Merge pull request #280 from ntumlgroup/random-fixes

Random fixes

Author: cjlin1

docs/examples/plot_linear_gridsearch_tutorial.py

@@ -11,9 +11,8 @@
 
 from sklearn.preprocessing import MultiLabelBinarizer
 import libmultilabel.linear as linear
-from libmultilabel.linear.preprocessor import read_libmultilabel_format
 
-train_data = read_libmultilabel_format('data/rcv1/train.txt')
+train_data = linear.read_libmultilabel_format('data/rcv1/train.txt')
 binarizer = MultiLabelBinarizer(sparse_output=True)
 binarizer.fit(train_data['label'])
 y = binarizer.transform(train_data['label']).astype('d')

libmultilabel/linear/__init__.py

@@ -1,5 +1,5 @@
 from .linear import *
-from .metrics import get_metrics, tabulate_metrics
+from .metrics import *
 from .preprocessor import *
 from .tree import *
 from .utils import *

libmultilabel/linear/linear.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import logging
 import os
 
@@ -11,13 +13,14 @@
            'train_cost_sensitive',
            'train_cost_sensitive_micro',
            'train_binary_and_multiclass',
-           'predict_values']
+           'predict_values',
+           'get_topk_labels']
 
 
 class FlatModel:
     def __init__(self, weights: np.matrix,
                  bias: float,
-                 thresholds: 'float | np.ndarray',
+                 thresholds: float | np.ndarray,
                  ):
         self.weights = weights
         self.bias = bias
@@ -68,7 +71,7 @@ def train_1vsrest(y: sparse.csr_matrix,
         A model which can be used in predict_values.
     """
     # Follows the MATLAB implementation at https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/multilabel/
-    x, options, bias = prepare_options(x, options)
+    x, options, bias = _prepare_options(x, options)
 
     y = y.tocsc()
     num_class = y.shape[1]
@@ -79,14 +82,14 @@ def train_1vsrest(y: sparse.csr_matrix,
         logging.info(f'Training one-vs-rest model on {num_class} labels')
     for i in tqdm(range(num_class), disable=not verbose):
         yi = y[:, i].toarray().reshape(-1)
-        weights[:, i] = do_train(2*yi - 1, x, options).ravel()
+        weights[:, i] = _do_train(2*yi - 1, x, options).ravel()
 
     return FlatModel(weights=np.asmatrix(weights),
                      bias=bias,
                      thresholds=0)
 
 
-def prepare_options(x: sparse.csr_matrix, options: str) -> 'tuple[sparse.csr_matrix, str, float]':
+def _prepare_options(x: sparse.csr_matrix, options: str) -> tuple[sparse.csr_matrix, str, float]:
     """Prepare options and x for multi-label training. Called in the first line of
     any training function.
 
@@ -150,7 +153,7 @@ def train_thresholding(y: sparse.csr_matrix,
         A model which can be used in predict_values.
     """
     # Follows the MATLAB implementation at https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/multilabel/
-    x, options, bias = prepare_options(x, options)
+    x, options, bias = _prepare_options(x, options)
 
     y = y.tocsc()
     num_class = y.shape[1]
@@ -162,7 +165,7 @@ def train_thresholding(y: sparse.csr_matrix,
         logging.info(f'Training thresholding model on {num_class} labels')
     for i in tqdm(range(num_class), disable=not verbose):
         yi = y[:, i].toarray().reshape(-1)
-        w, t = thresholding_one_label(2*yi - 1, x, options)
+        w, t = _thresholding_one_label(2*yi - 1, x, options)
         weights[:, i] = w.ravel()
         thresholds[i] = t
 
@@ -171,10 +174,10 @@ def train_thresholding(y: sparse.csr_matrix,
                      thresholds=thresholds)
 
 
-def thresholding_one_label(y: np.ndarray,
+def _thresholding_one_label(y: np.ndarray,
                            x: sparse.csr_matrix,
                            options: str
-                           ) -> 'tuple[np.ndarray, float]':
+                           ) -> tuple[np.ndarray, float]:
     """Outer cross-validation for thresholding on a single label.
 
     Args:
@@ -201,29 +204,29 @@ def thresholding_one_label(y: np.ndarray,
         val_idx = perm[mask]
         train_idx = perm[mask != True]
 
-        scutfbr_w, scutfbr_b_list = scutfbr(
+        scutfbr_w, scutfbr_b_list = _scutfbr(
             y[train_idx], x[train_idx], fbr_list, options)
         wTx = (x[val_idx] * scutfbr_w).A1
 
         for i in range(fbr_list.size):
-            F = fmeasure(y[val_idx], 2*(wTx > -scutfbr_b_list[i]) - 1)
+            F = _fmeasure(y[val_idx], 2*(wTx > -scutfbr_b_list[i]) - 1)
             f_list[i] += F
 
     best_fbr = fbr_list[::-1][np.argmax(f_list[::-1])]  # last largest
     if np.max(f_list) == 0:
         best_fbr = np.min(fbr_list)
 
     # final model
-    w, b_list = scutfbr(y, x, np.array([best_fbr]), options)
+    w, b_list = _scutfbr(y, x, np.array([best_fbr]), options)
 
     return w, b_list[0]
 
 
-def scutfbr(y: np.ndarray,
+def _scutfbr(y: np.ndarray,
             x: sparse.csr_matrix,
-            fbr_list: 'list[float]',
+            fbr_list: list[float],
             options: str
-            ) -> 'tuple[np.matrix, np.ndarray]':
+            ) -> tuple[np.matrix, np.ndarray]:
     """Inner cross-validation for SCutfbr heuristic.
 
     Args:
@@ -250,10 +253,10 @@ def scutfbr(y: np.ndarray,
         val_idx = perm[mask]
         train_idx = perm[mask != True]
 
-        w = do_train(y[train_idx], x[train_idx], options)
+        w = _do_train(y[train_idx], x[train_idx], options)
         wTx = (x[val_idx] * w).A1
         scut_b = 0.
-        start_F = fmeasure(y[val_idx], 2*(wTx > -scut_b) - 1)
+        start_F = _fmeasure(y[val_idx], 2*(wTx > -scut_b) - 1)
 
         # stableness to match the MATLAB implementation
         sorted_wTx_index = np.argsort(wTx, kind='stable')
@@ -291,7 +294,7 @@ def scutfbr(y: np.ndarray,
             else:
                 scut_b = -(sorted_wTx[cut] + sorted_wTx[cut + 1]) / 2
 
-        F = fmeasure(y_val, 2*(wTx > -scut_b) - 1)
+        F = _fmeasure(y_val, 2*(wTx > -scut_b) - 1)
 
         for i in range(fbr_list.size):
             if F > fbr_list[i]:
@@ -300,10 +303,13 @@ def scutfbr(y: np.ndarray,
                 b_list[i] -= np.max(wTx)
 
     b_list = b_list / nr_fold
-    return do_train(y, x, options), b_list
+    return _do_train(y, x, options), b_list
 
 
-def do_train(y: np.ndarray, x: sparse.csr_matrix, options: str) -> np.matrix:
+def _do_train(y: np.ndarray,
+              x: sparse.csr_matrix,
+              options: str
+              ) -> np.matrix:
     """Wrapper around liblinear.liblinearutil.train.
     Forcibly suppresses all IO regardless of options.
 
@@ -351,7 +357,7 @@ def __exit__(self, type, value, traceback):
         os.close(self.stderr)
 
 
-def fmeasure(y_true: np.ndarray, y_pred: np.ndarray) -> float:
+def _fmeasure(y_true: np.ndarray, y_pred: np.ndarray) -> float:
     """Calculate F1 score.
 
     Args:
@@ -393,7 +399,7 @@ def train_cost_sensitive(y: sparse.csr_matrix,
         A model which can be used in predict_values.
     """
     # Follows the MATLAB implementation at https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/multilabel/
-    x, options, bias = prepare_options(x, options)
+    x, options, bias = _prepare_options(x, options)
 
     y = y.tocsc()
     num_class = y.shape[1]
@@ -405,15 +411,15 @@ def train_cost_sensitive(y: sparse.csr_matrix,
             f'Training cost-sensitive model for Macro-F1 on {num_class} labels')
     for i in tqdm(range(num_class), disable=not verbose):
         yi = y[:, i].toarray().reshape(-1)
-        w = cost_sensitive_one_label(2*yi - 1, x, options)
+        w = _cost_sensitive_one_label(2*yi - 1, x, options)
         weights[:, i] = w.ravel()
 
     return FlatModel(weights=np.asmatrix(weights),
                      bias=bias,
                      thresholds=0)
 
 
-def cost_sensitive_one_label(y: np.ndarray,
+def _cost_sensitive_one_label(y: np.ndarray,
                              x: sparse.csr_matrix,
                              options: str
                              ) -> np.ndarray:
@@ -436,17 +442,17 @@ def cost_sensitive_one_label(y: np.ndarray,
     bestScore = -np.Inf
     for a in param_space:
         cv_options = f'{options} -w1 {a}'
-        pred = cross_validate(y, x, cv_options, perm)
-        score = fmeasure(y, pred)
+        pred = _cross_validate(y, x, cv_options, perm)
+        score = _fmeasure(y, pred)
         if bestScore < score:
             bestScore = score
             bestA = a
 
     final_options = f'{options} -w1 {bestA}'
-    return do_train(y, x, final_options)
+    return _do_train(y, x, final_options)
 
 
-def cross_validate(y: np.ndarray,
+def _cross_validate(y: np.ndarray,
                    x: sparse.csr_matrix,
                    options: str,
                    perm: np.ndarray
@@ -470,7 +476,7 @@ def cross_validate(y: np.ndarray,
         val_idx = perm[mask]
         train_idx = perm[mask != True]
 
-        w = do_train(y[train_idx], x[train_idx], options)
+        w = _do_train(y[train_idx], x[train_idx], options)
         pred[val_idx] = (x[val_idx] * w).A1 > 0
 
     return 2*pred - 1
@@ -498,7 +504,7 @@ def train_cost_sensitive_micro(y: sparse.csr_matrix,
         A model which can be used in predict_values.
     """
     # Follows the MATLAB implementation at https://www.csie.ntu.edu.tw/~cjlin/libsvmtools/multilabel/
-    x, options, bias = prepare_options(x, options)
+    x, options, bias = _prepare_options(x, options)
 
     y = y.tocsc()
     num_class = y.shape[1]
@@ -520,7 +526,7 @@ def train_cost_sensitive_micro(y: sparse.csr_matrix,
             yi = 2*yi - 1
 
             cv_options = f'{options} -w1 {a}'
-            pred = cross_validate(yi, x, cv_options, perm)
+            pred = _cross_validate(yi, x, cv_options, perm)
             tp = tp + np.sum(np.logical_and(yi == 1, pred == 1))
             fn = fn + np.sum(np.logical_and(yi == 1, pred == -1))
             fp = fp + np.sum(np.logical_and(yi == -1, pred == 1))
@@ -533,7 +539,7 @@ def train_cost_sensitive_micro(y: sparse.csr_matrix,
     final_options = f'{options} -w1 {bestA}'
     for i in range(num_class):
         yi = y[:, i].toarray().reshape(-1)
-        w = do_train(2*yi - 1, x, final_options)
+        w = _do_train(2*yi - 1, x, final_options)
         weights[:, i] = w.ravel()
 
     return FlatModel(weights=np.asmatrix(weights),
@@ -557,7 +563,7 @@ def train_binary_and_multiclass(y: sparse.csr_matrix,
     Returns:
         A model which can be used in predict_values.
     """
-    x, options, bias = prepare_options(x, options)
+    x, options, bias = _prepare_options(x, options)
     num_instances, num_labels = y.shape
     nonzero_instance_ids, nonzero_label_ids = y.nonzero()
     assert len(set(nonzero_instance_ids)) == num_instances, """
@@ -602,7 +608,10 @@ def predict_values(model, x: sparse.csr_matrix) -> np.ndarray:
     return model.predict_values(x)
 
 
-def get_topk_labels(label_mapping: np.ndarray, preds: np.ndarray, top_k: int = 5) -> 'list[list[str]]':
+def get_topk_labels(label_mapping: np.ndarray,
+                    preds: np.ndarray,
+                    top_k: int = 5
+                    ) -> list[list[str]]:
     """Get top k predictions from decision values.
 
     Args:

libmultilabel/linear/metrics.py

@@ -1,12 +1,10 @@
+from __future__ import annotations
+
 import re
 
 import numpy as np
 
-__all__ = ['RPrecision',
-           'Precision',
-           'F1',
-           'MetricCollection',
-           'get_metrics',
+__all__ = ['get_metrics',
            'tabulate_metrics']
 
 
@@ -100,20 +98,24 @@ def update(self, preds: np.ndarray, target: np.ndarray) -> None:
         for metric in self.metrics.values():
             metric.update(preds, target)
 
-    def compute(self) -> "dict[str, float]":
+    def compute(self) -> dict[str, float]:
         ret = {}
         for name, metric in self.metrics.items():
             ret[name] = metric.compute()
         return ret
 
 
-def get_metrics(metric_threshold: float, monitor_metrics: list, num_classes: int, multiclass=False):
+def get_metrics(metric_threshold: float,
+                monitor_metrics: list[str],
+                num_classes: int,
+                multiclass: bool = False
+                ) -> MetricCollection:
     """Get a collection of metrics by their names.
 
     Args:
         metric_threshold (float): The decision value threshold over which a
         label is predicted as positive.
-        monitor_metrics (list): A list of strings naming the metrics.
+        monitor_metrics (list[str]): A list metric names.
         num_classes (int): The number of classes.
         multiclass (bool, optional): Enable multiclass mode. Defaults to False.
 
@@ -139,7 +141,7 @@ def get_metrics(metric_threshold: float, monitor_metrics: list, num_classes: int
     return MetricCollection(metrics)
 
 
-def tabulate_metrics(metric_dict, split):
+def tabulate_metrics(metric_dict: dict[str, float], split: str) -> str:
     msg = f'====== {split} dataset evaluation result =======\n'
     header = '|'.join([f'{k:^18}' for k in metric_dict.keys()])
     values = '|'.join([f'{x * 100:^18.4f}' if isinstance(x, (np.floating,