55import numpy as np
66
77__all__ = ['get_metrics' ,
8- 'tabulate_metrics' ]
8+ 'compute_metrics' ,
9+ 'tabulate_metrics' ,
10+ 'MetricCollection' ]
911
1012
1113class RPrecision :
12- def __init__ (self , top_k : int ) -> None :
14+ def __init__ (self , top_k : int ):
1315 self .top_k = top_k
1416 self .score = 0
1517 self .num_sample = 0
1618
17- def update (self , preds : np .ndarray , target : np .ndarray ) -> None :
19+ def update (self , preds : np .ndarray , target : np .ndarray ):
1820 assert preds .shape == target .shape # (batch_size, num_classes)
1921 top_k_ind = np .argpartition (preds , - self .top_k )[:, - self .top_k :]
2022 num_relevant = np .take_along_axis (
@@ -28,14 +30,21 @@ def update(self, preds: np.ndarray, target: np.ndarray) -> None:
2830 def compute (self ) -> float :
2931 return self .score / self .num_sample
3032
33+ def reset (self ):
34+ self .score = 0
35+ self .num_sample = 0
36+
3137
3238class Precision :
33- def __init__ (self , num_classes : int , average : str , top_k : int ) -> None :
39+ def __init__ (self , num_classes : int , average : str , top_k : int ):
40+ if average != 'samples' :
41+ raise ValueError ('unsupported average' )
42+
3443 self .top_k = top_k
3544 self .score = 0
3645 self .num_sample = 0
3746
38- def update (self , preds : np .ndarray , target : np .ndarray ) -> None :
47+ def update (self , preds : np .ndarray , target : np .ndarray ):
3948 assert preds .shape == target .shape # (batch_size, num_classes)
4049 top_k_ind = np .argpartition (preds , - self .top_k )[:, - self .top_k :]
4150 num_relevant = np .take_along_axis (target , top_k_ind , - 1 ).sum ()
@@ -45,25 +54,28 @@ def update(self, preds: np.ndarray, target: np.ndarray) -> None:
4554 def compute (self ) -> float :
4655 return self .score / self .num_sample
4756
57+ def reset (self ):
58+ self .score = 0
59+ self .num_sample = 0
60+
4861
4962class F1 :
50- def __init__ (self , num_classes : int , metric_threshold : float , average : str , multiclass = False ) -> None :
63+ def __init__ (self , num_classes : int , average : str , multiclass = False ):
5164 self .num_classes = num_classes
52- self .metric_threshold = metric_threshold
5365 if average not in 'macro' , 'micro' , 'another-macro' }:
5466 raise ValueError ('unsupported average' )
5567 self .average = average
5668 self .multiclass = multiclass
5769 self .tp = self .fp = self .fn = 0
5870
59- def update (self , preds : np .ndarray , target : np .ndarray ) -> None :
71+ def update (self , preds : np .ndarray , target : np .ndarray ):
6072 assert preds .shape == target .shape # (batch_size, num_classes)
6173 if self .multiclass :
6274 max_idx = np .argmax (preds , axis = 1 ).reshape (- 1 , 1 )
6375 preds = np .zeros (preds .shape )
6476 np .put_along_axis (preds , max_idx , 1 , axis = 1 )
6577 else :
66- preds = preds > self . metric_threshold
78+ preds = preds > 0
6779 self .tp += np .logical_and (target == 1 , preds == 1 ).sum (axis = 0 )
6880 self .fn += np .logical_and (target == 1 , preds == 0 ).sum (axis = 0 )
6981 self .fp += np .logical_and (target == 0 , preds == 1 ).sum (axis = 0 )
@@ -88,34 +100,58 @@ def compute(self) -> float:
88100 np .seterr (** prev_settings )
89101 return score
90102
103+ def reset (self ):
104+ self .tp = self .fp = self .fn = 0
105+
91106
92107class MetricCollection (dict ):
93- def __init__ (self , metrics ) -> None :
108+ """A collection of metrics created by get_metrics.
109+ MetricCollection computes metric values in two steps. First, batches of
110+ decision values and labels are added with update(). After all instances have been
111+ added, compute() computes the metric values from the accumulated batches.
112+ """
113+
114+ def __init__ (self , metrics ):
94115 self .metrics = metrics
95116
96- def update (self , preds : np .ndarray , target : np .ndarray ) -> None :
117+ def update (self , preds : np .ndarray , target : np .ndarray ):
118+ """Adds a batch of decision values and labels.
119+
120+ Args:
121+ preds (np.ndarray): A matrix of decision values with dimensions number of instances * number of classes.
122+ target (np.ndarray): A 0/1 matrix of labels with dimensions number of instances * number of classes.
123+ """
97124 assert preds .shape == target .shape # (batch_size, num_classes)
98125 for metric in self .metrics .values ():
99126 metric .update (preds , target )
100127
101128 def compute (self ) -> dict [str , float ]:
129+ """Computes the metrics from the accumulated batches of decision values and labels.
130+
131+ Returns:
132+ dict[str, float]: A dictionary of metric values.
133+ """
102134 ret = {}
103135 for name , metric in self .metrics .items ():
104136 ret [name ] = metric .compute ()
105137 return ret
106138
139+ def reset (self ):
140+ """Clears the accumulated batches of decision values and labels.
141+ """
142+ for metric in self .metrics .values ():
143+ metric .reset ()
144+
107145
108- def get_metrics (metric_threshold : float ,
109- monitor_metrics : list [str ],
146+ def get_metrics (monitor_metrics : list [str ],
110147 num_classes : int ,
111148 multiclass : bool = False
112149 ) -> MetricCollection :
113150 """Get a collection of metrics by their names.
151+ See MetricCollection for more details.
114152
115153 Args:
116- metric_threshold (float): The decision value threshold over which a
117- label is predicted as positive.
118- monitor_metrics (list[str]): A list metric names.
154+ monitor_metrics (list[str]): A list of metric names.
119155 num_classes (int): The number of classes.
120156 multiclass (bool, optional): Enable multiclass mode. Defaults to False.
121157
@@ -132,19 +168,54 @@ def get_metrics(metric_threshold: float,
132168 elif re .match ('RP@\d+' , metric ):
133169 metrics [metric ] = RPrecision (top_k = int (metric [3 :]))
134170 elif metric in {'Another-Macro-F1' , 'Macro-F1' , 'Micro-F1' }:
135- metrics [metric ] = F1 (num_classes , metric_threshold ,
171+ metrics [metric ] = F1 (num_classes ,
136172 average = metric [:- 3 ].lower (),
137173 multiclass = multiclass )
138174 else :
139- raise ValueError (f'Invalid metric: { metric } )
175+ raise ValueError (f'invalid metric: { metric } )
140176
141177 return MetricCollection (metrics )
142178
143179
180+ def compute_metrics (preds : np .ndarray ,
181+ target : np .ndarray ,
182+ monitor_metrics : list [str ],
183+ multiclass : bool = False
184+ ) -> dict [str , float ]:
185+ """Compute metrics with decision values and labels.
186+ See get_metrics and MetricCollection if decision values and labels are too
187+ large to hold in memory.
188+
189+
190+ Args:
191+ preds (np.ndarray): A matrix of decision values with dimensions number of instances * number of classes.
192+ target (np.ndarray): A 0/1 matrix of labels with dimensions number of instances * number of classes.
193+ monitor_metrics (list[str]): A list of metric names.
194+ multiclass (bool, optional): Enable multiclass mode. Defaults to False.
195+
196+ Returns:
197+ dict[str, float]: A dictionary of metric values.
198+ """
199+ assert preds .shape == target .shape
200+
201+ metric = get_metrics (monitor_metrics , preds .shape [1 ], multiclass )
202+ metric .update (preds , target )
203+ return metric .compute ()
204+
205+
144206def tabulate_metrics (metric_dict : dict [str , float ], split : str ) -> str :
207+ """Convert a dictionary of metric values into a pretty formatted string for printing.
208+
209+ Args:
210+ metric_dict (dict[str, float]): A dictionary of metric values.
211+ split (str): Name of the data split.
212+
213+ Returns:
214+ str: Pretty formatted string.
215+ """
145216 msg = f'====== { split } \n '
146217 header = '|' .join ([f'{ k :^18} for k in metric_dict .keys ()])
147- values = '|' .join ([f'{ x * 100 :^18.4f} if isinstance (x , (np .floating ,
218+ values = '|' .join ([f'{ x :^18.4f} if isinstance (x , (np .floating ,
148219 float )) else f'{ x :^18} for x in metric_dict .values ()])
149220 msg += f"|{ header } \n |{ '-----------------:|' * len (metric_dict )} \n |{ values } \n "
150221 return msg
0 commit comments