2
2
3
3
from datetime import datetime
4
4
from enum import Enum
5
- from typing import List
5
+ from typing import List , Optional
6
6
7
7
from nextmv .base_model import BaseModel
8
8
9
9
10
10
class MetricType (str , Enum ):
11
11
"""Type of metric when doing a comparison."""
12
12
13
- absolute_threshold = "absolute-threshold"
14
- """Absolute threshold metric type."""
15
- difference_threshold = "difference-threshold"
16
- """Difference threshold metric type."""
17
13
direct_comparison = "direct-comparison"
18
14
"""Direct comparison metric type."""
19
15
20
16
17
+ class StatisticType (str , Enum ):
18
+ """
19
+ Type of statistical process for collapsing multiple values of a metric
20
+ (from multiple runs) into a single value.
21
+ """
22
+
23
+ min = "min"
24
+ """Minimum value."""
25
+ max = "max"
26
+ """Maximum value."""
27
+ mean = "mean"
28
+ """Mean value."""
29
+ std = "std"
30
+ """Standard deviation."""
31
+ shifted_geometric_mean = "shifted_geometric_mean"
32
+ """Shifted geometric mean."""
33
+ p01 = "p01"
34
+ """1st percentile."""
35
+ p05 = "p05"
36
+ """5th percentile."""
37
+ p10 = "p10"
38
+ """10th percentile."""
39
+ p25 = "p25"
40
+ """25th percentile."""
41
+ p50 = "p50"
42
+ """50th percentile."""
43
+ p75 = "p75"
44
+ """75th percentile."""
45
+ p90 = "p90"
46
+ """90th percentile."""
47
+ p95 = "p95"
48
+ """95th percentile."""
49
+ p99 = "p99"
50
+ """99th percentile."""
51
+
52
+
21
53
class Comparison (str , Enum ):
22
54
"""Comparison to use for two metrics."""
23
55
@@ -35,11 +67,50 @@ class Comparison(str, Enum):
35
67
"""Not equal to metric type."""
36
68
37
69
70
+ class ToleranceType (str , Enum ):
71
+ """Type of tolerance used for a metric."""
72
+
73
+ undefined = ""
74
+ """Undefined tolerance type."""
75
+ absolute = "absolute"
76
+ """Absolute tolerance type."""
77
+ relative = "relative"
78
+ """Relative tolerance type."""
79
+
80
+
81
+ class ExperimentStatus (str , Enum ):
82
+ """Status of an acceptance test."""
83
+
84
+ started = "started"
85
+ """The experiment has started."""
86
+ completed = "completed"
87
+ """The experiment was completed."""
88
+ failed = "failed"
89
+ """The experiment failed."""
90
+ draft = "draft"
91
+ """The experiment is a draft."""
92
+ canceled = "canceled"
93
+ """The experiment was canceled."""
94
+ unknown = "unknown"
95
+ """The experiment status is unknown."""
96
+
97
+
98
+ class MetricTolerance (BaseModel ):
99
+ """Tolerance used for a metric."""
100
+
101
+ type : ToleranceType
102
+ """Type of tolerance."""
103
+ value : float
104
+ """Value of the tolerance."""
105
+
106
+
38
107
class MetricParams (BaseModel ):
39
108
"""Parameters of an acceptance test."""
40
109
41
110
operator : Comparison
42
111
"""Operator used to compare two metrics."""
112
+ tolerance : MetricTolerance
113
+ """Tolerance used for the comparison."""
43
114
44
115
45
116
class Metric (BaseModel ):
@@ -52,8 +123,11 @@ class Metric(BaseModel):
52
123
"""Type of the metric."""
53
124
params : MetricParams
54
125
"""Parameters of the metric."""
55
- statistic : str
56
- """Statistic of the metric."""
126
+ statistic : StatisticType
127
+ """
128
+ Type of statistical process for collapsing multiple values of a metric
129
+ (from multiple runs) into a single value.
130
+ """
57
131
58
132
59
133
class ComparisonInstance (BaseModel ):
@@ -65,6 +139,94 @@ class ComparisonInstance(BaseModel):
65
139
"""ID of the version."""
66
140
67
141
142
+ class DistributionSummaryStatistics (BaseModel ):
143
+ """Statistics of a distribution summary."""
144
+
145
+ min : float
146
+ """Minimum value."""
147
+ max : float
148
+ """Maximum value."""
149
+ count : int
150
+ """Count of runs."""
151
+ mean : float
152
+ """Mean value."""
153
+ std : float
154
+ """Standard deviation."""
155
+ shifted_geometric_mean : float
156
+ """Shifted geometric mean."""
157
+ shift_parameter : float
158
+ """Shift parameter of the geometric mean."""
159
+
160
+
161
+ class DistributionPercentiles (BaseModel ):
162
+ """Percentiles of a distribution."""
163
+
164
+ p01 : float
165
+ """1st percentile."""
166
+ p05 : float
167
+ """5th percentile."""
168
+ p10 : float
169
+ """10th percentile."""
170
+ p25 : float
171
+ """25th percentile."""
172
+ p50 : float
173
+ """50th percentile."""
174
+ p75 : float
175
+ """75th percentile."""
176
+ p90 : float
177
+ """90th percentile."""
178
+ p95 : float
179
+ """95th percentile."""
180
+ p99 : float
181
+ """99th percentile."""
182
+
183
+
184
+ class ResultStatistics (BaseModel ):
185
+ """Statistics of a metric result."""
186
+
187
+ instance_id : str
188
+ """ID of the instance."""
189
+ version_id : str
190
+ """ID of the version."""
191
+ number_of_runs_total : int
192
+ """Number of runs."""
193
+ distribution_summary_statistics : DistributionSummaryStatistics
194
+ """Distribution summary statistics."""
195
+ distribution_percentiles : DistributionPercentiles
196
+ """Distribution percentiles."""
197
+
198
+
199
+ class MetricStatistics (BaseModel ):
200
+ """Statistics of a metric."""
201
+
202
+ control : ResultStatistics
203
+ """Control statistics."""
204
+ candidate : ResultStatistics
205
+ """Candidate statistics."""
206
+
207
+
208
+ class MetricResult (BaseModel ):
209
+ """Result of a metric."""
210
+
211
+ metric : Metric
212
+ """Metric of the result."""
213
+ statistics : MetricStatistics
214
+ """Statistics of the metric."""
215
+ passed : bool
216
+ """Whether the candidate passed for the metric (or not)."""
217
+
218
+
219
+ class AcceptanceTestResults (BaseModel ):
220
+ """Results of an acceptance test."""
221
+
222
+ passed : bool
223
+ """Whether the acceptance test passed (or not)."""
224
+ metric_results : Optional [List [MetricResult ]] = None
225
+ """Results of the metrics."""
226
+ error : Optional [str ] = None
227
+ """Error message if the acceptance test failed."""
228
+
229
+
68
230
class AcceptanceTest (BaseModel ):
69
231
"""An acceptance test gives a go/no-go decision criteria for a set of
70
232
metrics. It relies on a batch experiment."""
@@ -89,3 +251,7 @@ class AcceptanceTest(BaseModel):
89
251
"""Creation date of the acceptance test."""
90
252
updated_at : datetime
91
253
"""Last update date of the acceptance test."""
254
+ status : Optional [ExperimentStatus ] = ExperimentStatus .unknown
255
+ """Status of the acceptance test."""
256
+ results : Optional [AcceptanceTestResults ] = None
257
+ """Results of the acceptance test."""
0 commit comments