Skip to content

Commit 22c1067

Browse files
Merge pull request #55 from nextmv-io/feature/eng-5632-extend-nextmv-py-acceptance-test-implementation
Enhancements to acceptance test models and addition of result polling method
2 parents c93c427 + cc7f4c6 commit 22c1067

File tree

5 files changed

+557
-16
lines changed

5 files changed

+557
-16
lines changed

nextmv/cloud/__init__.py

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,25 @@
11
"""Functionality for interacting with the Nextmv Cloud."""
22

33
from .acceptance_test import AcceptanceTest as AcceptanceTest
4+
from .acceptance_test import AcceptanceTestResults as AcceptanceTestResults
45
from .acceptance_test import Comparison as Comparison
56
from .acceptance_test import ComparisonInstance as ComparisonInstance
7+
from .acceptance_test import DistributionPercentiles as DistributionPercentiles
8+
from .acceptance_test import DistributionSummaryStatistics as DistributionSummaryStatistics
9+
from .acceptance_test import ExperimentStatus as ExperimentStatus
610
from .acceptance_test import Metric as Metric
711
from .acceptance_test import MetricParams as MetricParams
12+
from .acceptance_test import MetricResult as MetricResult
13+
from .acceptance_test import MetricStatistics as MetricStatistics
14+
from .acceptance_test import MetricTolerance as MetricTolerance
815
from .acceptance_test import MetricType as MetricType
16+
from .acceptance_test import ResultStatistics as ResultStatistics
17+
from .acceptance_test import StatisticType as StatisticType
18+
from .acceptance_test import ToleranceType as ToleranceType
919
from .account import Account as Account
1020
from .account import Queue as Queue
1121
from .account import QueuedRun as QueuedRun
1222
from .application import Application as Application
13-
from .application import Configuration as Configuration
1423
from .application import DownloadURL as DownloadURL
1524
from .application import ErrorLog as ErrorLog
1625
from .application import Metadata as Metadata
@@ -24,6 +33,8 @@
2433
from .batch_experiment import BatchExperimentRun as BatchExperimentRun
2534
from .client import Client as Client
2635
from .input_set import InputSet as InputSet
36+
from .instance import Configuration as Configuration
37+
from .instance import Instance as Instance
2738
from .manifest import Manifest as Manifest
2839
from .manifest import ManifestBuild as ManifestBuild
2940
from .manifest import ManifestPython as ManifestPython
@@ -32,3 +43,6 @@
3243
from .manifest import ManifestType as ManifestType
3344
from .status import Status as Status
3445
from .status import StatusV2 as StatusV2
46+
from .version import Version as Version
47+
from .version import VersionExecutable as VersionExecutable
48+
from .version import VersionExecutableRequirements as VersionExecutableRequirements

nextmv/cloud/acceptance_test.py

Lines changed: 173 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,22 +2,54 @@
22

33
from datetime import datetime
44
from enum import Enum
5-
from typing import List
5+
from typing import List, Optional
66

77
from nextmv.base_model import BaseModel
88

99

1010
class MetricType(str, Enum):
1111
"""Type of metric when doing a comparison."""
1212

13-
absolute_threshold = "absolute-threshold"
14-
"""Absolute threshold metric type."""
15-
difference_threshold = "difference-threshold"
16-
"""Difference threshold metric type."""
1713
direct_comparison = "direct-comparison"
1814
"""Direct comparison metric type."""
1915

2016

17+
class StatisticType(str, Enum):
18+
"""
19+
Type of statistical process for collapsing multiple values of a metric
20+
(from multiple runs) into a single value.
21+
"""
22+
23+
min = "min"
24+
"""Minimum value."""
25+
max = "max"
26+
"""Maximum value."""
27+
mean = "mean"
28+
"""Mean value."""
29+
std = "std"
30+
"""Standard deviation."""
31+
shifted_geometric_mean = "shifted_geometric_mean"
32+
"""Shifted geometric mean."""
33+
p01 = "p01"
34+
"""1st percentile."""
35+
p05 = "p05"
36+
"""5th percentile."""
37+
p10 = "p10"
38+
"""10th percentile."""
39+
p25 = "p25"
40+
"""25th percentile."""
41+
p50 = "p50"
42+
"""50th percentile."""
43+
p75 = "p75"
44+
"""75th percentile."""
45+
p90 = "p90"
46+
"""90th percentile."""
47+
p95 = "p95"
48+
"""95th percentile."""
49+
p99 = "p99"
50+
"""99th percentile."""
51+
52+
2153
class Comparison(str, Enum):
2254
"""Comparison to use for two metrics."""
2355

@@ -35,11 +67,50 @@ class Comparison(str, Enum):
3567
"""Not equal to metric type."""
3668

3769

70+
class ToleranceType(str, Enum):
71+
"""Type of tolerance used for a metric."""
72+
73+
undefined = ""
74+
"""Undefined tolerance type."""
75+
absolute = "absolute"
76+
"""Absolute tolerance type."""
77+
relative = "relative"
78+
"""Relative tolerance type."""
79+
80+
81+
class ExperimentStatus(str, Enum):
82+
"""Status of an acceptance test."""
83+
84+
started = "started"
85+
"""The experiment has started."""
86+
completed = "completed"
87+
"""The experiment was completed."""
88+
failed = "failed"
89+
"""The experiment failed."""
90+
draft = "draft"
91+
"""The experiment is a draft."""
92+
canceled = "canceled"
93+
"""The experiment was canceled."""
94+
unknown = "unknown"
95+
"""The experiment status is unknown."""
96+
97+
98+
class MetricTolerance(BaseModel):
99+
"""Tolerance used for a metric."""
100+
101+
type: ToleranceType
102+
"""Type of tolerance."""
103+
value: float
104+
"""Value of the tolerance."""
105+
106+
38107
class MetricParams(BaseModel):
39108
"""Parameters of an acceptance test."""
40109

41110
operator: Comparison
42111
"""Operator used to compare two metrics."""
112+
tolerance: MetricTolerance
113+
"""Tolerance used for the comparison."""
43114

44115

45116
class Metric(BaseModel):
@@ -52,8 +123,11 @@ class Metric(BaseModel):
52123
"""Type of the metric."""
53124
params: MetricParams
54125
"""Parameters of the metric."""
55-
statistic: str
56-
"""Statistic of the metric."""
126+
statistic: StatisticType
127+
"""
128+
Type of statistical process for collapsing multiple values of a metric
129+
(from multiple runs) into a single value.
130+
"""
57131

58132

59133
class ComparisonInstance(BaseModel):
@@ -65,6 +139,94 @@ class ComparisonInstance(BaseModel):
65139
"""ID of the version."""
66140

67141

142+
class DistributionSummaryStatistics(BaseModel):
143+
"""Statistics of a distribution summary."""
144+
145+
min: float
146+
"""Minimum value."""
147+
max: float
148+
"""Maximum value."""
149+
count: int
150+
"""Count of runs."""
151+
mean: float
152+
"""Mean value."""
153+
std: float
154+
"""Standard deviation."""
155+
shifted_geometric_mean: float
156+
"""Shifted geometric mean."""
157+
shift_parameter: float
158+
"""Shift parameter of the geometric mean."""
159+
160+
161+
class DistributionPercentiles(BaseModel):
162+
"""Percentiles of a distribution."""
163+
164+
p01: float
165+
"""1st percentile."""
166+
p05: float
167+
"""5th percentile."""
168+
p10: float
169+
"""10th percentile."""
170+
p25: float
171+
"""25th percentile."""
172+
p50: float
173+
"""50th percentile."""
174+
p75: float
175+
"""75th percentile."""
176+
p90: float
177+
"""90th percentile."""
178+
p95: float
179+
"""95th percentile."""
180+
p99: float
181+
"""99th percentile."""
182+
183+
184+
class ResultStatistics(BaseModel):
185+
"""Statistics of a metric result."""
186+
187+
instance_id: str
188+
"""ID of the instance."""
189+
version_id: str
190+
"""ID of the version."""
191+
number_of_runs_total: int
192+
"""Number of runs."""
193+
distribution_summary_statistics: DistributionSummaryStatistics
194+
"""Distribution summary statistics."""
195+
distribution_percentiles: DistributionPercentiles
196+
"""Distribution percentiles."""
197+
198+
199+
class MetricStatistics(BaseModel):
200+
"""Statistics of a metric."""
201+
202+
control: ResultStatistics
203+
"""Control statistics."""
204+
candidate: ResultStatistics
205+
"""Candidate statistics."""
206+
207+
208+
class MetricResult(BaseModel):
209+
"""Result of a metric."""
210+
211+
metric: Metric
212+
"""Metric of the result."""
213+
statistics: MetricStatistics
214+
"""Statistics of the metric."""
215+
passed: bool
216+
"""Whether the candidate passed for the metric (or not)."""
217+
218+
219+
class AcceptanceTestResults(BaseModel):
220+
"""Results of an acceptance test."""
221+
222+
passed: bool
223+
"""Whether the acceptance test passed (or not)."""
224+
metric_results: Optional[List[MetricResult]] = None
225+
"""Results of the metrics."""
226+
error: Optional[str] = None
227+
"""Error message if the acceptance test failed."""
228+
229+
68230
class AcceptanceTest(BaseModel):
69231
"""An acceptance test gives a go/no-go decision criteria for a set of
70232
metrics. It relies on a batch experiment."""
@@ -89,3 +251,7 @@ class AcceptanceTest(BaseModel):
89251
"""Creation date of the acceptance test."""
90252
updated_at: datetime
91253
"""Last update date of the acceptance test."""
254+
status: Optional[ExperimentStatus] = ExperimentStatus.unknown
255+
"""Status of the acceptance test."""
256+
results: Optional[AcceptanceTestResults] = None
257+
"""Results of the acceptance test."""

0 commit comments

Comments
 (0)