Merge pull request #55 from nextmv-io/feature/eng-5632-extend-nextmv-py-acceptance-test-implementation

Enhancements to acceptance test models and addition of result polling method

Author: merschformann

nextmv/cloud/__init__.py

@@ -1,16 +1,25 @@
 """Functionality for interacting with the Nextmv Cloud."""
 
 from .acceptance_test import AcceptanceTest as AcceptanceTest
+from .acceptance_test import AcceptanceTestResults as AcceptanceTestResults
 from .acceptance_test import Comparison as Comparison
 from .acceptance_test import ComparisonInstance as ComparisonInstance
+from .acceptance_test import DistributionPercentiles as DistributionPercentiles
+from .acceptance_test import DistributionSummaryStatistics as DistributionSummaryStatistics
+from .acceptance_test import ExperimentStatus as ExperimentStatus
 from .acceptance_test import Metric as Metric
 from .acceptance_test import MetricParams as MetricParams
+from .acceptance_test import MetricResult as MetricResult
+from .acceptance_test import MetricStatistics as MetricStatistics
+from .acceptance_test import MetricTolerance as MetricTolerance
 from .acceptance_test import MetricType as MetricType
+from .acceptance_test import ResultStatistics as ResultStatistics
+from .acceptance_test import StatisticType as StatisticType
+from .acceptance_test import ToleranceType as ToleranceType
 from .account import Account as Account
 from .account import Queue as Queue
 from .account import QueuedRun as QueuedRun
 from .application import Application as Application
-from .application import Configuration as Configuration
 from .application import DownloadURL as DownloadURL
 from .application import ErrorLog as ErrorLog
 from .application import Metadata as Metadata
@@ -24,6 +33,8 @@
 from .batch_experiment import BatchExperimentRun as BatchExperimentRun
 from .client import Client as Client
 from .input_set import InputSet as InputSet
+from .instance import Configuration as Configuration
+from .instance import Instance as Instance
 from .manifest import Manifest as Manifest
 from .manifest import ManifestBuild as ManifestBuild
 from .manifest import ManifestPython as ManifestPython
@@ -32,3 +43,6 @@
 from .manifest import ManifestType as ManifestType
 from .status import Status as Status
 from .status import StatusV2 as StatusV2
+from .version import Version as Version
+from .version import VersionExecutable as VersionExecutable
+from .version import VersionExecutableRequirements as VersionExecutableRequirements

nextmv/cloud/acceptance_test.py

@@ -2,22 +2,54 @@
 
 from datetime import datetime
 from enum import Enum
-from typing import List
+from typing import List, Optional
 
 from nextmv.base_model import BaseModel
 
 
 class MetricType(str, Enum):
     """Type of metric when doing a comparison."""
 
-    absolute_threshold = "absolute-threshold"
-    """Absolute threshold metric type."""
-    difference_threshold = "difference-threshold"
-    """Difference threshold metric type."""
     direct_comparison = "direct-comparison"
     """Direct comparison metric type."""
 
 
+class StatisticType(str, Enum):
+    """
+    Type of statistical process for collapsing multiple values of a metric
+    (from multiple runs) into a single value.
+    """
+
+    min = "min"
+    """Minimum value."""
+    max = "max"
+    """Maximum value."""
+    mean = "mean"
+    """Mean value."""
+    std = "std"
+    """Standard deviation."""
+    shifted_geometric_mean = "shifted_geometric_mean"
+    """Shifted geometric mean."""
+    p01 = "p01"
+    """1st percentile."""
+    p05 = "p05"
+    """5th percentile."""
+    p10 = "p10"
+    """10th percentile."""
+    p25 = "p25"
+    """25th percentile."""
+    p50 = "p50"
+    """50th percentile."""
+    p75 = "p75"
+    """75th percentile."""
+    p90 = "p90"
+    """90th percentile."""
+    p95 = "p95"
+    """95th percentile."""
+    p99 = "p99"
+    """99th percentile."""
+
+
 class Comparison(str, Enum):
     """Comparison to use for two metrics."""
 
@@ -35,11 +67,50 @@ class Comparison(str, Enum):
     """Not equal to metric type."""
 
 
+class ToleranceType(str, Enum):
+    """Type of tolerance used for a metric."""
+
+    undefined = ""
+    """Undefined tolerance type."""
+    absolute = "absolute"
+    """Absolute tolerance type."""
+    relative = "relative"
+    """Relative tolerance type."""
+
+
+class ExperimentStatus(str, Enum):
+    """Status of an acceptance test."""
+
+    started = "started"
+    """The experiment has started."""
+    completed = "completed"
+    """The experiment was completed."""
+    failed = "failed"
+    """The experiment failed."""
+    draft = "draft"
+    """The experiment is a draft."""
+    canceled = "canceled"
+    """The experiment was canceled."""
+    unknown = "unknown"
+    """The experiment status is unknown."""
+
+
+class MetricTolerance(BaseModel):
+    """Tolerance used for a metric."""
+
+    type: ToleranceType
+    """Type of tolerance."""
+    value: float
+    """Value of the tolerance."""
+
+
 class MetricParams(BaseModel):
     """Parameters of an acceptance test."""
 
     operator: Comparison
     """Operator used to compare two metrics."""
+    tolerance: MetricTolerance
+    """Tolerance used for the comparison."""
 
 
 class Metric(BaseModel):
@@ -52,8 +123,11 @@ class Metric(BaseModel):
     """Type of the metric."""
     params: MetricParams
     """Parameters of the metric."""
-    statistic: str
-    """Statistic of the metric."""
+    statistic: StatisticType
+    """
+    Type of statistical process for collapsing multiple values of a metric
+    (from multiple runs) into a single value.
+    """
 
 
 class ComparisonInstance(BaseModel):
@@ -65,6 +139,94 @@ class ComparisonInstance(BaseModel):
     """ID of the version."""
 
 
+class DistributionSummaryStatistics(BaseModel):
+    """Statistics of a distribution summary."""
+
+    min: float
+    """Minimum value."""
+    max: float
+    """Maximum value."""
+    count: int
+    """Count of runs."""
+    mean: float
+    """Mean value."""
+    std: float
+    """Standard deviation."""
+    shifted_geometric_mean: float
+    """Shifted geometric mean."""
+    shift_parameter: float
+    """Shift parameter of the geometric mean."""
+
+
+class DistributionPercentiles(BaseModel):
+    """Percentiles of a distribution."""
+
+    p01: float
+    """1st percentile."""
+    p05: float
+    """5th percentile."""
+    p10: float
+    """10th percentile."""
+    p25: float
+    """25th percentile."""
+    p50: float
+    """50th percentile."""
+    p75: float
+    """75th percentile."""
+    p90: float
+    """90th percentile."""
+    p95: float
+    """95th percentile."""
+    p99: float
+    """99th percentile."""
+
+
+class ResultStatistics(BaseModel):
+    """Statistics of a metric result."""
+
+    instance_id: str
+    """ID of the instance."""
+    version_id: str
+    """ID of the version."""
+    number_of_runs_total: int
+    """Number of runs."""
+    distribution_summary_statistics: DistributionSummaryStatistics
+    """Distribution summary statistics."""
+    distribution_percentiles: DistributionPercentiles
+    """Distribution percentiles."""
+
+
+class MetricStatistics(BaseModel):
+    """Statistics of a metric."""
+
+    control: ResultStatistics
+    """Control statistics."""
+    candidate: ResultStatistics
+    """Candidate statistics."""
+
+
+class MetricResult(BaseModel):
+    """Result of a metric."""
+
+    metric: Metric
+    """Metric of the result."""
+    statistics: MetricStatistics
+    """Statistics of the metric."""
+    passed: bool
+    """Whether the candidate passed for the metric (or not)."""
+
+
+class AcceptanceTestResults(BaseModel):
+    """Results of an acceptance test."""
+
+    passed: bool
+    """Whether the acceptance test passed (or not)."""
+    metric_results: Optional[List[MetricResult]] = None
+    """Results of the metrics."""
+    error: Optional[str] = None
+    """Error message if the acceptance test failed."""
+
+
 class AcceptanceTest(BaseModel):
     """An acceptance test gives a go/no-go decision criteria for a set of
     metrics. It relies on a batch experiment."""
@@ -89,3 +251,7 @@ class AcceptanceTest(BaseModel):
     """Creation date of the acceptance test."""
     updated_at: datetime
     """Last update date of the acceptance test."""
+    status: Optional[ExperimentStatus] = ExperimentStatus.unknown
+    """Status of the acceptance test."""
+    results: Optional[AcceptanceTestResults] = None
+    """Results of the acceptance test."""