More functional tests for GetModelStatus (#77)

* more functional tests for getmodelstatus

Author: mzegla

Makefile

@@ -70,6 +70,7 @@ test_local_only: venv
 	@. $(ACTIVATE); py.test $(TEST_DIRS)/functional/test_batching.py
 	@. $(ACTIVATE); py.test $(TEST_DIRS)/functional/test_mapping.py
 	@. $(ACTIVATE); py.test $(TEST_DIRS)/functional/test_single_model.py
+	@. $(ACTIVATE); py.test $(TEST_DIRS)/functional/test_update.py
 
 style: venv
 	@echo "Style-checking codebase..."

tests/functional/conftest.py

@@ -199,6 +199,13 @@ def create_channel_for_port_single_server():
     return stub
 
 
+@pytest.fixture(autouse=True, scope="session")
+def create_channel_for_port_multi_server_status():
+    channel = grpc.insecure_channel('localhost:9001')
+    stub = model_service_pb2_grpc.ModelServiceStub(channel)
+    return stub
+
+
 @pytest.fixture(autouse=True, scope="session")
 def create_channel_for_port_multi_server():
     channel = grpc.insecure_channel('localhost:9001')
@@ -234,20 +241,41 @@ def create_channel_for_batching_server_auto():
     return stub
 
 
+@pytest.fixture(scope="session")
+def create_channel_for_model_ver_pol_server_status():
+    channel = grpc.insecure_channel('localhost:9006')
+    stub = model_service_pb2_grpc.ModelServiceStub(channel)
+    return stub
+
+
 @pytest.fixture(scope="session")
 def create_channel_for_model_ver_pol_server():
     channel = grpc.insecure_channel('localhost:9006')
     stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
     return stub
 
 
+@pytest.fixture(scope="session")
+def create_channel_for_update_flow_latest_status():
+    channel = grpc.insecure_channel('localhost:9007')
+    stub = model_service_pb2_grpc.ModelServiceStub(channel)
+    return stub
+
+
 @pytest.fixture(scope="session")
 def create_channel_for_update_flow_latest():
     channel = grpc.insecure_channel('localhost:9007')
     stub = prediction_service_pb2_grpc.PredictionServiceStub(channel)
     return stub
 
 
+@pytest.fixture(scope="session")
+def create_channel_for_update_flow_specific_status():
+    channel = grpc.insecure_channel('localhost:9008')
+    stub = model_service_pb2_grpc.ModelServiceStub(channel)
+    return stub
+
+
 @pytest.fixture(scope="session")
 def create_channel_for_update_flow_specific():
     channel = grpc.insecure_channel('localhost:9008')
@@ -519,7 +547,7 @@ def start_server_model_ver_policy(request, get_image, get_test_dir,
     return container
 
 
-@pytest.fixture(scope="class")
+@pytest.fixture(scope="function")
 def start_server_update_flow_latest(request, get_image, get_test_dir,
                                     get_docker_context):
     client = get_docker_context
@@ -544,7 +572,7 @@ def start_server_update_flow_latest(request, get_image, get_test_dir,
     return container
 
 
-@pytest.fixture(scope="class")
+@pytest.fixture(scope="function")
 def start_server_update_flow_specific(request, get_image, get_test_dir,
                                       get_docker_context):
     client = get_docker_context

tests/functional/test_model_version_policy.py

@@ -14,13 +14,19 @@
 # limitations under the License.
 #
 
+from conftest import get_model_metadata, model_metadata_response, \
+    get_model_status, get_model_status_response_rest
+from tensorflow_serving.apis import get_model_metadata_pb2, \
+    get_model_status_pb2  # noqa
+from ie_serving.models.models_utils import ModelVersionState, ErrorCode, \
+    _ERROR_MESSAGE
+from google.protobuf.json_format import Parse
+
 import sys
 import requests
 import pytest
-from google.protobuf.json_format import Parse
+
 sys.path.append(".")
-from conftest import get_model_metadata, model_metadata_response  # noqa
-from tensorflow_serving.apis import get_model_metadata_pb2  # noqa
 
 
 class TestModelVerPolicy():
@@ -94,6 +100,51 @@ def test_get_model_metadata(self, model_version_policy_models,
                     response = stub.GetModelMetadata(request, 10)
                 assert "Servable not found for request" in str(e.value)
 
+    @pytest.mark.parametrize("model_name, throw_error", [
+        ('all', [False, False, False]),
+        ('specific', [False, True, False]),
+        ('latest', [True, False, False]),
+    ])
+    def test_get_model_status(self, model_version_policy_models,
+                              start_server_model_ver_policy,
+                              create_channel_for_model_ver_pol_server_status,
+                              model_name, throw_error):
+
+        print("Downloaded model files:", model_version_policy_models)
+
+        # Connect to grpc service
+        stub = create_channel_for_model_ver_pol_server_status
+
+        versions = [1, 2, 3]
+        for x in range(len(versions)):
+            request = get_model_status(model_name=model_name,
+                                       version=versions[x])
+            if not throw_error[x]:
+                response = stub.GetModelStatus(request, 10)
+                versions_statuses = response.model_version_status
+                version_status = versions_statuses[0]
+                assert version_status.version == versions[x]
+                assert version_status.state == ModelVersionState.AVAILABLE
+                assert version_status.status.error_code == ErrorCode.OK
+                assert version_status.status.error_message == _ERROR_MESSAGE[
+                    ModelVersionState.AVAILABLE][ErrorCode.OK]
+            else:
+                with pytest.raises(Exception) as e:
+                    response = stub.GetModelStatus(request, 10)
+                assert "Servable not found for request" in str(e.value)
+
+        #   aggregated results check
+        if model_name == 'all':
+            request = get_model_status(model_name=model_name)
+            response = stub.GetModelStatus(request, 10)
+            versions_statuses = response.model_version_status
+            assert len(versions_statuses) == 3
+            for version_status in versions_statuses:
+                assert version_status.state == ModelVersionState.AVAILABLE
+                assert version_status.status.error_code == ErrorCode.OK
+                assert version_status.status.error_message == _ERROR_MESSAGE[
+                    ModelVersionState.AVAILABLE][ErrorCode.OK]
+
     @pytest.mark.parametrize("model_name, throw_error", [
         ('all', [False, False, False]),
         ('specific', [False, True, False]),
@@ -162,3 +213,46 @@ def test_get_model_metadata_rest(self, model_version_policy_models,
                 assert expected_output_metadata == output_metadata
             else:
                 assert 404 == result.status_code
+
+    @pytest.mark.parametrize("model_name, throw_error", [
+        ('all', [False, False, False]),
+        ('specific', [False, True, False]),
+        ('latest', [True, False, False]),
+    ])
+    def test_get_model_status_rest(self, model_version_policy_models,
+                                   start_server_model_ver_policy,
+                                   model_name, throw_error):
+
+        print("Downloaded model files:", model_version_policy_models)
+
+        versions = [1, 2, 3]
+        for x in range(len(versions)):
+            rest_url = 'http://localhost:5560/v1/models/{}/' \
+                       'versions/{}'.format(model_name, versions[x])
+            result = requests.get(rest_url)
+            if not throw_error[x]:
+                output_json = result.text
+                status_pb = get_model_status_pb2.GetModelStatusResponse()
+                response = Parse(output_json, status_pb,
+                                 ignore_unknown_fields=False)
+                versions_statuses = response.model_version_status
+                version_status = versions_statuses[0]
+                assert version_status.version == versions[x]
+                assert version_status.state == ModelVersionState.AVAILABLE
+                assert version_status.status.error_code == ErrorCode.OK
+                assert version_status.status.error_message == _ERROR_MESSAGE[
+                    ModelVersionState.AVAILABLE][ErrorCode.OK]
+            else:
+                assert 404 == result.status_code
+
+                #   aggregated results check
+        if model_name == 'all':
+            rest_url = 'http://localhost:5560/v1/models/all'
+            response = get_model_status_response_rest(rest_url)
+            versions_statuses = response.model_version_status
+            assert len(versions_statuses) == 3
+            for version_status in versions_statuses:
+                assert version_status.state == ModelVersionState.AVAILABLE
+                assert version_status.status.error_code == ErrorCode.OK
+                assert version_status.status.error_message == _ERROR_MESSAGE[
+                    ModelVersionState.AVAILABLE][ErrorCode.OK]

tests/functional/test_model_versions_handling.py

@@ -14,11 +14,16 @@
 # limitations under the License.
 #
 
+from conftest import infer, get_model_metadata, model_metadata_response, \
+    infer_rest, get_model_metadata_response_rest, get_model_status, \
+    get_model_status_response_rest
+from ie_serving.models.models_utils import ModelVersionState, _ERROR_MESSAGE, \
+    ErrorCode
 import numpy as np
 import sys
+
+
 sys.path.append(".")
-from conftest import infer, get_model_metadata, model_metadata_response,\
-    infer_rest, get_model_metadata_response_rest  # noqa
 
 
 class TestModelVersionHandling():
@@ -80,30 +85,11 @@ def test_run_inference(self, download_two_model_versions,
             assert output[out_name_v1].shape == (1, 1000),\
                 'resnet model with latest version has invalid output'
 
-    def test_get_model_metadata(self, download_two_models,
+    def test_get_model_metadata(self, download_two_model_versions,
                                 start_server_multi_model,
                                 create_channel_for_port_multi_server):
-        """
-        <b>Description</b>
-        Execute inference request using gRPC interface hosting multiple models
-
-        <b>input data</b>
-        - directory with 2 models in IR format
-        - docker image
-
-        <b>fixtures used</b>
-        - model downloader
-        - input data downloader
-        - service launching
 
-        <b>Expected results</b>
-        - response contains proper response about model metadata for both
-        models set in config file:
-        model resnet_v1_50, pnasnet_large
-        - both served models handles appropriate input formats
-
-        """
-        print("Downloaded model files:", download_two_models)
+        print("Downloaded model files:", download_two_model_versions)
 
         # Connect to grpc service
         stub = create_channel_for_port_multi_server
@@ -133,6 +119,32 @@ def test_get_model_metadata(self, download_two_models,
             assert expected_input_metadata == input_metadata
             assert expected_output_metadata == output_metadata
 
+    def test_get_model_status(self, download_two_model_versions,
+                              start_server_multi_model,
+                              create_channel_for_port_multi_server_status):
+
+        print("Downloaded model files:", download_two_model_versions)
+
+        # Connect to grpc service
+        stub = create_channel_for_port_multi_server_status
+        versions = [None, 1]
+        for x in range(len(versions)):
+            model_name = 'resnet'
+            request = get_model_status(model_name=model_name,
+                                       version=versions[x])
+            response = stub.GetModelStatus(request, 10)
+
+            versions_statuses = response.model_version_status
+            version_status = versions_statuses[0]
+            if x == 0:
+                assert len(versions_statuses) == 2
+            else:
+                assert version_status.version == 1
+            assert version_status.state == ModelVersionState.AVAILABLE
+            assert version_status.status.error_code == ErrorCode.OK
+            assert version_status.status.error_message == _ERROR_MESSAGE[
+                ModelVersionState.AVAILABLE][ErrorCode.OK]
+
     def test_run_inference_rest(self, download_two_model_versions,
                                 input_data_downloader_v1_224,
                                 start_server_multi_model):
@@ -188,30 +200,10 @@ def test_run_inference_rest(self, download_two_model_versions,
             assert output[out_name_v1].shape == (1, 1000), \
                 'resnet model with latest version has invalid output'
 
-    def test_get_model_metadata_rest(self, download_two_models,
+    def test_get_model_metadata_rest(self, download_two_model_versions,
                                      start_server_multi_model):
-        """
-        <b>Description</b>
-        Execute inference request using REST API interface hosting multiple
-        models
-
-        <b>input data</b>
-        - directory with 2 models in IR format
-        - docker image
-
-        <b>fixtures used</b>
-        - model downloader
-        - input data downloader
-        - service launching
 
-        <b>Expected results</b>
-        - response contains proper response about model metadata for both
-        models set in config file:
-        model resnet_v1_50, pnasnet_large
-        - both served models handles appropriate input formats
-
-        """
-        print("Downloaded model files:", download_two_models)
+        print("Downloaded model files:", download_two_model_versions)
 
         urls = ['http://localhost:5561/v1/models/resnet/metadata',
                 'http://localhost:5561/v1/models/resnet/versions/1/metadata']
@@ -237,3 +229,24 @@ def test_get_model_metadata_rest(self, download_two_models,
             assert model_name == response.model_spec.name
             assert expected_input_metadata == input_metadata
             assert expected_output_metadata == output_metadata
+
+    def test_get_model_status_rest(self, download_two_model_versions,
+                                   start_server_multi_model):
+
+        print("Downloaded model files:", download_two_model_versions)
+
+        urls = ['http://localhost:5561/v1/models/resnet',
+                'http://localhost:5561/v1/models/resnet/versions/1']
+
+        for x in range(len(urls)):
+            response = get_model_status_response_rest(urls[x])
+            versions_statuses = response.model_version_status
+            version_status = versions_statuses[0]
+            if x == 0:
+                assert len(versions_statuses) == 2
+            else:
+                assert version_status.version == 1
+            assert version_status.state == ModelVersionState.AVAILABLE
+            assert version_status.status.error_code == ErrorCode.OK
+            assert version_status.status.error_message == _ERROR_MESSAGE[
+                ModelVersionState.AVAILABLE][ErrorCode.OK]