Skip to content

Commit d98fdef

Browse files
committed
Mark failing tests with pytest.xfail
1 parent a8636b3 commit d98fdef

File tree

3 files changed

+36
-66
lines changed

3 files changed

+36
-66
lines changed

pyproject.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ packages = ["elasticsearch_serverless"]
101101
[tool.pytest]
102102
junit_family = "legacy"
103103
addopts = "-vvv -p no:logging --cov-report=term-missing --cov=elasticsearch_serverless --cov-config=.pyproject.toml"
104+
xfail_strict=true
104105

105106
[tool.isort]
106107
profile = "black"

test_elasticsearch_serverless/test_async/test_server/test_rest_api_spec.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,7 @@ def async_runner(async_client):
249249

250250
@pytest.mark.parametrize("test_spec", YAML_TEST_SPECS)
251251
async def test_rest_api_spec(test_spec, async_runner):
252-
if test_spec.get("skip", False):
253-
pytest.skip("Manually skipped in 'SKIP_TESTS'")
252+
if test_spec.get("fail", False):
253+
pytest.xfail("Manually marked as failing in 'FAILING_TESTS'")
254254
async_runner.use_spec(test_spec)
255255
await async_runner.run()

test_elasticsearch_serverless/test_server/test_rest_api_spec.py

Lines changed: 33 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -76,61 +76,30 @@
7676
}
7777

7878
# broken YAML tests on some releases
79-
SKIP_TESTS = {
80-
# Warning about date_histogram.interval deprecation is raised randomly
81-
"search/aggregation/250_moving_fn[1]",
82-
# body: null
83-
"indices/simulate_index_template/10_basic[2]",
84-
# No ML node with sufficient capacity / random ML failing
85-
"ml/start_stop_datafeed",
86-
"ml/post_data",
87-
"ml/jobs_crud",
88-
"ml/datafeeds_crud",
89-
"ml/set_upgrade_mode",
90-
"ml/reset_job[2]",
91-
"ml/jobs_get_stats",
92-
"ml/get_datafeed_stats",
93-
"ml/get_trained_model_stats",
94-
"ml/delete_job_force",
95-
"ml/jobs_get_result_overall_buckets",
96-
"ml/bucket_correlation_agg[0]",
97-
"ml/job_groups",
98-
"transform/transforms_stats_continuous[0]",
99-
# Fails bad request instead of 404?
100-
"ml/inference_crud",
101-
# rollup/security_tests time out?
102-
"rollup/security_tests",
103-
# Our TLS certs are custom
104-
"ssl/10_basic[0]",
105-
# Our user is custom
106-
"users/10_basic[3]",
107-
# License warning not sent?
108-
"license/30_enterprise_license[0]",
109-
# Shards/snapshots aren't right?
110-
"searchable_snapshots/10_usage[1]",
111-
# flaky data streams?
112-
"data_stream/10_basic[1]",
113-
"data_stream/80_resolve_index_data_streams[1]",
114-
# bad formatting?
115-
"cat/allocation/10_basic",
116-
"runtime_fields/10_keyword[8]",
117-
# service account number not right?
118-
"service_accounts/10_basic[1]",
119-
# doesn't use 'contains' properly?
120-
"xpack/10_basic[0]",
121-
"privileges/40_get_user_privs[0]",
122-
"privileges/40_get_user_privs[1]",
123-
"features/get_features/10_basic[0]",
124-
"features/reset_features/10_basic[0]",
125-
# bad use of 'is_false'?
126-
"indices/get_alias/10_basic[22]",
127-
# unique usage of 'set'
128-
"indices/stats/50_disk_usage[0]",
129-
"indices/stats/60_field_usage[0]",
130-
# actual Elasticsearch failure?
131-
"transform/transforms_stats",
132-
"transform/transforms_cat_apis",
133-
"transform/transforms_update",
79+
FAILING_TESTS = {
80+
# TODO: bulk call in setup fails due to "malformed action/metadata line"
81+
# bulk body is being sent as a Buffer, unsure if related.
82+
"transform/10_basic",
83+
# TODO: wait_for_active_shards and rollover with conditions are not supported on serverless
84+
# see https://github.com/elastic/elasticsearch-clients-tests/issues/55
85+
"indices/rollover",
86+
# TODO: test runner needs to support ignoring 410 errors
87+
"indices/data_lifecycle",
88+
# TODO: test runner needs to support ignoring 410 errors
89+
"enrich/10_basic.yml",
90+
# TODO: parameter `enabled` is not allowed in source
91+
# Same underlying problem as https://github.com/elastic/elasticsearch-clients-tests/issues/55
92+
"cluster/component_templates",
93+
# TODO: expecting `ct_field` field mapping to be returned, but instead only finds `field`
94+
"indices/simulate_template",
95+
# Fixed by https://github.com/elastic/elasticsearch-clients-tests/pull/56
96+
"cat/aliases",
97+
"cat/component_templates",
98+
"cat/count",
99+
"cat/help",
100+
"cat/indices",
101+
"cat/ml",
102+
"cat/transform",
134103
}
135104

136105

@@ -578,11 +547,11 @@ def remove_implicit_resolver(cls, tag_to_remove):
578547
# Now we combine setup, teardown, and test_steps into
579548
# a set of pytest.param() instances
580549
for test_number, test_step in test_numbers_and_steps:
581-
# Build the id from the name of the YAML file and
582-
# the number within that file. Most important step
583-
# is to remove most of the file path prefixes and
584-
# the .yml suffix.
585-
pytest_test_name = yaml_file.rpartition(".")[0].replace(".", "/")
550+
# Build the id from the name of the YAML file and the number within
551+
# that file. Most important step is to remove most of the file path
552+
# prefixes and the .yml suffix.
553+
test_path = "/".join(yaml_file.split("/")[2:])
554+
pytest_test_name = test_path.rpartition(".")[0].replace(".", "/")
586555
for prefix in ("rest-api-spec/", "test/", "free/", "platinum/"):
587556
if pytest_test_name.startswith(prefix):
588557
pytest_test_name = pytest_test_name[len(prefix) :]
@@ -594,8 +563,8 @@ def remove_implicit_resolver(cls, tag_to_remove):
594563
"teardown": teardown_steps,
595564
}
596565
# Skip either 'test_name' or 'test_name[x]'
597-
if pytest_test_name in SKIP_TESTS or pytest_param_id in SKIP_TESTS:
598-
pytest_param["skip"] = True
566+
if pytest_test_name in FAILING_TESTS or pytest_param_id in FAILING_TESTS:
567+
pytest_param["fail"] = True
599568

600569
YAML_TEST_SPECS.append(pytest.param(pytest_param, id=pytest_param_id))
601570

@@ -615,7 +584,7 @@ def _pytest_param_sort_key(param: pytest.param) -> Tuple[Union[str, int], ...]:
615584

616585
@pytest.mark.parametrize("test_spec", YAML_TEST_SPECS)
617586
def test_rest_api_spec(test_spec, sync_runner):
618-
if test_spec.get("skip", False):
619-
pytest.skip("Manually skipped in 'SKIP_TESTS'")
587+
if test_spec.get("fail", False):
588+
pytest.xfail("Manually marked as failing in 'FAILING_TESTS'")
620589
sync_runner.use_spec(test_spec)
621590
sync_runner.run()

0 commit comments

Comments
 (0)