diff --git a/test/fixtures/xctest_runner/BUILD b/test/fixtures/xctest_runner/BUILD index d9f5f1d63..b3291480f 100644 --- a/test/fixtures/xctest_runner/BUILD +++ b/test/fixtures/xctest_runner/BUILD @@ -26,3 +26,21 @@ swift_test( name = "EmptyUnitTests", tags = FIXTURE_TAGS, ) + +swift_test( + name = "FailingSwiftTestingTests", + srcs = ["FailingSwiftTestingTests.swift"], + tags = FIXTURE_TAGS, +) + +swift_test( + name = "EmptySwiftTestingSuite", + srcs = ["EmptySwiftTestingSuite.swift"], + tags = FIXTURE_TAGS, +) + +swift_test( + name = "PassingSwiftTestingTests", + srcs = ["PassingSwiftTestingTests.swift"], + tags = FIXTURE_TAGS, +) diff --git a/test/fixtures/xctest_runner/EmptySwiftTestingSuite.swift b/test/fixtures/xctest_runner/EmptySwiftTestingSuite.swift new file mode 100644 index 000000000..adcda7768 --- /dev/null +++ b/test/fixtures/xctest_runner/EmptySwiftTestingSuite.swift @@ -0,0 +1,6 @@ +import Testing + +@Suite("Empty test suite") +class EmptyTestSuite { + +} \ No newline at end of file diff --git a/test/fixtures/xctest_runner/FailingSwiftTestingTests.swift b/test/fixtures/xctest_runner/FailingSwiftTestingTests.swift new file mode 100644 index 000000000..c4bf3f7a7 --- /dev/null +++ b/test/fixtures/xctest_runner/FailingSwiftTestingTests.swift @@ -0,0 +1,6 @@ +import Testing + +@Test("A failing test") +func aFalingTest(){ + #expect(false, "This assertion will always fail") +} diff --git a/test/fixtures/xctest_runner/PassingSwiftTestingTests.swift b/test/fixtures/xctest_runner/PassingSwiftTestingTests.swift new file mode 100644 index 000000000..5cca7e61a --- /dev/null +++ b/test/fixtures/xctest_runner/PassingSwiftTestingTests.swift @@ -0,0 +1,6 @@ +import Testing + +@Test("A passing test") +func aPassingTest(){ + #expect(true, "This assertion should pass") +} diff --git a/test/rules/swift_shell_runner.sh.template b/test/rules/swift_shell_runner.sh.template index 37d90dc6d..da997d18f 100755 --- a/test/rules/swift_shell_runner.sh.template +++ b/test/rules/swift_shell_runner.sh.template @@ -31,7 +31,7 @@ set -euxo pipefail executable="%executable%" expected_return_code="%expected_return_code%" expected_logs=%expected_logs% -not_expected_logs=%expected_logs% +not_expected_logs=%not_expected_logs% # execute the target under test while recording its outputs and return code tmp_dir=$(mktemp -d) diff --git a/test/xctest_runner_tests.bzl b/test/xctest_runner_tests.bzl index 93156d4a4..a6cb2520b 100644 --- a/test/xctest_runner_tests.bzl +++ b/test/xctest_runner_tests.bzl @@ -22,6 +22,7 @@ def xctest_runner_test_suite(name, tags = []): "Test Suite 'PassingUnitTests.xctest' passed", "Executed 3 tests, with 0 failures", ], + not_expected_logs = ["error: no tests were executed, is the test bundle empty"], tags = all_tags, target_under_test = "@build_bazel_rules_swift//test/fixtures/xctest_runner:PassingUnitTests", target_compatible_with = ["@platforms//os:macos"], @@ -35,6 +36,7 @@ def xctest_runner_test_suite(name, tags = []): "Test Suite 'FailingUnitTests.xctest' failed", "Executed 1 test, with 1 failure", ], + not_expected_logs = ["error: no tests were executed, is the test bundle empty"], tags = all_tags, target_under_test = "@build_bazel_rules_swift//test/fixtures/xctest_runner:FailingUnitTests", target_compatible_with = ["@platforms//os:macos"], @@ -52,6 +54,40 @@ def xctest_runner_test_suite(name, tags = []): target_compatible_with = ["@platforms//os:macos"], ) + swift_shell_test( + name = "{}_swift_testing_no_tests".format(name), + expected_return_code = 1, + expected_logs = [ + "Test run with 0 tests passed after", + "error: no tests were executed", + ], + tags = all_tags, + target_under_test = "@build_bazel_rules_swift//test/fixtures/xctest_runner:EmptySwiftTestingSuite", + target_compatible_with = ["@platforms//os:macos"], + ) + + swift_shell_test( + name = "{}_swift_testing_pass".format(name), + expected_return_code = 0, + expected_logs = [ + "Test run with 1 test passed after", + ], + not_expected_logs = ["error: no tests were executed, is the test bundle empty"], + tags = all_tags, + target_under_test = "@build_bazel_rules_swift//test/fixtures/xctest_runner:PassingSwiftTestingTests", + ) + + swift_shell_test( + name = "{}_swift_testing_fail".format(name), + expected_return_code = 1, + expected_logs = [ + "Test run with 1 test failed after", + ], + not_expected_logs = ["error: no tests were executed, is the test bundle empty"], + tags = all_tags, + target_under_test = "@build_bazel_rules_swift//test/fixtures/xctest_runner:FailingSwiftTestingTests", + ) + native.test_suite( name = name, tags = all_tags, diff --git a/tools/xctest_runner/xctest_runner.sh.template b/tools/xctest_runner/xctest_runner.sh.template index 746c2d0f6..a7429c731 100644 --- a/tools/xctest_runner/xctest_runner.sh.template +++ b/tools/xctest_runner/xctest_runner.sh.template @@ -66,9 +66,21 @@ if [[ "$exit_status" -ne 0 ]]; then fi # Fail when bundle executes nothing -test_target_execution_count=$(grep -e "Executed [[:digit:]]\{1,\} tests*," "$testlog" | tail -n1) -if echo "$test_target_execution_count" | grep -q -e "Executed 0 tests, with 0 failures"; then - echo "error: no tests were executed, is the test bundle empty?" >&2 +xctest_target_execution_count=$(grep -e "Executed [[:digit:]]\{1,\} tests*," "$testlog" | tail -n1) +swift_testing_target_execution_count=$(grep -e "Test run with [[:digit:]]\{1,\} tests*" "$testlog" | tail -n1 || true) +error_msg="error: no tests were executed, is the test bundle empty?" +echo "xctest_target_execution_count: $xctest_target_execution_count" +echo "swift_testing_target_execution_count: $swift_testing_target_execution_count" + +if echo "$xctest_target_execution_count" | grep -q -e "Executed 0 tests, with 0 failures" && \ + [ -z "$swift_testing_target_execution_count" ] ; then + echo "$error_msg" >&2 + exit 1 +fi + +if echo "$xctest_target_execution_count" | grep -q -e "Executed 0 tests, with 0 failures" && \ + echo "$swift_testing_target_execution_count" | grep -q -e "Test run with 0 tests" ; then + echo "$error_msg" >&2 exit 1 fi