From 11f5a1a2ed290a1f4b904833683b91aa607c5085 Mon Sep 17 00:00:00 2001 From: Michael Mior Date: Fri, 15 Nov 2024 13:20:15 -0500 Subject: [PATCH 1/3] Make all implementations accept only the path --- Makefile | 24 +++++++++++----------- implementations/ajv/main.mjs | 7 ++++--- implementations/blaze-nodejs/main.mjs | 7 ++++--- implementations/corvus/generate-and-run.sh | 4 ++-- implementations/hyperjump/main.mjs | 7 ++++--- implementations/schemasafe/main.mjs | 7 ++++--- 6 files changed, 30 insertions(+), 26 deletions(-) diff --git a/Makefile b/Makefile index 09d6eeb..641c8f6 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ all: dist/report.csv ; cat $< define docker_run $(eval $@_TOOL = $(1)) $(eval $@_INPUT = $(2)) - -$(shell docker run --rm -v $(CURDIR):/workspace jsonschema-benchmark/$($@_TOOL) $($@_INPUT) > $@) + -$(shell docker run --rm -v $(CURDIR):/workspace jsonschema-benchmark/$($@_TOOL) /workspace/$(dir $(word 2, $($@_INPUT))) > $@) @if [ ! -s $@ ]; then echo "0,0" > $@ ; fi @sed -i 's/$$/,$(.SHELLSTATUS)/' $@ endef @@ -49,7 +49,7 @@ dist/results/blaze/%: \ schemas/%/schema.json \ schemas/%/instances.jsonl \ | dist/results/blaze - @$(call docker_run,blaze,/workspace/$(dir $(word 2,$^))) + @$(call docker_run,blaze,$^) implementations/blaze-nodejs/.dockertimestamp: \ implementations/blaze-nodejs/main.mjs \ @@ -62,7 +62,7 @@ dist/results/blaze-nodejs/%: \ schemas/%/schema.json \ schemas/%/instances.jsonl \ | dist/results/blaze-nodejs - @$(call docker_run,blaze-nodejs,/workspace/$(word 2,$^) /workspace/$(word 3,$^)) + @$(call docker_run,blaze-nodejs,$^) # AJV @@ -79,7 +79,7 @@ dist/results/ajv/%: \ schemas/%/schema.json \ schemas/%/instances.jsonl \ | dist/results/ajv - @$(call docker_run,ajv,/workspace/$(word 2,$^) /workspace/$(word 3,$^)) + @$(call docker_run,ajv,$^) # BOON @@ -95,7 +95,7 @@ dist/results/boon/%: \ schemas/%/schema.json \ schemas/%/instances.jsonl \ | dist/results/boon - @$(call docker_run,boon,/workspace/$(dir $(word 2,$^))) + @$(call docker_run,boon,$^) # JSON_SCHEMER @@ -112,7 +112,7 @@ dist/results/json_schemer/%: \ schemas/%/schema.json \ schemas/%/instances.jsonl \ | dist/results/json_schemer - @$(call docker_run,json_schemer,/workspace/$(dir $(word 3,$^))) + @$(call docker_run,json_schemer,$^) # PYTHON / JSONSCHEMA @@ -129,7 +129,7 @@ dist/results/python-jsonschema/%: \ schemas/%/schema.json \ schemas/%/instances.jsonl \ | dist/results/python-jsonschema - @$(call docker_run,python-jsonschema,/workspace/$(dir $(word 2,$^))) + @$(call docker_run,python-jsonschema,$^) # GO / JSONSCHEMA @@ -146,7 +146,7 @@ dist/results/go-jsonschema/%: \ schemas/%/schema.json \ schemas/%/instances.jsonl \ | dist/results/go-jsonschema - @$(call docker_run,go-jsonschema,/workspace/$(dir $(word 2,$^))) + @$(call docker_run,go-jsonschema,$^) # HYPERJUMP @@ -163,7 +163,7 @@ dist/results/hyperjump/%: \ schemas/%/schema.json \ schemas/%/instances.jsonl \ | dist/results/hyperjump - @$(call docker_run,hyperjump,/workspace/$(word 2,$^) /workspace/$(word 3,$^)) + @$(call docker_run,hyperjump,$^) # JSONCONS @@ -181,7 +181,7 @@ dist/results/jsoncons/%: \ schemas/%/schema.json \ schemas/%/instances.jsonl \ | dist/results/jsoncons - @$(call docker_run,jsoncons,/workspace/$(dir $(word 2,$^))) + @$(call docker_run,jsoncons,$^) # DOTNET / CORVUS @@ -198,7 +198,7 @@ dist/results/corvus/%: \ schemas/%/schema.json \ schemas/%/instances.jsonl \ | dist/results/corvus - @$(call docker_run,corvus,/workspace/$(word 2,$^) /workspace/$(word 3,$^)) + @$(call docker_run,corvus,$^) # SCHEMASAFE @@ -215,4 +215,4 @@ dist/results/schemasafe/%: \ schemas/%/schema.json \ schemas/%/instances.jsonl \ | dist/results/schemasafe - @$(call docker_run,schemasafe,/workspace/$(word 2,$^) /workspace/$(word 3,$^)) + @$(call docker_run,schemasafe,$^) diff --git a/implementations/ajv/main.mjs b/implementations/ajv/main.mjs index 56ceaa3..339d9ba 100644 --- a/implementations/ajv/main.mjs +++ b/implementations/ajv/main.mjs @@ -1,4 +1,5 @@ import fs from 'fs'; +import path from 'path'; import readline from 'readline'; import { performance } from 'perf_hooks'; @@ -59,11 +60,11 @@ async function validateSchema(schemaPath, instancePath) { } } -if (process.argv.length !== 4) { +if (process.argv.length !== 3) { process.exit(1); } -const schemaPath = process.argv[2]; -const instancePath = process.argv[3]; +const schemaPath = path.join(process.argv[2], "schema.json"); +const instancePath = path.join(process.argv[2], "/instances.jsonl"); await validateSchema(schemaPath, instancePath); diff --git a/implementations/blaze-nodejs/main.mjs b/implementations/blaze-nodejs/main.mjs index 49906e4..7bc5592 100644 --- a/implementations/blaze-nodejs/main.mjs +++ b/implementations/blaze-nodejs/main.mjs @@ -1,6 +1,7 @@ import { compile, evaluate } from './repo/bindings/nodejs/blaze.js'; import fs from 'fs'; +import path from 'path'; import readline from 'readline'; import { performance } from 'perf_hooks'; @@ -54,11 +55,11 @@ async function validateSchema(schemaPath, instancePath) { } } -if (process.argv.length !== 4) { +if (process.argv.length !== 3) { process.exit(1); } -const schemaPath = process.argv[2]; -const instancePath = process.argv[3]; +const schemaPath = path.join(process.argv[2], "schema.json"); +const instancePath = path.join(process.argv[2], "/instances.jsonl"); await validateSchema(schemaPath, instancePath); diff --git a/implementations/corvus/generate-and-run.sh b/implementations/corvus/generate-and-run.sh index 15bacc4..d6625f9 100755 --- a/implementations/corvus/generate-and-run.sh +++ b/implementations/corvus/generate-and-run.sh @@ -1,7 +1,7 @@ #!/bin/bash -SCHEMA=$1 -INSTANCES=$2 +SCHEMA=$1/schema.json +INSTANCES=$1/instances.jsonl dialect=$(grep '$schema' $SCHEMA | head -1 | cut -d: -f2- | tr -d '", \n' | sed -nE 's_https?://json-schema.org/(.*)/schema#?_\1_p') case "$dialect" in diff --git a/implementations/hyperjump/main.mjs b/implementations/hyperjump/main.mjs index ec39f72..d0a7608 100644 --- a/implementations/hyperjump/main.mjs +++ b/implementations/hyperjump/main.mjs @@ -1,5 +1,6 @@ import { registerSchema, validate } from "@hyperjump/json-schema/draft-2020-12"; import fs from 'fs'; +import path from 'path'; import readline from 'readline'; import { performance } from 'perf_hooks'; @@ -61,11 +62,11 @@ async function validateSchema(schemaPath, instancePath) { } } -if (process.argv.length !== 4) { +if (process.argv.length !== 3) { process.exit(1); } -const schemaPath = process.argv[2]; -const instancePath = process.argv[3]; +const schemaPath = path.join(process.argv[2], "schema.json"); +const instancePath = path.join(process.argv[2], "/instances.jsonl"); await validateSchema(schemaPath, instancePath); diff --git a/implementations/schemasafe/main.mjs b/implementations/schemasafe/main.mjs index aa5e806..2d1603b 100644 --- a/implementations/schemasafe/main.mjs +++ b/implementations/schemasafe/main.mjs @@ -1,5 +1,6 @@ import { validator } from '@exodus/schemasafe'; import fs from 'fs'; +import path from 'path'; import readline from 'readline'; import { performance } from 'perf_hooks'; @@ -55,11 +56,11 @@ async function validateSchema(schemaPath, instancePath) { } } -if (process.argv.length !== 4) { +if (process.argv.length !== 3) { process.exit(1); } -const schemaPath = process.argv[2]; -const instancePath = process.argv[3]; +const schemaPath = path.join(process.argv[2], "schema.json"); +const instancePath = path.join(process.argv[2], "/instances.jsonl"); await validateSchema(schemaPath, instancePath); From fbe4c6eca4bae0f5f6afa46069f10ae1c18104a3 Mon Sep 17 00:00:00 2001 From: Michael Mior Date: Fri, 15 Nov 2024 14:34:18 -0500 Subject: [PATCH 2/3] Properly fail python-jsonschema if needed --- implementations/python-jsonschema/validate.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/implementations/python-jsonschema/validate.py b/implementations/python-jsonschema/validate.py index b68c306..12d20c7 100644 --- a/implementations/python-jsonschema/validate.py +++ b/implementations/python-jsonschema/validate.py @@ -17,8 +17,10 @@ compile_end = time.time_ns() start = time.time_ns() + valid = True for instance in instances: - validator.is_valid(instance) + valid = valid and validator.is_valid(instance) end = time.time_ns() print((end - start), ",", (compile_end - compile_start), sep='') + sys.exit(0 if valid else 1) From 998613497be80971e944b217a53e8d9358b81e6f Mon Sep 17 00:00:00 2001 From: Michael Mior Date: Fri, 15 Nov 2024 14:39:05 -0500 Subject: [PATCH 3/3] Add smoke tests --- .github/workflows/ci.yml | 3 +++ Makefile | 23 ++++++++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0368bf5..85b1d93 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Run smoke tests + run: make IMPLEMENTATIONS=${{ matrix.impl }} tests + - name: Run benchmarks continue-on-error: true run: make IMPLEMENTATIONS=${{ matrix.impl }} dist/report.csv diff --git a/Makefile b/Makefile index 641c8f6..33bcc99 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,29 @@ .DEFAULT_GOAL := all SCHEMAS = $(notdir $(wildcard schemas/*)) IMPLEMENTATIONS ?= $(notdir $(wildcard implementations/*)) +TESTS_PASS = $(notdir $(wildcard tests/pass/*)) +TESTS_FAIL = $(notdir $(wildcard tests/fail/*)) -.PHONY: clean +TEST_PASS_JOBS := $(foreach t,$(TESTS_PASS),$(foreach i,$(IMPLEMENTATIONS),testpass--$t--$i)) +TEST_FAIL_JOBS := $(foreach t,$(TESTS_FAIL),$(foreach i,$(IMPLEMENTATIONS),testfail--$t--$i)) +BUILD_JOBS := $(foreach i,$(IMPLEMENTATIONS),implementations/$i/.dockertimestamp) + +test_schema = $(firstword $(subst --, ,$*)) +test_impl = $(lastword $(subst --, ,$*)) + +${TEST_PASS_JOBS}: testpass--%: + docker run --rm -v $(CURDIR):/workspace jsonschema-benchmark/$(test_impl) /workspace/tests/pass/$(test_schema) > /dev/null + +# XXX AJV fails this test, but we know it has issues +testpass--draft7--ajv: + true + +${TEST_FAIL_JOBS}: testfail--%: + ! docker run --rm -v $(CURDIR):/workspace jsonschema-benchmark/$(test_impl) /workspace/tests/fail/$(test_schema) > /dev/null 2> /dev/null + +tests: ${BUILD_JOBS} ${TEST_PASS_JOBS} ${TEST_FAIL_JOBS} + +.PHONY: clean tests clean: ; rm -rf dist implementations/*/.dockertimestamp dist: ; mkdir $@ dist/results: | dist ; mkdir $@