Merge pull request #1119 from pq-code-package/acvp-on-the-fly

ACVP: Download testvectors transparently on the fly

Author: hanno-becker

.github/workflows/base.yml

@@ -61,6 +61,31 @@ jobs:
       - name: tests func
         run: |
           ./scripts/tests func --check-namespace
+  quickcheck-acvp:
+    strategy:
+      fail-fast: false
+      matrix:
+        external:
+         - ${{ github.repository_owner != 'pq-code-package' }}
+        target:
+         - runner: pqcp-arm64
+           name: 'aarch64'
+         - runner: ubuntu-latest
+           name: 'x86_64'
+        acvp-version: [v1.1.0.38, v1.1.0.39, v1.1.0.40]
+        exclude:
+          - {external: true,
+             target: {
+               runner: pqcp-arm64,
+               name: 'aarch64'
+             }}
+    name: Quickcheck ACVP (${{ matrix.target.name }}, ${{ matrix.acvp-version }})
+    runs-on: ${{ matrix.target.runner }}
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Run ACVP test
+        run: |
+          ./scripts/tests acvp --version ${{ matrix.acvp-version }}
   quickcheck_bench:
     strategy:
       fail-fast: false

.gitignore

@@ -5,3 +5,5 @@
 .idea
 test/build
 __pycache__/
+# Downloaded ACVP test data
+test/.acvp-data/

BIBLIOGRAPHY.md

@@ -7,24 +7,6 @@
 This file lists the citations made throughout the mlkem-native 
 source code and documentation.
 
-### `ACVP_Server`
-
-* ACVP Test Vectors
-* Author(s):
-  - National Institute of Standards and Technology
-* URL: https://github.com/usnistgov/ACVP-Server/tree/master/gen-val/json-files
-* Referenced from:
-  - [test/acvp_data/README.md](test/acvp_data/README.md)
-
-### `ACVP_Spec`
-
-* ACVP ML-KEM JSON Specification
-* Author(s):
-  - Christopher Celi
-* URL: https://pages.nist.gov/ACVP/draft-celi-acvp-ml-kem.html
-* Referenced from:
-  - [test/acvp_data/README.md](test/acvp_data/README.md)
-
 ### `AVX2_NTT`
 
 * Faster AVX2 optimized NTT multiplication for Ring-LWE lattice cryptography.

BIBLIOGRAPHY.yml

@@ -150,16 +150,6 @@
   name: Cycle counting on Apple M1
   url: https://gist.github.com/dougallj/5bafb113492047c865c0c8cfbc930155#file-m1_robsize-c-L390
 
-- id: ACVP_Server
-  author: National Institute of Standards and Technology
-  name: ACVP Test Vectors
-  url: https://github.com/usnistgov/ACVP-Server/tree/master/gen-val/json-files
-
-- id: ACVP_Spec
-  author: Celi, Christopher
-  name: ACVP ML-KEM JSON Specification
-  url: https://pages.nist.gov/ACVP/draft-celi-acvp-ml-kem.html
-
 - id: tiny_sha3
   name: tiny_sha3
   author: Saarinen, Markku-Juhani O.

Makefile

@@ -50,7 +50,7 @@ run_func_1024: func_1024
 run_func: run_func_512 run_func_768 run_func_1024
 
 run_acvp: acvp
-	python3 ./test/acvp_client.py
+	python3 ./test/acvp_client.py $(if $(ACVP_VERSION),--version $(ACVP_VERSION))
 
 func_512:  $(MLKEM512_DIR)/bin/test_mlkem512
 	$(Q)echo "  FUNC       ML-KEM-512:   $^"

README.md

@@ -50,7 +50,10 @@ See [BUILDING.md](BUILDING.md) for more information.
 
 ## Applications
 
-mlkem-native is used in [libOQS](https://github.com/open-quantum-safe/liboqs/) of the Open Quantum Safe project, and in AWS' Cryptography library [AWS-LC](https://github.com/aws/aws-lc/).
+mlkem-native is used in
+ - [libOQS](https://github.com/open-quantum-safe/liboqs/) of the Open Quantum Safe project since [0.13.0](https://github.com/open-quantum-safe/liboqs/releases/tag/0.13.0) (as the default ML-KEM implementation)
+ - AWS' Cryptography library [AWS-LC](https://github.com/aws/aws-lc/) since [v1.50.0](https://github.com/aws/aws-lc/releases/tag/v1.50.0)
+ - The [rustls](https://github.com/rustls/rustls) TLS library written in Rust since [0.23.28](https://github.com/rustls/rustls/releases/tag/v%2F0.23.28) (through AWS-LC as the default cryptography provider)
 
 ## Formal Verification
 

scripts/tests

@@ -506,6 +506,10 @@ class Tests:
             if stack_flags:
                 env_update["STACK_ANALYSIS_FLAGS"] = " ".join(stack_flags)
 
+        # Add ACVP version for ACVP tests
+        if test_type == TEST_TYPES.ACVP and hasattr(self.args, "version"):
+            env_update["ACVP_VERSION"] = self.args.version
+
         env = os.environ.copy()
         env.update(env_update)
 
@@ -1054,6 +1058,11 @@ def cli():
     acvp_parser = cmd_subparsers.add_parser(
         "acvp", help="Run ACVP client", parents=[common_parser]
     )
+    acvp_parser.add_argument(
+        "--version",
+        default="v1.1.0.40",
+        help="ACVP test vector version (default: v1.1.0.40)",
+    )
 
     # examples arguments
     examples_parser = cmd_subparsers.add_parser(

test/acvp_client.py

@@ -3,49 +3,64 @@
 # SPDX-License-Identifier: Apache-2.0 OR ISC OR MIT
 
 # ACVP client for ML-KEM
-#
-# Processes json files from
-# https://github.com/usnistgov/ACVP-Server/blob/master/gen-val/json-files
-#
+# See https://pages.nist.gov/ACVP/draft-celi-acvp-ml-kem.html and
+# https://github.com/usnistgov/ACVP-Server/tree/master/gen-val/json-files
 # Invokes `acvp_mlkem{lvl}` under the hood.
 
 import argparse
 import os
 import json
 import sys
 import subprocess
+import urllib.request
+from pathlib import Path
 
 # Check if we need to use a wrapper for execution (e.g. QEMU)
 exec_prefix = os.environ.get("EXEC_WRAPPER", "")
 exec_prefix = exec_prefix.split(" ") if exec_prefix != "" else []
 
-acvp_dir = "test/acvp_data"
-acvp_jsons = [
-    (
-        f"{acvp_dir}/acvp_v1.1.0.38_keyGen_prompt.json",
-        f"{acvp_dir}/acvp_v1.1.0.38_keyGen_expectedResults.json",
-    ),
-    (
-        f"{acvp_dir}/acvp_v1.1.0.39_keyGen_prompt.json",
-        f"{acvp_dir}/acvp_v1.1.0.39_keyGen_expectedResults.json",
-    ),
-    (
-        f"{acvp_dir}/acvp_v1.1.0.40_keyGen_prompt.json",
-        f"{acvp_dir}/acvp_v1.1.0.40_keyGen_expectedResults.json",
-    ),
-    (
-        f"{acvp_dir}/acvp_v1.1.0.38_encapDecap_prompt.json",
-        f"{acvp_dir}/acvp_v1.1.0.38_encapDecap_expectedResults.json",
-    ),
-    (
-        f"{acvp_dir}/acvp_v1.1.0.39_encapDecap_prompt.json",
-        f"{acvp_dir}/acvp_v1.1.0.39_encapDecap_expectedResults.json",
-    ),
-    (
-        f"{acvp_dir}/acvp_v1.1.0.40_encapDecap_prompt.json",
-        f"{acvp_dir}/acvp_v1.1.0.40_encapDecap_expectedResults.json",
-    ),
-]
+
+def download_acvp_files(version="v1.1.0.40"):
+    """Download ACVP test files for the specified version if not present."""
+    base_url = f"https://raw.githubusercontent.com/usnistgov/ACVP-Server/{version}/gen-val/json-files"
+
+    # Files we need to download for ML-KEM
+    files_to_download = [
+        "ML-KEM-keyGen-FIPS203/prompt.json",
+        "ML-KEM-keyGen-FIPS203/expectedResults.json",
+        "ML-KEM-encapDecap-FIPS203/prompt.json",
+        "ML-KEM-encapDecap-FIPS203/expectedResults.json",
+    ]
+
+    # Create directory structure
+    data_dir = Path(f"test/.acvp-data/{version}/files")
+    data_dir.mkdir(parents=True, exist_ok=True)
+
+    for file_path in files_to_download:
+        local_file = data_dir / file_path
+        local_file.parent.mkdir(parents=True, exist_ok=True)
+
+        if not local_file.exists():
+            url = f"{base_url}/{file_path}"
+            print(f"Downloading {file_path}...", file=sys.stderr)
+            try:
+                urllib.request.urlretrieve(url, local_file)
+                # Verify the file is valid JSON
+                with open(local_file, "r") as f:
+                    json.load(f)
+            except json.JSONDecodeError as e:
+                print(
+                    f"Error: Downloaded file {file_path} is not valid JSON: {e}",
+                    file=sys.stderr,
+                )
+                local_file.unlink(missing_ok=True)
+                return False
+            except Exception as e:
+                print(f"Error downloading {file_path}: {e}", file=sys.stderr)
+                local_file.unlink(missing_ok=True)
+                return False
+
+    return True
 
 
 def loadAcvpData(prompt, expectedResults):
@@ -59,9 +74,21 @@ def loadAcvpData(prompt, expectedResults):
     return (prompt, promptData, expectedResults, expectedResultsData)
 
 
-def loadDefaultAcvpData():
+def loadDefaultAcvpData(version="v1.1.0.40"):
+
+    data_dir = f"test/.acvp-data/{version}/files"
+    acvp_jsons_for_version = [
+        (
+            f"{data_dir}/ML-KEM-keyGen-FIPS203/prompt.json",
+            f"{data_dir}/ML-KEM-keyGen-FIPS203/expectedResults.json",
+        ),
+        (
+            f"{data_dir}/ML-KEM-encapDecap-FIPS203/prompt.json",
+            f"{data_dir}/ML-KEM-encapDecap-FIPS203/expectedResults.json",
+        ),
+    ]
     acvp_data = []
-    for prompt, expectedResults in acvp_jsons:
+    for prompt, expectedResults in acvp_jsons_for_version:
         acvp_data.append(loadAcvpData(prompt, expectedResults))
     return acvp_data
 
@@ -280,7 +307,7 @@ def runTest(data, output):
     info("ALL GOOD!")
 
 
-def test(prompt, expected, output):
+def test(prompt, expected, output, version="v1.1.0.40"):
     assert (
         prompt is not None or output is None
     ), "cannot produce output if there is no input"
@@ -293,8 +320,8 @@ def test(prompt, expected, output):
     if prompt is not None:
         data = [loadAcvpData(prompt, expected)]
     else:
-        # otherwise, load default data from acvp_data
-        data = loadDefaultAcvpData()
+        # load data from downloaded files
+        data = loadDefaultAcvpData(version)
 
     runTest(data, output)
 
@@ -312,5 +339,20 @@ def test(prompt, expected, output):
 parser.add_argument(
     "-o", "--output", help="Path to output file in json format", required=False
 )
+parser.add_argument(
+    "--version",
+    "-v",
+    default="v1.1.0.40",
+    help="ACVP test vector version (default: v1.1.0.40)",
+)
 args = parser.parse_args()
-test(args.prompt, args.expected, args.output)
+
+if args.prompt is None:
+    print(f"Using ACVP test vectors version {args.version}", file=sys.stderr)
+
+    # Download files if needed
+    if not download_acvp_files(args.version):
+        print("Failed to download ACVP test files", file=sys.stderr)
+        sys.exit(1)
+
+test(args.prompt, args.expected, args.output, args.version)