From d1b33b0c9a5727d01cf5a3b10410175beb885b16 Mon Sep 17 00:00:00 2001 From: willieyz Date: Wed, 18 Jun 2025 14:31:25 +0800 Subject: [PATCH] bench: integrate size measurement into /scripts/tests This commit adds the ability to measure code size for tests and examples: - For tests: run `./scripts/tests size` - For examples: run `./scripts/tests examples -l --verbose` The function is implemented using the `size` command: - Adds the size command and corresponding Makefile configuration for both tests and each example. - Filter out empty size results to keep the output clean. - Sort results in descending order by size to enhance readability. Signed-off-by: willieyz --- Makefile | 28 ++++++++++++++- examples/basic/Makefile | 17 +++++++-- examples/bring_your_own_fips202/Makefile | 17 +++++++-- examples/custom_backend/Makefile | 17 +++++++-- examples/monolithic_build/Makefile | 15 ++++++-- examples/monolithic_build_multilevel/Makefile | 15 ++++++-- .../Makefile | 15 ++++++-- examples/monolithic_build_native/Makefile | 16 +++++++-- examples/multilevel_build/Makefile | 19 ++++++++-- examples/multilevel_build_native/Makefile | 19 ++++++++-- scripts/tests | 35 +++++++++++++++++++ 11 files changed, 193 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 376fa2c28..e6938756f 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,9 @@ bench_components_512 bench_components_768 bench_components_1024 bench_components \ run_bench_components_512 run_bench_components_768 run_bench_components_1024 run_bench_components \ build test all \ - clean quickcheck check-defined-CYCLES + clean quickcheck check-defined-CYCLES \ + size_512 size_768 size_1024 size \ + run_size_512 run_size_768 run_size_1024 run_size .DEFAULT_GOAL := build all: build @@ -142,6 +144,30 @@ run_bench_components: \ run_bench_components_768 .WAIT\ run_bench_components_1024 + +size_512: $(BUILD_DIR)/libmlkem512.a +size_768: $(BUILD_DIR)/libmlkem768.a +size_1024: $(BUILD_DIR)/libmlkem1024.a +size: size_512 size_768 size_1024 + +run_size_512: size_512 + $(Q)echo "size $(BUILD_DIR)/libmlkem512.a" + $(Q)$(SIZE) $(BUILD_DIR)/libmlkem512.a | (read header; echo "$$header"; awk '$$5 != 0' | sort -k5 -n -r) + +run_size_768: size_768 + $(Q)echo "size $(BUILD_DIR)/libmlkem768.a" + $(Q)$(SIZE) $(BUILD_DIR)/libmlkem768.a | (read header; echo "$$header"; awk '$$5 != 0' | sort -k5 -n -r) + +run_size_1024: size_1024 + $(Q)echo "size $(BUILD_DIR)/libmlkem1024.a" + $(Q)$(SIZE) $(BUILD_DIR)/libmlkem1024.a | (read header; echo "$$header"; awk '$$5 != 0' | sort -k5 -n -r) + + +run_size: \ + run_size_512 \ + run_size_768 \ + run_size_1024 + clean: -$(RM) -rf *.gcno *.gcda *.lcov *.o *.so -$(RM) -rf $(BUILD_DIR) diff --git a/examples/basic/Makefile b/examples/basic/Makefile index 683535085..ce5e34953 100644 --- a/examples/basic/Makefile +++ b/examples/basic/Makefile @@ -1,16 +1,21 @@ # (SPDX-License-Identifier: CC-BY-4.0) -.PHONY: build run clean +.PHONY: build run clean size .DEFAULT_GOAL := all # Append cross-prefix for cross compilation # Remove or ignore for native builds CC ?= gcc +SIZE ?= size # When called from the root Makefile, CROSS_PREFIX has already been added here ifeq (,$(findstring $(CROSS_PREFIX),$(CC))) CC := $(CROSS_PREFIX)$(CC) endif +ifeq (,$(findstring $(CROSS_PREFIX),$(SIZE))) +SIZE := $(CROSS_PREFIX)$(SIZE) +endif + # Part A: # # mlkem-native source and header files @@ -80,7 +85,7 @@ $(BINARIES_FULL): $(ALL_SOURCE) mkdir -p $(BUILD_DIR) $(CC) $(CFLAGS) $^ -o $@ -all: build +all: build size build: $(BINARIES_FULL) @@ -89,5 +94,13 @@ run: $(BINARIES_FULL) $(EXEC_WRAPPER) ./$(BINARY_NAME_FULL_768) $(EXEC_WRAPPER) ./$(BINARY_NAME_FULL_1024) +size: build + @echo "=== Size info for $(BINARY_NAME_FULL_512) ===" + @$(SIZE) $(BINARY_NAME_FULL_512) + @echo "=== Size info for $(BINARY_NAME_FULL_768) ===" + @$(SIZE) $(BINARY_NAME_FULL_768) + @echo "=== Size info for $(BINARY_NAME_FULL_1024) ===" + @$(SIZE) $(BINARY_NAME_FULL_1024) + clean: rm -rf $(BUILD_DIR) diff --git a/examples/bring_your_own_fips202/Makefile b/examples/bring_your_own_fips202/Makefile index 8524a3e9d..b5b409aae 100644 --- a/examples/bring_your_own_fips202/Makefile +++ b/examples/bring_your_own_fips202/Makefile @@ -1,16 +1,21 @@ # (SPDX-License-Identifier: CC-BY-4.0) -.PHONY: build run clean +.PHONY: build run clean size .DEFAULT_GOAL := all # Append cross-prefix for cross compilation # Remove or ignore for native builds CC ?= gcc +SIZE ?= size # When called from the root Makefile, CROSS_PREFIX has already been added here ifeq (,$(findstring $(CROSS_PREFIX),$(CC))) CC := $(CROSS_PREFIX)$(CC) endif +ifeq (,$(findstring $(CROSS_PREFIX),$(SIZE))) +SIZE := $(CROSS_PREFIX)$(SIZE) +endif + # Part A: # # mlkem-native source and header files @@ -87,7 +92,7 @@ $(BINARIES_FULL): $(ALL_SOURCE) mkdir -p $(BUILD_DIR) $(CC) $(CFLAGS) $(INC) $^ -o $@ -all: build +all: build size build: $(BINARIES_FULL) @@ -96,5 +101,13 @@ run: $(BINARIES_FULL) $(EXEC_WRAPPER) ./$(BINARY_NAME_FULL_768) $(EXEC_WRAPPER) ./$(BINARY_NAME_FULL_1024) +size: build + @echo "=== Size info for binaries $(BINARY_NAME_FULL_512) ===" + @$(SIZE) $(BINARY_NAME_FULL_512) + @echo "=== Size info for binaries $(BINARY_NAME_FULL_768) ===" + @$(SIZE) $(BINARY_NAME_FULL_768) + @echo "=== Size info for binaries $(BINARY_NAME_FULL_1024) ===" + @$(SIZE) $(BINARY_NAME_FULL_1024) + clean: rm -rf $(BUILD_DIR) diff --git a/examples/custom_backend/Makefile b/examples/custom_backend/Makefile index f5268b685..a44ee775c 100644 --- a/examples/custom_backend/Makefile +++ b/examples/custom_backend/Makefile @@ -1,16 +1,21 @@ # (SPDX-License-Identifier: CC-BY-4.0) -.PHONY: build run clean +.PHONY: build run clean size .DEFAULT_GOAL := all # Append cross-prefix for cross compilation # Remove or ignore for native builds CC ?= gcc +SIZE ?= size # When called from the root Makefile, CROSS_PREFIX has already been added here ifeq (,$(findstring $(CROSS_PREFIX),$(CC))) CC := $(CROSS_PREFIX)$(CC) endif +ifeq (,$(findstring $(CROSS_PREFIX),$(SIZE))) +SIZE := $(CROSS_PREFIX)$(SIZE) +endif + # Part A: # # mlkem-native source and header files @@ -83,7 +88,7 @@ $(BINARIES_FULL): $(ALL_SOURCE) mkdir -p $(BUILD_DIR) $(CC) $(CFLAGS) $(INC) $^ -o $@ -all: build +all: build size build: $(BINARIES_FULL) @@ -92,5 +97,13 @@ run: $(BINARIES_FULL) $(EXEC_WRAPPER) ./$(BINARY_NAME_FULL_768) $(EXEC_WRAPPER) ./$(BINARY_NAME_FULL_1024) +size: build + @echo "=== Size info for binaries $(BINARY_NAME_FULL_512) ===" + @$(SIZE) $(BINARY_NAME_FULL_512) + @echo "=== Size info for binaries $(BINARY_NAME_FULL_768) ===" + @$(SIZE) $(BINARY_NAME_FULL_768) + @echo "=== Size info for binaries $(BINARY_NAME_FULL_1024) ===" + @$(SIZE) $(BINARY_NAME_FULL_1024) + clean: rm -rf $(BUILD_DIR) diff --git a/examples/monolithic_build/Makefile b/examples/monolithic_build/Makefile index a2d262633..2d686467f 100644 --- a/examples/monolithic_build/Makefile +++ b/examples/monolithic_build/Makefile @@ -1,12 +1,13 @@ # (SPDX-License-Identifier: CC-BY-4.0) -.PHONY: build run clean +.PHONY: build run clean size .DEFAULT_GOAL := all # Append cross-prefix for cross compilation # Remove or ignore for native builds CC ?= gcc AR ?= ar +SIZE ?= size # When called from the root Makefile, CROSS_PREFIX has already been added here ifeq (,$(findstring $(CROSS_PREFIX),$(CC))) CC := $(CROSS_PREFIX)$(CC) @@ -16,6 +17,10 @@ ifeq (,$(findstring $(CROSS_PREFIX),$(AR))) AR := $(CROSS_PREFIX)$(AR) endif +ifeq (,$(findstring $(CROSS_PREFIX),$(SIZE))) +SIZE := $(CROSS_PREFIX)$(SIZE) +endif + Q ?= @ # Part A: @@ -116,7 +121,7 @@ $(BIN1024_FULL): $(APP_SOURCE) $(LIB1024_FULL) $(Q)[ -d $(@) ] || mkdir -p $(@D) $(Q)$(CC) $(CFLAGS) -DMLK_CONFIG_API_PARAMETER_SET=1024 $(INC) $^ -o $@ -all: build +all: build size build: $(BIN512_FULL) $(BIN768_FULL) $(BIN1024_FULL) @@ -125,5 +130,11 @@ run: $(BIN512_FULL) $(BIN768_FULL) $(BIN1024_FULL) $(Q)$(EXEC_WRAPPER) ./$(BIN768_FULL) $(Q)$(EXEC_WRAPPER) ./$(BIN1024_FULL) +size: build + @echo "=== Size info for static libs ===" + @$(Q)$(SIZE) $(LIB512_FULL) + @$(Q)$(SIZE) $(LIB768_FULL) + @$(Q)$(SIZE) $(LIB1024_FULL) + clean: rm -rf $(BUILD_DIR) diff --git a/examples/monolithic_build_multilevel/Makefile b/examples/monolithic_build_multilevel/Makefile index 00b5801b1..65236479c 100644 --- a/examples/monolithic_build_multilevel/Makefile +++ b/examples/monolithic_build_multilevel/Makefile @@ -1,12 +1,14 @@ # (SPDX-License-Identifier: CC-BY-4.0) -.PHONY: build run clean +.PHONY: build run clean size .DEFAULT_GOAL := all # Append cross-prefix for cross compilation # Remove or ignore for native builds CC ?= gcc AR ?= ar +SIZE ?= size +Q ?= @ # When called from the root Makefile, CROSS_PREFIX has already been added here ifeq (,$(findstring $(CROSS_PREFIX),$(CC))) CC := $(CROSS_PREFIX)$(CC) @@ -16,6 +18,11 @@ ifeq (,$(findstring $(CROSS_PREFIX),$(AR))) AR := $(CROSS_PREFIX)$(AR) endif +ifeq (,$(findstring $(CROSS_PREFIX),$(SIZE))) +SIZE := $(CROSS_PREFIX)$(SIZE) +endif + + # Part A: # # mlkem-native source and header files @@ -80,12 +87,16 @@ $(BINARY_NAME_FULL): $(APP_SOURCE) $(LIB_NAME_FULL) mkdir -p $(BUILD_DIR) $(CC) $(CFLAGS) $(INC) $^ -o $@ -all: build +all: build size build: $(BINARY_NAME_FULL) run: $(BINARY_NAME_FULL) $(EXEC_WRAPPER) ./$(BINARY_NAME_FULL) +size: build + $(Q)echo "=== Size info for static lib ===" + $(Q)$(SIZE) ${LIB_NAME_FULL} + clean: rm -rf $(BUILD_DIR) diff --git a/examples/monolithic_build_multilevel_native/Makefile b/examples/monolithic_build_multilevel_native/Makefile index e301d6eaf..908311de1 100644 --- a/examples/monolithic_build_multilevel_native/Makefile +++ b/examples/monolithic_build_multilevel_native/Makefile @@ -1,12 +1,13 @@ # (SPDX-License-Identifier: CC-BY-4.0) -.PHONY: build run clean +.PHONY: build run clean size .DEFAULT_GOAL := all # Append cross-prefix for cross compilation # Remove or ignore for native builds CC ?= gcc AR ?= ar +SIZE ?= size # When called from the root Makefile, CROSS_PREFIX has already been added here ifeq (,$(findstring $(CROSS_PREFIX),$(CC))) CC := $(CROSS_PREFIX)$(CC) @@ -16,6 +17,10 @@ ifeq (,$(findstring $(CROSS_PREFIX),$(AR))) AR := $(CROSS_PREFIX)$(AR) endif +ifeq (,$(findstring $(CROSS_PREFIX),$(SIZE))) +SIZE := $(CROSS_PREFIX)$(SIZE) +endif + # Part A: # # mlkem-native source and header files @@ -123,12 +128,18 @@ $(BINARY_NAME_FULL): $(APP_SOURCE) $(MLK_OBJ_ASM) $(Q)$(CC) $(CFLAGS) $(INC) $^ -o $@ $(Q)strip -S $@ -all: build +all: build size build: $(BINARY_NAME_FULL) run: $(BINARY_NAME_FULL) $(EXEC_WRAPPER) ./$(BINARY_NAME_FULL) +size: build + $(Q)echo "=== Size info for binaries $(BINARY_NAME_FULL)===" + $(Q)$(SIZE) $(BINARY_NAME_FULL) + $(Q)echo " " + + clean: rm -rf $(BUILD_DIR) diff --git a/examples/monolithic_build_native/Makefile b/examples/monolithic_build_native/Makefile index 70f288eae..e27909053 100644 --- a/examples/monolithic_build_native/Makefile +++ b/examples/monolithic_build_native/Makefile @@ -1,12 +1,13 @@ # (SPDX-License-Identifier: CC-BY-4.0) -.PHONY: build run clean +.PHONY: build run clean size .DEFAULT_GOAL := all # Append cross-prefix for cross compilation # Remove or ignore for native builds CC ?= gcc AR ?= ar +SIZE ?= size # When called from the root Makefile, CROSS_PREFIX has already been added here ifeq (,$(findstring $(CROSS_PREFIX),$(CC))) CC := $(CROSS_PREFIX)$(CC) @@ -16,8 +17,11 @@ ifeq (,$(findstring $(CROSS_PREFIX),$(AR))) AR := $(CROSS_PREFIX)$(AR) endif -Q ?= @ +ifeq (,$(findstring $(CROSS_PREFIX),$(SIZE))) +SIZE := $(CROSS_PREFIX)$(SIZE) +endif +Q ?= @ # Part A: # # mlkem-native source and header files @@ -129,7 +133,7 @@ $(BIN1024_FULL): $(APP_SOURCE) $(LIB1024_FULL) $(Q)[ -d $(@) ] || mkdir -p $(@D) $(Q)$(CC) $(CFLAGS) -DMLK_CONFIG_API_PARAMETER_SET=1024 $(INC) $^ -o $@ -all: build +all: build size build: $(BIN512_FULL) $(BIN768_FULL) $(BIN1024_FULL) @@ -138,5 +142,11 @@ run: $(BIN512_FULL) $(BIN768_FULL) $(BIN1024_FULL) $(Q)$(EXEC_WRAPPER) ./$(BIN768_FULL) $(Q)$(EXEC_WRAPPER) ./$(BIN1024_FULL) +size: build + @echo "=== Size info for static libs ===" + $(Q)$(SIZE) $(LIB512_FULL) | (read header; echo "$$header"; awk '$$5 != 0' | sort -k5 -n -r) + $(Q)$(SIZE) $(LIB768_FULL) | (read header; echo "$$header"; awk '$$5 != 0' | sort -k5 -n -r) + $(Q)$(SIZE) $(LIB1024_FULL) | (read header; echo "$$header"; awk '$$5 != 0' | sort -k5 -n -r) + clean: rm -rf $(BUILD_DIR) diff --git a/examples/multilevel_build/Makefile b/examples/multilevel_build/Makefile index d58eea0ff..870b2996c 100644 --- a/examples/multilevel_build/Makefile +++ b/examples/multilevel_build/Makefile @@ -1,17 +1,22 @@ # (SPDX-License-Identifier: CC-BY-4.0) -.PHONY: build run clean mlkem512_objs mlkem768_objs mlkem1024_objs mlkem_objs +.PHONY: build run clean mlkem512_objs mlkem768_objs mlkem1024_objs mlkem_objs size size_objs .DEFAULT_GOAL := all Q ?= @ # Append cross-prefix for cross compilation # Remove or ignore for native builds CC ?= gcc +SIZE ?= size # When called from the root Makefile, CROSS_PREFIX has already been added here ifeq (,$(findstring $(CROSS_PREFIX),$(CC))) CC := $(CROSS_PREFIX)$(CC) endif +ifeq (,$(findstring $(CROSS_PREFIX),$(SIZE))) +SIZE := $(CROSS_PREFIX)$(SIZE) +endif + # Part A: # # mlkem-native source and header files @@ -98,12 +103,22 @@ $(BINARY_NAME_FULL): $(APP_SOURCE) $(RNG_SOURCE) $(MLKEM512_OBJS) $(MLKEM768_OBJ mkdir -p $(BUILD_DIR) $(CC) $(CFLAGS) $^ -o $@ -all: build +all: build size_objs build: $(BINARY_NAME_FULL) run: $(BINARY_NAME_FULL) $(EXEC_WRAPPER) ./$(BINARY_NAME_FULL) +size: build + @echo "=== Size info for $(BINARY_NAME_FULL) ===" + $(Q)$(SIZE) $(BINARY_NAME_FULL) + +size_objs: size + $(Q)echo "=== Object size summary ===" + $(Q)$(SIZE) $(shell find $(BUILD_DIR)/mlkem512 -name '*.o') | (read header; echo "$$header"; awk '$$5 != 0' | sort -k5 -n -r) + $(Q)$(SIZE) $(shell find $(BUILD_DIR)/mlkem768 -name '*.o') | (read header; echo "$$header"; awk '$$5 != 0' | sort -k5 -n -r) + $(Q)$(SIZE) $(shell find $(BUILD_DIR)/mlkem1024 -name '*.o') | (read header; echo "$$header"; awk '$$5 != 0' | sort -k5 -n -r) + clean: rm -rf $(BUILD_DIR) diff --git a/examples/multilevel_build_native/Makefile b/examples/multilevel_build_native/Makefile index 01b869fd5..7d0645551 100644 --- a/examples/multilevel_build_native/Makefile +++ b/examples/multilevel_build_native/Makefile @@ -1,17 +1,22 @@ # (SPDX-License-Identifier: CC-BY-4.0) -.PHONY: build run clean mlkem512_objs mlkem768_objs mlkem1024_objs mlkem_objs +.PHONY: build run clean mlkem512_objs mlkem768_objs mlkem1024_objs mlkem_objs size size_objs .DEFAULT_GOAL := all Q ?= @ # Append cross-prefix for cross compilation # Remove or ignore for native builds CC ?= gcc +SIZE ?= size # When called from the root Makefile, CROSS_PREFIX has already been added here ifeq (,$(findstring $(CROSS_PREFIX),$(CC))) CC := $(CROSS_PREFIX)$(CC) endif +ifeq (,$(findstring $(CROSS_PREFIX),$(SIZE))) +SIZE := $(CROSS_PREFIX)$(SIZE) +endif + # Automatically detect system architecture and set preprocessor etc accordingly HOST_PLATFORM := $(shell uname -s)-$(shell uname -m) @@ -138,12 +143,22 @@ $(BINARY_NAME_FULL): $(APP_SOURCE) $(RNG_SOURCE) $(MLKEM512_OBJS) $(MLKEM768_OBJ $(Q)mkdir -p $(BUILD_DIR) $(Q)$(CC) $(CFLAGS) $^ -o $@ -all: build +all: build size_objs build: $(BINARY_NAME_FULL) run: $(BINARY_NAME_FULL) $(EXEC_WRAPPER) ./$(BINARY_NAME_FULL) +size: build + @echo "=== Size info for $(BINARY_NAME_FULL) ===" + $(Q)$(SIZE) $(BINARY_NAME_FULL) + +size_objs: size + $(Q)echo "=== Object size summary ===" + $(Q)$(SIZE) $(shell find $(BUILD_DIR)/mlkem512 -name '*.o') | (read header; echo "$$header"; awk '$$5 != 0' | sort -k5 -n -r) + $(Q)$(SIZE) $(shell find $(BUILD_DIR)/mlkem768 -name '*.o') | (read header; echo "$$header"; awk '$$5 != 0' | sort -k5 -n -r) + $(Q)$(SIZE) $(shell find $(BUILD_DIR)/mlkem1024 -name '*.o') | (read header; echo "$$header"; awk '$$5 != 0' | sort -k5 -n -r) + clean: rm -rf $(BUILD_DIR) diff --git a/scripts/tests b/scripts/tests index cdf6500a9..40b35da7f 100755 --- a/scripts/tests +++ b/scripts/tests @@ -207,6 +207,7 @@ class TEST_TYPES(Enum): MONOLITHIC_BUILD_MULTILEVEL_NATIVE = 13 MONOLITHIC_BUILD_NATIVE = 14 STACK = 15 + SIZE = 16 def is_benchmark(self): return self in [TEST_TYPES.BENCH, TEST_TYPES.BENCH_COMPONENTS] @@ -271,6 +272,8 @@ class TEST_TYPES(Enum): return "Example (multilevel build)" if self == TEST_TYPES.MULTILEVEL_BUILD_NATIVE: return "Example (multilevel build, native)" + if self == TEST_TYPES.SIZE: + return "Measurement Code Size" def make_dir(self): if self == TEST_TYPES.BRING_YOUR_OWN_FIPS202: @@ -324,6 +327,8 @@ class TEST_TYPES(Enum): return "" if self == TEST_TYPES.MULTILEVEL_BUILD_NATIVE: return "" + if self == TEST_TYPES.SIZE: + return "size" def make_run_target(self, scheme): t = self.make_target() @@ -703,6 +708,29 @@ class Tests: self.check_fail() + def size(self): + + test_type = TEST_TYPES.SIZE + + resultss = None + + if self.do_opt_all(): + self._compile_schemes(test_type, False) + if self.args.run: + self._run_schemes(test_type, False, suppress_output=False) + self._compile_schemes(test_type, True) + if self.args.run: + resultss = self._run_schemes(test_type, True, suppress_output=False) + else: + self._compile_schemes(test_type, self.do_opt()) + if self.args.run: + resultss = self._run_schemes( + test_type, self.do_opt(), suppress_output=False + ) + + if resultss is None: + self.check_fail() + def all(self): func = self.args.func kat = self.args.kat @@ -1118,6 +1146,11 @@ def cli(): action="store_true", default=False, ) + size_parser = cmd_subparsers.add_parser( + "size", + help="Run the code size measurement for all object file", + parents=[common_parser], + ) # cbmc arguments cbmc_parser = cmd_subparsers.add_parser( @@ -1265,6 +1298,8 @@ def cli(): Tests(args).kat() elif args.cmd == "stack": Tests(args).stack() + elif args.cmd == "size": + Tests(args).size() if __name__ == "__main__":