Create a proper Python project for the Python scripts (#470)

* Start on improving Python project structure

* Finish migrating to pyproject.toml

* Add find program for python scripts

* Remove existing mbed-tools, re-add dependency

* Remove dependency on future

* Print package dir

* Augh missing quote

* Oh neat reordering the commands already fixed it

* Install pyocd and cysecuretools lazily

* mbed-tools needs setuptools

* Set min version of setuptools

* Bump click version

* Install cysecuretools in CI

* Install xml apt packages

* augh no sudo

* Allow tests to be run from anywhere

* Update tools/run_python_tests.sh

Co-authored-by: VictorWTang <33789988+VictorWTang@users.noreply.github.com>

---------

Co-authored-by: VictorWTang <33789988+VictorWTang@users.noreply.github.com>

Author: multiplemonomials

.github/workflows/basic_checks.yml

@@ -29,6 +29,8 @@ jobs:
         # detection output format: https://github.com/nexB/scancode-toolkit/releases/tag/v32.0.0
         # Need to update Mbed's scripts for the new format.
         run: |
+          apt-get update
+          apt-get install -y libxml2-dev libxslt1-dev
           pip install -U "scancode-toolkit<32.0" "click>=7,<8"
 
       - 
@@ -169,20 +171,16 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y python3-venv
 
-      -
-        name: install dependencies
+      - name: Install Python packages
         run: |
           python3 -m venv venv
           source venv/bin/activate
-          pip install -r tools/requirements.txt
-          pip install -r tools/python/python_tests/requirements.txt
+          pip install -e ./tools[unit-tests]
 
-      -
-        name: Python Tests
+      - name: Python Tests
         run: |
           source venv/bin/activate
-          cd tools/python
-          ./run_python_tests.sh
+          tools/run_python_tests.sh
 
   check-cmsis-mcu-descriptions-matches-target-list:
     runs-on: ubuntu-latest
@@ -195,18 +193,18 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y python3-venv
 
-      - name: install dependencies
+      - name: Install Python packages
         run: |
           python3 -m venv venv
           source venv/bin/activate
-          pip install -r tools/requirements.txt
+          pip install -e ./tools
 
       - name: Verify that cmsis_mcu_descriptions.json5 is in sync with targets.json5
         run: |
           source venv/bin/activate
           cd tools/python
-          python -m mbed_tools.cli.main cmsis-mcu-descr find-unused
-          python -m mbed_tools.cli.main cmsis-mcu-descr check-missing
+          mbed-tools cmsis-mcu-descr find-unused
+          mbed-tools cmsis-mcu-descr check-missing
 
 
   pin-validation:

.github/workflows/test_building_multiple_executables.yml

@@ -14,7 +14,14 @@ jobs:
         # Note: pip>=20.3 is needed to install dependencies of cysecuretools
         run: |
           python3 -m pip install --upgrade pip
-          python3 -m pip install -r tools/requirements.txt
+          
+          # Remove mbed-tools package that comes with the docker image, it conflicts with the one we want to install
+          python3 -m pip uninstall -y mbed-tools
+          
+          python3 -m pip install -e ./tools
+          
+          # Also install cysecuretools which is not installed automatically
+          python3 -m pip install 'cysecuretools~=6.0'
 
       # Note: For this CI job we use MBED_CREATE_PYTHON_VENV=FALSE so that we can make sure
       # this mode works.

targets/TARGET_Cypress/scripts/mbed_set_post_build_cypress.cmake

@@ -3,6 +3,12 @@
 
 include(mbed_target_functions)
 
+# Make sure we have the python packages we need
+mbed_check_or_install_python_package(HAVE_PYTHON_CYSECURETOOLS cysecuretools "cysecuretools~=6.0")
+if(NOT HAVE_PYTHON_CYSECURETOOLS)
+    message(FATAL_ERROR "Python package required for signing not found.")
+endif()
+
 #
 # Merge Cortex-M4 HEX and a Cortex-M0 HEX.
 #
@@ -63,8 +69,6 @@ macro(mbed_post_build_psoc6_sign_image
 )
     if("${cypress_psoc6_target}" STREQUAL "${MBED_TARGET}")
         function(mbed_post_build_function target)
-            find_package(Python3)
-
             set(post_build_command
                 ${Python3_EXECUTABLE} ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/PSOC6.py
                 sign

targets/upload_method_cfg/SFE_ARTEMIS.cmake

@@ -6,7 +6,7 @@
 # 1. This board does not have an onboard debugger. You must use an external debugger, e.g. a PicoProbe
 #    or J-Link, if you wish to debug code.
 # 2. Support for this device exists in PyOCD main branch but has not been released yet (as of Jun 2025).
-#   To use PyOCD, you need to manually install it from the git repository by running (inside the Mbed OS venv):
+#   This version will be used automatically by Mbed if the python venv is enabled. If not, you need to install it via:
 #   pip install git+https://github.com/pyocd/pyOCD.git
 
 set(UPLOAD_METHOD_DEFAULT NONE)

tools/cmake/mbed-run-greentea-test.in.cmake

@@ -14,9 +14,8 @@ set(MBEDHTRUN_ARGS --skip-flashing @MBED_HTRUN_ARGUMENTS@) # filled in by config
 
 # Print out command
 string(REPLACE ";" " " MBEDHTRUN_ARGS_FOR_DISPLAY "${MBEDHTRUN_ARGS}")
-message("Executing: @Python3_EXECUTABLE@ -m mbed_host_tests.mbedhtrun ${MBEDHTRUN_ARGS_FOR_DISPLAY}")
+message("Executing: @mbedhtrun@ ${MBEDHTRUN_ARGS_FOR_DISPLAY}")
 
 execute_process(
-	COMMAND @Python3_EXECUTABLE@ -m mbed_host_tests.mbedhtrun ${MBEDHTRUN_ARGS}
-	WORKING_DIRECTORY "@mbed-os_SOURCE_DIR@/tools/python"
+	COMMAND @mbedhtrun@ ${MBEDHTRUN_ARGS}
 	COMMAND_ERROR_IS_FATAL ANY)

tools/cmake/mbed_generate_configuration.cmake

@@ -82,8 +82,7 @@ if(MBED_NEED_TO_RECONFIGURE)
     # Make sure an old config file doesn't stick around
     file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/mbed_config.cmake)
 
-    set(MBEDTOOLS_CONFIGURE_COMMAND ${Python3_EXECUTABLE}
-        -m mbed_tools.cli.main
+    set(MBEDTOOLS_CONFIGURE_COMMAND ${mbed_tools}
         -v -v # without at least -v, warnings (e.g. "you have tried to override a nonexistent parameter") do not get printed
         configure
         -t GCC_ARM # GCC_ARM is currently the only supported toolchain
@@ -96,7 +95,6 @@ if(MBED_NEED_TO_RECONFIGURE)
 
     execute_process(
         COMMAND ${MBEDTOOLS_CONFIGURE_COMMAND}
-        WORKING_DIRECTORY ${CMAKE_CURRENT_LIST_DIR}/../python
         RESULT_VARIABLE MBEDTOOLS_CONFIGURE_RESULT
         OUTPUT_VARIABLE MBEDTOOLS_CONFIGURE_OUTPUT
         ERROR_VARIABLE MBEDTOOLS_CONFIGURE_ERROR_OUTPUT
@@ -106,7 +104,7 @@ if(MBED_NEED_TO_RECONFIGURE)
 
     if((NOT MBEDTOOLS_CONFIGURE_RESULT EQUAL 0) OR (NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/mbed_config.cmake))
         string(JOIN " " MBEDTOOLS_COMMAND_SPC_SEP ${MBEDTOOLS_CONFIGURE_COMMAND})
-        message(FATAL_ERROR "mbedtools configure failed!  Cannot build this project.  Command was cd ${CMAKE_CURRENT_LIST_DIR}/../python && ${MBEDTOOLS_COMMAND_SPC_SEP}")
+        message(FATAL_ERROR "mbedtools configure failed!  Cannot build this project. Command was ${MBEDTOOLS_COMMAND_SPC_SEP}")
     endif()
 
 endif()

tools/cmake/mbed_python_interpreter.cmake

@@ -4,32 +4,64 @@
 # CMake script to find the Python interpreter and either install or find
 # Mbed's dependencies.
 
+include(CheckPythonPackage)
+
 option(MBED_CREATE_PYTHON_VENV "If true, Mbed OS will create its own virtual environment (venv) and install its Python packages there.  This removes the need to manually install Python packages." TRUE)
 
+get_filename_component(MBED_CE_TOOLS_BASE_DIR "${CMAKE_CURRENT_LIST_DIR}/.." ABSOLUTE)
+
 if(MBED_CREATE_PYTHON_VENV)
     # Use the venv.
 
     # Note: venv is stored in the source directory as it can be shared between all the build directories
     # (not target specific)
     set(MBED_VENV_LOCATION ${MBED_SOURCE_DIR}/venv)
     set(VENV_STAMP_FILE ${MBED_VENV_LOCATION}/mbed-venv.stamp)
-    set(MBED_REQUIREMENTS_TXT_LOCATION "${CMAKE_CURRENT_LIST_DIR}/../requirements.txt")
+    set(MBED_PYPROJECT_TOML_LOCATION "${MBED_CE_TOOLS_BASE_DIR}/pyproject.toml")
 
-    # Make it so modifying requirements.txt will trigger a reconfigure
-    set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${MBED_REQUIREMENTS_TXT_LOCATION})
+    # Make it so modifying pyproject.toml will trigger a reconfigure
+    set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${MBED_PYPROJECT_TOML_LOCATION})
 
-    # Find Python3, using the venv if it already exists
-    set (ENV{VIRTUAL_ENV} ${MBED_VENV_LOCATION})
-    set (Python3_FIND_VIRTUALENV FIRST)
-    find_package(Python3 REQUIRED COMPONENTS Interpreter)
+    # Find Python3 (this will get the one in the venv if we already found it)
+    set(ENV{VIRTUAL_ENV} ${MBED_VENV_LOCATION})
+    set(Python3_FIND_VIRTUALENV FIRST)
+    find_package(Python3 COMPONENTS Interpreter)
     include(CheckPythonPackage)
 
     set(NEED_TO_CREATE_VENV FALSE)
     set(NEED_TO_INSTALL_PACKAGES FALSE)
-    if(NOT EXISTS "${VENV_STAMP_FILE}")
+
+    # Special situation: if we have a cached interpreter location in the venv dir, but Python could be found,
+    # this means that the venv was deleted or symlinks to a missing python install location.
+    # So, use the system python and recreate it.
+    if("${Python3_EXECUTABLE}" MATCHES "${MBED_VENV_LOCATION}" AND NOT Python3_FOUND)
+        message(STATUS "Python venv deleted or unusable. Recreating using system Python...")
+
+        # Launch a new search for Python3
+        unset(Python3_EXECUTABLE)
+        unset(_Python3_EXECUTABLE CACHE)
+        unset(_Python3_INTERPRETER_PROPERTIES CACHE)
+        unset(_Python3_INTERPRETER_SIGNATURE CACHE)
+        set (Python3_FIND_VIRTUALENV STANDARD)
+        unset(ENV{VIRTUAL_ENV})
+        find_package(Python3 REQUIRED COMPONENTS Interpreter)
+
+        # Reset venv configuration as above
+        set(Python3_FIND_VIRTUALENV FIRST)
+        set(ENV{VIRTUAL_ENV} ${MBED_VENV_LOCATION})
+
         set(NEED_TO_CREATE_VENV TRUE)
         set(NEED_TO_INSTALL_PACKAGES TRUE)
-    elseif("${MBED_REQUIREMENTS_TXT_LOCATION}" IS_NEWER_THAN "${VENV_STAMP_FILE}")
+    elseif(NOT EXISTS "${VENV_STAMP_FILE}")
+        set(NEED_TO_CREATE_VENV TRUE)
+        set(NEED_TO_INSTALL_PACKAGES TRUE)
+    elseif(NOT ("${Python3_EXECUTABLE}" MATCHES "${MBED_VENV_LOCATION}"))
+        # Alternately if we think we have the venv but FindPython didn't use it, that likely means it's
+        # missing or corrupted and we need to recreate it
+        message(STATUS "Python venv deleted or unusable. Recreating using system Python...")
+        set(NEED_TO_CREATE_VENV TRUE)
+        set(NEED_TO_INSTALL_PACKAGES TRUE)
+    elseif("${MBED_PYPROJECT_TOML_LOCATION}" IS_NEWER_THAN "${VENV_STAMP_FILE}")
         set(NEED_TO_INSTALL_PACKAGES TRUE)
     endif()
 
@@ -47,7 +79,8 @@ if(MBED_CREATE_PYTHON_VENV)
         unset(_Python3_EXECUTABLE CACHE)
         unset(_Python3_INTERPRETER_PROPERTIES CACHE)
         unset(_Python3_INTERPRETER_SIGNATURE CACHE)
-        ## Launch a new search for Python3
+
+        ## Launch a new search for Python3 in the venv
         find_package (Python3 REQUIRED COMPONENTS Interpreter)
     endif()
 
@@ -59,41 +92,80 @@ if(MBED_CREATE_PYTHON_VENV)
             COMMAND_ERROR_IS_FATAL ANY
         )
         execute_process(
-            COMMAND ${Python3_EXECUTABLE} -m pip install -r ${MBED_REQUIREMENTS_TXT_LOCATION}
+            COMMAND ${Python3_EXECUTABLE} -m pip install -e ${MBED_CE_TOOLS_BASE_DIR}
             COMMAND_ERROR_IS_FATAL ANY
         )
 
         message(STATUS "Mbed: venv created successfully")
         file(TOUCH ${VENV_STAMP_FILE})
     endif()
 
-    # We always have the memap deps with the venv
-    set(HAVE_MEMAP_DEPS TRUE)
+    # When using the venv, scripts will always be installed to the directory where Python itself is installed
+    # (venv\Scripts on Windows, venv/bin on Linux/Mac)
+    get_filename_component(PYTHON3_INTERP_DIR ${Python3_EXECUTABLE} DIRECTORY)
+    set(PYTHON_SCRIPT_LOC_HINTS ${PYTHON3_INTERP_DIR})
 
 else()
 
     find_package(Python3 REQUIRED COMPONENTS Interpreter)
-    include(CheckPythonPackage)
 
+    # The cmsis_mcu_descr module was written from scratch by Mbed CE.
+    # So, this check will ensure that the user has installed the Mbed CE version of mbed_tools
+    # and not the PyPI version (which we cannot update because it's owned by ARM)
+    check_python_package(mbed_tools.cli.cmsis_mcu_descr HAVE_MBED_CE_TOOLS)
 
-    # Check python packages
-    set(PYTHON_PACKAGES_TO_CHECK intelhex prettytable future jinja2)
-    foreach(PACKAGE_NAME ${PYTHON_PACKAGES_TO_CHECK})
-        string(TOUPPER ${PACKAGE_NAME} PACKAGE_NAME_UCASE) # Ucase name needed for CMake variable
-        string(TOLOWER ${PACKAGE_NAME} PACKAGE_NAME_LCASE) # Lcase name needed for import statement
-
-        check_python_package(${PACKAGE_NAME_LCASE} HAVE_PYTHON_${PACKAGE_NAME_UCASE})
-        if(NOT HAVE_PYTHON_${PACKAGE_NAME_UCASE})
-            message(WARNING "Missing Python dependency ${PACKAGE_NAME}")
-        endif()
-    endforeach()
-
-    # Check deps for memap
-    if(Python3_FOUND AND HAVE_PYTHON_INTELHEX AND HAVE_PYTHON_PRETTYTABLE)
-        set(HAVE_MEMAP_DEPS TRUE)
-    else()
-        set(HAVE_MEMAP_DEPS FALSE)
-        message(STATUS "Missing Python dependencies (at least one of: python3, intelhex, prettytable) so the memory map cannot be printed")
+    if(NOT HAVE_MBED_CE_TOOLS)
+        message(FATAL_ERROR "Did not detect the Mbed CE Python tools installed into the python interpreter ${Python3_EXECUTABLE}. Install them with a command like: ${Python3_EXECUTABLE} -m pip install -e ${MBED_CE_TOOLS_BASE_DIR}")
     endif()
 
+    # For now, don't supply any hints and assume that the script install dir is correctly on PATH
+    set(PYTHON_SCRIPT_LOC_HINTS)
 endif()
+
+# Find scripts provided by the Python package
+find_program(mbed_tools
+    NAMES mbed-tools
+    HINTS ${PYTHON_SCRIPT_LOC_HINTS}
+    DOC "Path to mbed-tools Python script."
+    REQUIRED)
+
+find_program(mbedhtrun
+    NAMES mbedhtrun
+    HINTS ${PYTHON_SCRIPT_LOC_HINTS}
+    DOC "Path to mbedhtrun Python script."
+    REQUIRED)
+
+find_program(memap
+    NAMES memap
+    HINTS ${PYTHON_SCRIPT_LOC_HINTS}
+    DOC "Path to memap Python script."
+    REQUIRED)
+
+#
+# Utility function to check for a Python package with the given import name.
+# If the package is not found and the Mbed venv is in use,
+# then the package will be installed by passing PACKAGE_INSTALL_CONSTRAINT to Pip.
+# If the install fails or the venv is not being used, FOUND_VAR will be set to false.
+#
+function(mbed_check_or_install_python_package FOUND_VAR PACKAGE_IMPORT_NAME PACKAGE_INSTALL_CONSTRAINT)
+    check_python_package(${PACKAGE_IMPORT_NAME} ${FOUND_VAR})
+
+    if(NOT ${FOUND_VAR})
+        # If we are using the Mbed venv, we can install the package automatically.
+        if(MBED_CREATE_PYTHON_VENV)
+            message(STATUS "Mbed: Installing ${PACKAGE_INSTALL_CONSTRAINT} into Mbed's Python virtualenv")
+            execute_process(
+                    COMMAND ${Python3_EXECUTABLE} -m pip install ${PACKAGE_INSTALL_CONSTRAINT}
+                    RESULT_VARIABLE PIP_INSTALL_RESULT
+            )
+            if(NOT PIP_INSTALL_RESULT EQUAL 0)
+                message(WARNING "Installation of ${PACKAGE_INSTALL_CONSTRAINT} via pip failed.")
+            else()
+                # Redo the check to confirm it's installed
+                check_python_package(${PACKAGE_IMPORT_NAME} ${FOUND_VAR})
+            endif()
+        else()
+            message(WARNING "Mbed: ${PACKAGE_IMPORT_NAME} cannot be installed because the Mbed virtualenv is not being used. Please install ${PACKAGE_INSTALL_CONSTRAINT} into Mbed's Python interpeter manually.")
+        endif()
+    endif()
+endfunction(mbed_check_or_install_python_package)

tools/cmake/mbed_target_functions.cmake

@@ -65,12 +65,12 @@ function(mbed_generate_map_file target)
         TARGET
             ${target}
         POST_BUILD
-        COMMAND ${Python3_EXECUTABLE} -m memap.memap
-            -t ${MBED_TOOLCHAIN} ${CMAKE_CURRENT_BINARY_DIR}/${target}${CMAKE_EXECUTABLE_SUFFIX}.map 
+        COMMAND ${memap}
+            -t ${MBED_TOOLCHAIN} ${target}${CMAKE_EXECUTABLE_SUFFIX}.map
             --depth ${MBED_MEMAP_DEPTH}
             ${MEMORY_BANKS_ARG}
         WORKING_DIRECTORY
-			${mbed-os_SOURCE_DIR}/tools/python
+            ${CMAKE_CURRENT_BINARY_DIR}
     )
 
     # generate json file
@@ -79,14 +79,14 @@ function(mbed_generate_map_file target)
             TARGET
                 ${target}
             POST_BUILD
-            COMMAND ${Python3_EXECUTABLE} -m memap.memap
-            -t ${MBED_TOOLCHAIN} ${CMAKE_CURRENT_BINARY_DIR}/${target}${CMAKE_EXECUTABLE_SUFFIX}.map 
+            COMMAND ${memap}
+            -t ${MBED_TOOLCHAIN} ${target}${CMAKE_EXECUTABLE_SUFFIX}.map
             --depth ${MBED_MEMAP_DEPTH} 
             -e json
-            -o ${CMAKE_CURRENT_BINARY_DIR}/${target}${CMAKE_EXECUTABLE_SUFFIX}.memmap.json
+            -o ${target}${CMAKE_EXECUTABLE_SUFFIX}.memmap.json
             ${MEMORY_BANKS_ARG}
             WORKING_DIRECTORY
-			    ${mbed-os_SOURCE_DIR}/tools/python
+                ${CMAKE_CURRENT_BINARY_DIR}
     )
     endif()
 
@@ -96,14 +96,14 @@ function(mbed_generate_map_file target)
             TARGET
                 ${target}
             POST_BUILD
-            COMMAND ${Python3_EXECUTABLE} -m memap.memap
-            -t ${MBED_TOOLCHAIN} ${CMAKE_CURRENT_BINARY_DIR}/${target}${CMAKE_EXECUTABLE_SUFFIX}.map 
+            COMMAND ${memap}
+            -t ${MBED_TOOLCHAIN} ${target}${CMAKE_EXECUTABLE_SUFFIX}.map
             --depth ${MBED_MEMAP_DEPTH} 
             -e html
-            -o ${CMAKE_CURRENT_BINARY_DIR}/${target}${CMAKE_EXECUTABLE_SUFFIX}.memmap.html
+            -o ${target}${CMAKE_EXECUTABLE_SUFFIX}.memmap.html
             ${MEMORY_BANKS_ARG}
             WORKING_DIRECTORY
-			    ${mbed-os_SOURCE_DIR}/tools/python
+                ${CMAKE_CURRENT_BINARY_DIR}
     )
     endif()
 endfunction()