Merge branch 'release/v6.0.1'

Author: valeros

README.md

@@ -2,7 +2,7 @@
 
 [![Build Status](https://github.com/platformio/platform-espressif32/workflows/Examples/badge.svg)](https://github.com/platformio/platform-espressif32/actions)
 
-Espressif Systems is a privately held fabless semiconductor company. They provide wireless communications and Wi-Fi chips which are widely used in mobile devices and the Internet of Things applications.
+ESP32 is a series of low-cost, low-power system on a chip microcontrollers with integrated Wi-Fi and Bluetooth. ESP32 integrates an antenna switch, RF balun, power amplifier, low-noise receive amplifier, filters, and power management modules.
 
 * [Home](https://registry.platformio.org/platforms/platformio/espressif32) (home page in the PlatformIO Registry)
 * [Documentation](https://docs.platformio.org/page/platforms/espressif32.html) (advanced usage, packages, boards, frameworks, etc.)

boards/adafruit_feather_esp32s3_nopsram.json

@@ -2,7 +2,7 @@
   "build": {
     "arduino":{
       "ldscript": "esp32s3_out.ld",
-      "partitions": "partitions-4MB-tinyuf2.csv",
+      "partitions": "partitions-8MB-tinyuf2.csv",
       "memory_type": "qio_qspi"
     },
     "core": "esp32",

boards/adafruit_qtpy_esp32s3_nopsram.json

@@ -1,7 +1,8 @@
 {
   "build": {
     "arduino": {
-      "ldscript": "esp32s3_out.ld"
+      "ldscript": "esp32s3_out.ld",
+      "partitions": "partitions-8MB-tinyuf2.csv"
     },
     "core": "esp32",
     "extra_flags": [

builder/frameworks/espidf.py

@@ -37,11 +37,17 @@
 )
 
 from platformio import fs
+from platformio.compat import IS_WINDOWS
 from platformio.proc import exec_command
-from platformio.util import get_systype
 from platformio.builder.tools.piolib import ProjectAsLibBuilder
 from platformio.package.version import get_original_version, pepver_to_semver
 
+# Added to avoid conflicts between installed Python packages from
+# the IDF virtual environment and PlatformIO Core
+# Note: This workaround can be safely deleted when PlatformIO 6.1.7 is released
+if os.environ.get("PYTHONPATH"):
+    del os.environ["PYTHONPATH"]
+
 env = DefaultEnvironment()
 env.SConscript("_embed_files.py", exports="env")
 
@@ -51,8 +57,7 @@
 idf_variant = mcu.lower()
 
 # Required until Arduino switches to v5
-IDF5 = platform.get_package_version(
-    "framework-espidf").split(".")[1].startswith("5")
+IDF5 = platform.get_package_version("framework-espidf").split(".")[1].startswith("5")
 FRAMEWORK_DIR = platform.get_package_dir("framework-espidf")
 TOOLCHAIN_DIR = platform.get_package_dir(
     "toolchain-%s" % ("riscv32-esp" if mcu == "esp32c3" else ("xtensa-%s" % mcu))
@@ -224,15 +229,15 @@ def populate_idf_env_vars(idf_env):
         os.path.join(TOOLCHAIN_DIR, "bin"),
         platform.get_package_dir("tool-ninja"),
         os.path.join(platform.get_package_dir("tool-cmake"), "bin"),
-        os.path.dirname(env.subst("$PYTHONEXE")),
+        os.path.dirname(get_python_exe()),
     ]
 
     if mcu != "esp32c3":
         additional_packages.append(
             os.path.join(platform.get_package_dir("toolchain-esp32ulp"), "bin"),
         )
 
-    if "windows" in get_systype():
+    if IS_WINDOWS:
         additional_packages.append(platform.get_package_dir("tool-mconf"))
 
     idf_env["PATH"] = os.pathsep.join(additional_packages + [idf_env["PATH"]])
@@ -559,7 +564,7 @@ def generate_project_ld_script(sdk_config, ignore_targets=None):
     }
 
     cmd = (
-        '"$PYTHONEXE" "{script}" --input $SOURCE '
+        '"$ESPIDF_PYTHONEXE" "{script}" --input $SOURCE '
         '--config "{config}" --fragments {fragments} --output $TARGET '
         '--kconfig "{kconfig}" --env-file "{env_file}" '
         '--libraries-file "{libraries_list}" '
@@ -763,7 +768,7 @@ def build_bootloader(sdk_config):
         [
             "-DIDF_TARGET=" + idf_variant,
             "-DPYTHON_DEPS_CHECKED=1",
-            "-DPYTHON=" + env.subst("$PYTHONEXE"),
+            "-DPYTHON=" + get_python_exe(),
             "-DIDF_PATH=" + FRAMEWORK_DIR,
             "-DSDKCONFIG=" + SDKCONFIG_PATH,
             "-DLEGACY_INCLUDE_COMMON_HEADERS=",
@@ -918,7 +923,7 @@ def generate_empty_partition_image(binary_path, image_size):
         binary_path,
         None,
         env.VerboseAction(
-            '"$PYTHONEXE" "%s" %s $TARGET'
+            '"$ESPIDF_PYTHONEXE" "%s" %s $TARGET'
             % (
                 os.path.join(
                     FRAMEWORK_DIR,
@@ -943,7 +948,7 @@ def get_partition_info(pt_path, pt_offset, pt_params):
         env.Exit(1)
 
     cmd = [
-        env.subst("$PYTHONEXE"),
+        get_python_exe(),
         os.path.join(FRAMEWORK_DIR, "components", "partition_table", "parttool.py"),
         "-q",
         "--partition-table-offset",
@@ -1000,7 +1005,7 @@ def generate_mbedtls_bundle(sdk_config):
         FRAMEWORK_DIR, "components", "mbedtls", "esp_crt_bundle"
     )
 
-    cmd = [env.subst("$PYTHONEXE"), os.path.join(default_crt_dir, "gen_crt_bundle.py")]
+    cmd = [get_python_exe(), os.path.join(default_crt_dir, "gen_crt_bundle.py")]
 
     crt_args = ["--input"]
     if sdk_config.get("MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_FULL", False):
@@ -1051,12 +1056,12 @@ def generate_mbedtls_bundle(sdk_config):
 
 
 def install_python_deps():
-    def _get_installed_pip_packages():
+    def _get_installed_pip_packages(python_exe_path):
         result = {}
         packages = {}
         pip_output = subprocess.check_output(
             [
-                env.subst("$PYTHONEXE"),
+                python_exe_path,
                 "-m",
                 "pip",
                 "list",
@@ -1087,7 +1092,8 @@ def _get_installed_pip_packages():
         # Remove specific versions for IDF5 as not required
         deps = {dep: "" for dep in deps}
 
-    installed_packages = _get_installed_pip_packages()
+    python_exe_path = get_python_exe()
+    installed_packages = _get_installed_pip_packages(python_exe_path)
     packages_to_install = []
     for package, spec in deps.items():
         if package not in installed_packages:
@@ -1101,22 +1107,17 @@ def _get_installed_pip_packages():
         env.Execute(
             env.VerboseAction(
                 (
-                    '"$PYTHONEXE" -m pip install -U '
-                    + " ".join(
-                        [
-                            '"%s%s"' % (p, deps[p])
-                            for p in packages_to_install
-                        ]
-                    )
+                    '"%s" -m pip install -U ' % python_exe_path
+                    + " ".join(['"%s%s"' % (p, deps[p]) for p in packages_to_install])
                 ),
                 "Installing ESP-IDF's Python dependencies",
             )
         )
 
-    if "windows" in get_systype() and "windows-curses" not in installed_packages:
+    if IS_WINDOWS and "windows-curses" not in installed_packages:
         env.Execute(
             env.VerboseAction(
-                "$PYTHONEXE -m pip install windows-curses",
+                '"%s" -m pip install windows-curses' % python_exe_path,
                 "Installing windows-curses package",
             )
         )
@@ -1128,15 +1129,59 @@ def _get_installed_pip_packages():
         }:
             env.Execute(
                 env.VerboseAction(
-                    '$PYTHONEXE -m pip install "file://%s/tools/kconfig_new/esp-windows-curses"'
-                    % FRAMEWORK_DIR,
+                    '"%s" -m pip install "file://%s/tools/kconfig_new/esp-windows-curses"'
+                    % (python_exe_path, FRAMEWORK_DIR),
                     "Installing windows-curses package",
                 )
             )
 
 
+def get_python_exe():
+    def _create_venv(venv_dir):
+        pip_path = os.path.join(
+            venv_dir,
+            "Scripts" if IS_WINDOWS else "bin",
+            "pip" + (".exe" if IS_WINDOWS else ""),
+        )
+        if not os.path.isfile(pip_path):
+            # Use the built-in PlatformIO Python to create a standalone IDF virtual env
+            env.Execute(
+                env.VerboseAction(
+                    '"$PYTHONEXE" -m venv --clear "%s"' % venv_dir,
+                    "Creating a virtual environment for IDF Python dependencies",
+                )
+            )
+
+        assert os.path.isfile(
+            pip_path
+        ), "Error: Failed to create a proper virtual environment. Missing the pip binary!"
+
+    # The name of the IDF venv contains the IDF version to avoid possible conflicts and
+    # unnecessary reinstallation of Python dependencies in cases when Arduino
+    # as an IDF component requires a different version of the IDF package and
+    # hence a different set of Python deps or their versions
+    idf_version = get_original_version(platform.get_package_version("framework-espidf"))
+    venv_dir = os.path.join(
+        env.subst("$PROJECT_CORE_DIR"), "penv", ".espidf-" + idf_version)
+
+    if not os.path.isdir(venv_dir):
+        _create_venv(venv_dir)
+
+    python_exe_path = os.path.join(
+        venv_dir,
+        "Scripts" if IS_WINDOWS else "bin",
+        "python" + (".exe" if IS_WINDOWS else ""),
+    )
+
+    assert os.path.isfile(python_exe_path), (
+        "Error: Missing Python executable file `%s`" % python_exe_path
+    )
+
+    return python_exe_path
+
+
 #
-# ESP-IDF requires Python packages with specific versions
+# ESP-IDF requires Python packages with specific versions in a virtual environment
 #
 
 install_python_deps()
@@ -1235,7 +1280,7 @@ def _get_installed_pip_packages():
         "-DIDF_TARGET=" + idf_variant,
         "-DPYTHON_DEPS_CHECKED=1",
         "-DEXTRA_COMPONENT_DIRS:PATH=" + ";".join(extra_components),
-        "-DPYTHON=" + env.subst("$PYTHONEXE"),
+        "-DPYTHON=" + get_python_exe(),
         "-DSDKCONFIG=" + SDKCONFIG_PATH,
     ]
     + click.parser.split_arg_string(board.get("build.cmake_extra_args", "")),
@@ -1373,7 +1418,7 @@ def _skip_prj_source_files(node):
     os.path.join("$BUILD_DIR", "partitions.bin"),
     "$PARTITIONS_TABLE_CSV",
     env.VerboseAction(
-        '"$PYTHONEXE" "%s" -q --offset "%s" --flash-size "%s" $SOURCE $TARGET'
+        '"$ESPIDF_PYTHONEXE" "%s" -q --offset "%s" --flash-size "%s" $SOURCE $TARGET'
         % (
             os.path.join(
                 FRAMEWORK_DIR, "components", "partition_table", "gen_esp32part.py"
@@ -1396,6 +1441,7 @@ def _skip_prj_source_files(node):
 env.Prepend(
     CPPPATH=app_includes["plain_includes"],
     CPPDEFINES=project_defines,
+    ESPIDF_PYTHONEXE=get_python_exe(),
     LINKFLAGS=extra_flags,
     LIBS=libs,
     FLASH_EXTRA_IMAGES=[