Split generate/configure stages for sd-generator compliance

.github/workflows/autopkgtest.yml

@@ -47,8 +47,9 @@ jobs:
           pull-lp-source netplan.io
           cp -r netplan.io-*/debian .
           rm -r debian/patches/  # clear any distro patches
-          sed -i 's|iproute2,|iproute2, ethtool,|' debian/control  # add ethtool as a dependency of netplan.io temporarily
           sed -i 's|systemd (>= 257.2-3ubuntu1~),|systemd (>= 248~),|g' debian/control  # see https://github.com/canonical/netplan/pull/535
+          echo 'usr/libexec/netplan/configure' >> debian/netplan-generator.install
+          echo 'usr/lib/systemd/system/netplan-configure.service' >> debian/netplan-generator.install
           TAG=$(git describe --tags $(git rev-list --tags --max-count=1))  # find latest (stable) tag
           REV=$(git rev-parse --short HEAD)  # get current git revision
           VER="$TAG+git~$REV"
@@ -58,5 +59,6 @@ jobs:
           autopkgtest . -U \
             --env=DPKG_GENSYMBOLS_CHECK_LEVEL=0 \
             --env=NETPLAN_PARSER_IGNORE_ERRORS=1 \
+            --setup-commands='mkdir -p /etc/systemd/system/NetworkManager.service.d && echo "[Service]\nCapabilityBoundingSet=CAP_CHOWN\n" > /etc/systemd/system/NetworkManager.service.d/override.conf && systemctl daemon-reload' \
             -- lxd autopkgtest/ubuntu/noble/amd64 \
             || test $? -eq 2  # allow wifi test to be skipped (exit code = 2)

.github/workflows/debci.yml

@@ -50,7 +50,8 @@ jobs:
           dget -u "https://deb.debian.org/debian/pool/main/n/netplan.io/netplan.io_$V.dsc"
           cp -r netplan.io-*/debian .
           rm -r debian/patches/  # clear any distro patches
-          sed -i 's|iproute2,|iproute2, ethtool,|' debian/control  # add ethtool as a dependency of netplan.io temporarily
+          echo 'usr/libexec/netplan/configure' >> debian/netplan-generator.install
+          echo 'usr/lib/systemd/system/netplan-configure.service' >> debian/netplan-generator.install
           TAG=$(git describe --tags $(git rev-list --tags --max-count=1))  # find latest (stable) tag
           REV=$(git rev-parse --short HEAD)  # get current git revision
           VER="$TAG+git~$REV"

.github/workflows/network-manager.yml

@@ -42,6 +42,8 @@ jobs:
           cp -r netplan.io-*/debian .
           rm -r debian/patches/  # clear any distro patches
           sed -i 's|systemd (>= 257.2-3ubuntu1~),|systemd (>= 248~),|g' debian/control  # see https://github.com/canonical/netplan/pull/535
+          echo 'usr/libexec/netplan/configure' >> debian/netplan-generator.install
+          echo 'usr/lib/systemd/system/netplan-configure.service' >> debian/netplan-generator.install
           echo "3.0 (native)" > debian/source/format  # force native build
           TAG=$(git describe --tags $(git rev-list --tags --max-count=1))  # find latest (stable) tag
           REV=$(git rev-parse --short HEAD)  # get current git revision
@@ -64,8 +66,10 @@ jobs:
       - name: Run autopkgtest
         run: |
           pull-lp-source network-manager noble
+          cd network-manager*/
+          patch src/core/settings/plugins/keyfile/nms-keyfile-utils.c ../.github/workflows/nm-netplan-configure.diff
           sudo autopkgtest -U \
-            --copy=debian/artifacts:/root/ --setup-commands='dpkg -i /root/*.deb' \
+            --copy=../debian/artifacts:/root/ --setup-commands='dpkg -i /root/*.deb' \
             --env=DEB_BUILD_OPTIONS=nocheck \
             --apt-pocket=proposed=src:network-manager \
-            network-manager_*.dsc -- lxd autopkgtest/ubuntu/noble/amd64 || test $? -eq 2  # allow for skipped tests (exit code = 2)
+            . -- lxd autopkgtest/ubuntu/noble/amd64 || test $? -eq 2  # allow for skipped tests (exit code = 2)

.github/workflows/nm-netplan-configure.diff

@@ -0,0 +1,13 @@
+diff --git a/src/core/settings/plugins/keyfile/nms-keyfile-utils.c b/src/core/settings/plugins/keyfile/nms-keyfile-utils.c
+index a91ee6997..5135aed4c 100644
+--- a/src/core/settings/plugins/keyfile/nms-keyfile-utils.c
++++ b/src/core/settings/plugins/keyfile/nms-keyfile-utils.c
+@@ -408,7 +408,7 @@ generate_netplan(const char* rootdir)
+      * finding a way to pass the --root-dir parameter via DBus, to make it work
+      * inside NM's unit-tests where netplan needs to read & generate outside of
+      * /etc/netplan and /run/{systemd,NetworkManager} */
+-    const gchar *argv[] = { "netplan", "generate", NULL , NULL, NULL };
++    const gchar *argv[] = { "/usr/libexec/netplan/configure", NULL , NULL, NULL };
+     if (rootdir) {
+         argv[2] = "--root-dir";
+         argv[3] = rootdir;

doc/security.md

@@ -30,11 +30,11 @@ units that might contain arbitrary content, for example using the `other-config`
 or `external-ids`. Make sure not to put any secrets into those fields, as those
 will become world-readable.
 
-* `/run/systemd/system/netplan-ovs-*.service`
-* `/run/systemd/system/netplan-sriov-*.service`
-* `/run/systemd/system/netplan-regdom.service`
-* `/run/systemd/system/netplan-wpa-*.service`
-* `/run/systemd/system/systemd-networkd-wait-online.service.d/10-netplan*.conf`
+* `/run/systemd/generator.late/netplan-ovs-*.service`
+* `/run/systemd/generator.late/netplan-sriov-*.service`
+* `/run/systemd/generator.late/netplan-regdom.service`
+* `/run/systemd/generator.late/netplan-wpa-*.service`
+* `/run/systemd/generator.late/systemd-networkd-wait-online.service.d/10-netplan*.conf`
 
 ## Cryptography
 

meson.build

@@ -61,13 +61,20 @@ install_data(
     rename: 'netplan',
     install_dir: bash_completions_dir)
 
+service_dir = systemd.get_pkgconfig_variable('systemdsystemunitdir')
+# XXX: should we use configure_file() instead?
+install_data(
+    'netplan-configure.service',
+    install_dir: service_dir)
+
 ###########
 # Testing #
 ###########
 test_env = [
     'PYTHONPATH=' + join_paths(meson.current_build_dir(), 'python-cffi') + ':' + meson.current_source_dir(),
     'LD_LIBRARY_PATH=' + join_paths(meson.current_build_dir(), 'src'),
     'NETPLAN_GENERATE_PATH=' + join_paths(meson.current_build_dir(), 'src', 'generate'),
+    'NETPLAN_CONFIGURE_PATH=' + join_paths(meson.current_build_dir(), 'src', 'configure'),
     'NETPLAN_DBUS_CMD=' + join_paths(meson.current_build_dir(), 'dbus', 'netplan-dbus'),
     'COVERAGE_PROCESS_START=' + join_paths(meson.current_source_dir(), '.coveragerc'),
     'G_DEBUG=fatal_criticals',

netplan-configure.service

@@ -0,0 +1,16 @@
+[Unit]
+Description=Netplan Backend Configuration
+Documentation=man:netplan(8)
+Documentation=https://netplan.readthedocs.io/en/stable/netplan-yaml/
+
+DefaultDependencies=no
+After=systemd-userdbd.service sysinit.target
+Before=network-pre.target network.target network-online.target nss-lookup.target shutdown.target systemd-networkd.service NetworkManager.service systemd-udevd.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=true
+ExecStart=/usr/libexec/netplan/configure
+
+[Install]
+WantedBy=sysinit.target

netplan_cli/cli/commands/apply.py

@@ -107,9 +107,9 @@ def command_apply(self, run_generate=True, sync=False, exit_on_error=True, state
             else:
                 return
 
-        ovs_cleanup_service = '/run/systemd/system/netplan-ovs-cleanup.service'
+        ovs_cleanup_service = self.generator_late_dir + 'netplan-ovs-cleanup.service'
         old_files_networkd = bool(glob.glob('/run/systemd/network/*netplan-*'))
-        old_ovs_glob = glob.glob('/run/systemd/system/netplan-ovs-*')
+        old_ovs_glob = glob.glob(self.generator_late_dir + 'netplan-ovs-*')
         # Ignore netplan-ovs-cleanup.service, as it can always be there
         if ovs_cleanup_service in old_ovs_glob:
             old_ovs_glob.remove(ovs_cleanup_service)
@@ -118,18 +118,23 @@ def command_apply(self, run_generate=True, sync=False, exit_on_error=True, state
         nm_ifaces = utils.nm_interfaces(old_nm_glob, utils.get_interfaces())
         old_files_nm = bool(old_nm_glob)
 
-        generator_call = []
-        generate_out = None
+        configure = []
+        configure_out = None
         if 'NETPLAN_PROFILE' in os.environ:
-            generator_call.extend(['valgrind', '--leak-check=full'])
-            generate_out = subprocess.STDOUT
+            configure.extend(['valgrind', '--leak-check=full'])
+            configure_out = subprocess.STDOUT
 
-        generator_call.append(utils.get_generator_path())
-        if run_generate and subprocess.call(generator_call, stderr=generate_out) != 0:
-            if exit_on_error:
-                sys.exit(os.EX_CONFIG)
-            else:
-                raise ConfigurationError("the configuration could not be generated")
+        configure.append(utils.get_configure_path())
+        if run_generate:
+            logging.debug('command configure: running %s', configure)
+            if subprocess.call(configure, stderr=configure_out) != 0:
+                if exit_on_error:
+                    sys.exit(os.EX_CONFIG)
+                else:
+                    raise ConfigurationError("the configuration could not be generated")
+            # Running 'systemctl daemon-reload' will re-run the netplan systemd generator.
+            logging.debug('executing Netplan systemd-generator via daemon-reload')
+            utils.systemctl_daemon_reload()
 
         devices = utils.get_interfaces()
 
@@ -142,7 +147,7 @@ def command_apply(self, run_generate=True, sync=False, exit_on_error=True, state
         restart_networkd = bool(glob.glob('/run/systemd/network/*netplan-*'))
         if not restart_networkd and old_files_networkd:
             restart_networkd = True
-        restart_ovs_glob = glob.glob('/run/systemd/system/netplan-ovs-*')
+        restart_ovs_glob = glob.glob(self.generator_late_dir + 'netplan-ovs-*')
         # Ignore netplan-ovs-cleanup.service, as it can always be there
         if ovs_cleanup_service in restart_ovs_glob:
             restart_ovs_glob.remove(ovs_cleanup_service)
@@ -157,10 +162,6 @@ def command_apply(self, run_generate=True, sync=False, exit_on_error=True, state
         if not restart_nm and old_files_nm:
             restart_nm = True
 
-        # Running 'systemctl daemon-reload' will re-run the netplan systemd generator,
-        # so let's make sure we only run it iff we're willing to run 'netplan generate'
-        if run_generate:
-            utils.systemctl_daemon_reload()
         # stop backends
         if restart_networkd:
             logging.debug('netplan generated networkd configuration changed, reloading networkd')
@@ -264,13 +265,13 @@ def command_apply(self, run_generate=True, sync=False, exit_on_error=True, state
         NetplanApply.process_sriov_config(config_manager, exit_on_error)
 
         # (re)set global regulatory domain
-        if os.path.exists('/run/systemd/system/netplan-regdom.service'):
+        if os.path.exists(self.generator_late_dir + 'netplan-regdom.service'):
             utils.systemctl('start', ['netplan-regdom.service'])
         # (re)start backends
         if restart_networkd:
-            netplan_wpa = [os.path.basename(f) for f in glob.glob('/run/systemd/system/*.wants/netplan-wpa-*.service')]
+            netplan_wpa = [os.path.basename(f) for f in glob.glob(self.generator_late_dir + '*.wants/netplan-wpa-*.service')]
             # exclude the special 'netplan-ovs-cleanup.service' unit
-            netplan_ovs = [os.path.basename(f) for f in glob.glob('/run/systemd/system/*.wants/netplan-ovs-*.service')
+            netplan_ovs = [os.path.basename(f) for f in glob.glob(self.generator_late_dir + '*.wants/netplan-ovs-*.service')
                            if not f.endswith('/' + OVS_CLEANUP_SERVICE)]
             # Run 'systemctl start' command synchronously, to avoid race conditions
             # with 'oneshot' systemd service units, e.g. netplan-ovs-*.service.

netplan_cli/cli/commands/generate.py

@@ -75,24 +75,36 @@ def command_generate(self):
             else:
                 return
 
-        argv = [utils.get_generator_path()]
+        argv = [utils.get_configure_path()]
         if self.root_dir:
             argv += ['--root-dir', self.root_dir]
         if self.mapping:
             argv += ['--mapping', self.mapping]
-        logging.debug('command generate: running %s', argv)
-        res = subprocess.call(argv)
-        try:
-            subprocess.check_call(['udevadm', 'control', '--reload'])
-        except subprocess.CalledProcessError as e:
-            logging.debug(f'Could not call "udevadm control --reload": {str(e)}')
-        # reload systemd, as we might have changed service units, such as
-        # /run/systemd/system/systemd-networkd-wait-online.service.d/10-netplan.conf
-        # Skip it if --mapping is used as nothing will be generated
-        if self.mapping is None:
-            try:
+
+        if self.mapping:  # XXX: get rid of the legacy "--mapping" option
+            argv[0] = utils.get_generator_path()
+            res = subprocess.call(argv)
+        else:
+            logging.debug('executing Netplan systemd-generator via daemon-reload')
+            if self.root_dir:  # for testing purposes
+                sd_generator = os.path.join(self.root_dir, 'usr', 'lib', 'systemd', 'system-generators', 'netplan')
+                generator_dir = os.path.join(self.root_dir, 'run', 'systemd', 'generator')
+                generator_early_dir = os.path.join(self.root_dir, 'run', 'systemd', 'generator.early')
+                generator_late_dir = os.path.join(self.root_dir, 'run', 'systemd', 'generator.late')
+                subprocess.check_call([sd_generator, '--root-dir', self.root_dir,
+                                       generator_dir, generator_early_dir, generator_late_dir])
+            else:  # pragma: nocover (covered by autopkgtests)
+                # automatically reloads systemd, as we might have changed
+                # service units, such as
+                # /run/systemd/generator.late/systemd-networkd-wait-online.service.d/10-netplan.conf
                 utils.systemctl_daemon_reload()
+
+            logging.debug('command configure: running %s', argv)
+            res = subprocess.call(argv)
+            try:
+                subprocess.check_call(['udevadm', 'control', '--reload'])
             except subprocess.CalledProcessError as e:
-                logging.warning(e)
+                logging.debug(f'Could not call "udevadm control --reload": {str(e)}')
+
         # FIXME: os.execv(argv[0], argv) would be better but fails coverage
         sys.exit(res)

netplan_cli/cli/commands/ip.py

@@ -132,7 +132,7 @@ def lease_method_nm_connection():
                       file=sys.stderr)
                 sys.exit(1)
 
-        argv = [utils.get_generator_path()]
+        argv = [utils.get_generator_path()]  # FIXME: should this be moved to the 'configure' binary?
         if self.root_dir:
             argv += ['--root-dir', self.root_dir]
         argv += ['--mapping', self.interface]

netplan_cli/cli/utils.py

@@ -44,6 +44,10 @@ def get_generator_path():
     return os.environ.get('NETPLAN_GENERATE_PATH', '/usr/libexec/netplan/generate')
 
 
+def get_configure_path():
+    return os.environ.get('NETPLAN_CONFIGURE_PATH', '/usr/libexec/netplan/configure')
+
+
 def is_nm_snap_enabled():
     return subprocess.call(['systemctl', '--quiet', 'is-enabled', NM_SNAP_SERVICE_NAME], stderr=subprocess.DEVNULL) == 0
 
@@ -293,6 +297,9 @@ def __init__(self, command_id, description, leaf=True, testing=False):
         self.subcommands = {}
         self.subcommand = None
         self.func = None
+        self.generator_dir = '/run/systemd/generator/'
+        self.generator_early_dir = '/run/systemd/generator.early/'
+        self.generator_late_dir = '/run/systemd/generator.late/'
 
         self.parser = argparse.ArgumentParser(prog="%s %s" % (sys.argv[0], command_id),
                                               description=description,

rpm/netplan.spec

@@ -95,6 +95,8 @@ Currently supported backends are NetworkManager and systemd-networkd.
 %dir %{_sysconfdir}/%{name}
 %dir %{_libexecdir}/%{name}
 %{_libexecdir}/%{name}/generate
+%{_libexecdir}/%{name}/configure
+%{_prefix}/lib/systemd/system/netplan-configure.service
 %{_libexecdir}/%{name}/%{name}-dbus
 %{_datadir}/bash-completion/completions/%{name}