From fbead99292d5e05f79b90228ad102dda59e15228 Mon Sep 17 00:00:00 2001 From: apostasie Date: Mon, 26 May 2025 10:44:20 -0700 Subject: [PATCH 1/2] Remove useless MkdirAll Signed-off-by: apostasie --- pkg/clientutil/client.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/pkg/clientutil/client.go b/pkg/clientutil/client.go index 7304f7a44ec..d50f6f16852 100644 --- a/pkg/clientutil/client.go +++ b/pkg/clientutil/client.go @@ -82,9 +82,6 @@ func NewClientWithPlatform(ctx context.Context, namespace, address, platform str // "1935db9" is from `$(echo -n "/run/containerd/containerd.sock" | sha256sum | cut -c1-8)` // on Windows it will return "%PROGRAMFILES%/nerdctl/1935db59" func DataStore(dataRoot, address string) (string, error) { - if err := os.MkdirAll(dataRoot, 0700); err != nil { - return "", err - } addrHash, err := getAddrHash(address) if err != nil { return "", err From 1dccbdee3c5172ffa48d33e10c4a2159ccb9325e Mon Sep 17 00:00:00 2001 From: apostasie Date: Mon, 26 May 2025 15:49:05 -0700 Subject: [PATCH 2/2] Cleanup annotations: move to separate pkg Signed-off-by: apostasie --- pkg/annotations/annotations.go | 359 ++++++++++++++- pkg/annotations/keys.go | 125 ++++++ pkg/bypass4netnsutil/bypass.go | 20 +- pkg/bypass4netnsutil/bypass4netnsutil.go | 19 - pkg/cmd/container/create.go | 410 +++++------------- pkg/cmd/container/run_cgroup_linux.go | 5 +- pkg/cmd/container/run_linux.go | 11 +- pkg/cmd/container/run_unix_nolinux.go | 3 +- pkg/cmd/container/run_windows.go | 3 +- .../container_network_manager_windows.go | 8 +- pkg/ocihook/ocihook.go | 138 ++---- 11 files changed, 642 insertions(+), 459 deletions(-) create mode 100644 pkg/annotations/keys.go diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index 97c1edd56ef..2792d232281 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -14,31 +14,350 @@ limitations under the License. */ -// Package annotations defines OCI annotations +// Package annotations provides a high-level interface to retrieve and manipulate containers OCI annotations. +// Any parsing, encoding or transformation of the struct into concrete containers annotations must be handled here, +// and the containers underlying annotations map should never be accessed directly by consuming code, to ensure +// we can evolve the storage format in a single place and consuming code to be fully isolated from the details of +// how it is being handled. +// Container annotations generally should be used to store state information or other container properties that +// we may want to inspect at different stages of the container lifecycle. package annotations -const ( - // Prefix is the common prefix of nerdctl annotations - Prefix = "nerdctl/" +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" - // Bypass4netns is the flag for acceleration with bypass4netns - // Boolean value which can be parsed with strconv.ParseBool() is required. - // (like "nerdctl/bypass4netns=true" or "nerdctl/bypass4netns=false") - Bypass4netns = Prefix + "bypass4netns" + "github.com/opencontainers/runtime-spec/specs-go" - // Bypass4netnsIgnoreSubnets is a JSON of []string that is appended to - // the `bypass4netns --ignore` list. - Bypass4netnsIgnoreSubnets = Bypass4netns + "-ignore-subnets" + containerd "github.com/containerd/containerd/v2/client" + "github.com/containerd/go-cni" - // Bypass4netnsIgnoreBind disables acceleration for bind. - // Boolean value which can be parsed with strconv.ParseBool() is required. - Bypass4netnsIgnoreBind = Bypass4netns + "-ignore-bind" + "github.com/containerd/nerdctl/v2/pkg/inspecttypes/dockercompat" + "github.com/containerd/nerdctl/v2/pkg/mountutil" + "github.com/containerd/nerdctl/v2/pkg/platformutil" ) -var ShellCompletions = []string{ - Bypass4netns + "=true", - Bypass4netns + "=false", - Bypass4netnsIgnoreSubnets + "=", - Bypass4netnsIgnoreBind + "=true", - Bypass4netnsIgnoreBind + "=false", +func New(ctx context.Context, con containerd.Container) (*Annotations, error) { + an := &Annotations{} + err := an.Unmarshall(ctx, con) + return an, err +} + +func NewFromState(state *specs.State) (*Annotations, error) { + an := &Annotations{} + if state == nil || state.Annotations == nil { + return nil, fmt.Errorf("invalid state") + } + err := an.UnmarshallFromMap(state.Annotations) + return an, err +} + +type Log struct { + Driver string + Opts map[string]string + Address string +} + +type Annotations struct { + AnonVolumes []string + Bypass4netns bool + Bypass4netnsIgnoreBind bool + Bypass4netnsIgnoreSubnets []string + CidFile string + DeviceMapping []dockercompat.DeviceMapping + DNSResolvConfOptions []string + DNSSearchDomains []string + DNSServers []string + DomainName string + ExtraHosts map[string]string + // GroupAdd []string + HostName string + IPC string + LogConfig *Log + LogURI string + MountPoints []*mountutil.Processed + Name string + Namespace string + NetworkNamespace string + Networks []string + Platform string + StateDir string + PidContainer string + PidFile string + Ports []cni.PortMapping + Rm bool + User string + + // FIXME: these should be replaced by a richer Network label allowing per network ip and mac + IP6Address string + IPAddress string + MACAddress string +} + +func (an *Annotations) UnmarshallFromMap(source map[string]string) error { + hostConfig := &dockercompat.HostConfig{} + dnsConfig := &dockercompat.DNSSettings{} + for k, v := range source { + switch k { + case AnonymousVolumes: + _ = json.Unmarshal([]byte(v), &an.AnonVolumes) + case Bypass4netns: + an.Bypass4netns, _ = strconv.ParseBool(v) + case Bypass4netnsIgnoreBind: + an.Bypass4netnsIgnoreBind, _ = strconv.ParseBool(v) + case Bypass4netnsIgnoreSubnets: + _ = json.Unmarshal([]byte(v), &an.Bypass4netnsIgnoreSubnets) + case DNSSetting: + _ = json.Unmarshal([]byte(v), dnsConfig) + an.DNSServers = dnsConfig.DNSServers + an.DNSSearchDomains = dnsConfig.DNSSearchDomains + an.DNSResolvConfOptions = dnsConfig.DNSResolvConfOptions + case Domainname: + an.DomainName = v + case ExtraHosts: + hosts := []string{} + _ = json.Unmarshal([]byte(v), &hosts) + if an.ExtraHosts == nil { + an.ExtraHosts = map[string]string{} + } + for _, host := range hosts { + if v := strings.SplitN(host, ":", 2); len(v) == 2 { + an.ExtraHosts[v[0]] = v[1] + } + } + + case HostConfig: + _ = json.Unmarshal([]byte(v), hostConfig) + an.CidFile = hostConfig.ContainerIDFile + // = hostConfig.CgroupnsMode + case Hostname: + an.HostName = v + case IP6Address: + an.IP6Address = v + case IPAddress: + an.IPAddress = v + case IPC: + an.IPC = v + case LogConfig: + _ = json.Unmarshal([]byte(v), &an.LogConfig) + case LogURI: + an.LogURI = v + case MACAddress: + an.MACAddress = v + case Mounts: + _ = json.Unmarshal([]byte(v), &an.MountPoints) + case Name: + an.Name = v + case Namespace: + an.Namespace = v + case NetworkNamespace: + an.NetworkNamespace = v + case Networks: + _ = json.Unmarshal([]byte(v), &an.Networks) + case PIDContainer: + an.PidContainer = v + case PIDFile: + an.PidFile = v + case Platform: + an.Platform = v + case Ports: + _ = json.Unmarshal([]byte(v), &an.Ports) + case ContainerAutoRemove: + an.Rm, _ = strconv.ParseBool(v) + case StateDir: + an.StateDir = v + // FIXME: add missing, including NetworkNamespace + case User: + an.User = v + default: + } + } + + return nil +} + +func (an *Annotations) Unmarshall(ctx context.Context, con containerd.Container) error { + conSpecs, err := con.Spec(ctx) + if err != nil || conSpecs.Annotations == nil { + return err + } + + return an.UnmarshallFromMap(conSpecs.Annotations) +} + +func (an *Annotations) Marshall() (map[string]string, error) { + annot := make(map[string]string) + + var ( + err error + hostConfig dockercompat.HostConfigLabel + dnsSettings dockercompat.DNSSettings + ) + + if len(an.AnonVolumes) > 0 { + anonVolumeJSON, err := json.Marshal(an.AnonVolumes) + if err != nil { + return nil, err + } + annot[AnonymousVolumes] = string(anonVolumeJSON) + } + + if an.CidFile != "" { + hostConfig.CidFile = an.CidFile + } + + if len(an.DeviceMapping) > 0 { + hostConfig.Devices = append(hostConfig.Devices, an.DeviceMapping...) + } + + if len(an.DNSResolvConfOptions) > 0 { + dnsSettings.DNSResolvConfOptions = an.DNSResolvConfOptions + } + + if len(an.DNSServers) > 0 { + dnsSettings.DNSServers = an.DNSServers + } + + if len(an.DNSSearchDomains) > 0 { + dnsSettings.DNSSearchDomains = an.DNSSearchDomains + } + + dnsSettingsJSON, err := json.Marshal(dnsSettings) + if err != nil { + return nil, err + } + annot[DNSSetting] = string(dnsSettingsJSON) + + annot[Domainname] = an.DomainName + + //if len(an.GroupAdd) > 0 { + //} + + hosts := []string{} + for k, v := range an.ExtraHosts { + hosts = append(hosts, fmt.Sprintf("%s:%s", k, v)) + } + extraHostsJSON, err := json.Marshal(hosts) + if err != nil { + return nil, err + } + annot[ExtraHosts] = string(extraHostsJSON) + + hostConfigJSON, err := json.Marshal(hostConfig) + if err != nil { + return nil, err + } + annot[HostConfig] = string(hostConfigJSON) + + annot[Hostname] = an.HostName + + if an.IP6Address != "" { + annot[IP6Address] = an.IP6Address + } + + if an.IPAddress != "" { + annot[IPAddress] = an.IPAddress + } + + if an.IPC != "" { + annot[IPC] = an.IPC + } + + if an.LogURI != "" { + annot[LogURI] = an.LogURI + + if an.LogConfig != nil { + logConfigJSON, err := json.Marshal(an.LogConfig) + if err != nil { + return nil, err + } + + annot[LogConfig] = string(logConfigJSON) + } + } + + if an.MACAddress != "" { + annot[MACAddress] = an.MACAddress + } + + if len(an.MountPoints) > 0 { + mounts := dockercompatMounts(an.MountPoints) + mountPointsJSON, err := json.Marshal(mounts) + if err != nil { + return nil, err + } + annot[Mounts] = string(mountPointsJSON) + } + + annot[Name] = an.Name + annot[Namespace] = an.Namespace + + networksJSON, err := json.Marshal(an.Networks) + if err != nil { + return nil, err + } + annot[Networks] = string(networksJSON) + + annot[Platform], err = platformutil.NormalizeString(an.Platform) + if err != nil { + return nil, err + } + + if an.PidFile != "" { + annot[PIDFile] = an.PidFile + } + + if len(an.Ports) > 0 { + portsJSON, err := json.Marshal(an.Ports) + if err != nil { + return nil, err + } + + annot[Ports] = string(portsJSON) + } + + if an.PidContainer != "" { + annot[PIDContainer] = an.PidContainer + } + + annot[ContainerAutoRemove] = fmt.Sprintf("%t", an.Rm) + + annot[StateDir] = an.StateDir + + if an.User != "" { + annot[User] = an.User + } + + return annot, nil +} + +func dockercompatMounts(mountPoints []*mountutil.Processed) []dockercompat.MountPoint { + result := make([]dockercompat.MountPoint, len(mountPoints)) + for i := range mountPoints { + mp := mountPoints[i] + result[i] = dockercompat.MountPoint{ + Type: mp.Type, + Name: mp.Name, + Source: mp.Mount.Source, + Destination: mp.Mount.Destination, + Driver: "", + Mode: mp.Mode, + } + result[i].RW, result[i].Propagation = dockercompat.ParseMountProperties(strings.Split(mp.Mode, ",")) + + // it's an anonymous volume + if mp.AnonymousVolume != "" { + result[i].Name = mp.AnonymousVolume + } + + // volume only support local driver + if mp.Type == "volume" { + result[i].Driver = "local" + } + } + + return result } diff --git a/pkg/annotations/keys.go b/pkg/annotations/keys.go new file mode 100644 index 00000000000..0a378690eb6 --- /dev/null +++ b/pkg/annotations/keys.go @@ -0,0 +1,125 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// Package annotations defines OCI annotations +package annotations + +const ( + // prefix is the common prefix of nerdctl annotations + prefix = "nerdctl/" + + // AnonymousVolumes is a JSON-marshalled string of []string + AnonymousVolumes = prefix + "anonymous-volumes" + + // Bypass4netns is the flag for acceleration with bypass4netns + // Boolean value which can be parsed with strconv.ParseBool() is required. + // (like "nerdctl/bypass4netns=true" or "nerdctl/bypass4netns=false") + Bypass4netns = prefix + "bypass4netns" + + // Bypass4netnsIgnoreSubnets is a JSON of []string that is appended to + // the `bypass4netns --ignore` list. + Bypass4netnsIgnoreSubnets = Bypass4netns + "-ignore-subnets" + + // Bypass4netnsIgnoreBind disables acceleration for bind. + // Boolean value which can be parsed with strconv.ParseBool() is required. + Bypass4netnsIgnoreBind = Bypass4netns + "-ignore-bind" + + // ContainerAutoRemove is to check whether the --rm option is specified. + ContainerAutoRemove = prefix + "auto-remove" + + // Domainname + Domainname = prefix + "domainname" + + // DNSSettings sets the dockercompat DNS config values + DNSSetting = prefix + "dns" + + // ExtraHosts are HostIPs to appended to /etc/hosts + ExtraHosts = prefix + "extraHosts" + + // HostConfig sets the dockercompat host config values + HostConfig = prefix + "host-config" + + // Hostname + Hostname = prefix + "hostname" + + // IP6Address is the static IP6 address of the container assigned by the user + IP6Address = prefix + "ip6" + + // IPAddress is the static IP address of the container assigned by the user + IPAddress = prefix + "ip" + + // IPC is the `nerectl run --ipc` for restrating + // IPC indicates ipc victim container. + IPC = prefix + "ipc" + + // LogConfig defines the logging configuration passed to the container + LogConfig = prefix + "log-config" + + // LogURI is the log URI + LogURI = prefix + "log-uri" + + MACAddress = prefix + "mac-address" + + // Mounts is the mount points for the container. + Mounts = prefix + "mounts" + + Name = prefix + "name" + + // Namespace is the containerd namespace such as "default", "k8s.io" + Namespace = prefix + "namespace" + + // NetworkNamespace is the network namespace path to be passed to the CNI plugins. + // When this annotation is set from the runtime spec.State payload, it takes + // precedence over the PID based resolution (/proc//ns/net) where pid is + // spec.State.Pid. + // This is mostly used for VM based runtime, where the spec.State PID does not + // necessarily lives in the created container networking namespace. + // + // On Windows, this label will contain the UUID of a namespace managed by + // the Host Compute Network Service (HCN) API. + NetworkNamespace = prefix + "network-namespace" + + // Networks is a JSON-marshalled string of []string, e.g. []string{"bridge"}. + // Currently, the length of the slice must be 1. + Networks = prefix + "networks" + + // PIDContainer is the `nerdctl run --pid` for restarting + PIDContainer = prefix + "pid-container" + + // PIDFile is the `nerdctl run --pidfile` + // (CLI flag is "pidfile", not "pid-file", for Podman compatibility) + PIDFile = prefix + "pid-file" + + // Platform is the normalized platform string like "linux/ppc64le". + Platform = prefix + "platform" + + // Ports is a JSON-marshalled string of []cni.PortMapping . + Ports = prefix + "ports" + + // StateDir is "/var/lib/nerdctl//containers//" + StateDir = prefix + "state-dir" + + // User is the username of the container + User = prefix + "user" +) + +var ShellCompletions = []string{ + Bypass4netns + "=true", + Bypass4netns + "=false", + Bypass4netnsIgnoreSubnets + "=", + Bypass4netnsIgnoreBind + "=true", + Bypass4netnsIgnoreBind + "=false", +} diff --git a/pkg/bypass4netnsutil/bypass.go b/pkg/bypass4netnsutil/bypass.go index bc9eed11f9d..4765ee34f14 100644 --- a/pkg/bypass4netnsutil/bypass.go +++ b/pkg/bypass4netnsutil/bypass.go @@ -18,7 +18,6 @@ package bypass4netnsutil import ( "context" - "encoding/json" "fmt" "net" "path/filepath" @@ -33,28 +32,21 @@ import ( "github.com/containerd/nerdctl/v2/pkg/annotations" ) -func NewBypass4netnsCNIBypassManager(client client.Client, rlkClient rlkclient.Client, annotationsMap map[string]string) (*Bypass4netnsCNIBypassManager, error) { +func NewBypass4netnsCNIBypassManager(client client.Client, rlkClient rlkclient.Client, containerAnnotations *annotations.Annotations) (*Bypass4netnsCNIBypassManager, error) { if client == nil || rlkClient == nil { return nil, errdefs.ErrInvalidArgument } - enabled, bindEnabled, err := IsBypass4netnsEnabled(annotationsMap) - if err != nil { - return nil, err - } + + enabled := containerAnnotations.Bypass4netns if !enabled { return nil, errdefs.ErrInvalidArgument } - var ignoreSubnets []string - if v := annotationsMap[annotations.Bypass4netnsIgnoreSubnets]; v != "" { - if err := json.Unmarshal([]byte(v), &ignoreSubnets); err != nil { - return nil, fmt.Errorf("failed to unmarshal annotation %q: %q: %w", annotations.Bypass4netnsIgnoreSubnets, v, err) - } - } + pm := &Bypass4netnsCNIBypassManager{ Client: client, rlkClient: rlkClient, - ignoreSubnets: ignoreSubnets, - ignoreBind: !bindEnabled, + ignoreSubnets: containerAnnotations.Bypass4netnsIgnoreSubnets, + ignoreBind: containerAnnotations.Bypass4netnsIgnoreBind, } return pm, nil } diff --git a/pkg/bypass4netnsutil/bypass4netnsutil.go b/pkg/bypass4netnsutil/bypass4netnsutil.go index 29af68dc366..29a0d4d28e0 100644 --- a/pkg/bypass4netnsutil/bypass4netnsutil.go +++ b/pkg/bypass4netnsutil/bypass4netnsutil.go @@ -133,22 +133,3 @@ func GetPidFilePathByID(id string) (string, error) { return pidPath, nil } - -func IsBypass4netnsEnabled(annotationsMap map[string]string) (enabled, bindEnabled bool, err error) { - if b4nn, ok := annotationsMap[annotations.Bypass4netns]; ok { - enabled, err = strconv.ParseBool(b4nn) - if err != nil { - return - } - bindEnabled = enabled - if s, ok := annotationsMap[annotations.Bypass4netnsIgnoreBind]; ok { - var bindDisabled bool - bindDisabled, err = strconv.ParseBool(s) - if err != nil { - return - } - bindEnabled = !bindDisabled - } - } - return -} diff --git a/pkg/cmd/container/create.go b/pkg/cmd/container/create.go index a0c8fc2bf9b..9c25386dca9 100644 --- a/pkg/cmd/container/create.go +++ b/pkg/cmd/container/create.go @@ -89,9 +89,10 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa newArg = append(newArg, args[2:]...) args = newArg } - var internalLabels internalLabels - internalLabels.platform = options.Platform - internalLabels.namespace = options.GOptions.Namespace + + internalLabels := &annotations.Annotations{} + internalLabels.Platform = options.Platform + internalLabels.Namespace = options.GOptions.Namespace var ( id = idgen.GenerateID() @@ -103,18 +104,18 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa if err := writeCIDFile(options.CidFile, id); err != nil { return nil, nil, err } - internalLabels.cidFile = options.CidFile + internalLabels.CidFile = options.CidFile } dataStore, err := clientutil.DataStore(options.GOptions.DataRoot, options.GOptions.Address) if err != nil { return nil, nil, err } - internalLabels.stateDir, err = containerutil.ContainerStateDirPath(options.GOptions.Namespace, dataStore, id) + internalLabels.StateDir, err = containerutil.ContainerStateDirPath(options.GOptions.Namespace, dataStore, id) if err != nil { return nil, nil, err } - if err := os.MkdirAll(internalLabels.stateDir, 0700); err != nil { + if err := os.MkdirAll(internalLabels.StateDir, 0700); err != nil { return nil, nil, err } @@ -122,7 +123,7 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa oci.WithDefaultSpec(), ) - platformOpts, err := setPlatformOptions(ctx, client, id, netManager.NetworkOptions().UTSNamespace, &internalLabels, options) + platformOpts, err := setPlatformOptions(ctx, client, id, netManager.NetworkOptions().UTSNamespace, internalLabels, options) if err != nil { return nil, generateRemoveStateDirFunc(ctx, id, internalLabels), err } @@ -186,12 +187,12 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa } if ensuredImage != nil && ensuredImage.ImageConfig.User != "" { - internalLabels.user = ensuredImage.ImageConfig.User + internalLabels.User = ensuredImage.ImageConfig.User } // Override it if User is passed if options.User != "" { - internalLabels.user = options.User + internalLabels.User = options.User } rootfsOpts, rootfsCOpts, err := generateRootfsOpts(args, id, ensuredImage, options) @@ -250,7 +251,7 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa } var mountOpts []oci.SpecOpts - mountOpts, internalLabels.anonVolumes, internalLabels.mountPoints, err = generateMountOpts(ctx, client, ensuredImage, volStore, options) + mountOpts, internalLabels.AnonVolumes, internalLabels.MountPoints, err = generateMountOpts(ctx, client, ensuredImage, volStore, options) if err != nil { return nil, generateRemoveStateDirFunc(ctx, id, internalLabels), err } @@ -266,10 +267,14 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa if err != nil { return nil, generateRemoveStateDirFunc(ctx, id, internalLabels), err } - internalLabels.logURI = logConfig.LogURI - internalLabels.logConfig = logConfig + internalLabels.LogURI = logConfig.LogURI + internalLabels.LogConfig = &annotations.Log{ + Driver: logConfig.Driver, + Opts: logConfig.Opts, + Address: logConfig.Address, + } if logConfig.Driver == "" && logConfig.Address == options.GOptions.Address { - internalLabels.logConfig.Driver = "json-file" + internalLabels.LogConfig.Driver = "json-file" } restartOpts, err := generateRestartOpts(ctx, client, options.Restart, logConfig.LogURI, options.InRun) @@ -297,7 +302,16 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa envs = append(envs, "HOSTNAME="+netLabelOpts.Hostname) opts = append(opts, oci.WithEnv(envs)) - internalLabels.loadNetOpts(netLabelOpts) + internalLabels.HostName = netLabelOpts.Hostname + internalLabels.DomainName = netLabelOpts.Domainname + internalLabels.Ports = netLabelOpts.PortMappings + internalLabels.IPAddress = netLabelOpts.IPAddress + internalLabels.IP6Address = netLabelOpts.IP6Address + internalLabels.Networks = netLabelOpts.NetworkSlice + internalLabels.MACAddress = netLabelOpts.MACAddress + internalLabels.DNSServers = netLabelOpts.DNSServers + internalLabels.DNSSearchDomains = netLabelOpts.DNSSearchDomains + internalLabels.DNSResolvConfOptions = netLabelOpts.DNSResolvConfOptions // NOTE: OCI hooks are currently not supported on Windows so we skip setting them altogether. // The OCI hooks we define (whose logic can be found in pkg/ocihook) primarily @@ -318,7 +332,6 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa opts = append(opts, uOpts...) gOpts, err := generateGroupsOpts(options.GroupAdd) - internalLabels.groupAdd = options.GroupAdd if err != nil { return nil, generateRemoveOrphanedDirsFunc(ctx, id, dataStore, internalLabels), err } @@ -373,23 +386,30 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa return nil, generateRemoveOrphanedDirsFunc(ctx, id, dataStore, internalLabels), err } - internalLabels.name = options.Name - internalLabels.pidFile = options.PidFile + internalLabels.Name = options.Name + internalLabels.PidFile = options.PidFile extraHosts, err := containerutil.ParseExtraHosts(netManager.NetworkOptions().AddHost, options.GOptions.HostGatewayIP, ":") if err != nil { return nil, generateRemoveOrphanedDirsFunc(ctx, id, dataStore, internalLabels), err } - internalLabels.extraHosts = extraHosts + if internalLabels.ExtraHosts == nil { + internalLabels.ExtraHosts = map[string]string{} + } + for _, extraHost := range extraHosts { + if v := strings.SplitN(extraHost, ":", 2); len(v) == 2 { + internalLabels.ExtraHosts[v[0]] = v[1] + } + } - internalLabels.rm = containerutil.EncodeContainerRmOptLabel(options.Rm) + internalLabels.Rm = options.Rm // TODO: abolish internal labels and only use annotations - ilOpt, err := withInternalLabels(internalLabels) + ilOpt, err := internalLabels.Marshall() if err != nil { return nil, generateRemoveOrphanedDirsFunc(ctx, id, dataStore, internalLabels), err } - cOpts = append(cOpts, ilOpt) + cOpts = append(cOpts, containerd.WithAdditionalContainerLabels(ilOpt)) netConf := networkstore.NetworkConfig{ PortMappings: netLabelOpts.PortMappings, @@ -427,6 +447,62 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa return c, nil, nil } +func withHealthcheck(options types.ContainerCreateOptions, ensuredImage *imgutil.EnsuredImage) (string, error) { + // If explicitly disabled + if options.NoHealthcheck { + hc := &healthcheck.Healthcheck{ + Test: []string{"NONE"}, + } + hcJSON, err := hc.ToJSONString() + if err != nil { + return "", fmt.Errorf("failed to serialize disabled healthcheck config: %w", err) + } + return hcJSON, nil + } + + // Start with health checks in image if present + hc := &healthcheck.Healthcheck{} + if ensuredImage != nil && ensuredImage.ImageConfig.Labels != nil { + if label := ensuredImage.ImageConfig.Labels[labels.HealthCheck]; label != "" { + parsed, err := healthcheck.HealthCheckFromJSON(label) + if err != nil { + return "", fmt.Errorf("failed to parse healthcheck label in image: %w", err) + } + hc = parsed + } + } + + // Apply CLI overrides + if options.HealthCmd != "" { + hc.Test = []string{"CMD-SHELL", options.HealthCmd} + } + if options.HealthInterval != 0 { + hc.Interval = options.HealthInterval + } + if options.HealthTimeout != 0 { + hc.Timeout = options.HealthTimeout + } + if options.HealthRetries != 0 { + hc.Retries = options.HealthRetries + } + if options.HealthStartPeriod != 0 { + hc.StartPeriod = options.HealthStartPeriod + } + if options.HealthStartInterval != 0 { + hc.StartInterval = options.HealthStartInterval + } + + // If no healthcheck config is set (via CLI or image), return empty string so we skip adding to container config. + if reflect.DeepEqual(hc, &healthcheck.Healthcheck{}) { + return "", nil + } + hcJSON, err := hc.ToJSONString() + if err != nil { + return "", fmt.Errorf("failed to serialize healthcheck config: %w", err) + } + return hcJSON, nil +} + func generateRootfsOpts(args []string, id string, ensured *imgutil.EnsuredImage, options types.ContainerCreateOptions) (opts []oci.SpecOpts, cOpts []containerd.NewContainerOpts, err error) { if !options.Rootfs { cOpts = append(cOpts, @@ -682,270 +758,6 @@ func withStop(stopSignal string, stopTimeout int, ensuredImage *imgutil.EnsuredI } } -type internalLabels struct { - // labels from cmd options - namespace string - platform string - extraHosts []string - pidFile string - // labels from cmd options or automatically set - name string - hostname string - domainname string - // automatically generated - stateDir string - // network - networks []string - ipAddress string - ip6Address string - macAddress string - dnsServers []string - dnsSearchDomains []string - dnsResolvConfOptions []string - // volume - mountPoints []*mountutil.Processed - anonVolumes []string - // pid namespace - pidContainer string - // ipc namespace & dev/shm - ipc string - // log - logURI string - // a label to check whether the --rm option is specified. - rm string - logConfig logging.LogConfig - - // a label to chek if --cidfile is set - cidFile string - - // label to check if --group-add is set - groupAdd []string - - // label for device mapping set by the --device flag - deviceMapping []dockercompat.DeviceMapping - - user string - - healthcheck string -} - -// WithInternalLabels sets the internal labels for a container. -func withInternalLabels(internalLabels internalLabels) (containerd.NewContainerOpts, error) { - m := make(map[string]string) - var hostConfigLabel dockercompat.HostConfigLabel - var dnsSettings dockercompat.DNSSettings - m[labels.Namespace] = internalLabels.namespace - m[labels.Name] = internalLabels.name - m[labels.Hostname] = internalLabels.hostname - m[labels.Domainname] = internalLabels.domainname - extraHostsJSON, err := json.Marshal(internalLabels.extraHosts) - if err != nil { - return nil, err - } - m[labels.ExtraHosts] = string(extraHostsJSON) - m[labels.StateDir] = internalLabels.stateDir - networksJSON, err := json.Marshal(internalLabels.networks) - if err != nil { - return nil, err - } - m[labels.Networks] = string(networksJSON) - if internalLabels.logURI != "" { - m[labels.LogURI] = internalLabels.logURI - logConfigJSON, err := json.Marshal(internalLabels.logConfig) - if err != nil { - return nil, err - } - m[labels.LogConfig] = string(logConfigJSON) - } - if len(internalLabels.anonVolumes) > 0 { - anonVolumeJSON, err := json.Marshal(internalLabels.anonVolumes) - if err != nil { - return nil, err - } - m[labels.AnonymousVolumes] = string(anonVolumeJSON) - } - - if internalLabels.pidFile != "" { - m[labels.PIDFile] = internalLabels.pidFile - } - - if internalLabels.ipAddress != "" { - m[labels.IPAddress] = internalLabels.ipAddress - } - - if internalLabels.ip6Address != "" { - m[labels.IP6Address] = internalLabels.ip6Address - } - - m[labels.Platform], err = platformutil.NormalizeString(internalLabels.platform) - if err != nil { - return nil, err - } - - if len(internalLabels.mountPoints) > 0 { - mounts := dockercompatMounts(internalLabels.mountPoints) - mountPointsJSON, err := json.Marshal(mounts) - if err != nil { - return nil, err - } - m[labels.Mounts] = string(mountPointsJSON) - } - - if internalLabels.macAddress != "" { - m[labels.MACAddress] = internalLabels.macAddress - } - - if internalLabels.pidContainer != "" { - m[labels.PIDContainer] = internalLabels.pidContainer - } - - if internalLabels.ipc != "" { - m[labels.IPC] = internalLabels.ipc - } - - if internalLabels.rm != "" { - m[labels.ContainerAutoRemove] = internalLabels.rm - } - - if internalLabels.cidFile != "" { - hostConfigLabel.CidFile = internalLabels.cidFile - } - - if len(internalLabels.dnsServers) > 0 { - dnsSettings.DNSServers = internalLabels.dnsServers - } - - if len(internalLabels.dnsSearchDomains) > 0 { - dnsSettings.DNSSearchDomains = internalLabels.dnsSearchDomains - } - - if len(internalLabels.dnsResolvConfOptions) > 0 { - dnsSettings.DNSResolvConfOptions = internalLabels.dnsResolvConfOptions - } - - if len(internalLabels.deviceMapping) > 0 { - hostConfigLabel.Devices = append(hostConfigLabel.Devices, internalLabels.deviceMapping...) - } - - hostConfigJSON, err := json.Marshal(hostConfigLabel) - if err != nil { - return nil, err - } - m[labels.HostConfigLabel] = string(hostConfigJSON) - - dnsSettingsJSON, err := json.Marshal(dnsSettings) - if err != nil { - return nil, err - } - m[labels.DNSSetting] = string(dnsSettingsJSON) - - if internalLabels.user != "" { - m[labels.User] = internalLabels.user - } - - if len(internalLabels.healthcheck) > 0 { - m[labels.HealthCheck] = internalLabels.healthcheck - } - - return containerd.WithAdditionalContainerLabels(m), nil -} - -func withHealthcheck(options types.ContainerCreateOptions, ensuredImage *imgutil.EnsuredImage) (string, error) { - // If explicitly disabled - if options.NoHealthcheck { - hc := &healthcheck.Healthcheck{ - Test: []string{"NONE"}, - } - hcJSON, err := hc.ToJSONString() - if err != nil { - return "", fmt.Errorf("failed to serialize disabled healthcheck config: %w", err) - } - return hcJSON, nil - } - - // Start with health checks in image if present - hc := &healthcheck.Healthcheck{} - if ensuredImage != nil && ensuredImage.ImageConfig.Labels != nil { - if label := ensuredImage.ImageConfig.Labels[labels.HealthCheck]; label != "" { - parsed, err := healthcheck.HealthCheckFromJSON(label) - if err != nil { - return "", fmt.Errorf("failed to parse healthcheck label in image: %w", err) - } - hc = parsed - } - } - - // Apply CLI overrides - if options.HealthCmd != "" { - hc.Test = []string{"CMD-SHELL", options.HealthCmd} - } - if options.HealthInterval != 0 { - hc.Interval = options.HealthInterval - } - if options.HealthTimeout != 0 { - hc.Timeout = options.HealthTimeout - } - if options.HealthRetries != 0 { - hc.Retries = options.HealthRetries - } - if options.HealthStartPeriod != 0 { - hc.StartPeriod = options.HealthStartPeriod - } - if options.HealthStartInterval != 0 { - hc.StartInterval = options.HealthStartInterval - } - - // If no healthcheck config is set (via CLI or image), return empty string so we skip adding to container config. - if reflect.DeepEqual(hc, &healthcheck.Healthcheck{}) { - return "", nil - } - hcJSON, err := hc.ToJSONString() - if err != nil { - return "", fmt.Errorf("failed to serialize healthcheck config: %w", err) - } - return hcJSON, nil -} - -// loadNetOpts loads network options into InternalLabels. -func (il *internalLabels) loadNetOpts(opts types.NetworkOptions) { - il.hostname = opts.Hostname - il.domainname = opts.Domainname - il.ipAddress = opts.IPAddress - il.ip6Address = opts.IP6Address - il.networks = opts.NetworkSlice - il.macAddress = opts.MACAddress - il.dnsServers = opts.DNSServers - il.dnsSearchDomains = opts.DNSSearchDomains - il.dnsResolvConfOptions = opts.DNSResolvConfOptions -} - -func dockercompatMounts(mountPoints []*mountutil.Processed) []dockercompat.MountPoint { - result := make([]dockercompat.MountPoint, len(mountPoints)) - for i := range mountPoints { - mp := mountPoints[i] - result[i] = dockercompat.MountPoint{ - Type: mp.Type, - Name: mp.Name, - Source: mp.Mount.Source, - Destination: mp.Mount.Destination, - Driver: "", - Mode: mp.Mode, - } - result[i].RW, result[i].Propagation = dockercompat.ParseMountProperties(strings.Split(mp.Mode, ",")) - - // it's an anonymous volume - if mp.AnonymousVolume != "" { - result[i].Name = mp.AnonymousVolume - } - - // volume only support local driver - if mp.Type == "volume" { - result[i].Driver = "local" - } - } - return result -} - func processeds(mountPoints []dockercompat.MountPoint) []*mountutil.Processed { result := make([]*mountutil.Processed, len(mountPoints)) for i := range mountPoints { @@ -1041,30 +853,30 @@ func generateLogConfig(dataStore string, id string, logDriver string, logOpt []s return logConfig, nil } -func generateRemoveStateDirFunc(ctx context.Context, id string, internalLabels internalLabels) func() { +func generateRemoveStateDirFunc(ctx context.Context, id string, internalLabels *annotations.Annotations) func() { return func() { - if rmErr := os.RemoveAll(internalLabels.stateDir); rmErr != nil { - log.G(ctx).WithError(rmErr).Warnf("failed to remove container %q state dir %q", id, internalLabels.stateDir) + if rmErr := os.RemoveAll(internalLabels.StateDir); rmErr != nil { + log.G(ctx).WithError(rmErr).Warnf("failed to remove container %q state dir %q", id, internalLabels.StateDir) } } } -func generateRemoveOrphanedDirsFunc(ctx context.Context, id, dataStore string, internalLabels internalLabels) func() { +func generateRemoveOrphanedDirsFunc(ctx context.Context, id, dataStore string, internalLabels *annotations.Annotations) func() { return func() { - if rmErr := os.RemoveAll(internalLabels.stateDir); rmErr != nil { - log.G(ctx).WithError(rmErr).Warnf("failed to remove container %q state dir %q", id, internalLabels.stateDir) + if rmErr := os.RemoveAll(internalLabels.StateDir); rmErr != nil { + log.G(ctx).WithError(rmErr).Warnf("failed to remove container %q state dir %q", id, internalLabels.StateDir) } - hs, err := hostsstore.New(dataStore, internalLabels.namespace) + hs, err := hostsstore.New(dataStore, internalLabels.Namespace) if err != nil { - log.G(ctx).WithError(err).Warnf("failed to instantiate hostsstore for %q", internalLabels.namespace) + log.G(ctx).WithError(err).Warnf("failed to instantiate hostsstore for %q", internalLabels.Namespace) } else if err = hs.Delete(id); err != nil { log.G(ctx).WithError(err).Warnf("failed to remove an etchosts directory for container %q", id) } } } -func generateGcFunc(ctx context.Context, container containerd.Container, ns, id, name, dataStore string, containerErr error, containerNameStore namestore.NameStore, netManager containerutil.NetworkOptionsManager, internalLabels internalLabels) func() { +func generateGcFunc(ctx context.Context, container containerd.Container, ns, id, name, dataStore string, containerErr error, containerNameStore namestore.NameStore, netManager containerutil.NetworkOptionsManager, internalLabels *annotations.Annotations) func() { return func() { if containerErr == nil { netGcErr := netManager.CleanupNetworking(ctx, container) @@ -1072,9 +884,9 @@ func generateGcFunc(ctx context.Context, container containerd.Container, ns, id, log.G(ctx).WithError(netGcErr).Warnf("failed to revert container %q networking settings", id) } } else { - hs, err := hostsstore.New(dataStore, internalLabels.namespace) + hs, err := hostsstore.New(dataStore, internalLabels.Namespace) if err != nil { - log.G(ctx).WithError(err).Warnf("failed to instantiate hostsstore for %q", internalLabels.namespace) + log.G(ctx).WithError(err).Warnf("failed to instantiate hostsstore for %q", internalLabels.Namespace) } else { if _, err := hs.HostsPath(id); err != nil { log.G(ctx).WithError(err).Warnf("an etchosts directory for container %q dosen't exist", id) @@ -1084,15 +896,15 @@ func generateGcFunc(ctx context.Context, container containerd.Container, ns, id, } } - ipc, ipcErr := ipcutil.DecodeIPCLabel(internalLabels.ipc) + ipc, ipcErr := ipcutil.DecodeIPCLabel(internalLabels.IPC) if ipcErr != nil { log.G(ctx).WithError(ipcErr).Warnf("failed to decode ipc label for container %q", id) } if ipcErr := ipcutil.CleanUp(ipc); ipcErr != nil { log.G(ctx).WithError(ipcErr).Warnf("failed to clean up ipc for container %q", id) } - if rmErr := os.RemoveAll(internalLabels.stateDir); rmErr != nil { - log.G(ctx).WithError(rmErr).Warnf("failed to remove container %q state dir %q", id, internalLabels.stateDir) + if rmErr := os.RemoveAll(internalLabels.StateDir); rmErr != nil { + log.G(ctx).WithError(rmErr).Warnf("failed to remove container %q state dir %q", id, internalLabels.StateDir) } var errE error diff --git a/pkg/cmd/container/run_cgroup_linux.go b/pkg/cmd/container/run_cgroup_linux.go index 75eb5f6405b..505eaa77439 100644 --- a/pkg/cmd/container/run_cgroup_linux.go +++ b/pkg/cmd/container/run_cgroup_linux.go @@ -30,6 +30,7 @@ import ( "github.com/containerd/containerd/v2/pkg/oci" "github.com/containerd/log" + "github.com/containerd/nerdctl/v2/pkg/annotations" "github.com/containerd/nerdctl/v2/pkg/api/types" "github.com/containerd/nerdctl/v2/pkg/infoutil" "github.com/containerd/nerdctl/v2/pkg/inspecttypes/dockercompat" @@ -42,7 +43,7 @@ type customMemoryOptions struct { disableOOMKiller *bool } -func generateCgroupOpts(id string, options types.ContainerCreateOptions, internalLabels *internalLabels) ([]oci.SpecOpts, error) { +func generateCgroupOpts(id string, options types.ContainerCreateOptions, internalLabels *annotations.Annotations) ([]oci.SpecOpts, error) { if options.KernelMemory != "" { log.L.Warnf("The --kernel-memory flag is no longer supported. This flag is a noop.") } @@ -221,7 +222,7 @@ func generateCgroupOpts(id string, options types.ContainerCreateOptions, interna deviceMap.PathOnHost = devPath deviceMap.PathInContainer = conPath deviceMap.CgroupPermissions = mode - internalLabels.deviceMapping = append(internalLabels.deviceMapping, deviceMap) + internalLabels.DeviceMapping = append(internalLabels.DeviceMapping, deviceMap) } return opts, nil diff --git a/pkg/cmd/container/run_linux.go b/pkg/cmd/container/run_linux.go index 3280d3e532d..06a3a71589f 100644 --- a/pkg/cmd/container/run_linux.go +++ b/pkg/cmd/container/run_linux.go @@ -30,6 +30,7 @@ import ( "github.com/containerd/containerd/v2/pkg/oci" "github.com/containerd/log" + "github.com/containerd/nerdctl/v2/pkg/annotations" "github.com/containerd/nerdctl/v2/pkg/api/types" "github.com/containerd/nerdctl/v2/pkg/bypass4netnsutil" "github.com/containerd/nerdctl/v2/pkg/containerutil" @@ -44,7 +45,7 @@ func WithoutRunMount() func(ctx context.Context, client oci.Client, c *container return oci.WithoutRunMount } -func setPlatformOptions(ctx context.Context, client *containerd.Client, id, uts string, internalLabels *internalLabels, options types.ContainerCreateOptions) ([]oci.SpecOpts, error) { +func setPlatformOptions(ctx context.Context, client *containerd.Client, id, uts string, internalLabels *annotations.Annotations, options types.ContainerCreateOptions) ([]oci.SpecOpts, error) { var opts []oci.SpecOpts opts = append(opts, oci.WithDefaultUnixDevices, @@ -128,7 +129,7 @@ func generateNamespaceOpts( ctx context.Context, client *containerd.Client, uts string, - internalLabels *internalLabels, + internalLabels *annotations.Annotations, options types.ContainerCreateOptions, ) ([]oci.SpecOpts, error) { var opts []oci.SpecOpts @@ -142,19 +143,19 @@ func generateNamespaceOpts( return nil, fmt.Errorf("unknown uts value. valid value(s) are 'host', got: %q", uts) } - stateDir := internalLabels.stateDir + stateDir := internalLabels.StateDir ipcOpts, ipcLabel, err := generateIPCOpts(ctx, client, options.IPC, options.ShmSize, stateDir) if err != nil { return nil, err } - internalLabels.ipc = ipcLabel + internalLabels.IPC = ipcLabel opts = append(opts, ipcOpts...) pidOpts, pidLabel, err := generatePIDOpts(ctx, client, options.Pid) if err != nil { return nil, err } - internalLabels.pidContainer = pidLabel + internalLabels.PidContainer = pidLabel opts = append(opts, pidOpts...) return opts, nil diff --git a/pkg/cmd/container/run_unix_nolinux.go b/pkg/cmd/container/run_unix_nolinux.go index fde2629b27d..96ae9c6ce69 100644 --- a/pkg/cmd/container/run_unix_nolinux.go +++ b/pkg/cmd/container/run_unix_nolinux.go @@ -25,6 +25,7 @@ import ( "github.com/containerd/containerd/v2/core/containers" "github.com/containerd/containerd/v2/pkg/oci" + "github.com/containerd/nerdctl/v2/pkg/annotations" "github.com/containerd/nerdctl/v2/pkg/api/types" ) @@ -37,7 +38,7 @@ func setPlatformOptions( ctx context.Context, client *containerd.Client, id, uts string, - internalLabels *internalLabels, + internalLabels *annotations.Annotations, options types.ContainerCreateOptions, ) ([]oci.SpecOpts, error) { return []oci.SpecOpts{}, nil diff --git a/pkg/cmd/container/run_windows.go b/pkg/cmd/container/run_windows.go index 6bc5b42e699..3f8b71abcb9 100644 --- a/pkg/cmd/container/run_windows.go +++ b/pkg/cmd/container/run_windows.go @@ -29,6 +29,7 @@ import ( "github.com/containerd/containerd/v2/core/containers" "github.com/containerd/containerd/v2/pkg/oci" + "github.com/containerd/nerdctl/v2/pkg/annotations" "github.com/containerd/nerdctl/v2/pkg/api/types" ) @@ -46,7 +47,7 @@ func setPlatformOptions( ctx context.Context, client *containerd.Client, id, uts string, - internalLabels *internalLabels, + internalLabels *annotations.Annotations, options types.ContainerCreateOptions, ) ([]oci.SpecOpts, error) { opts := []oci.SpecOpts{} diff --git a/pkg/containerutil/container_network_manager_windows.go b/pkg/containerutil/container_network_manager_windows.go index 90079f12907..c7bbb2fe88e 100644 --- a/pkg/containerutil/container_network_manager_windows.go +++ b/pkg/containerutil/container_network_manager_windows.go @@ -25,9 +25,9 @@ import ( "github.com/containerd/containerd/v2/pkg/oci" "github.com/containerd/go-cni" + "github.com/containerd/nerdctl/v2/pkg/annotations" "github.com/containerd/nerdctl/v2/pkg/api/types" "github.com/containerd/nerdctl/v2/pkg/netutil" - "github.com/containerd/nerdctl/v2/pkg/ocihook" ) type cniNetworkManagerPlatform struct { @@ -119,9 +119,9 @@ func (m *cniNetworkManager) CleanupNetworking(ctx context.Context, container con return fmt.Errorf("failed to get container specs for networking cleanup: %w", err) } - netNsID, found := spec.Annotations[ocihook.NetworkNamespace] + netNsID, found := spec.Annotations[annotations.NetworkNamespace] if !found { - return fmt.Errorf("no %q annotation present on container with ID %s", ocihook.NetworkNamespace, containerID) + return fmt.Errorf("no %q annotation present on container with ID %s", annotations.NetworkNamespace, containerID) } return cni.Remove(ctx, containerID, netNsID, m.getCNINamespaceOpts()...) @@ -148,7 +148,7 @@ func (m *cniNetworkManager) ContainerNetworkingOpts(_ context.Context, container cOpts := []containerd.NewContainerOpts{ containerd.WithAdditionalContainerLabels( map[string]string{ - ocihook.NetworkNamespace: ns.GetPath(), + annotations.NetworkNamespace: ns.GetPath(), }, ), } diff --git a/pkg/ocihook/ocihook.go b/pkg/ocihook/ocihook.go index 89b6c6b1410..d7ab07f323f 100644 --- a/pkg/ocihook/ocihook.go +++ b/pkg/ocihook/ocihook.go @@ -37,10 +37,10 @@ import ( "github.com/containerd/go-cni" "github.com/containerd/log" + "github.com/containerd/nerdctl/v2/pkg/annotations" "github.com/containerd/nerdctl/v2/pkg/bypass4netnsutil" "github.com/containerd/nerdctl/v2/pkg/dnsutil/hostsstore" "github.com/containerd/nerdctl/v2/pkg/internal/filesystem" - "github.com/containerd/nerdctl/v2/pkg/labels" "github.com/containerd/nerdctl/v2/pkg/namestore" "github.com/containerd/nerdctl/v2/pkg/netutil" "github.com/containerd/nerdctl/v2/pkg/netutil/nettype" @@ -50,19 +50,6 @@ import ( "github.com/containerd/nerdctl/v2/pkg/store" ) -const ( - // NetworkNamespace is the network namespace path to be passed to the CNI plugins. - // When this annotation is set from the runtime spec.State payload, it takes - // precedence over the PID based resolution (/proc//ns/net) where pid is - // spec.State.Pid. - // This is mostly used for VM based runtime, where the spec.State PID does not - // necessarily lives in the created container networking namespace. - // - // On Windows, this label will contain the UUID of a namespace managed by - // the Host Compute Network Service (HCN) API. - NetworkNamespace = labels.Prefix + "network-namespace" -) - func Run(stdin io.Reader, stderr io.Writer, event, dataStore, cniPath, cniNetconfPath, bridgeIP string) error { if stdin == nil || event == "" || dataStore == "" || cniPath == "" || cniNetconfPath == "" { return errors.New("got insufficient args") @@ -73,7 +60,11 @@ func Run(stdin io.Reader, stderr io.Writer, event, dataStore, cniPath, cniNetcon return err } - containerStateDir := state.Annotations[labels.StateDir] + containerAnnotations, err := annotations.NewFromState(&state) + if err != nil { + return err + } + containerStateDir := containerAnnotations.StateDir if containerStateDir == "" { return errors.New("state dir must be set") } @@ -117,33 +108,27 @@ func Run(stdin io.Reader, stderr io.Writer, event, dataStore, cniPath, cniNetcon } defer filesystem.Unlock(lock) - opts, err := newHandlerOpts(&state, dataStore, cniPath, cniNetconfPath, bridgeIP) + opts, err := newHandlerOpts(&state, containerAnnotations, dataStore, cniPath, cniNetconfPath, bridgeIP) if err != nil { return err } switch event { case "createRuntime": - return onCreateRuntime(opts) + return onCreateRuntime(opts, containerAnnotations) case "postStop": - return onPostStop(opts) + return onPostStop(opts, containerAnnotations) default: return fmt.Errorf("unexpected event %q", event) } } -func newHandlerOpts(state *specs.State, dataStore, cniPath, cniNetconfPath, bridgeIP string) (*handlerOpts, error) { +func newHandlerOpts(state *specs.State, containerAnnotations *annotations.Annotations, dataStore, cniPath, cniNetconfPath, bridgeIP string) (*handlerOpts, error) { o := &handlerOpts{ state: state, dataStore: dataStore, } - extraHosts, err := getExtraHosts(state) - if err != nil { - return nil, err - } - o.extraHosts = extraHosts - hs, err := loadSpec(o.state.Bundle) if err != nil { return nil, err @@ -153,7 +138,7 @@ func newHandlerOpts(state *specs.State, dataStore, cniPath, cniNetconfPath, brid o.rootfs = filepath.Join(o.state.Bundle, o.rootfs) } - namespace := o.state.Annotations[labels.Namespace] + namespace := containerAnnotations.Namespace if namespace == "" { return nil, errors.New("namespace must be set") } @@ -162,11 +147,7 @@ func newHandlerOpts(state *specs.State, dataStore, cniPath, cniNetconfPath, brid } o.fullID = namespace + "-" + o.state.ID - networksJSON := o.state.Annotations[labels.Networks] - var networks []string - if err := json.Unmarshal([]byte(networksJSON), &networks); err != nil { - return nil, err - } + networks := containerAnnotations.Networks netType, err := nettype.Detect(networks) if err != nil { @@ -203,7 +184,7 @@ func newHandlerOpts(state *specs.State, dataStore, cniPath, cniNetconfPath, brid return nil, fmt.Errorf("unexpected network type %v", netType) } - if pidFile := o.state.Annotations[labels.PIDFile]; pidFile != "" { + if pidFile := containerAnnotations.PidFile; pidFile != "" { if err := writePidFile(pidFile, state.Pid); err != nil { return nil, err } @@ -215,27 +196,17 @@ func newHandlerOpts(state *specs.State, dataStore, cniPath, cniNetconfPath, brid } o.ports = ports - if ipAddress, ok := o.state.Annotations[labels.IPAddress]; ok { - o.containerIP = ipAddress - } - - if macAddress, ok := o.state.Annotations[labels.MACAddress]; ok { - o.containerMAC = macAddress - } + o.containerIP = containerAnnotations.IPAddress + o.containerMAC = containerAnnotations.MACAddress - if ip6Address, ok := o.state.Annotations[labels.IP6Address]; ok { - o.containerIP6 = ip6Address - } + o.containerIP6 = containerAnnotations.IP6Address if rootlessutil.IsRootlessChild() { o.rootlessKitClient, err = rootlessutil.NewRootlessKitClient() if err != nil { return nil, err } - b4nnEnabled, _, err := bypass4netnsutil.IsBypass4netnsEnabled(o.state.Annotations) - if err != nil { - return nil, err - } + b4nnEnabled := containerAnnotations.Bypass4netns if b4nnEnabled { socketPath, err := bypass4netnsutil.GetBypass4NetnsdDefaultSocketPath() if err != nil { @@ -260,7 +231,6 @@ type handlerOpts struct { fullID string rootlessKitClient rlkclient.Client bypassClient b4nndclient.Client - extraHosts map[string]string // host:ip containerIP string containerMAC string containerIP6 string @@ -287,26 +257,10 @@ func loadSpec(bundle string) (*hookSpec, error) { return &s, nil } -func getExtraHosts(state *specs.State) (map[string]string, error) { - extraHostsJSON := state.Annotations[labels.ExtraHosts] - var extraHosts []string - if err := json.Unmarshal([]byte(extraHostsJSON), &extraHosts); err != nil { - return nil, err - } - - hosts := make(map[string]string) - for _, host := range extraHosts { - if v := strings.SplitN(host, ":", 2); len(v) == 2 { - hosts[v[0]] = v[1] - } - } - return hosts, nil -} - -func getNetNSPath(state *specs.State) (string, error) { +func getNetNSPath(state *specs.State, containerAnnotations *annotations.Annotations) (string, error) { // If we have a network-namespace annotation we use it over the passed Pid. - netNsPath, netNsFound := state.Annotations[NetworkNamespace] - if netNsFound { + netNsPath := containerAnnotations.NetworkNamespace + if netNsPath != "" { if _, err := os.Stat(netNsPath); err != nil { return "", err } @@ -420,17 +374,17 @@ func getIP6AddressOpts(opts *handlerOpts) ([]cni.NamespaceOpts, error) { return nil, nil } -func applyNetworkSettings(opts *handlerOpts) (err error) { +func applyNetworkSettings(opts *handlerOpts, containerAnnotations *annotations.Annotations) error { portMapOpts, err := getPortMapOpts(opts) if err != nil { return err } - nsPath, err := getNetNSPath(opts.state) + nsPath, err := getNetNSPath(opts.state, containerAnnotations) if err != nil { return err } ctx := context.Background() - hs, err := hostsstore.New(opts.dataStore, opts.state.Annotations[labels.Namespace]) + hs, err := hostsstore.New(opts.dataStore, containerAnnotations.Namespace) if err != nil { return err } @@ -455,15 +409,15 @@ func applyNetworkSettings(opts *handlerOpts) (err error) { cni.WithLabels(map[string]string{ "IgnoreUnknown": "1", }), - cni.WithArgs("NERDCTL_CNI_DHCP_HOSTNAME", opts.state.Annotations[labels.Hostname]), + cni.WithArgs("NERDCTL_CNI_DHCP_HOSTNAME", containerAnnotations.HostName), ) hsMeta := hostsstore.Meta{ ID: opts.state.ID, Networks: make(map[string]*types100.Result, len(opts.cniNames)), - Hostname: opts.state.Annotations[labels.Hostname], - Domainname: opts.state.Annotations[labels.Domainname], - ExtraHosts: opts.extraHosts, - Name: opts.state.Annotations[labels.Name], + Hostname: containerAnnotations.HostName, + Domainname: containerAnnotations.DomainName, + ExtraHosts: containerAnnotations.ExtraHosts, + Name: containerAnnotations.Name, } // When containerd gets bounced, containers that were previously running and that are restarted will go again @@ -495,10 +449,8 @@ func applyNetworkSettings(opts *handlerOpts) (err error) { hsMeta.Networks[cniName] = cniResRaw[i] } - b4nnEnabled, b4nnBindEnabled, err := bypass4netnsutil.IsBypass4netnsEnabled(opts.state.Annotations) - if err != nil { - return err - } + b4nnEnabled := containerAnnotations.Bypass4netns + b4nnBindEnabled := !containerAnnotations.Bypass4netnsIgnoreBind if err := hs.Acquire(hsMeta); err != nil { return err @@ -506,11 +458,11 @@ func applyNetworkSettings(opts *handlerOpts) (err error) { if rootlessutil.IsRootlessChild() { if b4nnEnabled { - bm, err := bypass4netnsutil.NewBypass4netnsCNIBypassManager(opts.bypassClient, opts.rootlessKitClient, opts.state.Annotations) + bm, err := bypass4netnsutil.NewBypass4netnsCNIBypassManager(opts.bypassClient, opts.rootlessKitClient, containerAnnotations) if err != nil { return err } - err = bm.StartBypass(ctx, opts.ports, opts.state.ID, opts.state.Annotations[labels.StateDir]) + err = bm.StartBypass(ctx, opts.ports, opts.state.ID, containerAnnotations.StateDir) if err != nil { return fmt.Errorf("bypass4netnsd not running? (Hint: run `containerd-rootless-setuptool.sh install-bypass4netnsd`): %w", err) } @@ -524,11 +476,11 @@ func applyNetworkSettings(opts *handlerOpts) (err error) { return nil } -func onCreateRuntime(opts *handlerOpts) error { +func onCreateRuntime(opts *handlerOpts, containerAnnotations *annotations.Annotations) error { loadAppArmor() - name := opts.state.Annotations[labels.Name] - ns := opts.state.Annotations[labels.Namespace] + name := containerAnnotations.Name + ns := containerAnnotations.Namespace namst, err := namestore.New(opts.dataStore, ns) if err != nil { log.L.WithError(err).Error("failed opening the namestore in onCreateRuntime") @@ -538,11 +490,11 @@ func onCreateRuntime(opts *handlerOpts) error { var netError error if opts.cni != nil { - netError = applyNetworkSettings(opts) + netError = applyNetworkSettings(opts, containerAnnotations) } // Set StartedAt and CreateError - lf, err := state.New(opts.state.Annotations[labels.StateDir]) + lf, err := state.New(containerAnnotations.StateDir) if err != nil { return err } @@ -559,8 +511,8 @@ func onCreateRuntime(opts *handlerOpts) error { return netError } -func onPostStop(opts *handlerOpts) error { - lf, err := state.New(opts.state.Annotations[labels.StateDir]) +func onPostStop(opts *handlerOpts, containerAnnotations *annotations.Annotations) error { + lf, err := state.New(containerAnnotations.StateDir) if err != nil { return err } @@ -584,16 +536,14 @@ func onPostStop(opts *handlerOpts) error { } ctx := context.Background() - ns := opts.state.Annotations[labels.Namespace] + ns := containerAnnotations.Namespace if opts.cni != nil { var err error - b4nnEnabled, b4nnBindEnabled, err := bypass4netnsutil.IsBypass4netnsEnabled(opts.state.Annotations) - if err != nil { - return err - } + b4nnEnabled := containerAnnotations.Bypass4netns + b4nnBindEnabled := !containerAnnotations.Bypass4netnsIgnoreBind if rootlessutil.IsRootlessChild() { if b4nnEnabled { - bm, err := bypass4netnsutil.NewBypass4netnsCNIBypassManager(opts.bypassClient, opts.rootlessKitClient, opts.state.Annotations) + bm, err := bypass4netnsutil.NewBypass4netnsCNIBypassManager(opts.bypassClient, opts.rootlessKitClient, containerAnnotations) if err != nil { return err } @@ -654,7 +604,7 @@ func onPostStop(opts *handlerOpts) error { if err != nil { return err } - name := opts.state.Annotations[labels.Name] + name := containerAnnotations.Name // Double-releasing may happen with containers started with --rm, so, ignore NotFound errors if err := namst.Release(name, opts.state.ID); err != nil && !errors.Is(err, store.ErrNotFound) { return fmt.Errorf("failed to release container name %s: %w", name, err)