MEDIUM: add ingress.class annotation to TCP CR

Author: hdurand0710

documentation/custom-resource-tcp.md

@@ -35,6 +35,8 @@ kind: TCP
 metadata:
   name: tcp-1
   namespace: test
+  annotations:
+    ingress.class: haproxy
 spec:
   - name: tcp-http-echo-8443
     frontend:
@@ -103,6 +105,30 @@ Note that in the TCP CR :
 
 Except the frontend keyword `default_backend`, all other lines are not automatically generated but are in a flexible way handled by the `frontend` section in the TCP CR.
 
+## ingress.class
+
+Starting `3.1`, the TCP Custom Resource managed by the Ingress Controller can be filtered using the `ingress.class` annotation.
+It behaves the same way as `Ingress`:
+
+| ingress.class controller flag | TCP CR ingress.class annotation | Behavior |
+|------------------|---------------------|-----------------|
+| '' (not set) |  * (any value) | TCP CR managed by IC |
+| \<igclass\> | Same value as controller  | TCP CR managed by IC |
+| \<igclass\> | Value different from controller  | TCP CR not managed by IC, frontend and backend deleted if existing |
+| \<igclass\> | '' (empty, not set)| if controller `empty-ingress-class` flag is set, TCP CR managed by IC, otherwise ignored (and frontend and backend are deleted)|
+
+
+### Migration 3.0 to 3.1: action required regarding ingress.class annotation
+
+If some TCP CRs were deployed with Ingress Controller version <= v3.0, and the Ingress Controller has a `ingress.class` flag for the controller, the TCP CRs need to have the same value for the `ingress.class` annotation in the TCP CR.
+
+If the annotation is not set, the corresponding backends and frontends in the haproxy configuration would be deleted:
+- except if the controller `empty-ingress-class` flag is set (same behavior as for `Ingress`).
+
+The setting of the `ingress.class` to the TCP CRs should be done **prior to the upgrade to** `v3.1`. It will not be used in v3.0 but needs to be there starting v3.1.
+
+
+
 ## Pod and Service definitions
 
 with the following Kubernetes Service and Pod manifests:
@@ -258,6 +284,8 @@ kind: TCP
 metadata:
   name: tcp-2
   namespace: test
+  annotations:
+    ingress.class: haproxy
 spec:
   - name: tcp-http-echo-test2-8443
     frontend:
@@ -297,6 +325,8 @@ kind: TCP
 metadata:
   name: tcp-1
   namespace: test
+  annotations:
+    ingress.class: haproxy
 spec:
   - name: tcp-http-echo-443
     frontend:

documentation/tcp-cr-full-example/tcp-cr-full.yaml

@@ -2,6 +2,8 @@ apiVersion: ingress.v1.haproxy.org/v1
 kind: TCP
 metadata:
   name: tcp-1
+  annotations:
+    ingress.class: haproxy
 spec:
   - name: tcp-test
     frontend:

pkg/controller/handler.go

@@ -52,7 +52,7 @@ func (c *HAProxyController) initHandlers() {
 		&handler.PatternFiles{},
 		annotations.ConfigSnippetHandler{},
 		c.updateStatusManager,
-		handler.TCPCustomResource{},
+		handler.NewTCPCustomResource(c.osArgs.IngressClass, c.osArgs.EmptyIngressClass),
 	}
 
 	defer func() { c.updateHandlers = append(c.updateHandlers, handler.Refresh{}) }()

pkg/handler/tcp-cr.go

@@ -17,6 +17,7 @@ package handler
 import (
 	"fmt"
 	"strings"
+	"sync"
 
 	"github.com/haproxytech/client-native/v5/models"
 	v1 "github.com/haproxytech/kubernetes-ingress/crs/api/ingress/v1"
@@ -42,19 +43,63 @@ import (
 
 const tcpServicePrefix = "tcpcr"
 
-type TCPCustomResource struct{}
+type TCPCustomResource struct {
+	controllerIngressClass string
+	allowEmptyIngressClass bool
+}
 
 type tcpcontext struct {
 	k         store.K8s
 	namespace string
 	h         haproxy.HAProxy
 }
 
+var syncIngressClassLog sync.Once
+
+func NewTCPCustomResource(controllerIngressClass string, allowEmptyIngressClass bool) TCPCustomResource {
+	return TCPCustomResource{
+		controllerIngressClass: controllerIngressClass,
+		allowEmptyIngressClass: allowEmptyIngressClass,
+	}
+}
+
+func logTCPMigration30To31Warning() {
+	logger := utils.GetLogger()
+	// For 3.0, (WARNING)
+	// Starting from 3.1, if ingress.class is set for controller, you will need to set the ingress.class annotation in the TCP CRD
+	// - Setting the ingress.class annotation in the TCP CRD in 3.0 is highly recommended before migration to 3.1
+	// - empty-ingress-class controller option will also impact TCP CRD starting 3.1
+	logger.Warning("Using TCP CRD without ingress.class annotation will work only in 3.0")
+	logger.Warning("If you are using TCP CRDS without ingress.class annotation and ingress.class is set for the controller,an action is required before migrating to 3.1")
+	logger.Warning("Please read https://github.com/haproxytech/kubernetes-ingress/blob/master/documentation/custom-resource-tcp.md for more information")
+}
+
 func (handler TCPCustomResource) Update(k store.K8s, h haproxy.HAProxy, a annotations.Annotations) (err error) {
 	var errs utils.Errors
 
 	for _, ns := range k.Namespaces {
 		for _, tcpCR := range ns.CRs.TCPsPerCR {
+			//----------------------------------
+			// ingress.class migration
+			// To log in 3.0
+			// Not in 3.1
+			syncIngressClassLog.Do(func() {
+				logTCPMigration30To31Warning()
+			})
+
+			// >= v3.1 for ingress.class
+			// Not in 3.0
+			// supported := handler.isSupportedIngressClass(tcpCR)
+			// if !supported {
+			// 	for _, atcp := range tcpCR.Items {
+			// 		owner := atcp.Owner()
+			// 		k.FrontendRC.RemoveOwner(owner)
+			// 	}
+			// 	continue
+			// }
+			// end ingress.class migration
+			//----------------------------
+
 			// Cleanup will done after Haproxy config transaction succeeds
 			if tcpCR.Status == store.DELETED {
 				continue
@@ -302,3 +347,25 @@ func (handler TCPCustomResource) reconcileAdditionalBackends(ctx tcpcontext, ser
 	}
 	return errors.Result()
 }
+
+// func (handler TCPCustomResource) isSupportedIngressClass(tcps *store.TCPs) bool {
+// 	var supported bool
+// 	tcpIgClassAnn := tcps.IngressClass
+
+// 	switch handler.controllerIngressClass {
+// 	case "", tcpIgClassAnn:
+// 		supported = true
+// 	default: // mismatch osArgs.Ingress and TCP IngressClass annotation
+// 		if tcpIgClassAnn == "" {
+// 			supported = handler.allowEmptyIngressClass
+// 			if !supported {
+// 				utils.GetLogger().Warningf("[SKIP] TCP %s/%s ingress.class annotation='%s' does not match with controller ingress.class flag '%s' and controller flag 'empty-ingress-class' is false",
+// 					tcps.Namespace, tcps.Name, tcpIgClassAnn, handler.controllerIngressClass)
+// 			}
+// 		} else {
+// 			utils.GetLogger().Warningf("[SKIP] TCP %s/%s ingress.class annotation='%s' does not match with controller ingress.class flag '%s'",
+// 				tcps.Namespace, tcps.Name, tcpIgClassAnn, handler.controllerIngressClass)
+// 		}
+// 	}
+// 	return supported
+// }

pkg/k8s/cr-tcp.go

@@ -71,10 +71,11 @@ func convertToStoreTCP(k8sData interface{}, status store.Status) *store.TCPs {
 		return nil
 	}
 	storeTCP := store.TCPs{
-		Status:    status,
-		Namespace: data.GetNamespace(),
-		Name:      data.GetName(),
-		Items:     make([]*store.TCPResource, 0),
+		Status:       status,
+		Namespace:    data.GetNamespace(),
+		IngressClass: data.Annotations["ingress.class"],
+		Name:         data.GetName(),
+		Items:        make([]*store.TCPResource, 0),
 	}
 	for _, tcp := range data.Spec {
 		storeTCP.Items = append(storeTCP.Items, &store.TCPResource{

pkg/store/types-tcp-cr.go

@@ -45,10 +45,11 @@ type TCPResource struct {
 type TCPResourceList []*TCPResource
 
 type TCPs struct {
-	Status    Status          `json:"status,omitempty"`
-	Namespace string          `json:"namespace,omitempty"`
-	Name      string          `json:"name,omitempty"`
-	Items     TCPResourceList `json:"items"`
+	Status       Status          `json:"status,omitempty"`
+	IngressClass string          `json:"ingress_class,omitempty"`
+	Namespace    string          `json:"namespace,omitempty"`
+	Name         string          `json:"name,omitempty"`
+	Items        TCPResourceList `json:"items"`
 }
 
 func (a *TCPs) Equal(b *TCPs, opt ...models.Options) bool {
@@ -64,6 +65,9 @@ func (a *TCPs) Equal(b *TCPs, opt ...models.Options) bool {
 	if a.Namespace != b.Namespace {
 		return false
 	}
+	if a.IngressClass != b.IngressClass {
+		return false
+	}
 	// Always ordered before being added into the store, so no need to order here
 	a.Items.Order()
 	b.Items.Order()