Enable TestClientPublishEventKerberosAware and update kerberos docker image

libbeat/common/transport/kerberos/client_fips.go

@@ -24,6 +24,6 @@ import (
 	"net/http"
 )
 
-func NewClient(config *Config, httpClient *http.Client, esurl string) (Client, error) {
+func NewClient(config *Config, httpClient *http.Client) (Client, error) {
 	return nil, errors.New("kerberos is not supported in fips mode")
 }

libbeat/common/transport/kerberos/client_fips_test.go

@@ -33,7 +33,7 @@ func TestNewClient(t *testing.T) {
 	c, err := NewClient(&Config{
 		AuthType:   authPassword,
 		ConfigPath: cfg.Name(),
-	}, http.DefaultClient, "")
+	}, http.DefaultClient)
 	require.Nil(t, c)
 	require.EqualError(t, err, "kerberos is not supported in fips mode")
 }

libbeat/common/transport/kerberos/client_nofips.go

@@ -29,7 +29,7 @@ import (
 	"github.com/elastic/gokrb5/v8/spnego"
 )
 
-func NewClient(config *Config, httpClient *http.Client, esurl string) (Client, error) {
+func NewClient(config *Config, httpClient *http.Client) (Client, error) {
 	var krbClient *krbclient.Client
 	krbConf, err := krbconfig.Load(config.ConfigPath)
 	if err != nil {

libbeat/common/transport/kerberos/client_nofips_test.go

@@ -33,7 +33,7 @@ func TestNewClient(t *testing.T) {
 	c, err := NewClient(&Config{
 		AuthType:   authPassword,
 		ConfigPath: cfg.Name(),
-	}, http.DefaultClient, "")
+	}, http.DefaultClient)
 	require.Nil(t, err)
 	require.NotNil(t, c)
 }

libbeat/docker-compose.yml

@@ -6,6 +6,7 @@ services:
     depends_on:
       elasticsearch:                  { condition: service_healthy }
       elasticsearchssl:               { condition: service_healthy }
+      elasticsearch_kerberos.elastic: { condition: service_healthy }
       logstash:                       { condition: service_healthy }
       kafka:                          { condition: service_healthy }
       redis:                          { condition: service_healthy }
@@ -57,6 +58,37 @@ services:
     ports:
       - 9201:9200
 
+
+  elasticsearch_kerberos.elastic:
+    build: ${ES_BEATS}/testing/environments/docker/elasticsearch_kerberos
+    healthcheck:
+      test: bash -c "/healthcheck.sh"
+      retries: 1200
+      interval: 5s
+      start_period: 60s
+    environment:
+      - "TERM=linux"
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m -Djava.security.krb5.conf=/etc/krb5.conf -Dsun.security.krb5.debug=true"
+      - "transport.host=127.0.0.1"
+      - "http.host=0.0.0.0"
+      - "xpack.license.self_generated.type=trial"
+      - "xpack.security.enabled=true"
+      - "indices.id_field_data.enabled=true"
+      - "xpack.security.audit.enabled=true"
+      - "xpack.security.authc.realms.kerberos.elastic.order=0"
+      - "xpack.security.authc.realms.kerberos.elastic.keytab.path=/usr/share/elasticsearch/config/HTTP_localhost.keytab"
+      - "xpack.security.authc.realms.kerberos.elastic.remove_realm_name=false"
+      - "xpack.security.authc.realms.kerberos.elastic.krb.debug=true"
+    volumes:
+      # This is needed otherwise there won't be enough entropy to generate a new kerberos realm
+      - /dev/urandom:/dev/random
+    ports:
+      - 1088:1088
+      - 1749:1749
+      - 9203:9200
+    command: bash -c "/start.sh"
+
+
   # This host name is static because of the certificate.
   logstash:
     extends:

libbeat/esleg/eslegclient/connection.go

@@ -176,7 +176,7 @@ func NewConnection(s ConnectionSettings, log *logp.Logger) (*Connection, error)
 
 	esClient := esHTTPClient(httpClient)
 	if s.Kerberos.IsEnabled() {
-		esClient, err = kerberos.NewClient(s.Kerberos, httpClient, s.URL)
+		esClient, err = kerberos.NewClient(s.Kerberos, httpClient)
 		if err != nil {
 			return nil, err
 		}
@@ -227,7 +227,6 @@ func NewClients(cfg *cfg.C, beatname string, log *logp.Logger) ([]Connection, er
 			return nil, err
 		}
 
-		// TODO: use local logger here
 		client, err := NewConnection(ConnectionSettings{
 			URL:              esURL,
 			Beatname:         beatname,
@@ -444,13 +443,14 @@ func (conn *Connection) getVersion() error {
 		conn.version = *v
 	}
 
-	if versionData.Version.BuildFlavor == "serverless" {
+	switch versionData.Version.BuildFlavor {
+	case "serverless":
 		conn.log.Info("build flavor of es is serverless, marking connection as serverless")
 		conn.isServerless = true
-	} else if versionData.Version.BuildFlavor == "default" {
+	case "default":
 		conn.isServerless = false
 		// not sure if this is even possible, just being defensive
-	} else {
+	default:
 		conn.log.Infof("Got unexpected build flavor '%s'", versionData.Version.BuildFlavor)
 	}
 

libbeat/outputs/elasticsearch/client_integration_test.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"go.elastic.co/apm/v2/apmtest"
+	"go.uber.org/zap"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -53,25 +54,22 @@ func TestClientPublishEvent(t *testing.T) {
 }
 
 func TestClientPublishEventKerberosAware(t *testing.T) {
-	t.Skip("Flaky test: https://github.com/elastic/beats/issues/21295")
-
-	err := setupRoleMapping(t, eslegtest.GetEsKerberosHost())
+	kerberosURL := "http://localhost:9203"
+	err := setupRoleMapping(t, kerberosURL)
 	if err != nil {
 		t.Fatal(err)
 	}
 
 	index := "beat-int-pub-single-event-behind-kerb"
 	cfg := map[string]interface{}{
-		"hosts":    eslegtest.GetEsKerberosHost(),
-		"index":    index,
-		"username": "",
-		"password": "",
+		"hosts": kerberosURL,
+		"index": index,
 		"kerberos": map[string]interface{}{
 			"auth_type":   "password",
 			"config_path": "testdata/krb5.conf",
-			"username":    eslegtest.GetUser(),
-			"password":    eslegtest.GetPass(),
-			"realm":       "ELASTIC",
+			"username":    "beats",
+			"password":    "testing",
+			"realm":       "elastic",
 		},
 	}
 
@@ -405,7 +403,7 @@ func connectTestEs(t *testing.T, cfg interface{}, stats outputs.Observer) (outpu
 		t.Fatal(err)
 	}
 
-	logger := logptest.NewTestingLogger(t, "elasticsearch")
+	logger := logptest.NewTestingLogger(t, "elasticsearch", zap.AddCallerSkip(1))
 	info := beat.Info{Beat: "libbeat", Logger: logger}
 	// disable ILM if using specified index name
 	im, _ := idxmgmt.DefaultSupport(info, conf.MustNewConfigFrom(map[string]interface{}{"setup.ilm.enabled": "false"}))
@@ -431,12 +429,12 @@ func connectTestEs(t *testing.T, cfg interface{}, stats outputs.Observer) (outpu
 	return client, client
 }
 
-// setupRoleMapping sets up role mapping for the Kerberos user beats@ELASTIC
+// setupRoleMapping sets up role mapping for the Kerberos user beats@elastic
 func setupRoleMapping(t *testing.T, host string) error {
 	_, client := connectTestEs(t, map[string]interface{}{
 		"hosts":    host,
-		"username": "elastic",
-		"password": "changeme",
+		"username": "admin",
+		"password": "testing",
 	}, nil)
 
 	roleMappingURL := client.conn.URL + "/_security/role_mapping/kerbrolemapping"
@@ -446,7 +444,7 @@ func setupRoleMapping(t *testing.T, host string) error {
 		"enabled": true,
 		"rules": map[string]interface{}{
 			"field": map[string]interface{}{
-				"username": "beats@ELASTIC",
+				"username": "beats@elastic",
 			},
 		},
 	})

libbeat/outputs/elasticsearch/testdata/krb5.conf

@@ -16,7 +16,7 @@
 # under the License.
 
 [libdefaults]
-    default_realm = ELASTIC
+    default_realm = elastic
     dns_canonicalize_hostname = false
     dns_lookup_kdc = false
     dns_lookup_realm = false
@@ -31,13 +31,13 @@
     kdc_timeout = 3000
 
 [realms]
-    ELASTIC = {
-        kdc = elasticsearch_kerberos.elastic:1088
-        admin_server = elasticsearch_kerberos.elastic:1749
+    elastic = {
+        kdc = localhost:1088
+        admin_server = localhost:1749
         default_domain = elastic
     }
 
 [domain_realm]
-    .elastic = ELASTIC
-    elastic = ELASTIC
+    .elastic = elastic
+    elastic = elastic
 

testing/environments/docker/elasticsearch_kerberos/Dockerfile

@@ -5,10 +5,10 @@ ADD config /config
 ADD healthcheck.sh /healthcheck.sh
 ADD start.sh /start.sh
 
-ENV REALM_NAME ELASTIC
-ENV KDC_NAME elasticsearch_kerberos.elastic
-ENV BUILD_ZONE elastic
-ENV ELASTIC_ZONE $BUILD_ZONE
+ENV REALM_NAME=elastic
+ENV KDC_NAME=elasticsearch_kerberos.elastic
+ENV BUILD_ZONE=elastic
+ENV ELASTIC_ZONE=$BUILD_ZONE
 
 USER root
 RUN /scripts/installkdc.sh && /scripts/addprincs.sh

testing/environments/docker/elasticsearch_kerberos/config/kdc.conf.template

@@ -16,19 +16,24 @@
 # under the License.
 
 [kdcdefaults]
-    kdc_listen = 1088
-    kdc_tcp_listen = 1088
+    kdc_ports = 1088
+    kdc_tcp_ports = 1088
 
 [realms]
     ${REALM_NAME} = {
         kadmind_port = 1749
         max_life = 12h 0m 0s
         max_renewable_life = 7d 0h 0m 0s
         master_key_type = aes256-cts
-        supported_enctypes = aes128-cts:normal des3-hmac-sha1:normal arcfour-hmac:normal des-hmac-sha1:normal des-cbc-md5:normal des-cbc-crc:normal
+        acl_file = /etc/krb5kdc/kadm5.acl
+        database_name = /var/lib/krb5kdc/principal
+        key_stash_file = /etc/krb5kdc/stash
+        admin_keytab = FILE:/etc/admin.keytab
+        kdc_ports = 1088
+        default_principal_flags = +preauth
     }
 
 [logging]
-    kdc = FILE:/var/log/krb5/krb5kdc.log
-    admin_server = FILE:/var/log/krb5/kadmin.log
-    default = FILE:/var/log/krb5/krb5lib.log
+    kdc = STDERR
+    admin_server = STDERR
+    default = STDERR

testing/environments/docker/elasticsearch_kerberos/config/krb5.conf.template

@@ -18,9 +18,9 @@
 [libdefaults]
     default_realm = ${REALM_NAME}
     dns_canonicalize_hostname = false
-    dns_lookup_kdc = false
-    dns_lookup_realm = false
-    dns_uri_lookup = false
+    dns_lookup_kdc = true
+    dns_lookup_realm = true
+    dns_uri_lookup = true
     forwardable = true
     ignore_acceptor_hostname = true
     rdns = false

testing/environments/docker/elasticsearch_kerberos/healthcheck.sh

@@ -1,11 +1,11 @@
 #!/bin/sh
 
 # check if service principal is OK
-KRB5_CONFIG=/etc/krb5.conf \
-    kinit -k -t /etc/HTTP_elasticsearch_kerberos.elastic.keytab HTTP/elasticsearch_kerberos.elastic@ELASTIC
-
+export KRB5_CONFIG=/etc/krb5.conf 
+kinit -k -t /etc/HTTP_localhost.keytab HTTP/localhost@$REALM
 
 # check if beats user can connect
-echo testing | KRB5_CONFIG=/etc/krb5.conf kinit beats@ELASTIC
+kinit beats@$REALM
 klist
-curl --negotiate -u : -XGET http://elasticsearch_kerberos.elastic:9200/
+
+curl --negotiate -u : -XGET http://localhost:9200/

testing/environments/docker/elasticsearch_kerberos/scripts/addprinc.sh

@@ -29,17 +29,16 @@ fi
 PRINC="$1"
 PASSWD="$2"
 USER=$(echo $PRINC | tr "/" "_")
-REALM=ELASTIC
 
 VDIR=/usr/share/kerberos
 BUILD_DIR=/var/build
 LOCALSTATEDIR=/etc
 LOGDIR=/var/log/krb5
 
-ADMIN_PRIN=admin/admin@$REALM
+ADMIN_PRIN=root/admin
 ADMIN_KTAB=$LOCALSTATEDIR/admin.keytab
 
-USER_PRIN=$PRINC@$REALM
+USER_PRIN=$PRINC
 USER_KTAB=$LOCALSTATEDIR/$USER.keytab
 
 if [ -f $USER_KTAB ] && [ -z "$PASSWD" ]; then
@@ -48,14 +47,14 @@ if [ -f $USER_KTAB ] && [ -z "$PASSWD" ]; then
 else
   if [ -z "$PASSWD" ]; then
     echo "Provisioning '${PRINC}@${REALM}' principal and keytab..."
-    sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -randkey $USER_PRIN"
-    sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "ktadd -k $USER_KTAB $USER_PRIN"
+    sudo kadmin.local -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -randkey $USER_PRIN"
+    sudo kadmin.local -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "ktadd -k $USER_KTAB $USER_PRIN"
     sudo chmod 777 $USER_KTAB
     sudo cp $USER_KTAB /usr/share/elasticsearch/config
     sudo chown elasticsearch:elasticsearch /usr/share/elasticsearch/config/$USER.keytab
   else
     echo "Provisioning '${PRINC}@${REALM}' principal with password..."
-    sudo kadmin -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -pw $PASSWD $PRINC"
+    sudo kadmin.local -p $ADMIN_PRIN -kt $ADMIN_KTAB -q "addprinc -pw $PASSWD $PRINC"
   fi
 fi
 

testing/environments/docker/elasticsearch_kerberos/scripts/addprincs.sh

@@ -1,7 +1,10 @@
 set -e
 
-krb5kdc
+export KRB5_KDC_PROFILE="/var/kerberos/krb5kdc/kdc.conf"
+krb5kdc 
 kadmind
 
-addprinc.sh HTTP/elasticsearch_kerberos.elastic
+
+## set principal and user
+addprinc.sh HTTP/localhost
 addprinc.sh beats testing