[3.2.4 Backport] CBG-4556: Detect loading non-xattr config document in xattr bootstrap mode (#7468)

Author: bbrks

base/config_persistence.go

@@ -107,7 +107,8 @@ func (xbp *XattrBootstrapPersistence) loadRawConfig(ctx context.Context, c *gocb
 		// config
 		xattrContErr := res.ContentAt(0, &rawValue)
 		if xattrContErr != nil {
-			DebugfCtx(ctx, KeyCRUD, "No xattr config found for key=%s, path=%s: %v", key, cfgXattrConfigPath, xattrContErr)
+			SyncGatewayStats.GlobalStats.ConfigStat.XattrFormatMismatches.Add(1)
+			DebugfCtx(ctx, KeyCRUD, "Found config document but No xattr config found for key=%s, path=%s: %v", key, cfgXattrConfigPath, xattrContErr)
 			return rawValue, 0, ErrNotFound
 		}
 		return rawValue, res.Cas(), nil
@@ -181,7 +182,8 @@ func (xbp *XattrBootstrapPersistence) loadConfig(ctx context.Context, c *gocb.Co
 		// config
 		xattrContErr := res.ContentAt(0, valuePtr)
 		if xattrContErr != nil {
-			DebugfCtx(ctx, KeyCRUD, "No xattr config found for key=%s, path=%s: %v", key, cfgXattrConfigPath, xattrContErr)
+			SyncGatewayStats.GlobalStats.ConfigStat.XattrFormatMismatches.Add(1)
+			DebugfCtx(ctx, KeyCRUD, "Found config document but No xattr config found for key=%s, path=%s: %v", key, cfgXattrConfigPath, xattrContErr)
 			return 0, ErrNotFound
 		}
 		casOut := res.Cas()

base/config_persistence_test.go

@@ -16,6 +16,7 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+// TestConfigPersistence ensures that all implementations of ConfigPersistence behave as expected for common operations.
 func TestConfigPersistence(t *testing.T) {
 
 	if UnitTestUrlIsWalrus() {
@@ -198,3 +199,63 @@ func TestXattrConfigPersistence(t *testing.T) {
 	assert.Equal(t, configBody["sampleConfig"], loadedConfig["sampleConfig"])
 
 }
+
+// TestConfigPersistenceXattrFormatMismatches ensures that the XattrFormatMismatches stat is incremented when loading a config of a different format.
+func TestConfigPersistenceXattrFormatMismatches(t *testing.T) {
+
+	if UnitTestUrlIsWalrus() {
+		t.Skip("This test only works against Couchbase Server")
+	}
+
+	ctx := TestCtx(t)
+	bucket := GetTestBucket(t)
+	defer bucket.Close(ctx)
+
+	dataStore := bucket.GetSingleDataStore()
+
+	sgCollection, ok := dataStore.(*Collection)
+	require.True(t, ok)
+	c := sgCollection.Collection
+
+	nonXattrConfigPersistence := &DocumentBootstrapPersistence{}
+	nonXattrConfigKey := "testNonXattrConfigKey"
+	nonXattrConfigBody := map[string]interface{}{
+		"sampleConfig": "value",
+	}
+
+	xattrConfigKey := "testXattrConfigKey"
+	xattrConfigPersistence := &XattrBootstrapPersistence{}
+	xattrConfigBody := map[string]interface{}{
+		"sampleConfig": "value",
+	}
+
+	// create config without xattrs
+	_, marshalErr := JSONMarshal(nonXattrConfigBody)
+	require.NoError(t, marshalErr)
+	_, insertErr := nonXattrConfigPersistence.insertConfig(c, nonXattrConfigKey, nonXattrConfigBody)
+	require.NoError(t, insertErr)
+
+	existingStat := SyncGatewayStats.GlobalStats.ConfigStat.XattrFormatMismatches.Value()
+
+	// load config in xattr mode
+	data, _, err := xattrConfigPersistence.loadRawConfig(ctx, c, nonXattrConfigKey)
+	require.Error(t, err)
+	assert.Nil(t, data)
+	assert.Equal(t, existingStat+1, SyncGatewayStats.GlobalStats.ConfigStat.XattrFormatMismatches.Value())
+
+	// we can try the other way around (xattr config loading in non-xattr mode) - but it won't have any effect on the stat. This works in one way only.
+	_, marshalErr = JSONMarshal(xattrConfigBody)
+	require.NoError(t, marshalErr)
+	_, insertErr = xattrConfigPersistence.insertConfig(c, xattrConfigKey, xattrConfigBody)
+	require.NoError(t, insertErr)
+
+	existingStat = SyncGatewayStats.GlobalStats.ConfigStat.XattrFormatMismatches.Value()
+
+	// load config in non-xattr mode
+	data, _, err = nonXattrConfigPersistence.loadRawConfig(ctx, c, xattrConfigKey)
+	require.NoError(t, err)
+	assert.Equal(t, []byte(`{"cfgVersion":1}`), data) // whitespace stripped cfgXattrBody
+
+	// this stat isn't effective since it only works when loading a non-xattr config in xattr mode
+	assert.Equal(t, existingStat, SyncGatewayStats.GlobalStats.ConfigStat.XattrFormatMismatches.Value())
+}

base/stats.go

@@ -204,6 +204,10 @@ func (g *GlobalStat) initConfigStats() error {
 	if err != nil {
 		return err
 	}
+	configStat.XattrFormatMismatches, err = NewIntStat(ConfigSubsystem, "xattr_format_mismatches", StatUnitBytes, XattrFormatMismatchesDesc, StatAddedVersion3dot2dot4, StatDeprecatedVersionNotDeprecated, StatStabilityCommitted, nil, nil, prometheus.CounterValue, 0)
+	if err != nil {
+		return err
+	}
 	g.ConfigStat = configStat
 	return nil
 }
@@ -403,6 +407,8 @@ type ConfigStat struct {
 	DatabaseBucketMismatches *SgwIntStat `json:"database_config_bucket_mismatches"`
 	// The number of times the config was rolled back to an invalid state (conflicting collections)
 	DatabaseRollbackCollectionCollisions *SgwIntStat `json:"database_config_rollback_collection_collisions"`
+	// The number of times a non-xattr config or registry document was loaded in xattr mode
+	XattrFormatMismatches *SgwIntStat `json:"xattr_format_mismatches"`
 }
 
 type AuditStat struct {

base/stats_descriptions.go

@@ -74,6 +74,7 @@ const (
 const (
 	DatabaseBucketMismatchesDesc   = "The total number of times a database config is polled from a bucket that doesn't match the bucket specified in the database config."
 	DatabaseCollectionConflictDesc = "The total number of times a database config is rolled back to an invalid state (collection conflicts)."
+	XattrFormatMismatchesDesc      = "The total number of times a non-xattr config or registry document was loaded in xattr mode."
 )
 
 // audit stat