diff --git a/cmd/config/app_config_test.go b/cmd/config/app_config_test.go index 691d24ca..15679417 100644 --- a/cmd/config/app_config_test.go +++ b/cmd/config/app_config_test.go @@ -34,18 +34,18 @@ import ( var ( defaultServerTLSConfig = connection.TLSConfig{ Mode: connection.MutualTLSMode, - CertPath: "/server-certs/public-key", - KeyPath: "/server-certs/private-key", + CertPath: "/server-certs/public-key.crt", + KeyPath: "/server-certs/private-key.key", CACertPaths: []string{ - "/server-certs/ca-certificate", + "/server-certs/ca-certificate.crt", }, } defaultClientTLSConfig = connection.TLSConfig{ Mode: connection.MutualTLSMode, - CertPath: "/client-certs/public-key", - KeyPath: "/client-certs/private-key", + CertPath: "/client-certs/public-key.crt", + KeyPath: "/client-certs/private-key.key", CACertPaths: []string{ - "/client-certs/ca-certificate", + "/client-certs/ca-certificate.crt", }, } ) @@ -443,10 +443,14 @@ func defaultDBConfig() *vc.DatabaseConfig { func defaultSampleDBConfig() *vc.DatabaseConfig { return &vc.DatabaseConfig{ - Endpoints: []*connection.Endpoint{newEndpoint("db", 5433)}, - Username: "yugabyte", - Password: "yugabyte", - Database: "yugabyte", + Endpoints: []*connection.Endpoint{newEndpoint("db", 5433)}, + Username: "yugabyte", + Password: "yugabyte", + Database: "yugabyte", + TLS: connection.DatabaseTLS{ + Activate: true, + CACertPath: "/server-certs/ca-certificate.crt", + }, MaxConnections: 10, MinConnections: 5, LoadBalance: false, diff --git a/cmd/config/cobra_test_exports.go b/cmd/config/cobra_test_exports.go index beaed2ae..82299f47 100644 --- a/cmd/config/cobra_test_exports.go +++ b/cmd/config/cobra_test_exports.go @@ -64,8 +64,8 @@ func StartDefaultSystem(t *testing.T) SystemConfig { }, DB: DatabaseConfig{ Name: conn.Database, - LoadBalance: false, Endpoints: conn.Endpoints, + LoadBalance: false, }, Policy: &workload.PolicyProfile{ ChannelID: "channel1", diff --git a/cmd/config/create_config_file.go b/cmd/config/create_config_file.go index 268762a9..a0e67629 100644 --- a/cmd/config/create_config_file.go +++ b/cmd/config/create_config_file.go @@ -72,8 +72,10 @@ type ( // DatabaseConfig represents the used DB. DatabaseConfig struct { Name string + Password string LoadBalance bool Endpoints []*connection.Endpoint + TLS connection.DatabaseTLS } ) diff --git a/cmd/config/samples/coordinator.yaml b/cmd/config/samples/coordinator.yaml index 9a401ac2..4bc70ed0 100644 --- a/cmd/config/samples/coordinator.yaml +++ b/cmd/config/samples/coordinator.yaml @@ -6,10 +6,10 @@ server: endpoint: :9001 tls: mode: mtls - cert-path: /server-certs/public-key - key-path: /server-certs/private-key + cert-path: /server-certs/public-key.crt + key-path: /server-certs/private-key.key ca-cert-paths: - - /server-certs/ca-certificate + - /server-certs/ca-certificate.crt monitoring: server: endpoint: :2119 @@ -19,10 +19,10 @@ verifier: - verifier:5001 tls: &ClientTLS mode: mtls - cert-path: /client-certs/public-key - key-path: /client-certs/private-key + cert-path: /client-certs/public-key.crt + key-path: /client-certs/private-key.key ca-cert-paths: - - /client-certs/ca-certificate + - /client-certs/ca-certificate.crt validator-committer: endpoints: - vc:6001 diff --git a/cmd/config/samples/loadgen.yaml b/cmd/config/samples/loadgen.yaml index ef97fe63..4fef972b 100644 --- a/cmd/config/samples/loadgen.yaml +++ b/cmd/config/samples/loadgen.yaml @@ -6,10 +6,10 @@ server: endpoint: :8001 tls: mode: mtls - cert-path: /server-certs/public-key - key-path: /server-certs/private-key + cert-path: /server-certs/public-key.crt + key-path: /server-certs/private-key.key ca-cert-paths: - - /server-certs/ca-certificate + - /server-certs/ca-certificate.crt monitoring: server: endpoint: :2118 @@ -26,10 +26,10 @@ orderer-client: endpoint: sidecar:4001 tls: mode: mtls - cert-path: /client-certs/public-key - key-path: /client-certs/private-key + cert-path: /client-certs/public-key.crt + key-path: /client-certs/private-key.key ca-cert-paths: - - /client-certs/ca-certificate + - /client-certs/ca-certificate.crt orderer: connection: endpoints: diff --git a/cmd/config/samples/query.yaml b/cmd/config/samples/query.yaml index b2011ddc..2c57e9f0 100644 --- a/cmd/config/samples/query.yaml +++ b/cmd/config/samples/query.yaml @@ -8,11 +8,10 @@ server: endpoint: :7001 tls: mode: mtls - cert-path: /server-certs/public-key - key-path: /server-certs/private-key + cert-path: /server-certs/public-key.crt + key-path: /server-certs/private-key.key ca-cert-paths: - - /server-certs/ca-certificate - # Credentials for the server + - /server-certs/ca-certificate.crt monitoring: server: endpoint: :2117 @@ -24,6 +23,9 @@ database: # TODO: pass password via environment variable password: "yugabyte" # The password for the database database: "yugabyte" # The database name + tls: + activate: true + ca-cert-path: /server-certs/ca-certificate.crt max-connections: 10 # The maximum size of the connection pool min-connections: 5 # The minimum size of the connection pool load-balance: false # Should be enabled for DB cluster diff --git a/cmd/config/samples/sidecar.yaml b/cmd/config/samples/sidecar.yaml index bb7b722f..ba1d5690 100644 --- a/cmd/config/samples/sidecar.yaml +++ b/cmd/config/samples/sidecar.yaml @@ -6,10 +6,10 @@ server: endpoint: :4001 tls: mode: mtls - cert-path: /server-certs/public-key - key-path: /server-certs/private-key + cert-path: /server-certs/public-key.crt + key-path: /server-certs/private-key.key ca-cert-paths: - - /server-certs/ca-certificate + - /server-certs/ca-certificate.crt keep-alive: params: time: 300s @@ -40,10 +40,10 @@ committer: endpoint: coordinator:9001 tls: mode: mtls - cert-path: /client-certs/public-key - key-path: /client-certs/private-key + cert-path: /client-certs/public-key.crt + key-path: /client-certs/private-key.key ca-cert-paths: - - /client-certs/ca-certificate + - /client-certs/ca-certificate.crt ledger: path: /root/sc/ledger notification: diff --git a/cmd/config/samples/vc.yaml b/cmd/config/samples/vc.yaml index 54afaf09..2f61054d 100644 --- a/cmd/config/samples/vc.yaml +++ b/cmd/config/samples/vc.yaml @@ -8,11 +8,10 @@ server: endpoint: :6001 tls: mode: mtls - cert-path: /server-certs/public-key - key-path: /server-certs/private-key + cert-path: /server-certs/public-key.crt + key-path: /server-certs/private-key.key ca-cert-paths: - - /server-certs/ca-certificate - # Credentials for the server + - /server-certs/ca-certificate.crt monitoring: server: endpoint: :2116 @@ -23,6 +22,9 @@ database: # TODO: pass password via environment variable password: "yugabyte" # The password for the database database: "yugabyte" # The database name + tls: + activate: true + ca-cert-path: /server-certs/ca-certificate.crt max-connections: 10 # The maximum size of the connection pool min-connections: 5 # The minimum size of the connection pool. load-balance: false # Should be enabled for DB cluster. diff --git a/cmd/config/samples/verifier.yaml b/cmd/config/samples/verifier.yaml index 99fdb8c5..7f48b981 100644 --- a/cmd/config/samples/verifier.yaml +++ b/cmd/config/samples/verifier.yaml @@ -6,10 +6,10 @@ server: endpoint: :5001 tls: mode: mtls - cert-path: /server-certs/public-key - key-path: /server-certs/private-key + cert-path: /server-certs/public-key.crt + key-path: /server-certs/private-key.key ca-cert-paths: - - /server-certs/ca-certificate + - /server-certs/ca-certificate.crt monitoring: server: endpoint: :2115 diff --git a/cmd/config/templates/query.yaml b/cmd/config/templates/query.yaml index 46b5ecba..6e538a2c 100644 --- a/cmd/config/templates/query.yaml +++ b/cmd/config/templates/query.yaml @@ -24,9 +24,12 @@ database: {{- end }} username: "yugabyte" # TODO: pass password via environment variable - password: "yugabyte" + password: {{ .DB.Password }} database: {{ .DB.Name }} load-balance: {{ .DB.LoadBalance }} + tls: + activate: {{ .DB.TLS.Activate }} + ca-cert-path: {{ .DB.TLS.CACertPath }} max-connections: 10 min-connections: 5 retry: diff --git a/cmd/config/templates/vc.yaml b/cmd/config/templates/vc.yaml index 6e229546..7d613627 100644 --- a/cmd/config/templates/vc.yaml +++ b/cmd/config/templates/vc.yaml @@ -23,9 +23,12 @@ database: {{- end }} username: "yugabyte" # TODO: pass password via environment variable - password: "yugabyte" + password: {{ .DB.Password }} database: {{ .DB.Name }} load-balance: {{ .DB.LoadBalance }} + tls: + activate: {{ .DB.TLS.Activate }} + ca-cert-path: {{ .DB.TLS.CACertPath }} max-connections: 10 min-connections: 5 retry: diff --git a/docker/images/test_node/Dockerfile b/docker/images/test_node/Dockerfile index 8fef9ab4..189763fe 100644 --- a/docker/images/test_node/Dockerfile +++ b/docker/images/test_node/Dockerfile @@ -35,6 +35,10 @@ ENV SC_SIDECAR_COMMITTER_TLS_MODE="none" ENV SC_VC_SERVER_TLS_MODE="none" ENV SC_VERIFIER_SERVER_TLS_MODE="none" +# Disable TLS usage for db. +ENV SC_VC_DATABASE_TLS_ACTIVATE=false +ENV SC_QUERY_DATABASE_TLS_ACTIVATE=false + COPY ${ARCHBIN_PATH}/${TARGETOS}-${TARGETARCH}/* ${BINS_PATH}/ COPY ./docker/images/test_node/run ${BINS_PATH}/ COPY ./cmd/config/samples $CONFIGS_PATH diff --git a/docker/test/common.go b/docker/test/common.go index c30e2455..ea4e5cf6 100644 --- a/docker/test/common.go +++ b/docker/test/common.go @@ -134,7 +134,7 @@ func getContainerMappedHostPort( info, err := createDockerClient(t).ContainerInspect(ctx, containerName) require.NoError(t, err) require.NotNil(t, info) - portKey := nat.Port(fmt.Sprintf("%s/%s", containerPort, "tcp")) + portKey := nat.Port(fmt.Sprintf("%s/tcp", containerPort)) bindings, ok := info.NetworkSettings.Ports[portKey] require.True(t, ok) require.NotEmpty(t, bindings) diff --git a/docker/test/container_release_image_test.go b/docker/test/container_release_image_test.go index e2bb2661..2270dd19 100644 --- a/docker/test/container_release_image_test.go +++ b/docker/test/container_release_image_test.go @@ -21,6 +21,8 @@ import ( "github.com/hyperledger/fabric-x-committer/cmd/config" "github.com/hyperledger/fabric-x-committer/loadgen/workload" + "github.com/hyperledger/fabric-x-committer/service/vc/dbtest" + "github.com/hyperledger/fabric-x-committer/utils/connection" testutils "github.com/hyperledger/fabric-x-committer/utils/test" ) @@ -30,6 +32,8 @@ type startNodeParameters struct { networkName string tlsMode string configBlockPath string + dbType string + dbPassword string } const ( @@ -42,6 +46,25 @@ const ( containerConfigPath = "/root/config" // localConfigPath is the path to the sample YAML configuration of each service. localConfigPath = "../../cmd/config/samples" + + // containerPathForYugabytePassword holds the path to the database credentials inside the docker container. + // This work-around is needed due to a Yugabyte behavior that prevents using default passwords in secure mode. + // Instead, Yugabyte generates a random password, and this path points to the output file containing it. + containerPathForYugabytePassword = "/root/var/data/yugabyted_credentials.txt" //nolint:gosec +) + +var ( + // enforcePostgresSSLScript enforces SSL-only client connections to a PostgreSQL instance by updating pg_hba.conf. + enforcePostgresSSLScript = []string{ + "sh", "-c", + `sed -i 's/^host all all all scram-sha-256$/hostssl all all 0.0.0.0\/0 scram-sha-256/' ` + + `/var/lib/postgresql/data/pg_hba.conf`, + } + + // reloadPostgresConfigScript reloads the PostgreSQL server configuration without restarting the instance. + reloadPostgresConfigScript = []string{ + "psql", "-U", "yugabyte", "-c", "SELECT pg_reload_conf();", + } ) // TestCommitterReleaseImagesWithTLS runs the committer components in different Docker containers with different TLS @@ -71,22 +94,25 @@ func TestCommitterReleaseImagesWithTLS(t *testing.T) { testutils.RemoveDockerNetwork(t, networkName) }) + params := startNodeParameters{ + credsFactory: credsFactory, + networkName: networkName, + tlsMode: mode, + configBlockPath: configBlockPath, + dbType: testutils.YugaDBType, + } + for _, node := range []string{ "db", "verifier", "vc", "query", "coordinator", "sidecar", "orderer", "loadgen", } { - params := startNodeParameters{ - credsFactory: credsFactory, - node: node, - networkName: networkName, - tlsMode: mode, - configBlockPath: configBlockPath, - } - + params.node = node // stop and remove the container if it already exists. stopAndRemoveContainersByName(ctx, t, createDockerClient(t), assembleContainerName(node, mode)) switch node { - case "db", "orderer": + case "db": + params.dbPassword = startSecuredDatabaseNode(ctx, t, params).Password + case "orderer": startCommitterNodeWithTestImage(ctx, t, params) case "loadgen": startLoadgenNodeWithReleaseImage(ctx, t, params) @@ -101,12 +127,67 @@ func TestCommitterReleaseImagesWithTLS(t *testing.T) { } } +// CreateAndStartSecuredDatabaseNode creates a containerized YugabyteDB or PostgreSQL +// database instance in a secure mode. +func startSecuredDatabaseNode(ctx context.Context, t *testing.T, params startNodeParameters) *dbtest.Connection { + t.Helper() + + tlsConfig, credsPath := params.credsFactory.CreateServerCredentials(t, params.tlsMode, params.dbType, params.node) + + node := &dbtest.DatabaseContainer{ + DatabaseType: params.dbType, + Network: params.networkName, + Hostname: params.node, + TLSConfig: tlsConfig, + CredsPathDir: credsPath, + UseTLS: true, + } + + node.StartContainer(ctx, t) + conn := node.GetConnectionOptions(ctx, t) + + // This is relevant if a different CA was used to issue the DB's TLS certificates. + require.NotEmpty(t, node.TLSConfig.CACertPaths) + conn.TLS = connection.DatabaseTLS{ + Activate: true, + CACertPath: node.TLSConfig.CACertPaths[0], + } + + // post start container tweaking + switch node.DatabaseType { + case testutils.YugaDBType: + // Ensure proper root ownership and permissions for the TLS certificate files. + node.ExecuteCommand(t, []string{ + "chown", "root:root", + fmt.Sprintf("/creds/node.%s.crt", node.Hostname), + fmt.Sprintf("/creds/node.%s.key", node.Hostname), + }) + node.EnsureNodeReadiness(t, "Data placement constraint successfully verified") + conn.Password = node.ReadPasswordFromContainer(t, containerPathForYugabytePassword) + case testutils.PostgresDBType: + // Ensure proper root ownership and permissions for the TLS certificate files. + node.ExecuteCommand(t, []string{ + "chown", "postgres:postgres", + "/creds/server.crt", + "/creds/server.key", + }) + node.EnsureNodeReadiness(t, dbtest.PostgresReadinessOutput) + node.ExecuteCommand(t, enforcePostgresSSLScript) + node.ExecuteCommand(t, reloadPostgresConfigScript) + default: + t.Fatalf("Unsupported database type: %s", node.DatabaseType) + } + + t.Cleanup( + func() { + node.StopAndRemoveContainer(t) + }) + + return conn +} + // startCommitterNodeWithReleaseImage starts a committer node using the release image. -func startCommitterNodeWithReleaseImage( - ctx context.Context, - t *testing.T, - params startNodeParameters, -) { +func startCommitterNodeWithReleaseImage(ctx context.Context, t *testing.T, params startNodeParameters) { t.Helper() configPath := filepath.Join(containerConfigPath, params.node) @@ -129,6 +210,8 @@ func startCommitterNodeWithReleaseImage( "SC_SIDECAR_COMMITTER_TLS_MODE=" + params.tlsMode, "SC_VC_SERVER_TLS_MODE=" + params.tlsMode, "SC_VERIFIER_SERVER_TLS_MODE=" + params.tlsMode, + "SC_VC_DATABASE_PASSWORD=" + params.dbPassword, + "SC_QUERY_DATABASE_PASSWORD=" + params.dbPassword, }, Tty: true, }, @@ -222,7 +305,9 @@ func assembleContainerName(node, tlsMode string) string { func assembleBinds(t *testing.T, params startNodeParameters, additionalBinds ...string) []string { t.Helper() - _, serverCredsPath := params.credsFactory.CreateServerCredentials(t, params.tlsMode, params.node) + _, serverCredsPath := params.credsFactory.CreateServerCredentials( + t, params.tlsMode, testutils.DefaultCertStyle, params.node, + ) require.NotEmpty(t, serverCredsPath) _, clientCredsPath := params.credsFactory.CreateClientCredentials(t, params.tlsMode) require.NotEmpty(t, clientCredsPath) diff --git a/integration/runner/postgres.go b/integration/runner/postgres.go index 60c1b6db..afb6798d 100644 --- a/integration/runner/postgres.go +++ b/integration/runner/postgres.go @@ -16,6 +16,7 @@ import ( "github.com/stretchr/testify/require" "github.com/hyperledger/fabric-x-committer/service/vc/dbtest" + "github.com/hyperledger/fabric-x-committer/utils/test" ) const ( @@ -99,7 +100,7 @@ func (cc *PostgresClusterController) addPrimaryNode(ctx context.Context, t *test }, }) node.StartContainer(ctx, t) - node.EnsureNodeReadiness(t, "database system is ready to accept connections") + node.EnsureNodeReadiness(t, dbtest.PostgresReadinessOutput) return node } @@ -132,7 +133,7 @@ func (cc *PostgresClusterController) createNode( Role: nodeCreationOpts.role, Image: postgresImage, Tag: defaultBitnamiPostgresTag, - DatabaseType: dbtest.PostgresDBType, + DatabaseType: test.PostgresDBType, Env: append([]string{ "POSTGRESQL_REPLICATION_USER=repl_user", "POSTGRESQL_REPLICATION_PASSWORD=repl_password", diff --git a/integration/runner/runtime.go b/integration/runner/runtime.go index 2ce3e33e..8676ac0e 100644 --- a/integration/runner/runtime.go +++ b/integration/runner/runtime.go @@ -95,8 +95,8 @@ type ( BlockTimeout time.Duration LoadgenBlockLimit uint64 - // DBCluster configures the cluster to operate in DB cluster mode. - DBCluster *dbtest.Connection + // DBConnection configures the runtime to operate with a custom database connection. + DBConnection *dbtest.Connection // TLS configures the secure level between the components: none | tls | mtls TLSMode string @@ -159,16 +159,18 @@ func NewRuntime(t *testing.T, conf *Config) *CommitterRuntime { c.AddOrUpdateNamespaces(t, types.MetaNamespaceID, workload.GeneratedNamespaceID, "1", "2", "3") t.Log("Making DB env") - if conf.DBCluster == nil { + if conf.DBConnection == nil { c.dbEnv = vc.NewDatabaseTestEnv(t) } else { - c.dbEnv = vc.NewDatabaseTestEnvWithCluster(t, conf.DBCluster) + c.dbEnv = vc.NewDatabaseTestEnvWithCustomConnection(t, conf.DBConnection) } s := &c.SystemConfig s.DB.Name = c.dbEnv.DBConf.Database + s.DB.Password = c.dbEnv.DBConf.Password s.DB.LoadBalance = c.dbEnv.DBConf.LoadBalance s.DB.Endpoints = c.dbEnv.DBConf.Endpoints + s.DB.TLS = c.dbEnv.DBConf.TLS s.LedgerPath = t.TempDir() s.ConfigBlockPath = filepath.Join(t.TempDir(), "config-block.pb.bin") @@ -581,7 +583,12 @@ func (c *CommitterRuntime) createSystemConfigWithServerTLS( ) *config.SystemConfig { t.Helper() serviceCfg := c.SystemConfig - serviceCfg.ServiceTLS, _ = c.CredFactory.CreateServerCredentials(t, c.config.TLSMode, endpoints.Server.Host) + serviceCfg.ServiceTLS, _ = c.CredFactory.CreateServerCredentials( + t, + c.config.TLSMode, + test.DefaultCertStyle, + endpoints.Server.Host, + ) serviceCfg.ServiceEndpoints = endpoints return &serviceCfg } diff --git a/integration/runner/yugabyte.go b/integration/runner/yugabyte.go index 7b0d9bf9..c8761900 100644 --- a/integration/runner/yugabyte.go +++ b/integration/runner/yugabyte.go @@ -125,7 +125,7 @@ func (cc *YugaClusterController) createNode(role string) { Name: fmt.Sprintf("yuga-%s-%s", role, uuid.New().String()), Image: defaultImage, Role: role, - DatabaseType: dbtest.YugaDBType, + DatabaseType: test.YugaDBType, Network: cc.networkName, } cc.nodes = append(cc.nodes, node) diff --git a/integration/test/db_node_failure_handling_test.go b/integration/test/db_node_failure_handling_test.go index 5a111d5f..1794b6bc 100644 --- a/integration/test/db_node_failure_handling_test.go +++ b/integration/test/db_node_failure_handling_test.go @@ -109,7 +109,7 @@ func registerAndCreateRuntime(t *testing.T, clusterConnection *dbtest.Connection NumVCService: 2, BlockTimeout: 2 * time.Second, BlockSize: 500, - DBCluster: clusterConnection, + DBConnection: clusterConnection, }) c.Start(t, runner.FullTxPathWithLoadGen) diff --git a/loadgen/client_test.go b/loadgen/client_test.go index 4d39c427..8afa7074 100644 --- a/loadgen/client_test.go +++ b/loadgen/client_test.go @@ -453,7 +453,7 @@ func createServerAndClientTLSConfig(t *testing.T, tlsMode string) ( ) { t.Helper() credsFactory := test.NewCredentialsFactory(t) - clientTLSConfig, _ = credsFactory.CreateServerCredentials(t, tlsMode, defaultServerSAN) + clientTLSConfig, _ = credsFactory.CreateServerCredentials(t, tlsMode, test.DefaultCertStyle, defaultServerSAN) serverTLSConfig, _ = credsFactory.CreateClientCredentials(t, tlsMode) return clientTLSConfig, serverTLSConfig } diff --git a/service/vc/config.go b/service/vc/config.go index 21a91d72..1de3e22d 100644 --- a/service/vc/config.go +++ b/service/vc/config.go @@ -32,13 +32,23 @@ type DatabaseConfig struct { MinConnections int32 `mapstructure:"min-connections"` LoadBalance bool `mapstructure:"load-balance"` Retry *connection.RetryProfile `mapstructure:"retry"` + TLS connection.DatabaseTLS `mapstructure:"tls"` } // DataSourceName returns the data source name of the database. func (d *DatabaseConfig) DataSourceName() string { - ret := fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", + ret := fmt.Sprintf("postgres://%s:%s@%s/%s?", d.Username, d.Password, d.EndpointsString(), d.Database) + if d.TLS.UseTLS() { + // Enforce full SSL verification: + // requires an encrypted connection (TLS), + // and ensures the server hostname matches the certificate. + ret += "sslmode=verify-full" + ret += fmt.Sprintf("&sslrootcert=%s", d.TLS.CACertPath) + } else { + ret += "sslmode=disable" + } // The load balancing flag is only available when the server supports it (having multiple nodes). // Thus, we only add it when explicitly required. Otherwise, an error will occur. if d.LoadBalance { diff --git a/service/vc/dbtest/connection.go b/service/vc/dbtest/connection.go index 8e02fbb8..250bf330 100644 --- a/service/vc/dbtest/connection.go +++ b/service/vc/dbtest/connection.go @@ -44,6 +44,7 @@ type Connection struct { Password string Database string LoadBalance bool + TLS connection.DatabaseTLS } // NewConnection returns a connection parameters with the specified host:port, and the default values @@ -58,8 +59,24 @@ func NewConnection(endpoints ...*connection.Endpoint) *Connection { // dataSourceName returns the dataSourceName to be used by the database/sql package. func (c *Connection) dataSourceName() string { - return fmt.Sprintf("postgres://%s:%s@%s/%s?sslmode=disable", + ret := fmt.Sprintf("postgres://%s:%s@%s/%s?", c.User, c.Password, c.endpointsString(), c.Database) + + if c.TLS.UseTLS() { + // Enforce full SSL verification: + // requires an encrypted connection (TLS), + // and ensures the server hostname matches the certificate. + ret += "sslmode=verify-full" + ret += fmt.Sprintf("&sslrootcert=%s", c.TLS.CACertPath) + } else { + ret += "sslmode=disable" + } + // The load balancing flag is only available when the server supports it (having multiple nodes). + // Thus, we only add it when explicitly required. Otherwise, an error will occur. + if c.LoadBalance { + ret += "&load_balance=true" + } + return ret } // endpointsString returns the address:port as a string with comma as a separator between endpoints. diff --git a/service/vc/dbtest/container.go b/service/vc/dbtest/container.go index 3260cbac..bb9199b6 100644 --- a/service/vc/dbtest/container.go +++ b/service/vc/dbtest/container.go @@ -7,10 +7,12 @@ SPDX-License-Identifier: Apache-2.0 package dbtest import ( + "bufio" "bytes" "context" "fmt" "os" + "regexp" "strconv" "strings" "testing" @@ -18,6 +20,7 @@ import ( "github.com/cockroachdb/errors" docker "github.com/fsouza/go-dockerclient" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -36,9 +39,12 @@ const ( // container's Memory and CPU management. gb = 1 << 30 // gb is the number of bytes needed to represent 1 GB. memorySwap = -1 // memorySwap disable memory swaps (don't store data on disk) + + // PostgresReadinessOutput is the output indicating that a Postgres node is ready. + PostgresReadinessOutput = "database system is ready to accept connections" ) -// YugabyteCMD starts yugabyte without SSL and fault tolerance (single server). +// YugabyteCMD starts yugabyte without fault tolerance (single server). var YugabyteCMD = []string{ "bin/yugabyted", "start", "--callhome", "false", @@ -50,7 +56,6 @@ var YugabyteCMD = []string{ "yb_num_shards_per_tserver=1," + "minloglevel=3," + "yb_enable_read_committed_isolation=true", - "--insecure", } // DatabaseContainer manages the execution of an instance of a dockerized DB for tests. @@ -59,17 +64,22 @@ type DatabaseContainer struct { Image string HostIP string Network string + Hostname string DatabaseType string Tag string Role string + CredsPathDir string Cmd []string Env []string + Binds []string HostPort int DbPort docker.Port PortMap docker.Port PortBinds map[docker.Port][]docker.PortBinding NetToIP map[string]*docker.EndpointConfig AutoRm bool + UseTLS bool + TLSConfig connection.TLSConfig client *docker.Client containerID string @@ -100,19 +110,24 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit t.Helper() switch dc.DatabaseType { - case YugaDBType: + case test.YugaDBType: if dc.Image == "" { dc.Image = defaultYugabyteImage } if dc.Cmd == nil { dc.Cmd = YugabyteCMD + if dc.UseTLS { + dc.Cmd = append(dc.Cmd, "--secure", "--certs_dir=/creds", "--advertise_address", dc.Hostname) + } else { + dc.Cmd = append(dc.Cmd, "--insecure") + } } if dc.DbPort == "" { dc.DbPort = docker.Port(fmt.Sprintf("%s/tcp", yugaDBPort)) } - case PostgresDBType: + case test.PostgresDBType: if dc.Image == "" { dc.Image = defaultPostgresImage } @@ -127,6 +142,13 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit if dc.DbPort == "" { dc.DbPort = docker.Port(fmt.Sprintf("%s/tcp", postgresDBPort)) } + if dc.UseTLS { + dc.Cmd = []string{ + "-c", "ssl=on", + "-c", "ssl_cert_file=/creds/server.crt", + "-c", "ssl_key_file=/creds/server.key", + } + } default: t.Fatalf("Unsupported database type: %s", dc.DatabaseType) } @@ -154,6 +176,10 @@ func (dc *DatabaseContainer) initDefaults(t *testing.T) { //nolint:gocognit if dc.client == nil { dc.client = test.GetDockerClient(t) } + if dc.UseTLS { + dc.Binds = append(dc.Binds, fmt.Sprintf("%s:/creds", dc.CredsPathDir)) + dc.Name += fmt.Sprintf("_with_tls_%s", uuid.NewString()[0:8]) + } } // createContainer attempts to create a container instance, or attach to an existing one. @@ -179,14 +205,16 @@ func (dc *DatabaseContainer) createContainer(ctx context.Context, t *testing.T) Context: ctx, Name: dc.Name, Config: &docker.Config{ - Image: dc.Image, - Cmd: dc.Cmd, - Env: dc.Env, + Image: dc.Image, + Cmd: dc.Cmd, + Env: dc.Env, + Hostname: dc.Hostname, }, HostConfig: &docker.HostConfig{ AutoRemove: dc.AutoRm, PortBindings: dc.PortBinds, NetworkMode: dc.Network, + Binds: dc.Binds, Memory: 4 * gb, MemorySwap: memorySwap, }, @@ -223,8 +251,8 @@ func (dc *DatabaseContainer) findContainer(t *testing.T) error { return errors.Errorf("cannot find container '%s'. Containers: %v", dc.Name, names) } -// getConnectionOptions inspect the container and fetches the available connection options. -func (dc *DatabaseContainer) getConnectionOptions(ctx context.Context, t *testing.T) *Connection { +// GetConnectionOptions inspect the container and fetches the available connection options. +func (dc *DatabaseContainer) GetConnectionOptions(ctx context.Context, t *testing.T) *Connection { t.Helper() container, err := dc.client.InspectContainerWithOptions(docker.InspectContainerOptions{ Context: ctx, @@ -233,7 +261,7 @@ func (dc *DatabaseContainer) getConnectionOptions(ctx context.Context, t *testin require.NoError(t, err) endpoints := []*connection.Endpoint{ - connection.CreateEndpointHP(container.NetworkSettings.IPAddress, dc.DbPort.Port()), + dc.GetContainerConnectionDetails(t), } for _, p := range container.NetworkSettings.Ports[dc.DbPort] { endpoints = append(endpoints, connection.CreateEndpointHP(p.HostIP, p.HostPort)) @@ -317,6 +345,30 @@ func (dc *DatabaseContainer) ContainerID() string { return dc.containerID } +// ReadPasswordFromContainer extracts the randomly generated password from a file inside the container. +// This is required because YugabyteDB, when running in secure mode, doesn't allow default passwords +// and instead generates a random one at startup. +// If no password is found, the default one will be returned. +func (dc *DatabaseContainer) ReadPasswordFromContainer(t *testing.T, filePath string) string { + t.Helper() + output := dc.ExecuteCommand(t, []string{"cat", filePath}) + + scanner := bufio.NewScanner(strings.NewReader(output)) + re := regexp.MustCompile(`(?i)^password:\s*(.+)$`) + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if matches := re.FindStringSubmatch(line); len(matches) == 2 { + return matches[1] + } + } + + require.NoError(t, scanner.Err(), "error scanning command output") + t.Log("password not found in output, returning default password.") + + return defaultPassword +} + // ExecuteCommand executes a command and returns the container output. func (dc *DatabaseContainer) ExecuteCommand(t *testing.T, cmd []string) string { t.Helper() diff --git a/service/vc/dbtest/database_setup.go b/service/vc/dbtest/database_setup.go index 78b16f05..2aa1efd1 100644 --- a/service/vc/dbtest/database_setup.go +++ b/service/vc/dbtest/database_setup.go @@ -22,6 +22,7 @@ import ( "github.com/hyperledger/fabric-x-committer/utils" "github.com/hyperledger/fabric-x-committer/utils/connection" + "github.com/hyperledger/fabric-x-committer/utils/test" ) const ( @@ -31,9 +32,6 @@ const ( deploymentLocal = "local" deploymentContainer = "container" - YugaDBType = "yugabyte" //nolint:revive - PostgresDBType = "postgres" - yugaDBPort = "5433" postgresDBPort = "5432" @@ -73,7 +71,7 @@ func getDBTypeFromEnv() string { if found { return strings.ToLower(val) } - return YugaDBType + return test.YugaDBType } // PrepareTestEnv initializes a test environment for an existing or uncontrollable db instance. @@ -120,7 +118,7 @@ func StartAndConnect(ctx context.Context, t *testing.T) *Connection { DatabaseType: getDBTypeFromEnv(), } container.StartContainer(ctx, t) - connOptions = container.getConnectionOptions(ctx, t) + connOptions = container.GetConnectionOptions(ctx, t) case deploymentLocal: connOptions = NewConnection(connection.CreateEndpointHP("localhost", defaultLocalDBPort)) default: diff --git a/service/vc/test_exports.go b/service/vc/test_exports.go index 2e6c59d7..f39f9d8b 100644 --- a/service/vc/test_exports.go +++ b/service/vc/test_exports.go @@ -131,8 +131,8 @@ func NewDatabaseTestEnv(t *testing.T) *DatabaseTestEnv { return newDatabaseTestEnv(t, dbtest.PrepareTestEnv(t), false) } -// NewDatabaseTestEnvWithCluster creates a new db cluster test environment. -func NewDatabaseTestEnvWithCluster(t *testing.T, dbConnections *dbtest.Connection) *DatabaseTestEnv { +// NewDatabaseTestEnvWithCustomConnection creates a new db test environment given a db connection. +func NewDatabaseTestEnvWithCustomConnection(t *testing.T, dbConnections *dbtest.Connection) *DatabaseTestEnv { t.Helper() require.NotNil(t, dbConnections) return newDatabaseTestEnv(t, dbtest.PrepareTestEnvWithConnection(t, dbConnections), dbConnections.LoadBalance) @@ -148,6 +148,7 @@ func newDatabaseTestEnv(t *testing.T, cs *dbtest.Connection, loadBalance bool) * MaxConnections: 10, MinConnections: 1, LoadBalance: loadBalance, + TLS: cs.TLS, Retry: &connection.RetryProfile{ MaxElapsedTime: 5 * time.Minute, InitialInterval: time.Duration(rand.Intn(900)+100) * time.Millisecond, diff --git a/utils/connection/config.go b/utils/connection/config.go index 8194d510..ee63e813 100644 --- a/utils/connection/config.go +++ b/utils/connection/config.go @@ -77,6 +77,12 @@ type ( KeyPath string `mapstructure:"key-path"` CACertPaths []string `mapstructure:"ca-cert-paths"` } + + // DatabaseTLS holds the database connection credentials. + DatabaseTLS struct { + Activate bool `mapstructure:"activate"` + CACertPath string `mapstructure:"ca-cert-path"` + } ) const ( @@ -90,6 +96,11 @@ const ( DefaultTLSMinVersion = tls.VersionTLS12 ) +// UseTLS sets the option of using TLS configuration for database connection. +func (dc *DatabaseTLS) UseTLS() bool { + return dc.Activate && dc.CACertPath != "" +} + // ServerCredentials returns the gRPC transport credentials to be used by a server, // based on the provided TLS configuration. func (c TLSConfig) ServerCredentials() (credentials.TransportCredentials, error) { diff --git a/utils/test/secure_connection.go b/utils/test/secure_connection.go index a0123c2d..8f654c7a 100644 --- a/utils/test/secure_connection.go +++ b/utils/test/secure_connection.go @@ -36,6 +36,12 @@ type ( shouldFail bool } + createTLSConfigParameters struct { + connectionMode string + keyPair *tlsgen.CertKeyPair + namingStyle string + } + // ServerStarter is a function that receives a TLS configuration, starts the server, // and returns a RPCAttempt function for initiating a client connection and attempting an RPC call. ServerStarter func(t *testing.T, serverTLS connection.TLSConfig) RPCAttempt @@ -45,7 +51,20 @@ type ( RPCAttempt func(ctx context.Context, t *testing.T, cfg connection.TLSConfig) error ) -const defaultHostName = "localhost" +const ( + defaultHostName = "localhost" + + // YugaDBType represents the usage of Yugabyte DB. + YugaDBType = "yugabyte" + // PostgresDBType represents the usage of PostgreSQL DB. + PostgresDBType = "postgres" + // DefaultCertStyle represents the default TLS certificate style creation. + DefaultCertStyle = "default" + //nolint:revive // KeyPrivate, KeyPublic and KeyCACert represents the chosen key in the naming function. + KeyPrivate = "private-key" + KeyPublic = "public-key" + KeyCACert = "ca-certificate" +) // ServerModes is a list of server-side TLS modes used for testing. var ServerModes = []string{connection.MutualTLSMode, connection.OneSideTLSMode, connection.NoneTLSMode} @@ -65,12 +84,17 @@ func NewCredentialsFactory(t *testing.T) *CredentialsFactory { func (scm *CredentialsFactory) CreateServerCredentials( t *testing.T, tlsMode string, + namingStyle string, san ...string, ) (connection.TLSConfig, string) { t.Helper() serverKeypair, err := scm.CertificateAuthority.NewServerCertKeyPair(san...) require.NoError(t, err) - return createTLSConfig(t, tlsMode, serverKeypair, scm.CertificateAuthority.CertBytes()) + return scm.createTLSConfig(t, createTLSConfigParameters{ + connectionMode: tlsMode, + keyPair: serverKeypair, + namingStyle: namingStyle, + }) } // CreateClientCredentials creates a client key pair, @@ -79,7 +103,11 @@ func (scm *CredentialsFactory) CreateClientCredentials(t *testing.T, tlsMode str t.Helper() clientKeypair, err := scm.CertificateAuthority.NewClientCertKeyPair() require.NoError(t, err) - return createTLSConfig(t, tlsMode, clientKeypair, scm.CertificateAuthority.CertBytes()) + return scm.createTLSConfig(t, createTLSConfigParameters{ + connectionMode: tlsMode, + keyPair: clientKeypair, + namingStyle: DefaultCertStyle, + }) } /* @@ -136,7 +164,7 @@ func RunSecureConnectionTest( t.Run(fmt.Sprintf("server-tls:%s", tc.serverMode), func(t *testing.T) { t.Parallel() // create server's tls config and start it according to the server tls mode. - serverTLS, _ := tlsMgr.CreateServerCredentials(t, tc.serverMode, defaultHostName) + serverTLS, _ := tlsMgr.CreateServerCredentials(t, tc.serverMode, DefaultCertStyle, defaultHostName) rpcAttemptFunc := starter(t, serverTLS) // for each server secure mode, build the client's test cases. for _, clientTestCase := range tc.cases { @@ -186,28 +214,74 @@ func CreateClientWithTLS[T any]( // createTLSConfig creates a TLS configuration based on the // given TLS mode and credential bytes, and returns it along with the certificates' path. -func createTLSConfig( +func (scm *CredentialsFactory) createTLSConfig( t *testing.T, - connectionMode string, - keyPair *tlsgen.CertKeyPair, - caCertificate []byte, + params createTLSConfigParameters, ) (connection.TLSConfig, string) { t.Helper() tmpDir := t.TempDir() + namingFunction := selectFileNames(params.namingStyle) - privateKeyPath := filepath.Join(tmpDir, "private-key") - require.NoError(t, os.WriteFile(privateKeyPath, keyPair.Key, 0o600)) + privateKeyPath := filepath.Join(tmpDir, namingFunction("private-key")) + require.NoError(t, os.WriteFile(privateKeyPath, params.keyPair.Key, 0o600)) - publicKeyPath := filepath.Join(tmpDir, "public-key") - require.NoError(t, os.WriteFile(publicKeyPath, keyPair.Cert, 0o600)) + publicKeyPath := filepath.Join(tmpDir, namingFunction("public-key")) + require.NoError(t, os.WriteFile(publicKeyPath, params.keyPair.Cert, 0o600)) - caCertificatePath := filepath.Join(tmpDir, "ca-certificate") - require.NoError(t, os.WriteFile(caCertificatePath, caCertificate, 0o600)) + caCertificatePath := filepath.Join(tmpDir, namingFunction("ca-certificate")) + require.NoError(t, os.WriteFile(caCertificatePath, scm.CertificateAuthority.CertBytes(), 0o600)) return connection.TLSConfig{ - Mode: connectionMode, + Mode: params.connectionMode, KeyPath: privateKeyPath, CertPath: publicKeyPath, CACertPaths: []string{caCertificatePath}, }, tmpDir } + +func selectFileNames(style string) func(string) string { + switch style { + case YugaDBType: + return func(key string) string { + switch key { + // We currently use YugabyteDB with the hostname "db" only. + // To support additional instances with different hostnames, + // replace "db" with the desired hostname when creating the instance. + case KeyPublic: + return "node.db.crt" + case KeyPrivate: + return "node.db.key" + case KeyCACert: + return "ca.crt" + default: + return "" + } + } + case PostgresDBType: + return func(key string) string { + switch key { + case KeyPublic: + return "server.crt" + case KeyPrivate: + return "server.key" + case KeyCACert: + return "ca-certificate.crt" + default: + return "" + } + } + default: + return func(key string) string { + switch key { + case KeyPublic: + return "public-key.crt" + case KeyPrivate: + return "private-key.key" + case KeyCACert: + return "ca-certificate.crt" + default: + return "" + } + } + } +}