Skip to content

Commit 13739d8

Browse files
meroton-benjaminmeroton-benjamin
committed
Switch sharding struct from array to map
Improves the ability to perform resharding by switching the sharding struct from array to map. Each map entry has a key which is used in rendezvous hashing to deterministically select which shard to use from the collection of keys. When a shard is removed it is guaranteed that only blobs which belonged to the removed shard will resolve to a new shard. In combination with ReadFallbackConfigurations this allows adding and removing shards with minimal need to rebalance the blobs between the shards. For more details and migration instructions, see: https://github.com/buildbarn/bb-adrs/blob/bf2066633e1712a3ef7c295d37cd52e65867391d/0011-rendezvous-hashing.md
1 parent e8d8188 commit 13739d8

21 files changed

+1380
-304
lines changed

‎internal/mock/BUILD.bazel‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,13 +86,23 @@ gomock(
8686
gomock(
8787
name = "blobstore_sharding",
8888
out = "blobstore_sharding.go",
89-
interfaces = ["ShardPermuter"],
89+
interfaces = ["ShardSelector"],
9090
library = "//pkg/blobstore/sharding",
9191
mockgen_model_library = "@org_uber_go_mock//mockgen/model",
9292
mockgen_tool = "@org_uber_go_mock//mockgen",
9393
package = "mock",
9494
)
9595

96+
gomock(
97+
name = "blobstore_legacy_sharding",
98+
out = "blobstore_legacy_sharding.go",
99+
interfaces = ["ShardPermuter"],
100+
library = "//pkg/blobstore/sharding/legacy",
101+
mockgen_model_library = "@org_uber_go_mock//mockgen/model",
102+
mockgen_tool = "@org_uber_go_mock//mockgen",
103+
package = "mock",
104+
)
105+
96106
gomock(
97107
name = "blobstore_slicing",
98108
out = "blobstore_slicing.go",
@@ -357,6 +367,7 @@ go_library(
357367
"aliases.go",
358368
"auth.go",
359369
"blobstore.go",
370+
"blobstore_legacy_sharding.go",
360371
"blobstore_local.go",
361372
"blobstore_replication.go",
362373
"blobstore_sharding.go",
@@ -391,6 +402,7 @@ go_library(
391402
"//pkg/blobstore/buffer",
392403
"//pkg/blobstore/local",
393404
"//pkg/blobstore/sharding",
405+
"//pkg/blobstore/sharding/legacy",
394406
"//pkg/blobstore/slicing",
395407
"//pkg/builder",
396408
"//pkg/clock",

‎pkg/blobstore/configuration/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ go_library(
2929
"//pkg/blobstore/readfallback",
3030
"//pkg/blobstore/replication",
3131
"//pkg/blobstore/sharding",
32+
"//pkg/blobstore/sharding/legacy",
3233
"//pkg/blockdevice",
3334
"//pkg/capabilities",
3435
"//pkg/clock",

‎pkg/blobstore/configuration/new_blob_access.go‎

Lines changed: 76 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/buildbarn/bb-storage/pkg/blobstore/readcaching"
1414
"github.com/buildbarn/bb-storage/pkg/blobstore/readfallback"
1515
"github.com/buildbarn/bb-storage/pkg/blobstore/sharding"
16+
"github.com/buildbarn/bb-storage/pkg/blobstore/sharding/legacy"
1617
"github.com/buildbarn/bb-storage/pkg/blockdevice"
1718
"github.com/buildbarn/bb-storage/pkg/clock"
1819
"github.com/buildbarn/bb-storage/pkg/digest"
@@ -85,43 +86,95 @@ func (nc *simpleNestedBlobAccessCreator) newNestedBlobAccessBare(configuration *
8586
DigestKeyFormat: slow.DigestKeyFormat,
8687
}, "read_caching", nil
8788
case *pb.BlobAccessConfiguration_Sharding:
88-
backends := make([]blobstore.BlobAccess, 0, len(backend.Sharding.Shards))
89-
weights := make([]uint32, 0, len(backend.Sharding.Shards))
90-
var combinedDigestKeyFormat *digest.KeyFormat
91-
for _, shard := range backend.Sharding.Shards {
92-
if shard.Backend == nil {
93-
// Drained backend.
94-
backends = append(backends, nil)
95-
} else {
96-
// Undrained backend.
89+
if backend.Sharding.Legacy == nil {
90+
backends := make([]sharding.ShardBackend, 0, len(backend.Sharding.Shards))
91+
shards := make([]sharding.Shard, 0, len(backend.Sharding.Shards))
92+
keys := make([]string, 0, len(backend.Sharding.Shards))
93+
var combinedDigestKeyFormat *digest.KeyFormat
94+
for key, shard := range backend.Sharding.Shards {
95+
if shard.Backend == nil {
96+
return BlobAccessInfo{}, "", status.Errorf(codes.InvalidArgument, "Shard '%s' has an undefined backend, drained backends are only allowed when running in Legacy mode", key)
97+
}
9798
backend, err := nc.NewNestedBlobAccess(shard.Backend, creator)
9899
if err != nil {
99100
return BlobAccessInfo{}, "", err
100101
}
101-
backends = append(backends, backend.BlobAccess)
102+
backends = append(backends, sharding.ShardBackend{Backend: backend.BlobAccess, Key: key})
102103
if combinedDigestKeyFormat == nil {
103104
combinedDigestKeyFormat = &backend.DigestKeyFormat
104105
} else {
105106
newDigestKeyFormat := combinedDigestKeyFormat.Combine(backend.DigestKeyFormat)
106107
combinedDigestKeyFormat = &newDigestKeyFormat
107108
}
109+
110+
if shard.Weight == 0 {
111+
return BlobAccessInfo{}, "", status.Errorf(codes.InvalidArgument, "Shards must have positive weights")
112+
}
113+
shards = append(shards, sharding.Shard{
114+
Key: key,
115+
Weight: shard.Weight,
116+
})
117+
keys = append(keys, key)
118+
}
119+
if combinedDigestKeyFormat == nil {
120+
return BlobAccessInfo{}, "", status.Errorf(codes.InvalidArgument, "Cannot create sharding blob access without any backends")
121+
}
122+
shardSelector, err := sharding.NewRendezvousShardSelector(shards)
123+
if err != nil {
124+
return BlobAccessInfo{}, "", status.Errorf(codes.InvalidArgument, "Could not create rendezvous shard selector")
125+
}
126+
return BlobAccessInfo{
127+
BlobAccess: sharding.NewShardingBlobAccess(
128+
backends,
129+
shardSelector,
130+
),
131+
DigestKeyFormat: *combinedDigestKeyFormat,
132+
}, "sharding", nil
133+
} else {
134+
backends := make([]blobstore.BlobAccess, 0, len(backend.Sharding.Legacy.ShardOrder))
135+
weights := make([]uint32, 0, len(backend.Sharding.Legacy.ShardOrder))
136+
var combinedDigestKeyFormat *digest.KeyFormat
137+
for _, key := range backend.Sharding.Legacy.ShardOrder {
138+
shard, exists := backend.Sharding.Shards[key]
139+
if !exists {
140+
return BlobAccessInfo{}, "", status.Errorf(codes.InvalidArgument, "Legacy sharding blob access refers to non-existing key %s", key)
141+
}
142+
if shard.Backend == nil {
143+
// Drained backend
144+
backends = append(backends, nil)
145+
} else {
146+
// Undrained backend
147+
backend, err := nc.NewNestedBlobAccess(shard.Backend, creator)
148+
if err != nil {
149+
return BlobAccessInfo{}, "", err
150+
}
151+
backends = append(backends, backend.BlobAccess)
152+
if combinedDigestKeyFormat == nil {
153+
combinedDigestKeyFormat = &backend.DigestKeyFormat
154+
} else {
155+
newDigestKeyFormat := combinedDigestKeyFormat.Combine(backend.DigestKeyFormat)
156+
combinedDigestKeyFormat = &newDigestKeyFormat
157+
}
158+
}
159+
160+
if shard.Weight == 0 {
161+
return BlobAccessInfo{}, "", status.Errorf(codes.InvalidArgument, "Shards must have positive weights")
162+
}
163+
weights = append(weights, shard.Weight)
108164
}
109165

110-
if shard.Weight == 0 {
111-
return BlobAccessInfo{}, "", status.Errorf(codes.InvalidArgument, "Shards must have positive weights")
166+
if combinedDigestKeyFormat == nil {
167+
return BlobAccessInfo{}, "", status.Errorf(codes.InvalidArgument, "Cannot create sharding blob access without any undrained backends")
112168
}
113-
weights = append(weights, shard.Weight)
169+
return BlobAccessInfo{
170+
BlobAccess: legacy.NewShardingBlobAccess(
171+
backends,
172+
legacy.NewWeightedShardPermuter(weights),
173+
backend.Sharding.Legacy.HashInitialization,
174+
),
175+
DigestKeyFormat: *combinedDigestKeyFormat,
176+
}, "sharding", nil
114177
}
115-
if combinedDigestKeyFormat == nil {
116-
return BlobAccessInfo{}, "", status.Errorf(codes.InvalidArgument, "Cannot create sharding blob access without any undrained backends")
117-
}
118-
return BlobAccessInfo{
119-
BlobAccess: sharding.NewShardingBlobAccess(
120-
backends,
121-
sharding.NewWeightedShardPermuter(weights),
122-
backend.Sharding.HashInitialization),
123-
DigestKeyFormat: *combinedDigestKeyFormat,
124-
}, "sharding", nil
125178
case *pb.BlobAccessConfiguration_Mirrored:
126179
backendA, err := nc.NewNestedBlobAccess(backend.Mirrored.BackendA, creator)
127180
if err != nil {

‎pkg/blobstore/sharding/BUILD.bazel‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ load("@rules_go//go:def.bzl", "go_library", "go_test")
33
go_library(
44
name = "sharding",
55
srcs = [
6-
"shard_permuter.go",
6+
"rendezvous_shard_selector.go",
7+
"shard_selector.go",
78
"sharding_blob_access.go",
8-
"weighted_shard_permuter.go",
99
],
1010
importpath = "github.com/buildbarn/bb-storage/pkg/blobstore/sharding",
1111
visibility = ["//visibility:public"],
@@ -16,21 +16,19 @@ go_library(
1616
"//pkg/digest",
1717
"//pkg/util",
1818
"@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto",
19-
"@com_github_lazybeaver_xorshift//:xorshift",
2019
"@org_golang_x_sync//errgroup",
2120
],
2221
)
2322

2423
go_test(
2524
name = "sharding_test",
2625
srcs = [
26+
"rendezvous_shard_selector_test.go",
2727
"sharding_blob_access_test.go",
28-
"weighted_shard_permuter_test.go",
2928
],
3029
deps = [
3130
":sharding",
3231
"//internal/mock",
33-
"//pkg/blobstore",
3432
"//pkg/blobstore/buffer",
3533
"//pkg/digest",
3634
"//pkg/testutil",

‎pkg/blobstore/sharding/integration/BUILD.bazel‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
load("@rules_go//go:def.bzl", "go_test")
2+
3+
go_test(
4+
name = "integration",
5+
srcs = ["benchmarking_integration_test.go"],
6+
data = ["//cmd/bb_storage"],
7+
deps = [
8+
"@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto",
9+
"@com_github_stretchr_testify//require",
10+
"@org_golang_google_grpc//:grpc",
11+
"@org_golang_google_grpc//credentials/insecure",
12+
"@rules_go//go/runfiles",
13+
],
14+
)
15+
16+
go_test(
17+
name = "integration_test",
18+
srcs = ["benchmarking_integration_test.go"],
19+
deps = [
20+
"@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto",
21+
"@org_golang_google_grpc//:grpc",
22+
"@org_golang_google_grpc//credentials/insecure",
23+
"@rules_go//go/runfiles",
24+
],
25+
)

0 commit comments

Comments
 (0)