Skip to content

Commit abf0196

Browse files
meroton-benjaminmeroton-benjamin
committed
Switch sharding struct from array to map
Improves the ability to perform resharding by switching the sharding struct from array to map. Each map entry has a key which is used in rendezvous hashing to deterministically select which shard to use from the collection of keys. When a shard is removed it is guaranteed that only blobs which belonged to the removed shard will resolve to a new shard. In combination with ReadFallbackConfigurations this allows adding and removing shards with minimal need to rebalance the blobs between the shards. For more details and migration instructions, see: https://github.com/buildbarn/bb-adrs/blob/bf2066633e1712a3ef7c295d37cd52e65867391d/0011-rendezvous-hashing.md
1 parent e8d8188 commit abf0196

21 files changed

+1387
-302
lines changed

‎internal/mock/BUILD.bazel‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,13 +86,23 @@ gomock(
8686
gomock(
8787
name = "blobstore_sharding",
8888
out = "blobstore_sharding.go",
89-
interfaces = ["ShardPermuter"],
89+
interfaces = ["ShardSelector"],
9090
library = "//pkg/blobstore/sharding",
9191
mockgen_model_library = "@org_uber_go_mock//mockgen/model",
9292
mockgen_tool = "@org_uber_go_mock//mockgen",
9393
package = "mock",
9494
)
9595

96+
gomock(
97+
name = "blobstore_legacy_sharding",
98+
out = "blobstore_legacy_sharding.go",
99+
interfaces = ["ShardPermuter"],
100+
library = "//pkg/blobstore/sharding/legacy",
101+
mockgen_model_library = "@org_uber_go_mock//mockgen/model",
102+
mockgen_tool = "@org_uber_go_mock//mockgen",
103+
package = "mock",
104+
)
105+
96106
gomock(
97107
name = "blobstore_slicing",
98108
out = "blobstore_slicing.go",
@@ -357,6 +367,7 @@ go_library(
357367
"aliases.go",
358368
"auth.go",
359369
"blobstore.go",
370+
"blobstore_legacy_sharding.go",
360371
"blobstore_local.go",
361372
"blobstore_replication.go",
362373
"blobstore_sharding.go",
@@ -391,6 +402,7 @@ go_library(
391402
"//pkg/blobstore/buffer",
392403
"//pkg/blobstore/local",
393404
"//pkg/blobstore/sharding",
405+
"//pkg/blobstore/sharding/legacy",
394406
"//pkg/blobstore/slicing",
395407
"//pkg/builder",
396408
"//pkg/clock",

‎pkg/blobstore/configuration/BUILD.bazel‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ go_library(
2929
"//pkg/blobstore/readfallback",
3030
"//pkg/blobstore/replication",
3131
"//pkg/blobstore/sharding",
32+
"//pkg/blobstore/sharding/legacy",
3233
"//pkg/blockdevice",
3334
"//pkg/capabilities",
3435
"//pkg/clock",

‎pkg/blobstore/configuration/new_blob_access.go‎

Lines changed: 73 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ import (
1313
"github.com/buildbarn/bb-storage/pkg/blobstore/readcaching"
1414
"github.com/buildbarn/bb-storage/pkg/blobstore/readfallback"
1515
"github.com/buildbarn/bb-storage/pkg/blobstore/sharding"
16+
"github.com/buildbarn/bb-storage/pkg/blobstore/sharding/legacy"
1617
"github.com/buildbarn/bb-storage/pkg/blockdevice"
1718
"github.com/buildbarn/bb-storage/pkg/clock"
1819
"github.com/buildbarn/bb-storage/pkg/digest"
@@ -85,41 +86,92 @@ func (nc *simpleNestedBlobAccessCreator) newNestedBlobAccessBare(configuration *
8586
DigestKeyFormat: slow.DigestKeyFormat,
8687
}, "read_caching", nil
8788
case *pb.BlobAccessConfiguration_Sharding:
88-
backends := make([]blobstore.BlobAccess, 0, len(backend.Sharding.Shards))
89-
weights := make([]uint32, 0, len(backend.Sharding.Shards))
90-
var combinedDigestKeyFormat *digest.KeyFormat
91-
for _, shard := range backend.Sharding.Shards {
92-
if shard.Backend == nil {
93-
// Drained backend.
94-
backends = append(backends, nil)
95-
} else {
96-
// Undrained backend.
97-
backend, err := nc.NewNestedBlobAccess(shard.Backend, creator)
98-
if err != nil {
99-
return BlobAccessInfo{}, "", err
89+
if backend.Sharding.Legacy != nil {
90+
backends := make([]blobstore.BlobAccess, 0, len(backend.Sharding.Legacy.ShardOrder))
91+
weights := make([]uint32, 0, len(backend.Sharding.Legacy.ShardOrder))
92+
var combinedDigestKeyFormat *digest.KeyFormat
93+
for _, key := range backend.Sharding.Legacy.ShardOrder {
94+
shard, exists := backend.Sharding.Shards[key]
95+
if !exists {
96+
return BlobAccessInfo{}, "", status.Errorf(codes.InvalidArgument, "Legacy sharding blob access refers to non-existing key %s", key)
10097
}
101-
backends = append(backends, backend.BlobAccess)
102-
if combinedDigestKeyFormat == nil {
103-
combinedDigestKeyFormat = &backend.DigestKeyFormat
98+
if shard.Backend == nil {
99+
// Drained backend
100+
backends = append(backends, nil)
104101
} else {
105-
newDigestKeyFormat := combinedDigestKeyFormat.Combine(backend.DigestKeyFormat)
106-
combinedDigestKeyFormat = &newDigestKeyFormat
102+
// Undrained backend
103+
backend, err := nc.NewNestedBlobAccess(shard.Backend, creator)
104+
if err != nil {
105+
return BlobAccessInfo{}, "", err
106+
}
107+
backends = append(backends, backend.BlobAccess)
108+
if combinedDigestKeyFormat == nil {
109+
combinedDigestKeyFormat = &backend.DigestKeyFormat
110+
} else {
111+
newDigestKeyFormat := combinedDigestKeyFormat.Combine(backend.DigestKeyFormat)
112+
combinedDigestKeyFormat = &newDigestKeyFormat
113+
}
107114
}
115+
116+
if shard.Weight == 0 {
117+
return BlobAccessInfo{}, "", status.Errorf(codes.InvalidArgument, "Shards must have positive weights")
118+
}
119+
weights = append(weights, shard.Weight)
120+
}
121+
122+
if combinedDigestKeyFormat == nil {
123+
return BlobAccessInfo{}, "", status.Errorf(codes.InvalidArgument, "Cannot create sharding blob access without any undrained backends")
124+
}
125+
return BlobAccessInfo{
126+
BlobAccess: legacy.NewShardingBlobAccess(
127+
backends,
128+
legacy.NewWeightedShardPermuter(weights),
129+
backend.Sharding.Legacy.HashInitialization,
130+
),
131+
DigestKeyFormat: *combinedDigestKeyFormat,
132+
}, "sharding", nil
133+
}
134+
backends := make([]sharding.ShardBackend, 0, len(backend.Sharding.Shards))
135+
shards := make([]sharding.Shard, 0, len(backend.Sharding.Shards))
136+
keys := make([]string, 0, len(backend.Sharding.Shards))
137+
var combinedDigestKeyFormat *digest.KeyFormat
138+
for key, shard := range backend.Sharding.Shards {
139+
if shard.Backend == nil {
140+
return BlobAccessInfo{}, "", status.Errorf(codes.InvalidArgument, "Shard '%s' has an undefined backend, drained backends are only allowed when running in Legacy mode", key)
141+
}
142+
backend, err := nc.NewNestedBlobAccess(shard.Backend, creator)
143+
if err != nil {
144+
return BlobAccessInfo{}, "", err
145+
}
146+
backends = append(backends, sharding.ShardBackend{Backend: backend.BlobAccess, Key: key})
147+
if combinedDigestKeyFormat == nil {
148+
combinedDigestKeyFormat = &backend.DigestKeyFormat
149+
} else {
150+
newDigestKeyFormat := combinedDigestKeyFormat.Combine(backend.DigestKeyFormat)
151+
combinedDigestKeyFormat = &newDigestKeyFormat
108152
}
109153

110154
if shard.Weight == 0 {
111155
return BlobAccessInfo{}, "", status.Errorf(codes.InvalidArgument, "Shards must have positive weights")
112156
}
113-
weights = append(weights, shard.Weight)
157+
shards = append(shards, sharding.Shard{
158+
Key: key,
159+
Weight: shard.Weight,
160+
})
161+
keys = append(keys, key)
114162
}
115163
if combinedDigestKeyFormat == nil {
116-
return BlobAccessInfo{}, "", status.Errorf(codes.InvalidArgument, "Cannot create sharding blob access without any undrained backends")
164+
return BlobAccessInfo{}, "", status.Errorf(codes.InvalidArgument, "Cannot create sharding blob access without any backends")
165+
}
166+
shardSelector, err := sharding.NewRendezvousShardSelector(shards)
167+
if err != nil {
168+
return BlobAccessInfo{}, "", status.Errorf(codes.InvalidArgument, "Could not create rendezvous shard selector")
117169
}
118170
return BlobAccessInfo{
119171
BlobAccess: sharding.NewShardingBlobAccess(
120172
backends,
121-
sharding.NewWeightedShardPermuter(weights),
122-
backend.Sharding.HashInitialization),
173+
shardSelector,
174+
),
123175
DigestKeyFormat: *combinedDigestKeyFormat,
124176
}, "sharding", nil
125177
case *pb.BlobAccessConfiguration_Mirrored:

‎pkg/blobstore/sharding/BUILD.bazel‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@ load("@rules_go//go:def.bzl", "go_library", "go_test")
33
go_library(
44
name = "sharding",
55
srcs = [
6-
"shard_permuter.go",
6+
"rendezvous_shard_selector.go",
7+
"shard_selector.go",
78
"sharding_blob_access.go",
8-
"weighted_shard_permuter.go",
99
],
1010
importpath = "github.com/buildbarn/bb-storage/pkg/blobstore/sharding",
1111
visibility = ["//visibility:public"],
@@ -16,21 +16,19 @@ go_library(
1616
"//pkg/digest",
1717
"//pkg/util",
1818
"@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto",
19-
"@com_github_lazybeaver_xorshift//:xorshift",
2019
"@org_golang_x_sync//errgroup",
2120
],
2221
)
2322

2423
go_test(
2524
name = "sharding_test",
2625
srcs = [
26+
"rendezvous_shard_selector_test.go",
2727
"sharding_blob_access_test.go",
28-
"weighted_shard_permuter_test.go",
2928
],
3029
deps = [
3130
":sharding",
3231
"//internal/mock",
33-
"//pkg/blobstore",
3432
"//pkg/blobstore/buffer",
3533
"//pkg/digest",
3634
"//pkg/testutil",

‎pkg/blobstore/sharding/integration/BUILD.bazel‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
load("@rules_go//go:def.bzl", "go_test")
2+
3+
go_test(
4+
name = "integration",
5+
srcs = ["benchmarking_integration_test.go"],
6+
data = ["//cmd/bb_storage"],
7+
deps = [
8+
"@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto",
9+
"@com_github_stretchr_testify//require",
10+
"@org_golang_google_grpc//:grpc",
11+
"@org_golang_google_grpc//credentials/insecure",
12+
"@rules_go//go/runfiles",
13+
],
14+
)
15+
16+
go_test(
17+
name = "integration_test",
18+
srcs = ["benchmarking_integration_test.go"],
19+
deps = [
20+
"@bazel_remote_apis//build/bazel/remote/execution/v2:remote_execution_go_proto",
21+
"@org_golang_google_grpc//:grpc",
22+
"@org_golang_google_grpc//credentials/insecure",
23+
"@rules_go//go/runfiles",
24+
],
25+
)

0 commit comments

Comments
 (0)