feat(storage): support HNS-enabled buckets (#13753)

Add accessors and modifiers for the `hierarchical_namespace()` field in
a bucket. Parse the field in bucket metadata responses for JSON and
gRPC.  Add support to set this field on create, update, and patch
requests. Also for JSON and gRPC. Include and example (which doubles as
an integration test), and the usual unit tests.

Author: coryan

google/cloud/storage/bucket_hierarchical_namespace.cc

@@ -0,0 +1,35 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "google/cloud/storage/bucket_hierarchical_namespace.h"
+#include "google/cloud/internal/ios_flags_saver.h"
+#include <iomanip>
+#include <iostream>
+
+namespace google {
+namespace cloud {
+namespace storage {
+GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
+
+std::ostream& operator<<(std::ostream& os,
+                         BucketHierarchicalNamespace const& rhs) {
+  google::cloud::internal::IosFlagsSaver flags(os);
+  return os << "BucketHierarchicalNamespace={enabled=" << std::boolalpha
+            << rhs.enabled << "}";
+}
+
+GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
+}  // namespace storage
+}  // namespace cloud
+}  // namespace google

google/cloud/storage/bucket_hierarchical_namespace.h

@@ -0,0 +1,51 @@
+// Copyright 2024 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_BUCKET_HIERARCHICAL_NAMESPACE_H
+#define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_BUCKET_HIERARCHICAL_NAMESPACE_H
+
+#include "google/cloud/storage/version.h"
+#include <iosfwd>
+
+namespace google {
+namespace cloud {
+namespace storage {
+GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
+
+/**
+ * Configuration for a bucket's hierarchical namespace feature.
+ */
+struct BucketHierarchicalNamespace {
+  bool enabled;
+};
+
+inline bool operator==(BucketHierarchicalNamespace const& lhs,
+                       BucketHierarchicalNamespace const& rhs) {
+  return lhs.enabled == rhs.enabled;
+}
+
+inline bool operator!=(BucketHierarchicalNamespace const& lhs,
+                       BucketHierarchicalNamespace const& rhs) {
+  return !(lhs == rhs);
+}
+
+std::ostream& operator<<(std::ostream& os,
+                         BucketHierarchicalNamespace const& rhs);
+
+GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
+}  // namespace storage
+}  // namespace cloud
+}  // namespace google
+
+#endif  // GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_STORAGE_BUCKET_HIERARCHICAL_NAMESPACE_H

google/cloud/storage/bucket_metadata.cc

@@ -91,6 +91,7 @@ bool operator==(BucketMetadata const& lhs, BucketMetadata const& rhs) {
          && lhs.default_event_based_hold_ == rhs.default_event_based_hold_  //
          && lhs.encryption_ == rhs.encryption_                              //
          && lhs.etag_ == rhs.etag_                                          //
+         && lhs.hierarchical_namespace_ == rhs.hierarchical_namespace_      //
          && lhs.iam_configuration_ == rhs.iam_configuration_                //
          && lhs.id_ == rhs.id_                                              //
          && lhs.kind_ == rhs.kind_                                          //
@@ -151,6 +152,10 @@ std::ostream& operator<<(std::ostream& os, BucketMetadata const& rhs) {
 
   os << ", etag=" << rhs.etag();
 
+  if (rhs.has_hierarchical_namespace()) {
+    os << ", hierarchical_namespace=" << rhs.hierarchical_namespace();
+  }
+
   if (rhs.has_iam_configuration()) {
     os << ", iam_configuration=" << rhs.iam_configuration();
   }
@@ -390,6 +395,21 @@ BucketMetadataPatchBuilder::ResetIamConfiguration() {
   return *this;
 }
 
+BucketMetadataPatchBuilder&
+BucketMetadataPatchBuilder::SetHierarchicalNamespace(
+    BucketHierarchicalNamespace const& v) {
+  internal::PatchBuilder subpatch;
+  subpatch.SetBoolField("enabled", v.enabled);
+  impl_.AddSubPatch("hierarchicalNamespace", subpatch);
+  return *this;
+}
+
+BucketMetadataPatchBuilder&
+BucketMetadataPatchBuilder::ResetHierarchicalNamespace() {
+  impl_.RemoveField("hierarchicalNamespace");
+  return *this;
+}
+
 BucketMetadataPatchBuilder& BucketMetadataPatchBuilder::SetLabel(
     std::string const& label, std::string const& value) {
   labels_subpatch_.SetStringField(label.c_str(), value);

google/cloud/storage/bucket_metadata.h

@@ -21,6 +21,7 @@
 #include "google/cloud/storage/bucket_cors_entry.h"
 #include "google/cloud/storage/bucket_custom_placement_config.h"
 #include "google/cloud/storage/bucket_encryption.h"
+#include "google/cloud/storage/bucket_hierarchical_namespace.h"
 #include "google/cloud/storage/bucket_iam_configuration.h"
 #include "google/cloud/storage/bucket_lifecycle.h"
 #include "google/cloud/storage/bucket_logging.h"
@@ -212,6 +213,30 @@ class BucketMetadata {
     return *this;
   }
 
+  /**
+   * @name Get and set the hierarchical namespaces configuration.
+   */
+  ///@{
+  bool has_hierarchical_namespace() const {
+    return hierarchical_namespace_.has_value();
+  }
+  BucketHierarchicalNamespace const& hierarchical_namespace() const {
+    return *hierarchical_namespace_;
+  }
+  absl::optional<BucketHierarchicalNamespace> const&
+  hierarchical_namespace_as_optional() const {
+    return hierarchical_namespace_;
+  }
+  BucketMetadata& set_hierarchical_namespace(BucketHierarchicalNamespace v) {
+    hierarchical_namespace_ = std::move(v);
+    return *this;
+  }
+  BucketMetadata& reset_hierarchical_namespace() {
+    hierarchical_namespace_.reset();
+    return *this;
+  }
+  ///@}
+
   /**
    * @name Get and set the IAM configuration.
    *
@@ -612,6 +637,7 @@ class BucketMetadata {
   bool default_event_based_hold_ = false;
   absl::optional<BucketEncryption> encryption_;
   std::string etag_;
+  absl::optional<BucketHierarchicalNamespace> hierarchical_namespace_;
   absl::optional<BucketIamConfiguration> iam_configuration_;
   std::string id_;
   std::string kind_;
@@ -690,6 +716,12 @@ class BucketMetadataPatchBuilder {
       BucketIamConfiguration const& v);
   BucketMetadataPatchBuilder& ResetIamConfiguration();
 
+  /// Sets a new hierarchical namespace configuration.
+  BucketMetadataPatchBuilder& SetHierarchicalNamespace(
+      BucketHierarchicalNamespace const& v);
+  /// Resets the hierarchical namespace configuration
+  BucketMetadataPatchBuilder& ResetHierarchicalNamespace();
+
   BucketMetadataPatchBuilder& SetEncryption(BucketEncryption const& v);
   BucketMetadataPatchBuilder& ResetEncryption();
 

google/cloud/storage/bucket_metadata_test.cc

@@ -107,6 +107,9 @@ BucketMetadata CreateBucketMetadataForTest() {
         "defaultKmsKeyName": "projects/test-project-name/locations/us-central1/keyRings/test-keyring-name/cryptoKeys/test-key-name"
       },
       "etag": "XYZ=",
+      "hierarchicalNamespace": {
+        "enabled": true
+      },
       "iamConfiguration": {
         "uniformBucketLevelAccess": {
           "enabled": true,
@@ -216,6 +219,10 @@ TEST(BucketMetadataTest, Parse) {
       "test-keyring-name/cryptoKeys/test-key-name",
       actual.encryption().default_kms_key_name);
   EXPECT_EQ("XYZ=", actual.etag());
+  // hierarchicalNamespace
+  ASSERT_TRUE(actual.has_hierarchical_namespace());
+  ASSERT_TRUE(actual.hierarchical_namespace_as_optional().has_value());
+  ASSERT_EQ(actual.hierarchical_namespace(), BucketHierarchicalNamespace{true});
   ASSERT_TRUE(actual.has_iam_configuration());
   ASSERT_TRUE(
       actual.iam_configuration().uniform_bucket_level_access.has_value());
@@ -361,6 +368,12 @@ TEST(BucketMetadataTest, IOStream) {
               HasSubstr("projects/test-project-name/locations/us-central1/"
                         "keyRings/test-keyring-name/cryptoKeys/test-key-name"));
 
+  // hierarchical_namespace()
+  EXPECT_THAT(
+      actual,
+      HasSubstr(
+          "hierarchical_namespace=BucketHierarchicalNamespace={enabled=true}"));
+
   // iam_policy()
   EXPECT_THAT(actual, HasSubstr("BucketIamConfiguration={"));
   EXPECT_THAT(actual, HasSubstr("locked_time=2020-01-02T03:04:05Z"));
@@ -466,6 +479,11 @@ TEST(BucketMetadataTest, ToJsonString) {
       "test-keyring-name/cryptoKeys/test-key-name",
       actual["encryption"].value("defaultKmsKeyName", ""));
 
+  // hierarchical_namespace()
+  ASSERT_EQ(1, actual.count("hierarchicalNamespace"));
+  EXPECT_EQ(actual["hierarchicalNamespace"],
+            nlohmann::json({{"enabled", true}}));
+
   // iam_configuration()
   ASSERT_EQ(1U, actual.count("iamConfiguration"));
   nlohmann::json expected_iam_configuration{
@@ -747,6 +765,25 @@ TEST(BucketMetadataTest, SetDefaultObjectAcl) {
   EXPECT_NE(expected, copy);
 }
 
+/// @test Verify we can change the Hierarchical Namespace configuration.
+TEST(BucketMetadataTest, SetHierarchicalNamespace) {
+  auto expected = CreateBucketMetadataForTest();
+  auto copy = expected;
+  copy.set_hierarchical_namespace(BucketHierarchicalNamespace{false});
+  ASSERT_TRUE(copy.has_hierarchical_namespace());
+  EXPECT_EQ(copy.hierarchical_namespace(), BucketHierarchicalNamespace{false});
+  EXPECT_NE(expected, copy);
+}
+
+/// @test Verify we can reset the Hierarchical Namespace configuration.
+TEST(BucketMetadataTest, ResetHierarchicalNamespace) {
+  auto expected = CreateBucketMetadataForTest();
+  auto copy = expected;
+  copy.reset_hierarchical_namespace();
+  ASSERT_FALSE(copy.has_hierarchical_namespace());
+  EXPECT_NE(expected, copy);
+}
+
 /// @test Verify we can change the IAM Configuration in BucketMetadata.
 TEST(BucketMetadataTest, SetIamConfigurationUBLA) {
   auto expected = CreateBucketMetadataForTest();
@@ -1226,6 +1263,26 @@ TEST(BucketMetadataPatchBuilder, ResetIamConfiguration) {
   ASSERT_TRUE(json["iamConfiguration"].is_null()) << json;
 }
 
+TEST(BucketMetadataPatchBuilder, SetHierarchicalNamespace) {
+  BucketMetadataPatchBuilder builder;
+  builder.SetHierarchicalNamespace(BucketHierarchicalNamespace{true});
+
+  auto actual = builder.BuildPatch();
+  auto json = nlohmann::json::parse(actual);
+  ASSERT_EQ(1U, json.count("hierarchicalNamespace")) << json;
+  ASSERT_EQ(json["hierarchicalNamespace"], nlohmann::json({{"enabled", true}}));
+}
+
+TEST(BucketMetadataPatchBuilder, ResetHierarchicalNamespace) {
+  BucketMetadataPatchBuilder builder;
+  builder.ResetHierarchicalNamespace();
+
+  auto actual = builder.BuildPatch();
+  auto json = nlohmann::json::parse(actual);
+  ASSERT_EQ(1U, json.count("hierarchicalNamespace")) << json;
+  ASSERT_TRUE(json["hierarhicalNamespace"].is_null()) << json;
+}
+
 TEST(BucketMetadataPatchBuilder, SetEncryption) {
   BucketMetadataPatchBuilder builder;
   std::string expected =

google/cloud/storage/examples/storage_bucket_samples.cc

@@ -144,6 +144,26 @@ void CreateBucketDualRegion(google::cloud::storage::Client client,
   (std::move(client), argv.at(0), argv.at(1), argv.at(2));
 }
 
+void CreateBucketWithHNS(google::cloud::storage::Client client,
+                         std::vector<std::string> const& argv) {
+  namespace gcs = ::google::cloud::storage;
+  using ::google::cloud::StatusOr;
+  [](gcs::Client client, std::string const& bucket_name) {
+    auto metadata = client.CreateBucket(
+        bucket_name,
+        gcs::BucketMetadata()
+            .set_hierarchical_namespace(
+                gcs::BucketHierarchicalNamespace{/*.enabled=*/true})
+            .set_iam_configuration(gcs::BucketIamConfiguration{
+                gcs::UniformBucketLevelAccess{/*.enabled=*/true, {}},
+                absl::nullopt}));
+    if (!metadata) throw std::move(metadata).status();
+
+    std::cout << "Bucket " << metadata->name() << " created."
+              << "\nFull Metadata: " << *metadata << "\n";
+  }(std::move(client), argv.at(0));
+}
+
 void GetBucketMetadata(google::cloud::storage::Client client,
                        std::vector<std::string> const& argv) {
   //! [get bucket metadata]
@@ -675,6 +695,11 @@ void RunAll(std::vector<std::string> const& argv) {
             << std::endl;
   CreateBucketWithStorageClassLocation(client, {bucket_name, "STANDARD", "US"});
 
+  auto const hns_bucket_name = examples::MakeRandomBucketName(generator);
+  std::cout << "\nRunning CreateBucketWithHNS() example" << std::endl;
+  CreateBucketWithHNS(client, {hns_bucket_name});
+  (void)client.DeleteBucket(hns_bucket_name);
+
   std::cout << "\nRunning DeleteBucket() example [3]" << std::endl;
   DeleteBucket(client, {bucket_name});
 
@@ -705,6 +730,7 @@ int main(int argc, char* argv[]) {
                  CreateBucketWithStorageClassLocation),
       make_entry("create-bucket-dual-region", {"<region-a>", "<region-b>"},
                  CreateBucketDualRegion),
+      make_entry("create-bucket-with-hns", {}, CreateBucketWithHNS),
       make_entry("get-bucket-metadata", {}, GetBucketMetadata),
       make_entry("delete-bucket", {}, DeleteBucket),
       make_entry("change-default-storage-class", {"<new-class>"},

google/cloud/storage/google_cloud_cpp_storage.bzl

@@ -24,6 +24,7 @@ google_cloud_cpp_storage_hdrs = [
     "bucket_cors_entry.h",
     "bucket_custom_placement_config.h",
     "bucket_encryption.h",
+    "bucket_hierarchical_namespace.h",
     "bucket_iam_configuration.h",
     "bucket_lifecycle.h",
     "bucket_logging.h",
@@ -158,6 +159,7 @@ google_cloud_cpp_storage_srcs = [
     "bucket_autoclass.cc",
     "bucket_cors_entry.cc",
     "bucket_custom_placement_config.cc",
+    "bucket_hierarchical_namespace.cc",
     "bucket_iam_configuration.cc",
     "bucket_logging.cc",
     "bucket_metadata.cc",

google/cloud/storage/google_cloud_cpp_storage.cmake

@@ -32,6 +32,8 @@ add_library(
     bucket_custom_placement_config.cc
     bucket_custom_placement_config.h
     bucket_encryption.h
+    bucket_hierarchical_namespace.cc
+    bucket_hierarchical_namespace.h
     bucket_iam_configuration.cc
     bucket_iam_configuration.h
     bucket_lifecycle.h