Implement updated fragment metadata footer design.

Author: teo-tsirpanis

tiledb/sm/fragment/fragment_metadata.cc

@@ -388,6 +388,8 @@ void FragmentMetadata::set_tile_global_order_bounds_fixed(
         loaded_metadata_ptr_->tile_global_order_max_buffer()[dim].data();
     memcpy(&max_data[offset], tile_max.value().data(), fixed_size);
   }
+
+  has_tile_global_order_bounds_ = true;
 }
 
 /**
@@ -1321,7 +1323,7 @@ void FragmentMetadata::store_v15_or_higher(
   }
 
   if (!dense_ &&
-      version_ >= constants::fragment_metadata_global_order_bounds_version) {
+      version_ >= constants::fragment_metadata_data_directory_version) {
     const auto num_dims = array_schema_->dim_num();
     // Store global order mins
     gt_offsets_.tile_global_order_min_offsets_.resize(num_dims);
@@ -1846,6 +1848,10 @@ void FragmentMetadata::write_footer(Serializer& serializer) const {
   write_file_var_sizes(serializer);
   write_file_validity_sizes(serializer);
   write_generic_tile_offsets(serializer);
+
+  if (version_ >= constants::fragment_metadata_data_directory_version) {
+    write_data_directories(serializer);
+  }
 }
 
 /* ****************************** */
@@ -2395,10 +2401,8 @@ void FragmentMetadata::load_generic_tile_offsets(Deserializer& deserializer) {
     load_generic_tile_offsets_v11(deserializer);
   } else if (version_ >= 12 && version_ < 16) {
     load_generic_tile_offsets_v12_v15(deserializer);
-  } else if (version_ >= 16 && version_ < 23) {
-    load_generic_tile_offsets_v16_v22(deserializer);
   } else {
-    load_generic_tile_offsets_v23_or_higher(deserializer);
+    load_generic_tile_offsets_v16_or_higher(deserializer);
   }
 }
 
@@ -2552,7 +2556,7 @@ void FragmentMetadata::load_generic_tile_offsets_v12_v15(
       deserializer.read<uint64_t>();
 }
 
-void FragmentMetadata::load_generic_tile_offsets_v16_v22(
+void FragmentMetadata::load_generic_tile_offsets_v16_or_higher(
     Deserializer& deserializer) {
   // Load R-Tree offset
   gt_offsets_.rtree_ = deserializer.read<uint64_t>();
@@ -2598,63 +2602,41 @@ void FragmentMetadata::load_generic_tile_offsets_v16_v22(
   gt_offsets_.processed_conditions_offsets_ = deserializer.read<uint64_t>();
 }
 
-void FragmentMetadata::load_generic_tile_offsets_v23_or_higher(
-    Deserializer& deserializer) {
-  // Load R-Tree offset
-  gt_offsets_.rtree_ = deserializer.read<uint64_t>();
-
-  // Load offsets for tile offsets
-  auto num = num_dims_and_attrs();
-  gt_offsets_.tile_offsets_.resize(num);
-  deserializer.read(&gt_offsets_.tile_offsets_[0], num * sizeof(uint64_t));
-
-  // Load offsets for tile var offsets
-  gt_offsets_.tile_var_offsets_.resize(num);
-  deserializer.read(&gt_offsets_.tile_var_offsets_[0], num * sizeof(uint64_t));
+void FragmentMetadata::load_data_directories(Deserializer& deserializer) {
+  auto count = deserializer.read<uint32_t>();
 
-  // Load offsets for tile var sizes
-  gt_offsets_.tile_var_sizes_.resize(num);
-  deserializer.read(&gt_offsets_.tile_var_sizes_[0], num * sizeof(uint64_t));
-
-  // Load offsets for tile validity offsets
-  gt_offsets_.tile_validity_offsets_.resize(num);
-  deserializer.read(
-      &gt_offsets_.tile_validity_offsets_[0], num * sizeof(uint64_t));
-
-  // Load offsets for tile min offsets
-  gt_offsets_.tile_min_offsets_.resize(num);
-  deserializer.read(&gt_offsets_.tile_min_offsets_[0], num * sizeof(uint64_t));
-
-  // Load offsets for tile max offsets
-  gt_offsets_.tile_max_offsets_.resize(num);
-  deserializer.read(&gt_offsets_.tile_max_offsets_[0], num * sizeof(uint64_t));
-
-  if (!dense_) {
-    // Load offsets for the tile global order bounds
-    const auto num_dims = array_schema_->dim_num();
-    gt_offsets_.tile_global_order_min_offsets_.resize(num_dims);
-    gt_offsets_.tile_global_order_max_offsets_.resize(num_dims);
-    deserializer.read(
-        gt_offsets_.tile_global_order_min_offsets_.data(),
-        num_dims * sizeof(uint64_t));
-    deserializer.read(
-        gt_offsets_.tile_global_order_max_offsets_.data(),
-        num_dims * sizeof(uint64_t));
+  for (uint32_t i = 0; i < count; i++) {
+    auto identifier = deserializer.read<DataDirectoryIdentifier>();
+    auto data_size = deserializer.read<uint32_t>();
+    switch (identifier) {
+      case DataDirectoryIdentifier::tile_global_order_offsets:
+        if (data_size != 2 * array_schema_->dim_num() * sizeof(uint64_t)) {
+          throw FragmentMetadataStatusException(
+              "Invalid size of tile global order bound offsets data directory");
+        }
+        load_tile_global_order_offsets(deserializer);
+        has_tile_global_order_bounds_ = true;
+        break;
+      default:
+        // Ignore unknown data directories per the storage format spec.
+        deserializer.skip(data_size);
+        break;
+    }
   }
+}
 
-  // Load offsets for tile sum offsets
-  gt_offsets_.tile_sum_offsets_.resize(num);
-  deserializer.read(&gt_offsets_.tile_sum_offsets_[0], num * sizeof(uint64_t));
-
-  // Load offsets for tile null count offsets
-  gt_offsets_.tile_null_count_offsets_.resize(num);
+void FragmentMetadata::load_tile_global_order_offsets(
+    Deserializer& deserializer) {
+  // Load offsets for the tile global order bounds
+  const auto num_dims = array_schema_->dim_num();
+  gt_offsets_.tile_global_order_min_offsets_.resize(num_dims);
+  gt_offsets_.tile_global_order_max_offsets_.resize(num_dims);
   deserializer.read(
-      &gt_offsets_.tile_null_count_offsets_[0], num * sizeof(uint64_t));
-
-  gt_offsets_.fragment_min_max_sum_null_count_offset_ =
-      deserializer.read<uint64_t>();
-
-  gt_offsets_.processed_conditions_offsets_ = deserializer.read<uint64_t>();
+      gt_offsets_.tile_global_order_min_offsets_.data(),
+      num_dims * sizeof(uint64_t));
+  deserializer.read(
+      gt_offsets_.tile_global_order_max_offsets_.data(),
+      num_dims * sizeof(uint64_t));
 }
 
 void FragmentMetadata::load_array_schema_name(Deserializer& deserializer) {
@@ -2803,6 +2785,10 @@ void FragmentMetadata::load_footer(
 
   load_generic_tile_offsets(deserializer);
 
+  if (version_ >= 23) {
+    load_data_directories(deserializer);
+  }
+
   // If the footer_size is not set lets calculate from how much of the
   // buffer we read
   if (footer_size_ == 0) {
@@ -2883,18 +2869,6 @@ void FragmentMetadata::write_generic_tile_offsets(
     serializer.write(&gt_offsets_.tile_max_offsets_[0], num * sizeof(uint64_t));
   }
 
-  if (!dense_ &&
-      version_ >= constants::fragment_metadata_global_order_bounds_version) {
-    // Write the tile global order bound offsets
-    const auto num_dims = array_schema_->dim_num();
-    serializer.write(
-        gt_offsets_.tile_global_order_min_offsets_.data(),
-        num_dims * sizeof(uint64_t));
-    serializer.write(
-        gt_offsets_.tile_global_order_max_offsets_.data(),
-        num_dims * sizeof(uint64_t));
-  }
-
   // Write tile sum offsets
   if (version_ >= 11) {
     serializer.write(&gt_offsets_.tile_sum_offsets_[0], num * sizeof(uint64_t));
@@ -2916,6 +2890,29 @@ void FragmentMetadata::write_generic_tile_offsets(
   }
 }
 
+void FragmentMetadata::write_data_directories(Serializer& serializer) const {
+  uint32_t count = 0;
+
+  if (has_tile_global_order_bounds_) {
+    count++;
+  }
+
+  serializer.write(count);
+
+  if (has_tile_global_order_bounds_) {
+    serializer.write(DataDirectoryIdentifier::tile_global_order_offsets);
+    const auto num_dims = array_schema_->dim_num();
+    serializer.write<uint32_t>(2 * num_dims * sizeof(uint64_t));
+
+    serializer.write(
+        gt_offsets_.tile_global_order_min_offsets_.data(),
+        num_dims * sizeof(uint64_t));
+    serializer.write(
+        gt_offsets_.tile_global_order_max_offsets_.data(),
+        num_dims * sizeof(uint64_t));
+  }
+}
+
 void FragmentMetadata::write_array_schema_name(Serializer& serializer) const {
   uint64_t size = array_schema_name_.size();
   if (size == 0) {

tiledb/sm/fragment/fragment_metadata.h

@@ -72,6 +72,13 @@ class FragmentMetadataStatusException : public StatusException {
   }
 };
 
+/**
+ * Contains identifiers for fragment metadata footer data directories.
+ */
+enum class DataDirectoryIdentifier : uint64_t {
+  tile_global_order_offsets = 0,
+};
+
 /** Stores the metadata structures of a fragment. */
 class FragmentMetadata {
  public:
@@ -947,6 +954,9 @@ class FragmentMetadata {
   /** The format version of this metadata. */
   uint32_t version_;
 
+  /** Whether this metadata contains tile global order bounds. */
+  bool has_tile_global_order_bounds_ = false;
+
   /** The timestamp range of the fragment. */
   std::pair<uint64_t, uint64_t> timestamp_range_;
 
@@ -1053,15 +1063,19 @@ class FragmentMetadata {
 
   /**
    * Loads the generic tile offsets from the buffer. Applicable to
-   * versions 16 to 22.
+   * versions 16 or higher.
    */
-  void load_generic_tile_offsets_v16_v22(Deserializer& deserializer);
+  void load_generic_tile_offsets_v16_or_higher(Deserializer& deserializer);
 
   /**
-   * Loads the generic tile offsets from the buffer. Applicable to
-   * versions 23 or higher.
+   * Loads the footer's data directories.
    */
-  void load_generic_tile_offsets_v23_or_higher(Deserializer& deserializer);
+  void load_data_directories(Deserializer& deserializer);
+
+  /**
+   * Loads the offsets for the tile global order bounds.
+   */
+  void load_tile_global_order_offsets(Deserializer& deserializer);
 
   /**
    * Loads the array schema name.
@@ -1209,6 +1223,9 @@ class FragmentMetadata {
   /** Writes the generic tile offsets to the buffer. */
   void write_generic_tile_offsets(Serializer& serializer) const;
 
+  /** Writes the footer's data directories to the buffer. */
+  void write_data_directories(Serializer& serializer) const;
+
   /** Writes the array schema name. */
   void write_array_schema_name(Serializer& serializer) const;
 

tiledb/sm/fragment/ondemand_fragment_metadata.cc

@@ -115,8 +115,7 @@ void OndemandFragmentMetadata::load_fragment_min_max_sum_null_count(
 
 void OndemandFragmentMetadata::load_fragment_tile_global_order_bounds(
     const EncryptionKey& encryption_key) {
-  if (parent_fragment_.version_ <
-      constants::fragment_metadata_global_order_bounds_version) {
+  if (!parent_fragment_.has_tile_global_order_bounds_) {
     return;
   }
 
@@ -623,8 +622,7 @@ void OndemandFragmentMetadata::load_tile_max_values(
 
 void OndemandFragmentMetadata::load_tile_global_order_min_values(
     const EncryptionKey& encryption_key, unsigned dimension) {
-  if (parent_fragment_.version_ <
-      constants::fragment_metadata_global_order_bounds_version) {
+  if (!parent_fragment_.has_tile_global_order_bounds_) {
     return;
   }
 
@@ -648,8 +646,7 @@ void OndemandFragmentMetadata::load_tile_global_order_min_values(
 
 void OndemandFragmentMetadata::load_tile_global_order_max_values(
     const EncryptionKey& encryption_key, unsigned dimension) {
-  if (parent_fragment_.version_ <
-      constants::fragment_metadata_global_order_bounds_version) {
+  if (!parent_fragment_.has_tile_global_order_bounds_) {
     return;
   }
 

tiledb/sm/misc/constants.cc

@@ -722,9 +722,9 @@ const format_version_t current_domain_version = 0;
 /** The NDRectangle current domain */
 const std::string current_domain_ndrectangle_str = "NDRECTANGLE";
 
-/** The lowest version where fragment metadata contains per-tile global order
- * bounds */
-const format_version_t fragment_metadata_global_order_bounds_version = 23;
+/** The lowest version where fragment metadata footer contains extensible data
+ * directories. */
+const format_version_t fragment_metadata_data_directory_version = 23;
 
 /** The maximum size of a tile chunk (unit of compression) in bytes. */
 const uint64_t max_tile_chunk_size = 64 * 1024;

tiledb/sm/misc/constants.h

@@ -717,9 +717,9 @@ extern const format_version_t current_domain_version;
 /** The NDRectangle current_domain */
 extern const std::string current_domain_ndrectangle_str;
 
-/** The lowest version where fragment metadata contains per-tile global order
- * bounds */
-extern const format_version_t fragment_metadata_global_order_bounds_version;
+/** The lowest version where fragment metadata footer contains extensible data
+ * directories. */
+extern const format_version_t fragment_metadata_data_directory_version;
 
 /** The maximum size of a tile chunk (unit of compression) in bytes. */
 extern const uint64_t max_tile_chunk_size;

tiledb/sm/query/writers/writer_base.cc

@@ -1069,7 +1069,7 @@ Status WriterBase::write_tiles(
         }
       }
 
-      if (array_schema_.is_dim(attr)) {
+      if (!array_schema_.dense() && array_schema_.is_dim(attr)) {
         if (array_schema_.var_size(attr)) {
           frag_meta->convert_tile_global_order_bounds_sizes_to_offsets(attr);
         }
@@ -1155,7 +1155,7 @@ Status WriterBase::write_tiles(
         frag_meta->set_tile_min(name, tile_id, tile.min());
         frag_meta->set_tile_max(name, tile_id, tile.max());
       }
-      if (array_schema_.is_dim(name)) {
+      if (!array_schema_.dense() && array_schema_.is_dim(name)) {
         frag_meta->set_tile_global_order_bounds_fixed(name, tile_id, tile);
       }