diff --git a/config/gnu-warnings/4.8 b/config/gnu-warnings/4.8 index 8a1928991b8..6f04105985f 100644 --- a/config/gnu-warnings/4.8 +++ b/config/gnu-warnings/4.8 @@ -1,5 +1,5 @@ # warning flags added for GCC >= 4.3 --Wlarger-than=2560 +-Wlarger-than=2584 -Wlogical-op # warning flags added for GCC >= 4.4 diff --git a/config/gnu-warnings/cxx-4.8 b/config/gnu-warnings/cxx-4.8 index 387c41ebd31..83df3b3e6d1 100644 --- a/config/gnu-warnings/cxx-4.8 +++ b/config/gnu-warnings/cxx-4.8 @@ -1,5 +1,5 @@ # warning flags added for GCC >= 4.3 --Wlarger-than=2560 +-Wlarger-than=2584 -Wlogical-op # warning flags added for GCC >= 4.4 diff --git a/release_docs/CHANGELOG.md b/release_docs/CHANGELOG.md index ab659df20e5..3abfc9c9df2 100644 --- a/release_docs/CHANGELOG.md +++ b/release_docs/CHANGELOG.md @@ -193,6 +193,10 @@ All other HDF5 library CMake options are prefixed with `HDF5_` ## Library +### Added support for large chunks + + The library now supports chunks larger than 4 GiB using 64 bit addressing. Creating chunks with size >= 4 GiB will upgrade the file format and prevent the dataset from being opened with earlier versions of the library. 32 bit systems will not be able to use these chunks in all circumstances, such as with data filters or a fill value. + ### Changed default chunk cache hash table size to 8191 In order to reduce hash collisions and take advantage of modern memory capacity, the default hash table size for the chunk cache has been increased from 521 to 8191. This means the hash table will consume approximately 64 KiB per open dataset. This value can be changed with `H5Pset_cache()` or `H5Pset_chunk_cache()`. This value was chosen because it is a prime number close to 8K. diff --git a/src/H5D.c b/src/H5D.c index 394d9a3f569..7cb8e2ab59f 100644 --- a/src/H5D.c +++ b/src/H5D.c @@ -1491,10 +1491,9 @@ herr_t H5Dwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, const hsize_t *offset, size_t data_size, const void *buf) { - H5VL_object_t *vol_obj; /* Dataset for this operation */ - H5VL_optional_args_t vol_cb_args; /* Arguments to VOL callback */ - H5VL_native_dataset_optional_args_t dset_opt_args; /* Arguments for optional operation */ - uint32_t data_size_32; /* Chunk data size (limited to 32-bits currently) */ + H5VL_object_t *vol_obj; /* Dataset for this operation */ + H5VL_optional_args_t vol_cb_args; /* Arguments to VOL callback */ + H5VL_native_dataset_optional_args_t dset_opt_args; /* Arguments for optional operation */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) @@ -1509,11 +1508,6 @@ H5Dwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, const hsize_t *of if (0 == data_size) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "data_size cannot be zero"); - /* Make sure data size is less than 4 GiB */ - data_size_32 = (uint32_t)data_size; - if (data_size != (size_t)data_size_32) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid data_size - chunks cannot be > 4 GiB"); - /* Get the default dataset transfer property list if the user didn't provide one */ if (H5P_DEFAULT == dxpl_id) dxpl_id = H5P_DATASET_XFER_DEFAULT; @@ -1523,7 +1517,7 @@ H5Dwrite_chunk(hid_t dset_id, hid_t dxpl_id, uint32_t filters, const hsize_t *of /* Set up VOL callback arguments */ dset_opt_args.chunk_write.offset = offset; dset_opt_args.chunk_write.filters = filters; - dset_opt_args.chunk_write.size = data_size_32; + dset_opt_args.chunk_write.size = data_size; dset_opt_args.chunk_write.buf = buf; vol_cb_args.op_type = H5VL_NATIVE_DATASET_CHUNK_WRITE; vol_cb_args.args = &dset_opt_args; diff --git a/src/H5Dbtree.c b/src/H5Dbtree.c index 0587f77b16a..539ba1b3dfb 100644 --- a/src/H5Dbtree.c +++ b/src/H5Dbtree.c @@ -1072,16 +1072,14 @@ H5D__btree_idx_iterate_cb(H5F_t H5_ATTR_UNUSED *f, const void *_lt_key, haddr_t FUNC_ENTER_PACKAGE_NOERR /* Sanity check for memcpy() */ - HDcompile_assert(offsetof(H5D_chunk_rec_t, nbytes) == offsetof(H5D_btree_key_t, nbytes)); - HDcompile_assert(sizeof(chunk_rec.nbytes) == sizeof(lt_key->nbytes)); HDcompile_assert(offsetof(H5D_chunk_rec_t, scaled) == offsetof(H5D_btree_key_t, scaled)); HDcompile_assert(sizeof(chunk_rec.scaled) == sizeof(lt_key->scaled)); - HDcompile_assert(offsetof(H5D_chunk_rec_t, filter_mask) == offsetof(H5D_btree_key_t, filter_mask)); - HDcompile_assert(sizeof(chunk_rec.filter_mask) == sizeof(lt_key->filter_mask)); /* Compose generic chunk record for callback */ - H5MM_memcpy(&chunk_rec, lt_key, sizeof(*lt_key)); - chunk_rec.chunk_addr = addr; + H5MM_memcpy(&(chunk_rec.scaled), &(lt_key->scaled), sizeof(lt_key->scaled)); + chunk_rec.nbytes = (hsize_t)lt_key->nbytes; + chunk_rec.filter_mask = (uint32_t)lt_key->filter_mask; + chunk_rec.chunk_addr = addr; /* Make "generic chunk" callback */ if ((ret_value = (udata->cb)(&chunk_rec, udata->udata)) < 0) diff --git a/src/H5Dbtree2.c b/src/H5Dbtree2.c index ab02a2b8ed0..94188b53f39 100644 --- a/src/H5Dbtree2.c +++ b/src/H5Dbtree2.c @@ -67,19 +67,19 @@ /* User data for creating callback context */ typedef struct H5D_bt2_ctx_ud_t { const H5F_t *f; /* Pointer to file info */ - uint32_t chunk_size; /* Size of chunk (bytes; for filtered object) */ + hsize_t chunk_size; /* Size of chunk (bytes; for filtered object) */ unsigned ndims; /* Number of dimensions */ size_t chunk_size_len; /* Size of chunk sizes in the file (bytes) */ - uint32_t *dim; /* Size of chunk in elements */ + hsize_t *dim; /* Size of chunk in elements */ } H5D_bt2_ctx_ud_t; /* The callback context */ typedef struct H5D_bt2_ctx_t { - uint32_t chunk_size; /* Size of chunk (bytes; constant for unfiltered object) */ - size_t sizeof_addr; /* Size of file addresses in the file (bytes) */ - size_t chunk_size_len; /* Size of chunk sizes in the file (bytes) */ - unsigned ndims; /* Number of dimensions in chunk */ - uint32_t *dim; /* Size of chunk in elements */ + hsize_t chunk_size; /* Size of chunk (bytes; constant for unfiltered object) */ + size_t sizeof_addr; /* Size of file addresses in the file (bytes) */ + size_t chunk_size_len; /* Size of chunk sizes in the file (bytes) */ + unsigned ndims; /* Number of dimensions in chunk */ + hsize_t *dim; /* Size of chunk in elements */ } H5D_bt2_ctx_t; /* Callback info for iteration over chunks in v2 B-tree */ @@ -228,7 +228,7 @@ const H5B2_class_t H5D_BT2_FILT[1] = {{ H5FL_DEFINE_STATIC(H5D_bt2_ctx_t); /* Declare a free list to manage the page elements */ -H5FL_ARR_DEFINE_STATIC(uint32_t, H5O_LAYOUT_NDIMS); +H5FL_ARR_DEFINE_STATIC(hsize_t, H5O_LAYOUT_NDIMS); /*------------------------------------------------------------------------- * Function: H5D__bt2_crt_context @@ -245,7 +245,7 @@ H5D__bt2_crt_context(void *_udata) { H5D_bt2_ctx_ud_t *udata = (H5D_bt2_ctx_ud_t *)_udata; /* User data for building callback context */ H5D_bt2_ctx_t *ctx; /* Callback context structure */ - uint32_t *my_dim = NULL; /* Pointer to copy of chunk dimension size */ + hsize_t *my_dim = NULL; /* Pointer to copy of chunk dimension size */ void *ret_value = NULL; /* Return value */ FUNC_ENTER_PACKAGE @@ -267,9 +267,9 @@ H5D__bt2_crt_context(void *_udata) ctx->chunk_size_len = udata->chunk_size_len; /* Set up the "local" information for this dataset's chunk dimension sizes */ - if (NULL == (my_dim = (uint32_t *)H5FL_ARR_MALLOC(uint32_t, H5O_LAYOUT_NDIMS))) + if (NULL == (my_dim = (hsize_t *)H5FL_ARR_MALLOC(hsize_t, H5O_LAYOUT_NDIMS))) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, NULL, "can't allocate chunk dims"); - H5MM_memcpy(my_dim, udata->dim, H5O_LAYOUT_NDIMS * sizeof(uint32_t)); + H5MM_memcpy(my_dim, udata->dim, H5O_LAYOUT_NDIMS * sizeof(hsize_t)); ctx->dim = my_dim; /* Set return value */ @@ -301,7 +301,7 @@ H5D__bt2_dst_context(void *_ctx) /* Free array for chunk dimension sizes */ if (ctx->dim) - H5FL_ARR_FREE(uint32_t, ctx->dim); + H5FL_ARR_FREE(hsize_t, ctx->dim); /* Release callback context */ ctx = H5FL_FREE(H5D_bt2_ctx_t, ctx); @@ -561,7 +561,7 @@ H5D__bt2_filt_debug(FILE *stream, int indent, int fwidth, const void *_record, c assert(0 != record->nbytes); fprintf(stream, "%*s%-*s %" PRIuHADDR "\n", indent, "", fwidth, "Chunk address:", record->chunk_addr); - fprintf(stream, "%*s%-*s %u bytes\n", indent, "", fwidth, "Chunk size:", (unsigned)record->nbytes); + fprintf(stream, "%*s%-*s %" PRIuHSIZE " bytes\n", indent, "", fwidth, "Chunk size:", record->nbytes); fprintf(stream, "%*s%-*s 0x%08x\n", indent, "", fwidth, "Filter mask:", record->filter_mask); fprintf(stream, "%*s%-*s {", indent, "", fwidth, "Logical offset:"); @@ -953,7 +953,7 @@ H5D__bt2_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata, bt2_udata.ndims = idx_info->layout->u.chunk.ndims - 1; bt2_udata.rec.chunk_addr = udata->chunk_block.offset; if (idx_info->pline->nused > 0) { /* filtered chunk */ - H5_CHECKED_ASSIGN(bt2_udata.rec.nbytes, uint32_t, udata->chunk_block.length, hsize_t); + bt2_udata.rec.nbytes = udata->chunk_block.length; bt2_udata.rec.filter_mask = udata->filter_mask; } /* end if */ else { /* non-filtered chunk */ @@ -1237,8 +1237,7 @@ H5D__bt2_remove_cb(const void *_record, void *_udata) assert(f); /* Free the space in the file for the object being removed */ - H5_CHECK_OVERFLOW(record->nbytes, uint32_t, hsize_t); - if (H5MF_xfree(f, H5FD_MEM_DRAW, record->chunk_addr, (hsize_t)record->nbytes) < 0) + if (H5MF_xfree(f, H5FD_MEM_DRAW, record->chunk_addr, record->nbytes) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk"); done: diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index f291fcb47b3..f2a0e85c032 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -88,6 +88,46 @@ (H5D_CHUNK_IDX_SINGLE == (storage)->idx_type && H5D_COPS_SINGLE == (storage)->ops) || \ (H5D_CHUNK_IDX_NONE == (storage)->idx_type && H5D_COPS_NONE == (storage)->ops)); \ } while (0) + +/* Macro to Check for chunk size being too big to encode. Early versions were simply limited to 32 bits. + * Version 4, except for the single chunk index, was limited using a formula described below. Version 5 uses + * 64 bits, as does the single chunk index (with all versions). We additionally impose the restriction that + * version 4 cannot encode more than 32 bits, even though it is not precluded by the file format, because + * those versions of the library cannot handle chunks larger than 32 bits internally. */ +#define H5D_CHUNK_ENCODE_SIZE_CHECK(layout, length, err) \ + do { \ + if ((layout)->version <= H5O_LAYOUT_VERSION_4) { \ + if ((length) > UINT32_MAX) \ + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, err, \ + "chunk size is greater than UINT32_MAX and can't be encoded with this file " \ + "format version - see H5Pset_libver_bounds()"); \ + \ + if ((layout)->version == H5O_LAYOUT_VERSION_4 && \ + (layout)->storage.u.chunk.idx_type != H5D_CHUNK_IDX_SINGLE) { \ + unsigned allow_chunk_size_len; /* Allowed size of encoded chunk size */ \ + unsigned new_chunk_size_len; /* Size of encoded chunk size */ \ + \ + /* Compute the size required for encoding the size of a chunk, allowing */ \ + /* for an extra byte, in case the filter makes the chunk larger. */ \ + allow_chunk_size_len = 1 + ((H5VM_log2_gen((uint64_t)((layout)->u.chunk.size)) + 8) / 8); \ + if (allow_chunk_size_len > 8) \ + allow_chunk_size_len = 8; \ + \ + /* Compute encoded size of chunk */ \ + new_chunk_size_len = (H5VM_log2_gen((uint64_t)(length)) + 8) / 8; \ + if (new_chunk_size_len > 8) \ + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, err, \ + "encoded chunk size is more than 8 bytes?!?"); \ + \ + /* Check if the chunk became too large to be encoded */ \ + if (new_chunk_size_len > allow_chunk_size_len) \ + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, \ + "filter increased chunk size by too much and it cannot be encoded with " \ + "this file format version - see H5Pset_libver_bounds()"); \ + } \ + } \ + } while (0) + /* * Feature: If this constant is defined then every cache preemption and load * causes a character to be printed on the standard error stream: @@ -131,8 +171,8 @@ typedef struct H5D_rdcc_ent_t { bool deleted; /*chunk about to be deleted */ unsigned edge_chunk_state; /*states related to edge chunks (see above) */ hsize_t scaled[H5O_LAYOUT_NDIMS]; /*scaled chunk 'name' (coordinates) */ - uint32_t rd_count; /*bytes remaining to be read */ - uint32_t wr_count; /*bytes remaining to be written */ + hsize_t rd_count; /*bytes remaining to be read */ + hsize_t wr_count; /*bytes remaining to be written */ H5F_block_t chunk_block; /*offset/length of chunk in file */ hsize_t chunk_idx; /*index of chunk in dataset */ uint8_t *chunk; /*the unfiltered chunk data */ @@ -153,7 +193,7 @@ typedef struct H5D_chunk_it_ud1_t { const hsize_t *space_dim; /* New dataset dimensions */ const bool *shrunk_dim; /* Dimensions which have been shrunk */ H5S_t *chunk_space; /* Dataspace for a chunk */ - uint32_t elmts_per_chunk; /* Elements in chunk */ + hsize_t elmts_per_chunk; /* Elements in chunk */ hsize_t *hyper_start; /* Starting location of hyperslab */ H5D_fill_buf_info_t fb_info; /* Dataset's fill buffer info */ bool fb_info_init; /* Whether the fill value buffer has been initialized */ @@ -187,7 +227,7 @@ typedef struct H5D_chunk_it_ud3_t { H5T_path_t *tpath_mem_dst; /* Datatype conversion path from memory to dest. file */ void *reclaim_buf; /* Buffer for reclaiming data */ size_t reclaim_buf_size; /* Reclaim buffer size */ - uint32_t nelmts; /* Number of elements in buffer */ + size_t nelmts; /* Number of elements in buffer */ H5S_t *buf_space; /* Dataspace describing buffer */ /* needed for compressed variable-length data */ @@ -205,10 +245,10 @@ typedef struct H5D_chunk_it_ud3_t { /* Callback info for iteration to dump index */ typedef struct H5D_chunk_it_ud4_t { - FILE *stream; /* Output stream */ - bool header_displayed; /* Node's header is displayed? */ - unsigned ndims; /* Number of dimensions for chunk/dataset */ - uint32_t *chunk_dim; /* Chunk dimensions */ + FILE *stream; /* Output stream */ + bool header_displayed; /* Node's header is displayed? */ + unsigned ndims; /* Number of dimensions for chunk/dataset */ + hsize_t *chunk_dim; /* Chunk dimensions */ } H5D_chunk_it_ud4_t; /* Callback info for iteration to format convert chunks */ @@ -228,7 +268,7 @@ typedef struct H5D_chunk_readvv_ud_t { typedef struct H5D_chunk_info_iter_ud_t { hsize_t scaled[H5O_LAYOUT_NDIMS]; /* Logical offset of the chunk */ hsize_t ndims; /* Number of dimensions in the dataset */ - uint32_t nbytes; /* Size of stored data in the chunk */ + hsize_t nbytes; /* Size of stored data in the chunk */ unsigned filter_mask; /* Excluded filters */ haddr_t chunk_addr; /* Address of the chunk in file */ hsize_t chunk_idx; /* Chunk index, where the iteration needs to stop */ @@ -262,7 +302,7 @@ typedef struct H5D_chunk_iter_ud_t { /* Chunked layout operation callbacks */ static herr_t H5D__chunk_construct(H5F_t *f, H5D_t *dset); -static herr_t H5D__chunk_init(H5F_t *f, const H5D_t *dset, hid_t dapl_id); +static herr_t H5D__chunk_init(H5F_t *f, H5D_t *dset, hid_t dapl_id, bool open_op); static herr_t H5D__chunk_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo); static herr_t H5D__chunk_io_init_selections(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo); static herr_t H5D__chunk_mdio_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo); @@ -290,6 +330,7 @@ static int H5D__chunk_format_convert_cb(const H5D_chunk_rec_t *chunk_rec, void * /* Helper routines */ static herr_t H5D__chunk_set_info_real(H5O_layout_chunk_t *layout, unsigned ndims, const hsize_t *curr_dims, const hsize_t *max_dims); +static herr_t H5D__chunk_set_sizes(H5D_t *dset); static herr_t H5D__chunk_cinfo_cache_reset(H5D_chunk_cached_t *last); static herr_t H5D__chunk_cinfo_cache_update(H5D_chunk_cached_t *last, const H5D_chunk_ud_t *udata); static bool H5D__chunk_cinfo_cache_found(const H5D_chunk_cached_t *last, H5D_chunk_ud_t *udata); @@ -309,7 +350,7 @@ static herr_t H5D__chunk_cache_evict(const H5D_t *dset, H5D_rdcc_ent_t *ent, b static void *H5D__chunk_lock(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info, H5D_chunk_ud_t *udata, bool relax, bool prev_unfilt_chunk); static herr_t H5D__chunk_unlock(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info, - const H5D_chunk_ud_t *udata, bool dirty, void *chunk, uint32_t naccessed); + const H5D_chunk_ud_t *udata, bool dirty, void *chunk, hsize_t naccessed); static herr_t H5D__chunk_cache_prune(const H5D_t *dset, size_t size); static herr_t H5D__chunk_prune_fill(H5D_chunk_it_ud1_t *udata, bool new_unfilt_chunk); #ifdef H5_HAVE_PARALLEL @@ -378,7 +419,7 @@ H5FL_EXTERN(H5S_sel_iter_t); *------------------------------------------------------------------------- */ herr_t -H5D__chunk_direct_write(H5D_t *dset, uint32_t filters, hsize_t *offset, uint32_t data_size, const void *buf) +H5D__chunk_direct_write(H5D_t *dset, uint32_t filters, hsize_t *offset, size_t data_size, const void *buf) { const H5O_layout_t *layout = &(dset->shared->layout); /* Dataset layout */ H5D_chunk_ud_t udata; /* User data for querying chunk info */ @@ -427,6 +468,10 @@ H5D__chunk_direct_write(H5D_t *dset, uint32_t filters, hsize_t *offset, uint32_t /* Set up the size of chunk for user data */ udata.chunk_block.length = data_size; + /* Check for hsize_t overflow */ + if (H5_UNLIKELY((size_t)udata.chunk_block.length != data_size)) + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk size too big to fit in hsize_t"); + if (0 == idx_info.pline->nused && H5_addr_defined(old_chunk.offset)) /* If there are no filters and we are overwriting the chunk we can just set values */ need_insert = false; @@ -522,6 +567,10 @@ H5D__chunk_direct_read(const H5D_t *dset, hsize_t *offset, uint32_t *filters, vo assert((H5_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length > 0) || (!H5_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length == 0)); + /* Check for size_t overflow */ + if (H5_UNLIKELY((hsize_t)((size_t)udata.chunk_block.length) != udata.chunk_block.length)) + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk size does not fit in size_t"); + /* Check if the requested chunk exists in the chunk cache */ if (UINT_MAX != udata.idx_hint) { H5D_rdcc_ent_t *ent = rdcc->slot[udata.idx_hint]; @@ -558,12 +607,12 @@ H5D__chunk_direct_read(const H5D_t *dset, hsize_t *offset, uint32_t *filters, vo if (udata.chunk_block.length > 0 && buf && (!nalloc || *nalloc >= udata.chunk_block.length)) /* Read the chunk data into the supplied buffer */ if (H5F_shared_block_read(H5F_SHARED(dset->oloc.file), H5FD_MEM_DRAW, udata.chunk_block.offset, - udata.chunk_block.length, buf) < 0) + (size_t)udata.chunk_block.length, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "unable to read raw data chunk"); /* Return the size of the chunk block in *nalloc if nalloc is provided */ if (nalloc) - *nalloc = udata.chunk_block.length; + *nalloc = (size_t)udata.chunk_block.length; /* Return the filter mask */ *filters = udata.filter_mask; @@ -758,7 +807,7 @@ H5D__chunk_set_info(const H5D_t *dset) * *------------------------------------------------------------------------- */ -herr_t +static herr_t H5D__chunk_set_sizes(H5D_t *dset) { uint64_t chunk_size; /* Size of chunk in bytes */ @@ -772,9 +821,18 @@ H5D__chunk_set_sizes(H5D_t *dset) assert(dset); assert(dset->shared->layout.u.chunk.ndims > 0); + /* In this function, some of these sizes may have already been set since they are sometimes stored in the + * file. If this is the case, verify the calculated sizes match the stored sizes. */ /* Set the last dimension of the chunk size to the size of the datatype */ - dset->shared->layout.u.chunk.dim[dset->shared->layout.u.chunk.ndims - 1] = - (uint32_t)H5T_GET_SIZE(dset->shared->type); + if (dset->shared->layout.u.chunk.dim[dset->shared->layout.u.chunk.ndims - 1]) { + if (dset->shared->layout.u.chunk.dim[dset->shared->layout.u.chunk.ndims - 1] != + (hsize_t)H5T_GET_SIZE(dset->shared->type)) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, + "stored datatype size in chunk layout does not match datatype description"); + } + else + dset->shared->layout.u.chunk.dim[dset->shared->layout.u.chunk.ndims - 1] = + (hsize_t)H5T_GET_SIZE(dset->shared->type); /* Compute number of bytes to use for encoding chunk dimensions */ max_enc_bytes_per_dim = 0; @@ -789,7 +847,16 @@ H5D__chunk_set_sizes(H5D_t *dset) max_enc_bytes_per_dim = enc_bytes_per_dim; } /* end for */ assert(max_enc_bytes_per_dim > 0 && max_enc_bytes_per_dim <= 8); - dset->shared->layout.u.chunk.enc_bytes_per_dim = max_enc_bytes_per_dim; + + /* Set encoding length in layout */ + if (dset->shared->layout.u.chunk.enc_bytes_per_dim) { + if (dset->shared->layout.u.chunk.enc_bytes_per_dim != max_enc_bytes_per_dim) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, + "stored chunk dimension encoding length does not match value calculated from chunk " + "dimensions"); + } + else + dset->shared->layout.u.chunk.enc_bytes_per_dim = max_enc_bytes_per_dim; /* Compute and store the total size of a chunk */ /* (Use 64-bit value to ensure that we can detect >4GB chunks) */ @@ -797,12 +864,12 @@ H5D__chunk_set_sizes(H5D_t *dset) u < dset->shared->layout.u.chunk.ndims; u++) chunk_size *= (uint64_t)dset->shared->layout.u.chunk.dim[u]; - /* Check for chunk larger than can be represented in 32-bits */ - /* (Chunk size is encoded in 32-bit value in v1 B-tree records) */ - if (chunk_size > (uint64_t)0xffffffff) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "chunk size must be < 4GB"); + /* Store chunk size in layout */ + dset->shared->layout.u.chunk.size = (hsize_t)chunk_size; - H5_CHECKED_ASSIGN(dset->shared->layout.u.chunk.size, uint32_t, chunk_size, uint64_t); + /* Check for overflow converting uint64_t to hsize_t */ + if ((uint64_t)dset->shared->layout.u.chunk.size != chunk_size) + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk size too big to fit in hsize_t"); done: FUNC_LEAVE_NOAPI(ret_value) @@ -811,46 +878,196 @@ H5D__chunk_set_sizes(H5D_t *dset) /*------------------------------------------------------------------------- * Function: H5D__chunk_construct * - * Purpose: Constructs new chunked layout information for dataset + * Purpose: Constructs new chunked layout information for dataset and upgrade layout version if appropriate * * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ static herr_t -H5D__chunk_construct(H5F_t H5_ATTR_UNUSED *f, H5D_t *dset) +H5D__chunk_construct(H5F_t *f, H5D_t *dset) { - unsigned u; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ + H5O_layout_t *layout; + unsigned version_req = H5O_LAYOUT_VERSION_1; /* Required layout version */ + unsigned version_perf = H5O_LAYOUT_VERSION_1; /* Version we would like to upgrade to for performance */ + unsigned u; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE /* Sanity checks */ assert(f); assert(dset); + assert(dset->shared); + + /* Set converience pointer */ + layout = &dset->shared->layout; /* Check for invalid chunk dimension rank */ if (0 == dset->shared->layout.u.chunk.ndims) HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "no chunk information set?"); - if (dset->shared->layout.u.chunk.ndims != dset->shared->ndims) + if (layout->u.chunk.ndims != dset->shared->ndims) HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "dimensionality of chunks doesn't match the dataspace"); /* Increment # of chunk dimensions, to account for datatype size as last element */ - dset->shared->layout.u.chunk.ndims++; + layout->u.chunk.ndims++; /* Set chunk sizes */ if (H5D__chunk_set_sizes(dset) < 0) HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "unable to set chunk sizes"); - assert((unsigned)(dset->shared->layout.u.chunk.ndims) <= NELMTS(dset->shared->layout.u.chunk.dim)); + assert((unsigned)(layout->u.chunk.ndims) <= NELMTS(layout->u.chunk.dim)); + + /* Calculate layout version */ + /* Do not downgrade version */ + version_req = layout->version; + + /* Always prefer at least version 4 for performance due to new indexes */ + version_perf = H5O_LAYOUT_VERSION_4; + + /* First check for chunk larger than can be represented in 32-bits - this requires layout version 5. While + * it could be encoded as version 4, those versions of the library would not be able to read it. */ + if (layout->u.chunk.size > (hsize_t)0xffffffff) { + if (H5O_layout_ver_bounds[H5F_HIGH_BOUND(f)] < H5O_LAYOUT_VERSION_5) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "chunk size > 4GB requires H5F_LIBVER_V200 or later format version - see " + "H5Pset_libver_bounds()"); + + /* Set required version */ + version_req = MAX(version_req, H5O_LAYOUT_VERSION_5); + } +#ifndef NDEBUG + else { + /* Make sure no individual chunk dimension is greater than 32 bits. Should be impossible since the + * chunk size is less than 32 bits. */ + for (u = 0; u < dset->shared->ndims; u++) + assert(layout->u.chunk.dim[u] <= (hsize_t)0xffffffff); + } +#endif /* NDEBUG */ + + /* Now check if we can or must upgrade to version or above - if so set the new indexing */ + if ((H5O_layout_ver_bounds[H5F_LOW_BOUND(f)] >= H5O_LAYOUT_VERSION_4) || + (version_req >= H5O_LAYOUT_VERSION_4)) { + unsigned unlim_count = 0; /* Count of unlimited max. dimensions */ + bool single = true; /* Fulfill single chunk indexing */ + + /* It should be impossible to create a chunked dataset with a scalar/null dataspace */ + assert(dset->shared->ndims > 0); + + /* Spin through the max. dimensions, looking for unlimited dimensions */ + for (u = 0; u < dset->shared->ndims; u++) { + if (dset->shared->max_dims[u] == H5S_UNLIMITED) + unlim_count++; + if (dset->shared->curr_dims[u] != dset->shared->max_dims[u] || + dset->shared->curr_dims[u] != layout->u.chunk.dim[u]) + single = false; + } + + /* Chunked datasets with unlimited dimension(s) */ + if (unlim_count) { /* dataset with unlimited dimension(s) must be chunked */ + if (1 == unlim_count) { /* Chunked dataset with only 1 unlimited dimension */ + /* Set the chunk index type to an extensible array */ + layout->u.chunk.idx_type = H5D_CHUNK_IDX_EARRAY; + layout->storage.u.chunk.idx_type = H5D_CHUNK_IDX_EARRAY; + layout->storage.u.chunk.ops = H5D_COPS_EARRAY; + + /* Set the extensible array creation parameters */ + /* (use hard-coded defaults for now, until we give applications + * control over this with a property list - QAK) + */ + layout->u.chunk.u.earray.cparam.max_nelmts_bits = H5D_EARRAY_MAX_NELMTS_BITS; + layout->u.chunk.u.earray.cparam.idx_blk_elmts = H5D_EARRAY_IDX_BLK_ELMTS; + layout->u.chunk.u.earray.cparam.sup_blk_min_data_ptrs = H5D_EARRAY_SUP_BLK_MIN_DATA_PTRS; + layout->u.chunk.u.earray.cparam.data_blk_min_elmts = H5D_EARRAY_DATA_BLK_MIN_ELMTS; + layout->u.chunk.u.earray.cparam.max_dblk_page_nelmts_bits = + H5D_EARRAY_MAX_DBLOCK_PAGE_NELMTS_BITS; + + /* If there are filters, we prefer version 5 since that version can handle greatly expanding + * filters */ + if (dset->shared->dcpl_cache.pline.nused) + version_perf = H5O_LAYOUT_VERSION_5; + } + else { /* Chunked dataset with > 1 unlimited dimensions */ + /* Set the chunk index type to v2 B-tree */ + layout->u.chunk.idx_type = H5D_CHUNK_IDX_BT2; + layout->storage.u.chunk.idx_type = H5D_CHUNK_IDX_BT2; + layout->storage.u.chunk.ops = H5D_COPS_BT2; + + /* Set the v2 B-tree creation parameters */ + /* (use hard-coded defaults for now, until we give applications + * control over this with a property list - QAK) + */ + layout->u.chunk.u.btree2.cparam.node_size = H5D_BT2_NODE_SIZE; + layout->u.chunk.u.btree2.cparam.split_percent = H5D_BT2_SPLIT_PERC; + layout->u.chunk.u.btree2.cparam.merge_percent = H5D_BT2_MERGE_PERC; + + /* If there are filters, we prefer version 5 since that version can handle greatly expanding + * filters */ + if (dset->shared->dcpl_cache.pline.nused) + version_perf = H5O_LAYOUT_VERSION_5; + } + } + else { /* Chunked dataset with fixed dimensions */ + /* Check for correct condition for using "single chunk" chunk index */ + if (single) { + layout->u.chunk.idx_type = H5D_CHUNK_IDX_SINGLE; + layout->storage.u.chunk.idx_type = H5D_CHUNK_IDX_SINGLE; + layout->storage.u.chunk.ops = H5D_COPS_SINGLE; + + /* If there are filters, we prefer version 5 since that version can handle greatly expanding + * filters (>4 GiB) (technically allowed by the v4 format but disallowed by library since + * older library version can't handle >4 GiB chunks even if the format can) */ + if (dset->shared->dcpl_cache.pline.nused) + version_perf = H5O_LAYOUT_VERSION_5; + } + else if (!dset->shared->dcpl_cache.pline.nused && + dset->shared->dcpl_cache.fill.alloc_time == H5D_ALLOC_TIME_EARLY) { + + /* Set the chunk index type to "none" Index */ + layout->u.chunk.idx_type = H5D_CHUNK_IDX_NONE; + layout->storage.u.chunk.idx_type = H5D_CHUNK_IDX_NONE; + layout->storage.u.chunk.ops = H5D_COPS_NONE; + + /* No need to upgrade to version 5 since the none index isn't used with filters */ + } + else { + /* Set the chunk index type to Fixed Array */ + layout->u.chunk.idx_type = H5D_CHUNK_IDX_FARRAY; + layout->storage.u.chunk.idx_type = H5D_CHUNK_IDX_FARRAY; + layout->storage.u.chunk.ops = H5D_COPS_FARRAY; + + /* Set the fixed array creation parameters */ + /* (use hard-coded defaults for now, until we give applications + * control over this with a property list - QAK) + */ + layout->u.chunk.u.farray.cparam.max_dblk_page_nelmts_bits = + H5D_FARRAY_MAX_DBLK_PAGE_NELMTS_BITS; + + /* If there are filters, we prefer version 5 since that version can handle greatly expanding + * filters */ + if (dset->shared->dcpl_cache.pline.nused) + version_perf = H5O_LAYOUT_VERSION_5; + } + } + } + + /* Calculate final version - choose the maximum of the current version, the required version, and the + * minimum of the low bound and the version we would like to upgrade to for performance. This ensures that + * we are never below the current version or the required version, and that we always upgrade to at least + * the low bound when useful, but never farther unless required otherwise. */ + layout->version = + MAX3(layout->version, version_req, MIN(H5O_layout_ver_bounds[H5F_LOW_BOUND(f)], version_perf)); + + /* The logic above should guarantee the calculated layout version is not above the high bound */ + assert(layout->version <= H5O_layout_ver_bounds[H5F_HIGH_BOUND(f)]); /* Chunked storage is not compatible with external storage (currently) */ if (dset->shared->dcpl_cache.efl.nused > 0) HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "external storage not supported with chunked layout"); /* Sanity check dimensions */ - for (u = 0; u < dset->shared->layout.u.chunk.ndims - 1; u++) { + for (u = 0; u < layout->u.chunk.ndims - 1; u++) { /* Don't allow zero-sized chunk dimensions */ - if (0 == dset->shared->layout.u.chunk.dim[u]) + if (0 == layout->u.chunk.dim[u]) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "chunk size must be > 0, dim = %u ", u); /* @@ -859,13 +1076,13 @@ H5D__chunk_construct(H5F_t H5_ATTR_UNUSED *f, H5D_t *dset) * will be no such restriction. */ if (dset->shared->curr_dims[u] && dset->shared->max_dims[u] != H5S_UNLIMITED && - dset->shared->max_dims[u] < dset->shared->layout.u.chunk.dim[u]) + dset->shared->max_dims[u] < layout->u.chunk.dim[u]) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "chunk size must be <= maximum dimension size for fixed-sized dimensions"); } /* end for */ /* Reset address and pointer of the array struct for the chunked storage index */ - if (H5D_chunk_idx_reset(&dset->shared->layout.storage.u.chunk, true) < 0) + if (H5D_chunk_idx_reset(&layout->storage.u.chunk, true) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to reset chunked storage index"); done: @@ -883,7 +1100,7 @@ H5D__chunk_construct(H5F_t H5_ATTR_UNUSED *f, H5D_t *dset) *------------------------------------------------------------------------- */ static herr_t -H5D__chunk_init(H5F_t *f, const H5D_t *const dset, hid_t dapl_id) +H5D__chunk_init(H5F_t *f, H5D_t *dset, hid_t dapl_id, bool open_op) { H5D_chk_idx_info_t idx_info; /* Chunked index info */ H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /* Convenience pointer to dataset's chunk cache */ @@ -970,6 +1187,17 @@ H5D__chunk_init(H5F_t *f, const H5D_t *const dset, hid_t dapl_id) if (H5D__chunk_set_info(dset) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set # of chunks for dataset"); + /* Set chunk sizes if not done already (during a create operation the construct callback does this) */ + if (open_op && (H5D__chunk_set_sizes(dset) < 0)) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "unable to set chunk sizes"); + + /* Check for chunk larger than can be represented in 32-bits encoded as v1 b-tree. We don't allow creation + * of files with layout version < 5 with 64 bit chunks but we'll try to open them if they exist because + * they're not disallowed by the file format. */ + if (dset->shared->layout.u.chunk.size > (hsize_t)0xffffffff && + dset->shared->layout.u.chunk.idx_type == H5D_CHUNK_IDX_BTREE) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "chunk size must be < 4GB with v1 b-tree index"); + done: if (FAIL == ret_value) { if (rdcc->slot) @@ -2546,8 +2774,11 @@ H5D__chunk_cacheable(const H5D_io_info_t H5_ATTR_PARALLEL_USED *io_info, H5D_dse * need to write the fill value, then don't load the chunk into the * cache, just write the data to it directly. */ - H5_CHECK_OVERFLOW(dataset->shared->layout.u.chunk.size, uint32_t, size_t); - if ((size_t)dataset->shared->layout.u.chunk.size > dataset->shared->cache.chunk.nbytes_max) { + /* If the chunk is too big to fit in size_t, assume it is too big to fit in cache */ + if (!(((hsize_t)((size_t)dataset->shared->layout.u.chunk.size) == + dataset->shared->layout.u.chunk.size) && + ((size_t)dataset->shared->layout.u.chunk.size <= + dataset->shared->cache.chunk.nbytes_max))) { if (write_op && !H5_addr_defined(caddr)) { const H5O_fill_t *fill = &(dataset->shared->dcpl_cache.fill); /* Fill value info */ H5D_fill_value_t fill_status; /* Fill value status */ @@ -2561,8 +2792,15 @@ H5D__chunk_cacheable(const H5D_io_info_t H5_ATTR_PARALLEL_USED *io_info, H5D_dse if (fill->fill_time == H5D_FILL_TIME_ALLOC || (fill->fill_time == H5D_FILL_TIME_IFSET && (fill_status == H5D_FILL_VALUE_USER_DEFINED || - fill_status == H5D_FILL_VALUE_DEFAULT))) + fill_status == H5D_FILL_VALUE_DEFAULT))) { + /* Check for size_t overflow */ + if (H5_UNLIKELY((hsize_t)((size_t)dataset->shared->layout.u.chunk.size) != + dataset->shared->layout.u.chunk.size)) + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, + "must fill chunk but it is too big to fit in size_t - try using " + "early allocation"); ret_value = true; + } else ret_value = false; } @@ -2634,8 +2872,11 @@ H5D__chunk_may_use_select_io(H5D_io_info_t *io_info, const H5D_dset_io_info_t *d if (!(io_info->using_mpi_vfd && (H5F_ACC_RDWR & H5F_INTENT(dataset->oloc.file)))) { #endif /* H5_HAVE_PARALLEL */ /* Check if the chunk is too large to keep in the cache */ - H5_CHECK_OVERFLOW(dataset->shared->layout.u.chunk.size, uint32_t, size_t); - if ((size_t)dataset->shared->layout.u.chunk.size <= dataset->shared->cache.chunk.nbytes_max) { + /* If the chunk is too big to fit in size_t, assume it is too big to fit in cache */ + if (((hsize_t)((size_t)dataset->shared->layout.u.chunk.size) == + dataset->shared->layout.u.chunk.size) && + ((size_t)dataset->shared->layout.u.chunk.size <= + dataset->shared->cache.chunk.nbytes_max)) { io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; io_info->no_selection_io_cause |= H5D_SEL_IO_CHUNK_CACHE; } @@ -2666,7 +2907,7 @@ H5D__chunk_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) H5D_dset_io_info_t nonexistent_dset_info; /* "nonexistent" I/O dset info object */ H5D_dset_io_info_t ctg_dset_info; /* Contiguous I/O dset info object */ H5D_dset_io_info_t cpt_dset_info; /* Compact I/O dset info object */ - uint32_t src_accessed_bytes = 0; /* Total accessed size in a chunk */ + hsize_t src_accessed_bytes = 0; /* Total accessed size in a chunk */ bool skip_missing_chunks = false; /* Whether to skip missing chunks */ H5S_t **chunk_mem_spaces = NULL; /* Array of chunk memory spaces */ H5S_t *chunk_mem_spaces_local[8]; /* Local buffer for chunk_mem_spaces */ @@ -2808,7 +3049,8 @@ H5D__chunk_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) * usually does not use strip mining (H5D__scatgath_write), and instead allocates buffers * large enough for the entire I/O. Set request_nelmts to be large enough for all selected * elements in this chunk because it must be at least that large */ - nonexistent_dset_info.type_info.request_nelmts = nonexistent_dset_info.nelmts; + H5_CHECKED_ASSIGN(nonexistent_dset_info.type_info.request_nelmts, size_t, + nonexistent_dset_info.nelmts, hsize_t); /* Perform the actual read operation from the nonexistent chunk */ @@ -2864,8 +3106,7 @@ H5D__chunk_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) ctg_io_info.count = 1; /* Initialize temporary contiguous storage info */ - H5_CHECKED_ASSIGN(ctg_store.contig.dset_size, hsize_t, dset_info->dset->shared->layout.u.chunk.size, - uint32_t); + ctg_store.contig.dset_size = dset_info->dset->shared->layout.u.chunk.size; /* Set up compact I/O info object */ H5MM_memcpy(&cpt_io_info, io_info, sizeof(cpt_io_info)); @@ -2913,10 +3154,9 @@ H5D__chunk_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) /* Load the chunk into cache and lock it. */ /* Compute # of bytes accessed in chunk */ - H5_CHECK_OVERFLOW(dset_info->type_info.src_type_size, /*From:*/ size_t, /*To:*/ uint32_t); - H5_CHECK_OVERFLOW(chunk_info->piece_points, /*From:*/ size_t, /*To:*/ uint32_t); + H5_CHECK_OVERFLOW(dset_info->type_info.src_type_size, /*From:*/ size_t, /*To:*/ hsize_t); src_accessed_bytes = - (uint32_t)chunk_info->piece_points * (uint32_t)dset_info->type_info.src_type_size; + chunk_info->piece_points * (hsize_t)dset_info->type_info.src_type_size; /* Lock the chunk into the cache */ if (NULL == (chunk = H5D__chunk_lock(io_info, dset_info, &udata, false, false))) @@ -2999,7 +3239,7 @@ H5D__chunk_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) H5D_dset_io_info_t cpt_dset_info; /* Compact I/O dset info object */ H5D_storage_t cpt_store; /* Chunk storage information as compact dataset */ bool cpt_dirty; /* Temporary placeholder for compact storage "dirty" flag */ - uint32_t dst_accessed_bytes = 0; /* Total accessed size in a chunk */ + hsize_t dst_accessed_bytes = 0; /* Total accessed size in a chunk */ H5S_t **chunk_mem_spaces = NULL; /* Array of chunk memory spaces */ H5S_t *chunk_mem_spaces_local[8]; /* Local buffer for chunk_mem_spaces */ H5S_t **chunk_file_spaces = NULL; /* Array of chunk file spaces */ @@ -3024,8 +3264,7 @@ H5D__chunk_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) ctg_io_info.count = 1; /* Initialize temporary contiguous storage info */ - H5_CHECKED_ASSIGN(ctg_store.contig.dset_size, hsize_t, dset_info->dset->shared->layout.u.chunk.size, - uint32_t); + ctg_store.contig.dset_size = dset_info->dset->shared->layout.u.chunk.size; /* Set up compact I/O info object */ H5MM_memcpy(&cpt_io_info, io_info, sizeof(cpt_io_info)); @@ -3113,10 +3352,8 @@ H5D__chunk_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) bool entire_chunk = true; /* Whether whole chunk is selected */ /* Compute # of bytes accessed in chunk */ - H5_CHECK_OVERFLOW(dset_info->type_info.dst_type_size, /*From:*/ size_t, /*To:*/ uint32_t); - H5_CHECK_OVERFLOW(chunk_info->piece_points, /*From:*/ size_t, /*To:*/ uint32_t); - dst_accessed_bytes = - (uint32_t)chunk_info->piece_points * (uint32_t)dset_info->type_info.dst_type_size; + H5_CHECK_OVERFLOW(dset_info->type_info.dst_type_size, /*From:*/ size_t, /*To:*/ hsize_t); + dst_accessed_bytes = chunk_info->piece_points * (hsize_t)dset_info->type_info.dst_type_size; /* Determine if we will access all the data in the chunk */ if (dst_accessed_bytes != ctg_store.contig.dset_size || @@ -3142,7 +3379,8 @@ H5D__chunk_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) * usually does not use strip mining (H5D__scatgath_write), and instead allocates buffers * large enough for the entire I/O. Set request_nelmts to be large enough for all selected * elements in this chunk because it must be at least that large */ - cpt_dset_info.type_info.request_nelmts = cpt_dset_info.nelmts; + H5_CHECKED_ASSIGN(cpt_dset_info.type_info.request_nelmts, size_t, cpt_dset_info.nelmts, + hsize_t); /* Perform the actual write operation */ if ((dset_info->io_ops.single_write)(&cpt_io_info, &cpt_dset_info) < 0) @@ -3281,10 +3519,8 @@ H5D__chunk_write(H5D_io_info_t *io_info, H5D_dset_io_info_t *dset_info) bool entire_chunk = true; /* Whether whole chunk is selected */ /* Compute # of bytes accessed in chunk */ - H5_CHECK_OVERFLOW(dset_info->type_info.dst_type_size, /*From:*/ size_t, /*To:*/ uint32_t); - H5_CHECK_OVERFLOW(chunk_info->piece_points, /*From:*/ size_t, /*To:*/ uint32_t); - dst_accessed_bytes = - (uint32_t)chunk_info->piece_points * (uint32_t)dset_info->type_info.dst_type_size; + H5_CHECK_OVERFLOW(dset_info->type_info.dst_type_size, /*From:*/ size_t, /*To:*/ hsize_t); + dst_accessed_bytes = chunk_info->piece_points * (hsize_t)dset_info->type_info.dst_type_size; /* Determine if we will access all the data in the chunk */ if (dst_accessed_bytes != ctg_store.contig.dset_size || @@ -3610,8 +3846,8 @@ H5D__chunk_cinfo_cache_update(H5D_chunk_cached_t *last, const H5D_chunk_ud_t *ud /* Stored the information to cache */ H5MM_memcpy(last->scaled, udata->common.scaled, sizeof(hsize_t) * udata->common.layout->ndims); - last->addr = udata->chunk_block.offset; - H5_CHECKED_ASSIGN(last->nbytes, uint32_t, udata->chunk_block.length, hsize_t); + last->addr = udata->chunk_block.offset; + last->nbytes = udata->chunk_block.length; last->chunk_idx = udata->chunk_idx; last->filter_mask = udata->filter_mask; @@ -3962,10 +4198,13 @@ H5D__chunk_flush_entry(const H5D_t *dset, H5D_rdcc_ent_t *ent, bool reset) /* Should the chunk be filtered before writing it to disk? */ if (dset->shared->dcpl_cache.pline.nused && !(ent->edge_chunk_state & H5D_RDCC_DISABLE_FILTERS)) { - H5Z_EDC_t err_detect; /* Error detection info */ - H5Z_cb_t filter_cb; /* I/O filter callback function */ - size_t alloc = udata.chunk_block.length; /* Bytes allocated for BUF */ - size_t nbytes; /* Chunk size (in bytes) */ + H5Z_EDC_t err_detect; /* Error detection info */ + H5Z_cb_t filter_cb; /* I/O filter callback function */ + size_t alloc; /* Bytes allocated for BUF */ + size_t nbytes; /* Chunk size (in bytes) */ + + /* Assign alloc and check for overflow */ + H5_CHECKED_ASSIGN(alloc, size_t, udata.chunk_block.length, hsize_t); /* Retrieve filter settings from API context */ if (H5CX_get_err_detect(&err_detect) < 0) @@ -3998,11 +4237,7 @@ H5D__chunk_flush_entry(const H5D_t *dset, H5D_rdcc_ent_t *ent, bool reset) if (H5Z_pipeline(&(dset->shared->dcpl_cache.pline), 0, &(udata.filter_mask), err_detect, filter_cb, &nbytes, &alloc, &buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTFILTER, FAIL, "output pipeline failed"); -#if H5_SIZEOF_SIZE_T > 4 - /* Check for the chunk expanding too much to encode in a 32-bit value */ - if (nbytes > ((size_t)0xffffffff)) - HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk too large for 32-bit length"); -#endif /* H5_SIZEOF_SIZE_T > 4 */ + H5_CHECKED_ASSIGN(udata.chunk_block.length, hsize_t, nbytes, size_t); /* Indicate that the chunk must be allocated */ @@ -4173,7 +4408,7 @@ H5D__chunk_cache_evict(const H5D_t *dset, H5D_rdcc_ent_t *ent, bool flush) /* Remove from cache */ assert(rdcc->slot[ent->idx] != ent); ent->idx = UINT_MAX; - rdcc->nbytes_used -= dset->shared->layout.u.chunk.size; + rdcc->nbytes_used -= (size_t)dset->shared->layout.u.chunk.size; --rdcc->nused; /* Free */ @@ -4351,7 +4586,9 @@ H5D__chunk_lock(const H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, const H5D_ds /* Get the chunk's size */ assert(layout->u.chunk.size > 0); - H5_CHECKED_ASSIGN(chunk_size, size_t, layout->u.chunk.size, uint32_t); + if (H5_UNLIKELY((hsize_t)((size_t)layout->u.chunk.size) != layout->u.chunk.size)) + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, NULL, "chunk too big to fit in size_t"); + chunk_size = (size_t)layout->u.chunk.size; /* Check if the chunk is in the cache */ if (UINT_MAX != udata->idx_hint) { @@ -4518,8 +4755,12 @@ H5D__chunk_lock(const H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, const H5D_ds /* Check if the chunk exists on disk */ if (H5_addr_defined(chunk_addr)) { - size_t my_chunk_alloc = chunk_alloc; /* Allocated buffer size */ - size_t buf_alloc = chunk_alloc; /* [Re-]allocated buffer size */ + size_t my_chunk_alloc; /* Allocated buffer size */ + size_t buf_alloc; /* [Re-]allocated buffer size */ + + /* Assign above variables and check for overflow */ + H5_CHECKED_ASSIGN(my_chunk_alloc, size_t, chunk_alloc, hsize_t); + H5_CHECKED_ASSIGN(buf_alloc, size_t, chunk_alloc, hsize_t); /* Chunk size on disk isn't [likely] the same size as the final chunk * size in memory, so allocate memory big enough. */ @@ -4636,8 +4877,8 @@ H5D__chunk_lock(const H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, const H5D_ds ent->chunk_block.length = chunk_alloc; ent->chunk_idx = udata->chunk_idx; H5MM_memcpy(ent->scaled, udata->common.scaled, sizeof(hsize_t) * layout->u.chunk.ndims); - H5_CHECKED_ASSIGN(ent->rd_count, uint32_t, chunk_size, size_t); - H5_CHECKED_ASSIGN(ent->wr_count, uint32_t, chunk_size, size_t); + H5_CHECKED_ASSIGN(ent->rd_count, hsize_t, chunk_size, size_t); + H5_CHECKED_ASSIGN(ent->wr_count, hsize_t, chunk_size, size_t); ent->chunk = (uint8_t *)chunk; /* Add it to the cache */ @@ -4718,7 +4959,7 @@ H5D__chunk_lock(const H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, const H5D_ds */ static herr_t H5D__chunk_unlock(const H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, const H5D_dset_io_info_t *dset_info, - const H5D_chunk_ud_t *udata, bool dirty, void *chunk, uint32_t naccessed) + const H5D_chunk_ud_t *udata, bool dirty, void *chunk, hsize_t naccessed) { const H5O_layout_t *layout; /* Dataset layout */ const H5D_rdcc_t *rdcc; @@ -4901,8 +5142,8 @@ H5D__chunk_allocate(const H5D_t *dset, bool full_overwrite, const hsize_t old_di hsize_t max_unalloc[H5O_LAYOUT_NDIMS]; /* Last chunk in each dimension that is unallocated (in scaled coordinates) */ hsize_t scaled[H5O_LAYOUT_NDIMS]; /* Offset of current chunk (in scaled coordinates) */ - size_t orig_chunk_size; /* Original size of chunk in bytes */ - size_t chunk_size; /* Actual size of chunk in bytes, possibly filtered */ + size_t orig_chunk_size = 0; /* Original size of chunk in bytes */ + hsize_t chunk_size; /* Actual size of chunk in bytes, possibly filtered */ unsigned filter_mask = 0; /* Filter mask for chunks that have them */ H5O_layout_t *layout = &(dset->shared->layout); /* Dataset layout */ H5O_pline_t *pline = &(dset->shared->dcpl_cache.pline); /* I/O pipeline info */ @@ -4921,7 +5162,7 @@ H5D__chunk_allocate(const H5D_t *dset, bool full_overwrite, const hsize_t old_di bool carry; /* Flag to indicate that chunk increment carrys to higher dimension (sorta) */ unsigned space_ndims; /* Dataset's space rank */ const hsize_t *space_dim; /* Dataset's dataspace dimensions */ - const uint32_t *chunk_dim = layout->u.chunk.dim; /* Convenience pointer to chunk dimensions */ + const hsize_t *chunk_dim = layout->u.chunk.dim; /* Convenience pointer to chunk dimensions */ unsigned op_dim; /* Current operating dimension */ H5D_fill_buf_info_t fb_info; /* Dataset's fill buffer info */ bool fb_info_init = false; /* Whether the fill value buffer has been initialized */ @@ -4993,9 +5234,6 @@ H5D__chunk_allocate(const H5D_t *dset, bool full_overwrite, const hsize_t old_di unfilt_edge_chunk_dim[op_dim] = false; } /* end for */ - /* Get original chunk size */ - H5_CHECKED_ASSIGN(orig_chunk_size, size_t, layout->u.chunk.size, uint32_t); - /* Check the dataset's fill-value status */ if (H5P_is_fill_value_defined(fill, &fill_status) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't tell if fill value defined"); @@ -5014,6 +5252,13 @@ H5D__chunk_allocate(const H5D_t *dset, bool full_overwrite, const hsize_t old_di /* Check if fill values should be written to chunks */ if (should_fill) { + /* Get original chunk size */ + orig_chunk_size = (size_t)layout->u.chunk.size; + + /* Check for overflow */ + if (H5_UNLIKELY((hsize_t)orig_chunk_size != layout->u.chunk.size)) + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk size too big to fit in size_t"); + /* Initialize the fill value buffer */ /* (delay allocating fill buffer for VL datatypes until refilling) */ if (H5D__fill_init(&fb_info, NULL, H5D__chunk_mem_alloc, pline, H5D__chunk_mem_free, pline, @@ -5055,13 +5300,14 @@ H5D__chunk_allocate(const H5D_t *dset, bool full_overwrite, const hsize_t old_di if (H5Z_pipeline(pline, 0, &filter_mask, err_detect, filter_cb, &orig_chunk_size, &buf_size, &fb_info.fill_buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "output pipeline failed"); -#if H5_SIZEOF_SIZE_T > 4 - /* Check for the chunk expanding too much to encode in a 32-bit value */ - if (orig_chunk_size > ((size_t)0xffffffff)) - HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk too large for 32-bit length"); -#endif /* H5_SIZEOF_SIZE_T > 4 */ } /* end if */ - } /* end if */ + + /* Set chunk size */ + H5_CHECKED_ASSIGN(chunk_size, hsize_t, orig_chunk_size, size_t); + } /* end if */ + else + /* Set chunk size */ + chunk_size = layout->u.chunk.size; /* Compose chunked index info struct */ idx_info.f = dset->oloc.file; @@ -5089,7 +5335,6 @@ H5D__chunk_allocate(const H5D_t *dset, bool full_overwrite, const hsize_t old_di * Note that min_unalloc & max_unalloc are in scaled coordinates. * */ - chunk_size = orig_chunk_size; for (op_dim = 0; op_dim < space_ndims; op_dim++) { H5D_chunk_ud_t udata; /* User data for querying chunk info */ unsigned u; /* Local index variable */ @@ -5114,6 +5359,7 @@ H5D__chunk_allocate(const H5D_t *dset, bool full_overwrite, const hsize_t old_di if (should_fill && !fb_info.has_vlen_fill_type) { assert(fb_info_init); assert(unfilt_fill_buf); + assert(orig_chunk_size); if (nunfilt_edge_chunk_dims) { fill_buf = &unfilt_fill_buf; chunk_size = layout->u.chunk.size; @@ -5121,6 +5367,7 @@ H5D__chunk_allocate(const H5D_t *dset, bool full_overwrite, const hsize_t old_di else { fill_buf = &fb_info.fill_buf; chunk_size = orig_chunk_size; + assert(chunk_size == (size_t)orig_chunk_size); } /* end else */ } /* end if */ } /* end if */ @@ -5159,6 +5406,7 @@ H5D__chunk_allocate(const H5D_t *dset, bool full_overwrite, const hsize_t old_di /* Sanity check */ assert(should_fill); assert(!unfilt_fill_buf); + assert(orig_chunk_size); #ifdef H5_HAVE_PARALLEL assert(!using_mpi); /* Can't write VL datatypes in parallel currently */ #endif @@ -5196,14 +5444,8 @@ H5D__chunk_allocate(const H5D_t *dset, bool full_overwrite, const hsize_t old_di &fb_info.fill_buf_size, &fb_info.fill_buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "output pipeline failed"); -#if H5_SIZEOF_SIZE_T > 4 - /* Check for the chunk expanding too much to encode in a 32-bit value */ - if (nbytes > ((size_t)0xffffffff)) - HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk too large for 32-bit length"); -#endif /* H5_SIZEOF_SIZE_T > 4 */ - /* Keep the number of bytes the chunk turned in to */ - chunk_size = nbytes; + H5_CHECKED_ASSIGN(chunk_size, hsize_t, nbytes, size_t); } /* end if */ else chunk_size = layout->u.chunk.size; @@ -5216,8 +5458,8 @@ H5D__chunk_allocate(const H5D_t *dset, bool full_overwrite, const hsize_t old_di udata.common.storage = sc; udata.common.scaled = scaled; udata.chunk_block.offset = HADDR_UNDEF; - H5_CHECKED_ASSIGN(udata.chunk_block.length, uint32_t, chunk_size, size_t); - udata.filter_mask = filter_mask; + udata.chunk_block.length = chunk_size; + udata.filter_mask = filter_mask; /* Allocate the chunk (with all processes) */ if (H5D__chunk_file_alloc(&idx_info, NULL, &udata.chunk_block, &need_insert, scaled) < 0) @@ -5252,7 +5494,8 @@ H5D__chunk_allocate(const H5D_t *dset, bool full_overwrite, const hsize_t old_di /* Store info about the chunk for later */ chunk_fill_info.chunk_info[chunk_fill_info.num_chunks].addr = udata.chunk_block.offset; - chunk_fill_info.chunk_info[chunk_fill_info.num_chunks].chunk_size = chunk_size; + H5_CHECKED_ASSIGN(chunk_fill_info.chunk_info[chunk_fill_info.num_chunks].chunk_size, + size_t, chunk_size, hsize_t); chunk_fill_info.chunk_info[chunk_fill_info.num_chunks].unfiltered_partial_chunk = (*fill_buf == unfilt_fill_buf); chunk_fill_info.num_chunks++; @@ -5260,15 +5503,15 @@ H5D__chunk_allocate(const H5D_t *dset, bool full_overwrite, const hsize_t old_di /* Indicate that blocks will be written */ blocks_written = true; } /* end if */ - else { + else #endif /* H5_HAVE_PARALLEL */ + { + H5_CHECK_OVERFLOW(chunk_size, hsize_t, size_t); if (H5F_shared_block_write(H5F_SHARED(dset->oloc.file), H5FD_MEM_DRAW, - udata.chunk_block.offset, chunk_size, *fill_buf) < 0) + udata.chunk_block.offset, (size_t)chunk_size, *fill_buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file"); -#ifdef H5_HAVE_PARALLEL - } /* end else */ -#endif /* H5_HAVE_PARALLEL */ - } /* end if */ + } + } /* end if */ /* Insert the chunk record into the index */ if (need_insert && ops->insert) @@ -5292,8 +5535,9 @@ H5D__chunk_allocate(const H5D_t *dset, bool full_overwrite, const hsize_t old_di if (should_fill && nunfilt_edge_chunk_dims == 0 && !fb_info.has_vlen_fill_type) { assert( !H5D__chunk_is_partial_edge_chunk(space_ndims, chunk_dim, scaled, space_dim)); - fill_buf = &fb_info.fill_buf; - chunk_size = orig_chunk_size; + assert(orig_chunk_size); + fill_buf = &fb_info.fill_buf; + H5_CHECKED_ASSIGN(chunk_size, hsize_t, orig_chunk_size, size_t); } /* end if */ } /* end if */ } /* end if */ @@ -5375,7 +5619,7 @@ H5D__chunk_update_old_edge_chunks(H5D_t *dset, hsize_t old_dim[]) modified */ const H5O_layout_t *layout = &(dset->shared->layout); /* Dataset layout */ hsize_t chunk_sc[H5O_LAYOUT_NDIMS]; /* Offset of current chunk */ - const uint32_t *chunk_dim = layout->u.chunk.dim; /* Convenience pointer to chunk dimensions */ + const hsize_t *chunk_dim = layout->u.chunk.dim; /* Convenience pointer to chunk dimensions */ unsigned space_ndims; /* Dataset's space rank */ const hsize_t *space_dim; /* Dataset's dataspace dimensions */ unsigned op_dim; /* Current operating dimension */ @@ -5492,7 +5736,7 @@ H5D__chunk_update_old_edge_chunks(H5D_t *dset, hsize_t old_dim[]) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to lock raw data chunk"); /* Unlock the chunk */ - if (H5D__chunk_unlock(&chk_io_info, &chk_dset_info, &chk_udata, true, chunk, (uint32_t)0) < 0) + if (H5D__chunk_unlock(&chk_io_info, &chk_dset_info, &chk_udata, true, chunk, (hsize_t)0) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to unlock raw data chunk"); } /* end if */ @@ -5769,7 +6013,7 @@ H5D__chunk_prune_fill(H5D_chunk_it_ud1_t *udata, bool new_unfilt_chunk) size_t chunk_size; /*size of a chunk */ void *chunk; /* The file chunk */ H5D_chunk_ud_t chk_udata; /* User data for locking chunk */ - uint32_t bytes_accessed; /* Bytes accessed in chunk */ + hsize_t bytes_accessed; /* Bytes accessed in chunk */ unsigned u; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ @@ -5777,7 +6021,7 @@ H5D__chunk_prune_fill(H5D_chunk_it_ud1_t *udata, bool new_unfilt_chunk) /* Get the chunk's size */ assert(layout->u.chunk.size > 0); - H5_CHECKED_ASSIGN(chunk_size, size_t, layout->u.chunk.size, uint32_t); + H5_CHECKED_ASSIGN(chunk_size, size_t, layout->u.chunk.size, hsize_t); /* Get the info for the chunk in the file */ if (H5D__chunk_lookup(dset, scaled, &chk_udata) < 0) @@ -5790,7 +6034,7 @@ H5D__chunk_prune_fill(H5D_chunk_it_ud1_t *udata, bool new_unfilt_chunk) /* Initialize the fill value buffer, if necessary */ if (!udata->fb_info_init) { - H5_CHECK_OVERFLOW(udata->elmts_per_chunk, uint32_t, size_t); + H5_CHECK_OVERFLOW(udata->elmts_per_chunk, hsize_t, size_t); if (H5D__fill_init(&udata->fb_info, NULL, NULL, NULL, NULL, NULL, &dset->shared->dcpl_cache.fill, dset->shared->type, (size_t)udata->elmts_per_chunk, chunk_size) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize fill buffer info"); @@ -5835,7 +6079,8 @@ H5D__chunk_prune_fill(H5D_chunk_it_ud1_t *udata, bool new_unfilt_chunk) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "can't allocate chunk selection iterator"); /* Create a selection iterator for scattering the elements to memory buffer */ - if (H5S_select_iter_init(chunk_iter, udata->chunk_space, layout->u.chunk.dim[rank], 0) < 0) + H5_CHECK_OVERFLOW(layout->u.chunk.dim[rank], hsize_t, size_t); + if (H5S_select_iter_init(chunk_iter, udata->chunk_space, (size_t)layout->u.chunk.dim[rank], 0) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize chunk selection information"); chunk_iter_init = true; @@ -5845,8 +6090,7 @@ H5D__chunk_prune_fill(H5D_chunk_it_ud1_t *udata, bool new_unfilt_chunk) /* The number of bytes accessed in the chunk */ /* (i.e. the bytes replaced with fill values) */ - H5_CHECK_OVERFLOW(sel_nelmts, hsize_t, uint32_t); - bytes_accessed = (uint32_t)sel_nelmts * layout->u.chunk.dim[rank]; + bytes_accessed = sel_nelmts * layout->u.chunk.dim[rank]; /* Release lock on chunk */ if (H5D__chunk_unlock(io_info, udata->dset_info, &chk_udata, true, chunk, bytes_accessed) < 0) @@ -5987,7 +6231,7 @@ H5D__chunk_prune_by_extent(H5D_t *dset, const hsize_t *old_dim) hsize_t chunk_dim[H5O_LAYOUT_NDIMS]; /* Chunk dimensions */ hsize_t scaled[H5O_LAYOUT_NDIMS]; /* Scaled offset of current chunk */ hsize_t hyper_start[H5O_LAYOUT_NDIMS]; /* Starting location of hyperslab */ - uint32_t elmts_per_chunk; /* Elements in chunk */ + hsize_t elmts_per_chunk; /* Elements in chunk */ bool disable_edge_filters = false; /* Whether to disable filters on partial edge chunks */ bool new_unfilt_chunk = false; /* Whether the chunk is newly unfiltered */ unsigned u; /* Local index variable */ @@ -6570,7 +6814,7 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) FUNC_ENTER_PACKAGE /* Get 'size_t' local value for number of bytes in chunk */ - H5_CHECKED_ASSIGN(nbytes, size_t, chunk_rec->nbytes, uint32_t); + H5_CHECKED_ASSIGN(nbytes, size_t, chunk_rec->nbytes, hsize_t); /* Initialize the filter callback struct */ filter_cb.op_data = NULL; @@ -6660,7 +6904,7 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) assert(H5_addr_defined(chunk_rec->chunk_addr)); assert(H5_addr_defined(ent->chunk_block.offset)); - H5_CHECKED_ASSIGN(nbytes, size_t, shared_fo->layout.u.chunk.size, uint32_t); + H5_CHECKED_ASSIGN(nbytes, size_t, shared_fo->layout.u.chunk.size, hsize_t); H5MM_memcpy(buf, ent->chunk, nbytes); } else { @@ -6691,9 +6935,7 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) size_t reclaim_buf_size = udata->reclaim_buf_size; /* Convert from source file to memory */ - H5_CHECK_OVERFLOW(udata->nelmts, uint32_t, size_t); - if (H5T_convert(tpath_src_mem, dt_src, dt_mem, (size_t)udata->nelmts, (size_t)0, (size_t)0, buf, - bkg) < 0) + if (H5T_convert(tpath_src_mem, dt_src, dt_mem, udata->nelmts, (size_t)0, (size_t)0, buf, bkg) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, H5_ITER_ERROR, "datatype conversion failed"); /* Copy into another buffer, to reclaim memory later */ @@ -6738,12 +6980,8 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) if (H5Z_pipeline(pline, 0, &(udata_dst.filter_mask), H5Z_NO_EDC, filter_cb, &nbytes, &buf_size, &buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, H5_ITER_ERROR, "output pipeline failed"); -#if H5_SIZEOF_SIZE_T > 4 - /* Check for the chunk expanding too much to encode in a 32-bit value */ - if (nbytes > ((size_t)0xffffffff)) - HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, H5_ITER_ERROR, "chunk too large for 32-bit length"); -#endif /* H5_SIZEOF_SIZE_T > 4 */ - H5_CHECKED_ASSIGN(udata_dst.chunk_block.length, uint32_t, nbytes, size_t); + + H5_CHECKED_ASSIGN(udata_dst.chunk_block.length, hsize_t, nbytes, size_t); udata->buf = buf; udata->buf_size = buf_size; } /* end if */ @@ -6814,7 +7052,7 @@ H5D__chunk_copy(H5F_t *f_src, H5O_layout_t *layout_src, H5F_t *f_dst, H5O_layout void *bkg = NULL; /* Buffer for background during type conversion */ void *reclaim_buf = NULL; /* Buffer for reclaiming data */ H5S_t *buf_space = NULL; /* Dataspace describing buffer */ - uint32_t nelmts = 0; /* Number of elements in buffer */ + size_t nelmts = 0; /* Number of elements in buffer */ bool do_convert = false; /* Indicate that type conversions should be performed */ bool copy_setup_done = false; /* Indicate that 'copy setup' is done */ herr_t ret_value = SUCCEED; /* Return value */ @@ -6882,6 +7120,7 @@ H5D__chunk_copy(H5F_t *f_src, H5O_layout_t *layout_src, H5F_t *f_dst, H5O_layout size_t tmp_dt_size; /* Temp. datatype size */ size_t max_dt_size; /* Max atatype size */ hsize_t buf_dim; /* Dimension for buffer */ + uint64_t nelmts_64; /* nelmts as a uint64_t */ unsigned u; /* create a memory copy of the variable-length datatype */ @@ -6913,9 +7152,12 @@ H5D__chunk_copy(H5F_t *f_src, H5O_layout_t *layout_src, H5F_t *f_dst, H5O_layout max_dt_size = MAX(max_dt_size, tmp_dt_size); /* Compute the number of elements per chunk */ - nelmts = 1; + nelmts_64 = 1; for (u = 0; u < (layout_src->u.chunk.ndims - 1); u++) - nelmts *= layout_src->u.chunk.dim[u]; + nelmts_64 *= layout_src->u.chunk.dim[u]; + nelmts = (size_t)nelmts_64; + if ((uint64_t)nelmts != nelmts_64) + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "number of chunk elements too big to fit in size_t"); /* Create the space and set the initial extent */ buf_dim = nelmts; @@ -6939,7 +7181,7 @@ H5D__chunk_copy(H5F_t *f_src, H5O_layout_t *layout_src, H5F_t *f_dst, H5O_layout do_convert = true; } /* end if */ - H5_CHECKED_ASSIGN(buf_size, size_t, layout_src->u.chunk.size, uint32_t); + H5_CHECKED_ASSIGN(buf_size, size_t, layout_src->u.chunk.size, hsize_t); reclaim_buf_size = 0; } /* end else */ @@ -7144,7 +7386,7 @@ H5D__chunk_dump_index_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) } /* end if */ /* Print information about this chunk */ - fprintf(udata->stream, " 0x%08x %8" PRIu32 " %10" PRIuHADDR " [", chunk_rec->filter_mask, + fprintf(udata->stream, " 0x%08x %8" PRIuHSIZE " %10" PRIuHADDR " [", chunk_rec->filter_mask, chunk_rec->nbytes, chunk_rec->chunk_addr); for (u = 0; u < udata->ndims; u++) fprintf(udata->stream, "%s%" PRIuHSIZE, (u ? ", " : ""), @@ -7377,7 +7619,7 @@ H5D__nonexistent_readvv(const H5D_io_info_t H5_ATTR_NDEBUG_UNUSED *io_info, *------------------------------------------------------------------------- */ bool -H5D__chunk_is_partial_edge_chunk(unsigned dset_ndims, const uint32_t *chunk_dims, const hsize_t scaled[], +H5D__chunk_is_partial_edge_chunk(unsigned dset_ndims, const hsize_t *chunk_dims, const hsize_t scaled[], const hsize_t *dset_dims) { unsigned u; /* Local index variable */ @@ -7433,31 +7675,13 @@ H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info, const H5F_block_t *old /* Check for filters on chunks */ if (idx_info->pline->nused > 0) { - /* If we are using a layout version of 4 or earlier, we must make sure the filter did not create a - * chunk that's too large to have its size encoded. Version 5 always uses 64 bits to encode the chunk - * size. Single chunk works regardless of version. */ + /* The none index shouldn't be used with filtered datasets */ assert(idx_info->layout->storage.u.chunk.idx_type != H5D_CHUNK_IDX_NONE); - if (idx_info->layout->version <= H5O_LAYOUT_VERSION_4 && - idx_info->layout->storage.u.chunk.idx_type != H5D_CHUNK_IDX_SINGLE) { - unsigned allow_chunk_size_len; /* Allowed size of encoded chunk size */ - unsigned new_chunk_size_len; /* Size of encoded chunk size */ - /* Compute the size required for encoding the size of a chunk, allowing - * for an extra byte, in case the filter makes the chunk larger. - */ - allow_chunk_size_len = 1 + ((H5VM_log2_gen((uint64_t)(idx_info->layout->u.chunk.size)) + 8) / 8); - if (allow_chunk_size_len > 8) - allow_chunk_size_len = 8; - - /* Compute encoded size of chunk */ - new_chunk_size_len = (H5VM_log2_gen((uint64_t)(new_chunk->length)) + 8) / 8; - if (new_chunk_size_len > 8) - HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "encoded chunk size is more than 8 bytes?!?"); - - /* Check if the chunk became too large to be encoded */ - if (new_chunk_size_len > allow_chunk_size_len) - HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk size can't be encoded"); - } + /* Check for chunk size overflowing format limitations */ + /* Only needed for filtered datasets because the unfiltered chunk size + * was already checked in H5D__chunk_construct() */ + H5D_CHUNK_ENCODE_SIZE_CHECK(idx_info->layout, new_chunk->length, FAIL); if (old_chunk && H5_addr_defined(old_chunk->offset)) { /* Sanity check */ @@ -7512,8 +7736,7 @@ H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info, const H5F_block_t *old case H5D_CHUNK_IDX_BTREE: case H5D_CHUNK_IDX_SINGLE: assert(new_chunk->length > 0); - H5_CHECK_OVERFLOW(new_chunk->length, /*From: */ uint32_t, /*To: */ hsize_t); - new_chunk->offset = H5MF_alloc(idx_info->f, H5FD_MEM_DRAW, (hsize_t)new_chunk->length); + new_chunk->offset = H5MF_alloc(idx_info->f, H5FD_MEM_DRAW, new_chunk->length); if (!H5_addr_defined(new_chunk->offset)) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "file allocation failed"); *need_insert = true; @@ -7558,7 +7781,7 @@ H5D__chunk_format_convert_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) /* Set up */ new_idx_info = udata->new_idx_info; - H5_CHECKED_ASSIGN(nbytes, size_t, chunk_rec->nbytes, uint32_t); + H5_CHECKED_ASSIGN(nbytes, size_t, chunk_rec->nbytes, hsize_t); chunk_addr = chunk_rec->chunk_addr; if (new_idx_info->pline->nused && @@ -8125,9 +8348,8 @@ H5D__chunk_iter_cb(const H5D_chunk_rec_t *chunk_rec, void *udata) /* Prepare & restore library for user callback */ H5_BEFORE_USER_CB_NOERR(FAIL) { - ret_value = - (data->op)(offset, (unsigned)chunk_rec->filter_mask, data->base_addr + chunk_rec->chunk_addr, - (hsize_t)chunk_rec->nbytes, data->op_data); + ret_value = (data->op)(offset, (unsigned)chunk_rec->filter_mask, + data->base_addr + chunk_rec->chunk_addr, chunk_rec->nbytes, data->op_data); } H5_AFTER_USER_CB_NOERR(FAIL) diff --git a/src/H5Dcompact.c b/src/H5Dcompact.c index aa18f255586..17fc4f34779 100644 --- a/src/H5Dcompact.c +++ b/src/H5Dcompact.c @@ -58,7 +58,7 @@ typedef struct H5D_compact_iovv_memmanage_ud_t { /* Layout operation callbacks */ static herr_t H5D__compact_construct(H5F_t *f, H5D_t *dset); -static herr_t H5D__compact_init(H5F_t *f, const H5D_t *dset, hid_t dapl_id); +static herr_t H5D__compact_init(H5F_t *f, H5D_t *dset, hid_t dapl_id, bool open_op); static bool H5D__compact_is_space_alloc(const H5O_storage_t *storage); static herr_t H5D__compact_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo); static herr_t H5D__compact_iovv_memmanage_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *_udata); @@ -148,11 +148,12 @@ H5D__compact_fill(const H5D_t *dset) } /* end H5D__compact_fill() */ /*------------------------------------------------------------------------- - * Function: H5D__compact_construct + * Function: H5D__compact_construct * - * Purpose: Constructs new compact layout information for dataset + * Purpose: Constructs new compact layout information for dataset and + * upgrades layout version if appropriate * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -162,6 +163,7 @@ H5D__compact_construct(H5F_t *f, H5D_t *dset) hssize_t stmp_size; /* Temporary holder for raw data size */ hsize_t tmp_size; /* Temporary holder for raw data size */ hsize_t max_comp_data_size; /* Max. allowed size of compact data */ + unsigned version; /* Message version */ unsigned u; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ @@ -170,6 +172,7 @@ H5D__compact_construct(H5F_t *f, H5D_t *dset) /* Sanity checks */ assert(f); assert(dset); + assert(dset->shared); /* Check for invalid dataset dimensions */ for (u = 0; u < dset->shared->ndims; u++) @@ -195,6 +198,19 @@ H5D__compact_construct(H5F_t *f, H5D_t *dset) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "compact dataset size is bigger than header message maximum size"); + /* If the layout is below version 3, upgrade to version 3 if allowed. Do not upgrade past version 3 since + * there is no benefit. */ + if (dset->shared->layout.version < H5O_LAYOUT_VERSION_3) { + version = MAX(dset->shared->layout.version, + MIN(H5O_layout_ver_bounds[H5F_LOW_BOUND(f)], H5O_LAYOUT_VERSION_3)); + + /* Version bounds check */ + if (version > H5O_layout_ver_bounds[H5F_HIGH_BOUND(f)]) + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "layout version out of bounds"); + + dset->shared->layout.version = version; + } + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__compact_construct() */ @@ -210,7 +226,8 @@ H5D__compact_construct(H5F_t *f, H5D_t *dset) *------------------------------------------------------------------------- */ static herr_t -H5D__compact_init(H5F_t H5_ATTR_UNUSED *f, const H5D_t *dset, hid_t H5_ATTR_UNUSED dapl_id) +H5D__compact_init(H5F_t H5_ATTR_UNUSED *f, H5D_t *dset, hid_t H5_ATTR_UNUSED dapl_id, + bool H5_ATTR_UNUSED open_op) { hssize_t snelmts; /* Temporary holder for number of elements in dataspace */ hsize_t nelmts; /* Number of elements in dataspace */ diff --git a/src/H5Dcontig.c b/src/H5Dcontig.c index 902b512d2c2..067675915b2 100644 --- a/src/H5Dcontig.c +++ b/src/H5Dcontig.c @@ -84,7 +84,7 @@ typedef struct H5D_contig_writevv_ud_t { /* Layout operation callbacks */ static herr_t H5D__contig_construct(H5F_t *f, H5D_t *dset); -static herr_t H5D__contig_init(H5F_t *f, const H5D_t *dset, hid_t dapl_id); +static herr_t H5D__contig_init(H5F_t *f, H5D_t *dset, hid_t dapl_id, bool open_op); static herr_t H5D__contig_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo); static herr_t H5D__contig_mdio_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo); static ssize_t H5D__contig_readvv(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dinfo, @@ -416,11 +416,12 @@ H5D__contig_check(const H5F_t *f, const H5O_layout_t *layout, const H5S_extent_t } /* end H5D__contig_check() */ /*------------------------------------------------------------------------- - * Function: H5D__contig_construct + * Function: H5D__contig_construct * - * Purpose: Constructs new contiguous layout information for dataset + * Purpose: Constructs new contiguous layout information for dataset + * and upgrades layout version if appropriate * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -432,6 +433,7 @@ H5D__contig_construct(H5F_t *f, H5D_t *dset) size_t dt_size; /* Size of datatype */ hsize_t tmp_size; /* Temporary holder for raw data size */ size_t tmp_sieve_buf_size; /* Temporary holder for sieve buffer size */ + unsigned version; /* Message version */ unsigned u; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ @@ -440,6 +442,7 @@ H5D__contig_construct(H5F_t *f, H5D_t *dset) /* Sanity checks */ assert(f); assert(dset); + assert(dset->shared); /* * The maximum size of the dataset cannot exceed the storage size. @@ -482,6 +485,19 @@ H5D__contig_construct(H5F_t *f, H5D_t *dset) else dset->shared->cache.contig.sieve_buf_size = tmp_sieve_buf_size; + /* If the layout is below version 3, upgrade to version 3 if allowed. Do not upgrade past version 3 since + * there is no benefit. */ + if (dset->shared->layout.version < H5O_LAYOUT_VERSION_3) { + version = MAX(dset->shared->layout.version, + MIN(H5O_layout_ver_bounds[H5F_LOW_BOUND(f)], H5O_LAYOUT_VERSION_3)); + + /* Version bounds check */ + if (version > H5O_layout_ver_bounds[H5F_HIGH_BOUND(f)]) + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "layout version out of bounds"); + + dset->shared->layout.version = version; + } + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__contig_construct() */ @@ -497,7 +513,7 @@ H5D__contig_construct(H5F_t *f, H5D_t *dset) *------------------------------------------------------------------------- */ static herr_t -H5D__contig_init(H5F_t *f, const H5D_t *dset, hid_t H5_ATTR_UNUSED dapl_id) +H5D__contig_init(H5F_t *f, H5D_t *dset, hid_t H5_ATTR_UNUSED dapl_id, bool H5_ATTR_UNUSED open_op) { size_t tmp_sieve_buf_size; /* Temporary holder for sieve buffer size */ herr_t ret_value = SUCCEED; /* Return value */ diff --git a/src/H5Dearray.c b/src/H5Dearray.c index 2283f2bbea7..7c9d0bd1e5f 100644 --- a/src/H5Dearray.c +++ b/src/H5Dearray.c @@ -100,7 +100,7 @@ typedef struct H5D_earray_it_ud_t { /* Native extensible array element for chunks w/filters */ typedef struct H5D_earray_filt_elmt_t { haddr_t addr; /* Address of chunk */ - uint32_t nbytes; /* Size of chunk (in file) */ + hsize_t nbytes; /* Size of chunk (in file) */ uint32_t filter_mask; /* Excluded filters for chunk */ } H5D_earray_filt_elmt_t; @@ -567,8 +567,8 @@ H5D__earray_filt_debug(FILE *stream, int indent, int fwidth, hsize_t idx, const /* Print element */ snprintf(temp_str, sizeof(temp_str), "Element #%" PRIuHSIZE ":", idx); - fprintf(stream, "%*s%-*s {%" PRIuHADDR ", %u, %0x}\n", indent, "", fwidth, temp_str, elmt->addr, - elmt->nbytes, elmt->filter_mask); + fprintf(stream, "%*s%-*s {%" PRIuHADDR ", %" PRIuHSIZE ", %0x}\n", indent, "", fwidth, temp_str, + elmt->addr, elmt->nbytes, elmt->filter_mask); FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5D__earray_filt_debug() */ @@ -1096,8 +1096,8 @@ H5D__earray_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata if (idx_info->pline->nused > 0) { H5D_earray_filt_elmt_t elmt; /* Extensible array element */ - elmt.addr = udata->chunk_block.offset; - H5_CHECKED_ASSIGN(elmt.nbytes, uint32_t, udata->chunk_block.length, hsize_t); + elmt.addr = udata->chunk_block.offset; + elmt.nbytes = udata->chunk_block.length; elmt.filter_mask = udata->filter_mask; /* Set the info for the chunk */ @@ -1279,7 +1279,7 @@ H5D__earray_idx_resize(H5O_layout_chunk_t *layout) /* Get the swizzled chunk dimensions */ H5MM_memcpy(layout->u.earray.swizzled_dim, layout->dim, (layout->ndims - 1) * sizeof(layout->dim[0])); - H5VM_swizzle_coords(uint32_t, layout->u.earray.swizzled_dim, layout->u.earray.unlim_dim); + H5VM_swizzle_coords(hsize_t, layout->u.earray.swizzled_dim, layout->u.earray.unlim_dim); /* Get the swizzled number of chunks in each dimension */ H5MM_memcpy(swizzled_chunks, layout->chunks, (layout->ndims - 1) * sizeof(swizzled_chunks[0])); @@ -1496,11 +1496,9 @@ H5D__earray_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t /* Remove raw data chunk from file if not doing SWMR writes */ assert(H5_addr_defined(elmt.addr)); - if (!(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)) { - H5_CHECK_OVERFLOW(elmt.nbytes, /*From: */ uint32_t, /*To: */ hsize_t); - if (H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, elmt.addr, (hsize_t)elmt.nbytes) < 0) + if (!(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)) + if (H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, elmt.addr, elmt.nbytes) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk"); - } /* end if */ /* Reset the info about the chunk for the index */ elmt.addr = HADDR_UNDEF; @@ -1518,11 +1516,9 @@ H5D__earray_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t /* Remove raw data chunk from file if not doing SWMR writes */ assert(H5_addr_defined(addr)); - if (!(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)) { - H5_CHECK_OVERFLOW(idx_info->layout->u.chunk.size, /*From: */ uint32_t, /*To: */ hsize_t); - if (H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, addr, (hsize_t)idx_info->layout->u.chunk.size) < 0) + if (!(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)) + if (H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, addr, idx_info->layout->u.chunk.size) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk"); - } /* end if */ /* Reset the address of the chunk for the index */ addr = HADDR_UNDEF; @@ -1559,8 +1555,7 @@ H5D__earray_idx_delete_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) assert(f); /* Remove raw data chunk from file */ - H5_CHECK_OVERFLOW(chunk_rec->nbytes, /*From: */ uint32_t, /*To: */ hsize_t); - if (H5MF_xfree(f, H5FD_MEM_DRAW, chunk_rec->chunk_addr, (hsize_t)chunk_rec->nbytes) < 0) + if (H5MF_xfree(f, H5FD_MEM_DRAW, chunk_rec->chunk_addr, chunk_rec->nbytes) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, H5_ITER_ERROR, "unable to free chunk"); done: diff --git a/src/H5Defl.c b/src/H5Defl.c index 7f25e69d92c..c4314ef3e72 100644 --- a/src/H5Defl.c +++ b/src/H5Defl.c @@ -55,7 +55,7 @@ typedef struct H5D_efl_writevv_ud_t { /* Layout operation callbacks */ static herr_t H5D__efl_construct(H5F_t *f, H5D_t *dset); -static herr_t H5D__efl_init(H5F_t *f, const H5D_t *dset, hid_t dapl_id); +static herr_t H5D__efl_init(H5F_t *f, H5D_t *dset, hid_t dapl_id, bool open_op); static herr_t H5D__efl_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo); static ssize_t H5D__efl_readvv(const H5D_io_info_t *io_info, const H5D_dset_io_info_t *dset_info, size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_len_arr[], @@ -174,7 +174,7 @@ H5D__efl_construct(H5F_t *f, H5D_t *dset) *------------------------------------------------------------------------- */ static herr_t -H5D__efl_init(H5F_t H5_ATTR_UNUSED *f, const H5D_t *dset, hid_t H5_ATTR_UNUSED dapl_id) +H5D__efl_init(H5F_t H5_ATTR_UNUSED *f, H5D_t *dset, hid_t H5_ATTR_UNUSED dapl_id, bool H5_ATTR_UNUSED open_op) { size_t dt_size; /* Size of datatype */ hssize_t snelmts; /* Temporary holder for number of elements in dataspace */ diff --git a/src/H5Dfarray.c b/src/H5Dfarray.c index 1f83367e08c..3c92f11aa53 100644 --- a/src/H5Dfarray.c +++ b/src/H5Dfarray.c @@ -98,7 +98,7 @@ typedef struct H5D_farray_it_ud_t { /* Native fixed array element for chunks w/filters */ typedef struct H5D_farray_filt_elmt_t { haddr_t addr; /* Address of chunk */ - uint32_t nbytes; /* Size of chunk (in file) */ + hsize_t nbytes; /* Size of chunk (in file) */ uint32_t filter_mask; /* Excluded filters for chunk */ } H5D_farray_filt_elmt_t; @@ -636,8 +636,8 @@ H5D__farray_filt_debug(FILE *stream, int indent, int fwidth, hsize_t idx, const /* Print element */ snprintf(temp_str, sizeof(temp_str), "Element #%" PRIuHSIZE ":", idx); - fprintf(stream, "%*s%-*s {%" PRIuHADDR ", %u, %0x}\n", indent, "", fwidth, temp_str, elmt->addr, - elmt->nbytes, elmt->filter_mask); + fprintf(stream, "%*s%-*s {%" PRIuHADDR ", %" PRIuHSIZE ", %0x}\n", indent, "", fwidth, temp_str, + elmt->addr, elmt->nbytes, elmt->filter_mask); FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5D__farray_filt_debug() */ @@ -1043,8 +1043,8 @@ H5D__farray_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata if (idx_info->pline->nused > 0) { H5D_farray_filt_elmt_t elmt; /* Fixed array element */ - elmt.addr = udata->chunk_block.offset; - H5_CHECKED_ASSIGN(elmt.nbytes, uint32_t, udata->chunk_block.length, hsize_t); + elmt.addr = udata->chunk_block.offset; + elmt.nbytes = udata->chunk_block.length; elmt.filter_mask = udata->filter_mask; /* Set the info for the chunk */ @@ -1360,11 +1360,9 @@ H5D__farray_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t /* Remove raw data chunk from file if not doing SWMR writes */ assert(H5_addr_defined(elmt.addr)); - if (!(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)) { - H5_CHECK_OVERFLOW(elmt.nbytes, /*From: */ uint32_t, /*To: */ hsize_t); - if (H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, elmt.addr, (hsize_t)elmt.nbytes) < 0) + if (!(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)) + if (H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, elmt.addr, elmt.nbytes) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk"); - } /* end if */ /* Reset the info about the chunk for the index */ elmt.addr = HADDR_UNDEF; @@ -1382,11 +1380,9 @@ H5D__farray_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t /* Remove raw data chunk from file if not doing SWMR writes */ assert(H5_addr_defined(addr)); - if (!(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)) { - H5_CHECK_OVERFLOW(idx_info->layout->u.chunk.size, /*From: */ uint32_t, /*To: */ hsize_t); - if (H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, addr, (hsize_t)idx_info->layout->u.chunk.size) < 0) + if (!(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)) + if (H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, addr, idx_info->layout->u.chunk.size) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk"); - } /* end if */ /* Reset the address of the chunk for the index */ addr = HADDR_UNDEF; @@ -1423,8 +1419,7 @@ H5D__farray_idx_delete_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) assert(f); /* Remove raw data chunk from file */ - H5_CHECK_OVERFLOW(chunk_rec->nbytes, /*From: */ uint32_t, /*To: */ hsize_t); - if (H5MF_xfree(f, H5FD_MEM_DRAW, chunk_rec->chunk_addr, (hsize_t)chunk_rec->nbytes) < 0) + if (H5MF_xfree(f, H5FD_MEM_DRAW, chunk_rec->chunk_addr, chunk_rec->nbytes) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, H5_ITER_ERROR, "unable to free chunk"); done: diff --git a/src/H5Dint.c b/src/H5Dint.c index 0bcd9249586..6ab3de39037 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -1326,16 +1326,6 @@ H5D__create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id, hid_t if (H5O_fill_set_version(file, &new_dset->shared->dcpl_cache.fill) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set latest version of fill value"); - /* Set the latest version for the layout message */ - if (H5D__layout_set_version(file, &new_dset->shared->layout) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set latest version of layout"); - - if (new_dset->shared->layout.version >= H5O_LAYOUT_VERSION_4) { - /* Use latest indexing type for layout message version >= 4 */ - if (H5D__layout_set_latest_indexing(new_dset) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set latest indexing"); - } /* end if */ - /* Check if the file driver would like to force early space allocation */ if (H5F_HAS_FEATURE(file, H5FD_FEAT_ALLOCATE_EARLY)) new_dset->shared->dcpl_cache.fill.alloc_time = H5D_ALLOC_TIME_EARLY; @@ -1358,6 +1348,10 @@ H5D__create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id, hid_t (new_dset->shared->layout.ops->construct)(file, new_dset) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to construct layout information"); + /* Check if the layout version is above the high bound for the file */ + if (new_dset->shared->layout.version > H5O_layout_ver_bounds[H5F_HIGH_BOUND(file)]) + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, NULL, "layout version out of bounds"); + /* Update the dataset's object header info. */ if (H5D__update_oh_info(file, new_dset, new_dset->shared->dapl_id) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "can't update the metadata cache"); diff --git a/src/H5Dio.c b/src/H5Dio.c index dc930eaf0f6..637aa5bae1f 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -1193,9 +1193,15 @@ H5D__typeinfo_init_phase2(H5D_io_info_t *io_info) /* Check for background buffer */ if (type_info->need_bkg) { - /* Add size of this dataset's background buffer to the global background buffer size - */ - io_info->bkg_buf_size += io_info->dsets_info[i].nelmts * type_info->dst_type_size; + hsize_t bkg_buf_hsize; + + /* Add size of this dataset's background buffer to the global background buffer size. Make + * sure to check for overflow and disable selection I/O if it happens. */ + H5_CHECKED_ASSIGN(bkg_buf_hsize, hsize_t, io_info->bkg_buf_size, size_t); + bkg_buf_hsize += io_info->dsets_info[i].nelmts * type_info->dst_type_size; + io_info->bkg_buf_size = (size_t)bkg_buf_hsize; + if (H5_UNLIKELY((hsize_t)io_info->bkg_buf_size != bkg_buf_hsize)) + io_info->tconv_buf_overflow = true; /* Check if we need to fill the background buffer with the destination contents */ if (type_info->need_bkg == H5T_BKG_YES) @@ -1208,14 +1214,14 @@ H5D__typeinfo_init_phase2(H5D_io_info_t *io_info) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't retrieve max. temp. buf size"); /* Check if the needed type conversion or background buffer size is too big */ - if (io_info->tconv_buf_size > max_temp_buf) { + if (io_info->tconv_buf_size > max_temp_buf || io_info->tconv_buf_overflow) { io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; io_info->no_selection_io_cause |= H5D_SEL_IO_TCONV_BUF_TOO_SMALL; io_info->tconv_buf_size = 0; io_info->bkg_buf_size = 0; io_info->must_fill_bkg = false; } - if (io_info->bkg_buf_size > max_temp_buf) { + if (io_info->bkg_buf_size > max_temp_buf || io_info->tconv_buf_overflow) { io_info->use_select_io = H5D_SELECTION_IO_MODE_OFF; io_info->no_selection_io_cause |= H5D_SEL_IO_BKG_BUF_TOO_SMALL; io_info->tconv_buf_size = 0; diff --git a/src/H5Dlayout.c b/src/H5Dlayout.c index 67d2732873e..7bdb0d9eb7a 100644 --- a/src/H5Dlayout.c +++ b/src/H5Dlayout.c @@ -271,191 +271,6 @@ H5D__layout_meta_size(const H5F_t *f, const H5O_layout_t *layout, bool include_c FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__layout_meta_size() */ -/*------------------------------------------------------------------------- - * Function: H5D__layout_set_version - * - * Purpose: Set the version to encode a layout with. - * - * Return: Non-negative on success/Negative on failure - *------------------------------------------------------------------------- - */ -herr_t -H5D__layout_set_version(H5F_t *f, H5O_layout_t *layout) -{ - unsigned version; /* Message version */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Sanity check */ - assert(layout); - assert(f); - - /* Upgrade to the version indicated by the file's low bound if higher */ - /* This will be downgraded in H5D__layout_set_latest_indexing() if there is no benefit to the newer - * version */ - version = MAX(layout->version, H5O_layout_ver_bounds[H5F_LOW_BOUND(f)]); - - /* Version bounds check */ - if (version > H5O_layout_ver_bounds[H5F_HIGH_BOUND(f)]) - HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "layout version out of bounds"); - - /* Set the message version */ - layout->version = version; - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__layout_set_version() */ - -/*------------------------------------------------------------------------- - * Function: H5D__layout_set_latest_indexing - * - * Purpose: Set the latest indexing type for a layout message - * - * Return: Non-negative on success/Negative on failure - * - *------------------------------------------------------------------------- - */ -herr_t -H5D__layout_set_latest_indexing(H5D_t *dset) -{ - H5O_layout_t *layout; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* Sanity check */ - assert(dset); - assert(dset->shared); - assert(dset->shared->space); - - /* Set convenience pointer */ - layout = &dset->shared->layout; - - /* The indexing methods only apply to chunked datasets (currently) */ - if (layout->type == H5D_CHUNKED) { - int sndims; /* Rank of dataspace */ - unsigned ndims; /* Rank of dataspace */ - - /* Query the dimensionality of the dataspace */ - if ((sndims = H5S_GET_EXTENT_NDIMS(dset->shared->space)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "invalid dataspace rank"); - ndims = (unsigned)sndims; - - /* Avoid scalar/null dataspace */ - if (ndims > 0) { - hsize_t max_dims[H5O_LAYOUT_NDIMS]; /* Maximum dimension sizes */ - hsize_t cur_dims[H5O_LAYOUT_NDIMS]; /* Current dimension sizes */ - unsigned unlim_count = 0; /* Count of unlimited max. dimensions */ - bool single = true; /* Fulfill single chunk indexing */ - unsigned u; /* Local index variable */ - - /* Query the dataspace's dimensions */ - if (H5S_get_simple_extent_dims(dset->shared->space, cur_dims, max_dims) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataspace max. dimensions"); - - /* Spin through the max. dimensions, looking for unlimited dimensions */ - for (u = 0; u < ndims; u++) { - if (max_dims[u] == H5S_UNLIMITED) - unlim_count++; - if (cur_dims[u] != max_dims[u] || cur_dims[u] != layout->u.chunk.dim[u]) - single = false; - } - - /* Chunked datasets with unlimited dimension(s) */ - if (unlim_count) { /* dataset with unlimited dimension(s) must be chunked */ - if (1 == unlim_count) { /* Chunked dataset with only 1 unlimited dimension */ - /* Set the chunk index type to an extensible array */ - layout->u.chunk.idx_type = H5D_CHUNK_IDX_EARRAY; - layout->storage.u.chunk.idx_type = H5D_CHUNK_IDX_EARRAY; - layout->storage.u.chunk.ops = H5D_COPS_EARRAY; - - /* Set the extensible array creation parameters */ - /* (use hard-coded defaults for now, until we give applications - * control over this with a property list - QAK) - */ - layout->u.chunk.u.earray.cparam.max_nelmts_bits = H5D_EARRAY_MAX_NELMTS_BITS; - layout->u.chunk.u.earray.cparam.idx_blk_elmts = H5D_EARRAY_IDX_BLK_ELMTS; - layout->u.chunk.u.earray.cparam.sup_blk_min_data_ptrs = H5D_EARRAY_SUP_BLK_MIN_DATA_PTRS; - layout->u.chunk.u.earray.cparam.data_blk_min_elmts = H5D_EARRAY_DATA_BLK_MIN_ELMTS; - layout->u.chunk.u.earray.cparam.max_dblk_page_nelmts_bits = - H5D_EARRAY_MAX_DBLOCK_PAGE_NELMTS_BITS; - - /* If there are no filters, downgrade version to 4 since version 5 doesn't improve - * anything */ - if (!dset->shared->dcpl_cache.pline.nused) - layout->version = H5O_LAYOUT_VERSION_4; - } - else { /* Chunked dataset with > 1 unlimited dimensions */ - /* Set the chunk index type to v2 B-tree */ - layout->u.chunk.idx_type = H5D_CHUNK_IDX_BT2; - layout->storage.u.chunk.idx_type = H5D_CHUNK_IDX_BT2; - layout->storage.u.chunk.ops = H5D_COPS_BT2; - - /* Set the v2 B-tree creation parameters */ - /* (use hard-coded defaults for now, until we give applications - * control over this with a property list - QAK) - */ - layout->u.chunk.u.btree2.cparam.node_size = H5D_BT2_NODE_SIZE; - layout->u.chunk.u.btree2.cparam.split_percent = H5D_BT2_SPLIT_PERC; - layout->u.chunk.u.btree2.cparam.merge_percent = H5D_BT2_MERGE_PERC; - - /* If there are no filters, downgrade version to 4 since version 5 doesn't improve - * anything */ - if (!dset->shared->dcpl_cache.pline.nused) - layout->version = H5O_LAYOUT_VERSION_4; - } - } - else { /* Chunked dataset with fixed dimensions */ - /* Check for correct condition for using "single chunk" chunk index */ - if (single) { - layout->u.chunk.idx_type = H5D_CHUNK_IDX_SINGLE; - layout->storage.u.chunk.idx_type = H5D_CHUNK_IDX_SINGLE; - layout->storage.u.chunk.ops = H5D_COPS_SINGLE; - - /* Downgrade version to 4 since version 5 doesn't improve anything */ - layout->version = H5O_LAYOUT_VERSION_4; - } - else if (!dset->shared->dcpl_cache.pline.nused && - dset->shared->dcpl_cache.fill.alloc_time == H5D_ALLOC_TIME_EARLY) { - - /* Set the chunk index type to "none" Index */ - layout->u.chunk.idx_type = H5D_CHUNK_IDX_NONE; - layout->storage.u.chunk.idx_type = H5D_CHUNK_IDX_NONE; - layout->storage.u.chunk.ops = H5D_COPS_NONE; - - /* Downgrade version to 4 since version 5 doesn't improve anything */ - layout->version = H5O_LAYOUT_VERSION_4; - } - else { - /* Set the chunk index type to Fixed Array */ - layout->u.chunk.idx_type = H5D_CHUNK_IDX_FARRAY; - layout->storage.u.chunk.idx_type = H5D_CHUNK_IDX_FARRAY; - layout->storage.u.chunk.ops = H5D_COPS_FARRAY; - - /* Set the fixed array creation parameters */ - /* (use hard-coded defaults for now, until we give applications - * control over this with a property list - QAK) - */ - layout->u.chunk.u.farray.cparam.max_dblk_page_nelmts_bits = - H5D_FARRAY_MAX_DBLK_PAGE_NELMTS_BITS; - - /* If there are no filters, downgrade version to 4 since version 5 doesn't improve - * anything */ - if (!dset->shared->dcpl_cache.pline.nused) - layout->version = H5O_LAYOUT_VERSION_4; - } - } - } - else - /* Rank 0 -> v1 b-tree. Downgrade version to 4 since version 5 doesn't improve anything */ - layout->version = H5O_LAYOUT_VERSION_4; - } - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__layout_set_latest_indexing() */ - /*------------------------------------------------------------------------- * Function: H5D__layout_oh_create * @@ -497,7 +312,7 @@ H5D__layout_oh_create(H5F_t *file, H5O_t *oh, H5D_t *dset, hid_t dapl_id) } /* end if */ /* Initialize the layout information for the new dataset */ - if (dset->shared->layout.ops->init && (dset->shared->layout.ops->init)(file, dset, dapl_id) < 0) + if (dset->shared->layout.ops->init && (dset->shared->layout.ops->init)(file, dset, dapl_id, false) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize layout information"); /* Indicate that the layout information was initialized */ @@ -658,7 +473,7 @@ H5D__layout_oh_read(H5D_t *dataset, hid_t dapl_id, H5P_genplist_t *plist) /* Initialize the layout information for the dataset */ if (dataset->shared->layout.ops->init && - (dataset->shared->layout.ops->init)(dataset->oloc.file, dataset, dapl_id) < 0) + (dataset->shared->layout.ops->init)(dataset->oloc.file, dataset, dapl_id, true) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize layout information"); #ifndef NDEBUG @@ -673,11 +488,6 @@ H5D__layout_oh_read(H5D_t *dataset, hid_t dapl_id, H5P_genplist_t *plist) HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "unable to setup placeholder layout"); #endif - /* Set chunk sizes */ - if (H5D_CHUNKED == dataset->shared->layout.type) - if (H5D__chunk_set_sizes(dataset) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "unable to set chunk sizes"); - done: if (ret_value < 0) { if (pline_copied) diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c index ded4686b8a8..7fcaeadc613 100644 --- a/src/H5Dmpio.c +++ b/src/H5Dmpio.c @@ -3267,7 +3267,7 @@ H5D__mpio_collective_filtered_chunk_io_setup(const H5D_io_info_t *io_info, const assert(di[dset_idx].dset->shared->ndims == di[dset_idx].dset->shared->layout.u.chunk.ndims - 1); for (size_t dim_idx = 0; dim_idx < di[dset_idx].dset->shared->layout.u.chunk.ndims - 1; dim_idx++) - chunk_dims[dim_idx] = (hsize_t)di[dset_idx].dset->shared->layout.u.chunk.dim[dim_idx]; + chunk_dims[dim_idx] = di[dset_idx].dset->shared->layout.u.chunk.dim[dim_idx]; /* Get a dataspace for filling chunk memory buffers */ if (NULL == (curr_dset_info->fill_space = H5S_create_simple( diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 08429e5890f..e28199cfe48 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -105,10 +105,20 @@ } \ \ /* If we're not using in-place type conversion, add this piece to global type conversion buffer \ - * size. This will only be used if we must allocate a type conversion buffer for the entire I/O. */ \ - if (!(PIECE_INFO)->in_place_tconv) \ - (IO_INFO)->tconv_buf_size += (PIECE_INFO)->piece_points * MAX((DINFO)->type_info.src_type_size, \ - (DINFO)->type_info.dst_type_size); \ + * size. This will only be used if we must allocate a type conversion buffer for the entire I/O. \ + * Make sure to check for overflow and disable selection I/O if it happens. */ \ + if (!(PIECE_INFO)->in_place_tconv) { \ + hsize_t tconv_buf_hsize; \ + hsize_t old_size; \ + H5_CHECKED_ASSIGN(tconv_buf_hsize, hsize_t, (IO_INFO)->tconv_buf_size, size_t); \ + old_size = tconv_buf_hsize; \ + tconv_buf_hsize += (PIECE_INFO)->piece_points * \ + MAX((DINFO)->type_info.src_type_size, (DINFO)->type_info.dst_type_size); \ + if (H5_UNLIKELY(tconv_buf_hsize < old_size || tconv_buf_hsize > SIZE_MAX)) \ + (IO_INFO)->tconv_buf_overflow = true; \ + else \ + (IO_INFO)->tconv_buf_size = (size_t)tconv_buf_hsize; \ + } \ } /* Macro to add a virtual dataset source file or dataset name to a hash table for storing these names */ @@ -178,7 +188,7 @@ typedef struct H5D_shared_t H5D_shared_t; /* Function pointers for I/O on particular types of dataset layouts */ typedef herr_t (*H5D_layout_construct_func_t)(H5F_t *f, H5D_t *dset); -typedef herr_t (*H5D_layout_init_func_t)(H5F_t *f, const H5D_t *dset, hid_t dapl_id); +typedef herr_t (*H5D_layout_init_func_t)(H5F_t *f, H5D_t *dset, hid_t dapl_id, bool open_op); typedef bool (*H5D_layout_is_space_alloc_func_t)(const H5O_storage_t *storage); typedef bool (*H5D_layout_is_data_cached_func_t)(const H5D_shared_t *shared_dset); typedef herr_t (*H5D_layout_io_init_func_t)(struct H5D_io_info_t *io_info, struct H5D_dset_io_info_t *dinfo); @@ -352,6 +362,7 @@ typedef struct H5D_io_info_t { uint8_t *bkg_buf; /* Background buffer */ bool bkg_buf_allocated; /* Whether the background buffer was allocated */ size_t bkg_buf_size; /* Size of background buffer */ + bool tconv_buf_overflow; /* Whether the tconv or bkg buf overflowed size_t */ size_t max_tconv_type_size; /* Largest of all source and destination type sizes involved in type conversion */ bool must_fill_bkg; /* Whether any datasets need a background buffer filled with destination contents */ @@ -393,7 +404,7 @@ typedef struct H5D_chk_idx_info_t { */ typedef struct H5D_chunk_rec_t { hsize_t scaled[H5O_LAYOUT_NDIMS]; /* Logical offset to start */ - uint32_t nbytes; /* Size of stored data */ + hsize_t nbytes; /* Size of stored data */ uint32_t filter_mask; /* Excluded filters */ haddr_t chunk_addr; /* Address of chunk in file */ } H5D_chunk_rec_t; @@ -505,7 +516,7 @@ typedef struct H5D_chunk_cached_t { bool valid; /*whether cache info is valid*/ hsize_t scaled[H5O_LAYOUT_NDIMS]; /*scaled offset of chunk*/ haddr_t addr; /*file address of chunk */ - uint32_t nbytes; /*size of stored data */ + hsize_t nbytes; /*size of stored data */ hsize_t chunk_idx; /*index of chunk in dataset */ unsigned filter_mask; /*excluded filters */ } H5D_chunk_cached_t; @@ -732,8 +743,6 @@ H5_DLL herr_t H5D__scatgath_write_select(H5D_io_info_t *io_info); /* Functions that operate on dataset's layout information */ H5_DLL herr_t H5D__layout_set_io_ops(const H5D_t *dataset); H5_DLL size_t H5D__layout_meta_size(const H5F_t *f, const H5O_layout_t *layout, bool include_compact_data); -H5_DLL herr_t H5D__layout_set_version(H5F_t *f, H5O_layout_t *layout); -H5_DLL herr_t H5D__layout_set_latest_indexing(H5D_t *dset); H5_DLL herr_t H5D__layout_oh_create(H5F_t *file, H5O_t *oh, H5D_t *dset, hid_t dapl_id); H5_DLL herr_t H5D__layout_oh_read(H5D_t *dset, hid_t dapl_id, H5P_genplist_t *plist); H5_DLL herr_t H5D__layout_oh_write(const H5D_t *dataset, H5O_t *oh, unsigned update_flags); @@ -768,10 +777,9 @@ H5_DLL void H5D__chunk_mem_free(void *chk, void *pline); H5_DLL void *H5D__chunk_mem_xfree(void *chk, const void *pline); H5_DLL void *H5D__chunk_mem_realloc(void *chk, size_t size, const H5O_pline_t *pline); H5_DLL herr_t H5D__chunk_update_old_edge_chunks(H5D_t *dset, hsize_t old_dim[]); -H5_DLL bool H5D__chunk_is_partial_edge_chunk(unsigned dset_ndims, const uint32_t *chunk_dims, +H5_DLL bool H5D__chunk_is_partial_edge_chunk(unsigned dset_ndims, const hsize_t *chunk_dims, const hsize_t *chunk_scaled, const hsize_t *dset_dims); H5_DLL herr_t H5D__chunk_prune_by_extent(H5D_t *dset, const hsize_t *old_dim); -H5_DLL herr_t H5D__chunk_set_sizes(H5D_t *dset); #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5D__chunk_addrmap(const H5D_t *dset, haddr_t chunk_addr[]); #endif /* H5_HAVE_PARALLEL */ @@ -783,7 +791,7 @@ H5_DLL herr_t H5D__chunk_bh_info(const H5O_loc_t *loc, H5O_t *oh, H5O_layout_t * H5_DLL herr_t H5D__chunk_dump_index(H5D_t *dset, FILE *stream); H5_DLL herr_t H5D__chunk_delete(H5F_t *f, H5O_t *oh, H5O_layout_t *layout); H5_DLL herr_t H5D__chunk_get_offset_copy(const H5D_t *dset, const hsize_t *offset, hsize_t *offset_copy); -H5_DLL herr_t H5D__chunk_direct_write(H5D_t *dset, uint32_t filters, hsize_t *offset, uint32_t data_size, +H5_DLL herr_t H5D__chunk_direct_write(H5D_t *dset, uint32_t filters, hsize_t *offset, size_t data_size, const void *buf); H5_DLL herr_t H5D__chunk_direct_read(const H5D_t *dset, hsize_t *offset, uint32_t *filters, void *buf, size_t *nalloc); @@ -808,8 +816,6 @@ H5_DLL herr_t H5D__virtual_set_extent_unlim(const H5D_t *dset); H5_DLL herr_t H5D__virtual_reset_layout(H5O_layout_t *layout); H5_DLL herr_t H5D__virtual_delete(H5F_t *f, H5O_storage_t *storage); H5_DLL herr_t H5D__virtual_copy(H5F_t *f_src, H5O_layout_t *layout_dst); -H5_DLL herr_t H5D__virtual_init(H5F_t *f, const H5D_t *dset, hid_t dapl_id); -H5_DLL bool H5D__virtual_is_space_alloc(const H5O_storage_t *storage); H5_DLL herr_t H5D__virtual_hold_source_dset_files(const H5D_t *dset, H5D_virtual_held_file_t **head); H5_DLL herr_t H5D__virtual_refresh_source_dsets(H5D_t *dset); H5_DLL herr_t H5D__virtual_release_source_dset_files(H5D_virtual_held_file_t *head); diff --git a/src/H5Dsingle.c b/src/H5Dsingle.c index f4bda7c0030..994270bacc1 100644 --- a/src/H5Dsingle.c +++ b/src/H5Dsingle.c @@ -281,8 +281,7 @@ H5D__single_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata idx_info->layout->storage.u.chunk.idx_addr = udata->chunk_block.offset; if (idx_info->pline->nused > 0) { - H5_CHECKED_ASSIGN(idx_info->layout->storage.u.chunk.u.single.nbytes, uint32_t, - udata->chunk_block.length, hsize_t); + idx_info->layout->storage.u.chunk.u.single.nbytes = udata->chunk_block.length; idx_info->layout->storage.u.chunk.u.single.filter_mask = udata->filter_mask; } /* end if */ diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 9322ec2b120..7c925b889d2 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -98,6 +98,9 @@ /********************/ /* Layout operation callbacks */ +static herr_t H5D__virtual_construct(H5F_t *f, H5D_t *dset); +static herr_t H5D__virtual_init(H5F_t *f, H5D_t *dset, hid_t dapl_id, bool open_op); +static bool H5D__virtual_is_space_alloc(const H5O_storage_t *storage); static bool H5D__virtual_is_data_cached(const H5D_shared_t *shared_dset); static herr_t H5D__virtual_io_init(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo); static herr_t H5D__virtual_read(H5D_io_info_t *io_info, H5D_dset_io_info_t *dinfo); @@ -149,7 +152,7 @@ static herr_t H5D__virtual_not_in_tree_add(H5O_storage_virtual_ent_t ***list, si /* Contiguous storage layout I/O ops */ const H5D_layout_ops_t H5D_LOPS_VIRTUAL[1] = {{ - NULL, /* construct */ + H5D__virtual_construct, /* construct */ H5D__virtual_init, /* init */ H5D__virtual_is_space_alloc, /* is_space_alloc */ H5D__virtual_is_data_cached, /* is_data_cached */ @@ -2647,6 +2650,46 @@ H5D__virtual_init_all(const H5D_t *dset) FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__virtual_init_all() */ +/*------------------------------------------------------------------------- + * Function: H5D__virtual_construct + * + * Purpose: Constructs new virtual layout information for dataset and + * upgrades layout version if appropriate + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_construct(H5F_t *f, H5D_t *dset) +{ + unsigned version; /* Message version */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity checks */ + assert(f); + assert(dset); + assert(dset->shared); + + /* Currently only handles layout version */ + /* If the layout is below version 4, upgrade to version 4 if allowed. If not allowed throw an error, since + * virtual datasets require layout version 4. Do not upgrade past version 3 since there is no benefit. */ + if (dset->shared->layout.version < H5O_LAYOUT_VERSION_4) { + version = MAX(dset->shared->layout.version, H5O_LAYOUT_VERSION_4); + + /* Version bounds check */ + if (version > H5O_layout_ver_bounds[H5F_HIGH_BOUND(f)]) + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "layout version out of bounds"); + + dset->shared->layout.version = version; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_construct() */ + /*------------------------------------------------------------------------- * Function: H5D__virtual_init * @@ -2657,8 +2700,8 @@ H5D__virtual_init_all(const H5D_t *dset) * *------------------------------------------------------------------------- */ -herr_t -H5D__virtual_init(H5F_t *f, const H5D_t *dset, hid_t dapl_id) +static herr_t +H5D__virtual_init(H5F_t *f, H5D_t *dset, hid_t dapl_id, bool H5_ATTR_UNUSED open_op) { H5O_storage_virtual_t *storage; /* Convenience pointer */ H5P_genplist_t *dapl; /* Data access property list object pointer */ @@ -2781,7 +2824,7 @@ H5D__virtual_init(H5F_t *f, const H5D_t *dset, hid_t dapl_id) * *------------------------------------------------------------------------- */ -bool +static bool H5D__virtual_is_space_alloc(const H5O_storage_t H5_ATTR_UNUSED *storage) { bool ret_value = false; /* Return value */ diff --git a/src/H5Olayout.c b/src/H5Olayout.c index 2e44712be44..d230feb9925 100644 --- a/src/H5Olayout.c +++ b/src/H5Olayout.c @@ -187,7 +187,7 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU if (mesg->u.chunk.dim[u] == 0) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "bad chunk dimension value when parsing layout message - chunk dimension " - "must be positive: mesg->u.chunk.dim[%u] = %u", + "must be positive: mesg->u.chunk.dim[%u] = %" PRIuHSIZE, u, mesg->u.chunk.dim[u]); } @@ -299,7 +299,7 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU if (mesg->u.chunk.dim[u] == 0) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "bad chunk dimension value when parsing layout message - chunk " - "dimension must be positive: mesg->u.chunk.dim[%u] = %u", + "dimension must be positive: mesg->u.chunk.dim[%u] = %" PRIuHSIZE, u, mesg->u.chunk.dim[u]); } @@ -360,7 +360,7 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU if (mesg->u.chunk.dim[u] == 0) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "bad chunk dimension value when parsing layout message - chunk " - "dimension must be positive: mesg->u.chunk.dim[%u] = %u", + "dimension must be positive: mesg->u.chunk.dim[%u] = %" PRIuHSIZE, u, mesg->u.chunk.dim[u]); } @@ -1144,7 +1144,7 @@ H5O__layout_debug(H5F_t H5_ATTR_UNUSED *f, const void *_mesg, FILE *stream, int "Number of dimensions:", (unsigned long)(mesg->u.chunk.ndims)); fprintf(stream, "%*s%-*s {", indent, "", fwidth, "Size:"); for (u = 0; u < (size_t)mesg->u.chunk.ndims; u++) - fprintf(stream, "%s%lu", u ? ", " : "", (unsigned long)(mesg->u.chunk.dim[u])); + fprintf(stream, "%s%" PRIuHSIZE, u ? ", " : "", mesg->u.chunk.dim[u]); fprintf(stream, "}\n"); /* Index information */ diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index f5bffa61fe7..12aadb3cba4 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -486,7 +486,7 @@ typedef struct H5O_storage_chunk_earray_t { /* Filtered info for single chunk index */ typedef struct H5O_storage_chunk_single_filt_t { - uint32_t nbytes; /* Size of chunk (in file) */ + hsize_t nbytes; /* Size of chunk (in file) */ uint32_t filter_mask; /* Excluded filters for chunk */ } H5O_storage_chunk_single_filt_t; @@ -650,7 +650,7 @@ typedef struct H5O_layout_chunk_earray_t { } cparam; unsigned unlim_dim; /* Rank of unlimited dimension for dataset */ - uint32_t swizzled_dim[H5O_LAYOUT_NDIMS]; /* swizzled chunk dimensions */ + hsize_t swizzled_dim[H5O_LAYOUT_NDIMS]; /* swizzled chunk dimensions */ hsize_t swizzled_down_chunks[H5O_LAYOUT_NDIMS]; /* swizzled "down" size of number of chunks in each dimension */ hsize_t swizzled_max_down_chunks[H5O_LAYOUT_NDIMS]; /* swizzled max "down" size of number of chunks in @@ -670,9 +670,9 @@ typedef struct H5O_layout_chunk_t { H5D_chunk_index_t idx_type; /* Type of chunk index */ uint8_t flags; /* Chunk layout flags */ unsigned ndims; /* Num dimensions in chunk */ - uint32_t dim[H5O_LAYOUT_NDIMS]; /* Size of chunk in elements */ + hsize_t dim[H5O_LAYOUT_NDIMS]; /* Size of chunk in elements */ unsigned enc_bytes_per_dim; /* Encoded # of bytes for storing each chunk dimension */ - uint32_t size; /* Size of chunk in bytes */ + hsize_t size; /* Size of chunk in bytes */ hsize_t nchunks; /* Number of chunks in dataset */ hsize_t max_nchunks; /* Max. number of chunks in dataset */ hsize_t chunks[H5O_LAYOUT_NDIMS]; /* # of chunks in each dataset dimension */ diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c index 0e0689e2df4..2317c317f73 100644 --- a/src/H5Pdcpl.c +++ b/src/H5Pdcpl.c @@ -142,16 +142,19 @@ /* ======== Dataset creation properties ======== */ /* Definitions for storage layout property */ -#define H5D_CRT_LAYOUT_SIZE sizeof(H5O_layout_t) -#define H5D_CRT_LAYOUT_DEF H5D_DEF_LAYOUT_CONTIG -#define H5D_CRT_LAYOUT_SET H5P__dcrt_layout_set -#define H5D_CRT_LAYOUT_GET H5P__dcrt_layout_get -#define H5D_CRT_LAYOUT_ENC H5P__dcrt_layout_enc -#define H5D_CRT_LAYOUT_DEC H5P__dcrt_layout_dec -#define H5D_CRT_LAYOUT_DEL H5P__dcrt_layout_del -#define H5D_CRT_LAYOUT_COPY H5P__dcrt_layout_copy -#define H5D_CRT_LAYOUT_CMP H5P__dcrt_layout_cmp -#define H5D_CRT_LAYOUT_CLOSE H5P__dcrt_layout_close +#define H5D_CRT_LAYOUT_SIZE sizeof(H5O_layout_t) +#define H5D_CRT_LAYOUT_DEF H5D_DEF_LAYOUT_CONTIG +#define H5D_CRT_LAYOUT_SET H5P__dcrt_layout_set +#define H5D_CRT_LAYOUT_GET H5P__dcrt_layout_get +#define H5D_CRT_LAYOUT_ENC H5P__dcrt_layout_enc +#define H5D_CRT_LAYOUT_DEC H5P__dcrt_layout_dec +#define H5D_CRT_LAYOUT_DEL H5P__dcrt_layout_del +#define H5D_CRT_LAYOUT_COPY H5P__dcrt_layout_copy +#define H5D_CRT_LAYOUT_CMP H5P__dcrt_layout_cmp +#define H5D_CRT_LAYOUT_CLOSE H5P__dcrt_layout_close +#define H5D_CRT_LAYOUT_VERSION_0 0 +#define H5D_CRT_LAYOUT_VERSION_1 1 +#define H5D_CRT_LAYOUT_MAGIC_TYPE (uint8_t)0xff /* Definitions for fill value. size=0 means fill value will be 0 as * library default; size=-1 means fill value is undefined. */ #define H5D_CRT_FILL_VALUE_SIZE sizeof(H5O_fill_t) @@ -425,6 +428,9 @@ H5P__dcrt_layout_enc(const void *value, void **_pp, size_t *size) uint8_t **pp = (uint8_t **)_pp; uint8_t *tmp_p; size_t tmp_size; + unsigned version = H5D_CRT_LAYOUT_VERSION_0; + H5F_libver_t low_bound; + H5F_libver_t high_bound; size_t u; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ @@ -434,7 +440,29 @@ H5P__dcrt_layout_enc(const void *value, void **_pp, size_t *size) assert(layout); assert(size); + /* Get the file's low_bound and high_bound */ + if (H5CX_get_libver_bounds(&low_bound, &high_bound) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get low/high bounds from API context"); + + /* Determine version - only upgrade for chunked datasets that have a dimension size >32 bits */ + if (H5D_CHUNKED == layout->type) + for (u = 0; u < (size_t)layout->u.chunk.ndims; u++) + if (layout->u.chunk.dim[u] > (hsize_t)0xffffffff) { + if (high_bound < H5F_LIBVER_V200) + HGOTO_ERROR(H5E_PLIST, H5E_CANTENCODE, FAIL, + "can't encode chunk dimensions >= 2^32 with old encoding format - see " + "H5Pset_libver_bounds()"); + version = H5D_CRT_LAYOUT_VERSION_1; + } + if (NULL != *pp) { + /* If using version 1 or above, encode a magic value (0xff) instead of the type, then encode the + * version */ + if (version > H5D_CRT_LAYOUT_VERSION_0) { + *(*pp)++ = H5D_CRT_LAYOUT_MAGIC_TYPE; + *(*pp)++ = (uint8_t)version; + } + /* Encode layout type */ *(*pp)++ = (uint8_t)layout->type; *size += sizeof(uint8_t); @@ -446,12 +474,18 @@ H5P__dcrt_layout_enc(const void *value, void **_pp, size_t *size) *size += sizeof(uint8_t); /* Encode chunk dims */ - HDcompile_assert(sizeof(uint32_t) == sizeof(layout->u.chunk.dim[0])); - for (u = 0; u < (size_t)layout->u.chunk.ndims; u++) { - UINT32ENCODE(*pp, layout->u.chunk.dim[u]); - *size += sizeof(uint32_t); - } /* end for */ - } /* end if */ + if (version >= H5D_CRT_LAYOUT_VERSION_1) { + for (u = 0; u < (size_t)layout->u.chunk.ndims; u++) { + UINT64ENCODE(*pp, (uint64_t)layout->u.chunk.dim[u]); + *size += sizeof(uint64_t); + } /* end for */ + } + else + for (u = 0; u < (size_t)layout->u.chunk.ndims; u++) { + UINT32ENCODE(*pp, (uint32_t)layout->u.chunk.dim[u]); + *size += sizeof(uint32_t); + } /* end for */ + } /* end if */ else if (H5D_VIRTUAL == layout->type) { uint64_t nentries = (uint64_t)layout->storage.u.virt.list_nused; @@ -499,7 +533,8 @@ H5P__dcrt_layout_enc(const void *value, void **_pp, size_t *size) /* If layout is chunked, calculate chunking structure */ if (H5D_CHUNKED == layout->type) { *size += sizeof(uint8_t); - *size += layout->u.chunk.ndims * sizeof(uint32_t); + *size += layout->u.chunk.ndims * + ((version >= H5D_CRT_LAYOUT_VERSION_1) ? sizeof(uint64_t) : sizeof(uint32_t)); } /* end if */ else if (H5D_VIRTUAL == layout->type) { /* Calculate size of virtual layout info */ @@ -553,9 +588,10 @@ H5P__dcrt_layout_enc(const void *value, void **_pp, size_t *size) static herr_t H5P__dcrt_layout_dec(const void **_pp, void *value) { - const H5O_layout_t *layout; /* Storage layout */ - H5O_layout_t tmp_layout; /* Temporary local layout structure */ - H5D_layout_t type; /* Layout type */ + const H5O_layout_t *layout; /* Storage layout */ + H5O_layout_t tmp_layout; /* Temporary local layout structure */ + H5D_layout_t type; /* Layout type */ + unsigned version = H5D_CRT_LAYOUT_VERSION_0; /* Version of this property list encoding */ const uint8_t **pp = (const uint8_t **)_pp; herr_t ret_value = SUCCEED; /* Return value */ @@ -569,6 +605,12 @@ H5P__dcrt_layout_dec(const void **_pp, void *value) /* Decode layout type */ type = (H5D_layout_t) * (*pp)++; + /* Check if the version was encoded, and decode the version and the real type if so */ + if (type == H5D_CRT_LAYOUT_MAGIC_TYPE) { + version = (unsigned)*(*pp)++; + type = (H5D_layout_t) * (*pp)++; + } + /* set default layout in case the type is compact or contiguous, otherwise * decode the chunked structure and set chunked layout */ switch (type) { @@ -595,10 +637,23 @@ H5P__dcrt_layout_dec(const void **_pp, void *value) /* Initialize to default values */ tmp_layout = H5D_def_layout_chunk_g; - /* Set rank & dimensions */ + /* Set rank */ tmp_layout.u.chunk.ndims = (unsigned)ndims; - for (u = 0; u < ndims; u++) - UINT32DECODE(*pp, tmp_layout.u.chunk.dim[u]); + + /* Decode dimensions */ + if (version >= H5D_CRT_LAYOUT_VERSION_1) { + for (u = 0; u < ndims; u++) { + uint64_t dim64; + UINT64DECODE(*pp, dim64); + tmp_layout.u.chunk.dim[u] = (hsize_t)dim64; + if ((uint64_t)tmp_layout.u.chunk.dim[u] != dim64) + HGOTO_ERROR(H5E_PLIST, H5E_BADRANGE, FAIL, + "chunk dimension too big to fit in hsize_t"); + } + } + else + for (u = 0; u < ndims; u++) + UINT32DECODE(*pp, tmp_layout.u.chunk.dim[u]); /* Point at the newly set up struct */ layout = &tmp_layout; @@ -1952,7 +2007,6 @@ H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/]) { H5P_genplist_t *plist; /* Property list pointer */ H5O_layout_t chunk_layout; /* Layout information for setting chunk info */ - uint64_t chunk_nelmts; /* Number of elements in chunk */ unsigned u; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ @@ -1969,17 +2023,11 @@ H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/]) /* Verify & initialize property's chunk dims */ H5MM_memcpy(&chunk_layout, &H5D_def_layout_chunk_g, sizeof(H5D_def_layout_chunk_g)); memset(&chunk_layout.u.chunk.dim, 0, sizeof(chunk_layout.u.chunk.dim)); - chunk_nelmts = 1; for (u = 0; u < (unsigned)ndims; u++) { if (dim[u] == 0) HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "all chunk dimensions must be positive"); - if (dim[u] != (dim[u] & 0xffffffff)) - HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "all chunk dimensions must be less than 2^32"); - chunk_nelmts *= dim[u]; - if (chunk_nelmts > (uint64_t)0xffffffff) - HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "number of elements in chunk must be < 4GB"); - chunk_layout.u.chunk.dim[u] = (uint32_t)dim[u]; /* Store user's chunk dimensions */ - } /* end for */ + chunk_layout.u.chunk.dim[u] = dim[u]; /* Store user's chunk dimensions */ + } /* end for */ /* Get the plist structure */ if (NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE, false))) diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 636bd5e756e..f28cd6d7b08 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -6445,14 +6445,21 @@ H5_DLL herr_t H5Pset_alloc_time(hid_t plist_id, H5D_alloc_time_t alloc_time); * * \note Chunk size cannot exceed the size of a fixed-size dataset. For * example, a dataset consisting of a 5x4 fixed-size array cannot be - * defined with 10x10 chunks. Chunk maximums: - * - The maximum number of elements in a chunk is 232-1 which - * is equal to 4,294,967,295. If the number of elements in a chunk is - * set via H5Pset_chunk() to a value greater than 232-1, - * then H5Pset_chunk() will fail. - * - The maximum size for any chunk is 4GB. If a chunk that is larger - * than 4GB attempts to be written with H5Dwrite(), then H5Dwrite() - * will fail. + * defined with 10x10 chunks. + * + * \note With HDF5 version 2.0.0, creation of datasets with chunks larger than + * 4 GiB is now supported. However, doing so will upgrade the file format + * and prevent earlier versions of the library from being able to open the + * dataset. Users must also be aware that some operations will require the + * entire chunk be brought into memory, such as when there is a fill value + * or data filter set. These operations will not work on 32 bit systems + * when using chunks with size >= 4 GiB. The file format will be upgraded + * when the size of an unfiltered chunk is greater than 232-1 + * which is equal to 4,294,967,295. If a filter grows a chunk from below + * this value to above it, the write may fail since the file format was + * not automatically upgraded. To fix this, users can call + * H5Pset_libver_bounds() with #H5F_LIBVER_V200 as the low bound. With the + * new file format, chunk sizes are now limited to 264-1. * * \see H5Pset_layout(), H5Dwrite() * diff --git a/src/H5VLnative.h b/src/H5VLnative.h index 32f8035cb9a..96c969839ca 100644 --- a/src/H5VLnative.h +++ b/src/H5VLnative.h @@ -91,7 +91,7 @@ typedef struct H5VL_native_dataset_chunk_read_t { typedef struct H5VL_native_dataset_chunk_write_t { const hsize_t *offset; uint32_t filters; - uint32_t size; + size_t size; const void *buf; } H5VL_native_dataset_chunk_write_t; diff --git a/src/H5VM.c b/src/H5VM.c index 8d3874ee058..758f706a517 100644 --- a/src/H5VM.c +++ b/src/H5VM.c @@ -988,7 +988,7 @@ H5VM_array_calc(hsize_t offset, unsigned n, const hsize_t *total_size, hsize_t * *------------------------------------------------------------------------- */ hsize_t -H5VM_chunk_index(unsigned ndims, const hsize_t *coord, const uint32_t *chunk, const hsize_t *down_nchunks) +H5VM_chunk_index(unsigned ndims, const hsize_t *coord, const hsize_t *chunk, const hsize_t *down_nchunks) { hsize_t scaled_coord[H5VM_HYPER_NDIMS]; /* Scaled, coordinates, in terms of chunks */ hsize_t chunk_idx; /* Chunk index computed */ @@ -1017,7 +1017,7 @@ H5VM_chunk_index(unsigned ndims, const hsize_t *coord, const uint32_t *chunk, co *------------------------------------------------------------------------- */ void -H5VM_chunk_scaled(unsigned ndims, const hsize_t *coord, const uint32_t *chunk, hsize_t *scaled) +H5VM_chunk_scaled(unsigned ndims, const hsize_t *coord, const hsize_t *chunk, hsize_t *scaled) { unsigned u; /* Local index variable */ @@ -1078,7 +1078,7 @@ H5VM_chunk_scaled(unsigned ndims, const hsize_t *coord, const uint32_t *chunk, h *------------------------------------------------------------------------- */ hsize_t -H5VM_chunk_index_scaled(unsigned ndims, const hsize_t *coord, const uint32_t *chunk, +H5VM_chunk_index_scaled(unsigned ndims, const hsize_t *coord, const hsize_t *chunk, const hsize_t *down_nchunks, hsize_t *scaled) { hsize_t chunk_idx; /* Computed chunk index */ diff --git a/src/H5VMprivate.h b/src/H5VMprivate.h index 88a277eac43..f352d9eacfc 100644 --- a/src/H5VMprivate.h +++ b/src/H5VMprivate.h @@ -105,10 +105,10 @@ H5_DLL hsize_t H5VM_array_offset_pre(unsigned n, const hsize_t *acc, const hsize H5_DLL hsize_t H5VM_array_offset(unsigned n, const hsize_t *total_size, const hsize_t *offset); H5_DLL herr_t H5VM_array_calc_pre(hsize_t offset, unsigned n, const hsize_t *down, hsize_t *coords); H5_DLL herr_t H5VM_array_calc(hsize_t offset, unsigned n, const hsize_t *total_size, hsize_t *coords); -H5_DLL hsize_t H5VM_chunk_index(unsigned ndims, const hsize_t *coord, const uint32_t *chunk, +H5_DLL hsize_t H5VM_chunk_index(unsigned ndims, const hsize_t *coord, const hsize_t *chunk, const hsize_t *down_nchunks); -H5_DLL void H5VM_chunk_scaled(unsigned ndims, const hsize_t *coord, const uint32_t *chunk, hsize_t *scaled); -H5_DLL hsize_t H5VM_chunk_index_scaled(unsigned ndims, const hsize_t *coord, const uint32_t *chunk, +H5_DLL void H5VM_chunk_scaled(unsigned ndims, const hsize_t *coord, const hsize_t *chunk, hsize_t *scaled); +H5_DLL hsize_t H5VM_chunk_index_scaled(unsigned ndims, const hsize_t *coord, const hsize_t *chunk, const hsize_t *down_nchunks, hsize_t *scaled); H5_DLL ssize_t H5VM_opvv(size_t dst_max_nseq, size_t *dst_curr_seq, size_t dst_len_arr[], hsize_t dst_off_arr[], size_t src_max_nseq, size_t *src_curr_seq, diff --git a/test/big.c b/test/big.c index 48f1b3e406c..2dc7b944235 100644 --- a/test/big.c +++ b/test/big.c @@ -61,6 +61,7 @@ * NO_FILE stands for "no file" to be tested. */ typedef enum fsizes_t { SFILE, LFILE, XLFILE, HUGEFILE, NO_FILE } fsizes_t; +typedef enum dset_layout_t { CONTIG, SINGLE_CHUNK, FARRAY, EARRAY, BTREE2 } dset_layout_t; fsizes_t file_size = NO_FILE; const char *FILENAME[] = {"big", "sec2", "stdio", NULL}; @@ -297,18 +298,23 @@ enough_room(hid_t fapl) *------------------------------------------------------------------------- */ static int -writer(char *filename, hid_t fapl, fsizes_t testsize, int wrt_n) +writer(char *filename, hid_t fapl, fsizes_t testsize, int wrt_n, dset_layout_t layout) { hsize_t size1[4] = {8, 1024, 1024, 1024}; - hsize_t size2[1] = {8LL * 1024LL * 1024LL * 1024LL}; - hsize_t hs_start[1]; - hsize_t hs_size[1]; + hsize_t size1_max[4]; + hsize_t chunk_dim1[4]; + hsize_t size2[2] = {8LL * 1024LL * 1024LL * 1024LL, 0}; + hsize_t size2_max[2]; + hsize_t chunk_dim2[2]; + int rank2 = 1; + hsize_t hs_start[2] = {0, 0}; + hsize_t hs_size[2] = {WRT_SIZE, 1}; hid_t file = H5I_INVALID_HID, space1 = H5I_INVALID_HID, space2 = H5I_INVALID_HID, mem_space = H5I_INVALID_HID, d1 = H5I_INVALID_HID, d2 = H5I_INVALID_HID; int *buf = (int *)malloc(sizeof(int) * WRT_SIZE); int i, j; - FILE *out = fopen(DNAME, "w"); - hid_t dcpl; + FILE *out = fopen(DNAME, "w"); + hid_t dcpl1 = H5I_INVALID_HID, dcpl2 = H5I_INVALID_HID; switch (testsize) { case LFILE: @@ -349,6 +355,12 @@ writer(char *filename, hid_t fapl, fsizes_t testsize, int wrt_n) break; } + /* Initialize other arrays */ + memcpy(size1_max, size1, sizeof(size1)); + memcpy(chunk_dim1, size1, sizeof(size1)); + memcpy(size2_max, size2, sizeof(size2)); + memcpy(chunk_dim2, size2, sizeof(size2)); + /* * We might be on a machine that has 32-bit files, so create an HDF5 file * which is a family of files. Each member of the family will be 1GB @@ -357,13 +369,7 @@ writer(char *filename, hid_t fapl, fsizes_t testsize, int wrt_n) goto error; } - /* Create simple data spaces according to the size specified above. */ - if ((space1 = H5Screate_simple(4, size1, size1)) < 0 || - (space2 = H5Screate_simple(1, size2, size2)) < 0) { - goto error; - } - - /* Create the datasets */ + /* Create DCPLs */ /* * The fix below is provided for bug#921 * H5Dcreate with H5P_DEFAULT creation properties @@ -372,17 +378,58 @@ writer(char *filename, hid_t fapl, fsizes_t testsize, int wrt_n) * We should create a dataset allocating space late and never writing fill values. * EIP 4/8/03 */ - dcpl = H5Pcreate(H5P_DATASET_CREATE); - H5Pset_alloc_time(dcpl, H5D_ALLOC_TIME_LATE); - H5Pset_fill_time(dcpl, H5D_FILL_TIME_NEVER); - if ((d1 = H5Dcreate2(file, "d1", H5T_NATIVE_INT, space1, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0 || - (d2 = H5Dcreate2(file, "d2", H5T_NATIVE_INT, space2, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) { + dcpl1 = H5Pcreate(H5P_DATASET_CREATE); + H5Pset_alloc_time(dcpl1, H5D_ALLOC_TIME_LATE); + H5Pset_fill_time(dcpl1, H5D_FILL_TIME_NEVER); + dcpl2 = H5Pcopy(dcpl1); + + /* Set up chunking */ + if (layout != CONTIG) { + /* If we're not using the single chunk index, set the chunk dimensions so there are 2 chunks */ + if (layout != SINGLE_CHUNK) { + chunk_dim1[0] /= 2; + chunk_dim2[0] /= 2; + + /* If we're using the earray index, use one unlimited dimension */ + if (layout == EARRAY) { + size1_max[0] = H5S_UNLIMITED; + size2_max[0] = H5S_UNLIMITED; + } + /* If we're using the btree2 index, use two unlimited dimensions and convert dataset 2 to 2-D */ + else if (layout == BTREE2) { + size1_max[0] = H5S_UNLIMITED; + size1_max[1] = H5S_UNLIMITED; + rank2 = 2; + chunk_dim2[0] /= 2; + chunk_dim2[1] = 2; + size2[0] /= 2; + size2[1] = 2; + size2_max[0] = H5S_UNLIMITED; + size2_max[1] = H5S_UNLIMITED; + } + } + + /* Set chunk dimensions */ + if (H5Pset_chunk(dcpl1, 4, size1) < 0) + goto error; + if (H5Pset_chunk(dcpl2, rank2, size2) < 0) + goto error; + } + + /* Create simple data spaces according to the size specified above. */ + if ((space1 = H5Screate_simple(4, size1, size1_max)) < 0 || + (space2 = H5Screate_simple(rank2, size2, size2_max)) < 0) { + goto error; + } + + /* Create the datasets */ + if ((d1 = H5Dcreate2(file, "d1", H5T_NATIVE_INT, space1, H5P_DEFAULT, dcpl1, H5P_DEFAULT)) < 0 || + (d2 = H5Dcreate2(file, "d2", H5T_NATIVE_INT, space2, H5P_DEFAULT, dcpl2, H5P_DEFAULT)) < 0) { goto error; } /* Write some things to them randomly */ - hs_size[0] = WRT_SIZE; - if ((mem_space = H5Screate_simple(1, hs_size, hs_size)) < 0) + if ((mem_space = H5Screate_simple(rank2, hs_size, hs_size)) < 0) goto error; for (i = 0; i < wrt_n; i++) { /* start position must be at least hs_size from the end */ @@ -401,6 +448,10 @@ writer(char *filename, hid_t fapl, fsizes_t testsize, int wrt_n) goto error; if (H5Dclose(d2) < 0) goto error; + if (H5Pclose(dcpl1) < 0) + goto error; + if (H5Pclose(dcpl2) < 0) + goto error; if (H5Sclose(mem_space) < 0) goto error; if (H5Sclose(space1) < 0) @@ -419,6 +470,8 @@ writer(char *filename, hid_t fapl, fsizes_t testsize, int wrt_n) { H5Dclose(d1); H5Dclose(d2); + H5Pclose(dcpl1); + H5Pclose(dcpl2); H5Sclose(space1); H5Sclose(space2); H5Sclose(mem_space); @@ -449,9 +502,10 @@ reader(char *filename, hid_t fapl) FILE *script = NULL; hid_t file = H5I_INVALID_HID, mspace = H5I_INVALID_HID, fspace = H5I_INVALID_HID, d2 = H5I_INVALID_HID; char ln[128], *s; - hsize_t hs_offset[1]; - hsize_t hs_size[1] = {WRT_SIZE}; - int *buf = (int *)malloc(sizeof(int) * WRT_SIZE); + hsize_t hs_offset[2] = {0, 0}; + hsize_t hs_size[2] = {WRT_SIZE, 1}; + int *buf = (int *)malloc(sizeof(int) * WRT_SIZE); + int rank; int i, j, zero, wrong, nerrors = 0; /* Open script file */ @@ -459,17 +513,21 @@ reader(char *filename, hid_t fapl) /* Open HDF5 file */ if ((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* Open the dataset */ if ((d2 = H5Dopen2(file, "d2", H5P_DEFAULT)) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if ((fspace = H5Dget_space(d2)) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; + + /* Get rank of dataset */ + if ((rank = H5Sget_simple_extent_ndims(fspace)) < 0) + TEST_ERROR; /* Describe `buf' */ - if ((mspace = H5Screate_simple(1, hs_size, hs_size)) < 0) - FAIL_STACK_ERROR; + if ((mspace = H5Screate_simple(rank, hs_size, hs_size)) < 0) + TEST_ERROR; /* Read each region */ while (fgets(ln, (int)sizeof(ln), script)) { @@ -483,7 +541,7 @@ reader(char *filename, hid_t fapl) if (H5Sselect_hyperslab(fspace, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL) < 0) FAIL_STACK_ERROR; if (H5Dread(d2, H5T_NATIVE_INT, mspace, fspace, H5P_DEFAULT, buf) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* Check */ for (j = zero = wrong = 0; j < WRT_SIZE; j++) { @@ -507,13 +565,13 @@ reader(char *filename, hid_t fapl) } if (H5Dclose(d2) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if (H5Sclose(mspace) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if (H5Sclose(fspace) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; if (H5Fclose(file) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; free(buf); fclose(script); @@ -561,6 +619,48 @@ usage(void) (hsize_t)FAMILY_SIZE); } +static int +run_tests(char *filename, hid_t fapl, fsizes_t testsize, int wrt_n) +{ + if (writer(filename, fapl, testsize, wrt_n, CONTIG)) + goto error; + if (reader(filename, fapl)) + goto error; + puts(" Test passed with contiguous datasets."); + + if (writer(filename, fapl, testsize, wrt_n, SINGLE_CHUNK)) + goto error; + if (reader(filename, fapl)) + goto error; + puts(" Test passed with single chunk index."); + + if (writer(filename, fapl, testsize, wrt_n, FARRAY)) + goto error; + if (reader(filename, fapl)) + goto error; + puts(" Test passed with fixed array chunk index."); + + if (writer(filename, fapl, testsize, wrt_n, EARRAY)) + goto error; + if (reader(filename, fapl)) + goto error; + puts(" Test passed with extensible array chunk index."); + + if (writer(filename, fapl, testsize, wrt_n, BTREE2)) + goto error; + if (reader(filename, fapl)) + goto error; + puts(" Test passed with v2 b-tree chunk index."); + + /* Clean up */ + h5_delete_all_test_files(FILENAME, fapl); + HDremove(DNAME); + return 0; + +error: + return 1; +} + static int test_sec2(hid_t fapl) { @@ -573,23 +673,16 @@ test_sec2(hid_t fapl) goto quit; } /* Test big file with the SEC2 driver */ - puts("Testing big file with the SEC2 Driver "); + puts("Testing big file with the SEC2 Driver:"); h5_fixname(FILENAME[1], fapl, filename, sizeof filename); - if (writer(filename, fapl, testsize, WRT_N)) + if (run_tests(filename, fapl, testsize, WRT_N)) goto error; - if (reader(filename, fapl)) - goto error; - - puts("Test passed with the SEC2 Driver."); quit: /* End with normal return code */ - /* Clean up the test file */ - h5_delete_all_test_files(FILENAME, fapl); H5Pclose(fapl); - HDremove(DNAME); return 0; error: @@ -612,11 +705,8 @@ test_stdio(hid_t fapl) h5_fixname(FILENAME[2], fapl, filename, sizeof filename); - if (writer(filename, fapl, testsize, WRT_N)) + if (run_tests(filename, fapl, testsize, WRT_N)) goto error; - if (reader(filename, fapl)) - goto error; - puts("Test passed with the STDIO Driver."); /* Flush stdout at the end of this test routine to ensure later * output to stderr will not come out before it. @@ -625,9 +715,7 @@ test_stdio(hid_t fapl) quit: /* End with normal return code */ /* Clean up the test file */ - h5_delete_all_test_files(FILENAME, fapl); H5Pclose(fapl); - HDremove(DNAME); fflush(stdout); return 0; @@ -671,19 +759,13 @@ test_family(hid_t fapl) /* Do the test with the Family Driver */ h5_fixname(FILENAME[0], fapl, filename, sizeof filename); - if (writer(filename, fapl, HUGEFILE, WRT_N)) - goto error; - if (reader(filename, fapl)) + if (run_tests(filename, fapl, HUGEFILE, WRT_N)) goto error; - puts("Test passed with the Family Driver."); - quit: /* End with normal return code */ /* Clean up the test file */ - h5_delete_all_test_files(FILENAME, fapl); H5Pclose(fapl); - HDremove(DNAME); return 0; error: @@ -758,6 +840,7 @@ main(int ac, char **av) #endif srand((unsigned)seed); + /* Loop over dataset configuration */ /* run VFD-specific test */ if (H5FD_SEC2 == driver) { if (test_sec2(fapl) != 0) diff --git a/test/dsets.c b/test/dsets.c index 8f72b2ebae0..e89f154312b 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -80,6 +80,7 @@ static const char *FILENAME[] = {"dataset", /* 0 */ "h5s_plist", /* 28 */ "vds_strings", /* 29 */ "chunk_expand2", /* 30 */ + "scalar_datasets", /* 31 */ NULL}; #define OHMIN_FILENAME_A "ohdr_min_a" @@ -96,6 +97,7 @@ static const char *FILENAME[] = {"dataset", /* 0 */ #define DSET_CHUNKED_NAME "chunked" #define DSET_COMPACT_NAME "compact" #define DSET_SIMPLE_IO_NAME "simple_io" +#define DSET_SCALAR_IO_NAME "scalar_io" #define DSET_USERBLOCK_IO_NAME "userblock_io" #define DSET_COMPACT_IO_NAME "compact_io" #define DSET_COMPACT_MAX_NAME "max_compact" @@ -698,6 +700,118 @@ test_simple_io(const char *driver_name, hid_t fapl) return FAIL; } /* end test_simple_io() */ +/*------------------------------------------------------------------------- + * Function: test_scalar_io + * + * Purpose: Tests scalar datasets + * + * Return: Success: 0 + * Failure: -1 + *------------------------------------------------------------------------- + */ +static herr_t +test_scalar_io(hid_t fapl) +{ + char filename[FILENAME_BUF_SIZE]; + hid_t file = H5I_INVALID_HID, dataset = H5I_INVALID_HID, space = H5I_INVALID_HID, dcpl = H5I_INVALID_HID; + int rdata = 0; + int wdata = 1; + + TESTING("scalar datasets"); + + h5_fixname(FILENAME[31], fapl, filename, sizeof filename); + + /* Create the data space */ + if ((space = H5Screate(H5S_SCALAR)) < 0) + TEST_ERROR; + + /* Create the file */ + if ((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + TEST_ERROR; + + /* Create the dataset */ + if ((dataset = H5Dcreate2(file, DSET_SCALAR_IO_NAME, H5T_NATIVE_INT, space, H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Write the data to the dataset */ + if (H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &wdata) < 0) + TEST_ERROR; + + /* Read the dataset back */ + if (H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdata) < 0) + TEST_ERROR; + + /* Check that the value read is the same as the value written */ + if (rdata != wdata) + TEST_ERROR; + + /* Close */ + if (H5Dclose(dataset) < 0) + goto error; + dataset = -1; + if (H5Fclose(file) < 0) + goto error; + file = -1; + + /* + * Now test with a compact dataset + */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + if (H5Pset_layout(dcpl, H5D_COMPACT) < 0) + TEST_ERROR; + + /* Create the file */ + if ((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + TEST_ERROR; + + /* Create the dataset */ + if ((dataset = H5Dcreate2(file, DSET_SCALAR_IO_NAME, H5T_NATIVE_INT, space, H5P_DEFAULT, dcpl, + H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Write the data to the dataset */ + if (H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &wdata) < 0) + TEST_ERROR; + + /* Read the dataset back */ + if (H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &rdata) < 0) + TEST_ERROR; + + /* Check that the value read is the same as the value written */ + if (rdata != wdata) + TEST_ERROR; + + /* Close */ + if (H5Dclose(dataset) < 0) + TEST_ERROR; + dataset = -1; + if (H5Fclose(file) < 0) + TEST_ERROR; + file = -1; + if (H5Pclose(dcpl) < 0) + TEST_ERROR; + dcpl = -1; + if (H5Sclose(space) < 0) + TEST_ERROR; + space = -1; + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Sclose(space); + H5Pclose(dcpl); + H5Dclose(dataset); + H5Fclose(file); + } + H5E_END_TRY + + return FAIL; +} /* end test_scalar_io() */ + /*------------------------------------------------------------------------- * Function: test_userblock_offset * @@ -1016,7 +1130,9 @@ test_compact_io(hid_t fapl) TEST_ERROR; } else { - if (dsetp->shared->layout.version != H5O_layout_ver_bounds[fp->shared->low_bound]) + /* We no longer upgrade the layout for compact datasets past version 3 since there is no + * benefit to doing so */ + if (dsetp->shared->layout.version > H5O_layout_ver_bounds[fp->shared->low_bound]) TEST_ERROR; if (dsetp->shared->dcpl_cache.fill.version != H5O_fill_ver_bounds[fp->shared->low_bound]) TEST_ERROR; @@ -8874,9 +8990,9 @@ test_deprec(hid_t file) #endif /* H5_NO_DEPRECATED_SYMBOLS */ /*------------------------------------------------------------------------- - * Function: test_huge_chunks + * Function: test_huge_chunks * - * Purpose: Tests that datasets with chunks >4GB can't be created. + * Purpose: Tests that datasets with chunks >4GB can't be created, unless we're using the 2.0 file format. * * Return: Success: 0 * Failure: -1 @@ -8884,106 +9000,163 @@ test_deprec(hid_t file) *------------------------------------------------------------------------- */ static herr_t -test_huge_chunks(hid_t fapl) +test_huge_chunks(hid_t fapl, H5F_libver_t low) { char filename[FILENAME_BUF_SIZE]; - hid_t fid = H5I_INVALID_HID; /* File ID */ - hid_t dcpl = H5I_INVALID_HID; /* Dataset creation property list ID */ - hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ - hid_t dsid = H5I_INVALID_HID; /* Dataset ID */ - hsize_t dim, chunk_dim; /* Dataset and chunk dimensions */ - hsize_t dim2[3], chunk_dim2[3]; /* Dataset and chunk dimensions */ - herr_t ret; /* Generic return value */ + hid_t fail_fapl = H5I_INVALID_HID; /* File creation property list ID */ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t dcpl = H5I_INVALID_HID; /* Dataset creation property list ID */ + hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ + hid_t dsid = H5I_INVALID_HID; /* Dataset ID */ + hsize_t dim, chunk_dim; /* Dataset and chunk dimensions */ + hsize_t dim2[3], chunk_dim2[3]; /* Dataset and chunk dimensions */ TESTING("creating dataset with >4GB chunks"); + /* Set high bound of 1.14 so huge chunk dataset creation fails */ h5_fixname(FILENAME[7], fapl, filename, sizeof filename); - /* Create file */ - if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) - FAIL_STACK_ERROR; - /* Create dataset creation property list */ if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; - /* Try to set too large of a chunk for 1-D dataset (# of elements) */ + /* Set a very large chunk size for 1-D dataset (# of elements) */ chunk_dim = TOO_HUGE_CHUNK_DIM; - H5E_BEGIN_TRY - { - ret = H5Pset_chunk(dcpl, 1, &chunk_dim); - } - H5E_END_TRY - if (ret >= 0) - FAIL_PUTS_ERROR(" Set chunk size with too large of chunk dimensions."); + if (H5Pset_chunk(dcpl, 1, &chunk_dim) < 0) + TEST_ERROR; - /* Try to set too large of a chunk for n-D dataset (# of elements) */ + /* Set a very large chunk size for n-D dataset (# of elements) */ chunk_dim2[0] = TOO_HUGE_CHUNK_DIM2_0; chunk_dim2[1] = TOO_HUGE_CHUNK_DIM2_1; chunk_dim2[2] = TOO_HUGE_CHUNK_DIM2_2; - H5E_BEGIN_TRY - { - ret = H5Pset_chunk(dcpl, 3, chunk_dim2); - } - H5E_END_TRY - if (ret >= 0) - FAIL_PUTS_ERROR(" Set chunk size with too large of chunk dimensions."); + if (H5Pset_chunk(dcpl, 3, chunk_dim2) < 0) + TEST_ERROR; + + /* Create file */ + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + TEST_ERROR; /* Set 1-D chunk size */ chunk_dim = HUGE_CHUNK_DIM; if (H5Pset_chunk(dcpl, 1, &chunk_dim) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* Create 1-D dataspace */ dim = HUGE_DIM; if ((sid = H5Screate_simple(1, &dim, NULL)) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; - /* Try to create dataset */ - H5E_BEGIN_TRY - { - dsid = H5Dcreate2(fid, HUGE_DATASET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); - } - H5E_END_TRY - if (dsid >= 0) - FAIL_PUTS_ERROR(" 1-D Dataset with too large of chunk dimensions created."); + /* Create dataset */ + if ((dsid = H5Dcreate2(fid, HUGE_DATASET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + TEST_ERROR; - /* Close 1-D dataspace */ + /* Close dataset and 1-D dataspace */ + if (H5Dclose(dsid) < 0) + TEST_ERROR; if (H5Sclose(sid) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* Set n-D chunk size */ chunk_dim2[0] = HUGE_CHUNK_DIM2_0; chunk_dim2[1] = HUGE_CHUNK_DIM2_1; chunk_dim2[2] = HUGE_CHUNK_DIM2_2; if (H5Pset_chunk(dcpl, 3, chunk_dim2) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; /* Create n-D dataspace */ dim2[0] = HUGE_DIM2_0; dim2[1] = HUGE_DIM2_1; dim2[2] = HUGE_DIM2_2; if ((sid = H5Screate_simple(3, dim2, NULL)) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; - /* Try to create dataset */ - H5E_BEGIN_TRY - { - dsid = H5Dcreate2(fid, HUGE_DATASET2, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); - } - H5E_END_TRY - if (dsid >= 0) - FAIL_PUTS_ERROR(" n-D Dataset with too large of chunk dimensions created."); + /* Create dataset */ + if ((dsid = H5Dcreate2(fid, HUGE_DATASET2, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + TEST_ERROR; - /* Close n-D dataspace */ + /* Close dataset and n-D dataspace */ + if (H5Dclose(dsid) < 0) + TEST_ERROR; if (H5Sclose(sid) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; - /* Close everything else */ - if (H5Pclose(dcpl) < 0) - FAIL_STACK_ERROR; + /* Close file */ if (H5Fclose(fid) < 0) - FAIL_STACK_ERROR; + TEST_ERROR; + + /* Only check for failure if the low bounds is less than 2.0 */ + if (low < H5F_LIBVER_V200) { + /* Create a FAPL that will force huge chunk dataset creation to fail by setting a high bound of 1.14 + */ + if ((fail_fapl = H5Pcopy(fapl)) < 0) + TEST_ERROR; + if (H5Pset_libver_bounds(fail_fapl, low, H5F_LIBVER_V114) < 0) + TEST_ERROR; + + /* Create file */ + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fail_fapl)) < 0) + TEST_ERROR; + + /* Set 1-D chunk size */ + chunk_dim = HUGE_CHUNK_DIM; + if (H5Pset_chunk(dcpl, 1, &chunk_dim) < 0) + TEST_ERROR; + + /* Create 1-D dataspace */ + dim = HUGE_DIM; + if ((sid = H5Screate_simple(1, &dim, NULL)) < 0) + TEST_ERROR; + + /* Try to create dataset */ + H5E_BEGIN_TRY + { + dsid = H5Dcreate2(fid, HUGE_DATASET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); + } + H5E_END_TRY + if (dsid >= 0) + FAIL_PUTS_ERROR(" 1-D Dataset with too large of chunk dimensions created."); + + /* Close 1-D dataspace */ + if (H5Sclose(sid) < 0) + TEST_ERROR; + + /* Set n-D chunk size */ + chunk_dim2[0] = HUGE_CHUNK_DIM2_0; + chunk_dim2[1] = HUGE_CHUNK_DIM2_1; + chunk_dim2[2] = HUGE_CHUNK_DIM2_2; + if (H5Pset_chunk(dcpl, 3, chunk_dim2) < 0) + TEST_ERROR; + + /* Create n-D dataspace */ + dim2[0] = HUGE_DIM2_0; + dim2[1] = HUGE_DIM2_1; + dim2[2] = HUGE_DIM2_2; + if ((sid = H5Screate_simple(3, dim2, NULL)) < 0) + TEST_ERROR; + + /* Try to create dataset */ + H5E_BEGIN_TRY + { + dsid = H5Dcreate2(fid, HUGE_DATASET2, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); + } + H5E_END_TRY + if (dsid >= 0) + FAIL_PUTS_ERROR(" n-D Dataset with too large of chunk dimensions created."); + + /* Close n-D dataspace */ + if (H5Sclose(sid) < 0) + TEST_ERROR; + + /* Close file and FAPL */ + if (H5Fclose(fid) < 0) + TEST_ERROR; + if (H5Pclose(fail_fapl) < 0) + TEST_ERROR; + } + + /* Close DCPL */ + if (H5Pclose(dcpl) < 0) + TEST_ERROR; PASSED(); return SUCCEED; @@ -8995,6 +9168,7 @@ test_huge_chunks(hid_t fapl) H5Dclose(dsid); H5Sclose(sid); H5Fclose(fid); + H5Pclose(fail_fapl); } H5E_END_TRY return FAIL; @@ -16363,8 +16537,8 @@ test_dcpl_layout_caching(H5D_layout_t layout_type) hid_t type_id = H5I_INVALID_HID; hid_t dcpl_id = H5I_INVALID_HID; - H5O_layout_t layout; - H5O_layout_t default_layout; + H5O_layout_t *layout = NULL; + H5O_layout_t *default_layout = NULL; H5D_t *dset_int = NULL; H5P_genplist_t *dcpl_int = NULL; @@ -16402,6 +16576,11 @@ test_dcpl_layout_caching(H5D_layout_t layout_type) snprintf(test_str, sizeof(test_str), "delayed DCPL layout copy for %s", layout_msg); TESTING(test_str); + if (NULL == (layout = (H5O_layout_t *)malloc(sizeof(H5O_layout_t)))) + TEST_ERROR; + if (NULL == (default_layout = (H5O_layout_t *)malloc(sizeof(H5O_layout_t)))) + TEST_ERROR; + if ((file_id = H5Fcreate(DCPL_LAYOUT_FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR; @@ -16520,23 +16699,23 @@ test_dcpl_layout_caching(H5D_layout_t layout_type) if ((dcpl_int = H5P_object_verify(dset_int->shared->dcpl_id, H5P_DATASET_CREATE, false)) == NULL) TEST_ERROR; - if (H5P_peek(dcpl_int, H5D_CRT_LAYOUT_NAME, &layout) < 0) + if (H5P_peek(dcpl_int, H5D_CRT_LAYOUT_NAME, layout) < 0) TEST_ERROR; if ((default_dcpl_int = H5P_object_verify(H5P_DATASET_CREATE_DEFAULT, H5P_DATASET_CREATE, true)) == NULL) TEST_ERROR; - if (H5P_peek(default_dcpl_int, H5D_CRT_LAYOUT_NAME, &default_layout) < 0) + if (H5P_peek(default_dcpl_int, H5D_CRT_LAYOUT_NAME, default_layout) < 0) TEST_ERROR; #ifdef NDEBUG /* When NDEBUG is enabled, layout stored on internal DCPL should be equivalent to the default DCPL layout */ - if (memcmp(&layout, &default_layout, sizeof(H5O_layout_t)) != 0) + if (memcmp(layout, default_layout, sizeof(H5O_layout_t)) != 0) TEST_ERROR; #else /* NDEBUG disabled */ /* When NDEBUG is disabled, the internal layout should have an invalid layout to detect bad accesses */ - if (layout.type != H5D_LAYOUT_ERROR) + if (layout->type != H5D_LAYOUT_ERROR) TEST_ERROR; #endif /* NDEBUG */ @@ -16550,10 +16729,10 @@ test_dcpl_layout_caching(H5D_layout_t layout_type) if ((dcpl_int = H5P_object_verify(dset_int->shared->dcpl_id, H5P_DATASET_CREATE, false)) == NULL) TEST_ERROR; - if (H5P_peek(dcpl_int, H5D_CRT_LAYOUT_NAME, &layout) < 0) + if (H5P_peek(dcpl_int, H5D_CRT_LAYOUT_NAME, layout) < 0) TEST_ERROR; - if (layout.type != layout_type) + if (layout->type != layout_type) TEST_ERROR; /* Clean up */ @@ -16576,6 +16755,8 @@ test_dcpl_layout_caching(H5D_layout_t layout_type) TEST_ERROR; } } + free(layout); + free(default_layout); PASSED(); return 0; @@ -16593,6 +16774,8 @@ test_dcpl_layout_caching(H5D_layout_t layout_type) H5Dclose(src_dsets[i]); } } + free(layout); + free(default_layout); } H5E_END_TRY; @@ -17716,6 +17899,7 @@ main(void) nerrors += (test_create(file) < 0 ? 1 : 0); nerrors += (test_simple_io(driver_name, fapl) < 0 ? 1 : 0); + nerrors += (test_scalar_io(fapl) < 0 ? 1 : 0); nerrors += (test_compact_io(fapl) < 0 ? 1 : 0); nerrors += (test_max_compact(fapl) < 0 ? 1 : 0); nerrors += (test_compact_open_close_dirty(fapl) < 0 ? 1 : 0); @@ -17768,7 +17952,7 @@ main(void) nerrors += (test_deprec(file) < 0 ? 1 : 0); #endif /* H5_NO_DEPRECATED_SYMBOLS */ - nerrors += (test_huge_chunks(fapl) < 0 ? 1 : 0); + nerrors += (test_huge_chunks(fapl, low) < 0 ? 1 : 0); nerrors += (test_chunk_cache(fapl) < 0 ? 1 : 0); nerrors += (test_big_chunks_bypass_cache(fapl) < 0 ? 1 : 0); nerrors += (test_chunk_fast(driver_name, fapl) < 0 ? 1 : 0); diff --git a/test/tmisc.c b/test/tmisc.c index 3914492935a..0218aaac4c4 100644 --- a/test/tmisc.c +++ b/test/tmisc.c @@ -3851,20 +3851,10 @@ test_misc20(void) return; } - /* Verify that chunks with dimensions that are too large get rejected */ - /* Create a dataset creation property list */ dcpl = H5Pcreate(H5P_DATASET_CREATE); CHECK(dcpl, FAIL, "H5Pcreate"); - /* Try to use chunked storage for this dataset */ - H5E_BEGIN_TRY - { - ret = H5Pset_chunk(dcpl, rank, big_dims); - } - H5E_END_TRY - VERIFY(ret, FAIL, "H5Pset_chunk"); - /* Verify that the storage for the dataset is the correct size and hasn't * been truncated. */