17 #include "nc4internal.h"
18 #include "nc4dispatch.h"
21 #include <H5DSpublic.h>
34 extern int num_plists;
35 extern int num_spaces;
38 #define MIN_DEFLATE_LEVEL 0
39 #define MAX_DEFLATE_LEVEL 9
43 #define REFERENCE_LIST "REFERENCE_LIST"
45 #define DIMENSION_LIST "DIMENSION_LIST"
79 static int NC4_enddef(
int ncid);
80 static int nc4_rec_read_metadata(NC_GRP_INFO_T *grp);
81 static int close_netcdf4_file(NC_HDF5_FILE_INFO_T *h5,
int abort);
85 size_t nc4_chunk_cache_size = CHUNK_CACHE_SIZE;
86 size_t nc4_chunk_cache_nelems = CHUNK_CACHE_NELEMS;
87 float nc4_chunk_cache_preemption = CHUNK_CACHE_PREEMPTION;
91 static int virgin = 1;
96 static hid_t h5_native_type_constant_g[NUM_TYPES];
97 static const char nc_type_name_g[NUM_TYPES][
NC_MAX_NAME + 1] = {
"char",
"byte",
"short",
98 "int",
"float",
"double",
"ubyte",
99 "ushort",
"uint",
"int64",
105 static const int nc_type_size_g[NUM_TYPES] = {
sizeof(char),
sizeof(
char),
sizeof(short),
106 sizeof(
int),
sizeof(float),
sizeof(
double),
sizeof(
unsigned char),
107 sizeof(
unsigned short),
sizeof(
unsigned int),
sizeof(
long long),
108 sizeof(
unsigned long long),
sizeof(
char *)};
113 nc_set_chunk_cache(
size_t size,
size_t nelems,
float preemption)
115 if (preemption < 0 || preemption > 1)
117 nc4_chunk_cache_size = size;
118 nc4_chunk_cache_nelems = nelems;
119 nc4_chunk_cache_preemption = preemption;
126 nc_get_chunk_cache(
size_t *sizep,
size_t *nelemsp,
float *preemptionp)
129 *sizep = nc4_chunk_cache_size;
132 *nelemsp = nc4_chunk_cache_nelems;
135 *preemptionp = nc4_chunk_cache_preemption;
141 nc_set_chunk_cache_ints(
int size,
int nelems,
int preemption)
143 if (size <= 0 || nelems <= 0 || preemption < 0 || preemption > 100)
145 nc4_chunk_cache_size = size;
146 nc4_chunk_cache_nelems = nelems;
147 nc4_chunk_cache_preemption = (float)preemption / 100;
152 nc_get_chunk_cache_ints(
int *sizep,
int *nelemsp,
int *preemptionp)
155 *sizep = (int)nc4_chunk_cache_size;
157 *nelemsp = (int)nc4_chunk_cache_nelems;
159 *preemptionp = (int)(nc4_chunk_cache_preemption * 100);
189 #define MAGIC_NUMBER_LEN 4
190 #define NC_HDF5_FILE 1
191 #define NC_HDF4_FILE 2
193 nc_check_for_hdf(
const char *path,
int flags,
void* parameters,
int *hdf_file)
195 char blob[MAGIC_NUMBER_LEN];
198 NC_MPI_INFO* mpiinfo = (NC_MPI_INFO*)parameters;
199 MPI_Comm comm = MPI_COMM_WORLD;
200 MPI_Info info = MPI_INFO_NULL;
204 NC_MEM_INFO* meminfo = (NC_MEM_INFO*)parameters;
209 comm = mpiinfo->comm;
210 info = mpiinfo->info;
215 LOG((3,
"%s: path %s", __func__, path));
218 if(!inmemory && H5Fis_hdf5(path))
220 *hdf_file = NC_HDF5_FILE;
226 if (!inmemory && use_parallel)
231 if ((retval = MPI_File_open(comm, (
char *)path, MPI_MODE_RDONLY,
232 info, &fh)) != MPI_SUCCESS)
234 if ((retval = MPI_File_read(fh, blob, MAGIC_NUMBER_LEN, MPI_CHAR,
235 &status)) != MPI_SUCCESS)
237 if ((retval = MPI_File_close(&fh)) != MPI_SUCCESS)
244 if (!(fp = fopen(path,
"r")) ||
245 fread(blob, MAGIC_NUMBER_LEN, 1, fp) != 1) {
252 if(meminfo->size < MAGIC_NUMBER_LEN)
254 memcpy(blob,meminfo->memory,MAGIC_NUMBER_LEN);
258 if (memcmp(blob,
"\016\003\023\001", 4)==0)
259 *hdf_file = NC_HDF4_FILE;
260 else if (memcmp(blob,
"HDF", 3)==0)
261 *hdf_file = NC_HDF5_FILE;
271 nc4_create_file(
const char *path,
int cmode, MPI_Comm comm, MPI_Info info,
274 hid_t fcpl_id, fapl_id = -1;
278 NC_HDF5_FILE_INFO_T* nc4_info = NULL;
289 flags = H5F_ACC_TRUNC;
291 flags = H5F_ACC_EXCL;
293 flags = H5F_ACC_TRUNC;
295 LOG((3,
"%s: path %s mode 0x%x", __func__, path, cmode));
300 if (cmode & NC_DISKLESS) {
301 #ifndef USE_PARALLEL4
305 }
else if ((cmode & NC_NOCLOBBER) && (fp = fopen(path,
"r"))) {
311 if ((retval = nc4_nc4f_list_add(nc, path, (
NC_WRITE | cmode))))
313 nc4_info = NC4_DATA(nc);
314 assert(nc4_info && nc4_info->root_grp);
324 if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
330 if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI))
333 if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_STRONG))
342 nc4_info->parallel = NC_TRUE;
343 if (cmode & NC_MPIIO)
345 LOG((4,
"creating parallel file with MPI/IO"));
346 if (H5Pset_fapl_mpio(fapl_id, comm, info) < 0)
349 #ifdef USE_PARALLEL_POSIX
352 LOG((4,
"creating parallel file with MPI/posix"));
353 if (H5Pset_fapl_mpiposix(fapl_id, comm, 0) < 0)
366 if (MPI_SUCCESS != MPI_Comm_dup(comm, &nc4_info->comm))
369 if (MPI_INFO_NULL != info)
371 if (MPI_SUCCESS != MPI_Info_dup(info, &nc4_info->info))
378 nc4_info->info = info;
382 if(cmode & NC_DISKLESS) {
383 if (H5Pset_fapl_core(fapl_id, 4096, persist))
386 if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size,
387 nc4_chunk_cache_preemption) < 0)
389 LOG((4,
"%s: set HDF raw chunk cache to size %d nelems %d preemption %f",
390 __func__, nc4_chunk_cache_size, nc4_chunk_cache_nelems, nc4_chunk_cache_preemption));
393 if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
397 if ((fcpl_id = H5Pcreate(H5P_FILE_CREATE)) < 0)
404 if (H5Pset_obj_track_times(fcpl_id,0)<0)
410 if (H5Pset_link_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED |
411 H5P_CRT_ORDER_INDEXED)) < 0)
413 if (H5Pset_attr_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED |
414 H5P_CRT_ORDER_INDEXED)) < 0)
418 if ((nc4_info->hdfid = H5Fcreate(path, flags, fcpl_id, fapl_id)) < 0)
424 if ((nc4_info->root_grp->hdf_grpid = H5Gopen2(nc4_info->hdfid,
"/",
429 if (H5Pclose(fapl_id) < 0 ||
430 H5Pclose(fcpl_id) < 0)
438 nc4_info->flags |= NC_INDEF;
444 if (comm_duped) MPI_Comm_free(&nc4_info->comm);
445 if (info_duped) MPI_Info_free(&nc4_info->info);
450 if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id);
451 if(!nc4_info)
return retval;
452 close_netcdf4_file(nc4_info,1);
472 NC4_create(
const char* path,
int cmode,
size_t initialsz,
int basepe,
473 size_t *chunksizehintp,
int use_parallel,
void *parameters,
474 NC_Dispatch *dispatch, NC* nc_file)
476 MPI_Comm comm = MPI_COMM_WORLD;
477 MPI_Info info = MPI_INFO_NULL;
480 assert(nc_file && path);
482 LOG((1,
"%s: path %s cmode 0x%x comm %d info %d",
483 __func__, path, cmode, comm, info));
488 comm = ((NC_MPI_INFO *)parameters)->comm;
489 info = ((NC_MPI_INFO *)parameters)->info;
496 if (H5Eset_auto(NULL, NULL) < 0)
497 LOG((0,
"Couldn't turn off HDF5 error messages!"));
498 LOG((1,
"HDF5 error messages have been turned off."));
503 if((cmode & ILLEGAL_CREATE_FLAGS) != 0)
507 if((cmode & (NC_MPIIO|NC_MPIPOSIX)) == (NC_MPIIO|NC_MPIPOSIX))
511 if((cmode & (NC_MPIIO | NC_MPIPOSIX)) && (cmode & NC_DISKLESS))
514 #ifndef USE_PARALLEL_POSIX
518 if(cmode & NC_MPIPOSIX)
520 cmode &= ~NC_MPIPOSIX;
535 LOG((2,
"cmode after applying default format: 0x%x", cmode));
537 nc_file->int_ncid = nc_file->ext_ncid;
538 res = nc4_create_file(path, cmode, comm, info, nc_file);
541 if (cmode & NC_PNETCDF)
543 NC_HDF5_FILE_INFO_T* nc4_info;
544 nc4_info = NC4_DATA(nc_file);
547 nc4_info->pnetcdf_file++;
548 res = ncmpi_create(comm, path, cmode, info, &(nc_file->int_ncid));
561 read_scale(NC_GRP_INFO_T *grp, hid_t datasetid,
const char *obj_name,
562 const H5G_stat_t *statbuf, hsize_t scale_size, hsize_t max_scale_size,
565 NC_DIM_INFO_T *new_dim;
567 htri_t attr_exists = -1;
569 int dimscale_created = 0;
570 int initial_grp_ndims = grp->ndims;
571 short initial_next_dimid = grp->nc4_info->next_dimid;
575 if ((retval = nc4_dim_list_add(&grp->dim, &new_dim)))
581 if ((attr_exists = H5Aexists(datasetid, NC_DIMID_ATT_NAME)) < 0)
585 if ((attid = H5Aopen_by_name(datasetid,
".", NC_DIMID_ATT_NAME,
586 H5P_DEFAULT, H5P_DEFAULT)) < 0)
589 if (H5Aread(attid, H5T_NATIVE_INT, &new_dim->dimid) < 0)
593 if (new_dim->dimid >= grp->nc4_info->next_dimid)
594 grp->nc4_info->next_dimid = new_dim->dimid + 1;
599 new_dim->dimid = grp->nc4_info->next_dimid++;
605 if (!(new_dim->name = strdup(obj_name)))
610 new_dim->too_long = NC_TRUE;
613 new_dim->len = scale_size;
614 new_dim->hdf5_objid.fileno[0] = statbuf->fileno[0];
615 new_dim->hdf5_objid.fileno[1] = statbuf->fileno[1];
616 new_dim->hdf5_objid.objno[0] = statbuf->objno[0];
617 new_dim->hdf5_objid.objno[1] = statbuf->objno[1];
621 if (max_scale_size == H5S_UNLIMITED)
622 new_dim->unlimited = NC_TRUE;
627 if (H5DSget_scale_name(datasetid, dimscale_name_att,
NC_MAX_NAME) >= 0)
629 if (!strncmp(dimscale_name_att, DIM_WITHOUT_VARIABLE,
630 strlen(DIM_WITHOUT_VARIABLE)))
632 if (new_dim->unlimited)
634 size_t len = 0, *lenp = &len;
636 if ((retval = nc4_find_dim_len(grp, new_dim->dimid, &lenp)))
638 new_dim->len = *lenp;
642 new_dim->hdf_dimscaleid = datasetid;
643 H5Iinc_ref(new_dim->hdf_dimscaleid);
652 if (attid > 0 && H5Aclose(attid) < 0)
656 if (retval < 0 && dimscale_created)
659 if ((retval = nc4_dim_list_del(&grp->dim, new_dim)))
663 grp->ndims = initial_grp_ndims;
664 grp->nc4_info->next_dimid = initial_next_dimid;
673 read_coord_dimids(NC_VAR_INFO_T *var)
675 hid_t coord_att_typeid = -1, coord_attid = -1, spaceid = -1;
682 if ((coord_attid = H5Aopen_name(var->hdf_datasetid, COORDINATES)) < 0) ret++;
683 if (!ret && (coord_att_typeid = H5Aget_type(coord_attid)) < 0) ret++;
686 if (!ret && (spaceid = H5Aget_space(coord_attid)) < 0) ret++;
690 if (!ret && (npoints = H5Sget_simple_extent_npoints(spaceid)) < 0) ret++;
694 if (!ret && npoints != var->ndims) ret++;
696 if (!ret && H5Aread(coord_attid, coord_att_typeid, var->dimids) < 0) ret++;
697 LOG((4,
"dimscale %s is multidimensional and has coords", var->name));
700 if (spaceid >= 0 && H5Sclose(spaceid) < 0) ret++;
704 if (coord_att_typeid >= 0 && H5Tclose(coord_att_typeid) < 0) ret++;
705 if (coord_attid >= 0 && H5Aclose(coord_attid) < 0) ret++;
712 dimscale_visitor(hid_t did,
unsigned dim, hid_t dsid,
713 void *dimscale_hdf5_objids)
718 if (H5Gget_objinfo(dsid,
".", 1, &statbuf) < 0)
722 (*(HDF5_OBJID_T *)dimscale_hdf5_objids).fileno[0] = statbuf.fileno[0];
723 (*(HDF5_OBJID_T *)dimscale_hdf5_objids).fileno[1] = statbuf.fileno[1];
724 (*(HDF5_OBJID_T *)dimscale_hdf5_objids).objno[0] = statbuf.objno[0];
725 (*(HDF5_OBJID_T *)dimscale_hdf5_objids).objno[1] = statbuf.objno[1];
731 get_netcdf_type(NC_HDF5_FILE_INFO_T *h5, hid_t native_typeid,
734 NC_TYPE_INFO_T *type;
736 htri_t is_str, equal = 0;
740 if ((
class = H5Tget_class(native_typeid)) < 0)
745 if (
class == H5T_STRING)
747 if ((is_str = H5Tis_variable_str(native_typeid)) < 0)
755 else if (
class == H5T_INTEGER ||
class == H5T_FLOAT)
759 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_SCHAR)) < 0)
766 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_SHORT)) < 0)
773 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_INT)) < 0)
780 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_FLOAT)) < 0)
787 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_DOUBLE)) < 0)
794 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_UCHAR)) < 0)
801 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_USHORT)) < 0)
808 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_UINT)) < 0)
815 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_LLONG)) < 0)
822 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_ULLONG)) < 0)
833 if((type = nc4_rec_find_hdf_type(h5->root_grp, native_typeid)))
835 *xtype = type->nc_typeid;
847 get_type_info2(NC_HDF5_FILE_INFO_T *h5, hid_t datasetid,
848 NC_TYPE_INFO_T **type_info)
850 htri_t is_str, equal = 0;
852 hid_t native_typeid, hdf_typeid;
856 assert(h5 && type_info);
864 if (!h5_native_type_constant_g[1])
866 h5_native_type_constant_g[1] = H5T_NATIVE_SCHAR;
867 h5_native_type_constant_g[2] = H5T_NATIVE_SHORT;
868 h5_native_type_constant_g[3] = H5T_NATIVE_INT;
869 h5_native_type_constant_g[4] = H5T_NATIVE_FLOAT;
870 h5_native_type_constant_g[5] = H5T_NATIVE_DOUBLE;
871 h5_native_type_constant_g[6] = H5T_NATIVE_UCHAR;
872 h5_native_type_constant_g[7] = H5T_NATIVE_USHORT;
873 h5_native_type_constant_g[8] = H5T_NATIVE_UINT;
874 h5_native_type_constant_g[9] = H5T_NATIVE_LLONG;
875 h5_native_type_constant_g[10] = H5T_NATIVE_ULLONG;
879 if ((hdf_typeid = H5Dget_type(datasetid)) < 0)
884 if ((native_typeid = H5Tget_native_type(hdf_typeid, H5T_DIR_DEFAULT)) < 0)
888 if ((
class = H5Tget_class(native_typeid)) < 0)
892 if (
class == H5T_STRING ||
class == H5T_INTEGER ||
class == H5T_FLOAT)
895 if (!(*type_info = calloc(1,
sizeof(NC_TYPE_INFO_T))))
900 if (
class == H5T_STRING)
902 if ((is_str = H5Tis_variable_str(native_typeid)) < 0)
905 if (is_str || H5Tget_size(hdf_typeid) > 1)
915 (*type_info)->nc_type_class =
NC_CHAR;
918 else if (
class == H5T_INTEGER ||
class == H5T_FLOAT)
920 for (t = 1; t < NUM_TYPES - 1; t++)
922 if ((equal = H5Tequal(native_typeid, h5_native_type_constant_g[t])) < 0)
934 if((order = H5Tget_order(hdf_typeid)) < 0)
937 if(order == H5T_ORDER_LE)
939 else if(order == H5T_ORDER_BE)
944 if(
class == H5T_INTEGER)
945 (*type_info)->nc_type_class =
NC_INT;
947 (*type_info)->nc_type_class =
NC_FLOAT;
949 (*type_info)->nc_typeid = nc_type_constant_g[t];
950 (*type_info)->size = nc_type_size_g[t];
951 if (!((*type_info)->name = strdup(nc_type_name_g[t])))
953 (*type_info)->hdf_typeid = hdf_typeid;
954 (*type_info)->native_hdf_typeid = native_typeid;
959 NC_TYPE_INFO_T *type;
962 if((type = nc4_rec_find_hdf_type(h5->root_grp, native_typeid)))
968 if (H5Tclose(native_typeid) < 0)
970 if (H5Tclose(hdf_typeid) < 0)
982 read_hdf5_att(NC_GRP_INFO_T *grp, hid_t attid, NC_ATT_INFO_T *att)
984 hid_t spaceid = 0, file_typeid = 0;
985 hsize_t dims[1] = {0};
989 hssize_t att_npoints;
990 H5T_class_t att_class;
991 int fixed_len_string = 0;
992 size_t fixed_size = 0;
995 LOG((5,
"%s: att->attnum %d att->name %s att->nc_typeid %d att->len %d",
996 __func__, att->attnum, att->name, (
int)att->nc_typeid, att->len));
999 if ((file_typeid = H5Aget_type(attid)) < 0)
1001 if ((att->native_hdf_typeid = H5Tget_native_type(file_typeid, H5T_DIR_DEFAULT)) < 0)
1003 if ((att_class = H5Tget_class(att->native_hdf_typeid)) < 0)
1005 if (att_class == H5T_STRING && !H5Tis_variable_str(att->native_hdf_typeid))
1008 if (!(fixed_size = H5Tget_size(att->native_hdf_typeid)))
1011 if ((retval = get_netcdf_type(grp->nc4_info, att->native_hdf_typeid, &(att->nc_typeid))))
1016 if ((spaceid = H5Aget_space(attid)) < 0)
1021 if ((att_ndims = H5Sget_simple_extent_ndims(spaceid)) < 0)
1023 if ((att_npoints = H5Sget_simple_extent_npoints(spaceid)) < 0)
1028 if (att_ndims == 0 && att_npoints == 0)
1031 dims[0] = att_npoints;
1032 else if (att->nc_typeid ==
NC_CHAR)
1038 if (!(dims[0] = H5Tget_size(file_typeid)))
1045 dims[0] = att_npoints;
1050 H5S_class_t space_class;
1057 if ((space_class = H5Sget_simple_extent_type(spaceid)) < 0)
1061 if (H5S_NULL == space_class)
1065 if (H5S_SCALAR == space_class)
1070 if (H5Sget_simple_extent_dims(spaceid, dims, NULL) < 0)
1082 if ((retval = nc4_get_typelen_mem(grp->nc4_info, att->nc_typeid, 0,
1085 if (att_class == H5T_VLEN)
1087 if (!(att->vldata = malloc((
unsigned int)(att->len *
sizeof(hvl_t)))))
1089 if (H5Aread(attid, att->native_hdf_typeid, att->vldata) < 0)
1094 if (!(att->stdata = calloc(att->len,
sizeof(
char *))))
1107 if (fixed_len_string)
1110 char *contig_buf, *cur;
1113 if (!(contig_buf = malloc(att->len * fixed_size *
sizeof(
char))))
1117 if (H5Aread(attid, att->native_hdf_typeid, contig_buf) < 0) {
1126 for (i = 0; i < att->len; i++)
1128 if (!(att->stdata[i] = malloc(fixed_size))) {
1132 strncpy(att->stdata[i], cur, fixed_size);
1142 if (H5Aread(attid, att->native_hdf_typeid, att->stdata) < 0)
1148 if (!(att->data = malloc((
unsigned int)(att->len * type_size))))
1150 if (H5Aread(attid, att->native_hdf_typeid, att->data) < 0)
1155 if (H5Tclose(file_typeid) < 0)
1157 if (H5Sclose(spaceid) < 0)
1166 if (H5Tclose(file_typeid) < 0)
1168 if (spaceid > 0 && H5Sclose(spaceid) < 0)
1179 read_type(NC_GRP_INFO_T *grp, hid_t hdf_typeid,
char *type_name)
1181 NC_TYPE_INFO_T *type;
1183 hid_t native_typeid;
1187 assert(grp && type_name);
1189 LOG((4,
"%s: type_name %s grp->name %s", __func__, type_name, grp->name));
1192 if ((native_typeid = H5Tget_native_type(hdf_typeid, H5T_DIR_DEFAULT)) < 0)
1196 if (!(type_size = H5Tget_size(native_typeid)))
1198 LOG((5,
"type_size %d", type_size));
1201 if ((retval = nc4_type_list_add(grp, type_size, type_name, &type)))
1205 type->committed = NC_TRUE;
1206 type->hdf_typeid = hdf_typeid;
1207 H5Iinc_ref(type->hdf_typeid);
1208 type->native_hdf_typeid = native_typeid;
1211 if ((
class = H5Tget_class(hdf_typeid)) < 0)
1223 char* member_name = NULL;
1230 if ((nmembers = H5Tget_nmembers(hdf_typeid)) < 0)
1232 LOG((5,
"compound type has %d members", nmembers));
1233 for (m = 0; m < nmembers; m++)
1235 hid_t member_hdf_typeid;
1236 hid_t member_native_typeid;
1237 size_t member_offset;
1238 H5T_class_t mem_class;
1244 if ((member_hdf_typeid = H5Tget_member_type(type->native_hdf_typeid, m)) < 0)
1247 if ((member_native_typeid = H5Tget_native_type(member_hdf_typeid, H5T_DIR_DEFAULT)) < 0)
1251 member_name = H5Tget_member_name(type->native_hdf_typeid, m);
1252 if (!member_name || strlen(member_name) >
NC_MAX_NAME) {
1258 strncpy(jna,member_name,1000);
1264 member_offset = H5Tget_member_offset(type->native_hdf_typeid, m);
1267 if ((mem_class = H5Tget_class(member_hdf_typeid)) < 0)
1269 if (mem_class == H5T_ARRAY)
1275 if ((ndims = H5Tget_array_ndims(member_hdf_typeid)) < 0) {
1279 if (H5Tget_array_dims(member_hdf_typeid, dims, NULL) != ndims) {
1283 for (d = 0; d < ndims; d++)
1284 dim_size[d] = dims[d];
1287 if ((retval = get_netcdf_type(grp->nc4_info, H5Tget_super(member_hdf_typeid),
1292 if ((retval = nc4_field_list_add(&type->u.c.field, type->u.c.num_fields++, member_name,
1293 member_offset, H5Tget_super(member_hdf_typeid),
1294 H5Tget_super(member_native_typeid),
1295 member_xtype, ndims, dim_size)))
1301 if ((retval = get_netcdf_type(grp->nc4_info, member_native_typeid,
1306 if ((retval = nc4_field_list_add(&type->u.c.field, type->u.c.num_fields++, member_name,
1307 member_offset, member_hdf_typeid, member_native_typeid,
1308 member_xtype, 0, NULL)))
1319 #ifdef HDF5_HAS_H5FREE
1320 if(member_name != NULL) H5free_memory(member_name);
1323 if(member_name != NULL) free(member_name);
1330 if(member_name != NULL)
1346 if ((ret = H5Tis_variable_str(hdf_typeid)) < 0)
1352 hid_t base_hdf_typeid;
1355 type->nc_type_class =
NC_VLEN;
1359 if (!(base_hdf_typeid = H5Tget_super(native_typeid)))
1363 if (!(type_size = H5Tget_size(base_hdf_typeid)))
1367 if ((retval = get_netcdf_type(grp->nc4_info, base_hdf_typeid,
1370 LOG((5,
"base_hdf_typeid 0x%x type_size %d base_nc_type %d",
1371 base_hdf_typeid, type_size, base_nc_type));
1374 type->u.v.base_nc_typeid = base_nc_type;
1375 type->u.v.base_hdf_typeid = base_hdf_typeid;
1386 hid_t base_hdf_typeid;
1390 char *member_name = NULL;
1395 type->nc_type_class =
NC_ENUM;
1399 if (!(base_hdf_typeid = H5Tget_super(hdf_typeid)))
1402 if (!(type_size = H5Tget_size(base_hdf_typeid)))
1405 if ((retval = get_netcdf_type(grp->nc4_info, base_hdf_typeid,
1408 LOG((5,
"base_hdf_typeid 0x%x type_size %d base_nc_type %d",
1409 base_hdf_typeid, type_size, base_nc_type));
1412 type->u.e.base_nc_typeid = base_nc_type;
1413 type->u.e.base_hdf_typeid = base_hdf_typeid;
1416 if ((type->u.e.num_members = H5Tget_nmembers(hdf_typeid)) < 0)
1420 if (!(value = calloc(1, type_size)))
1424 for (i = 0; i < type->u.e.num_members; i++)
1428 if (!(member_name = H5Tget_member_name(hdf_typeid, i)))
1434 strncpy(jna,member_name,1000);
1443 if (H5Tget_member_value(hdf_typeid, i, value) < 0)
1450 if ((retval = nc4_enum_member_add(&type->u.e.enum_member, type->size,
1451 member_name, value)))
1458 if(member_name != NULL) free(member_name);
1464 if(member_name != NULL)
1467 if(value) free(value);
1474 LOG((0,
"unknown class"));
1485 read_var(NC_GRP_INFO_T *grp, hid_t datasetid,
const char *obj_name,
1486 size_t ndims, NC_DIM_INFO_T *dim)
1488 NC_VAR_INFO_T *var = NULL;
1489 hid_t access_pid = 0;
1495 char att_name[NC_MAX_HDF5_NAME + 1];
1497 #define CD_NELEMS_ZLIB 1
1498 #define CD_NELEMS_SZIP 4
1499 H5Z_filter_t filter;
1501 unsigned int cd_values[CD_NELEMS_SZIP];
1502 size_t cd_nelems = CD_NELEMS_SZIP;
1504 H5D_fill_value_t fill_status;
1505 H5D_layout_t layout;
1511 assert(obj_name && grp);
1512 LOG((4,
"%s: obj_name %s", __func__, obj_name));
1515 if ((retval = nc4_var_list_add(&grp->var, &var)))
1519 var->hdf_datasetid = datasetid;
1520 H5Iinc_ref(var->hdf_datasetid);
1522 var->varid = grp->nvars++;
1523 var->created = NC_TRUE;
1530 if (!(var->dim = calloc(var->ndims,
sizeof(NC_DIM_INFO_T *))))
1532 if (!(var->dimids = calloc(var->ndims,
sizeof(
int))))
1537 if ((access_pid = H5Dget_access_plist(datasetid)) < 0)
1544 if ((H5Pget_chunk_cache(access_pid, &(var->chunk_cache_nelems),
1545 &(var->chunk_cache_size), &rdcc_w0)) < 0)
1547 var->chunk_cache_preemption = rdcc_w0;
1552 if (strlen(obj_name) > strlen(NON_COORD_PREPEND) &&
1553 !strncmp(obj_name, NON_COORD_PREPEND, strlen(NON_COORD_PREPEND)))
1556 if (!(var->name = malloc(((strlen(obj_name) - strlen(NON_COORD_PREPEND))+ 1) *
sizeof(
char))))
1559 strcpy(var->name, &obj_name[strlen(NON_COORD_PREPEND)]);
1562 if (!(var->hdf5_name = malloc((strlen(obj_name) + 1) *
sizeof(
char))))
1565 strcpy(var->hdf5_name, obj_name);
1570 if (!(var->name = malloc((strlen(obj_name) + 1) *
sizeof(
char))))
1573 strcpy(var->name, obj_name);
1579 if ((propid = H5Dget_create_plist(datasetid)) < 0)
1586 if ((layout = H5Pget_layout(propid)) < -1)
1588 if (layout == H5D_CHUNKED)
1592 if (!(var->chunksizes = malloc(var->ndims *
sizeof(
size_t))))
1594 for (d = 0; d < var->ndims; d++)
1595 var->chunksizes[d] = chunksize[d];
1597 else if (layout == H5D_CONTIGUOUS || layout == H5D_COMPACT)
1598 var->contiguous = NC_TRUE;
1602 if ((num_filters = H5Pget_nfilters(propid)) < 0)
1604 for (f = 0; f < num_filters; f++)
1606 if ((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems,
1607 cd_values, 0, NULL, NULL)) < 0)
1611 case H5Z_FILTER_SHUFFLE:
1612 var->shuffle = NC_TRUE;
1615 case H5Z_FILTER_FLETCHER32:
1616 var->fletcher32 = NC_TRUE;
1619 case H5Z_FILTER_DEFLATE:
1620 var->deflate = NC_TRUE;
1621 if (cd_nelems != CD_NELEMS_ZLIB || cd_values[0] > MAX_DEFLATE_LEVEL)
1623 var->deflate_level = cd_values[0];
1626 case H5Z_FILTER_SZIP:
1627 var->szip = NC_TRUE;
1628 if (cd_nelems != CD_NELEMS_SZIP)
1630 var->options_mask = cd_values[0];
1631 var->pixels_per_block = cd_values[1];
1635 LOG((1,
"Yikes! Unknown filter type found on dataset!"));
1641 if ((retval = get_type_info2(grp->nc4_info, datasetid,
1646 var->type_info->rc++;
1649 if (H5Pfill_value_defined(propid, &fill_status) < 0)
1653 if (fill_status == H5D_FILL_VALUE_USER_DEFINED)
1656 if (!var->fill_value)
1658 if (var->type_info->nc_type_class ==
NC_VLEN)
1660 if (!(var->fill_value = malloc(
sizeof(
nc_vlen_t))))
1663 else if (var->type_info->nc_type_class ==
NC_STRING)
1665 if (!(var->fill_value = malloc(
sizeof(
char *))))
1670 assert(var->type_info->size);
1671 if (!(var->fill_value = malloc(var->type_info->size)))
1677 if (H5Pget_fill_value(propid, var->type_info->native_hdf_typeid,
1678 var->fill_value) < 0)
1682 var->no_fill = NC_TRUE;
1688 var->dimscale = NC_TRUE;
1691 if ((retval = read_coord_dimids(var)))
1697 assert(0 == strcmp(var->name, dim->name));
1699 var->dimids[0] = dim->dimid;
1702 dim->coord_var = var;
1714 num_scales = H5DSget_num_scales(datasetid, 0);
1718 if (num_scales && ndims)
1722 if (NULL == (var->dimscale_attached = calloc(ndims,
sizeof(nc_bool_t))))
1727 if (NULL == (var->dimscale_hdf5_objids = malloc(ndims *
sizeof(
struct hdf5_objid))))
1729 for (d = 0; d < var->ndims; d++)
1731 if (H5DSiterate_scales(var->hdf_datasetid, d, NULL, dimscale_visitor,
1732 &(var->dimscale_hdf5_objids[d])) < 0)
1734 var->dimscale_attached[d] = NC_TRUE;
1741 if ((natts = H5Aget_num_attrs(var->hdf_datasetid)) < 0)
1743 for (a = 0; a < natts; a++)
1748 if (attid && H5Aclose(attid) < 0)
1752 if ((attid = H5Aopen_idx(var->hdf_datasetid, (
unsigned int)a)) < 0)
1754 if (H5Aget_name(attid, NC_MAX_HDF5_NAME, att_name) < 0)
1756 LOG((4,
"%s:: a %d att_name %s", __func__, a, att_name));
1759 if (strcmp(att_name, REFERENCE_LIST) &&
1760 strcmp(att_name, CLASS) &&
1761 strcmp(att_name, DIMENSION_LIST) &&
1762 strcmp(att_name, NAME) &&
1763 strcmp(att_name, COORDINATES) &&
1764 strcmp(att_name, NC_DIMID_ATT_NAME))
1767 if ((retval = nc4_att_list_add(&var->att, &att)))
1771 att->attnum = var->natts++;
1772 if (!(att->name = strdup(att_name)))
1777 if ((retval = read_hdf5_att(grp, attid, att)))
1781 if ((retval = nc4_att_list_del(&var->att, att)))
1789 att->created = NC_TRUE;
1795 if ((retval = nc4_adjust_var_cache(grp, var)))
1801 if (incr_id_rc && H5Idec_ref(datasetid) < 0)
1803 if (var && nc4_var_list_del(&grp->var, var))
1806 if (access_pid && H5Pclose(access_pid) < 0)
1811 if (propid > 0 && H5Pclose(propid) < 0)
1816 if (attid > 0 && H5Aclose(attid) < 0)
1824 read_grp_atts(NC_GRP_INFO_T *grp)
1829 NC_TYPE_INFO_T *type;
1830 char obj_name[NC_MAX_HDF5_NAME + 1];
1834 num_obj = H5Aget_num_attrs(grp->hdf_grpid);
1835 for (i = 0; i < num_obj; i++)
1839 if (attid && H5Aclose(attid) < 0)
1842 if ((attid = H5Aopen_idx(grp->hdf_grpid, (
unsigned int)i)) < 0)
1844 if (H5Aget_name(attid,
NC_MAX_NAME + 1, obj_name) < 0)
1846 LOG((3,
"reading attribute of _netCDF group, named %s", obj_name));
1852 if (!strcmp(obj_name, NC3_STRICT_ATT_NAME))
1857 if ((retval = nc4_att_list_add(&grp->att, &att)))
1862 if (!(att->name = malloc((max_len + 1) *
sizeof(
char))))
1864 strncpy(att->name, obj_name, max_len);
1865 att->name[max_len] = 0;
1866 att->attnum = grp->natts++;
1867 if ((retval = read_hdf5_att(grp, attid, att)))
1871 if ((retval = nc4_att_list_del(&grp->att, att)))
1878 att->created = NC_TRUE;
1879 if ((retval = nc4_find_type(grp->nc4_info, att->nc_typeid, &type)))
1885 if (attid > 0 && H5Aclose(attid) < 0)
1893 read_dataset(NC_GRP_INFO_T *grp, hid_t datasetid,
const char *obj_name,
1894 const H5G_stat_t *statbuf)
1896 NC_DIM_INFO_T *dim = NULL;
1903 if ((spaceid = H5Dget_space(datasetid)) < 0)
1908 if ((ndims = H5Sget_simple_extent_ndims(spaceid)) < 0)
1912 if ((is_scale = H5DSis_scale(datasetid)) < 0)
1916 hsize_t dims[H5S_MAX_RANK];
1917 hsize_t max_dims[H5S_MAX_RANK];
1920 if (H5Sget_simple_extent_dims(spaceid, dims, max_dims) < 0)
1924 if ((retval = read_scale(grp, datasetid, obj_name, statbuf, dims[0],
1925 max_dims[0], &dim)))
1932 if (NULL == dim || (dim && !dim->hdf_dimscaleid))
1933 if ((retval = read_var(grp, datasetid, obj_name, ndims, dim)))
1937 if (spaceid && H5Sclose(spaceid) <0)
1954 if (!(new_oinfo = calloc(1,
sizeof(*new_oinfo))))
1958 memcpy(new_oinfo, oinfo,
sizeof(*oinfo));
1963 (*tail)->next = new_oinfo;
1968 assert(NULL == *head);
1969 *head = *tail = new_oinfo;
1976 nc4_rec_read_metadata_cb(hid_t grpid,
const char *name,
const H5L_info_t *info,
1981 int retval = H5_ITER_CONT;
1984 memset(&oinfo, 0,
sizeof(oinfo));
1987 if ((oinfo.oid = H5Oopen(grpid, name, H5P_DEFAULT)) < 0)
1988 BAIL(H5_ITER_ERROR);
1991 if (H5Gget_objinfo(oinfo.oid,
".", 1, &oinfo.statbuf) < 0)
1992 BAIL(H5_ITER_ERROR);
1997 switch(oinfo.statbuf.type)
2000 LOG((3,
"found group %s", oinfo.oname));
2006 if (nc4_rec_read_metadata_cb_list_add(&udata->grps_head, &udata->grps_tail, &oinfo))
2007 BAIL(H5_ITER_ERROR);
2011 LOG((3,
"found dataset %s", oinfo.oname));
2015 if ((retval = read_dataset(udata->grp, oinfo.oid, oinfo.oname, &oinfo.statbuf)))
2023 BAIL(H5_ITER_ERROR);
2025 retval = H5_ITER_CONT;
2029 if (H5Oclose(oinfo.oid) < 0)
2030 BAIL(H5_ITER_ERROR);
2034 LOG((3,
"found datatype %s", oinfo.oname));
2037 if (read_type(udata->grp, oinfo.oid, oinfo.oname))
2038 BAIL(H5_ITER_ERROR);
2041 if (H5Oclose(oinfo.oid) < 0)
2042 BAIL(H5_ITER_ERROR);
2046 LOG((0,
"Unknown object class %d in %s!", oinfo.statbuf.type, __func__));
2047 BAIL(H5_ITER_ERROR);
2053 if (oinfo.oid > 0 && H5Oclose(oinfo.oid) < 0)
2054 BAIL2(H5_ITER_ERROR);
2068 nc4_rec_read_metadata(NC_GRP_INFO_T *grp)
2074 unsigned crt_order_flags = 0;
2075 H5_index_t iter_index;
2078 assert(grp && grp->name);
2079 LOG((3,
"%s: grp->name %s", __func__, grp->name));
2082 memset(&udata, 0,
sizeof(udata));
2086 if (!grp->hdf_grpid)
2090 if ((grp->hdf_grpid = H5Gopen2(grp->parent->hdf_grpid,
2091 grp->name, H5P_DEFAULT)) < 0)
2096 if ((grp->hdf_grpid = H5Gopen2(grp->nc4_info->hdfid,
2097 "/", H5P_DEFAULT)) < 0)
2101 assert(grp->hdf_grpid > 0);
2104 pid = H5Gget_create_plist(grp->hdf_grpid);
2105 H5Pget_link_creation_order(pid, &crt_order_flags);
2106 if (H5Pclose(pid) < 0)
2110 if (crt_order_flags & H5P_CRT_ORDER_TRACKED)
2111 iter_index = H5_INDEX_CRT_ORDER;
2114 NC_HDF5_FILE_INFO_T *h5 = grp->nc4_info;
2120 iter_index = H5_INDEX_NAME;
2129 if (H5Literate(grp->hdf_grpid, iter_index, H5_ITER_INC, &idx,
2130 nc4_rec_read_metadata_cb, (
void *)&udata) < 0)
2137 for (oinfo = udata.grps_head; oinfo; oinfo = udata.grps_head)
2139 NC_GRP_INFO_T *child_grp;
2140 NC_HDF5_FILE_INFO_T *h5 = grp->nc4_info;
2143 if ((retval = nc4_grp_list_add(&(grp->children), h5->next_nc_grpid++,
2144 grp, grp->nc4_info->controller, oinfo->oname, &child_grp)))
2148 if ((retval = nc4_rec_read_metadata(child_grp)))
2152 if (H5Oclose(oinfo->oid) < 0)
2156 udata.grps_head = oinfo->next;
2161 if ((retval = read_grp_atts(grp)))
2168 for (oinfo = udata.grps_head; oinfo; oinfo = udata.grps_head)
2171 if (H5Oclose(oinfo->oid) < 0)
2175 udata.grps_head = oinfo->next;
2187 nc4_open_file(
const char *path,
int mode,
void* parameters, NC *nc)
2189 hid_t fapl_id = H5P_DEFAULT;
2190 unsigned flags = (mode &
NC_WRITE) ?
2191 H5F_ACC_RDWR : H5F_ACC_RDONLY;
2193 NC_HDF5_FILE_INFO_T* nc4_info = NULL;
2196 NC_MEM_INFO* meminfo = (NC_MEM_INFO*)parameters;
2198 #ifdef USE_PARALLEL4
2199 NC_MPI_INFO* mpiinfo = (NC_MPI_INFO*)parameters;
2204 LOG((3,
"%s: path %s mode %d", __func__, path, mode));
2208 if ((retval = nc4_nc4f_list_add(nc, path, mode)))
2210 nc4_info = NC4_DATA(nc);
2211 assert(nc4_info && nc4_info->root_grp);
2216 if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
2222 if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI))
2225 if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_STRONG))
2229 #ifdef USE_PARALLEL4
2232 if (mode & NC_MPIIO || mode & NC_MPIPOSIX)
2234 nc4_info->parallel = NC_TRUE;
2235 if (mode & NC_MPIIO)
2237 LOG((4,
"opening parallel file with MPI/IO"));
2238 if (H5Pset_fapl_mpio(fapl_id, mpiinfo->comm, mpiinfo->info) < 0)
2241 #ifdef USE_PARALLEL_POSIX
2244 LOG((4,
"opening parallel file with MPI/posix"));
2245 if (H5Pset_fapl_mpiposix(fapl_id, mpiinfo->comm, 0) < 0)
2258 if (MPI_SUCCESS != MPI_Comm_dup(mpiinfo->comm, &nc4_info->comm))
2261 if (MPI_INFO_NULL != mpiinfo->info)
2263 if (MPI_SUCCESS != MPI_Info_dup(mpiinfo->info, &nc4_info->info))
2270 nc4_info->info = mpiinfo->info;
2274 if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size,
2275 nc4_chunk_cache_preemption) < 0)
2277 LOG((4,
"%s: set HDF raw chunk cache to size %d nelems %d preemption %f",
2278 __func__, nc4_chunk_cache_size, nc4_chunk_cache_nelems, nc4_chunk_cache_preemption));
2285 if((nc4_info->hdfid = H5LTopen_file_image(meminfo->memory,meminfo->size,
2286 H5LT_FILE_IMAGE_DONT_COPY|H5LT_FILE_IMAGE_DONT_RELEASE
2289 nc4_info->no_write = NC_TRUE;
2290 }
else if ((nc4_info->hdfid = H5Fopen(path, flags, fapl_id)) < 0)
2295 nc4_info->no_write = NC_TRUE;
2301 if ((retval = nc4_rec_read_metadata(nc4_info->root_grp)))
2306 if ((retval = nc4_rec_match_dimscales(nc4_info->root_grp)))
2312 log_metadata_nc(nc);
2316 if (H5Pclose(fapl_id) < 0)
2325 #ifdef USE_PARALLEL4
2326 if (comm_duped) MPI_Comm_free(&nc4_info->comm);
2327 if (info_duped) MPI_Info_free(&nc4_info->info);
2332 if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id);
2333 if (!nc4_info)
return retval;
2334 close_netcdf4_file(nc4_info,1);
2353 get_netcdf_type_from_hdf4(NC_HDF5_FILE_INFO_T *h5, int32 hdf4_typeid,
2354 nc_type *xtype, NC_TYPE_INFO_T *type_info)
2366 assert(h5 && xtype);
2454 if (hdf4_typeid == DFNT_FLOAT32)
2455 type_info->nc_type_class =
NC_FLOAT;
2456 else if (hdf4_typeid == DFNT_FLOAT64)
2458 else if (hdf4_typeid == DFNT_CHAR)
2461 type_info->nc_type_class =
NC_INT;
2462 type_info->endianness = endianness;
2463 type_info->nc_typeid = *xtype;
2464 type_info->size = nc_type_size_g[t];
2465 if (!(type_info->name = strdup(nc_type_name_g[t])))
2475 nc4_open_hdf4_file(
const char *path,
int mode, NC *nc)
2477 NC_HDF5_FILE_INFO_T *h5;
2480 int32 num_datasets, num_gatts;
2484 NC_HDF5_FILE_INFO_T* nc4_info = NULL;
2486 LOG((3,
"%s: path %s mode %d", __func__, path, mode));
2490 if (mode & NC_WRITE)
2494 if ((retval = nc4_nc4f_list_add(nc, path, mode)))
2496 nc4_info = NC4_DATA(nc);
2497 assert(nc4_info && nc4_info->root_grp);
2501 h5->no_write = NC_TRUE;
2504 if ((h5->sdid = SDstart(path, DFACC_READ)) == FAIL)
2508 if (SDfileinfo(h5->sdid, &num_datasets, &num_gatts))
2512 for (a = 0; a < num_gatts; a++)
2514 int32 att_data_type, att_count;
2515 size_t att_type_size;
2518 if ((retval = nc4_att_list_add(&h5->root_grp->att, &att)))
2520 att->attnum = grp->natts++;
2521 att->created = NC_TRUE;
2526 if (SDattrinfo(h5->sdid, a, att->name, &att_data_type, &att_count))
2528 if ((retval = get_netcdf_type_from_hdf4(h5, att_data_type,
2529 &att->nc_typeid, NULL)))
2531 att->len = att_count;
2534 if ((retval = nc4_get_typelen_mem(h5, att->nc_typeid, 0, &att_type_size)))
2536 if (!(att->data = malloc(att_type_size * att->len)))
2540 if (SDreadattr(h5->sdid, a, att->data))
2545 for (v = 0; v < num_datasets; v++)
2548 int32 data_type, num_atts;
2557 int32 *dimsize = NULL;
2558 size_t var_type_size;
2562 if ((retval = nc4_var_list_add(&grp->var, &var)))
2565 var->varid = grp->nvars++;
2566 var->created = NC_TRUE;
2567 var->written_to = NC_TRUE;
2570 if ((var->sdsid = SDselect(h5->sdid, v)) == FAIL)
2578 if (SDgetinfo(var->sdsid, var->name, &rank, NULL, &data_type, &num_atts))
2581 if(!(dimsize = (int32*)malloc(
sizeof(int32)*rank)))
2584 if (SDgetinfo(var->sdsid, var->name, &rank, dimsize, &data_type, &num_atts)) {
2585 if(dimsize) free(dimsize);
2590 var->hdf4_data_type = data_type;
2593 if (!(var->type_info = calloc(1,
sizeof(NC_TYPE_INFO_T)))) {
2594 if(dimsize) free(dimsize);
2598 if ((retval = get_netcdf_type_from_hdf4(h5, data_type, &var->type_info->nc_typeid, var->type_info))) {
2599 if(dimsize) free(dimsize);
2604 var->type_info->rc++;
2606 if ((retval = nc4_get_typelen_mem(h5, var->type_info->nc_typeid, 0, &var_type_size))) {
2607 if(dimsize) free(dimsize);
2611 var->type_info->size = var_type_size;
2612 LOG((3,
"reading HDF4 dataset %s, rank %d netCDF type %d", var->name,
2613 rank, var->type_info->nc_typeid));
2616 if (!(var->fill_value = malloc(var_type_size))) {
2617 if(dimsize) free(dimsize);
2621 if (SDgetfillvalue(var->sdsid, var->fill_value))
2624 free(var->fill_value);
2625 var->fill_value = NULL;
2631 if (!(var->dim = malloc(
sizeof(NC_DIM_INFO_T *) * var->ndims))) {
2632 if(dimsize) free(dimsize);
2636 if (!(var->dimids = malloc(
sizeof(
int) * var->ndims))) {
2637 if(dimsize) free(dimsize);
2644 for (d = 0; d < var->ndims; d++)
2646 int32 dimid, dim_len, dim_data_type, dim_num_attrs;
2650 if ((dimid = SDgetdimid(var->sdsid, d)) == FAIL) {
2651 if(dimsize) free(dimsize);
2654 if (SDdiminfo(dimid, dim_name, &dim_len, &dim_data_type,
2657 if(dimsize) free(dimsize);
2663 for (dim = grp->dim; dim; dim = dim->l.next)
2664 if (!strcmp(dim->name, dim_name))
2670 LOG((4,
"adding dimension %s for HDF4 dataset %s",
2671 dim_name, var->name));
2672 if ((retval = nc4_dim_list_add(&grp->dim, &dim)))
2675 dim->dimid = grp->nc4_info->next_dimid++;
2678 if (!(dim->name = strdup(dim_name)))
2683 dim->len = *dimsize;
2687 var->dimids[d] = dim->dimid;
2691 for (a = 0; a < num_atts; a++)
2693 int32 att_data_type, att_count;
2694 size_t att_type_size;
2697 if ((retval = nc4_att_list_add(&var->att, &att))) {
2698 if(dimsize) free(dimsize);
2701 att->attnum = var->natts++;
2702 att->created = NC_TRUE;
2706 if(dimsize) free(dimsize);
2709 if (SDattrinfo(var->sdsid, a, att->name, &att_data_type, &att_count)) {
2710 if(dimsize) free(dimsize);
2713 if ((retval = get_netcdf_type_from_hdf4(h5, att_data_type,
2714 &att->nc_typeid, NULL))) {
2715 if(dimsize) free(dimsize);
2719 att->len = att_count;
2722 if ((retval = nc4_get_typelen_mem(h5, att->nc_typeid, 0, &att_type_size))) {
2723 if(dimsize) free(dimsize);
2726 if (!(att->data = malloc(att_type_size * att->len))) {
2727 if(dimsize) free(dimsize);
2732 if (SDreadattr(var->sdsid, a, att->data)) {
2733 if(dimsize) free(dimsize);
2737 if(dimsize) free(dimsize);
2741 HDF_CHUNK_DEF chunkdefs;
2743 if(!SDgetchunkinfo(var->sdsid, &chunkdefs, &flag)) {
2744 if(flag == HDF_NONE)
2745 var->contiguous = NC_TRUE;
2746 else if((flag & HDF_CHUNK) != 0) {
2747 var->contiguous = NC_FALSE;
2748 if (!(var->chunksizes = malloc(var->ndims *
sizeof(
size_t))))
2750 for (d = 0; d < var->ndims; d++) {
2751 var->chunksizes[d] = chunkdefs.chunk_lengths[d];
2762 log_metadata_nc(h5->root_grp->nc4_info->controller);
2770 NC4_open(
const char *path,
int mode,
int basepe,
size_t *chunksizehintp,
2771 int use_parallel,
void *parameters, NC_Dispatch *dispatch, NC *nc_file)
2775 #ifdef USE_PARALLEL4
2776 NC_MPI_INFO mpidfalt = {MPI_COMM_WORLD, MPI_INFO_NULL};
2778 #if defined USE_PARALLEL4 || defined USE_HDF4
2782 assert(nc_file && path);
2784 LOG((1,
"%s: path %s mode %d params %x",
2785 __func__, path, mode, parameters));
2787 #ifdef USE_PARALLEL4
2788 if (!inmemory && use_parallel && parameters == NULL)
2789 parameters = &mpidfalt;
2795 if (H5Eset_auto(NULL, NULL) < 0)
2796 LOG((0,
"Couldn't turn off HDF5 error messages!"));
2797 LOG((1,
"HDF5 error messages turned off!"));
2803 if((mode & ILLEGAL_OPEN_FLAGS) != 0)
2807 if((mode & (NC_MPIIO|NC_MPIPOSIX)) == (NC_MPIIO|NC_MPIPOSIX))
2810 #ifndef USE_PARALLEL_POSIX
2814 if(mode & NC_MPIPOSIX)
2816 mode &= ~NC_MPIPOSIX;
2822 if ((res = nc_check_for_hdf(path, use_parallel, parameters, &hdf_file)))
2826 nc_file->int_ncid = nc_file->ext_ncid;
2827 if (hdf_file == NC_HDF5_FILE)
2828 res = nc4_open_file(path, mode, parameters, nc_file);
2830 else if (hdf_file == NC_HDF4_FILE && inmemory)
2832 else if (hdf_file == NC_HDF4_FILE)
2833 res = nc4_open_hdf4_file(path, mode, nc_file);
2848 NC4_set_fill(
int ncid,
int fillmode,
int *old_modep)
2851 NC_HDF5_FILE_INFO_T* nc4_info;
2853 LOG((2,
"%s: ncid 0x%x fillmode %d", __func__, ncid, fillmode));
2855 if (!(nc = nc4_find_nc_file(ncid,&nc4_info)))
2860 if (nc4_info->no_write)
2869 *old_modep = nc4_info->fill_mode;
2871 nc4_info->fill_mode = fillmode;
2875 if (nc4_info->pnetcdf_file)
2876 return ncmpi_set_fill(nc->int_ncid, fillmode, old_modep);
2888 NC_HDF5_FILE_INFO_T* nc4_info;
2890 LOG((1,
"%s: ncid 0x%x", __func__, ncid));
2893 if (!(nc4_find_nc_file(ncid,&nc4_info)))
2899 if (nc4_info->pnetcdf_file)
2900 return ncmpi_redef(nc->int_ncid);
2904 if (nc4_info->flags & NC_INDEF)
2908 if (nc4_info->no_write)
2912 nc4_info->flags |= NC_INDEF;
2916 nc4_info->redef = NC_TRUE;
2924 NC4__enddef(
int ncid,
size_t h_minfree,
size_t v_align,
2925 size_t v_minfree,
size_t r_align)
2927 if (nc4_find_nc_file(ncid,NULL) == NULL)
2930 return NC4_enddef(ncid);
2935 static int NC4_enddef(
int ncid)
2938 NC_HDF5_FILE_INFO_T* nc4_info;
2940 LOG((1,
"%s: ncid 0x%x", __func__, ncid));
2942 if (!(nc = nc4_find_nc_file(ncid,&nc4_info)))
2947 if (nc4_info->pnetcdf_file)
2950 res = ncmpi_enddef(nc->int_ncid);
2953 if (nc4_info->pnetcdf_access_mode == NC_INDEPENDENT)
2954 res = ncmpi_begin_indep_data(nc->int_ncid);
2960 return nc4_enddef_netcdf4_file(nc4_info);
2966 sync_netcdf4_file(NC_HDF5_FILE_INFO_T *h5)
2971 LOG((3,
"%s", __func__));
2975 if (h5->flags & NC_INDEF)
2981 h5->flags ^= NC_INDEF;
2984 h5->redef = NC_FALSE;
2990 log_metadata_nc(h5->root_grp->nc4_info->controller);
2996 nc_bool_t bad_coord_order = NC_FALSE;
2998 if ((retval = nc4_rec_write_groups_types(h5->root_grp)))
3000 if ((retval = nc4_rec_detect_need_to_preserve_dimids(h5->root_grp, &bad_coord_order)))
3002 if ((retval = nc4_rec_write_metadata(h5->root_grp, bad_coord_order)))
3006 if (H5Fflush(h5->hdfid, H5F_SCOPE_GLOBAL) < 0)
3019 NC_HDF5_FILE_INFO_T* nc4_info;
3021 LOG((2,
"%s: ncid 0x%x", __func__, ncid));
3023 if (!(nc = nc4_find_nc_file(ncid,&nc4_info)))
3029 if (nc4_info->pnetcdf_file)
3030 return ncmpi_sync(nc->int_ncid);
3034 if (nc4_info && nc4_info->flags & NC_INDEF)
3038 if ((retval = NC4_enddef(ncid)))
3042 return sync_netcdf4_file(nc4_info);
3049 close_netcdf4_file(NC_HDF5_FILE_INFO_T *h5,
int abort)
3053 assert(h5 && h5->root_grp);
3054 LOG((3,
"%s: h5->path %s abort %d", __func__, h5->controller->path, abort));
3057 if (h5->flags & NC_INDEF)
3058 h5->flags ^= NC_INDEF;
3062 if (!h5->no_write && !abort)
3063 if ((retval = sync_netcdf4_file(h5)))
3068 if ((retval = nc4_rec_grp_del(&h5->root_grp, h5->root_grp)))
3075 if (SDend(h5->sdid))
3081 #ifdef USE_PARALLEL4
3085 MPI_Comm_free(&h5->comm);
3086 if(MPI_INFO_NULL != h5->info)
3087 MPI_Info_free(&h5->info);
3090 if (H5Fclose(h5->hdfid) < 0)
3094 nobjs = H5Fget_obj_count(h5->hdfid, H5F_OBJ_ALL);
3098 }
else if(nobjs > 0) {
3104 LOG((0,
"There are %d HDF5 objects open!", nobjs));
3130 int delete_file = 0;
3133 NC_HDF5_FILE_INFO_T* nc4_info;
3135 LOG((2,
"%s: ncid 0x%x", __func__, ncid));
3138 if (!(nc = nc4_find_nc_file(ncid,&nc4_info)))
3145 if (nc4_info->pnetcdf_file)
3146 return ncmpi_abort(nc->int_ncid);
3150 if (nc4_info->flags & NC_INDEF && !nc4_info->redef)
3158 if ((retval = close_netcdf4_file(nc4_info, 1)))
3163 if (
remove(path) < 0)
3175 NC_HDF5_FILE_INFO_T *h5;
3178 LOG((1,
"%s: ncid 0x%x", __func__, ncid));
3181 if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5)))
3184 assert(nc && h5 && grp);
3192 if (h5->pnetcdf_file)
3193 return ncmpi_close(nc->int_ncid);
3197 if ((retval = close_netcdf4_file(grp->nc4_info, 0)))
3206 NC4_inq(
int ncid,
int *ndimsp,
int *nvarsp,
int *nattsp,
int *unlimdimidp)
3209 NC_HDF5_FILE_INFO_T *h5;
3216 LOG((2,
"%s: ncid 0x%x", __func__, ncid));
3219 if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5)))
3222 assert(h5 && grp && nc);
3226 if (h5->pnetcdf_file)
3227 return ncmpi_inq(nc->int_ncid, ndimsp, nvarsp, nattsp, unlimdimidp);
3234 for (dim = grp->dim; dim; dim = dim->l.next)
3240 for (var = grp->var; var; var= var->l.next)
3246 for (att = grp->att; att; att = att->l.next)
3259 for (dim = grp->dim; dim; dim = dim->l.next)
3262 *unlimdimidp = dim->dimid;
3272 NC4_set_content(
int ncid,
size_t size,
void* memory)
3277 NC_HDF5_FILE_INFO_T *h5;
3280 LOG((4,
"%s: ncid 0x%x size %ld memory 0x%x", __func__, ncid, size, memory));
3283 if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5)))
3285 assert(h5 && grp && nc);
3288 herr = H5Pset_file_image(h5->hdfid,memory,size);
3302 nc4_enddef_netcdf4_file(NC_HDF5_FILE_INFO_T *h5)
3305 LOG((3,
"%s", __func__));
3308 if (!(h5->flags & NC_INDEF))
3312 h5->flags ^= NC_INDEF;
3315 h5->redef = NC_FALSE;
3317 return sync_netcdf4_file(h5);
3324 if (num_plists || num_spaces)
3331 #ifdef USE_PARALLEL4
3333 nc_use_parallel_enabled()
#define NC_PNETCDF
Use parallel-netcdf library; alias for NC_MPIIO.
#define NC_ENOMEM
Memory allocation (malloc) failure.
#define NC_CHAR
ISO/ASCII character.
#define NC_ECANTWRITE
Can't write.
int NC4_create(const char *path, int cmode, size_t initialsz, int basepe, size_t *chunksizehintp, int use_parallel, void *parameters, NC_Dispatch *dispatch, NC *nc_file)
Create a netCDF-4/HDF5 file.
#define NC_UBYTE
unsigned 1 byte int
#define NC_CLASSIC_MODEL
Enforce classic model on netCDF-4.
#define NC_MAX_VAR_DIMS
max per variable dimensions
#define NC_UINT
unsigned 4-byte int
#define NC_NOCLOBBER
Don't destroy existing file.
#define NC_INMEMORY
Read from memory.
#define NC_EHDFERR
Error at HDF5 layer.
#define NC_OPAQUE
opaque types
#define NC_MPIIO
Turn on MPI I/O.
#define NC_INT64
signed 8-byte int
#define NC_ENOTINDEFINE
Operation not allowed in data mode.
#define NC_DOUBLE
double precision floating point number
#define NC_EBADCLASS
Bad class.
int nc_type
The nc_type type is just an int.
#define NC_64BIT_OFFSET
Use large (64-bit) file offsets.
#define NC_NOWRITE
Set read-only access for nc_open().
#define NC_BYTE
signed 1 byte integer
#define NC_EINDEFINE
Operation not allowed in define mode.
#define NC_ENOTNC
Not a netcdf file.
#define NC_FORMAT_CDF5
Format specifier for nc_set_default_format() and returned by nc_inq_format.
#define NC_ENOTBUILT
Attempt to use feature that was not turned on when netCDF was built.
#define NC_ENDIAN_LITTLE
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
#define NC_EATTMETA
Problem with attribute metadata.
#define NC_VLEN
vlen (variable-length) types
#define NC_EMPI
MPI operation failed.
#define NC_EDISKLESS
Error in using diskless access.
#define NC_EFILEMETA
Problem with file metadata.
This is the type of arrays of vlens.
#define NC_EBADTYPE
Not a netcdf data type.
#define NC_EBADNAME
Attribute or variable name contains illegal characters.
#define NC_EDIMMETA
Problem with dimension metadata.
#define NC_EINVAL
Invalid Argument.
#define NC_INT
signed 4 byte integer
#define NC_EBADGRPID
Bad group ID.
#define NC_NOFILL
Argument to nc_set_fill() to turn off filling of data.
#define NC_ENDIAN_BIG
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
#define NC_MAX_NAME
Maximum for classic library.
#define NC_ECANTREMOVE
Can't remove file.
#define NC_NAT
Not A Type.
#define NC_EBADTYPID
Bad type ID.
#define NC_USHORT
unsigned 2-byte int
#define NC_EPARINIT
Error initializing for parallel access.
#define NC_NETCDF4
Use netCDF-4/HDF5 format.
#define NC_EEXIST
netcdf file exists && NC_NOCLOBBER
#define NC_FORMAT_NETCDF4_CLASSIC
Format specifier for nc_set_default_format() and returned by nc_inq_format.
#define NC_EBADID
Not a netcdf id.
#define NC_EVARMETA
Problem with variable metadata.
struct NC4_rec_read_metadata_ud NC4_rec_read_metadata_ud_t
User data struct for call to H5Literate() in nc4_rec_read_metadata()
#define NC_MAX_UINT
Max or min values for a type.
#define NC_SHORT
signed 2 byte integer
#define NC_CDF5
Alias NC_CDF5 to NC_64BIT_DATA.
#define NC_WRITE
Set read-write access for nc_open().
#define NC_EMAXNAME
NC_MAX_NAME exceeded.
#define NC_EPERM
Write to read only.
#define NC_MAX_HDF4_NAME
This is the max size of an SD dataset name in HDF4 (from HDF4 documentation).
#define NC_NOERR
No Error.
#define NC_ENUM
enum types
#define NC_DISKLESS
Use diskless file.
struct NC4_rec_read_metadata_obj_info NC4_rec_read_metadata_obj_info_t
Struct to track information about objects in a group, for nc4_rec_read_metadata() ...
#define NC_COMPOUND
compound types
#define NC_FORMAT_64BIT_OFFSET
Format specifier for nc_set_default_format() and returned by nc_inq_format.
#define NC_MMAP
Use diskless file with mmap.
#define NC_FILL
Argument to nc_set_fill() to clear NC_NOFILL.
#define NC_FLOAT
single precision floating point number
#define NC_UINT64
unsigned 8-byte int
#define NC_MPIPOSIX
Turn on MPI POSIX I/O.