NetCDF  4.6.3
 All Data Structures Files Functions Variables Typedefs Macros Modules Pages
nc4hdf.c
Go to the documentation of this file.
1 /* Copyright 2018, University Corporation for Atmospheric
2  * Research. See the COPYRIGHT file for copying and redistribution
3  * conditions. */
17 #include "config.h"
18 #include "hdf5internal.h"
19 #include <math.h>
20 
21 #ifdef HAVE_INTTYPES_H
22 #define __STDC_FORMAT_MACROS
23 #include <inttypes.h>
24 #endif
25 
26 #define NC_HDF5_MAX_NAME 1024
36 static int
37 flag_atts_dirty(NCindex *attlist) {
38 
39  NC_ATT_INFO_T *att = NULL;
40  int i;
41 
42  if(attlist == NULL) {
43  return NC_NOERR;
44  }
45 
46  for(i=0;i<ncindexsize(attlist);i++) {
47  att = (NC_ATT_INFO_T*)ncindexith(attlist,i);
48  if(att == NULL) continue;
49  att->dirty = NC_TRUE;
50  }
51 
52  return NC_NOERR;
53 }
54 
71 int
72 rec_reattach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid)
73 {
74  NC_VAR_INFO_T *var;
75  NC_GRP_INFO_T *child_grp;
76  int d, i;
77  int retval;
78 
79  assert(grp && grp->hdr.name && dimid >= 0 && dimscaleid >= 0);
80  LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
81 
82  /* If there are any child groups, attach dimscale there, if needed. */
83  for (i = 0; i < ncindexsize(grp->children); i++)
84  {
85  child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children, i);
86  assert(child_grp);
87  if ((retval = rec_reattach_scales(child_grp, dimid, dimscaleid)))
88  return retval;
89  }
90 
91  /* Find any vars that use this dimension id. */
92  for (i = 0; i < ncindexsize(grp->vars); i++)
93  {
94  NC_HDF5_VAR_INFO_T *hdf5_var;
95 
96  var = (NC_VAR_INFO_T*)ncindexith(grp->vars,i);
97  assert(var && var->format_var_info);
98  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
99 
100  for (d = 0; d < var->ndims; d++)
101  {
102  if (var->dimids[d] == dimid && !var->dimscale)
103  {
104  LOG((2, "%s: attaching scale for dimid %d to var %s",
105  __func__, var->dimids[d], var->hdr.name));
106  if (var->created)
107  {
108  if (H5DSattach_scale(hdf5_var->hdf_datasetid,
109  dimscaleid, d) < 0)
110  return NC_EHDFERR;
111  var->dimscale_attached[d] = NC_TRUE;
112  }
113  }
114  }
115  }
116  return NC_NOERR;
117 }
118 
135 int
136 rec_detach_scales(NC_GRP_INFO_T *grp, int dimid, hid_t dimscaleid)
137 {
138  NC_VAR_INFO_T *var;
139  NC_GRP_INFO_T *child_grp;
140  int d, i;
141  int retval;
142 
143  assert(grp && grp->hdr.name && dimid >= 0 && dimscaleid >= 0);
144  LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
145 
146  /* If there are any child groups, detach dimscale there, if needed. */
147  for(i=0;i<ncindexsize(grp->children);i++) {
148  child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children,i);
149  if(child_grp == NULL) continue;
150  if ((retval = rec_detach_scales(child_grp, dimid, dimscaleid)))
151  return retval;
152  }
153 
154  /* Find any vars that use this dimension id. */
155  for (i = 0; i < ncindexsize(grp->vars); i++)
156  {
157  NC_HDF5_VAR_INFO_T *hdf5_var;
158  var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
159  assert(var && var->format_var_info);
160  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
161 
162  for (d = 0; d < var->ndims; d++)
163  {
164  if (var->dimids[d] == dimid && !var->dimscale)
165  {
166  LOG((2, "%s: detaching scale for dimid %d to var %s",
167  __func__, var->dimids[d], var->hdr.name));
168  if (var->created)
169  {
170  if (var->dimscale_attached && var->dimscale_attached[d])
171  {
172  if (H5DSdetach_scale(hdf5_var->hdf_datasetid,
173  dimscaleid, d) < 0)
174  return NC_EHDFERR;
175  var->dimscale_attached[d] = NC_FALSE;
176  }
177  }
178  }
179  }
180  }
181  return NC_NOERR;
182 }
183 
195 int
196 nc4_open_var_grp2(NC_GRP_INFO_T *grp, int varid, hid_t *dataset)
197 {
198  NC_VAR_INFO_T *var;
199  NC_HDF5_VAR_INFO_T *hdf5_var;
200 
201  assert(grp && grp->format_grp_info && dataset);
202 
203  /* Find the requested varid. */
204  if (!(var = (NC_VAR_INFO_T *)ncindexith(grp->vars, varid)))
205  return NC_ENOTVAR;
206  assert(var && var->hdr.id == varid && var->format_var_info);
207  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
208 
209  /* Open this dataset if necessary. */
210  if (!hdf5_var->hdf_datasetid)
211  {
212  NC_HDF5_GRP_INFO_T *hdf5_grp;
213  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
214 
215  if ((hdf5_var->hdf_datasetid = H5Dopen2(hdf5_grp->hdf_grpid,
216  var->hdr.name, H5P_DEFAULT)) < 0)
217  return NC_ENOTVAR;
218  }
219 
220  *dataset = hdf5_var->hdf_datasetid;
221 
222  return NC_NOERR;
223 }
224 
236 int
237 nc4_get_fill_value(NC_FILE_INFO_T *h5, NC_VAR_INFO_T *var, void **fillp)
238 {
239  size_t size;
240  int retval;
241 
242  /* Find out how much space we need for this type's fill value. */
243  if (var->type_info->nc_type_class == NC_VLEN)
244  size = sizeof(nc_vlen_t);
245  else if (var->type_info->nc_type_class == NC_STRING)
246  size = sizeof(char *);
247  else
248  {
249  if ((retval = nc4_get_typelen_mem(h5, var->type_info->hdr.id, &size)))
250  return retval;
251  }
252  assert(size);
253 
254  /* Allocate the space. */
255  if (!((*fillp) = calloc(1, size)))
256  return NC_ENOMEM;
257 
258  /* If the user has set a fill_value for this var, use, otherwise
259  * find the default fill value. */
260  if (var->fill_value)
261  {
262  LOG((4, "Found a fill value for var %s", var->hdr.name));
263  if (var->type_info->nc_type_class == NC_VLEN)
264  {
265  nc_vlen_t *in_vlen = (nc_vlen_t *)(var->fill_value), *fv_vlen = (nc_vlen_t *)(*fillp);
266  size_t basetypesize = 0;
267 
268  if((retval=nc4_get_typelen_mem(h5, var->type_info->u.v.base_nc_typeid, &basetypesize)))
269  return retval;
270 
271  fv_vlen->len = in_vlen->len;
272  if (!(fv_vlen->p = malloc(basetypesize * in_vlen->len)))
273  {
274  free(*fillp);
275  *fillp = NULL;
276  return NC_ENOMEM;
277  }
278  memcpy(fv_vlen->p, in_vlen->p, in_vlen->len * basetypesize);
279  }
280  else if (var->type_info->nc_type_class == NC_STRING)
281  {
282  if (*(char **)var->fill_value)
283  if (!(**(char ***)fillp = strdup(*(char **)var->fill_value)))
284  {
285  free(*fillp);
286  *fillp = NULL;
287  return NC_ENOMEM;
288  }
289  }
290  else
291  memcpy((*fillp), var->fill_value, size);
292  }
293  else
294  {
295  if (nc4_get_default_fill_value(var->type_info, *fillp))
296  {
297  /* Note: release memory, but don't return error on failure */
298  free(*fillp);
299  *fillp = NULL;
300  }
301  }
302 
303  return NC_NOERR;
304 }
305 
323 int
324 nc4_get_hdf_typeid(NC_FILE_INFO_T *h5, nc_type xtype,
325  hid_t *hdf_typeid, int endianness)
326 {
327  NC_TYPE_INFO_T *type;
328  hid_t typeid = 0;
329  int retval = NC_NOERR;
330 
331  assert(hdf_typeid && h5);
332 
333  *hdf_typeid = -1;
334 
335  /* Determine an appropriate HDF5 datatype */
336  if (xtype == NC_NAT)
337  return NC_EBADTYPE;
338  else if (xtype == NC_CHAR || xtype == NC_STRING)
339  {
340  /* NC_CHAR & NC_STRING types create a new HDF5 datatype */
341  if (xtype == NC_CHAR)
342  {
343  if ((typeid = H5Tcopy(H5T_C_S1)) < 0)
344  return NC_EHDFERR;
345  if (H5Tset_strpad(typeid, H5T_STR_NULLTERM) < 0)
346  BAIL(NC_EVARMETA);
347  if(H5Tset_cset(typeid, H5T_CSET_ASCII) < 0)
348  BAIL(NC_EVARMETA);
349 
350  /* Take ownership of the newly created HDF5 datatype */
351  *hdf_typeid = typeid;
352  typeid = 0;
353  }
354  else
355  {
356  if ((typeid = H5Tcopy(H5T_C_S1)) < 0)
357  return NC_EHDFERR;
358  if (H5Tset_size(typeid, H5T_VARIABLE) < 0)
359  BAIL(NC_EVARMETA);
360  if(H5Tset_cset(typeid, H5T_CSET_UTF8) < 0)
361  BAIL(NC_EVARMETA);
362 
363  /* Take ownership of the newly created HDF5 datatype */
364  *hdf_typeid = typeid;
365  typeid = 0;
366  }
367  }
368  else
369  {
370  /* All other types use an existing HDF5 datatype */
371  switch (xtype)
372  {
373  case NC_BYTE: /* signed 1 byte integer */
374  if (endianness == NC_ENDIAN_LITTLE)
375  typeid = H5T_STD_I8LE;
376  else if (endianness == NC_ENDIAN_BIG)
377  typeid = H5T_STD_I8BE;
378  else
379  typeid = H5T_NATIVE_SCHAR;
380  break;
381 
382  case NC_SHORT: /* signed 2 byte integer */
383  if (endianness == NC_ENDIAN_LITTLE)
384  typeid = H5T_STD_I16LE;
385  else if (endianness == NC_ENDIAN_BIG)
386  typeid = H5T_STD_I16BE;
387  else
388  typeid = H5T_NATIVE_SHORT;
389  break;
390 
391  case NC_INT:
392  if (endianness == NC_ENDIAN_LITTLE)
393  typeid = H5T_STD_I32LE;
394  else if (endianness == NC_ENDIAN_BIG)
395  typeid = H5T_STD_I32BE;
396  else
397  typeid = H5T_NATIVE_INT;
398  break;
399 
400  case NC_UBYTE:
401  if (endianness == NC_ENDIAN_LITTLE)
402  typeid = H5T_STD_U8LE;
403  else if (endianness == NC_ENDIAN_BIG)
404  typeid = H5T_STD_U8BE;
405  else
406  typeid = H5T_NATIVE_UCHAR;
407  break;
408 
409  case NC_USHORT:
410  if (endianness == NC_ENDIAN_LITTLE)
411  typeid = H5T_STD_U16LE;
412  else if (endianness == NC_ENDIAN_BIG)
413  typeid = H5T_STD_U16BE;
414  else
415  typeid = H5T_NATIVE_USHORT;
416  break;
417 
418  case NC_UINT:
419  if (endianness == NC_ENDIAN_LITTLE)
420  typeid = H5T_STD_U32LE;
421  else if (endianness == NC_ENDIAN_BIG)
422  typeid = H5T_STD_U32BE;
423  else
424  typeid = H5T_NATIVE_UINT;
425  break;
426 
427  case NC_INT64:
428  if (endianness == NC_ENDIAN_LITTLE)
429  typeid = H5T_STD_I64LE;
430  else if (endianness == NC_ENDIAN_BIG)
431  typeid = H5T_STD_I64BE;
432  else
433  typeid = H5T_NATIVE_LLONG;
434  break;
435 
436  case NC_UINT64:
437  if (endianness == NC_ENDIAN_LITTLE)
438  typeid = H5T_STD_U64LE;
439  else if (endianness == NC_ENDIAN_BIG)
440  typeid = H5T_STD_U64BE;
441  else
442  typeid = H5T_NATIVE_ULLONG;
443  break;
444 
445  case NC_FLOAT:
446  if (endianness == NC_ENDIAN_LITTLE)
447  typeid = H5T_IEEE_F32LE;
448  else if (endianness == NC_ENDIAN_BIG)
449  typeid = H5T_IEEE_F32BE;
450  else
451  typeid = H5T_NATIVE_FLOAT;
452  break;
453 
454  case NC_DOUBLE:
455  if (endianness == NC_ENDIAN_LITTLE)
456  typeid = H5T_IEEE_F64LE;
457  else if (endianness == NC_ENDIAN_BIG)
458  typeid = H5T_IEEE_F64BE;
459  else
460  typeid = H5T_NATIVE_DOUBLE;
461  break;
462 
463  default:
464  /* Maybe this is a user defined type? */
465  if (nc4_find_type(h5, xtype, &type))
466  return NC_EBADTYPE;
467  if (!type)
468  return NC_EBADTYPE;
469  typeid = ((NC_HDF5_TYPE_INFO_T *)type->format_type_info)->hdf_typeid;
470  break;
471  }
472  assert(typeid);
473 
474  /* Copy the HDF5 datatype, so the function operates uniformly */
475  if ((*hdf_typeid = H5Tcopy(typeid)) < 0)
476  return NC_EHDFERR;
477  typeid = 0;
478  }
479  assert(*hdf_typeid != -1);
480 
481 exit:
482  if (typeid > 0 && H5Tclose(typeid) < 0)
483  BAIL2(NC_EHDFERR);
484  return retval;
485 }
486 
501 static int
502 put_att_grpa(NC_GRP_INFO_T *grp, int varid, NC_ATT_INFO_T *att)
503 {
504  NC_HDF5_GRP_INFO_T *hdf5_grp;
505  hid_t datasetid = 0, locid;
506  hid_t attid = 0, spaceid = 0, file_typeid = 0;
507  hid_t existing_att_typeid = 0, existing_attid = 0, existing_spaceid = 0;
508  hsize_t dims[1]; /* netcdf attributes always 1-D. */
509  htri_t attr_exists;
510  int reuse_att = 0; /* Will be true if we can re-use an existing att. */
511  void *data;
512  int phoney_data = 99;
513  int retval = NC_NOERR;
514 
515  assert(att->hdr.name && grp && grp->format_grp_info);
516  LOG((3, "%s: varid %d att->hdr.id %d att->hdr.name %s att->nc_typeid %d "
517  "att->len %d", __func__, varid, att->hdr.id, att->hdr.name,
518  att->nc_typeid, att->len));
519 
520  /* Get HDF5-specific group info. */
521  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
522 
523  /* If the file is read-only, return an error. */
524  if (grp->nc4_info->no_write)
525  BAIL(NC_EPERM);
526 
527  /* Get the hid to attach the attribute to, or read it from. */
528  if (varid == NC_GLOBAL)
529  locid = hdf5_grp->hdf_grpid;
530  else
531  {
532  if ((retval = nc4_open_var_grp2(grp, varid, &datasetid)))
533  BAIL(retval);
534  locid = datasetid;
535  }
536 
537  /* Get the length ready, and find the HDF type we'll be
538  * writing. */
539  dims[0] = att->len;
540  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, att->nc_typeid,
541  &file_typeid, 0)))
542  BAIL(retval);
543 
544  /* Even if the length is zero, HDF5 won't let me write with a
545  * NULL pointer. So if the length of the att is zero, point to
546  * some phoney data (which won't be written anyway.)*/
547  if (!dims[0])
548  data = &phoney_data;
549  else if (att->data)
550  data = att->data;
551  else if (att->stdata)
552  data = att->stdata;
553  else
554  data = att->vldata;
555 
556  /* NC_CHAR types require some extra work. The space ID is set to
557  * scalar, and the type is told how long the string is. If it's
558  * really zero length, set the size to 1. (The fact that it's
559  * really zero will be marked by the NULL dataspace, but HDF5
560  * doesn't allow me to set the size of the type to zero.)*/
561  if (att->nc_typeid == NC_CHAR)
562  {
563  size_t string_size = dims[0];
564  if (!string_size)
565  {
566  string_size = 1;
567  if ((spaceid = H5Screate(H5S_NULL)) < 0)
568  BAIL(NC_EATTMETA);
569  }
570  else
571  {
572  if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
573  BAIL(NC_EATTMETA);
574  }
575  if (H5Tset_size(file_typeid, string_size) < 0)
576  BAIL(NC_EATTMETA);
577  if (H5Tset_strpad(file_typeid, H5T_STR_NULLTERM) < 0)
578  BAIL(NC_EATTMETA);
579  }
580  else
581  {
582  if (!att->len)
583  {
584  if ((spaceid = H5Screate(H5S_NULL)) < 0)
585  BAIL(NC_EATTMETA);
586  }
587  else
588  {
589  if ((spaceid = H5Screate_simple(1, dims, NULL)) < 0)
590  BAIL(NC_EATTMETA);
591  }
592  }
593 
594  /* Does the att exists already? */
595  if ((attr_exists = H5Aexists(locid, att->hdr.name)) < 0)
596  BAIL(NC_EHDFERR);
597  if (attr_exists)
598  {
599  hssize_t npoints;
600 
601  /* Open the attribute. */
602  if ((existing_attid = H5Aopen(locid, att->hdr.name, H5P_DEFAULT)) < 0)
603  BAIL(NC_EATTMETA);
604 
605  /* Find the type of the existing attribute. */
606  if ((existing_att_typeid = H5Aget_type(existing_attid)) < 0)
607  BAIL(NC_EATTMETA);
608 
609  /* How big is the attribute? */
610  if ((existing_spaceid = H5Aget_space(existing_attid)) < 0)
611  BAIL(NC_EATTMETA);
612  if ((npoints = H5Sget_simple_extent_npoints(existing_spaceid)) < 0)
613  BAIL(NC_EATTMETA);
614 
615  /* Delete the attribute. */
616  if (file_typeid != existing_att_typeid || npoints != att->len)
617  {
618  if (H5Adelete(locid, att->hdr.name) < 0)
619  BAIL(NC_EHDFERR);
620  }
621  else
622  {
623  reuse_att++;
624  }
625  }
626 
627  /* Create the attribute. */
628  if ((attid = H5Acreate(locid, att->hdr.name, file_typeid, spaceid,
629  H5P_DEFAULT)) < 0)
630  BAIL(NC_EATTMETA);
631 
632  /* Write the values, (even if length is zero). */
633  if (H5Awrite(attid, file_typeid, data) < 0)
634  BAIL(NC_EATTMETA);
635 
636 exit:
637  if (file_typeid && H5Tclose(file_typeid))
638  BAIL2(NC_EHDFERR);
639  if (attid > 0 && H5Aclose(attid) < 0)
640  BAIL2(NC_EHDFERR);
641  if (existing_att_typeid && H5Tclose(existing_att_typeid))
642  BAIL2(NC_EHDFERR);
643  if (existing_attid > 0 && H5Aclose(existing_attid) < 0)
644  BAIL2(NC_EHDFERR);
645  if (spaceid > 0 && H5Sclose(spaceid) < 0)
646  BAIL2(NC_EHDFERR);
647  if (existing_spaceid > 0 && H5Sclose(existing_spaceid) < 0)
648  BAIL2(NC_EHDFERR);
649  return retval;
650 }
651 
663 static int
664 write_attlist(NCindex *attlist, int varid, NC_GRP_INFO_T *grp)
665 {
666  NC_ATT_INFO_T *att;
667  int retval;
668  int i;
669 
670  for(i = 0; i < ncindexsize(attlist); i++)
671  {
672  att = (NC_ATT_INFO_T *)ncindexith(attlist, i);
673  assert(att);
674  if (att->dirty)
675  {
676  LOG((4, "%s: writing att %s to varid %d", __func__, att->hdr.name, varid));
677  if ((retval = put_att_grpa(grp, varid, att)))
678  return retval;
679  att->dirty = NC_FALSE;
680  att->created = NC_TRUE;
681  }
682  }
683  return NC_NOERR;
684 }
685 
699 static int
700 write_coord_dimids(NC_VAR_INFO_T *var)
701 {
702  NC_HDF5_VAR_INFO_T *hdf5_var;
703  hsize_t coords_len[1];
704  hid_t c_spaceid = -1, c_attid = -1;
705  int retval = NC_NOERR;
706 
707  assert(var && var->format_var_info);
708 
709  /* Get HDF5-specific var info. */
710  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
711 
712  /* Set up space for attribute. */
713  coords_len[0] = var->ndims;
714  if ((c_spaceid = H5Screate_simple(1, coords_len, coords_len)) < 0)
715  BAIL(NC_EHDFERR);
716 
717  /* Create the attribute. */
718  if ((c_attid = H5Acreate(hdf5_var->hdf_datasetid, COORDINATES,
719  H5T_NATIVE_INT, c_spaceid, H5P_DEFAULT)) < 0)
720  BAIL(NC_EHDFERR);
721 
722  /* Write our attribute. */
723  if (H5Awrite(c_attid, H5T_NATIVE_INT, var->dimids) < 0)
724  BAIL(NC_EHDFERR);
725 
726 exit:
727  if (c_spaceid >= 0 && H5Sclose(c_spaceid) < 0)
728  BAIL2(NC_EHDFERR);
729  if (c_attid >= 0 && H5Aclose(c_attid) < 0)
730  BAIL2(NC_EHDFERR);
731  return retval;
732 }
733 
744 static int
745 write_netcdf4_dimid(hid_t datasetid, int dimid)
746 {
747  hid_t dimid_spaceid = -1, dimid_attid = -1;
748  htri_t attr_exists;
749  int retval = NC_NOERR;
750 
751  /* Create the space. */
752  if ((dimid_spaceid = H5Screate(H5S_SCALAR)) < 0)
753  BAIL(NC_EHDFERR);
754 
755  /* Does the attribute already exist? If so, don't try to create it. */
756  if ((attr_exists = H5Aexists(datasetid, NC_DIMID_ATT_NAME)) < 0)
757  BAIL(NC_EHDFERR);
758  if (attr_exists)
759  dimid_attid = H5Aopen_by_name(datasetid, ".", NC_DIMID_ATT_NAME,
760  H5P_DEFAULT, H5P_DEFAULT);
761  else
762  /* Create the attribute if needed. */
763  dimid_attid = H5Acreate(datasetid, NC_DIMID_ATT_NAME,
764  H5T_NATIVE_INT, dimid_spaceid, H5P_DEFAULT);
765  if (dimid_attid < 0)
766  BAIL(NC_EHDFERR);
767 
768 
769  /* Write it. */
770  LOG((4, "%s: writing secret dimid %d", __func__, dimid));
771  if (H5Awrite(dimid_attid, H5T_NATIVE_INT, &dimid) < 0)
772  BAIL(NC_EHDFERR);
773 
774 exit:
775  /* Close stuff*/
776  if (dimid_spaceid >= 0 && H5Sclose(dimid_spaceid) < 0)
777  BAIL2(NC_EHDFERR);
778  if (dimid_attid >= 0 && H5Aclose(dimid_attid) < 0)
779  BAIL2(NC_EHDFERR);
780 
781  return retval;
782 }
783 
798 static int
799 var_create_dataset(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var, nc_bool_t write_dimid)
800 {
801  NC_HDF5_GRP_INFO_T *hdf5_grp;
802  NC_HDF5_VAR_INFO_T *hdf5_var;
803  hid_t plistid = 0, access_plistid = 0, typeid = 0, spaceid = 0;
804  hsize_t chunksize[H5S_MAX_RANK], dimsize[H5S_MAX_RANK], maxdimsize[H5S_MAX_RANK];
805  int d;
806  void *fillp = NULL;
807  NC_DIM_INFO_T *dim = NULL;
808  char *name_to_use;
809  int retval;
810 
811  assert(grp && grp->format_grp_info && var && var->format_var_info);
812 
813  LOG((3, "%s:: name %s", __func__, var->hdr.name));
814 
815  /* Get HDF5-specific group and var info. */
816  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
817  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
818 
819  /* Scalar or not, we need a creation property list. */
820  if ((plistid = H5Pcreate(H5P_DATASET_CREATE)) < 0)
821  BAIL(NC_EHDFERR);
822  if ((access_plistid = H5Pcreate(H5P_DATASET_ACCESS)) < 0)
823  BAIL(NC_EHDFERR);
824 
825  /* Turn off object tracking times in HDF5. */
826  if (H5Pset_obj_track_times(plistid, 0) < 0)
827  BAIL(NC_EHDFERR);
828 
829  /* Find the HDF5 type of the dataset. */
830  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, var->type_info->hdr.id, &typeid,
831  var->type_info->endianness)))
832  BAIL(retval);
833 
834  /* Figure out what fill value to set, if any. */
835  if (var->no_fill)
836  {
837  /* Required to truly turn HDF5 fill values off */
838  if (H5Pset_fill_time(plistid, H5D_FILL_TIME_NEVER) < 0)
839  BAIL(NC_EHDFERR);
840  }
841  else
842  {
843  if ((retval = nc4_get_fill_value(grp->nc4_info, var, &fillp)))
844  BAIL(retval);
845 
846  /* If there is a fill value, set it. */
847  if (fillp)
848  {
849  if (var->type_info->nc_type_class == NC_STRING)
850  {
851  if (H5Pset_fill_value(plistid, typeid, fillp) < 0)
852  BAIL(NC_EHDFERR);
853  }
854  else
855  {
856  /* The fill value set in HDF5 must always be presented as
857  * a native type, even if the endianness for this dataset
858  * is non-native. HDF5 will translate the fill value to
859  * the target endiannesss. */
860  hid_t fill_typeid = 0;
861 
862  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, var->type_info->hdr.id, &fill_typeid,
864  BAIL(retval);
865  if (H5Pset_fill_value(plistid, fill_typeid, fillp) < 0)
866  {
867  if (H5Tclose(fill_typeid) < 0)
868  BAIL(NC_EHDFERR);
869  BAIL(NC_EHDFERR);
870  }
871  if (H5Tclose(fill_typeid) < 0)
872  BAIL(NC_EHDFERR);
873  }
874  }
875  }
876 
877  /* If the user wants to shuffle the data, set that up now. */
878  if (var->shuffle) {
879  if (H5Pset_shuffle(plistid) < 0)
880  BAIL(NC_EHDFERR);
881  }
882 
883  /* If the user wants to deflate the data, set that up now. */
884  if (var->deflate) {
885  if (H5Pset_deflate(plistid, var->deflate_level) < 0)
886  BAIL(NC_EHDFERR);
887  } else if(var->filterid) {
888  /* Handle szip case here */
889  if(var->filterid == H5Z_FILTER_SZIP) {
890  int options_mask;
891  int bits_per_pixel;
892  if(var->nparams != 2)
893  BAIL(NC_EFILTER);
894  options_mask = (int)var->params[0];
895  bits_per_pixel = (int)var->params[1];
896  if(H5Pset_szip(plistid, options_mask, bits_per_pixel) < 0)
897  BAIL(NC_EFILTER);
898  } else {
899  herr_t code = H5Pset_filter(plistid, var->filterid, H5Z_FLAG_MANDATORY, var->nparams, var->params);
900  if(code < 0) {
901  BAIL(NC_EFILTER);
902  }
903  }
904  }
905 
906  /* If the user wants to fletcher error correcton, set that up now. */
907  if (var->fletcher32)
908  if (H5Pset_fletcher32(plistid) < 0)
909  BAIL(NC_EHDFERR);
910 
911  /* If ndims non-zero, get info for all dimensions. We look up the
912  dimids and get the len of each dimension. We need this to create
913  the space for the dataset. In netCDF a dimension length of zero
914  means an unlimited dimension. */
915  if (var->ndims)
916  {
917  int unlimdim = 0;
918 
919  /* Check to see if any unlimited dimensions are used in this var. */
920  for (d = 0; d < var->ndims; d++) {
921  dim = var->dim[d];
922  assert(dim && dim->hdr.id == var->dimids[d]);
923  if (dim->unlimited)
924  unlimdim++;
925  }
926 
927  /* If there are no unlimited dims, and no filters, and the user
928  * has not specified chunksizes, use contiguous variable for
929  * better performance. */
930  if (!var->shuffle && !var->deflate && !var->fletcher32 &&
931  (var->chunksizes == NULL || !var->chunksizes[0]) && !unlimdim)
932  var->contiguous = NC_TRUE;
933 
934  /* Gather current & maximum dimension sizes, along with chunk sizes */
935  for (d = 0; d < var->ndims; d++)
936  {
937  dim = var->dim[d];
938  assert(dim && dim->hdr.id == var->dimids[d]);
939  dimsize[d] = dim->unlimited ? NC_HDF5_UNLIMITED_DIMSIZE : dim->len;
940  maxdimsize[d] = dim->unlimited ? H5S_UNLIMITED : (hsize_t)dim->len;
941  if (!var->contiguous) {
942  if (var->chunksizes[d])
943  chunksize[d] = var->chunksizes[d];
944  else
945  {
946  size_t type_size;
947  if (var->type_info->nc_type_class == NC_STRING)
948  type_size = sizeof(char *);
949  else
950  type_size = var->type_info->size;
951 
952  /* Unlimited dim always gets chunksize of 1. */
953  if (dim->unlimited)
954  chunksize[d] = 1;
955  else
956  chunksize[d] = pow((double)DEFAULT_CHUNK_SIZE/type_size,
957  1/(double)(var->ndims - unlimdim));
958 
959  /* If the chunksize is greater than the dim
960  * length, make it the dim length. */
961  if (!dim->unlimited && chunksize[d] > dim->len)
962  chunksize[d] = dim->len;
963 
964  /* Remember the computed chunksize */
965  var->chunksizes[d] = chunksize[d];
966  }
967  }
968  }
969 
970  if (var->contiguous)
971  {
972  if (H5Pset_layout(plistid, H5D_CONTIGUOUS) < 0)
973  BAIL(NC_EHDFERR);
974  }
975  else
976  {
977  if (H5Pset_chunk(plistid, var->ndims, chunksize) < 0)
978  BAIL(NC_EHDFERR);
979  }
980 
981  /* Create the dataspace. */
982  if ((spaceid = H5Screate_simple(var->ndims, dimsize, maxdimsize)) < 0)
983  BAIL(NC_EHDFERR);
984  }
985  else
986  {
987  if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
988  BAIL(NC_EHDFERR);
989  }
990 
991  /* Turn on creation order tracking. */
992  if (H5Pset_attr_creation_order(plistid, H5P_CRT_ORDER_TRACKED|
993  H5P_CRT_ORDER_INDEXED) < 0)
994  BAIL(NC_EHDFERR);
995 
996  /* Set per-var chunk cache, for chunked datasets. */
997  if (!var->contiguous && var->chunk_cache_size)
998  if (H5Pset_chunk_cache(access_plistid, var->chunk_cache_nelems,
999  var->chunk_cache_size, var->chunk_cache_preemption) < 0)
1000  BAIL(NC_EHDFERR);
1001 
1002  /* At long last, create the dataset. */
1003  name_to_use = var->hdf5_name ? var->hdf5_name : var->hdr.name;
1004  LOG((4, "%s: about to H5Dcreate2 dataset %s of type 0x%x", __func__,
1005  name_to_use, typeid));
1006  if ((hdf5_var->hdf_datasetid = H5Dcreate2(hdf5_grp->hdf_grpid, name_to_use, typeid,
1007  spaceid, H5P_DEFAULT, plistid, access_plistid)) < 0)
1008  BAIL(NC_EHDFERR);
1009  var->created = NC_TRUE;
1010  var->is_new_var = NC_FALSE;
1011 
1012  /* Always write the hidden coordinates attribute, which lists the
1013  * dimids of this var. When present, this speeds opens. When no
1014  * present, dimscale matching is used. */
1015  if (var->ndims > 1)
1016  if ((retval = write_coord_dimids(var)))
1017  BAIL(retval);
1018 
1019  /* If this is a dimscale, mark it as such in the HDF5 file. Also
1020  * find the dimension info and store the dataset id of the dimscale
1021  * dataset. */
1022  if (var->dimscale)
1023  {
1024  if (H5DSset_scale(hdf5_var->hdf_datasetid, var->hdr.name) < 0)
1025  BAIL(NC_EHDFERR);
1026 
1027  /* If this is a multidimensional coordinate variable, write a
1028  * coordinates attribute. */
1029  /* if (var->ndims > 1) */
1030  /* if ((retval = write_coord_dimids(var))) */
1031  /* BAIL(retval); */
1032 
1033  /* If desired, write the netCDF dimid. */
1034  if (write_dimid)
1035  if ((retval = write_netcdf4_dimid(hdf5_var->hdf_datasetid, var->dimids[0])))
1036  BAIL(retval);
1037  }
1038 
1039  /* Write attributes for this var. */
1040  if ((retval = write_attlist(var->att, var->hdr.id, grp)))
1041  BAIL(retval);
1042  var->attr_dirty = NC_FALSE;
1043 
1044 exit:
1045  if (typeid > 0 && H5Tclose(typeid) < 0)
1046  BAIL2(NC_EHDFERR);
1047  if (plistid > 0 && H5Pclose(plistid) < 0)
1048  BAIL2(NC_EHDFERR);
1049  if (access_plistid > 0 && H5Pclose(access_plistid) < 0)
1050  BAIL2(NC_EHDFERR);
1051  if (spaceid > 0 && H5Sclose(spaceid) < 0)
1052  BAIL2(NC_EHDFERR);
1053  if (fillp)
1054  {
1055  if (var->type_info->nc_type_class == NC_VLEN)
1056  nc_free_vlen((nc_vlen_t *)fillp);
1057  else if (var->type_info->nc_type_class == NC_STRING && *(char **)fillp)
1058  free(*(char **)fillp);
1059  free(fillp);
1060  }
1061 
1062  return retval;
1063 }
1064 
1075 int
1076 nc4_adjust_var_cache(NC_GRP_INFO_T *grp, NC_VAR_INFO_T *var)
1077 {
1078  size_t chunk_size_bytes = 1;
1079  int d;
1080  int retval;
1081 
1082  /* Nothing to be done. */
1083  if (var->contiguous)
1084  return NC_NOERR;
1085 #ifdef USE_PARALLEL4
1086  return NC_NOERR;
1087 #endif
1088 
1089  /* How many bytes in the chunk? */
1090  for (d = 0; d < var->ndims; d++)
1091  chunk_size_bytes *= var->chunksizes[d];
1092  if (var->type_info->size)
1093  chunk_size_bytes *= var->type_info->size;
1094  else
1095  chunk_size_bytes *= sizeof(char *);
1096 
1097  /* If the chunk cache is too small, and the user has not changed
1098  * the default value of the chunk cache size, then increase the
1099  * size of the cache. */
1100  if (var->chunk_cache_size == CHUNK_CACHE_SIZE)
1101  if (chunk_size_bytes > var->chunk_cache_size)
1102  {
1103  var->chunk_cache_size = chunk_size_bytes * DEFAULT_CHUNKS_IN_CACHE;
1104  if (var->chunk_cache_size > MAX_DEFAULT_CACHE_SIZE)
1105  var->chunk_cache_size = MAX_DEFAULT_CACHE_SIZE;
1106  if ((retval = nc4_reopen_dataset(grp, var)))
1107  return retval;
1108  }
1109 
1110  return NC_NOERR;
1111 }
1112 
1128 static int
1129 commit_type(NC_GRP_INFO_T *grp, NC_TYPE_INFO_T *type)
1130 {
1131  NC_HDF5_GRP_INFO_T *hdf5_grp;
1132  NC_HDF5_TYPE_INFO_T *hdf5_type;
1133  hid_t base_hdf_typeid;
1134  int retval;
1135 
1136  assert(grp && grp->format_grp_info && type && type->format_type_info);
1137 
1138  /* Get HDF5-specific group and type info. */
1139  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1140  hdf5_type = (NC_HDF5_TYPE_INFO_T *)type->format_type_info;
1141 
1142  /* Did we already record this type? */
1143  if (type->committed)
1144  return NC_NOERR;
1145 
1146  /* Is this a compound type? */
1147  if (type->nc_type_class == NC_COMPOUND)
1148  {
1149  NC_FIELD_INFO_T *field;
1150  hid_t hdf_base_typeid, hdf_typeid;
1151  int i;
1152 
1153  if ((hdf5_type->hdf_typeid = H5Tcreate(H5T_COMPOUND, type->size)) < 0)
1154  return NC_EHDFERR;
1155  LOG((4, "creating compound type %s hdf_typeid 0x%x", type->hdr.name,
1156  hdf5_type->hdf_typeid));
1157 
1158  for(i=0;i<nclistlength(type->u.c.field);i++)
1159  {
1160  field = (NC_FIELD_INFO_T *)nclistget(type->u.c.field, i);
1161  assert(field);
1162  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, field->nc_typeid,
1163  &hdf_base_typeid, type->endianness)))
1164  return retval;
1165 
1166  /* If this is an array, create a special array type. */
1167  if (field->ndims)
1168  {
1169  int d;
1170  hsize_t dims[NC_MAX_VAR_DIMS];
1171 
1172  for (d = 0; d < field->ndims; d++)
1173  dims[d] = field->dim_size[d];
1174  if ((hdf_typeid = H5Tarray_create(hdf_base_typeid, field->ndims,
1175  dims, NULL)) < 0)
1176  {
1177  if (H5Tclose(hdf_base_typeid) < 0)
1178  return NC_EHDFERR;
1179  return NC_EHDFERR;
1180  }
1181  if (H5Tclose(hdf_base_typeid) < 0)
1182  return NC_EHDFERR;
1183  }
1184  else
1185  hdf_typeid = hdf_base_typeid;
1186  LOG((4, "inserting field %s offset %d hdf_typeid 0x%x", field->hdr.name,
1187  field->offset, hdf_typeid));
1188  if (H5Tinsert(hdf5_type->hdf_typeid, field->hdr.name, field->offset,
1189  hdf_typeid) < 0)
1190  return NC_EHDFERR;
1191  if (H5Tclose(hdf_typeid) < 0)
1192  return NC_EHDFERR;
1193  }
1194  }
1195  else if (type->nc_type_class == NC_VLEN)
1196  {
1197  /* Find the HDF typeid of the base type of this vlen. */
1198  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, type->u.v.base_nc_typeid,
1199  &base_hdf_typeid, type->endianness)))
1200  return retval;
1201 
1202  /* Create a vlen type. */
1203  if ((hdf5_type->hdf_typeid = H5Tvlen_create(base_hdf_typeid)) < 0)
1204  return NC_EHDFERR;
1205  }
1206  else if (type->nc_type_class == NC_OPAQUE)
1207  {
1208  /* Create the opaque type. */
1209  if ((hdf5_type->hdf_typeid = H5Tcreate(H5T_OPAQUE, type->size)) < 0)
1210  return NC_EHDFERR;
1211  }
1212  else if (type->nc_type_class == NC_ENUM)
1213  {
1214  NC_ENUM_MEMBER_INFO_T *enum_m;
1215  int i;
1216 
1217  if (nclistlength(type->u.e.enum_member) == 0)
1218  return NC_EINVAL;
1219 
1220  /* Find the HDF typeid of the base type of this enum. */
1221  if ((retval = nc4_get_hdf_typeid(grp->nc4_info, type->u.e.base_nc_typeid,
1222  &base_hdf_typeid, type->endianness)))
1223  return retval;
1224 
1225  /* Create an enum type. */
1226  if ((hdf5_type->hdf_typeid = H5Tenum_create(base_hdf_typeid)) < 0)
1227  return NC_EHDFERR;
1228 
1229  /* Add all the members to the HDF5 type. */
1230  for(i=0;i<nclistlength(type->u.e.enum_member);i++) {
1231  enum_m = (NC_ENUM_MEMBER_INFO_T*)nclistget(type->u.e.enum_member,i);
1232  if (H5Tenum_insert(hdf5_type->hdf_typeid, enum_m->name, enum_m->value) < 0)
1233  return NC_EHDFERR;
1234  }
1235  }
1236  else
1237  {
1238  LOG((0, "Unknown class: %d", type->nc_type_class));
1239  return NC_EBADTYPE;
1240  }
1241 
1242  /* Commit the type. */
1243  if (H5Tcommit(hdf5_grp->hdf_grpid, type->hdr.name, hdf5_type->hdf_typeid) < 0)
1244  return NC_EHDFERR;
1245  type->committed = NC_TRUE;
1246  LOG((4, "just committed type %s, HDF typeid: 0x%x", type->hdr.name,
1247  hdf5_type->hdf_typeid));
1248 
1249  /* Later we will always use the native typeid. In this case, it is
1250  * a copy of the same type pointed to by hdf_typeid, but it's
1251  * easier to maintain a copy. */
1252  if ((hdf5_type->native_hdf_typeid = H5Tget_native_type(hdf5_type->hdf_typeid,
1253  H5T_DIR_DEFAULT)) < 0)
1254  return NC_EHDFERR;
1255 
1256  return NC_NOERR;
1257 }
1258 
1269 static int
1270 write_nc3_strict_att(hid_t hdf_grpid)
1271 {
1272  hid_t attid = 0, spaceid = 0;
1273  int one = 1;
1274  int retval = NC_NOERR;
1275  htri_t attr_exists;
1276 
1277  /* If the attribute already exists, call that a success and return
1278  * NC_NOERR. */
1279  if ((attr_exists = H5Aexists(hdf_grpid, NC3_STRICT_ATT_NAME)) < 0)
1280  return NC_EHDFERR;
1281  if (attr_exists)
1282  return NC_NOERR;
1283 
1284  /* Create the attribute to mark this as a file that needs to obey
1285  * strict netcdf-3 rules. */
1286  if ((spaceid = H5Screate(H5S_SCALAR)) < 0)
1287  BAIL(NC_EFILEMETA);
1288  if ((attid = H5Acreate(hdf_grpid, NC3_STRICT_ATT_NAME,
1289  H5T_NATIVE_INT, spaceid, H5P_DEFAULT)) < 0)
1290  BAIL(NC_EFILEMETA);
1291  if (H5Awrite(attid, H5T_NATIVE_INT, &one) < 0)
1292  BAIL(NC_EFILEMETA);
1293 
1294 exit:
1295  if (spaceid > 0 && (H5Sclose(spaceid) < 0))
1296  BAIL2(NC_EFILEMETA);
1297  if (attid > 0 && (H5Aclose(attid) < 0))
1298  BAIL2(NC_EFILEMETA);
1299  return retval;
1300 }
1301 
1314 static int
1315 create_group(NC_GRP_INFO_T *grp)
1316 {
1317  NC_HDF5_GRP_INFO_T *hdf5_grp, *parent_hdf5_grp;
1318  hid_t gcpl_id = -1;
1319  int retval = NC_NOERR;;
1320 
1321  assert(grp && grp->format_grp_info && grp->parent &&
1322  grp->parent->format_grp_info);
1323 
1324  /* Get HDF5 specific group info for group and parent. */
1325  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1326  parent_hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->parent->format_grp_info;
1327  assert(parent_hdf5_grp->hdf_grpid);
1328 
1329  /* Create group, with link_creation_order set in the group
1330  * creation property list. */
1331  if ((gcpl_id = H5Pcreate(H5P_GROUP_CREATE)) < 0)
1332  BAIL(NC_EHDFERR);
1333 
1334  /* Set track_times to be FALSE. */
1335  if (H5Pset_obj_track_times(gcpl_id, 0) < 0)
1336  BAIL(NC_EHDFERR);
1337 
1338  /* Tell HDF5 to keep track of objects in creation order. */
1339  if (H5Pset_link_creation_order(gcpl_id, H5P_CRT_ORDER_TRACKED|H5P_CRT_ORDER_INDEXED) < 0)
1340  BAIL(NC_EHDFERR);
1341 
1342  /* Tell HDF5 to keep track of attributes in creation order. */
1343  if (H5Pset_attr_creation_order(gcpl_id, H5P_CRT_ORDER_TRACKED|H5P_CRT_ORDER_INDEXED) < 0)
1344  BAIL(NC_EHDFERR);
1345 
1346  /* Create the group. */
1347  if ((hdf5_grp->hdf_grpid = H5Gcreate2(parent_hdf5_grp->hdf_grpid, grp->hdr.name,
1348  H5P_DEFAULT, gcpl_id, H5P_DEFAULT)) < 0)
1349  BAIL(NC_EHDFERR);
1350 
1351 exit:
1352  if (gcpl_id > -1 && H5Pclose(gcpl_id) < 0)
1353  BAIL2(NC_EHDFERR);
1354  if (retval)
1355  if (hdf5_grp->hdf_grpid > 0 && H5Gclose(hdf5_grp->hdf_grpid) < 0)
1356  BAIL2(NC_EHDFERR);
1357  return retval;
1358 }
1359 
1371 static int
1372 attach_dimscales(NC_GRP_INFO_T *grp)
1373 {
1374  NC_VAR_INFO_T *var;
1375  NC_HDF5_VAR_INFO_T *hdf5_var;
1376  int d, v;
1377 
1378  /* Attach dimension scales. */
1379  for (v = 0; v < ncindexsize(grp->vars); v++)
1380  {
1381  /* Get pointer to var and HDF5-specific var info. */
1382  var = (NC_VAR_INFO_T *)ncindexith(grp->vars, v);
1383  assert(var && var->format_var_info);
1384  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
1385 
1386  /* Scales themselves do not attach. But I really wish they
1387  * would. */
1388  if (var->dimscale)
1389  continue;
1390 
1391  /* Find the scale for each dimension, if any, and attach it. */
1392  for (d = 0; d < var->ndims; d++)
1393  {
1394  /* Is there a dimscale for this dimension? */
1395  if (var->dimscale_attached)
1396  {
1397  if (!var->dimscale_attached[d])
1398  {
1399  hid_t dsid; /* Dataset ID for dimension */
1400  assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d] &&
1401  var->dim[d]->format_dim_info);
1402 
1403  LOG((2, "%s: attaching scale for dimid %d to var %s",
1404  __func__, var->dimids[d], var->hdr.name));
1405 
1406  /* Find dataset ID for dimension */
1407  if (var->dim[d]->coord_var)
1408  dsid = ((NC_HDF5_VAR_INFO_T *)(var->dim[d]->coord_var->format_var_info))->hdf_datasetid;
1409  else
1410  dsid = ((NC_HDF5_DIM_INFO_T *)var->dim[d]->format_dim_info)->hdf_dimscaleid;
1411  assert(dsid > 0);
1412 
1413  /* Attach the scale. */
1414  if (H5DSattach_scale(hdf5_var->hdf_datasetid, dsid, d) < 0)
1415  return NC_EHDFERR;
1416  var->dimscale_attached[d] = NC_TRUE;
1417  }
1418  }
1419  }
1420  }
1421 
1422  return NC_NOERR;
1423 }
1424 
1435 static int
1436 var_exists(hid_t grpid, char *name, nc_bool_t *exists)
1437 {
1438  htri_t link_exists;
1439 
1440  /* Reset the boolean */
1441  *exists = NC_FALSE;
1442 
1443  /* Check if the object name exists in the group */
1444  if ((link_exists = H5Lexists(grpid, name, H5P_DEFAULT)) < 0)
1445  return NC_EHDFERR;
1446  if (link_exists)
1447  {
1448  H5G_stat_t statbuf;
1449 
1450  /* Get info about the object */
1451  if (H5Gget_objinfo(grpid, name, 1, &statbuf) < 0)
1452  return NC_EHDFERR;
1453 
1454  if (H5G_DATASET == statbuf.type)
1455  *exists = NC_TRUE;
1456  }
1457 
1458  return NC_NOERR;
1459 }
1460 
1476 static int
1477 remove_coord_atts(hid_t hdf_datasetid)
1478 {
1479  htri_t attr_exists;
1480 
1481  /* If the variable dataset has an optional NC_DIMID_ATT_NAME
1482  * attribute, delete it. */
1483  if ((attr_exists = H5Aexists(hdf_datasetid, NC_DIMID_ATT_NAME)) < 0)
1484  return NC_EHDFERR;
1485  if (attr_exists)
1486  {
1487  if (H5Adelete(hdf_datasetid, NC_DIMID_ATT_NAME) < 0)
1488  return NC_EHDFERR;
1489  }
1490 
1491  /* Remove the dimension scale 'CLASS' & 'NAME' attributes. */
1492  if ((attr_exists = H5Aexists(hdf_datasetid,
1493  HDF5_DIMSCALE_CLASS_ATT_NAME)) < 0)
1494  return NC_EHDFERR;
1495  if (attr_exists)
1496  {
1497  if (H5Adelete(hdf_datasetid, HDF5_DIMSCALE_CLASS_ATT_NAME) < 0)
1498  return NC_EHDFERR;
1499  }
1500  if ((attr_exists = H5Aexists(hdf_datasetid,
1501  HDF5_DIMSCALE_NAME_ATT_NAME)) < 0)
1502  return NC_EHDFERR;
1503  if (attr_exists)
1504  {
1505  if (H5Adelete(hdf_datasetid, HDF5_DIMSCALE_NAME_ATT_NAME) < 0)
1506  return NC_EHDFERR;
1507  }
1508  return NC_NOERR;
1509 }
1510 
1525 static int
1526 write_var(NC_VAR_INFO_T *var, NC_GRP_INFO_T *grp, nc_bool_t write_dimid)
1527 {
1528  NC_HDF5_GRP_INFO_T *hdf5_grp;
1529  NC_HDF5_VAR_INFO_T *hdf5_var;
1530  nc_bool_t replace_existing_var = NC_FALSE;
1531  int retval;
1532 
1533  assert(var && var->format_var_info && grp && grp->format_grp_info);
1534 
1535  LOG((4, "%s: writing var %s", __func__, var->hdr.name));
1536 
1537  /* Get HDF5-specific group and var info. */
1538  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1539  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
1540 
1541  /* If the variable has already been created & the fill value changed,
1542  * indicate that the existing variable should be replaced. */
1543  if (var->created && var->fill_val_changed)
1544  {
1545  replace_existing_var = NC_TRUE;
1546  var->fill_val_changed = NC_FALSE;
1547  /* If the variable is going to be replaced, we need to flag any
1548  other attributes associated with the variable as 'dirty', or
1549  else *only* the fill value attribute will be copied over and
1550  the rest will be lost. See
1551  https://github.com/Unidata/netcdf-c/issues/239 */
1552  flag_atts_dirty(var->att);
1553  }
1554 
1555  /* Is this a coordinate var that has already been created in
1556  * the HDF5 file as a dimscale dataset? Check for dims with the
1557  * same name in this group. If there is one, check to see if
1558  * this object exists in the HDF group. */
1559  if (var->became_coord_var)
1560  {
1561  if ((NC_DIM_INFO_T *)ncindexlookup(grp->dim, var->hdr.name))
1562  {
1563  nc_bool_t exists;
1564 
1565  if ((retval = var_exists(hdf5_grp->hdf_grpid, var->hdr.name, &exists)))
1566  return retval;
1567  if (exists)
1568  {
1569  /* Indicate that the variable already exists, and should
1570  * be replaced. */
1571  replace_existing_var = NC_TRUE;
1572  flag_atts_dirty(var->att);
1573  }
1574  }
1575  }
1576 
1577  /* Check dims if the variable will be replaced, so that the
1578  * dimensions will be de-attached and re-attached correctly. */
1579  if (replace_existing_var)
1580  {
1581  NC_DIM_INFO_T *d1;
1582 
1583  /* Is there a dim with this var's name? */
1584  if ((d1 = (NC_DIM_INFO_T *)ncindexlookup(grp->dim, var->hdr.name)))
1585  {
1586  nc_bool_t exists;
1587  assert(d1->format_dim_info && d1->hdr.name);
1588 
1589  if ((retval = var_exists(hdf5_grp->hdf_grpid, var->hdr.name, &exists)))
1590  return retval;
1591  if (exists)
1592  {
1593  hid_t dsid;
1594 
1595  /* Find dataset ID for dimension */
1596  if (d1->coord_var)
1597  dsid = ((NC_HDF5_VAR_INFO_T *)d1->coord_var->format_var_info)->hdf_datasetid;
1598  else
1599  dsid = ((NC_HDF5_DIM_INFO_T *)d1->format_dim_info)->hdf_dimscaleid;
1600  assert(dsid > 0);
1601 
1602  /* If we're replacing an existing dimscale dataset, go to
1603  * every var in the file and detach this dimension scale,
1604  * because we have to delete it. */
1605  if ((retval = rec_detach_scales(grp->nc4_info->root_grp,
1606  var->dimids[0], dsid)))
1607  return retval;
1608  }
1609  }
1610  }
1611 
1612  /* If this is not a dimension scale, remove any attached scales,
1613  * and delete dimscale attributes from the var. */
1614  if (var->was_coord_var && var->dimscale_attached)
1615  {
1616  int d;
1617 
1618  /* If the variable already exists in the file, Remove any dimension scale
1619  * attributes from it, if they exist. */
1620  if (var->created)
1621  if ((retval = remove_coord_atts(hdf5_var->hdf_datasetid)))
1622  return retval;
1623 
1624  /* If this is a regular var, detach all its dim scales. */
1625  for (d = 0; d < var->ndims; d++)
1626  {
1627  if (var->dimscale_attached[d])
1628  {
1629  hid_t dsid; /* Dataset ID for dimension */
1630  assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d] &&
1631  var->dim[d]->format_dim_info);
1632 
1633  /* Find dataset ID for dimension */
1634  if (var->dim[d]->coord_var)
1635  dsid = ((NC_HDF5_VAR_INFO_T *)var->dim[d]->coord_var->format_var_info)->hdf_datasetid;
1636  else
1637  dsid = ((NC_HDF5_DIM_INFO_T *)var->dim[d]->format_dim_info)->hdf_dimscaleid;
1638  assert(dsid > 0);
1639 
1640  /* Detach this dim scale. */
1641  if (H5DSdetach_scale(hdf5_var->hdf_datasetid, dsid, d) < 0)
1642  return NC_EHDFERR;
1643  var->dimscale_attached[d] = NC_FALSE;
1644  }
1645  }
1646  }
1647 
1648  /* Delete the HDF5 dataset that is to be replaced. */
1649  if (replace_existing_var)
1650  {
1651  /* Free the HDF5 dataset id. */
1652  if (hdf5_var->hdf_datasetid && H5Dclose(hdf5_var->hdf_datasetid) < 0)
1653  return NC_EHDFERR;
1654  hdf5_var->hdf_datasetid = 0;
1655 
1656  /* Now delete the variable. */
1657  if (H5Gunlink(hdf5_grp->hdf_grpid, var->hdr.name) < 0)
1658  return NC_EDIMMETA;
1659  }
1660 
1661  /* Create the dataset. */
1662  if (var->is_new_var || replace_existing_var)
1663  {
1664  if ((retval = var_create_dataset(grp, var, write_dimid)))
1665  return retval;
1666  }
1667  else
1668  {
1669  if (write_dimid && var->ndims)
1670  if ((retval = write_netcdf4_dimid(hdf5_var->hdf_datasetid,
1671  var->dimids[0])))
1672  return retval;
1673  }
1674 
1675  if (replace_existing_var)
1676  {
1677  /* If this is a dimension scale, reattach the scale everywhere it
1678  * is used. (Recall that netCDF dimscales are always 1-D). */
1679  if(var->dimscale)
1680  {
1681  if ((retval = rec_reattach_scales(grp->nc4_info->root_grp,
1682  var->dimids[0], hdf5_var->hdf_datasetid)))
1683  return retval;
1684  }
1685  /* If it's not a dimension scale, clear the dimscale attached flags,
1686  * so the dimensions are re-attached. */
1687  else
1688  {
1689  if (var->dimscale_attached)
1690  memset(var->dimscale_attached, 0, sizeof(nc_bool_t) * var->ndims);
1691  }
1692  }
1693 
1694  /* Clear coord. var state transition flags */
1695  var->was_coord_var = NC_FALSE;
1696  var->became_coord_var = NC_FALSE;
1697 
1698  /* Now check the attributes for this var. */
1699  if (var->attr_dirty)
1700  {
1701  /* Write attributes for this var. */
1702  if ((retval = write_attlist(var->att, var->hdr.id, grp)))
1703  return retval;
1704  var->attr_dirty = NC_FALSE;
1705  }
1706 
1707  return NC_NOERR;
1708 }
1709 
1723 int
1724 nc4_create_dim_wo_var(NC_DIM_INFO_T *dim)
1725 {
1726  NC_HDF5_DIM_INFO_T *hdf5_dim;
1727  NC_HDF5_GRP_INFO_T *hdf5_grp;
1728  hid_t spaceid = -1, create_propid = -1;
1729  hsize_t dims[1], max_dims[1], chunk_dims[1] = {1};
1730  char dimscale_wo_var[NC_MAX_NAME];
1731  int retval = NC_NOERR;
1732 
1733  LOG((4, "%s: creating dim %s", __func__, dim->hdr.name));
1734 
1735  /* Sanity check */
1736  assert(!dim->coord_var);
1737 
1738  /* Get HDF5-specific dim and group info. */
1739  hdf5_grp = (NC_HDF5_GRP_INFO_T *)dim->container->format_grp_info;
1740  hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
1741 
1742  /* Create a property list. */
1743  if ((create_propid = H5Pcreate(H5P_DATASET_CREATE)) < 0)
1744  BAIL(NC_EHDFERR);
1745 
1746  /* Turn off recording of times associated with this object. */
1747  if (H5Pset_obj_track_times(create_propid, 0) < 0)
1748  BAIL(NC_EHDFERR);
1749 
1750  /* Set size of dataset to size of dimension. */
1751  dims[0] = dim->len;
1752  max_dims[0] = dim->len;
1753 
1754  /* If this dimension scale is unlimited (i.e. it's an unlimited
1755  * dimension), then set up chunking, with a chunksize of 1. */
1756  if (dim->unlimited)
1757  {
1758  max_dims[0] = H5S_UNLIMITED;
1759  if (H5Pset_chunk(create_propid, 1, chunk_dims) < 0)
1760  BAIL(NC_EHDFERR);
1761  }
1762 
1763  /* Set up space. */
1764  if ((spaceid = H5Screate_simple(1, dims, max_dims)) < 0)
1765  BAIL(NC_EHDFERR);
1766 
1767  /* Turn on creation-order tracking. */
1768  if (H5Pset_attr_creation_order(create_propid, H5P_CRT_ORDER_TRACKED|
1769  H5P_CRT_ORDER_INDEXED) < 0)
1770  BAIL(NC_EHDFERR);
1771 
1772  /* Create the dataset that will be the dimension scale. */
1773  LOG((4, "%s: about to H5Dcreate1 a dimscale dataset %s", __func__,
1774  dim->hdr.name));
1775  if ((hdf5_dim->hdf_dimscaleid = H5Dcreate2(hdf5_grp->hdf_grpid, dim->hdr.name,
1776  H5T_IEEE_F32BE, spaceid,
1777  H5P_DEFAULT, create_propid,
1778  H5P_DEFAULT)) < 0)
1779  BAIL(NC_EHDFERR);
1780 
1781  /* Indicate that this is a scale. Also indicate that not
1782  * be shown to the user as a variable. It is hidden. It is
1783  * a DIM WITHOUT A VARIABLE! */
1784  sprintf(dimscale_wo_var, "%s%10d", DIM_WITHOUT_VARIABLE, (int)dim->len);
1785  if (H5DSset_scale(hdf5_dim->hdf_dimscaleid, dimscale_wo_var) < 0)
1786  BAIL(NC_EHDFERR);
1787 
1788  /* Since this dimension was created out of order, we cannot rely on
1789  * it getting the correct dimid on file open. We must assign it
1790  * explicitly. */
1791  if ((retval = write_netcdf4_dimid(hdf5_dim->hdf_dimscaleid, dim->hdr.id)))
1792  BAIL(retval);
1793 
1794 exit:
1795  if (spaceid > 0 && H5Sclose(spaceid) < 0)
1796  BAIL2(NC_EHDFERR);
1797  if (create_propid > 0 && H5Pclose(create_propid) < 0)
1798  BAIL2(NC_EHDFERR);
1799  return retval;
1800 }
1801 
1814 static int
1815 write_dim(NC_DIM_INFO_T *dim, NC_GRP_INFO_T *grp, nc_bool_t write_dimid)
1816 {
1817  NC_HDF5_DIM_INFO_T *hdf5_dim;
1818  int retval = NC_NOERR;
1819 
1820  assert(dim && dim->format_dim_info && grp && grp->format_grp_info);
1821 
1822  /* Get HDF5-specific dim and group info. */
1823  hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
1824 
1825  /* If there's no dimscale dataset for this dim, create one,
1826  * and mark that it should be hidden from netCDF as a
1827  * variable. (That is, it should appear as a dimension
1828  * without an associated variable.) */
1829  if (!hdf5_dim->hdf_dimscaleid)
1830  if ((retval = nc4_create_dim_wo_var(dim)))
1831  BAIL(retval);
1832 
1833  /* Did we extend an unlimited dimension? */
1834  if (dim->extended)
1835  {
1836  NC_VAR_INFO_T *v1 = NULL;
1837 
1838  assert(dim->unlimited);
1839  /* If this is a dimension without a variable, then update
1840  * the secret length information at the end of the NAME
1841  * attribute. */
1842  v1 = (NC_VAR_INFO_T *)ncindexlookup(grp->vars, dim->hdr.name);
1843  if (v1)
1844  {
1845  NC_HDF5_VAR_INFO_T *hdf5_v1;
1846  hsize_t *new_size;
1847  int d1;
1848 
1849  hdf5_v1 = (NC_HDF5_VAR_INFO_T *)v1->format_var_info;
1850 
1851  /* Extend the dimension scale dataset to reflect the new
1852  * length of the dimension. */
1853  if (!(new_size = malloc(v1->ndims * sizeof(hsize_t))))
1854  BAIL(NC_ENOMEM);
1855  for (d1 = 0; d1 < v1->ndims; d1++)
1856  {
1857  assert(v1->dim[d1] && v1->dim[d1]->hdr.id == v1->dimids[d1]);
1858  new_size[d1] = v1->dim[d1]->len;
1859  }
1860  if (H5Dset_extent(hdf5_v1->hdf_datasetid, new_size) < 0)
1861  BAIL(NC_EHDFERR);
1862  free(new_size);
1863  }
1864  }
1865 
1866  /* If desired, write the secret dimid. This will be used instead of
1867  * the dimid that the dimension would otherwise receive based on
1868  * creation order. This can be necessary when dims and their
1869  * coordinate variables were created in different order. */
1870  if (write_dimid && hdf5_dim->hdf_dimscaleid)
1871  if ((retval = write_netcdf4_dimid(hdf5_dim->hdf_dimscaleid, dim->hdr.id)))
1872  BAIL(retval);
1873 
1874 exit:
1875 
1876  return retval;
1877 }
1878 
1891 int
1892 nc4_rec_write_metadata(NC_GRP_INFO_T *grp, nc_bool_t bad_coord_order)
1893 {
1894  NC_DIM_INFO_T *dim = NULL;
1895  NC_VAR_INFO_T *var = NULL;
1896  NC_GRP_INFO_T *child_grp = NULL;
1897  int coord_varid = -1;
1898  int var_index = 0;
1899  int dim_index = 0;
1900  int retval;
1901  int i;
1902 
1903  assert(grp && grp->hdr.name &&
1904  ((NC_HDF5_GRP_INFO_T *)(grp->format_grp_info))->hdf_grpid);
1905  LOG((3, "%s: grp->hdr.name %s, bad_coord_order %d", __func__, grp->hdr.name,
1906  bad_coord_order));
1907 
1908  /* Write global attributes for this group. */
1909  if ((retval = write_attlist(grp->att, NC_GLOBAL, grp)))
1910  return retval;
1911 
1912  /* Set the pointers to the beginning of the list of dims & vars in this
1913  * group. */
1914  dim = (NC_DIM_INFO_T *)ncindexith(grp->dim, dim_index);
1915  var = (NC_VAR_INFO_T *)ncindexith(grp->vars, var_index);
1916 
1917  /* Because of HDF5 ordering the dims and vars have to be stored in
1918  * this way to ensure that the dims and coordinate vars come out in
1919  * the correct order. */
1920  while (dim || var)
1921  {
1922  nc_bool_t found_coord, wrote_coord;
1923 
1924  /* Write non-coord dims in order, stopping at the first one that
1925  * has an associated coord var. */
1926  for (found_coord = NC_FALSE; dim && !found_coord; )
1927  {
1928  if (!dim->coord_var)
1929  {
1930  if ((retval = write_dim(dim, grp, bad_coord_order)))
1931  return retval;
1932  }
1933  else
1934  {
1935  coord_varid = dim->coord_var->hdr.id;
1936  found_coord = NC_TRUE;
1937  }
1938  dim = (NC_DIM_INFO_T *)ncindexith(grp->dim, ++dim_index);
1939  }
1940 
1941  /* Write each var. When we get to the coord var we are waiting
1942  * for (if any), then we break after writing it. */
1943  for (wrote_coord = NC_FALSE; var && !wrote_coord; )
1944  {
1945  if ((retval = write_var(var, grp, bad_coord_order)))
1946  return retval;
1947  if (found_coord && var->hdr.id == coord_varid)
1948  wrote_coord = NC_TRUE;
1949  var = (NC_VAR_INFO_T *)ncindexith(grp->vars, ++var_index);
1950  }
1951  } /* end while */
1952 
1953  /* Attach dimscales to vars in this group. */
1954  if ((retval = attach_dimscales(grp)))
1955  return retval;
1956 
1957  /* If there are any child groups, write their metadata. */
1958  for (i = 0; i < ncindexsize(grp->children); i++)
1959  {
1960  child_grp = (NC_GRP_INFO_T *)ncindexith(grp->children, i);
1961  assert(child_grp);
1962  if ((retval = nc4_rec_write_metadata(child_grp, bad_coord_order)))
1963  return retval;
1964  }
1965  return NC_NOERR;
1966 }
1967 
1977 int
1978 nc4_rec_write_groups_types(NC_GRP_INFO_T *grp)
1979 {
1980  NC_GRP_INFO_T *child_grp;
1981  NC_HDF5_GRP_INFO_T *hdf5_grp;
1982  NC_TYPE_INFO_T *type;
1983  int retval;
1984  int i;
1985 
1986  assert(grp && grp->hdr.name && grp->format_grp_info);
1987  LOG((3, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
1988 
1989  /* Get HDF5-specific group info. */
1990  hdf5_grp = (NC_HDF5_GRP_INFO_T *)grp->format_grp_info;
1991 
1992  /* Create the group in the HDF5 file if it doesn't exist. */
1993  if (!hdf5_grp->hdf_grpid)
1994  if ((retval = create_group(grp)))
1995  return retval;
1996 
1997  /* If this is the root group of a file with strict NC3 rules, write
1998  * an attribute. But don't leave the attribute open. */
1999  if (!grp->parent && (grp->nc4_info->cmode & NC_CLASSIC_MODEL))
2000  if ((retval = write_nc3_strict_att(hdf5_grp->hdf_grpid)))
2001  return retval;
2002 
2003  /* If there are any user-defined types, write them now. */
2004  for(i=0;i<ncindexsize(grp->type);i++) {
2005  type = (NC_TYPE_INFO_T *)ncindexith(grp->type, i);
2006  assert(type);
2007  if ((retval = commit_type(grp, type)))
2008  return retval;
2009  }
2010 
2011  /* If there are any child groups, write their groups and types. */
2012  for(i=0;i<ncindexsize(grp->children);i++) {
2013  if((child_grp = (NC_GRP_INFO_T*)ncindexith(grp->children,i)) == NULL) continue;
2014  if ((retval = nc4_rec_write_groups_types(child_grp)))
2015  return retval;
2016  }
2017  return NC_NOERR;
2018 }
2019 
2033 int
2034 nc4_rec_match_dimscales(NC_GRP_INFO_T *grp)
2035 {
2036  NC_GRP_INFO_T *g;
2037  NC_VAR_INFO_T *var;
2038  NC_DIM_INFO_T *dim;
2039  int retval = NC_NOERR;
2040  int i;
2041 
2042  assert(grp && grp->hdr.name);
2043  LOG((4, "%s: grp->hdr.name %s", __func__, grp->hdr.name));
2044 
2045  /* Perform var dimscale match for child groups. */
2046  for (i = 0; i < ncindexsize(grp->children); i++)
2047  {
2048  g = (NC_GRP_INFO_T*)ncindexith(grp->children, i);
2049  assert(g);
2050  if ((retval = nc4_rec_match_dimscales(g)))
2051  return retval;
2052  }
2053 
2054  /* Check all the vars in this group. If they have dimscale info,
2055  * try and find a dimension for them. */
2056  for (i = 0; i < ncindexsize(grp->vars); i++)
2057  {
2058  NC_HDF5_VAR_INFO_T *hdf5_var;
2059  int ndims;
2060  int d;
2061 
2062  /* Get pointer to var and to the HDF5-specific var info. */
2063  var = (NC_VAR_INFO_T*)ncindexith(grp->vars, i);
2064  assert(var && var->format_var_info);
2065  hdf5_var = (NC_HDF5_VAR_INFO_T *)var->format_var_info;
2066 
2067  /* Check all vars and see if dim[i] != NULL if dimids[i] valid. */
2068  /* This loop is very odd. Under normal circumstances, var->dimid[d] is zero
2069  (from the initial calloc) which is a legitimate dimid. The code does not
2070  distinquish this case from the dimscale case where the id might actually
2071  be defined.
2072  The original nc4_find_dim searched up the group tree looking for the given
2073  dimid in one of the dim lists associated with each ancestor group.
2074  I changed nc4_fnd_dim to use the dimid directly using h5->alldims.
2075  However, here that is incorrect because it will find the dimid 0 always
2076  (if any dimensions were defined). Except that when dimscale dimids have
2077  been defined, one or more of the values in var->dimids will have a
2078  legitimate value.
2079  The solution I choose is to modify nc4_var_list_add to initialize dimids to
2080  illegal values (-1). This is another example of the problems with dimscales.
2081  */
2082  ndims = var->ndims;
2083  for (d = 0; d < ndims; d++)
2084  {
2085  if (var->dim[d] == NULL) {
2086  nc4_find_dim(grp, var->dimids[d], &var->dim[d], NULL);
2087  }
2088  /* assert(var->dim[d] && var->dim[d]->hdr.id == var->dimids[d]); */
2089  }
2090 
2091  /* Skip dimension scale variables */
2092  if (!var->dimscale)
2093  {
2094  int d;
2095  int j;
2096 
2097  /* Are there dimscales for this variable? */
2098  if (hdf5_var->dimscale_hdf5_objids)
2099  {
2100  for (d = 0; d < var->ndims; d++)
2101  {
2102  nc_bool_t finished = NC_FALSE;
2103  LOG((5, "%s: var %s has dimscale info...", __func__, var->hdr.name));
2104 
2105  /* Check this and parent groups. */
2106  for (g = grp; g && !finished; g = g->parent)
2107  {
2108  /* Check all dims in this group. */
2109  for (j = 0; j < ncindexsize(g->dim); j++)
2110  {
2111  /* Get the HDF5 specific dim info. */
2112  NC_HDF5_DIM_INFO_T *hdf5_dim;
2113  dim = (NC_DIM_INFO_T *)ncindexith(g->dim, j);
2114  assert(dim && dim->format_dim_info);
2115  hdf5_dim = (NC_HDF5_DIM_INFO_T *)dim->format_dim_info;
2116 
2117  /* Check for exact match of fileno/objid arrays
2118  * to find identical objects in HDF5 file. */
2119  if (hdf5_var->dimscale_hdf5_objids[d].fileno[0] == hdf5_dim->hdf5_objid.fileno[0] &&
2120  hdf5_var->dimscale_hdf5_objids[d].objno[0] == hdf5_dim->hdf5_objid.objno[0] &&
2121  hdf5_var->dimscale_hdf5_objids[d].fileno[1] == hdf5_dim->hdf5_objid.fileno[1] &&
2122  hdf5_var->dimscale_hdf5_objids[d].objno[1] == hdf5_dim->hdf5_objid.objno[1])
2123  {
2124  LOG((4, "%s: for dimension %d, found dim %s", __func__,
2125  d, dim->hdr.name));
2126  var->dimids[d] = dim->hdr.id;
2127  var->dim[d] = dim;
2128  finished = NC_TRUE;
2129  break;
2130  }
2131  } /* next dim */
2132  } /* next grp */
2133  LOG((5, "%s: dimid for this dimscale is %d", __func__,
2134  var->type_info->hdr.id));
2135  } /* next var->dim */
2136  }
2137  /* No dimscales for this var! Invent phony dimensions. */
2138  else
2139  {
2140  hid_t spaceid = 0;
2141  hsize_t *h5dimlen = NULL, *h5dimlenmax = NULL;
2142  int dataset_ndims;
2143 
2144  /* Find the space information for this dimension. */
2145  if ((spaceid = H5Dget_space(hdf5_var->hdf_datasetid)) < 0)
2146  return NC_EHDFERR;
2147 
2148  /* Get the len of each dim in the space. */
2149  if (var->ndims)
2150  {
2151  if (!(h5dimlen = malloc(var->ndims * sizeof(hsize_t))))
2152  return NC_ENOMEM;
2153  if (!(h5dimlenmax = malloc(var->ndims * sizeof(hsize_t))))
2154  {
2155  free(h5dimlen);
2156  return NC_ENOMEM;
2157  }
2158  if ((dataset_ndims = H5Sget_simple_extent_dims(spaceid, h5dimlen,
2159  h5dimlenmax)) < 0) {
2160  free(h5dimlenmax);
2161  free(h5dimlen);
2162  return NC_EHDFERR;
2163  }
2164  if (dataset_ndims != var->ndims) {
2165  free(h5dimlenmax);
2166  free(h5dimlen);
2167  return NC_EHDFERR;
2168  }
2169  }
2170  else
2171  {
2172  /* Make sure it's scalar. */
2173  if (H5Sget_simple_extent_type(spaceid) != H5S_SCALAR)
2174  return NC_EHDFERR;
2175  }
2176 
2177  /* Release the space object. */
2178  if (H5Sclose(spaceid) < 0) {
2179  free(h5dimlen);
2180  free(h5dimlenmax);
2181  return NC_EHDFERR;
2182  }
2183 
2184  /* Create a phony dimension for each dimension in the
2185  * dataset, unless there already is one the correct
2186  * size. */
2187  for (d = 0; d < var->ndims; d++)
2188  {
2189  int k;
2190  int match;
2191  /* Is there already a phony dimension of the correct size? */
2192  for(match=-1,k=0;k<ncindexsize(grp->dim);k++) {
2193  if((dim = (NC_DIM_INFO_T*)ncindexith(grp->dim,k)) == NULL) continue;
2194  if ((dim->len == h5dimlen[d]) &&
2195  ((h5dimlenmax[d] == H5S_UNLIMITED && dim->unlimited) ||
2196  (h5dimlenmax[d] != H5S_UNLIMITED && !dim->unlimited)))
2197  {match = k; break;}
2198  }
2199 
2200  /* Didn't find a phony dim? Then create one. */
2201  if (match < 0)
2202  {
2203  char phony_dim_name[NC_MAX_NAME + 1];
2204  sprintf(phony_dim_name, "phony_dim_%d", grp->nc4_info->next_dimid);
2205  LOG((3, "%s: creating phony dim for var %s", __func__, var->hdr.name));
2206  if ((retval = nc4_dim_list_add(grp, phony_dim_name, h5dimlen[d], -1, &dim)))
2207  {
2208  free(h5dimlenmax);
2209  free(h5dimlen);
2210  return retval;
2211  }
2212  /* Create struct for HDF5-specific dim info. */
2213  if (!(dim->format_dim_info = calloc(1, sizeof(NC_HDF5_DIM_INFO_T))))
2214  return NC_ENOMEM;
2215  if (h5dimlenmax[d] == H5S_UNLIMITED)
2216  dim->unlimited = NC_TRUE;
2217  }
2218 
2219  /* The variable must remember the dimid. */
2220  var->dimids[d] = dim->hdr.id;
2221  var->dim[d] = dim;
2222  } /* next dim */
2223 
2224  /* Free the memory we malloced. */
2225  free(h5dimlen);
2226  free(h5dimlenmax);
2227  }
2228  }
2229  }
2230 
2231  return retval;
2232 }
2233 
2246 int
2247 nc4_get_typeclass(const NC_FILE_INFO_T *h5, nc_type xtype, int *type_class)
2248 {
2249  int retval = NC_NOERR;
2250 
2251  LOG((4, "%s xtype: %d", __func__, xtype));
2252  assert(type_class);
2253 
2254  /* If this is an atomic type, the answer is easy. */
2255  if (xtype <= NC_STRING)
2256  {
2257  switch (xtype)
2258  {
2259  case NC_BYTE:
2260  case NC_UBYTE:
2261  case NC_SHORT:
2262  case NC_USHORT:
2263  case NC_INT:
2264  case NC_UINT:
2265  case NC_INT64:
2266  case NC_UINT64:
2267  /* NC_INT is class used for all integral types */
2268  *type_class = NC_INT;
2269  break;
2270 
2271  case NC_FLOAT:
2272  case NC_DOUBLE:
2273  /* NC_FLOAT is class used for all floating-point types */
2274  *type_class = NC_FLOAT;
2275  break;
2276 
2277  case NC_CHAR:
2278  *type_class = NC_CHAR;
2279  break;
2280 
2281  case NC_STRING:
2282  *type_class = NC_STRING;
2283  break;
2284 
2285  default:
2286  BAIL(NC_EBADTYPE);
2287  }
2288  }
2289  else
2290  {
2291  NC_TYPE_INFO_T *type;
2292 
2293  /* See if it's a used-defined type */
2294  if ((retval = nc4_find_type(h5, xtype, &type)))
2295  BAIL(retval);
2296  if (!type)
2297  BAIL(NC_EBADTYPE);
2298 
2299  *type_class = type->nc_type_class;
2300  }
2301 
2302 exit:
2303  return retval;
2304 }
2305 
2317 void
2318 reportobject(int uselog, hid_t id, unsigned int type)
2319 {
2320  char name[NC_HDF5_MAX_NAME];
2321  ssize_t len;
2322  const char* typename = NULL;
2323  long long printid = (long long)id;
2324 
2325  len = H5Iget_name(id, name, NC_HDF5_MAX_NAME);
2326  if(len < 0) return;
2327  name[len] = '\0';
2328 
2329  switch (type) {
2330  case H5F_OBJ_FILE: typename = "File"; break;
2331  case H5F_OBJ_DATASET: typename = "Dataset"; break;
2332  case H5F_OBJ_GROUP: typename = "Group"; break;
2333  case H5F_OBJ_DATATYPE: typename = "Datatype"; break;
2334  case H5F_OBJ_ATTR:
2335  typename = "Attribute";
2336  len = H5Aget_name(id, NC_HDF5_MAX_NAME, name);
2337  if(len < 0) len = 0;
2338  name[len] = '\0';
2339  break;
2340  default: typename = "<unknown>"; break;
2341  }
2342 #ifdef LOGGING
2343  if(uselog) {
2344  LOG((0,"Type = %s(%lld) name='%s'",typename,printid,name));
2345  } else
2346 #endif
2347  {
2348  fprintf(stderr,"Type = %s(%lld) name='%s'",typename,printid,name);
2349  }
2350 
2351 }
2352 
2363 static void
2364 reportopenobjectsT(int uselog, hid_t fid, int ntypes, unsigned int* otypes)
2365 {
2366  int t,i;
2367  ssize_t ocount;
2368  size_t maxobjs = -1;
2369  hid_t* idlist = NULL;
2370 
2371  /* Always report somehow */
2372 #ifdef LOGGING
2373  if(uselog)
2374  LOG((0,"\nReport: open objects on %lld",(long long)fid));
2375  else
2376 #endif
2377  fprintf(stdout,"\nReport: open objects on %lld\n",(long long)fid);
2378  maxobjs = H5Fget_obj_count(fid,H5F_OBJ_ALL);
2379  if(idlist != NULL) free(idlist);
2380  idlist = (hid_t*)malloc(sizeof(hid_t)*maxobjs);
2381  for(t=0;t<ntypes;t++) {
2382  unsigned int ot = otypes[t];
2383  ocount = H5Fget_obj_ids(fid,ot,maxobjs,idlist);
2384  for(i=0;i<ocount;i++) {
2385  hid_t o = idlist[i];
2386  reportobject(uselog,o,ot);
2387  }
2388  }
2389  if(idlist != NULL) free(idlist);
2390 }
2391 
2400 void
2401 reportopenobjects(int uselog, hid_t fid)
2402 {
2403  unsigned int OTYPES[5] = {H5F_OBJ_FILE, H5F_OBJ_DATASET, H5F_OBJ_GROUP,
2404  H5F_OBJ_DATATYPE, H5F_OBJ_ATTR};
2405 
2406  reportopenobjectsT(uselog, fid ,5, OTYPES);
2407 }
2408 
2416 void
2417 showopenobjects5(NC_FILE_INFO_T* h5)
2418 {
2419  NC_HDF5_FILE_INFO_T *hdf5_info;
2420 
2421  assert(h5 && h5->format_file_info);
2422  hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info;
2423 
2424  fprintf(stderr,"===== begin showopenobjects =====\n");
2425  reportopenobjects(0,hdf5_info->hdfid);
2426  fprintf(stderr,"===== end showopenobjects =====\n");
2427  fflush(stderr);
2428 }
2429 
2438 void
2439 showopenobjects(int ncid)
2440 {
2441  NC_FILE_INFO_T* h5 = NULL;
2442 
2443  /* Find our metadata for this file. */
2444  if (nc4_find_nc_grp_h5(ncid, NULL, NULL, &h5) != NC_NOERR)
2445  fprintf(stderr,"failed\n");
2446  else
2447  showopenobjects5(h5);
2448  fflush(stderr);
2449 }
2450 
2462 int
2463 NC4_hdf5get_libversion(unsigned* major,unsigned* minor,unsigned* release)
2464 {
2465  if(H5get_libversion(major,minor,release) < 0)
2466  return NC_EHDFERR;
2467  return NC_NOERR;
2468 }
2469 
2480 int
2481 NC4_hdf5get_superblock(struct NC_FILE_INFO* h5, int* idp)
2482 {
2483  NC_HDF5_FILE_INFO_T *hdf5_info;
2484  int stat = NC_NOERR;
2485  unsigned super;
2486  hid_t plist = -1;
2487 
2488  assert(h5 && h5->format_file_info);
2489  hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info;
2490 
2491  if((plist = H5Fget_create_plist(hdf5_info->hdfid)) < 0)
2492  {stat = NC_EHDFERR; goto done;}
2493  if(H5Pget_version(plist, &super, NULL, NULL, NULL) < 0)
2494  {stat = NC_EHDFERR; goto done;}
2495  if(idp) *idp = (int)super;
2496 done:
2497  if(plist >= 0) H5Pclose(plist);
2498  return stat;
2499 }
2500 
2501 static int NC4_get_strict_att(NC_FILE_INFO_T*);
2502 static int NC4_walk(hid_t, int*);
2503 
2528 int
2529 NC4_isnetcdf4(struct NC_FILE_INFO* h5)
2530 {
2531  int stat;
2532  int isnc4 = 0;
2533  int count;
2534 
2535  /* Look for NC3_STRICT_ATT_NAME */
2536  isnc4 = NC4_get_strict_att(h5);
2537  if(isnc4 > 0)
2538  goto done;
2539  /* attribute did not exist */
2540  /* => last resort: walk the HDF5 file looking for markers */
2541  count = 0;
2542  stat = NC4_walk(((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid,
2543  &count);
2544  if(stat != NC_NOERR)
2545  isnc4 = 0;
2546  else /* Threshold is at least two matches */
2547  isnc4 = (count >= 2);
2548 
2549 done:
2550  return isnc4;
2551 }
2552 
2561 static int
2562 NC4_get_strict_att(NC_FILE_INFO_T *h5)
2563 {
2564  hid_t grpid = -1;
2565  hid_t attid = -1;
2566 
2567  /* Get root group ID. */
2568  grpid = ((NC_HDF5_GRP_INFO_T *)(h5->root_grp->format_grp_info))->hdf_grpid;
2569 
2570  /* Try to extract the NC3_STRICT_ATT_NAME attribute */
2571  attid = H5Aopen_name(grpid, NC3_STRICT_ATT_NAME);
2572  H5Aclose(attid);
2573  return attid;
2574 }
2575 
2585 static int
2586 NC4_walk(hid_t gid, int* countp)
2587 {
2588  int ncstat = NC_NOERR;
2589  int i,j,na;
2590  ssize_t len;
2591  hsize_t nobj;
2592  herr_t err;
2593  int otype;
2594  hid_t grpid, dsid;
2595  char name[NC_HDF5_MAX_NAME];
2596 
2597  /* walk group members of interest */
2598  err = H5Gget_num_objs(gid, &nobj);
2599  if(err < 0) return err;
2600 
2601  for(i = 0; i < nobj; i++) {
2602  /* Get name & kind of object in the group */
2603  len = H5Gget_objname_by_idx(gid,(hsize_t)i,name,(size_t)NC_HDF5_MAX_NAME);
2604  if(len < 0) return len;
2605 
2606  otype = H5Gget_objtype_by_idx(gid,(size_t)i);
2607  switch(otype) {
2608  case H5G_GROUP:
2609  grpid = H5Gopen(gid,name);
2610  NC4_walk(grpid,countp);
2611  H5Gclose(grpid);
2612  break;
2613  case H5G_DATASET: /* variables */
2614  /* Check for phony_dim */
2615  if(strcmp(name,"phony_dim")==0)
2616  *countp = *countp + 1;
2617  dsid = H5Dopen(gid,name);
2618  na = H5Aget_num_attrs(dsid);
2619  for(j = 0; j < na; j++) {
2620  hid_t aid = H5Aopen_idx(dsid,(unsigned int) j);
2621  if(aid >= 0) {
2622  const NC_reservedatt* ra;
2623  ssize_t len = H5Aget_name(aid, NC_HDF5_MAX_NAME, name);
2624  if(len < 0) return len;
2625  /* Is this a netcdf-4 marker attribute */
2626  /* Is this a netcdf-4 marker attribute */
2627  ra = NC_findreserved(name);
2628  if(ra != NULL)
2629  *countp = *countp + 1;
2630  }
2631  H5Aclose(aid);
2632  }
2633  H5Dclose(dsid);
2634  break;
2635  default:/* ignore */
2636  break;
2637  }
2638  }
2639  return ncstat;
2640 }
#define NC_ENOMEM
Memory allocation (malloc) failure.
Definition: netcdf.h:403
#define NC_CHAR
ISO/ASCII character.
Definition: netcdf.h:36
#define NC_UBYTE
unsigned 1 byte int
Definition: netcdf.h:42
#define NC_CLASSIC_MODEL
Enforce classic model on netCDF-4.
Definition: netcdf.h:138
#define NC_MAX_VAR_DIMS
max per variable dimensions
Definition: netcdf.h:274
#define NC_UINT
unsigned 4-byte int
Definition: netcdf.h:44
#define NC_EHDFERR
Error at HDF5 layer.
Definition: netcdf.h:436
#define NC_OPAQUE
opaque types
Definition: netcdf.h:54
#define NC_INT64
signed 8-byte int
Definition: netcdf.h:45
#define NC_STRING
string
Definition: netcdf.h:47
#define NC_DOUBLE
double precision floating point number
Definition: netcdf.h:41
int nc_type
The nc_type type is just an int.
Definition: netcdf.h:25
#define H5Z_FILTER_SZIP
ID of HDF SZIP filter.
Definition: dvarinq.c:16
#define NC_BYTE
signed 1 byte integer
Definition: netcdf.h:35
size_t len
Length of VL data (in base type units)
Definition: netcdf.h:686
#define NC_ENDIAN_LITTLE
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:284
#define NC_EATTMETA
Problem with attribute metadata.
Definition: netcdf.h:442
#define NC_VLEN
vlen (variable-length) types
Definition: netcdf.h:53
#define NC_EFILEMETA
Problem with file metadata.
Definition: netcdf.h:440
#define NC_EFILTER
Filter operation failed.
Definition: netcdf.h:468
#define NC_EBADTYPE
Not a netcdf data type.
Definition: netcdf.h:365
#define NC_EDIMMETA
Problem with dimension metadata.
Definition: netcdf.h:441
#define NC_EINVAL
Invalid Argument.
Definition: netcdf.h:333
#define NC_INT
signed 4 byte integer
Definition: netcdf.h:38
#define NC_ENDIAN_BIG
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:285
#define NC_MAX_NAME
Maximum for classic library.
Definition: netcdf.h:273
void * p
Pointer to VL data.
Definition: netcdf.h:687
#define NC_NAT
Not A Type.
Definition: netcdf.h:34
EXTERNL int nc_free_vlen(nc_vlen_t *vl)
Free memory in a VLEN object.
Definition: dvlen.c:41
#define NC_USHORT
unsigned 2-byte int
Definition: netcdf.h:43
#define NC_EVARMETA
Problem with variable metadata.
Definition: netcdf.h:443
This is the type of arrays of vlens.
Definition: netcdf.h:685
#define NC_SHORT
signed 2 byte integer
Definition: netcdf.h:37
#define NC_ENDIAN_NATIVE
In HDF5 files you can set the endianness of variables with nc_def_var_endian().
Definition: netcdf.h:283
#define NC_ENOTVAR
Variable not found.
Definition: netcdf.h:377
#define NC_EPERM
Write to read only.
Definition: netcdf.h:334
#define NC_NOERR
No Error.
Definition: netcdf.h:323
#define NC_ENUM
enum types
Definition: netcdf.h:55
#define NC_COMPOUND
compound types
Definition: netcdf.h:56
#define NC_GLOBAL
Attribute id to put/get a global attribute.
Definition: netcdf.h:246
#define NC_FLOAT
single precision floating point number
Definition: netcdf.h:40
#define NC_UINT64
unsigned 8-byte int
Definition: netcdf.h:46

Return to the Main Unidata NetCDF page.
Generated on Sat Apr 6 2019 08:19:00 for NetCDF. NetCDF is a Unidata library.