NetCDF  4.6.3
 All Data Structures Files Functions Variables Typedefs Macros Modules Pages
hdf5file.c
Go to the documentation of this file.
1 /* Copyright 2003-2018, University Corporation for Atmospheric
2  * Research. See COPYRIGHT file for copying and redistribution
3  * conditions. */
14 #include "config.h"
15 #include "hdf5internal.h"
16 #include "ncrc.h"
17 
18 extern int NC4_extract_file_image(NC_FILE_INFO_T* h5); /* In nc4memcb.c */
19 
20 static void dumpopenobjects(NC_FILE_INFO_T* h5);
21 
24 #define LOGOPEN 1
25 
29 #define NRESERVED 11 /*|NC_reservedatt|*/
30 
33 static const NC_reservedatt NC_reserved[NRESERVED] = {
34  {NC_ATT_CLASS, READONLYFLAG|DIMSCALEFLAG}, /*CLASS*/
35  {NC_ATT_DIMENSION_LIST, READONLYFLAG|DIMSCALEFLAG}, /*DIMENSION_LIST*/
36  {NC_ATT_NAME, READONLYFLAG|DIMSCALEFLAG}, /*NAME*/
37  {NC_ATT_REFERENCE_LIST, READONLYFLAG|DIMSCALEFLAG}, /*REFERENCE_LIST*/
38  {NC_ATT_FORMAT, READONLYFLAG}, /*_Format*/
39  {ISNETCDF4ATT, READONLYFLAG|NAMEONLYFLAG}, /*_IsNetcdf4*/
40  {NCPROPS, READONLYFLAG|NAMEONLYFLAG|MATERIALIZEDFLAG},/*_NCProperties*/
41  {NC_ATT_COORDINATES, READONLYFLAG|DIMSCALEFLAG|MATERIALIZEDFLAG},/*_Netcdf4Coordinates*/
42  {NC_DIMID_ATT_NAME, READONLYFLAG|DIMSCALEFLAG|MATERIALIZEDFLAG},/*_Netcdf4Dimid*/
43  {SUPERBLOCKATT, READONLYFLAG|NAMEONLYFLAG},/*_SuperblockVersion*/
44  {NC3_STRICT_ATT_NAME, READONLYFLAG|MATERIALIZEDFLAG}, /*_nc3_strict*/
45 };
46 
47 /* Forward */
48 static int NC4_enddef(int ncid);
49 static void dumpopenobjects(NC_FILE_INFO_T* h5);
50 
58 const NC_reservedatt*
59 NC_findreserved(const char* name)
60 {
61  int n = NRESERVED;
62  int L = 0;
63  int R = (n - 1);
64  for(;;) {
65  if(L > R) break;
66  int m = (L + R) / 2;
67  const NC_reservedatt* p = &NC_reserved[m];
68  int cmp = strcmp(p->name,name);
69  if(cmp == 0) return p;
70  if(cmp < 0)
71  L = (m + 1);
72  else /*cmp > 0*/
73  R = (m - 1);
74  }
75  return NULL;
76 }
77 
94 static int
95 detect_preserve_dimids(NC_GRP_INFO_T *grp, nc_bool_t *bad_coord_orderp)
96 {
97  NC_VAR_INFO_T *var;
98  NC_GRP_INFO_T *child_grp;
99  int last_dimid = -1;
100  int retval;
101  int i;
102 
103  /* Iterate over variables in this group */
104  for (i=0; i < ncindexsize(grp->vars); i++)
105  {
106  var = (NC_VAR_INFO_T*)ncindexith(grp->vars,i);
107  if (var == NULL) continue;
108  /* Only matters for dimension scale variables, with non-scalar dimensionality */
109  if (var->dimscale && var->ndims)
110  {
111  /* If the user writes coord vars in a different order then he
112  * defined their dimensions, then, when the file is reopened, the
113  * order of the dimids will change to match the order of the coord
114  * vars. Detect if this is about to happen. */
115  if (var->dimids[0] < last_dimid)
116  {
117  LOG((5, "%s: %s is out of order coord var", __func__, var->hdr.name));
118  *bad_coord_orderp = NC_TRUE;
119  return NC_NOERR;
120  }
121  last_dimid = var->dimids[0];
122 
123  /* If there are multidimensional coordinate variables defined, then
124  * it's also necessary to preserve dimension IDs when the file is
125  * reopened ... */
126  if (var->ndims > 1)
127  {
128  LOG((5, "%s: %s is multidimensional coord var", __func__, var->hdr.name));
129  *bad_coord_orderp = NC_TRUE;
130  return NC_NOERR;
131  }
132 
133  /* Did the user define a dimension, end define mode, reenter define
134  * mode, and then define a coordinate variable for that dimension?
135  * If so, dimensions will be out of order. */
136  if (var->is_new_var || var->became_coord_var)
137  {
138  LOG((5, "%s: coord var defined after enddef/redef", __func__));
139  *bad_coord_orderp = NC_TRUE;
140  return NC_NOERR;
141  }
142  }
143  }
144 
145  /* If there are any child groups, check them also for this condition. */
146  for (i = 0; i < ncindexsize(grp->children); i++)
147  {
148  if (!(child_grp = (NC_GRP_INFO_T *)ncindexith(grp->children, i)))
149  continue;
150  if ((retval = detect_preserve_dimids(child_grp, bad_coord_orderp)))
151  return retval;
152  }
153  return NC_NOERR;
154 }
155 
167 static int
168 sync_netcdf4_file(NC_FILE_INFO_T *h5)
169 {
170  NC_HDF5_FILE_INFO_T *hdf5_info;
171  int retval;
172 
173  assert(h5 && h5->format_file_info);
174  LOG((3, "%s", __func__));
175 
176  /* If we're in define mode, that's an error, for strict nc3 rules,
177  * otherwise, end define mode. */
178  if (h5->flags & NC_INDEF)
179  {
180  if (h5->cmode & NC_CLASSIC_MODEL)
181  return NC_EINDEFINE;
182 
183  /* Turn define mode off. */
184  h5->flags ^= NC_INDEF;
185 
186  /* Redef mode needs to be tracked separately for nc_abort. */
187  h5->redef = NC_FALSE;
188  }
189 
190 #ifdef LOGGING
191  /* This will print out the names, types, lens, etc of the vars and
192  atts in the file, if the logging level is 2 or greater. */
193  log_metadata_nc(h5);
194 #endif
195 
196  /* Write any metadata that has changed. */
197  if (!h5->no_write)
198  {
199  nc_bool_t bad_coord_order = NC_FALSE;
200 
201  /* Write any user-defined types. */
202  if ((retval = nc4_rec_write_groups_types(h5->root_grp)))
203  return retval;
204 
205  /* Check to see if the coordinate order is messed up. If
206  * detected, propagate to all groups to consistently store
207  * dimids. */
208  if ((retval = detect_preserve_dimids(h5->root_grp, &bad_coord_order)))
209  return retval;
210 
211  /* Write all the metadata. */
212  if ((retval = nc4_rec_write_metadata(h5->root_grp, bad_coord_order)))
213  return retval;
214 
215  /* Write out _NCProperties */
216  if((retval = NC4_write_ncproperties(h5)))
217  return retval;
218  }
219 
220  /* Tell HDF5 to flush all changes to the file. */
221  hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info;
222  if (H5Fflush(hdf5_info->hdfid, H5F_SCOPE_GLOBAL) < 0)
223  return NC_EHDFERR;
224 
225  return NC_NOERR;
226 }
227 
243 int
244 nc4_close_netcdf4_file(NC_FILE_INFO_T *h5, int abort, NC_memio *memio)
245 {
246  NC_HDF5_FILE_INFO_T *hdf5_info;
247  int retval;
248 
249  assert(h5 && h5->root_grp && h5->format_file_info);
250  LOG((3, "%s: h5->path %s abort %d", __func__, h5->controller->path, abort));
251 
252  /* Get HDF5 specific info. */
253  hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info;
254 
255  /* Delete all the list contents for vars, dims, and atts, in each
256  * group. */
257  if ((retval = nc4_rec_grp_del(h5->root_grp)))
258  return retval;
259 
260  /* Free lists of dims, groups, and types in the root group. */
261  nclistfree(h5->alldims);
262  nclistfree(h5->allgroups);
263  nclistfree(h5->alltypes);
264 
265 #ifdef USE_PARALLEL4
266  /* Free the MPI Comm & Info objects, if we opened the file in
267  * parallel. */
268  if (h5->parallel)
269  {
270  if (h5->comm != MPI_COMM_NULL)
271  MPI_Comm_free(&h5->comm);
272  if (h5->info != MPI_INFO_NULL)
273  MPI_Info_free(&h5->info);
274  }
275 #endif
276 
277  /* Free the fileinfo struct, which holds info from the fileinfo
278  * hidden attribute. */
279  if (h5->provenance)
280  NC4_free_provenance(h5->provenance);
281  h5->provenance = NULL; /* Avoid double dealloc */
282 
283  /* Close hdf file. It may not be open, since this function is also
284  * called by NC_create() when a file opening is aborted. */
285  if (hdf5_info->hdfid > 0 && H5Fclose(hdf5_info->hdfid) < 0)
286  {
287  dumpopenobjects(h5);
288  return NC_EHDFERR;
289  }
290 
291  /* If inmemory is used and user wants the final memory block,
292  then capture and return the final memory block else free it */
293  if(h5->mem.inmemory) {
294  /* Pull out the final memory */
295  (void)NC4_extract_file_image(h5);
296  if(!abort && memio != NULL) {
297  *memio = h5->mem.memio; /* capture it */
298  h5->mem.memio.memory = NULL; /* avoid duplicate free */
299  }
300  /* If needed, reclaim extraneous memory */
301  if(h5->mem.memio.memory != NULL) {
302  /* If the original block of memory is not resizeable, then
303  it belongs to the caller and we should not free it. */
304  if(!h5->mem.locked)
305  free(h5->mem.memio.memory);
306  }
307  h5->mem.memio.memory = NULL;
308  h5->mem.memio.size = 0;
309  NC4_image_finalize(h5->mem.udata);
310  }
311 
312  /* Free the HDF5-specific info. */
313  if (h5->format_file_info)
314  free(h5->format_file_info);
315 
316  /* Free the nc4_info struct; above code should have reclaimed
317  everything else */
318  free(h5);
319 
320  return NC_NOERR;
321 }
322 
336 int
337 nc4_close_hdf5_file(NC_FILE_INFO_T *h5, int abort, NC_memio *memio)
338 {
339  int retval;
340 
341  assert(h5 && h5->root_grp && h5->format_file_info);
342  LOG((3, "%s: h5->path %s abort %d", __func__, h5->controller->path, abort));
343 
344  /* According to the docs, always end define mode on close. */
345  if (h5->flags & NC_INDEF)
346  h5->flags ^= NC_INDEF;
347 
348  /* Sync the file, unless we're aborting, or this is a read-only
349  * file. */
350  if (!h5->no_write && !abort)
351  if ((retval = sync_netcdf4_file(h5)))
352  return retval;
353 
354  /* Close all open HDF5 objects within the file. */
355  if ((retval = nc4_rec_grp_HDF5_del(h5->root_grp)))
356  return retval;
357 
358  /* Release all intarnal lists and metadata associated with this
359  * file. All HDF5 objects have already been released. */
360  if ((retval = nc4_close_netcdf4_file(h5, abort, memio)))
361  return retval;
362 
363  return NC_NOERR;
364 }
365 
374 static void
375 dumpopenobjects(NC_FILE_INFO_T* h5)
376 {
377  NC_HDF5_FILE_INFO_T *hdf5_info;
378  int nobjs;
379 
380  assert(h5 && h5->format_file_info);
381  hdf5_info = (NC_HDF5_FILE_INFO_T *)h5->format_file_info;
382 
383  if(hdf5_info->hdfid <= 0)
384  return; /* File was never opened */
385 
386  nobjs = H5Fget_obj_count(hdf5_info->hdfid, H5F_OBJ_ALL);
387 
388  /* Apparently we can get an error even when nobjs == 0 */
389  if(nobjs < 0) {
390  return;
391  } else if(nobjs > 0) {
392  char msg[1024];
393  int logit = 0;
394  /* If the close doesn't work, probably there are still some HDF5
395  * objects open, which means there's a bug in the library. So
396  * print out some info on to help the poor programmer figure it
397  * out. */
398  snprintf(msg,sizeof(msg),"There are %d HDF5 objects open!", nobjs);
399 #ifdef LOGGING
400 #ifdef LOGOPEN
401  LOG((0, msg));
402  logit = 1;
403 #endif
404 #else
405  fprintf(stdout,"%s\n",msg);
406  logit = 0;
407 #endif
408  reportopenobjects(logit,hdf5_info->hdfid);
409  fflush(stderr);
410  }
411 
412  return;
413 }
414 
429 int
430 NC4_set_fill(int ncid, int fillmode, int *old_modep)
431 {
432  NC_FILE_INFO_T *nc4_info;
433  int retval;
434 
435  LOG((2, "%s: ncid 0x%x fillmode %d", __func__, ncid, fillmode));
436 
437  /* Get pointer to file info. */
438  if ((retval = nc4_find_grp_h5(ncid, NULL, &nc4_info)))
439  return retval;
440  assert(nc4_info);
441 
442  /* Trying to set fill on a read-only file? You sicken me! */
443  if (nc4_info->no_write)
444  return NC_EPERM;
445 
446  /* Did you pass me some weird fillmode? */
447  if (fillmode != NC_FILL && fillmode != NC_NOFILL)
448  return NC_EINVAL;
449 
450  /* If the user wants to know, tell him what the old mode was. */
451  if (old_modep)
452  *old_modep = nc4_info->fill_mode;
453 
454  nc4_info->fill_mode = fillmode;
455 
456  return NC_NOERR;
457 }
458 
468 int
469 NC4_redef(int ncid)
470 {
471  NC_FILE_INFO_T *nc4_info;
472  int retval;
473 
474  LOG((1, "%s: ncid 0x%x", __func__, ncid));
475 
476  /* Find this file's metadata. */
477  if ((retval = nc4_find_grp_h5(ncid, NULL, &nc4_info)))
478  return retval;
479  assert(nc4_info);
480 
481  /* If we're already in define mode, return an error. */
482  if (nc4_info->flags & NC_INDEF)
483  return NC_EINDEFINE;
484 
485  /* If the file is read-only, return an error. */
486  if (nc4_info->no_write)
487  return NC_EPERM;
488 
489  /* Set define mode. */
490  nc4_info->flags |= NC_INDEF;
491 
492  /* For nc_abort, we need to remember if we're in define mode as a
493  redef. */
494  nc4_info->redef = NC_TRUE;
495 
496  return NC_NOERR;
497 }
498 
512 int
513 NC4__enddef(int ncid, size_t h_minfree, size_t v_align,
514  size_t v_minfree, size_t r_align)
515 {
516  return NC4_enddef(ncid);
517 }
518 
530 static int
531 NC4_enddef(int ncid)
532 {
533  NC_FILE_INFO_T *nc4_info;
534  NC_GRP_INFO_T *grp;
535  NC_VAR_INFO_T *var;
536  int i;
537  int retval;
538 
539  LOG((1, "%s: ncid 0x%x", __func__, ncid));
540 
541  /* Find pointer to group and nc4_info. */
542  if ((retval = nc4_find_grp_h5(ncid, &grp, &nc4_info)))
543  return retval;
544 
545  /* When exiting define mode, mark all variable written. */
546  for (i = 0; i < ncindexsize(grp->vars); i++)
547  {
548  var = (NC_VAR_INFO_T *)ncindexith(grp->vars, i);
549  assert(var);
550  var->written_to = NC_TRUE;
551  }
552 
553  return nc4_enddef_netcdf4_file(nc4_info);
554 }
555 
567 int
568 NC4_sync(int ncid)
569 {
570  NC_FILE_INFO_T *nc4_info;
571  int retval;
572 
573  LOG((2, "%s: ncid 0x%x", __func__, ncid));
574 
575  if ((retval = nc4_find_grp_h5(ncid, NULL, &nc4_info)))
576  return retval;
577  assert(nc4_info);
578 
579  /* If we're in define mode, we can't sync. */
580  if (nc4_info->flags & NC_INDEF)
581  {
582  if (nc4_info->cmode & NC_CLASSIC_MODEL)
583  return NC_EINDEFINE;
584  if ((retval = NC4_enddef(ncid)))
585  return retval;
586  }
587 
588  return sync_netcdf4_file(nc4_info);
589 }
590 
604 int
605 NC4_abort(int ncid)
606 {
607  NC *nc;
608  NC_FILE_INFO_T *nc4_info;
609  int delete_file = 0;
610  char path[NC_MAX_NAME + 1];
611  int retval;
612 
613  LOG((2, "%s: ncid 0x%x", __func__, ncid));
614 
615  /* Find metadata for this file. */
616  if ((retval = nc4_find_nc_grp_h5(ncid, &nc, NULL, &nc4_info)))
617  return retval;
618  assert(nc4_info);
619 
620  /* If we're in define mode, but not redefing the file, delete it. */
621  if (nc4_info->flags & NC_INDEF && !nc4_info->redef)
622  {
623  delete_file++;
624  strncpy(path, nc->path, NC_MAX_NAME);
625  }
626 
627  /* Free any resources the netcdf-4 library has for this file's
628  * metadata. */
629  if ((retval = nc4_close_hdf5_file(nc4_info, 1, NULL)))
630  return retval;
631 
632  /* Delete the file, if we should. */
633  if (delete_file)
634  if (remove(path) < 0)
635  return NC_ECANTREMOVE;
636 
637  return NC_NOERR;
638 }
639 
649 int
650 NC4_close(int ncid, void* params)
651 {
652  NC_GRP_INFO_T *grp;
653  NC_FILE_INFO_T *h5;
654  int retval;
655  int inmemory;
656  NC_memio* memio = NULL;
657 
658  LOG((1, "%s: ncid 0x%x", __func__, ncid));
659 
660  /* Find our metadata for this file. */
661  if ((retval = nc4_find_grp_h5(ncid, &grp, &h5)))
662  return retval;
663 
664  assert(h5 && grp);
665 
666  /* This must be the root group. */
667  if (grp->parent)
668  return NC_EBADGRPID;
669 
670  inmemory = ((h5->cmode & NC_INMEMORY) == NC_INMEMORY);
671 
672  if(inmemory && params != NULL) {
673  memio = (NC_memio*)params;
674  }
675 
676  /* Call the nc4 close. */
677  if ((retval = nc4_close_hdf5_file(grp->nc4_info, 0, memio)))
678  return retval;
679 
680  return NC_NOERR;
681 }
682 
700 int
701 NC4_inq(int ncid, int *ndimsp, int *nvarsp, int *nattsp, int *unlimdimidp)
702 {
703  NC *nc;
704  NC_FILE_INFO_T *h5;
705  NC_GRP_INFO_T *grp;
706  int retval;
707  int i;
708 
709  LOG((2, "%s: ncid 0x%x", __func__, ncid));
710 
711  /* Find file metadata. */
712  if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5)))
713  return retval;
714 
715  assert(h5 && grp && nc);
716 
717  /* Count the number of dims, vars, and global atts; need to iterate
718  * because of possible nulls. */
719  if (ndimsp)
720  {
721  *ndimsp = ncindexcount(grp->dim);
722  }
723  if (nvarsp)
724  {
725  *nvarsp = ncindexcount(grp->vars);
726  }
727  if (nattsp)
728  {
729  /* Do we need to read the atts? */
730  if (!grp->atts_read)
731  if ((retval = nc4_read_atts(grp, NULL)))
732  return retval;
733 
734  *nattsp = ncindexcount(grp->att);
735  }
736 
737  if (unlimdimidp)
738  {
739  /* Default, no unlimited dimension */
740  *unlimdimidp = -1;
741 
742  /* If there's more than one unlimited dim, which was not possible
743  with netcdf-3, then only the last unlimited one will be reported
744  back in xtendimp. */
745  /* Note that this code is inconsistent with nc_inq_unlimid() */
746  for(i=0;i<ncindexsize(grp->dim);i++) {
747  NC_DIM_INFO_T* d = (NC_DIM_INFO_T*)ncindexith(grp->dim,i);
748  if(d == NULL) continue;
749  if(d->unlimited) {
750  *unlimdimidp = d->hdr.id;
751  break;
752  }
753  }
754  }
755 
756  return NC_NOERR;
757 }
758 
768 int
769 nc4_enddef_netcdf4_file(NC_FILE_INFO_T *h5)
770 {
771  assert(h5);
772  LOG((3, "%s", __func__));
773 
774  /* If we're not in define mode, return an error. */
775  if (!(h5->flags & NC_INDEF))
776  return NC_ENOTINDEFINE;
777 
778  /* Turn define mode off. */
779  h5->flags ^= NC_INDEF;
780 
781  /* Redef mode needs to be tracked separately for nc_abort. */
782  h5->redef = NC_FALSE;
783 
784  return sync_netcdf4_file(h5);
785 }
#define NC_CLASSIC_MODEL
Enforce classic model on netCDF-4.
Definition: netcdf.h:138
#define NC_INMEMORY
Read from memory.
Definition: netcdf.h:161
#define NC_EHDFERR
Error at HDF5 layer.
Definition: netcdf.h:436
#define NC_ENOTINDEFINE
Operation not allowed in data mode.
Definition: netcdf.h:339
#define NC_EINDEFINE
Operation not allowed in define mode.
Definition: netcdf.h:348
#define NC_EINVAL
Invalid Argument.
Definition: netcdf.h:333
#define NC_EBADGRPID
Bad group ID.
Definition: netcdf.h:451
#define NC_NOFILL
Argument to nc_set_fill() to turn off filling of data.
Definition: netcdf.h:115
#define NC_MAX_NAME
Maximum for classic library.
Definition: netcdf.h:273
#define NC_ECANTREMOVE
Can't remove file.
Definition: netcdf.h:428
#define NC_EPERM
Write to read only.
Definition: netcdf.h:334
#define NC_NOERR
No Error.
Definition: netcdf.h:323
#define NC_FILL
Argument to nc_set_fill() to clear NC_NOFILL.
Definition: netcdf.h:114

Return to the Main Unidata NetCDF page.
Generated on Sat Apr 6 2019 08:19:00 for NetCDF. NetCDF is a Unidata library.