NetCDF  4.6.3
 All Data Structures Files Functions Variables Typedefs Macros Modules Pages
filter_example.c
Go to the documentation of this file.
1 /*
2  Copyright 2018, UCAR/Unidata
3  See COPYRIGHT file for copying and redistribution conditions.
4 */
5 /*
6 This file is the same as nc_test4/test_filter.c
7 */
8 
44 #include "config.h"
45 #include <stdio.h>
46 #include <string.h>
47 #include <stdlib.h>
48 
49 #include <hdf5.h>
50 #include "netcdf.h"
51 
52 /* The HDF assigned id for bzip compression */
53 #define BZIP2_ID 307
54 /* The compression level used in this example */
55 #define BZIP2_LEVEL 9
56 
57 #define TESTFILE "bzip2.nc"
58 
59 /* Point at which we give up */
60 #define MAXERRS 8
61 
62 #define NDIMS 4
63 #define DIMSIZE 4
64 #define CHUNKSIZE 4 /* Note: not the total size of the chunk, but size wrt a dim*/
65 
66 static size_t dimsize = DIMSIZE;
67 static size_t chunksize = CHUNKSIZE;
68 static size_t actualdims = NDIMS;
69 
70 static size_t actualproduct = 1; /* x-product over dim sizes */
71 static size_t chunkproduct = 1; /* x-product over chunksizes */
72 
73 static size_t dims[NDIMS];
74 static size_t chunks[NDIMS];
75 
76 static int nerrs = 0;
77 
78 static int ncid, varid;
79 static int dimids[NDIMS];
80 static float* array = NULL;
81 static float* expected = NULL;
82 static unsigned int filterid = 0;
83 static unsigned int* params = NULL;
84 
85 /* Forward */
86 static void init(int argc, char** argv);
87 static int test_bzip2(void);
88 static int verifychunks(void);
89 
90 #define ERRR do { \
91 fflush(stdout); /* Make sure our stdout is synced with stderr. */ \
92 fprintf(stderr, "Sorry! Unexpected result, %s, line: %d\n", \
93  __FILE__, __LINE__); \
94 nerrs++;\
95 } while (0)
96 
97 static int
98 check(int err,int line)
99 {
100  if(err != NC_NOERR) {
101  fprintf(stderr,"fail (%d): %s\n",line,nc_strerror(err));
102  fflush(stderr);
103  exit(1);
104  }
105  return NC_NOERR;
106 }
107 
108 #define CHECK(x) check(x,__LINE__)
109 
110 /*
111 Read the chunking information about the variable
112 and verify that it is as expected.
113 */
114 
115 static int
116 verifychunks(void)
117 {
118  int i;
119  int store = -1;
120  size_t chunksizes[NDIMS];
121  memset(chunksizes,0,sizeof(chunksizes));
122  CHECK(nc_inq_var_chunking(ncid, varid, &store, chunksizes));
123  /* Storate must be chunked, not contiguous */
124  if(store != NC_CHUNKED) {
125  fprintf(stderr,"bad chunk store\n");
126  return NC_ESTORAGE;
127  }
128  /* Chunk sizes must match our predefined set */
129  for(i=0;i<actualdims;i++) {
130  if(chunksizes[i] != chunks[i]) {
131  fprintf(stderr,"bad chunk size: %d\n",i);
132  return NC_EBADCHUNK;
133  }
134  }
135  return 1;
136 }
137 
138 /*
139 Compare the data we wrote against the data we read.
140 */
141 
142 static int
143 compare(void)
144 {
145  int errs = 0;
146  int i;
147  printf("data comparison: |array|=%ld\n",(unsigned long)actualproduct);
148  for(i=0;i<actualproduct;i++) {
149  if(expected[i] != array[i]) {
150  printf("mismatch: array[%d]=%f expected[%d]=%f\n",
151  i,array[i],i,expected[i]);
152  errs++;
153  if(errs >= MAXERRS)
154  break;
155  }
156  }
157  if(errs == 0)
158  printf("no data errors\n");
159  if(actualproduct <= 1)
160  return NC_EBADDIM;
161  return (errs == 0 ? NC_NOERR: NC_EINVAL);
162 }
163 
164 /*
165 Create the file, write it, then re-read for comparison.
166 */
167 static int
168 test_bzip2(void)
169 {
170  int i;
171  unsigned int level = BZIP2_LEVEL;
172  unsigned int id=0;
173  size_t nparams = 0;
174 
175  printf("\n*** Testing API: bzip2 compression.\n");
176 
177  /* Clear the data array */
178  memset(array,0,sizeof(float)*actualproduct);
179 
180  /* Create a file */
181  CHECK(nc_create(TESTFILE, NC_NETCDF4|NC_CLOBBER, &ncid));
182 
183  /* Do not use fill for this file */
184  CHECK(nc_set_fill(ncid, NC_NOFILL, NULL));
185 
186  /* Define the dimensions */
187  for(i=0;i<actualdims;i++) {
188  char dimname[1024];
189  snprintf(dimname,sizeof(dimname),"dim%d",i);
190  CHECK(nc_def_dim(ncid, dimname, dims[i], &dimids[i]));
191  }
192 
193  /* Define the variable */
194  CHECK(nc_def_var(ncid, "var", NC_FLOAT, actualdims, dimids, &varid));
195 
196  /* Set chunking on the variable */
197  CHECK(nc_def_var_chunking(ncid,varid,NC_CHUNKED,chunks));
198 
199  /* Verify that chunking succeeded */
200  if(!verifychunks())
201  return NC_EINVAL;
202  /* Set bzip2 compression for the variable: takes one parameter == level */
203  CHECK(nc_def_var_filter(ncid,varid,BZIP2_ID,1,&level));
204 
205  /* Read back the compression info and verify it */
206  level = 0;
207  CHECK(nc_inq_var_filter(ncid,varid,&id,&nparams,&level));
208  if(id != BZIP2_ID || nparams != 1 || level != BZIP2_LEVEL) {
209  printf("test_filter: filter def/inq mismatch\n");
210  return NC_EFILTER;
211  }
212  /* Show the level */
213  printf("show parameters for bzip2: level=%u\n",level);
214  /* Show chunking */
215  printf("show chunks:");
216  for(i=0;i<actualdims;i++)
217  printf("%s%ld",(i==0?" chunks=":","),(unsigned long)chunks[i]);
218  printf("\n");
219 
220  /* prepare to write */
221  CHECK(nc_enddef(ncid));
222 
223  /* Fill in the array */
224  for(i=0;i<actualproduct;i++)
225  expected[i] = (float)i;
226 
227  /* write array */
228  CHECK(nc_put_var(ncid,varid,expected));
229 
230  /* Close file */
231  CHECK(nc_close(ncid));
232 
233  /* Now re-open and verify */
234  printf("\n*** Testing API: bzip2 decompression.\n");
235 
236  /* Clear the data array */
237  memset(array,0,sizeof(float)*actualproduct);
238 
239  /* Open the file */
240  CHECK(nc_open(TESTFILE, NC_NOWRITE, &ncid));
241 
242  /* Get the variable id */
243  CHECK(nc_inq_varid(ncid, "var", &varid));
244 
245  /* Check the compression algorithm */
246  filterid = 0;
247  nparams = 0;
248  params = NULL;
249  CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,NULL));
250  if(nparams > 0) {
251  params = (unsigned int*)malloc(sizeof(unsigned int)*nparams);
252  if(params == NULL)
253  return NC_ENOMEM;
254  CHECK(nc_inq_var_filter(ncid,varid,&filterid,&nparams,params));
255  }
256  if(filterid != BZIP2_ID) {
257  printf("Bzip2 id mismatch: %d\n",filterid);
258  return NC_EFILTER;
259  }
260  if(nparams != 1 && params != NULL && params[0] != BZIP2_LEVEL) {
261  printf("Compression parameter mismatch\n");
262  return NC_EFILTER;
263  }
264 
265  /* Verify chunking */
266  if(!verifychunks())
267  return 0;
268 
269  /* Read the data */
270  CHECK(nc_get_var_float(ncid, varid, array));
271 
272  /* Close the file */
273  CHECK(nc_close(ncid));
274  return (compare() == NC_NOERR ? 0 : 1);
275 }
276 
277 /**************************************************/
278 /* Utilities */
279 
280 static void
281 init(int argc, char** argv)
282 {
283  int i;
284  /* Setup various variables */
285  actualproduct = 1;
286  chunkproduct = 1;
287  for(i=0;i<NDIMS;i++) {
288  dims[i] = dimsize;
289  chunks[i] = chunksize;
290  if(i < actualdims) {
291  actualproduct *= dims[i];
292  chunkproduct *= chunks[i];
293  }
294  }
295  /* Allocate max size */
296  array = (float*)calloc(1,sizeof(float)*actualproduct);
297  expected = (float*)calloc(1,sizeof(float)*actualproduct);
298 }
299 
300 /**************************************************/
301 int
302 main(int argc, char **argv)
303 {
304  H5Eprint(stderr);
305  init(argc,argv);
306  if(test_bzip2() != NC_NOERR) ERRR;
307  exit(nerrs > 0?1:0);
308 }
309 
#define NC_ENOMEM
Memory allocation (malloc) failure.
Definition: netcdf.h:403
#define NC_CHUNKED
In HDF5 files you can set storage for each variable to be either contiguous or chunked, with nc_def_var_chunking().
Definition: netcdf.h:292
EXTERNL int nc_def_var(int ncid, const char *name, nc_type xtype, int ndims, const int *dimidsp, int *varidp)
Define a new variable.
Definition: dvar.c:207
EXTERNL int nc_def_dim(int ncid, const char *name, size_t len, int *idp)
Define a new dimension.
Definition: ddim.c:121
Main header file for the C API.
EXTERNL int nc_put_var(int ncid, int varid, const void *op)
Write an entire variable with one call.
Definition: dvarput.c:931
EXTERNL int nc_inq_var_chunking(int ncid, int varid, int *storagep, size_t *chunksizesp)
This is a wrapper for nc_inq_var_all().
Definition: dvarinq.c:422
#define NC_NOWRITE
Set read-only access for nc_open().
Definition: netcdf.h:123
#define NC_EBADDIM
Invalid dimension id or name.
Definition: netcdf.h:366
EXTERNL int nc_close(int ncid)
Close an open netCDF dataset.
Definition: dfile.c:1470
#define NC_EFILTER
Filter operation failed.
Definition: netcdf.h:468
#define NC_EINVAL
Invalid Argument.
Definition: netcdf.h:333
EXTERNL int nc_def_var_chunking(int ncid, int varid, int storage, const size_t *chunksizesp)
Define chunking parameters for a variable.
Definition: dvar.c:545
EXTERNL int nc_def_var_filter(int ncid, int varid, unsigned int id, size_t nparams, const unsigned int *parms)
Define a new variable filter.
Definition: dvar.c:646
#define NC_NOFILL
Argument to nc_set_fill() to turn off filling of data.
Definition: netcdf.h:115
EXTERNL int nc_set_fill(int ncid, int fillmode, int *old_modep)
Change the fill-value mode to improve write performance.
Definition: dfile.c:1656
EXTERNL int nc_inq_var_filter(int ncid, int varid, unsigned int *idp, size_t *nparams, unsigned int *params)
Find the filter (if any) associated with a variable.
Definition: dvarinq.c:638
#define NC_NETCDF4
Use netCDF-4/HDF5 format.
Definition: netcdf.h:151
EXTERNL int nc_open(const char *path, int mode, int *ncidp)
Open an existing netCDF file.
Definition: dfile.c:828
EXTERNL int nc_inq_varid(int ncid, const char *name, int *varidp)
Find the ID of a variable, from the name.
Definition: dvarinq.c:62
int nc_get_var_float(int ncid, int varid, float *ip)
Read an entire variable in one call.
Definition: dvarget.c:1087
#define NC_ESTORAGE
Can't specify both contiguous and chunking.
Definition: netcdf.h:461
#define NC_CLOBBER
Destroy existing file.
Definition: netcdf.h:126
#define NC_NOERR
No Error.
Definition: netcdf.h:323
EXTERNL int nc_enddef(int ncid)
Leave define mode.
Definition: dfile.c:1191
EXTERNL const char * nc_strerror(int ncerr)
Given an error number, return an error message.
Definition: derror.c:86
#define NC_FLOAT
single precision floating point number
Definition: netcdf.h:40
EXTERNL int nc_create(const char *path, int cmode, int *ncidp)
Create a new netCDF file.
Definition: dfile.c:562
#define NC_EBADCHUNK
Bad chunksize.
Definition: netcdf.h:462

Return to the Main Unidata NetCDF page.
Generated on Sat Apr 6 2019 08:19:00 for NetCDF. NetCDF is a Unidata library.