Reading and writing HDF5

NCL now fully support all HDF5 features. The features added to NCL including: support group, and support new data types as: string, vlen, compound, enum, and opaque. Here are some examples of these new features:

Read and write with groups

If run command:

ncl_filedump ncl_wrt_uvt.h5

show something like:

......
      group 
      float T ( DIM_000, DIM_002, DIM_003, DIM_001 )
....

Then it means this file have a group GROUPNAME in it, where GROUPNAME is a string for the name of that group. User can use:

f = addfile("ncl_wrt_uvt.h5", "r")
g = f=>GROUPNAME

	to access the group, where "=>" is an operator, which to read this group. After this, variable "g" is a
	group, which functions vary like a file.



	Below is a sample to read HDF5 file with group in it:
 setfileoption("h5", "FileStructure", "Advanced")

 fn = "ncl_wrt_uvt.h5"

 fi = addfile(fn, "r")

;printVarSummary(fi)
;print(fi)

 time = fi->time
 lev = fi->lev
 lat = fi->lat
 lon = fi->lon

 t = fi->T
 u = fi->U
 v = fi->V

;===================================================================

 g1 = fi=>/grp1

;print(g1)
;printVarSummary(g1)

;time = g1->time
;lev = g1->lev
;lat = g1->lat
;lon = g1->lon

 t = g1->T
 u = g1->U
 v = g1->V

;===================================================================

 wks  = gsn_open_wks("x11", "xy")

 res              = True
 res@tiMainString = "Basic plot" 

 uf = fi->U

 uf!0 = "time"
 uf!1 = "lev"
 uf!2 = "lat"
 uf!3 = "lon"

 plot1 = gsn_csm_contour(wks, uf(0, 0, :, :), False)

 ug = g1->U

 ug!0 = "time"
 ug!1 = "lev"
 ug!2 = "lat"
 ug!3 = "lon"

 plot2 = gsn_csm_contour(wks, ug(0, 0, :, :), False)

 print("uf(0, 0, 0, 0) = " + uf(0, 0, 0, 0))
 print("ug(0, 0, 0, 0) = " + ug(0, 0, 0, 0))

;print(time)
;print(lev)
;print(lat)
 print(lon(::18))


	


	
	Write HDF5 file with group:
 setfileoption("nc", "FileStructure", "Advanced")

 fn = "/Users/huangwei/myncl/nc4_work/wrtgroup/nc4uvt.nc"

 fi = addfile(fn, "r")

;printVarSummary(fi)
;print(fi)

 time = fi->time
 lev = fi->lev
 lat = fi->lat
 lon = fi->lon

 t = fi->T
 u = fi->U
 v = fi->V

;printVarSummary(t)
;printVarSummary(u)
;printVarSummary(v)

;print("t(0,0,0,0) = " + t(0,0,0,0))
;print("u(0,1,1,1) = " + u(0,1,1,1))
;print("v(0,2,2,2) = " + v(0,2,2,2))

;print("t&lat(0) = " + t&lat(0))
;print("t@units = " + t@units)
;print("u@units = " + u@units)
;print("v@units = " + v@units)

;u1 = fi->U(::2)
;v1 = fi->V(3:121:3)

 ntim  = dimsizes(time)	; get dimension sizes
 nlev  = dimsizes(lev)
 nlat  = dimsizes(lat)
 nlon  = dimsizes(lon)

;------------------------------------------------------------
 setfileoption("h5", "FileStructure", "Advanced")

 fon = "ncl_wrt_uvt.h5"
 system("/bin/rm -f " + fon) ; remove if exists
 fo = addfile(fon, "c")

;===================================================================
; explicitly declare file definition mode. Improve efficiency.
;===================================================================
;setfileoption(fo,"CompressionLevel", 6)
;setfileoption(fo,"CacheSize", 3200000)
;setfileoption(fo,"CacheNelems", 1027)
;setfileoption(fo,"CachePreemption", 0.25)

; create global attributes of the file
;===================================================================
 fAtt               = True            ; assign file attributes
 fAtt@title         = "NCL generated netCDF file"
 fAtt@source_file   = fn
 fAtt@Conventions   = "None"
;fAtt@creation_date = systemfunc ("date")
 fileattdef(fo, fAtt)      ; copy file attributes

;===================================================================
; predefine the coordinate variables and their dimensionality
; Note: to get an UNLIMITED record dimension, we set the dimensionality
; to -1 (or the actual size) and set the dimension name to True.
;===================================================================
 dimNames = (/"time", "lev", "lat", "lon"/)
 dimSizes = (/ 1 ,   nlev,  nlat,  nlon /)
 dimUnlim = (/ True , False, False, False/)
 filedimdef(fo, dimNames, dimSizes, dimUnlim)

;===================================================================

 mtim = 1
 mlat = nlat/2
 mlon = nlon/2
 mlev = nlev/2
 chunkSizes = (/ mtim, mlev, mlat, mlon /)
;dimUnlim(0) = False
 filechunkdimdef(fo,dimNames,chunkSizes,dimUnlim)

;===================================================================

 grpnames = (/"grp1", "group2", "g3"/)

 filegrpdef(fo, grpnames)

; predefine the the dimensionality of the variables to be written out
;===================================================================
; Here we are using NCL functions to facilitate defining
; each variable's dimension name(s) and type.
;===================================================================
 filevardef(fo, "time", typeof(time), getvardims(time))
 filevarattdef(fo,"time", time)                   ; copy time attributes
 fo->time   = (/time/)

 filevardef(fo, "lev", typeof(lev), getvardims(lev) )
 filevarattdef(fo,"lev", lev)                     ; copy lev attributes
 fo->lev    = (/lev/)

 filevardef(fo, "lat", typeof(lat), getvardims(lat))
 filevarattdef(fo,"lat", lat)                     ; copy lat attributes
 fo->lat    = (/lat/)

 filevardef(fo, "lon", typeof(lon), getvardims(lon))
 filevarattdef(fo,"lon", lon)                     ; copy lon attributes
 fo->lon    = (/lon/)

 filevardef(fo, "T", typeof(t), getvardims(t))
 filevarattdef(fo,"T", t)                         ; copy T attributes
 filevarchunkdef(fo, "T", chunkSizes)
 filevarcompressleveldef(fo, "T", 2)
 fo->T      = (/t/)
;print(fo)
;printVarSummary(t)
;exit

 filevardef(fo, "U", typeof(u), getvardims(u))
 filevarattdef(fo,"U", u)                         ; copy U attributes
;filevarchunkdef(fo, "U", chunkSizes)
;filevarcompressleveldef(fo, "U", 4)
 fo->U      = (/u/)

 filevardef(fo, "V", typeof(v), getvardims(v))
 filevarattdef(fo,"V", v)                         ; copy V attributes
;filevarchunkdef(fo, "V", chunkSizes)
;filevarcompressleveldef(fo, "V", 6)
;filevarchunkcachedef(fo, "V", 3200000, 1027, 0.75)
 fo->V      = (/v/)

;printVarSummary(fo)
;print(fo)

;===================================================================

 g1 = fo=>/grp1

 fileattdef(g1, fAtt)
 filedimdef(g1, dimNames, dimSizes, dimUnlim)
 filechunkdimdef(g1,dimNames,chunkSizes,dimUnlim)

 filevardef(g1, "time", typeof(time), getvardims(time))
 filevarattdef(g1,"time", time)                   ; copy time attributes
 g1->time   = (/time/)

;print(g1)

 filevardef(g1, "lev", typeof(lev), getvardims(lev) )
 filevarattdef(g1,"lev", lev)                     ; copy lev attributes
 g1->lev    = (/lev/)

 filevardef(g1, "lat", typeof(lat), getvardims(lat))
 filevarattdef(g1,"lat", lat)                     ; copy lat attributes
 g1->lat    = (/lat/)

 filevardef(g1, "lon", typeof(lon), getvardims(lon))
 filevarattdef(g1,"lon", lon)                     ; copy lon attributes
 g1->lon    = (/lon/)

 filevardef(g1, "T", typeof(t), getvardims(t))
 filevarattdef(g1,"T", t)                         ; copy T attributes
 filevarchunkdef(g1, "T", chunkSizes)
 filevarcompressleveldef(g1, "T", 2)
 g1->T      = (/t/)

 filevardef(g1, "U", typeof(u), getvardims(u))
 filevarattdef(g1,"U", u)                         ; copy U attributes
;filevarchunkdef(g1, "U", chunkSizes)
;filevarcompressleveldef(g1, "U", 4)
 g1->U      = (/u/)

 filevardef(g1, "V", typeof(v), getvardims(v))
 filevarattdef(g1,"V", v)                         ; copy V attributes
;filevarchunkdef(g1, "V", chunkSizes)
;filevarcompressleveldef(g1, "V", 6)
;filevarchunkcachedef(g1, "V", 3200000, 1027, 0.75)
 g1->V      = (/v/)

;===================================================================
; output only the data values since the dimensionality and such have
; been predefined. The "(/", "/)" syntax tells NCL to only output the
; data values to the predefined locations on the file.
;====================================================================

;printVarSummary(g1)
;print(g1)

	

	Read and write string

	
	Write string:
 setfileoption("nc", "FileStructure", "Advanced")

 fn = "./tst_strings.nc"

 f = addfile(fn, "r")

;atts = getvaratts(f)
;print(atts)

;print(f)
;printVarSummary(f)

 strs = f->universal_declaration_of_human_rights

;print(strs)

 delete(f)

;===================================================================

 fn = "ncl_wrt_string.h5"
 system("/bin/rm -f " + fn)
 fo = addfile(fn, "c")

 fAtt               = True
 fAtt@title         = "NCL generated HDF5 file"
 fAtt@source_file   = fn
 fAtt@Conventions   = "None"
 fAtt@creation_date = systemfunc("date")

;print(fAtt)

 fileattdef(fo, fAtt)

;===================================================================
 nstrs    = dimsizes(strs)
 dimNames = (/"Sentence"/)
 dimSizes = (/ nstrs    /)
 dimUnlim = (/ False    /)
 filedimdef(fo, dimNames, dimSizes, dimUnlim)

 strsdims = getvardims(strs)

 strsdims = "Sentence"

 filevardef(fo, "h5_string", typeof(strs), strsdims)

 fo->h5_string = (/strs/)


	

	
	Read string:
 setfileoption("h5", "FileStructure", "Advanced")

 fn = "./ncl_wrt_string.h5"

 f = addfile(fn, "r")

 print(f)
 printVarSummary(f)

 strs = f->h5_string

 print(strs)

	

	Read and write compound

	
	If command:
ncl_filedump SDScompound.h5

	produces something like:
......
   dimensions:
      ......
   variables:
      compound       (a_name, c_name, b_name) (......)
......

	Then this file contains compound data. where COMPOUNDNAME is name of this compound data.
        The names within the parentheses are called compound data components.
        To accesee these components, NCL uses "." to access those components, as demostrated below.


	Read compound in HDF5:
 f = addfile("SDScompound.h5", "r")

;print(f)

 da = f->/ArrayOfStructures.a_name
 db = f->/ArrayOfStructures.b_name
 dc = f->/ArrayOfStructures.c_name

	


	
	Write compound in HDF5:
 setfileoption("h5", "FileStructure", "Advanced")

 f = addfile("SDScompound.h5", "r")

 print(f)

 ia = f->/ArrayOfStructures.a_name
 fb = f->/ArrayOfStructures.b_name
 dc = f->/ArrayOfStructures.c_name

;print(ia)
;print(fb)
;print(dc)

 delete(f)

;===================================================================

 fon = "ncl_wrtH5comp.h5"
 system("/bin/rm -f " + fon)
 fo = addfile(fon, "c")

 fAtt               = True            ; assign file attributes
 fAtt@title         = "NCL generated HDF5 file with compound"
 fAtt@source_file   = fon
 fAtt@Conventions   = "None"
 fAtt@creation_date = systemfunc("date")

;print(fAtt)

 fileattdef(fo, fAtt)

;===================================================================

 nvals   = dimsizes(ia)

 dimNames = (/"DimName"/)
 dimSizes = (/ nvals   /)
 dimUnlim = (/ False    /)
 filedimdef(fo, dimNames, dimSizes, dimUnlim)

 var_name = "h5_compound"
 mem_name = (/"ia", "fb", "dc"/)
 mem_type = (/"integer", "float", "double"/)
 mem_size = (/1, 1, 1/)
 filecompounddef(fo, "h5_compound_var", var_name, dimNames, \
                 mem_name, mem_type, mem_size)

 complist = new(dimSizes, list)
;printVarSummary(complist)

 do n = 0, nvals - 1
    ListPush(complist(n), (/ia(n)/))
    ListPush(complist(n), (/fb(n)/))
    ListPush(complist(n), (/dc(n)/))
 end do

;printVarSummary(complist)
;print(complist)

 fo->$var_name$ = complist

 delete(fo)


	

	
	
Read and write ENUM
	
	

	
	Write ENUM:
 setfileoption("h5", "FileStructure", "Advanced")

 fn = "./tst_enums.nc"

 fi = addfile(fn, "r")

 print(fi)
;printVarSummary(fi)

 v = fi->primary_cloud

 enum_name = v@enum_name
 enum_value = v@enum_value

 print(enum_value)

 print(v)

 print(enum_name)
 print(enum_value)

 nv = dimsizes(v)
 
 print(nv)

 tc = dimsizes(enum_value)

;do n = 0, nv - 1
;   m = ind(v(n) .eq. enum_value)
;   print("v{key, value} = {" + enum_value(m) + "," + enum_name(m) + "}")
;end do

;===================================================================

 fn = "ncl_wrt_enum.h5"
 system("/bin/rm -f " + fn)
 fo = addfile(fn, "c")

 fAtt               = True            ; assign file attributes
 fAtt@title         = "NCL generated netCDF file with enum"
 fAtt@source_file   = fn
 fAtt@Conventions   = "None"
 fAtt@creation_date = systemfunc("date")

;print(fAtt)

 fileattdef(fo, fAtt)

;===================================================================
 nvals   = dimsizes(v)
 dimNames = (/"types"/)
 dimSizes = (/ nvals   /)
 dimUnlim = (/ False    /)
 filedimdef(fo, dimNames, dimSizes, dimUnlim)

 var_name = "cloud"
 fileenumdef(fo, "enum_var", var_name, dimNames, enum_name, enum_value)

 fo->$var_name$ = (/v/)


	

	
	Read ENUM:
 setfileoption("h5", "FileStructure", "Advanced")

 fn = "ncl_wrt_enum.h5"

 f = addfile(fn, "r")

 print(f)
;printVarSummary(f)

 v = f->cloud
 print(v)

 exit

 enum_name = v@enum_name
 enum_value = v@enum_value

 print(enum_name)
 print(enum_value)

	

	Read and write opaque

	
	read opaque data:
 setfileoption("h5", "FileStructure", "Advanced")

 fn = "./ncl_wrt_opaque.h5"

 f = addfile(fn, "r")

 print(f)
;printVarSummary(f)

 v = f->opaque_variable

 print(v)


	

	
	write opaque data:
 fn = "./tst_opaques.nc"

 fi = addfile(fn, "r")

 print(fi)
;printVarSummary(fi)

 v = fi->var_defined_by_netcdf_user

 nv = dimsizes(v)
 
 print(v)
 print(nv)

 var_size = 1
 do n = 1, dimsizes(nv) - 1
    var_size = nv(n) * var_size
 end do

 print(var_size)

;===================================================================
 setfileoption("h5", "FileStructure", "Advanced")

 fn = "ncl_wrt_opaque.h5"
 system("/bin/rm -f " + fn)
 fo = addfile(fn, "c")

 fAtt               = True            ; assign file attributes
 fAtt@title         = "NCL generated netCDF file with enum"
 fAtt@source_file   = fn
 fAtt@Conventions   = "None"
 fAtt@creation_date = systemfunc("date")

;print(fAtt)

 fileattdef(fo, fAtt)

;===================================================================
 nvals   = nv(0)
 dimNames = (/"opaque_base_size"/)
 dimSizes = (/ nvals   /)
 dimUnlim = (/ False    /)
 filedimdef(fo, dimNames, dimSizes, dimUnlim)

 var_name = "opaque_variable"
 fileopaquedef(fo, "opaque_type", var_name, var_size, dimNames)

 fo->$var_name$ = (/v/)

	

	Read and write opaque

	
	Write VLEN:
 setfileoption("h5", "FileStructure", "Advanced")

 fn = "h5ex_t_vlen.h5"

 f = addfile(fn, "r")

;print(f)
;printVarSummary(f)

 v = f->DS1

;print(v)

 a = v[0]
;print(a)

 b = v[1]
;print(b)

;===================================================================
 setfileoption("h5", "FileStructure", "Advanced")

 fn = "ncl_wrt_vlen.h5"
 system("/bin/rm -f " + fn)
 fo = addfile(fn, "c")

 fAtt               = True
 fAtt@title         = "NCL generated HDF5 file"
 fAtt@source_file   = fn
 fAtt@Conventions   = "None"
 fAtt@creation_date = systemfunc("date")

;print(fAtt)

 fileattdef(fo, fAtt)

;===================================================================
 nelm = ListCount(v)
 print(nelm)

 dimNames = (/"dimension"/)
 dimSizes = (/ nelm  /)
 dimUnlim = (/ False /)
 filedimdef(fo, dimNames, dimSizes, dimUnlim)

 vlen_name = "vlen_name"
 var_name = "vlen_var"
 filevlendef(fo, vlen_name, var_name, "integer", dimNames)

 fo->$var_name$ = v

	

        
        Read VLEN:
 setfileoption("h5", "FileStructure", "Advanced")

 fn = "h5ex_t_vlen.h5"

 f = addfile(fn, "r")

;print(f)
;printVarSummary(f)

 v = f->DS1

 print(v)

 a = v[0]
 print(a)

 b = v[1]
 print(b)