auto chunk size, tolerate ~ filename

Author: scivision

auto_chunk_size.m

@@ -0,0 +1,52 @@
+function csize = auto_chunk_size(dims)
+% automatically determine HDF5 / NetCDF4 chunk size.
+% based on https://github.com/h5py/h5py/blob/master/h5py/_hl/filters.py
+% refer to https://support.hdfgroup.org/HDF5/Tutor/layout.html
+narginchk(1,1)
+validateattributes(dims, {'numeric'}, {'vector', 'integer', 'positive'})
+
+CHUNK_BASE = 16000;  % Multiplier by which chunks are adjusted
+CHUNK_MIN = 8000;    % lower limit: 8 kbyte
+CHUNK_MAX = 1000000; % upper limit: 1 Mbyte
+TYPESIZE = 8;        % bytes, assume real64 for simplicity
+
+csize = dims;
+Ndim = length(dims);
+if Ndim == 1 || prod(dims) * TYPESIZE < CHUNK_MIN
+  return
+end
+
+dset_size = prod(csize) * TYPESIZE;
+target_size = CHUNK_BASE * (2.*log10(dset_size / 1e6));
+if (target_size > CHUNK_MAX)
+  target_size = CHUNK_MAX;
+end
+
+% print *,'target_size [bytes]: ',target_size
+
+i = 0;
+while true
+  % Repeatedly loop over the axes, dividing them by 2.
+ % Stop when:
+  %  1a. We're smaller than the target chunk size, OR
+  %  1b. We're within 50% of the target chunk size, AND
+  %   2. The chunk is smaller than the maximum chunk size
+
+  chunk_bytes = prod(csize) * TYPESIZE;
+
+  if chunk_bytes < target_size || ...
+     2*(abs(chunk_bytes-target_size) / target_size) < 1 && ...
+     chunk_bytes < CHUNK_MAX
+     break
+  end
+
+  if prod(csize) == 1
+    break
+  end
+  % Element size larger than CHUNK_MAX
+  j = mod(i, Ndim) + 1;
+  csize(j) = ceil(csize(j) / 2);
+  i = i+1;
+end
+
+end % function

h5save.m

@@ -2,6 +2,9 @@ function h5save(filename, varname, A, sizeA, dtype)
 
 narginchk(3, 5)
 
+% Matlab can't cope with tilde in many cases, especially on Windows
+filename = expanduser(filename);
+
 if nargin < 4 || isempty(sizeA)
   if isvector(A)
     sizeA = length(A);
@@ -18,8 +21,6 @@ function h5save(filename, varname, A, sizeA, dtype)
   sizeA = size(A);
 end
 
-
-
 if h5exists(filename, varname)
   exist_file(filename, varname, A, sizeA)
 else
@@ -54,26 +55,20 @@ function exist_file(filename, varname, A, sizeA)
 
 function new_file(filename, varname, A, sizeA)
 
-if ~ismatrix(A)
+if isvector(A)
+  h5create(filename, varname, sizeA, 'DataType', class(A))
+else
   % enable Gzip compression--remember Matlab's dim order is flipped from
   % C / Python
-  switch length(sizeA)
-    case 4, chunksize = [sizeA(1), sizeA(2), 1, sizeA(4)];
-    case 3, chunksize = [sizeA(1), sizeA(2), 1];
-    otherwise, error('h5save:fixme', '%s is bigger than 4 dims', varname)
-  end
   h5create(filename, varname, sizeA, 'DataType', class(A), ...
-    'Deflate', 1, 'Fletcher32', true, 'Shuffle', true, 'ChunkSize', chunksize)
-else
-  h5create(filename, varname, sizeA, 'DataType', class(A))
+    'Deflate', 1, 'Fletcher32', true, 'Shuffle', true, 'ChunkSize', auto_chunk_size(sizeA))
 end % if
 
 h5write(filename, varname, A)
 
 end % function
 
 
-
 % Copyright 2020 Michael Hirsch, Ph.D.
 
 % Licensed under the Apache License, Version 2.0 (the "License");

h5variables.m

@@ -2,6 +2,8 @@
 % get dataset names and groups in an HDF5 file
 narginchk(1,1)
 
+filename = expanduser(filename);
+
 % use temporary variable to be R2017b OK
 finf = h5info(filename);
 ds = finf.Datasets;

is_file.m

@@ -12,11 +12,10 @@
 try
   ret = isfile(path);
 catch excp
-  if any(strcmp(excp.identifier, {'MATLAB:UndefinedFunction', 'Octave:undefined-function'}))
-    ret = exist(path, 'file') == 2;
-  else
+  if ~any(strcmp(excp.identifier, {'MATLAB:UndefinedFunction', 'Octave:undefined-function'}))
     rethrow(excp)
   end
+  ret = exist(path, 'file') == 2;
 end
 
 end % function

ncsave.m

@@ -2,6 +2,8 @@ function ncsave(filename, varname, A, ncdims, dtype)
 
 narginchk(3, 5)
 
+filename = expanduser(filename);
+
 if nargin >= 4 && ~isempty(ncdims)
   for i = 2:2:length(ncdims)
     sizeA(i/2) = ncdims{i};
@@ -37,24 +39,13 @@ function ncsave(filename, varname, A, ncdims, dtype)
 
 if any(strcmp(vars, varname))
   exist_file(filename, varname, A, sizeA)
-% catch excp
-%   if any(strcmp(excp.identifier, {'MATLAB:imagesci:netcdf:unableToOpenFileforRead', 'MATLAB:imagesci:netcdf:unknownLocation'}))
-%     % pass Matlab
-%   elseif any(strcmp(excp.message, {'No such file or directory', 'NetCDF: Variable not found'}))
-%     % pass Octave
-%   else
-%     disp(['failed create ', varname])
-%     rethrow(excp)
-%   end
 else
   new_file(filename, varname, A, sizeA, ncdims)
 end
 
 end % function
 
 
-
-
 function exist_file(filename, varname, A, sizeA)
 narginchk(4,4)
 
@@ -76,20 +67,15 @@ function new_file(filename, varname, A, sizeA, ncdims)
 
 if isscalar(A)
   nccreate(filename, varname, 'Datatype', class(A), 'Format', 'netcdf4')
-elseif isvector(A) || ismatrix(A)
+elseif isvector(A)
   nccreate(filename, varname, 'Dimensions', ncdims, 'Datatype', class(A), 'Format', 'netcdf4')
 else
   % enable Gzip compression--remember Matlab's dim order is flipped from
   % C / Python
-  switch length(sizeA)
-    case 4, chunksize = [sizeA(1), sizeA(2), 1, sizeA(4)];
-    case 3, chunksize = [sizeA(1), sizeA(2), 1];
-    otherwise, error('ncsave:fixme', '%s is bigger than 4 dims', varname)
-  end
   % "Datatype" to be Octave case-sensitive keyword compatible
   nccreate(filename, varname, 'Dimensions', ncdims, ...
     'Datatype', class(A), ...
-    'DeflateLevel', 1, 'Shuffle', true, 'ChunkSize', chunksize, 'Format', 'netcdf4')
+    'DeflateLevel', 1, 'Shuffle', true, 'ChunkSize', auto_chunk_size(sizeA), 'Format', 'netcdf4')
 end
 
 ncwrite(filename, varname, A)

ncsize.m

@@ -9,6 +9,8 @@
 narginchk(2,2)
 validateattributes(varname, {'char'}, {'vector'}, 2)
 
+filename = expanduser(filename);
+
 vinf = ncinfo(filename, varname);
 fsize = vinf.Size;
 
@@ -17,4 +19,4 @@
   fsize = 1;
 end
 
-end
+end

ncvariables.m

@@ -3,8 +3,8 @@
 narginchk(1,1)
 
 % use temporary variable to be R2017b OK
-finf = ncinfo(filename);
+finf = ncinfo(expanduser(filename));
 ds = finf.Variables(:);
 names = {ds(:).Name};
 
-end % function
+end % function

test_hdf5.m

@@ -9,7 +9,10 @@
 
 basic = fullfile(tempdir, 'basic.h5');
 if is_file(basic), delete(basic), end
-
+%% test_auto_chunk_size
+assert(isequal(auto_chunk_size([1500,2500,1000,500,100]), [12,20,8,8,2]), '5D chunk fail')
+assert(isequal(auto_chunk_size([15,250,100]), [2,32,25]), '3D chunk fail')
+assert(isequal(auto_chunk_size([15,250]), [15,250]), '2D small chunk fail')
 %% test_write_basic
 h5save(basic, '/A0', A0)
 h5save(basic, '/A1', A1)