Add read!

NEWS.md

@@ -1,12 +1,18 @@
 ClimaUtilities.jl Release Notes
 ===============================
 
-v0.1.10
+v0.1.11
 ------
 
-- Reduced alloctions in regridding. Now method `regridded_snapshot!`. PR
+- Reduced alloctions in regridding. New method `regridded_snapshot!`. PR
   [#72](https://github.com/CliMA/ClimaUtilities.jl/pull/72)
 
+v0.1.10
+------
+
+- Reduced alloctions in regridding. New method `read!`. PR
+  [#83](https://github.com/CliMA/ClimaUtilities.jl/pull/83)
+
 v0.1.9
 ------
 

Project.toml

@@ -1,7 +1,7 @@
 name = "ClimaUtilities"
 uuid = "b3f4f4ca-9299-4f7f-bd9b-81e1242a7513"
 authors = ["Gabriele Bozzola <gbozzola@caltech.edu>", "Julia Sloan <jsloan@caltech.edu>"]
-version = "0.1.10"
+version = "0.1.11"
 
 [deps]
 Artifacts = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"

docs/src/filereaders.md

@@ -78,6 +78,7 @@ close(v_var)
 ```@docs
 ClimaUtilities.FileReaders.NCFileReader
 ClimaUtilities.FileReaders.read
+ClimaUtilities.FileReaders.read!
 ClimaUtilities.FileReaders.available_dates
 ClimaUtilities.FileReaders.close_all_ncfiles
 Base.close

ext/DataHandlingExt.jl

@@ -8,7 +8,7 @@ import ClimaCore: ClimaComms
 
 import ClimaUtilities.DataStructures
 import ClimaUtilities.Regridders
-import ClimaUtilities.FileReaders: AbstractFileReader, NCFileReader, read
+import ClimaUtilities.FileReaders: AbstractFileReader, NCFileReader, read, read!
 import ClimaUtilities.Regridders: AbstractRegridder, regrid
 
 import ClimaUtilities.DataHandling
@@ -24,6 +24,7 @@ import ClimaUtilities.DataHandling
          DIMS,
          TIMES <: AbstractArray{<:AbstractFloat},
          CACHE <: DataStructures.LRUCache{<:Dates.DateTime, ClimaCore.Fields.Field},
+         PR
      }
 
 Currently, the `DataHandler` works with one variable at the time. This might not be the most
@@ -48,6 +49,7 @@ struct DataHandler{
     DIMS,
     TIMES <: AbstractArray{<:AbstractFloat},
     CACHE <: DataStructures.LRUCache{<:Dates.DateTime, ClimaCore.Fields.Field},
+    PR,
 }
     """Object responsible for getting the data from disk to memory"""
     file_reader::FR
@@ -76,6 +78,9 @@ struct DataHandler{
 
     """Private field where cached data is stored"""
     _cached_regridded_fields::CACHE
+
+    """Preallocated memory for storing read dataset"""
+    preallocated_read_data::PR
 end
 
 """
@@ -93,6 +98,8 @@ Create a `DataHandler` to read `varname` from `file_path` and remap it to `targe
 
 The DataHandler maintains an LRU cache of Fields that were previously computed.
 
+Creating this object results in the file being accessed (to preallocate some memory).
+
 Positional arguments
 =====================
 
@@ -181,6 +188,9 @@ function DataHandling.DataHandler(
     times_s = Second.(available_dates .- reference_date) ./ Second(1)
     available_times = times_s .- t_start
 
+    one_date = isempty(available_dates) ? () : (first(available_dates),)
+    preallocated_read_data = read(file_reader, one_date...)
+
     return DataHandler(
         file_reader,
         regridder,
@@ -191,6 +201,7 @@ function DataHandling.DataHandler(
         reference_date,
         available_times,
         _cached_regridded_fields,
+        preallocated_read_data,
     )
 end
 
@@ -318,8 +329,13 @@ function DataHandling.regridded_snapshot(
         if regridder_type == :TempestRegridder
             regrid_args = (date,)
         elseif regridder_type == :InterpolationsRegridder
+            read!(
+                data_handler.preallocated_read_data,
+                data_handler.file_reader,
+                date,
+            )
             regrid_args =
-                (read(data_handler.file_reader, date), data_handler.dimensions)
+                (data_handler.preallocated_read_data, data_handler.dimensions)
         else
             error("Uncaught case")
         end

ext/NCFileReaderExt.jl

@@ -226,4 +226,28 @@ function FileReaders.read(file_reader::NCFileReader)
     end
 end
 
+"""
+    read!(dest, file_reader::NCFileReader)
+
+Read and preprocess data (for static datasets), saving the output to `dest`.
+"""
+function FileReaders.read!(dest, file_reader::NCFileReader)
+    dest .= FileReaders.read(file_reader)
+    return nothing
+end
+
+"""
+    read!(dest, file_reader::NCFileReader, date::Dates.DateTime)
+
+Read and preprocess the data at the given `date`, saving the output to `dest`.
+"""
+function FileReaders.read!(
+    dest,
+    file_reader::NCFileReader,
+    date::Dates.DateTime,
+)
+    dest .= FileReaders.read(file_reader, date)
+    return nothing
+end
+
 end

src/FileReaders.jl

@@ -17,6 +17,8 @@ function NCFileReader end
 
 function read end
 
+function read! end
+
 function available_dates end
 
 function close_all_ncfiles end

test/file_readers.jl

@@ -41,6 +41,12 @@ using NCDatasets
         @test FileReaders.read(ncreader_u, DateTime(2021, 01, 01, 01)) ==
               nc["u10n"][:, :, 2]
 
+        # Test read!
+        dest = copy(nc["u10n"][:, :, 2])
+        fill!(dest, 0)
+        FileReaders.read!(dest, ncreader_u, DateTime(2021, 01, 01, 01))
+        @test dest == nc["u10n"][:, :, 2]
+
         # Test that we need to close all the variables to close the file
         open_ncfiles =
             Base.get_extension(

test/output_path_generator.jl

@@ -133,5 +133,9 @@ end
         context = context,
     )
 
-    Base.rm(base_output_path, force = true, recursive = true)
+    if ClimaComms.iamroot(context)
+        Base.rm(base_output_path, force = true, recursive = true)
+    end
+    ClimaComms.barrier(context)
+    let_filesystem_catch_up()
 end