Skip to content
Merged
Show file tree
Hide file tree
Changes from 40 commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
bcf43f0
Wrap GMT's standard data type GMT_IMAGE for images
seisman Mar 18, 2024
a052a1a
Initial implementation of to_dataarray method for _GMT_IMAGE class
weiji14 Mar 20, 2024
56a6d65
Merge branch 'main' into datatypes/gmtimage
seisman Apr 17, 2024
f71e79c
Merge branch 'main' into datatypes/gmtimage
weiji14 Jun 18, 2024
4cce4a2
Small typo fixes and add output type-hint for to_dataarray
weiji14 Jun 18, 2024
e02b650
Fix mypy error using np.array([0, 1, 2]) instead of np.arange
weiji14 Jun 18, 2024
f3d4b1f
Parse name and data_attrs from grid/image header
weiji14 Jun 18, 2024
4390136
Transpose array to (band, y, x) order and add doctest for to_dataarray
weiji14 Jun 20, 2024
5f25669
Set registration and gtype from header
weiji14 Jun 20, 2024
a3c6c14
Print basic shape and padding info in _GMT_IMAGE doctest
weiji14 Jun 20, 2024
5888e10
Only set Conventions = CF-1.7 attribute for NetCDF grid type
weiji14 Jun 20, 2024
798e658
Merge branch 'main' into datatypes/gmtimage
weiji14 Jun 20, 2024
3dbf2f2
Remove rioxarray import
weiji14 Jun 20, 2024
6b860bf
Merge branch 'main' into datatypes/gmtimage
seisman Jul 27, 2024
0bf9368
Merge branch 'main' into datatypes/gmtimage
seisman Sep 19, 2024
7d437be
Use enum for grid ids
seisman Sep 19, 2024
268e34e
Fix the band. Starting from 1
seisman Sep 19, 2024
86765e1
Refactor the tests for images
seisman Sep 19, 2024
86f3ffa
In np.reshape, a is a position-only parameter
seisman Sep 20, 2024
cc28247
Improve tests
seisman Sep 20, 2024
1e2c973
Fix one failing doctest due to xarray changes
seisman Sep 20, 2024
734dc28
The np.reshape's newshape parameter is deprecated
seisman Sep 20, 2024
919dc00
Define grid IDs using IntEnum instead of Enum
seisman Sep 20, 2024
b1eacf1
Pass the new shape as a positional parameter
seisman Sep 20, 2024
aa4fdc9
Fix failing tests
seisman Sep 20, 2024
c87a3ec
One more fix
seisman Sep 20, 2024
a20d8a2
One more fix
seisman Sep 20, 2024
926427b
Simplify a doctest
seisman Sep 20, 2024
c73328e
Improve the tests
seisman Sep 20, 2024
fb97daa
Convert ctypes array to numpy array using np.ctypeslib.as_array
seisman Sep 20, 2024
15b8d53
Fix the incorrect value due to floating number conversion in sphinter…
seisman Sep 20, 2024
8433e78
Merge branch 'ctypesarray' into datatypes/gmtimage
seisman Sep 20, 2024
3e3a6f3
Update the to_dataarray method to match the codes in GMT_GRID
seisman Sep 20, 2024
12ef40a
image data should has uint8 dtype
seisman Sep 20, 2024
f64fbb8
Further improve the tests
seisman Sep 21, 2024
4f2ae48
Merge branch 'main' into datatypes/gmtimage
seisman Sep 24, 2024
d49afed
Add a note that currently only 3-band images are supported
seisman Sep 24, 2024
f70bec0
Merge branch 'main' into datatypes/gmtimage
seisman Sep 28, 2024
2fd13fb
Remove the old GMTGridID enums from pygmt/datatypes/header.py
seisman Sep 28, 2024
9972ba1
A minor fix
seisman Sep 28, 2024
62f0ce0
Set CF-1.7-specific attributes for netCDF formats only
seisman Sep 29, 2024
f715aee
Improve the logic for determine grid/image gtype based on upstream GM…
seisman Sep 29, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 124 additions & 1 deletion pygmt/datatypes/image.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@
import ctypes as ctp
from typing import ClassVar

from pygmt.datatypes.grid import _GMT_GRID_HEADER
import numpy as np
import xarray as xr
from pygmt.datatypes.header import _GMT_GRID_HEADER


class _GMT_IMAGE(ctp.Structure): # noqa: N801
Expand Down Expand Up @@ -63,6 +65,8 @@ class _GMT_IMAGE(ctp.Structure): # noqa: N801
array([-179.5, -178.5, ..., 178.5, 179.5])
>>> y # doctest: +NORMALIZE_WHITESPACE, +ELLIPSIS
array([ 89.5, 88.5, ..., -88.5, -89.5])
>>> data.dtype
dtype('uint8')
>>> data.shape
(180, 360, 3)
>>> data.min(), data.max()
Expand Down Expand Up @@ -91,3 +95,122 @@ class _GMT_IMAGE(ctp.Structure): # noqa: N801
# Book-keeping variables "hidden" from the API
("hidden", ctp.c_void_p),
]

def to_dataarray(self) -> xr.DataArray:
"""
Convert a _GMT_IMAGE object to an :class:`xarray.DataArray` object.

Returns
-------
dataarray
A :class:`xarray.DataArray` object.

Examples
--------
>>> from pygmt.clib import Session
>>> with Session() as lib:
... with lib.virtualfile_out(kind="image") as voutimg:
... lib.call_module("read", ["@earth_day_01d_p", voutimg, "-Ti"])
... # Read the image from the virtual file
... image = lib.read_virtualfile(voutimg, kind="image")
... # Convert to xarray.DataArray and use it later
... da = image.contents.to_dataarray()
>>> da # doctest: +NORMALIZE_WHITESPACE, +ELLIPSIS
<xarray.DataArray 'z' (band: 3, y: 180, x: 360)>...
array([[[ 10, 10, 10, ..., 10, 10, 10],
[ 10, 10, 10, ..., 10, 10, 10],
[ 10, 10, 10, ..., 10, 10, 10],
...,
[192, 193, 193, ..., 193, 192, 191],
[204, 206, 206, ..., 205, 206, 204],
[208, 210, 210, ..., 210, 210, 208]],
<BLANKLINE>
[[ 10, 10, 10, ..., 10, 10, 10],
[ 10, 10, 10, ..., 10, 10, 10],
[ 10, 10, 10, ..., 10, 10, 10],
...,
[186, 187, 188, ..., 187, 186, 185],
[196, 198, 198, ..., 197, 197, 196],
[199, 201, 201, ..., 201, 202, 199]],
<BLANKLINE>
[[ 51, 51, 51, ..., 51, 51, 51],
[ 51, 51, 51, ..., 51, 51, 51],
[ 51, 51, 51, ..., 51, 51, 51],
...,
[177, 179, 179, ..., 178, 177, 177],
[185, 187, 187, ..., 187, 186, 185],
[189, 191, 191, ..., 191, 191, 189]]], dtype=uint8)
Coordinates:
* y (y) float64... 89.5 88.5 87.5 86.5 ... -86.5 -87.5 -88.5 -89.5
* x (x) float64... -179.5 -178.5 -177.5 -176.5 ... 177.5 178.5 179.5
* band (band) uint8... 1 2 3
Attributes:
title:
history:
description:

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe don't add these attributes if they're empty?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These come from the header no? For GeoTIFF files, they don't make sense since GMT is not parsing the TIFF tags, but the title/history/description might show up for 3-band NetCDF files (if those can be passed into this GMT_IMAGE struct).

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right, title/history/description are recommended attributes of CF-1.7. So, for netCDF files, I guess we should always show them even if they're empty.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If those are only needed for CF-1.7, then maybe we can indent the 3 attrs["..."] lines here to be inside the if self.type in {GridFormat.NX} block?

if self.type in {
GridFormat.NB,
GridFormat.NS,
GridFormat.NI,
GridFormat.NF,
GridFormat.ND,
}: # Only set the 'Conventions' attribute for netCDF.
attrs["Conventions"] = "CF-1.7"
attrs["title"] = self.title.decode()
attrs["history"] = self.command.decode()
attrs["description"] = self.remark.decode()

(2) actual_range is the standard attribution defined by CF-1.7 convention, but it's only for netCDF files, not for images;

Same here, we can put attrs["actual_range"] under the if-block above:

attrs["actual_range"] = np.array([self.z_min, self.z_max])

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done at 62f0ce0.

long_name: z
actual_range: [ 1.79769313e+308 -1.79769313e+308]

>>> da.coords["x"] # doctest: +NORMALIZE_WHITESPACE, +ELLIPSIS
<xarray.DataArray 'x' (x: 360)>...
array([-179.5, -178.5, -177.5, ..., 177.5, 178.5, 179.5])
Coordinates:
* x (x) float64... -179.5 -178.5 -177.5 -176.5 ... 177.5 178.5 179.5
Attributes:
long_name: x
axis: X
actual_range: [-180. 180.]
>>> da.coords["y"] # doctest: +NORMALIZE_WHITESPACE, +ELLIPSIS
<xarray.DataArray 'y' (y: 180)>...
array([ 89.5, 88.5, 87.5, 86.5, ..., -87.5, -88.5, -89.5])
Coordinates:
* y (y) float64... 89.5 88.5 87.5 86.5 ... -86.5 -87.5 -88.5 -89.5
Attributes:
long_name: y
axis: Y
actual_range: [-90. 90.]
>>> da.gmt.registration, da.gmt.gtype
(1, 0)

@weiji14 weiji14 Sep 29, 2024 •

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The gtype for earth_day_01d should be Geographic (1), not Cartesian (0).

Suggested change
>>> da.gmt.registration, da.gmt.gtype
(1, 0)
>>> da.gmt.registration, da.gmt.gtype
(1, 1)

according to:

$ gmt grdinfo @earth_day_01d
~/.gmt/server/earth/earth_day/earth_day_01d_p.tif: Title: Grid imported via GDAL
~/.gmt/server/earth/earth_day/earth_day_01d_p.tif: Command: 
~/.gmt/server/earth/earth_day/earth_day_01d_p.tif: Remark: 
~/.gmt/server/earth/earth_day/earth_day_01d_p.tif: Pixel node registration used [Geographic grid]
~/.gmt/server/earth/earth_day/earth_day_01d_p.tif: Grid file format: gd = Import/export through GDAL
~/.gmt/server/earth/earth_day/earth_day_01d_p.tif: x_min: -180 x_max: 180 x_inc: 1 name: x n_columns: 360
~/.gmt/server/earth/earth_day/earth_day_01d_p.tif: y_min: -90 y_max: 90 y_inc: 1 name: y n_rows: 180
~/.gmt/server/earth/earth_day/earth_day_01d_p.tif: v_min: 1.79769313486e+308 v_max: -1.79769313486e+308 name: z
~/.gmt/server/earth/earth_day/earth_day_01d_p.tif: scale_factor: 1 add_offset: 0
~/.gmt/server/earth/earth_day/earth_day_01d_p.tif: Default CPT: 

We either need to modify the logic here:

gtype = 1 if dims[0] == "lat" and dims[1] == "lon" else 0

Or I can manually override the gtype in the load_blue_marble function which I'm trying to finish up at #2235.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://github.com/GenericMappingTools/gmt/blob/7809736ba32d87a4a96b15444419eb176c6a35f3/src/gmt_grdio.c#L1360

This is the function (again, not a public API function) that GMT uses to determine the grid type. The logic here is not that complicated to duplicate in Python.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How easy would it be to make that C function public actually?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How easy would it be to make that C function public actually?

It doesn't seem too difficult. But we still need to implement our own functions if we want to be compatible with GMT 6.4-6.5.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, it'll be some time before GMT 6.6 comes out. I just want the gtype determination to be consistent between GMT and PyGMT, and the best case is to bind to the GMT C function. But we can also reimplement it in PyGMT for now (in a separate PR), maybe in header.py?

@seisman seisman Sep 29, 2024 •

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Another possibility is to wrap the "hidden", private GMT_GRID_HEADER_HIDDEN structure https://github.com/GenericMappingTools/gmt/blob/7809736ba32d87a4a96b15444419eb176c6a35f3/src/gmt_hidden.h#L151. The structure has many members but we can only wrap the first few members. The members that are most useful to us are: grdtype (Cartesian or geographic), BC (boundary condition), varname (the actual variable name so that we don't have to hardcode the grid name to z), and cpt (the default CPT for this grid).

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In f715aee, I've updated the logic for determining the grid/image gtype based on ProjRefPROJ4.

The logic of codes come from: https://github.com/GenericMappingTools/gmt/blob/7809736ba32d87a4a96b15444419eb176c6a35f3/src/gmt_grdio.c#L3778

"""
# The image header
header = self.header.contents

if header.n_bands != 3:
msg = (
f"The raster image has {header.n_bands} band(s). "
"Currently only 3-band images are supported. "
"Please consider submitting a feature request to us. "
)
raise NotImplementedError(msg)

# Get dimensions and their attributes from the header.
dims, dim_attrs = header.dims, header.dim_attrs
# The coordinates, given as a tuple of the form (dims, data, attrs)
x = np.ctypeslib.as_array(self.x, shape=(header.n_columns,)).copy()
y = np.ctypeslib.as_array(self.y, shape=(header.n_rows,)).copy()
coords = [
(dims[0], y, dim_attrs[0]),
(dims[1], x, dim_attrs[1]),
("band", np.array([1, 2, 3], dtype=np.uint8), None),
]

# Get DataArray without padding
data = np.ctypeslib.as_array(
self.data, shape=(header.my, header.mx, header.n_bands)
).copy()
pad = header.pad[:]
data = data[pad[2] : header.my - pad[3], pad[0] : header.mx - pad[1], :]

# Create the xarray.DataArray object
image = xr.DataArray(
data=data,
coords=coords,
name=header.name,
attrs=header.data_attrs,
Comment on lines +204 to +205

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The image name is currently hardcoded to z, is that ok for an RGB image?

@property
def name(self) -> str:
"""
Name of the grid.
"""
return "z"

The attrs fields might need some work. I'm getting 'actual_range': array([ 1.79769313e+308, -1.79769313e+308])} when loading the @earth_day_01d image.

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree they make no sense, but they're consistent with the behavior in GMT.

gmt grdinfo @earth_day_01d
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: Title: Grid imported via GDAL
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: Command:
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: Remark:
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: Pixel node registration used [Geographic grid]
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: Grid file format: gd = Import/export through GDAL
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: x_min: -180 x_max: 180 x_inc: 1 name: x n_columns: 360
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: y_min: -90 y_max: 90 y_inc: 1 name: y n_rows: 180
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: v_min: 1.79769313486e+308 v_max: -1.79769313486e+308 name: z
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: scale_factor: 1 add_offset: 0
/Users/seisman/.gmt/server/earth/earth_day/earth_day_01d_p.tif: Default CPT:
+proj=longlat +R=6378137 +no_defs

The GMT's image support was likely added by Joaquim so that you may ping him for more information.

).transpose("band", dims[0], dims[1])

# Set GMT accessors.
# Must put at the end, otherwise info gets lost after certain image operations.
image.gmt.registration = header.registration
image.gmt.gtype = header.gtype
return image
88 changes: 60 additions & 28 deletions pygmt/tests/test_clib_read_data.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

from pathlib import Path

import numpy as np
import pandas as pd
import pytest
import xarray as xr
Expand All @@ -14,7 +15,7 @@
from pygmt.src import which

try:
import rioxarray # noqa: F401
import rioxarray

_HAS_RIOXARRAY = True
except ImportError:
Expand All @@ -29,6 +30,18 @@ def fixture_expected_xrgrid():
return load_dataarray(which("@static_earth_relief.nc"))


@pytest.fixture(scope="module", name="expected_xrimage")
def fixture_expected_xrimage():
"""
The expected xr.DataArray object for the @earth_day_01d_p file.
"""
if _HAS_RIOXARRAY:
with rioxarray.open_rasterio(which("@earth_day_01d_p")) as da:
dataarray = da.load().drop_vars("spatial_ref")
return dataarray
return None


def test_clib_read_data_dataset():
"""
Test the Session.read_data method for datasets.
Expand Down Expand Up @@ -98,56 +111,63 @@ def test_clib_read_data_grid_two_steps(expected_xrgrid):

# Read the data
lib.read_data(infile, kind="grid", mode="GMT_DATA_ONLY", data=data_ptr)

# Full check
xrgrid = data_ptr.contents.to_dataarray()
xr.testing.assert_equal(xrgrid, expected_xrgrid)


def test_clib_read_data_grid_actual_image():
def test_clib_read_data_grid_actual_image(expected_xrimage):
"""
Test the Session.read_data method for grid, but actually the file is an image.
"""
with Session() as lib:
data_ptr = lib.read_data(
"@earth_day_01d_p", kind="grid", mode="GMT_CONTAINER_AND_DATA"
)
image = data_ptr.contents
header = image.header.contents
assert header.n_rows == 180
assert header.n_columns == 360
assert header.wesn[:] == [-180.0, 180.0, -90.0, 90.0]
image = lib.read_data("@earth_day_01d_p", kind="grid").contents
# Explicitly check n_bands. Only one band is read for 3-band images.
assert header.n_bands == 1
assert image.header.contents.n_bands == 1

xrimage = image.to_dataarray()
assert xrimage.shape == (180, 360)
assert xrimage.coords["x"].data.min() == -179.5
assert xrimage.coords["x"].data.max() == 179.5
assert xrimage.coords["y"].data.min() == -89.5
assert xrimage.coords["y"].data.max() == 89.5
assert xrimage.data.min() == 10.0
assert xrimage.data.max() == 255.0
# Data are stored as uint8 in images but are converted to float32 when reading
# into a GMT_GRID container.
assert xrimage.data.dtype == np.float32

if _HAS_RIOXARRAY: # Full check if rioxarray is installed.
xrimage = image.to_dataarray()
expected_xrimage = xr.open_dataarray(
which("@earth_day_01d_p"), engine="rasterio"
)
assert expected_xrimage.band.size == 3 # 3-band image.
xr.testing.assert_equal(
xrimage,
expected_xrimage.isel(band=0)
.drop_vars(["band", "spatial_ref"])
.sortby("y"),
expected_xrimage.isel(band=0).drop_vars(["band"]).sortby("y"),
)


# Note: Simplify the tests for images after GMT_IMAGE.to_dataarray() is implemented.
def test_clib_read_data_image():
def test_clib_read_data_image(expected_xrimage):
"""
Test the Session.read_data method for images.
"""
with Session() as lib:
image = lib.read_data("@earth_day_01d_p", kind="image").contents
header = image.header.contents
assert header.n_rows == 180
assert header.n_columns == 360
assert header.n_bands == 3
assert header.wesn[:] == [-180.0, 180.0, -90.0, 90.0]
assert image.data

xrimage = image.to_dataarray()
assert xrimage.shape == (3, 180, 360)
assert xrimage.coords["x"].data.min() == -179.5
assert xrimage.coords["x"].data.max() == 179.5
assert xrimage.coords["y"].data.min() == -89.5
assert xrimage.coords["y"].data.max() == 89.5
assert xrimage.data.min() == 10
assert xrimage.data.max() == 255
assert xrimage.data.dtype == np.uint8

if _HAS_RIOXARRAY: # Full check if rioxarray is installed.
xr.testing.assert_equal(xrimage, expected_xrimage)


def test_clib_read_data_image_two_steps():
def test_clib_read_data_image_two_steps(expected_xrimage):
"""
Test the Session.read_data method for images in two steps, first reading the header
and then the data.
Expand All @@ -166,7 +186,19 @@ def test_clib_read_data_image_two_steps():

# Read the data
lib.read_data(infile, kind="image", mode="GMT_DATA_ONLY", data=data_ptr)
assert image.data

xrimage = image.to_dataarray()
assert xrimage.shape == (3, 180, 360)
assert xrimage.coords["x"].data.min() == -179.5
assert xrimage.coords["x"].data.max() == 179.5
assert xrimage.coords["y"].data.min() == -89.5
assert xrimage.coords["y"].data.max() == 89.5
assert xrimage.data.min() == 10
assert xrimage.data.max() == 255
assert xrimage.data.dtype == np.uint8

if _HAS_RIOXARRAY: # Full check if rioxarray is installed.
xr.testing.assert_equal(xrimage, expected_xrimage)


def test_clib_read_data_fails():
Expand Down