diff --git a/autotest/gcore/cog.py b/autotest/gcore/cog.py
index 8056e244caed..4b6e4b6a6aba 100755
--- a/autotest/gcore/cog.py
+++ b/autotest/gcore/cog.py
@@ -18,6 +18,7 @@
import gdaltest
import pytest
+import webserver
from test_py_scripts import samples_path
from osgeo import gdal, osr
@@ -1990,3 +1991,300 @@ def test_cog_preserve_ALPHA_PREMULTIPLIED_on_copy(tmp_vsimem):
ds.GetRasterBand(4).GetMetadataItem("ALPHA", "IMAGE_STRUCTURE")
== "PREMULTIPLIED"
)
+
+
+###############################################################################
+#
+
+
+@gdaltest.enable_exceptions()
+def test_cog_write_interleave_tile(tmp_vsimem):
+ out_filename = str(tmp_vsimem / "out.tif")
+
+ with gdal.quiet_errors():
+ gdal.GetDriverByName("COG").CreateCopy(
+ out_filename,
+ gdal.Open("data/rgbsmall.tif"),
+ options=["INTERLEAVE=TILE", "BLOCKSIZE=32"],
+ )
+
+ ds = gdal.Open(out_filename)
+ assert ds.GetMetadataItem("INTERLEAVE", "IMAGE_STRUCTURE") == "TILE"
+ assert ds.GetMetadataItem("LAYOUT", "IMAGE_STRUCTURE") == "COG"
+
+ assert [ds.GetRasterBand(band + 1).Checksum() for band in range(3)] == [
+ 21212,
+ 21053,
+ 21349,
+ ]
+
+ _check_cog(out_filename)
+
+
+###############################################################################
+#
+
+
+@gdaltest.enable_exceptions()
+def test_cog_write_interleave_band(tmp_vsimem):
+ out_filename = str(tmp_vsimem / "out.tif")
+
+ with gdal.quiet_errors():
+ gdal.GetDriverByName("COG").CreateCopy(
+ out_filename,
+ gdal.Open("data/rgbsmall.tif"),
+ options=["INTERLEAVE=BAND", "BLOCKSIZE=32"],
+ )
+
+ ds = gdal.Open(out_filename)
+ assert ds.GetMetadataItem("INTERLEAVE", "IMAGE_STRUCTURE") == "BAND"
+ assert ds.GetMetadataItem("LAYOUT", "IMAGE_STRUCTURE") == "COG"
+
+ assert [ds.GetRasterBand(band + 1).Checksum() for band in range(3)] == [
+ 21212,
+ 21053,
+ 21349,
+ ]
+
+ _check_cog(out_filename)
+
+
+###############################################################################
+#
+
+
+@gdaltest.enable_exceptions()
+def test_cog_write_interleave_tile_with_mask(tmp_vsimem):
+ out_filename = str(tmp_vsimem / "out.tif")
+
+ with gdal.quiet_errors():
+ gdal.GetDriverByName("COG").CreateCopy(
+ out_filename,
+ gdal.Translate(
+ "", "data/stefan_full_rgba.tif", options="-f MEM -b 1 -b 2 -b 3 -mask 4"
+ ),
+ options=["INTERLEAVE=TILE", "BLOCKSIZE=32"],
+ )
+
+ ds = gdal.Open(out_filename)
+ assert ds.GetMetadataItem("INTERLEAVE", "IMAGE_STRUCTURE") == "TILE"
+ assert ds.GetMetadataItem("LAYOUT", "IMAGE_STRUCTURE") == "COG"
+
+ assert [ds.GetRasterBand(band + 1).Checksum() for band in range(3)] == [
+ 12603,
+ 58561,
+ 36064,
+ ]
+ assert ds.GetRasterBand(1).GetMaskBand().Checksum() == 22499
+
+ _check_cog(out_filename)
+
+ # Check that the tiles are in the expected order in the file
+ last_offset = 0
+ for y in range(2):
+ for x in range(2):
+ for band in range(3):
+ offset = int(
+ ds.GetRasterBand(band + 1).GetMetadataItem(
+ f"BLOCK_OFFSET_{x}_{y}", "TIFF"
+ )
+ )
+ assert offset > last_offset
+ last_offset = offset
+ offset = int(
+ ds.GetRasterBand(1)
+ .GetMaskBand()
+ .GetMetadataItem(f"BLOCK_OFFSET_{x}_{y}", "TIFF")
+ )
+ assert offset > last_offset
+ last_offset = offset
+
+
+###############################################################################
+#
+
+
+@gdaltest.enable_exceptions()
+def test_cog_write_interleave_tile_with_mask_and_ovr(tmp_vsimem):
+ out_filename = str(tmp_vsimem / "out.tif")
+ out2_filename = str(tmp_vsimem / "out2.tif")
+
+ ds = gdal.Translate(
+ out_filename,
+ "data/stefan_full_rgba.tif",
+ options="-b 1 -b 2 -b 3 -mask 4 -outsize 1024 0",
+ )
+ ds.BuildOverviews("NEAR", [2])
+ ds.Close()
+
+ ds = gdal.Open(out_filename)
+ expected_md = [ds.GetRasterBand(band + 1).Checksum() for band in range(3)]
+ expected_md += [ds.GetRasterBand(1).GetMaskBand().Checksum()]
+ expected_ovr_md = [
+ ds.GetRasterBand(band + 1).GetOverview(0).Checksum() for band in range(3)
+ ]
+ expected_ovr_md += [ds.GetRasterBand(1).GetMaskBand().GetOverview(0).Checksum()]
+
+ gdal.GetDriverByName("COG").CreateCopy(
+ out2_filename,
+ ds,
+ options=["INTERLEAVE=TILE", "OVERVIEW_RESAMPLING=NEAREST"],
+ )
+
+ ds = gdal.Open(out2_filename)
+ assert ds.GetMetadataItem("INTERLEAVE", "IMAGE_STRUCTURE") == "TILE"
+ assert ds.GetMetadataItem("LAYOUT", "IMAGE_STRUCTURE") == "COG"
+
+ _check_cog(out2_filename)
+
+ got_md = [ds.GetRasterBand(band + 1).Checksum() for band in range(3)]
+ got_md += [ds.GetRasterBand(1).GetMaskBand().Checksum()]
+ assert got_md == expected_md
+ got_ovr_md = [
+ ds.GetRasterBand(band + 1).GetOverview(0).Checksum() for band in range(3)
+ ]
+ got_ovr_md += [ds.GetRasterBand(1).GetMaskBand().GetOverview(0).Checksum()]
+ assert got_ovr_md == expected_ovr_md
+
+
+###############################################################################
+# Check that our reading of a COG with /vsicurl is efficient
+
+
+@pytest.mark.require_curl()
+@pytest.mark.skipif(
+ not check_libtiff_internal_or_at_least(4, 0, 11),
+ reason="libtiff >= 4.0.11 required",
+)
+@pytest.mark.parametrize("INTERLEAVE", ["BAND", "TILE"])
+def test_cog_interleave_tile_or_band_vsicurl(tmp_vsimem, INTERLEAVE):
+
+ gdal.VSICurlClearCache()
+
+ webserver_process = None
+ webserver_port = 0
+
+ (webserver_process, webserver_port) = webserver.launch(
+ handler=webserver.DispatcherHttpHandler
+ )
+ if webserver_port == 0:
+ pytest.skip()
+
+ in_filename = str(tmp_vsimem / "in.tif")
+ cog_filename = str(tmp_vsimem / "cog.tif")
+
+ ds = gdal.Translate(
+ in_filename,
+ "data/stefan_full_rgba.tif",
+ options="-b 1 -b 2 -b 3 -mask 4 -outsize 1024 0",
+ )
+ ds.BuildOverviews("NEAR", [2])
+ ds.Close()
+
+ src_ds = gdal.Open(in_filename)
+ gdal.GetDriverByName("COG").CreateCopy(
+ cog_filename,
+ src_ds,
+ options=["INTERLEAVE=" + INTERLEAVE, "OVERVIEW_RESAMPLING=NEAREST"],
+ )
+
+ def extract(offset, size):
+ f = gdal.VSIFOpenL(cog_filename, "rb")
+ gdal.VSIFSeekL(f, offset, 0)
+ data = gdal.VSIFReadL(size, 1, f)
+ gdal.VSIFCloseL(f)
+ return data
+
+ try:
+ filesize = gdal.VSIStatL(cog_filename).size
+
+ handler = webserver.SequentialHandler()
+ handler.add("HEAD", "/cog.tif", 200, {"Content-Length": "%d" % filesize})
+ handler.add(
+ "GET",
+ "/cog.tif",
+ 206,
+ {"Content-Length": "16384"},
+ extract(0, 16384),
+ expected_headers={"Range": "bytes=0-16383"},
+ )
+ with webserver.install_http_handler(handler):
+ ds = gdal.Open("/vsicurl/http://localhost:%d/cog.tif" % webserver_port)
+
+ handler = webserver.SequentialHandler()
+
+ def method(request):
+ # sys.stderr.write('%s\n' % str(request.headers))
+
+ if request.headers["Range"].startswith("bytes="):
+ rng = request.headers["Range"][len("bytes=") :]
+ assert len(rng.split("-")) == 2
+ start = int(rng.split("-")[0])
+ end = int(rng.split("-")[1])
+
+ request.protocol_version = "HTTP/1.1"
+ request.send_response(206)
+ request.send_header("Content-type", "application/octet-stream")
+ request.send_header(
+ "Content-Range", "bytes %d-%d/%d" % (start, end, filesize)
+ )
+ request.send_header("Content-Length", end - start + 1)
+ request.send_header("Connection", "close")
+ request.end_headers()
+
+ request.wfile.write(extract(start, end - start + 1))
+
+ handler.add("GET", "/cog.tif", custom_method=method)
+ with webserver.install_http_handler(handler):
+ ret = ds.ReadRaster()
+ assert ret == src_ds.ReadRaster()
+
+ finally:
+ webserver.server_stop(webserver_process, webserver_port)
+
+ gdal.VSICurlClearCache()
+
+
+###############################################################################
+
+
+@pytest.mark.require_creation_option("COG", "JPEG")
+@gdaltest.enable_exceptions()
+def test_cog_write_interleave_tile_jpeg(tmp_vsimem):
+ out_filename = str(tmp_vsimem / "out.tif")
+
+ gdal.GetDriverByName("GTiff").CreateCopy(
+ out_filename,
+ gdal.Open("data/rgbsmall.tif"),
+ options=["INTERLEAVE=BAND", "COMPRESS=JPEG"],
+ )
+ with gdal.Open(out_filename) as ds:
+ expected_md = [ds.GetRasterBand(band + 1).Checksum() for band in range(3)]
+
+ gdal.GetDriverByName("COG").CreateCopy(
+ out_filename,
+ gdal.Open("data/rgbsmall.tif"),
+ options=["INTERLEAVE=TILE", "COMPRESS=JPEG"],
+ )
+ with gdal.Open(out_filename) as ds:
+ got_md = [ds.GetRasterBand(band + 1).Checksum() for band in range(3)]
+
+ assert got_md == expected_md
+
+
+###############################################################################
+
+
+@pytest.mark.require_creation_option("COG", "WEBP")
+@gdaltest.enable_exceptions()
+def test_cog_write_interleave_tile_webp_error(tmp_vsimem):
+ out_filename = str(tmp_vsimem / "out.tif")
+
+ with pytest.raises(
+ Exception, match="COMPRESS=WEBP only supported for INTERLEAVE=PIXEL"
+ ):
+ gdal.GetDriverByName("COG").CreateCopy(
+ out_filename,
+ gdal.Open("data/rgbsmall.tif"),
+ options=["INTERLEAVE=TILE", "COMPRESS=WEBP"],
+ )
diff --git a/autotest/gcore/test_driver_metadata.py b/autotest/gcore/test_driver_metadata.py
index d36628be4055..b8907c47e938 100644
--- a/autotest/gcore/test_driver_metadata.py
+++ b/autotest/gcore/test_driver_metadata.py
@@ -130,6 +130,7 @@
+
diff --git a/autotest/gcore/tiff_write.py b/autotest/gcore/tiff_write.py
index 25f58a827f9c..c589fdf6b200 100755
--- a/autotest/gcore/tiff_write.py
+++ b/autotest/gcore/tiff_write.py
@@ -11988,3 +11988,55 @@ def test_tiff_write_warn_ignore_predictor_option(tmp_vsimem):
out_filename, 1, 1, options=["PREDICTOR=2"]
)
assert "PREDICTOR option is ignored" in gdal.GetLastErrorMsg()
+
+
+###############################################################################
+#
+
+
+@gdaltest.enable_exceptions()
+@pytest.mark.parametrize("COPY_SRC_OVERVIEWS", ["YES", "NO"])
+def test_tiff_write_interleave_tile(tmp_vsimem, COPY_SRC_OVERVIEWS):
+ out_filename = str(tmp_vsimem / "out.tif")
+ with pytest.raises(
+ Exception, match="INTERLEAVE=TILE is only supported for CreateCopy"
+ ):
+ gdal.GetDriverByName("GTiff").Create(
+ out_filename, 1, 1, 2, options=["INTERLEAVE=TILE"]
+ )
+
+ ds = gdal.GetDriverByName("GTiff").CreateCopy(
+ out_filename,
+ gdal.Open("data/rgbsmall.tif"),
+ options=[
+ "INTERLEAVE=TILE",
+ "TILED=YES",
+ "BLOCKXSIZE=32",
+ "BLOCKYSIZE=32",
+ "COPY_SRC_OVERVIEWS=" + COPY_SRC_OVERVIEWS,
+ ],
+ )
+ assert ds.GetMetadataItem("INTERLEAVE", "IMAGE_STRUCTURE") == "TILE"
+ ds.Close()
+
+ ds = gdal.Open(out_filename)
+ assert ds.GetMetadataItem("INTERLEAVE", "IMAGE_STRUCTURE") == "TILE"
+
+ assert [ds.GetRasterBand(band + 1).Checksum() for band in range(3)] == [
+ 21212,
+ 21053,
+ 21349,
+ ]
+
+ # Check that the tiles are in the expected order in the file
+ last_offset = 0
+ for y in range(2):
+ for x in range(2):
+ for band in range(3):
+ offset = int(
+ ds.GetRasterBand(band + 1).GetMetadataItem(
+ f"BLOCK_OFFSET_{x}_{y}", "TIFF"
+ )
+ )
+ assert offset > last_offset
+ last_offset = offset
diff --git a/doc/source/drivers/raster/cog.rst b/doc/source/drivers/raster/cog.rst
index c46bac1f5703..d9a6f06fca1b 100644
--- a/doc/source/drivers/raster/cog.rst
+++ b/doc/source/drivers/raster/cog.rst
@@ -43,6 +43,105 @@ General creation options
Sets the tile width and height in pixels. Must be divisible by 16.
+- .. co:: INTERLEAVE
+ :choices: BAND, PIXEL, TILE
+ :since: 3.11
+
+ Set the interleaving to use
+
+ * ``PIXEL``: for each spatial block, one TIFF tile/strip gathering values for
+ all bands is used . This matches the ``contiguous`` planar configuration in
+ TIFF terminology.
+ This is also known as a ``BIP (Band Interleaved Per Pixel)`` organization.
+ Such method is the default, and may be slightly less
+ efficient than BAND interleaving for some purposes, but some applications
+ only support pixel interleaved TIFF files. On the other hand, image-based
+ compression methods may perform better using PIXEL interleaving. For JPEG
+ PHOTOMETRIC=YCbCr, pixel interleaving is required. It is also required for
+ WebP compression.
+
+ Prior to GDAL 3.11, INTERLEAVE=PIXEL was the only possible interleaving
+ output by the COG driver.
+
+ Assuming a pixel[band][y][x] indexed array, when using CreateCopy(),
+ the pseudo code for the file disposition is:
+
+ ::
+
+ for y in 0 ... numberOfBlocksInHeight - 1:
+ for x in 0 ... numberOfTilesInWidth - 1:
+ for j in 0 ... blockHeight - 1:
+ for i in 0 ... blockWidth -1:
+ start_new_strip_or_tile()
+ for band in 0 ... numberBands -1:
+ write(pixel[band][y*blockHeight+j][x*blockWidth+i])
+ end_new_strip_or_tile()
+ end_for
+ end_for
+ end_for
+ end_for
+
+
+ * ``BAND``: for each spatial block, one TIFF tile/strip is used for each band.
+ This matches the contiguous ``separate`` configuration in TIFF terminology.
+ This is also known as a ``BSQ (Band SeQuential)`` organization.
+
+ In addition to that, when using CreateCopy(), data for the first band is
+ written first, followed by data for the second band, etc.
+ The pseudo code for the file disposition is:
+
+ ::
+
+ for y in 0 ... numberOfBlocksInHeight - 1:
+ for x in 0 ... numberOfTilesInWidth - 1:
+ start_new_strip_or_tile()
+ for band in 0 ... numberBands -1:
+ for j in 0 ... blockHeight - 1:
+ for i in 0 ... blockWidth -1:
+ write(pixel[band][y*blockHeight+j][x*blockWidth+i])
+ end_for
+ end_for
+ end_new_strip_or_tile()
+ end_for
+ end_for
+
+
+ * ``TILE`` (added in 3.11, only for CreateCopy()): this is a sort of
+ compromise between PIXEL and BAND, using the ``separate`` configuration,
+ but where data for a same spatial block is written for all bands, before
+ the data of the next spatial block.
+ When the block height is 1, this is also known as a
+ ``BIL (Band Interleaved per Line)`` organization.
+
+ Such a layout may be useful for writing hyperspectral datasets (several
+ hundred of bands), to get a compromise between efficient spatial query,
+ and partial band selection.
+
+ Assuming a pixel[band][y][x] indexed array, when using CreateCopy(),
+ the pseudo code for the file disposition is:
+
+ ::
+
+ for y in 0 ... numberOfBlocksInHeight - 1:
+ for x in 0 ... numberOfTilesInWidth - 1:
+ for band in 0 ... numberBands -1:
+ start_new_strip_or_tile()
+ for j in 0 ... blockHeight - 1:
+ for i in 0 ... blockWidth -1:
+ write(pixel[band][y*blockHeight+j][x*blockWidth+i])
+ end_for
+ end_for
+ end_new_strip_or_tile()
+ end_for
+ end_for
+ end_for
+
+
+ Starting with GDAL 3.5, when copying from a source dataset with multiple bands
+ which advertises a INTERLEAVE metadata item, if the INTERLEAVE creation option
+ is not specified, the source dataset INTERLEAVE will be automatically taken
+ into account, unless the COMPRESS creation option is specified.
+
- .. co:: COMPRESS
:choices: NONE, LZW, JPEG, DEFLATE, ZSTD, WEBP, LERC, LERC_DEFLATE, LERC_ZSTD, LZMA
:default: LZW
@@ -56,7 +155,7 @@ General creation options
files (seen as UInt16 bands with NBITS=12).
For the COG driver, JPEG compression for 3 or 4-band images automatically
selects the PHOTOMETRIC=YCBCR colorspace with a 4:2:2 subsampling of the Y,Cb,Cr
- components.
+ components with the default INTERLEAVE=PIXEL.
For a input dataset (single-band or 3-band), plus an alpha band,
the alpha band will be converted as a 1-bit DEFLATE compressed mask.
@@ -65,6 +164,10 @@ General creation options
* ``ZSTD`` is available when using internal libtiff and if GDAL built against
libzstd >=1.0, or if built against external libtiff with zstd support.
+ * ``WEBP`` is available when using internal libtiff and if GDAL built against
+ libwebp, or if built against external libtiff with WebP support.
+ It can only be used with the default INTERLEAVE=PIXEL.
+
* ``LERC`` is available when using internal libtiff.
* ``LERC_ZSTD`` is available when ``LERC`` and ``ZSTD`` are available.
@@ -559,11 +662,18 @@ line).
mask.TileByteCounts[i], but none of them actually need to be read)
* trailer of mask data (4 bytes)
+ This is only written if INTERLEAVE=PIXEL.
+
+- ``INTERLEAVE=BAND`` or ``INTERLEAVE=TILE``: (GDAL >= 3.11)
+ Reflects the value of the INTERLEAVE creation option.
+ Omission implies INTERLEAVE=PIXEL.
+
+
.. note::
The content of the header ghost area can be retrieved by getting the
``GDAL_STRUCTURAL_METADATA`` metadata item of the ``TIFF`` metadata domain
- on the datasett object (with GetMetadataItem())
+ on the dataset object (with GetMetadataItem())
.. _cog.tile_data_leader_trailer:
@@ -576,7 +686,8 @@ that follows it. This leader is *ghost* in the sense that the
TileOffsets[] array does not point to it, but points to the real payload. Hence
the offset of the leader is TileOffsets[i]-4.
-An optimized reader seeing the ``BLOCK_LEADER=SIZE_AS_UINT4`` metadata item will thus look for TileOffset[i]
+For INTERLEAVE=PIXEL or INTERLEAVE=TILE, an optimized reader seeing the
+``BLOCK_LEADER=SIZE_AS_UINT4`` metadata item will thus look for TileOffset[i]
and TileOffset[i+1] to deduce it must fetch the data starting at
offset=TileOffset[i] - 4 and of size=TileOffset[i+1]-TileOffset[i]+4. It then
checks the 4 first bytes to see if the size in this leader marker is
@@ -587,6 +698,9 @@ MASK_INTERLEAVED_WITH_IMAGERY=YES, then the tile size indicated in the leader
will be < TileOffset[i+1]-TileOffset[i] since the data for the mask will
follow the imagery data (see MASK_INTERLEAVED_WITH_IMAGERY=YES)
+For INTERLEAVE=BAND, the above paragraph applies but the successor of tile i
+is not tile i+1, but tile i+nTilesPerBand.
+
Each tile data is immediately followed by a trailer, consisting of the repetition
of the last 4 bytes of the payload of the tile data. The size of this trailer is
*not* included in the TileByteCounts[] array. The purpose of this trailer is forces
@@ -613,3 +727,9 @@ See Also
- If your source dataset is an internally tiled geotiff with the desired georeferencing and compression,
using `cogger `__ (possibly along with gdaladdo to create overviews) will
be much faster than the COG driver.
+
+
+.. below is an allow-list for spelling checker.
+
+.. spelling:word-list::
+ nTilesPerBand
diff --git a/doc/source/drivers/raster/gtiff.rst b/doc/source/drivers/raster/gtiff.rst
index f6d4f722c42a..cdc14a6e3634 100644
--- a/doc/source/drivers/raster/gtiff.rst
+++ b/doc/source/drivers/raster/gtiff.rst
@@ -420,13 +420,95 @@ This driver supports the following creation options:
RPC information is available.
- .. co:: INTERLEAVE
- :choices: BAND, PIXEL
+ :choices: BAND, PIXEL, TILE
+
+ Set the interleaving to use
+
+ * ``PIXEL``: for each spatial block, one TIFF tile/strip gathering values for
+ all bands is used . This matches the ``contiguous`` planar configuration in
+ TIFF terminology.
+ This is also known as a ``BIP (Band Interleaved Per Pixel)`` organization.
+ Such method is the default, and may be slightly less
+ efficient than BAND interleaving for some purposes, but some applications
+ only support pixel interleaved TIFF files. On the other hand, image-based
+ compression methods may perform better using PIXEL interleaving. For JPEG
+ PHOTOMETRIC=YCbCr, pixel interleaving is required. It is also required for
+ WebP compression.
+
+ Assuming a pixel[band][y][x] indexed array, when using CreateCopy(),
+ the pseudo code for the file disposition is:
+
+ ::
+
+ for y in 0 ... numberOfBlocksInHeight - 1:
+ for x in 0 ... numberOfTilesInWidth - 1:
+ for j in 0 ... blockHeight - 1:
+ for i in 0 ... blockWidth -1:
+ start_new_strip_or_tile()
+ for band in 0 ... numberBands -1:
+ write(pixel[band][y*blockHeight+j][x*blockWidth+i])
+ end_new_strip_or_tile()
+ end_for
+ end_for
+ end_for
+ end_for
+
+
+ * ``BAND``: for each spatial block, one TIFF tile/strip is used for each band.
+ This matches the contiguous ``separate`` configuration in TIFF terminology.
+ This is also known as a ``BSQ (Band SeQuential)`` organization.
+
+ In addition to that, when using CreateCopy(), data for the first band is
+ written first, followed by data for the second band, etc.
+ The pseudo code for the file disposition is:
+
+ ::
+
+ for y in 0 ... numberOfBlocksInHeight - 1:
+ for x in 0 ... numberOfTilesInWidth - 1:
+ start_new_strip_or_tile()
+ for band in 0 ... numberBands -1:
+ for j in 0 ... blockHeight - 1:
+ for i in 0 ... blockWidth -1:
+ write(pixel[band][y*blockHeight+j][x*blockWidth+i])
+ end_for
+ end_for
+ end_new_strip_or_tile()
+ end_for
+ end_for
+
+
+ * ``TILE`` (added in 3.11, only for CreateCopy()): this is a sort of
+ compromise between PIXEL and BAND, using the ``separate`` configuration,
+ but where data for a same spatial block is written for all bands, before
+ the data of the next spatial block.
+ When the block height is 1, this is also known as a
+ ``BIL (Band Interleaved per Line)`` organization.
+
+ Such a layout may be useful for writing hyperspectral datasets (several
+ hundred of bands), to get a compromise between efficient spatial query,
+ and partial band selection.
+
+ Assuming a pixel[band][y][x] indexed array, when using CreateCopy(),
+ the pseudo code for the file disposition is:
+
+ ::
+
+ for y in 0 ... numberOfBlocksInHeight - 1:
+ for x in 0 ... numberOfTilesInWidth - 1:
+ for band in 0 ... numberBands -1:
+ start_new_strip_or_tile()
+ for j in 0 ... blockHeight - 1:
+ for i in 0 ... blockWidth -1:
+ write(pixel[band][y*blockHeight+j][x*blockWidth+i])
+ end_for
+ end_for
+ end_new_strip_or_tile()
+ end_for
+ end_for
+ end_for
+
- By default TIFF files with pixel
- interleaving (PLANARCONFIG_CONTIG in TIFF terminology) are created.
- These are slightly less efficient than BAND interleaving for some
- purposes, but some applications only support pixel interleaved TIFF
- files.
Starting with GDAL 3.5, when copying from a source dataset with multiple bands
which advertises a INTERLEAVE metadata item, if the INTERLEAVE creation option
is not specified, the source dataset INTERLEAVE will be automatically taken
diff --git a/frmts/gtiff/cogdriver.cpp b/frmts/gtiff/cogdriver.cpp
index ac56ca0d2845..366955c7fb8a 100644
--- a/frmts/gtiff/cogdriver.cpp
+++ b/frmts/gtiff/cogdriver.cpp
@@ -763,6 +763,21 @@ GDALDataset *GDALCOGCreator::Create(const char *pszFilename,
return nullptr;
}
+ const CPLString osCompress = CSLFetchNameValueDef(
+ papszOptions, "COMPRESS", gbHasLZW ? "LZW" : "NONE");
+
+ const char *pszInterleave =
+ CSLFetchNameValueDef(papszOptions, "INTERLEAVE", "PIXEL");
+ if (EQUAL(osCompress, "WEBP"))
+ {
+ if (!EQUAL(pszInterleave, "PIXEL"))
+ {
+ CPLError(CE_Failure, CPLE_NotSupported,
+ "COMPRESS=WEBP only supported for INTERLEAVE=PIXEL");
+ return nullptr;
+ }
+ }
+
CPLConfigOptionSetter oSetterReportDirtyBlockFlushing(
"GDAL_REPORT_DIRTY_BLOCK_FLUSHING", "NO", true);
@@ -877,9 +892,7 @@ GDALDataset *GDALCOGCreator::Create(const char *pszFilename,
}
}
- CPLString osCompress = CSLFetchNameValueDef(papszOptions, "COMPRESS",
- gbHasLZW ? "LZW" : "NONE");
- if (EQUAL(osCompress, "JPEG") &&
+ if (EQUAL(osCompress, "JPEG") && EQUAL(pszInterleave, "PIXEL") &&
(poCurDS->GetRasterCount() == 2 || poCurDS->GetRasterCount() == 4) &&
poCurDS->GetRasterBand(poCurDS->GetRasterCount())
->GetColorInterpretation() == GCI_AlphaBand)
@@ -1198,7 +1211,7 @@ GDALDataset *GDALCOGCreator::Create(const char *pszFilename,
if (EQUAL(osCompress, "JPEG"))
{
aosOptions.SetNameValue("JPEG_QUALITY", pszQuality);
- if (nBands == 3)
+ if (nBands == 3 && EQUAL(pszInterleave, "PIXEL"))
aosOptions.SetNameValue("PHOTOMETRIC", "YCBCR");
}
else if (EQUAL(osCompress, "WEBP"))
@@ -1316,7 +1329,8 @@ GDALDataset *GDALCOGCreator::Create(const char *pszFilename,
}
std::unique_ptr poPhotometricSetter;
- if (nBands == 3 && EQUAL(pszOverviewCompress, "JPEG"))
+ if (nBands == 3 && EQUAL(pszOverviewCompress, "JPEG") &&
+ EQUAL(pszInterleave, "PIXEL"))
{
poPhotometricSetter.reset(
new CPLConfigOptionSetter("PHOTOMETRIC_OVERVIEW", "YCBCR", true));
@@ -1347,6 +1361,8 @@ GDALDataset *GDALCOGCreator::Create(const char *pszFilename,
aosOptions.AddNameValue("SRC_MDD", *papszSrcMDDIter);
CSLDestroy(papszSrcMDD);
+ aosOptions.SetNameValue("INTERLEAVE", pszInterleave);
+
CPLDebug("COG", "Generating final product: start");
auto poRet =
poGTiffDrv->CreateCopy(pszFilename, poCurDS, false, aosOptions.List(),
@@ -1523,6 +1539,11 @@ void GDALCOGDriver::InitializeCreationOptionList()
"(16)'/>"
" "
+ " "
" "
" "
diff --git a/frmts/gtiff/gtiffdataset.cpp b/frmts/gtiff/gtiffdataset.cpp
index ee65ece7df1e..2cd7b391d38f 100644
--- a/frmts/gtiff/gtiffdataset.cpp
+++ b/frmts/gtiff/gtiffdataset.cpp
@@ -99,7 +99,7 @@ GTiffDataset::GTiffDataset()
m_bLeaderSizeAsUInt4(false), m_bTrailerRepeatedLast4BytesRepeated(false),
m_bMaskInterleavedWithImagery(false), m_bKnownIncompatibleEdition(false),
m_bWriteKnownIncompatibleEdition(false), m_bHasUsedReadEncodedAPI(false),
- m_bWriteCOGLayout(false)
+ m_bWriteCOGLayout(false), m_bTileInterleave(false)
{
// CPLDebug("GDAL", "sizeof(GTiffDataset) = %d bytes", static_cast(
// sizeof(GTiffDataset)));
@@ -483,8 +483,8 @@ CPLErr GTiffDataset::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff,
const int nBlockY2 = (nYOff + nYSize - 1) / m_nBlockYSize;
const int nXBlocks = nBlockX2 - nBlockX1 + 1;
const int nYBlocks = nBlockY2 - nBlockY1 + 1;
- const int nBlocks =
- nXBlocks * nYBlocks *
+ const size_t nBlocks =
+ static_cast(nXBlocks) * nYBlocks *
(m_nPlanarConfig == PLANARCONFIG_CONTIG ? 1 : nBandCount);
if (nBlocks > 1)
{
@@ -497,13 +497,23 @@ CPLErr GTiffDataset::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff,
const auto eDataType = poFirstBand->GetRasterDataType();
if (eAccess == GA_ReadOnly && eRWFlag == GF_Read &&
- (nBands == 1 || m_nPlanarConfig == PLANARCONFIG_CONTIG) &&
HasOptimizedReadMultiRange() &&
!(bCanUseMultiThreadedRead &&
VSI_TIFFGetVSILFile(TIFFClientdata(m_hTIFF))->HasPRead()))
{
- pBufferedData = poFirstBand->CacheMultiRange(
- nXOff, nYOff, nXSize, nYSize, nBufXSize, nBufYSize, psExtraArg);
+ if (nBands == 1 || m_nPlanarConfig == PLANARCONFIG_CONTIG)
+ {
+ const int nBandOne = 1;
+ pBufferedData =
+ CacheMultiRange(nXOff, nYOff, nXSize, nYSize, nBufXSize,
+ nBufYSize, &nBandOne, 1, psExtraArg);
+ }
+ else
+ {
+ pBufferedData =
+ CacheMultiRange(nXOff, nYOff, nXSize, nYSize, nBufXSize,
+ nBufYSize, panBandMap, nBandCount, psExtraArg);
+ }
}
else if (bCanUseMultiThreadedRead)
{
@@ -1003,6 +1013,7 @@ void GTiffDataset::SetStructuralMDFromParent(GTiffDataset *poParentDS)
poParentDS->m_bTrailerRepeatedLast4BytesRepeated;
m_bMaskInterleavedWithImagery = poParentDS->m_bMaskInterleavedWithImagery;
m_bWriteEmptyTiles = poParentDS->m_bWriteEmptyTiles;
+ m_bTileInterleave = poParentDS->m_bTileInterleave;
}
/************************************************************************/
diff --git a/frmts/gtiff/gtiffdataset.h b/frmts/gtiff/gtiffdataset.h
index 3bca95d0ad33..804b1ec56541 100644
--- a/frmts/gtiff/gtiffdataset.h
+++ b/frmts/gtiff/gtiffdataset.h
@@ -305,6 +305,7 @@ class GTiffDataset final : public GDALPamDataset
bool m_bWriteKnownIncompatibleEdition : 1;
bool m_bHasUsedReadEncodedAPI : 1; // for debugging
bool m_bWriteCOGLayout : 1;
+ bool m_bTileInterleave : 1;
void ScanDirectories();
bool ReadStrile(int nBlockId, void *pOutputBuffer,
@@ -440,6 +441,10 @@ class GTiffDataset final : public GDALPamDataset
bool IsWholeBlock(int nXOff, int nYOff, int nXSize, int nYSize) const;
+ void *CacheMultiRange(int nXOff, int nYOff, int nXSize, int nYSize,
+ int nBufXSize, int nBufYSize, const int *panBandMap,
+ int nBandCount, GDALRasterIOExtraArg *psExtraArg);
+
static void ThreadDecompressionFunc(void *pData);
static GTIF *GTIFNew(TIFF *hTIFF);
@@ -542,7 +547,8 @@ class GTiffDataset final : public GDALPamDataset
int nBands, GDALDataType eType,
double dfExtraSpaceForOverviews,
int nColorTableMultiplier, char **papszParamList,
- VSILFILE **pfpL, CPLString &osTmpFilename);
+ VSILFILE **pfpL, CPLString &osTmpFilename,
+ bool bCreateCopy, bool &bTileInterleavingOut);
CPLErr WriteEncodedTileOrStrip(uint32_t tile_or_strip, void *data,
int bPreserveDataBuffer);
diff --git a/frmts/gtiff/gtiffdataset_read.cpp b/frmts/gtiff/gtiffdataset_read.cpp
index 81c587b0f167..21526ee03daf 100644
--- a/frmts/gtiff/gtiffdataset_read.cpp
+++ b/frmts/gtiff/gtiffdataset_read.cpp
@@ -3893,13 +3893,20 @@ GDALDataset *GTiffDataset::Open(GDALOpenInfo *poOpenInfo)
poDS->m_bBlockOrderRowMajor =
strstr(pszStructuralMD, "BLOCK_ORDER=ROW_MAJOR") != nullptr;
poDS->m_bLeaderSizeAsUInt4 =
- strstr(pszStructuralMD, "BLOCK_LEADER=SIZE_AS_UINT4") != nullptr;
+ strstr(pszStructuralMD, "BLOCK_LEADER=SIZE_AS_UINT4") != nullptr &&
+ (strstr(pszStructuralMD, "INTERLEAVE=") == nullptr ||
+ strstr(pszStructuralMD, "INTERLEAVE=BAND") != nullptr ||
+ strstr(pszStructuralMD, "INTERLEAVE=TILE") != nullptr);
poDS->m_bTrailerRepeatedLast4BytesRepeated =
strstr(pszStructuralMD, "BLOCK_TRAILER=LAST_4_BYTES_REPEATED") !=
- nullptr;
+ nullptr &&
+ (strstr(pszStructuralMD, "INTERLEAVE=") == nullptr ||
+ strstr(pszStructuralMD, "INTERLEAVE=BAND") != nullptr ||
+ strstr(pszStructuralMD, "INTERLEAVE=TILE") != nullptr);
poDS->m_bMaskInterleavedWithImagery =
strstr(pszStructuralMD, "MASK_INTERLEAVED_WITH_IMAGERY=YES") !=
- nullptr;
+ nullptr &&
+ strstr(pszStructuralMD, "INTERLEAVE=") == nullptr;
poDS->m_bKnownIncompatibleEdition =
strstr(pszStructuralMD, "KNOWN_INCOMPATIBLE_EDITION=YES") !=
nullptr;
@@ -5532,8 +5539,24 @@ CPLErr GTiffDataset::OpenOffset(TIFF *hTIFFIn, toff_t nDirOffsetIn,
continue;
if (EQUAL(pszDomain, "IMAGE_STRUCTURE"))
{
- if (m_nCompression == COMPRESSION_WEBP &&
- EQUAL(pszKey, "COMPRESSION_REVERSIBILITY"))
+ if (EQUAL(pszKey, "INTERLEAVE"))
+ {
+ if (EQUAL(pszValue, "TILE"))
+ {
+ m_bTileInterleave = true;
+ m_oGTiffMDMD.SetMetadataItem("INTERLEAVE", "TILE",
+ "IMAGE_STRUCTURE");
+ }
+ else
+ {
+ CPLDebug("GTiff",
+ "Unhandled INTERLEAVE=%s found in "
+ "GDAL_METADATA tag",
+ pszValue);
+ }
+ }
+ else if (m_nCompression == COMPRESSION_WEBP &&
+ EQUAL(pszKey, "COMPRESSION_REVERSIBILITY"))
{
if (EQUAL(pszValue, "LOSSLESS"))
m_bWebPLossless = true;
@@ -6701,3 +6724,482 @@ bool GTiffDataset::HasOptimizedReadMultiRange()
"GTIFF_HAS_OPTIMIZED_READ_MULTI_RANGE", "NO")));
return m_nHasOptimizedReadMultiRange != 0;
}
+
+/************************************************************************/
+/* CacheMultiRange() */
+/************************************************************************/
+
+static bool CheckTrailer(const GByte *strileData, vsi_l_offset nStrileSize)
+{
+ GByte abyTrailer[4];
+ memcpy(abyTrailer, strileData + nStrileSize, 4);
+ GByte abyLastBytes[4] = {};
+ if (nStrileSize >= 4)
+ memcpy(abyLastBytes, strileData + nStrileSize - 4, 4);
+ else
+ {
+ // The last bytes will be zero due to the above {} initialization,
+ // and that's what should be in abyTrailer too when the trailer is
+ // correct.
+ memcpy(abyLastBytes, strileData, static_cast(nStrileSize));
+ }
+ return memcmp(abyTrailer, abyLastBytes, 4) == 0;
+}
+
+void *GTiffDataset::CacheMultiRange(int nXOff, int nYOff, int nXSize,
+ int nYSize, int nBufXSize, int nBufYSize,
+ const int *panBandMap, int nBandCount,
+ GDALRasterIOExtraArg *psExtraArg)
+{
+ void *pBufferedData = nullptr;
+ // Same logic as in GDALRasterBand::IRasterIO()
+ double dfXOff = nXOff;
+ double dfYOff = nYOff;
+ double dfXSize = nXSize;
+ double dfYSize = nYSize;
+ if (psExtraArg->bFloatingPointWindowValidity)
+ {
+ dfXOff = psExtraArg->dfXOff;
+ dfYOff = psExtraArg->dfYOff;
+ dfXSize = psExtraArg->dfXSize;
+ dfYSize = psExtraArg->dfYSize;
+ }
+ const double dfSrcXInc = dfXSize / static_cast(nBufXSize);
+ const double dfSrcYInc = dfYSize / static_cast(nBufYSize);
+ const double EPS = 1e-10;
+ const int nBlockX1 =
+ static_cast(std::max(0.0, (0 + 0.5) * dfSrcXInc + dfXOff + EPS)) /
+ m_nBlockXSize;
+ const int nBlockY1 =
+ static_cast(std::max(0.0, (0 + 0.5) * dfSrcYInc + dfYOff + EPS)) /
+ m_nBlockYSize;
+ const int nBlockX2 =
+ static_cast(
+ std::min(static_cast(nRasterXSize - 1),
+ (nBufXSize - 1 + 0.5) * dfSrcXInc + dfXOff + EPS)) /
+ m_nBlockXSize;
+ const int nBlockY2 =
+ static_cast(
+ std::min(static_cast(nRasterYSize - 1),
+ (nBufYSize - 1 + 0.5) * dfSrcYInc + dfYOff + EPS)) /
+ m_nBlockYSize;
+
+ struct StrileData
+ {
+ vsi_l_offset nOffset;
+ vsi_l_offset nByteCount;
+ bool bTryMask;
+ };
+
+ std::map oMapStrileToOffsetByteCount;
+
+ // Dedicated method to retrieved the offset and size in an efficient way
+ // when m_bBlockOrderRowMajor and m_bLeaderSizeAsUInt4 conditions are
+ // met.
+ // Except for the last block, we just read the offset from the TIFF offset
+ // array, and retrieve the size in the leader 4 bytes that come before the
+ // payload.
+ auto OptimizedRetrievalOfOffsetSize =
+ [&](int nBand, int nBlockId, vsi_l_offset &nOffset, vsi_l_offset &nSize,
+ size_t nTotalSize, size_t nMaxRawBlockCacheSize)
+ {
+ bool bTryMask = m_bMaskInterleavedWithImagery;
+ nOffset = TIFFGetStrileOffset(m_hTIFF, nBlockId);
+ if (nOffset >= 4)
+ {
+ if ((m_nPlanarConfig == PLANARCONFIG_CONTIG &&
+ nBlockId == m_nBlocksPerBand - 1) ||
+ (m_nPlanarConfig == PLANARCONFIG_SEPARATE &&
+ nBlockId == m_nBlocksPerBand * nBands - 1))
+ {
+ // Special case for the last block. As there is no next block
+ // from which to retrieve an offset, use the good old method
+ // that consists in reading the ByteCount array.
+ if (m_nPlanarConfig == PLANARCONFIG_CONTIG && bTryMask &&
+ GetRasterBand(1)->GetMaskBand() && m_poMaskDS)
+ {
+ auto nMaskOffset =
+ TIFFGetStrileOffset(m_poMaskDS->m_hTIFF, nBlockId);
+ if (nMaskOffset)
+ {
+ nSize = nMaskOffset +
+ TIFFGetStrileByteCount(m_poMaskDS->m_hTIFF,
+ nBlockId) -
+ nOffset;
+ }
+ else
+ {
+ bTryMask = false;
+ }
+ }
+ if (nSize == 0)
+ {
+ nSize = TIFFGetStrileByteCount(m_hTIFF, nBlockId);
+ }
+ if (nSize && m_bTrailerRepeatedLast4BytesRepeated)
+ {
+ nSize += 4;
+ }
+ }
+ else
+ {
+ const auto nNextBlockId =
+ (m_bTileInterleave && nBand < nBands)
+ ? nBlockId + m_nBlocksPerBand
+ : (m_bTileInterleave && nBand == nBands)
+ ? nBlockId - (nBand - 1) * m_nBlocksPerBand + 1
+ : nBlockId + 1;
+ auto nOffsetNext = TIFFGetStrileOffset(m_hTIFF, nNextBlockId);
+ if (nOffsetNext > nOffset)
+ {
+ nSize = nOffsetNext - nOffset;
+ }
+ else
+ {
+ // Shouldn't happen for a compliant file
+ if (nOffsetNext != 0)
+ {
+ CPLDebug("GTiff", "Tile %d is not located after %d",
+ nNextBlockId, nBlockId);
+ }
+ bTryMask = false;
+ nSize = TIFFGetStrileByteCount(m_hTIFF, nBlockId);
+ if (m_bTrailerRepeatedLast4BytesRepeated)
+ nSize += 4;
+ }
+ }
+ if (nSize)
+ {
+ nOffset -= 4;
+ nSize += 4;
+ if (nTotalSize + nSize < nMaxRawBlockCacheSize)
+ {
+ StrileData data;
+ data.nOffset = nOffset;
+ data.nByteCount = nSize;
+ data.bTryMask = bTryMask;
+ oMapStrileToOffsetByteCount[nBlockId] = data;
+ }
+ }
+ }
+ else
+ {
+ // Sparse tile
+ StrileData data;
+ data.nOffset = 0;
+ data.nByteCount = 0;
+ data.bTryMask = false;
+ oMapStrileToOffsetByteCount[nBlockId] = data;
+ }
+ };
+
+ // This lambda fills m_poDS->m_oCacheStrileToOffsetByteCount (and
+ // m_poDS->m_poMaskDS->m_oCacheStrileToOffsetByteCount, when there is a
+ // mask) from the temporary oMapStrileToOffsetByteCount.
+ auto FillCacheStrileToOffsetByteCount =
+ [&](const std::vector &anOffsets,
+ const std::vector &anSizes,
+ const std::vector &apData)
+ {
+ CPLAssert(m_bLeaderSizeAsUInt4);
+ size_t i = 0;
+ vsi_l_offset nLastOffset = 0;
+ for (const auto &entry : oMapStrileToOffsetByteCount)
+ {
+ const auto nBlockId = entry.first;
+ const auto nOffset = entry.second.nOffset;
+ const auto nSize = entry.second.nByteCount;
+ if (nOffset == 0)
+ {
+ // Sparse tile
+ m_oCacheStrileToOffsetByteCount.insert(nBlockId,
+ std::pair(0, 0));
+ continue;
+ }
+
+ if (nOffset < nLastOffset)
+ {
+ // shouldn't happen normally if tiles are sorted
+ i = 0;
+ }
+ nLastOffset = nOffset;
+ while (i < anOffsets.size() &&
+ !(nOffset >= anOffsets[i] &&
+ nOffset + nSize <= anOffsets[i] + anSizes[i]))
+ {
+ i++;
+ }
+ CPLAssert(i < anOffsets.size());
+ CPLAssert(nOffset >= anOffsets[i]);
+ CPLAssert(nOffset + nSize <= anOffsets[i] + anSizes[i]);
+ GUInt32 nSizeFromLeader;
+ memcpy(&nSizeFromLeader,
+ // cppcheck-suppress containerOutOfBounds
+ static_cast(apData[i]) + nOffset - anOffsets[i],
+ sizeof(nSizeFromLeader));
+ CPL_LSBPTR32(&nSizeFromLeader);
+ bool bOK = true;
+ constexpr int nLeaderSize = 4;
+ const int nTrailerSize =
+ (m_bTrailerRepeatedLast4BytesRepeated ? 4 : 0);
+ if (nSizeFromLeader > nSize - nLeaderSize - nTrailerSize)
+ {
+ CPLDebug("GTiff",
+ "Inconsistent block size from in leader of block %d",
+ nBlockId);
+ bOK = false;
+ }
+ else if (m_bTrailerRepeatedLast4BytesRepeated)
+ {
+ // Check trailer consistency
+ const GByte *strileData = static_cast(apData[i]) +
+ nOffset - anOffsets[i] + nLeaderSize;
+ if (!CheckTrailer(strileData, nSizeFromLeader))
+ {
+ CPLDebug("GTiff", "Inconsistent trailer of block %d",
+ nBlockId);
+ bOK = false;
+ }
+ }
+ if (!bOK)
+ {
+ return false;
+ }
+
+ {
+ const vsi_l_offset nRealOffset = nOffset + nLeaderSize;
+ const vsi_l_offset nRealSize = nSizeFromLeader;
+#ifdef DEBUG_VERBOSE
+ CPLDebug("GTiff",
+ "Block %d found at offset " CPL_FRMT_GUIB
+ " with size " CPL_FRMT_GUIB,
+ nBlockId, nRealOffset, nRealSize);
+#endif
+ m_oCacheStrileToOffsetByteCount.insert(
+ nBlockId, std::pair(nRealOffset, nRealSize));
+ }
+
+ // Processing of mask
+ if (!(entry.second.bTryMask && m_bMaskInterleavedWithImagery &&
+ GetRasterBand(1)->GetMaskBand() && m_poMaskDS))
+ {
+ continue;
+ }
+
+ bOK = false;
+ const vsi_l_offset nMaskOffsetWithLeader =
+ nOffset + nLeaderSize + nSizeFromLeader + nTrailerSize;
+ if (nMaskOffsetWithLeader + nLeaderSize <=
+ anOffsets[i] + anSizes[i])
+ {
+ GUInt32 nMaskSizeFromLeader;
+ memcpy(&nMaskSizeFromLeader,
+ static_cast(apData[i]) + nMaskOffsetWithLeader -
+ anOffsets[i],
+ sizeof(nMaskSizeFromLeader));
+ CPL_LSBPTR32(&nMaskSizeFromLeader);
+ if (nMaskOffsetWithLeader + nLeaderSize + nMaskSizeFromLeader +
+ nTrailerSize <=
+ anOffsets[i] + anSizes[i])
+ {
+ bOK = true;
+ if (m_bTrailerRepeatedLast4BytesRepeated)
+ {
+ // Check trailer consistency
+ const GByte *strileMaskData =
+ static_cast(apData[i]) + nOffset -
+ anOffsets[i] + nLeaderSize + nSizeFromLeader +
+ nTrailerSize + nLeaderSize;
+ if (!CheckTrailer(strileMaskData, nMaskSizeFromLeader))
+ {
+ CPLDebug("GTiff",
+ "Inconsistent trailer of mask of block %d",
+ nBlockId);
+ bOK = false;
+ }
+ }
+ }
+ if (bOK)
+ {
+ const vsi_l_offset nRealOffset = nOffset + nLeaderSize +
+ nSizeFromLeader +
+ nTrailerSize + nLeaderSize;
+ const vsi_l_offset nRealSize = nMaskSizeFromLeader;
+#ifdef DEBUG_VERBOSE
+ CPLDebug("GTiff",
+ "Mask of block %d found at offset " CPL_FRMT_GUIB
+ " with size " CPL_FRMT_GUIB,
+ nBlockId, nRealOffset, nRealSize);
+#endif
+
+ m_poMaskDS->m_oCacheStrileToOffsetByteCount.insert(
+ nBlockId, std::pair(nRealOffset, nRealSize));
+ }
+ }
+ if (!bOK)
+ {
+ CPLDebug("GTiff",
+ "Mask for block %d is not properly interleaved with "
+ "imagery block",
+ nBlockId);
+ }
+ }
+ return true;
+ };
+
+ thandle_t th = TIFFClientdata(m_hTIFF);
+ if (!VSI_TIFFHasCachedRanges(th))
+ {
+ std::vector> aOffsetSize;
+ size_t nTotalSize = 0;
+ const unsigned int nMaxRawBlockCacheSize = atoi(
+ CPLGetConfigOption("GDAL_MAX_RAW_BLOCK_CACHE_SIZE", "10485760"));
+ bool bGoOn = true;
+ for (int iBand = 0; iBand < nBandCount; ++iBand)
+ {
+ const int nBand = panBandMap[iBand];
+ GTiffRasterBand *poBand =
+ cpl::down_cast(papoBands[nBand - 1]);
+ for (int iY = nBlockY1; bGoOn && iY <= nBlockY2; iY++)
+ {
+ for (int iX = nBlockX1; bGoOn && iX <= nBlockX2; iX++)
+ {
+ GDALRasterBlock *poBlock =
+ poBand->TryGetLockedBlockRef(iX, iY);
+ if (poBlock != nullptr)
+ {
+ poBlock->DropLock();
+ continue;
+ }
+ int nBlockId = iX + iY * m_nBlocksPerRow;
+ if (m_nPlanarConfig == PLANARCONFIG_SEPARATE)
+ nBlockId += (nBand - 1) * m_nBlocksPerBand;
+ vsi_l_offset nOffset = 0;
+ vsi_l_offset nSize = 0;
+
+ if (!m_bStreamingIn && m_bBlockOrderRowMajor &&
+ m_bLeaderSizeAsUInt4)
+ {
+ OptimizedRetrievalOfOffsetSize(nBand, nBlockId, nOffset,
+ nSize, nTotalSize,
+ nMaxRawBlockCacheSize);
+ }
+ else
+ {
+ CPL_IGNORE_RET_VAL(
+ IsBlockAvailable(nBlockId, &nOffset, &nSize));
+ }
+ if (nSize)
+ {
+ if (nTotalSize + nSize < nMaxRawBlockCacheSize)
+ {
+#ifdef DEBUG_VERBOSE
+ CPLDebug(
+ "GTiff",
+ "Precaching for block (%d, %d), " CPL_FRMT_GUIB
+ "-" CPL_FRMT_GUIB,
+ iX, iY, nOffset,
+ nOffset + static_cast(nSize) - 1);
+#endif
+ aOffsetSize.push_back(
+ std::pair(nOffset, static_cast(nSize)));
+ nTotalSize += static_cast(nSize);
+ }
+ else
+ {
+ bGoOn = false;
+ }
+ }
+ }
+ }
+ }
+
+ std::sort(aOffsetSize.begin(), aOffsetSize.end());
+
+ if (nTotalSize > 0)
+ {
+ pBufferedData = VSI_MALLOC_VERBOSE(nTotalSize);
+ if (pBufferedData)
+ {
+ std::vector anOffsets;
+ std::vector anSizes;
+ std::vector apData;
+ anOffsets.push_back(aOffsetSize[0].first);
+ apData.push_back(static_cast(pBufferedData));
+ size_t nChunkSize = aOffsetSize[0].second;
+ size_t nAccOffset = 0;
+ // Try to merge contiguous or slightly overlapping ranges
+ for (size_t i = 0; i < aOffsetSize.size() - 1; i++)
+ {
+ if (aOffsetSize[i].first < aOffsetSize[i + 1].first &&
+ aOffsetSize[i].first + aOffsetSize[i].second >=
+ aOffsetSize[i + 1].first)
+ {
+ const auto overlap = aOffsetSize[i].first +
+ aOffsetSize[i].second -
+ aOffsetSize[i + 1].first;
+ // That should always be the case for well behaved
+ // TIFF files.
+ if (aOffsetSize[i + 1].second > overlap)
+ {
+ nChunkSize += static_cast(
+ aOffsetSize[i + 1].second - overlap);
+ }
+ }
+ else
+ {
+ // terminate current block
+ anSizes.push_back(nChunkSize);
+#ifdef DEBUG_VERBOSE
+ CPLDebug("GTiff",
+ "Requesting range [" CPL_FRMT_GUIB
+ "-" CPL_FRMT_GUIB "]",
+ anOffsets.back(),
+ anOffsets.back() + anSizes.back() - 1);
+#endif
+ nAccOffset += nChunkSize;
+ // start a new range
+ anOffsets.push_back(aOffsetSize[i + 1].first);
+ apData.push_back(static_cast(pBufferedData) +
+ nAccOffset);
+ nChunkSize = aOffsetSize[i + 1].second;
+ }
+ }
+ // terminate last block
+ anSizes.push_back(nChunkSize);
+#ifdef DEBUG_VERBOSE
+ CPLDebug(
+ "GTiff",
+ "Requesting range [" CPL_FRMT_GUIB "-" CPL_FRMT_GUIB "]",
+ anOffsets.back(), anOffsets.back() + anSizes.back() - 1);
+#endif
+
+ VSILFILE *fp = VSI_TIFFGetVSILFile(th);
+
+ if (VSIFReadMultiRangeL(static_cast(anSizes.size()),
+ &apData[0], &anOffsets[0], &anSizes[0],
+ fp) == 0)
+ {
+ if (!oMapStrileToOffsetByteCount.empty() &&
+ !FillCacheStrileToOffsetByteCount(anOffsets, anSizes,
+ apData))
+ {
+ // Retry without optimization
+ CPLFree(pBufferedData);
+ m_bLeaderSizeAsUInt4 = false;
+ void *pRet = CacheMultiRange(
+ nXOff, nYOff, nXSize, nYSize, nBufXSize, nBufYSize,
+ panBandMap, nBandCount, psExtraArg);
+ m_bLeaderSizeAsUInt4 = true;
+ return pRet;
+ }
+
+ VSI_TIFFSetCachedRanges(
+ th, static_cast(anSizes.size()), &apData[0],
+ &anOffsets[0], &anSizes[0]);
+ }
+ }
+ }
+ }
+ return pBufferedData;
+}
diff --git a/frmts/gtiff/gtiffdataset_write.cpp b/frmts/gtiff/gtiffdataset_write.cpp
index 095b9d19a9e3..06f8431a2c49 100644
--- a/frmts/gtiff/gtiffdataset_write.cpp
+++ b/frmts/gtiff/gtiffdataset_write.cpp
@@ -598,7 +598,6 @@ void GTiffDataset::WriteDealWithLercAndNan(T *pBuffer, int nActualBlockWidth,
bool GTiffDataset::WriteEncodedTile(uint32_t tile, GByte *pabyData,
int bPreserveDataBuffer)
{
-
const int iColumn = (tile % m_nBlocksPerBand) % m_nBlocksPerRow;
const int iRow = (tile % m_nBlocksPerBand) / m_nBlocksPerRow;
@@ -4281,6 +4280,14 @@ bool GTiffDataset::WriteMetadata(GDALDataset *poSrcDS, TIFF *l_hTIFF,
if (CPLTestBool(
CPLGetConfigOption("GTIFF_WRITE_IMAGE_STRUCTURE_METADATA", "YES")))
{
+ const char *pszInterleave =
+ CSLFetchNameValue(papszCreationOptions, "INTERLEAVE");
+ if (pszInterleave && EQUAL(pszInterleave, "TILE"))
+ {
+ AppendMetadataItem(&psRoot, &psTail, "INTERLEAVE", "TILE", 0,
+ nullptr, "IMAGE_STRUCTURE");
+ }
+
const char *pszCompress =
CSLFetchNameValue(papszCreationOptions, "COMPRESS");
if (pszCompress && EQUAL(pszCompress, "WEBP"))
@@ -5034,9 +5041,12 @@ TIFF *GTiffDataset::CreateLL(const char *pszFilename, int nXSize, int nYSize,
int l_nBands, GDALDataType eType,
double dfExtraSpaceForOverviews,
int nColorTableMultiplier, char **papszParamList,
- VSILFILE **pfpL, CPLString &l_osTmpFilename)
+ VSILFILE **pfpL, CPLString &l_osTmpFilename,
+ bool bCreateCopy, bool &bTileInterleavingOut)
{
+ bTileInterleavingOut = false;
+
GTiffOneTimeInit();
/* -------------------------------------------------------------------- */
@@ -5116,11 +5126,30 @@ TIFF *GTiffDataset::CreateLL(const char *pszFilename, int nXSize, int nYSize,
{
nPlanar = PLANARCONFIG_SEPARATE;
}
+ else if (EQUAL(pszValue, "BAND"))
+ {
+ nPlanar = PLANARCONFIG_SEPARATE;
+ }
+ else if (EQUAL(pszValue, "TILE"))
+ {
+ bTileInterleavingOut = true;
+ nPlanar = PLANARCONFIG_SEPARATE;
+ if (!bCreateCopy)
+ {
+ CPLError(CE_Failure, CPLE_NotSupported,
+ "INTERLEAVE=TILE is only supported for CreateCopy(), "
+ "not Create()");
+ return nullptr;
+ }
+ }
else
{
ReportError(
pszFilename, CE_Failure, CPLE_IllegalArg,
- "INTERLEAVE=%s unsupported, value must be PIXEL or BAND.",
+ bCreateCopy
+ ? "INTERLEAVE=%s unsupported, value must be PIXEL, BAND or "
+ "TILE."
+ : "INTERLEAVE=%s unsupported, value must be PIXEL or BAND.",
pszValue);
return nullptr;
}
@@ -6169,10 +6198,11 @@ int GTiffDataset::GuessJPEGQuality(bool &bOutHasQuantizationTable,
CPLPushErrorHandler(CPLQuietErrorHandler);
CPLString osTmp;
- TIFF *hTIFFTmp =
- CreateLL(osTmpFilenameIn, 16, 16, (nBands <= 4) ? nBands : 1,
- GetRasterBand(1)->GetRasterDataType(), 0.0, 0,
- papszLocalParameters, &fpTmp, osTmp);
+ bool bTileInterleaving;
+ TIFF *hTIFFTmp = CreateLL(
+ osTmpFilenameIn, 16, 16, (nBands <= 4) ? nBands : 1,
+ GetRasterBand(1)->GetRasterDataType(), 0.0, 0, papszLocalParameters,
+ &fpTmp, osTmp, /* bCreateCopy=*/false, bTileInterleaving);
CPLPopErrorHandler();
if (!hTIFFTmp)
{
@@ -6331,9 +6361,11 @@ GDALDataset *GTiffDataset::Create(const char *pszFilename, int nXSize,
/* -------------------------------------------------------------------- */
/* Create the underlying TIFF file. */
/* -------------------------------------------------------------------- */
- TIFF *l_hTIFF = CreateLL(pszFilename, nXSize, nYSize, l_nBands, eType, 0,
- nColorTableMultiplier, papszParamList, &l_fpL,
- l_osTmpFilename);
+ bool bTileInterleaving;
+ TIFF *l_hTIFF =
+ CreateLL(pszFilename, nXSize, nYSize, l_nBands, eType, 0,
+ nColorTableMultiplier, papszParamList, &l_fpL, l_osTmpFilename,
+ /* bCreateCopy=*/false, bTileInterleaving);
const bool bStreaming = !l_osTmpFilename.empty();
if (l_hTIFF == nullptr)
@@ -6565,19 +6597,15 @@ CPLErr GTiffDataset::CopyImageryAndMask(GTiffDataset *poDstDS,
const auto eType = poDstDS->GetRasterBand(1)->GetRasterDataType();
const int nDataTypeSize = GDALGetDataTypeSizeBytes(eType);
const int l_nBands = poDstDS->GetRasterCount();
- void *pBlockBuffer =
+ GByte *pBlockBuffer = static_cast(
VSI_MALLOC3_VERBOSE(poDstDS->m_nBlockXSize, poDstDS->m_nBlockYSize,
- cpl::fits_on(l_nBands * nDataTypeSize));
+ cpl::fits_on(l_nBands * nDataTypeSize)));
if (pBlockBuffer == nullptr)
{
eErr = CE_Failure;
}
const int nYSize = poDstDS->nRasterYSize;
const int nXSize = poDstDS->nRasterXSize;
- const int nBlocks = poDstDS->m_nBlocksPerBand;
-
- CPLAssert(l_nBands == 1 || poDstDS->m_nPlanarConfig == PLANARCONFIG_CONTIG);
-
const bool bIsOddBand =
dynamic_cast(poDstDS->GetRasterBand(1)) != nullptr;
@@ -6587,134 +6615,276 @@ CPLErr GTiffDataset::CopyImageryAndMask(GTiffDataset *poDstDS,
CPLAssert(poDstDS->m_poMaskDS->m_nBlockYSize == poDstDS->m_nBlockYSize);
}
- int iBlock = 0;
- for (int iY = 0, nYBlock = 0; iY < nYSize && eErr == CE_None;
- iY = ((nYSize - iY < poDstDS->m_nBlockYSize)
- ? nYSize
- : iY + poDstDS->m_nBlockYSize),
- nYBlock++)
+ if (poDstDS->m_nPlanarConfig == PLANARCONFIG_SEPARATE &&
+ !poDstDS->m_bTileInterleave)
{
- const int nReqYSize = std::min(nYSize - iY, poDstDS->m_nBlockYSize);
- for (int iX = 0, nXBlock = 0; iX < nXSize && eErr == CE_None;
- iX = ((nXSize - iX < poDstDS->m_nBlockXSize)
- ? nXSize
- : iX + poDstDS->m_nBlockXSize),
- nXBlock++)
+ int iBlock = 0;
+ const int nBlocks = poDstDS->m_nBlocksPerBand *
+ (l_nBands + (poDstDS->m_poMaskDS ? 1 : 0));
+ for (int i = 0; eErr == CE_None && i < l_nBands; i++)
{
- const int nReqXSize = std::min(nXSize - iX, poDstDS->m_nBlockXSize);
- if (nReqXSize < poDstDS->m_nBlockXSize ||
- nReqYSize < poDstDS->m_nBlockYSize)
+ for (int iY = 0, nYBlock = 0; iY < nYSize && eErr == CE_None;
+ iY = ((nYSize - iY < poDstDS->m_nBlockYSize)
+ ? nYSize
+ : iY + poDstDS->m_nBlockYSize),
+ nYBlock++)
{
- memset(pBlockBuffer, 0,
- static_cast(poDstDS->m_nBlockXSize) *
- poDstDS->m_nBlockYSize * l_nBands * nDataTypeSize);
- }
-
- if (!bIsOddBand)
- {
- eErr = poSrcDS->RasterIO(
- GF_Read, iX, iY, nReqXSize, nReqYSize, pBlockBuffer,
- nReqXSize, nReqYSize, eType, l_nBands, nullptr,
- static_cast(nDataTypeSize) * l_nBands,
- static_cast(nDataTypeSize) * l_nBands *
- poDstDS->m_nBlockXSize,
- nDataTypeSize, nullptr);
- if (eErr == CE_None)
+ const int nReqYSize =
+ std::min(nYSize - iY, poDstDS->m_nBlockYSize);
+ for (int iX = 0, nXBlock = 0; iX < nXSize && eErr == CE_None;
+ iX = ((nXSize - iX < poDstDS->m_nBlockXSize)
+ ? nXSize
+ : iX + poDstDS->m_nBlockXSize),
+ nXBlock++)
{
- eErr = poDstDS->WriteEncodedTileOrStrip(
- iBlock, pBlockBuffer, false);
+ const int nReqXSize =
+ std::min(nXSize - iX, poDstDS->m_nBlockXSize);
+ if (nReqXSize < poDstDS->m_nBlockXSize ||
+ nReqYSize < poDstDS->m_nBlockYSize)
+ {
+ memset(pBlockBuffer, 0,
+ static_cast(poDstDS->m_nBlockXSize) *
+ poDstDS->m_nBlockYSize * nDataTypeSize);
+ }
+ eErr = poSrcDS->GetRasterBand(i + 1)->RasterIO(
+ GF_Read, iX, iY, nReqXSize, nReqYSize, pBlockBuffer,
+ nReqXSize, nReqYSize, eType, nDataTypeSize,
+ static_cast(nDataTypeSize) *
+ poDstDS->m_nBlockXSize,
+ nullptr);
+ if (eErr == CE_None)
+ {
+ eErr = poDstDS->WriteEncodedTileOrStrip(
+ iBlock, pBlockBuffer, false);
+ }
+
+ iBlock++;
+ if (pfnProgress &&
+ !pfnProgress(static_cast(iBlock) / nBlocks,
+ nullptr, pProgressData))
+ {
+ eErr = CE_Failure;
+ }
+
+ if (poDstDS->m_bWriteError)
+ eErr = CE_Failure;
}
}
- else
+ }
+ if (poDstDS->m_poMaskDS && eErr == CE_None)
+ {
+ int iBlockMask = 0;
+ for (int iY = 0, nYBlock = 0; iY < nYSize && eErr == CE_None;
+ iY = ((nYSize - iY < poDstDS->m_nBlockYSize)
+ ? nYSize
+ : iY + poDstDS->m_nBlockYSize),
+ nYBlock++)
{
- // In the odd bit case, this is a bit messy to ensure
- // the strile gets written synchronously.
- // We load the content of the n-1 bands in the cache,
- // and for the last band we invoke WriteBlock() directly
- // We also force FlushBlockBuf()
- std::vector apoLockedBlocks;
- for (int i = 0; eErr == CE_None && i < l_nBands - 1; i++)
+ const int nReqYSize =
+ std::min(nYSize - iY, poDstDS->m_nBlockYSize);
+ for (int iX = 0, nXBlock = 0; iX < nXSize && eErr == CE_None;
+ iX = ((nXSize - iX < poDstDS->m_nBlockXSize)
+ ? nXSize
+ : iX + poDstDS->m_nBlockXSize),
+ nXBlock++)
{
- auto poBlock =
- poDstDS->GetRasterBand(i + 1)->GetLockedBlockRef(
- nXBlock, nYBlock, TRUE);
- if (poBlock)
+ const int nReqXSize =
+ std::min(nXSize - iX, poDstDS->m_nBlockXSize);
+ if (nReqXSize < poDstDS->m_nBlockXSize ||
+ nReqYSize < poDstDS->m_nBlockYSize)
{
- eErr = poSrcDS->GetRasterBand(i + 1)->RasterIO(
- GF_Read, iX, iY, nReqXSize, nReqYSize,
- poBlock->GetDataRef(), nReqXSize, nReqYSize, eType,
- nDataTypeSize,
- static_cast(nDataTypeSize) *
- poDstDS->m_nBlockXSize,
- nullptr);
- poBlock->MarkDirty();
- apoLockedBlocks.emplace_back(poBlock);
+ memset(pBlockBuffer, 0,
+ static_cast(poDstDS->m_nBlockXSize) *
+ poDstDS->m_nBlockYSize * nDataTypeSize);
}
- else
+ eErr = poSrcMaskBand->RasterIO(
+ GF_Read, iX, iY, nReqXSize, nReqYSize, pBlockBuffer,
+ nReqXSize, nReqYSize, eType, nDataTypeSize,
+ static_cast(nDataTypeSize) *
+ poDstDS->m_nBlockXSize,
+ nullptr);
+ if (eErr == CE_None)
+ {
+ eErr = poDstDS->m_poMaskDS->WriteEncodedTileOrStrip(
+ iBlockMask, pBlockBuffer, false);
+ }
+
+ iBlockMask++;
+ if (pfnProgress &&
+ !pfnProgress(static_cast(iBlock + iBlockMask) /
+ nBlocks,
+ nullptr, pProgressData))
{
eErr = CE_Failure;
}
+
+ if (poDstDS->m_poMaskDS->m_bWriteError)
+ eErr = CE_Failure;
+ }
+ }
+ }
+ }
+ else
+ {
+ int iBlock = 0;
+ const int nBlocks = poDstDS->m_nBlocksPerBand;
+ for (int iY = 0, nYBlock = 0; iY < nYSize && eErr == CE_None;
+ iY = ((nYSize - iY < poDstDS->m_nBlockYSize)
+ ? nYSize
+ : iY + poDstDS->m_nBlockYSize),
+ nYBlock++)
+ {
+ const int nReqYSize = std::min(nYSize - iY, poDstDS->m_nBlockYSize);
+ for (int iX = 0, nXBlock = 0; iX < nXSize && eErr == CE_None;
+ iX = ((nXSize - iX < poDstDS->m_nBlockXSize)
+ ? nXSize
+ : iX + poDstDS->m_nBlockXSize),
+ nXBlock++)
+ {
+ const int nReqXSize =
+ std::min(nXSize - iX, poDstDS->m_nBlockXSize);
+ if (nReqXSize < poDstDS->m_nBlockXSize ||
+ nReqYSize < poDstDS->m_nBlockYSize)
+ {
+ memset(pBlockBuffer, 0,
+ static_cast(poDstDS->m_nBlockXSize) *
+ poDstDS->m_nBlockYSize * l_nBands *
+ nDataTypeSize);
}
- if (eErr == CE_None)
+
+ if (poDstDS->m_bTileInterleave)
{
- eErr = poSrcDS->GetRasterBand(l_nBands)->RasterIO(
+ eErr = poSrcDS->RasterIO(
GF_Read, iX, iY, nReqXSize, nReqYSize, pBlockBuffer,
- nReqXSize, nReqYSize, eType, nDataTypeSize,
+ nReqXSize, nReqYSize, eType, l_nBands, nullptr,
+ nDataTypeSize,
static_cast(nDataTypeSize) *
poDstDS->m_nBlockXSize,
+ static_cast(nDataTypeSize) *
+ poDstDS->m_nBlockXSize * poDstDS->m_nBlockYSize,
nullptr);
+ if (eErr == CE_None)
+ {
+ for (int i = 0; eErr == CE_None && i < l_nBands; i++)
+ {
+ eErr = poDstDS->WriteEncodedTileOrStrip(
+ iBlock + i * poDstDS->m_nBlocksPerBand,
+ pBlockBuffer + static_cast(i) *
+ poDstDS->m_nBlockXSize *
+ poDstDS->m_nBlockYSize *
+ nDataTypeSize,
+ false);
+ }
+ }
}
- if (eErr == CE_None)
+ else if (!bIsOddBand)
{
- // Avoid any attempt to load from disk
- poDstDS->m_nLoadedBlock = iBlock;
- eErr = poDstDS->GetRasterBand(l_nBands)->WriteBlock(
- nXBlock, nYBlock, pBlockBuffer);
+ eErr = poSrcDS->RasterIO(
+ GF_Read, iX, iY, nReqXSize, nReqYSize, pBlockBuffer,
+ nReqXSize, nReqYSize, eType, l_nBands, nullptr,
+ static_cast(nDataTypeSize) * l_nBands,
+ static_cast(nDataTypeSize) * l_nBands *
+ poDstDS->m_nBlockXSize,
+ nDataTypeSize, nullptr);
if (eErr == CE_None)
- eErr = poDstDS->FlushBlockBuf();
+ {
+ eErr = poDstDS->WriteEncodedTileOrStrip(
+ iBlock, pBlockBuffer, false);
+ }
}
- for (auto poBlock : apoLockedBlocks)
+ else
{
- poBlock->MarkClean();
- poBlock->DropLock();
+ // In the odd bit case, this is a bit messy to ensure
+ // the strile gets written synchronously.
+ // We load the content of the n-1 bands in the cache,
+ // and for the last band we invoke WriteBlock() directly
+ // We also force FlushBlockBuf()
+ std::vector apoLockedBlocks;
+ for (int i = 0; eErr == CE_None && i < l_nBands - 1; i++)
+ {
+ auto poBlock =
+ poDstDS->GetRasterBand(i + 1)->GetLockedBlockRef(
+ nXBlock, nYBlock, TRUE);
+ if (poBlock)
+ {
+ eErr = poSrcDS->GetRasterBand(i + 1)->RasterIO(
+ GF_Read, iX, iY, nReqXSize, nReqYSize,
+ poBlock->GetDataRef(), nReqXSize, nReqYSize,
+ eType, nDataTypeSize,
+ static_cast(nDataTypeSize) *
+ poDstDS->m_nBlockXSize,
+ nullptr);
+ poBlock->MarkDirty();
+ apoLockedBlocks.emplace_back(poBlock);
+ }
+ else
+ {
+ eErr = CE_Failure;
+ }
+ }
+ if (eErr == CE_None)
+ {
+ eErr = poSrcDS->GetRasterBand(l_nBands)->RasterIO(
+ GF_Read, iX, iY, nReqXSize, nReqYSize, pBlockBuffer,
+ nReqXSize, nReqYSize, eType, nDataTypeSize,
+ static_cast(nDataTypeSize) *
+ poDstDS->m_nBlockXSize,
+ nullptr);
+ }
+ if (eErr == CE_None)
+ {
+ // Avoid any attempt to load from disk
+ poDstDS->m_nLoadedBlock = iBlock;
+ eErr = poDstDS->GetRasterBand(l_nBands)->WriteBlock(
+ nXBlock, nYBlock, pBlockBuffer);
+ if (eErr == CE_None)
+ eErr = poDstDS->FlushBlockBuf();
+ }
+ for (auto poBlock : apoLockedBlocks)
+ {
+ poBlock->MarkClean();
+ poBlock->DropLock();
+ }
}
- }
- if (eErr == CE_None && poDstDS->m_poMaskDS)
- {
- if (nReqXSize < poDstDS->m_nBlockXSize ||
- nReqYSize < poDstDS->m_nBlockYSize)
+ if (eErr == CE_None && poDstDS->m_poMaskDS)
{
- memset(pBlockBuffer, 0,
- static_cast(poDstDS->m_nBlockXSize) *
- poDstDS->m_nBlockYSize);
- }
- eErr = poSrcMaskBand->RasterIO(
- GF_Read, iX, iY, nReqXSize, nReqYSize, pBlockBuffer,
- nReqXSize, nReqYSize, GDT_Byte, 1, poDstDS->m_nBlockXSize,
- nullptr);
- if (eErr == CE_None)
- {
- // Avoid any attempt to load from disk
- poDstDS->m_poMaskDS->m_nLoadedBlock = iBlock;
- eErr = poDstDS->m_poMaskDS->GetRasterBand(1)->WriteBlock(
- nXBlock, nYBlock, pBlockBuffer);
+ if (nReqXSize < poDstDS->m_nBlockXSize ||
+ nReqYSize < poDstDS->m_nBlockYSize)
+ {
+ memset(pBlockBuffer, 0,
+ static_cast(poDstDS->m_nBlockXSize) *
+ poDstDS->m_nBlockYSize);
+ }
+ eErr = poSrcMaskBand->RasterIO(
+ GF_Read, iX, iY, nReqXSize, nReqYSize, pBlockBuffer,
+ nReqXSize, nReqYSize, GDT_Byte, 1,
+ poDstDS->m_nBlockXSize, nullptr);
if (eErr == CE_None)
- eErr = poDstDS->m_poMaskDS->FlushBlockBuf();
+ {
+ // Avoid any attempt to load from disk
+ poDstDS->m_poMaskDS->m_nLoadedBlock = iBlock;
+ eErr =
+ poDstDS->m_poMaskDS->GetRasterBand(1)->WriteBlock(
+ nXBlock, nYBlock, pBlockBuffer);
+ if (eErr == CE_None)
+ eErr = poDstDS->m_poMaskDS->FlushBlockBuf();
+ }
}
- }
- if (poDstDS->m_bWriteError)
- eErr = CE_Failure;
+ if (poDstDS->m_bWriteError)
+ eErr = CE_Failure;
- iBlock++;
- if (pfnProgress &&
- !pfnProgress(static_cast(iBlock) / nBlocks, nullptr,
- pProgressData))
- {
- eErr = CE_Failure;
+ iBlock++;
+ if (pfnProgress &&
+ !pfnProgress(static_cast(iBlock) / nBlocks, nullptr,
+ pProgressData))
+ {
+ eErr = CE_Failure;
+ }
}
}
}
+
poDstDS->FlushCache(false); // mostly to wait for thread completion
VSIFree(pBlockBuffer);
@@ -7037,9 +7207,11 @@ GDALDataset *GTiffDataset::CreateCopy(const char *pszFilename,
papszOptions, "COLOR_TABLE_MULTIPLIER",
CPLSPrintf("%d", DEFAULT_COLOR_TABLE_MULTIPLIER_257)))));
+ bool bTileInterleaving = false;
TIFF *l_hTIFF = CreateLL(pszFilename, nXSize, nYSize, l_nBands, eType,
dfExtraSpaceForOverviews, nColorTableMultiplier,
- papszCreateOptions, &l_fpL, l_osTmpFilename);
+ papszCreateOptions, &l_fpL, l_osTmpFilename,
+ /* bCreateCopy = */ true, bTileInterleaving);
const bool bStreaming = !l_osTmpFilename.empty();
CSLDestroy(papszCreateOptions);
@@ -7251,13 +7423,20 @@ GDALDataset *GTiffDataset::CreateCopy(const char *pszFilename,
const int nMaskFlags = poSrcDS->GetRasterBand(1)->GetMaskFlags();
bool bCreateMask = false;
CPLString osHiddenStructuralMD;
- if ((l_nBands == 1 || l_nPlanarConfig == PLANARCONFIG_CONTIG) &&
- bCopySrcOverviews)
+ const char *pszInterleave =
+ CSLFetchNameValueDef(papszOptions, "INTERLEAVE", "PIXEL");
+ if (bCopySrcOverviews)
{
osHiddenStructuralMD += "LAYOUT=IFDS_BEFORE_DATA\n";
osHiddenStructuralMD += "BLOCK_ORDER=ROW_MAJOR\n";
osHiddenStructuralMD += "BLOCK_LEADER=SIZE_AS_UINT4\n";
osHiddenStructuralMD += "BLOCK_TRAILER=LAST_4_BYTES_REPEATED\n";
+ if (l_nBands > 1 && !EQUAL(pszInterleave, "PIXEL"))
+ {
+ osHiddenStructuralMD += "INTERLEAVE=";
+ osHiddenStructuralMD += CPLString(pszInterleave).toupper();
+ osHiddenStructuralMD += "\n";
+ }
osHiddenStructuralMD +=
"KNOWN_INCOMPATIBLE_EDITION=NO\n "; // Final space intended, so
// this can be replaced by YES
@@ -7267,7 +7446,7 @@ GDALDataset *GTiffDataset::CreateCopy(const char *pszFilename,
{
bCreateMask = true;
if (GTiffDataset::MustCreateInternalMask() &&
- !osHiddenStructuralMD.empty())
+ !osHiddenStructuralMD.empty() && EQUAL(pszInterleave, "PIXEL"))
{
osHiddenStructuralMD += "MASK_INTERLEAVED_WITH_IMAGERY=YES\n";
}
@@ -7620,6 +7799,13 @@ GDALDataset *GTiffDataset::CreateCopy(const char *pszFilename,
poDS->m_fpL = l_fpL;
poDS->m_bIMDRPCMetadataLoaded = true;
poDS->m_nColorTableMultiplier = nColorTableMultiplier;
+ poDS->m_bTileInterleave = bTileInterleaving;
+
+ if (bTileInterleaving)
+ {
+ poDS->m_oGTiffMDMD.SetMetadataItem("INTERLEAVE", "TILE",
+ "IMAGE_STRUCTURE");
+ }
const bool bAppend = CPLFetchBool(papszOptions, "APPEND_SUBDATASET", false);
if (poDS->OpenOffset(l_hTIFF,
@@ -8052,10 +8238,6 @@ GDALDataset *GTiffDataset::CreateCopy(const char *pszFilename,
}
}
- char *papszCopyWholeRasterOptions[2] = {nullptr, nullptr};
- if (l_nCompression != COMPRESSION_NONE)
- papszCopyWholeRasterOptions[0] =
- const_cast("COMPRESSED=YES");
// Now copy the imagery.
// Begin with the smallest overview.
for (int iOvrLevel = nSrcOverviews - 1;
@@ -8091,6 +8273,7 @@ GDALDataset *GTiffDataset::CreateCopy(const char *pszFilename,
poDstDS->m_bFillEmptyTilesAtClosing =
poDS->m_bFillEmptyTilesAtClosing;
poDstDS->m_bWriteEmptyTiles = poDS->m_bWriteEmptyTiles;
+ poDstDS->m_bTileInterleave = poDS->m_bTileInterleave;
GDALRasterBand *poSrcMaskBand = nullptr;
if (poDstDS->m_poMaskDS)
{
@@ -8112,68 +8295,22 @@ GDALDataset *GTiffDataset::CreateCopy(const char *pszFilename,
: poSrcOvrBand->GetMaskBand();
}
- if (l_nBands == 1 ||
- poDstDS->m_nPlanarConfig == PLANARCONFIG_CONTIG)
+ if (poDstDS->m_poMaskDS)
{
- if (poDstDS->m_poMaskDS)
- {
- dfNextCurPixels +=
- static_cast(poSrcOvrBand->GetXSize()) *
- poSrcOvrBand->GetYSize();
- }
- void *pScaledData = GDALCreateScaledProgress(
- dfCurPixels / dfTotalPixels,
- dfNextCurPixels / dfTotalPixels, pfnProgress,
- pProgressData);
-
- eErr =
- CopyImageryAndMask(poDstDS, poSrcOvrDS, poSrcMaskBand,
- GDALScaledProgress, pScaledData);
-
- dfCurPixels = dfNextCurPixels;
- GDALDestroyScaledProgress(pScaledData);
+ dfNextCurPixels +=
+ static_cast(poSrcOvrBand->GetXSize()) *
+ poSrcOvrBand->GetYSize();
}
- else
- {
- void *pScaledData = GDALCreateScaledProgress(
- dfCurPixels / dfTotalPixels,
- dfNextCurPixels / dfTotalPixels, pfnProgress,
- pProgressData);
-
- eErr = GDALDatasetCopyWholeRaster(
- GDALDataset::ToHandle(poSrcOvrDS),
- GDALDataset::ToHandle(poDstDS),
- papszCopyWholeRasterOptions, GDALScaledProgress,
- pScaledData);
-
- dfCurPixels = dfNextCurPixels;
- GDALDestroyScaledProgress(pScaledData);
+ void *pScaledData =
+ GDALCreateScaledProgress(dfCurPixels / dfTotalPixels,
+ dfNextCurPixels / dfTotalPixels,
+ pfnProgress, pProgressData);
- poDstDS->FlushCache(false);
+ eErr = CopyImageryAndMask(poDstDS, poSrcOvrDS, poSrcMaskBand,
+ GDALScaledProgress, pScaledData);
- // Copy mask of the overview.
- if (eErr == CE_None &&
- (poMaskOvrDS ||
- poSrcOvrBand->GetMaskFlags() == GMF_PER_DATASET) &&
- poDstDS->m_poMaskDS != nullptr)
- {
- dfNextCurPixels +=
- static_cast(poSrcOvrBand->GetXSize()) *
- poSrcOvrBand->GetYSize();
- pScaledData = GDALCreateScaledProgress(
- dfCurPixels / dfTotalPixels,
- dfNextCurPixels / dfTotalPixels, pfnProgress,
- pProgressData);
- eErr = GDALRasterBandCopyWholeRaster(
- poSrcMaskBand,
- poDstDS->m_poMaskDS->GetRasterBand(1),
- papszCopyWholeRasterOptions, GDALScaledProgress,
- pScaledData);
- dfCurPixels = dfNextCurPixels;
- GDALDestroyScaledProgress(pScaledData);
- poDstDS->m_poMaskDS->FlushCache(false);
- }
- }
+ dfCurPixels = dfNextCurPixels;
+ GDALDestroyScaledProgress(pScaledData);
if (poSrcOvrDS != poOvrDS.get())
delete poSrcOvrDS;
@@ -8328,6 +8465,7 @@ GDALDataset *GTiffDataset::CreateCopy(const char *pszFilename,
{
papszCopyWholeRasterOptions[iNextOption++] = "COMPRESSED=YES";
}
+
// For streaming with separate, we really want that bands are written
// after each other, even if the source is pixel interleaved.
else if (bStreaming && poDS->m_nPlanarConfig == PLANARCONFIG_SEPARATE)
@@ -8335,21 +8473,17 @@ GDALDataset *GTiffDataset::CreateCopy(const char *pszFilename,
papszCopyWholeRasterOptions[iNextOption++] = "INTERLEAVE=BAND";
}
- if (bCopySrcOverviews &&
- (l_nBands == 1 || poDS->m_nPlanarConfig == PLANARCONFIG_CONTIG))
+ if (bCopySrcOverviews || bTileInterleaving)
{
poDS->m_bBlockOrderRowMajor = true;
- poDS->m_bLeaderSizeAsUInt4 = true;
- poDS->m_bTrailerRepeatedLast4BytesRepeated = true;
+ poDS->m_bLeaderSizeAsUInt4 = bCopySrcOverviews;
+ poDS->m_bTrailerRepeatedLast4BytesRepeated = bCopySrcOverviews;
if (poDS->m_poMaskDS)
{
poDS->m_poMaskDS->m_bBlockOrderRowMajor = true;
- poDS->m_poMaskDS->m_bLeaderSizeAsUInt4 = true;
- poDS->m_poMaskDS->m_bTrailerRepeatedLast4BytesRepeated = true;
- }
-
- if (poDS->m_poMaskDS)
- {
+ poDS->m_poMaskDS->m_bLeaderSizeAsUInt4 = bCopySrcOverviews;
+ poDS->m_poMaskDS->m_bTrailerRepeatedLast4BytesRepeated =
+ bCopySrcOverviews;
GDALDestroyScaledProgress(pScaledData);
pScaledData =
GDALCreateScaledProgress(dfCurPixels / dfTotalPixels, 1.0,
diff --git a/frmts/gtiff/gtiffrasterband.cpp b/frmts/gtiff/gtiffrasterband.cpp
index cbb2a451ccda..ab77f9cb910d 100644
--- a/frmts/gtiff/gtiffrasterband.cpp
+++ b/frmts/gtiff/gtiffrasterband.cpp
@@ -315,19 +315,21 @@ CPLErr GTiffRasterBand::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff,
else
{
bCanUseMultiThreadedRead = false;
- GTiffRasterBand *poBandForCache = this;
+ GTiffDataset *poDSForCache = m_poGDS;
+ int nBandForCache = nBand;
if (!m_poGDS->m_bStreamingIn && m_poGDS->m_bBlockOrderRowMajor &&
m_poGDS->m_bLeaderSizeAsUInt4 &&
m_poGDS->m_bMaskInterleavedWithImagery &&
m_poGDS->m_poImageryDS)
{
- poBandForCache = cpl::down_cast(
- m_poGDS->m_poImageryDS->GetRasterBand(1));
+ poDSForCache = m_poGDS->m_poImageryDS;
+ nBandForCache = 1;
}
- bufferedDataFreer.Init(poBandForCache->CacheMultiRange(
- nXOff, nYOff, nXSize, nYSize, nBufXSize,
- nBufYSize, psExtraArg),
- poBandForCache->m_poGDS->m_hTIFF);
+ bufferedDataFreer.Init(
+ poDSForCache->CacheMultiRange(nXOff, nYOff, nXSize, nYSize,
+ nBufXSize, nBufYSize,
+ &nBandForCache, 1, psExtraArg),
+ poDSForCache->m_hTIFF);
}
}
@@ -379,7 +381,8 @@ CPLErr GTiffRasterBand::IRasterIO(GDALRWFlag eRWFlag, int nXOff, int nYOff,
!m_poGDS->m_bLoadedBlockDirty &&
(m_poGDS->nBands == 1 ||
m_poGDS->m_nPlanarConfig == PLANARCONFIG_SEPARATE) &&
- (nXOff % nBlockXSize) == 0 && (nYOff % nBlockYSize) == 0 &&
+ !m_poGDS->m_bLeaderSizeAsUInt4 && (nXOff % nBlockXSize) == 0 &&
+ (nYOff % nBlockYSize) == 0 &&
(nXOff + nXSize == nRasterXSize || (nXSize % nBlockXSize) == 0) &&
(nYOff + nYSize == nRasterYSize || (nYSize % nBlockYSize) == 0))
{
diff --git a/frmts/gtiff/gtiffrasterband.h b/frmts/gtiff/gtiffrasterband.h
index 8aab99209a79..ff250d51f5c8 100644
--- a/frmts/gtiff/gtiffrasterband.h
+++ b/frmts/gtiff/gtiffrasterband.h
@@ -55,10 +55,6 @@ class GTiffRasterBand CPL_NON_FINAL : public GDALPamRasterBand
GIntBig *pnLineSpace,
char **papszOptions);
- void *CacheMultiRange(int nXOff, int nYOff, int nXSize, int nYSize,
- int nBufXSize, int nBufYSize,
- GDALRasterIOExtraArg *psExtraArg);
-
protected:
GTiffDataset *m_poGDS = nullptr;
GDALMultiDomainMetadata m_oGTiffMDMD{};
diff --git a/frmts/gtiff/gtiffrasterband_read.cpp b/frmts/gtiff/gtiffrasterband_read.cpp
index 63b381671375..325bf39123d0 100644
--- a/frmts/gtiff/gtiffrasterband_read.cpp
+++ b/frmts/gtiff/gtiffrasterband_read.cpp
@@ -630,475 +630,6 @@ CPLVirtualMem *GTiffRasterBand::GetVirtualMemAutoInternal(GDALRWFlag eRWFlag,
return pVMem;
}
-/************************************************************************/
-/* CacheMultiRange() */
-/************************************************************************/
-
-static bool CheckTrailer(const GByte *strileData, vsi_l_offset nStrileSize)
-{
- GByte abyTrailer[4];
- memcpy(abyTrailer, strileData + nStrileSize, 4);
- GByte abyLastBytes[4] = {};
- if (nStrileSize >= 4)
- memcpy(abyLastBytes, strileData + nStrileSize - 4, 4);
- else
- {
- // The last bytes will be zero due to the above {} initialization,
- // and that's what should be in abyTrailer too when the trailer is
- // correct.
- memcpy(abyLastBytes, strileData, static_cast(nStrileSize));
- }
- return memcmp(abyTrailer, abyLastBytes, 4) == 0;
-}
-
-void *GTiffRasterBand::CacheMultiRange(int nXOff, int nYOff, int nXSize,
- int nYSize, int nBufXSize, int nBufYSize,
- GDALRasterIOExtraArg *psExtraArg)
-{
- void *pBufferedData = nullptr;
- // Same logic as in GDALRasterBand::IRasterIO()
- double dfXOff = nXOff;
- double dfYOff = nYOff;
- double dfXSize = nXSize;
- double dfYSize = nYSize;
- if (psExtraArg->bFloatingPointWindowValidity)
- {
- dfXOff = psExtraArg->dfXOff;
- dfYOff = psExtraArg->dfYOff;
- dfXSize = psExtraArg->dfXSize;
- dfYSize = psExtraArg->dfYSize;
- }
- const double dfSrcXInc = dfXSize / static_cast(nBufXSize);
- const double dfSrcYInc = dfYSize / static_cast(nBufYSize);
- const double EPS = 1e-10;
- const int nBlockX1 =
- static_cast(std::max(0.0, (0 + 0.5) * dfSrcXInc + dfXOff + EPS)) /
- nBlockXSize;
- const int nBlockY1 =
- static_cast(std::max(0.0, (0 + 0.5) * dfSrcYInc + dfYOff + EPS)) /
- nBlockYSize;
- const int nBlockX2 =
- static_cast(
- std::min(static_cast(nRasterXSize - 1),
- (nBufXSize - 1 + 0.5) * dfSrcXInc + dfXOff + EPS)) /
- nBlockXSize;
- const int nBlockY2 =
- static_cast(
- std::min(static_cast(nRasterYSize - 1),
- (nBufYSize - 1 + 0.5) * dfSrcYInc + dfYOff + EPS)) /
- nBlockYSize;
-
- const int nBlockCount = nBlocksPerRow * nBlocksPerColumn;
-
- struct StrileData
- {
- vsi_l_offset nOffset;
- vsi_l_offset nByteCount;
- bool bTryMask;
- };
-
- std::map oMapStrileToOffsetByteCount;
-
- // Dedicated method to retrieved the offset and size in an efficient way
- // when m_bBlockOrderRowMajor and m_bLeaderSizeAsUInt4 conditions are
- // met.
- // Except for the last block, we just read the offset from the TIFF offset
- // array, and retrieve the size in the leader 4 bytes that come before the
- // payload.
- auto OptimizedRetrievalOfOffsetSize =
- [&](int nBlockId, vsi_l_offset &nOffset, vsi_l_offset &nSize,
- size_t nTotalSize, size_t nMaxRawBlockCacheSize)
- {
- bool bTryMask = m_poGDS->m_bMaskInterleavedWithImagery;
- nOffset = TIFFGetStrileOffset(m_poGDS->m_hTIFF, nBlockId);
- if (nOffset >= 4)
- {
- if (nBlockId == nBlockCount - 1)
- {
- // Special case for the last block. As there is no next block
- // from which to retrieve an offset, use the good old method
- // that consists in reading the ByteCount array.
- if (bTryMask && m_poGDS->GetRasterBand(1)->GetMaskBand() &&
- m_poGDS->m_poMaskDS)
- {
- auto nMaskOffset = TIFFGetStrileOffset(
- m_poGDS->m_poMaskDS->m_hTIFF, nBlockId);
- if (nMaskOffset)
- {
- nSize = nMaskOffset +
- TIFFGetStrileByteCount(
- m_poGDS->m_poMaskDS->m_hTIFF, nBlockId) -
- nOffset;
- }
- else
- {
- bTryMask = false;
- }
- }
- if (nSize == 0)
- {
- nSize = TIFFGetStrileByteCount(m_poGDS->m_hTIFF, nBlockId);
- }
- if (nSize && m_poGDS->m_bTrailerRepeatedLast4BytesRepeated)
- {
- nSize += 4;
- }
- }
- else
- {
- auto nOffsetNext =
- TIFFGetStrileOffset(m_poGDS->m_hTIFF, nBlockId + 1);
- if (nOffsetNext > nOffset)
- {
- nSize = nOffsetNext - nOffset;
- }
- else
- {
- // Shouldn't happen for a compliant file
- if (nOffsetNext != 0)
- {
- CPLDebug("GTiff", "Tile %d is not located after %d",
- nBlockId + 1, nBlockId);
- }
- bTryMask = false;
- nSize = TIFFGetStrileByteCount(m_poGDS->m_hTIFF, nBlockId);
- if (m_poGDS->m_bTrailerRepeatedLast4BytesRepeated)
- nSize += 4;
- }
- }
- if (nSize)
- {
- nOffset -= 4;
- nSize += 4;
- if (nTotalSize + nSize < nMaxRawBlockCacheSize)
- {
- StrileData data;
- data.nOffset = nOffset;
- data.nByteCount = nSize;
- data.bTryMask = bTryMask;
- oMapStrileToOffsetByteCount[nBlockId] = data;
- }
- }
- }
- else
- {
- // Sparse tile
- StrileData data;
- data.nOffset = 0;
- data.nByteCount = 0;
- data.bTryMask = false;
- oMapStrileToOffsetByteCount[nBlockId] = data;
- }
- };
-
- // This lambda fills m_poDS->m_oCacheStrileToOffsetByteCount (and
- // m_poDS->m_poMaskDS->m_oCacheStrileToOffsetByteCount, when there is a
- // mask) from the temporary oMapStrileToOffsetByteCount.
- auto FillCacheStrileToOffsetByteCount =
- [&](const std::vector &anOffsets,
- const std::vector &anSizes,
- const std::vector &apData)
- {
- CPLAssert(m_poGDS->m_bLeaderSizeAsUInt4);
- size_t i = 0;
- vsi_l_offset nLastOffset = 0;
- for (const auto &entry : oMapStrileToOffsetByteCount)
- {
- const auto nBlockId = entry.first;
- const auto nOffset = entry.second.nOffset;
- const auto nSize = entry.second.nByteCount;
- if (nOffset == 0)
- {
- // Sparse tile
- m_poGDS->m_oCacheStrileToOffsetByteCount.insert(
- nBlockId, std::pair(0, 0));
- continue;
- }
-
- if (nOffset < nLastOffset)
- {
- // shouldn't happen normally if tiles are sorted
- i = 0;
- }
- nLastOffset = nOffset;
- while (i < anOffsets.size() &&
- !(nOffset >= anOffsets[i] &&
- nOffset + nSize <= anOffsets[i] + anSizes[i]))
- {
- i++;
- }
- CPLAssert(i < anOffsets.size());
- CPLAssert(nOffset >= anOffsets[i]);
- CPLAssert(nOffset + nSize <= anOffsets[i] + anSizes[i]);
- GUInt32 nSizeFromLeader;
- memcpy(&nSizeFromLeader,
- // cppcheck-suppress containerOutOfBounds
- static_cast(apData[i]) + nOffset - anOffsets[i],
- sizeof(nSizeFromLeader));
- CPL_LSBPTR32(&nSizeFromLeader);
- bool bOK = true;
- constexpr int nLeaderSize = 4;
- const int nTrailerSize =
- (m_poGDS->m_bTrailerRepeatedLast4BytesRepeated ? 4 : 0);
- if (nSizeFromLeader > nSize - nLeaderSize - nTrailerSize)
- {
- CPLDebug("GTiff",
- "Inconsistent block size from in leader of block %d",
- nBlockId);
- bOK = false;
- }
- else if (m_poGDS->m_bTrailerRepeatedLast4BytesRepeated)
- {
- // Check trailer consistency
- const GByte *strileData = static_cast(apData[i]) +
- nOffset - anOffsets[i] + nLeaderSize;
- if (!CheckTrailer(strileData, nSizeFromLeader))
- {
- CPLDebug("GTiff", "Inconsistent trailer of block %d",
- nBlockId);
- bOK = false;
- }
- }
- if (!bOK)
- {
- return false;
- }
-
- {
- const vsi_l_offset nRealOffset = nOffset + nLeaderSize;
- const vsi_l_offset nRealSize = nSizeFromLeader;
-#ifdef DEBUG_VERBOSE
- CPLDebug("GTiff",
- "Block %d found at offset " CPL_FRMT_GUIB
- " with size " CPL_FRMT_GUIB,
- nBlockId, nRealOffset, nRealSize);
-#endif
- m_poGDS->m_oCacheStrileToOffsetByteCount.insert(
- nBlockId, std::pair(nRealOffset, nRealSize));
- }
-
- // Processing of mask
- if (!(entry.second.bTryMask &&
- m_poGDS->m_bMaskInterleavedWithImagery &&
- m_poGDS->GetRasterBand(1)->GetMaskBand() &&
- m_poGDS->m_poMaskDS))
- {
- continue;
- }
-
- bOK = false;
- const vsi_l_offset nMaskOffsetWithLeader =
- nOffset + nLeaderSize + nSizeFromLeader + nTrailerSize;
- if (nMaskOffsetWithLeader + nLeaderSize <=
- anOffsets[i] + anSizes[i])
- {
- GUInt32 nMaskSizeFromLeader;
- memcpy(&nMaskSizeFromLeader,
- static_cast(apData[i]) + nMaskOffsetWithLeader -
- anOffsets[i],
- sizeof(nMaskSizeFromLeader));
- CPL_LSBPTR32(&nMaskSizeFromLeader);
- if (nMaskOffsetWithLeader + nLeaderSize + nMaskSizeFromLeader +
- nTrailerSize <=
- anOffsets[i] + anSizes[i])
- {
- bOK = true;
- if (m_poGDS->m_bTrailerRepeatedLast4BytesRepeated)
- {
- // Check trailer consistency
- const GByte *strileMaskData =
- static_cast(apData[i]) + nOffset -
- anOffsets[i] + nLeaderSize + nSizeFromLeader +
- nTrailerSize + nLeaderSize;
- if (!CheckTrailer(strileMaskData, nMaskSizeFromLeader))
- {
- CPLDebug("GTiff",
- "Inconsistent trailer of mask of block %d",
- nBlockId);
- bOK = false;
- }
- }
- }
- if (bOK)
- {
- const vsi_l_offset nRealOffset = nOffset + nLeaderSize +
- nSizeFromLeader +
- nTrailerSize + nLeaderSize;
- const vsi_l_offset nRealSize = nMaskSizeFromLeader;
-#ifdef DEBUG_VERBOSE
- CPLDebug("GTiff",
- "Mask of block %d found at offset " CPL_FRMT_GUIB
- " with size " CPL_FRMT_GUIB,
- nBlockId, nRealOffset, nRealSize);
-#endif
-
- m_poGDS->m_poMaskDS->m_oCacheStrileToOffsetByteCount.insert(
- nBlockId, std::pair(nRealOffset, nRealSize));
- }
- }
- if (!bOK)
- {
- CPLDebug("GTiff",
- "Mask for block %d is not properly interleaved with "
- "imagery block",
- nBlockId);
- }
- }
- return true;
- };
-
- thandle_t th = TIFFClientdata(m_poGDS->m_hTIFF);
- if (!VSI_TIFFHasCachedRanges(th))
- {
- std::vector> aOffsetSize;
- size_t nTotalSize = 0;
- const unsigned int nMaxRawBlockCacheSize = atoi(
- CPLGetConfigOption("GDAL_MAX_RAW_BLOCK_CACHE_SIZE", "10485760"));
- bool bGoOn = true;
- for (int iY = nBlockY1; bGoOn && iY <= nBlockY2; iY++)
- {
- for (int iX = nBlockX1; bGoOn && iX <= nBlockX2; iX++)
- {
- GDALRasterBlock *poBlock = TryGetLockedBlockRef(iX, iY);
- if (poBlock != nullptr)
- {
- poBlock->DropLock();
- continue;
- }
- int nBlockId = iX + iY * nBlocksPerRow;
- if (m_poGDS->m_nPlanarConfig == PLANARCONFIG_SEPARATE)
- nBlockId += (nBand - 1) * m_poGDS->m_nBlocksPerBand;
- vsi_l_offset nOffset = 0;
- vsi_l_offset nSize = 0;
-
- if ((m_poGDS->m_nPlanarConfig == PLANARCONFIG_CONTIG ||
- m_poGDS->nBands == 1) &&
- !m_poGDS->m_bStreamingIn &&
- m_poGDS->m_bBlockOrderRowMajor &&
- m_poGDS->m_bLeaderSizeAsUInt4)
- {
- OptimizedRetrievalOfOffsetSize(nBlockId, nOffset, nSize,
- nTotalSize,
- nMaxRawBlockCacheSize);
- }
- else
- {
- CPL_IGNORE_RET_VAL(
- m_poGDS->IsBlockAvailable(nBlockId, &nOffset, &nSize));
- }
- if (nSize)
- {
- if (nTotalSize + nSize < nMaxRawBlockCacheSize)
- {
-#ifdef DEBUG_VERBOSE
- CPLDebug("GTiff",
- "Precaching for block (%d, %d), " CPL_FRMT_GUIB
- "-" CPL_FRMT_GUIB,
- iX, iY, nOffset,
- nOffset + static_cast(nSize) - 1);
-#endif
- aOffsetSize.push_back(
- std::pair(nOffset, static_cast(nSize)));
- nTotalSize += static_cast(nSize);
- }
- else
- {
- bGoOn = false;
- }
- }
- }
- }
-
- std::sort(aOffsetSize.begin(), aOffsetSize.end());
-
- if (nTotalSize > 0)
- {
- pBufferedData = VSI_MALLOC_VERBOSE(nTotalSize);
- if (pBufferedData)
- {
- std::vector anOffsets;
- std::vector anSizes;
- std::vector apData;
- anOffsets.push_back(aOffsetSize[0].first);
- apData.push_back(static_cast(pBufferedData));
- size_t nChunkSize = aOffsetSize[0].second;
- size_t nAccOffset = 0;
- // Try to merge contiguous or slightly overlapping ranges
- for (size_t i = 0; i < aOffsetSize.size() - 1; i++)
- {
- if (aOffsetSize[i].first < aOffsetSize[i + 1].first &&
- aOffsetSize[i].first + aOffsetSize[i].second >=
- aOffsetSize[i + 1].first)
- {
- const auto overlap = aOffsetSize[i].first +
- aOffsetSize[i].second -
- aOffsetSize[i + 1].first;
- // That should always be the case for well behaved
- // TIFF files.
- if (aOffsetSize[i + 1].second > overlap)
- {
- nChunkSize += static_cast(
- aOffsetSize[i + 1].second - overlap);
- }
- }
- else
- {
- // terminate current block
- anSizes.push_back(nChunkSize);
-#ifdef DEBUG_VERBOSE
- CPLDebug("GTiff",
- "Requesting range [" CPL_FRMT_GUIB
- "-" CPL_FRMT_GUIB "]",
- anOffsets.back(),
- anOffsets.back() + anSizes.back() - 1);
-#endif
- nAccOffset += nChunkSize;
- // start a new range
- anOffsets.push_back(aOffsetSize[i + 1].first);
- apData.push_back(static_cast(pBufferedData) +
- nAccOffset);
- nChunkSize = aOffsetSize[i + 1].second;
- }
- }
- // terminate last block
- anSizes.push_back(nChunkSize);
-#ifdef DEBUG_VERBOSE
- CPLDebug(
- "GTiff",
- "Requesting range [" CPL_FRMT_GUIB "-" CPL_FRMT_GUIB "]",
- anOffsets.back(), anOffsets.back() + anSizes.back() - 1);
-#endif
-
- VSILFILE *fp = VSI_TIFFGetVSILFile(th);
-
- if (VSIFReadMultiRangeL(static_cast(anSizes.size()),
- &apData[0], &anOffsets[0], &anSizes[0],
- fp) == 0)
- {
- if (!oMapStrileToOffsetByteCount.empty() &&
- !FillCacheStrileToOffsetByteCount(anOffsets, anSizes,
- apData))
- {
- // Retry without optimization
- CPLFree(pBufferedData);
- m_poGDS->m_bLeaderSizeAsUInt4 = false;
- void *pRet =
- CacheMultiRange(nXOff, nYOff, nXSize, nYSize,
- nBufXSize, nBufYSize, psExtraArg);
- m_poGDS->m_bLeaderSizeAsUInt4 = true;
- return pRet;
- }
-
- VSI_TIFFSetCachedRanges(
- th, static_cast(anSizes.size()), &apData[0],
- &anOffsets[0], &anSizes[0]);
- }
- }
- }
- }
- return pBufferedData;
-}
-
/************************************************************************/
/* IGetDataCoverageStatus() */
/************************************************************************/
diff --git a/port/cpl_known_config_options.h b/port/cpl_known_config_options.h
index 3d57944c2ce5..c1f6124e1f3c 100644
--- a/port/cpl_known_config_options.h
+++ b/port/cpl_known_config_options.h
@@ -324,7 +324,7 @@ constexpr static const char* const apszKnownConfigOptions[] =
"GDAL_MAX_CONNECTIONS", // from gdalogcapidataset.cpp, gdalwmsdataset.cpp
"GDAL_MAX_DATASET_POOL_RAM_USAGE", // from gdalproxypool.cpp
"GDAL_MAX_DATASET_POOL_SIZE", // from gdal_translate_bin.cpp, gdalproxypool.cpp, gdalwarp_bin.cpp
- "GDAL_MAX_RAW_BLOCK_CACHE_SIZE", // from gtiffrasterband_read.cpp
+ "GDAL_MAX_RAW_BLOCK_CACHE_SIZE", // from gtiffdataset_read.cpp
"GDAL_MEM_ENABLE_OPEN", // from memdataset.cpp
"GDAL_NETCDF_ASSUME_LONGLAT", // from netcdfdataset.cpp
"GDAL_NETCDF_BOTTOMUP", // from netcdfdataset.cpp
diff --git a/swig/python/gdal-utils/osgeo_utils/samples/validate_cloud_optimized_geotiff.py b/swig/python/gdal-utils/osgeo_utils/samples/validate_cloud_optimized_geotiff.py
index 52e087e55128..1855d93e75aa 100644
--- a/swig/python/gdal-utils/osgeo_utils/samples/validate_cloud_optimized_geotiff.py
+++ b/swig/python/gdal-utils/osgeo_utils/samples/validate_cloud_optimized_geotiff.py
@@ -38,6 +38,7 @@ class ValidateCloudOptimizedGeoTIFFException(Exception):
def full_check_band(
f,
+ interleave,
band_name,
band,
errors,
@@ -45,6 +46,7 @@ def full_check_band(
block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated,
mask_interleaved_with_imagery,
+ last_offset,
):
block_size = band.GetBlockSize()
@@ -60,7 +62,6 @@ def full_check_band(
yblocks = (band.YSize + block_size[1] - 1) // block_size[1]
xblocks = (band.XSize + block_size[0] - 1) // block_size[0]
- last_offset = 0
for y in range(yblocks):
for x in range(xblocks):
@@ -98,7 +99,13 @@ def full_check_band(
% (x, y)
]
- if mask_band:
+ if mask_band and (
+ interleave == "PIXEL"
+ or (
+ interleave == "TILE"
+ and band.GetBand() == band.GetDataset().RasterCount
+ )
+ ):
offset_mask = mask_band.GetMetadataItem(
"BLOCK_OFFSET_%d_%d" % (x, y), "TIFF"
)
@@ -131,6 +138,33 @@ def full_check_band(
last_offset = offset
+ return last_offset
+
+
+def check_tile_interleave(ds, ds_name, block_order_row_major, errors):
+
+ block_size = ds.GetRasterBand(1).GetBlockSize()
+ yblocks = (ds.RasterYSize + block_size[1] - 1) // block_size[1]
+ xblocks = (ds.RasterXSize + block_size[0] - 1) // block_size[0]
+ last_offset = 0
+ for y in range(yblocks):
+ for x in range(xblocks):
+ for band_idx in range(ds.RasterCount):
+ offset = ds.GetRasterBand(band_idx + 1).GetMetadataItem(
+ "BLOCK_OFFSET_%d_%d" % (x, y), "TIFF"
+ )
+ offset = int(offset) if offset is not None else 0
+
+ if offset > 0:
+ if block_order_row_major and offset < last_offset:
+ errors += [
+ ds_name
+ + ": offset of block (%d, %d) is smaller than previous block"
+ % (x, y)
+ ]
+
+ last_offset = offset
+
def validate(ds, check_tiled=True, full_check=False):
"""Check if a file is a (Geo)TIFF with cloud optimized compatible structure.
@@ -331,6 +365,8 @@ def get_block_offset(band):
"should be after the one of the overview of index %d" % (ovr_count - 1)
]
+ interleave = ds.GetMetadataItem("INTERLEAVE", "IMAGE_STRUCTURE")
+
if full_check and (
block_order_row_major
or block_leader_size_as_uint4
@@ -341,22 +377,49 @@ def get_block_offset(band):
if not f:
raise ValidateCloudOptimizedGeoTIFFException("Cannot open file")
- full_check_band(
- f,
- "Main resolution image",
- main_band,
- errors,
- block_order_row_major,
- block_leader_size_as_uint4,
- block_trailer_last_4_bytes_repeated,
- mask_interleaved_with_imagery,
- )
+ if interleave == "PIXEL":
+ full_check_band(
+ f,
+ interleave,
+ "Main resolution image",
+ main_band,
+ errors,
+ block_order_row_major,
+ block_leader_size_as_uint4,
+ block_trailer_last_4_bytes_repeated,
+ mask_interleaved_with_imagery,
+ 0,
+ )
+ else:
+ last_offset = 0
+ for band_idx in range(ds.RasterCount):
+ if interleave != "BAND":
+ last_offset = 0
+ last_offset = full_check_band(
+ f,
+ interleave,
+ "Band %d of main resolution image" % (band_idx + 1),
+ ds.GetRasterBand(band_idx + 1),
+ errors,
+ block_order_row_major,
+ block_leader_size_as_uint4,
+ block_trailer_last_4_bytes_repeated,
+ mask_interleaved_with_imagery,
+ last_offset,
+ )
+
+ if interleave == "TILE":
+ check_tile_interleave(
+ ds, "Main resolution image", block_order_row_major, errors
+ )
+
if (
main_band.GetMaskFlags() == gdal.GMF_PER_DATASET
and (filename + ".msk") not in ds.GetFileList()
):
full_check_band(
f,
+ interleave,
"Mask band of main resolution image",
main_band.GetMaskBand(),
errors,
@@ -364,25 +427,56 @@ def get_block_offset(band):
block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated,
False,
+ 0,
)
for i in range(ovr_count):
ovr_band = ds.GetRasterBand(1).GetOverview(i)
- full_check_band(
- f,
- "Overview %d" % i,
- ovr_band,
- errors,
- block_order_row_major,
- block_leader_size_as_uint4,
- block_trailer_last_4_bytes_repeated,
- mask_interleaved_with_imagery,
- )
+ if interleave == "PIXEL":
+ full_check_band(
+ f,
+ interleave,
+ "Overview %d" % i,
+ ovr_band,
+ errors,
+ block_order_row_major,
+ block_leader_size_as_uint4,
+ block_trailer_last_4_bytes_repeated,
+ mask_interleaved_with_imagery,
+ 0,
+ )
+ else:
+ last_offset = 0
+ for band_idx in range(ds.RasterCount):
+ if interleave != "BAND":
+ last_offset = 0
+ last_offset = full_check_band(
+ f,
+ interleave,
+ "Band %d of overview %d" % (band_idx + 1, i),
+ ds.GetRasterBand(band_idx + 1).GetOverview(i),
+ errors,
+ block_order_row_major,
+ block_leader_size_as_uint4,
+ block_trailer_last_4_bytes_repeated,
+ mask_interleaved_with_imagery,
+ last_offset,
+ )
+
+ if interleave == "TILE":
+ check_tile_interleave(
+ ds.GetRasterBand(1).GetDataset(),
+ "Overview %d" % i,
+ block_order_row_major,
+ errors,
+ )
+
if (
ovr_band.GetMaskFlags() == gdal.GMF_PER_DATASET
and (filename + ".msk") not in ds.GetFileList()
):
full_check_band(
f,
+ interleave,
"Mask band of overview %d" % i,
ovr_band.GetMaskBand(),
errors,
@@ -390,6 +484,7 @@ def get_block_offset(band):
block_leader_size_as_uint4,
block_trailer_last_4_bytes_repeated,
False,
+ 0,
)
gdal.VSIFCloseL(f)