From 6d91bc9c18b88245cd6baf1164feccee2694162a Mon Sep 17 00:00:00 2001 From: Even Rouault Date: Mon, 20 May 2024 20:16:23 +0200 Subject: [PATCH] /vsiaz/: add BLOB_TYPE=BLOCK and CHUNK_SIZE options to VSIFOpenEx2L() to be able to overcome the 195 GiB file size limit when using append blocks, which is the default mode. --- autotest/gcore/vsiaz.py | 124 ++++++++++++++++++++++ port/cpl_azure.h | 2 +- port/cpl_vsil.cpp | 24 +++-- port/cpl_vsil_adls.cpp | 2 +- port/cpl_vsil_az.cpp | 56 ++++++++-- port/cpl_vsil_curl_class.h | 47 ++++++++- port/cpl_vsil_gs.cpp | 103 +++++++++++------- port/cpl_vsil_s3.cpp | 208 ++++++++++++++++++++++++------------- 8 files changed, 434 insertions(+), 132 deletions(-) diff --git a/autotest/gcore/vsiaz.py b/autotest/gcore/vsiaz.py index 23da34c4cb47..660658192929 100755 --- a/autotest/gcore/vsiaz.py +++ b/autotest/gcore/vsiaz.py @@ -831,6 +831,130 @@ def test_vsiaz_write_appendblob_retry(): gdal.VSIFCloseL(f) +############################################################################### +# Test writing a block blob + + +def test_vsiaz_write_blockblob_chunk_size_1(): + + if gdaltest.webserver_port == 0: + pytest.skip() + + gdal.VSICurlClearCache() + + f = gdal.VSIFOpenExL( + "/vsiaz/test_create/file.bin", "wb", False, ["BLOB_TYPE=BLOCK", "CHUNK_SIZE=1"] + ) + assert f is not None + + assert gdal.VSIFWriteL(b"x", 1, 1, f) == 1 + + handler = webserver.SequentialHandler() + handler.add( + "PUT", + "/azure/blob/myaccount/test_create/file.bin?blockid=000000000001&comp=block", + 201, + expected_headers={"Content-Length": str(1024 * 1024)}, + ) + + with webserver.install_http_handler(handler): + assert gdal.VSIFWriteL(b"x" * (1024 * 1024 - 1), 1024 * 1024 - 1, 1, f) == 1 + + assert gdal.VSIFWriteL(b"x", 1, 1, f) == 1 + + handler = webserver.SequentialHandler() + handler.add( + "PUT", + "/azure/blob/myaccount/test_create/file.bin?blockid=000000000002&comp=block", + 201, + expected_headers={"Content-Length": "1"}, + ) + + handler.add( + "PUT", + "/azure/blob/myaccount/test_create/file.bin?comp=blocklist", + 201, + expected_body=b'\n\n000000000001\n000000000002\n\n', + ) + + with webserver.install_http_handler(handler): + gdal.VSIFCloseL(f) + + +############################################################################### +# Test writing a block blob + + +def test_vsiaz_write_blockblob_default_chunk_size(): + + if gdaltest.webserver_port == 0: + pytest.skip() + + gdal.VSICurlClearCache() + + f = gdal.VSIFOpenExL( + "/vsiaz/test_create/file.bin", "wb", False, ["BLOB_TYPE=BLOCK"] + ) + assert f is not None + + handler = webserver.SequentialHandler() + handler.add( + "PUT", + "/azure/blob/myaccount/test_create/file.bin?blockid=000000000001&comp=block", + 201, + expected_headers={"Content-Length": str(50 * 1024 * 1024)}, + ) + handler.add( + "PUT", + "/azure/blob/myaccount/test_create/file.bin?blockid=000000000002&comp=block", + 201, + expected_headers={"Content-Length": "1"}, + ) + + handler.add( + "PUT", + "/azure/blob/myaccount/test_create/file.bin?comp=blocklist", + 201, + expected_body=b'\n\n000000000001\n000000000002\n\n', + ) + + with webserver.install_http_handler(handler): + assert ( + gdal.VSIFWriteL(b"x" * (50 * 1024 * 1024 + 1), 50 * 1024 * 1024 + 1, 1, f) + == 1 + ) + gdal.VSIFCloseL(f) + + +############################################################################### +# Test writing a block blob + + +def test_vsiaz_write_blockblob_single_put(): + + if gdaltest.webserver_port == 0: + pytest.skip() + + gdal.VSICurlClearCache() + + f = gdal.VSIFOpenExL( + "/vsiaz/test_create/file.bin", "wb", False, ["BLOB_TYPE=BLOCK"] + ) + assert f is not None + + handler = webserver.SequentialHandler() + handler.add( + "PUT", + "/azure/blob/myaccount/test_create/file.bin", + 201, + expected_headers={"Content-Length": "1"}, + ) + + with webserver.install_http_handler(handler): + assert gdal.VSIFWriteL(b"x", 1, 1, f) == 1 + gdal.VSIFCloseL(f) + + ############################################################################### # Test Unlink() diff --git a/port/cpl_azure.h b/port/cpl_azure.h index b9b1f359eeb7..45d8416d900e 100644 --- a/port/cpl_azure.h +++ b/port/cpl_azure.h @@ -121,7 +121,7 @@ class VSIAzureBlobHandleHelper final : public IVSIS3LikeHandleHelper namespace cpl { -int GetAzureBufferSize(); +int GetAzureAppendBufferSize(); } #endif /* HAVE_CURL */ diff --git a/port/cpl_vsil.cpp b/port/cpl_vsil.cpp index 427fa532711d..1bc4545aeb2b 100644 --- a/port/cpl_vsil.cpp +++ b/port/cpl_vsil.cpp @@ -2023,10 +2023,10 @@ bool VSIFilesystemHandler::SetFileMetadata(const char * /* pszFilename*/, /************************************************************************/ /** - * \brief Open file. + * \brief Open/create file. * - * This function opens a file with the desired access. Large files (larger - * than 2GB) should be supported. Binary access is always implied and + * This function opens (or creates) a file with the desired access. + * Binary access is always implied and * the "b" does not need to be included in the pszAccess string. * * Note that the "VSILFILE *" returned by this function is @@ -2064,10 +2064,10 @@ VSILFILE *VSIFOpenExL(const char *pszFilename, const char *pszAccess, /************************************************************************/ /** - * \brief Open file. + * \brief Open/create file. * - * This function opens a file with the desired access. Large files (larger - * than 2GB) should be supported. Binary access is always implied and + * This function opens (or creates) a file with the desired access. + * Binary access is always implied and * the "b" does not need to be included in the pszAccess string. * * Note that the "VSILFILE *" returned by this function is @@ -2094,6 +2094,18 @@ VSILFILE *VSIFOpenExL(const char *pszFilename, const char *pszAccess, * delay. * * + * Options specifics for /vsiaz/ in "w" mode: + * + * * Analog of the POSIX fopen() function. * * @param pszFilename the file to open. UTF-8 encoded. diff --git a/port/cpl_vsil_adls.cpp b/port/cpl_vsil_adls.cpp index 3feff87fe5f7..fb9a3e0bcca2 100644 --- a/port/cpl_vsil_adls.cpp +++ b/port/cpl_vsil_adls.cpp @@ -1159,7 +1159,7 @@ VSIADLSWriteHandle::VSIADLSWriteHandle(VSIADLSFSHandler *poFS, const char *pszFilename, VSIAzureBlobHandleHelper *poHandleHelper) : VSIAppendWriteHandle(poFS, poFS->GetFSPrefix().c_str(), pszFilename, - GetAzureBufferSize()), + GetAzureAppendBufferSize()), m_poHandleHelper(poHandleHelper) { } diff --git a/port/cpl_vsil_az.cpp b/port/cpl_vsil_az.cpp index 3b0118bec365..183c5e89006e 100644 --- a/port/cpl_vsil_az.cpp +++ b/port/cpl_vsil_az.cpp @@ -660,6 +660,32 @@ class VSIAzureFSHandler final : public IVSIS3LikeFSHandler { return new VSIAzureFSHandler(pszPrefix); } + + //! Maximum number of parts for multipart upload + // Cf https://learn.microsoft.com/en-us/rest/api/storageservices/understanding-block-blobs--append-blobs--and-page-blobs + int GetMaximumPartCount() override + { + return 50000; + } + + //! Minimum size of a part for multipart upload (except last one), in MiB. + int GetMinimumPartSizeInMiB() override + { + return 0; + } + + //! Maximum size of a part for multipart upload, in MiB. + // Cf https://learn.microsoft.com/en-us/rest/api/storageservices/understanding-block-blobs--append-blobs--and-page-blobs + int GetMaximumPartSizeInMiB() override + { +#if SIZEOF_VOIDP == 8 + return 4 * 1024; +#else + // Cannot be larger than 4, otherwise integer overflow would occur + // 1 GiB is the maximum reasonable value on a 32-bit machine + return 1 * 1024; +#endif + } }; /************************************************************************/ @@ -735,13 +761,27 @@ VSIAzureFSHandler::CreateWriteHandle(const char *pszFilename, pszFilename + GetFSPrefix().size(), GetFSPrefix().c_str()); if (poHandleHelper == nullptr) return nullptr; - auto poHandle = std::make_unique( - this, pszFilename, poHandleHelper, papszOptions); - if (!poHandle->IsOK()) + const char *pszBlobType = CSLFetchNameValue(papszOptions, "BLOB_TYPE"); + if (pszBlobType && EQUAL(pszBlobType, "BLOCK")) { - return nullptr; + auto poHandle = std::make_unique( + this, pszFilename, poHandleHelper, false, papszOptions); + if (!poHandle->IsOK()) + { + return nullptr; + } + return VSIVirtualHandleUniquePtr(poHandle.release()); + } + else + { + auto poHandle = std::make_unique( + this, pszFilename, poHandleHelper, papszOptions); + if (!poHandle->IsOK()) + { + return nullptr; + } + return VSIVirtualHandleUniquePtr(poHandle.release()); } - return VSIVirtualHandleUniquePtr(poHandle.release()); } /************************************************************************/ @@ -1162,10 +1202,10 @@ VSIAzureFSHandler::GetStreamingFilename(const std::string &osFilename) const } /************************************************************************/ -/* GetAzureBufferSize() */ +/* GetAzureAppendBufferSize() */ /************************************************************************/ -int GetAzureBufferSize() +int GetAzureAppendBufferSize() { int nBufferSize; int nChunkSizeMB = atoi(CPLGetConfigOption("VSIAZ_CHUNK_SIZE", "4")); @@ -1192,7 +1232,7 @@ VSIAzureWriteHandle::VSIAzureWriteHandle( VSIAzureFSHandler *poFS, const char *pszFilename, VSIAzureBlobHandleHelper *poHandleHelper, CSLConstList papszOptions) : VSIAppendWriteHandle(poFS, poFS->GetFSPrefix().c_str(), pszFilename, - GetAzureBufferSize()), + GetAzureAppendBufferSize()), m_poHandleHelper(poHandleHelper), m_aosOptions(papszOptions), m_aosHTTPOptions(CPLHTTPGetOptionsFromEnv(pszFilename)) { diff --git a/port/cpl_vsil_curl_class.h b/port/cpl_vsil_curl_class.h index 418bc35edbec..99752bd2e4ee 100644 --- a/port/cpl_vsil_curl_class.h +++ b/port/cpl_vsil_curl_class.h @@ -672,8 +672,47 @@ class IVSIS3LikeFSHandler : public VSICurlFilesystemHandlerBaseWritable bool AbortPendingUploads(const char *pszFilename) override; - int GetUploadChunkSizeInBytes(const char *pszFilename, - const char *pszSpecifiedValInBytes); + size_t GetUploadChunkSizeInBytes(const char *pszFilename, + const char *pszSpecifiedValInBytes); + + //! Maximum number of parts for multipart upload + // Limit currently used by S3 and GS. + // Cf https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html + // and https://cloud.google.com/storage/quotas#requests + virtual int GetMaximumPartCount() + { + return 10000; + } + + //! Minimum size of a part for multipart upload (except last one), in MiB. + // Limit currently used by S3 and GS. + // Cf https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html + // and https://cloud.google.com/storage/quotas#requests + virtual int GetMinimumPartSizeInMiB() + { + return 5; + } + + //! Maximum size of a part for multipart upload, in MiB. + // Limit currently used by S3 and GS. + // Cf https://docs.aws.amazon.com/AmazonS3/latest/userguide/qfacts.html + // and https://cloud.google.com/storage/quotas#requests + virtual int GetMaximumPartSizeInMiB() + { +#if SIZEOF_VOIDP == 8 + return 5 * 1024; +#else + // Cannot be larger than 4, otherwise integer overflow would occur + // 1 GiB is the maximum reasonable value on a 32-bit machine + return 1 * 1024; +#endif + } + + //! Default size of a part for multipart upload, in MiB. + virtual int GetDefaultPartSizeInMiB() + { + return 50; + } }; /************************************************************************/ @@ -731,8 +770,8 @@ class VSIS3LikeWriteHandle final : public VSIVirtualHandle CPLStringList m_aosHTTPOptions{}; vsi_l_offset m_nCurOffset = 0; - int m_nBufferOff = 0; - int m_nBufferSize = 0; + size_t m_nBufferOff = 0; + size_t m_nBufferSize = 0; bool m_bClosed = false; GByte *m_pabyBuffer = nullptr; std::string m_osUploadID{}; diff --git a/port/cpl_vsil_gs.cpp b/port/cpl_vsil_gs.cpp index 516ad3214b1b..113cf04aacee 100644 --- a/port/cpl_vsil_gs.cpp +++ b/port/cpl_vsil_gs.cpp @@ -198,45 +198,70 @@ VSICurlHandle *VSIGSFSHandler::CreateFileHandle(const char *pszFilename) const char *VSIGSFSHandler::GetOptions() { static std::string osOptions( - std::string("") + - " "); + std::string("") + .append( + " ")); return osOptions.c_str(); } diff --git a/port/cpl_vsil_s3.cpp b/port/cpl_vsil_s3.cpp index a9161de798f3..cd8840cf488e 100644 --- a/port/cpl_vsil_s3.cpp +++ b/port/cpl_vsil_s3.cpp @@ -64,11 +64,12 @@ void VSIInstallS3FileHandler(void) #define ENABLE_DEBUG 0 -constexpr int knMAX_PART_NUMBER = 10000; // Limitation from S3 - #define unchecked_curl_easy_setopt(handle, opt, param) \ CPL_IGNORE_RET_VAL(curl_easy_setopt(handle, opt, param)) +// MebIByte +constexpr int MIB_CONSTANT = 1024 * 1024; + namespace cpl { @@ -765,7 +766,15 @@ VSIS3LikeWriteHandle::VSIS3LikeWriteHandle( if (!m_bUseChunkedTransfer) { - m_nBufferSize = poFS->GetUploadChunkSizeInBytes(pszFilename, nullptr); + const char *pszChunkSize = m_aosOptions.FetchNameValue("CHUNK_SIZE"); + if (pszChunkSize) + m_nBufferSize = poFS->GetUploadChunkSizeInBytes( + pszFilename, + CPLSPrintf(CPL_FRMT_GIB, + CPLAtoGIntBig(pszChunkSize) * MIB_CONSTANT)); + else + m_nBufferSize = + poFS->GetUploadChunkSizeInBytes(pszFilename, nullptr); m_pabyBuffer = static_cast(VSIMalloc(m_nBufferSize)); if (m_pabyBuffer == nullptr) @@ -781,19 +790,10 @@ VSIS3LikeWriteHandle::VSIS3LikeWriteHandle( /* GetUploadChunkSizeInBytes() */ /************************************************************************/ -int IVSIS3LikeFSHandler::GetUploadChunkSizeInBytes( +size_t IVSIS3LikeFSHandler::GetUploadChunkSizeInBytes( const char *pszFilename, const char *pszSpecifiedValInBytes) { - int nChunkSize = 0; - - const int nChunkSizeMB = atoi(VSIGetPathSpecificOption( - pszFilename, - std::string("VSI").append(GetDebugKey()).append("_CHUNK_SIZE").c_str(), - "50")); - if (nChunkSizeMB <= 0 || nChunkSizeMB > 1000) - nChunkSize = 0; - else - nChunkSize = nChunkSizeMB * 1024 * 1024; + size_t nChunkSize = 0; const char *pszChunkSizeBytes = pszSpecifiedValInBytes ? pszSpecifiedValInBytes : @@ -805,10 +805,44 @@ int IVSIS3LikeFSHandler::GetUploadChunkSizeInBytes( .c_str(), nullptr); if (pszChunkSizeBytes) - nChunkSize = atoi(pszChunkSizeBytes); - - if (nChunkSize <= 0 || nChunkSize > 1000 * 1024 * 1024) - nChunkSize = 50 * 1024 * 1024; + { + const auto nChunkSizeInt = CPLAtoGIntBig(pszChunkSizeBytes); + if (nChunkSizeInt <= 0) + nChunkSize = + static_cast(GetDefaultPartSizeInMiB()) * MIB_CONSTANT; + else if (nChunkSizeInt / MIB_CONSTANT >= GetMaximumPartSizeInMiB()) + { + CPLError(CE_Warning, CPLE_AppDefined, + "Specified chunk size too large. Clamping to %d MiB", + GetMaximumPartSizeInMiB()); + nChunkSize = + static_cast(GetMaximumPartSizeInMiB()) * MIB_CONSTANT; + } + else + nChunkSize = nChunkSizeInt; + } + else + { + const int nChunkSizeMiB = atoi(VSIGetPathSpecificOption( + pszFilename, + std::string("VSI") + .append(GetDebugKey()) + .append("_CHUNK_SIZE") + .c_str(), + CPLSPrintf("%d", GetDefaultPartSizeInMiB()))); + if (nChunkSizeMiB <= 0) + nChunkSize = 0; + else if (nChunkSizeMiB > GetMaximumPartSizeInMiB()) + { + CPLError(CE_Warning, CPLE_AppDefined, + "Specified chunk size too large. Clamping to %d MiB", + GetMaximumPartSizeInMiB()); + nChunkSize = + static_cast(GetMaximumPartSizeInMiB()) * MIB_CONSTANT; + } + else + nChunkSize = static_cast(nChunkSizeMiB) * MIB_CONSTANT; + } return nChunkSize; } @@ -990,15 +1024,16 @@ std::string IVSIS3LikeFSHandler::InitiateMultipartUpload( bool VSIS3LikeWriteHandle::UploadPart() { ++m_nPartNumber; - if (m_nPartNumber > knMAX_PART_NUMBER) + if (m_nPartNumber > m_poFS->GetMaximumPartCount()) { m_bError = true; - CPLError( - CE_Failure, CPLE_AppDefined, - "%d parts have been uploaded for %s failed. " - "This is the maximum. " - "Increase VSIS3_CHUNK_SIZE to a higher value (e.g. 500 for 500 MB)", - knMAX_PART_NUMBER, m_osFilename.c_str()); + CPLError(CE_Failure, CPLE_AppDefined, + "%d parts have been uploaded for %s failed. " + "This is the maximum. " + "Increase VSI%s_CHUNK_SIZE to a higher value (e.g. 500 for " + "500 MiB)", + m_poFS->GetMaximumPartCount(), m_osFilename.c_str(), + m_poFS->GetDebugKey()); return false; } const std::string osEtag = m_poFS->UploadPart( @@ -1447,8 +1482,8 @@ size_t VSIS3LikeWriteHandle::Write(const void *pBuffer, size_t nSize, const GByte *pabySrcBuffer = reinterpret_cast(pBuffer); while (nBytesToWrite > 0) { - const int nToWriteInBuffer = static_cast(std::min( - static_cast(m_nBufferSize - m_nBufferOff), nBytesToWrite)); + const size_t nToWriteInBuffer = + std::min(m_nBufferSize - m_nBufferOff, nBytesToWrite); memcpy(m_pabyBuffer + m_nBufferOff, pabySrcBuffer, nToWriteInBuffer); pabySrcBuffer += nToWriteInBuffer; m_nBufferOff += nToWriteInBuffer; @@ -1456,7 +1491,7 @@ size_t VSIS3LikeWriteHandle::Write(const void *pBuffer, size_t nSize, nBytesToWrite -= nToWriteInBuffer; if (m_nBufferOff == m_nBufferSize) { - if (m_nCurOffset == static_cast(m_nBufferSize)) + if (m_nCurOffset == m_nBufferSize) { m_osUploadID = m_poFS->InitiateMultipartUpload( m_osFilename, m_poS3HandleHelper, m_nMaxRetry, @@ -2226,40 +2261,56 @@ void VSIS3FSHandler::ClearCache() const char *VSIS3FSHandler::GetOptions() { static std::string osOptions( - std::string("") + - " "); + std::string("") + .append( + " ")); return osOptions.c_str(); } @@ -3842,7 +3893,7 @@ int IVSIS3LikeFSHandler::CopyFileRestartable( } const char *pszChunkSize = CSLFetchNameValue(papszOptions, "CHUNK_SIZE"); - int nChunkSize = GetUploadChunkSizeInBytes(pszTarget, pszChunkSize); + size_t nChunkSize = GetUploadChunkSizeInBytes(pszTarget, pszChunkSize); VSIStatBufL sStatBuf; if (VSIStatL(pszSource, &sStatBuf) != 0) @@ -3909,13 +3960,23 @@ int IVSIS3LikeFSHandler::CopyFileRestartable( return -1; } - nChunkSize = oRoot.GetInteger("chunk_size"); - if (nChunkSize <= 0) + const auto nChunkSizeLong = oRoot.GetLong("chunk_size"); + if (nChunkSizeLong <= 0) { CPLError(CE_Failure, CPLE_AppDefined, "'chunk_size' field in input payload missing or invalid"); return -1; } +#if SIZEOF_VOIDP < 8 + if (static_cast(nChunkSizeLong) > + std::numeric_limits::max()) + { + CPLError(CE_Failure, CPLE_AppDefined, + "'chunk_size' field in input payload is too large"); + return -1; + } +#endif + nChunkSize = static_cast(nChunkSizeLong); auto oEtags = oRoot.GetArray("chunk_etags"); if (!oEtags.IsValid()) @@ -3927,7 +3988,7 @@ int IVSIS3LikeFSHandler::CopyFileRestartable( const auto nChunkCountLarge = (sStatBuf.st_size + nChunkSize - 1) / nChunkSize; - if (nChunkCountLarge != oEtags.Size()) + if (nChunkCountLarge != static_cast(oEtags.Size())) { CPLError( CE_Failure, CPLE_AppDefined, @@ -3945,11 +4006,11 @@ int IVSIS3LikeFSHandler::CopyFileRestartable( // Compute the number of chunks auto nChunkCountLarge = (sStatBuf.st_size + nChunkSize - 1) / nChunkSize; - if (nChunkCountLarge > knMAX_PART_NUMBER) + if (nChunkCountLarge > static_cast(GetMaximumPartCount())) { // Re-adjust the chunk size if needed - constexpr int nWishedChunkCount = knMAX_PART_NUMBER / 10; - const auto nMinChunkSizeLarge = + const int nWishedChunkCount = GetMaximumPartCount() / 10; + const uint64_t nMinChunkSizeLarge = (sStatBuf.st_size + nWishedChunkCount - 1) / nWishedChunkCount; if (pszChunkSize) { @@ -3960,12 +4021,13 @@ int IVSIS3LikeFSHandler::CopyFileRestartable( static_cast(nMinChunkSizeLarge)); return -1; } - if (nMinChunkSizeLarge > 1000 * 1024 * 1024) + if (nMinChunkSizeLarge > + static_cast(GetMaximumPartSizeInMiB()) * MIB_CONSTANT) { CPLError(CE_Failure, CPLE_AppDefined, "Too large file"); return -1; } - nChunkSize = static_cast(nMinChunkSizeLarge); + nChunkSize = static_cast(nMinChunkSizeLarge); nChunkCountLarge = (sStatBuf.st_size + nChunkSize - 1) / nChunkSize; } nChunkCount = static_cast(nChunkCountLarge); @@ -4477,7 +4539,7 @@ bool IVSIS3LikeFSHandler::Sync(const char *pszSource, const char *pszTarget, CPLTestBool(CPLGetConfigOption("VSIS3_SIMULATE_THREADING", "NO")); const int nMinSizeChunk = bSupportsParallelMultipartUpload && !bSimulateThreading - ? 8 * 1024 * 1024 + ? 8 * MIB_CONSTANT : 1; // 5242880 defined by S3 API as the minimum, but 8 MB used by // default by the Python s3transfer library const int nMinThreads = bSimulateThreading ? 0 : 1; @@ -4486,7 +4548,7 @@ bool IVSIS3LikeFSHandler::Sync(const char *pszSource, const char *pszTarget, (bDownloadFromNetworkToLocal || bSupportsParallelMultipartUpload) ? static_cast( - std::min(1024 * 1024 * 1024, + std::min(1024 * MIB_CONSTANT, std::max(nMinSizeChunk, atoi(pszChunkSize)))) : 0;