From db9dfe74b424a75911aef50e480add88c4950de5 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Sun, 31 Oct 2021 03:09:11 +0300 Subject: [PATCH 01/10] [PGPRO-5771] Replace macro COMP_CRC32C with handler function. Tags: ptrack --- engine.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/engine.c b/engine.c index 42fa65a..649f7fa 100644 --- a/engine.c +++ b/engine.c @@ -31,7 +31,7 @@ #include "access/xlog.h" #include "catalog/pg_tablespace.h" #include "miscadmin.h" -#include "port/pg_crc32c.h" +#include "common/checksum_helper.h" #include "storage/copydir.h" #if PG_VERSION_NUM >= 120000 #include "storage/md.h" @@ -73,7 +73,7 @@ ptrack_file_exists(const char *path) static void ptrack_write_chunk(int fd, pg_crc32c *crc, char *chunk, size_t size) { - COMP_CRC32C(*crc, (char *) chunk, size); + pg_comp_crc32c_handler(crc, (char *) chunk, size); if (write(fd, chunk, size) != size) { @@ -193,7 +193,7 @@ ptrackMapReadFromFile(const char *ptrack_path) pg_crc32c *file_crc; INIT_CRC32C(crc); - COMP_CRC32C(crc, (char *) ptrack_map, PtrackCrcOffset); + pg_comp_crc32c_handler(&crc, (char *) ptrack_map, PtrackCrcOffset); FIN_CRC32C(crc); file_crc = (pg_crc32c *) ((char *) ptrack_map + PtrackCrcOffset); From c5d59068160ab0c7c8d7fd5b3bcf75852cab7db5 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Sun, 31 Oct 2021 16:11:26 +0300 Subject: [PATCH 02/10] [PGPRO-5771] Add pg_comp_crc32c_handler and restore pg_comp_crc32c pointer definition in REL_14_STABLE core patch. Tags: ptrack --- patches/REL_14_STABLE-ptrack-core.diff | 99 ++++++++++++++++---------- 1 file changed, 61 insertions(+), 38 deletions(-) diff --git a/patches/REL_14_STABLE-ptrack-core.diff b/patches/REL_14_STABLE-ptrack-core.diff index 3491700..f08081a 100644 --- a/patches/REL_14_STABLE-ptrack-core.diff +++ b/patches/REL_14_STABLE-ptrack-core.diff @@ -1,14 +1,17 @@ -commit a14ac459d71528c64df00c693e9c71ac70d3ba29 -Author: anastasia -Date: Mon Oct 19 14:53:06 2020 +0300 +commit 31aad7adad5d9f977e3a3623108ff81b82181d68 +Author: Anton A. Melnikov +Date: Fri Oct 29 08:45:13 2021 +0300 - add ptrack 2.0 + [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c + pointer definition. + + Tags: ptrack diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c -index 50ae1f16d0..721b926ad2 100644 +index e09108d0ec..6858c7b670 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c -@@ -233,6 +233,13 @@ static const struct exclude_list_item excludeFiles[] = +@@ -225,6 +225,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, @@ -22,7 +25,7 @@ index 50ae1f16d0..721b926ad2 100644 /* end of list */ {NULL, false} }; -@@ -248,6 +255,11 @@ static const struct exclude_list_item noChecksumFiles[] = { +@@ -240,6 +247,11 @@ static const struct exclude_list_item noChecksumFiles[] = { {"pg_filenode.map", false}, {"pg_internal.init", true}, {"PG_VERSION", false}, @@ -35,7 +38,7 @@ index 50ae1f16d0..721b926ad2 100644 {"config_exec_params", true}, #endif diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c -index 0cf598dd0c..c9c44a4ae7 100644 +index da8b7cbeca..6a817f45a5 100644 --- a/src/backend/storage/file/copydir.c +++ b/src/backend/storage/file/copydir.c @@ -27,6 +27,8 @@ @@ -58,7 +61,7 @@ index 0cf598dd0c..c9c44a4ae7 100644 * Be paranoid here and fsync all files to ensure the copy is really done. * But if fsync is disabled, we're done. diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c -index 0eacd461cd..c2ef404a1a 100644 +index b4bca7eed6..90d479526c 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -87,6 +87,8 @@ typedef struct _MdfdVec @@ -70,7 +73,7 @@ index 0eacd461cd..c2ef404a1a 100644 /* Populate a file tag describing an md.c segment file. */ #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ -@@ -435,6 +437,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -463,6 +465,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); @@ -80,7 +83,7 @@ index 0eacd461cd..c2ef404a1a 100644 } /* -@@ -721,6 +726,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -749,6 +754,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); @@ -91,10 +94,10 @@ index 0eacd461cd..c2ef404a1a 100644 /* diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c -index 3ded2cdd71..3a596a59f7 100644 +index bc3ceb2712..6c72d24c95 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c -@@ -75,6 +75,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ +@@ -78,6 +78,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ static CycleCtr sync_cycle_ctr = 0; static CycleCtr checkpoint_cycle_ctr = 0; @@ -103,7 +106,7 @@ index 3ded2cdd71..3a596a59f7 100644 /* Intervals for calling AbsorbSyncRequests */ #define FSYNCS_PER_ABSORB 10 #define UNLINKS_PER_ABSORB 10 -@@ -420,6 +422,9 @@ ProcessSyncRequests(void) +@@ -441,6 +443,9 @@ ProcessSyncRequests(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; @@ -114,10 +117,10 @@ index 3ded2cdd71..3a596a59f7 100644 sync_in_progress = false; } diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c -index 1683629ee3..d2fc154576 100644 +index 6652a60ec3..051cd6359d 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c -@@ -620,7 +620,6 @@ static char *recovery_target_xid_string; +@@ -659,7 +659,6 @@ static char *recovery_target_xid_string; static char *recovery_target_name_string; static char *recovery_target_lsn_string; @@ -126,7 +129,7 @@ index 1683629ee3..d2fc154576 100644 char *role_string; diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c -index ffdc23945c..7ae95866ce 100644 +index 831cf42d3a..891302ba2f 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -114,6 +114,11 @@ static const struct exclude_list_item skip[] = { @@ -142,10 +145,10 @@ index ffdc23945c..7ae95866ce 100644 {"config_exec_params", true}, #endif diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c -index 233441837f..cf7bd073bf 100644 +index f8d5ecb6ad..4027bdf3df 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c -@@ -84,6 +84,7 @@ static void RewriteControlFile(void); +@@ -85,6 +85,7 @@ static void RewriteControlFile(void); static void FindEndOfXLOG(void); static void KillExistingXLOG(void); static void KillExistingArchiveStatus(void); @@ -153,7 +156,7 @@ index 233441837f..cf7bd073bf 100644 static void WriteEmptyXLOG(void); static void usage(void); -@@ -513,6 +514,7 @@ main(int argc, char *argv[]) +@@ -522,6 +523,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); @@ -161,7 +164,7 @@ index 233441837f..cf7bd073bf 100644 WriteEmptyXLOG(); printf(_("Write-ahead log reset\n")); -@@ -1102,6 +1104,53 @@ KillExistingArchiveStatus(void) +@@ -1111,6 +1113,53 @@ KillExistingArchiveStatus(void) } } @@ -216,10 +219,10 @@ index 233441837f..cf7bd073bf 100644 /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c -index fbb97b5cf1..6cd7f2ae3e 100644 +index 2618b4c957..bbba813fe7 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c -@@ -124,6 +124,10 @@ static const struct exclude_list_item excludeFiles[] = +@@ -158,6 +158,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, @@ -230,11 +233,37 @@ index fbb97b5cf1..6cd7f2ae3e 100644 /* end of list */ {NULL, false} }; +diff --git a/src/common/checksum_helper.c b/src/common/checksum_helper.c +index 431e247d59..3d042ebb77 100644 +--- a/src/common/checksum_helper.c ++++ b/src/common/checksum_helper.c +@@ -230,3 +230,9 @@ pg_checksum_final(pg_checksum_context *context, uint8 *output) + Assert(retval <= PG_CHECKSUM_MAX_LENGTH); + return retval; + } ++ ++pg_crc32c pg_comp_crc32c_handler(pg_crc32c *crc, const void *data, size_t len) ++{ ++ COMP_CRC32C(*crc, data, len); ++ return *crc; ++} +diff --git a/src/include/common/checksum_helper.h b/src/include/common/checksum_helper.h +index cac7570ea1..3f7287371b 100644 +--- a/src/include/common/checksum_helper.h ++++ b/src/include/common/checksum_helper.h +@@ -68,5 +68,7 @@ extern int pg_checksum_init(pg_checksum_context *, pg_checksum_type); + extern int pg_checksum_update(pg_checksum_context *, const uint8 *input, + size_t len); + extern int pg_checksum_final(pg_checksum_context *, uint8 *output); ++extern pg_crc32c ++pg_comp_crc32c_handler(pg_crc32c *crc, const void *data, size_t len); + + #endif diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h -index 72e3352398..5c2e016501 100644 +index 3f155ce4f8..ce8dc0bee5 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h -@@ -388,7 +388,7 @@ typedef enum ProcessingMode +@@ -401,7 +401,7 @@ typedef enum ProcessingMode NormalProcessing /* normal processing */ } ProcessingMode; @@ -244,25 +273,19 @@ index 72e3352398..5c2e016501 100644 #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h -index 3c6f906683..a7355f7ad1 100644 +index f3c4107ff9..05b2e63775 100644 --- a/src/include/port/pg_crc32c.h +++ b/src/include/port/pg_crc32c.h -@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le - #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) +@@ -70,7 +70,6 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); --extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); + extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); - -+extern -+#ifndef FRONTEND -+PGDLLIMPORT -+#endif -+pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); #endif diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h -index 5d28f59c1d..0d3f04d8af 100644 +index 2c3936b0da..c6691c0be0 100644 --- a/src/include/storage/copydir.h +++ b/src/include/storage/copydir.h @@ -13,6 +13,9 @@ @@ -276,7 +299,7 @@ index 5d28f59c1d..0d3f04d8af 100644 extern void copy_file(char *fromfile, char *tofile); diff --git a/src/include/storage/md.h b/src/include/storage/md.h -index 07fd1bb7d0..5294811bc8 100644 +index 752b440864..61c18a169f 100644 --- a/src/include/storage/md.h +++ b/src/include/storage/md.h @@ -19,6 +19,13 @@ @@ -294,10 +317,10 @@ index 07fd1bb7d0..5294811bc8 100644 extern void mdinit(void); extern void mdopen(SMgrRelation reln); diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h -index e16ab8e711..88da9686eb 100644 +index fbdf34f762..455de202b6 100644 --- a/src/include/storage/sync.h +++ b/src/include/storage/sync.h -@@ -50,6 +50,9 @@ typedef struct FileTag +@@ -55,6 +55,9 @@ typedef struct FileTag uint32 segno; } FileTag; From dc3530b69eec224771c03b3c485cdc3deeacc411 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Tue, 2 Nov 2021 14:04:33 +0300 Subject: [PATCH 03/10] [PGPRO-5771] Add macros to use pg_comp_crc32c_handler() in windows only. Tags: ptrack --- engine.c | 13 +++++++++++++ pg_probackup | 1 + 2 files changed, 14 insertions(+) create mode 160000 pg_probackup diff --git a/engine.c b/engine.c index 649f7fa..e9b6bf7 100644 --- a/engine.c +++ b/engine.c @@ -31,7 +31,11 @@ #include "access/xlog.h" #include "catalog/pg_tablespace.h" #include "miscadmin.h" +#ifdef WIN32 #include "common/checksum_helper.h" +#else +#include "port/pg_crc32c.h" +#endif #include "storage/copydir.h" #if PG_VERSION_NUM >= 120000 #include "storage/md.h" @@ -73,7 +77,12 @@ ptrack_file_exists(const char *path) static void ptrack_write_chunk(int fd, pg_crc32c *crc, char *chunk, size_t size) { + +#ifdef WIN32 pg_comp_crc32c_handler(crc, (char *) chunk, size); +#else + COMP_CRC32C(*crc, (char *) chunk, size); +#endif if (write(fd, chunk, size) != size) { @@ -193,7 +202,11 @@ ptrackMapReadFromFile(const char *ptrack_path) pg_crc32c *file_crc; INIT_CRC32C(crc); +#ifdef WIN32 pg_comp_crc32c_handler(&crc, (char *) ptrack_map, PtrackCrcOffset); +#else + COMP_CRC32C(crc, (char *) ptrack_map, PtrackCrcOffset); +#endif FIN_CRC32C(crc); file_crc = (pg_crc32c *) ((char *) ptrack_map + PtrackCrcOffset); diff --git a/pg_probackup b/pg_probackup new file mode 160000 index 0000000..5b6ca62 --- /dev/null +++ b/pg_probackup @@ -0,0 +1 @@ +Subproject commit 5b6ca624170e1b7955c293ce0173a812b6402d80 From f9e9dcda0fa3e889aa609fa5900f33c580691288 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Sun, 7 Nov 2021 23:27:46 +0300 Subject: [PATCH 04/10] [PGPRO-5771] Rename pg_comp_crc32c_handler() to comp_crc32c() Tags: ptrack --- engine.c | 4 ++-- patches/REL_14_STABLE-ptrack-core.diff | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/engine.c b/engine.c index e9b6bf7..d6733c4 100644 --- a/engine.c +++ b/engine.c @@ -79,7 +79,7 @@ ptrack_write_chunk(int fd, pg_crc32c *crc, char *chunk, size_t size) { #ifdef WIN32 - pg_comp_crc32c_handler(crc, (char *) chunk, size); + comp_crc32c(crc, (char *) chunk, size); #else COMP_CRC32C(*crc, (char *) chunk, size); #endif @@ -203,7 +203,7 @@ ptrackMapReadFromFile(const char *ptrack_path) INIT_CRC32C(crc); #ifdef WIN32 - pg_comp_crc32c_handler(&crc, (char *) ptrack_map, PtrackCrcOffset); + comp_crc32c(&crc, (char *) ptrack_map, PtrackCrcOffset); #else COMP_CRC32C(crc, (char *) ptrack_map, PtrackCrcOffset); #endif diff --git a/patches/REL_14_STABLE-ptrack-core.diff b/patches/REL_14_STABLE-ptrack-core.diff index f08081a..bb2d95b 100644 --- a/patches/REL_14_STABLE-ptrack-core.diff +++ b/patches/REL_14_STABLE-ptrack-core.diff @@ -1,9 +1,9 @@ -commit 31aad7adad5d9f977e3a3623108ff81b82181d68 +commit 84a43938a670336fbb203b85c7f5ea15d642aec2 Author: Anton A. Melnikov Date: Fri Oct 29 08:45:13 2021 +0300 - [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c - pointer definition. + [PGPRO-5771] Add wrapper function for COMP_CRC32C and restore + pg_comp_crc32c pointer definition. Tags: ptrack @@ -234,7 +234,7 @@ index 2618b4c957..bbba813fe7 100644 {NULL, false} }; diff --git a/src/common/checksum_helper.c b/src/common/checksum_helper.c -index 431e247d59..3d042ebb77 100644 +index 431e247d59..405247a597 100644 --- a/src/common/checksum_helper.c +++ b/src/common/checksum_helper.c @@ -230,3 +230,9 @@ pg_checksum_final(pg_checksum_context *context, uint8 *output) @@ -242,13 +242,13 @@ index 431e247d59..3d042ebb77 100644 return retval; } + -+pg_crc32c pg_comp_crc32c_handler(pg_crc32c *crc, const void *data, size_t len) ++pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) +{ + COMP_CRC32C(*crc, data, len); + return *crc; +} diff --git a/src/include/common/checksum_helper.h b/src/include/common/checksum_helper.h -index cac7570ea1..3f7287371b 100644 +index cac7570ea1..8c002860cf 100644 --- a/src/include/common/checksum_helper.h +++ b/src/include/common/checksum_helper.h @@ -68,5 +68,7 @@ extern int pg_checksum_init(pg_checksum_context *, pg_checksum_type); @@ -256,7 +256,7 @@ index cac7570ea1..3f7287371b 100644 size_t len); extern int pg_checksum_final(pg_checksum_context *, uint8 *output); +extern pg_crc32c -+pg_comp_crc32c_handler(pg_crc32c *crc, const void *data, size_t len); ++comp_crc32c(pg_crc32c *crc, const void *data, size_t len); #endif diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h From 06b27cf1cada1d9e022ab25e0015c5e59f6e7595 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Mon, 8 Nov 2021 07:29:38 +0300 Subject: [PATCH 05/10] [PGPRO-5771] Move comp_crc32c() to checksum_impl.h There are no checksum_helper interface in 11 and 12 versions. Tags: ptrack --- engine.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine.c b/engine.c index d6733c4..9fbec3e 100644 --- a/engine.c +++ b/engine.c @@ -32,7 +32,7 @@ #include "catalog/pg_tablespace.h" #include "miscadmin.h" #ifdef WIN32 -#include "common/checksum_helper.h" +#include "storage/checksum.h" #else #include "port/pg_crc32c.h" #endif From 099b0b0e928fd3220cb2ca93a5f94fdd198fa3a1 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Mon, 8 Nov 2021 08:32:26 +0300 Subject: [PATCH 06/10] [PGPRO-5771] Update core patches for REL_11_STABLE, REL_12_STABLE, REL_13_STABLE and REL_14_STABLE. Tags: ptrack --- patches/REL_11_STABLE-ptrack-core.diff | 138 +++++++++++++--------- patches/REL_12_STABLE-ptrack-core.diff | 152 +++++++++++++++---------- patches/REL_13_STABLE-ptrack-core.diff | 142 +++++++++++++---------- patches/REL_14_STABLE-ptrack-core.diff | 141 +++++++++++------------ 4 files changed, 321 insertions(+), 252 deletions(-) diff --git a/patches/REL_11_STABLE-ptrack-core.diff b/patches/REL_11_STABLE-ptrack-core.diff index a8207f5..bf4a834 100644 --- a/patches/REL_11_STABLE-ptrack-core.diff +++ b/patches/REL_11_STABLE-ptrack-core.diff @@ -1,11 +1,17 @@ +commit 5255ff7a6051f56689e0db2a0fa7e9a6e4086d66 +Author: Anton A. Melnikov +Date: Mon Nov 8 11:11:45 2021 +0300 + + [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c pointer definition. + diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c -index 3e53b3df6fb..f76bfc2a646 100644 +index 3e53b3df6f..f76bfc2a64 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -209,6 +209,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + /* + * Skip all transient ptrack files, but do copy ptrack.map, since it may + * be successfully used immediately after backup. TODO: check, test? @@ -28,13 +34,13 @@ index 3e53b3df6fb..f76bfc2a646 100644 {"config_exec_params", true}, #endif diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c -index 4a0d23b11e3..d59009a4c8c 100644 +index 4a0d23b11e..d59009a4c8 100644 --- a/src/backend/storage/file/copydir.c +++ b/src/backend/storage/file/copydir.c @@ -27,6 +27,8 @@ #include "miscadmin.h" #include "pgstat.h" - + +copydir_hook_type copydir_hook = NULL; + /* @@ -43,7 +49,7 @@ index 4a0d23b11e3..d59009a4c8c 100644 @@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) } FreeDir(xldir); - + + if (copydir_hook) + copydir_hook(todir); + @@ -51,50 +57,50 @@ index 4a0d23b11e3..d59009a4c8c 100644 * Be paranoid here and fsync all files to ensure the copy is really done. * But if fsync is disabled, we're done. diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c -index 200cc7f657a..d0dcb5c0287 100644 +index fa29e7041f..8bda639eda 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -39,6 +39,7 @@ #include "utils/memutils.h" #include "pg_trace.h" - + +ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; - + /* intervals for calling AbsorbFsyncRequests in mdsync and mdpostckpt */ #define FSYNCS_PER_ABSORB 10 @@ -114,6 +115,8 @@ typedef struct _MdfdVec - + static MemoryContext MdCxt; /* context for all MdfdVec objects */ - + +mdextend_hook_type mdextend_hook = NULL; +mdwrite_hook_type mdwrite_hook = NULL; - + /* * In some contexts (currently, standalone backends and the checkpointer) -@@ -558,6 +561,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -600,6 +603,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); - + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + + if (mdextend_hook) + mdextend_hook(reln->smgr_rnode, forknum, blocknum); } - + /* -@@ -851,6 +857,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, - +@@ -893,6 +899,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); + + if (mdwrite_hook) + mdwrite_hook(reln->smgr_rnode, forknum, blocknum); } - + /* -@@ -1329,6 +1338,9 @@ mdsync(void) +@@ -1371,6 +1380,9 @@ mdsync(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; - + + if (ProcessSyncRequests_hook) + ProcessSyncRequests_hook(); + @@ -102,29 +108,29 @@ index 200cc7f657a..d0dcb5c0287 100644 mdsync_in_progress = false; } diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c -index 6fb403a5a8a..6e31ccb3e0f 100644 +index 611edf6ade..ec1c1212bd 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c -@@ -84,6 +84,7 @@ static void RewriteControlFile(void); +@@ -85,6 +85,7 @@ static void RewriteControlFile(void); static void FindEndOfXLOG(void); static void KillExistingXLOG(void); static void KillExistingArchiveStatus(void); +static void KillExistingPtrack(void); static void WriteEmptyXLOG(void); static void usage(void); - -@@ -516,6 +517,7 @@ main(int argc, char *argv[]) + +@@ -525,6 +526,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); + KillExistingPtrack(); WriteEmptyXLOG(); - + printf(_("Write-ahead log reset\n")); -@@ -1201,6 +1203,57 @@ KillExistingArchiveStatus(void) +@@ -1210,6 +1212,57 @@ KillExistingArchiveStatus(void) } } - + +/* + * Remove existing ptrack files + */ @@ -176,17 +182,17 @@ index 6fb403a5a8a..6e31ccb3e0f 100644 + } +} + - + /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c -index 197163d5544..fc846e78175 100644 +index 8ea7fafa27..997168fcac 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -118,6 +118,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + {"ptrack.map.mmap", false}, + {"ptrack.map", false}, + {"ptrack.map.tmp", false}, @@ -195,57 +201,77 @@ index 197163d5544..fc846e78175 100644 {NULL, false} }; diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h -index 80241455357..50dca7bf6f4 100644 +index 7c9f319b67..1e29827030 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h -@@ -367,7 +367,7 @@ typedef enum ProcessingMode +@@ -379,7 +379,7 @@ typedef enum ProcessingMode NormalProcessing /* normal processing */ } ProcessingMode; - + -extern ProcessingMode Mode; +extern PGDLLIMPORT ProcessingMode Mode; - + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h -index 9a26295c8e8..dc72b27a10d 100644 ---- a/src/include/port/pg_crc32c.h -+++ b/src/include/port/pg_crc32c.h -@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le - #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) - - extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); --extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); -- -+extern -+#ifndef FRONTEND -+PGDLLIMPORT -+#endif -+pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); - #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK - extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); - #endif +diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h +index 433755e279..de06d3b0cf 100644 +--- a/src/include/storage/checksum.h ++++ b/src/include/storage/checksum.h +@@ -14,6 +14,7 @@ + #define CHECKSUM_H + + #include "storage/block.h" ++#include "port/pg_crc32c.h" + + /* + * Compute the checksum for a Postgres page. The page must be aligned on a +@@ -21,4 +22,11 @@ + */ + extern uint16 pg_checksum_page(char *page, BlockNumber blkno); + ++/* ++* Wrapper function for COMP_CRC32C macro. Was added to avoid ++* FRONTEND macro use for pg_comp_crc32c pointer in windows build. ++*/ ++extern pg_crc32c ++comp_crc32c(pg_crc32c *crc, const void *data, size_t len); ++ + #endif /* CHECKSUM_H */ +diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h +index a49d27febb..459c938018 100644 +--- a/src/include/storage/checksum_impl.h ++++ b/src/include/storage/checksum_impl.h +@@ -213,3 +213,9 @@ pg_checksum_page(char *page, BlockNumber blkno) + */ + return (checksum % 65535) + 1; + } ++ ++pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) ++{ ++ COMP_CRC32C(*crc, data, len); ++ return *crc; ++} diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h -index 4fef3e21072..e55430879c3 100644 +index 4fef3e2107..e55430879c 100644 --- a/src/include/storage/copydir.h +++ b/src/include/storage/copydir.h @@ -13,6 +13,9 @@ #ifndef COPYDIR_H #define COPYDIR_H - + +typedef void (*copydir_hook_type) (const char *path); +extern PGDLLIMPORT copydir_hook_type copydir_hook; + extern void copydir(char *fromdir, char *todir, bool recurse); extern void copy_file(char *fromfile, char *tofile); - + diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h -index 0298ed1a2bc..24c684771d0 100644 +index 0298ed1a2b..24c684771d 100644 --- a/src/include/storage/smgr.h +++ b/src/include/storage/smgr.h @@ -116,6 +116,17 @@ extern void AtEOXact_SMgr(void); /* internals: move me elsewhere -- ay 7/94 */ - + /* in md.c */ + +typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, diff --git a/patches/REL_12_STABLE-ptrack-core.diff b/patches/REL_12_STABLE-ptrack-core.diff index d8c00e0..ba246ce 100644 --- a/patches/REL_12_STABLE-ptrack-core.diff +++ b/patches/REL_12_STABLE-ptrack-core.diff @@ -1,11 +1,17 @@ +commit 0504f29dfe157db0bb24c3a1196476e58b1ee263 +Author: Anton A. Melnikov +Date: Mon Nov 8 10:53:11 2021 +0300 + + [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c pointer definition. + diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c -index 3bc26568eb7..aa282bfe0ab 100644 +index efc458d80b..3479214b00 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c -@@ -210,6 +210,13 @@ static const struct exclude_list_item excludeFiles[] = +@@ -209,6 +209,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + /* + * Skip all transient ptrack files, but do copy ptrack.map, since it may + * be successfully used immediately after backup. TODO: check, test? @@ -16,7 +22,7 @@ index 3bc26568eb7..aa282bfe0ab 100644 /* end of list */ {NULL, false} }; -@@ -225,6 +232,11 @@ static const struct exclude_list_item noChecksumFiles[] = { +@@ -224,6 +231,11 @@ static const struct exclude_list_item noChecksumFiles[] = { {"pg_filenode.map", false}, {"pg_internal.init", true}, {"PG_VERSION", false}, @@ -29,13 +35,13 @@ index 3bc26568eb7..aa282bfe0ab 100644 {"config_exec_params", true}, #endif diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c -index 30f6200a86f..53e3b22c3e4 100644 +index 30f6200a86..53e3b22c3e 100644 --- a/src/backend/storage/file/copydir.c +++ b/src/backend/storage/file/copydir.c @@ -27,6 +27,8 @@ #include "miscadmin.h" #include "pgstat.h" - + +copydir_hook_type copydir_hook = NULL; + /* @@ -44,7 +50,7 @@ index 30f6200a86f..53e3b22c3e4 100644 @@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) } FreeDir(xldir); - + + if (copydir_hook) + copydir_hook(todir); + @@ -52,46 +58,46 @@ index 30f6200a86f..53e3b22c3e4 100644 * Be paranoid here and fsync all files to ensure the copy is really done. * But if fsync is disabled, we're done. diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c -index 050cee5f9a9..75cf67d464f 100644 +index 0c0e26b522..7d70db2778 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -86,6 +86,8 @@ typedef struct _MdfdVec - + static MemoryContext MdCxt; /* context for all MdfdVec objects */ - + +mdextend_hook_type mdextend_hook = NULL; +mdwrite_hook_type mdwrite_hook = NULL; - + /* Populate a file tag describing an md.c segment file. */ #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ -@@ -422,6 +424,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -461,6 +463,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); - + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + + if (mdextend_hook) + mdextend_hook(reln->smgr_rnode, forknum, blocknum); } - + /* -@@ -692,6 +697,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, - +@@ -731,6 +736,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); + + if (mdwrite_hook) + mdwrite_hook(reln->smgr_rnode, forknum, blocknum); } - + /* diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c -index aff3e885f36..4fffa5df17c 100644 +index aff3e885f3..4fffa5df17 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c @@ -75,6 +75,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ static CycleCtr sync_cycle_ctr = 0; static CycleCtr checkpoint_cycle_ctr = 0; - + +ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; + /* Intervals for calling AbsorbSyncRequests */ @@ -100,7 +106,7 @@ index aff3e885f36..4fffa5df17c 100644 @@ -420,6 +422,9 @@ ProcessSyncRequests(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; - + + if (ProcessSyncRequests_hook) + ProcessSyncRequests_hook(); + @@ -108,19 +114,19 @@ index aff3e885f36..4fffa5df17c 100644 sync_in_progress = false; } diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c -index a70e79c4891..712f985f3e8 100644 +index 1dd8c5674f..741e0c4e92 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -581,7 +581,6 @@ static char *recovery_target_xid_string; static char *recovery_target_name_string; static char *recovery_target_lsn_string; - + - /* should be static, but commands/variable.c needs to get at this */ char *role_string; - + diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c -index 03c3da3d730..fdfe5c1318e 100644 +index 2a749266ad..d8184e6ddc 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -113,6 +113,11 @@ static const struct exclude_list_item skip[] = { @@ -136,29 +142,29 @@ index 03c3da3d730..fdfe5c1318e 100644 {"config_exec_params", true}, #endif diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c -index 349347593cf..99d1e0a4fc0 100644 +index fc79896126..129c587f7e 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c -@@ -84,6 +84,7 @@ static void RewriteControlFile(void); +@@ -85,6 +85,7 @@ static void RewriteControlFile(void); static void FindEndOfXLOG(void); static void KillExistingXLOG(void); static void KillExistingArchiveStatus(void); +static void KillExistingPtrack(void); static void WriteEmptyXLOG(void); static void usage(void); - -@@ -513,6 +514,7 @@ main(int argc, char *argv[]) + +@@ -522,6 +523,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); + KillExistingPtrack(); WriteEmptyXLOG(); - + printf(_("Write-ahead log reset\n")); -@@ -1121,6 +1123,53 @@ KillExistingArchiveStatus(void) +@@ -1130,6 +1132,53 @@ KillExistingArchiveStatus(void) } } - + +/* + * Remove existing ptrack files + */ @@ -206,17 +212,17 @@ index 349347593cf..99d1e0a4fc0 100644 + } +} + - + /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c -index 56f83d2fb2f..60bb7bf7a3b 100644 +index 72299f00ea..2548143da5 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -117,6 +117,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + {"ptrack.map.mmap", false}, + {"ptrack.map", false}, + {"ptrack.map.tmp", false}, @@ -225,58 +231,78 @@ index 56f83d2fb2f..60bb7bf7a3b 100644 {NULL, false} }; diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h -index 61a24c2e3c6..cbd46d0cb02 100644 +index 11e9f5fcc9..045a20f3f4 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h -@@ -369,7 +369,7 @@ typedef enum ProcessingMode +@@ -381,7 +381,7 @@ typedef enum ProcessingMode NormalProcessing /* normal processing */ } ProcessingMode; - + -extern ProcessingMode Mode; +extern PGDLLIMPORT ProcessingMode Mode; - + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h -index fbd079d2439..01682035e0b 100644 ---- a/src/include/port/pg_crc32c.h -+++ b/src/include/port/pg_crc32c.h -@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le - #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) - - extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); --extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); -- -+extern -+#ifndef FRONTEND -+PGDLLIMPORT -+#endif -+pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); - #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK - extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); - #endif +diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h +index 7ef32a3baa..3778c8f773 100644 +--- a/src/include/storage/checksum.h ++++ b/src/include/storage/checksum.h +@@ -14,6 +14,7 @@ + #define CHECKSUM_H + + #include "storage/block.h" ++#include "port/pg_crc32c.h" + + /* + * Compute the checksum for a Postgres page. The page must be aligned on a +@@ -21,4 +22,11 @@ + */ + extern uint16 pg_checksum_page(char *page, BlockNumber blkno); + ++/* ++* Wrapper function for COMP_CRC32C macro. Was added to avoid ++* FRONTEND macro use for pg_comp_crc32c pointer in windows build. ++*/ ++extern pg_crc32c ++comp_crc32c(pg_crc32c *crc, const void *data, size_t len); ++ + #endif /* CHECKSUM_H */ +diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h +index 4b87610796..e202b25ccc 100644 +--- a/src/include/storage/checksum_impl.h ++++ b/src/include/storage/checksum_impl.h +@@ -213,3 +213,9 @@ pg_checksum_page(char *page, BlockNumber blkno) + */ + return (checksum % 65535) + 1; + } ++ ++pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) ++{ ++ COMP_CRC32C(*crc, data, len); ++ return *crc; ++} diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h -index 525cc6203e1..9481e1c5a88 100644 +index 525cc6203e..9481e1c5a8 100644 --- a/src/include/storage/copydir.h +++ b/src/include/storage/copydir.h @@ -13,6 +13,9 @@ #ifndef COPYDIR_H #define COPYDIR_H - + +typedef void (*copydir_hook_type) (const char *path); +extern PGDLLIMPORT copydir_hook_type copydir_hook; + extern void copydir(char *fromdir, char *todir, bool recurse); extern void copy_file(char *fromfile, char *tofile); - + diff --git a/src/include/storage/md.h b/src/include/storage/md.h -index df24b931613..b32c1e9500f 100644 +index df24b93161..b32c1e9500 100644 --- a/src/include/storage/md.h +++ b/src/include/storage/md.h @@ -19,6 +19,13 @@ #include "storage/smgr.h" #include "storage/sync.h" - + +typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, + ForkNumber forknum, BlockNumber blocknum); +extern PGDLLIMPORT mdextend_hook_type mdextend_hook; @@ -288,13 +314,13 @@ index df24b931613..b32c1e9500f 100644 extern void mdinit(void); extern void mdclose(SMgrRelation reln, ForkNumber forknum); diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h -index 16428c5f5fb..6b0cd8f8eea 100644 +index 16428c5f5f..6b0cd8f8ee 100644 --- a/src/include/storage/sync.h +++ b/src/include/storage/sync.h @@ -50,6 +50,9 @@ typedef struct FileTag uint32 segno; } FileTag; - + +typedef void (*ProcessSyncRequests_hook_type) (void); +extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook; + diff --git a/patches/REL_13_STABLE-ptrack-core.diff b/patches/REL_13_STABLE-ptrack-core.diff index 3491700..6cde3fa 100644 --- a/patches/REL_13_STABLE-ptrack-core.diff +++ b/patches/REL_13_STABLE-ptrack-core.diff @@ -1,8 +1,8 @@ -commit a14ac459d71528c64df00c693e9c71ac70d3ba29 -Author: anastasia -Date: Mon Oct 19 14:53:06 2020 +0300 +commit 2ffe8b4a2c2cb65a327ac85a02f52d8b74f244f9 +Author: Anton A. Melnikov +Date: Mon Nov 8 10:36:20 2021 +0300 - add ptrack 2.0 + [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c pointer definition. diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 50ae1f16d0..721b926ad2 100644 @@ -11,7 +11,7 @@ index 50ae1f16d0..721b926ad2 100644 @@ -233,6 +233,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + /* + * Skip all transient ptrack files, but do copy ptrack.map, since it may + * be successfully used immediately after backup. TODO: check, test? @@ -41,7 +41,7 @@ index 0cf598dd0c..c9c44a4ae7 100644 @@ -27,6 +27,8 @@ #include "storage/copydir.h" #include "storage/fd.h" - + +copydir_hook_type copydir_hook = NULL; + /* @@ -50,7 +50,7 @@ index 0cf598dd0c..c9c44a4ae7 100644 @@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) } FreeDir(xldir); - + + if (copydir_hook) + copydir_hook(todir); + @@ -58,55 +58,55 @@ index 0cf598dd0c..c9c44a4ae7 100644 * Be paranoid here and fsync all files to ensure the copy is really done. * But if fsync is disabled, we're done. diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c -index 0eacd461cd..c2ef404a1a 100644 +index 51617e17c1..ce9938e533 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -87,6 +87,8 @@ typedef struct _MdfdVec - + static MemoryContext MdCxt; /* context for all MdfdVec objects */ - + +mdextend_hook_type mdextend_hook = NULL; +mdwrite_hook_type mdwrite_hook = NULL; - + /* Populate a file tag describing an md.c segment file. */ #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ -@@ -435,6 +437,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -474,6 +476,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); - + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + + if (mdextend_hook) + mdextend_hook(reln->smgr_rnode, forknum, blocknum); } - + /* -@@ -721,6 +726,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, - +@@ -760,6 +765,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); + + if (mdwrite_hook) + mdwrite_hook(reln->smgr_rnode, forknum, blocknum); } - + /* diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c -index 3ded2cdd71..3a596a59f7 100644 +index 4ce2e687d3..e9ba3fe903 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c -@@ -75,6 +75,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ +@@ -76,6 +76,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ static CycleCtr sync_cycle_ctr = 0; static CycleCtr checkpoint_cycle_ctr = 0; - + +ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; + /* Intervals for calling AbsorbSyncRequests */ #define FSYNCS_PER_ABSORB 10 #define UNLINKS_PER_ABSORB 10 -@@ -420,6 +422,9 @@ ProcessSyncRequests(void) +@@ -444,6 +446,9 @@ ProcessSyncRequests(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; - + + if (ProcessSyncRequests_hook) + ProcessSyncRequests_hook(); + @@ -114,19 +114,19 @@ index 3ded2cdd71..3a596a59f7 100644 sync_in_progress = false; } diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c -index 1683629ee3..d2fc154576 100644 +index f7a3297161..ff9157697f 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -620,7 +620,6 @@ static char *recovery_target_xid_string; static char *recovery_target_name_string; static char *recovery_target_lsn_string; - + - /* should be static, but commands/variable.c needs to get at this */ char *role_string; - + diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c -index ffdc23945c..7ae95866ce 100644 +index 52e873159d..5cc334b44c 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -114,6 +114,11 @@ static const struct exclude_list_item skip[] = { @@ -142,29 +142,29 @@ index ffdc23945c..7ae95866ce 100644 {"config_exec_params", true}, #endif diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c -index 233441837f..cf7bd073bf 100644 +index c8cbd11fc2..f318d4c1a3 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c -@@ -84,6 +84,7 @@ static void RewriteControlFile(void); +@@ -85,6 +85,7 @@ static void RewriteControlFile(void); static void FindEndOfXLOG(void); static void KillExistingXLOG(void); static void KillExistingArchiveStatus(void); +static void KillExistingPtrack(void); static void WriteEmptyXLOG(void); static void usage(void); - -@@ -513,6 +514,7 @@ main(int argc, char *argv[]) + +@@ -522,6 +523,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); + KillExistingPtrack(); WriteEmptyXLOG(); - + printf(_("Write-ahead log reset\n")); -@@ -1102,6 +1104,53 @@ KillExistingArchiveStatus(void) +@@ -1111,6 +1113,53 @@ KillExistingArchiveStatus(void) } } - + +/* + * Remove existing ptrack files + */ @@ -212,7 +212,7 @@ index 233441837f..cf7bd073bf 100644 + } +} + - + /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c @@ -222,7 +222,7 @@ index fbb97b5cf1..6cd7f2ae3e 100644 @@ -124,6 +124,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + {"ptrack.map.mmap", false}, + {"ptrack.map", false}, + {"ptrack.map.tmp", false}, @@ -231,36 +231,56 @@ index fbb97b5cf1..6cd7f2ae3e 100644 {NULL, false} }; diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h -index 72e3352398..5c2e016501 100644 +index 6b2b4343a0..66b6d24e0a 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h -@@ -388,7 +388,7 @@ typedef enum ProcessingMode +@@ -400,7 +400,7 @@ typedef enum ProcessingMode NormalProcessing /* normal processing */ } ProcessingMode; - + -extern ProcessingMode Mode; +extern PGDLLIMPORT ProcessingMode Mode; - + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h -index 3c6f906683..a7355f7ad1 100644 ---- a/src/include/port/pg_crc32c.h -+++ b/src/include/port/pg_crc32c.h -@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le - #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) - - extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); --extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); -- -+extern -+#ifndef FRONTEND -+PGDLLIMPORT -+#endif -+pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); - #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK - extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); - #endif +diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h +index 6e77744cbc..81a11cffcc 100644 +--- a/src/include/storage/checksum.h ++++ b/src/include/storage/checksum.h +@@ -14,6 +14,7 @@ + #define CHECKSUM_H + + #include "storage/block.h" ++#include "port/pg_crc32c.h" + + /* + * Compute the checksum for a Postgres page. The page must be aligned on a +@@ -21,4 +22,11 @@ + */ + extern uint16 pg_checksum_page(char *page, BlockNumber blkno); + ++/* ++* Wrapper function for COMP_CRC32C macro. Was added to avoid ++* FRONTEND macro use for pg_comp_crc32c pointer in windows build. ++*/ ++extern pg_crc32c ++comp_crc32c(pg_crc32c *crc, const void *data, size_t len); ++ + #endif /* CHECKSUM_H */ +diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h +index 364acfafd2..f4813e22aa 100644 +--- a/src/include/storage/checksum_impl.h ++++ b/src/include/storage/checksum_impl.h +@@ -213,3 +213,9 @@ pg_checksum_page(char *page, BlockNumber blkno) + */ + return (uint16) ((checksum % 65535) + 1); + } ++ ++pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) ++{ ++ COMP_CRC32C(*crc, data, len); ++ return *crc; ++} diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h index 5d28f59c1d..0d3f04d8af 100644 --- a/src/include/storage/copydir.h @@ -268,13 +288,13 @@ index 5d28f59c1d..0d3f04d8af 100644 @@ -13,6 +13,9 @@ #ifndef COPYDIR_H #define COPYDIR_H - + +typedef void (*copydir_hook_type) (const char *path); +extern PGDLLIMPORT copydir_hook_type copydir_hook; + extern void copydir(char *fromdir, char *todir, bool recurse); extern void copy_file(char *fromfile, char *tofile); - + diff --git a/src/include/storage/md.h b/src/include/storage/md.h index 07fd1bb7d0..5294811bc8 100644 --- a/src/include/storage/md.h @@ -282,7 +302,7 @@ index 07fd1bb7d0..5294811bc8 100644 @@ -19,6 +19,13 @@ #include "storage/smgr.h" #include "storage/sync.h" - + +typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, + ForkNumber forknum, BlockNumber blocknum); +extern PGDLLIMPORT mdextend_hook_type mdextend_hook; @@ -300,7 +320,7 @@ index e16ab8e711..88da9686eb 100644 @@ -50,6 +50,9 @@ typedef struct FileTag uint32 segno; } FileTag; - + +typedef void (*ProcessSyncRequests_hook_type) (void); +extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook; + diff --git a/patches/REL_14_STABLE-ptrack-core.diff b/patches/REL_14_STABLE-ptrack-core.diff index bb2d95b..8dfbac4 100644 --- a/patches/REL_14_STABLE-ptrack-core.diff +++ b/patches/REL_14_STABLE-ptrack-core.diff @@ -1,11 +1,8 @@ -commit 84a43938a670336fbb203b85c7f5ea15d642aec2 +commit 6b7ba7c634fb34bfaad63b0b9184fb57584880db Author: Anton A. Melnikov -Date: Fri Oct 29 08:45:13 2021 +0300 +Date: Mon Nov 8 09:19:07 2021 +0300 - [PGPRO-5771] Add wrapper function for COMP_CRC32C and restore - pg_comp_crc32c pointer definition. - - Tags: ptrack + [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c pointer definition. diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index e09108d0ec..6858c7b670 100644 @@ -14,7 +11,7 @@ index e09108d0ec..6858c7b670 100644 @@ -225,6 +225,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + /* + * Skip all transient ptrack files, but do copy ptrack.map, since it may + * be successfully used immediately after backup. TODO: check, test? @@ -44,7 +41,7 @@ index da8b7cbeca..6a817f45a5 100644 @@ -27,6 +27,8 @@ #include "storage/copydir.h" #include "storage/fd.h" - + +copydir_hook_type copydir_hook = NULL; + /* @@ -53,7 +50,7 @@ index da8b7cbeca..6a817f45a5 100644 @@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) } FreeDir(xldir); - + + if (copydir_hook) + copydir_hook(todir); + @@ -65,51 +62,51 @@ index b4bca7eed6..90d479526c 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -87,6 +87,8 @@ typedef struct _MdfdVec - + static MemoryContext MdCxt; /* context for all MdfdVec objects */ - + +mdextend_hook_type mdextend_hook = NULL; +mdwrite_hook_type mdwrite_hook = NULL; - + /* Populate a file tag describing an md.c segment file. */ #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ @@ -463,6 +465,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); - + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + + if (mdextend_hook) + mdextend_hook(reln->smgr_rnode, forknum, blocknum); } - + /* @@ -749,6 +754,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, - + if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); + + if (mdwrite_hook) + mdwrite_hook(reln->smgr_rnode, forknum, blocknum); } - + /* diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c -index bc3ceb2712..6c72d24c95 100644 +index a12b357275..e52c2910f7 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c -@@ -78,6 +78,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ +@@ -79,6 +79,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ static CycleCtr sync_cycle_ctr = 0; static CycleCtr checkpoint_cycle_ctr = 0; - + +ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; + /* Intervals for calling AbsorbSyncRequests */ #define FSYNCS_PER_ABSORB 10 #define UNLINKS_PER_ABSORB 10 -@@ -441,6 +443,9 @@ ProcessSyncRequests(void) +@@ -465,6 +467,9 @@ ProcessSyncRequests(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; - + + if (ProcessSyncRequests_hook) + ProcessSyncRequests_hook(); + @@ -123,11 +120,11 @@ index 6652a60ec3..051cd6359d 100644 @@ -659,7 +659,6 @@ static char *recovery_target_xid_string; static char *recovery_target_name_string; static char *recovery_target_lsn_string; - + - /* should be static, but commands/variable.c needs to get at this */ char *role_string; - + diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c index 831cf42d3a..891302ba2f 100644 --- a/src/bin/pg_checksums/pg_checksums.c @@ -155,19 +152,19 @@ index f8d5ecb6ad..4027bdf3df 100644 +static void KillExistingPtrack(void); static void WriteEmptyXLOG(void); static void usage(void); - + @@ -522,6 +523,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); + KillExistingPtrack(); WriteEmptyXLOG(); - + printf(_("Write-ahead log reset\n")); @@ -1111,6 +1113,53 @@ KillExistingArchiveStatus(void) } } - + +/* + * Remove existing ptrack files + */ @@ -215,7 +212,7 @@ index f8d5ecb6ad..4027bdf3df 100644 + } +} + - + /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c @@ -225,7 +222,7 @@ index 2618b4c957..bbba813fe7 100644 @@ -158,6 +158,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + {"ptrack.map.mmap", false}, + {"ptrack.map", false}, + {"ptrack.map.tmp", false}, @@ -233,32 +230,6 @@ index 2618b4c957..bbba813fe7 100644 /* end of list */ {NULL, false} }; -diff --git a/src/common/checksum_helper.c b/src/common/checksum_helper.c -index 431e247d59..405247a597 100644 ---- a/src/common/checksum_helper.c -+++ b/src/common/checksum_helper.c -@@ -230,3 +230,9 @@ pg_checksum_final(pg_checksum_context *context, uint8 *output) - Assert(retval <= PG_CHECKSUM_MAX_LENGTH); - return retval; - } -+ -+pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) -+{ -+ COMP_CRC32C(*crc, data, len); -+ return *crc; -+} -diff --git a/src/include/common/checksum_helper.h b/src/include/common/checksum_helper.h -index cac7570ea1..8c002860cf 100644 ---- a/src/include/common/checksum_helper.h -+++ b/src/include/common/checksum_helper.h -@@ -68,5 +68,7 @@ extern int pg_checksum_init(pg_checksum_context *, pg_checksum_type); - extern int pg_checksum_update(pg_checksum_context *, const uint8 *input, - size_t len); - extern int pg_checksum_final(pg_checksum_context *, uint8 *output); -+extern pg_crc32c -+comp_crc32c(pg_crc32c *crc, const void *data, size_t len); - - #endif diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h index 3f155ce4f8..ce8dc0bee5 100644 --- a/src/include/miscadmin.h @@ -266,24 +237,50 @@ index 3f155ce4f8..ce8dc0bee5 100644 @@ -401,7 +401,7 @@ typedef enum ProcessingMode NormalProcessing /* normal processing */ } ProcessingMode; - + -extern ProcessingMode Mode; +extern PGDLLIMPORT ProcessingMode Mode; - + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h -index f3c4107ff9..05b2e63775 100644 ---- a/src/include/port/pg_crc32c.h -+++ b/src/include/port/pg_crc32c.h -@@ -70,7 +70,6 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le - - extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); - extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); -- - #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK - extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); - #endif +diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h +index 80d2359192..ac55286088 100644 +--- a/src/include/storage/checksum.h ++++ b/src/include/storage/checksum.h +@@ -14,6 +14,7 @@ + #define CHECKSUM_H + + #include "storage/block.h" ++#include "port/pg_crc32c.h" + + /* + * Compute the checksum for a Postgres page. The page must be aligned on a +@@ -21,4 +22,11 @@ + */ + extern uint16 pg_checksum_page(char *page, BlockNumber blkno); + ++/* ++* Wrapper function for COMP_CRC32C macro. Was added to avoid ++* FRONTEND macro use for pg_comp_crc32c pointer in windows build. ++*/ ++extern pg_crc32c ++comp_crc32c(pg_crc32c *crc, const void *data, size_t len); ++ + #endif /* CHECKSUM_H */ +diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h +index ef30268330..19a0248538 100644 +--- a/src/include/storage/checksum_impl.h ++++ b/src/include/storage/checksum_impl.h +@@ -213,3 +213,9 @@ pg_checksum_page(char *page, BlockNumber blkno) + */ + return (uint16) ((checksum % 65535) + 1); + } ++ ++pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) ++{ ++ COMP_CRC32C(*crc, data, len); ++ return *crc; ++} diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h index 2c3936b0da..c6691c0be0 100644 --- a/src/include/storage/copydir.h @@ -291,13 +288,13 @@ index 2c3936b0da..c6691c0be0 100644 @@ -13,6 +13,9 @@ #ifndef COPYDIR_H #define COPYDIR_H - + +typedef void (*copydir_hook_type) (const char *path); +extern PGDLLIMPORT copydir_hook_type copydir_hook; + extern void copydir(char *fromdir, char *todir, bool recurse); extern void copy_file(char *fromfile, char *tofile); - + diff --git a/src/include/storage/md.h b/src/include/storage/md.h index 752b440864..61c18a169f 100644 --- a/src/include/storage/md.h @@ -305,7 +302,7 @@ index 752b440864..61c18a169f 100644 @@ -19,6 +19,13 @@ #include "storage/smgr.h" #include "storage/sync.h" - + +typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, + ForkNumber forknum, BlockNumber blocknum); +extern PGDLLIMPORT mdextend_hook_type mdextend_hook; @@ -323,7 +320,7 @@ index fbdf34f762..455de202b6 100644 @@ -55,6 +55,9 @@ typedef struct FileTag uint32 segno; } FileTag; - + +typedef void (*ProcessSyncRequests_hook_type) (void); +extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook; + From 0e000fa5a796cda3df6f2040adf5c6996400df7d Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Sat, 12 Mar 2022 23:36:03 +0300 Subject: [PATCH 07/10] [PGPRO-5771] Revert changes for stable branches (last 6 commits). At first we decided to solve this task for master branch only. Tags: ptrack --- engine.c | 13 -- patches/REL_11_STABLE-ptrack-core.diff | 138 +++++++++------------ patches/REL_12_STABLE-ptrack-core.diff | 152 ++++++++++------------- patches/REL_13_STABLE-ptrack-core.diff | 142 +++++++++------------ patches/REL_14_STABLE-ptrack-core.diff | 164 +++++++++++-------------- pg_probackup | 1 - 6 files changed, 252 insertions(+), 358 deletions(-) delete mode 160000 pg_probackup diff --git a/engine.c b/engine.c index 9fbec3e..42fa65a 100644 --- a/engine.c +++ b/engine.c @@ -31,11 +31,7 @@ #include "access/xlog.h" #include "catalog/pg_tablespace.h" #include "miscadmin.h" -#ifdef WIN32 -#include "storage/checksum.h" -#else #include "port/pg_crc32c.h" -#endif #include "storage/copydir.h" #if PG_VERSION_NUM >= 120000 #include "storage/md.h" @@ -77,12 +73,7 @@ ptrack_file_exists(const char *path) static void ptrack_write_chunk(int fd, pg_crc32c *crc, char *chunk, size_t size) { - -#ifdef WIN32 - comp_crc32c(crc, (char *) chunk, size); -#else COMP_CRC32C(*crc, (char *) chunk, size); -#endif if (write(fd, chunk, size) != size) { @@ -202,11 +193,7 @@ ptrackMapReadFromFile(const char *ptrack_path) pg_crc32c *file_crc; INIT_CRC32C(crc); -#ifdef WIN32 - comp_crc32c(&crc, (char *) ptrack_map, PtrackCrcOffset); -#else COMP_CRC32C(crc, (char *) ptrack_map, PtrackCrcOffset); -#endif FIN_CRC32C(crc); file_crc = (pg_crc32c *) ((char *) ptrack_map + PtrackCrcOffset); diff --git a/patches/REL_11_STABLE-ptrack-core.diff b/patches/REL_11_STABLE-ptrack-core.diff index bf4a834..a8207f5 100644 --- a/patches/REL_11_STABLE-ptrack-core.diff +++ b/patches/REL_11_STABLE-ptrack-core.diff @@ -1,17 +1,11 @@ -commit 5255ff7a6051f56689e0db2a0fa7e9a6e4086d66 -Author: Anton A. Melnikov -Date: Mon Nov 8 11:11:45 2021 +0300 - - [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c pointer definition. - diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c -index 3e53b3df6f..f76bfc2a64 100644 +index 3e53b3df6fb..f76bfc2a646 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c @@ -209,6 +209,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + /* + * Skip all transient ptrack files, but do copy ptrack.map, since it may + * be successfully used immediately after backup. TODO: check, test? @@ -34,13 +28,13 @@ index 3e53b3df6f..f76bfc2a64 100644 {"config_exec_params", true}, #endif diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c -index 4a0d23b11e..d59009a4c8 100644 +index 4a0d23b11e3..d59009a4c8c 100644 --- a/src/backend/storage/file/copydir.c +++ b/src/backend/storage/file/copydir.c @@ -27,6 +27,8 @@ #include "miscadmin.h" #include "pgstat.h" - + +copydir_hook_type copydir_hook = NULL; + /* @@ -49,7 +43,7 @@ index 4a0d23b11e..d59009a4c8 100644 @@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) } FreeDir(xldir); - + + if (copydir_hook) + copydir_hook(todir); + @@ -57,50 +51,50 @@ index 4a0d23b11e..d59009a4c8 100644 * Be paranoid here and fsync all files to ensure the copy is really done. * But if fsync is disabled, we're done. diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c -index fa29e7041f..8bda639eda 100644 +index 200cc7f657a..d0dcb5c0287 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -39,6 +39,7 @@ #include "utils/memutils.h" #include "pg_trace.h" - + +ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; - + /* intervals for calling AbsorbFsyncRequests in mdsync and mdpostckpt */ #define FSYNCS_PER_ABSORB 10 @@ -114,6 +115,8 @@ typedef struct _MdfdVec - + static MemoryContext MdCxt; /* context for all MdfdVec objects */ - + +mdextend_hook_type mdextend_hook = NULL; +mdwrite_hook_type mdwrite_hook = NULL; - + /* * In some contexts (currently, standalone backends and the checkpointer) -@@ -600,6 +603,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -558,6 +561,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); - + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + + if (mdextend_hook) + mdextend_hook(reln->smgr_rnode, forknum, blocknum); } - + /* -@@ -893,6 +899,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, - +@@ -851,6 +857,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); + + if (mdwrite_hook) + mdwrite_hook(reln->smgr_rnode, forknum, blocknum); } - + /* -@@ -1371,6 +1380,9 @@ mdsync(void) +@@ -1329,6 +1338,9 @@ mdsync(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; - + + if (ProcessSyncRequests_hook) + ProcessSyncRequests_hook(); + @@ -108,29 +102,29 @@ index fa29e7041f..8bda639eda 100644 mdsync_in_progress = false; } diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c -index 611edf6ade..ec1c1212bd 100644 +index 6fb403a5a8a..6e31ccb3e0f 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c -@@ -85,6 +85,7 @@ static void RewriteControlFile(void); +@@ -84,6 +84,7 @@ static void RewriteControlFile(void); static void FindEndOfXLOG(void); static void KillExistingXLOG(void); static void KillExistingArchiveStatus(void); +static void KillExistingPtrack(void); static void WriteEmptyXLOG(void); static void usage(void); - -@@ -525,6 +526,7 @@ main(int argc, char *argv[]) + +@@ -516,6 +517,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); + KillExistingPtrack(); WriteEmptyXLOG(); - + printf(_("Write-ahead log reset\n")); -@@ -1210,6 +1212,57 @@ KillExistingArchiveStatus(void) +@@ -1201,6 +1203,57 @@ KillExistingArchiveStatus(void) } } - + +/* + * Remove existing ptrack files + */ @@ -182,17 +176,17 @@ index 611edf6ade..ec1c1212bd 100644 + } +} + - + /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c -index 8ea7fafa27..997168fcac 100644 +index 197163d5544..fc846e78175 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -118,6 +118,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + {"ptrack.map.mmap", false}, + {"ptrack.map", false}, + {"ptrack.map.tmp", false}, @@ -201,77 +195,57 @@ index 8ea7fafa27..997168fcac 100644 {NULL, false} }; diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h -index 7c9f319b67..1e29827030 100644 +index 80241455357..50dca7bf6f4 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h -@@ -379,7 +379,7 @@ typedef enum ProcessingMode +@@ -367,7 +367,7 @@ typedef enum ProcessingMode NormalProcessing /* normal processing */ } ProcessingMode; - + -extern ProcessingMode Mode; +extern PGDLLIMPORT ProcessingMode Mode; - + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h -index 433755e279..de06d3b0cf 100644 ---- a/src/include/storage/checksum.h -+++ b/src/include/storage/checksum.h -@@ -14,6 +14,7 @@ - #define CHECKSUM_H - - #include "storage/block.h" -+#include "port/pg_crc32c.h" - - /* - * Compute the checksum for a Postgres page. The page must be aligned on a -@@ -21,4 +22,11 @@ - */ - extern uint16 pg_checksum_page(char *page, BlockNumber blkno); - -+/* -+* Wrapper function for COMP_CRC32C macro. Was added to avoid -+* FRONTEND macro use for pg_comp_crc32c pointer in windows build. -+*/ -+extern pg_crc32c -+comp_crc32c(pg_crc32c *crc, const void *data, size_t len); -+ - #endif /* CHECKSUM_H */ -diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h -index a49d27febb..459c938018 100644 ---- a/src/include/storage/checksum_impl.h -+++ b/src/include/storage/checksum_impl.h -@@ -213,3 +213,9 @@ pg_checksum_page(char *page, BlockNumber blkno) - */ - return (checksum % 65535) + 1; - } -+ -+pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) -+{ -+ COMP_CRC32C(*crc, data, len); -+ return *crc; -+} +diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h +index 9a26295c8e8..dc72b27a10d 100644 +--- a/src/include/port/pg_crc32c.h ++++ b/src/include/port/pg_crc32c.h +@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le + #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) + + extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); +-extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); +- ++extern ++#ifndef FRONTEND ++PGDLLIMPORT ++#endif ++pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); + #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK + extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); + #endif diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h -index 4fef3e2107..e55430879c 100644 +index 4fef3e21072..e55430879c3 100644 --- a/src/include/storage/copydir.h +++ b/src/include/storage/copydir.h @@ -13,6 +13,9 @@ #ifndef COPYDIR_H #define COPYDIR_H - + +typedef void (*copydir_hook_type) (const char *path); +extern PGDLLIMPORT copydir_hook_type copydir_hook; + extern void copydir(char *fromdir, char *todir, bool recurse); extern void copy_file(char *fromfile, char *tofile); - + diff --git a/src/include/storage/smgr.h b/src/include/storage/smgr.h -index 0298ed1a2b..24c684771d 100644 +index 0298ed1a2bc..24c684771d0 100644 --- a/src/include/storage/smgr.h +++ b/src/include/storage/smgr.h @@ -116,6 +116,17 @@ extern void AtEOXact_SMgr(void); /* internals: move me elsewhere -- ay 7/94 */ - + /* in md.c */ + +typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, diff --git a/patches/REL_12_STABLE-ptrack-core.diff b/patches/REL_12_STABLE-ptrack-core.diff index ba246ce..d8c00e0 100644 --- a/patches/REL_12_STABLE-ptrack-core.diff +++ b/patches/REL_12_STABLE-ptrack-core.diff @@ -1,17 +1,11 @@ -commit 0504f29dfe157db0bb24c3a1196476e58b1ee263 -Author: Anton A. Melnikov -Date: Mon Nov 8 10:53:11 2021 +0300 - - [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c pointer definition. - diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c -index efc458d80b..3479214b00 100644 +index 3bc26568eb7..aa282bfe0ab 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c -@@ -209,6 +209,13 @@ static const struct exclude_list_item excludeFiles[] = +@@ -210,6 +210,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + /* + * Skip all transient ptrack files, but do copy ptrack.map, since it may + * be successfully used immediately after backup. TODO: check, test? @@ -22,7 +16,7 @@ index efc458d80b..3479214b00 100644 /* end of list */ {NULL, false} }; -@@ -224,6 +231,11 @@ static const struct exclude_list_item noChecksumFiles[] = { +@@ -225,6 +232,11 @@ static const struct exclude_list_item noChecksumFiles[] = { {"pg_filenode.map", false}, {"pg_internal.init", true}, {"PG_VERSION", false}, @@ -35,13 +29,13 @@ index efc458d80b..3479214b00 100644 {"config_exec_params", true}, #endif diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c -index 30f6200a86..53e3b22c3e 100644 +index 30f6200a86f..53e3b22c3e4 100644 --- a/src/backend/storage/file/copydir.c +++ b/src/backend/storage/file/copydir.c @@ -27,6 +27,8 @@ #include "miscadmin.h" #include "pgstat.h" - + +copydir_hook_type copydir_hook = NULL; + /* @@ -50,7 +44,7 @@ index 30f6200a86..53e3b22c3e 100644 @@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) } FreeDir(xldir); - + + if (copydir_hook) + copydir_hook(todir); + @@ -58,46 +52,46 @@ index 30f6200a86..53e3b22c3e 100644 * Be paranoid here and fsync all files to ensure the copy is really done. * But if fsync is disabled, we're done. diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c -index 0c0e26b522..7d70db2778 100644 +index 050cee5f9a9..75cf67d464f 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -86,6 +86,8 @@ typedef struct _MdfdVec - + static MemoryContext MdCxt; /* context for all MdfdVec objects */ - + +mdextend_hook_type mdextend_hook = NULL; +mdwrite_hook_type mdwrite_hook = NULL; - + /* Populate a file tag describing an md.c segment file. */ #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ -@@ -461,6 +463,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -422,6 +424,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); - + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + + if (mdextend_hook) + mdextend_hook(reln->smgr_rnode, forknum, blocknum); } - + /* -@@ -731,6 +736,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, - +@@ -692,6 +697,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); + + if (mdwrite_hook) + mdwrite_hook(reln->smgr_rnode, forknum, blocknum); } - + /* diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c -index aff3e885f3..4fffa5df17 100644 +index aff3e885f36..4fffa5df17c 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c @@ -75,6 +75,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ static CycleCtr sync_cycle_ctr = 0; static CycleCtr checkpoint_cycle_ctr = 0; - + +ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; + /* Intervals for calling AbsorbSyncRequests */ @@ -106,7 +100,7 @@ index aff3e885f3..4fffa5df17 100644 @@ -420,6 +422,9 @@ ProcessSyncRequests(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; - + + if (ProcessSyncRequests_hook) + ProcessSyncRequests_hook(); + @@ -114,19 +108,19 @@ index aff3e885f3..4fffa5df17 100644 sync_in_progress = false; } diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c -index 1dd8c5674f..741e0c4e92 100644 +index a70e79c4891..712f985f3e8 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -581,7 +581,6 @@ static char *recovery_target_xid_string; static char *recovery_target_name_string; static char *recovery_target_lsn_string; - + - /* should be static, but commands/variable.c needs to get at this */ char *role_string; - + diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c -index 2a749266ad..d8184e6ddc 100644 +index 03c3da3d730..fdfe5c1318e 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -113,6 +113,11 @@ static const struct exclude_list_item skip[] = { @@ -142,29 +136,29 @@ index 2a749266ad..d8184e6ddc 100644 {"config_exec_params", true}, #endif diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c -index fc79896126..129c587f7e 100644 +index 349347593cf..99d1e0a4fc0 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c -@@ -85,6 +85,7 @@ static void RewriteControlFile(void); +@@ -84,6 +84,7 @@ static void RewriteControlFile(void); static void FindEndOfXLOG(void); static void KillExistingXLOG(void); static void KillExistingArchiveStatus(void); +static void KillExistingPtrack(void); static void WriteEmptyXLOG(void); static void usage(void); - -@@ -522,6 +523,7 @@ main(int argc, char *argv[]) + +@@ -513,6 +514,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); + KillExistingPtrack(); WriteEmptyXLOG(); - + printf(_("Write-ahead log reset\n")); -@@ -1130,6 +1132,53 @@ KillExistingArchiveStatus(void) +@@ -1121,6 +1123,53 @@ KillExistingArchiveStatus(void) } } - + +/* + * Remove existing ptrack files + */ @@ -212,17 +206,17 @@ index fc79896126..129c587f7e 100644 + } +} + - + /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c -index 72299f00ea..2548143da5 100644 +index 56f83d2fb2f..60bb7bf7a3b 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c @@ -117,6 +117,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + {"ptrack.map.mmap", false}, + {"ptrack.map", false}, + {"ptrack.map.tmp", false}, @@ -231,78 +225,58 @@ index 72299f00ea..2548143da5 100644 {NULL, false} }; diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h -index 11e9f5fcc9..045a20f3f4 100644 +index 61a24c2e3c6..cbd46d0cb02 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h -@@ -381,7 +381,7 @@ typedef enum ProcessingMode +@@ -369,7 +369,7 @@ typedef enum ProcessingMode NormalProcessing /* normal processing */ } ProcessingMode; - + -extern ProcessingMode Mode; +extern PGDLLIMPORT ProcessingMode Mode; - + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h -index 7ef32a3baa..3778c8f773 100644 ---- a/src/include/storage/checksum.h -+++ b/src/include/storage/checksum.h -@@ -14,6 +14,7 @@ - #define CHECKSUM_H - - #include "storage/block.h" -+#include "port/pg_crc32c.h" - - /* - * Compute the checksum for a Postgres page. The page must be aligned on a -@@ -21,4 +22,11 @@ - */ - extern uint16 pg_checksum_page(char *page, BlockNumber blkno); - -+/* -+* Wrapper function for COMP_CRC32C macro. Was added to avoid -+* FRONTEND macro use for pg_comp_crc32c pointer in windows build. -+*/ -+extern pg_crc32c -+comp_crc32c(pg_crc32c *crc, const void *data, size_t len); -+ - #endif /* CHECKSUM_H */ -diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h -index 4b87610796..e202b25ccc 100644 ---- a/src/include/storage/checksum_impl.h -+++ b/src/include/storage/checksum_impl.h -@@ -213,3 +213,9 @@ pg_checksum_page(char *page, BlockNumber blkno) - */ - return (checksum % 65535) + 1; - } -+ -+pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) -+{ -+ COMP_CRC32C(*crc, data, len); -+ return *crc; -+} +diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h +index fbd079d2439..01682035e0b 100644 +--- a/src/include/port/pg_crc32c.h ++++ b/src/include/port/pg_crc32c.h +@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le + #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) + + extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); +-extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); +- ++extern ++#ifndef FRONTEND ++PGDLLIMPORT ++#endif ++pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); + #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK + extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); + #endif diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h -index 525cc6203e..9481e1c5a8 100644 +index 525cc6203e1..9481e1c5a88 100644 --- a/src/include/storage/copydir.h +++ b/src/include/storage/copydir.h @@ -13,6 +13,9 @@ #ifndef COPYDIR_H #define COPYDIR_H - + +typedef void (*copydir_hook_type) (const char *path); +extern PGDLLIMPORT copydir_hook_type copydir_hook; + extern void copydir(char *fromdir, char *todir, bool recurse); extern void copy_file(char *fromfile, char *tofile); - + diff --git a/src/include/storage/md.h b/src/include/storage/md.h -index df24b93161..b32c1e9500 100644 +index df24b931613..b32c1e9500f 100644 --- a/src/include/storage/md.h +++ b/src/include/storage/md.h @@ -19,6 +19,13 @@ #include "storage/smgr.h" #include "storage/sync.h" - + +typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, + ForkNumber forknum, BlockNumber blocknum); +extern PGDLLIMPORT mdextend_hook_type mdextend_hook; @@ -314,13 +288,13 @@ index df24b93161..b32c1e9500 100644 extern void mdinit(void); extern void mdclose(SMgrRelation reln, ForkNumber forknum); diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h -index 16428c5f5f..6b0cd8f8ee 100644 +index 16428c5f5fb..6b0cd8f8eea 100644 --- a/src/include/storage/sync.h +++ b/src/include/storage/sync.h @@ -50,6 +50,9 @@ typedef struct FileTag uint32 segno; } FileTag; - + +typedef void (*ProcessSyncRequests_hook_type) (void); +extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook; + diff --git a/patches/REL_13_STABLE-ptrack-core.diff b/patches/REL_13_STABLE-ptrack-core.diff index 6cde3fa..3491700 100644 --- a/patches/REL_13_STABLE-ptrack-core.diff +++ b/patches/REL_13_STABLE-ptrack-core.diff @@ -1,8 +1,8 @@ -commit 2ffe8b4a2c2cb65a327ac85a02f52d8b74f244f9 -Author: Anton A. Melnikov -Date: Mon Nov 8 10:36:20 2021 +0300 +commit a14ac459d71528c64df00c693e9c71ac70d3ba29 +Author: anastasia +Date: Mon Oct 19 14:53:06 2020 +0300 - [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c pointer definition. + add ptrack 2.0 diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c index 50ae1f16d0..721b926ad2 100644 @@ -11,7 +11,7 @@ index 50ae1f16d0..721b926ad2 100644 @@ -233,6 +233,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + /* + * Skip all transient ptrack files, but do copy ptrack.map, since it may + * be successfully used immediately after backup. TODO: check, test? @@ -41,7 +41,7 @@ index 0cf598dd0c..c9c44a4ae7 100644 @@ -27,6 +27,8 @@ #include "storage/copydir.h" #include "storage/fd.h" - + +copydir_hook_type copydir_hook = NULL; + /* @@ -50,7 +50,7 @@ index 0cf598dd0c..c9c44a4ae7 100644 @@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) } FreeDir(xldir); - + + if (copydir_hook) + copydir_hook(todir); + @@ -58,55 +58,55 @@ index 0cf598dd0c..c9c44a4ae7 100644 * Be paranoid here and fsync all files to ensure the copy is really done. * But if fsync is disabled, we're done. diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c -index 51617e17c1..ce9938e533 100644 +index 0eacd461cd..c2ef404a1a 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -87,6 +87,8 @@ typedef struct _MdfdVec - + static MemoryContext MdCxt; /* context for all MdfdVec objects */ - + +mdextend_hook_type mdextend_hook = NULL; +mdwrite_hook_type mdwrite_hook = NULL; - + /* Populate a file tag describing an md.c segment file. */ #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ -@@ -474,6 +476,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -435,6 +437,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); - + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + + if (mdextend_hook) + mdextend_hook(reln->smgr_rnode, forknum, blocknum); } - + /* -@@ -760,6 +765,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, - +@@ -721,6 +726,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); + + if (mdwrite_hook) + mdwrite_hook(reln->smgr_rnode, forknum, blocknum); } - + /* diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c -index 4ce2e687d3..e9ba3fe903 100644 +index 3ded2cdd71..3a596a59f7 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c -@@ -76,6 +76,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ +@@ -75,6 +75,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ static CycleCtr sync_cycle_ctr = 0; static CycleCtr checkpoint_cycle_ctr = 0; - + +ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; + /* Intervals for calling AbsorbSyncRequests */ #define FSYNCS_PER_ABSORB 10 #define UNLINKS_PER_ABSORB 10 -@@ -444,6 +446,9 @@ ProcessSyncRequests(void) +@@ -420,6 +422,9 @@ ProcessSyncRequests(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; - + + if (ProcessSyncRequests_hook) + ProcessSyncRequests_hook(); + @@ -114,19 +114,19 @@ index 4ce2e687d3..e9ba3fe903 100644 sync_in_progress = false; } diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c -index f7a3297161..ff9157697f 100644 +index 1683629ee3..d2fc154576 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -620,7 +620,6 @@ static char *recovery_target_xid_string; static char *recovery_target_name_string; static char *recovery_target_lsn_string; - + - /* should be static, but commands/variable.c needs to get at this */ char *role_string; - + diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c -index 52e873159d..5cc334b44c 100644 +index ffdc23945c..7ae95866ce 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -114,6 +114,11 @@ static const struct exclude_list_item skip[] = { @@ -142,29 +142,29 @@ index 52e873159d..5cc334b44c 100644 {"config_exec_params", true}, #endif diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c -index c8cbd11fc2..f318d4c1a3 100644 +index 233441837f..cf7bd073bf 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c -@@ -85,6 +85,7 @@ static void RewriteControlFile(void); +@@ -84,6 +84,7 @@ static void RewriteControlFile(void); static void FindEndOfXLOG(void); static void KillExistingXLOG(void); static void KillExistingArchiveStatus(void); +static void KillExistingPtrack(void); static void WriteEmptyXLOG(void); static void usage(void); - -@@ -522,6 +523,7 @@ main(int argc, char *argv[]) + +@@ -513,6 +514,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); + KillExistingPtrack(); WriteEmptyXLOG(); - + printf(_("Write-ahead log reset\n")); -@@ -1111,6 +1113,53 @@ KillExistingArchiveStatus(void) +@@ -1102,6 +1104,53 @@ KillExistingArchiveStatus(void) } } - + +/* + * Remove existing ptrack files + */ @@ -212,7 +212,7 @@ index c8cbd11fc2..f318d4c1a3 100644 + } +} + - + /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c @@ -222,7 +222,7 @@ index fbb97b5cf1..6cd7f2ae3e 100644 @@ -124,6 +124,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + {"ptrack.map.mmap", false}, + {"ptrack.map", false}, + {"ptrack.map.tmp", false}, @@ -231,56 +231,36 @@ index fbb97b5cf1..6cd7f2ae3e 100644 {NULL, false} }; diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h -index 6b2b4343a0..66b6d24e0a 100644 +index 72e3352398..5c2e016501 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h -@@ -400,7 +400,7 @@ typedef enum ProcessingMode +@@ -388,7 +388,7 @@ typedef enum ProcessingMode NormalProcessing /* normal processing */ } ProcessingMode; - + -extern ProcessingMode Mode; +extern PGDLLIMPORT ProcessingMode Mode; - + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h -index 6e77744cbc..81a11cffcc 100644 ---- a/src/include/storage/checksum.h -+++ b/src/include/storage/checksum.h -@@ -14,6 +14,7 @@ - #define CHECKSUM_H - - #include "storage/block.h" -+#include "port/pg_crc32c.h" - - /* - * Compute the checksum for a Postgres page. The page must be aligned on a -@@ -21,4 +22,11 @@ - */ - extern uint16 pg_checksum_page(char *page, BlockNumber blkno); - -+/* -+* Wrapper function for COMP_CRC32C macro. Was added to avoid -+* FRONTEND macro use for pg_comp_crc32c pointer in windows build. -+*/ -+extern pg_crc32c -+comp_crc32c(pg_crc32c *crc, const void *data, size_t len); -+ - #endif /* CHECKSUM_H */ -diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h -index 364acfafd2..f4813e22aa 100644 ---- a/src/include/storage/checksum_impl.h -+++ b/src/include/storage/checksum_impl.h -@@ -213,3 +213,9 @@ pg_checksum_page(char *page, BlockNumber blkno) - */ - return (uint16) ((checksum % 65535) + 1); - } -+ -+pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) -+{ -+ COMP_CRC32C(*crc, data, len); -+ return *crc; -+} +diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h +index 3c6f906683..a7355f7ad1 100644 +--- a/src/include/port/pg_crc32c.h ++++ b/src/include/port/pg_crc32c.h +@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le + #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) + + extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); +-extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); +- ++extern ++#ifndef FRONTEND ++PGDLLIMPORT ++#endif ++pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); + #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK + extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); + #endif diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h index 5d28f59c1d..0d3f04d8af 100644 --- a/src/include/storage/copydir.h @@ -288,13 +268,13 @@ index 5d28f59c1d..0d3f04d8af 100644 @@ -13,6 +13,9 @@ #ifndef COPYDIR_H #define COPYDIR_H - + +typedef void (*copydir_hook_type) (const char *path); +extern PGDLLIMPORT copydir_hook_type copydir_hook; + extern void copydir(char *fromdir, char *todir, bool recurse); extern void copy_file(char *fromfile, char *tofile); - + diff --git a/src/include/storage/md.h b/src/include/storage/md.h index 07fd1bb7d0..5294811bc8 100644 --- a/src/include/storage/md.h @@ -302,7 +282,7 @@ index 07fd1bb7d0..5294811bc8 100644 @@ -19,6 +19,13 @@ #include "storage/smgr.h" #include "storage/sync.h" - + +typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, + ForkNumber forknum, BlockNumber blocknum); +extern PGDLLIMPORT mdextend_hook_type mdextend_hook; @@ -320,7 +300,7 @@ index e16ab8e711..88da9686eb 100644 @@ -50,6 +50,9 @@ typedef struct FileTag uint32 segno; } FileTag; - + +typedef void (*ProcessSyncRequests_hook_type) (void); +extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook; + diff --git a/patches/REL_14_STABLE-ptrack-core.diff b/patches/REL_14_STABLE-ptrack-core.diff index 8dfbac4..3491700 100644 --- a/patches/REL_14_STABLE-ptrack-core.diff +++ b/patches/REL_14_STABLE-ptrack-core.diff @@ -1,17 +1,17 @@ -commit 6b7ba7c634fb34bfaad63b0b9184fb57584880db -Author: Anton A. Melnikov -Date: Mon Nov 8 09:19:07 2021 +0300 +commit a14ac459d71528c64df00c693e9c71ac70d3ba29 +Author: anastasia +Date: Mon Oct 19 14:53:06 2020 +0300 - [PGPRO-5771] Add handler for COMP_CRC32C and restore pg_comp_crc32c pointer definition. + add ptrack 2.0 diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c -index e09108d0ec..6858c7b670 100644 +index 50ae1f16d0..721b926ad2 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c -@@ -225,6 +225,13 @@ static const struct exclude_list_item excludeFiles[] = +@@ -233,6 +233,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + /* + * Skip all transient ptrack files, but do copy ptrack.map, since it may + * be successfully used immediately after backup. TODO: check, test? @@ -22,7 +22,7 @@ index e09108d0ec..6858c7b670 100644 /* end of list */ {NULL, false} }; -@@ -240,6 +247,11 @@ static const struct exclude_list_item noChecksumFiles[] = { +@@ -248,6 +255,11 @@ static const struct exclude_list_item noChecksumFiles[] = { {"pg_filenode.map", false}, {"pg_internal.init", true}, {"PG_VERSION", false}, @@ -35,13 +35,13 @@ index e09108d0ec..6858c7b670 100644 {"config_exec_params", true}, #endif diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c -index da8b7cbeca..6a817f45a5 100644 +index 0cf598dd0c..c9c44a4ae7 100644 --- a/src/backend/storage/file/copydir.c +++ b/src/backend/storage/file/copydir.c @@ -27,6 +27,8 @@ #include "storage/copydir.h" #include "storage/fd.h" - + +copydir_hook_type copydir_hook = NULL; + /* @@ -50,7 +50,7 @@ index da8b7cbeca..6a817f45a5 100644 @@ -78,6 +80,9 @@ copydir(char *fromdir, char *todir, bool recurse) } FreeDir(xldir); - + + if (copydir_hook) + copydir_hook(todir); + @@ -58,55 +58,55 @@ index da8b7cbeca..6a817f45a5 100644 * Be paranoid here and fsync all files to ensure the copy is really done. * But if fsync is disabled, we're done. diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c -index b4bca7eed6..90d479526c 100644 +index 0eacd461cd..c2ef404a1a 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -87,6 +87,8 @@ typedef struct _MdfdVec - + static MemoryContext MdCxt; /* context for all MdfdVec objects */ - + +mdextend_hook_type mdextend_hook = NULL; +mdwrite_hook_type mdwrite_hook = NULL; - + /* Populate a file tag describing an md.c segment file. */ #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ -@@ -463,6 +465,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -435,6 +437,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); - + Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); + + if (mdextend_hook) + mdextend_hook(reln->smgr_rnode, forknum, blocknum); } - + /* -@@ -749,6 +754,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, - +@@ -721,6 +726,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, + if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); + + if (mdwrite_hook) + mdwrite_hook(reln->smgr_rnode, forknum, blocknum); } - + /* diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c -index a12b357275..e52c2910f7 100644 +index 3ded2cdd71..3a596a59f7 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c -@@ -79,6 +79,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ +@@ -75,6 +75,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ static CycleCtr sync_cycle_ctr = 0; static CycleCtr checkpoint_cycle_ctr = 0; - + +ProcessSyncRequests_hook_type ProcessSyncRequests_hook = NULL; + /* Intervals for calling AbsorbSyncRequests */ #define FSYNCS_PER_ABSORB 10 #define UNLINKS_PER_ABSORB 10 -@@ -465,6 +467,9 @@ ProcessSyncRequests(void) +@@ -420,6 +422,9 @@ ProcessSyncRequests(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; - + + if (ProcessSyncRequests_hook) + ProcessSyncRequests_hook(); + @@ -114,19 +114,19 @@ index a12b357275..e52c2910f7 100644 sync_in_progress = false; } diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c -index 6652a60ec3..051cd6359d 100644 +index 1683629ee3..d2fc154576 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c -@@ -659,7 +659,6 @@ static char *recovery_target_xid_string; +@@ -620,7 +620,6 @@ static char *recovery_target_xid_string; static char *recovery_target_name_string; static char *recovery_target_lsn_string; - + - /* should be static, but commands/variable.c needs to get at this */ char *role_string; - + diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c -index 831cf42d3a..891302ba2f 100644 +index ffdc23945c..7ae95866ce 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c @@ -114,6 +114,11 @@ static const struct exclude_list_item skip[] = { @@ -142,29 +142,29 @@ index 831cf42d3a..891302ba2f 100644 {"config_exec_params", true}, #endif diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c -index f8d5ecb6ad..4027bdf3df 100644 +index 233441837f..cf7bd073bf 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c -@@ -85,6 +85,7 @@ static void RewriteControlFile(void); +@@ -84,6 +84,7 @@ static void RewriteControlFile(void); static void FindEndOfXLOG(void); static void KillExistingXLOG(void); static void KillExistingArchiveStatus(void); +static void KillExistingPtrack(void); static void WriteEmptyXLOG(void); static void usage(void); - -@@ -522,6 +523,7 @@ main(int argc, char *argv[]) + +@@ -513,6 +514,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); + KillExistingPtrack(); WriteEmptyXLOG(); - + printf(_("Write-ahead log reset\n")); -@@ -1111,6 +1113,53 @@ KillExistingArchiveStatus(void) +@@ -1102,6 +1104,53 @@ KillExistingArchiveStatus(void) } } - + +/* + * Remove existing ptrack files + */ @@ -212,17 +212,17 @@ index f8d5ecb6ad..4027bdf3df 100644 + } +} + - + /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c -index 2618b4c957..bbba813fe7 100644 +index fbb97b5cf1..6cd7f2ae3e 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c -@@ -158,6 +158,10 @@ static const struct exclude_list_item excludeFiles[] = +@@ -124,6 +124,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, - + + {"ptrack.map.mmap", false}, + {"ptrack.map", false}, + {"ptrack.map.tmp", false}, @@ -231,78 +231,58 @@ index 2618b4c957..bbba813fe7 100644 {NULL, false} }; diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h -index 3f155ce4f8..ce8dc0bee5 100644 +index 72e3352398..5c2e016501 100644 --- a/src/include/miscadmin.h +++ b/src/include/miscadmin.h -@@ -401,7 +401,7 @@ typedef enum ProcessingMode +@@ -388,7 +388,7 @@ typedef enum ProcessingMode NormalProcessing /* normal processing */ } ProcessingMode; - + -extern ProcessingMode Mode; +extern PGDLLIMPORT ProcessingMode Mode; - + #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h -index 80d2359192..ac55286088 100644 ---- a/src/include/storage/checksum.h -+++ b/src/include/storage/checksum.h -@@ -14,6 +14,7 @@ - #define CHECKSUM_H - - #include "storage/block.h" -+#include "port/pg_crc32c.h" - - /* - * Compute the checksum for a Postgres page. The page must be aligned on a -@@ -21,4 +22,11 @@ - */ - extern uint16 pg_checksum_page(char *page, BlockNumber blkno); - -+/* -+* Wrapper function for COMP_CRC32C macro. Was added to avoid -+* FRONTEND macro use for pg_comp_crc32c pointer in windows build. -+*/ -+extern pg_crc32c -+comp_crc32c(pg_crc32c *crc, const void *data, size_t len); -+ - #endif /* CHECKSUM_H */ -diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h -index ef30268330..19a0248538 100644 ---- a/src/include/storage/checksum_impl.h -+++ b/src/include/storage/checksum_impl.h -@@ -213,3 +213,9 @@ pg_checksum_page(char *page, BlockNumber blkno) - */ - return (uint16) ((checksum % 65535) + 1); - } -+ -+pg_crc32c comp_crc32c(pg_crc32c *crc, const void *data, size_t len) -+{ -+ COMP_CRC32C(*crc, data, len); -+ return *crc; -+} +diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h +index 3c6f906683..a7355f7ad1 100644 +--- a/src/include/port/pg_crc32c.h ++++ b/src/include/port/pg_crc32c.h +@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le + #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) + + extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); +-extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); +- ++extern ++#ifndef FRONTEND ++PGDLLIMPORT ++#endif ++pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); + #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK + extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); + #endif diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h -index 2c3936b0da..c6691c0be0 100644 +index 5d28f59c1d..0d3f04d8af 100644 --- a/src/include/storage/copydir.h +++ b/src/include/storage/copydir.h @@ -13,6 +13,9 @@ #ifndef COPYDIR_H #define COPYDIR_H - + +typedef void (*copydir_hook_type) (const char *path); +extern PGDLLIMPORT copydir_hook_type copydir_hook; + extern void copydir(char *fromdir, char *todir, bool recurse); extern void copy_file(char *fromfile, char *tofile); - + diff --git a/src/include/storage/md.h b/src/include/storage/md.h -index 752b440864..61c18a169f 100644 +index 07fd1bb7d0..5294811bc8 100644 --- a/src/include/storage/md.h +++ b/src/include/storage/md.h @@ -19,6 +19,13 @@ #include "storage/smgr.h" #include "storage/sync.h" - + +typedef void (*mdextend_hook_type) (RelFileNodeBackend smgr_rnode, + ForkNumber forknum, BlockNumber blocknum); +extern PGDLLIMPORT mdextend_hook_type mdextend_hook; @@ -314,13 +294,13 @@ index 752b440864..61c18a169f 100644 extern void mdinit(void); extern void mdopen(SMgrRelation reln); diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h -index fbdf34f762..455de202b6 100644 +index e16ab8e711..88da9686eb 100644 --- a/src/include/storage/sync.h +++ b/src/include/storage/sync.h -@@ -55,6 +55,9 @@ typedef struct FileTag +@@ -50,6 +50,9 @@ typedef struct FileTag uint32 segno; } FileTag; - + +typedef void (*ProcessSyncRequests_hook_type) (void); +extern PGDLLIMPORT ProcessSyncRequests_hook_type ProcessSyncRequests_hook; + diff --git a/pg_probackup b/pg_probackup deleted file mode 160000 index 5b6ca62..0000000 --- a/pg_probackup +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5b6ca624170e1b7955c293ce0173a812b6402d80 From 6ad12a5db489c9baf619bd399f491fc10f025c8d Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Sat, 12 Mar 2022 21:49:23 +0300 Subject: [PATCH 08/10] [PGPRO-5771] Fix make errors with current PG master. Tags: ptrack --- engine.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/engine.c b/engine.c index 42fa65a..b0860b1 100644 --- a/engine.c +++ b/engine.c @@ -39,6 +39,10 @@ #endif #include "storage/reinit.h" #include "storage/smgr.h" +#if PG_VERSION_NUM >= 150000 +#include "storage/fd.h" +#include "access/xlogrecovery.h" +#endif #include "utils/array.h" #include "utils/builtins.h" #include "utils/pg_lsn.h" From 511e341a510d29b88c1dfe97ba093c5e42c75e91 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Sat, 12 Mar 2022 22:16:45 +0300 Subject: [PATCH 09/10] [PGPRO-5771] Use common macro for Windows and Linux. Tags: ptrack --- engine.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/engine.c b/engine.c index b0860b1..0d7933f 100644 --- a/engine.c +++ b/engine.c @@ -31,7 +31,7 @@ #include "access/xlog.h" #include "catalog/pg_tablespace.h" #include "miscadmin.h" -#include "port/pg_crc32c.h" +#include "storage/checksum.h" #include "storage/copydir.h" #if PG_VERSION_NUM >= 120000 #include "storage/md.h" @@ -77,7 +77,7 @@ ptrack_file_exists(const char *path) static void ptrack_write_chunk(int fd, pg_crc32c *crc, char *chunk, size_t size) { - COMP_CRC32C(*crc, (char *) chunk, size); + COMP_CRC32_COMMON(*crc, (char *) chunk, size); if (write(fd, chunk, size) != size) { @@ -197,7 +197,7 @@ ptrackMapReadFromFile(const char *ptrack_path) pg_crc32c *file_crc; INIT_CRC32C(crc); - COMP_CRC32C(crc, (char *) ptrack_map, PtrackCrcOffset); + COMP_CRC32_COMMON(crc, (char *) ptrack_map, PtrackCrcOffset); FIN_CRC32C(crc); file_crc = (pg_crc32c *) ((char *) ptrack_map + PtrackCrcOffset); From 1ad0308c0a6932ee1f3b3282193b4569b1191e34 Mon Sep 17 00:00:00 2001 From: "Anton A. Melnikov" Date: Sat, 12 Mar 2022 22:37:12 +0300 Subject: [PATCH 10/10] [PGPRO-5771] Use common crc32 macro with PG master only. Tags: ptrack --- engine.c | 16 +++- patches/master-ptrack-core.diff | 136 ++++++++++++++++---------------- 2 files changed, 84 insertions(+), 68 deletions(-) diff --git a/engine.c b/engine.c index 0d7933f..8f65557 100644 --- a/engine.c +++ b/engine.c @@ -31,7 +31,11 @@ #include "access/xlog.h" #include "catalog/pg_tablespace.h" #include "miscadmin.h" +#if PG_VERSION_NUM >= 150000 #include "storage/checksum.h" +#else +#include "port/pg_crc32c.h" +#endif #include "storage/copydir.h" #if PG_VERSION_NUM >= 120000 #include "storage/md.h" @@ -77,7 +81,11 @@ ptrack_file_exists(const char *path) static void ptrack_write_chunk(int fd, pg_crc32c *crc, char *chunk, size_t size) { - COMP_CRC32_COMMON(*crc, (char *) chunk, size); +#if PG_VERSION_NUM >= 150000 + COMP_CRC32C_COMMON(*crc, (char *) chunk, size); +#else + COMP_CRC32C(*crc, (char *) chunk, size); +#endif if (write(fd, chunk, size) != size) { @@ -197,7 +205,11 @@ ptrackMapReadFromFile(const char *ptrack_path) pg_crc32c *file_crc; INIT_CRC32C(crc); - COMP_CRC32_COMMON(crc, (char *) ptrack_map, PtrackCrcOffset); +#if PG_VERSION_NUM >= 150000 + COMP_CRC32C_COMMON(crc, (char *) ptrack_map, PtrackCrcOffset); +#else + COMP_CRC32C(crc, (char *) ptrack_map, PtrackCrcOffset); +#endif FIN_CRC32C(crc); file_crc = (pg_crc32c *) ((char *) ptrack_map + PtrackCrcOffset); diff --git a/patches/master-ptrack-core.diff b/patches/master-ptrack-core.diff index 3491700..a756ca0 100644 --- a/patches/master-ptrack-core.diff +++ b/patches/master-ptrack-core.diff @@ -1,14 +1,14 @@ -commit a14ac459d71528c64df00c693e9c71ac70d3ba29 -Author: anastasia -Date: Mon Oct 19 14:53:06 2020 +0300 +commit 98d98792c0dbe589e8baee4c4100e0dc174ccacb +Author: Anton A. Melnikov +Date: Wed Apr 13 09:49:33 2022 +0300 - add ptrack 2.0 + Apply core patch on PG master diff --git a/src/backend/replication/basebackup.c b/src/backend/replication/basebackup.c -index 50ae1f16d0..721b926ad2 100644 +index 67489192a2..5e22017bd1 100644 --- a/src/backend/replication/basebackup.c +++ b/src/backend/replication/basebackup.c -@@ -233,6 +233,13 @@ static const struct exclude_list_item excludeFiles[] = +@@ -197,6 +197,13 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, @@ -22,7 +22,7 @@ index 50ae1f16d0..721b926ad2 100644 /* end of list */ {NULL, false} }; -@@ -248,6 +255,11 @@ static const struct exclude_list_item noChecksumFiles[] = { +@@ -212,6 +219,11 @@ static const struct exclude_list_item noChecksumFiles[] = { {"pg_filenode.map", false}, {"pg_internal.init", true}, {"PG_VERSION", false}, @@ -35,7 +35,7 @@ index 50ae1f16d0..721b926ad2 100644 {"config_exec_params", true}, #endif diff --git a/src/backend/storage/file/copydir.c b/src/backend/storage/file/copydir.c -index 0cf598dd0c..c9c44a4ae7 100644 +index 658fd95ba9..eee38eba17 100644 --- a/src/backend/storage/file/copydir.c +++ b/src/backend/storage/file/copydir.c @@ -27,6 +27,8 @@ @@ -58,7 +58,7 @@ index 0cf598dd0c..c9c44a4ae7 100644 * Be paranoid here and fsync all files to ensure the copy is really done. * But if fsync is disabled, we're done. diff --git a/src/backend/storage/smgr/md.c b/src/backend/storage/smgr/md.c -index 0eacd461cd..c2ef404a1a 100644 +index 286dd3f755..2c496b3a6d 100644 --- a/src/backend/storage/smgr/md.c +++ b/src/backend/storage/smgr/md.c @@ -87,6 +87,8 @@ typedef struct _MdfdVec @@ -70,7 +70,7 @@ index 0eacd461cd..c2ef404a1a 100644 /* Populate a file tag describing an md.c segment file. */ #define INIT_MD_FILETAG(a,xx_rnode,xx_forknum,xx_segno) \ -@@ -435,6 +437,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -465,6 +467,9 @@ mdextend(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, register_dirty_segment(reln, forknum, v); Assert(_mdnblocks(reln, forknum, v) <= ((BlockNumber) RELSEG_SIZE)); @@ -80,7 +80,7 @@ index 0eacd461cd..c2ef404a1a 100644 } /* -@@ -721,6 +726,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, +@@ -757,6 +762,9 @@ mdwrite(SMgrRelation reln, ForkNumber forknum, BlockNumber blocknum, if (!skipFsync && !SmgrIsTemp(reln)) register_dirty_segment(reln, forknum, v); @@ -91,10 +91,10 @@ index 0eacd461cd..c2ef404a1a 100644 /* diff --git a/src/backend/storage/sync/sync.c b/src/backend/storage/sync/sync.c -index 3ded2cdd71..3a596a59f7 100644 +index c695d816fc..77e2ab1996 100644 --- a/src/backend/storage/sync/sync.c +++ b/src/backend/storage/sync/sync.c -@@ -75,6 +75,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ +@@ -81,6 +81,8 @@ static MemoryContext pendingOpsCxt; /* context for the above */ static CycleCtr sync_cycle_ctr = 0; static CycleCtr checkpoint_cycle_ctr = 0; @@ -103,7 +103,7 @@ index 3ded2cdd71..3a596a59f7 100644 /* Intervals for calling AbsorbSyncRequests */ #define FSYNCS_PER_ABSORB 10 #define UNLINKS_PER_ABSORB 10 -@@ -420,6 +422,9 @@ ProcessSyncRequests(void) +@@ -478,6 +480,9 @@ ProcessSyncRequests(void) CheckpointStats.ckpt_longest_sync = longest; CheckpointStats.ckpt_agg_sync_time = total_elapsed; @@ -113,23 +113,11 @@ index 3ded2cdd71..3a596a59f7 100644 /* Flag successful completion of ProcessSyncRequests */ sync_in_progress = false; } -diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c -index 1683629ee3..d2fc154576 100644 ---- a/src/backend/utils/misc/guc.c -+++ b/src/backend/utils/misc/guc.c -@@ -620,7 +620,6 @@ static char *recovery_target_xid_string; - static char *recovery_target_name_string; - static char *recovery_target_lsn_string; - -- - /* should be static, but commands/variable.c needs to get at this */ - char *role_string; - diff --git a/src/bin/pg_checksums/pg_checksums.c b/src/bin/pg_checksums/pg_checksums.c -index ffdc23945c..7ae95866ce 100644 +index 21dfe1b6ee..266ac1ef40 100644 --- a/src/bin/pg_checksums/pg_checksums.c +++ b/src/bin/pg_checksums/pg_checksums.c -@@ -114,6 +114,11 @@ static const struct exclude_list_item skip[] = { +@@ -118,6 +118,11 @@ static const struct exclude_list_item skip[] = { {"pg_filenode.map", false}, {"pg_internal.init", true}, {"PG_VERSION", false}, @@ -142,10 +130,10 @@ index ffdc23945c..7ae95866ce 100644 {"config_exec_params", true}, #endif diff --git a/src/bin/pg_resetwal/pg_resetwal.c b/src/bin/pg_resetwal/pg_resetwal.c -index 233441837f..cf7bd073bf 100644 +index d4772a2965..66456f7e44 100644 --- a/src/bin/pg_resetwal/pg_resetwal.c +++ b/src/bin/pg_resetwal/pg_resetwal.c -@@ -84,6 +84,7 @@ static void RewriteControlFile(void); +@@ -85,6 +85,7 @@ static void RewriteControlFile(void); static void FindEndOfXLOG(void); static void KillExistingXLOG(void); static void KillExistingArchiveStatus(void); @@ -153,7 +141,7 @@ index 233441837f..cf7bd073bf 100644 static void WriteEmptyXLOG(void); static void usage(void); -@@ -513,6 +514,7 @@ main(int argc, char *argv[]) +@@ -488,6 +489,7 @@ main(int argc, char *argv[]) RewriteControlFile(); KillExistingXLOG(); KillExistingArchiveStatus(); @@ -161,8 +149,8 @@ index 233441837f..cf7bd073bf 100644 WriteEmptyXLOG(); printf(_("Write-ahead log reset\n")); -@@ -1102,6 +1104,53 @@ KillExistingArchiveStatus(void) - } +@@ -1036,6 +1038,53 @@ KillExistingArchiveStatus(void) + pg_fatal("could not close directory \"%s\": %m", ARCHSTATDIR); } +/* @@ -216,10 +204,10 @@ index 233441837f..cf7bd073bf 100644 /* * Write an empty XLOG file, containing only the checkpoint record diff --git a/src/bin/pg_rewind/filemap.c b/src/bin/pg_rewind/filemap.c -index fbb97b5cf1..6cd7f2ae3e 100644 +index d61067f6b2..27ec23137d 100644 --- a/src/bin/pg_rewind/filemap.c +++ b/src/bin/pg_rewind/filemap.c -@@ -124,6 +124,10 @@ static const struct exclude_list_item excludeFiles[] = +@@ -157,6 +157,10 @@ static const struct exclude_list_item excludeFiles[] = {"postmaster.pid", false}, {"postmaster.opts", false}, @@ -230,39 +218,55 @@ index fbb97b5cf1..6cd7f2ae3e 100644 /* end of list */ {NULL, false} }; -diff --git a/src/include/miscadmin.h b/src/include/miscadmin.h -index 72e3352398..5c2e016501 100644 ---- a/src/include/miscadmin.h -+++ b/src/include/miscadmin.h -@@ -388,7 +388,7 @@ typedef enum ProcessingMode - NormalProcessing /* normal processing */ - } ProcessingMode; +diff --git a/src/include/storage/checksum.h b/src/include/storage/checksum.h +index 1904fabd5a..f9306a6d01 100644 +--- a/src/include/storage/checksum.h ++++ b/src/include/storage/checksum.h +@@ -14,6 +14,7 @@ + #define CHECKSUM_H --extern ProcessingMode Mode; -+extern PGDLLIMPORT ProcessingMode Mode; + #include "storage/block.h" ++#include "port/pg_crc32c.h" - #define IsBootstrapProcessingMode() (Mode == BootstrapProcessing) - #define IsInitProcessingMode() (Mode == InitProcessing) -diff --git a/src/include/port/pg_crc32c.h b/src/include/port/pg_crc32c.h -index 3c6f906683..a7355f7ad1 100644 ---- a/src/include/port/pg_crc32c.h -+++ b/src/include/port/pg_crc32c.h -@@ -69,8 +69,11 @@ extern pg_crc32c pg_comp_crc32c_armv8(pg_crc32c crc, const void *data, size_t le - #define FIN_CRC32C(crc) ((crc) ^= 0xFFFFFFFF) + /* + * Compute the checksum for a Postgres page. The page must be aligned on a +@@ -21,4 +22,18 @@ + */ + extern uint16 pg_checksum_page(char *page, BlockNumber blkno); - extern pg_crc32c pg_comp_crc32c_sb8(pg_crc32c crc, const void *data, size_t len); --extern pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); -- -+extern -+#ifndef FRONTEND -+PGDLLIMPORT ++#ifdef WIN32 ++/* ++ * Wrapper function for COMP_CRC32C macro. Was added to avoid ++ * FRONTEND macro use for pg_comp_crc32c pointer in windows build. ++ */ ++extern void ++comp_crc32c(pg_crc32c *crc, const void *data, size_t len); ++ ++#define COMP_CRC32C_COMMON(crc, data, len) \ ++ comp_crc32c(&(crc), data, len) ++#else ++#define COMP_CRC32C_COMMON COMP_CRC32C ++#endif /* WIN32 */ ++ + #endif /* CHECKSUM_H */ +diff --git a/src/include/storage/checksum_impl.h b/src/include/storage/checksum_impl.h +index 015f0f1f83..1c5a0ba351 100644 +--- a/src/include/storage/checksum_impl.h ++++ b/src/include/storage/checksum_impl.h +@@ -213,3 +213,11 @@ pg_checksum_page(char *page, BlockNumber blkno) + */ + return (uint16) ((checksum % 65535) + 1); + } ++ ++#ifdef WIN32 ++void ++comp_crc32c(pg_crc32c *crc, const void *data, size_t len) ++{ ++ COMP_CRC32C(*crc, data, len); ++} +#endif -+pg_crc32c (*pg_comp_crc32c) (pg_crc32c crc, const void *data, size_t len); - #ifdef USE_SSE42_CRC32C_WITH_RUNTIME_CHECK - extern pg_crc32c pg_comp_crc32c_sse42(pg_crc32c crc, const void *data, size_t len); - #endif diff --git a/src/include/storage/copydir.h b/src/include/storage/copydir.h -index 5d28f59c1d..0d3f04d8af 100644 +index 50a26edeb0..af1602f515 100644 --- a/src/include/storage/copydir.h +++ b/src/include/storage/copydir.h @@ -13,6 +13,9 @@ @@ -276,7 +280,7 @@ index 5d28f59c1d..0d3f04d8af 100644 extern void copy_file(char *fromfile, char *tofile); diff --git a/src/include/storage/md.h b/src/include/storage/md.h -index 07fd1bb7d0..5294811bc8 100644 +index 6e46d8d96a..f0967ef484 100644 --- a/src/include/storage/md.h +++ b/src/include/storage/md.h @@ -19,6 +19,13 @@ @@ -294,10 +298,10 @@ index 07fd1bb7d0..5294811bc8 100644 extern void mdinit(void); extern void mdopen(SMgrRelation reln); diff --git a/src/include/storage/sync.h b/src/include/storage/sync.h -index e16ab8e711..88da9686eb 100644 +index 9737e1eb67..914ad86328 100644 --- a/src/include/storage/sync.h +++ b/src/include/storage/sync.h -@@ -50,6 +50,9 @@ typedef struct FileTag +@@ -55,6 +55,9 @@ typedef struct FileTag uint32 segno; } FileTag;