From 99eec6e4e22d162b0bff536d93300ff0450d95e1 Mon Sep 17 00:00:00 2001 From: vshepard Date: Thu, 2 May 2024 10:12:43 +0200 Subject: [PATCH 1/4] Add s3 tests --- tests/CVE_2018_1058_test.py | 53 +- tests/Readme.md | 14 +- tests/__init__.py | 10 +- tests/archive_test.py | 1596 ++++---- tests/auth_test.py | 398 +- tests/backup_test.py | 2799 +++++--------- tests/catchup_test.py | 1062 ++++-- tests/cfs_backup_test.py | 1216 ------ tests/cfs_catchup_test.py | 117 - tests/cfs_restore_test.py | 447 --- tests/cfs_validate_backup_test.py | 24 - tests/checkdb_test.py | 607 ++- tests/compatibility_test.py | 696 ++-- tests/compression_test.py | 508 +-- tests/config_test.py | 152 +- tests/delete_test.py | 498 ++- tests/delta_test.py | 565 +-- tests/exclude_test.py | 116 +- tests/expected/option_help.out | 94 +- tests/expected/option_help_ru.out | 94 +- tests/expected/option_version.out | 1 - tests/external_test.py | 1238 ++---- tests/false_positive_test.py | 253 +- tests/helpers/__init__.py | 4 +- tests/helpers/cfs_helpers.py | 93 - tests/helpers/data_helpers.py | 198 +- tests/helpers/enums/date_time_enum.py | 7 + tests/helpers/ptrack_helpers.py | 2037 +++------- tests/helpers/state_helper.py | 25 + tests/helpers/validators/show_validator.py | 141 + tests/incr_restore_test.py | 1489 ++++---- tests/init_test.py | 293 +- tests/locking_test.py | 991 +++-- tests/logging_test.py | 195 +- tests/merge_test.py | 2820 +++++++++----- tests/option_test.py | 312 +- tests/page_test.py | 775 ++-- tests/pgpro2068_test.py | 171 - tests/pgpro560_test.py | 105 +- tests/pgpro589_test.py | 45 +- tests/ptrack_load_test.py | 150 + tests/ptrack_test.py | 1987 +++------- tests/remote_test.py | 40 +- tests/replica_test.py | 1053 ++--- tests/requirements.txt | 16 + tests/restore_test.py | 2560 ++++++------- tests/retention_test.py | 1473 ++++--- tests/set_backup_test.py | 437 +-- tests/show_test.py | 702 ++-- tests/test_utils/__init__.py | 0 tests/test_utils/config_provider.py | 8 + tests/test_utils/s3_backup.py | 208 + tests/time_consuming_test.py | 23 +- tests/time_stamp_test.py | 136 +- tests/validate_test.py | 4022 +++++++------------- 55 files changed, 14440 insertions(+), 20634 deletions(-) delete mode 100644 tests/cfs_backup_test.py delete mode 100644 tests/cfs_catchup_test.py delete mode 100644 tests/cfs_restore_test.py delete mode 100644 tests/cfs_validate_backup_test.py delete mode 100644 tests/expected/option_version.out delete mode 100644 tests/helpers/cfs_helpers.py create mode 100644 tests/helpers/enums/date_time_enum.py create mode 100644 tests/helpers/state_helper.py create mode 100644 tests/helpers/validators/show_validator.py create mode 100644 tests/ptrack_load_test.py create mode 100644 tests/requirements.txt create mode 100644 tests/test_utils/__init__.py create mode 100644 tests/test_utils/config_provider.py create mode 100644 tests/test_utils/s3_backup.py diff --git a/tests/CVE_2018_1058_test.py b/tests/CVE_2018_1058_test.py index cfd55cc60..3fa28ded1 100644 --- a/tests/CVE_2018_1058_test.py +++ b/tests/CVE_2018_1058_test.py @@ -1,19 +1,16 @@ import os import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest -class CVE_2018_1058(ProbackupTest, unittest.TestCase): +class CVE_2018_1058(ProbackupTest): # @unittest.skip("skip") def test_basic_default_search_path(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True) + node = self.pg_node.make_simple('node', checksum=False, set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -26,19 +23,16 @@ def test_basic_default_search_path(self): "END " "$$ LANGUAGE plpgsql") - self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + self.pb.backup_node('node', node, backup_type='full', options=['--stream']) # @unittest.skip("skip") def test_basic_backup_modified_search_path(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True) - self.set_auto_conf(node, options={'search_path': 'public,pg_catalog'}) + node = self.pg_node.make_simple('node', checksum=False, set_replication=True) + node.set_auto_conf(options={'search_path': 'public,pg_catalog'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -62,7 +56,7 @@ def test_basic_backup_modified_search_path(self): "$$ LANGUAGE plpgsql; " "CREATE VIEW public.pg_proc AS SELECT proname FROM public.pg_proc()") - self.backup_node(backup_dir, 'node', node, backup_type='full', options=['--stream']) + self.pb.backup_node('node', node, backup_type='full', options=['--stream']) log_file = os.path.join(node.logs_dir, 'postgresql.log') with open(log_file, 'r') as f: @@ -73,10 +67,8 @@ def test_basic_backup_modified_search_path(self): # @unittest.skip("skip") def test_basic_checkdb_modified_search_path(self): """""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - self.set_auto_conf(node, options={'search_path': 'public,pg_catalog'}) + node = self.pg_node.make_simple('node') + node.set_auto_conf(options={'search_path': 'public,pg_catalog'}) node.slow_start() node.safe_psql( @@ -110,20 +102,11 @@ def test_basic_checkdb_modified_search_path(self): "CREATE VIEW public.pg_namespace AS SELECT * FROM public.pg_namespace();" ) - try: - self.checkdb_node( + self.pb.checkdb_node( options=[ '--amcheck', '--skip-block-validation', - '-d', 'postgres', '-p', str(node.port)]) - self.assertEqual( - 1, 0, - "Expecting Error because amcheck{,_next} not installed\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "WARNING: Extension 'amcheck' or 'amcheck_next' are not installed in database postgres", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + '-d', 'postgres', '-p', str(node.port)], + expect_error="because amcheck{,_next} not installed") + self.assertMessage(contains= + "WARNING: Extension 'amcheck' or 'amcheck_next' are not installed in database postgres") diff --git a/tests/Readme.md b/tests/Readme.md index 11c5272f9..589c00bad 100644 --- a/tests/Readme.md +++ b/tests/Readme.md @@ -45,12 +45,22 @@ Run long (time consuming) tests: export PG_PROBACKUP_LONG=ON Usage: - sudo echo 0 > /proc/sys/kernel/yama/ptrace_scope + + sudo sysctl kernel.yama.ptrace_scope=0 # only active until the next reboot +or + sudo sed -i 's/ptrace_scope = 1/ptrace_scope = 0/' /etc/sysctl.d/10-ptrace.conf # set permanently, needs a reboot to take effect + # see https://www.kernel.org/doc/Documentation/security/Yama.txt for possible implications of setting this parameter permanently pip install testgres export PG_CONFIG=/path/to/pg_config python -m unittest [-v] tests[.specific_module][.class.test] ``` - +# Environment variables +| Variable | Possible values | Required | Default value | Description | +| - |------------------------------------| - | - |------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| KEEP_LOGS | Any | No | Not set | If this variable is set to '1', 'y' or 'Y' then test logs are kept after the successful execution of a test, otherwise test logs are deleted upon the successful execution of a test | +| PG_PROBACKUP_S3_TEST | Not set, minio, VK | No | Not set | If set, specifies the type of the S3 storage for the tests, otherwise signifies to the tests that the storage is not an S3 one | +| PG_PROBACKUP_S3_CONFIG_FILE | Not set, path to config file, True | No | Not set | Specifies the path to an S3 configuration file. If set, all commands will include --s3-config-file='path'. If 'True', the default configuration file at ./s3/tests/s3.conf will be used | +| PGPROBACKUP_TMP_DIR | File path | No | tmp_dirs | The root of the temporary directory hierarchy where tests store data and logs. Relative paths start from the `tests` directory. | # Troubleshooting FAQ ## Python tests failure diff --git a/tests/__init__.py b/tests/__init__.py index c8d2c70c3..89f0c47a7 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -3,9 +3,9 @@ from . import init_test, merge_test, option_test, show_test, compatibility_test, \ backup_test, delete_test, delta_test, restore_test, validate_test, \ - retention_test, pgpro560_test, pgpro589_test, pgpro2068_test, false_positive_test, replica_test, \ - compression_test, page_test, ptrack_test, archive_test, exclude_test, cfs_backup_test, cfs_restore_test, \ - cfs_validate_backup_test, auth_test, time_stamp_test, logging_test, \ + retention_test, pgpro560_test, pgpro589_test, false_positive_test, replica_test, \ + compression_test, page_test, ptrack_test, archive_test, exclude_test, \ + auth_test, time_stamp_test, logging_test, \ locking_test, remote_test, external_test, config_test, checkdb_test, set_backup_test, incr_restore_test, \ catchup_test, CVE_2018_1058_test, time_consuming_test @@ -35,9 +35,6 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(compatibility_test)) suite.addTests(loader.loadTestsFromModule(checkdb_test)) suite.addTests(loader.loadTestsFromModule(config_test)) - suite.addTests(loader.loadTestsFromModule(cfs_backup_test)) - suite.addTests(loader.loadTestsFromModule(cfs_restore_test)) - suite.addTests(loader.loadTestsFromModule(cfs_validate_backup_test)) suite.addTests(loader.loadTestsFromModule(compression_test)) suite.addTests(loader.loadTestsFromModule(delete_test)) suite.addTests(loader.loadTestsFromModule(delta_test)) @@ -53,7 +50,6 @@ def load_tests(loader, tests, pattern): suite.addTests(loader.loadTestsFromModule(page_test)) suite.addTests(loader.loadTestsFromModule(pgpro560_test)) suite.addTests(loader.loadTestsFromModule(pgpro589_test)) - suite.addTests(loader.loadTestsFromModule(pgpro2068_test)) suite.addTests(loader.loadTestsFromModule(remote_test)) suite.addTests(loader.loadTestsFromModule(replica_test)) suite.addTests(loader.loadTestsFromModule(restore_test)) diff --git a/tests/archive_test.py b/tests/archive_test.py index 00fd1f592..034223aa4 100644 --- a/tests/archive_test.py +++ b/tests/archive_test.py @@ -1,33 +1,30 @@ import os import shutil -import gzip import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, GdbException +from .helpers.ptrack_helpers import ProbackupTest, fs_backup_class, get_relative_path +from pg_probackup2.gdb import needs_gdb from .helpers.data_helpers import tail_file -from datetime import datetime, timedelta import subprocess from sys import exit from time import sleep -from distutils.dir_util import copy_tree +from testgres import ProcessType -class ArchiveTest(ProbackupTest, unittest.TestCase): +class ArchiveTest(ProbackupTest): # @unittest.expectedFailure # @unittest.skip("skip") def test_pgpro434_1(self): """Description in jira issue PGPRO-434""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -37,25 +34,23 @@ def test_pgpro434_1(self): "generate_series(0,100) i") result = node.table_checksum("t_heap") - self.backup_node( - backup_dir, 'node', node) + self.pb.backup_node('node', node) node.cleanup() - self.restore_node( - backup_dir, 'node', node) + self.pb.restore_node('node', node=node) node.slow_start() # Recreate backup catalog - self.clean_pb(backup_dir) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir.cleanup() + self.pb.init() + self.pb.add_instance('node', node) # Make backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.cleanup() # Restore Database - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) node.slow_start() self.assertEqual( @@ -69,22 +64,15 @@ def test_pgpro434_2(self): Check that timelines are correct. WAITING PGPRO-1053 for --immediate """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s'} ) - if self.get_version(node) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because pg_control_checkpoint() is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FIRST TIMELINE @@ -93,7 +81,7 @@ def test_pgpro434_2(self): "create table t_heap as select 1 as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,100) i") - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) node.safe_psql( "postgres", "insert into t_heap select 100501 as id, md5(i::text) as text, " @@ -102,8 +90,7 @@ def test_pgpro434_2(self): # SECOND TIMELIN node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=['--immediate', '--recovery-target-action=promote']) node.slow_start() @@ -124,7 +111,7 @@ def test_pgpro434_2(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(100,200) i") - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) node.safe_psql( "postgres", @@ -134,8 +121,7 @@ def test_pgpro434_2(self): # THIRD TIMELINE node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=['--immediate', '--recovery-target-action=promote']) node.slow_start() @@ -151,7 +137,7 @@ def test_pgpro434_2(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(200,300) i") - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) result = node.table_checksum("t_heap") node.safe_psql( @@ -162,8 +148,7 @@ def test_pgpro434_2(self): # FOURTH TIMELINE node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=['--immediate', '--recovery-target-action=promote']) node.slow_start() @@ -175,8 +160,7 @@ def test_pgpro434_2(self): # FIFTH TIMELINE node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=['--immediate', '--recovery-target-action=promote']) node.slow_start() @@ -188,8 +172,7 @@ def test_pgpro434_2(self): # SIXTH TIMELINE node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=['--immediate', '--recovery-target-action=promote']) node.slow_start() @@ -209,29 +192,25 @@ def test_pgpro434_2(self): 'data after restore not equal to original data') # @unittest.skip("skip") + @needs_gdb def test_pgpro434_3(self): """ Check pg_stop_backup_timeout, needed backup_timeout Fixed in commit d84d79668b0c139 and assert fixed by ptrack 1.7 """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - gdb = self.backup_node( - backup_dir, 'node', node, + gdb = self.pb.backup_node('node', node, options=[ - "--archive-timeout=60", + "--archive-timeout=10", "--log-level-file=LOG"], gdb=True) @@ -239,26 +218,20 @@ def test_pgpro434_3(self): gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() - self.set_auto_conf(node, {'archive_command': 'exit 1'}) + node.set_auto_conf({'archive_command': 'exit 1'}) node.reload() + sleep(1) + gdb.continue_execution_until_exit() sleep(1) - log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(log_file, 'r') as f: - log_content = f.read() + log_content = self.read_pb_log() - # in PG =< 9.6 pg_stop_backup always wait - if self.get_version(node) < 100000: - self.assertIn( - "ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it", - log_content) - else: - self.assertIn( - "ERROR: WAL segment 000000010000000000000003 could not be archived in 60 seconds", - log_content) + self.assertIn( + "ERROR: WAL segment 000000010000000000000003 could not be archived in 10 seconds", + log_content) log_file = os.path.join(node.logs_dir, 'postgresql.log') with open(log_file, 'r') as f: @@ -270,29 +243,25 @@ def test_pgpro434_3(self): 'PostgreSQL crashed because of a failed assert') # @unittest.skip("skip") + @needs_gdb def test_pgpro434_4(self): """ Check pg_stop_backup_timeout, libpq-timeout requested. Fixed in commit d84d79668b0c139 and assert fixed by ptrack 1.7 """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - gdb = self.backup_node( - backup_dir, 'node', node, + gdb = self.pb.backup_node('node', node, options=[ - "--archive-timeout=60", + "--archive-timeout=10", "--log-level-file=info"], gdb=True) @@ -300,7 +269,7 @@ def test_pgpro434_4(self): gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() - self.set_auto_conf(node, {'archive_command': 'exit 1'}) + node.set_auto_conf({'archive_command': 'exit 1'}) node.reload() os.environ["PGAPPNAME"] = "foo" @@ -314,26 +283,23 @@ def test_pgpro434_4(self): os.environ["PGAPPNAME"] = "pg_probackup" postgres_gdb = self.gdb_attach(pid) - if self.get_version(node) < 150000: + if self.pg_config_version < 150000: postgres_gdb.set_breakpoint('do_pg_stop_backup') else: postgres_gdb.set_breakpoint('do_pg_backup_stop') postgres_gdb.continue_execution_until_running() gdb.continue_execution_until_exit() - # gdb._execute('detach') - log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(log_file, 'r') as f: - log_content = f.read() + log_content = self.read_pb_log() - if self.get_version(node) < 150000: + if self.pg_config_version < 150000: self.assertIn( - "ERROR: pg_stop_backup doesn't answer in 60 seconds, cancel it", + "ERROR: pg_stop_backup doesn't answer in 10 seconds, cancel it", log_content) else: self.assertIn( - "ERROR: pg_backup_stop doesn't answer in 60 seconds, cancel it", + "ERROR: pg_backup_stop doesn't answer in 10 seconds, cancel it", log_content) log_file = os.path.join(node.logs_dir, 'postgresql.log') @@ -348,30 +314,20 @@ def test_pgpro434_4(self): # @unittest.skip("skip") def test_archive_push_file_exists(self): """Archive-push if file exists""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) - wals_dir = os.path.join(backup_dir, 'wal', 'node') - if self.archive_compress: - filename = '000000010000000000000001.gz' - file = os.path.join(wals_dir, filename) - else: - filename = '000000010000000000000001' - file = os.path.join(wals_dir, filename) - - with open(file, 'a+b') as f: - f.write(b"blablablaadssaaaaaaaaaaaaaaa") - f.flush() - f.close() + suffix = self.compress_suffix + walfile = '000000010000000000000001'+suffix + self.write_instance_wal(backup_dir, 'node', walfile, + b"blablablaadssaaaaaaaaaaaaaaa") node.slow_start() node.safe_psql( @@ -382,7 +338,6 @@ def test_archive_push_file_exists(self): log_file = os.path.join(node.logs_dir, 'postgresql.log') self.switch_wal_segment(node) - sleep(1) log = tail_file(log_file, linetimeout=30, totaltimeout=120, collect=True) @@ -400,61 +355,50 @@ def test_archive_push_file_exists(self): 'pg_probackup archive-push WAL file', log.content) - self.assertIn( - 'WAL file already exists in archive with different checksum', - log.content) + if self.archive_compress: + self.assertIn( + 'WAL file already exists and looks like it is damaged', + log.content) + else: + self.assertIn( + 'WAL file already exists in archive with different checksum', + log.content) self.assertNotIn( 'pg_probackup archive-push completed successfully', log.content) # btw check that console coloring codes are not slipped into log file self.assertNotIn('[0m', log.content) + log.stop_collect() - if self.get_version(node) < 100000: - wal_src = os.path.join( - node.data_dir, 'pg_xlog', '000000010000000000000001') - else: - wal_src = os.path.join( - node.data_dir, 'pg_wal', '000000010000000000000001') + wal_src = os.path.join( + node.data_dir, 'pg_wal', '000000010000000000000001') + with open(wal_src, 'rb') as f_in: + file_content = f_in.read() - if self.archive_compress: - with open(wal_src, 'rb') as f_in, gzip.open( - file, 'wb', compresslevel=1) as f_out: - shutil.copyfileobj(f_in, f_out) - else: - shutil.copyfile(wal_src, file) + self.write_instance_wal(backup_dir, 'node', walfile, file_content, + compress = self.archive_compress) self.switch_wal_segment(node) - log.stop_collect() log.wait(contains = 'pg_probackup archive-push completed successfully') # @unittest.skip("skip") def test_archive_push_file_exists_overwrite(self): """Archive-push if file exists""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={'checkpoint_timeout': '30s'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) - wals_dir = os.path.join(backup_dir, 'wal', 'node') - if self.archive_compress: - filename = '000000010000000000000001.gz' - file = os.path.join(wals_dir, filename) - else: - filename = '000000010000000000000001' - file = os.path.join(wals_dir, filename) - - with open(file, 'a+b') as f: - f.write(b"blablablaadssaaaaaaaaaaaaaaa") - f.flush() - f.close() + suffix = self.compress_suffix + walfile = '000000010000000000000001'+suffix + self.write_instance_wal(backup_dir, 'node', walfile, + b"blablablaadssaaaaaaaaaaaaaaa") node.slow_start() node.safe_psql( @@ -465,7 +409,6 @@ def test_archive_push_file_exists_overwrite(self): log_file = os.path.join(node.logs_dir, 'postgresql.log') self.switch_wal_segment(node) - sleep(1) log = tail_file(log_file, linetimeout=30, collect=True) log.wait(contains = 'The failed archive command was') @@ -476,118 +419,47 @@ def test_archive_push_file_exists_overwrite(self): 'DETAIL: The failed archive command was:', log.content) self.assertIn( 'pg_probackup archive-push WAL file', log.content) - self.assertNotIn( - 'WAL file already exists in archive with ' - 'different checksum, overwriting', log.content) - self.assertIn( - 'WAL file already exists in archive with ' - 'different checksum', log.content) + self.assertNotIn('overwriting', log.content) + if self.archive_compress: + self.assertIn( + 'WAL file already exists and looks like ' + 'it is damaged', log.content) + else: + self.assertIn( + 'WAL file already exists in archive with ' + 'different checksum', log.content) self.assertNotIn( 'pg_probackup archive-push completed successfully', log.content) - self.set_archiving(backup_dir, 'node', node, overwrite=True) + self.pb.set_archiving('node', node, overwrite=True) node.reload() self.switch_wal_segment(node) log.drop_content() log.wait(contains = 'pg_probackup archive-push completed successfully') - self.assertIn( - 'WAL file already exists in archive with ' - 'different checksum, overwriting', log.content) - - # @unittest.skip("skip") - def test_archive_push_partial_file_exists(self): - """Archive-push if stale '.part' file exists""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving( - backup_dir, 'node', node, - log_level='verbose', archive_timeout=60) - - node.slow_start() - - # this backup is needed only for validation to xid - self.backup_node(backup_dir, 'node', node) - - node.safe_psql( - "postgres", - "create table t1(a int)") - - xid = node.safe_psql( - "postgres", - "INSERT INTO t1 VALUES (1) RETURNING (xmin)").decode('utf-8').rstrip() - - if self.get_version(node) < 100000: - filename_orig = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_xlogfile_name_offset(pg_current_xlog_location());").rstrip() - else: - filename_orig = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() - - filename_orig = filename_orig.decode('utf-8') - - # form up path to next .part WAL segment - wals_dir = os.path.join(backup_dir, 'wal', 'node') if self.archive_compress: - filename = filename_orig + '.gz' + '.part' - file = os.path.join(wals_dir, filename) + self.assertIn( + 'WAL file already exists and looks like ' + 'it is damaged, overwriting', log.content) else: - filename = filename_orig + '.part' - file = os.path.join(wals_dir, filename) - - # emulate stale .part file - with open(file, 'a+b') as f: - f.write(b"blahblah") - f.flush() - f.close() - - self.switch_wal_segment(node) - sleep(70) - - # check that segment is archived - if self.archive_compress: - filename_orig = filename_orig + '.gz' - - file = os.path.join(wals_dir, filename_orig) - self.assertTrue(os.path.isfile(file)) - - # successful validate means that archive-push reused stale wal segment - self.validate_pb( - backup_dir, 'node', - options=['--recovery-target-xid={0}'.format(xid)]) - - log_file = os.path.join(node.logs_dir, 'postgresql.log') - with open(log_file, 'r') as f: - log_content = f.read() - self.assertIn( - 'Reusing stale temp WAL file', - log_content) + 'WAL file already exists in archive with ' + 'different checksum, overwriting', log.content) - # @unittest.skip("skip") + @unittest.skip("should be redone with file locking") def test_archive_push_part_file_exists_not_stale(self): """Archive-push if .part file exists and it is not stale""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + # TODO: this test is not completely obsolete, but should be rewritten + # with use of file locking when push_file_internal will use it. + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node, archive_timeout=60) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node, archive_timeout=60) node.slow_start() @@ -600,27 +472,17 @@ def test_archive_push_part_file_exists_not_stale(self): "postgres", "create table t2()") - if self.get_version(node) < 100000: - filename_orig = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_xlogfile_name_offset(pg_current_xlog_location());").rstrip() - else: - filename_orig = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() + filename_orig = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn());").rstrip() filename_orig = filename_orig.decode('utf-8') # form up path to next .part WAL segment wals_dir = os.path.join(backup_dir, 'wal', 'node') - if self.archive_compress: - filename = filename_orig + '.gz' + '.part' - file = os.path.join(wals_dir, filename) - else: - filename = filename_orig + '.part' - file = os.path.join(wals_dir, filename) + filename = filename_orig + self.compress_suffix + '.part' + file = os.path.join(wals_dir, filename) with open(file, 'a+b') as f: f.write(b"blahblah") @@ -638,8 +500,7 @@ def test_archive_push_part_file_exists_not_stale(self): sleep(40) # check that segment is NOT archived - if self.archive_compress: - filename_orig = filename_orig + '.gz' + filename_orig += self.compress_suffix file = os.path.join(wals_dir, filename_orig) @@ -654,33 +515,27 @@ def test_archive_push_part_file_exists_not_stale(self): # @unittest.expectedFailure # @unittest.skip("skip") + @needs_gdb def test_replica_archive(self): """ make node without archiving, take stream backup and turn it into replica, set replica with archiving, make archive backup from replica """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + backup_dir = self.backup_dir + master = self.pg_node.make_simple('master', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '10s', 'checkpoint_timeout': '30s', 'max_wal_size': '32MB'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) + self.pb.init() # ADD INSTANCE 'MASTER' - self.add_instance(backup_dir, 'master', master) + self.pb.add_instance('master', master) master.slow_start() - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() master.psql( @@ -689,15 +544,15 @@ def test_replica_archive(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,2560) i") - self.backup_node(backup_dir, 'master', master, options=['--stream']) + self.pb.backup_node('master', master, options=['--stream']) before = master.table_checksum("t_heap") # Settings for Replica - self.restore_node(backup_dir, 'master', replica) + self.pb.restore_node('master', node=replica) self.set_replica(master, replica, synchronous=True) - self.add_instance(backup_dir, 'replica', replica) - self.set_archiving(backup_dir, 'replica', replica, replica=True) + self.pb.add_instance('replica', replica) + self.pb.set_archiving('replica', replica, replica=True) replica.slow_start(replica=True) # Check data correctness on replica @@ -714,26 +569,21 @@ def test_replica_archive(self): "from generate_series(256,512) i") before = master.table_checksum("t_heap") - backup_id = self.backup_node( - backup_dir, 'replica', replica, + backup_id = self.pb.backup_node('replica', replica, options=[ '--archive-timeout=30', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), '--stream']) - self.validate_pb(backup_dir, 'replica') + self.pb.validate('replica') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + 'OK', self.pb.show('replica', backup_id)['status']) # RESTORE FULL BACKUP TAKEN FROM replica - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node')) + node = self.pg_node.make_simple('node') node.cleanup() - self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + self.pb.restore_node('replica', node=node) - self.set_auto_conf(node, {'port': node.port}) + node.set_auto_conf({'port': node.port}) node.slow_start() # CHECK DATA CORRECTNESS after = node.table_checksum("t_heap") @@ -752,26 +602,22 @@ def test_replica_archive(self): self.wait_until_replica_catch_with_master(master, replica) - backup_id = self.backup_node( - backup_dir, 'replica', + backup_id, _ = self.pb.backup_replica_node('replica', replica, backup_type='page', + master=master, options=[ '--archive-timeout=60', - '--master-db=postgres', - '--master-host=localhost', - '--master-port={0}'.format(master.port), '--stream']) - self.validate_pb(backup_dir, 'replica') + self.pb.validate('replica') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + 'OK', self.pb.show('replica', backup_id)['status']) # RESTORE PAGE BACKUP TAKEN FROM replica node.cleanup() - self.restore_node( - backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + self.pb.restore_node('replica', node, backup_id=backup_id) - self.set_auto_conf(node, {'port': node.port}) + node.set_auto_conf({'port': node.port}) node.slow_start() # CHECK DATA CORRECTNESS @@ -787,27 +633,19 @@ def test_master_and_replica_parallel_archiving(self): set replica with archiving, make archive backup from replica, make archive backup from master """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + master = self.pg_node.make_simple('master', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '10s'} ) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.init_pb(backup_dir) + self.pb.init() # ADD INSTANCE 'MASTER' - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) + self.pb.add_instance('master', master) + self.pb.set_archiving('master', master) master.slow_start() master.psql( @@ -817,23 +655,23 @@ def test_master_and_replica_parallel_archiving(self): "from generate_series(0,10000) i") # TAKE FULL ARCHIVE BACKUP FROM MASTER - self.backup_node(backup_dir, 'master', master) + self.pb.backup_node('master', master) # GET LOGICAL CONTENT FROM MASTER before = master.table_checksum("t_heap") # GET PHYSICAL CONTENT FROM MASTER pgdata_master = self.pgdata_content(master.data_dir) # Settings for Replica - self.restore_node(backup_dir, 'master', replica) + self.pb.restore_node('master', node=replica) # CHECK PHYSICAL CORRECTNESS on REPLICA pgdata_replica = self.pgdata_content(replica.data_dir) self.compare_pgdata(pgdata_master, pgdata_replica) self.set_replica(master, replica) # ADD INSTANCE REPLICA - self.add_instance(backup_dir, 'replica', replica) + self.pb.add_instance('replica', replica) # SET ARCHIVING FOR REPLICA - self.set_archiving(backup_dir, 'replica', replica, replica=True) + self.pb.set_archiving('replica', replica, replica=True) replica.slow_start(replica=True) # CHECK LOGICAL CORRECTNESS on REPLICA @@ -846,27 +684,24 @@ def test_master_and_replica_parallel_archiving(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0, 60000) i") - backup_id = self.backup_node( - backup_dir, 'replica', replica, + backup_id = self.pb.backup_node('replica', replica, options=[ '--archive-timeout=30', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), '--stream']) - self.validate_pb(backup_dir, 'replica') + self.pb.validate('replica') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + 'OK', self.pb.show('replica', backup_id)['status']) # TAKE FULL ARCHIVE BACKUP FROM MASTER - backup_id = self.backup_node(backup_dir, 'master', master) - self.validate_pb(backup_dir, 'master') + backup_id = self.pb.backup_node('master', master) + self.pb.validate('master') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) + 'OK', self.pb.show('master', backup_id)['status']) # @unittest.expectedFailure # @unittest.skip("skip") + @needs_gdb def test_basic_master_and_replica_concurrent_archiving(self): """ make node 'master 'with archiving, @@ -874,30 +709,19 @@ def test_basic_master_and_replica_concurrent_archiving(self): set replica with archiving, make sure that archiving on both node is working. """ - if self.pg_config_version < self.version_to_num('9.6.0'): - self.skipTest('You need PostgreSQL >= 9.6 for this test') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + master = self.pg_node.make_simple('master', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', 'archive_timeout': '10s'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.init_pb(backup_dir) + self.pb.init() # ADD INSTANCE 'MASTER' - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) + self.pb.add_instance('master', master) + self.pb.set_archiving('master', master) master.slow_start() master.psql( @@ -909,29 +733,33 @@ def test_basic_master_and_replica_concurrent_archiving(self): master.pgbench_init(scale=5) # TAKE FULL ARCHIVE BACKUP FROM MASTER - self.backup_node(backup_dir, 'master', master) - # GET LOGICAL CONTENT FROM MASTER - before = master.table_checksum("t_heap") + self.pb.backup_node('master', master) # GET PHYSICAL CONTENT FROM MASTER - pgdata_master = self.pgdata_content(master.data_dir) + master.stop() + pgdata_master = self.pgdata_content(master.data_dir, exclude_dirs = ['pg_stat']) + master.start() # Settings for Replica - self.restore_node( - backup_dir, 'master', replica) + self.pb.restore_node('master', node=replica) # CHECK PHYSICAL CORRECTNESS on REPLICA - pgdata_replica = self.pgdata_content(replica.data_dir) - self.compare_pgdata(pgdata_master, pgdata_replica) + pgdata_replica = self.pgdata_content(replica.data_dir, exclude_dirs = ['pg_stat']) self.set_replica(master, replica, synchronous=False) # ADD INSTANCE REPLICA - # self.add_instance(backup_dir, 'replica', replica) + # self.pb.add_instance('replica', replica) # SET ARCHIVING FOR REPLICA - self.set_archiving(backup_dir, 'master', replica, replica=True) + self.pb.set_archiving('master', replica, replica=True) replica.slow_start(replica=True) + + # GET LOGICAL CONTENT FROM MASTER + before = master.table_checksum("t_heap") # CHECK LOGICAL CORRECTNESS on REPLICA after = replica.table_checksum("t_heap") - self.assertEqual(before, after) + + # self.assertEqual(before, after) + if before != after: + self.compare_pgdata(pgdata_master, pgdata_replica) master.psql( "postgres", @@ -939,18 +767,22 @@ def test_basic_master_and_replica_concurrent_archiving(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,10000) i") + # freeze bgwriter to get rid of RUNNING XACTS records + bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] + gdb_bgwriter = self.gdb_attach(bgwriter_pid) + # TAKE FULL ARCHIVE BACKUP FROM REPLICA - backup_id = self.backup_node(backup_dir, 'master', replica) + backup_id = self.pb.backup_node('master', replica) - self.validate_pb(backup_dir, 'master') + self.pb.validate('master') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) + 'OK', self.pb.show('master', backup_id)['status']) # TAKE FULL ARCHIVE BACKUP FROM MASTER - backup_id = self.backup_node(backup_dir, 'master', master) - self.validate_pb(backup_dir, 'master') + backup_id = self.pb.backup_node('master', master) + self.pb.validate('master') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'master', backup_id)['status']) + 'OK', self.pb.show('master', backup_id)['status']) master.pgbench_init(scale=10) @@ -961,8 +793,11 @@ def test_basic_master_and_replica_concurrent_archiving(self): master.pgbench_init(scale=10) replica.pgbench_init(scale=10) - self.backup_node(backup_dir, 'master', master) - self.backup_node(backup_dir, 'master', replica) + self.pb.backup_node('master', master) + self.pb.backup_node('master', replica, data_dir=replica.data_dir) + + # Clean after yourself + gdb_bgwriter.detach() # @unittest.expectedFailure # @unittest.skip("skip") @@ -977,55 +812,50 @@ def test_concurrent_archiving(self): if self.pg_config_version < self.version_to_num('11.0'): self.skipTest('You need PostgreSQL >= 11 for this test') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums']) + master = self.pg_node.make_simple('master', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', master) - self.set_archiving(backup_dir, 'node', master, replica=True) + self.pb.init() + self.pb.add_instance('node', master) + self.pb.set_archiving('node', master, replica=True) master.slow_start() master.pgbench_init(scale=10) # TAKE FULL ARCHIVE BACKUP FROM MASTER - self.backup_node(backup_dir, 'node', master) + self.pb.backup_node('node', master) # Settings for Replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'node', replica) + self.pb.restore_node('node', node=replica) self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'node', replica, replica=True) - self.set_auto_conf(replica, {'port': replica.port}) + self.pb.set_archiving('node', replica, replica=True) + replica.set_auto_conf({'port': replica.port}) replica.slow_start(replica=True) # create cascade replicas - replica1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica1')) + replica1 = self.pg_node.make_simple('replica1') replica1.cleanup() # Settings for casaced replica - self.restore_node(backup_dir, 'node', replica1) + self.pb.restore_node('node', node=replica1) self.set_replica(replica, replica1, synchronous=False) - self.set_auto_conf(replica1, {'port': replica1.port}) + replica1.set_auto_conf({'port': replica1.port}) replica1.slow_start(replica=True) # Take full backup from master - self.backup_node(backup_dir, 'node', master) + self.pb.backup_node('node', master) pgbench = master.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, options=['-T', '30', '-c', '1']) # Take several incremental backups from master - self.backup_node(backup_dir, 'node', master, backup_type='page', options=['--no-validate']) + self.pb.backup_node('node', master, backup_type='page', options=['--no-validate']) - self.backup_node(backup_dir, 'node', master, backup_type='page', options=['--no-validate']) + self.pb.backup_node('node', master, backup_type='page', options=['--no-validate']) pgbench.wait() pgbench.stdout.close() @@ -1046,23 +876,21 @@ def test_concurrent_archiving(self): # @unittest.skip("skip") def test_archive_pg_receivexlog(self): """Test backup with pg_receivexlog wal delivary method""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + if not backup_dir.is_file_based: + self.skipTest('test has no meaning for cloud storage') + + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - if self.get_version(node) < 100000: - pg_receivexlog_path = self.get_bin_path('pg_receivexlog') - else: - pg_receivexlog_path = self.get_bin_path('pg_receivewal') + pg_receivexlog_path = self.get_bin_path('pg_receivewal') - pg_receivexlog = self.run_binary( + pg_receivexlog = self.pb.run_binary( [ pg_receivexlog_path, '-p', str(node.port), '--synchronous', '-D', os.path.join(backup_dir, 'wal', 'node') @@ -1080,7 +908,7 @@ def test_archive_pg_receivexlog(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,10000) i") - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # PAGE node.safe_psql( @@ -1089,18 +917,17 @@ def test_archive_pg_receivexlog(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(10000,20000) i") - self.backup_node( - backup_dir, + self.pb.backup_node( 'node', node, backup_type='page' ) result = node.table_checksum("t_heap") - self.validate_pb(backup_dir) + self.pb.validate() # Check data correctness node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) node.slow_start() self.assertEqual( @@ -1115,23 +942,21 @@ def test_archive_pg_receivexlog(self): # @unittest.skip("skip") def test_archive_pg_receivexlog_compression_pg10(self): """Test backup with pg_receivewal compressed wal delivary method""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + if not backup_dir.is_file_based: + self.skipTest('test has no meaning for cloud storage') + + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s'} ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - if self.get_version(node) < self.version_to_num('10.0'): - self.skipTest('You need PostgreSQL >= 10 for this test') - else: - pg_receivexlog_path = self.get_bin_path('pg_receivewal') - pg_receivexlog = self.run_binary( + pg_receivexlog_path = self.get_bin_path('pg_receivewal') + pg_receivexlog = self.pb.run_binary( [ pg_receivexlog_path, '-p', str(node.port), '--synchronous', '-Z', '9', '-D', os.path.join(backup_dir, 'wal', 'node') @@ -1149,7 +974,7 @@ def test_archive_pg_receivexlog_compression_pg10(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,10000) i") - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # PAGE node.safe_psql( @@ -1158,16 +983,15 @@ def test_archive_pg_receivexlog_compression_pg10(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(10000,20000) i") - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='page' ) result = node.table_checksum("t_heap") - self.validate_pb(backup_dir) + self.pb.validate() # Check data correctness node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) node.slow_start() self.assertEqual( @@ -1196,22 +1020,16 @@ def test_archive_catalog(self): ARCHIVE master: t1 -Z1--Z2--- """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + backup_dir = self.backup_dir + master = self.pg_node.make_simple('master', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '30s', 'checkpoint_timeout': '30s'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) + self.pb.init() + self.pb.add_instance('master', master) + self.pb.set_archiving('master', master) master.slow_start() @@ -1222,7 +1040,7 @@ def test_archive_catalog(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,10000) i") - self.backup_node(backup_dir, 'master', master) + self.pb.backup_node('master', master) # PAGE master.safe_psql( @@ -1231,42 +1049,33 @@ def test_archive_catalog(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(10000,20000) i") - self.backup_node( - backup_dir, 'master', master, backup_type='page') + self.pb.backup_node('master', master, backup_type='page') - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'master', replica) + self.pb.restore_node('master', node=replica) self.set_replica(master, replica) - self.add_instance(backup_dir, 'replica', replica) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - - copy_tree( - os.path.join(backup_dir, 'wal', 'master'), - os.path.join(backup_dir, 'wal', 'replica')) + self.pb.add_instance('replica', replica) + self.pb.set_archiving('replica', replica, replica=True) replica.slow_start(replica=True) # FULL backup replica - Y1 = self.backup_node( - backup_dir, 'replica', replica, + Y1 = self.pb.backup_node('replica', replica, options=['--stream', '--archive-timeout=60s']) master.pgbench_init(scale=5) # PAGE backup replica - Y2 = self.backup_node( - backup_dir, 'replica', replica, + Y2 = self.pb.backup_node('replica', replica, backup_type='page', options=['--stream', '--archive-timeout=60s']) # create timeline t2 replica.promote() # FULL backup replica - A1 = self.backup_node( - backup_dir, 'replica', replica) + A1 = self.pb.backup_node('replica', replica) replica.pgbench_init(scale=5) @@ -1282,13 +1091,11 @@ def test_archive_catalog(self): target_xid = res[0][0] # DELTA backup replica - A2 = self.backup_node( - backup_dir, 'replica', replica, backup_type='delta') + A2 = self.pb.backup_node('replica', replica, backup_type='delta') # create timeline t3 replica.cleanup() - self.restore_node( - backup_dir, 'replica', replica, + self.pb.restore_node('replica', replica, options=[ '--recovery-target-xid={0}'.format(target_xid), '--recovery-target-timeline=2', @@ -1296,13 +1103,11 @@ def test_archive_catalog(self): replica.slow_start() - B1 = self.backup_node( - backup_dir, 'replica', replica) + B1 = self.pb.backup_node('replica', replica) replica.pgbench_init(scale=2) - B2 = self.backup_node( - backup_dir, 'replica', replica, backup_type='page') + B2 = self.pb.backup_node('replica', replica, backup_type='page') replica.pgbench_init(scale=2) @@ -1313,15 +1118,13 @@ def test_archive_catalog(self): con.commit() target_xid = res[0][0] - B3 = self.backup_node( - backup_dir, 'replica', replica, backup_type='page') + B3 = self.pb.backup_node('replica', replica, backup_type='page') replica.pgbench_init(scale=2) # create timeline t4 replica.cleanup() - self.restore_node( - backup_dir, 'replica', replica, + self.pb.restore_node('replica', replica, options=[ '--recovery-target-xid={0}'.format(target_xid), '--recovery-target-timeline=3', @@ -1352,8 +1155,7 @@ def test_archive_catalog(self): # create timeline t5 replica.cleanup() - self.restore_node( - backup_dir, 'replica', replica, + self.pb.restore_node('replica', replica, options=[ '--recovery-target-xid={0}'.format(target_xid), '--recovery-target-timeline=4', @@ -1371,8 +1173,7 @@ def test_archive_catalog(self): # create timeline t6 replica.cleanup() - self.restore_node( - backup_dir, 'replica', replica, backup_id=A1, + self.pb.restore_node('replica', replica, backup_id=A1, options=[ '--recovery-target=immediate', '--recovery-target-action=promote']) @@ -1382,8 +1183,8 @@ def test_archive_catalog(self): sleep(5) - show = self.show_archive(backup_dir, as_text=True) - show = self.show_archive(backup_dir) + show = self.pb.show_archive(as_text=True) + show = self.pb.show_archive() for instance in show: if instance['instance'] == 'replica': @@ -1400,36 +1201,19 @@ def test_archive_catalog(self): for timeline in master_timelines: self.assertTrue(timeline['status'], 'OK') - # create holes in t3 - wals_dir = os.path.join(backup_dir, 'wal', 'replica') - wals = [ - f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) - and not f.endswith('.backup') and not f.endswith('.history') and f.startswith('00000003') - ] - wals.sort() - # check that t3 is ok - self.show_archive(backup_dir) + self.pb.show_archive() - file = os.path.join(backup_dir, 'wal', 'replica', '000000030000000000000017') - if self.archive_compress: - file = file + '.gz' - os.remove(file) - - file = os.path.join(backup_dir, 'wal', 'replica', '000000030000000000000012') - if self.archive_compress: - file = file + '.gz' - os.remove(file) - - file = os.path.join(backup_dir, 'wal', 'replica', '000000030000000000000013') - if self.archive_compress: - file = file + '.gz' - os.remove(file) + # create holes in t3 + suffix = self.compress_suffix + self.remove_instance_wal(backup_dir, 'replica', '000000030000000000000017' + suffix) + self.remove_instance_wal(backup_dir, 'replica', '000000030000000000000012' + suffix) + self.remove_instance_wal(backup_dir, 'replica', '000000030000000000000013' + suffix) # check that t3 is not OK - show = self.show_archive(backup_dir) + show = self.pb.show_archive() - show = self.show_archive(backup_dir) + show = self.pb.show_archive() for instance in show: if instance['instance'] == 'replica': @@ -1514,37 +1298,33 @@ def test_archive_catalog_1(self): self.skipTest('You need to enable ARCHIVE_COMPRESSION ' 'for this test to run') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '30s', 'checkpoint_timeout': '30s'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node, compress=True) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node, compress=True) node.slow_start() # FULL - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.pgbench_init(scale=2) + tailer = tail_file(os.path.join(node.logs_dir, 'postgresql.log')) + tailer.wait_archive_push_completed() + node.stop() - wals_dir = os.path.join(backup_dir, 'wal', 'node') - original_file = os.path.join(wals_dir, '000000010000000000000001.gz') - tmp_file = os.path.join(wals_dir, '000000010000000000000001') - - with gzip.open(original_file, 'rb') as f_in, open(tmp_file, 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) - - os.rename( - os.path.join(wals_dir, '000000010000000000000001'), - os.path.join(wals_dir, '000000010000000000000002')) + file_content = self.read_instance_wal(backup_dir, 'node', + '000000010000000000000001'+self.compress_suffix, + decompress=True) + self.write_instance_wal(backup_dir, 'node', '000000010000000000000002', + file_content) - show = self.show_archive(backup_dir) + show = self.pb.show_archive() for instance in show: timelines = instance['timelines'] @@ -1566,39 +1346,37 @@ def test_archive_catalog_2(self): self.skipTest('You need to enable ARCHIVE_COMPRESSION ' 'for this test to run') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '30s', 'checkpoint_timeout': '30s'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node, compress=True) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node, compress=True) node.slow_start() # FULL - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.pgbench_init(scale=2) + tailer = tail_file(os.path.join(node.logs_dir, "postgresql.log")) + tailer.wait_archive_push_completed() + node.stop() - wals_dir = os.path.join(backup_dir, 'wal', 'node') - original_file = os.path.join(wals_dir, '000000010000000000000001.gz') - tmp_file = os.path.join(wals_dir, '000000010000000000000001') - - with gzip.open(original_file, 'rb') as f_in, open(tmp_file, 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) - - os.rename( - os.path.join(wals_dir, '000000010000000000000001'), - os.path.join(wals_dir, '000000010000000000000002')) + suffix = self.compress_suffix + file_content = self.read_instance_wal(backup_dir, 'node', + '000000010000000000000001'+suffix, + decompress=True) + self.write_instance_wal(backup_dir, 'node', '000000010000000000000002', + file_content) - os.remove(original_file) + self.remove_instance_wal(backup_dir, 'node', + '000000010000000000000001'+suffix) - show = self.show_archive(backup_dir) + show = self.pb.show_archive() for instance in show: timelines = instance['timelines'] @@ -1620,35 +1398,35 @@ def test_archive_options(self): if not self.remote: self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + if not backup_dir.is_file_based: + self.skipTest("Test has no meaning for cloud storage") + + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node, compress=True) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node, compress=True) node.slow_start() # FULL - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.pgbench_init(scale=1) node.cleanup() wal_dir = os.path.join(backup_dir, 'wal', 'node') - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ '--restore-command="cp {0}/%f %p"'.format(wal_dir), '--archive-host=localhost', '--archive-port=22', - '--archive-user={0}'.format(self.user) + '--archive-user={0}'.format(self.username) ]) - if self.get_version(node) >= self.version_to_num('12.0'): + if self.pg_config_version >= self.version_to_num('12.0'): recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') else: recovery_conf = os.path.join(node.data_dir, 'recovery.conf') @@ -1662,12 +1440,11 @@ def test_archive_options(self): node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ '--archive-host=localhost', '--archive-port=22', - '--archive-user={0}'.format(self.user)]) + '--archive-user={0}'.format(self.username)]) with open(recovery_conf, 'r') as f: recovery_content = f.read() @@ -1676,7 +1453,7 @@ def test_archive_options(self): "restore_command = '\"{0}\" archive-get -B \"{1}\" --instance \"{2}\" " "--wal-file-path=%p --wal-file-name=%f --remote-host=localhost " "--remote-port=22 --remote-user={3}'".format( - self.probackup_path, backup_dir, 'node', self.user), + self.probackup_path, backup_dir, 'node', self.username), recovery_content) node.slow_start() @@ -1692,35 +1469,35 @@ def test_archive_options_1(self): check that '--archive-host', '--archive-user', '--archiver-port' and '--restore-command' are working as expected with set-config """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + if not backup_dir.is_file_based: + self.skipTest("Test has no meaning for cloud storage") + + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node, compress=True) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node, compress=True) node.slow_start() # FULL - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.pgbench_init(scale=1) node.cleanup() wal_dir = os.path.join(backup_dir, 'wal', 'node') - self.set_config( - backup_dir, 'node', + self.pb.set_config('node', options=[ '--restore-command="cp {0}/%f %p"'.format(wal_dir), '--archive-host=localhost', '--archive-port=22', - '--archive-user={0}'.format(self.user)]) - self.restore_node(backup_dir, 'node', node) + '--archive-user={0}'.format(self.username)]) + self.pb.restore_node('node', node=node) - if self.get_version(node) >= self.version_to_num('12.0'): + if self.pg_config_version >= self.version_to_num('12.0'): recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') else: recovery_conf = os.path.join(node.data_dir, 'recovery.conf') @@ -1734,13 +1511,12 @@ def test_archive_options_1(self): node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ '--restore-command=none', '--archive-host=localhost1', '--archive-port=23', - '--archive-user={0}'.format(self.user) + '--archive-user={0}'.format(self.username) ]) with open(recovery_conf, 'r') as f: @@ -1750,7 +1526,7 @@ def test_archive_options_1(self): "restore_command = '\"{0}\" archive-get -B \"{1}\" --instance \"{2}\" " "--wal-file-path=%p --wal-file-name=%f --remote-host=localhost1 " "--remote-port=23 --remote-user={3}'".format( - self.probackup_path, backup_dir, 'node', self.user), + self.probackup_path, backup_dir, 'node', self.username), recovery_content) # @unittest.skip("skip") @@ -1760,27 +1536,23 @@ def test_undefined_wal_file_path(self): check that archive-push works correct with undefined --wal-file-path """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + archive_command = " ".join([f'"{self.probackup_path}"', 'archive-push', + *self.backup_dir.pb_args, '--instance=node', + '--wal-file-name=%f']) if os.name == 'posix': - archive_command = '\"{0}\" archive-push -B \"{1}\" --instance \"{2}\" --wal-file-name=%f'.format( - self.probackup_path, backup_dir, 'node') - elif os.name == 'nt': - archive_command = '\"{0}\" archive-push -B \"{1}\" --instance \"{2}\" --wal-file-name=%f'.format( - self.probackup_path, backup_dir, 'node').replace("\\","\\\\") - else: - self.assertTrue(False, 'Unexpected os family') + # Dash produces a core dump when it gets a SIGQUIT from its + # child process so replace the shell with pg_probackup + archive_command = 'exec ' + archive_command + elif os.name == "nt": + archive_command = archive_command.replace("\\","\\\\") - self.set_auto_conf( - node, - {'archive_command': archive_command}) + self.pb.set_archiving('node', node, custom_archive_command=archive_command) node.slow_start() node.safe_psql( @@ -1788,9 +1560,15 @@ def test_undefined_wal_file_path(self): "create table t_heap as select i" " as id from generate_series(0, 10) i") self.switch_wal_segment(node) + tailer = tail_file(os.path.join(node.logs_dir, "postgresql.log")) + tailer.wait_archive_push_completed() + node.stop() + + log = tail_file(os.path.join(node.logs_dir, 'postgresql.log'), collect=True) + log.wait(contains='archive-push completed successfully') # check - self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], '000000010000000000000001') + self.assertEqual(self.pb.show_archive(instance='node', tli=1)['min-segno'], '000000010000000000000001') # @unittest.skip("skip") # @unittest.expectedFailure @@ -1798,29 +1576,25 @@ def test_intermediate_archiving(self): """ check that archive-push works correct with --wal-file-path setting by user """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') node_pg_options = {} if node.major_version >= 13: node_pg_options['wal_keep_size'] = '0MB' else: node_pg_options['wal_keep_segments'] = '0' - self.set_auto_conf(node, node_pg_options) + node.set_auto_conf(node_pg_options) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) - wal_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'intermediate_dir') + wal_dir = os.path.join(self.test_path, 'intermediate_dir') shutil.rmtree(wal_dir, ignore_errors=True) os.makedirs(wal_dir) if os.name == 'posix': - self.set_archiving(backup_dir, 'node', node, custom_archive_command='cp -v %p {0}/%f'.format(wal_dir)) + self.pb.set_archiving('node', node, custom_archive_command='cp -v %p {0}/%f'.format(wal_dir)) elif os.name == 'nt': - self.set_archiving(backup_dir, 'node', node, custom_archive_command='copy /Y "%p" "{0}\\\\%f"'.format(wal_dir.replace("\\","\\\\"))) + self.pb.set_archiving('node', node, custom_archive_command='copy /Y "%p" "{0}\\\\%f"'.format(wal_dir.replace("\\","\\\\"))) else: self.assertTrue(False, 'Unexpected os family') @@ -1833,11 +1607,10 @@ def test_intermediate_archiving(self): wal_segment = '000000010000000000000001' - self.run_pb(["archive-push", "-B", backup_dir, - "--instance=node", "-D", node.data_dir, - "--wal-file-path", "{0}/{1}".format(wal_dir, wal_segment), "--wal-file-name", wal_segment]) + self.pb.archive_push('node', node, wal_file_path="{0}/{1}".format(wal_dir, wal_segment), + wal_file_name=wal_segment) - self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], wal_segment) + self.assertEqual(self.pb.show_archive(instance='node', tli=1)['min-segno'], wal_segment) # @unittest.skip("skip") # @unittest.expectedFailure @@ -1845,21 +1618,15 @@ def test_waldir_outside_pgdata_archiving(self): """ check that archive-push works correct with symlinked waldir """ - if self.pg_config_version < self.version_to_num('10.0'): - self.skipTest( - 'Skipped because waldir outside pgdata is supported since PG 10') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - external_wal_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'ext_wal_dir') + backup_dir = self.backup_dir + external_wal_dir = os.path.join(self.test_path, 'ext_wal_dir') shutil.rmtree(external_wal_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums', '--waldir={0}'.format(external_wal_dir)]) + node = self.pg_node.make_simple('node', initdb_params=['--waldir={0}'.format(external_wal_dir)]) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -1868,8 +1635,12 @@ def test_waldir_outside_pgdata_archiving(self): " as id from generate_series(0, 10) i") self.switch_wal_segment(node) + tailer = tail_file(os.path.join(node.logs_dir, 'postgresql.log')) + tailer.wait_archive_push_completed() + node.stop() + # check - self.assertEqual(self.show_archive(backup_dir, instance='node', tli=1)['min-segno'], '000000010000000000000001') + self.assertEqual(self.pb.show_archive(instance='node', tli=1)['min-segno'], '000000010000000000000001') # @unittest.skip("skip") # @unittest.expectedFailure @@ -1877,33 +1648,30 @@ def test_hexadecimal_timeline(self): """ Check that timelines are correct. """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node, log_level='verbose') + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) node.pgbench_init(scale=2) # create timelines for i in range(1, 13): # print(i) node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=['--recovery-target-timeline={0}'.format(i)]) node.slow_start() node.pgbench_init(scale=2) sleep(5) - show = self.show_archive(backup_dir) + show = self.pb.show_archive() timelines = show[0]['timelines'] @@ -1934,29 +1702,27 @@ def test_archiving_and_slots(self): Check that archiving don`t break slot guarantee. """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + if not backup_dir.is_file_based: + self.skipTest("Test has no meaning for cloud storage") + + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', 'max_wal_size': '64MB'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node, log_level='verbose') + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - if self.get_version(node) < 100000: - pg_receivexlog_path = self.get_bin_path('pg_receivexlog') - else: - pg_receivexlog_path = self.get_bin_path('pg_receivewal') + pg_receivexlog_path = self.get_bin_path('pg_receivewal') # "pg_receivewal --create-slot --slot archive_slot --if-not-exists " # "&& pg_receivewal --synchronous -Z 1 /tmp/wal --slot archive_slot --no-loop" - self.run_binary( + self.pb.run_binary( [ pg_receivexlog_path, '-p', str(node.port), '--synchronous', '--create-slot', '--slot', 'archive_slot', '--if-not-exists' @@ -1964,7 +1730,7 @@ def test_archiving_and_slots(self): node.pgbench_init(scale=10) - pg_receivexlog = self.run_binary( + pg_receivexlog = self.pb.run_binary( [ pg_receivexlog_path, '-p', str(node.port), '--synchronous', '-D', os.path.join(backup_dir, 'wal', 'node'), @@ -1982,56 +1748,51 @@ def test_archiving_and_slots(self): pg_receivexlog.kill() - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) node.pgbench_init(scale=20) exit(1) def test_archive_push_sanity(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'archive_mode': 'on', 'archive_command': 'exit 1'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.pgbench_init(scale=50) node.stop() - self.set_archiving(backup_dir, 'node', node) + self.pb.set_archiving('node', node) os.remove(os.path.join(node.logs_dir, 'postgresql.log')) node.slow_start() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) with open(os.path.join(node.logs_dir, 'postgresql.log'), 'r') as f: postgres_log_content = cleanup_ptrack(f.read()) # print(postgres_log_content) # make sure that .backup file is not compressed - self.assertNotIn('.backup.gz', postgres_log_content) + if self.archive_compress: + self.assertNotIn('.backup'+self.compress_suffix, postgres_log_content) self.assertNotIn('WARNING', postgres_log_content) - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node( - backup_dir, 'node', replica, - data_dir=replica.data_dir, options=['-R']) + self.pb.restore_node('node', replica, options=['-R']) - # self.set_archiving(backup_dir, 'replica', replica, replica=True) - self.set_auto_conf(replica, {'port': replica.port}) - self.set_auto_conf(replica, {'archive_mode': 'always'}) - self.set_auto_conf(replica, {'hot_standby': 'on'}) + # self.pb.set_archiving('replica', replica, replica=True) + replica.set_auto_conf({'port': replica.port}) + replica.set_auto_conf({'archive_mode': 'always'}) + replica.set_auto_conf({'hot_standby': 'on'}) replica.slow_start(replica=True) self.wait_until_replica_catch_with_master(node, replica) @@ -2042,24 +1803,25 @@ def test_archive_push_sanity(self): replica.pgbench_init(scale=10) log = tail_file(os.path.join(replica.logs_dir, 'postgresql.log'), - collect=True) + collect=True, linetimeout=30) log.wait(regex=r"pushing file.*history") - log.wait(contains='archive-push completed successfully') + log.wait_archive_push_completed() log.wait(regex=r"pushing file.*partial") - log.wait(contains='archive-push completed successfully') + log.wait_archive_push_completed() - # make sure that .partial file is not compressed - self.assertNotIn('.partial.gz', log.content) - # make sure that .history file is not compressed - self.assertNotIn('.history.gz', log.content) + if self.archive_compress: + # make sure that .partial file is not compressed + self.assertNotIn('.partial'+self.compress_suffix, log.content) + # make sure that .history file is not compressed + self.assertNotIn('.history'+self.compress_suffix, log.content) replica.stop() log.wait_shutdown() self.assertNotIn('WARNING', cleanup_ptrack(log.content)) - output = self.show_archive( - backup_dir, 'node', as_json=False, as_text=True, + output = self.pb.show_archive( + 'node', as_json=False, as_text=True, options=['--log-level-console=INFO']) self.assertNotIn('WARNING', output) @@ -2068,27 +1830,20 @@ def test_archive_push_sanity(self): # @unittest.skip("skip") def test_archive_pg_receivexlog_partial_handling(self): """check that archive-get delivers .partial and .gz.partial files""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + if not backup_dir.is_file_based: + self.skipTest("Test has no meaning for cloud storage") - if self.get_version(node) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - if self.get_version(node) < 100000: - app_name = 'pg_receivexlog' - pg_receivexlog_path = self.get_bin_path('pg_receivexlog') - else: - app_name = 'pg_receivewal' - pg_receivexlog_path = self.get_bin_path('pg_receivewal') + app_name = 'pg_receivewal' + pg_receivexlog_path = self.get_bin_path('pg_receivewal') cmdline = [ pg_receivexlog_path, '-p', str(node.port), '--synchronous', @@ -2099,7 +1854,7 @@ def test_archive_pg_receivexlog_partial_handling(self): env = self.test_env env["PGAPPNAME"] = app_name - pg_receivexlog = self.run_binary(cmdline, asynchronous=True, env=env) + pg_receivexlog = self.pb.run_binary(cmdline, asynchronous=True, env=env) if pg_receivexlog.returncode: self.assertFalse( @@ -2107,12 +1862,12 @@ def test_archive_pg_receivexlog_partial_handling(self): 'Failed to start pg_receivexlog: {0}'.format( pg_receivexlog.communicate()[1])) - self.set_auto_conf(node, {'synchronous_standby_names': app_name}) - self.set_auto_conf(node, {'synchronous_commit': 'on'}) + node.set_auto_conf({'synchronous_standby_names': app_name}) + node.set_auto_conf({'synchronous_commit': 'on'}) node.reload() # FULL - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) node.safe_psql( "postgres", @@ -2121,8 +1876,7 @@ def test_archive_pg_receivexlog_partial_handling(self): "from generate_series(0,1000000) i") # PAGE - self.backup_node( - backup_dir, 'node', node, backup_type='page', options=['--stream']) + self.pb.backup_node('node', node, backup_type='page', options=['--stream']) node.safe_psql( "postgres", @@ -2132,15 +1886,13 @@ def test_archive_pg_receivexlog_partial_handling(self): pg_receivexlog.kill() - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, node_restored.data_dir, + self.pb.restore_node('node', node_restored, options=['--recovery-target=latest', '--recovery-target-action=promote']) - self.set_auto_conf(node_restored, {'port': node_restored.port}) - self.set_auto_conf(node_restored, {'hot_standby': 'off'}) + node_restored.set_auto_conf({'port': node_restored.port}) + node_restored.set_auto_conf({'hot_standby': 'off'}) node_restored.slow_start() @@ -2152,19 +1904,17 @@ def test_archive_pg_receivexlog_partial_handling(self): @unittest.skip("skip") def test_multi_timeline_recovery_prefetching(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.pgbench_init(scale=50) @@ -2177,8 +1927,7 @@ def test_multi_timeline_recovery_prefetching(self): node.stop() node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ '--recovery-target-xid={0}'.format(target_xid), '--recovery-target-action=promote']) @@ -2194,8 +1943,7 @@ def test_multi_timeline_recovery_prefetching(self): node.stop(['-m', 'immediate', '-D', node.data_dir]) node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ # '--recovery-target-xid={0}'.format(target_xid), '--recovery-target-timeline=2', @@ -2208,8 +1956,7 @@ def test_multi_timeline_recovery_prefetching(self): node.stop() node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ # '--recovery-target-xid=100500', '--recovery-target-timeline=3', @@ -2217,7 +1964,7 @@ def test_multi_timeline_recovery_prefetching(self): '--no-validate']) os.remove(os.path.join(node.logs_dir, 'postgresql.log')) - restore_command = self.get_restore_command(backup_dir, 'node', node) + restore_command = self.get_restore_command(backup_dir, 'node') restore_command += ' -j 2 --batch-size=10 --log-level-console=VERBOSE' if node.major_version >= 12: @@ -2259,50 +2006,42 @@ def test_archive_get_batching_sanity(self): .gz file is corrupted and uncompressed is not, check that both corruption detected and uncompressed file is used. """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - if self.get_version(node) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) node.pgbench_init(scale=50) - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node( - backup_dir, 'node', replica, replica.data_dir) + self.pb.restore_node('node', node=replica) self.set_replica(node, replica, log_shipping=True) if node.major_version >= 12: - self.set_auto_conf(replica, {'restore_command': 'exit 1'}) + replica.set_auto_conf({'restore_command': 'exit 1'}) else: replica.append_conf('recovery.conf', "restore_command = 'exit 1'") replica.slow_start(replica=True) # at this point replica is consistent - restore_command = self.get_restore_command(backup_dir, 'node', replica) + restore_command = self.get_restore_command(backup_dir, 'node') restore_command += ' -j 2 --batch-size=10' # print(restore_command) if node.major_version >= 12: - self.set_auto_conf(replica, {'restore_command': restore_command}) + replica.set_auto_conf({'restore_command': restore_command}) else: replica.append_conf( 'recovery.conf', "restore_command = '{0}'".format(restore_command)) @@ -2326,45 +2065,49 @@ def test_archive_get_prefetch_corruption(self): Make sure that WAL corruption is detected. And --prefetch-dir is honored. """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) + node.set_auto_conf({ + 'wal_compression': 'off', + }) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) - node.pgbench_init(scale=50) + node.pgbench_init(scale=20) - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node( - backup_dir, 'node', replica, replica.data_dir) + self.pb.restore_node('node', node=replica) self.set_replica(node, replica, log_shipping=True) if node.major_version >= 12: - self.set_auto_conf(replica, {'restore_command': 'exit 1'}) + replica.set_auto_conf({'restore_command': 'exit 1'}) else: replica.append_conf('recovery.conf', "restore_command = 'exit 1'") + log = tail_file(os.path.join(node.logs_dir, 'postgresql.log'), + linetimeout=30) + log.wait(regex=r"pushing file.*000000D") + log.wait_archive_push_completed() + replica.slow_start(replica=True) # at this point replica is consistent - restore_command = self.get_restore_command(backup_dir, 'node', replica) + restore_command = self.get_restore_command(backup_dir, 'node') restore_command += ' -j5 --batch-size=10 --log-level-console=VERBOSE' #restore_command += ' --batch-size=2 --log-level-console=VERBOSE' if node.major_version >= 12: - self.set_auto_conf(replica, {'restore_command': restore_command}) + replica.set_auto_conf({'restore_command': restore_command}) else: replica.append_conf( 'recovery.conf', "restore_command = '{0}'".format(restore_command)) @@ -2387,35 +2130,31 @@ def test_archive_get_prefetch_corruption(self): # generate WAL, copy it into prefetch directory, then corrupt # some segment - node.pgbench_init(scale=20) + node.pgbench_init(scale=5) sleep(20) # now copy WAL files into prefetch directory and corrupt some of them - archive_dir = os.path.join(backup_dir, 'wal', 'node') - files = os.listdir(archive_dir) - files.sort() + files = self.get_instance_wal_list(backup_dir, 'node') + suffix = self.compress_suffix for filename in [files[-4], files[-3], files[-2], files[-1]]: - src_file = os.path.join(archive_dir, filename) + content = self.read_instance_wal(backup_dir, 'node', filename, + decompress=True) if node.major_version >= 10: wal_dir = 'pg_wal' else: wal_dir = 'pg_xlog' - if filename.endswith('.gz'): - dst_file = os.path.join(replica.data_dir, wal_dir, 'pbk_prefetch', filename[:-3]) - with gzip.open(src_file, 'rb') as f_in, open(dst_file, 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) - else: - dst_file = os.path.join(replica.data_dir, wal_dir, 'pbk_prefetch', filename) - shutil.copyfile(src_file, dst_file) - - # print(dst_file) + if suffix and filename.endswith(suffix): + filename = filename[:-len(suffix)] + dst_file = os.path.join(replica.data_dir, wal_dir, 'pbk_prefetch', filename) + with open(dst_file, 'wb') as f_out: + f_out.write(content) # corrupt file - if files[-2].endswith('.gz'): - filename = files[-2][:-3] + if suffix and files[-2].endswith(suffix): + filename = files[-2][:-len(suffix)] else: filename = files[-2] @@ -2425,14 +2164,13 @@ def test_archive_get_prefetch_corruption(self): f.seek(8192*2) f.write(b"SURIKEN") f.flush() - f.close # enable restore_command - restore_command = self.get_restore_command(backup_dir, 'node', replica) + restore_command = self.get_restore_command(backup_dir, 'node') restore_command += ' --batch-size=2 --log-level-console=VERBOSE' if node.major_version >= 12: - self.set_auto_conf(replica, {'restore_command': restore_command}) + replica.set_auto_conf({'restore_command': restore_command}) else: replica.append_conf( 'recovery.conf', "restore_command = '{0}'".format(restore_command)) @@ -2442,6 +2180,9 @@ def test_archive_get_prefetch_corruption(self): prefetch_line = 'Prefetched WAL segment {0} is invalid, cannot use it'.format(filename) restored_line = 'LOG: restored log file "{0}" from archive'.format(filename) + + self.wait_server_wal_exists(replica.data_dir, wal_dir, filename) + tailer = tail_file(os.path.join(replica.logs_dir, 'postgresql.log')) tailer.wait(contains=prefetch_line) tailer.wait(contains=restored_line) @@ -2452,122 +2193,102 @@ def test_archive_show_partial_files_handling(self): check that files with '.part', '.part.gz', '.partial' and '.partial.gz' siffixes are handled correctly """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node, compress=False) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node, compress=False) node.slow_start() - self.backup_node(backup_dir, 'node', node) - - wals_dir = os.path.join(backup_dir, 'wal', 'node') + self.pb.backup_node('node', node) # .part file - node.safe_psql( - "postgres", - "create table t1()") + if backup_dir.is_file_based: + wals_dir = os.path.join(backup_dir, 'wal', 'node') - if self.get_version(node) < 100000: - filename = node.safe_psql( + node.safe_psql( "postgres", - "SELECT file_name " - "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() - else: + "create table t1()") + filename = node.safe_psql( "postgres", "SELECT file_name " "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() - filename = filename.decode('utf-8') + filename = filename.decode('utf-8') - self.switch_wal_segment(node) - - os.rename( - os.path.join(wals_dir, filename), - os.path.join(wals_dir, '{0}.part'.format(filename))) + self.switch_wal_segment(node) - # .gz.part file - node.safe_psql( - "postgres", - "create table t2()") + self.wait_instance_wal_exists(backup_dir, 'node', filename) + os.rename( + os.path.join(wals_dir, filename), + os.path.join(wals_dir, '{0}~tmp123451'.format(filename))) - if self.get_version(node) < 100000: - filename = node.safe_psql( + # .gz.part file + node.safe_psql( "postgres", - "SELECT file_name " - "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() - else: + "create table t2()") + filename = node.safe_psql( "postgres", "SELECT file_name " "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() - filename = filename.decode('utf-8') + filename = filename.decode('utf-8') - self.switch_wal_segment(node) + self.switch_wal_segment(node) - os.rename( - os.path.join(wals_dir, filename), - os.path.join(wals_dir, '{0}.gz.part'.format(filename))) + self.wait_instance_wal_exists(backup_dir, 'node', filename) + os.rename( + os.path.join(wals_dir, filename), + os.path.join(wals_dir, f'{filename}{self.compress_suffix}~tmp234513')) # .partial file node.safe_psql( "postgres", "create table t3()") - if self.get_version(node) < 100000: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() - else: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() filename = filename.decode('utf-8') self.switch_wal_segment(node) - os.rename( - os.path.join(wals_dir, filename), - os.path.join(wals_dir, '{0}.partial'.format(filename))) + self.wait_instance_wal_exists(backup_dir, 'node', filename) + file_content = self.read_instance_wal(backup_dir, 'node', filename) + self.write_instance_wal(backup_dir, 'node', f'{filename}.partial', + file_content) + self.remove_instance_wal(backup_dir, 'node', filename) # .gz.partial file node.safe_psql( "postgres", "create table t4()") - if self.get_version(node) < 100000: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_xlogfile_name_offset(pg_current_xlog_location())").rstrip() - else: - filename = node.safe_psql( - "postgres", - "SELECT file_name " - "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() + filename = node.safe_psql( + "postgres", + "SELECT file_name " + "FROM pg_walfile_name_offset(pg_current_wal_flush_lsn())").rstrip() filename = filename.decode('utf-8') self.switch_wal_segment(node) - os.rename( - os.path.join(wals_dir, filename), - os.path.join(wals_dir, '{0}.gz.partial'.format(filename))) + self.wait_instance_wal_exists(backup_dir, 'node', filename) + file_content = self.read_instance_wal(backup_dir, 'node', filename) + self.write_instance_wal(backup_dir, 'node', f'{filename}{self.compress_suffix}.partial', + file_content) + self.remove_instance_wal(backup_dir, 'node', filename) - self.show_archive(backup_dir, 'node', options=['--log-level-file=VERBOSE']) + self.pb.show_archive('node', options=['--log-level-file=VERBOSE']) - with open(os.path.join(backup_dir, 'log', 'pg_probackup.log'), 'r') as f: - log_content = f.read() + log_content = self.read_pb_log() self.assertNotIn( 'WARNING', @@ -2579,27 +2300,24 @@ def test_archive_empty_history_file(self): """ https://github.com/postgrespro/pg_probackup/issues/326 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=5) # FULL - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.pgbench_init(scale=5) node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ '--recovery-target=latest', '--recovery-target-action=promote']) @@ -2610,8 +2328,7 @@ def test_archive_empty_history_file(self): node.pgbench_init(scale=5) node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ '--recovery-target=latest', '--recovery-target-timeline=2', @@ -2623,8 +2340,7 @@ def test_archive_empty_history_file(self): node.pgbench_init(scale=5) node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ '--recovery-target=latest', '--recovery-target-timeline=3', @@ -2636,32 +2352,192 @@ def test_archive_empty_history_file(self): # Truncate history files for tli in range(2, 5): - file = os.path.join( - backup_dir, 'wal', 'node', '0000000{0}.history'.format(tli)) - with open(file, "w+") as f: - f.truncate() + self.write_instance_wal(backup_dir, 'node', f'0000000{tli}.history', + b'') - timelines = self.show_archive(backup_dir, 'node', options=['--log-level-file=INFO']) + timelines = self.pb.show_archive('node', options=['--log-level-file=INFO']) # check that all timelines has zero switchpoint for timeline in timelines: self.assertEqual(timeline['switchpoint'], '0/0') - log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(log_file, 'r') as f: - log_content = f.read() - wal_dir = os.path.join(backup_dir, 'wal', 'node') + log_content = self.read_pb_log() - self.assertIn( - 'WARNING: History file is corrupted or missing: "{0}"'.format(os.path.join(wal_dir, '00000002.history')), - log_content) - self.assertIn( - 'WARNING: History file is corrupted or missing: "{0}"'.format(os.path.join(wal_dir, '00000003.history')), - log_content) - self.assertIn( - 'WARNING: History file is corrupted or missing: "{0}"'.format(os.path.join(wal_dir, '00000004.history')), - log_content) + self.assertRegex( + log_content, + 'WARNING: History file is corrupted or missing: "[^"]*00000002.history"') + self.assertRegex( + log_content, + 'WARNING: History file is corrupted or missing: "[^"]*00000003.history"') + self.assertRegex( + log_content, + 'WARNING: History file is corrupted or missing: "[^"]*00000004.history"') + + def test_archive_get_relative_path(self): + """ + Take a backup in archive mode, restore it and run the cluster + on it with relative pgdata path, archive-get should be ok with + relative pgdata path as well. + """ + + # initialize basic node + node = self.pg_node.make_simple( + base_dir='node', + pg_options={ + 'archive_timeout': '10s'} + ) + + # initialize the node to restore to + restored = self.pg_node.make_empty(base_dir='restored') + + # initialize pg_probackup setup including archiving + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + + # the job + node.slow_start() + self.pb.backup_node('node', node) + node.stop() + self.pb.restore_node('node', restored) + restored.set_auto_conf({"port": restored.port}) + + run_path = os.getcwd() + relative_pgdata = get_relative_path(run_path, restored.data_dir) + + restored.start(params=["-D", relative_pgdata]) + # cleanup + restored.stop() + + def test_archive_push_alot_of_files(self): + """ + Test archive-push pushes files in-order. + PBCKP-911 + """ + if self.pg_config_version < 130000: + self.skipTest("too costly to test with 16MB wal segment") + + node = self.pg_node.make_simple(base_dir='node', + initdb_params=['--wal-segsize','1'], + pg_options={ + 'archive_mode': 'on', + }) + + self.pb.init() + self.pb.add_instance('node', node) + + pg_wal_dir = os.path.join(node.data_dir, 'pg_wal') + + node.slow_start() + # create many segments + for i in range(30): + node.execute("select pg_logical_emit_message(False, 'z', repeat('0', 1024*1024))") + # EXT4 always stores in hash table, so test could skip following two + # loops if it runs on EXT4. + # + # But for XFS we have to disturb file order manually. + # 30-30-30 is empirically obtained: pg_wal/archive_status doesn't overflow + # to B+Tree yet, but already reuses some of removed items + for i in range(1,30): + fl = f'{1:08x}{0:08x}{i:08X}' + if os.path.exists(os.path.join(pg_wal_dir, fl)): + os.remove(os.path.join(pg_wal_dir, fl)) + os.remove(os.path.join(pg_wal_dir, f'archive_status/{fl}.ready')) + for i in range(30): + node.execute("select pg_logical_emit_message(False, 'z', repeat('0', 1024*1024))") + + node.stop() + + files = os.listdir(pg_wal_dir) + files.sort() + n = int(len(files)/2) + + self.pb.archive_push("node", node, wal_file_name=files[0], wal_file_path=pg_wal_dir, + options=['--threads', '10', + '--batch-size', str(n), + '--log-level-file', 'VERBOSE']) + + archived = self.get_instance_wal_list(self.backup_dir, 'node') + + self.assertListEqual(files[:n], archived) + +################################################################# +# dry-run +################################################################# + + @unittest.skipUnless(fs_backup_class.is_file_based, "AccessPath check is always true on s3") + def test_dry_run_archive_push(self): + """ Check archive-push command with dry_run option""" + node = self.pg_node.make_simple('node', + set_replication=True) + self.pb.init() + self.pb.add_instance('node', node) + + node.slow_start() + node.pgbench_init(scale=10) + + walfile = node.safe_psql( + 'postgres', + 'select pg_walfile_name(pg_current_wal_lsn())').decode('utf-8').rstrip() + self.pb.archive_push('node', node=node, wal_file_name=walfile, options=['--dry-run']) + + self.assertTrue(len(self.backup_dir.list_dirs((os.path.join(self.backup_dir, 'wal/node')))) == 0) + # Access check suit if disk mounted as read_only + if fs_backup_class.is_file_based: #AccessPath check is always true on s3 + dir_path = os.path.join(self.backup_dir, 'wal/node') + dir_mode = os.stat(dir_path).st_mode + os.chmod(dir_path, 0o400) + print(self.backup_dir) + + error_message = self.pb.archive_push('node', node=node, wal_file_name=walfile, options=['--dry-run'], + expect_error="because of changed permissions") + try: + self.assertMessage(error_message, contains='ERROR: Check permissions') + finally: + # Cleanup + os.chmod(dir_path, dir_mode) + + node.stop() + + @unittest.skipUnless(fs_backup_class.is_file_based, "AccessPath check is always true on s3") + def test_archive_get_dry_run(self): + """ + Check archive-get command with dry-ryn option + """ + # initialize basic node + node = self.pg_node.make_simple( + base_dir='node', + pg_options={ + 'archive_timeout': '3s'} + ) + + # initialize the node to restore to + restored = self.pg_node.make_empty(base_dir='restored') + + # initialize pg_probackup setup including archiving + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + + # the job + node.slow_start() + node.pgbench_init(scale=10) + + self.pb.backup_node('node', node) + self.pb.restore_node('node', restored, options=['--recovery-target=latest']) + restored.set_auto_conf({"port": restored.port}) + + files = self.get_instance_wal_list(self.backup_dir, 'node') + cwd = os.getcwd() + os.chdir(restored.data_dir) + wal_dir = self.pgdata_content(os.path.join(restored.data_dir, 'pg_wal')) + self.pb.archive_get('node', wal_file_name=files[-1], wal_file_path="{0}/{1}".format('pg_wal', files[-1]), + options=['--dry-run', "-D", restored.data_dir]) + restored_wal = self.pgdata_content(os.path.join(restored.data_dir, 'pg_wal')) + self.compare_pgdata(wal_dir, restored_wal) + os.chdir(cwd) + node.stop() def cleanup_ptrack(log_content): # PBCKP-423 - need to clean ptrack warning diff --git a/tests/auth_test.py b/tests/auth_test.py index 32cabc4a1..d1a7c707c 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -1,388 +1,20 @@ -""" -The Test suite check behavior of pg_probackup utility, if password is required for connection to PostgreSQL instance. - - https://confluence.postgrespro.ru/pages/viewpage.action?pageId=16777522 -""" +from .helpers.ptrack_helpers import ProbackupTest -import os -import unittest -import signal -import time -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from testgres import StartNodeException +class AuthorizationTest(ProbackupTest): + """ + Check connect to S3 via pre_start_checks() function + calling pg_probackup init --s3 -module_name = 'auth_test' -skip_test = False + test that s3 keys allow to connect to all types of storages + """ + def test_s3_auth_test(self): + console_output = self.pb.init(options=["--log-level-console=VERBOSE"]) -try: - from pexpect import * -except ImportError: - skip_test = True - - -class SimpleAuthTest(ProbackupTest, unittest.TestCase): - - # @unittest.skip("skip") - def test_backup_via_unprivileged_user(self): - """ - Make node, create unprivileged user, try to - run a backups without EXECUTE rights on - certain functions - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - if self.ptrack: - node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - node.safe_psql("postgres", "CREATE ROLE backup with LOGIN") - - try: - self.backup_node( - backup_dir, 'node', node, options=['-U', 'backup']) - self.assertEqual( - 1, 0, - "Expecting Error due to missing grant on EXECUTE.") - except ProbackupException as e: - if self.get_version(node) < 150000: - self.assertIn( - "ERROR: query failed: ERROR: permission denied " - "for function pg_start_backup", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - else: - self.assertIn( - "ERROR: query failed: ERROR: permission denied " - "for function pg_backup_start", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - if self.get_version(node) < 150000: - node.safe_psql( - "postgres", - "GRANT EXECUTE ON FUNCTION" - " pg_start_backup(text, boolean, boolean) TO backup;") - else: - node.safe_psql( - "postgres", - "GRANT EXECUTE ON FUNCTION" - " pg_backup_start(text, boolean) TO backup;") - - if self.get_version(node) < 100000: - node.safe_psql( - 'postgres', - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup") - else: - node.safe_psql( - 'postgres', - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup") - - try: - self.backup_node( - backup_dir, 'node', node, options=['-U', 'backup']) - self.assertEqual( - 1, 0, - "Expecting Error due to missing grant on EXECUTE.") - except ProbackupException as e: - self.assertIn( - "ERROR: query failed: ERROR: permission denied for function " - "pg_create_restore_point\nquery was: " - "SELECT pg_catalog.pg_create_restore_point($1)", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - node.safe_psql( - "postgres", - "GRANT EXECUTE ON FUNCTION" - " pg_create_restore_point(text) TO backup;") - - try: - self.backup_node( - backup_dir, 'node', node, options=['-U', 'backup']) - self.assertEqual( - 1, 0, - "Expecting Error due to missing grant on EXECUTE.") - except ProbackupException as e: - if self.get_version(node) < 150000: - self.assertIn( - "ERROR: Query failed: ERROR: permission denied " - "for function pg_stop_backup", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - else: - self.assertIn( - "ERROR: Query failed: ERROR: permission denied " - "for function pg_backup_stop", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - if self.get_version(node) < self.version_to_num('10.0'): - node.safe_psql( - "postgres", - "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup") - elif self.get_version(node) < self.version_to_num('15.0'): - node.safe_psql( - "postgres", - "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean, boolean) TO backup;") - else: - node.safe_psql( - "postgres", - "GRANT EXECUTE ON FUNCTION pg_backup_stop(boolean) TO backup;") - - self.backup_node( - backup_dir, 'node', node, options=['-U', 'backup']) - - node.safe_psql("postgres", "CREATE DATABASE test1") - - self.backup_node( - backup_dir, 'node', node, options=['-U', 'backup']) - - node.safe_psql( - "test1", "create table t1 as select generate_series(0,100)") - - node.stop() - node.slow_start() - - node.safe_psql( - "postgres", - "ALTER ROLE backup REPLICATION") - - # FULL - self.backup_node( - backup_dir, 'node', node, options=['-U', 'backup']) - - # PTRACK - if self.ptrack: - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['-U', 'backup']) - - -class AuthTest(unittest.TestCase): - pb = None - node = None - - # TODO move to object scope, replace module_name - @classmethod - def setUpClass(cls): - - super(AuthTest, cls).setUpClass() - - cls.pb = ProbackupTest() - cls.backup_dir = os.path.join(cls.pb.tmp_path, module_name, 'backup') - - cls.node = cls.pb.make_simple_node( - base_dir="{}/node".format(module_name), - set_replication=True, - initdb_params=['--data-checksums', '--auth-host=md5'] - ) - - cls.username = cls.pb.get_username() - - cls.modify_pg_hba(cls.node) - - cls.pb.init_pb(cls.backup_dir) - cls.pb.add_instance(cls.backup_dir, cls.node.name, cls.node) - cls.pb.set_archiving(cls.backup_dir, cls.node.name, cls.node) - try: - cls.node.slow_start() - except StartNodeException: - raise unittest.skip("Node hasn't started") - - if cls.pb.get_version(cls.node) < 100000: - cls.node.safe_psql( - "postgres", - "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") - elif cls.pb.get_version(cls.node) < 150000: - cls.node.safe_psql( - "postgres", - "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_switch_wal() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") - else: - cls.node.safe_psql( - "postgres", - "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_backup_start(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_backup_stop(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_switch_wal() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") - - cls.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass') - - # TODO move to object scope, replace module_name - @classmethod - def tearDownClass(cls): - cls.node.cleanup() - cls.pb.del_test_dir(module_name, '') - - @unittest.skipIf(skip_test, "Module pexpect isn't installed. You need to install it.") - def setUp(self): - self.pb_cmd = ['backup', - '-B', self.backup_dir, - '--instance', self.node.name, - '-h', '127.0.0.1', - '-p', str(self.node.port), - '-U', 'backup', - '-d', 'postgres', - '-b', 'FULL' - ] - - def tearDown(self): - if "PGPASSWORD" in self.pb.test_env.keys(): - del self.pb.test_env["PGPASSWORD"] - - if "PGPASSWORD" in self.pb.test_env.keys(): - del self.pb.test_env["PGPASSFILE"] - - try: - os.remove(self.pgpass_file) - except OSError: - pass - - def test_empty_password(self): - """ Test case: PGPB_AUTH03 - zero password length """ - try: - self.assertIn("ERROR: no password supplied", - self.run_pb_with_auth('\0\r\n')) - except (TIMEOUT, ExceptionPexpect) as e: - self.fail(e.value) - - def test_wrong_password(self): - """ Test case: PGPB_AUTH04 - incorrect password """ - self.assertIn("password authentication failed", - self.run_pb_with_auth('wrong_password\r\n')) - - def test_right_password(self): - """ Test case: PGPB_AUTH01 - correct password """ - self.assertIn("completed", - self.run_pb_with_auth('password\r\n')) - - def test_right_password_and_wrong_pgpass(self): - """ Test case: PGPB_AUTH05 - correct password and incorrect .pgpass (-W)""" - line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'wrong_password']) - self.create_pgpass(self.pgpass_file, line) - self.assertIn("completed", - self.run_pb_with_auth('password\r\n', add_args=["-W"])) - - def test_ctrl_c_event(self): - """ Test case: PGPB_AUTH02 - send interrupt signal """ - try: - self.run_pb_with_auth(kill=True) - except TIMEOUT: - self.fail("Error: CTRL+C event ignored") - - def test_pgpassfile_env(self): - """ Test case: PGPB_AUTH06 - set environment var PGPASSFILE """ - path = os.path.join(self.pb.tmp_path, module_name, 'pgpass.conf') - line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'password']) - self.create_pgpass(path, line) - self.pb.test_env["PGPASSFILE"] = path - self.assertEqual( - "OK", - self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.pb_cmd + ['-w']))["status"], - "ERROR: Full backup status is not valid." - ) - - def test_pgpass(self): - """ Test case: PGPB_AUTH07 - Create file .pgpass in home dir. """ - line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'password']) - self.create_pgpass(self.pgpass_file, line) - self.assertEqual( - "OK", - self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.pb_cmd + ['-w']))["status"], - "ERROR: Full backup status is not valid." - ) - - def test_pgpassword(self): - """ Test case: PGPB_AUTH08 - set environment var PGPASSWORD """ - self.pb.test_env["PGPASSWORD"] = "password" - self.assertEqual( - "OK", - self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.pb_cmd + ['-w']))["status"], - "ERROR: Full backup status is not valid." - ) - - def test_pgpassword_and_wrong_pgpass(self): - """ Test case: PGPB_AUTH09 - Check priority between PGPASSWORD and .pgpass file""" - line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'wrong_password']) - self.create_pgpass(self.pgpass_file, line) - self.pb.test_env["PGPASSWORD"] = "password" - self.assertEqual( - "OK", - self.pb.show_pb(self.backup_dir, self.node.name, self.pb.run_pb(self.pb_cmd + ['-w']))["status"], - "ERROR: Full backup status is not valid." - ) - - def run_pb_with_auth(self, password=None, add_args = [], kill=False): - with spawn(self.pb.probackup_path, self.pb_cmd + add_args, encoding='utf-8', timeout=10) as probackup: - result = probackup.expect(u"Password for user .*:", 5) - if kill: - probackup.kill(signal.SIGINT) - elif result == 0: - probackup.sendline(password) - probackup.expect(EOF) - return str(probackup.before) - else: - raise ExceptionPexpect("Other pexpect errors.") - - - @classmethod - def modify_pg_hba(cls, node): - """ - Description: - Add trust authentication for user postgres. Need for add new role and set grant. - :param node: - :return None: - """ - hba_conf = os.path.join(node.data_dir, "pg_hba.conf") - with open(hba_conf, 'r+') as fio: - data = fio.read() - fio.seek(0) - fio.write('host\tall\t%s\t127.0.0.1/0\ttrust\n%s' % (cls.username, data)) - - - def create_pgpass(self, path, line): - with open(path, 'w') as passfile: - # host:port:db:username:password - passfile.write(line) - os.chmod(path, 0o600) + self.assertNotIn(': 403', console_output) # Because we can have just '403' substring in timestamp + self.assertMessage(console_output, contains='S3_pre_start_check successful') + self.assertMessage(console_output, contains='HTTP response: 200') + self.assertIn( + f"INFO: Backup catalog '{self.backup_dir}' successfully initialized", + console_output) diff --git a/tests/backup_test.py b/tests/backup_test.py index dc60228b5..2e0695b6c 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -2,29 +2,28 @@ import os import re from time import sleep, time -from .helpers.ptrack_helpers import base36enc, ProbackupTest, ProbackupException -import shutil -from distutils.dir_util import copy_tree -from testgres import ProcessType, QueryException +from datetime import datetime + +from pg_probackup2.gdb import needs_gdb + +from .helpers.ptrack_helpers import base36enc, ProbackupTest +from .helpers.ptrack_helpers import fs_backup_class import subprocess -class BackupTest(ProbackupTest, unittest.TestCase): +class BackupTest(ProbackupTest): def test_full_backup(self): """ Just test full backup with at least two segments """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', # we need to write a lot. Lets speedup a bit. pg_options={"fsync": "off", "synchronous_commit": "off"}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Fill with data @@ -32,9 +31,9 @@ def test_full_backup(self): node.pgbench_init(scale=100, no_vacuum=True) # FULL - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) - out = self.validate_pb(backup_dir, 'node', backup_id) + out = self.pb.validate('node', backup_id) self.assertIn( "INFO: Backup {0} is valid".format(backup_id), out) @@ -43,15 +42,12 @@ def test_full_backup_stream(self): """ Just test full backup with at least two segments in stream mode """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', # we need to write a lot. Lets speedup a bit. pg_options={"fsync": "off", "synchronous_commit": "off"}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # Fill with data @@ -59,10 +55,10 @@ def test_full_backup_stream(self): node.pgbench_init(scale=100, no_vacuum=True) # FULL - backup_id = self.backup_node(backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, options=["--stream"]) - out = self.validate_pb(backup_dir, 'node', backup_id) + out = self.pb.validate('node', backup_id) self.assertIn( "INFO: Backup {0} is valid".format(backup_id), out) @@ -72,216 +68,152 @@ def test_full_backup_stream(self): # PGPRO-707 def test_backup_modes_archive(self): """standart backup modes with ARCHIVE WAL method""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir - full_backup_id = self.backup_node(backup_dir, 'node', node) - show_backup = self.show_pb(backup_dir, 'node')[0] + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + full_backup_id = self.pb.backup_node('node', node) + show_backup = self.pb.show('node')[0] self.assertEqual(show_backup['status'], "OK") self.assertEqual(show_backup['backup-mode'], "FULL") # postmaster.pid and postmaster.opts shouldn't be copied - excluded = True - db_dir = os.path.join( - backup_dir, "backups", 'node', full_backup_id, "database") - - for f in os.listdir(db_dir): - if ( - os.path.isfile(os.path.join(db_dir, f)) and - ( - f == "postmaster.pid" or - f == "postmaster.opts" - ) - ): - excluded = False - self.assertEqual(excluded, True) + pms = {"postmaster.pid", "postmaster.opts"} + files = self.get_backup_listdir(backup_dir, 'node', full_backup_id, + 'database') + self.assertFalse(pms.intersection(files)) + files = self.get_backup_filelist(backup_dir, 'node', full_backup_id) + self.assertFalse(pms.intersection(files.keys())) # page backup mode - page_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="page") + page_backup_id = self.pb.backup_node('node', node, backup_type="page") - show_backup_1 = self.show_pb(backup_dir, 'node')[1] + show_backup_1 = self.pb.show('node')[1] self.assertEqual(show_backup_1['status'], "OK") self.assertEqual(show_backup_1['backup-mode'], "PAGE") # delta backup mode - delta_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="delta") + delta_backup_id = self.pb.backup_node('node', node, backup_type="delta") - show_backup_2 = self.show_pb(backup_dir, 'node')[2] + show_backup_2 = self.pb.show('node')[2] self.assertEqual(show_backup_2['status'], "OK") self.assertEqual(show_backup_2['backup-mode'], "DELTA") # Check parent backup self.assertEqual( full_backup_id, - self.show_pb( - backup_dir, 'node', + self.pb.show('node', backup_id=show_backup_1['id'])["parent-backup-id"]) self.assertEqual( page_backup_id, - self.show_pb( - backup_dir, 'node', + self.pb.show('node', backup_id=show_backup_2['id'])["parent-backup-id"]) # @unittest.skip("skip") def test_smooth_checkpoint(self): """full backup with smooth checkpoint""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=["-C"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + self.assertEqual(self.pb.show('node')[0]['status'], "OK") node.stop() # @unittest.skip("skip") def test_incremental_backup_without_full(self): """page backup without validated full backup""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - try: - self.backup_node(backup_dir, 'node', node, backup_type="page") - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "without valid full backup.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "WARNING: Valid full backup on current timeline 1 is not found" in e.message and - "ERROR: Create new full backup before an incremental one" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, backup_type="page", + expect_error="because page backup should not be possible") + self.assertMessage(contains="WARNING: Valid full backup on current timeline 1 is not found") + self.assertMessage(contains="ERROR: Create new full backup before an incremental one") self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['status'], + self.pb.show('node')[0]['status'], "ERROR") # @unittest.skip("skip") def test_incremental_backup_corrupt_full(self): """page-level backup with corrupted full backup""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - backup_id = self.backup_node(backup_dir, 'node', node) - file = os.path.join( - backup_dir, "backups", "node", backup_id, - "database", "postgresql.conf") - os.remove(file) + backup_id = self.pb.backup_node('node', node) + self.remove_backup_file(backup_dir, "node", backup_id, "database/postgresql.conf") - try: - self.validate_pb(backup_dir, 'node') - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of validation of corrupted backup.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "INFO: Validate backups of the instance 'node'" in e.message and - "WARNING: Backup file" in e.message and "is not found" in e.message and - "WARNING: Backup {0} data files are corrupted".format( - backup_id) in e.message and - "WARNING: Some backups are not valid" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - try: - self.backup_node(backup_dir, 'node', node, backup_type="page") - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "without valid full backup.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "WARNING: Valid full backup on current timeline 1 is not found" in e.message and - "ERROR: Create new full backup before an incremental one" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + self.pb.validate('node', + expect_error="because of validation of corrupted backup") + self.assertMessage(contains="INFO: Validate backups of the instance 'node'") + self.assertMessage(contains="WARNING: Validating ") + self.assertMessage(contains="No such file") + self.assertMessage(contains=f"WARNING: Backup {backup_id} data files are corrupted") + self.assertMessage(contains="WARNING: Some backups are not valid") + + self.pb.backup_node('node', node, backup_type="page", + expect_error="because page backup should not be possible") + self.assertMessage(contains="WARNING: Valid full backup on current timeline 1 is not found") + self.assertMessage(contains="ERROR: Create new full backup before an incremental one") self.assertEqual( - self.show_pb(backup_dir, 'node', backup_id)['status'], "CORRUPT") + self.pb.show('node', backup_id)['status'], "CORRUPT") self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['status'], "ERROR") + self.pb.show('node')[1]['status'], "ERROR") # @unittest.skip("skip") def test_delta_threads_stream(self): """delta multi thread backup mode and stream""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - self.backup_node( - backup_dir, 'node', node, backup_type="full", + self.pb.backup_node('node', node, backup_type="full", options=["-j", "4", "--stream"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") - self.backup_node( - backup_dir, 'node', node, + self.assertEqual(self.pb.show('node')[0]['status'], "OK") + self.pb.backup_node('node', node, backup_type="delta", options=["-j", "4", "--stream"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") + self.assertEqual(self.pb.show('node')[1]['status'], "OK") # @unittest.skip("skip") def test_page_detect_corruption(self): """make node, corrupt some page, check that backup failed""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) + ptrack_enable=self.ptrack) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type="full", options=["-j", "4", "--stream"]) node.safe_psql( @@ -303,42 +235,28 @@ def test_page_detect_corruption(self): f.seek(9000) f.write(b"bla") f.flush() - f.close - try: - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4", "--stream", "--log-level-file=VERBOSE"]) - self.assertEqual( - 1, 0, - "Expecting Error because data file is corrupted" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: Corruption detected in file "{0}", ' - 'block 1: page verification failed, calculated checksum'.format(path), - e.message) + self.pb.backup_node('node', node, backup_type="full", + options=["-j", "4", "--stream", "--log-level-file=VERBOSE"], + expect_error="because data file is corrupted") + self.assertMessage(contains=f'ERROR: Corruption detected in file "{path}", ' + 'block 1: page verification failed, calculated checksum') self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['status'], + self.pb.show('node')[1]['status'], 'ERROR', "Backup Status should be ERROR") # @unittest.skip("skip") def test_backup_detect_corruption(self): """make node, corrupt some page, check that backup failed""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + ptrack_enable=self.ptrack) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() if self.ptrack: @@ -346,8 +264,7 @@ def test_backup_detect_corruption(self): "postgres", "create extension ptrack") - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type="full", options=["-j", "4", "--stream"]) node.safe_psql( @@ -360,8 +277,7 @@ def test_backup_detect_corruption(self): "postgres", "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type="full", options=["-j", "4", "--stream"]) node.safe_psql( @@ -384,240 +300,31 @@ def test_backup_detect_corruption(self): node.slow_start() - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page verification failed, calculated checksum'.format( - heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="delta", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page verification failed, calculated checksum'.format( - heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="page", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page verification failed, calculated checksum'.format( - heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - + modes = "full,delta,page" if self.ptrack: - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="ptrack", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page verification failed, calculated checksum'.format( - heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + modes += ",ptrack" + for mode in modes.split(','): + with self.subTest(mode): + self.pb.backup_node('node', node, + backup_type=mode, + options=["-j", "4", "--stream"], + expect_error="because of block corruption") + self.assertMessage(contains= + 'ERROR: Corruption detected in file "{0}", block 1: ' + 'page verification failed, calculated checksum'.format( + heap_fullpath)) + sleep(1) # @unittest.skip("skip") def test_backup_detect_invalid_block_header(self): """make node, corrupt some page, check that backup failed""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - if self.ptrack: - node.safe_psql( - "postgres", - "create extension ptrack") - - node.safe_psql( - "postgres", - "create table t_heap as select 1 as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,10000) i") - - heap_path = node.safe_psql( - "postgres", - "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - - self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4", "--stream"]) - - node.safe_psql( - "postgres", - "select count(*) from t_heap") - - node.safe_psql( - "postgres", - "update t_heap set id = id + 10000") - - node.stop() - - heap_fullpath = os.path.join(node.data_dir, heap_path) - with open(heap_fullpath, "rb+", 0) as f: - f.seek(8193) - f.write(b"blahblahblahblah") - f.flush() - f.close - - node.slow_start() - -# self.backup_node( -# backup_dir, 'node', node, -# backup_type="full", options=["-j", "4", "--stream"]) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="delta", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="page", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - if self.ptrack: - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="ptrack", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # @unittest.skip("skip") - def test_backup_detect_missing_permissions(self): - """make node, corrupt some page, check that backup failed""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) + ptrack_enable=self.ptrack) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() if self.ptrack: @@ -635,8 +342,7 @@ def test_backup_detect_missing_permissions(self): "postgres", "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type="full", options=["-j", "4", "--stream"]) node.safe_psql( @@ -654,92 +360,21 @@ def test_backup_detect_missing_permissions(self): f.seek(8193) f.write(b"blahblahblahblah") f.flush() - f.close node.slow_start() -# self.backup_node( -# backup_dir, 'node', node, -# backup_type="full", options=["-j", "4", "--stream"]) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="full", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="delta", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="page", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - sleep(1) - + modes = "full,delta,page" if self.ptrack: - try: - self.backup_node( - backup_dir, 'node', node, - backup_type="ptrack", options=["-j", "4", "--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of block corruption" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Corruption detected in file "{0}", block 1: ' - 'page header invalid, pd_lower'.format(heap_fullpath), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + modes += ",ptrack" + for mode in modes.split(','): + with self.subTest(mode): + self.pb.backup_node('node', node, + backup_type=mode, options=["-j", "4", "--stream"], + expect_error="because of block corruption") + self.assertMessage(contains='ERROR: Corruption detected in file ' + f'"{heap_fullpath}", block 1: ' + 'page header invalid, pd_lower') + sleep(1) # @unittest.skip("skip") def test_backup_truncate_misaligned(self): @@ -747,15 +382,11 @@ def test_backup_truncate_misaligned(self): make node, truncate file to size not even to BLCKSIZE, take backup """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -781,8 +412,7 @@ def test_backup_truncate_misaligned(self): f.flush() f.close - output = self.backup_node( - backup_dir, 'node', node, backup_type="full", + output = self.pb.backup_node('node', node, backup_type="full", options=["-j", "4", "--stream"], return_id=False) self.assertIn("WARNING: File", output) @@ -791,15 +421,13 @@ def test_backup_truncate_misaligned(self): # @unittest.skip("skip") def test_tablespace_in_pgdata_pgpro_1376(self): """PGPRO-1376 """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + backup_dir = self.backup_dir - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() self.create_tblspace_in_node( @@ -828,8 +456,7 @@ def test_tablespace_in_pgdata_pgpro_1376(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,1000) i") - backup_id_1 = self.backup_node( - backup_dir, 'node', node, backup_type="full", + backup_id_1 = self.pb.backup_node('node', node, backup_type="full", options=["-j", "4", "--stream"]) node.safe_psql( @@ -839,8 +466,7 @@ def test_tablespace_in_pgdata_pgpro_1376(self): "postgres", "drop tablespace tblspace2") - self.backup_node( - backup_dir, 'node', node, backup_type="full", + self.pb.backup_node('node', node, backup_type="full", options=["-j", "4", "--stream"]) pgdata = self.pgdata_content(node.data_dir) @@ -871,8 +497,7 @@ def test_tablespace_in_pgdata_pgpro_1376(self): node.cleanup() - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]) + self.pb.restore_node('node', node, options=["-j", "4"]) if self.paranoia: pgdata_restored = self.pgdata_content(node.data_dir) @@ -886,19 +511,14 @@ def test_basic_tablespace_handling(self): check that restore with tablespace mapping will end with success """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="full", + backup_id = self.pb.backup_node('node', node, backup_type="full", options=["-j", "4", "--stream"]) tblspace1_old_path = self.get_tblspace_path(node, 'tblspace1_old') @@ -932,32 +552,20 @@ def test_basic_tablespace_handling(self): tblspace1_new_path = self.get_tblspace_path(node, 'tblspace1_new') tblspace2_new_path = self.get_tblspace_path(node, 'tblspace2_new') - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format( - tblspace1_old_path, tblspace1_new_path), - "-T", "{0}={1}".format( - tblspace2_old_path, tblspace2_new_path)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because tablespace mapping is incorrect" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Backup {0} has no tablespaceses, ' - 'nothing to remap'.format(backup_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + tblspace1_old_path, tblspace1_new_path), + "-T", "{0}={1}".format( + tblspace2_old_path, tblspace2_new_path)], + expect_error="because tablespace mapping is incorrect") + self.assertMessage(contains= + f'ERROR: Backup {backup_id} has no tablespaceses, ' + 'nothing to remap') node.safe_psql( "postgres", @@ -967,12 +575,10 @@ def test_basic_tablespace_handling(self): "postgres", "drop tablespace some_lame_tablespace") - self.backup_node( - backup_dir, 'node', node, backup_type="delta", + self.pb.backup_node('node', node, backup_type="delta", options=["-j", "4", "--stream"]) - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=[ "-j", "4", "-T", "{0}={1}".format( @@ -993,15 +599,11 @@ def test_tablespace_handling_1(self): make node with tablespace A, take full backup, check that restore with tablespace mapping of tablespace B will end with error """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() tblspace1_old_path = self.get_tblspace_path(node, 'tblspace1_old') @@ -1013,33 +615,20 @@ def test_tablespace_handling_1(self): node, 'tblspace1', tblspc_path=tblspace1_old_path) - self.backup_node( - backup_dir, 'node', node, backup_type="full", + self.pb.backup_node('node', node, backup_type="full", options=["-j", "4", "--stream"]) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format( - tblspace2_old_path, tblspace_new_path)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because tablespace mapping is incorrect" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: --tablespace-mapping option' in e.message and - 'have an entry in tablespace_map file' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + tblspace2_old_path, tblspace_new_path)], + expect_error="because tablespace mapping is incorrect") + self.assertMessage(contains='ERROR: --tablespace-mapping option') + self.assertMessage(contains='have an entry in tablespace_map file') # @unittest.skip("skip") def test_tablespace_handling_2(self): @@ -1047,61 +636,40 @@ def test_tablespace_handling_2(self): make node without tablespaces, take full backup, check that restore with tablespace mapping will end with error """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() tblspace1_old_path = self.get_tblspace_path(node, 'tblspace1_old') tblspace_new_path = self.get_tblspace_path(node, 'tblspace_new') - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="full", + backup_id = self.pb.backup_node('node', node, backup_type="full", options=["-j", "4", "--stream"]) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format( - tblspace1_old_path, tblspace_new_path)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because tablespace mapping is incorrect" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Backup {0} has no tablespaceses, ' - 'nothing to remap'.format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + tblspace1_old_path, tblspace_new_path)], + expect_error="because tablespace mapping is incorrect") + self.assertMessage(contains=f'ERROR: Backup {backup_id} has no tablespaceses, ' + 'nothing to remap') # @unittest.skip("skip") + @needs_gdb def test_drop_rel_during_full_backup(self): """""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() for i in range(1, 512): @@ -1128,9 +696,8 @@ def test_drop_rel_during_full_backup(self): absolute_path_2 = os.path.join(node.data_dir, relative_path_2) # FULL backup - gdb = self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--log-level-file=LOG', '--log-level-console=LOG', '--progress'], + gdb = self.pb.backup_node('node', node, + options=['--stream', '--log-level-console=LOG', '--progress'], gdb=True) gdb.set_breakpoint('backup_files') @@ -1155,14 +722,13 @@ def test_drop_rel_during_full_backup(self): pgdata = self.pgdata_content(node.data_dir) - #with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: - # log_content = f.read() - # self.assertTrue( + #log_content = self.read_pb_log() + #self.assertTrue( # 'LOG: File "{0}" is not found'.format(absolute_path) in log_content, # 'File "{0}" should be deleted but it`s not'.format(absolute_path)) node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) # Physical comparison pgdata_restored = self.pgdata_content(node.data_dir) @@ -1171,14 +737,11 @@ def test_drop_rel_during_full_backup(self): @unittest.skip("skip") def test_drop_db_during_full_backup(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() for i in range(1, 2): @@ -1191,8 +754,7 @@ def test_drop_db_during_full_backup(self): "VACUUM") # FULL backup - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True, + gdb = self.pb.backup_node('node', node, gdb=True, options=[ '--stream', '--log-level-file=LOG', '--log-level-console=LOG', '--progress']) @@ -1219,33 +781,28 @@ def test_drop_db_during_full_backup(self): pgdata = self.pgdata_content(node.data_dir) - #with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: - # log_content = f.read() - # self.assertTrue( + #log_content = self.read_pb_log() + #self.assertTrue( # 'LOG: File "{0}" is not found'.format(absolute_path) in log_content, # 'File "{0}" should be deleted but it`s not'.format(absolute_path)) node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) # Physical comparison pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") + @needs_gdb def test_drop_rel_during_backup_delta(self): """""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=10) @@ -1262,11 +819,10 @@ def test_drop_rel_during_backup_delta(self): absolute_path = os.path.join(node.data_dir, relative_path) # FULL backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) # DELTA backup - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='delta', + gdb = self.pb.backup_node('node', node, backup_type='delta', gdb=True, options=['--log-level-file=LOG']) gdb.set_breakpoint('backup_files') @@ -1286,33 +842,30 @@ def test_drop_rel_during_backup_delta(self): pgdata = self.pgdata_content(node.data_dir) - with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: - log_content = f.read() - self.assertTrue( + log_content = self.read_pb_log() + self.assertTrue( 'LOG: File not found: "{0}"'.format(absolute_path) in log_content, 'File "{0}" should be deleted but it`s not'.format(absolute_path)) node.cleanup() - self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + self.pb.restore_node('node', node=node, options=["-j", "4"]) # Physical comparison pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") + @needs_gdb def test_drop_rel_during_backup_page(self): """""" - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -1327,7 +880,7 @@ def test_drop_rel_during_backup_page(self): absolute_path = os.path.join(node.data_dir, relative_path) # FULL backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) node.safe_psql( "postgres", @@ -1335,8 +888,7 @@ def test_drop_rel_during_backup_page(self): " as id from generate_series(101,102) i") # PAGE backup - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='page', + gdb = self.pb.backup_node('node', node, backup_type='page', gdb=True, options=['--log-level-file=LOG']) gdb.set_breakpoint('backup_files') @@ -1351,13 +903,13 @@ def test_drop_rel_during_backup_page(self): pgdata = self.pgdata_content(node.data_dir) - backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + backup_id = self.pb.show('node')[1]['id'] filelist = self.get_backup_filelist(backup_dir, 'node', backup_id) self.assertNotIn(relative_path, filelist) node.cleanup() - self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + self.pb.restore_node('node', node=node, options=["-j", "4"]) # Physical comparison pgdata_restored = self.pgdata_content(node.data_dir) @@ -1366,82 +918,76 @@ def test_drop_rel_during_backup_page(self): # @unittest.skip("skip") def test_persistent_slot_for_stream_backup(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'max_wal_size': '40MB'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( "postgres", "SELECT pg_create_physical_replication_slot('slot_1')") + # FULL backup. By default, --temp-slot=true. + self.pb.backup_node('node', node, + options=['--stream', '--slot=slot_1'], + expect_error="because replication slot already exist") + self.assertMessage(contains='ERROR: replication slot "slot_1" already exists') + # FULL backup - self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--slot=slot_1']) + self.pb.backup_node('node', node, + options=['--stream', '--slot=slot_1', '--temp-slot=false']) # FULL backup - self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--slot=slot_1']) + self.pb.backup_node('node', node, + options=['--stream', '--slot=slot_1', '--temp-slot=false']) # @unittest.skip("skip") def test_basic_temp_slot_for_stream_backup(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={'max_wal_size': '40MB'}) - if self.get_version(node) < self.version_to_num('10.0'): - self.skipTest('You need PostgreSQL >= 10 for this test') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL backup - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=['--stream', '--temp-slot']) + # FULL backup. By default, --temp-slot=true. + self.pb.backup_node('node', node, + options=['--stream', '--slot=slot_1']) + # FULL backup - self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--slot=slot_1', '--temp-slot']) + self.pb.backup_node('node', node, + options=['--stream', '--slot=slot_1', '--temp-slot=true']) # @unittest.skip("skip") + @needs_gdb def test_backup_concurrent_drop_table(self): """""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=1) # FULL backup - gdb = self.backup_node( - backup_dir, 'node', node, + gdb = self.pb.backup_node('node', node, options=['--stream', '--compress'], gdb=True) @@ -1457,217 +1003,191 @@ def test_backup_concurrent_drop_table(self): 'postgres', 'CHECKPOINT') - gdb.remove_all_breakpoints() gdb.continue_execution_until_exit() gdb.kill() - show_backup = self.show_pb(backup_dir, 'node')[0] + show_backup = self.pb.show('node')[0] self.assertEqual(show_backup['status'], "OK") - # @unittest.skip("skip") def test_pg_11_adjusted_wal_segment_size(self): """""" if self.pg_config_version < self.version_to_num('11.0'): self.skipTest('You need PostgreSQL >= 11 for this test') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=[ - '--data-checksums', - '--wal-segsize=64'], + initdb_params=['--wal-segsize=64'], pg_options={ 'min_wal_size': '128MB'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=5) # FULL STREAM backup - self.backup_node( - backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) pgbench = node.pgbench(options=['-T', '5', '-c', '2']) pgbench.wait() # PAGE STREAM backup - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='page', options=['--stream']) pgbench = node.pgbench(options=['-T', '5', '-c', '2']) pgbench.wait() # DELTA STREAM backup - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='delta', options=['--stream']) pgbench = node.pgbench(options=['-T', '5', '-c', '2']) pgbench.wait() # FULL ARCHIVE backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) pgbench = node.pgbench(options=['-T', '5', '-c', '2']) pgbench.wait() # PAGE ARCHIVE backup - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') pgbench = node.pgbench(options=['-T', '5', '-c', '2']) pgbench.wait() # DELTA ARCHIVE backup - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + backup_id = self.pb.backup_node('node', node, backup_type='delta') pgdata = self.pgdata_content(node.data_dir) # delete - output = self.delete_pb( - backup_dir, 'node', + output = self.pb.delete('node', options=[ '--expired', '--delete-wal', '--retention-redundancy=1']) # validate - self.validate_pb(backup_dir) + self.pb.validate() # merge - self.merge_backup(backup_dir, 'node', backup_id=backup_id) + self.pb.merge_backup('node', backup_id=backup_id) # restore node.cleanup() - self.restore_node( - backup_dir, 'node', node, backup_id=backup_id) + self.pb.restore_node('node', node, backup_id=backup_id) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") + @needs_gdb def test_sigint_handling(self): """""" - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # FULL backup - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True, + gdb = self.pb.backup_node('node', node, gdb=True, options=['--stream', '--log-level-file=LOG']) gdb.set_breakpoint('backup_non_data_file') gdb.run_until_break() + gdb.continue_execution_until_break(200) - gdb.continue_execution_until_break(20) gdb.remove_all_breakpoints() - - gdb._execute('signal SIGINT') + gdb.signal('SIGINT') gdb.continue_execution_until_error() gdb.kill() - backup_id = self.show_pb(backup_dir, 'node')[0]['id'] + backup_id = self.pb.show('node')[0]['id'] self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node', backup_id)['status'], + self.pb.show('node', backup_id)['status'], 'Backup STATUS should be "ERROR"') # @unittest.skip("skip") + @needs_gdb def test_sigterm_handling(self): """""" - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # FULL backup - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True, + gdb = self.pb.backup_node('node', node, gdb=True, options=['--stream', '--log-level-file=LOG']) gdb.set_breakpoint('backup_non_data_file') gdb.run_until_break() + gdb.continue_execution_until_break(200) - gdb.continue_execution_until_break(20) - gdb.remove_all_breakpoints() - - gdb._execute('signal SIGTERM') + gdb.signal('SIGTERM') gdb.continue_execution_until_error() - backup_id = self.show_pb(backup_dir, 'node')[0]['id'] + backup_id = self.pb.show('node')[0]['id'] self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node', backup_id)['status'], + self.pb.show('node', backup_id)['status'], 'Backup STATUS should be "ERROR"') # @unittest.skip("skip") + @needs_gdb def test_sigquit_handling(self): """""" - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # FULL backup - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True, options=['--stream']) + gdb = self.pb.backup_node('node', node, gdb=True, options=['--stream']) gdb.set_breakpoint('backup_non_data_file') gdb.run_until_break() + gdb.continue_execution_until_break(200) - gdb.continue_execution_until_break(20) - gdb.remove_all_breakpoints() - - gdb._execute('signal SIGQUIT') + gdb.signal('SIGQUIT') gdb.continue_execution_until_error() - backup_id = self.show_pb(backup_dir, 'node')[0]['id'] + backup_id = self.pb.show('node')[0]['id'] self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node', backup_id)['status'], + self.pb.show('node', backup_id)['status'], 'Backup STATUS should be "ERROR"') # @unittest.skip("skip") def test_drop_table(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() connect_1 = node.connect("postgres") @@ -1685,8 +1205,7 @@ def test_drop_table(self): connect_2.commit() # FULL backup - self.backup_node( - backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) # @unittest.skip("skip") def test_basic_missing_file_permissions(self): @@ -1694,14 +1213,12 @@ def test_basic_missing_file_permissions(self): if os.name == 'nt': self.skipTest('Skipped because it is POSIX only test') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() relative_path = node.safe_psql( @@ -1712,22 +1229,10 @@ def test_basic_missing_file_permissions(self): os.chmod(full_path, 000) - try: - # FULL backup - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of missing permissions" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Cannot open file', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + # FULL backup + self.pb.backup_node('node', node, options=['--stream'], + expect_error="because of missing permissions") + self.assertMessage(regex=r"ERROR: [^\n]*: Permission denied") os.chmod(full_path, 700) @@ -1737,53 +1242,37 @@ def test_basic_missing_dir_permissions(self): if os.name == 'nt': self.skipTest('Skipped because it is POSIX only test') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() full_path = os.path.join(node.data_dir, 'pg_twophase') os.chmod(full_path, 000) - try: # FULL backup - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of missing permissions" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Cannot open directory', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, options=['--stream'], + expect_error="because of missing permissions") + self.assertMessage(regex=r'ERROR:[^\n]*Cannot open dir') os.rmdir(full_path) # @unittest.skip("skip") def test_backup_with_least_privileges_role(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=self.ptrack, - initdb_params=['--data-checksums'], - pg_options={'archive_timeout': '30s'}) + pg_options={'archive_timeout': '10s'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -1796,81 +1285,7 @@ def test_backup_with_least_privileges_role(self): "CREATE SCHEMA ptrack; " "CREATE EXTENSION ptrack WITH SCHEMA ptrack") - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + if self.pg_config_version < 150000: node.safe_psql( 'backupdb', "REVOKE ALL ON DATABASE backupdb from PUBLIC; " @@ -1956,43 +1371,35 @@ def test_backup_with_least_privileges_role(self): "GRANT EXECUTE ON FUNCTION ptrack.ptrack_get_pagemapset(pg_lsn) TO backup; " "GRANT EXECUTE ON FUNCTION ptrack.ptrack_init_lsn() TO backup;") - if ProbackupTest.enterprise: + if ProbackupTest.pgpro: node.safe_psql( "backupdb", "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") # FULL backup - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, datname='backupdb', options=['--stream', '-U', 'backup']) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, datname='backupdb', options=['-U', 'backup']) # PAGE - self.backup_node( - backup_dir, 'node', node, backup_type='page', + self.pb.backup_node('node', node, backup_type='page', datname='backupdb', options=['-U', 'backup']) - self.backup_node( - backup_dir, 'node', node, backup_type='page', datname='backupdb', + self.pb.backup_node('node', node, backup_type='page', datname='backupdb', options=['--stream', '-U', 'backup']) # DELTA - self.backup_node( - backup_dir, 'node', node, backup_type='delta', + self.pb.backup_node('node', node, backup_type='delta', datname='backupdb', options=['-U', 'backup']) - self.backup_node( - backup_dir, 'node', node, backup_type='delta', + self.pb.backup_node('node', node, backup_type='delta', datname='backupdb', options=['--stream', '-U', 'backup']) # PTRACK if self.ptrack: - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + self.pb.backup_node('node', node, backup_type='ptrack', datname='backupdb', options=['-U', 'backup']) - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + self.pb.backup_node('node', node, backup_type='ptrack', datname='backupdb', options=['--stream', '-U', 'backup']) # @unittest.skip("skip") @@ -2003,38 +1410,31 @@ def test_parent_choosing(self): PAGE1 <- CORRUPT FULL """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - full_id = self.backup_node(backup_dir, 'node', node) + full_id = self.pb.backup_node('node', node) # PAGE1 - page1_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page1_id = self.pb.backup_node('node', node, backup_type='page') # PAGE2 - page2_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page2_id = self.pb.backup_node('node', node, backup_type='page') # Change PAGE1 to ERROR self.change_backup_status(backup_dir, 'node', page1_id, 'ERROR') # PAGE3 - page3_id = self.backup_node( - backup_dir, 'node', node, + page3_id = self.pb.backup_node('node', node, backup_type='page', options=['--log-level-file=LOG']) - log_file_path = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(log_file_path) as f: - log_file_content = f.read() + log_file_content = self.read_pb_log() self.assertIn( "WARNING: Backup {0} has invalid parent: {1}. " @@ -2051,8 +1451,7 @@ def test_parent_choosing(self): log_file_content) self.assertEqual( - self.show_pb( - backup_dir, 'node', backup_id=page3_id)['parent-backup-id'], + self.pb.show('node', backup_id=page3_id)['parent-backup-id'], full_id) # @unittest.skip("skip") @@ -2063,39 +1462,31 @@ def test_parent_choosing_1(self): PAGE1 <- (missing) FULL """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - full_id = self.backup_node(backup_dir, 'node', node) + full_id = self.pb.backup_node('node', node) # PAGE1 - page1_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page1_id = self.pb.backup_node('node', node, backup_type='page') # PAGE2 - page2_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page2_id = self.pb.backup_node('node', node, backup_type='page') # Delete PAGE1 - shutil.rmtree( - os.path.join(backup_dir, 'backups', 'node', page1_id)) + self.remove_one_backup(backup_dir, 'node', page1_id) # PAGE3 - page3_id = self.backup_node( - backup_dir, 'node', node, + page3_id = self.pb.backup_node('node', node, backup_type='page', options=['--log-level-file=LOG']) - log_file_path = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(log_file_path) as f: - log_file_content = f.read() + log_file_content = self.read_pb_log() self.assertIn( "WARNING: Backup {0} has missing parent: {1}. " @@ -2107,8 +1498,7 @@ def test_parent_choosing_1(self): log_file_content) self.assertEqual( - self.show_pb( - backup_dir, 'node', backup_id=page3_id)['parent-backup-id'], + self.pb.show('node', backup_id=page3_id)['parent-backup-id'], full_id) # @unittest.skip("skip") @@ -2119,76 +1509,57 @@ def test_parent_choosing_2(self): PAGE1 <- OK FULL <- (missing) """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - full_id = self.backup_node(backup_dir, 'node', node) + full_id = self.pb.backup_node('node', node) # PAGE1 - page1_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page1_id = self.pb.backup_node('node', node, backup_type='page') # PAGE2 - page2_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page2_id = self.pb.backup_node('node', node, backup_type='page') # Delete FULL - shutil.rmtree( - os.path.join(backup_dir, 'backups', 'node', full_id)) + self.remove_one_backup(backup_dir, 'node', full_id) # PAGE3 - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=['--log-level-file=LOG']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because FULL backup is missing" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'WARNING: Valid full backup on current timeline 1 is not found' in e.message and - 'ERROR: Create new full backup before an incremental one' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, + backup_type='page', options=['--log-level-file=LOG'], + expect_error="because FULL backup is missing") + self.assertMessage(contains='WARNING: Valid full backup on current timeline 1 is not found') + self.assertMessage(contains='ERROR: Create new full backup before an incremental one') self.assertEqual( - self.show_pb( - backup_dir, 'node')[2]['status'], + self.pb.show('node')[2]['status'], 'ERROR') # @unittest.skip("skip") + @needs_gdb def test_backup_with_less_privileges_role(self): """ check permissions correctness from documentation: https://github.com/postgrespro/pg_probackup/blob/master/Documentation.md#configuring-the-database-cluster """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=self.ptrack, - initdb_params=['--data-checksums'], pg_options={ - 'archive_timeout': '30s', + 'archive_timeout': '10s', 'archive_mode': 'always', - 'checkpoint_timeout': '60s', + 'checkpoint_timeout': '30s', 'wal_level': 'logical'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_config('node', options=['--archive-timeout=30s']) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -2200,43 +1571,10 @@ def test_backup_with_less_privileges_role(self): 'backupdb', 'CREATE EXTENSION ptrack') - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup; " - "COMMIT;" - ) - # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + if self.pg_config_version < 150000: node.safe_psql( 'backupdb', + "BEGIN; " "CREATE ROLE backup WITH LOGIN; " "GRANT USAGE ON SCHEMA pg_catalog TO backup; " "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " @@ -2277,106 +1615,84 @@ def test_backup_with_less_privileges_role(self): 'ALTER ROLE backup WITH REPLICATION;') # FULL backup - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, datname='backupdb', options=['--stream', '-U', 'backup']) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, datname='backupdb', options=['-U', 'backup']) # PAGE - self.backup_node( - backup_dir, 'node', node, backup_type='page', + self.pb.backup_node('node', node, backup_type='page', datname='backupdb', options=['-U', 'backup']) - self.backup_node( - backup_dir, 'node', node, backup_type='page', datname='backupdb', + self.pb.backup_node('node', node, backup_type='page', datname='backupdb', options=['--stream', '-U', 'backup']) # DELTA - self.backup_node( - backup_dir, 'node', node, backup_type='delta', + self.pb.backup_node('node', node, backup_type='delta', datname='backupdb', options=['-U', 'backup']) - self.backup_node( - backup_dir, 'node', node, backup_type='delta', + self.pb.backup_node('node', node, backup_type='delta', datname='backupdb', options=['--stream', '-U', 'backup']) # PTRACK if self.ptrack: - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + self.pb.backup_node('node', node, backup_type='ptrack', datname='backupdb', options=['-U', 'backup']) - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + self.pb.backup_node('node', node, backup_type='ptrack', datname='backupdb', options=['--stream', '-U', 'backup']) - if self.get_version(node) < 90600: - return - # Restore as replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'node', replica) + self.pb.restore_node('node', node=replica) self.set_replica(node, replica) - self.add_instance(backup_dir, 'replica', replica) - self.set_config( - backup_dir, 'replica', - options=['--archive-timeout=120s', '--log-level-console=LOG']) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - self.set_auto_conf(replica, {'hot_standby': 'on'}) + self.pb.add_instance('replica', replica) + self.pb.set_config('replica', + options=['--archive-timeout=60s', '--log-level-console=LOG']) + self.pb.set_archiving('replica', replica, replica=True) + replica.set_auto_conf({'hot_standby': 'on'}) # freeze bgwriter to get rid of RUNNING XACTS records # bgwriter_pid = node.auxiliary_pids[ProcessType.BackgroundWriter][0] # gdb_checkpointer = self.gdb_attach(bgwriter_pid) - copy_tree( - os.path.join(backup_dir, 'wal', 'node'), - os.path.join(backup_dir, 'wal', 'replica')) - replica.slow_start(replica=True) - # self.switch_wal_segment(node) - # self.switch_wal_segment(node) + # make sure replica will archive wal segment with backup start point + lsn = self.switch_wal_segment(node, and_tx=True) + self.wait_until_lsn_replayed(replica, lsn) + replica.execute('CHECKPOINT') + replica.poll_query_until(f"select redo_lsn >= '{lsn}' from pg_control_checkpoint()") - self.backup_node( - backup_dir, 'replica', replica, - datname='backupdb', options=['-U', 'backup']) + self.pb.backup_replica_node('replica', replica, master=node, + datname='backupdb', options=['-U', 'backup']) # stream full backup from replica - self.backup_node( - backup_dir, 'replica', replica, + self.pb.backup_node('replica', replica, datname='backupdb', options=['--stream', '-U', 'backup']) # self.switch_wal_segment(node) # PAGE backup from replica - self.switch_wal_segment(node) - self.backup_node( - backup_dir, 'replica', replica, backup_type='page', - datname='backupdb', options=['-U', 'backup', '--archive-timeout=30s']) + self.pb.backup_replica_node('replica', replica, master=node, + backup_type='page', datname='backupdb', + options=['-U', 'backup']) - self.backup_node( - backup_dir, 'replica', replica, backup_type='page', + self.pb.backup_node('replica', replica, backup_type='page', datname='backupdb', options=['--stream', '-U', 'backup']) # DELTA backup from replica - self.switch_wal_segment(node) - self.backup_node( - backup_dir, 'replica', replica, backup_type='delta', - datname='backupdb', options=['-U', 'backup']) - self.backup_node( - backup_dir, 'replica', replica, backup_type='delta', + self.pb.backup_replica_node('replica', replica, master=node, + backup_type='delta', datname='backupdb', + options=['-U', 'backup']) + self.pb.backup_node('replica', replica, backup_type='delta', datname='backupdb', options=['--stream', '-U', 'backup']) # PTRACK backup from replica if self.ptrack: - self.switch_wal_segment(node) - self.backup_node( - backup_dir, 'replica', replica, backup_type='ptrack', - datname='backupdb', options=['-U', 'backup']) - self.backup_node( - backup_dir, 'replica', replica, backup_type='ptrack', + self.pb.backup_replica_node('replica', replica, master=node, + backup_type='ptrack', datname='backupdb', + options=['-U', 'backup']) + self.pb.backup_node('replica', replica, backup_type='ptrack', datname='backupdb', options=['--stream', '-U', 'backup']) @unittest.skip("skip") @@ -2384,14 +1700,12 @@ def test_issue_132(self): """ https://github.com/postgrespro/pg_probackup/issues/132 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() with node.connect("postgres") as conn: @@ -2400,13 +1714,12 @@ def test_issue_132(self): "CREATE TABLE t_{0} as select 1".format(i)) conn.commit() - self.backup_node( - backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) pgdata = self.pgdata_content(node.data_dir) node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -2418,16 +1731,14 @@ def test_issue_132_1(self): """ https://github.com/postgrespro/pg_probackup/issues/132 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) # TODO: check version of old binary, it should be 2.1.4, 2.1.5 or 2.2.1 - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() with node.connect("postgres") as conn: @@ -2436,101 +1747,55 @@ def test_issue_132_1(self): "CREATE TABLE t_{0} as select 1".format(i)) conn.commit() - full_id = self.backup_node( - backup_dir, 'node', node, options=['--stream'], old_binary=True) + full_id = self.pb.backup_node('node', node, options=['--stream'], old_binary=True) - delta_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', + delta_id = self.pb.backup_node('node', node, backup_type='delta', options=['--stream'], old_binary=True) node.cleanup() # make sure that new binary can detect corruption - try: - self.validate_pb(backup_dir, 'node', backup_id=full_id) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because FULL backup is CORRUPT" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} is a victim of metadata corruption'.format(full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.validate_pb(backup_dir, 'node', backup_id=delta_id) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because FULL backup is CORRUPT" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} is a victim of metadata corruption'.format(full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.validate('node', backup_id=full_id, + expect_error="because FULL backup is CORRUPT") + self.assertMessage(contains= + f'WARNING: Backup {full_id} is a victim of metadata corruption') + + self.pb.validate('node', backup_id=delta_id, + expect_error="because FULL backup is CORRUPT") + self.assertMessage(contains= + f'WARNING: Backup {full_id} is a victim of metadata corruption') self.assertEqual( - 'CORRUPT', self.show_pb(backup_dir, 'node', full_id)['status'], + 'CORRUPT', self.pb.show('node', full_id)['status'], 'Backup STATUS should be "CORRUPT"') self.assertEqual( - 'ORPHAN', self.show_pb(backup_dir, 'node', delta_id)['status'], + 'ORPHAN', self.pb.show('node', delta_id)['status'], 'Backup STATUS should be "ORPHAN"') # check that revalidation is working correctly - try: - self.restore_node( - backup_dir, 'node', node, backup_id=delta_id) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because FULL backup is CORRUPT" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} is a victim of metadata corruption'.format(full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.validate('node', backup_id=delta_id, + expect_error="because FULL backup is CORRUPT") + self.assertMessage(contains= + f'WARNING: Backup {full_id} is a victim of metadata corruption') self.assertEqual( - 'CORRUPT', self.show_pb(backup_dir, 'node', full_id)['status'], + 'CORRUPT', self.pb.show('node', full_id)['status'], 'Backup STATUS should be "CORRUPT"') self.assertEqual( - 'ORPHAN', self.show_pb(backup_dir, 'node', delta_id)['status'], + 'ORPHAN', self.pb.show('node', delta_id)['status'], 'Backup STATUS should be "ORPHAN"') # check that '--no-validate' do not allow to restore ORPHAN backup -# try: -# self.restore_node( -# backup_dir, 'node', node, backup_id=delta_id, -# options=['--no-validate']) -# # we should die here because exception is what we expect to happen -# self.assertEqual( -# 1, 0, -# "Expecting Error because FULL backup is CORRUPT" -# "\n Output: {0} \n CMD: {1}".format( -# repr(self.output), self.cmd)) -# except ProbackupException as e: -# self.assertIn( -# 'Insert data', -# e.message, -# '\n Unexpected Error Message: {0}\n CMD: {1}'.format( -# repr(e.message), self.cmd)) +# self.pb.restore_node('node', node=node, backup_id=delta_id, +# options=['--no-validate'], +# expect_error="because FULL backup is CORRUPT") +# self.assertMessage(contains='Insert data') node.cleanup() - output = self.restore_node( - backup_dir, 'node', node, backup_id=full_id, options=['--force']) + output = self.pb.restore_node('node', node, backup_id=full_id, options=['--force']) self.assertIn( 'WARNING: Backup {0} has status: CORRUPT'.format(full_id), @@ -2550,8 +1815,7 @@ def test_issue_132_1(self): node.cleanup() - output = self.restore_node( - backup_dir, 'node', node, backup_id=delta_id, options=['--force']) + output = self.pb.restore_node('node', node, backup_id=delta_id, options=['--force']) self.assertIn( 'WARNING: Backup {0} is orphan.'.format(delta_id), @@ -2573,31 +1837,28 @@ def test_note_sanity(self): """ test that adding note to backup works as expected """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, options=['--stream', '--log-level-file=LOG', '--note=test_note']) - show_backups = self.show_pb(backup_dir, 'node') + show_backups = self.pb.show('node') - print(self.show_pb(backup_dir, as_text=True, as_json=True)) + print(self.pb.show(as_text=True, as_json=True)) self.assertEqual(show_backups[0]['note'], "test_note") - self.set_backup(backup_dir, 'node', backup_id, options=['--note=none']) + self.pb.set_backup('node', backup_id, options=['--note=none']) - backup_meta = self.show_pb(backup_dir, 'node', backup_id) + backup_meta = self.pb.show('node', backup_id) self.assertNotIn( 'note', @@ -2606,151 +1867,96 @@ def test_note_sanity(self): # @unittest.skip("skip") def test_parent_backup_made_by_newer_version(self): """incremental backup with parent made by newer version""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() + node = self.pg_node.make_simple('node') - backup_id = self.backup_node(backup_dir, 'node', node) + backup_dir = self.backup_dir - control_file = os.path.join( - backup_dir, "backups", "node", backup_id, - "backup.control") + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + backup_id = self.pb.backup_node('node', node) version = self.probackup_version fake_new_version = str(int(version.split('.')[0]) + 1) + '.0.0' - with open(control_file, 'r') as f: - data = f.read(); - - data = data.replace(version, fake_new_version) + with self.modify_backup_control(backup_dir, "node", backup_id) as cf: + cf.data = cf.data.replace(version, fake_new_version) - with open(control_file, 'w') as f: - f.write(data); - - try: - self.backup_node(backup_dir, 'node', node, backup_type="page") - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because incremental backup should not be possible " - "if parent made by newer version.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( + self.pb.backup_node('node', node, backup_type="page", + expect_error="because incremental backup should not be possible") + self.assertMessage(contains= "pg_probackup do not guarantee to be forward compatible. " - "Please upgrade pg_probackup binary.", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + "Please upgrade pg_probackup binary.") self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['status'], "ERROR") + self.pb.show('node')[1]['status'], "ERROR") # @unittest.skip("skip") def test_issue_289(self): """ https://github.com/postgrespro/pg_probackup/issues/289 """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=['--archive-timeout=10s']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because full backup is missing" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertNotIn( - "INFO: Wait for WAL segment", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertIn( - "ERROR: Create new full backup before an incremental one", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, + backup_type='page', options=['--archive-timeout=10s'], + expect_error="because full backup is missing") + self.assertMessage(has_no="INFO: Wait for WAL segment") + self.assertMessage(contains="ERROR: Create new full backup before an incremental one") self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") + self.pb.show('node')[0]['status'], "ERROR") # @unittest.skip("skip") def test_issue_290(self): """ + For archive backup make sure that archive dir exists. + https://github.com/postgrespro/pg_probackup/issues/290 """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + if not backup_dir.is_file_based: + self.skipTest("directories are not implemented on cloud storage") - os.rmdir( - os.path.join(backup_dir, "wal", "node")) + node = self.pg_node.make_simple('node') - node.slow_start() + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) - try: - self.backup_node( - backup_dir, 'node', node, - options=['--archive-timeout=10s']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because full backup is missing" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertNotIn( - "INFO: Wait for WAL segment", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + self.remove_instance_waldir(backup_dir, 'node') - self.assertIn( - "WAL archive directory is not accessible", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + node.slow_start() + + self.pb.backup_node('node', node, + options=['--archive-timeout=10s'], + expect_error="because full backup is missing") + self.assertMessage(has_no="INFO: Wait for WAL segment") + self.assertMessage(contains="WAL archive directory is not accessible") self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['status'], "ERROR") + self.pb.show('node')[0]['status'], "ERROR") @unittest.skip("skip") def test_issue_203(self): """ https://github.com/postgrespro/pg_probackup/issues/203 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() with node.connect("postgres") as conn: @@ -2759,17 +1965,14 @@ def test_issue_203(self): "CREATE TABLE t_{0} as select 1".format(i)) conn.commit() - full_id = self.backup_node( - backup_dir, 'node', node, options=['--stream', '-j2']) + full_id = self.pb.backup_node('node', node, options=['--stream', '-j2']) pgdata = self.pgdata_content(node.data_dir) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node(backup_dir, 'node', - node_restored, data_dir=node_restored.data_dir) + self.pb.restore_node('node', node=node_restored) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -2777,22 +1980,22 @@ def test_issue_203(self): # @unittest.skip("skip") def test_issue_231(self): """ + Backups get the same ID if they are created within the same second. https://github.com/postgrespro/pg_probackup/issues/231 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node')) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) datadir = os.path.join(node.data_dir, '123') t0 = time() while True: - with self.assertRaises(ProbackupException) as ctx: - self.backup_node(backup_dir, 'node', node) - pb1 = re.search(r' backup ID: ([^\s,]+),', ctx.exception.message).groups()[0] + output = self.pb.backup_node('node', node, + expect_error=True) + pb1 = re.search(r' backup ID: ([^\s,]+),', output).groups()[0] t = time() if int(pb1, 36) == int(t) and t % 1 < 0.5: @@ -2805,9 +2008,8 @@ def test_issue_231(self): # sleep to the second's end so backup will not sleep for a second. sleep(1 - t % 1) - with self.assertRaises(ProbackupException) as ctx: - self.backup_node(backup_dir, 'node', node) - pb2 = re.search(r' backup ID: ([^\s,]+),', ctx.exception.message).groups()[0] + output = self.pb.backup_node('node', node, expect_error=True) + pb2 = re.search(r' backup ID: ([^\s,]+),', output).groups()[0] self.assertNotEqual(pb1, pb2) @@ -2815,43 +2017,38 @@ def test_incr_backup_filenode_map(self): """ https://github.com/postgrespro/pg_probackup/issues/320 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), - initdb_params=['--data-checksums']) + node1 = self.pg_node.make_simple('node1') node1.cleanup() node.pgbench_init(scale=5) # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, options=['-T', '10', '-c', '1']) - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + backup_id = self.pb.backup_node('node', node, backup_type='delta') node.safe_psql( 'postgres', 'reindex index pg_type_oid_index') - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') + backup_id = self.pb.backup_node('node', node, backup_type='delta') # incremental restore into node1 node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) node.slow_start() node.safe_psql( @@ -2859,21 +2056,19 @@ def test_incr_backup_filenode_map(self): 'select 1') # @unittest.skip("skip") + @needs_gdb def test_missing_wal_segment(self): """""" - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=self.ptrack, - initdb_params=['--data-checksums'], pg_options={'archive_timeout': '30s'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=10) @@ -2891,8 +2086,7 @@ def test_missing_wal_segment(self): pg_wal_dir = os.path.join(node.data_dir, 'pg_xlog') # Full backup in streaming mode - gdb = self.backup_node( - backup_dir, 'node', node, datname='backupdb', + gdb = self.pb.backup_node('node', node, datname='backupdb', options=['--stream', '--log-level-file=INFO'], gdb=True) # break at streaming start @@ -2934,26 +2128,23 @@ def test_missing_wal_segment(self): # @unittest.skip("skip") def test_missing_replication_permission(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) + ptrack_enable=self.ptrack) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) -# self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) +# self.pb.set_archiving('node', node) node.slow_start() # FULL backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'node', replica) + self.pb.restore_node('node', node=replica) # Settings for Replica self.set_replica(node, replica) @@ -2963,50 +2154,7 @@ def test_missing_replication_permission(self): 'postgres', 'CREATE DATABASE backupdb') - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + if self.pg_config_version < 150000: node.safe_psql( 'backupdb', "CREATE ROLE backup WITH LOGIN; " @@ -3052,7 +2200,7 @@ def test_missing_replication_permission(self): "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" ) - if ProbackupTest.enterprise: + if ProbackupTest.pgpro: node.safe_psql( "backupdb", "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " @@ -3061,58 +2209,40 @@ def test_missing_replication_permission(self): sleep(2) replica.promote() - # Delta backup - try: - self.backup_node( - backup_dir, 'node', replica, backup_type='delta', - data_dir=replica.data_dir, datname='backupdb', options=['--stream', '-U', 'backup']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because incremental backup should not be possible " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - # 9.5: ERROR: must be superuser or replication role to run a backup - # >=9.6: FATAL: must be superuser or replication role to start walsender - if self.pg_config_version < 160000: - self.assertRegex( - e.message, - "ERROR: must be superuser or replication role to run a backup|" - "FATAL: must be superuser or replication role to start walsender", - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - else: - self.assertRegex( - e.message, - "FATAL: permission denied to start WAL sender\n" - "DETAIL: Only roles with the REPLICATION", - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + # Delta backup + self.pb.backup_node('node', replica, backup_type='delta', + data_dir=replica.data_dir, datname='backupdb', + options=['--stream', '-U', 'backup'], + expect_error="because incremental backup should not be possible") + + if self.pg_config_version < 160000: + self.assertMessage( + contains=r"FATAL: must be superuser or replication role to start walsender") + else: + self.assertMessage( + contains="FATAL: permission denied to start WAL sender\n" + "DETAIL: Only roles with the REPLICATION") # @unittest.skip("skip") def test_missing_replication_permission_1(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) + ptrack_enable=self.ptrack) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'node', replica) + self.pb.restore_node('node', node=replica) # Settings for Replica self.set_replica(node, replica) @@ -3122,51 +2252,8 @@ def test_missing_replication_permission_1(self): 'postgres', 'CREATE DATABASE backupdb') - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - "CREATE ROLE backup WITH LOGIN; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + if self.pg_config_version >= 100000 and self.pg_config_version < 150000: node.safe_psql( 'backupdb', "CREATE ROLE backup WITH LOGIN; " @@ -3212,7 +2299,7 @@ def test_missing_replication_permission_1(self): "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" ) - if ProbackupTest.enterprise: + if ProbackupTest.pgpro: node.safe_psql( "backupdb", "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " @@ -3221,11 +2308,10 @@ def test_missing_replication_permission_1(self): replica.promote() # PAGE - output = self.backup_node( - backup_dir, 'node', replica, backup_type='page', + output = self.pb.backup_node('node', replica, backup_type='page', data_dir=replica.data_dir, datname='backupdb', options=['-U', 'backup'], return_id=False) - + self.assertIn( 'WARNING: Valid full backup on current timeline 2 is not found, trying to look up on previous timelines', output) @@ -3237,7 +2323,6 @@ def test_missing_replication_permission_1(self): # 'WARNING: could not connect to database backupdb: connection to server at "localhost" (127.0.0.1), port 29732 failed: FATAL: must be superuser or replication role to start walsender' # OS-dependant messages: # 'WARNING: could not connect to database backupdb: connection to server at "localhost" (::1), port 12101 failed: Connection refused\n\tIs the server running on that host and accepting TCP/IP connections?\nconnection to server at "localhost" (127.0.0.1), port 12101 failed: FATAL: must be superuser or replication role to start walsender' - if self.pg_config_version < 160000: self.assertRegex( output, @@ -3252,69 +2337,50 @@ def test_missing_replication_permission_1(self): # @unittest.skip("skip") def test_basic_backup_default_transaction_read_only(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={'default_transaction_read_only': 'on'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - try: - node.safe_psql( - 'postgres', - 'create temp table t1()') - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because incremental backup should not be possible " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except QueryException as e: - self.assertIn( - "cannot execute CREATE TABLE in a read-only transaction", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + error_result = node.safe_psql('postgres', + 'create temp table t1()', expect_error=True) + + self.assertMessage(error_result, contains="cannot execute CREATE TABLE in a read-only transaction") # FULL backup - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=['--stream']) # DELTA backup - self.backup_node( - backup_dir, 'node', node, backup_type='delta', options=['--stream']) + self.pb.backup_node('node', node, backup_type='delta', options=['--stream']) # PAGE backup - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') # @unittest.skip("skip") + @needs_gdb def test_backup_atexit(self): """""" - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) + ptrack_enable=self.ptrack) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=5) # Full backup in streaming mode - gdb = self.backup_node( - backup_dir, 'node', node, + gdb = self.pb.backup_node('node', node, options=['--stream', '--log-level-file=VERBOSE'], gdb=True) # break at streaming start @@ -3322,61 +2388,54 @@ def test_backup_atexit(self): gdb.run_until_break() gdb.remove_all_breakpoints() - gdb._execute('signal SIGINT') - sleep(1) + gdb.signal('SIGINT') - self.assertEqual( - self.show_pb( - backup_dir, 'node')[0]['status'], 'ERROR') + timeout = 60 + status = self.pb.show('node')[0]['status'] + while status == 'RUNNING' or timeout > 0: + sleep(1) + timeout = timeout - 1 + status = self.pb.show('node')[0]['status'] - with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: - log_content = f.read() - #print(log_content) - self.assertIn( - 'WARNING: A backup is in progress, stopping it.', + self.assertEqual(status, 'ERROR') + + log_content = self.read_pb_log() + + self.assertIn( + 'WARNING: A backup is in progress, stopping it', log_content) - if self.get_version(node) < 150000: - self.assertIn( + if self.pg_config_version < 150000: + self.assertIn( 'FROM pg_catalog.pg_stop_backup', log_content) - else: - self.assertIn( + else: + self.assertIn( 'FROM pg_catalog.pg_backup_stop', log_content) - self.assertIn( + self.assertIn( 'setting its status to ERROR', log_content) # @unittest.skip("skip") def test_pg_stop_backup_missing_permissions(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) + ptrack_enable=self.ptrack) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=5) self.simple_bootstrap(node, 'backup') - if self.get_version(node) < 90600: - node.safe_psql( - 'postgres', - 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() FROM backup') - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'postgres', - 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) FROM backup') - elif self.get_version(node) < 150000: + if self.pg_config_version < 150000: node.safe_psql( 'postgres', 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean, boolean) FROM backup') @@ -3385,111 +2444,66 @@ def test_pg_stop_backup_missing_permissions(self): 'postgres', 'REVOKE EXECUTE ON FUNCTION pg_catalog.pg_backup_stop(boolean) FROM backup') - + if self.pg_config_version < 150000: + stop_backup = "pg_stop_backup" + else: + stop_backup = "pg_backup_stop" # Full backup in streaming mode - try: - self.backup_node( - backup_dir, 'node', node, - options=['--stream', '-U', 'backup']) - # we should die here because exception is what we expect to happen - if self.get_version(node) < 150000: - self.assertEqual( - 1, 0, - "Expecting Error because of missing permissions on pg_stop_backup " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - else: - self.assertEqual( - 1, 0, - "Expecting Error because of missing permissions on pg_backup_stop " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - if self.get_version(node) < 150000: - self.assertIn( - "ERROR: permission denied for function pg_stop_backup", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - else: - self.assertIn( - "ERROR: permission denied for function pg_backup_stop", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertIn( - "query was: SELECT pg_catalog.txid_snapshot_xmax", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, + options=['--stream', '-U', 'backup'], + expect_error=f"because of missing permissions on {stop_backup}") + self.assertMessage(contains=f"ERROR: permission denied for function {stop_backup}") + self.assertMessage(contains="query was: SELECT pg_catalog.txid_snapshot_xmax") # @unittest.skip("skip") def test_start_time(self): """Test, that option --start-time allows to set backup_id and restore""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) + ptrack_enable=self.ptrack) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL backup - startTime = int(time()) - self.backup_node( - backup_dir, 'node', node, backup_type='full', - options=['--stream', '--start-time={0}'.format(str(startTime))]) + startTimeFull = int(time()) + self.pb.backup_node('node', node, backup_type='full', + options=['--stream', '--start-time={0}'.format(startTimeFull)]) # restore FULL backup by backup_id calculated from start-time - self.restore_node( - backup_dir, 'node', - data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_full'), - backup_id=base36enc(startTime)) + # cleanup it if we have leftover from a failed test + node_restored_full = self.pg_node.make_empty('node_restored_full') + self.pb.restore_node('node', node_restored_full, + backup_id=base36enc(startTimeFull)) #FULL backup with incorrect start time - try: - startTime = str(int(time()-100000)) - self.backup_node( - backup_dir, 'node', node, backup_type='full', - options=['--stream', '--start-time={0}'.format(startTime)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - 'Expecting Error because start time for new backup must be newer ' - '\n Output: {0} \n CMD: {1}'.format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertRegex( - e.message, - r"ERROR: Can't assign backup_id from requested start_time \(\w*\), this time must be later that backup \w*\n", - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + startTime = startTimeFull-100000 + self.pb.backup_node('node', node, backup_type='full', + options=['--stream', '--start-time={0}'.format(startTime)], + expect_error="because start time for new backup must be newer") + self.assertMessage( + regex=r"ERROR: Can't assign backup_id from requested start_time " + r"\(\w*\), this time must be later that backup \w*\n") # DELTA backup - startTime = int(time()) - self.backup_node( - backup_dir, 'node', node, backup_type='delta', + startTime = max(int(time()), startTimeFull+1) + self.pb.backup_node('node', node, backup_type='delta', options=['--stream', '--start-time={0}'.format(str(startTime))]) # restore DELTA backup by backup_id calculated from start-time - self.restore_node( - backup_dir, 'node', - data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_delta'), + node_restored_delta = self.pg_node.make_empty('node_restored_delta') + self.pb.restore_node('node', node_restored_delta, backup_id=base36enc(startTime)) # PAGE backup - startTime = int(time()) - self.backup_node( - backup_dir, 'node', node, backup_type='page', + startTime = max(int(time()), startTime+1) + self.pb.backup_node('node', node, backup_type='page', options=['--stream', '--start-time={0}'.format(str(startTime))]) # restore PAGE backup by backup_id calculated from start-time - self.restore_node( - backup_dir, 'node', - data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_page'), - backup_id=base36enc(startTime)) + node_restored_page = self.pg_node.make_empty('node_restored_page') + self.pb.restore_node('node', node=node_restored_page, + backup_id=base36enc(startTime)) # PTRACK backup if self.ptrack: @@ -3497,77 +2511,63 @@ def test_start_time(self): 'postgres', 'create extension ptrack') - startTime = int(time()) - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + startTime = max(int(time()), startTime+1) + self.pb.backup_node('node', node, backup_type='ptrack', options=['--stream', '--start-time={0}'.format(str(startTime))]) # restore PTRACK backup by backup_id calculated from start-time - self.restore_node( - backup_dir, 'node', - data_dir=os.path.join(self.tmp_path, self.module_name, self.fname, 'node_restored_ptrack'), + node_restored_ptrack = self.pg_node.make_empty('node_restored_ptrack') + self.pb.restore_node('node', node_restored_ptrack, backup_id=base36enc(startTime)) # @unittest.skip("skip") def test_start_time_few_nodes(self): """Test, that we can synchronize backup_id's for different DBs""" - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), + node1 = self.pg_node.make_simple('node1', set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) + ptrack_enable=self.ptrack) - backup_dir1 = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup1') - self.init_pb(backup_dir1) - self.add_instance(backup_dir1, 'node1', node1) - self.set_archiving(backup_dir1, 'node1', node1) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node1', node1) + self.pb.set_archiving('node1', node1) node1.slow_start() - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2'), + node2 = self.pg_node.make_simple('node2', set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) + ptrack_enable=self.ptrack) - backup_dir2 = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup2') - self.init_pb(backup_dir2) - self.add_instance(backup_dir2, 'node2', node2) - self.set_archiving(backup_dir2, 'node2', node2) + self.pb.add_instance('node2', node2) + self.pb.set_archiving('node2', node2) node2.slow_start() # FULL backup - startTime = str(int(time())) - self.backup_node( - backup_dir1, 'node1', node1, backup_type='full', + startTime = int(time()) + self.pb.backup_node('node1', node1, backup_type='full', options=['--stream', '--start-time={0}'.format(startTime)]) - self.backup_node( - backup_dir2, 'node2', node2, backup_type='full', + self.pb.backup_node('node2', node2, backup_type='full', options=['--stream', '--start-time={0}'.format(startTime)]) - show_backup1 = self.show_pb(backup_dir1, 'node1')[0] - show_backup2 = self.show_pb(backup_dir2, 'node2')[0] + show_backup1 = self.pb.show('node1')[0] + show_backup2 = self.pb.show('node2')[0] self.assertEqual(show_backup1['id'], show_backup2['id']) # DELTA backup - startTime = str(int(time())) - self.backup_node( - backup_dir1, 'node1', node1, backup_type='delta', + startTime = max(int(time()), startTime+1) + self.pb.backup_node('node1', node1, backup_type='delta', options=['--stream', '--start-time={0}'.format(startTime)]) - self.backup_node( - backup_dir2, 'node2', node2, backup_type='delta', + self.pb.backup_node('node2', node2, backup_type='delta', options=['--stream', '--start-time={0}'.format(startTime)]) - show_backup1 = self.show_pb(backup_dir1, 'node1')[1] - show_backup2 = self.show_pb(backup_dir2, 'node2')[1] + show_backup1 = self.pb.show('node1')[1] + show_backup2 = self.pb.show('node2')[1] self.assertEqual(show_backup1['id'], show_backup2['id']) # PAGE backup - startTime = str(int(time())) - self.backup_node( - backup_dir1, 'node1', node1, backup_type='page', + startTime = max(int(time()), startTime+1) + self.pb.backup_node('node1', node1, backup_type='page', options=['--stream', '--start-time={0}'.format(startTime)]) - self.backup_node( - backup_dir2, 'node2', node2, backup_type='page', + self.pb.backup_node('node2', node2, backup_type='page', options=['--stream', '--start-time={0}'.format(startTime)]) - show_backup1 = self.show_pb(backup_dir1, 'node1')[2] - show_backup2 = self.show_pb(backup_dir2, 'node2')[2] + show_backup1 = self.pb.show('node1')[2] + show_backup2 = self.pb.show('node2')[2] self.assertEqual(show_backup1['id'], show_backup2['id']) # PTRACK backup @@ -3579,27 +2579,24 @@ def test_start_time_few_nodes(self): 'postgres', 'create extension ptrack') - startTime = str(int(time())) - self.backup_node( - backup_dir1, 'node1', node1, backup_type='ptrack', + startTime = max(int(time()), startTime+1) + self.pb.backup_node( + 'node1', node1, backup_type='ptrack', options=['--stream', '--start-time={0}'.format(startTime)]) - self.backup_node( - backup_dir2, 'node2', node2, backup_type='ptrack', + self.pb.backup_node('node2', node2, backup_type='ptrack', options=['--stream', '--start-time={0}'.format(startTime)]) - show_backup1 = self.show_pb(backup_dir1, 'node1')[3] - show_backup2 = self.show_pb(backup_dir2, 'node2')[3] + show_backup1 = self.pb.show('node1')[3] + show_backup2 = self.pb.show('node2')[3] self.assertEqual(show_backup1['id'], show_backup2['id']) def test_regress_issue_585(self): """https://github.com/postgrespro/pg_probackup/issues/585""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple( + base_dir='node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # create couple of files that looks like db files @@ -3608,12 +2605,10 @@ def test_regress_issue_585(self): with open(os.path.join(node.data_dir, 'pg_multixact/members/1000'),'wb') as f: pass - self.backup_node( - backup_dir, 'node', node, backup_type='full', + self.pb.backup_node('node', node, backup_type='full', options=['--stream']) - output = self.backup_node( - backup_dir, 'node', node, backup_type='delta', + output = self.pb.backup_node('node', node, backup_type='delta', options=['--stream'], return_id=False, ) @@ -3621,30 +2616,27 @@ def test_regress_issue_585(self): node.cleanup() - output = self.restore_node(backup_dir, 'node', node) + output = self.pb.restore_node('node', node) self.assertNotRegex(output, r'WARNING: [^\n]* was stored as .* but looks like') def test_2_delta_backups(self): """https://github.com/postgrespro/pg_probackup/issues/596""" - node = self.make_simple_node('node', - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + backup_dir = self.backup_dir - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - # self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + # self.pb.set_archiving('node', node) node.slow_start() # FULL - full_backup_id = self.backup_node(backup_dir, 'node', node, options=["--stream"]) + full_backup_id = self.pb.backup_node('node', node, options=["--stream"]) # delta backup mode - delta_backup_id1 = self.backup_node( - backup_dir, 'node', node, backup_type="delta", options=["--stream"]) + delta_backup_id1 = self.pb.backup_node('node', node, backup_type="delta", options=["--stream"]) - delta_backup_id2 = self.backup_node( - backup_dir, 'node', node, backup_type="delta", options=["--stream"]) + delta_backup_id2 = self.pb.backup_node('node', node, backup_type="delta", options=["--stream"]) # postgresql.conf and pg_hba.conf shouldn't be copied conf_file = os.path.join(backup_dir, 'backups', 'node', delta_backup_id1, 'database', 'postgresql.conf') @@ -3656,3 +2648,308 @@ def test_2_delta_backups(self): self.assertFalse( os.path.exists(conf_file), "File should not exist: {0}".format(conf_file)) + + + ######################################### + # --dry-run + ######################################### + + def test_dry_run_backup(self): + """ + Test dry-run option for full backup + """ + node = self.pg_node.make_simple('node', + ptrack_enable=self.ptrack, + # we need to write a lot. Lets speedup a bit. + pg_options={"fsync": "off", "synchronous_commit": "off"}) + external_dir = self.get_tblspace_path(node, 'somedirectory') + os.mkdir(external_dir) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + # Fill with data + # Have to use scale=100 to create second segment. + node.pgbench_init(scale=50, no_vacuum=True) + + backup_dir = self.backup_dir + + content_before = self.pgdata_content(os.path.join(backup_dir, 'backups', 'node')) + + # FULL archive + backup_id = self.pb.backup_node('node', node, options=['--dry-run', '--note=test_note', + '--external-dirs={0}'.format(external_dir)]) + + show_backups = self.pb.show('node') + self.assertEqual(len(show_backups), 0) + + self.compare_pgdata( + content_before, + self.pgdata_content(os.path.join(backup_dir, 'backups', 'node')) + ) + + # FULL stream + backup_id = self.pb.backup_node('node', node, options=['--stream', '--dry-run', '--note=test_note', + '--external-dirs={0}'.format(external_dir)]) + + show_backups = self.pb.show('node') + self.assertEqual(len(show_backups), 0) + + self.compare_pgdata( + content_before, + self.pgdata_content(os.path.join(backup_dir, 'backups', 'node')) + ) + + # do FULL + backup_id = self.pb.backup_node('node', node, + options=['--stream', '--external-dirs={0}'.format(external_dir), + '--note=test_note']) + # Add some data changes to better testing + pgbench = node.pgbench(options=['-T', '2', '--no-vacuum']) + pgbench.wait() + + content_before = self.pgdata_content(os.path.join(backup_dir, 'backups', 'node')) + + # DELTA + delta_backup_id = self.pb.backup_node('node', node, backup_type="delta", + options=['--stream', '--external-dirs={0}'.format(external_dir), + '--note=test_note', '--dry-run']) + # DELTA + delta_backup_id = self.pb.backup_node('node', node, backup_type="delta", + options=['--external-dirs={0}'.format(external_dir), + '--note=test_note', '--dry-run']) + show_backups = self.pb.show('node') + self.assertEqual(len(show_backups), 1) + + self.compare_pgdata( + content_before, + self.pgdata_content(os.path.join(backup_dir, 'backups', 'node')) + ) + + # do DELTA + delta_backup_id = self.pb.backup_node('node', node, backup_type="delta", + options=['--stream', '--external-dirs={0}'.format(external_dir), + '--note=test_note']) + # Add some data changes + pgbench = node.pgbench(options=['-T', '2', '--no-vacuum']) + pgbench.wait() + + instance_before = self.pgdata_content(os.path.join(backup_dir, 'backups', 'node')) + + # PAGE + page_backup_id = self.pb.backup_node('node', node, backup_type="page", + options=['--external-dirs={0}'.format(external_dir), + '--note=test_note', '--dry-run']) + show_backups = self.pb.show('node') + self.assertEqual(len(show_backups), 2) + + self.compare_pgdata( + instance_before, + self.pgdata_content(os.path.join(backup_dir, 'backups', 'node')) + ) + + # do PAGE + page_backup_id = self.pb.backup_node('node', node, backup_type="page", + options=['--external-dirs={0}'.format(external_dir), + '--note=test_note']) + instance_before = self.pgdata_content(os.path.join(backup_dir, 'backups', 'node')) + + # Add some data changes + pgbench = node.pgbench(options=['-T', '2', '--no-vacuum']) + pgbench.wait() + + if self.ptrack: + node.safe_psql( + "postgres", + "create extension ptrack") + + if self.ptrack: + backup_id = self.pb.backup_node('node', node, backup_type='ptrack', + options=['--stream', '--external-dirs={0}'.format(external_dir), + '--note=test_note', '--dry-run']) + if self.ptrack: + backup_id = self.pb.backup_node('node', node, backup_type='ptrack', + options=['--external-dirs={0}'.format(external_dir), + '--note=test_note', '--dry-run']) + + show_backups = self.pb.show('node') + self.assertEqual(len(show_backups), 3) + + self.compare_pgdata( + instance_before, + self.pgdata_content(os.path.join(backup_dir, 'backups', 'node')) + ) + + # do PTRACK + if self.ptrack: + backup_id = self.pb.backup_node('node', node, backup_type='ptrack', + options=['--stream', '--external-dirs={0}'.format(external_dir), + '--note=test_note']) + + out = self.pb.validate('node', backup_id) + self.assertIn( + "INFO: Backup {0} is valid".format(backup_id), + out) + # Cleanup + node.stop() + + @unittest.skipIf(not fs_backup_class.is_file_based, "AccessPath check is always true on s3") + def test_basic_dry_run_check_backup_with_access(self): + """ + Access check suite if disk mounted as read_only + """ + node = self.pg_node.make_simple('node', + # we need to write a lot. Lets speedup a bit. + pg_options={"fsync": "off", "synchronous_commit": "off"}) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + # Fill with data + # Have to use scale=100 to create second segment. + node.pgbench_init(scale=20, no_vacuum=True) + + # FULL backup + self.pb.backup_node('node', node, options=['--dry-run', '--stream', '--log-level-file=verbose']) + + check_permissions_dir = ['backups', 'wal'] + for dir in check_permissions_dir: + # Access check suit if disk mounted as read_only + dir_path = os.path.join(backup_dir, dir) + dir_mode = os.stat(dir_path).st_mode + os.chmod(dir_path, 0o400) + print(backup_dir) + + try: + error_message = self.pb.backup_node('node', node, backup_type='delta', + options=['--stream', '--dry-run'], + expect_error="because of changed permissions") + + self.assertMessage(error_message, contains='Permission denied') + finally: + # Cleanup + os.chmod(dir_path, dir_mode) + os.chmod(dir_path, 0o500) + print(backup_dir) + + try: + error_message = self.pb.backup_node('node', node, backup_type='delta', + options=['--stream', '--dry-run'], + expect_error="because of changed permissions") + + + self.assertMessage(error_message, contains='ERROR: Check permissions') + finally: + # Cleanup + os.chmod(dir_path, dir_mode) + + node.stop() + node.cleanup() + + def string_in_file(self, file_path, str): + with open(file_path, 'r') as file: + # read all content of a file + content = file.read() + # check if string present in a file + if str in content: + return True + else: + return False + + def test_dry_run_restore_point_absence(self): + node = self.pg_node.make_simple('node', + # we need to write a lot. Lets speedup a bit. + pg_options={"fsync": "off", "synchronous_commit": "off"}) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + # Fill with data + # Have to use scale=100 to create second segment. + node.pgbench_init(scale=100, no_vacuum=True) + + data_dir = node.data_dir + + backup_id = self.pb.backup_node('node', node, options=['--dry-run']) + + node.stop() + + restore_point = self.string_in_file(os.path.join(node.logs_dir, "postgresql.log"), "restore point") + self.assertFalse(restore_point, "String should not exist: {0}".format("restore point")) + + @needs_gdb + def test_dry_run_backup_kill_process(self): + node = self.pg_node.make_simple('node', + # we need to write a lot. Lets speedup a bit. + pg_options={"fsync": "off", "synchronous_commit": "off"}) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + # Fill with data + # Have to use scale=100 to create second segment. + node.pgbench_init(scale=20, no_vacuum=True) + + backup_dir = self.backup_dir + + content_before = self.pgdata_content(os.path.join(backup_dir, 'backups', 'node')) + # FULL backup + gdb = self.pb.backup_node('node', node, options=['--dry-run', '--stream', '--log-level-file=verbose'], + gdb=True) + + gdb.set_breakpoint('backup_files') + gdb.run_until_break() + gdb.signal('SIGTERM') + gdb.continue_execution_until_error() + + self.compare_pgdata( + content_before, + self.pgdata_content(os.path.join(backup_dir, 'backups', 'node')) + ) + + gdb.kill() + node.stop() + + def test_limit_rate_full_backup(self): + """ + Test full backup with slow down to 8MBps speed + """ + set_rate_limit = 8 + node = self.pg_node.make_simple('node', + # we need to write a lot. Lets speedup a bit. + pg_options={"fsync": "off", "synchronous_commit": "off"}) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=5, no_vacuum=True) + + # FULL backup with rate limit + backup_id = self.pb.backup_node("node", node, options=['--write-rate-limit='+str(set_rate_limit)]) + + # Validate backup + out = self.pb.validate('node', backup_id) + self.assertIn( + "INFO: Backup {0} is valid".format(backup_id), + out) + + # Calculate time from start to end of backup + show_backup = self.pb.show("node") + backup_time = (datetime.strptime(show_backup[0]["end-time"]+"00", "%Y-%m-%d %H:%M:%S%z") - + datetime.strptime(show_backup[0]["start-time"]+"00", "%Y-%m-%d %H:%M:%S%z") + ).seconds + + # Calculate rate limit we've got in MBps and round it down + get_rate_limit = int(show_backup[0]["data-bytes"] / (1024 * 1024 * backup_time)) + + # Check that we are NOT faseter than expexted + self.assertLessEqual(get_rate_limit, set_rate_limit) diff --git a/tests/catchup_test.py b/tests/catchup_test.py index cf8388dd2..117ac0407 100644 --- a/tests/catchup_test.py +++ b/tests/catchup_test.py @@ -1,10 +1,15 @@ import os +import subprocess from pathlib import Path -import signal -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest +from parameterized import parameterized -class CatchupTest(ProbackupTest, unittest.TestCase): +module_name = 'catchup' + + +class CatchupTest(ProbackupTest): + def setUp(self): + self.fname = self.id().split('.')[3] ######################################### # Basic tests @@ -14,8 +19,7 @@ def test_basic_full_catchup(self): Test 'multithreaded basebackup' mode (aka FULL catchup) """ # preparation - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True ) src_pg.slow_start() @@ -25,8 +29,8 @@ def test_basic_full_catchup(self): src_query_result = src_pg.table_checksum("ultimate_question") # do full catchup - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -43,7 +47,7 @@ def test_basic_full_catchup(self): src_pg.stop() dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start() # 2nd check: run verification query @@ -54,13 +58,165 @@ def test_basic_full_catchup(self): dst_pg.stop() #self.assertEqual(1, 0, 'Stop test') + + @parameterized.expand(("DELTA", "PTRACK")) + def test_cascade_catchup(self, test_input): + """ + Test catchup of catchup'ed node + """ + # preparation + + if test_input == "PTRACK" and not self.ptrack: + self.skipTest("Ptrack is disabled, test_cascade_catchup") + elif test_input == "PTRACK" and self.ptrack: + db1 = self.pg_node.make_simple('db1', set_replication = True, ptrack_enable=True) + else: + db1 = self.pg_node.make_simple('db1', set_replication = True) + + db1.slow_start() + + if test_input == "PTRACK": + db1.safe_psql("postgres", "CREATE EXTENSION ptrack") + + db1.safe_psql( + "postgres", + "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + db1_query_result = db1.table_checksum("ultimate_question") + + # full catchup db1 -> db2 + db2 = self.pg_node.make_empty('db2') + self.pb.catchup_node( + backup_mode = 'FULL', + source_pgdata = db1.data_dir, + destination_node = db2, + options = ['-d', 'postgres', '-p', str(db1.port), '--stream'] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(db1.data_dir), + self.pgdata_content(db2.data_dir) + ) + + # run&recover catchup'ed instance + self.set_replica(db1, db2) + db2_options = {} + db2_options['port'] = str(db2.port) + db2.set_auto_conf(db2_options) + db2.slow_start(replica = True) + + # 2nd check: run verification query + db2_query_result = db2.table_checksum("ultimate_question") + self.assertEqual(db1_query_result, db2_query_result, 'Different answer from copy 2') + + # full catchup db2 -> db3 + db3 = self.pg_node.make_empty('db3') + self.pb.catchup_node( + backup_mode = 'FULL', + source_pgdata = db2.data_dir, + destination_node = db3, + options = ['-d', 'postgres', '-p', str(db2.port), '--stream'] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(db2.data_dir), + self.pgdata_content(db3.data_dir) + ) + + # run&recover catchup'ed instance + self.set_replica(db2, db3) + db3_options = {} + db3_options['port'] = str(db3.port) + db3.set_auto_conf(db3_options) + db3.slow_start(replica = True) + + db3_query_result = db3.table_checksum("ultimate_question") + self.assertEqual(db2_query_result, db3_query_result, 'Different answer from copy 3') + + db2.stop() + db3.stop() + + # data modifications before incremental catchups + db1.safe_psql( + "postgres", + "UPDATE ultimate_question SET answer = -1") + db1.safe_psql("postgres", "CHECKPOINT") + + # do first incremental catchup + self.pb.catchup_node( + backup_mode = test_input, + source_pgdata = db1.data_dir, + destination_node = db2, + options = ['-d', 'postgres', '-p', str(db1.port), '--stream'] + ) + + self.compare_pgdata( + self.pgdata_content(db1.data_dir), + self.pgdata_content(db2.data_dir) + ) + + self.set_replica(db1, db2) + db2_options = {} + db2_options['port'] = str(db2.port) + db2.set_auto_conf(db2_options) + db2.slow_start(replica = True) + + # do second incremental catchup + self.pb.catchup_node( + backup_mode = test_input, + source_pgdata = db2.data_dir, + destination_node = db3, + options = ['-d', 'postgres', '-p', str(db2.port), '--stream'] + ) + + self.compare_pgdata( + self.pgdata_content(db2.data_dir), + self.pgdata_content(db3.data_dir) + ) + + self.set_replica(db2, db3) + db3_options = {} + db3_options['port'] = str(db3.port) + db3.set_auto_conf(db3_options) + self.pb.set_archiving('db3', db3, replica=True) + db3.slow_start(replica = True) + + # data modification for checking continuous archiving + db1.safe_psql( + "postgres", + "DROP TABLE ultimate_question") + db1.safe_psql("postgres", "CHECKPOINT") + + self.wait_until_replica_catch_with_master(db1, db2) + self.wait_until_replica_catch_with_master(db1, db3) + + db1_query_result = db1.table_checksum("pg_class") + db2_query_result = db2.table_checksum("pg_class") + db3_query_result = db3.table_checksum("pg_class") + + self.assertEqual(db1_query_result, db2_query_result, 'Different answer from copy 2') + self.assertEqual(db2_query_result, db3_query_result, 'Different answer from copy 3') + + db1_query_result = db1.table_checksum("pg_depend") + db2_query_result = db2.table_checksum("pg_depend") + db3_query_result = db3.table_checksum("pg_depend") + + self.assertEqual(db1_query_result, db2_query_result, 'Different answer from copy 2') + self.assertEqual(db2_query_result, db3_query_result, 'Different answer from copy 3') + + # cleanup + db3.stop() + db2.stop() + db1.stop() + + def test_full_catchup_with_tablespace(self): """ Test tablespace transfers """ # preparation - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True ) src_pg.slow_start() @@ -72,9 +228,9 @@ def test_full_catchup_with_tablespace(self): src_query_result = src_pg.table_checksum("ultimate_question") # do full catchup with tablespace mapping - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + dst_pg = self.pg_node.make_empty('dst') tblspace1_new_path = self.get_tblspace_path(dst_pg, 'tblspace1_new') - self.catchup_node( + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -101,7 +257,7 @@ def test_full_catchup_with_tablespace(self): # run&recover catchup'ed instance dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start() # 2nd check: run verification query @@ -116,8 +272,7 @@ def test_basic_delta_catchup(self): Test delta catchup """ # preparation 1: source - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True, pg_options = { 'wal_log_hints': 'on' } ) @@ -127,8 +282,8 @@ def test_basic_delta_catchup(self): "CREATE TABLE ultimate_question(answer int)") # preparation 2: make clean shutdowned lagging behind replica - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -137,7 +292,7 @@ def test_basic_delta_catchup(self): self.set_replica(src_pg, dst_pg) dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start(replica = True) dst_pg.stop() @@ -149,7 +304,7 @@ def test_basic_delta_catchup(self): src_query_result = src_pg.table_checksum("ultimate_question") # do delta catchup - self.catchup_node( + self.pb.catchup_node( backup_mode = 'DELTA', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -167,7 +322,7 @@ def test_basic_delta_catchup(self): self.set_replica(master = src_pg, replica = dst_pg) dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start(replica = True) # 2nd check: run verification query @@ -186,12 +341,7 @@ def test_basic_ptrack_catchup(self): self.skipTest('Skipped because ptrack support is disabled') # preparation 1: source - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), - set_replication = True, - ptrack_enable = True, - initdb_params = ['--data-checksums'] - ) + src_pg = self.pg_node.make_simple('src', set_replication=True, ptrack_enable=True) src_pg.slow_start() src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack") src_pg.safe_psql( @@ -199,8 +349,8 @@ def test_basic_ptrack_catchup(self): "CREATE TABLE ultimate_question(answer int)") # preparation 2: make clean shutdowned lagging behind replica - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -209,7 +359,7 @@ def test_basic_ptrack_catchup(self): self.set_replica(src_pg, dst_pg) dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start(replica = True) dst_pg.stop() @@ -221,7 +371,7 @@ def test_basic_ptrack_catchup(self): src_query_result = src_pg.table_checksum("ultimate_question") # do ptrack catchup - self.catchup_node( + self.pb.catchup_node( backup_mode = 'PTRACK', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -239,7 +389,7 @@ def test_basic_ptrack_catchup(self): self.set_replica(master = src_pg, replica = dst_pg) dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start(replica = True) # 2nd check: run verification query @@ -255,16 +405,15 @@ def test_tli_delta_catchup(self): Test that we correctly follow timeline change with delta catchup """ # preparation 1: source - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True, pg_options = { 'wal_log_hints': 'on' } ) src_pg.slow_start() # preparation 2: destination - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -272,7 +421,7 @@ def test_tli_delta_catchup(self): ) dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start() dst_pg.stop() @@ -285,7 +434,7 @@ def test_tli_delta_catchup(self): src_query_result = src_pg.table_checksum("ultimate_question") # do catchup (src_tli = 2, dst_tli = 1) - self.catchup_node( + self.pb.catchup_node( backup_mode = 'DELTA', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -301,7 +450,7 @@ def test_tli_delta_catchup(self): # run&recover catchup'ed instance dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) self.set_replica(master = src_pg, replica = dst_pg) dst_pg.slow_start(replica = True) @@ -312,7 +461,7 @@ def test_tli_delta_catchup(self): dst_pg.stop() # do catchup (src_tli = 2, dst_tli = 2) - self.catchup_node( + self.pb.catchup_node( backup_mode = 'DELTA', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -330,18 +479,13 @@ def test_tli_ptrack_catchup(self): self.skipTest('Skipped because ptrack support is disabled') # preparation 1: source - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), - set_replication = True, - ptrack_enable = True, - initdb_params = ['--data-checksums'] - ) + src_pg = self.pg_node.make_simple('src', set_replication=True, ptrack_enable=True) src_pg.slow_start() src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack") # preparation 2: destination - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -349,7 +493,7 @@ def test_tli_ptrack_catchup(self): ) dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start() dst_pg.stop() @@ -367,7 +511,7 @@ def test_tli_ptrack_catchup(self): src_query_result = src_pg.table_checksum("ultimate_question") # do catchup (src_tli = 2, dst_tli = 1) - self.catchup_node( + self.pb.catchup_node( backup_mode = 'PTRACK', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -383,7 +527,7 @@ def test_tli_ptrack_catchup(self): # run&recover catchup'ed instance dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) self.set_replica(master = src_pg, replica = dst_pg) dst_pg.slow_start(replica = True) @@ -394,7 +538,7 @@ def test_tli_ptrack_catchup(self): dst_pg.stop() # do catchup (src_tli = 2, dst_tli = 2) - self.catchup_node( + self.pb.catchup_node( backup_mode = 'PTRACK', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -412,8 +556,7 @@ def test_table_drop_with_delta(self): Test that dropped table in source will be dropped in delta catchup'ed instance too """ # preparation 1: source - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True, pg_options = { 'wal_log_hints': 'on' } ) @@ -423,8 +566,8 @@ def test_table_drop_with_delta(self): "CREATE TABLE ultimate_question AS SELECT 42 AS answer") # preparation 2: make clean shutdowned lagging behind replica - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -432,7 +575,7 @@ def test_table_drop_with_delta(self): ) dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start() dst_pg.stop() @@ -443,7 +586,7 @@ def test_table_drop_with_delta(self): src_pg.safe_psql("postgres", "CHECKPOINT") # do delta catchup - self.catchup_node( + self.pb.catchup_node( backup_mode = 'DELTA', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -467,12 +610,9 @@ def test_table_drop_with_ptrack(self): self.skipTest('Skipped because ptrack support is disabled') # preparation 1: source - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True, - ptrack_enable = True, - initdb_params = ['--data-checksums'] - ) + ptrack_enable = True) src_pg.slow_start() src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack") src_pg.safe_psql( @@ -480,8 +620,8 @@ def test_table_drop_with_ptrack(self): "CREATE TABLE ultimate_question AS SELECT 42 AS answer") # preparation 2: make clean shutdowned lagging behind replica - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -489,7 +629,7 @@ def test_table_drop_with_ptrack(self): ) dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start() dst_pg.stop() @@ -500,7 +640,7 @@ def test_table_drop_with_ptrack(self): src_pg.safe_psql("postgres", "CHECKPOINT") # do ptrack catchup - self.catchup_node( + self.pb.catchup_node( backup_mode = 'PTRACK', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -521,8 +661,7 @@ def test_tablefile_truncation_with_delta(self): Test that truncated table in source will be truncated in delta catchup'ed instance too """ # preparation 1: source - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True, pg_options = { 'wal_log_hints': 'on' } ) @@ -537,8 +676,8 @@ def test_tablefile_truncation_with_delta(self): src_pg.safe_psql("postgres", "VACUUM t_heap") # preparation 2: make clean shutdowned lagging behind replica - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -547,7 +686,7 @@ def test_tablefile_truncation_with_delta(self): dest_options = {} dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start() dst_pg.stop() @@ -556,7 +695,7 @@ def test_tablefile_truncation_with_delta(self): src_pg.safe_psql("postgres", "VACUUM t_heap") # do delta catchup - self.catchup_node( + self.pb.catchup_node( backup_mode = 'DELTA', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -580,12 +719,9 @@ def test_tablefile_truncation_with_ptrack(self): self.skipTest('Skipped because ptrack support is disabled') # preparation 1: source - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True, - ptrack_enable = True, - initdb_params = ['--data-checksums'] - ) + ptrack_enable = True) src_pg.slow_start() src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack") src_pg.safe_psql( @@ -598,8 +734,8 @@ def test_tablefile_truncation_with_ptrack(self): src_pg.safe_psql("postgres", "VACUUM t_heap") # preparation 2: make clean shutdowned lagging behind replica - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -608,7 +744,7 @@ def test_tablefile_truncation_with_ptrack(self): dest_options = {} dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start() dst_pg.stop() @@ -617,7 +753,7 @@ def test_tablefile_truncation_with_ptrack(self): src_pg.safe_psql("postgres", "VACUUM t_heap") # do ptrack catchup - self.catchup_node( + self.pb.catchup_node( backup_mode = 'PTRACK', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -643,7 +779,7 @@ def test_local_tablespace_without_mapping(self): if self.remote: self.skipTest('Skipped because this test tests local catchup error handling') - src_pg = self.make_simple_node(base_dir = os.path.join(self.module_name, self.fname, 'src')) + src_pg = self.pg_node.make_simple('src') src_pg.slow_start() tblspace_path = self.get_tblspace_path(src_pg, 'tblspace') @@ -655,9 +791,8 @@ def test_local_tablespace_without_mapping(self): "postgres", "CREATE TABLE ultimate_question TABLESPACE tblspace AS SELECT 42 AS answer") - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - try: - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -665,15 +800,11 @@ def test_local_tablespace_without_mapping(self): '-d', 'postgres', '-p', str(src_pg.port), '--stream', - ] + ], + expect_error="because '-T' parameter is not specified" ) - self.assertEqual(1, 0, "Expecting Error because '-T' parameter is not specified.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Local catchup executed, but source database contains tablespace', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertMessage(contains='ERROR: Local catchup executed, but source ' + 'database contains tablespace') # Cleanup src_pg.stop() @@ -683,16 +814,15 @@ def test_running_dest_postmaster(self): Test that we detect running postmaster in destination """ # preparation 1: source - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True, pg_options = { 'wal_log_hints': 'on' } ) src_pg.slow_start() # preparation 2: destination - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -700,26 +830,20 @@ def test_running_dest_postmaster(self): ) dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start() # leave running destination postmaster # so don't call dst_pg.stop() # try delta catchup - try: - self.catchup_node( + self.pb.catchup_node( backup_mode = 'DELTA', source_pgdata = src_pg.data_dir, destination_node = dst_pg, - options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'], + expect_error="because postmaster in destination is running" ) - self.assertEqual(1, 0, "Expecting Error because postmaster in destination is running.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Postmaster with pid ', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertMessage(contains='ERROR: Postmaster with pid ') # Cleanup src_pg.stop() @@ -730,14 +854,13 @@ def test_same_db_id(self): """ # preparation: # source - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True ) src_pg.slow_start() # destination - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -745,45 +868,33 @@ def test_same_db_id(self): ) dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start() dst_pg.stop() # fake destination - fake_dst_pg = self.make_simple_node(base_dir = os.path.join(self.module_name, self.fname, 'fake_dst')) + fake_dst_pg = self.pg_node.make_simple('fake_dst') # fake source - fake_src_pg = self.make_simple_node(base_dir = os.path.join(self.module_name, self.fname, 'fake_src')) + fake_src_pg = self.pg_node.make_simple('fake_src') # try delta catchup (src (with correct src conn), fake_dst) - try: - self.catchup_node( + self.pb.catchup_node( backup_mode = 'DELTA', source_pgdata = src_pg.data_dir, destination_node = fake_dst_pg, - options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'], + expect_error="because database identifiers mismatch" ) - self.assertEqual(1, 0, "Expecting Error because database identifiers mismatch.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Database identifiers mismatch: ', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertMessage(contains='ERROR: Database identifiers mismatch: ') # try delta catchup (fake_src (with wrong src conn), dst) - try: - self.catchup_node( + self.pb.catchup_node( backup_mode = 'DELTA', source_pgdata = fake_src_pg.data_dir, destination_node = dst_pg, - options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'], + expect_error="because database identifiers mismatch" ) - self.assertEqual(1, 0, "Expecting Error because database identifiers mismatch.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Database identifiers mismatch: ', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertMessage(contains='ERROR: Database identifiers mismatch: ') # Cleanup src_pg.stop() @@ -793,16 +904,15 @@ def test_tli_destination_mismatch(self): Test that we detect TLI mismatch in destination """ # preparation 1: source - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True, pg_options = { 'wal_log_hints': 'on' } ) src_pg.slow_start() # preparation 2: destination - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -810,7 +920,7 @@ def test_tli_destination_mismatch(self): ) dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) self.set_replica(src_pg, dst_pg) dst_pg.slow_start(replica = True) dst_pg.promote() @@ -818,28 +928,16 @@ def test_tli_destination_mismatch(self): # preparation 3: "useful" changes src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 42 AS answer") - src_query_result = src_pg.table_checksum("ultimate_question") # try catchup - try: - self.catchup_node( + self.pb.catchup_node( backup_mode = 'DELTA', source_pgdata = src_pg.data_dir, destination_node = dst_pg, - options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'], + expect_error="because of stale timeline", ) - dst_options = {} - dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) - dst_pg.slow_start() - dst_query_result = dst_pg.table_checksum("ultimate_question") - dst_pg.stop() - self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') - except ProbackupException as e: - self.assertIn( - 'ERROR: Source is behind destination in timeline history', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertMessage(contains='ERROR: Source is behind destination in timeline history') # Cleanup src_pg.stop() @@ -849,16 +947,15 @@ def test_tli_source_mismatch(self): Test that we detect TLI mismatch in source history """ # preparation 1: source - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True, pg_options = { 'wal_log_hints': 'on' } ) src_pg.slow_start() # preparation 2: fake source (promouted copy) - fake_src_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'fake_src')) - self.catchup_node( + fake_src_pg = self.pg_node.make_empty('fake_src') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = fake_src_pg, @@ -866,7 +963,7 @@ def test_tli_source_mismatch(self): ) fake_src_options = {} fake_src_options['port'] = str(fake_src_pg.port) - self.set_auto_conf(fake_src_pg, fake_src_options) + fake_src_pg.set_auto_conf(fake_src_options) self.set_replica(src_pg, fake_src_pg) fake_src_pg.slow_start(replica = True) fake_src_pg.promote() @@ -881,8 +978,8 @@ def test_tli_source_mismatch(self): fake_src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 'trash' AS garbage") # preparation 3: destination - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -890,34 +987,22 @@ def test_tli_source_mismatch(self): ) dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start() dst_pg.stop() # preparation 4: "useful" changes src_pg.safe_psql("postgres", "CREATE TABLE ultimate_question AS SELECT 42 AS answer") - src_query_result = src_pg.table_checksum("ultimate_question") # try catchup - try: - self.catchup_node( - backup_mode = 'DELTA', - source_pgdata = fake_src_pg.data_dir, - destination_node = dst_pg, - options = ['-d', 'postgres', '-p', str(fake_src_pg.port), '--stream'] - ) - dst_options = {} - dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) - dst_pg.slow_start() - dst_query_result = dst_pg.table_checksum("ultimate_question") - dst_pg.stop() - self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') - except ProbackupException as e: - self.assertIn( - 'ERROR: Destination is not in source timeline history', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.pb.catchup_node( + backup_mode = 'DELTA', + source_pgdata = fake_src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(fake_src_pg.port), '--stream'], + expect_error="because of future timeline", + ) + self.assertMessage(contains='ERROR: Destination is not in source timeline history') # Cleanup src_pg.stop() @@ -931,8 +1016,7 @@ def test_unclean_delta_catchup(self): Test that we correctly recover uncleanly shutdowned destination """ # preparation 1: source - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True, pg_options = { 'wal_log_hints': 'on' } ) @@ -942,8 +1026,8 @@ def test_unclean_delta_catchup(self): "CREATE TABLE ultimate_question(answer int)") # preparation 2: destination - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -951,25 +1035,19 @@ def test_unclean_delta_catchup(self): ) # try #1 - try: - self.catchup_node( + self.pb.catchup_node( backup_mode = 'DELTA', source_pgdata = src_pg.data_dir, destination_node = dst_pg, - options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'], + expect_error="because destination pg is not cleanly shutdowned" ) - self.assertEqual(1, 0, "Expecting Error because destination pg is not cleanly shutdowned.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Destination directory contains "backup_label" file', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertMessage(contains='ERROR: Destination directory contains "backup_label" file') # try #2 dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start() self.assertNotEqual(dst_pg.pid, 0, "Cannot detect pid of running postgres") dst_pg.kill() @@ -982,7 +1060,7 @@ def test_unclean_delta_catchup(self): src_query_result = src_pg.table_checksum("ultimate_question") # do delta catchup - self.catchup_node( + self.pb.catchup_node( backup_mode = 'DELTA', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -1000,7 +1078,7 @@ def test_unclean_delta_catchup(self): self.set_replica(master = src_pg, replica = dst_pg) dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start(replica = True) # 2nd check: run verification query @@ -1018,8 +1096,7 @@ def test_unclean_ptrack_catchup(self): self.skipTest('Skipped because ptrack support is disabled') # preparation 1: source - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True, ptrack_enable = True, pg_options = { 'wal_log_hints': 'on' } @@ -1031,8 +1108,8 @@ def test_unclean_ptrack_catchup(self): "CREATE TABLE ultimate_question(answer int)") # preparation 2: destination - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -1040,25 +1117,19 @@ def test_unclean_ptrack_catchup(self): ) # try #1 - try: - self.catchup_node( + self.pb.catchup_node( backup_mode = 'PTRACK', source_pgdata = src_pg.data_dir, destination_node = dst_pg, - options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream'], + expect_error="because destination pg is not cleanly shutdowned" ) - self.assertEqual(1, 0, "Expecting Error because destination pg is not cleanly shutdowned.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Destination directory contains "backup_label" file', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertMessage(contains='ERROR: Destination directory contains "backup_label" file') # try #2 dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start() self.assertNotEqual(dst_pg.pid, 0, "Cannot detect pid of running postgres") dst_pg.kill() @@ -1071,7 +1142,7 @@ def test_unclean_ptrack_catchup(self): src_query_result = src_pg.table_checksum("ultimate_question") # do delta catchup - self.catchup_node( + self.pb.catchup_node( backup_mode = 'PTRACK', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -1089,7 +1160,7 @@ def test_unclean_ptrack_catchup(self): self.set_replica(master = src_pg, replica = dst_pg) dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start(replica = True) # 2nd check: run verification query @@ -1117,48 +1188,41 @@ def test_catchup_with_replication_slot(self): """ """ # preparation - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True ) src_pg.slow_start() # 1a. --slot option - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_1a')) - try: - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst_1a') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, options = [ '-d', 'postgres', '-p', str(src_pg.port), '--stream', - '--slot=nonexistentslot_1a' - ] + '--slot=nonexistentslot_1a', '--temp-slot=false' + ], + expect_error="because replication slot does not exist" ) - self.assertEqual(1, 0, "Expecting Error because replication slot does not exist.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: replication slot "nonexistentslot_1a" does not exist', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertMessage(contains='ERROR: replication slot "nonexistentslot_1a" does not exist') # 1b. --slot option - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_1b')) + dst_pg = self.pg_node.make_empty('dst_1b') src_pg.safe_psql("postgres", "SELECT pg_catalog.pg_create_physical_replication_slot('existentslot_1b')") - self.catchup_node( + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, options = [ '-d', 'postgres', '-p', str(src_pg.port), '--stream', - '--slot=existentslot_1b' + '--slot=existentslot_1b', '--temp-slot=false' ] ) # 2a. --slot --perm-slot - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_2a')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst_2a') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -1170,10 +1234,9 @@ def test_catchup_with_replication_slot(self): ) # 2b. and 4. --slot --perm-slot - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_2b')) + dst_pg = self.pg_node.make_empty('dst_2b') src_pg.safe_psql("postgres", "SELECT pg_catalog.pg_create_physical_replication_slot('existentslot_2b')") - try: - self.catchup_node( + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -1181,19 +1244,14 @@ def test_catchup_with_replication_slot(self): '-d', 'postgres', '-p', str(src_pg.port), '--stream', '--slot=existentslot_2b', '--perm-slot' - ] + ], + expect_error="because replication slot already exist" ) - self.assertEqual(1, 0, "Expecting Error because replication slot already exist.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: replication slot "existentslot_2b" already exists', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.assertMessage(contains='ERROR: replication slot "existentslot_2b" already exists') # 3. --perm-slot --slot - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_3')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst_3') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -1210,29 +1268,47 @@ def test_catchup_with_replication_slot(self): ).decode('utf-8').rstrip() self.assertEqual(slot_name, 'pg_probackup_perm_slot', 'Slot name mismatch') - # 5. --perm-slot --temp-slot (PG>=10) - if self.get_version(src_pg) >= self.version_to_num('10.0'): - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst_5')) - try: - self.catchup_node( - backup_mode = 'FULL', - source_pgdata = src_pg.data_dir, - destination_node = dst_pg, - options = [ - '-d', 'postgres', '-p', str(src_pg.port), '--stream', - '--perm-slot', - '--temp-slot' - ] - ) - self.assertEqual(1, 0, "Expecting Error because conflicting options --perm-slot and --temp-slot used together\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: You cannot specify "--perm-slot" option with the "--temp-slot" option', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + # 5. --perm-slot --temp-slot + dst_pg = self.pg_node.make_empty('dst_5a') + self.pb.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--perm-slot', + '--temp-slot' + ], + expect_error="because conflicting options --perm-slot and --temp-slot used together" + ) + self.assertMessage(contains='ERROR: You cannot specify "--perm-slot" option with the "--temp-slot" option') + + dst_pg = self.pg_node.make_empty('dst_5b') + self.pb.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--perm-slot', + '--temp-slot=true' + ], + expect_error="because conflicting options --perm-slot and --temp-slot used together" + ) + self.assertMessage(contains='ERROR: You cannot specify "--perm-slot" option with the "--temp-slot" option') - #self.assertEqual(1, 0, 'Stop test') + dst_pg = self.pg_node.make_empty('dst_5c') + self.pb.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--perm-slot', + '--temp-slot=false', + '--slot=dst_5c' + ], + ) ######################################### # --exclude-path @@ -1242,8 +1318,7 @@ def test_catchup_with_exclude_path(self): various syntetic tests for --exclude-path option """ # preparation - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True ) src_pg.slow_start() @@ -1260,8 +1335,8 @@ def test_catchup_with_exclude_path(self): f.flush() f.close - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -1279,7 +1354,7 @@ def test_catchup_with_exclude_path(self): self.set_replica(src_pg, dst_pg) dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start(replica = True) dst_pg.stop() @@ -1291,7 +1366,7 @@ def test_catchup_with_exclude_path(self): f.flush() f.close - self.catchup_node( + self.pb.catchup_node( backup_mode = 'DELTA', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -1317,8 +1392,7 @@ def test_config_exclusion(self): Test that catchup can preserve dest replication config """ # preparation 1: source - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True, pg_options = { 'wal_log_hints': 'on' } ) @@ -1328,8 +1402,8 @@ def test_config_exclusion(self): "CREATE TABLE ultimate_question(answer int)") # preparation 2: make lagging behind replica - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -1338,7 +1412,7 @@ def test_config_exclusion(self): self.set_replica(src_pg, dst_pg) dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg._assign_master(src_pg) dst_pg.slow_start(replica = True) dst_pg.stop() @@ -1349,7 +1423,7 @@ def test_config_exclusion(self): pgbench.wait() # test 1: do delta catchup with relative exclusion paths - self.catchup_node( + self.pb.catchup_node( backup_mode = 'DELTA', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -1379,7 +1453,7 @@ def test_config_exclusion(self): pgbench.wait() # test 2: do delta catchup with absolute source exclusion paths - self.catchup_node( + self.pb.catchup_node( backup_mode = 'DELTA', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -1408,7 +1482,7 @@ def test_config_exclusion(self): pgbench.wait() # test 3: do delta catchup with absolute destination exclusion paths - self.catchup_node( + self.pb.catchup_node( backup_mode = 'DELTA', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -1444,14 +1518,13 @@ def test_dry_run_catchup_full(self): Test dry-run option for full catchup """ # preparation 1: source - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True ) src_pg.slow_start() # preparation 2: make clean shutdowned lagging behind replica - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) + dst_pg = self.pg_node.make_empty('dst') src_pg.pgbench_init(scale = 10) pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) @@ -1461,7 +1534,7 @@ def test_dry_run_catchup_full(self): content_before = self.pgdata_content(dst_pg.data_dir) # do full catchup - self.catchup_node( + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -1485,12 +1558,9 @@ def test_dry_run_catchup_ptrack(self): self.skipTest('Skipped because ptrack support is disabled') # preparation 1: source - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True, - ptrack_enable = True, - initdb_params = ['--data-checksums'] - ) + ptrack_enable = True) src_pg.slow_start() src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack") @@ -1499,8 +1569,8 @@ def test_dry_run_catchup_ptrack(self): pgbench.wait() # preparation 2: make clean shutdowned lagging behind replica - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -1509,7 +1579,7 @@ def test_dry_run_catchup_ptrack(self): self.set_replica(src_pg, dst_pg) dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start(replica = True) dst_pg.stop() @@ -1517,7 +1587,7 @@ def test_dry_run_catchup_ptrack(self): content_before = self.pgdata_content(dst_pg.data_dir) # do incremental catchup - self.catchup_node( + self.pb.catchup_node( backup_mode = 'PTRACK', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -1539,10 +1609,8 @@ def test_dry_run_catchup_delta(self): """ # preparation 1: source - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication = True, - initdb_params = ['--data-checksums'], pg_options = { 'wal_log_hints': 'on' } ) src_pg.slow_start() @@ -1552,8 +1620,8 @@ def test_dry_run_catchup_delta(self): pgbench.wait() # preparation 2: make clean shutdowned lagging behind replica - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -1562,7 +1630,7 @@ def test_dry_run_catchup_delta(self): self.set_replica(src_pg, dst_pg) dst_options = {} dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start(replica = True) dst_pg.stop() @@ -1570,7 +1638,7 @@ def test_dry_run_catchup_delta(self): content_before = self.pgdata_content(dst_pg.data_dir) # do delta catchup - self.catchup_node( + self.pb.catchup_node( backup_mode = 'DELTA', source_pgdata = src_pg.data_dir, destination_node = dst_pg, @@ -1591,14 +1659,14 @@ def test_pgdata_is_ignored(self): or from the env var. This test that PGDATA is actually ignored and --source-pgadta is used instead """ - node = self.make_simple_node('node', + node = self.pg_node.make_simple('node', set_replication = True ) node.slow_start() # do full catchup - dest = self.make_empty_node('dst') - self.catchup_node( + dest = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = node.data_dir, destination_node = dest, @@ -1610,10 +1678,10 @@ def test_pgdata_is_ignored(self): self.pgdata_content(dest.data_dir) ) - os.environ['PGDATA']='xxx' + self.test_env['PGDATA']='xxx' - dest2 = self.make_empty_node('dst') - self.catchup_node( + dest2 = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode = 'FULL', source_pgdata = node.data_dir, destination_node = dest2, @@ -1624,3 +1692,371 @@ def test_pgdata_is_ignored(self): self.pgdata_content(node.data_dir), self.pgdata_content(dest2.data_dir) ) + + def test_catchup_from_standby_single_wal(self): + """ Make a standby node, with a single WAL file in it and try to catchup """ + node = self.pg_node.make_simple('node', + pg_options={'hot_standby': 'on'}) + node.set_auto_conf({}, 'postgresql.conf', ['max_worker_processes']) + standby_signal = os.path.join(node.data_dir, 'standby.signal') + with open(standby_signal, 'w') as fout: + fout.flush() + fout.close() + node.start() + + # No inserts to keep WAL size small + + dest = self.pg_node.make_empty('dst') + + self.pb.catchup_node( + backup_mode='FULL', + source_pgdata=node.data_dir, + destination_node=dest, + options = ['-d', 'postgres', '-p', str(node.port), '--stream'] + ) + + dst_options = {} + dst_options['port'] = str(dest.port) + dest.set_auto_conf(dst_options) + + dest.slow_start() + res = dest.safe_psql("postgres", "select 1").decode('utf-8').strip() + self.assertEqual(res, "1") + + def test_catchup_ptrack_unlogged(self): + """ catchup + ptrack when unlogged tables exist """ + node = self.pg_node.make_simple('node', ptrack_enable = True) + node.slow_start() + node.safe_psql("postgres", "CREATE EXTENSION ptrack") + + dest = self.pg_node.make_empty('dst') + + self.pb.catchup_node( + backup_mode='FULL', + source_pgdata=node.data_dir, + destination_node=dest, + options = ['-d', 'postgres', '-p', str(node.port), '--stream'] + ) + + for i in range(1,7): + node.safe_psql('postgres', 'create unlogged table t' + str(i) + ' (id int, name text);') + + dst_options = {} + dst_options['port'] = str(dest.port) + dest.set_auto_conf(dst_options) + + dest.slow_start() + dest.stop() + + self.pb.catchup_node( + backup_mode = 'PTRACK', + source_pgdata = node.data_dir, + destination_node = dest, + options = ['-d', 'postgres', '-p', str(node.port), '--stream', '--dry-run'] + ) + + return + + def test_catchup_instance_from_the_past(self): + src_pg = self.pg_node.make_simple('src', + set_replication=True + ) + src_pg.slow_start() + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( + backup_mode='FULL', + source_pgdata=src_pg.data_dir, + destination_node=dst_pg, + options=['-d', 'postgres', '-p', str(src_pg.port), '--stream'] + ) + dst_options = {'port': str(dst_pg.port)} + dst_pg.set_auto_conf(dst_options) + dst_pg.slow_start() + dst_pg.pgbench_init(scale=10) + pgbench = dst_pg.pgbench( + stdout=subprocess.PIPE, + options=["-c", "4", "-T", "20"]) + pgbench.wait() + pgbench.stdout.close() + dst_pg.stop() + self.pb.catchup_node( + backup_mode='DELTA', + source_pgdata=src_pg.data_dir, + destination_node=dst_pg, + options=[ + '-d', 'postgres', + '-p', str(src_pg.port), + '--stream' + ], + expect_error="because instance is from the past" + ) + + self.assertMessage(regex='ERROR: Current START LSN .* is lower than SYNC LSN') + self.assertMessage(contains='it may indicate that we are trying to catchup ' + 'with PostgreSQL instance from the past') + + +######################################### +# --waldir +######################################### + + def test_waldir_option(self): + """ + Test waldir option for full catchup + """ + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + # preparation: source + src_pg = self.pg_node.make_simple('src', + set_replication = True, + ptrack_enable = True + ) + src_pg.slow_start() + src_pg.safe_psql("postgres", "CREATE EXTENSION ptrack") + + # do full catchup + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = [ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--waldir={0}'.format(os.path.join(self.test_path, 'tmp_new_wal_dir')), + ] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # 2nd check new waldir exists + self.assertTrue(Path(os.path.join(self.test_path, 'tmp_new_wal_dir')).exists()) + + #3rd check pg_wal is symlink + if src_pg.major_version >= 10: + wal_path = os.path.join(dst_pg.data_dir, "pg_wal") + else: + wal_path = os.path.join(dst_pg.data_dir, "pg_xlog") + + self.assertEqual(os.path.islink(wal_path), True) + print("FULL DONE ----------------------------------------------------------") + + """ + Test waldir otion for delta catchup to different directory from full catchup's wal directory + """ + # preparation 2: make clean shutdowned lagging behind replica + + self.set_replica(src_pg, dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + dst_pg.set_auto_conf(dst_options) + dst_pg.slow_start(replica=True) + dst_pg.stop() + + # do delta catchup + self.pb.catchup_node( + backup_mode='DELTA', + source_pgdata=src_pg.data_dir, + destination_node=dst_pg, + options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--waldir={0}'.format(os.path.join(self.test_path, 'tmp_another_wal_dir')), + ], + expect_error="because we perform DELTA catchup's WAL in a different dir from FULL catchup's WAL dir", + ) + self.assertMessage(contains='ERROR: WAL directory does not egual to symlinked pg_wal path') + + print("ANOTHER DIR DONE ------------------------------------------------") + + """ + Test waldir otion to delta catchup + """ + + self.set_replica(src_pg, dst_pg) + dst_pg._assign_master(src_pg) + dst_pg.slow_start(replica = True) + dst_pg.stop() + + # preparation 3: make changes on master (source) + src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '2', '--no-vacuum']) + pgbench.wait() + + # do delta catchup + self.pb.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--waldir={0}'.format(os.path.join(self.test_path, 'tmp_new_wal_dir')), + ], + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # 2nd check new waldir exists + self.assertTrue(Path(os.path.join(self.test_path, 'tmp_new_wal_dir')).exists()) + + #3rd check pg_wal is symlink + if src_pg.major_version >= 10: + wal_path = os.path.join(dst_pg.data_dir, "pg_wal") + else: + wal_path = os.path.join(dst_pg.data_dir, "pg_xlog") + + self.assertEqual(os.path.islink(wal_path), True) + + print ("DELTA DONE---------------------------------------------------------") + + """ + Test waldir option for catchup in incremental ptrack mode + """ + self.set_replica(src_pg, dst_pg) + dst_pg.slow_start(replica = True) + dst_pg.stop() + + # preparation 3: make changes on master (source) + src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '2', '--no-vacuum']) + pgbench.wait() + + # do incremental catchup + self.pb.catchup_node( + backup_mode = 'PTRACK', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--waldir={0}'.format(os.path.join(self.test_path, 'tmp_new_wal_dir')), + ] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir), + self.pgdata_content(dst_pg.data_dir) + ) + + # 2nd check new waldir exists + self.assertTrue(Path(os.path.join(self.test_path, 'tmp_new_wal_dir')).exists()) + + #3rd check pg_wal is symlink + if src_pg.major_version >= 10: + wal_path = os.path.join(dst_pg.data_dir, "pg_wal") + else: + wal_path = os.path.join(dst_pg.data_dir, "pg_xlog") + + self.assertEqual(os.path.islink(wal_path), True) + + print ("PTRACK DONE -----------------------------------------------------------") + + """ + Test waldir option for full catchup to not empty WAL directory + """ + + dst_pg = self.pg_node.make_empty('dst2') + self.pb.catchup_node( + backup_mode='FULL', + source_pgdata=src_pg.data_dir, + destination_node=dst_pg, + options=[ + '-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--waldir={0}'.format(os.path.join(self.test_path, 'tmp_new_wal_dir')), + ], + + expect_error="because full catchup's WAL must be perform into empty directory", + ) + self.assertMessage(contains='ERROR: Can\'t perform FULL catchup with non-empty pg_wal directory') + + print ("ANOTHER FULL DONE -----------------------------------------------------") + # Cleanup + src_pg.stop() + + + def test_waldir_delta_catchup_without_full(self): + """ + Test waldir otion with delta catchup without using it doing full + """ + # preparation 1: source + src_pg = self.pg_node.make_simple('src', + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream', + ], + ) + self.set_replica(src_pg, dst_pg) + dst_options = {} + dst_options['port'] = str(dst_pg.port) + dst_pg.set_auto_conf(dst_options) + dst_pg.slow_start(replica = True) + dst_pg.stop() + + # do delta catchup + self.pb.catchup_node( + backup_mode = 'DELTA', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream', + '--waldir={0}'.format(os.path.join(self.test_path, 'tmp_new_wal_dir')), + ], + expect_error="because we didn't perform FULL catchup's WAL before DELTA catchup", + ) + self.assertMessage(contains='ERROR: Unable to read pg_wal symbolic link') + + # Cleanup + src_pg.stop() + + + def test_waldir_dry_run_catchup_full(self): + """ + Test waldir with dry-run option for full catchup + """ + # preparation 1: source + src_pg = self.pg_node.make_simple('src', + set_replication = True, + pg_options = { 'wal_log_hints': 'on' } + ) + src_pg.slow_start() + + # preparation 2: make clean shutdowned lagging behind replica + dst_pg = self.pg_node.make_empty('dst') + + src_pg.pgbench_init(scale = 10) + pgbench = src_pg.pgbench(options=['-T', '10', '--no-vacuum']) + pgbench.wait() + + # save the condition before dry-run + content_before = self.pgdata_content(dst_pg.data_dir) + + # do full catchup + self.pb.catchup_node( + backup_mode = 'FULL', + source_pgdata = src_pg.data_dir, + destination_node = dst_pg, + options = ['-d', 'postgres', '-p', str(src_pg.port), '--stream', '--dry-run', + '--waldir={0}'.format(os.path.join(self.test_path, 'tmp_new_wal_dir')), + ] + ) + + # compare data dirs before and after catchup + self.compare_pgdata( + content_before, + self.pgdata_content(dst_pg.data_dir) + ) + self.assertFalse(Path(os.path.join(self.test_path, 'tmp_new_wal_dir')).exists()) + # Cleanup + src_pg.stop() + diff --git a/tests/cfs_backup_test.py b/tests/cfs_backup_test.py deleted file mode 100644 index fb4a6c6b8..000000000 --- a/tests/cfs_backup_test.py +++ /dev/null @@ -1,1216 +0,0 @@ -import os -import unittest -import random -import shutil - -from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - -tblspace_name = 'cfs_tblspace' - - -class CfsBackupNoEncTest(ProbackupTest, unittest.TestCase): - # --- Begin --- # - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def setUp(self): - self.backup_dir = os.path.join( - self.tmp_path, self.module_name, self.fname, 'backup') - self.node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'cfs_encryption': 'off', - 'max_wal_senders': '2', - 'shared_buffers': '200MB' - } - ) - - self.init_pb(self.backup_dir) - self.add_instance(self.backup_dir, 'node', self.node) - self.set_archiving(self.backup_dir, 'node', self.node) - - self.node.slow_start() - - self.node.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") - - self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) - - tblspace = self.node.safe_psql( - "postgres", - "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format( - tblspace_name)) - - self.assertIn( - tblspace_name, str(tblspace), - "ERROR: The tablespace not created " - "or it create without compressions") - - self.assertIn( - "compression=true", str(tblspace), - "ERROR: The tablespace not created " - "or it create without compressions") - - self.assertTrue( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression']), - "ERROR: File pg_compression not found" - ) - - # --- Section: Full --- # - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_empty_tablespace(self): - """Case: Check fullbackup empty compressed tablespace""" - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_empty_tablespace_stream(self): - """Case: Check fullbackup empty compressed tablespace with options stream""" - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - # PGPRO-1018 invalid file size - def test_fullbackup_after_create_table(self): - """Case: Make full backup after created table in the tablespace""" - if not self.enterprise: - return - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "\n ERROR: {0}\n CMD: {1}".format( - repr(e.message), - repr(self.cmd) - ) - ) - return False - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['pg_compression']), - "ERROR: File pg_compression not found in {0}".format( - os.path.join(self.backup_dir, 'node', backup_id)) - ) - - # check cfm size - cfms = find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']) - self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir") - for cfm in cfms: - size = os.stat(cfm).st_size - self.assertLessEqual(size, 4096, - "ERROR: {0} is not truncated (has size {1} > 4096)".format( - cfm, size - )) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - # PGPRO-1018 invalid file size - def test_fullbackup_after_create_table_stream(self): - """ - Case: Make full backup after created table in the tablespace with option --stream - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Full backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) - - # --- Section: Incremental from empty tablespace --- # - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_empty_tablespace_ptrack_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table - """ - - try: - self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='ptrack') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_empty_tablespace_ptrack_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table - """ - - try: - self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='ptrack', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) - self.assertFalse( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['_ptrack']), - "ERROR: _ptrack files was found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_empty_tablespace_page_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make page backup after create table - """ - - try: - self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='page') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) - - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_page_doesnt_store_unchanged_cfm(self): - """ - Case: Test page backup doesn't store cfm file if table were not modified - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - self.node.safe_psql("postgres", "checkpoint") - - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id_full)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) - - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='page') - - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertFalse( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files is found in backup dir" - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_empty_tablespace_page_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace. - Make page backup after create table - """ - - try: - self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - backup_id = None - try: - backup_id = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='page', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - show_backup = self.show_pb(self.backup_dir, 'node', backup_id) - self.assertEqual( - "OK", - show_backup["status"], - "ERROR: Incremental backup status is not valid. \n " - "Current backup status={0}".format(show_backup["status"]) - ) - self.assertTrue( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression']), - "ERROR: File pg_compression not found" - ) - self.assertTrue( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['.cfm']), - "ERROR: .cfm files not found in backup dir" - ) - self.assertFalse( - find_by_extensions( - [os.path.join(self.backup_dir, 'backups', 'node', backup_id)], - ['_ptrack']), - "ERROR: _ptrack files was found in backup dir" - ) - - # --- Section: Incremental from fill tablespace --- # - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_after_create_table_ptrack_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table. - Check: incremental backup will not greater as full - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,10) i".format('t2', tblspace_name) - ) - - backup_id_ptrack = None - try: - backup_id_ptrack = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='ptrack') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb( - self.backup_dir, 'node', backup_id_full) - show_backup_ptrack = self.show_pb( - self.backup_dir, 'node', backup_id_ptrack) - self.assertGreater( - show_backup_full["data-bytes"], - show_backup_ptrack["data-bytes"], - "ERROR: Size of incremental backup greater than full. \n " - "INFO: {0} >{1}".format( - show_backup_ptrack["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_after_create_table_ptrack_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace(--stream). - Make ptrack backup after create table(--stream). - Check: incremental backup size should not be greater than full - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,25) i".format('t2', tblspace_name) - ) - - backup_id_ptrack = None - try: - backup_id_ptrack = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='ptrack', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb( - self.backup_dir, 'node', backup_id_full) - show_backup_ptrack = self.show_pb( - self.backup_dir, 'node', backup_id_ptrack) - self.assertGreater( - show_backup_full["data-bytes"], - show_backup_ptrack["data-bytes"], - "ERROR: Size of incremental backup greater than full. \n " - "INFO: {0} >{1}".format( - show_backup_ptrack["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_after_create_table_page_after_create_table(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table. - Check: incremental backup size should not be greater than full - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,10) i".format('t2', tblspace_name) - ) - - backup_id_page = None - try: - backup_id_page = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='page') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb( - self.backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb( - self.backup_dir, 'node', backup_id_page) - self.assertGreater( - show_backup_full["data-bytes"], - show_backup_page["data-bytes"], - "ERROR: Size of incremental backup greater than full. \n " - "INFO: {0} >{1}".format( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_multiple_segments(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table. - Check: incremental backup will not greater as full - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format( - 't_heap', tblspace_name) - ) - - full_result = self.node.table_checksum("t_heap") - - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "INSERT INTO {0} " - "SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,10) i".format( - 't_heap') - ) - - page_result = self.node.table_checksum("t_heap") - - try: - backup_id_page = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='page') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb( - self.backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb( - self.backup_dir, 'node', backup_id_page) - self.assertGreater( - show_backup_full["data-bytes"], - show_backup_page["data-bytes"], - "ERROR: Size of incremental backup greater than full. \n " - "INFO: {0} >{1}".format( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # CHECK FULL BACKUP - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - self.restore_node( - self.backup_dir, 'node', self.node, backup_id=backup_id_full, - options=[ - "-j", "4", - "--recovery-target=immediate", - "--recovery-target-action=promote"]) - - self.node.slow_start() - self.assertEqual( - full_result, - self.node.table_checksum("t_heap"), - 'Lost data after restore') - - # CHECK PAGE BACKUP - self.node.stop() - self.node.cleanup() - shutil.rmtree( - self.get_tblspace_path(self.node, tblspace_name), - ignore_errors=True) - self.restore_node( - self.backup_dir, 'node', self.node, backup_id=backup_id_page, - options=[ - "-j", "4", - "--recovery-target=immediate", - "--recovery-target-action=promote"]) - - self.node.slow_start() - self.assertEqual( - page_result, - self.node.table_checksum("t_heap"), - 'Lost data after restore') - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_multiple_segments_in_multiple_tablespaces(self): - """ - Case: Make full backup before created table in the tablespace. - Make ptrack backup after create table. - Check: incremental backup will not greater as full - """ - tblspace_name_1 = 'tblspace_name_1' - tblspace_name_2 = 'tblspace_name_2' - - self.create_tblspace_in_node(self.node, tblspace_name_1, cfs=True) - self.create_tblspace_in_node(self.node, tblspace_name_2, cfs=True) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format( - 't_heap_1', tblspace_name_1)) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format( - 't_heap_2', tblspace_name_2)) - - full_result_1 = self.node.table_checksum("t_heap_1") - full_result_2 = self.node.table_checksum("t_heap_2") - - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "INSERT INTO {0} " - "SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,10) i".format( - 't_heap_1') - ) - - self.node.safe_psql( - "postgres", - "INSERT INTO {0} " - "SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,10) i".format( - 't_heap_2') - ) - - page_result_1 = self.node.table_checksum("t_heap_1") - page_result_2 = self.node.table_checksum("t_heap_2") - - try: - backup_id_page = self.backup_node( - self.backup_dir, 'node', self.node, backup_type='page') - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb( - self.backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb( - self.backup_dir, 'node', backup_id_page) - self.assertGreater( - show_backup_full["data-bytes"], - show_backup_page["data-bytes"], - "ERROR: Size of incremental backup greater than full. \n " - "INFO: {0} >{1}".format( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # CHECK FULL BACKUP - self.node.stop() - - self.restore_node( - self.backup_dir, 'node', self.node, - backup_id=backup_id_full, - options=[ - "-j", "4", "--incremental-mode=checksum", - "--recovery-target=immediate", - "--recovery-target-action=promote"]) - self.node.slow_start() - - self.assertEqual( - full_result_1, - self.node.table_checksum("t_heap_1"), - 'Lost data after restore') - self.assertEqual( - full_result_2, - self.node.table_checksum("t_heap_2"), - 'Lost data after restore') - - # CHECK PAGE BACKUP - self.node.stop() - - self.restore_node( - self.backup_dir, 'node', self.node, - backup_id=backup_id_page, - options=[ - "-j", "4", "--incremental-mode=checksum", - "--recovery-target=immediate", - "--recovery-target-action=promote"]) - self.node.slow_start() - - self.assertEqual( - page_result_1, - self.node.table_checksum("t_heap_1"), - 'Lost data after restore') - self.assertEqual( - page_result_2, - self.node.table_checksum("t_heap_2"), - 'Lost data after restore') - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_fullbackup_after_create_table_page_after_create_table_stream(self): - """ - Case: Make full backup before created table in the tablespace(--stream). - Make ptrack backup after create table(--stream). - Check: incremental backup will not greater as full - """ - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,1005000) i".format('t1', tblspace_name) - ) - - backup_id_full = None - try: - backup_id_full = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='full', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,10) i".format('t2', tblspace_name) - ) - - backup_id_page = None - try: - backup_id_page = self.backup_node( - self.backup_dir, 'node', self.node, - backup_type='page', options=['--stream']) - except ProbackupException as e: - self.fail( - "ERROR: Incremental backup failed.\n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - show_backup_full = self.show_pb( - self.backup_dir, 'node', backup_id_full) - show_backup_page = self.show_pb( - self.backup_dir, 'node', backup_id_page) - self.assertGreater( - show_backup_full["data-bytes"], - show_backup_page["data-bytes"], - "ERROR: Size of incremental backup greater than full. \n " - "INFO: {0} >{1}".format( - show_backup_page["data-bytes"], - show_backup_full["data-bytes"] - ) - ) - - # --- Make backup with not valid data(broken .cfm) --- # - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_delete_random_cfm_file_from_tablespace_dir(self): - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - self.node.safe_psql( - "postgres", - "CHECKPOINT" - ) - - list_cmf = find_by_extensions( - [self.get_tblspace_path(self.node, tblspace_name)], - ['.cfm']) - self.assertTrue( - list_cmf, - "ERROR: .cfm-files not found into tablespace dir" - ) - - os.remove(random.choice(list_cmf)) - - self.assertRaises( - ProbackupException, - self.backup_node, - self.backup_dir, - 'node', - self.node, - backup_type='full' - ) - - @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_delete_file_pg_compression_from_tablespace_dir(self): - os.remove( - find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression'])[0]) - - self.assertRaises( - ProbackupException, - self.backup_node, - self.backup_dir, - 'node', - self.node, - backup_type='full' - ) - - @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_delete_random_data_file_from_tablespace_dir(self): - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - self.node.safe_psql( - "postgres", - "CHECKPOINT" - ) - - list_data_files = find_by_pattern( - [self.get_tblspace_path(self.node, tblspace_name)], - '^.*/\d+$') - self.assertTrue( - list_data_files, - "ERROR: Files of data not found into tablespace dir" - ) - - os.remove(random.choice(list_data_files)) - - self.assertRaises( - ProbackupException, - self.backup_node, - self.backup_dir, - 'node', - self.node, - backup_type='full' - ) - - @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_broken_random_cfm_file_into_tablespace_dir(self): - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - list_cmf = find_by_extensions( - [self.get_tblspace_path(self.node, tblspace_name)], - ['.cfm']) - self.assertTrue( - list_cmf, - "ERROR: .cfm-files not found into tablespace dir" - ) - - corrupt_file(random.choice(list_cmf)) - - self.assertRaises( - ProbackupException, - self.backup_node, - self.backup_dir, - 'node', - self.node, - backup_type='full' - ) - - @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_broken_random_data_file_into_tablespace_dir(self): - self.node.safe_psql( - "postgres", - "CREATE TABLE {0} TABLESPACE {1} " - "AS SELECT i AS id, MD5(i::text) AS text, " - "MD5(repeat(i::text,10))::tsvector AS tsvector " - "FROM generate_series(0,256) i".format('t1', tblspace_name) - ) - - list_data_files = find_by_pattern( - [self.get_tblspace_path(self.node, tblspace_name)], - '^.*/\d+$') - self.assertTrue( - list_data_files, - "ERROR: Files of data not found into tablespace dir" - ) - - corrupt_file(random.choice(list_data_files)) - - self.assertRaises( - ProbackupException, - self.backup_node, - self.backup_dir, - 'node', - self.node, - backup_type='full' - ) - - @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_broken_file_pg_compression_into_tablespace_dir(self): - - corrupted_file = find_by_name( - [self.get_tblspace_path(self.node, tblspace_name)], - ['pg_compression'])[0] - - self.assertTrue( - corrupt_file(corrupted_file), - "ERROR: File is not corrupted or it missing" - ) - - self.assertRaises( - ProbackupException, - self.backup_node, - self.backup_dir, - 'node', - self.node, - backup_type='full' - ) - -# # --- End ---# - - -#class CfsBackupEncTest(CfsBackupNoEncTest): -# # --- Begin --- # -# def setUp(self): -# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" -# super(CfsBackupEncTest, self).setUp() diff --git a/tests/cfs_catchup_test.py b/tests/cfs_catchup_test.py deleted file mode 100644 index f6760b72c..000000000 --- a/tests/cfs_catchup_test.py +++ /dev/null @@ -1,117 +0,0 @@ -import os -import unittest -import random -import shutil - -from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - - -class CfsCatchupNoEncTest(ProbackupTest, unittest.TestCase): - - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_full_catchup_with_tablespace(self): - """ - Test tablespace transfers - """ - # preparation - src_pg = self.make_simple_node( - base_dir = os.path.join(self.module_name, self.fname, 'src'), - set_replication = True - ) - src_pg.slow_start() - tblspace1_old_path = self.get_tblspace_path(src_pg, 'tblspace1_old') - self.create_tblspace_in_node(src_pg, 'tblspace1', tblspc_path = tblspace1_old_path, cfs=True) - src_pg.safe_psql( - "postgres", - "CREATE TABLE ultimate_question TABLESPACE tblspace1 AS SELECT 42 AS answer") - src_query_result = src_pg.table_checksum("ultimate_question") - src_pg.safe_psql( - "postgres", - "CHECKPOINT") - - # do full catchup with tablespace mapping - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - tblspace1_new_path = self.get_tblspace_path(dst_pg, 'tblspace1_new') - self.catchup_node( - backup_mode = 'FULL', - source_pgdata = src_pg.data_dir, - destination_node = dst_pg, - options = [ - '-d', 'postgres', - '-p', str(src_pg.port), - '--stream', - '-T', '{0}={1}'.format(tblspace1_old_path, tblspace1_new_path) - ] - ) - - # 1st check: compare data directories - self.compare_pgdata( - self.pgdata_content(src_pg.data_dir), - self.pgdata_content(dst_pg.data_dir) - ) - - # check cfm size - cfms = find_by_extensions([os.path.join(dst_pg.data_dir)], ['.cfm']) - self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir") - for cfm in cfms: - size = os.stat(cfm).st_size - self.assertLessEqual(size, 4096, - "ERROR: {0} is not truncated (has size {1} > 4096)".format( - cfm, size - )) - - # make changes in master tablespace - src_pg.safe_psql( - "postgres", - "UPDATE ultimate_question SET answer = -1") - src_pg.safe_psql( - "postgres", - "CHECKPOINT") - - # run&recover catchup'ed instance - dst_options = {} - dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) - dst_pg.slow_start() - - # 2nd check: run verification query - dst_query_result = dst_pg.table_checksum("ultimate_question") - self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') - - # and now delta backup - dst_pg.stop() - - self.catchup_node( - backup_mode = 'DELTA', - source_pgdata = src_pg.data_dir, - destination_node = dst_pg, - options = [ - '-d', 'postgres', - '-p', str(src_pg.port), - '--stream', - '-T', '{0}={1}'.format(tblspace1_old_path, tblspace1_new_path) - ] - ) - - # check cfm size again - cfms = find_by_extensions([os.path.join(dst_pg.data_dir)], ['.cfm']) - self.assertTrue(cfms, "ERROR: .cfm files not found in backup dir") - for cfm in cfms: - size = os.stat(cfm).st_size - self.assertLessEqual(size, 4096, - "ERROR: {0} is not truncated (has size {1} > 4096)".format( - cfm, size - )) - - # run&recover catchup'ed instance - dst_options = {} - dst_options['port'] = str(dst_pg.port) - self.set_auto_conf(dst_pg, dst_options) - dst_pg.slow_start() - - - # 3rd check: run verification query - src_query_result = src_pg.table_checksum("ultimate_question") - dst_query_result = dst_pg.table_checksum("ultimate_question") - self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') diff --git a/tests/cfs_restore_test.py b/tests/cfs_restore_test.py deleted file mode 100644 index 2fa35e71a..000000000 --- a/tests/cfs_restore_test.py +++ /dev/null @@ -1,447 +0,0 @@ -""" -restore - Syntax: - - pg_probackup restore -B backupdir --instance instance_name - [-D datadir] - [ -i backup_id | [{--time=time | --xid=xid | --lsn=lsn } [--inclusive=boolean]]][--timeline=timeline] [-T OLDDIR=NEWDIR] - [-j num_threads] [--progress] [-q] [-v] - -""" -import os -import unittest -import shutil - -from .helpers.cfs_helpers import find_by_name -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - -tblspace_name = 'cfs_tblspace' -tblspace_name_new = 'cfs_tblspace_new' - - -class CfsRestoreBase(ProbackupTest, unittest.TestCase): - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def setUp(self): - self.backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - - self.node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ -# 'ptrack_enable': 'on', - 'cfs_encryption': 'off', - } - ) - - self.init_pb(self.backup_dir) - self.add_instance(self.backup_dir, 'node', self.node) - self.set_archiving(self.backup_dir, 'node', self.node) - - self.node.slow_start() - self.create_tblspace_in_node(self.node, tblspace_name, cfs=True) - - self.add_data_in_cluster() - - self.backup_id = None - try: - self.backup_id = self.backup_node(self.backup_dir, 'node', self.node, backup_type='full') - except ProbackupException as e: - self.fail( - "ERROR: Full backup failed \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - def add_data_in_cluster(self): - pass - - -class CfsRestoreNoencEmptyTablespaceTest(CfsRestoreBase): - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_empty_tablespace_from_fullbackup(self): - """ - Case: Restore empty tablespace from valid full backup. - """ - self.node.stop(["-m", "immediate"]) - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - try: - self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) - except ProbackupException as e: - self.fail( - "ERROR: Restore failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ["pg_compression"]), - "ERROR: Restored data is not valid. pg_compression not found in tablespace dir." - ) - - try: - self.node.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - tblspace = self.node.safe_psql( - "postgres", - "SELECT * FROM pg_tablespace WHERE spcname='{0}'".format(tblspace_name) - ).decode("UTF-8") - self.assertTrue( - tblspace_name in tblspace and "compression=true" in tblspace, - "ERROR: The tablespace not restored or it restored without compressions" - ) - - -class CfsRestoreNoencTest(CfsRestoreBase): - def add_data_in_cluster(self): - self.node.safe_psql( - "postgres", - 'CREATE TABLE {0} TABLESPACE {1} \ - AS SELECT i AS id, MD5(i::text) AS text, \ - MD5(repeat(i::text,10))::tsvector AS tsvector \ - FROM generate_series(0,1e5) i'.format('t1', tblspace_name) - ) - self.table_t1 = self.node.table_checksum("t1") - - # --- Restore from full backup ---# - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_from_fullbackup_to_old_location(self): - """ - Case: Restore instance from valid full backup to old location. - """ - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - try: - self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id) - except ProbackupException as e: - self.fail( - "ERROR: Restore from full backup failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found in tablespace dir" - ) - try: - self.node.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertEqual( - self.node.table_checksum("t1"), - self.table_t1 - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_from_fullbackup_to_old_location_3_jobs(self): - """ - Case: Restore instance from valid full backup to old location. - """ - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - try: - self.restore_node(self.backup_dir, 'node', self.node, backup_id=self.backup_id, options=['-j', '3']) - except ProbackupException as e: - self.fail( - "ERROR: Restore from full backup failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - try: - self.node.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertEqual( - self.node.table_checksum("t1"), - self.table_t1 - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_from_fullbackup_to_new_location(self): - """ - Case: Restore instance from valid full backup to new location. - """ - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(self.module_name, self.fname)) - node_new.cleanup() - - try: - self.restore_node(self.backup_dir, 'node', node_new, backup_id=self.backup_id) - self.set_auto_conf(node_new, {'port': node_new.port}) - except ProbackupException as e: - self.fail( - "ERROR: Restore from full backup failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - try: - node_new.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertEqual( - node_new.table_checksum("t1"), - self.table_t1 - ) - node_new.cleanup() - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_from_fullbackup_to_new_location_5_jobs(self): - """ - Case: Restore instance from valid full backup to new location. - """ - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - node_new = self.make_simple_node(base_dir="{0}/{1}/node_new_location".format(self.module_name, self.fname)) - node_new.cleanup() - - try: - self.restore_node(self.backup_dir, 'node', node_new, backup_id=self.backup_id, options=['-j', '5']) - self.set_auto_conf(node_new, {'port': node_new.port}) - except ProbackupException as e: - self.fail( - "ERROR: Restore from full backup failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name)], ['pg_compression']), - "ERROR: File pg_compression not found in backup dir" - ) - try: - node_new.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertEqual( - node_new.table_checksum("t1"), - self.table_t1 - ) - node_new.cleanup() - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_from_fullbackup_to_old_location_tablespace_new_location(self): - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - os.mkdir(self.get_tblspace_path(self.node, tblspace_name_new)) - - try: - self.restore_node( - self.backup_dir, - 'node', self.node, - backup_id=self.backup_id, - options=["-T", "{0}={1}".format( - self.get_tblspace_path(self.node, tblspace_name), - self.get_tblspace_path(self.node, tblspace_name_new) - ) - ] - ) - except ProbackupException as e: - self.fail( - "ERROR: Restore from full backup failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name_new)], ['pg_compression']), - "ERROR: File pg_compression not found in new tablespace location" - ) - try: - self.node.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertEqual( - self.node.table_checksum("t1"), - self.table_t1 - ) - - # @unittest.expectedFailure - # @unittest.skip("skip") - @unittest.skipUnless(ProbackupTest.enterprise, 'skip') - def test_restore_from_fullbackup_to_old_location_tablespace_new_location_3_jobs(self): - self.node.stop() - self.node.cleanup() - shutil.rmtree(self.get_tblspace_path(self.node, tblspace_name)) - - os.mkdir(self.get_tblspace_path(self.node, tblspace_name_new)) - - try: - self.restore_node( - self.backup_dir, - 'node', self.node, - backup_id=self.backup_id, - options=["-j", "3", "-T", "{0}={1}".format( - self.get_tblspace_path(self.node, tblspace_name), - self.get_tblspace_path(self.node, tblspace_name_new) - ) - ] - ) - except ProbackupException as e: - self.fail( - "ERROR: Restore from full backup failed. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - self.assertTrue( - find_by_name([self.get_tblspace_path(self.node, tblspace_name_new)], ['pg_compression']), - "ERROR: File pg_compression not found in new tablespace location" - ) - try: - self.node.slow_start() - except ProbackupException as e: - self.fail( - "ERROR: Instance not started after restore. \n {0} \n {1}".format( - repr(self.cmd), - repr(e.message) - ) - ) - - self.assertEqual( - self.node.table_checksum("t1"), - self.table_t1 - ) - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_fullbackup_to_new_location_tablespace_new_location(self): - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_fullbackup_to_new_location_tablespace_new_location_5_jobs(self): - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_ptrack(self): - """ - Case: Restore from backup to old location - """ - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_ptrack_jobs(self): - """ - Case: Restore from backup to old location, four jobs - """ - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_ptrack_new_jobs(self): - pass - -# --------------------------------------------------------- # - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_page(self): - """ - Case: Restore from backup to old location - """ - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_page_jobs(self): - """ - Case: Restore from backup to old location, four jobs - """ - pass - - # @unittest.expectedFailure - @unittest.skip("skip") - def test_restore_from_page_new_jobs(self): - """ - Case: Restore from backup to new location, four jobs - """ - pass - - -#class CfsRestoreEncEmptyTablespaceTest(CfsRestoreNoencEmptyTablespaceTest): -# # --- Begin --- # -# def setUp(self): -# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" -# super(CfsRestoreNoencEmptyTablespaceTest, self).setUp() -# -# -#class CfsRestoreEncTest(CfsRestoreNoencTest): -# # --- Begin --- # -# def setUp(self): -# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" -# super(CfsRestoreNoencTest, self).setUp() diff --git a/tests/cfs_validate_backup_test.py b/tests/cfs_validate_backup_test.py deleted file mode 100644 index 343020dfc..000000000 --- a/tests/cfs_validate_backup_test.py +++ /dev/null @@ -1,24 +0,0 @@ -import os -import unittest -import random - -from .helpers.cfs_helpers import find_by_extensions, find_by_name, find_by_pattern, corrupt_file -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException - -tblspace_name = 'cfs_tblspace' - - -class CfsValidateBackupNoenc(ProbackupTest,unittest.TestCase): - def setUp(self): - pass - - def test_validate_fullbackup_empty_tablespace_after_delete_pg_compression(self): - pass - - def tearDown(self): - pass - - -#class CfsValidateBackupNoenc(CfsValidateBackupNoenc): -# os.environ["PG_CIPHER_KEY"] = "super_secret_cipher_key" -# super(CfsValidateBackupNoenc).setUp() diff --git a/tests/checkdb_test.py b/tests/checkdb_test.py index eb46aea19..252b8757e 100644 --- a/tests/checkdb_test.py +++ b/tests/checkdb_test.py @@ -1,29 +1,23 @@ import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from datetime import datetime, timedelta -import subprocess +from .helpers.ptrack_helpers import ProbackupTest +from pg_probackup2.gdb import needs_gdb from testgres import QueryException -import shutil -import sys -import time +from parameterized import parameterized +from .helpers.data_helpers import corrupt_data_file, validate_data_file -class CheckdbTest(ProbackupTest, unittest.TestCase): +class CheckdbTest(ProbackupTest): # @unittest.skip("skip") + @needs_gdb def test_checkdb_amcheck_only_sanity(self): """""" - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -34,14 +28,14 @@ def test_checkdb_amcheck_only_sanity(self): node.safe_psql( "postgres", "create index on t_heap(id)") - + node.safe_psql( "postgres", "create table idxpart (a int) " "partition by range (a)") # there aren't partitioned indexes on 10 and lesser versions - if self.get_version(node) >= 110000: + if self.pg_config_version >= 110000: node.safe_psql( "postgres", "create index on idxpart(a)") @@ -55,29 +49,14 @@ def test_checkdb_amcheck_only_sanity(self): "postgres", "create extension amcheck_next") - log_file_path = os.path.join( - backup_dir, 'log', 'pg_probackup.log') - # simple sanity - try: - self.checkdb_node( - options=['--skip-block-validation']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because --amcheck option is missing\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Option '--skip-block-validation' must be " - "used with '--amcheck' option", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + self.pb.checkdb_node(options=['--skip-block-validation'], + expect_error="because --amcheck options is missing") + self.assertMessage(contains="ERROR: Option '--skip-block-validation' must be " + "used with '--amcheck' option") # simple sanity - output = self.checkdb_node( + output = self.pb.checkdb_node( options=[ '--amcheck', '--skip-block-validation', @@ -91,56 +70,37 @@ def test_checkdb_amcheck_only_sanity(self): output) # logging to file sanity - try: - self.checkdb_node( + self.pb.checkdb_node( options=[ '--amcheck', '--skip-block-validation', '--log-level-file=verbose', - '-d', 'postgres', '-p', str(node.port)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because log_directory missing\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( + '-d', 'postgres', '-p', str(node.port)], + skip_log_directory=True, + expect_error="because log_directory missing") + self.assertMessage(contains= "ERROR: Cannot save checkdb logs to a file. " "You must specify --log-directory option when " - "running checkdb with --log-level-file option enabled", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + "running checkdb with --log-level-file option enabled") # If backup_dir provided, then instance name must be # provided too - try: - self.checkdb_node( - backup_dir, + self.pb.checkdb_node( + use_backup_dir=True, options=[ '--amcheck', '--skip-block-validation', '--log-level-file=verbose', - '-d', 'postgres', '-p', str(node.port)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because log_directory missing\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Required parameter not specified: --instance", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + '-d', 'postgres', '-p', str(node.port)], + expect_error="because instance missing" + ) + self.assertMessage(contains="ERROR: Required parameter not specified: --instance") # checkdb can use default or set in config values, # if backup_dir and instance name are provided - self.checkdb_node( - backup_dir, - 'node', + self.pb.checkdb_node( + use_backup_dir=True, + instance='node', options=[ '--amcheck', '--skip-block-validation', @@ -148,49 +108,41 @@ def test_checkdb_amcheck_only_sanity(self): '-d', 'postgres', '-p', str(node.port)]) # check that file present and full of messages - os.path.isfile(log_file_path) - with open(log_file_path) as f: - log_file_content = f.read() - self.assertIn( - 'INFO: checkdb --amcheck finished successfully', - log_file_content) - self.assertIn( - 'VERBOSE: (query)', - log_file_content) - os.unlink(log_file_path) + log_file_content = self.read_pb_log() + self.assertIn( + 'INFO: checkdb --amcheck finished successfully', + log_file_content) + self.assertIn( + 'VERBOSE: (query)', + log_file_content) + self.unlink_pg_log() # log-level-file and log-directory are provided - self.checkdb_node( - backup_dir, - 'node', + self.pb.checkdb_node( + use_backup_dir=True, + instance='node', options=[ '--amcheck', '--skip-block-validation', '--log-level-file=verbose', - '--log-directory={0}'.format( - os.path.join(backup_dir, 'log')), '-d', 'postgres', '-p', str(node.port)]) # check that file present and full of messages - os.path.isfile(log_file_path) - with open(log_file_path) as f: - log_file_content = f.read() - self.assertIn( - 'INFO: checkdb --amcheck finished successfully', - log_file_content) - self.assertIn( - 'VERBOSE: (query)', - log_file_content) - os.unlink(log_file_path) - - gdb = self.checkdb_node( + log_file_content = self.read_pb_log() + self.assertIn( + 'INFO: checkdb --amcheck finished successfully', + log_file_content) + self.assertIn( + 'VERBOSE: (query)', + log_file_content) + self.unlink_pg_log() + + gdb = self.pb.checkdb_node( gdb=True, options=[ '--amcheck', '--skip-block-validation', '--log-level-file=verbose', - '--log-directory={0}'.format( - os.path.join(backup_dir, 'log')), '-d', 'postgres', '-p', str(node.port)]) gdb.set_breakpoint('amcheck_one_index') @@ -200,23 +152,20 @@ def test_checkdb_amcheck_only_sanity(self): "postgres", "drop table t_heap") - gdb.remove_all_breakpoints() - gdb.continue_execution_until_exit() # check that message about missing index is present - with open(log_file_path) as f: - log_file_content = f.read() - self.assertIn( - 'ERROR: checkdb --amcheck finished with failure', - log_file_content) - self.assertIn( - "WARNING: Thread [1]. Amcheck failed in database 'postgres' " - "for index: 'public.t_heap_id_idx':", - log_file_content) - self.assertIn( - 'ERROR: could not open relation with OID', - log_file_content) + log_file_content = self.read_pb_log() + self.assertIn( + 'ERROR: checkdb --amcheck finished with failure', + log_file_content) + self.assertIn( + "Amcheck failed in database 'postgres' " + "for index: 'public.t_heap_id_idx':", + log_file_content) + self.assertIn( + 'ERROR: could not open relation with OID', + log_file_content) # Clean after yourself gdb.kill() @@ -225,14 +174,12 @@ def test_checkdb_amcheck_only_sanity(self): # @unittest.skip("skip") def test_basic_checkdb_amcheck_only_sanity(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # create two databases @@ -276,24 +223,13 @@ def test_basic_checkdb_amcheck_only_sanity(self): "db2", "select pg_relation_filepath('some_index')").decode('utf-8').rstrip()) - try: - self.checkdb_node( + self.pb.checkdb_node( options=[ '--amcheck', '--skip-block-validation', - '-d', 'postgres', '-p', str(node.port)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because some db was not amchecked" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Some databases were not amchecked", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + '-d', 'postgres', '-p', str(node.port)], + expect_error="because some db was not amchecked") + self.assertMessage(contains="ERROR: Some databases were not amchecked") node.stop() @@ -302,59 +238,37 @@ def test_basic_checkdb_amcheck_only_sanity(self): f.seek(42000) f.write(b"blablahblahs") f.flush() - f.close with open(index_path_2, "rb+", 0) as f: f.seek(42000) f.write(b"blablahblahs") f.flush() - f.close node.slow_start() - log_file_path = os.path.join( - backup_dir, 'log', 'pg_probackup.log') - - try: - self.checkdb_node( + self.pb.checkdb_node( options=[ '--amcheck', '--skip-block-validation', '--log-level-file=verbose', - '--log-directory={0}'.format( - os.path.join(backup_dir, 'log')), - '-d', 'postgres', '-p', str(node.port)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because some db was not amchecked" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: checkdb --amcheck finished with failure", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + '-d', 'postgres', '-p', str(node.port)], + expect_error="because some file checks failed") + self.assertMessage(contains="ERROR: checkdb --amcheck finished with failure") # corruption of both indexes in db1 and db2 must be detected # also the that amcheck is not installed in 'postgres' # should be logged - with open(log_file_path) as f: - log_file_content = f.read() - self.assertIn( - "WARNING: Thread [1]. Amcheck failed in database 'db1' " - "for index: 'public.pgbench_accounts_pkey':", - log_file_content) - - self.assertIn( - "WARNING: Thread [1]. Amcheck failed in database 'db2' " - "for index: 'public.some_index':", - log_file_content) - - self.assertIn( - "ERROR: checkdb --amcheck finished with failure", - log_file_content) + log_file_content = self.read_pb_log() + self.assertMessage(log_file_content, contains= + "Amcheck failed in database 'db1' " + "for index: 'public.pgbench_accounts_pkey':") + + self.assertMessage(log_file_content, contains= + "Amcheck failed in database 'db2' " + "for index: 'public.some_index':") + + self.assertMessage(log_file_content, contains= + "ERROR: checkdb --amcheck finished with failure") # Clean after yourself node.stop() @@ -362,15 +276,11 @@ def test_basic_checkdb_amcheck_only_sanity(self): # @unittest.skip("skip") def test_checkdb_block_validation_sanity(self): """make node, corrupt some pages, check that checkdb failed""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -387,27 +297,14 @@ def test_checkdb_block_validation_sanity(self): "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # sanity - try: - self.checkdb_node() - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because pgdata must be specified\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Required parameter not specified: PGDATA (-D, --pgdata)", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.checkdb_node( + self.pb.checkdb_node(expect_error="because pgdata must be specified") + self.assertMessage(contains="No postgres data directory specified.\nPlease specify it either using environment variable PGDATA or\ncommand line option --pgdata (-D)") + + self.pb.checkdb_node( data_dir=node.data_dir, options=['-d', 'postgres', '-p', str(node.port)]) - self.checkdb_node( - backup_dir, 'node', + self.pb.checkdb_node(use_backup_dir=True, instance='node', options=['-d', 'postgres', '-p', str(node.port)]) heap_full_path = os.path.join(node.data_dir, heap_path) @@ -416,50 +313,78 @@ def test_checkdb_block_validation_sanity(self): f.seek(9000) f.write(b"bla") f.flush() - f.close with open(heap_full_path, "rb+", 0) as f: f.seek(42000) f.write(b"bla") f.flush() - f.close - try: - self.checkdb_node( - backup_dir, 'node', - options=['-d', 'postgres', '-p', str(node.port)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of data corruption\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Checkdb failed", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Corruption detected in file "{0}", block 1'.format( - os.path.normpath(heap_full_path)), - e.message) - - self.assertIn( - 'WARNING: Corruption detected in file "{0}", block 5'.format( - os.path.normpath(heap_full_path)), - e.message) + self.pb.checkdb_node(use_backup_dir=True, instance='node', + options=['-d', 'postgres', '-p', str(node.port)], + expect_error="because of data corruption") + self.assertMessage(contains="ERROR: Checkdb failed") + self.assertMessage(contains='WARNING: Corruption detected in file "{0}", block 1'.format( + os.path.normpath(heap_full_path))) + + self.assertMessage(contains='WARNING: Corruption detected in file "{0}", block 5'.format( + os.path.normpath(heap_full_path))) # Clean after yourself node.stop() + # @unittest.skip("skip") + @parameterized.expand(("vm", "fsm")) + def test_checkdb_nondatafile_validation(self, fork_kind): + """make node, corrupt vm file, check that checkdb failed""" + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select 1 as id, md5(i::text) as text, " + "md5(repeat(i::text,10))::tsvector as tsvector " + "from generate_series(0,1000) i") + node.safe_psql( + "postgres", + "CHECKPOINT;") + + node.safe_psql( + "postgres", + "VACUUM t_heap;") + + node.safe_psql( + "postgres", + "CHECKPOINT;") + + heap_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + heap_path += "_" + fork_kind + + self.pb.checkdb_node(use_backup_dir=True, instance='node', + options=['-d', 'postgres', '-p', str(node.port)]) + + heap_full_path = os.path.join(node.data_dir, heap_path) + self.assertTrue(os.path.exists(heap_full_path)) + + self.assertTrue(validate_data_file(heap_full_path)) + self.assertTrue(corrupt_data_file(heap_full_path), "corrupting file error") + + self.pb.checkdb_node(use_backup_dir=True, instance='node', + options=['-d', 'postgres', '-p', str(node.port)], + expect_error="because of data corruption") + self.assertMessage(contains="ERROR: Checkdb failed") + self.assertMessage(contains='WARNING: Corruption detected in file "{0}"'.format( + os.path.normpath(heap_full_path))) + def test_checkdb_checkunique(self): """Test checkunique parameter of amcheck.bt_index_check function""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') node.slow_start() try: @@ -494,7 +419,7 @@ def test_checkdb_checkunique(self): "DELETE FROM bttest_unique WHERE ctid::text='(9,3)';") # run without checkunique option (error will not detected) - output = self.checkdb_node( + output = self.pb.checkdb_node( options=[ '--amcheck', '--skip-block-validation', @@ -508,54 +433,48 @@ def test_checkdb_checkunique(self): output) # run with checkunique option - try: - self.checkdb_node( + if (ProbackupTest.enterprise and + (self.pg_config_version >= 111300 and self.pg_config_version < 120000 + or self.pg_config_version >= 120800 and self.pg_config_version < 130000 + or self.pg_config_version >= 130400 and self.pg_config_version < 160000 + or self.pg_config_version > 160000)): + self.pb.checkdb_node( + options=[ + '--amcheck', + '--skip-block-validation', + '--checkunique', + '-d', 'postgres', '-p', str(node.port)], + expect_error="because of index corruption") + self.assertMessage(contains= + "ERROR: checkdb --amcheck finished with failure. Not all checked indexes are valid. All databases were amchecked.") + + self.assertMessage(contains= + "Amcheck failed in database 'postgres' for index: 'public.bttest_unique_idx'") + + self.assertMessage(regex= + r"ERROR:[^\n]*(violating UNIQUE constraint|uniqueness is violated)") + else: + self.pb.checkdb_node( options=[ '--amcheck', '--skip-block-validation', '--checkunique', '-d', 'postgres', '-p', str(node.port)]) - if (ProbackupTest.enterprise and - (self.get_version(node) >= 111300 and self.get_version(node) < 120000 - or self.get_version(node) >= 120800 and self.get_version(node) < 130000 - or self.get_version(node) >= 130400)): - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of index corruption\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - else: - self.assertRegex( - self.output, - r"WARNING: Extension 'amcheck(|_next)' version [\d.]* in schema 'public' do not support 'checkunique' parameter") - except ProbackupException as e: - self.assertIn( - "ERROR: checkdb --amcheck finished with failure. Not all checked indexes are valid. All databases were amchecked.", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertIn( - "Amcheck failed in database 'postgres' for index: 'public.bttest_unique_idx': ERROR: index \"bttest_unique_idx\" is corrupted. There are tuples violating UNIQUE constraint", - e.message) + self.assertMessage(regex= + r"WARNING: Extension 'amcheck(|_next)' version [\d.]* in schema 'public' do not support 'checkunique' parameter") # Clean after yourself node.stop() # @unittest.skip("skip") + @needs_gdb def test_checkdb_sigint_handling(self): """""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() try: @@ -568,8 +487,7 @@ def test_checkdb_sigint_handling(self): "create extension amcheck_next") # FULL backup - gdb = self.checkdb_node( - backup_dir, 'node', gdb=True, + gdb = self.pb.checkdb_node(use_backup_dir=True, instance='node', gdb=True, options=[ '-d', 'postgres', '-j', '2', '--skip-block-validation', @@ -578,11 +496,9 @@ def test_checkdb_sigint_handling(self): gdb.set_breakpoint('amcheck_one_index') gdb.run_until_break() - gdb.continue_execution_until_break(20) - gdb.remove_all_breakpoints() - gdb._execute('signal SIGINT') + gdb.signal('SIGINT') gdb.continue_execution_until_error() with open(node.pg_log_file, 'r') as f: @@ -599,13 +515,11 @@ def test_checkdb_sigint_handling(self): # @unittest.skip("skip") def test_checkdb_with_least_privileges(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -637,66 +551,8 @@ def test_checkdb_with_least_privileges(self): "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC;") - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - 'CREATE ROLE backup WITH LOGIN; ' - 'GRANT CONNECT ON DATABASE backupdb to backup; ' - 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' - 'GRANT USAGE ON SCHEMA public TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') # amcheck-next function - - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - 'CREATE ROLE backup WITH LOGIN; ' - 'GRANT CONNECT ON DATABASE backupdb to backup; ' - 'GRANT USAGE ON SCHEMA pg_catalog TO backup; ' - 'GRANT USAGE ON SCHEMA public TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_am TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_class TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_index TO backup; ' - 'GRANT SELECT ON TABLE pg_catalog.pg_namespace TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.texteq(text, text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.namene(name, name) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.int8(integer) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.charne("char", "char") TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' -# 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') - # PG 10 - elif self.get_version(node) > 100000 and self.get_version(node) < 110000: + if self.pg_config_version < 110000: node.safe_psql( 'backupdb', 'CREATE ROLE backup WITH LOGIN; ' @@ -722,20 +578,15 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup;' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup;') - + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup;' + ) if ProbackupTest.enterprise: # amcheck-1.1 node.safe_psql( 'backupdb', 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup') - else: - # amcheck-1.0 - node.safe_psql( - 'backupdb', - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup') # >= 11 < 14 - elif self.get_version(node) > 110000 and self.get_version(node) < 140000: + elif self.pg_config_version > 110000 and self.pg_config_version < 140000: node.safe_psql( 'backupdb', 'CREATE ROLE backup WITH LOGIN; ' @@ -762,13 +613,13 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anyarray, anyelement) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') - + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' + ) # checkunique parameter if ProbackupTest.enterprise: - if (self.get_version(node) >= 111300 and self.get_version(node) < 120000 - or self.get_version(node) >= 120800 and self.get_version(node) < 130000 - or self.get_version(node) >= 130400): + if (self.pg_config_version >= 111300 and self.pg_config_version < 120000 + or self.pg_config_version >= 120800 and self.pg_config_version < 130000 + or self.pg_config_version >= 130400): node.safe_psql( "backupdb", "GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool, bool) TO backup") @@ -800,52 +651,30 @@ def test_checkdb_with_least_privileges(self): 'GRANT EXECUTE ON FUNCTION pg_catalog.string_to_array(text, text) TO backup; ' 'GRANT EXECUTE ON FUNCTION pg_catalog.array_position(anycompatiblearray, anycompatible) TO backup; ' 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass) TO backup; ' - 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;') - + 'GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool) TO backup;' + ) # checkunique parameter - if ProbackupTest.enterprise: + if ProbackupTest.enterprise and self.pg_config_version != 160000: node.safe_psql( "backupdb", "GRANT EXECUTE ON FUNCTION bt_index_check(regclass, bool, bool) TO backup") if ProbackupTest.pgpro: node.safe_psql( - 'backupdb', - 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;') + "backupdb", + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup;" + "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") # checkdb - try: - self.checkdb_node( - backup_dir, 'node', - options=[ - '--amcheck', '-U', 'backup', - '-d', 'backupdb', '-p', str(node.port)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because permissions are missing\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "INFO: Amcheck succeeded for database 'backupdb'", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertIn( - "WARNING: Extension 'amcheck' or 'amcheck_next' are " - "not installed in database postgres", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertIn( - "ERROR: Some databases were not amchecked", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + self.pb.checkdb_node(use_backup_dir=True, instance='node', + options=[ + '--amcheck', '-U', 'backup', + '-d', 'backupdb', '-p', str(node.port)], + expect_error="because permissions are missing") + self.assertMessage(contains="INFO: Amcheck succeeded for database 'backupdb'") + self.assertMessage(contains="WARNING: Extension 'amcheck' or 'amcheck_next' " + "are not installed in database postgres") + self.assertMessage(contains="ERROR: Some databases were not amchecked") # Clean after yourself node.stop() diff --git a/tests/compatibility_test.py b/tests/compatibility_test.py index 7ae8baf9f..3fa005a21 100644 --- a/tests/compatibility_test.py +++ b/tests/compatibility_test.py @@ -1,7 +1,7 @@ import unittest import subprocess import os -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest, fs_backup_class from sys import exit import shutil @@ -14,7 +14,9 @@ def check_ssh_agent_path_exists(): return 'PGPROBACKUP_SSH_AGENT_PATH' in os.environ -class CrossCompatibilityTest(ProbackupTest, unittest.TestCase): +class CrossCompatibilityTest(ProbackupTest): + auto_compress_alg = False + @unittest.skipUnless(check_manual_tests_enabled(), 'skip manual test') @unittest.skipUnless(check_ssh_agent_path_exists(), 'skip no ssh agent path exist') # @unittest.skip("skip") @@ -44,8 +46,7 @@ def test_catchup_with_different_remote_major_pg(self): # pgprobackup_ssh_agent_path = '/home/avaness/postgres/postgres.build.clean/bin/' pgprobackup_ssh_agent_path = os.environ['PGPROBACKUP_SSH_AGENT_PATH'] - src_pg = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'src'), + src_pg = self.pg_node.make_simple('src', set_replication=True, ) src_pg.slow_start() @@ -54,8 +55,8 @@ def test_catchup_with_different_remote_major_pg(self): "CREATE TABLE ultimate_question AS SELECT 42 AS answer") # do full catchup - dst_pg = self.make_empty_node(os.path.join(self.module_name, self.fname, 'dst')) - self.catchup_node( + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( backup_mode='FULL', source_pgdata=src_pg.data_dir, destination_node=dst_pg, @@ -63,7 +64,7 @@ def test_catchup_with_different_remote_major_pg(self): ) dst_options = {'port': str(dst_pg.port)} - self.set_auto_conf(dst_pg, dst_options) + dst_pg.set_auto_conf(dst_options) dst_pg.slow_start() dst_pg.stop() @@ -73,7 +74,7 @@ def test_catchup_with_different_remote_major_pg(self): # do delta catchup with remote pg_probackup agent with another postgres major version # this DELTA backup should fail without PBCKP-236 patch. - self.catchup_node( + self.pb.catchup_node( backup_mode='DELTA', source_pgdata=src_pg.data_dir, destination_node=dst_pg, @@ -82,53 +83,48 @@ def test_catchup_with_different_remote_major_pg(self): ) -class CompatibilityTest(ProbackupTest, unittest.TestCase): +class CompatibilityTest(ProbackupTest): def setUp(self): super().setUp() if not self.probackup_old_path: - self.skipTest('PGPROBACKUPBIN_OLD is not set') + self.skipTest('An old version of pg_probackup is not set up') # @unittest.expectedFailure # @unittest.skip("skip") def test_backward_compatibility_page(self): """Description in jira issue PGPRO-434""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir, old_binary=True) - self.show_pb(backup_dir) + self.pb.init(old_binary=True) + self.pb.show() - self.add_instance(backup_dir, 'node', node, old_binary=True) - self.show_pb(backup_dir) + self.pb.add_instance('node', node, old_binary=True) + self.pb.show() - self.set_archiving(backup_dir, 'node', node, old_binary=True) + self.pb.set_archiving('node', node, old_binary=True) node.slow_start() node.pgbench_init(scale=10) # FULL backup with old binary - self.backup_node( - backup_dir, 'node', node, old_binary=True) + self.pb.backup_node('node', node, old_binary=True) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - self.show_pb(backup_dir) + self.pb.show() - self.validate_pb(backup_dir) + self.pb.validate() # RESTORE old FULL with new binary - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) + self.pb.restore_node('node', node_restored, options=["-j", "4"]) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -143,16 +139,14 @@ def test_backward_compatibility_page(self): pgbench.wait() pgbench.stdout.close() - self.backup_node( - backup_dir, 'node', node, backup_type='page', + self.pb.backup_node('node', node, backup_type='page', old_binary=True) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) + self.pb.restore_node('node', node_restored, options=["-j", "4"]) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -167,16 +161,14 @@ def test_backward_compatibility_page(self): pgbench.wait() pgbench.stdout.close() - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) + self.pb.restore_node('node', node_restored, options=["-j", "4"]) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -194,13 +186,12 @@ def test_backward_compatibility_page(self): 'postgres', 'VACUUM') - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') pgdata = self.pgdata_content(node.data_dir) node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) + self.pb.restore_node('node', node_restored, options=["-j", "4"]) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -209,13 +200,12 @@ def test_backward_compatibility_page(self): 'postgres', 'insert into pgbench_accounts select * from pgbench_accounts') - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') pgdata = self.pgdata_content(node.data_dir) node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) + self.pb.restore_node('node', node_restored, options=["-j", "4"]) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -224,42 +214,37 @@ def test_backward_compatibility_page(self): # @unittest.skip("skip") def test_backward_compatibility_delta(self): """Description in jira issue PGPRO-434""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir, old_binary=True) - self.show_pb(backup_dir) + self.pb.init(old_binary=True) + self.pb.show() - self.add_instance(backup_dir, 'node', node, old_binary=True) - self.show_pb(backup_dir) + self.pb.add_instance('node', node, old_binary=True) + self.pb.show() - self.set_archiving(backup_dir, 'node', node, old_binary=True) + self.pb.set_archiving('node', node, old_binary=True) node.slow_start() node.pgbench_init(scale=10) # FULL backup with old binary - self.backup_node( - backup_dir, 'node', node, old_binary=True) + self.pb.backup_node('node', node, old_binary=True) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - self.show_pb(backup_dir) + self.pb.show() - self.validate_pb(backup_dir) + self.pb.validate() # RESTORE old FULL with new binary - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) + self.pb.restore_node('node', node_restored, options=["-j", "4"]) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -274,16 +259,14 @@ def test_backward_compatibility_delta(self): pgbench.wait() pgbench.stdout.close() - self.backup_node( - backup_dir, 'node', node, backup_type='delta', + self.pb.backup_node('node', node, backup_type='delta', old_binary=True) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) + self.pb.restore_node('node', node_restored, options=["-j", "4"]) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -298,15 +281,14 @@ def test_backward_compatibility_delta(self): pgbench.wait() pgbench.stdout.close() - self.backup_node(backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) + self.pb.restore_node('node', node_restored, options=["-j", "4"]) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -324,13 +306,12 @@ def test_backward_compatibility_delta(self): 'postgres', 'VACUUM') - self.backup_node(backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') pgdata = self.pgdata_content(node.data_dir) node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) + self.pb.restore_node('node', node_restored, options=["-j", "4"]) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -339,13 +320,12 @@ def test_backward_compatibility_delta(self): 'postgres', 'insert into pgbench_accounts select * from pgbench_accounts') - self.backup_node(backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') pgdata = self.pgdata_content(node.data_dir) node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) + self.pb.restore_node('node', node_restored, options=["-j", "4"]) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -358,20 +338,18 @@ def test_backward_compatibility_ptrack(self): if not self.ptrack: self.skipTest('Skipped because ptrack support is disabled') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - self.init_pb(backup_dir, old_binary=True) - self.show_pb(backup_dir) + self.pb.init(old_binary=True) + self.pb.show() - self.add_instance(backup_dir, 'node', node, old_binary=True) - self.show_pb(backup_dir) + self.pb.add_instance('node', node, old_binary=True) + self.pb.show() - self.set_archiving(backup_dir, 'node', node, old_binary=True) + self.pb.set_archiving('node', node, old_binary=True) node.slow_start() node.safe_psql( @@ -381,24 +359,21 @@ def test_backward_compatibility_ptrack(self): node.pgbench_init(scale=10) # FULL backup with old binary - self.backup_node( - backup_dir, 'node', node, old_binary=True) + self.pb.backup_node('node', node, old_binary=True) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - self.show_pb(backup_dir) + self.pb.show() - self.validate_pb(backup_dir) + self.pb.validate() # RESTORE old FULL with new binary - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) + self.pb.restore_node('node', node_restored, options=["-j", "4"]) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -413,16 +388,14 @@ def test_backward_compatibility_ptrack(self): pgbench.wait() pgbench.stdout.close() - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + self.pb.backup_node('node', node, backup_type='ptrack', old_binary=True) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=[ "-j", "4", "--recovery-target=latest", @@ -441,16 +414,14 @@ def test_backward_compatibility_ptrack(self): pgbench.wait() pgbench.stdout.close() - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack') + self.pb.backup_node('node', node, backup_type='ptrack') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=[ "-j", "4", "--recovery-target=latest", @@ -464,23 +435,20 @@ def test_backward_compatibility_ptrack(self): # @unittest.skip("skip") def test_backward_compatibility_compression(self): """Description in jira issue PGPRO-434""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) + self.pb.init(old_binary=True) + self.pb.add_instance('node', node, old_binary=True) - self.set_archiving(backup_dir, 'node', node, old_binary=True) + self.pb.set_archiving('node', node, old_binary=True) node.slow_start() node.pgbench_init(scale=10) # FULL backup with OLD binary - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, old_binary=True, options=['--compress']) @@ -488,13 +456,11 @@ def test_backward_compatibility_compression(self): pgdata = self.pgdata_content(node.data_dir) # restore OLD FULL with new binary - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=["-j", "4"]) if self.paranoia: @@ -509,8 +475,7 @@ def test_backward_compatibility_compression(self): pgbench.wait() pgbench.stdout.close() - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='page', old_binary=True, options=['--compress']) @@ -519,8 +484,7 @@ def test_backward_compatibility_compression(self): pgdata = self.pgdata_content(node.data_dir) node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=["-j", "4"]) if self.paranoia: @@ -535,8 +499,7 @@ def test_backward_compatibility_compression(self): pgbench.wait() pgbench.stdout.close() - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='page', options=['--compress']) @@ -545,8 +508,7 @@ def test_backward_compatibility_compression(self): node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=["-j", "4"]) if self.paranoia: @@ -554,10 +516,9 @@ def test_backward_compatibility_compression(self): self.compare_pgdata(pgdata, pgdata_restored) # Delta backup with old binary - self.delete_pb(backup_dir, 'node', backup_id) + self.pb.delete('node', backup_id) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, old_binary=True, options=['--compress']) @@ -569,8 +530,7 @@ def test_backward_compatibility_compression(self): pgbench.wait() pgbench.stdout.close() - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='delta', options=['--compress'], old_binary=True) @@ -580,8 +540,7 @@ def test_backward_compatibility_compression(self): node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=["-j", "4"]) if self.paranoia: @@ -597,8 +556,7 @@ def test_backward_compatibility_compression(self): pgbench.wait() pgbench.stdout.close() - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='delta', options=['--compress']) @@ -607,8 +565,7 @@ def test_backward_compatibility_compression(self): node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=["-j", "4"]) if self.paranoia: @@ -616,80 +573,70 @@ def test_backward_compatibility_compression(self): self.compare_pgdata(pgdata, pgdata_restored) # @unittest.expectedFailure - # @unittest.skip("skip") def test_backward_compatibility_merge(self): """ Create node, take FULL and PAGE backups with old binary, merge them with new binary """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) + self.pb.init(old_binary=True) + self.pb.add_instance('node', node, old_binary=True) - self.set_archiving(backup_dir, 'node', node, old_binary=True) + self.pb.set_archiving('node', node, old_binary=True) node.slow_start() # FULL backup with OLD binary - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, old_binary=True) node.pgbench_init(scale=1) # PAGE backup with OLD binary - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, backup_type='page', old_binary=True) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - self.merge_backup(backup_dir, "node", backup_id) + self.pb.merge_backup("node", backup_id) - self.show_pb(backup_dir, as_text=True, as_json=False) + self.pb.show(as_text=True, as_json=False) # restore OLD FULL with new binary - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) + self.pb.restore_node('node', node_restored, options=["-j", "4"]) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) # @unittest.expectedFailure - # @unittest.skip("skip") def test_backward_compatibility_merge_1(self): """ Create node, take FULL and PAGE backups with old binary, merge them with new binary. old binary version =< 2.2.7 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) + self.pb.init(old_binary=True) + self.pb.add_instance('node', node, old_binary=True) - self.set_archiving(backup_dir, 'node', node, old_binary=True) + self.pb.set_archiving('node', node, old_binary=True) node.slow_start() node.pgbench_init(scale=20) # FULL backup with OLD binary - self.backup_node(backup_dir, 'node', node, old_binary=True) + self.pb.backup_node('node', node, old_binary=True) pgbench = node.pgbench( stdout=subprocess.PIPE, @@ -699,8 +646,7 @@ def test_backward_compatibility_merge_1(self): pgbench.stdout.close() # PAGE1 backup with OLD binary - self.backup_node( - backup_dir, 'node', node, backup_type='page', old_binary=True) + self.pb.backup_node('node', node, backup_type='page', old_binary=True) node.safe_psql( 'postgres', @@ -711,13 +657,12 @@ def test_backward_compatibility_merge_1(self): 'VACUUM pgbench_accounts') # PAGE2 backup with OLD binary - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', old_binary=True) + backup_id = self.pb.backup_node('node', node, backup_type='page', old_binary=True) pgdata = self.pgdata_content(node.data_dir) # merge chain created by old binary with new binary - output = self.merge_backup(backup_dir, "node", backup_id) + output = self.pb.merge_backup("node", backup_id) # check that in-place is disabled self.assertIn( @@ -725,33 +670,29 @@ def test_backward_compatibility_merge_1(self): "because of storage format incompatibility", output) # restore merged backup - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored) + self.pb.restore_node('node', node=node_restored) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) # @unittest.expectedFailure - # @unittest.skip("skip") def test_backward_compatibility_merge_2(self): """ Create node, take FULL and PAGE backups with old binary, merge them with new binary. old binary version =< 2.2.7 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) + self.pb.init(old_binary=True) + self.pb.add_instance('node', node, old_binary=True) - self.set_archiving(backup_dir, 'node', node, old_binary=True) + self.pb.set_archiving('node', node, old_binary=True) node.slow_start() node.pgbench_init(scale=50) @@ -760,11 +701,10 @@ def test_backward_compatibility_merge_2(self): 'postgres', 'VACUUM pgbench_accounts') - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') # FULL backup with OLD binary - self.backup_node(backup_dir, 'node', node, old_binary=True) + self.pb.backup_node('node', node, old_binary=True) pgbench = node.pgbench( stdout=subprocess.PIPE, @@ -774,8 +714,7 @@ def test_backward_compatibility_merge_2(self): pgbench.stdout.close() # PAGE1 backup with OLD binary - page1 = self.backup_node( - backup_dir, 'node', node, + page1 = self.pb.backup_node('node', node, backup_type='page', old_binary=True) pgdata1 = self.pgdata_content(node.data_dir) @@ -785,15 +724,13 @@ def test_backward_compatibility_merge_2(self): "DELETE from pgbench_accounts where ctid > '(10,1)'") # PAGE2 backup with OLD binary - page2 = self.backup_node( - backup_dir, 'node', node, + page2 = self.pb.backup_node('node', node, backup_type='page', old_binary=True) pgdata2 = self.pgdata_content(node.data_dir) # PAGE3 backup with OLD binary - page3 = self.backup_node( - backup_dir, 'node', node, + page3 = self.pb.backup_node('node', node, backup_type='page', old_binary=True) pgdata3 = self.pgdata_content(node.data_dir) @@ -806,70 +743,64 @@ def test_backward_compatibility_merge_2(self): pgbench.stdout.close() # PAGE4 backup with NEW binary - page4 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page4 = self.pb.backup_node('node', node, backup_type='page') pgdata4 = self.pgdata_content(node.data_dir) # merge backups one by one and check data correctness # merge PAGE1 - self.merge_backup( - backup_dir, "node", page1, options=['--log-level-file=VERBOSE']) + self.pb.merge_backup("node", page1, options=['--log-level-file=VERBOSE']) # check data correctness for PAGE1 node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, backup_id=page1, + self.pb.restore_node('node', node_restored, backup_id=page1, options=['--log-level-file=VERBOSE']) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata1, pgdata_restored) # merge PAGE2 - self.merge_backup(backup_dir, "node", page2) + self.pb.merge_backup("node", page2) # check data correctness for PAGE2 node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored, backup_id=page2) + self.pb.restore_node('node', node=node_restored, backup_id=page2) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata2, pgdata_restored) # merge PAGE3 - self.show_pb(backup_dir, 'node', page3) - self.merge_backup(backup_dir, "node", page3) - self.show_pb(backup_dir, 'node', page3) + self.pb.show('node', page3) + self.pb.merge_backup("node", page3) + self.pb.show('node', page3) # check data correctness for PAGE3 node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored, backup_id=page3) + self.pb.restore_node('node', node=node_restored, backup_id=page3) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata3, pgdata_restored) # merge PAGE4 - self.merge_backup(backup_dir, "node", page4) + self.pb.merge_backup("node", page4) # check data correctness for PAGE4 node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored, backup_id=page4) + self.pb.restore_node('node', node_restored, backup_id=page4) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata4, pgdata_restored) # @unittest.expectedFailure - # @unittest.skip("skip") def test_backward_compatibility_merge_3(self): """ Create node, take FULL and PAGE backups with old binary, merge them with new binary. old binary version =< 2.2.7 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) + self.pb.init(old_binary=True) + self.pb.add_instance('node', node, old_binary=True) - self.set_archiving(backup_dir, 'node', node, old_binary=True) + self.pb.set_archiving('node', node, old_binary=True) node.slow_start() node.pgbench_init(scale=50) @@ -878,12 +809,10 @@ def test_backward_compatibility_merge_3(self): 'postgres', 'VACUUM pgbench_accounts') - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') # FULL backup with OLD binary - self.backup_node( - backup_dir, 'node', node, old_binary=True, options=['--compress']) + self.pb.backup_node('node', node, old_binary=True, options=['--compress']) pgbench = node.pgbench( stdout=subprocess.PIPE, @@ -893,8 +822,7 @@ def test_backward_compatibility_merge_3(self): pgbench.stdout.close() # PAGE1 backup with OLD binary - page1 = self.backup_node( - backup_dir, 'node', node, + page1 = self.pb.backup_node('node', node, backup_type='page', old_binary=True, options=['--compress']) pgdata1 = self.pgdata_content(node.data_dir) @@ -904,15 +832,13 @@ def test_backward_compatibility_merge_3(self): "DELETE from pgbench_accounts where ctid > '(10,1)'") # PAGE2 backup with OLD binary - page2 = self.backup_node( - backup_dir, 'node', node, + page2 = self.pb.backup_node('node', node, backup_type='page', old_binary=True, options=['--compress']) pgdata2 = self.pgdata_content(node.data_dir) # PAGE3 backup with OLD binary - page3 = self.backup_node( - backup_dir, 'node', node, + page3 = self.pb.backup_node('node', node, backup_type='page', old_binary=True, options=['--compress']) pgdata3 = self.pgdata_content(node.data_dir) @@ -925,54 +851,50 @@ def test_backward_compatibility_merge_3(self): pgbench.stdout.close() # PAGE4 backup with NEW binary - page4 = self.backup_node( - backup_dir, 'node', node, backup_type='page', options=['--compress']) + page4 = self.pb.backup_node('node', node, backup_type='page', options=['--compress']) pgdata4 = self.pgdata_content(node.data_dir) # merge backups one by one and check data correctness # merge PAGE1 - self.merge_backup( - backup_dir, "node", page1, options=['--log-level-file=VERBOSE']) + self.pb.merge_backup("node", page1, options=['--log-level-file=VERBOSE']) # check data correctness for PAGE1 node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, backup_id=page1, + self.pb.restore_node('node', node_restored, backup_id=page1, options=['--log-level-file=VERBOSE']) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata1, pgdata_restored) # merge PAGE2 - self.merge_backup(backup_dir, "node", page2) + self.pb.merge_backup("node", page2) # check data correctness for PAGE2 node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored, backup_id=page2) + self.pb.restore_node('node', node_restored, backup_id=page2) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata2, pgdata_restored) # merge PAGE3 - self.show_pb(backup_dir, 'node', page3) - self.merge_backup(backup_dir, "node", page3) - self.show_pb(backup_dir, 'node', page3) + self.pb.show('node', page3) + self.pb.merge_backup("node", page3) + self.pb.show('node', page3) # check data correctness for PAGE3 node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored, backup_id=page3) + self.pb.restore_node('node', node_restored, backup_id=page3) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata3, pgdata_restored) # merge PAGE4 - self.merge_backup(backup_dir, "node", page4) + self.pb.merge_backup("node", page4) # check data correctness for PAGE4 node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored, backup_id=page4) + self.pb.restore_node('node', node_restored, backup_id=page4) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata4, pgdata_restored) # @unittest.expectedFailure - # @unittest.skip("skip") def test_backward_compatibility_merge_4(self): """ Start merge between minor version, crash and retry it. @@ -982,16 +904,14 @@ def test_backward_compatibility_merge_4(self): self.assertTrue( False, 'You need pg_probackup old_binary =< 2.4.0 for this test') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) + self.pb.init(old_binary=True) + self.pb.add_instance('node', node, old_binary=True) - self.set_archiving(backup_dir, 'node', node, old_binary=True) + self.pb.set_archiving('node', node, old_binary=True) node.slow_start() node.pgbench_init(scale=20) @@ -1000,12 +920,10 @@ def test_backward_compatibility_merge_4(self): 'postgres', 'VACUUM pgbench_accounts') - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') # FULL backup with OLD binary - self.backup_node( - backup_dir, 'node', node, old_binary=True, options=['--compress']) + self.pb.backup_node('node', node, old_binary=True, options=['--compress']) pgbench = node.pgbench( stdout=subprocess.PIPE, @@ -1015,37 +933,28 @@ def test_backward_compatibility_merge_4(self): pgbench.stdout.close() # PAGE backup with NEW binary - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', options=['--compress']) + page_id = self.pb.backup_node('node', node, backup_type='page', options=['--compress']) pgdata = self.pgdata_content(node.data_dir) # merge PAGE4 - gdb = self.merge_backup(backup_dir, "node", page_id, gdb=True) + gdb = self.pb.merge_backup("node", page_id, gdb=True) gdb.set_breakpoint('rename') gdb.run_until_break() gdb.continue_execution_until_break(500) - gdb._execute('signal SIGKILL') - - try: - self.merge_backup(backup_dir, "node", page_id) - self.assertEqual( - 1, 0, - "Expecting Error because of format changes.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( + gdb.signal('SIGKILL') + + self.expire_locks(backup_dir, "node") + + self.pb.merge_backup("node", page_id, + expect_error="because of format changes") + self.assertMessage(contains= "ERROR: Retry of failed merge for backups with different " "between minor versions is forbidden to avoid data corruption " "because of storage format changes introduced in 2.4.0 version, " - "please take a new full backup", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + "please take a new full backup") # @unittest.expectedFailure - # @unittest.skip("skip") def test_backward_compatibility_merge_5(self): """ Create node, take FULL and PAGE backups with old binary, @@ -1060,22 +969,20 @@ def test_backward_compatibility_merge_5(self): self.version_to_num(self.old_probackup_version), self.version_to_num(self.probackup_version)) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) + self.pb.init(old_binary=True) + self.pb.add_instance('node', node, old_binary=True) - self.set_archiving(backup_dir, 'node', node, old_binary=True) + self.pb.set_archiving('node', node, old_binary=True) node.slow_start() node.pgbench_init(scale=20) # FULL backup with OLD binary - self.backup_node(backup_dir, 'node', node, old_binary=True) + self.pb.backup_node('node', node, old_binary=True) pgbench = node.pgbench( stdout=subprocess.PIPE, @@ -1085,8 +992,7 @@ def test_backward_compatibility_merge_5(self): pgbench.stdout.close() # PAGE1 backup with OLD binary - self.backup_node( - backup_dir, 'node', node, backup_type='page', old_binary=True) + self.pb.backup_node('node', node, backup_type='page', old_binary=True) node.safe_psql( 'postgres', @@ -1097,13 +1003,12 @@ def test_backward_compatibility_merge_5(self): 'VACUUM pgbench_accounts') # PAGE2 backup with OLD binary - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', old_binary=True) + backup_id = self.pb.backup_node('node', node, backup_type='page', old_binary=True) pgdata = self.pgdata_content(node.data_dir) # merge chain created by old binary with new binary - output = self.merge_backup(backup_dir, "node", backup_id) + output = self.pb.merge_backup("node", backup_id) # check that in-place is disabled self.assertNotIn( @@ -1111,11 +1016,10 @@ def test_backward_compatibility_merge_5(self): "because of storage format incompatibility", output) # restore merged backup - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored) + self.pb.restore_node('node', node_restored) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -1131,15 +1035,13 @@ def test_page_vacuum_truncate(self): and check data correctness old binary should be 2.2.x version """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) - self.set_archiving(backup_dir, 'node', node, old_binary=True) + self.pb.init(old_binary=True) + self.pb.add_instance('node', node, old_binary=True) + self.pb.set_archiving('node', node, old_binary=True) node.slow_start() node.safe_psql( @@ -1154,7 +1056,7 @@ def test_page_vacuum_truncate(self): "postgres", "vacuum t_heap") - id1 = self.backup_node(backup_dir, 'node', node, old_binary=True) + id1 = self.pb.backup_node('node', node, old_binary=True) pgdata1 = self.pgdata_content(node.data_dir) node.safe_psql( @@ -1165,8 +1067,7 @@ def test_page_vacuum_truncate(self): "postgres", "vacuum t_heap") - id2 = self.backup_node( - backup_dir, 'node', node, backup_type='page', old_binary=True) + id2 = self.pb.backup_node('node', node, backup_type='page', old_binary=True) pgdata2 = self.pgdata_content(node.data_dir) node.safe_psql( @@ -1176,47 +1077,39 @@ def test_page_vacuum_truncate(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,1) i") - id3 = self.backup_node( - backup_dir, 'node', node, backup_type='page', old_binary=True) + id3 = self.pb.backup_node('node', node, backup_type='page', old_binary=True) pgdata3 = self.pgdata_content(node.data_dir) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, - data_dir=node_restored.data_dir, backup_id=id1) + self.pb.restore_node('node', node_restored, backup_id=id1) # Physical comparison pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata1, pgdata_restored) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, - data_dir=node_restored.data_dir, backup_id=id2) + self.pb.restore_node('node', node_restored, backup_id=id2) # Physical comparison pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata2, pgdata_restored) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, - data_dir=node_restored.data_dir, backup_id=id3) + self.pb.restore_node('node', node_restored, backup_id=id3) # Physical comparison pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata3, pgdata_restored) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() node_restored.cleanup() @@ -1231,15 +1124,13 @@ def test_page_vacuum_truncate_compression(self): and check data correctness old binary should be 2.2.x version """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) - self.set_archiving(backup_dir, 'node', node, old_binary=True) + self.pb.init(old_binary=True) + self.pb.add_instance('node', node, old_binary=True) + self.pb.set_archiving('node', node, old_binary=True) node.slow_start() node.safe_psql( @@ -1254,8 +1145,7 @@ def test_page_vacuum_truncate_compression(self): "postgres", "vacuum t_heap") - self.backup_node( - backup_dir, 'node',node, old_binary=True, options=['--compress']) + self.pb.backup_node('node',node, old_binary=True, options=['--compress']) node.safe_psql( "postgres", @@ -1265,8 +1155,7 @@ def test_page_vacuum_truncate_compression(self): "postgres", "vacuum t_heap") - self.backup_node( - backup_dir, 'node', node, backup_type='page', + self.pb.backup_node('node', node, backup_type='page', old_binary=True, options=['--compress']) node.safe_psql( @@ -1276,23 +1165,21 @@ def test_page_vacuum_truncate_compression(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,1) i") - self.backup_node( - backup_dir, 'node', node, backup_type='page', + self.pb.backup_node('node', node, backup_type='page', old_binary=True, options=['--compress']) pgdata = self.pgdata_content(node.data_dir) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored) + self.pb.restore_node('node', node_restored) # Physical comparison pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() # @unittest.skip("skip") @@ -1306,15 +1193,13 @@ def test_page_vacuum_truncate_compressed_1(self): and check data correctness old binary should be 2.2.x version """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) - self.set_archiving(backup_dir, 'node', node, old_binary=True) + self.pb.init(old_binary=True) + self.pb.add_instance('node', node, old_binary=True) + self.pb.set_archiving('node', node, old_binary=True) node.slow_start() node.safe_psql( @@ -1329,8 +1214,7 @@ def test_page_vacuum_truncate_compressed_1(self): "postgres", "vacuum t_heap") - id1 = self.backup_node( - backup_dir, 'node', node, + id1 = self.pb.backup_node('node', node, old_binary=True, options=['--compress']) pgdata1 = self.pgdata_content(node.data_dir) @@ -1342,8 +1226,7 @@ def test_page_vacuum_truncate_compressed_1(self): "postgres", "vacuum t_heap") - id2 = self.backup_node( - backup_dir, 'node', node, backup_type='page', + id2 = self.pb.backup_node('node', node, backup_type='page', old_binary=True, options=['--compress']) pgdata2 = self.pgdata_content(node.data_dir) @@ -1354,48 +1237,40 @@ def test_page_vacuum_truncate_compressed_1(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,1) i") - id3 = self.backup_node( - backup_dir, 'node', node, backup_type='page', + id3 = self.pb.backup_node('node', node, backup_type='page', old_binary=True, options=['--compress']) pgdata3 = self.pgdata_content(node.data_dir) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, - data_dir=node_restored.data_dir, backup_id=id1) + self.pb.restore_node('node', node_restored, backup_id=id1) # Physical comparison pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata1, pgdata_restored) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, - data_dir=node_restored.data_dir, backup_id=id2) + self.pb.restore_node('node', node_restored, backup_id=id2) # Physical comparison pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata2, pgdata_restored) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, - data_dir=node_restored.data_dir, backup_id=id3) + self.pb.restore_node('node', node_restored, backup_id=id3) # Physical comparison pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata3, pgdata_restored) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() node_restored.cleanup() @@ -1407,41 +1282,35 @@ def test_hidden_files(self): with old binary, then try to delete backup with new binary """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) + self.pb.init(old_binary=True) + self.pb.add_instance('node', node, old_binary=True) node.slow_start() open(os.path.join(node.data_dir, ".hidden_stuff"), 'a').close() - backup_id = self.backup_node( - backup_dir, 'node',node, old_binary=True, options=['--stream']) + backup_id = self.pb.backup_node('node',node, old_binary=True, options=['--stream']) - self.delete_pb(backup_dir, 'node', backup_id) + self.pb.delete('node', backup_id) # @unittest.skip("skip") def test_compatibility_tablespace(self): """ https://github.com/postgrespro/pg_probackup/issues/348 """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + backup_dir = self.backup_dir - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node, old_binary=True) + self.pb.init() + self.pb.add_instance('node', node, old_binary=True) node.slow_start() - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="full", + backup_id = self.pb.backup_node('node', node, backup_type="full", options=["-j", "4", "--stream"], old_binary=True) tblspace_old_path = self.get_tblspace_path(node, 'tblspace_old') @@ -1459,37 +1328,23 @@ def test_compatibility_tablespace(self): tblspace_new_path = self.get_tblspace_path(node, 'tblspace_new') - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "-T", "{0}={1}".format( - tblspace_old_path, tblspace_new_path)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because tablespace mapping is incorrect" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( + self.pb.restore_node('node', node_restored, + options=[ + "-j", "4", + "-T", "{0}={1}".format( + tblspace_old_path, tblspace_new_path)], + expect_error="because tablespace mapping is incorrect") + self.assertMessage(contains= 'ERROR: Backup {0} has no tablespaceses, ' - 'nothing to remap'.format(backup_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + 'nothing to remap'.format(backup_id)) - self.backup_node( - backup_dir, 'node', node, backup_type="delta", + self.pb.backup_node('node', node, backup_type="delta", options=["-j", "4", "--stream"], old_binary=True) - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=[ "-j", "4", "-T", "{0}={1}".format( @@ -1501,3 +1356,44 @@ def test_compatibility_tablespace(self): if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_compatibility_master_options(self): + """ + Test correctness of handling of removed master-db, master-host, master-port, + master-user and replica-timeout options + """ + self.assertTrue( + self.version_to_num(self.old_probackup_version) <= self.version_to_num('2.6.0'), + 'You need pg_probackup old_binary =< 2.6.0 for this test') + + node = self.pg_node.make_simple('node') + backup_dir = self.backup_dir + + self.pb.init(old_binary=True) + self.pb.add_instance('node', node, old_binary=True) + + # add deprecated options (using probackup< 2.6) into pg_probackup.conf + # don't care about option values, we can use random values here + self.pb.set_config('node', + options=[ + '--master-db=postgres', + '--master-host=localhost', + '--master-port=5432', + '--master-user={0}'.format(self.username), + '--replica-timeout=100500'], + old_binary=True) + + # and try to show config with new binary (those options must be silently skipped) + self.pb.show_config('node', old_binary=False) + + # store config with new version (those options must disappear from config) + self.pb.set_config('node', + options=[], + old_binary=False) + + # and check absence + config_options = self.pb.show_config('node', old_binary=False) + self.assertFalse( + ['master-db', 'master-host', 'master-port', 'master-user', 'replica-timeout'] & config_options.keys(), + 'Obsolete options found in new config') \ No newline at end of file diff --git a/tests/compression_test.py b/tests/compression_test.py index 55924b9d2..74b044791 100644 --- a/tests/compression_test.py +++ b/tests/compression_test.py @@ -1,231 +1,41 @@ -import os import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack -from datetime import datetime, timedelta -import subprocess +from pg_probackup2.init_helpers import init_params +from .helpers.ptrack_helpers import ProbackupTest -class CompressionTest(ProbackupTest, unittest.TestCase): - # @unittest.skip("skip") - # @unittest.expectedFailure - def test_basic_compression_stream_zlib(self): - """ - make archive node, make full and page stream backups, - check data correctness in restored instance - """ - self.maxDiff = None - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,256) i") - full_result = node.table_checksum("t_heap") - full_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='full', - options=[ - '--stream', - '--compress-algorithm=zlib']) - - # PAGE BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(256,512) i") - page_result = node.table_checksum("t_heap") - page_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=[ - '--stream', '--compress-algorithm=zlib']) - - # DELTA BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(512,768) i") - delta_result = node.table_checksum("t_heap") - delta_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--stream', '--compress-algorithm=zlib']) - - # Drop Node - node.cleanup() +def have_alg(alg): + return alg in init_params.probackup_compressions - # Check full backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(full_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=full_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - - full_result_new = node.table_checksum("t_heap") - self.assertEqual(full_result, full_result_new) - node.cleanup() - # Check page backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(page_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=page_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() +class CompressionTest(ProbackupTest): - page_result_new = node.table_checksum("t_heap") - self.assertEqual(page_result, page_result_new) - node.cleanup() + def test_basic_compression_stream_pglz(self): + self._test_compression_stream(compression = 'pglz') - # Check delta backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(delta_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=delta_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - - delta_result_new = node.table_checksum("t_heap") - self.assertEqual(delta_result, delta_result_new) - - def test_compression_archive_zlib(self): - """ - make archive node, make full and page backups, - check data correctness in restored instance - """ - self.maxDiff = None - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") - full_result = node.table_checksum("t_heap") - full_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='full', - options=["--compress-algorithm=zlib"]) - - # PAGE BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector " - "from generate_series(0,2) i") - page_result = node.table_checksum("t_heap") - page_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=["--compress-algorithm=zlib"]) - - # DELTA BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,3) i") - delta_result = node.table_checksum("t_heap") - delta_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--compress-algorithm=zlib']) - - # Drop Node - node.cleanup() - - # Check full backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(full_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=full_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - - full_result_new = node.table_checksum("t_heap") - self.assertEqual(full_result, full_result_new) - node.cleanup() - - # Check page backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(page_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=page_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() - - page_result_new = node.table_checksum("t_heap") - self.assertEqual(page_result, page_result_new) - node.cleanup() + def test_basic_compression_stream_zlib(self): + self._test_compression_stream(compression = 'zlib') - # Check delta backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(delta_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=delta_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - node.slow_start() + @unittest.skipUnless(have_alg('lz4'), "pg_probackup is not compiled with lz4 support") + def test_basic_compression_stream_lz4(self): + self._test_compression_stream(compression = 'lz4') - delta_result_new = node.table_checksum("t_heap") - self.assertEqual(delta_result, delta_result_new) - node.cleanup() + @unittest.skipUnless(have_alg('zstd'), "pg_probackup is not compiled with zstd support") + def test_basic_compression_stream_zstd(self): + self._test_compression_stream(compression = 'zstd') - def test_compression_stream_pglz(self): + def _test_compression_stream(self, *, compression): """ make archive node, make full and page stream backups, check data correctness in restored instance """ self.maxDiff = None - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL BACKUP @@ -235,9 +45,8 @@ def test_compression_stream_pglz(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,256) i") full_result = node.table_checksum("t_heap") - full_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='full', - options=['--stream', '--compress-algorithm=pglz']) + full_backup_id = self.pb.backup_node('node', node, backup_type='full', + options=['--stream', f'--compress-algorithm={compression}']) # PAGE BACKUP node.safe_psql( @@ -246,9 +55,8 @@ def test_compression_stream_pglz(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(256,512) i") page_result = node.table_checksum("t_heap") - page_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--stream', '--compress-algorithm=pglz']) + page_backup_id = self.pb.backup_node('node', node, backup_type='page', + options=['--stream', f'--compress-algorithm={compression}']) # DELTA BACKUP node.safe_psql( @@ -257,23 +65,18 @@ def test_compression_stream_pglz(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(512,768) i") delta_result = node.table_checksum("t_heap") - delta_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--stream', '--compress-algorithm=pglz']) + delta_backup_id = self.pb.backup_node('node', node, backup_type='delta', + options=['--stream', f'--compress-algorithm={compression}']) # Drop Node node.cleanup() # Check full backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(full_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=full_backup_id, + restore_result = self.pb.restore_node('node', node, backup_id=full_backup_id, options=[ "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + "--recovery-target-action=promote"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(full_backup_id)) node.slow_start() full_result_new = node.table_checksum("t_heap") @@ -281,15 +84,11 @@ def test_compression_stream_pglz(self): node.cleanup() # Check page backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(page_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=page_backup_id, + restore_result = self.pb.restore_node('node', node, backup_id=page_backup_id, options=[ "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + "--recovery-target-action=promote"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(page_backup_id)) node.slow_start() page_result_new = node.table_checksum("t_heap") @@ -297,36 +96,44 @@ def test_compression_stream_pglz(self): node.cleanup() # Check delta backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(delta_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=delta_backup_id, + restore_result = self.pb.restore_node('node', node, backup_id=delta_backup_id, options=[ "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + "--recovery-target-action=promote"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(delta_backup_id)) node.slow_start() delta_result_new = node.table_checksum("t_heap") self.assertEqual(delta_result, delta_result_new) node.cleanup() - def test_compression_archive_pglz(self): + def test_basic_compression_archive_pglz(self): + self._test_compression_archive(compression = 'pglz') + + def test_basic_compression_archive_zlib(self): + self._test_compression_archive(compression = 'zlib') + + @unittest.skipUnless(have_alg('lz4'), "pg_probackup is not compiled with lz4 support") + def test_basic_compression_archive_lz4(self): + self._test_compression_archive(compression = 'lz4') + + @unittest.skipUnless(have_alg('zstd'), "pg_probackup is not compiled with zstd support") + def test_basic_compression_archive_zstd(self): + self._test_compression_archive(compression = 'zstd') + + def _test_compression_archive(self, *, compression): """ make archive node, make full and page backups, check data correctness in restored instance """ self.maxDiff = None - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL BACKUP @@ -336,9 +143,8 @@ def test_compression_archive_pglz(self): "md5(i::text)::tsvector as tsvector " "from generate_series(0,100) i") full_result = node.table_checksum("t_heap") - full_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='full', - options=['--compress-algorithm=pglz']) + full_backup_id = self.pb.backup_node('node', node, backup_type='full', + options=[f'--compress-algorithm={compression}']) # PAGE BACKUP node.safe_psql( @@ -347,9 +153,8 @@ def test_compression_archive_pglz(self): "md5(i::text)::tsvector as tsvector " "from generate_series(100,200) i") page_result = node.table_checksum("t_heap") - page_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', - options=['--compress-algorithm=pglz']) + page_backup_id = self.pb.backup_node('node', node, backup_type='page', + options=[f'--compress-algorithm={compression}']) # DELTA BACKUP node.safe_psql( @@ -358,23 +163,18 @@ def test_compression_archive_pglz(self): "md5(i::text)::tsvector as tsvector " "from generate_series(200,300) i") delta_result = node.table_checksum("t_heap") - delta_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', - options=['--compress-algorithm=pglz']) + delta_backup_id = self.pb.backup_node('node', node, backup_type='delta', + options=[f'--compress-algorithm={compression}']) # Drop Node node.cleanup() # Check full backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(full_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=full_backup_id, - options=[ - "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + restore_result = self.pb.restore_node('node', node, backup_id=full_backup_id, + options=[ + "-j", "4", "--immediate", + "--recovery-target-action=promote"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(full_backup_id)) node.slow_start() full_result_new = node.table_checksum("t_heap") @@ -382,15 +182,11 @@ def test_compression_archive_pglz(self): node.cleanup() # Check page backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(page_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=page_backup_id, + restore_result = self.pb.restore_node('node', node, backup_id=page_backup_id, options=[ "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + "--recovery-target-action=promote"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(page_backup_id)) node.slow_start() page_result_new = node.table_checksum("t_heap") @@ -398,15 +194,11 @@ def test_compression_archive_pglz(self): node.cleanup() # Check delta backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(delta_backup_id), - self.restore_node( - backup_dir, 'node', node, backup_id=delta_backup_id, + restore_result = self.pb.restore_node('node', node, backup_id=delta_backup_id, options=[ "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + "--recovery-target-action=promote"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(delta_backup_id)) node.slow_start() delta_result_new = node.table_checksum("t_heap") @@ -419,33 +211,19 @@ def test_compression_wrong_algorithm(self): check data correctness in restored instance """ self.maxDiff = None - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='full', options=['--compress-algorithm=bla-blah']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because compress-algorithm is invalid.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertEqual( - e.message, - 'ERROR: Invalid compress algorithm value "bla-blah"\n', - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, + backup_type='full', options=['--compress-algorithm=bla-blah'], + expect_error="because compress-algorithm is invalid") + self.assertMessage(contains='ERROR: Invalid compress algorithm value "bla-blah"') # @unittest.skip("skip") def test_incompressible_pages(self): @@ -454,28 +232,24 @@ def test_incompressible_pages(self): take backup with compression, make sure that page was not compressed, restore backup and check data correctness """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Full - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=[ '--compress-algorithm=zlib', '--compress-level=0']) node.pgbench_init(scale=3) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='delta', options=[ '--compress-algorithm=zlib', @@ -485,7 +259,7 @@ def test_incompressible_pages(self): node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) # Physical comparison if self.paranoia: @@ -493,3 +267,91 @@ def test_incompressible_pages(self): self.compare_pgdata(pgdata, pgdata_restored) node.slow_start() + + def test_compression_variant_alorithms_increment_chain(self): + """ + If any algorithm isn't supported -> skip backup + 1. Full compressed [zlib, 3] backup -> change data + 2. Delta compressed [pglz, 5] -> change data + 3. Delta compressed [lz4, 9] -> change data + 4. Page compressed [zstd, 3] -> change data + Restore and compare + """ + + # Initialize instance and backup directory + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=True) + total_backups = 0 + + self.pb.init() + self.pb.add_instance("node", node) + self.pb.set_archiving("node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=10) + + # Do pglz compressed FULL backup + self.pb.backup_node("node", node, options=['--stream', + '--compress-level', '5', + '--compress-algorithm', 'pglz']) + # Check backup + show_backup = self.pb.show("node")[0] + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Do zlib compressed DELTA backup + if have_alg('zlib'): + total_backups += 1 + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + # Do backup + self.pb.backup_node("node", node, + backup_type="delta", options=['--compress-level', '3', + '--compress-algorithm', 'zlib']) + # Check backup + show_backup = self.pb.show("node")[total_backups] + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "DELTA") + + # Do lz4 compressed DELTA backup + if have_alg('lz4'): + total_backups += 1 + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + # Do backup + self.pb.backup_node("node", node, + backup_type="delta", options=['--compress-level', '9', + '--compress-algorithm', 'lz4']) + # Check backup + show_backup = self.pb.show("node")[total_backups] + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "DELTA") + + # Do zstd compressed PAGE backup + if have_alg('zstd'): + total_backups += 1 + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + # Do backup + self.pb.backup_node("node", node, + backup_type="page", options=['--compress-level', '3', + '--compress-algorithm', 'zstd']) + # Check backup + show_backup = self.pb.show("node")[total_backups] + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + pgdata = self.pgdata_content(node.data_dir) + + # Drop node and restore it + node.cleanup() + self.pb.restore_node('node', node=node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + node.cleanup() diff --git a/tests/config_test.py b/tests/config_test.py index b1a0f9295..7989a4e04 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -1,113 +1,107 @@ import unittest import subprocess import os -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest, fs_backup_class from sys import exit from shutil import copyfile -class ConfigTest(ProbackupTest, unittest.TestCase): +class ConfigTest(ProbackupTest): # @unittest.expectedFailure # @unittest.skip("skip") def test_remove_instance_config(self): """remove pg_probackup.conself.f""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.show_pb(backup_dir) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.show() + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') - conf_file = os.path.join( - backup_dir, 'backups','node', 'pg_probackup.conf') + self.remove_backup_config(backup_dir, 'node') - os.unlink(os.path.join(backup_dir, 'backups','node', 'pg_probackup.conf')) - - try: - self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.assertEqual( - 1, 0, - "Expecting Error because pg_probackup.conf is missing. " - ".\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: could not open file "{0}": ' - 'No such file or directory'.format(conf_file), - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, backup_type='page', + expect_error="because pg_probackup.conf is missing") + self.assertMessage(regex=r'ERROR: Reading instance control.*No such file') # @unittest.expectedFailure # @unittest.skip("skip") def test_corrupt_backup_content(self): """corrupt backup_content.control""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - full1_id = self.backup_node(backup_dir, 'node', node) + full1_id = self.pb.backup_node('node', node) node.safe_psql( 'postgres', 'create table t1()') - fulle2_id = self.backup_node(backup_dir, 'node', node) + fulle2_id = self.pb.backup_node('node', node) + + content = self.read_backup_file(backup_dir, 'node', fulle2_id, + 'backup_content.control') + self.write_backup_file(backup_dir, 'node', full1_id, + 'backup_content.control', content) - fulle1_conf_file = os.path.join( - backup_dir, 'backups','node', full1_id, 'backup_content.control') + self.pb.validate('node', + expect_error="because pg_probackup.conf is missing") + self.assertMessage(regex="WARNING: Invalid CRC of backup control file " + fr".*{full1_id}") + self.assertMessage(contains=f"WARNING: Failed to get file list for backup {full1_id}") + self.assertMessage(contains=f"WARNING: Backup {full1_id} file list is corrupted") - fulle2_conf_file = os.path.join( - backup_dir, 'backups','node', fulle2_id, 'backup_content.control') + self.pb.show('node', full1_id)['status'] - copyfile(fulle2_conf_file, fulle1_conf_file) + self.assertEqual(self.pb.show('node')[0]['status'], 'CORRUPT') + self.assertEqual(self.pb.show('node')[1]['status'], 'OK') + @unittest.skipUnless(fs_backup_class.is_file_based, "AccessPath check is always true on s3") + def test_basic_dry_run_set_config(self): + """Check set-config command witch dry-run option""" + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.show() + self.pb.set_archiving('node', node) + node.slow_start() + + conf_file = os.path.join(backup_dir, 'backups', 'node', 'pg_probackup.conf') + with open(conf_file) as cf: + cf_before = cf.read() + self.pb.set_config('node', options=['--dry-run']) + with open(conf_file) as cf: + cf_after = cf.read() + # Compare content of conf_file after dry-run + self.assertTrue(cf_before==cf_after) + + #Check access suit - if disk mounted as read_only + dir_path = os.path.join(backup_dir, 'backups', 'node') + dir_mode = os.stat(dir_path).st_mode + os.chmod(dir_path, 0o500) + + error_message = self.pb.set_config('node', options=['--dry-run'], expect_error ='because of changed permissions') try: - self.validate_pb(backup_dir, 'node') - self.assertEqual( - 1, 0, - "Expecting Error because pg_probackup.conf is missing. " - ".\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "WARNING: Invalid CRC of backup control file '{0}':".format(fulle1_conf_file), - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertIn( - "WARNING: Failed to get file list for backup {0}".format(full1_id), - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertIn( - "WARNING: Backup {0} file list is corrupted".format(full1_id), - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.show_pb(backup_dir, 'node', full1_id)['status'] - - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], 'CORRUPT') - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], 'OK') + self.assertMessage(error_message, contains='ERROR: Check permissions ') + finally: + # Cleanup + os.chmod(dir_path, dir_mode) + + node.cleanup() diff --git a/tests/delete_test.py b/tests/delete_test.py index 10100887d..761aa36f3 100644 --- a/tests/delete_test.py +++ b/tests/delete_test.py @@ -1,48 +1,53 @@ -import unittest -import os -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest import subprocess -class DeleteTest(ProbackupTest, unittest.TestCase): +class DeleteTest(ProbackupTest): # @unittest.skip("skip") # @unittest.expectedFailure def test_delete_full_backups(self): """delete full backups""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # full backup - self.backup_node(backup_dir, 'node', node) + id_1 = self.pb.backup_node('node', node) + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + + id_2 = self.pb.backup_node('node', node) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - self.backup_node(backup_dir, 'node', node) + id_2_1 = self.pb.backup_node('node', node, backup_type = "delta") pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - self.backup_node(backup_dir, 'node', node) + id_3 = self.pb.backup_node('node', node) - show_backups = self.show_pb(backup_dir, 'node') - id_1 = show_backups[0]['id'] - id_2 = show_backups[1]['id'] - id_3 = show_backups[2]['id'] - self.delete_pb(backup_dir, 'node', id_2) - show_backups = self.show_pb(backup_dir, 'node') + show_backups = self.pb.show('node') + self.assertEqual(show_backups[0]['id'], id_1) + self.assertEqual(show_backups[1]['id'], id_2) + self.assertEqual(show_backups[2]['id'], id_2_1) + self.assertEqual(show_backups[3]['id'], id_3) + + self.pb.delete('node', id_2) + show_backups = self.pb.show('node') + self.assertEqual(len(show_backups), 2) self.assertEqual(show_backups[0]['id'], id_1) self.assertEqual(show_backups[1]['id'], id_3) @@ -50,63 +55,55 @@ def test_delete_full_backups(self): # @unittest.expectedFailure def test_del_instance_archive(self): """delete full backups""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # full backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # full backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # restore node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) node.slow_start() # Delete instance - self.del_instance(backup_dir, 'node') + self.pb.del_instance('node') # @unittest.skip("skip") # @unittest.expectedFailure def test_delete_archive_mix_compress_and_non_compressed_segments(self): """delete full backups""" - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving( - backup_dir, 'node', node, compress=False) + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node, compress=False) node.slow_start() # full backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.pgbench_init(scale=10) # Restart archiving with compression - self.set_archiving(backup_dir, 'node', node, compress=True) + self.pb.set_archiving('node', node, compress=True) node.restart() # full backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) pgbench = node.pgbench(options=['-T', '10', '-c', '2']) pgbench.wait() - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=[ '--retention-redundancy=3', '--delete-expired']) @@ -114,8 +111,7 @@ def test_delete_archive_mix_compress_and_non_compressed_segments(self): pgbench = node.pgbench(options=['-T', '10', '-c', '2']) pgbench.wait() - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=[ '--retention-redundancy=3', '--delete-expired']) @@ -123,41 +119,37 @@ def test_delete_archive_mix_compress_and_non_compressed_segments(self): pgbench = node.pgbench(options=['-T', '10', '-c', '2']) pgbench.wait() - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=[ '--retention-redundancy=3', '--delete-expired']) # @unittest.skip("skip") - def test_delete_increment_page(self): + def test_basic_delete_increment_page(self): """delete increment and all after him""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # full backup mode - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # page backup mode - self.backup_node(backup_dir, 'node', node, backup_type="page") + self.pb.backup_node('node', node, backup_type="page") # page backup mode - self.backup_node(backup_dir, 'node', node, backup_type="page") + self.pb.backup_node('node', node, backup_type="page") # full backup mode - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) - show_backups = self.show_pb(backup_dir, 'node') + show_backups = self.pb.show('node') self.assertEqual(len(show_backups), 4) # delete first page backup - self.delete_pb(backup_dir, 'node', show_backups[1]['id']) + self.pb.delete('node', show_backups[1]['id']) - show_backups = self.show_pb(backup_dir, 'node') + show_backups = self.pb.show('node') self.assertEqual(len(show_backups), 2) self.assertEqual(show_backups[0]['backup-mode'], "FULL") @@ -171,15 +163,12 @@ def test_delete_increment_ptrack(self): if not self.ptrack: self.skipTest('Skipped because ptrack support is disabled') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + ptrack_enable=self.ptrack) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -187,21 +176,21 @@ def test_delete_increment_ptrack(self): 'CREATE EXTENSION ptrack') # full backup mode - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # ptrack backup mode - self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + self.pb.backup_node('node', node, backup_type="ptrack") # ptrack backup mode - self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + self.pb.backup_node('node', node, backup_type="ptrack") # full backup mode - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) - show_backups = self.show_pb(backup_dir, 'node') + show_backups = self.pb.show('node') self.assertEqual(len(show_backups), 4) # delete first page backup - self.delete_pb(backup_dir, 'node', show_backups[1]['id']) + self.pb.delete('node', show_backups[1]['id']) - show_backups = self.show_pb(backup_dir, 'node') + show_backups = self.pb.show('node') self.assertEqual(len(show_backups), 2) self.assertEqual(show_backups[0]['backup-mode'], "FULL") @@ -210,71 +199,68 @@ def test_delete_increment_ptrack(self): self.assertEqual(show_backups[1]['status'], "OK") # @unittest.skip("skip") - def test_delete_orphaned_wal_segments(self): + def test_basic_delete_orphaned_wal_segments(self): """ make archive node, make three full backups, delete second backup without --wal option, then delete orphaned wals via --wal option """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( "postgres", "create table t_heap as select 1 as id, md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector from generate_series(0,10000) i") # first full backup - backup_1_id = self.backup_node(backup_dir, 'node', node) + backup_1_id = self.pb.backup_node('node', node) # second full backup - backup_2_id = self.backup_node(backup_dir, 'node', node) + backup_2_id = self.pb.backup_node('node', node) # third full backup - backup_3_id = self.backup_node(backup_dir, 'node', node) + backup_3_id = self.pb.backup_node('node', node) node.stop() # Check wals - wals_dir = os.path.join(backup_dir, 'wal', 'node') - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f))] + wals = self.get_instance_wal_list(backup_dir, 'node') original_wal_quantity = len(wals) # delete second full backup - self.delete_pb(backup_dir, 'node', backup_2_id) + self.pb.delete('node', backup_2_id) # check wal quantity - self.validate_pb(backup_dir) - self.assertEqual(self.show_pb(backup_dir, 'node', backup_1_id)['status'], "OK") - self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + self.pb.validate() + self.assertEqual(self.pb.show('node', backup_1_id)['status'], "OK") + self.assertEqual(self.pb.show('node', backup_3_id)['status'], "OK") # try to delete wals for second backup - self.delete_pb(backup_dir, 'node', options=['--wal']) + self.pb.delete('node', options=['--wal']) # check wal quantity - self.validate_pb(backup_dir) - self.assertEqual(self.show_pb(backup_dir, 'node', backup_1_id)['status'], "OK") - self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + self.pb.validate() + self.assertEqual(self.pb.show('node', backup_1_id)['status'], "OK") + self.assertEqual(self.pb.show('node', backup_3_id)['status'], "OK") # delete first full backup - self.delete_pb(backup_dir, 'node', backup_1_id) - self.validate_pb(backup_dir) - self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + self.pb.delete('node', backup_1_id) + self.pb.validate() + self.assertEqual(self.pb.show('node', backup_3_id)['status'], "OK") - result = self.delete_pb(backup_dir, 'node', options=['--wal']) + result = self.pb.delete('node', options=['--wal']) # delete useless wals self.assertTrue('On timeline 1 WAL segments between ' in result and 'will be removed' in result) - self.validate_pb(backup_dir) - self.assertEqual(self.show_pb(backup_dir, 'node', backup_3_id)['status'], "OK") + self.pb.validate() + self.assertEqual(self.pb.show('node', backup_3_id)['status'], "OK") # Check quantity, it should be lower than original - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f))] - self.assertTrue(original_wal_quantity > len(wals), "Number of wals not changed after 'delete --wal' which is illegal") + wals = self.get_instance_wal_list(backup_dir, 'node') + self.assertGreater(original_wal_quantity, len(wals), "Number of wals not changed after 'delete --wal' which is illegal") # Delete last backup - self.delete_pb(backup_dir, 'node', backup_3_id, options=['--wal']) - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f))] + self.pb.delete('node', backup_3_id, options=['--wal']) + wals = self.get_instance_wal_list(backup_dir, 'node') self.assertEqual (0, len(wals), "Number of wals should be equal to 0") # @unittest.skip("skip") @@ -287,45 +273,41 @@ def test_delete_wal_between_multiple_timelines(self): [A1, B1) are deleted and backups B1 and A2 keep their WAL """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - A1 = self.backup_node(backup_dir, 'node', node) + A1 = self.pb.backup_node('node', node) # load some data to node node.pgbench_init(scale=3) - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2 = self.pg_node.make_simple('node2') node2.cleanup() - self.restore_node(backup_dir, 'node', node2) - self.set_auto_conf(node2, {'port': node2.port}) + self.pb.restore_node('node', node=node2) + node2.set_auto_conf({'port': node2.port}) node2.slow_start() # load some more data to node node.pgbench_init(scale=3) # take A2 - A2 = self.backup_node(backup_dir, 'node', node) + A2 = self.pb.backup_node('node', node) # load some more data to node2 node2.pgbench_init(scale=2) - B1 = self.backup_node( - backup_dir, 'node', + B1 = self.pb.backup_node('node', node2, data_dir=node2.data_dir) - self.delete_pb(backup_dir, 'node', backup_id=A1, options=['--wal']) + self.pb.delete('node', backup_id=A1, options=['--wal']) - self.validate_pb(backup_dir) + self.pb.validate() # @unittest.skip("skip") def test_delete_backup_with_empty_control_file(self): @@ -333,53 +315,43 @@ def test_delete_backup_with_empty_control_file(self): take backup, truncate its control file, try to delete it via 'delete' command """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # full backup mode - self.backup_node( - backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) # page backup mode - self.backup_node( - backup_dir, 'node', node, backup_type="delta", options=['--stream']) + self.pb.backup_node('node', node, backup_type="delta", options=['--stream']) # page backup mode - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="delta", options=['--stream']) + backup_id = self.pb.backup_node('node', node, backup_type="delta", options=['--stream']) - with open( - os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.control'), - 'wt') as f: - f.flush() - f.close() + with self.modify_backup_control(backup_dir, 'node', backup_id) as cf: + cf.data = '' - show_backups = self.show_pb(backup_dir, 'node') + show_backups = self.pb.show('node') self.assertEqual(len(show_backups), 3) - self.delete_pb(backup_dir, 'node', backup_id=backup_id) + self.pb.delete('node', backup_id=backup_id) # @unittest.skip("skip") def test_delete_interleaved_incremental_chains(self): """complicated case of interleaved backup chains""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Take FULL BACKUPs - backup_id_a = self.backup_node(backup_dir, 'node', node) - backup_id_b = self.backup_node(backup_dir, 'node', node) + backup_id_a = self.pb.backup_node('node', node) + backup_id_b = self.pb.backup_node('node', node) # Change FULLb to ERROR self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') @@ -388,8 +360,7 @@ def test_delete_interleaved_incremental_chains(self): # FULLa OK # Take PAGEa1 backup - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a1 = self.pb.backup_node('node', node, backup_type='page') # PAGEa1 OK # FULLb ERROR @@ -405,8 +376,7 @@ def test_delete_interleaved_incremental_chains(self): # FULLb OK # FULLa OK - page_id_b1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b1 = self.pb.backup_node('node', node, backup_type='page') # PAGEb1 OK # PAGEa1 ERROR @@ -426,8 +396,7 @@ def test_delete_interleaved_incremental_chains(self): # FULLb ERROR # FULLa OK - page_id_a2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a2 = self.pb.backup_node('node', node, backup_type='page') # PAGEa2 OK # PAGEb1 ERROR @@ -449,8 +418,7 @@ def test_delete_interleaved_incremental_chains(self): # FULLb OK # FULLa ERROR - page_id_b2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b2 = self.pb.backup_node('node', node, backup_type='page') # Change PAGEa2 and FULLa status to OK self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') @@ -463,8 +431,8 @@ def test_delete_interleaved_incremental_chains(self): # FULLb OK # FULLa OK - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type='page') # PAGEc1 OK # FULLc OK @@ -476,17 +444,15 @@ def test_delete_interleaved_incremental_chains(self): # FULLa OK # Delete FULLb - self.delete_pb( - backup_dir, 'node', backup_id_b) + self.pb.delete('node', backup_id_b) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 5) + self.assertEqual(len(self.pb.show('node')), 5) - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) + print(self.pb.show('node', as_json=False, as_text=True)) # @unittest.skip("skip") def test_delete_multiple_descendants(self): - """ + r""" PAGEb3 | PAGEa3 PAGEb2 / @@ -496,25 +462,22 @@ def test_delete_multiple_descendants(self): FULLb | FULLa should be deleted """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Take FULL BACKUPs - backup_id_a = self.backup_node(backup_dir, 'node', node) - backup_id_b = self.backup_node(backup_dir, 'node', node) + backup_id_a = self.pb.backup_node('node', node) + backup_id_b = self.pb.backup_node('node', node) # Change FULLb to ERROR self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a1 = self.pb.backup_node('node', node, backup_type='page') # Change FULLb to OK self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') @@ -526,8 +489,7 @@ def test_delete_multiple_descendants(self): # FULLb OK # FULLa OK - page_id_b1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b1 = self.pb.backup_node('node', node, backup_type='page') # PAGEb1 OK # PAGEa1 ERROR @@ -546,8 +508,7 @@ def test_delete_multiple_descendants(self): # FULLb ERROR # FULLa OK - page_id_a2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a2 = self.pb.backup_node('node', node, backup_type='page') # PAGEa2 OK # PAGEb1 ERROR @@ -569,8 +530,7 @@ def test_delete_multiple_descendants(self): # FULLb OK # FULLa ERROR - page_id_b2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b2 = self.pb.backup_node('node', node, backup_type='page') # PAGEb2 OK # PAGEa2 ERROR @@ -594,8 +554,7 @@ def test_delete_multiple_descendants(self): # FULLb ERROR # FULLa OK - page_id_a3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a3 = self.pb.backup_node('node', node, backup_type='page') # PAGEa3 OK # PAGEb2 ERROR @@ -612,8 +571,7 @@ def test_delete_multiple_descendants(self): self.change_backup_status(backup_dir, 'node', page_id_b2, 'OK') self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - page_id_b3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b3 = self.pb.backup_node('node', node, backup_type='page') # PAGEb3 OK # PAGEa3 ERROR @@ -640,21 +598,21 @@ def test_delete_multiple_descendants(self): # Check that page_id_a3 and page_id_a2 are both direct descendants of page_id_a1 self.assertEqual( - self.show_pb(backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'], + self.pb.show('node', backup_id=page_id_a3)['parent-backup-id'], page_id_a1) self.assertEqual( - self.show_pb(backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'], + self.pb.show('node', backup_id=page_id_a2)['parent-backup-id'], page_id_a1) # Delete FULLa - self.delete_pb(backup_dir, 'node', backup_id_a) + self.pb.delete('node', backup_id_a) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + self.assertEqual(len(self.pb.show('node')), 4) # @unittest.skip("skip") def test_delete_multiple_descendants_dry_run(self): - """ + r""" PAGEa3 PAGEa2 / \ / @@ -662,29 +620,25 @@ def test_delete_multiple_descendants_dry_run(self): | FULLa """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Take FULL BACKUP node.pgbench_init(scale=1) - backup_id_a = self.backup_node(backup_dir, 'node', node) + backup_id_a = self.pb.backup_node('node', node) pgbench = node.pgbench(options=['-T', '10', '-c', '2']) pgbench.wait() - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a1 = self.pb.backup_node('node', node, backup_type='page') pgbench = node.pgbench(options=['-T', '10', '-c', '2']) pgbench.wait() - page_id_a2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a2 = self.pb.backup_node('node', node, backup_type='page') # Change PAGEa2 to ERROR @@ -692,15 +646,13 @@ def test_delete_multiple_descendants_dry_run(self): pgbench = node.pgbench(options=['-T', '10', '-c', '2']) pgbench.wait() - page_id_a3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a3 = self.pb.backup_node('node', node, backup_type='page') # Change PAGEa2 to ERROR self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') # Delete PAGEa1 - output = self.delete_pb( - backup_dir, 'node', page_id_a1, + output = self.pb.delete('node', page_id_a1, options=['--dry-run', '--log-level-console=LOG', '--delete-wal']) print(output) @@ -719,15 +671,13 @@ def test_delete_multiple_descendants_dry_run(self): 'delete of backup {0} :'.format(page_id_a1), output) - self.assertIn( - 'On timeline 1 WAL segments between 000000010000000000000001 ' - 'and 000000010000000000000003 can be removed', - output) + self.assertRegex(output, + r'On timeline 1 WAL segments between 000000010000000000000001 ' + r'and 00000001000000000000000\d can be removed') - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + self.assertEqual(len(self.pb.show('node')), 4) - output = self.delete_pb( - backup_dir, 'node', page_id_a1, + output = self.pb.delete('node', page_id_a1, options=['--log-level-console=LOG', '--delete-wal']) self.assertIn( @@ -744,60 +694,57 @@ def test_delete_multiple_descendants_dry_run(self): 'delete of backup {0} :'.format(page_id_a1), output) - self.assertIn( - 'On timeline 1 WAL segments between 000000010000000000000001 ' - 'and 000000010000000000000003 will be removed', - output) + self.assertRegex(output, + r'On timeline 1 WAL segments between 000000010000000000000001 ' + r'and 00000001000000000000000\d will be removed') - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 1) + self.assertEqual(len(self.pb.show('node')), 1) - self.validate_pb(backup_dir, 'node') + self.pb.validate('node') def test_delete_error_backups(self): """delete increment and all after him""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # full backup mode - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # page backup mode - self.backup_node(backup_dir, 'node', node, backup_type="page") + self.pb.backup_node('node', node, backup_type="page") # Take FULL BACKUP - backup_id_a = self.backup_node(backup_dir, 'node', node) + backup_id_a = self.pb.backup_node('node', node) # Take PAGE BACKUP - backup_id_b = self.backup_node(backup_dir, 'node', node, backup_type="page") + backup_id_b = self.pb.backup_node('node', node, backup_type="page") - backup_id_c = self.backup_node(backup_dir, 'node', node, backup_type="page") + backup_id_c = self.pb.backup_node('node', node, backup_type="page") - backup_id_d = self.backup_node(backup_dir, 'node', node, backup_type="page") + backup_id_d = self.pb.backup_node('node', node, backup_type="page") # full backup mode - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type="page") - backup_id_e = self.backup_node(backup_dir, 'node', node, backup_type="page") - self.backup_node(backup_dir, 'node', node, backup_type="page") + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type="page") + backup_id_e = self.pb.backup_node('node', node, backup_type="page") + self.pb.backup_node('node', node, backup_type="page") # Change status to ERROR self.change_backup_status(backup_dir, 'node', backup_id_a, 'ERROR') self.change_backup_status(backup_dir, 'node', backup_id_c, 'ERROR') self.change_backup_status(backup_dir, 'node', backup_id_e, 'ERROR') - print(self.show_pb(backup_dir, as_text=True, as_json=False)) + print(self.pb.show(as_text=True, as_json=False)) - show_backups = self.show_pb(backup_dir, 'node') + show_backups = self.pb.show('node') self.assertEqual(len(show_backups), 10) # delete error backups - output = self.delete_pb(backup_dir, 'node', options=['--status=ERROR', '--dry-run']) - show_backups = self.show_pb(backup_dir, 'node') + output = self.pb.delete('node', options=['--status=ERROR', '--dry-run']) + show_backups = self.pb.show('node') self.assertEqual(len(show_backups), 10) self.assertIn( @@ -808,15 +755,46 @@ def test_delete_error_backups(self): "INFO: Backup {0} with status OK can be deleted".format(backup_id_d), output) - print(self.show_pb(backup_dir, as_text=True, as_json=False)) + print(self.pb.show(as_text=True, as_json=False)) - show_backups = self.show_pb(backup_dir, 'node') - output = self.delete_pb(backup_dir, 'node', options=['--status=ERROR']) + show_backups = self.pb.show('node') + output = self.pb.delete('node', options=['--status=ERROR']) print(output) - show_backups = self.show_pb(backup_dir, 'node') + show_backups = self.pb.show('node') self.assertEqual(len(show_backups), 4) self.assertEqual(show_backups[0]['status'], "OK") self.assertEqual(show_backups[1]['status'], "OK") self.assertEqual(show_backups[2]['status'], "OK") self.assertEqual(show_backups[3]['status'], "OK") + +########################################################################### +# dry-run +########################################################################### + + def test_basic_dry_run_del_instance(self): + """ Check del-instance command with dry-run option""" + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + # full backup + self.pb.backup_node('node', node) + # restore + node.cleanup() + self.pb.restore_node('node', node=node) + node.slow_start() + + content_before = self.pgdata_content(self.backup_dir) + # Delete instance + self.pb.del_instance('node', options=['--dry-run']) + + self.compare_instance_dir( + content_before, + self.pgdata_content(self.backup_dir) + ) + + node.cleanup() \ No newline at end of file diff --git a/tests/delta_test.py b/tests/delta_test.py index 8736a079c..6bf7f9d9a 100644 --- a/tests/delta_test.py +++ b/tests/delta_test.py @@ -1,14 +1,11 @@ import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from datetime import datetime, timedelta -from testgres import QueryException -import subprocess -import time + +from .helpers.ptrack_helpers import ProbackupTest +from pg_probackup2.gdb import needs_gdb from threading import Thread -class DeltaTest(ProbackupTest, unittest.TestCase): +class DeltaTest(ProbackupTest): # @unittest.skip("skip") def test_basic_delta_vacuum_truncate(self): @@ -16,20 +13,17 @@ def test_basic_delta_vacuum_truncate(self): make node, create table, take full backup, delete last 3 pages, vacuum relation, take delta backup, take second delta backup, - restore latest delta backup and check data correctness + restore the latest delta backup and check data correctness """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node_restored.cleanup() node.slow_start() @@ -45,7 +39,7 @@ def test_basic_delta_vacuum_truncate(self): "postgres", "vacuum t_heap") - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) node.safe_psql( "postgres", @@ -55,22 +49,19 @@ def test_basic_delta_vacuum_truncate(self): "postgres", "vacuum t_heap") - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') pgdata = self.pgdata_content(node.data_dir) - self.restore_node( - backup_dir, 'node', node_restored) + self.pb.restore_node('node', node_restored) # Physical comparison pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() # @unittest.skip("skip") @@ -81,19 +72,16 @@ def test_delta_vacuum_truncate_1(self): take delta backup, take second delta backup, restore latest delta backup and check data correctness """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], ) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), + node_restored = self.pg_node.make_simple('node_restored', ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node_restored.cleanup() node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -112,7 +100,7 @@ def test_delta_vacuum_truncate_1(self): "vacuum t_heap" ) - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.safe_psql( "postgres", @@ -124,12 +112,10 @@ def test_delta_vacuum_truncate_1(self): "vacuum t_heap" ) - self.backup_node( - backup_dir, 'node', node, backup_type='delta' + self.pb.backup_node('node', node, backup_type='delta' ) - self.backup_node( - backup_dir, 'node', node, backup_type='delta' + self.pb.backup_node('node', node, backup_type='delta' ) pgdata = self.pgdata_content(node.data_dir) @@ -137,8 +123,7 @@ def test_delta_vacuum_truncate_1(self): old_tablespace = self.get_tblspace_path(node, 'somedata') new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') - self.restore_node( - backup_dir, + self.pb.restore_node( 'node', node_restored, options=[ @@ -150,7 +135,7 @@ def test_delta_vacuum_truncate_1(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() # @unittest.skip("skip") @@ -161,19 +146,16 @@ def test_delta_vacuum_truncate_2(self): take delta backup, take second delta backup, restore latest delta backup and check data correctness """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], ) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), + node_restored = self.pg_node.make_simple('node_restored', ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node_restored.cleanup() node.slow_start() @@ -188,27 +170,24 @@ def test_delta_vacuum_truncate_2(self): "select pg_relation_filepath('t_heap')" ).decode('utf-8').rstrip() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) print(os.path.join(node.data_dir, filepath + '.1')) os.unlink(os.path.join(node.data_dir, filepath + '.1')) - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') pgdata = self.pgdata_content(node.data_dir) - self.restore_node( - backup_dir, 'node', node_restored) + self.pb.restore_node('node', node_restored) # Physical comparison pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() # @unittest.skip("skip") @@ -217,19 +196,17 @@ def test_delta_stream(self): make archive node, take full and delta stream backups, restore them and check data correctness """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s' } ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL BACKUP @@ -240,8 +217,7 @@ def test_delta_stream(self): "from generate_series(0,100) i") full_result = node.table_checksum("t_heap") - full_backup_id = self.backup_node( - backup_dir, 'node', node, + full_backup_id = self.pb.backup_node('node', node, backup_type='full', options=['--stream']) # delta BACKUP @@ -251,40 +227,32 @@ def test_delta_stream(self): "md5(i::text)::tsvector as tsvector " "from generate_series(100,200) i") delta_result = node.table_checksum("t_heap") - delta_backup_id = self.backup_node( - backup_dir, 'node', node, + delta_backup_id = self.pb.backup_node('node', node, backup_type='delta', options=['--stream']) # Drop Node node.cleanup() # Check full backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(full_backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, backup_id=full_backup_id, options=[ "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(self.output), self.cmd)) + "--recovery-target-action=promote"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(full_backup_id)) node.slow_start() full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() # Check delta backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(delta_backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, backup_id=delta_backup_id, options=[ "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(self.output), self.cmd)) + "--recovery-target-action=promote"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(delta_backup_id)) + node.slow_start() delta_result_new = node.table_checksum("t_heap") self.assertEqual(delta_result, delta_result_new) @@ -297,15 +265,13 @@ def test_delta_archive(self): restore them and check data correctness """ self.maxDiff = None - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL BACKUP @@ -314,8 +280,7 @@ def test_delta_archive(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,1) i") full_result = node.table_checksum("t_heap") - full_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='full') + full_backup_id = self.pb.backup_node('node', node, backup_type='full') # delta BACKUP node.safe_psql( @@ -323,39 +288,30 @@ def test_delta_archive(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,2) i") delta_result = node.table_checksum("t_heap") - delta_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') + delta_backup_id = self.pb.backup_node('node', node, backup_type='delta') # Drop Node node.cleanup() # Restore and check full backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(full_backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, backup_id=full_backup_id, options=[ "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + "--recovery-target-action=promote"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(full_backup_id)) node.slow_start() full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() # Restore and check delta backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(delta_backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, backup_id=delta_backup_id, options=[ "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + "--recovery-target-action=promote"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(delta_backup_id)) node.slow_start() delta_result_new = node.table_checksum("t_heap") self.assertEqual(delta_result, delta_result_new) @@ -367,11 +323,9 @@ def test_delta_multiple_segments(self): Make node, create table with multiple segments, write some data to it, check delta and data correctness """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'fsync': 'off', 'shared_buffers': '1GB', @@ -380,9 +334,9 @@ def test_delta_multiple_segments(self): } ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - # self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + # self.pb.set_archiving('node', node) node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -392,7 +346,7 @@ def test_delta_multiple_segments(self): scale=100, options=['--tablespace=somedata', '--no-vacuum']) # FULL BACKUP - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) # PGBENCH STUFF pgbench = node.pgbench(options=['-T', '50', '-c', '1', '--no-vacuum']) @@ -402,22 +356,19 @@ def test_delta_multiple_segments(self): # GET LOGICAL CONTENT FROM NODE result = node.table_checksum("pgbench_accounts") # delta BACKUP - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='delta', options=['--stream']) # GET PHYSICAL CONTENT FROM NODE pgdata = self.pgdata_content(node.data_dir) # RESTORE NODE - restored_node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) + restored_node = self.pg_node.make_simple('restored_node') restored_node.cleanup() tblspc_path = self.get_tblspace_path(node, 'somedata') tblspc_path_new = self.get_tblspace_path( restored_node, 'somedata_restored') - self.restore_node( - backup_dir, 'node', restored_node, + self.pb.restore_node('node', restored_node, options=[ "-j", "4", "-T", "{0}={1}".format( tblspc_path, tblspc_path_new)]) @@ -426,7 +377,7 @@ def test_delta_multiple_segments(self): pgdata_restored = self.pgdata_content(restored_node.data_dir) # START RESTORED NODE - self.set_auto_conf(restored_node, {'port': restored_node.port}) + restored_node.set_auto_conf({'port': restored_node.port}) restored_node.slow_start() result_new = restored_node.table_checksum("pgbench_accounts") @@ -438,29 +389,26 @@ def test_delta_multiple_segments(self): self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") + @needs_gdb def test_delta_vacuum_full(self): """ make node, make full and delta stream backups, restore them and check data correctness """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() self.create_tblspace_in_node(node, 'somedata') - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) node.safe_psql( "postgres", @@ -479,27 +427,24 @@ def test_delta_vacuum_full(self): target=pg_connect.execute, args=["VACUUM FULL t_heap"]) process.start() - while not gdb.stopped_in_breakpoint: - time.sleep(1) + gdb.stopped_in_breakpoint() gdb.continue_execution_until_break(20) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='delta', options=['--stream']) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) gdb.remove_all_breakpoints() - gdb._execute('detach') + gdb.detach() process.join() old_tablespace = self.get_tblspace_path(node, 'somedata') new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=["-j", "4", "-T", "{0}={1}".format( old_tablespace, new_tablespace)]) @@ -508,7 +453,7 @@ def test_delta_vacuum_full(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() @@ -518,18 +463,16 @@ def test_create_db(self): Make node, take full backup, create database db1, take delta backup, restore database and check it presense """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'max_wal_size': '10GB', } ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # FULL BACKUP @@ -539,8 +482,7 @@ def test_create_db(self): "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") node.table_checksum("t_heap") - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=["--stream"]) # CREATE DATABASE DB1 @@ -551,8 +493,7 @@ def test_create_db(self): "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") # DELTA BACKUP - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, backup_type='delta', options=["--stream"] ) @@ -561,13 +502,10 @@ def test_create_db(self): pgdata = self.pgdata_content(node.data_dir) # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored') - ) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, + self.pb.restore_node( 'node', node_restored, backup_id=backup_id, @@ -582,15 +520,14 @@ def test_create_db(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() # DROP DATABASE DB1 node.safe_psql( "postgres", "drop database db1") # SECOND DELTA BACKUP - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, backup_type='delta', options=["--stream"] ) @@ -599,8 +536,7 @@ def test_create_db(self): # RESTORE SECOND DELTA BACKUP node_restored.cleanup() - self.restore_node( - backup_dir, + self.pb.restore_node( 'node', node_restored, backup_id=backup_id, @@ -616,22 +552,11 @@ def test_create_db(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() - try: - node_restored.safe_psql('db1', 'select 1') - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because we are connecting to deleted database" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except QueryException as e: - self.assertTrue( - 'FATAL: database "db1" does not exist' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + error_result = node_restored.safe_psql('db1', 'select 1', expect_error=True) + self.assertMessage(error_result, contains='FATAL: database "db1" does not exist') # @unittest.skip("skip") def test_exists_in_previous_backup(self): @@ -639,20 +564,18 @@ def test_exists_in_previous_backup(self): Make node, take full backup, create table, take page backup, take delta backup, check that file is no fully copied to delta backup """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'max_wal_size': '10GB', 'checkpoint_timeout': '5min', } ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL BACKUP @@ -665,23 +588,20 @@ def test_exists_in_previous_backup(self): filepath = node.safe_psql( "postgres", "SELECT pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - self.backup_node( - backup_dir, + self.pb.backup_node( 'node', node, options=["--stream"]) # PAGE BACKUP - backup_id = self.backup_node( - backup_dir, + backup_id = self.pb.backup_node( 'node', node, backup_type='page' ) - fullpath = os.path.join( - backup_dir, 'backups', 'node', backup_id, 'database', filepath) - self.assertFalse(os.path.exists(fullpath)) + self.assertFalse(self.backup_file_exists(backup_dir, 'node', backup_id, + f'database/{filepath}')) # if self.paranoia: # pgdata_page = self.pgdata_content( @@ -690,8 +610,7 @@ def test_exists_in_previous_backup(self): # 'node', backup_id, 'database')) # DELTA BACKUP - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, backup_type='delta', options=["--stream"] ) @@ -703,21 +622,18 @@ def test_exists_in_previous_backup(self): # self.compare_pgdata( # pgdata_page, pgdata_delta) - fullpath = os.path.join( - backup_dir, 'backups', 'node', backup_id, 'database', filepath) - self.assertFalse(os.path.exists(fullpath)) + self.assertFalse(self.backup_file_exists(backup_dir, 'node', backup_id, + f'database/{filepath}')) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored') + node_restored = self.pg_node.make_simple('node_restored' ) node_restored.cleanup() - self.restore_node( - backup_dir, + self.pb.restore_node( 'node', node_restored, backup_id=backup_id, @@ -732,7 +648,7 @@ def test_exists_in_previous_backup(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() # @unittest.skip("skip") @@ -741,17 +657,15 @@ def test_alter_table_set_tablespace_delta(self): Make node, create tablespace with table, take full backup, alter tablespace location, take delta backup, restore database. """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', + set_replication=True, pg_options={ 'checkpoint_timeout': '30s', } ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # FULL BACKUP @@ -763,7 +677,7 @@ def test_alter_table_set_tablespace_delta(self): " from generate_series(0,100) i") # FULL backup - self.backup_node(backup_dir, 'node', node, options=["--stream"]) + self.pb.backup_node('node', node, options=["--stream"]) # ALTER TABLESPACE self.create_tblspace_in_node(node, 'somedata_new') @@ -773,8 +687,7 @@ def test_alter_table_set_tablespace_delta(self): # DELTA BACKUP result = node.table_checksum("t_heap") - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='delta', options=["--stream"]) @@ -782,12 +695,10 @@ def test_alter_table_set_tablespace_delta(self): pgdata = self.pgdata_content(node.data_dir) # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=[ "-j", "4", "-T", "{0}={1}".format( @@ -807,7 +718,7 @@ def test_alter_table_set_tablespace_delta(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() result_new = node_restored.table_checksum("t_heap") @@ -821,20 +732,18 @@ def test_alter_database_set_tablespace_delta(self): take delta backup, alter database tablespace location, take delta backup restore last delta backup. """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() self.create_tblspace_in_node(node, 'somedata') # FULL backup - self.backup_node(backup_dir, 'node', node, options=["--stream"]) + self.pb.backup_node('node', node, options=["--stream"]) # CREATE DATABASE DB1 node.safe_psql( @@ -846,8 +755,7 @@ def test_alter_database_set_tablespace_delta(self): "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='delta', options=["--stream"] ) @@ -860,8 +768,7 @@ def test_alter_database_set_tablespace_delta(self): ) # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='delta', options=["--stream"] ) @@ -870,13 +777,11 @@ def test_alter_database_set_tablespace_delta(self): pgdata = self.pgdata_content(node.data_dir) # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored') + node_restored = self.pg_node.make_simple('node_restored' ) node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=[ "-j", "4", "-T", "{0}={1}".format( @@ -896,7 +801,7 @@ def test_alter_database_set_tablespace_delta(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() # @unittest.skip("skip") @@ -905,24 +810,22 @@ def test_delta_delete(self): Make node, create tablespace with table, take full backup, alter tablespace location, take delta backup, restore database. """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', + set_replication=True, pg_options={ 'checkpoint_timeout': '30s', } ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() self.create_tblspace_in_node(node, 'somedata') # FULL backup - self.backup_node(backup_dir, 'node', node, options=["--stream"]) + self.pb.backup_node('node', node, options=["--stream"]) node.safe_psql( "postgres", @@ -942,8 +845,7 @@ def test_delta_delete(self): ) # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='delta', options=["--stream"] ) @@ -952,13 +854,10 @@ def test_delta_delete(self): pgdata = self.pgdata_content(node.data_dir) # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored') - ) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=[ "-j", "4", "-T", "{0}={1}".format( @@ -974,7 +873,7 @@ def test_delta_delete(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() def test_delta_nullified_heap_page_backup(self): @@ -982,14 +881,12 @@ def test_delta_nullified_heap_page_backup(self): make node, take full backup, nullify some heap block, take delta backup, restore, physically compare pgdata`s """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=1) @@ -1002,8 +899,7 @@ def test_delta_nullified_heap_page_backup(self): "postgres", "CHECKPOINT") - self.backup_node( - backup_dir, 'node', node) + self.pb.backup_node('node', node) # Nullify some block in PostgreSQL file = os.path.join(node.data_dir, file_path).replace("\\", "/") @@ -1014,20 +910,18 @@ def test_delta_nullified_heap_page_backup(self): f.seek(8192) f.write(b"\x00"*8192) f.flush() - f.close - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='delta', options=["--log-level-file=verbose"]) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) + content = self.read_pb_log() + self.assertIn( + 'VERBOSE: File: {0} blknum 1, empty zeroed page'.format(file_path), + content) if not self.remote: - log_file_path = os.path.join(backup_dir, "log", "pg_probackup.log") - with open(log_file_path) as f: - content = f.read() - self.assertIn( 'VERBOSE: File: "{0}" blknum 1, empty page'.format(file), content) @@ -1036,12 +930,10 @@ def test_delta_nullified_heap_page_backup(self): content) # Restore DELTA backup - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored) + self.pb.restore_node('node', node_restored) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -1052,66 +944,49 @@ def test_delta_backup_from_past(self): make node, take FULL stream backup, take DELTA stream backup, restore FULL backup, try to take second DELTA stream backup """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + instance_name = 'node' + node = self.pg_node.make_simple(instance_name, + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance(instance_name, node) node.slow_start() - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) + backup_id = self.pb.backup_node(instance_name, node, options=['--stream']) node.pgbench_init(scale=3) # First DELTA - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node(instance_name, node, backup_type='delta', options=['--stream']) # Restore FULL backup node.cleanup() - self.restore_node(backup_dir, 'node', node, backup_id=backup_id) + self.pb.restore_node(instance_name, node, backup_id=backup_id) node.slow_start() # Second DELTA backup - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because we are backing up an instance from the past" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: Current START LSN ' in e.message and - 'is lower than START LSN ' in e.message and - 'of previous backup ' in e.message and - 'It may indicate that we are trying ' - 'to backup PostgreSQL instance from the past' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - @unittest.skip("skip") + error_result = self.pb.backup_node( instance_name, node, + backup_type='delta', + options=['--stream'], + expect_error=True) + + self.assertMessage(error_result, regex=r'Current START LSN (\d+)/(\d+) is lower than START LSN (\d+)/(\d+) ' + r'of previous backup \w{6}. It may indicate that we are trying ' + r'to backup PostgreSQL instance from the past.') + # @unittest.expectedFailure def test_delta_pg_resetxlog(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - 'shared_buffers': '512MB', - 'max_wal_size': '3GB'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + instance_name = 'node' + node = self.pg_node.make_simple(instance_name, + set_replication=True, + pg_options={'shared_buffers': '512MB', + 'max_wal_size': '3GB'}) + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance(instance_name, node) node.slow_start() # Create table @@ -1121,11 +996,9 @@ def test_delta_pg_resetxlog(self): "create table t_heap " "as select nextval('t_seq')::int as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " -# "from generate_series(0,25600) i") "from generate_series(0,2560) i") - self.backup_node( - backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node(instance_name, node, options=['--stream']) node.safe_psql( 'postgres', @@ -1140,12 +1013,10 @@ def test_delta_pg_resetxlog(self): # now smack it with sledgehammer if node.major_version >= 10: pg_resetxlog_path = self.get_bin_path('pg_resetwal') - wal_dir = 'pg_wal' else: pg_resetxlog_path = self.get_bin_path('pg_resetxlog') - wal_dir = 'pg_xlog' - self.run_binary( + self.pb.run_binary( [ pg_resetxlog_path, '-D', @@ -1161,37 +1032,23 @@ def test_delta_pg_resetxlog(self): print("Die! Die! Why won't you die?... Why won't you die?") exit(1) - # take ptrack backup -# self.backup_node( -# backup_dir, 'node', node, -# backup_type='delta', options=['--stream']) - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because instance was brutalized by pg_resetxlog" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd) - ) - except ProbackupException as e: - self.assertIn( - 'Insert error message', - e.message, - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(e.message), self.cmd)) - -# pgdata = self.pgdata_content(node.data_dir) -# -# node_restored = self.make_simple_node( -# base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) -# node_restored.cleanup() -# -# self.restore_node( -# backup_dir, 'node', node_restored) -# -# pgdata_restored = self.pgdata_content(node_restored.data_dir) -# self.compare_pgdata(pgdata, pgdata_restored) + backup_id = self.pb.backup_node(instance_name, + node, + backup_type='delta', + options=['--stream']) + self.pb.validate(instance_name, backup_id) + + def test_delta_backup_before_full_will_fail(self): + instance_name = 'node' + node = self.pg_node.make_simple( + base_dir=instance_name) + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance(instance_name, node) + self.pb.set_archiving(instance_name, node) + node.slow_start() + + error_result = self.pb.backup_node(instance_name, node, backup_type="delta", expect_error=True) + self.assertMessage(error_result, + contains='ERROR: could not open file "pg_wal/00000001.history": No such file or directory') diff --git a/tests/exclude_test.py b/tests/exclude_test.py index cb3530cd5..1d6485ce8 100644 --- a/tests/exclude_test.py +++ b/tests/exclude_test.py @@ -1,25 +1,23 @@ import os import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest -class ExcludeTest(ProbackupTest, unittest.TestCase): +class ExcludeTest(ProbackupTest): # @unittest.skip("skip") def test_exclude_temp_files(self): """ """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'logging_collector': 'on', 'log_filename': 'postgresql.log'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() oid = node.safe_psql( @@ -36,16 +34,13 @@ def test_exclude_temp_files(self): f.flush() f.close - full_id = self.backup_node( - backup_dir, 'node', node, backup_type='full', options=['--stream']) - - file = os.path.join( - backup_dir, 'backups', 'node', full_id, - 'database', 'base', 'pgsql_tmp', 'pgsql_tmp7351.16') + full_id = self.pb.backup_node('node', node, backup_type='full', options=['--stream']) self.assertFalse( - os.path.exists(file), - "File must be excluded: {0}".format(file)) + self.backup_file_exists(backup_dir, 'node', full_id, + 'database/base/pgsql_tmp/pgsql_tmp7351.16'), + "File must be excluded: database/base/pgsql_tmp/pgsql_tmp7351.16" + ) # TODO check temporary tablespaces @@ -56,14 +51,12 @@ def test_exclude_temp_tables(self): make node without archiving, create temp table, take full backup, check that temp table not present in backup catalogue """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() with node.connect("postgres") as conn: @@ -112,8 +105,7 @@ def test_exclude_temp_tables(self): temp_toast_filename = os.path.basename(toast_path) temp_idx_toast_filename = os.path.basename(toast_idx_path) - self.backup_node( - backup_dir, 'node', node, backup_type='full', options=['--stream']) + self.pb.backup_node('node', node, backup_type='full', options=['--stream']) for root, dirs, files in os.walk(backup_dir): for file in files: @@ -138,16 +130,14 @@ def test_exclude_unlogged_tables_1(self): alter table to unlogged, take delta backup, restore delta backup, check that PGDATA`s are physically the same """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ "shared_buffers": "10MB"}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() conn = node.connect() @@ -161,25 +151,21 @@ def test_exclude_unlogged_tables_1(self): conn.execute("create index test_idx on test (generate_series)") conn.commit() - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='full', options=['--stream']) node.safe_psql('postgres', "alter table test set logged") - self.backup_node( - backup_dir, 'node', node, backup_type='delta', + self.pb.backup_node('node', node, backup_type='delta', options=['--stream']) pgdata = self.pgdata_content(node.data_dir) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) + self.pb.restore_node('node', node_restored, options=["-j", "4"]) # Physical comparison pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -193,17 +179,15 @@ def test_exclude_unlogged_tables_2(self): 2. restore FULL, DELTA, PAGE to empty db, ensure unlogged table exist and is epmty """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ "shared_buffers": "10MB"}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() backup_ids = [] @@ -223,14 +207,12 @@ def test_exclude_unlogged_tables_2(self): 'postgres', "select pg_relation_filepath('test')")[0][0] - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, backup_type=backup_type, options=['--stream']) backup_ids.append(backup_id) - filelist = self.get_backup_filelist( - backup_dir, 'node', backup_id) + filelist = self.get_backup_filelist(backup_dir, 'node', backup_id) self.assertNotIn( rel_path, filelist, @@ -253,7 +235,7 @@ def test_exclude_unlogged_tables_2(self): node.stop() node.cleanup() - self.restore_node(backup_dir, 'node', node, backup_id=backup_id) + self.pb.restore_node('node', node=node, backup_id=backup_id) node.slow_start() @@ -268,21 +250,18 @@ def test_exclude_log_dir(self): """ check that by default 'log' and 'pg_log' directories are not backed up """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'logging_collector': 'on', 'log_filename': 'postgresql.log'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='full', options=['--stream']) log_dir = node.safe_psql( @@ -291,8 +270,7 @@ def test_exclude_log_dir(self): node.cleanup() - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]) + self.pb.restore_node('node', node, options=["-j", "4"]) # check that PGDATA/log or PGDATA/pg_log do not exists path = os.path.join(node.data_dir, log_dir) @@ -305,31 +283,27 @@ def test_exclude_log_dir_1(self): """ check that "--backup-pg-log" works correctly """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'logging_collector': 'on', 'log_filename': 'postgresql.log'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() log_dir = node.safe_psql( 'postgres', 'show log_directory').decode('utf-8').rstrip() - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='full', options=['--stream', '--backup-pg-log']) node.cleanup() - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]) + self.pb.restore_node('node', node, options=["-j", "4"]) # check that PGDATA/log or PGDATA/pg_log do not exists path = os.path.join(node.data_dir, log_dir) diff --git a/tests/expected/option_help.out b/tests/expected/option_help.out index 49f79607f..31990287b 100644 --- a/tests/expected/option_help.out +++ b/tests/expected/option_help.out @@ -5,9 +5,13 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. pg_probackup version - pg_probackup init -B backup-path + pg_probackup init -B backup-dir + [--s3=s3-interface-provider] + [--skip-if-exists] + [--dry-run] + [--help] - pg_probackup set-config -B backup-path --instance=instance_name + pg_probackup set-config -B backup-dir --instance=instance-name [-D pgdata-path] [--external-dirs=external-directories-paths] [--log-level-console=log-level-console] @@ -30,18 +34,22 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--ssh-options] [--restore-command=cmdline] [--archive-host=destination] [--archive-port=port] [--archive-user=username] + [--write-rate-limit=baudrate] + [--dry-run] [--help] - pg_probackup set-backup -B backup-path --instance=instance_name + pg_probackup set-backup -B backup-dir --instance=instance-name -i backup-id [--ttl=interval] [--expire-time=timestamp] [--note=text] + [--dry-run] [--help] - pg_probackup show-config -B backup-path --instance=instance_name + pg_probackup show-config -B backup-dir --instance=instance-name [--format=format] + [--no-scale-units] [--help] - pg_probackup backup -B backup-path -b backup-mode --instance=instance_name + pg_probackup backup -B backup-dir -b backup-mode --instance=instance-name [-D pgdata-path] [-C] [--stream [-S slot-name] [--temp-slot]] [--backup-pg-log] [-j num-threads] [--progress] @@ -71,9 +79,13 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--remote-port] [--remote-path] [--remote-user] [--ssh-options] [--ttl=interval] [--expire-time=timestamp] [--note=text] + [--write-rate-limit=baudrate] + [--s3=s3-interface-provider] + [--cfs-nondatafile-mode] + [--dry-run] [--help] - pg_probackup restore -B backup-path --instance=instance_name + pg_probackup restore -B backup-dir --instance=instance-name [-D pgdata-path] [-i backup-id] [-j num-threads] [--recovery-target-time=time|--recovery-target-xid=xid |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] @@ -98,29 +110,35 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--ssh-options] [--archive-host=hostname] [--archive-port=port] [--archive-user=username] + [--s3=s3-interface-provider] + [--dry-run] [--help] - pg_probackup validate -B backup-path [--instance=instance_name] + pg_probackup validate -B backup-dir [--instance=instance-name] [-i backup-id] [--progress] [-j num-threads] [--recovery-target-time=time|--recovery-target-xid=xid |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] [--recovery-target-timeline=timeline] [--recovery-target-name=target-name] [--skip-block-validation] + [--s3=s3-interface-provider] + [--wal] [--help] - pg_probackup checkdb [-B backup-path] [--instance=instance_name] + pg_probackup checkdb [-B backup-dir] [--instance=instance-name] [-D pgdata-path] [--progress] [-j num-threads] [--amcheck] [--skip-block-validation] [--heapallindexed] [--checkunique] [--help] - pg_probackup show -B backup-path - [--instance=instance_name [-i backup-id]] + pg_probackup show -B backup-dir + [--instance=instance-name [-i backup-id]] [--format=format] [--archive] - [--no-color] [--help] + [--no-color] [--show-symlinks] + [--s3=s3-interface-provider] + [--help] - pg_probackup delete -B backup-path --instance=instance_name + pg_probackup delete -B backup-dir --instance=instance-name [-j num-threads] [--progress] [--retention-redundancy=retention-redundancy] [--retention-window=retention-window] @@ -128,26 +146,35 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [-i backup-id | --delete-expired | --merge-expired | --status=backup_status] [--delete-wal] [--dry-run] [--no-validate] [--no-sync] + [--s3=s3-interface-provider] [--help] - pg_probackup merge -B backup-path --instance=instance_name + pg_probackup merge -B backup-dir --instance=instance-name -i backup-id [--progress] [-j num-threads] [--no-validate] [--no-sync] + [--dry-run] [--help] - pg_probackup add-instance -B backup-path -D pgdata-path - --instance=instance_name + pg_probackup add-instance -B backup-dir -D pgdata-path + --instance=instance-name [--external-dirs=external-directories-paths] [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] + [--s3=s3-interface-provider] + [--skip-if-exists] + [--dry-run] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] [--help] - pg_probackup del-instance -B backup-path - --instance=instance_name + pg_probackup del-instance -B backup-dir + --instance=instance-name + [--s3=s3-interface-provider] + [--dry-run] [--help] - pg_probackup archive-push -B backup-path --instance=instance_name + pg_probackup archive-push -B backup-dir --instance=instance-name --wal-file-name=wal-file-name [--wal-file-path=wal-file-path] [-j num-threads] [--batch-size=batch_size] @@ -159,9 +186,10 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] + [--dry-run] [--help] - pg_probackup archive-get -B backup-path --instance=instance_name + pg_probackup archive-get -B backup-dir --instance=instance-name --wal-file-path=wal-file-path --wal-file-name=wal-file-name [-j num-threads] [--batch-size=batch_size] @@ -169,6 +197,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] + [--dry-run] [--help] pg_probackup catchup -b catchup-mode @@ -178,6 +207,7 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [-j num-threads] [-T OLDDIR=NEWDIR] [--exclude-path=path_prefix] + [-X WALDIR | --waldir=WALDIR] [-d dbname] [-h host] [-p port] [-U username] [-w --no-password] [-W --password] [--remote-proto] [--remote-host] @@ -186,5 +216,31 @@ pg_probackup - utility to manage backup/recovery of PostgreSQL database. [--dry-run] [--help] + pg_probackup S3 Enviroment variables +PG_PROBACKUP_S3_HOST Host name of the S3 server +PG_PROBACKUP_S3_PORT Port of the S3 server +PG_PROBACKUP_S3_REGION Region of the S3 server +PG_PROBACKUP_S3_BUCKET_NAME Bucket on the S3 server +PG_PROBACKUP_S3_ACCESS_KEY, +PG_PROBACKUP_S3_SECRET_ACCESS_KEY Secure tokens on the S3 server +PG_PROBACKUP_S3_HTTPS S3 HTTP protocol + Set to ON or HTTPS for HTTPS or to any other + value for HTTP. + Default values: + HTTP if --s3=minio is specified + HTTPS otherwise +PG_PROBACKUP_S3_BUFFER_SIZE Size of the read/write buffer (in MiB) for + communicating with S3 (default: 16) +PG_PROBACKUP_S3_RETRIES Maximum number of attempts to execute an S3 + request in case of failures (default: 5) +PG_PROBACKUP_S3_TIMEOUT Maximum allowable amount of time (in seconds) + for transferring PG_PROBACKUP_S3_BUFFER_SIZE + of data to/from S3 (default: 300) +PG_PROBACKUP_S3_IGNORE_CERT_VER Don't verify the certificate host and peer +PG_PROBACKUP_S3_CA_CERTIFICATE Trust to the path to Certificate Authority (CA) bundle +PG_PROBACKUP_S3_CA_PATH Trust to the directory holding CA certificates +PG_PROBACKUP_S3_CLIENT_CERT Setup SSL client certificate +PG_PROBACKUP_S3_CLIENT_KEY Setup private key file for TLS and SSL client certificate + Read the website for details . Report bugs to . diff --git a/tests/expected/option_help_ru.out b/tests/expected/option_help_ru.out index 976932b9d..c152fe0b8 100644 --- a/tests/expected/option_help_ru.out +++ b/tests/expected/option_help_ru.out @@ -5,9 +5,13 @@ pg_probackup - утилита для управления резервным к pg_probackup version - pg_probackup init -B backup-path + pg_probackup init -B backup-dir + [--s3=s3-interface-provider] + [--skip-if-exists] + [--dry-run] + [--help] - pg_probackup set-config -B backup-path --instance=instance_name + pg_probackup set-config -B backup-dir --instance=instance-name [-D pgdata-path] [--external-dirs=external-directories-paths] [--log-level-console=log-level-console] @@ -30,18 +34,22 @@ pg_probackup - утилита для управления резервным к [--ssh-options] [--restore-command=cmdline] [--archive-host=destination] [--archive-port=port] [--archive-user=username] + [--write-rate-limit=baudrate] + [--dry-run] [--help] - pg_probackup set-backup -B backup-path --instance=instance_name + pg_probackup set-backup -B backup-dir --instance=instance-name -i backup-id [--ttl=interval] [--expire-time=timestamp] [--note=text] + [--dry-run] [--help] - pg_probackup show-config -B backup-path --instance=instance_name + pg_probackup show-config -B backup-dir --instance=instance-name [--format=format] + [--no-scale-units] [--help] - pg_probackup backup -B backup-path -b backup-mode --instance=instance_name + pg_probackup backup -B backup-dir -b backup-mode --instance=instance-name [-D pgdata-path] [-C] [--stream [-S slot-name] [--temp-slot]] [--backup-pg-log] [-j num-threads] [--progress] @@ -71,9 +79,13 @@ pg_probackup - утилита для управления резервным к [--remote-port] [--remote-path] [--remote-user] [--ssh-options] [--ttl=interval] [--expire-time=timestamp] [--note=text] + [--write-rate-limit=baudrate] + [--s3=s3-interface-provider] + [--cfs-nondatafile-mode] + [--dry-run] [--help] - pg_probackup restore -B backup-path --instance=instance_name + pg_probackup restore -B backup-dir --instance=instance-name [-D pgdata-path] [-i backup-id] [-j num-threads] [--recovery-target-time=time|--recovery-target-xid=xid |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] @@ -98,29 +110,35 @@ pg_probackup - утилита для управления резервным к [--ssh-options] [--archive-host=hostname] [--archive-port=port] [--archive-user=username] + [--s3=s3-interface-provider] + [--dry-run] [--help] - pg_probackup validate -B backup-path [--instance=instance_name] + pg_probackup validate -B backup-dir [--instance=instance-name] [-i backup-id] [--progress] [-j num-threads] [--recovery-target-time=time|--recovery-target-xid=xid |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]] [--recovery-target-timeline=timeline] [--recovery-target-name=target-name] [--skip-block-validation] + [--s3=s3-interface-provider] + [--wal] [--help] - pg_probackup checkdb [-B backup-path] [--instance=instance_name] + pg_probackup checkdb [-B backup-dir] [--instance=instance-name] [-D pgdata-path] [--progress] [-j num-threads] [--amcheck] [--skip-block-validation] [--heapallindexed] [--checkunique] [--help] - pg_probackup show -B backup-path - [--instance=instance_name [-i backup-id]] + pg_probackup show -B backup-dir + [--instance=instance-name [-i backup-id]] [--format=format] [--archive] - [--no-color] [--help] + [--no-color] [--show-symlinks] + [--s3=s3-interface-provider] + [--help] - pg_probackup delete -B backup-path --instance=instance_name + pg_probackup delete -B backup-dir --instance=instance-name [-j num-threads] [--progress] [--retention-redundancy=retention-redundancy] [--retention-window=retention-window] @@ -128,26 +146,35 @@ pg_probackup - утилита для управления резервным к [-i backup-id | --delete-expired | --merge-expired | --status=backup_status] [--delete-wal] [--dry-run] [--no-validate] [--no-sync] + [--s3=s3-interface-provider] [--help] - pg_probackup merge -B backup-path --instance=instance_name + pg_probackup merge -B backup-dir --instance=instance-name -i backup-id [--progress] [-j num-threads] [--no-validate] [--no-sync] + [--dry-run] [--help] - pg_probackup add-instance -B backup-path -D pgdata-path - --instance=instance_name + pg_probackup add-instance -B backup-dir -D pgdata-path + --instance=instance-name [--external-dirs=external-directories-paths] [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] + [--s3=s3-interface-provider] + [--skip-if-exists] + [--dry-run] + [--compress-algorithm=compress-algorithm] + [--compress-level=compress-level] [--help] - pg_probackup del-instance -B backup-path - --instance=instance_name + pg_probackup del-instance -B backup-dir + --instance=instance-name + [--s3=s3-interface-provider] + [--dry-run] [--help] - pg_probackup archive-push -B backup-path --instance=instance_name + pg_probackup archive-push -B backup-dir --instance=instance-name --wal-file-name=wal-file-name [--wal-file-path=wal-file-path] [-j num-threads] [--batch-size=batch_size] @@ -159,9 +186,10 @@ pg_probackup - утилита для управления резервным к [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] + [--dry-run] [--help] - pg_probackup archive-get -B backup-path --instance=instance_name + pg_probackup archive-get -B backup-dir --instance=instance-name --wal-file-path=wal-file-path --wal-file-name=wal-file-name [-j num-threads] [--batch-size=batch_size] @@ -169,6 +197,7 @@ pg_probackup - утилита для управления резервным к [--remote-proto] [--remote-host] [--remote-port] [--remote-path] [--remote-user] [--ssh-options] + [--dry-run] [--help] pg_probackup catchup -b catchup-mode @@ -178,6 +207,7 @@ pg_probackup - утилита для управления резервным к [-j num-threads] [-T OLDDIR=NEWDIR] [--exclude-path=path_prefix] + [-X WALDIR | --waldir=WALDIR] [-d dbname] [-h host] [-p port] [-U username] [-w --no-password] [-W --password] [--remote-proto] [--remote-host] @@ -186,5 +216,31 @@ pg_probackup - утилита для управления резервным к [--dry-run] [--help] + pg_probackup S3 Enviroment variables +PG_PROBACKUP_S3_HOST Host name of the S3 server +PG_PROBACKUP_S3_PORT Port of the S3 server +PG_PROBACKUP_S3_REGION Region of the S3 server +PG_PROBACKUP_S3_BUCKET_NAME Bucket on the S3 server +PG_PROBACKUP_S3_ACCESS_KEY, +PG_PROBACKUP_S3_SECRET_ACCESS_KEY Secure tokens on the S3 server +PG_PROBACKUP_S3_HTTPS S3 HTTP protocol + Set to ON or HTTPS for HTTPS or to any other + value for HTTP. + Default values: + HTTP if --s3=minio is specified + HTTPS otherwise +PG_PROBACKUP_S3_BUFFER_SIZE Size of the read/write buffer (in MiB) for + communicating with S3 (default: 16) +PG_PROBACKUP_S3_RETRIES Maximum number of attempts to execute an S3 + request in case of failures (default: 5) +PG_PROBACKUP_S3_TIMEOUT Maximum allowable amount of time (in seconds) + for transferring PG_PROBACKUP_S3_BUFFER_SIZE + of data to/from S3 (default: 300) +PG_PROBACKUP_S3_IGNORE_CERT_VER Don't verify the certificate host and peer +PG_PROBACKUP_S3_CA_CERTIFICATE Trust to the path to Certificate Authority (CA) bundle +PG_PROBACKUP_S3_CA_PATH Trust to the directory holding CA certificates +PG_PROBACKUP_S3_CLIENT_CERT Setup SSL client certificate +PG_PROBACKUP_S3_CLIENT_KEY Setup private key file for TLS and SSL client certificate + Подробнее читайте на сайте . Сообщайте об ошибках в . diff --git a/tests/expected/option_version.out b/tests/expected/option_version.out deleted file mode 100644 index 0d50cb268..000000000 --- a/tests/expected/option_version.out +++ /dev/null @@ -1 +0,0 @@ -pg_probackup 2.5.12 diff --git a/tests/external_test.py b/tests/external_test.py index 53f3c5449..dd6aa1b24 100644 --- a/tests/external_test.py +++ b/tests/external_test.py @@ -1,13 +1,12 @@ import unittest import os from time import sleep -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from .helpers.cfs_helpers import find_by_name +from .helpers.ptrack_helpers import ProbackupTest, fs_backup_class import shutil # TODO: add some ptrack tests -class ExternalTest(ProbackupTest, unittest.TestCase): +class ExternalTest(ProbackupTest): # @unittest.skip("skip") # @unittest.expectedFailure @@ -17,60 +16,41 @@ def test_basic_external(self): with external directory, restore backup, check that external directory was successfully copied """ - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + backup_dir = self.backup_dir external_dir = self.get_tblspace_path(node, 'somedirectory') # create directory in external_directory - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # take FULL backup with external directory pointing to a file - file_path = os.path.join(core_dir, 'file') + file_path = os.path.join(self.test_path, 'file') with open(file_path, "w+") as f: pass - try: - self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=[ - '--external-dirs={0}'.format(file_path)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because external dir point to a file" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: --external-dirs option' in e.message and - 'directory or symbolic link expected' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, backup_type="full", + options=[ + '--external-dirs={0}'.format(file_path)], + expect_error="because external dir point to a file") + self.assertMessage(contains='ERROR: --external-dirs option') + self.assertMessage(contains='directory or symbolic link expected') sleep(1) # FULL backup - self.backup_node( - backup_dir, 'node', node, backup_type="full", + self.pb.backup_node('node', node, backup_type="full", options=["-j", "4", "--stream"]) # Fill external directories - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir, options=["-j", "4"]) # Full backup with external dir - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=[ '--external-dirs={0}'.format(external_dir)]) @@ -80,8 +60,7 @@ def test_basic_external(self): node.cleanup() shutil.rmtree(node.base_dir, ignore_errors=True) - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]) + self.pb.restore_node('node', node=node, options=["-j", "4"]) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -97,39 +76,32 @@ def test_external_none(self): restore delta backup, check that external directory was not copied """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + backup_dir = self.backup_dir external_dir = self.get_tblspace_path(node, 'somedirectory') # create directory in external_directory - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # FULL backup - self.backup_node( - backup_dir, 'node', node, backup_type="full", + self.pb.backup_node('node', node, backup_type="full", options=["-j", "4", "--stream"]) # Fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir, options=["-j", "4"]) # Full backup with external dir - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=[ '--stream', '--external-dirs={0}'.format(external_dir)]) # Delta backup without external directory - self.backup_node( - backup_dir, 'node', node, backup_type="delta", + self.pb.backup_node('node', node, backup_type="delta", options=['--external-dirs=none', '--stream']) shutil.rmtree(external_dir, ignore_errors=True) @@ -139,8 +111,7 @@ def test_external_none(self): node.cleanup() shutil.rmtree(node.base_dir, ignore_errors=True) - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]) + self.pb.restore_node('node', node=node, options=["-j", "4"]) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -154,47 +125,34 @@ def test_external_dirs_overlapping(self): take backup with two external directories pointing to the same directory, backup should fail """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + backup_dir = self.backup_dir external_dir1 = self.get_tblspace_path(node, 'external_dir1') external_dir2 = self.get_tblspace_path(node, 'external_dir2') # create directory in external_directory - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() os.mkdir(external_dir1) os.mkdir(external_dir2) # Full backup with external dirs - try: - self.backup_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--stream", - "-E", "{0}{1}{2}{1}{0}".format( - external_dir1, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir2, - self.EXTERNAL_DIRECTORY_DELIMITER, - external_dir1)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because tablespace mapping is incorrect" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: External directory path (-E option)' in e.message and - 'contain another external directory' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, + options=[ + "-j", "4", "--stream", + "-E", "{0}{1}{2}{1}{0}".format( + external_dir1, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir2, + self.EXTERNAL_DIRECTORY_DELIMITER, + external_dir1)], + expect_error="because tablespace mapping is incorrect") + self.assertMessage(regex=r'ERROR: External directory path \(-E option\) ".*" ' + r'contain another external directory') # @unittest.skip("skip") def test_external_dir_mapping(self): @@ -204,64 +162,42 @@ def test_external_dir_mapping(self): check that restore with external-dir mapping will end with success """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - self.backup_node( - backup_dir, 'node', node, backup_type="full", + self.pb.backup_node('node', node, backup_type="full", options=["-j", "4", "--stream"]) external_dir1 = self.get_tblspace_path(node, 'external_dir1') external_dir2 = self.get_tblspace_path(node, 'external_dir2') # Fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir1, options=["-j", "4"]) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir2, options=["-j", "4"]) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2') - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=[ - "-j", "4", - "--external-mapping={0}={1}".format( - external_dir1, external_dir1_new), - "--external-mapping={0}={1}".format( - external_dir2, external_dir2_new)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because tablespace mapping is incorrect" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: --external-mapping option' in e.message and - 'have an entry in list of external directories' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.backup_node( - backup_dir, 'node', node, backup_type="delta", + self.pb.restore_node('node', node=node_restored, + options=[ + "-j", "4", + "--external-mapping={0}={1}".format( + external_dir1, external_dir1_new), + "--external-mapping={0}={1}".format( + external_dir2, external_dir2_new)], + expect_error="because tablespace mapping is incorrect") + self.assertMessage(contains=r"ERROR: --external-mapping option's old directory " + r"doesn't have an entry in list of external directories") + + self.pb.backup_node('node', node, backup_type="delta", options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( @@ -272,8 +208,7 @@ def test_external_dir_mapping(self): pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node=node_restored, options=[ "-j", "4", "--external-mapping={0}={1}".format( @@ -289,40 +224,30 @@ def test_external_dir_mapping(self): # @unittest.expectedFailure def test_backup_multiple_external(self): """check that cmdline has priority over config""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() external_dir1 = self.get_tblspace_path(node, 'external_dir1') external_dir2 = self.get_tblspace_path(node, 'external_dir2') # FULL backup - self.backup_node( - backup_dir, 'node', node, backup_type="full", + self.pb.backup_node('node', node, backup_type="full", options=["-j", "4", "--stream"]) # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir1, options=["-j", "4"]) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir2, options=["-j", "4"]) - self.set_config( - backup_dir, 'node', + self.pb.set_config('node', options=['-E', external_dir1]) # cmdline option MUST override options in config - self.backup_node( - backup_dir, 'node', node, backup_type="delta", + self.pb.backup_node('node', node, backup_type="delta", options=[ "-j", "4", "--stream", "-E", external_dir2]) @@ -334,8 +259,7 @@ def test_backup_multiple_external(self): shutil.rmtree(external_dir1, ignore_errors=True) shutil.rmtree(external_dir2, ignore_errors=True) - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node=node, options=["-j", "4"]) pgdata_restored = self.pgdata_content( @@ -354,64 +278,50 @@ def test_external_backward_compatibility(self): if not self.probackup_old_path: self.skipTest("You must specify PGPROBACKUPBIN_OLD" " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir, old_binary=True) - self.show_pb(backup_dir) + self.pb.init(old_binary=True) + self.pb.show() - self.add_instance(backup_dir, 'node', node, old_binary=True) - self.show_pb(backup_dir) + self.pb.add_instance('node', node, old_binary=True) + self.pb.show() node.slow_start() node.pgbench_init(scale=3) # FULL backup with old binary without external dirs support - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, old_binary=True, options=["-j", "4", "--stream"]) external_dir1 = self.get_tblspace_path(node, 'external_dir1') external_dir2 = self.get_tblspace_path(node, 'external_dir2') # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) + self.pb.restore_node('node', external_dir1, options=["-j", "4"]) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) + self.pb.restore_node('node', external_dir2, options=["-j", "4"]) pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) pgbench.wait() # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, old_binary=True, options=["-j", "4", "--stream"]) # fill external directories with changed data shutil.rmtree(external_dir1, ignore_errors=True) shutil.rmtree(external_dir2, ignore_errors=True) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) + self.pb.restore_node('node', external_dir1, options=["-j", "4"]) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) + self.pb.restore_node('node', external_dir2, options=["-j", "4"]) - self.delete_pb(backup_dir, 'node', backup_id=backup_id) + self.pb.delete('node', backup_id=backup_id) # delta backup with external directories using new binary - self.backup_node( - backup_dir, 'node', node, backup_type="delta", + self.pb.backup_node('node', node, backup_type="delta", options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( @@ -423,16 +333,14 @@ def test_external_backward_compatibility(self): node.base_dir, exclude_dirs=['logs']) # RESTORE chain with new binary - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2') - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=[ "-j", "4", "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), @@ -444,7 +352,6 @@ def test_external_backward_compatibility(self): self.compare_pgdata(pgdata, pgdata_restored) # @unittest.expectedFailure - # @unittest.skip("skip") def test_external_backward_compatibility_merge_1(self): """ take backup with old binary without external dirs support @@ -454,52 +361,42 @@ def test_external_backward_compatibility_merge_1(self): if not self.probackup_old_path: self.skipTest("You must specify PGPROBACKUPBIN_OLD" " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir, old_binary=True) - self.show_pb(backup_dir) + self.pb.init(old_binary=True) + self.pb.show() - self.add_instance(backup_dir, 'node', node, old_binary=True) - self.show_pb(backup_dir) + self.pb.add_instance('node', node, old_binary=True) + self.pb.show() node.slow_start() node.pgbench_init(scale=3) # tmp FULL backup with old binary - tmp_id = self.backup_node( - backup_dir, 'node', node, + tmp_id = self.pb.backup_node('node', node, old_binary=True, options=["-j", "4", "--stream"]) external_dir1 = self.get_tblspace_path(node, 'external_dir1') external_dir2 = self.get_tblspace_path(node, 'external_dir2') # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) + self.pb.restore_node('node', external_dir1, options=["-j", "4"]) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) + self.pb.restore_node('node', external_dir2, options=["-j", "4"]) - self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + self.pb.delete('node', backup_id=tmp_id) # FULL backup with old binary without external dirs support - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, old_binary=True, options=["-j", "4", "--stream"]) pgbench = node.pgbench(options=['-T', '30', '-c', '1']) pgbench.wait() # delta backup with external directories using new binary - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="delta", + backup_id = self.pb.backup_node('node', node, backup_type="delta", options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( @@ -511,19 +408,17 @@ def test_external_backward_compatibility_merge_1(self): node.base_dir, exclude_dirs=['logs']) # Merge chain chain with new binary - self.merge_backup(backup_dir, 'node', backup_id=backup_id) + self.pb.merge_backup('node', backup_id=backup_id) # Restore merged backup - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() external_dir1_new = self.get_tblspace_path(node_restored, 'external_dir1') external_dir2_new = self.get_tblspace_path(node_restored, 'external_dir2') - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=[ "-j", "4", "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), @@ -535,7 +430,6 @@ def test_external_backward_compatibility_merge_1(self): self.compare_pgdata(pgdata, pgdata_restored) # @unittest.expectedFailure - # @unittest.skip("skip") def test_external_backward_compatibility_merge_2(self): """ take backup with old binary without external dirs support @@ -545,52 +439,42 @@ def test_external_backward_compatibility_merge_2(self): if not self.probackup_old_path: self.skipTest("You must specify PGPROBACKUPBIN_OLD" " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir, old_binary=True) - self.show_pb(backup_dir) + self.pb.init(old_binary=True) + self.pb.show() - self.add_instance(backup_dir, 'node', node, old_binary=True) - self.show_pb(backup_dir) + self.pb.add_instance('node', node, old_binary=True) + self.pb.show() node.slow_start() node.pgbench_init(scale=3) # tmp FULL backup with old binary - tmp_id = self.backup_node( - backup_dir, 'node', node, + tmp_id = self.pb.backup_node('node', node, old_binary=True, options=["-j", "4", "--stream"]) external_dir1 = self.get_tblspace_path(node, 'external_dir1') external_dir2 = self.get_tblspace_path(node, 'external_dir2') # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) + self.pb.restore_node('node', external_dir1, options=["-j", "4"]) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) + self.pb.restore_node('node', external_dir2, options=["-j", "4"]) - self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + self.pb.delete('node', backup_id=tmp_id) # FULL backup with old binary without external dirs support - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, old_binary=True, options=["-j", "4", "--stream"]) pgbench = node.pgbench(options=['-T', '30', '-c', '1']) pgbench.wait() # delta backup with external directories using new binary - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type="delta", options=[ "-j", "4", "--stream", @@ -606,19 +490,14 @@ def test_external_backward_compatibility_merge_2(self): shutil.rmtree(external_dir1, ignore_errors=True) shutil.rmtree(external_dir2, ignore_errors=True) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, + self.pb.restore_node('node', external_dir1, options=['-j', '4', '--skip-external-dirs']) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, + self.pb.restore_node('node', external_dir2, options=['-j', '4', '--skip-external-dirs']) # delta backup without external directories using old binary - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, backup_type="delta", options=[ "-j", "4", "--stream", @@ -631,11 +510,10 @@ def test_external_backward_compatibility_merge_2(self): node.base_dir, exclude_dirs=['logs']) # Merge chain using new binary - self.merge_backup(backup_dir, 'node', backup_id=backup_id) + self.pb.merge_backup('node', backup_id=backup_id) # Restore merged backup - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() @@ -644,8 +522,7 @@ def test_external_backward_compatibility_merge_2(self): external_dir2_new = self.get_tblspace_path( node_restored, 'external_dir2') - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=[ "-j", "4", "--external-mapping={0}={1}".format( @@ -659,45 +536,45 @@ def test_external_backward_compatibility_merge_2(self): self.compare_pgdata(pgdata, pgdata_restored) # @unittest.expectedFailure - # @unittest.skip("skip") def test_external_merge(self): """""" if not self.probackup_old_path: self.skipTest("You must specify PGPROBACKUPBIN_OLD" " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node, old_binary=True) + self.pb.init() + self.pb.add_instance('node', node, old_binary=True) node.slow_start() node.pgbench_init(scale=3) # take temp FULL backup - tmp_id = self.backup_node( - backup_dir, 'node', node, options=["-j", "4", "--stream"]) + tmp_id = self.pb.backup_node('node', node, options=["-j", "4", "--stream"]) external_dir1 = self.get_tblspace_path(node, 'external_dir1') external_dir2 = self.get_tblspace_path(node, 'external_dir2') + self.create_tblspace_in_node(node, 'tblsp_1') + node.safe_psql( + "postgres", + "create table t_heap_lame tablespace tblsp_1 " + "as select 1 as id, md5(i::text) as text, " + "md5(i::text)::tsvector as tsvector " + "from generate_series(0,100) i") + # fill external directories with data - self.restore_node( - backup_dir, 'node', node, backup_id=tmp_id, - data_dir=external_dir1, options=["-j", "4"]) + self.pb.restore_node('node', external_dir1, backup_id=tmp_id, + options=["-j", "4"]) - self.restore_node( - backup_dir, 'node', node, backup_id=tmp_id, - data_dir=external_dir2, options=["-j", "4"]) + self.pb.restore_node('node', external_dir2, backup_id=tmp_id, + options=["-j", "4"]) - self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + self.pb.delete('node', backup_id=tmp_id) # FULL backup with old binary without external dirs support - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, old_binary=True, options=["-j", "4", "--stream"]) # change data a bit @@ -705,8 +582,7 @@ def test_external_merge(self): pgbench.wait() # delta backup with external directories using new binary - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="delta", + backup_id = self.pb.backup_node('node', node, backup_type="delta", options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( @@ -717,10 +593,10 @@ def test_external_merge(self): pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) - print(self.show_pb(backup_dir, 'node', as_json=False, as_text=True)) + print(self.pb.show('node', as_json=False, as_text=True)) # Merge - print(self.merge_backup(backup_dir, 'node', backup_id=backup_id, + print(self.pb.merge_backup('node', backup_id=backup_id, options=['--log-level-file=VERBOSE'])) # RESTORE @@ -730,8 +606,7 @@ def test_external_merge(self): external_dir1_new = self.get_tblspace_path(node, 'external_dir1') external_dir2_new = self.get_tblspace_path(node, 'external_dir2') - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ "-j", "4", "--external-mapping={0}={1}".format( @@ -748,43 +623,36 @@ def test_external_merge(self): # @unittest.skip("skip") def test_external_merge_skip_external_dirs(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.pgbench_init(scale=3) # FULL backup with old data - tmp_id = self.backup_node( - backup_dir, 'node', node, options=["-j", "4", "--stream"]) + tmp_id = self.pb.backup_node('node', node, options=["-j", "4", "--stream"]) external_dir1 = self.get_tblspace_path(node, 'external_dir1') external_dir2 = self.get_tblspace_path(node, 'external_dir2') # fill external directories with old data - self.restore_node( - backup_dir, 'node', node, backup_id=tmp_id, - data_dir=external_dir1, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir1, backup_id=tmp_id, + options=["-j", "4"]) - self.restore_node( - backup_dir, 'node', node, backup_id=tmp_id, - data_dir=external_dir2, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir2, backup_id=tmp_id, + options=["-j", "4"]) - self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + self.pb.delete('node', backup_id=tmp_id) # change data a bit pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) pgbench.wait() # FULL backup with external directories - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( @@ -797,19 +665,14 @@ def test_external_merge_skip_external_dirs(self): shutil.rmtree(external_dir2, ignore_errors=True) # fill external directories with new data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, + self.pb.restore_node('node', restore_dir=external_dir1, options=["-j", "4", "--skip-external-dirs"]) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, + self.pb.restore_node('node', restore_dir=external_dir2, options=["-j", "4", "--skip-external-dirs"]) # DELTA backup with external directories - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="delta", + backup_id = self.pb.backup_node('node', node, backup_type="delta", options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( @@ -821,16 +684,14 @@ def test_external_merge_skip_external_dirs(self): node.base_dir, exclude_dirs=['logs']) # merge backups without external directories - self.merge_backup( - backup_dir, 'node', + self.pb.merge_backup('node', backup_id=backup_id, options=['--skip-external-dirs']) # RESTORE node.cleanup() shutil.rmtree(node.base_dir, ignore_errors=True) - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node=node, options=["-j", "4"]) pgdata_restored = self.pgdata_content( @@ -839,24 +700,20 @@ def test_external_merge_skip_external_dirs(self): self.compare_pgdata(pgdata, pgdata_restored) # @unittest.expectedFailure - # @unittest.skip("skip") def test_external_merge_1(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.pgbench_init(scale=3) # FULL backup - self.backup_node( - backup_dir, 'node', node, options=["-j", "4", "--stream"]) + self.pb.backup_node('node', node, options=["-j", "4", "--stream"]) external_dir1 = self.get_tblspace_path(node, 'external_dir1') external_dir2 = self.get_tblspace_path(node, 'external_dir2') @@ -865,24 +722,18 @@ def test_external_merge_1(self): pgbench.wait() # FULL backup with changed data - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, options=["-j", "4", "--stream"]) # fill external directories with changed data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir1, options=["-j", "4"]) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir2, options=["-j", "4"]) - self.delete_pb(backup_dir, 'node', backup_id=backup_id) + self.pb.delete('node', backup_id=backup_id) # delta backup with external directories using new binary - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="delta", + backup_id = self.pb.backup_node('node', node, backup_type="delta", options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( @@ -890,7 +741,7 @@ def test_external_merge_1(self): self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2)]) - self.merge_backup(backup_dir, 'node', backup_id=backup_id) + self.pb.merge_backup('node', backup_id=backup_id) pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -902,8 +753,7 @@ def test_external_merge_1(self): external_dir1_new = self.get_tblspace_path(node, 'external_dir1') external_dir2_new = self.get_tblspace_path(node, 'external_dir2') - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node=node, options=[ "-j", "4", "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), @@ -918,21 +768,19 @@ def test_external_merge_1(self): # @unittest.skip("skip") def test_external_merge_3(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=3) # FULL backup - self.backup_node(backup_dir, 'node', node, options=["-j", "4"]) + self.pb.backup_node('node', node, options=["-j", "4"]) external_dir1 = self.get_tblspace_path(node, 'external_dir1') external_dir2 = self.get_tblspace_path(node, 'external_dir2') @@ -941,23 +789,17 @@ def test_external_merge_3(self): pgbench.wait() # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) # fill external directories with changed data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1) + self.pb.restore_node('node', restore_dir=external_dir1) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2) + self.pb.restore_node('node', restore_dir=external_dir2) - self.delete_pb(backup_dir, 'node', backup_id=backup_id) + self.pb.delete('node', backup_id=backup_id) # page backup with external directories - self.backup_node( - backup_dir, 'node', node, backup_type="page", + self.pb.backup_node('node', node, backup_type="page", options=[ "-j", "4", "-E", "{0}{1}{2}".format( @@ -966,8 +808,7 @@ def test_external_merge_3(self): external_dir2)]) # page backup with external directories - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="page", + backup_id = self.pb.backup_node('node', node, backup_type="page", options=[ "-j", "4", "-E", "{0}{1}{2}".format( @@ -978,8 +819,7 @@ def test_external_merge_3(self): pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) - self.merge_backup( - backup_dir, 'node', backup_id=backup_id, + self.pb.merge_backup('node', backup_id=backup_id, options=['--log-level-file=verbose']) # RESTORE @@ -989,8 +829,7 @@ def test_external_merge_3(self): external_dir1_new = self.get_tblspace_path(node, 'external_dir1') external_dir2_new = self.get_tblspace_path(node, 'external_dir2') - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ "-j", "4", "--external-mapping={0}={1}".format( @@ -1004,24 +843,20 @@ def test_external_merge_3(self): self.compare_pgdata(pgdata, pgdata_restored) # @unittest.expectedFailure - # @unittest.skip("skip") def test_external_merge_2(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.pgbench_init(scale=3) # FULL backup - self.backup_node( - backup_dir, 'node', node, options=["-j", "4", "--stream"]) + self.pb.backup_node('node', node, options=["-j", "4", "--stream"]) external_dir1 = self.get_tblspace_path(node, 'external_dir1') external_dir2 = self.get_tblspace_path(node, 'external_dir2') @@ -1030,24 +865,18 @@ def test_external_merge_2(self): pgbench.wait() # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, options=["-j", "4", "--stream"]) # fill external directories with changed data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir1, options=["-j", "4"]) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir2, options=["-j", "4"]) - self.delete_pb(backup_dir, 'node', backup_id=backup_id) + self.pb.delete('node', backup_id=backup_id) # delta backup with external directories - self.backup_node( - backup_dir, 'node', node, backup_type="delta", + self.pb.backup_node('node', node, backup_type="delta", options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( @@ -1056,8 +885,7 @@ def test_external_merge_2(self): external_dir2)]) # delta backup with external directories - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="delta", + backup_id = self.pb.backup_node('node', node, backup_type="delta", options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( @@ -1072,7 +900,7 @@ def test_external_merge_2(self): shutil.rmtree(external_dir2, ignore_errors=True) # delta backup without external directories - self.merge_backup(backup_dir, 'node', backup_id=backup_id) + self.pb.merge_backup('node', backup_id=backup_id) # RESTORE node.cleanup() @@ -1081,8 +909,7 @@ def test_external_merge_2(self): external_dir1_new = self.get_tblspace_path(node, 'external_dir1') external_dir2_new = self.get_tblspace_path(node, 'external_dir2') - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node=node, options=[ "-j", "4", "--external-mapping={0}={1}".format(external_dir1, external_dir1_new), @@ -1097,14 +924,12 @@ def test_external_merge_2(self): # @unittest.skip("skip") def test_restore_external_changed_data(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.pgbench_init(scale=2) @@ -1114,28 +939,22 @@ def test_restore_external_changed_data(self): external_dir2 = self.get_tblspace_path(node, 'external_dir2') # FULL backup - tmp_id = self.backup_node( - backup_dir, 'node', + tmp_id = self.pb.backup_node('node', node, options=["-j", "4", "--stream"]) # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir1, options=["-j", "4"]) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir2, options=["-j", "4"]) - self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + self.pb.delete('node', backup_id=tmp_id) # change data a bit pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) pgbench.wait() # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( @@ -1147,14 +966,10 @@ def test_restore_external_changed_data(self): shutil.rmtree(external_dir1, ignore_errors=True) shutil.rmtree(external_dir2, ignore_errors=True) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, backup_id=backup_id, + self.pb.restore_node('node', restore_dir=external_dir1, backup_id=backup_id, options=["-j", "4", "--skip-external-dirs"]) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, backup_id=backup_id, + self.pb.restore_node('node', restore_dir=external_dir2, backup_id=backup_id, options=["-j", "4", "--skip-external-dirs"]) # change data a bit more @@ -1162,8 +977,7 @@ def test_restore_external_changed_data(self): pgbench.wait() # Delta backup with external directories - self.backup_node( - backup_dir, 'node', node, backup_type="delta", + self.pb.backup_node('node', node, backup_type="delta", options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( @@ -1178,8 +992,7 @@ def test_restore_external_changed_data(self): node.cleanup() shutil.rmtree(node.base_dir, ignore_errors=True) - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=["-j", "4"]) pgdata_restored = self.pgdata_content( @@ -1191,16 +1004,14 @@ def test_restore_external_changed_data(self): # @unittest.skip("skip") def test_restore_external_changed_data_1(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'max_wal_size': '32MB'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.pgbench_init(scale=1) @@ -1210,28 +1021,22 @@ def test_restore_external_changed_data_1(self): external_dir2 = self.get_tblspace_path(node, 'external_dir2') # FULL backup - tmp_id = self.backup_node( - backup_dir, 'node', + tmp_id = self.pb.backup_node('node', node, options=["-j", "4", "--stream"]) # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir1, options=["-j", "4"]) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir2, options=["-j", "4"]) - self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + self.pb.delete('node', backup_id=tmp_id) # change data a bit pgbench = node.pgbench(options=['-T', '5', '-c', '1', '--no-vacuum']) pgbench.wait() # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( @@ -1243,14 +1048,10 @@ def test_restore_external_changed_data_1(self): shutil.rmtree(external_dir1, ignore_errors=True) shutil.rmtree(external_dir2, ignore_errors=True) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, backup_id=backup_id, + self.pb.restore_node('node', restore_dir=external_dir1, backup_id=backup_id, options=["-j", "4", "--skip-external-dirs"]) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, backup_id=backup_id, + self.pb.restore_node('node', restore_dir=external_dir2, backup_id=backup_id, options=["-j", "4", "--skip-external-dirs"]) # change data a bit more @@ -1258,8 +1059,7 @@ def test_restore_external_changed_data_1(self): pgbench.wait() # Delta backup with only one external directory - self.backup_node( - backup_dir, 'node', node, backup_type="delta", + self.pb.backup_node('node', node, backup_type="delta", options=[ "-j", "4", "--stream", "-E", external_dir1]) @@ -1268,17 +1068,15 @@ def test_restore_external_changed_data_1(self): node.base_dir, exclude_dirs=['logs', 'external_dir2']) # Restore - node.cleanup() - shutil.rmtree(node._base_dir) + node.stop() + shutil.rmtree(node.base_dir) # create empty file in external_dir2 - os.mkdir(node._base_dir) - os.mkdir(external_dir2) + os.makedirs(external_dir2) with open(os.path.join(external_dir2, 'file'), 'w+') as f: f.close() - output = self.restore_node( - backup_dir, 'node', node, + output = self.pb.restore_node('node', node=node, options=["-j", "4"]) self.assertNotIn( @@ -1291,19 +1089,16 @@ def test_restore_external_changed_data_1(self): self.compare_pgdata(pgdata, pgdata_restored) # @unittest.expectedFailure - # @unittest.skip("skip") def test_merge_external_changed_data(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'max_wal_size': '32MB'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.pgbench_init(scale=2) @@ -1313,28 +1108,22 @@ def test_merge_external_changed_data(self): external_dir2 = self.get_tblspace_path(node, 'external_dir2') # FULL backup - tmp_id = self.backup_node( - backup_dir, 'node', + tmp_id = self.pb.backup_node('node', node, options=["-j", "4", "--stream"]) # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir1, options=["-j", "4"]) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir2, options=["-j", "4"]) - self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + self.pb.delete('node', backup_id=tmp_id) # change data a bit pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) pgbench.wait() # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( @@ -1346,14 +1135,10 @@ def test_merge_external_changed_data(self): shutil.rmtree(external_dir1, ignore_errors=True) shutil.rmtree(external_dir2, ignore_errors=True) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, backup_id=backup_id, + self.pb.restore_node('node', restore_dir=external_dir1, backup_id=backup_id, options=["-j", "4", "--skip-external-dirs"]) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, backup_id=backup_id, + self.pb.restore_node('node', restore_dir=external_dir2, backup_id=backup_id, options=["-j", "4", "--skip-external-dirs"]) # change data a bit more @@ -1361,8 +1146,7 @@ def test_merge_external_changed_data(self): pgbench.wait() # Delta backup with external directories - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="delta", + backup_id = self.pb.backup_node('node', node, backup_type="delta", options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( @@ -1374,14 +1158,13 @@ def test_merge_external_changed_data(self): node.base_dir, exclude_dirs=['logs']) # Merge - self.merge_backup(backup_dir, 'node', backup_id) + self.pb.merge_backup('node', backup_id) # Restore node.cleanup() shutil.rmtree(node.base_dir, ignore_errors=True) - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node=node, options=["-j", "4"]) pgdata_restored = self.pgdata_content( @@ -1395,37 +1178,29 @@ def test_restore_skip_external(self): """ Check that --skip-external-dirs works correctly """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() external_dir1 = self.get_tblspace_path(node, 'external_dir1') external_dir2 = self.get_tblspace_path(node, 'external_dir2') # temp FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=["-j", "4", "--stream"]) + backup_id = self.pb.backup_node('node', node, options=["-j", "4", "--stream"]) # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir1, options=["-j", "4"]) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir2, options=["-j", "4"]) - self.delete_pb(backup_dir, 'node', backup_id=backup_id) + self.pb.delete('node', backup_id=backup_id) # FULL backup with external directories - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( @@ -1446,8 +1221,7 @@ def test_restore_skip_external(self): node.cleanup() shutil.rmtree(node.base_dir, ignore_errors=True) - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node=node, options=[ "-j", "4", "--skip-external-dirs"]) @@ -1467,41 +1241,32 @@ def test_external_dir_is_symlink(self): if os.name == 'nt': self.skipTest('Skipped for Windows') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() external_dir = self.get_tblspace_path(node, 'external_dir') # temp FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=["-j", "4", "--stream"]) + backup_id = self.pb.backup_node('node', node, options=["-j", "4", "--stream"]) # fill some directory with data - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - symlinked_dir = os.path.join(core_dir, 'symlinked') + symlinked_dir = os.path.join(self.test_path, 'symlinked') - self.restore_node( - backup_dir, 'node', node, - data_dir=symlinked_dir, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=symlinked_dir, options=["-j", "4"]) # drop temp FULL backup - self.delete_pb(backup_dir, 'node', backup_id=backup_id) + self.pb.delete('node', backup_id=backup_id) # create symlink to directory in external directory os.symlink(symlinked_dir, external_dir) # FULL backup with external directories - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, options=[ "-j", "4", "--stream", "-E", external_dir]) @@ -1509,8 +1274,7 @@ def test_external_dir_is_symlink(self): pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') # RESTORE node_restored.cleanup() @@ -1518,8 +1282,7 @@ def test_external_dir_is_symlink(self): external_dir_new = self.get_tblspace_path( node_restored, 'external_dir') - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node=node_restored, options=[ "-j", "4", "--external-mapping={0}={1}".format( external_dir, external_dir_new)]) @@ -1531,8 +1294,7 @@ def test_external_dir_is_symlink(self): self.assertEqual( external_dir, - self.show_pb( - backup_dir, 'node', + self.pb.show('node', backup_id=backup_id)['external-dirs']) # @unittest.expectedFailure @@ -1546,43 +1308,34 @@ def test_external_dir_contain_symlink_on_dir(self): if os.name == 'nt': self.skipTest('Skipped for Windows') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() external_dir = self.get_tblspace_path(node, 'external_dir') dir_in_external_dir = os.path.join(external_dir, 'dir') # temp FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=["-j", "4", "--stream"]) + backup_id = self.pb.backup_node('node', node, options=["-j", "4", "--stream"]) # fill some directory with data - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - symlinked_dir = os.path.join(core_dir, 'symlinked') + symlinked_dir = os.path.join(self.test_path, 'symlinked') - self.restore_node( - backup_dir, 'node', node, - data_dir=symlinked_dir, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=symlinked_dir, options=["-j", "4"]) # drop temp FULL backup - self.delete_pb(backup_dir, 'node', backup_id=backup_id) + self.pb.delete('node', backup_id=backup_id) # create symlink to directory in external directory os.mkdir(external_dir) os.symlink(symlinked_dir, dir_in_external_dir) # FULL backup with external directories - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node=node, options=[ "-j", "4", "--stream", "-E", external_dir]) @@ -1590,8 +1343,7 @@ def test_external_dir_contain_symlink_on_dir(self): pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') # RESTORE node_restored.cleanup() @@ -1599,8 +1351,7 @@ def test_external_dir_contain_symlink_on_dir(self): external_dir_new = self.get_tblspace_path( node_restored, 'external_dir') - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node=node_restored, options=[ "-j", "4", "--external-mapping={0}={1}".format( external_dir, external_dir_new)]) @@ -1612,8 +1363,7 @@ def test_external_dir_contain_symlink_on_dir(self): self.assertEqual( external_dir, - self.show_pb( - backup_dir, 'node', + self.pb.show('node', backup_id=backup_id)['external-dirs']) # @unittest.expectedFailure @@ -1627,35 +1377,27 @@ def test_external_dir_contain_symlink_on_file(self): if os.name == 'nt': self.skipTest('Skipped for Windows') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() external_dir = self.get_tblspace_path(node, 'external_dir') file_in_external_dir = os.path.join(external_dir, 'file') # temp FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=["-j", "4", "--stream"]) + backup_id = self.pb.backup_node('node', node, options=["-j", "4", "--stream"]) # fill some directory with data - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - symlinked_dir = os.path.join(core_dir, 'symlinked') + symlinked_dir = os.path.join(self.test_path, 'symlinked') - self.restore_node( - backup_dir, 'node', node, - data_dir=symlinked_dir, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=symlinked_dir, options=["-j", "4"]) # drop temp FULL backup - self.delete_pb(backup_dir, 'node', backup_id=backup_id) + self.pb.delete('node', backup_id=backup_id) # create symlink to directory in external directory src_file = os.path.join(symlinked_dir, 'postgresql.conf') @@ -1664,8 +1406,7 @@ def test_external_dir_contain_symlink_on_file(self): os.symlink(src_file, file_in_external_dir) # FULL backup with external directories - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, options=[ "-j", "4", "--stream", "-E", external_dir]) @@ -1673,8 +1414,7 @@ def test_external_dir_contain_symlink_on_file(self): pgdata = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') # RESTORE node_restored.cleanup() @@ -1682,8 +1422,7 @@ def test_external_dir_contain_symlink_on_file(self): external_dir_new = self.get_tblspace_path( node_restored, 'external_dir') - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node=node_restored, options=[ "-j", "4", "--external-mapping={0}={1}".format( external_dir, external_dir_new)]) @@ -1695,8 +1434,7 @@ def test_external_dir_contain_symlink_on_file(self): self.assertEqual( external_dir, - self.show_pb( - backup_dir, 'node', + self.pb.show('node', backup_id=backup_id)['external-dirs']) # @unittest.expectedFailure @@ -1706,16 +1444,12 @@ def test_external_dir_is_tablespace(self): Check that backup fails with error if external directory points to tablespace """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() external_dir = self.get_tblspace_path(node, 'external_dir') @@ -1726,24 +1460,10 @@ def test_external_dir_is_tablespace(self): node.pgbench_init(scale=1, tablespace='tblspace1') # FULL backup with external directories - try: - backup_id = self.backup_node( - backup_dir, 'node', node, - options=[ - "-j", "4", "--stream", - "-E", external_dir]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because external dir points to the tablespace" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'External directory path (-E option)', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, + options=["-j", "4", "--stream", "-E", external_dir], + expect_error="because external dir points to the tablespace") + self.assertMessage(contains='External directory path (-E option)') def test_restore_external_dir_not_empty(self): """ @@ -1751,16 +1471,12 @@ def test_restore_external_dir_not_empty(self): if external directory point to not empty tablespace and if remapped directory also isn`t empty """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() external_dir = self.get_tblspace_path(node, 'external_dir') @@ -1772,28 +1488,17 @@ def test_restore_external_dir_not_empty(self): f.close() # FULL backup with external directory - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=[ "-j", "4", "--stream", "-E", external_dir]) - node.cleanup() + node.stop() + shutil.rmtree(node.data_dir) - try: - self.restore_node(backup_dir, 'node', node) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because external dir is not empty" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'External directory is not empty', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node, + expect_error="because external dir is not empty") + self.assertMessage(contains='External directory is not empty') external_dir_new = self.get_tblspace_path(node, 'external_dir_new') @@ -1803,23 +1508,11 @@ def test_restore_external_dir_not_empty(self): with open(os.path.join(external_dir_new, 'file1'), 'w+') as f: f.close() - try: - self.restore_node( - backup_dir, 'node', node, - options=['--external-mapping={0}={1}'.format( - external_dir, external_dir_new)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because remapped external dir is not empty" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'External directory is not empty', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node, + options=[f'--external-mapping', + f'{external_dir}={external_dir_new}'], + expect_error="because remapped external dir is not empty") + self.assertMessage(contains='External directory is not empty') def test_restore_external_dir_is_missing(self): """ @@ -1828,16 +1521,12 @@ def test_restore_external_dir_is_missing(self): take DELTA backup with external directory, which should fail """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() external_dir = self.get_tblspace_path(node, 'external_dir') @@ -1849,8 +1538,7 @@ def test_restore_external_dir_is_missing(self): f.close() # FULL backup with external directory - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=[ "-j", "4", "--stream", "-E", external_dir]) @@ -1858,31 +1546,15 @@ def test_restore_external_dir_is_missing(self): # drop external directory shutil.rmtree(external_dir, ignore_errors=True) - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=[ - "-j", "4", "--stream", - "-E", external_dir]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because external dir is missing" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: External directory is not found:', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, backup_type='delta', + options=["-j", "4", "--stream", "-E", external_dir], + expect_error="because external dir is missing") + self.assertMessage(contains='ERROR: External directory is not found:') sleep(1) # take DELTA without external directories - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='delta', options=["-j", "4", "--stream"]) @@ -1893,7 +1565,7 @@ def test_restore_external_dir_is_missing(self): node.cleanup() shutil.rmtree(node.base_dir, ignore_errors=True) - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -1909,16 +1581,12 @@ def test_merge_external_dir_is_missing(self): merge it into FULL, restore and check data correctness """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() external_dir = self.get_tblspace_path(node, 'external_dir') @@ -1930,8 +1598,7 @@ def test_merge_external_dir_is_missing(self): f.close() # FULL backup with external directory - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=[ "-j", "4", "--stream", "-E", external_dir]) @@ -1939,31 +1606,15 @@ def test_merge_external_dir_is_missing(self): # drop external directory shutil.rmtree(external_dir, ignore_errors=True) - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', - options=[ - "-j", "4", "--stream", - "-E", external_dir]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because external dir is missing" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: External directory is not found:', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, backup_type='delta', + options=["-j", "4", "--stream", "-E", external_dir], + expect_error="because external dir is missing") + self.assertMessage(contains='ERROR: External directory is not found:') sleep(1) # take DELTA without external directories - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, backup_type='delta', options=["-j", "4", "--stream"]) @@ -1971,13 +1622,13 @@ def test_merge_external_dir_is_missing(self): node.base_dir, exclude_dirs=['logs']) # Merge - self.merge_backup(backup_dir, 'node', backup_id=backup_id) + self.pb.merge_backup('node', backup_id=backup_id) # Restore node.cleanup() shutil.rmtree(node.base_dir, ignore_errors=True) - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -1991,16 +1642,12 @@ def test_restore_external_dir_is_empty(self): restore DELRA backup, check that restored external directory is empty """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() external_dir = self.get_tblspace_path(node, 'external_dir') @@ -2013,8 +1660,7 @@ def test_restore_external_dir_is_empty(self): f.close() # FULL backup with external directory - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=[ "-j", "4", "--stream", "-E", external_dir]) @@ -2023,8 +1669,7 @@ def test_restore_external_dir_is_empty(self): os.remove(os.path.join(external_dir, 'file')) # take DELTA backup with empty external directory - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='delta', options=[ "-j", "4", "--stream", @@ -2037,7 +1682,7 @@ def test_restore_external_dir_is_empty(self): node.cleanup() shutil.rmtree(node.base_dir, ignore_errors=True) - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -2051,16 +1696,12 @@ def test_merge_external_dir_is_empty(self): merge backups and restore FULL, check that restored external directory is empty """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() external_dir = self.get_tblspace_path(node, 'external_dir') @@ -2073,8 +1714,7 @@ def test_merge_external_dir_is_empty(self): f.close() # FULL backup with external directory - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=[ "-j", "4", "--stream", "-E", external_dir]) @@ -2083,8 +1723,7 @@ def test_merge_external_dir_is_empty(self): os.remove(os.path.join(external_dir, 'file')) # take DELTA backup with empty external directory - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, backup_type='delta', options=[ "-j", "4", "--stream", @@ -2094,13 +1733,13 @@ def test_merge_external_dir_is_empty(self): node.base_dir, exclude_dirs=['logs']) # Merge - self.merge_backup(backup_dir, 'node', backup_id=backup_id) + self.pb.merge_backup('node', backup_id=backup_id) # Restore node.cleanup() shutil.rmtree(node.base_dir, ignore_errors=True) - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -2114,16 +1753,12 @@ def test_restore_external_dir_string_order(self): restore DELRA backup, check that restored external directory is empty """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() external_dir_1 = self.get_tblspace_path(node, 'external_dir_1') @@ -2141,8 +1776,7 @@ def test_restore_external_dir_string_order(self): f.close() # FULL backup with external directory - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( @@ -2158,8 +1792,7 @@ def test_restore_external_dir_string_order(self): # take DELTA backup and swap external_dir_2 and external_dir_1 # in external_dir_str - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='delta', options=[ "-j", "4", "--stream", @@ -2175,7 +1808,7 @@ def test_restore_external_dir_string_order(self): node.cleanup() shutil.rmtree(node.base_dir, ignore_errors=True) - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -2190,16 +1823,12 @@ def test_merge_external_dir_string_order(self): restore DELRA backup, check that restored external directory is empty """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - core_dir = os.path.join(self.tmp_path, self.module_name, self.fname) - shutil.rmtree(core_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() external_dir_1 = self.get_tblspace_path(node, 'external_dir_1') @@ -2217,8 +1846,7 @@ def test_merge_external_dir_string_order(self): f.close() # FULL backup with external directory - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=[ "-j", "4", "--stream", "-E", "{0}{1}{2}".format( @@ -2234,8 +1862,7 @@ def test_merge_external_dir_string_order(self): # take DELTA backup and swap external_dir_2 and external_dir_1 # in external_dir_str - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, backup_type='delta', options=[ "-j", "4", "--stream", @@ -2248,13 +1875,13 @@ def test_merge_external_dir_string_order(self): node.base_dir, exclude_dirs=['logs']) # Merge backups - self.merge_backup(backup_dir, 'node', backup_id=backup_id) + self.pb.merge_backup('node', backup_id=backup_id) # Restore node.cleanup() shutil.rmtree(node.base_dir, ignore_errors=True) - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) pgdata_restored = self.pgdata_content( node.base_dir, exclude_dirs=['logs']) @@ -2269,32 +1896,28 @@ def test_smart_restore_externals(self): make sure that files from externals are not copied during restore https://github.com/postgrespro/pg_probackup/issues/63 """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # fill external directories with data - tmp_id = self.backup_node(backup_dir, 'node', node) + tmp_id = self.pb.backup_node('node', node) external_dir_1 = self.get_tblspace_path(node, 'external_dir_1') external_dir_2 = self.get_tblspace_path(node, 'external_dir_2') - self.restore_node( - backup_dir, 'node', node, backup_id=tmp_id, - data_dir=external_dir_1, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir_1, backup_id=tmp_id, + options=["-j", "4"]) - self.restore_node( - backup_dir, 'node', node, backup_id=tmp_id, - data_dir=external_dir_2, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir_2, backup_id=tmp_id, + options=["-j", "4"]) - self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + self.pb.delete('node', backup_id=tmp_id) # create database node.safe_psql( @@ -2302,7 +1925,7 @@ def test_smart_restore_externals(self): "CREATE DATABASE testdb") # take FULL backup - full_id = self.backup_node(backup_dir, 'node', node) + full_id = self.pb.backup_node('node', node) # drop database node.safe_psql( @@ -2310,29 +1933,24 @@ def test_smart_restore_externals(self): "DROP DATABASE testdb") # take PAGE backup - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id = self.pb.backup_node('node', node=node, backup_type='page') # restore PAGE backup node.cleanup() - self.restore_node( - backup_dir, 'node', node, backup_id=page_id, + self.pb.restore_node('node', node=node, backup_id=page_id, options=['--no-validate', '--log-level-file=VERBOSE']) - logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(logfile, 'r') as f: - logfile_content = f.read() + logfile_content = self.read_pb_log() # get delta between FULL and PAGE filelists - filelist_full = self.get_backup_filelist( - backup_dir, 'node', full_id) + filelist_full = self.get_backup_filelist(backup_dir, 'node', full_id) - filelist_page = self.get_backup_filelist( - backup_dir, 'node', page_id) + filelist_page = self.get_backup_filelist(backup_dir, 'node', page_id) filelist_diff = self.get_backup_filelist_diff( filelist_full, filelist_page) + self.assertTrue(filelist_diff, 'There should be deleted files') for file in filelist_diff: self.assertNotIn(file, logfile_content) @@ -2344,32 +1962,27 @@ def test_external_validation(self): corrupt external file in backup, run validate which should fail """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # take temp FULL backup - tmp_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) + tmp_id = self.pb.backup_node('node', node, options=['--stream']) external_dir = self.get_tblspace_path(node, 'external_dir') # fill external directories with data - self.restore_node( - backup_dir, 'node', node, backup_id=tmp_id, - data_dir=external_dir, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir, backup_id=tmp_id, + options=["-j", "4"]) - self.delete_pb(backup_dir, 'node', backup_id=tmp_id) + self.pb.delete('node', backup_id=tmp_id) # take FULL backup - full_id = self.backup_node( - backup_dir, 'node', node, + full_id = self.pb.backup_node('node', node, options=[ '--stream', '-E', "{0}".format(external_dir)]) @@ -2378,28 +1991,15 @@ def test_external_validation(self): backup_dir, 'backups', 'node', full_id, 'external_directories', 'externaldir1', 'postgresql.auto.conf') - with open(file, "r+b", 0) as f: - f.seek(42) - f.write(b"blah") - f.flush() - f.close - - try: - self.validate_pb(backup_dir) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because file in external dir is corrupted" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Invalid CRC of backup file', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + to_corrupt = 'external_directories/externaldir1/postgresql.auto.conf' + self.corrupt_backup_file(backup_dir, 'node', full_id, to_corrupt, + damage=(42, b"blah")) + + self.pb.validate( + expect_error="because file in external dir is corrupted") + self.assertMessage(contains='WARNING: Invalid CRC of backup file') self.assertEqual( 'CORRUPT', - self.show_pb(backup_dir, 'node', full_id)['status'], + self.pb.show('node', full_id)['status'], 'Backup STATUS should be "CORRUPT"') diff --git a/tests/false_positive_test.py b/tests/false_positive_test.py index fbb785c60..e4e410fbf 100644 --- a/tests/false_positive_test.py +++ b/tests/false_positive_test.py @@ -2,12 +2,12 @@ import os from time import sleep -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from datetime import datetime, timedelta -import subprocess +from .helpers.ptrack_helpers import ProbackupTest +from pg_probackup2.gdb import needs_gdb +from datetime import datetime -class FalsePositive(ProbackupTest, unittest.TestCase): +class FalsePositive(ProbackupTest): # @unittest.skip("skip") @unittest.expectedFailure @@ -15,88 +15,54 @@ def test_validate_wal_lost_segment(self): """ Loose segment located between backups. ExpectedFailure. This is BUG """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # make some wals node.pgbench_init(scale=5) # delete last wal segment - wals_dir = os.path.join(backup_dir, "wal", 'node') - wals = [f for f in os.listdir(wals_dir) if os.path.isfile( - os.path.join(wals_dir, f)) and not f.endswith('.backup')] - wals = map(int, wals) - os.remove(os.path.join(wals_dir, '0000000' + str(max(wals)))) + wals = self.get_instance_wal_list(backup_dir, 'node') + self.remove_instance_wal(backup_dir, 'node', max(wals)) # We just lost a wal segment and know nothing about it - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) self.assertTrue( - 'validation completed successfully' in self.validate_pb( - backup_dir, 'node')) + 'validation completed successfully' in self.pb.validate('node')) ######## @unittest.expectedFailure # Need to force validation of ancestor-chain def test_incremental_backup_corrupt_full_1(self): """page-level backup with corrupted full backup""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - backup_id = self.backup_node(backup_dir, 'node', node) - file = os.path.join( - backup_dir, "backups", "node", - backup_id.decode("utf-8"), "database", "postgresql.conf") - os.remove(file) - - try: - self.backup_node(backup_dir, 'node', node, backup_type="page") - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be " - "possible without valid full backup.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertEqual( - e.message, - 'ERROR: Valid full backup on current timeline is not found. ' - 'Create new FULL backup before an incremental one.\n', - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertFalse( - True, - "Expecting Error because page backup should not be " - "possible without valid full backup.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertEqual( - e.message, + backup_id = self.pb.backup_node('node', node) + self.remove_backup_file(backup_dir, 'node', backup_id, + 'database/postgresql.conf') + + self.pb.backup_node('node', node, backup_type="page", + expect_error="because page backup without full is impossible") + self.assertMessage(contains= 'ERROR: Valid full backup on current timeline is not found. ' - 'Create new FULL backup before an incremental one.\n', - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + 'Create new FULL backup before an incremental one.') self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['Status'], "ERROR") + self.pb.show('node')[0]['Status'], "ERROR") # @unittest.skip("skip") @unittest.expectedFailure @@ -104,38 +70,28 @@ def test_pg_10_waldir(self): """ test group access for PG >= 11 """ - if self.pg_config_version < self.version_to_num('10.0'): - self.skipTest('You need PostgreSQL >= 10 for this test') - - wal_dir = os.path.join( - os.path.join(self.tmp_path, self.module_name, self.fname), 'wal_dir') + wal_dir = os.path.join(self.test_path, 'wal_dir') import shutil shutil.rmtree(wal_dir, ignore_errors=True) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=[ - '--data-checksums', - '--waldir={0}'.format(wal_dir)]) + initdb_params=['--waldir={0}'.format(wal_dir)]) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # take FULL backup - self.backup_node( - backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) pgdata = self.pgdata_content(node.data_dir) # restore backup - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored) + self.pb.restore_node('node', node_restored) # compare pgdata permissions pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -145,27 +101,29 @@ def test_pg_10_waldir(self): os.path.islink(os.path.join(node_restored.data_dir, 'pg_wal')), 'pg_wal should be symlink') - @unittest.expectedFailure + # @unittest.expectedFailure + @needs_gdb # @unittest.skip("skip") def test_recovery_target_time_backup_victim(self): """ Check that for validation to recovery target probackup chooses valid backup https://github.com/postgrespro/pg_probackup/issues/104 + + @y.sokolov: looks like this test should pass. + So I commented 'expectedFailure' """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.safe_psql( "postgres", @@ -173,9 +131,7 @@ def test_recovery_target_time_backup_victim(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,10000) i") - target_time = node.safe_psql( - "postgres", - "select now()").rstrip() + target_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f") node.safe_psql( "postgres", @@ -183,47 +139,48 @@ def test_recovery_target_time_backup_victim(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,100) i") - gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb = self.pb.backup_node('node', node, gdb=True) # Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() - gdb.remove_all_breakpoints() - gdb._execute('signal SIGINT') + gdb.signal('SIGINT') gdb.continue_execution_until_error() - backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + backup_id = self.pb.show('node')[1]['id'] self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node', backup_id)['status'], + self.pb.show('node', backup_id)['status'], 'Backup STATUS should be "ERROR"') - self.validate_pb( - backup_dir, 'node', + self.pb.validate( + 'node', options=['--recovery-target-time={0}'.format(target_time)]) - @unittest.expectedFailure + # @unittest.expectedFailure # @unittest.skip("skip") + @needs_gdb def test_recovery_target_lsn_backup_victim(self): """ Check that for validation to recovery target probackup chooses valid backup https://github.com/postgrespro/pg_probackup/issues/104 + + @y.sokolov: looks like this test should pass. + So I commented 'expectedFailure' """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.safe_psql( "postgres", @@ -237,56 +194,51 @@ def test_recovery_target_lsn_backup_victim(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,100) i") - gdb = self.backup_node( - backup_dir, 'node', node, + gdb = self.pb.backup_node('node', node, options=['--log-level-console=LOG'], gdb=True) # Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() - gdb.remove_all_breakpoints() - gdb._execute('signal SIGINT') + gdb.signal('SIGINT') gdb.continue_execution_until_error() - backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + backup_id = self.pb.show('node')[1]['id'] self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node', backup_id)['status'], + self.pb.show('node', backup_id)['status'], 'Backup STATUS should be "ERROR"') self.switch_wal_segment(node) - target_lsn = self.show_pb(backup_dir, 'node', backup_id)['start-lsn'] + target_lsn = self.pb.show('node', backup_id)['start-lsn'] - self.validate_pb( - backup_dir, 'node', + self.pb.validate( + 'node', options=['--recovery-target-lsn={0}'.format(target_lsn)]) # @unittest.skip("skip") - @unittest.expectedFailure + @needs_gdb def test_streaming_timeout(self): """ Illustrate the problem of loosing exact error message because our WAL streaming engine is "borrowed" from pg_receivexlog """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '1h', 'wal_sender_timeout': '5s'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # FULL backup - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True, + gdb = self.pb.backup_node('node', node, gdb=True, options=['--stream', '--log-level-file=LOG']) # Attention! This breakpoint is set to a probackup internal fuction, not a postgres core one @@ -295,16 +247,10 @@ def test_streaming_timeout(self): sleep(10) gdb.continue_execution_until_error() - gdb._execute('detach') + gdb.detach() sleep(2) - log_file_path = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(log_file_path) as f: - log_content = f.read() - - self.assertIn( - 'could not receive data from WAL stream', - log_content) + log_content = self.read_pb_log() self.assertIn( 'ERROR: Problem in receivexlog', @@ -315,23 +261,12 @@ def test_streaming_timeout(self): def test_validate_all_empty_catalog(self): """ """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - - try: - self.validate_pb(backup_dir) - self.assertEqual( - 1, 0, - "Expecting Error because backup_dir is empty.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: This backup catalog contains no backup instances', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + + self.pb.validate( + expect_error="because backup_dir is empty") + self.assertMessage(contains= + 'ERROR: This backup catalog contains no backup instances') diff --git a/tests/helpers/__init__.py b/tests/helpers/__init__.py index 2e5ed40e8..42f56e492 100644 --- a/tests/helpers/__init__.py +++ b/tests/helpers/__init__.py @@ -1,4 +1,4 @@ -__all__ = ['ptrack_helpers', 'cfs_helpers', 'data_helpers'] +__all__ = ['ptrack_helpers', 'data_helpers', 'fs_backup', 'init_helpers'] import unittest @@ -6,4 +6,4 @@ if not hasattr(unittest.TestCase, "skipTest"): def skipTest(self, reason): raise unittest.SkipTest(reason) - unittest.TestCase.skipTest = skipTest \ No newline at end of file + unittest.TestCase.skipTest = skipTest diff --git a/tests/helpers/cfs_helpers.py b/tests/helpers/cfs_helpers.py deleted file mode 100644 index 31af76f2e..000000000 --- a/tests/helpers/cfs_helpers.py +++ /dev/null @@ -1,93 +0,0 @@ -import os -import re -import random -import string - - -def find_by_extensions(dirs=None, extensions=None): - """ - find_by_extensions(['path1','path2'],['.txt','.log']) - :return: - Return list of files include full path by file extensions - """ - files = [] - new_dirs = [] - - if dirs is not None and extensions is not None: - for d in dirs: - try: - new_dirs += [os.path.join(d, f) for f in os.listdir(d)] - except OSError: - if os.path.splitext(d)[1] in extensions: - files.append(d) - - if new_dirs: - files.extend(find_by_extensions(new_dirs, extensions)) - - return files - - -def find_by_pattern(dirs=None, pattern=None): - """ - find_by_pattern(['path1','path2'],'^.*/*.txt') - :return: - Return list of files include full path by pattern - """ - files = [] - new_dirs = [] - - if dirs is not None and pattern is not None: - for d in dirs: - try: - new_dirs += [os.path.join(d, f) for f in os.listdir(d)] - except OSError: - if re.match(pattern,d): - files.append(d) - - if new_dirs: - files.extend(find_by_pattern(new_dirs, pattern)) - - return files - - -def find_by_name(dirs=None, filename=None): - files = [] - new_dirs = [] - - if dirs is not None and filename is not None: - for d in dirs: - try: - new_dirs += [os.path.join(d, f) for f in os.listdir(d)] - except OSError: - if os.path.basename(d) in filename: - files.append(d) - - if new_dirs: - files.extend(find_by_name(new_dirs, filename)) - - return files - - -def corrupt_file(filename): - file_size = None - try: - file_size = os.path.getsize(filename) - except OSError: - return False - - try: - with open(filename, "rb+") as f: - f.seek(random.randint(int(0.1*file_size),int(0.8*file_size))) - f.write(random_string(0.1*file_size)) - f.close() - except OSError: - return False - - return True - - -def random_string(n): - a = string.ascii_letters + string.digits - random_str = ''.join([random.choice(a) for i in range(int(n)+1)]) - return str.encode(random_str) -# return ''.join([random.choice(a) for i in range(int(n)+1)]) diff --git a/tests/helpers/data_helpers.py b/tests/helpers/data_helpers.py index 27cb66c3d..2acf2fddb 100644 --- a/tests/helpers/data_helpers.py +++ b/tests/helpers/data_helpers.py @@ -1,7 +1,200 @@ +import os import re +import random +import string import unittest -import functools import time +from array import array +import struct + + +def find_by_extension(dir, extensions, backup_dir=None): + """ + find_by_extensions('path1',['.txt','.log']) + + Add backup_dir if we need to check files from backup folder + :return: + Return list of files by file extensions. + If backup_dir is not passed, then file path include full path. + Otherwise file path is relative to backup_dir. + """ + if isinstance(extensions, str): + extensions = [extensions] + + if backup_dir is not None: + return [obj for obj in backup_dir.list_files(dir, recursive=True) + if os.path.splitext(obj)[1] in extensions] + + return [os.path.join(rootdir, obj) + for rootdir, dirs, objs in os.walk(dir, followlinks=True) + for obj in objs + if os.path.splitext(obj)[1] in extensions] + +def find_by_pattern(dir, pattern, backup_dir=None): + """ + find_by_pattern('path1','^.*/*.txt') + :return: + Return list of files include full path by pattern + """ + if backup_dir is not None: + return [obj for obj in backup_dir.list_files(dir, recursive=True) + if re.match(pattern, obj)] + + objs = (os.path.join(rootdir, obj) + for rootdir, dirs, objs in os.walk(dir, followlinks=True) + for obj in objs) + return [obj for obj in objs if re.match(pattern, obj)] + +def find_by_name(dir, filenames, backup_dir=None): + if isinstance(filenames, str): + filenames = [filenames] + + if backup_dir is not None: + return [obj for obj in backup_dir.list_files(dir, recursive=True) + if os.path.basename(obj) in filenames] + + return [os.path.join(rootdir, obj) + for rootdir, dirs, objs in os.walk(dir, followlinks=True) + for obj in objs + if obj in filenames] + + +def get_page_size(filename): + # fixed PostgreSQL page header size + PAGE_HEADER_SIZE = 24 + with open(filename, "rb+") as f: + page_header = f.read(PAGE_HEADER_SIZE) + assert len(page_header) == PAGE_HEADER_SIZE + + size = struct.unpack('H', page_header[18:20])[0] & 0xff00 + assert (size & (size - 1)) == 0 + + return size + + +def pg_checksum_block(raw_page, blkno): + N_SUMS = 32 + # prime multiplier of FNV-1a hash + FNV_PRIME = 16777619 + MASK = (1<<32) - 1 + + # Set pd_checksum to zero, so that the checksum calculation isn't + # affected by the old checksum stored on the page. + assert array('I').itemsize == 4 + page = array('I', raw_page[:8] + bytes([0, 0]) + raw_page[10:]) + + assert len(page) % N_SUMS == 0 + + sums = [ + 0x5B1F36E9, 0xB8525960, 0x02AB50AA, 0x1DE66D2A, + 0x79FF467A, 0x9BB9F8A3, 0x217E7CD2, 0x83E13D2C, + 0xF8D4474F, 0xE39EB970, 0x42C6AE16, 0x993216FA, + 0x7B093B5D, 0x98DAFF3C, 0xF718902A, 0x0B1C9CDB, + 0xE58F764B, 0x187636BC, 0x5D7B3BB1, 0xE73DE7DE, + 0x92BEC979, 0xCCA6C0B2, 0x304A0979, 0x85AA43D4, + 0x783125BB, 0x6CA8EAA2, 0xE407EAC6, 0x4B5CFC3E, + 0x9FBF8C76, 0x15CA20BE, 0xF2CA9FD3, 0x959BD756 + ] + + def mix2sum(s, v): + tmp = s ^ v + return ((tmp * FNV_PRIME) & MASK) ^ (tmp >> 17) + + def mix_chunk2sums(sums, values): + return [mix2sum(s, v) for s, v in zip(sums, values)] + + # main checksum calculation + for i in range(0, len(page), N_SUMS): + sums = mix_chunk2sums(sums, page[i:i+N_SUMS]) + + # finally add in two rounds of zeroes for additional mixing + for _ in range(2): + sums = mix_chunk2sums(sums, [0] * N_SUMS) + + # xor fold partial checksums together + result = blkno + for s in sums: + result ^= s + + return result % 65535 + 1 + + +def validate_data_file(filename, blcksz = 0) -> bool: + file_size = os.path.getsize(filename) + if blcksz == 0: + blcksz = get_page_size(filename) + assert file_size % blcksz == 0 + + # determine positional number of first page based on segment number + fname = os.path.basename(filename) + if '.' in fname: + segno = int(fname.rsplit('.', 1)[1]) + # Hardwired segments size 1GB + basepage = (1<<30) / blcksz * segno + else: + basepage = 0 + + with open(filename, "rb") as f: + for blckno in range(file_size // blcksz): + raw_page = f.read(blcksz) + if len(raw_page) == 0: + break + if len(raw_page) != blcksz: + return False + checksum = struct.unpack('H', raw_page[8:10])[0] + + calculated_checksum = pg_checksum_block(raw_page, basepage + blckno) + if checksum != calculated_checksum: + return False + + return True + + +def corrupt_data_file(filename): + blcksz = get_page_size(filename) + try: + while True: + if not corrupt_file(filename): + return False + if not validate_data_file(filename, blcksz): + return True + except OSError: + return False + + +def corrupt_file(filename): + file_size = None + try: + file_size = os.path.getsize(filename) + + with open(filename, "rb+") as f: + pos = random.randint(int(0.1*file_size),int(0.8*file_size)) + len = int(0.1 * file_size) + 1 + f.seek(pos) + old = f.read(len) + new = random_string(len, old) + f.seek(pos) + f.write(new) + except OSError: + return False + + return True + + +def random_string(n, old_bytes=b''): + """ + Generate random string so that it's not equal neither to old bytes nor + to casefold text of these bytes + """ + old_str = old_bytes.decode('latin-1', errors='replace').casefold() + template = string.ascii_letters + string.digits + random_bytes = old_bytes + random_str = old_str + while random_bytes == old_bytes or random_str.casefold() == old_str: + random_str = ''.join([random.choice(template) for i in range(int(n))]) + random_bytes = str.encode(random_str) + return random_bytes + def _tail_file(file, linetimeout, totaltimeout): start = time.time() @@ -76,3 +269,6 @@ def wait(self, *, contains:str = None, regex:str = None): def wait_shutdown(self): self.wait(contains='database system is shut down') + + def wait_archive_push_completed(self): + self.wait(contains='archive-push completed successfully') diff --git a/tests/helpers/enums/date_time_enum.py b/tests/helpers/enums/date_time_enum.py new file mode 100644 index 000000000..b96a58ec6 --- /dev/null +++ b/tests/helpers/enums/date_time_enum.py @@ -0,0 +1,7 @@ +from enum import Enum + + +class DateTimePattern(Enum): + # 2022-12-30 14:07:30+01 + Y_m_d_H_M_S_z_dash = '%Y-%m-%d %H:%M:%S%z' + Y_m_d_H_M_S_f_z_dash = '%Y-%m-%d %H:%M:%S.%f%z' diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index 6b665097c..f06629012 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -1,21 +1,34 @@ # you need os for unittest to work +import gzip +import io import os -import gc +import threading import unittest -from sys import exit, argv, version_info -import signal -import subprocess import shutil -import six +import sys + import testgres +from testgres.enums import NodeStatus import hashlib -import re -import getpass -import select -from time import sleep +import time import re import json -import random +import contextlib + +from pg_probackup2.gdb import GDBobj +from pg_probackup2.init_helpers import init_params +from pg_probackup2.app import ProbackupApp +from pg_probackup2.storage.fs_backup import TestBackupDir, FSTestBackupDir + +try: + import lz4.frame +except ImportError: + pass + +try: + import zstd +except ImportError: + pass idx_ptrack = { 't_heap': { @@ -63,349 +76,133 @@ } } -warning = """ -Wrong splint in show_pb -Original Header: -{header} -Original Body: -{body} -Splitted Header -{header_split} -Splitted Body -{body_split} -""" +def load_backup_class(fs_type): + fs_type = os.environ.get('PROBACKUP_FS_TYPE') + implementation = f"{__package__}.fs_backup.FSTestBackupDir" + if fs_type: + implementation = fs_type + + print("Using ", implementation) + module_name, class_name = implementation.rsplit(sep='.', maxsplit=1) + + module = importlib.import_module(module_name) + + return getattr(module, class_name) + + +fs_backup_class = FSTestBackupDir +if os.environ.get('PROBACKUP_FS_TYPE'): + fs_backup_class = load_backup_class(os.environ.get('PROBACKUP_FS_TYPE')) +# Run tests on s3 when we have PG_PROBACKUP_S3_TEST (minio, vk...) or PG_PROBACKUP_S3_CONFIG_FILE. +# If PG_PROBACKUP_S3_CONFIG_FILE is 'True', then using default conf file. Check config_provider.py +elif (os.environ.get('PG_PROBACKUP_S3_TEST') and os.environ.get('PG_PROBACKUP_S3_HOST') or + os.environ.get('PG_PROBACKUP_S3_CONFIG_FILE')): + root = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..')) + if root not in sys.path: + sys.path.append(root) + from tests.test_utils.s3_backup import S3TestBackupDir + fs_backup_class = S3TestBackupDir def dir_files(base_dir): out_list = [] for dir_name, subdir_list, file_list in os.walk(base_dir): - if dir_name != base_dir: - out_list.append(os.path.relpath(dir_name, base_dir)) + rel_dir = os.path.relpath(dir_name, base_dir) + if rel_dir != '.': + out_list.append(rel_dir) for fname in file_list: out_list.append( os.path.relpath(os.path.join( dir_name, fname), base_dir) - ) + ) out_list.sort() return out_list -def is_pgpro(): - # pg_config --help - cmd = [os.environ['PG_CONFIG'], '--help'] - - result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) - return b'postgrespro' in result.stdout - - -def is_enterprise(): - # pg_config --help - cmd = [os.environ['PG_CONFIG'], '--help'] - - p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) - # PostgresPro std or ent - if b'postgrespro' in p.stdout: - cmd = [os.environ['PG_CONFIG'], '--pgpro-edition'] - p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) - - return b'enterprise' in p.stdout - else: # PostgreSQL - return False - - -def is_nls_enabled(): - cmd = [os.environ['PG_CONFIG'], '--configure'] - - result = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True) - return b'enable-nls' in result.stdout - - def base36enc(number): """Converts an integer to a base36 string.""" - alphabet = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' - base36 = '' - sign = '' - if number < 0: - sign = '-' - number = -number - - if 0 <= number < len(alphabet): - return sign + alphabet[number] + return '-' + base36enc(-number) - while number != 0: + alphabet = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ' + base36 = '' + while number >= len(alphabet): number, i = divmod(number, len(alphabet)) - base36 = alphabet[i] + base36 - - return sign + base36 - - -class ProbackupException(Exception): - def __init__(self, message, cmd): - self.message = message - self.cmd = cmd - - def __str__(self): - return '\n ERROR: {0}\n CMD: {1}'.format(repr(self.message), self.cmd) - -class PostgresNodeExtended(testgres.PostgresNode): - - def __init__(self, base_dir=None, *args, **kwargs): - super(PostgresNodeExtended, self).__init__(name='test', base_dir=base_dir, *args, **kwargs) - self.is_started = False + base36 += alphabet[i] + base36 += alphabet[number] + return base36[::-1] - def slow_start(self, replica=False): - # wait for https://github.com/postgrespro/testgres/pull/50 - # self.start() - # self.poll_query_until( - # "postgres", - # "SELECT not pg_is_in_recovery()", - # suppress={testgres.NodeConnection}) - if replica: - query = 'SELECT pg_is_in_recovery()' - else: - query = 'SELECT not pg_is_in_recovery()' - - self.start() - while True: - try: - output = self.safe_psql('template1', query).decode("utf-8").rstrip() - - if output == 't': - break - - except testgres.QueryException as e: - if 'database system is starting up' in e.message: - pass - elif 'FATAL: the database system is not accepting connections' in e.message: - pass - elif replica and 'Hot standby mode is disabled' in e.message: - raise e - else: - raise e - - sleep(0.5) - - def start(self, *args, **kwargs): - if not self.is_started: - super(PostgresNodeExtended, self).start(*args, **kwargs) - self.is_started = True - return self - - def stop(self, *args, **kwargs): - if self.is_started: - result = super(PostgresNodeExtended, self).stop(*args, **kwargs) - self.is_started = False - return result - - def kill(self, someone = None): - if self.is_started: - sig = signal.SIGKILL if os.name != 'nt' else signal.SIGBREAK - if someone == None: - os.kill(self.pid, sig) - else: - os.kill(self.auxiliary_pids[someone][0], sig) - self.is_started = False - - def table_checksum(self, table, dbname="postgres"): - con = self.connect(dbname=dbname) - - curname = "cur_"+str(random.randint(0,2**48)) +def base36dec(id): + return int(id, 36) - con.execute(""" - DECLARE %s NO SCROLL CURSOR FOR - SELECT t::text FROM %s as t - """ % (curname, table)) - sum = hashlib.md5() - while True: - rows = con.execute("FETCH FORWARD 5000 FROM %s" % curname) - if not rows: - break - for row in rows: - # hash uses SipHash since Python3.4, therefore it is good enough - sum.update(row[0].encode('utf8')) - - con.execute("CLOSE %s; ROLLBACK;" % curname) - - con.close() - return sum.hexdigest() - -class ProbackupTest(object): +class ProbackupTest(unittest.TestCase): # Class attributes - enterprise = is_enterprise() - enable_nls = is_nls_enabled() - pgpro = is_pgpro() + enterprise = init_params.is_enterprise + shardman = init_params.is_shardman + enable_nls = init_params.is_nls_enabled + enable_lz4 = init_params.is_lz4_enabled + pgpro = init_params.is_pgpro + verbose = init_params.verbose + username = init_params.username + remote = init_params.remote + ptrack = init_params.ptrack + paranoia = init_params.paranoia + tests_source_path = os.path.join(init_params.source_path, 'tests') + archive_compress = init_params.archive_compress + compress_suffix = init_params.compress_suffix + pg_config_version = init_params.pg_config_version + probackup_path = init_params.probackup_path + probackup_old_path = init_params.probackup_old_path + probackup_version = init_params.probackup_version + old_probackup_version = init_params.old_probackup_version + cfs_compress_default = init_params.cfs_compress + EXTERNAL_DIRECTORY_DELIMITER = init_params.EXTERNAL_DIRECTORY_DELIMITER + s3_type = os.environ.get('PG_PROBACKUP_S3_TEST') + + auto_compress_alg = True def __init__(self, *args, **kwargs): - super(ProbackupTest, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) + self.output = None + self.cmd = None self.nodes_to_cleanup = [] if isinstance(self, unittest.TestCase): - self.module_name = self.id().split('.')[1] - self.fname = self.id().split('.')[3] - - if '-v' in argv or '--verbose' in argv: - self.verbose = True - else: - self.verbose = False - - self.test_env = os.environ.copy() - envs_list = [ - 'LANGUAGE', - 'LC_ALL', - 'PGCONNECT_TIMEOUT', - 'PGDATA', - 'PGDATABASE', - 'PGHOSTADDR', - 'PGREQUIRESSL', - 'PGSERVICE', - 'PGSSLMODE', - 'PGUSER', - 'PGPORT', - 'PGHOST' - ] - - for e in envs_list: try: - del self.test_env[e] - except: - pass - - self.test_env['LC_MESSAGES'] = 'C' - self.test_env['LC_TIME'] = 'C' - - self.gdb = 'PGPROBACKUP_GDB' in self.test_env and \ - self.test_env['PGPROBACKUP_GDB'] == 'ON' - - self.paranoia = 'PG_PROBACKUP_PARANOIA' in self.test_env and \ - self.test_env['PG_PROBACKUP_PARANOIA'] == 'ON' - - self.archive_compress = 'ARCHIVE_COMPRESSION' in self.test_env and \ - self.test_env['ARCHIVE_COMPRESSION'] == 'ON' - - try: - testgres.configure_testgres( - cache_initdb=False, - cached_initdb_dir=False, - cache_pg_config=False, - node_cleanup_full=False) - except: - pass - - self.helpers_path = os.path.dirname(os.path.realpath(__file__)) - self.dir_path = os.path.abspath( - os.path.join(self.helpers_path, os.pardir) - ) - self.tmp_path = os.path.abspath( - os.path.join(self.dir_path, 'tmp_dirs') - ) - try: - os.makedirs(os.path.join(self.dir_path, 'tmp_dirs')) - except: - pass - - self.user = self.get_username() - self.probackup_path = None - if 'PGPROBACKUPBIN' in self.test_env: - if shutil.which(self.test_env["PGPROBACKUPBIN"]): - self.probackup_path = self.test_env["PGPROBACKUPBIN"] - else: - if self.verbose: - print('PGPROBACKUPBIN is not an executable file') - - if not self.probackup_path: - probackup_path_tmp = os.path.join( - testgres.get_pg_config()['BINDIR'], 'pg_probackup') - - if os.path.isfile(probackup_path_tmp): - if not os.access(probackup_path_tmp, os.X_OK): - print('{0} is not an executable file'.format( - probackup_path_tmp)) - else: - self.probackup_path = probackup_path_tmp - - if not self.probackup_path: - probackup_path_tmp = os.path.abspath(os.path.join( - self.dir_path, '../pg_probackup')) - - if os.path.isfile(probackup_path_tmp): - if not os.access(probackup_path_tmp, os.X_OK): - print('{0} is not an executable file'.format( - probackup_path_tmp)) - else: - self.probackup_path = probackup_path_tmp - - if not self.probackup_path: - print('pg_probackup binary is not found') - exit(1) - - if os.name == 'posix': - self.EXTERNAL_DIRECTORY_DELIMITER = ':' - os.environ['PATH'] = os.path.dirname( - self.probackup_path) + ':' + os.environ['PATH'] - - elif os.name == 'nt': - self.EXTERNAL_DIRECTORY_DELIMITER = ';' - os.environ['PATH'] = os.path.dirname( - self.probackup_path) + ';' + os.environ['PATH'] - - self.probackup_old_path = None - - if 'PGPROBACKUPBIN_OLD' in self.test_env: - if ( - os.path.isfile(self.test_env['PGPROBACKUPBIN_OLD']) and - os.access(self.test_env['PGPROBACKUPBIN_OLD'], os.X_OK) - ): - self.probackup_old_path = self.test_env['PGPROBACKUPBIN_OLD'] - else: - if self.verbose: - print('PGPROBACKUPBIN_OLD is not an executable file') - - self.probackup_version = None - self.old_probackup_version = None - - try: - self.probackup_version_output = subprocess.check_output( - [self.probackup_path, "--version"], - stderr=subprocess.STDOUT, - ).decode('utf-8') - except subprocess.CalledProcessError as e: - raise ProbackupException(e.output.decode('utf-8')) - - if self.probackup_old_path: - old_probackup_version_output = subprocess.check_output( - [self.probackup_old_path, "--version"], - stderr=subprocess.STDOUT, - ).decode('utf-8') - self.old_probackup_version = re.search( - r"\d+\.\d+\.\d+", - subprocess.check_output( - [self.probackup_old_path, "--version"], - stderr=subprocess.STDOUT, - ).decode('utf-8') - ).group(0) - - self.probackup_version = re.search(r"\d+\.\d+\.\d+", self.probackup_version_output).group(0) - - self.remote = False - self.remote_host = None - self.remote_port = None - self.remote_user = None - - if 'PGPROBACKUP_SSH_REMOTE' in self.test_env: - if self.test_env['PGPROBACKUP_SSH_REMOTE'] == 'ON': - self.remote = True - - self.ptrack = False - if 'PG_PROBACKUP_PTRACK' in self.test_env: - if self.test_env['PG_PROBACKUP_PTRACK'] == 'ON': - if self.pg_config_version >= self.version_to_num('11.0'): - self.ptrack = True - - os.environ["PGAPPNAME"] = "pg_probackup" + self.module_name = self.id().split('.')[-2] + self.fname = self.id().split('.')[-1] + except IndexError: + print("Couldn't get module name and function name from self.id(): `{}`".format(self.id())) + self.module_name = self.module_name if self.module_name else str(self).split('(')[1].split('.')[1] + self.fname = str(self).split('(')[0] + + self.test_env = init_params.test_env() + + if self.s3_type != "minio": + if 'PG_PROBACKUP_S3_HOST' in self.test_env: + del(self.test_env['PG_PROBACKUP_S3_HOST']) + if 'PG_PROBACKUP_S3_PORT' in self.test_env: + del(self.test_env['PG_PROBACKUP_S3_PORT']) + + self.rel_path = os.path.join(self.module_name, self.fname) + self.test_path = os.path.join(init_params.tmp_path, self.rel_path) + + self.pg_node = testgres.NodeApp(self.test_path, self.nodes_to_cleanup) + self.pg_node.os_ops.set_env('LANGUAGE','en') + + # Cleanup FS dependent part first + self.backup_dir = self.build_backup_dir('backup') + self.backup_dir.cleanup() + # Recreate the rest which should reside on local file system only + shutil.rmtree(self.test_path, ignore_errors=True) + os.makedirs(self.test_path) + + self.pb_log_path = os.path.join(self.test_path, "pb_log") + self.pb = ProbackupApp(self, self.pg_node, self.pb_log_path, self.test_env, + self.auto_compress_alg, self.backup_dir) def is_test_result_ok(test_case): # sources of solution: @@ -414,29 +211,39 @@ def is_test_result_ok(test_case): # # 2. python versions 3.11+ mixin, verified on 3.11, taken from: https://stackoverflow.com/a/39606065 - if not isinstance(test_case, unittest.TestCase): - raise AssertionError("test_case is not instance of unittest.TestCase") - - if hasattr(test_case, '_outcome'): # Python 3.4+ - if hasattr(test_case._outcome, 'errors'): - # Python 3.4 - 3.10 (These two methods have no side effects) - result = test_case.defaultTestResult() # These two methods have no side effects - test_case._feedErrorsToResult(result, test_case._outcome.errors) - else: - # Python 3.11+ - result = test_case._outcome.result - else: # Python 2.7, 3.0-3.3 - result = getattr(test_case, '_outcomeForDoCleanups', test_case._resultForDoCleanups) + if hasattr(test_case._outcome, 'errors'): + # Python 3.4 - 3.10 (These two methods have no side effects) + result = test_case.defaultTestResult() # These two methods have no side effects + test_case._feedErrorsToResult(result, test_case._outcome.errors) + else: + # Python 3.11+ and pytest 5.3.5+ + result = test_case._outcome.result + if not hasattr(result, 'errors'): + result.errors = [] + if not hasattr(result, 'failures'): + result.failures = [] ok = all(test != test_case for test, text in result.errors + result.failures) + # check subtests as well + ok = ok and all(getattr(test, 'test_case', None) != test_case + for test, text in result.errors + result.failures) + + # for pytest 8+ + if hasattr(result, '_excinfo'): + if result._excinfo is not None and len(result._excinfo) > 0: + # if test was successful, _excinfo will be None, else it will be non-empty list + ok = False return ok def tearDown(self): + node_crashed = None if self.is_test_result_ok(): for node in self.nodes_to_cleanup: + if node.is_started and node.status() != NodeStatus.Running: + node_crashed = node node.cleanup() - self.del_test_dir(self.module_name, self.fname) + self.del_test_dirs() else: for node in self.nodes_to_cleanup: @@ -446,143 +253,27 @@ def tearDown(self): self.nodes_to_cleanup.clear() - @property - def pg_config_version(self): - return self.version_to_num( - testgres.get_pg_config()['VERSION'].split(" ")[1]) - -# if 'PGPROBACKUP_SSH_HOST' in self.test_env: -# self.remote_host = self.test_env['PGPROBACKUP_SSH_HOST'] -# else -# print('PGPROBACKUP_SSH_HOST is not set') -# exit(1) -# -# if 'PGPROBACKUP_SSH_PORT' in self.test_env: -# self.remote_port = self.test_env['PGPROBACKUP_SSH_PORT'] -# else -# print('PGPROBACKUP_SSH_PORT is not set') -# exit(1) -# -# if 'PGPROBACKUP_SSH_USER' in self.test_env: -# self.remote_user = self.test_env['PGPROBACKUP_SSH_USER'] -# else -# print('PGPROBACKUP_SSH_USER is not set') -# exit(1) - - def make_empty_node( - self, - base_dir=None): - real_base_dir = os.path.join(self.tmp_path, base_dir) - shutil.rmtree(real_base_dir, ignore_errors=True) - os.makedirs(real_base_dir) - - node = PostgresNodeExtended(base_dir=real_base_dir) - node.should_rm_dirs = True - self.nodes_to_cleanup.append(node) - - return node - - def make_simple_node( - self, - base_dir=None, - set_replication=False, - ptrack_enable=False, - initdb_params=[], - pg_options={}): - - node = self.make_empty_node(base_dir) - node.init( - initdb_params=initdb_params, allow_streaming=set_replication) - - # set major version - with open(os.path.join(node.data_dir, 'PG_VERSION')) as f: - node.major_version_str = str(f.read().rstrip()) - node.major_version = float(node.major_version_str) - - # Sane default parameters - options = {} - options['max_connections'] = 100 - options['shared_buffers'] = '10MB' - options['fsync'] = 'off' - - options['wal_level'] = 'logical' - options['hot_standby'] = 'off' - - options['log_line_prefix'] = '%t [%p]: [%l-1] ' - options['log_statement'] = 'none' - options['log_duration'] = 'on' - options['log_min_duration_statement'] = 0 - options['log_connections'] = 'on' - options['log_disconnections'] = 'on' - options['restart_after_crash'] = 'off' - options['autovacuum'] = 'off' - - # Allow replication in pg_hba.conf - if set_replication: - options['max_wal_senders'] = 10 - - if ptrack_enable: - options['ptrack.map_size'] = '128' - options['shared_preload_libraries'] = 'ptrack' - - if node.major_version >= 13: - options['wal_keep_size'] = '200MB' - else: - options['wal_keep_segments'] = '100' + if node_crashed: + self.fail(f"Node '{os.path.relpath(node.base_dir, self.test_path)}' unexpectingly crashed") - # set default values - self.set_auto_conf(node, options) + def build_backup_dir(self, backup='backup'): + return fs_backup_class(rel_path=self.rel_path, backup=backup) - # Apply given parameters - self.set_auto_conf(node, pg_options) + def read_pb_log(self): + with open(os.path.join(self.pb_log_path, 'pg_probackup.log')) as fl: + return fl.read() - # kludge for testgres - # https://github.com/postgrespro/testgres/issues/54 - # for PG >= 13 remove 'wal_keep_segments' parameter - if node.major_version >= 13: - self.set_auto_conf( - node, {}, 'postgresql.conf', ['wal_keep_segments']) + def unlink_pg_log(self): + os.unlink(os.path.join(self.pb_log_path, 'pg_probackup.log')) - return node - def simple_bootstrap(self, node, role) -> None: node.safe_psql( 'postgres', 'CREATE ROLE {0} WITH LOGIN REPLICATION'.format(role)) - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'postgres', - 'GRANT USAGE ON SCHEMA pg_catalog TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO {0};'.format(role)) - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'postgres', - 'GRANT USAGE ON SCHEMA pg_catalog TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO {0}; ' - 'GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_checkpoint() TO {0};'.format(role)) # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + if self.pg_config_version < 150000: node.safe_psql( 'postgres', 'GRANT USAGE ON SCHEMA pg_catalog TO {0}; ' @@ -620,12 +311,12 @@ def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False 'select exists' " (select 1 from pg_tablespace where spcname = '{0}')".format( tblspc_name) - ) + ) # Check that tablespace with name 'tblspc_name' do not exists already self.assertFalse( res[0][0], 'Tablespace "{0}" already exists'.format(tblspc_name) - ) + ) if not tblspc_path: tblspc_path = os.path.join( @@ -633,7 +324,13 @@ def create_tblspace_in_node(self, node, tblspc_name, tblspc_path=None, cfs=False cmd = "CREATE TABLESPACE {0} LOCATION '{1}'".format( tblspc_name, tblspc_path) if cfs: - cmd += ' with (compression=true)' + + if cfs is True and self.cfs_compress_default: + cfs = self.cfs_compress_default + if cfs is True or node.major_version < 12: + cmd += ' with (compression=true)' + else: + cmd += ' with (compression=' + cfs + ')' if not os.path.exists(tblspc_path): os.makedirs(tblspc_path) @@ -649,12 +346,12 @@ def drop_tblspace(self, node, tblspc_name): 'select exists' " (select 1 from pg_tablespace where spcname = '{0}')".format( tblspc_name) - ) + ) # Check that tablespace with name 'tblspc_name' do not exists already self.assertTrue( res[0][0], 'Tablespace "{0}" do not exists'.format(tblspc_name) - ) + ) rels = node.execute( "postgres", @@ -671,7 +368,6 @@ def drop_tblspace(self, node, tblspc_name): 'postgres', 'DROP TABLESPACE {0}'.format(tblspc_name)) - def get_tblspace_path(self, node, tblspc_name): return os.path.join(node.base_dir, tblspc_name) @@ -686,13 +382,13 @@ def get_fork_path(self, node, fork_name): 'postgres', "select pg_relation_filepath('{0}')".format( fork_name))[0][0] - ) + ) def get_md5_per_page_for_fork(self, file, size_in_pages): pages_per_segment = {} md5_per_page = {} size_in_pages = int(size_in_pages) - nsegments = int(size_in_pages/131072) + nsegments = int(size_in_pages / 131072) if size_in_pages % 131072 != 0: nsegments = nsegments + 1 @@ -712,9 +408,9 @@ def get_md5_per_page_for_fork(self, file, size_in_pages): end_page = pages_per_segment[segment_number] else: file_desc = os.open( - file+'.{0}'.format(segment_number), os.O_RDONLY - ) - start_page = max(md5_per_page)+1 + file + '.{0}'.format(segment_number), os.O_RDONLY + ) + start_page = max(md5_per_page) + 1 end_page = end_page + pages_per_segment[segment_number] for page in range(start_page, end_page): @@ -726,38 +422,34 @@ def get_md5_per_page_for_fork(self, file, size_in_pages): return md5_per_page - def get_ptrack_bits_per_page_for_fork(self, node, file, size=[]): + def get_ptrack_bits_per_page_for_fork(self, node, file, size=None): - if self.get_pgpro_edition(node) == 'enterprise': - if self.get_version(node) < self.version_to_num('10.0'): - header_size = 48 - else: - header_size = 24 - else: - header_size = 24 + if size is None: + size = [] + header_size = 24 ptrack_bits_for_fork = [] # TODO: use macro instead of hard coded 8KB - page_body_size = 8192-header_size + page_body_size = 8192 - header_size # Check that if main fork file size is 0, it`s ok # to not having a _ptrack fork if os.path.getsize(file) == 0: return ptrack_bits_for_fork byte_size = os.path.getsize(file + '_ptrack') - npages = int(byte_size/8192) + npages = int(byte_size / 8192) if byte_size % 8192 != 0: print('Ptrack page is not 8k aligned') - exit(1) + sys.exit(1) file = os.open(file + '_ptrack', os.O_RDONLY) for page in range(npages): - offset = 8192*page+header_size + offset = 8192 * page + header_size os.lseek(file, offset, 0) lots_of_bytes = os.read(file, page_body_size) byte_list = [ - lots_of_bytes[i:i+1] for i in range(len(lots_of_bytes)) - ] + lots_of_bytes[i:i + 1] for i in range(len(lots_of_bytes)) + ] for byte in byte_list: # byte_inverted = bin(int(byte, base=16))[2:][::-1] # bits = (byte >> x) & 1 for x in range(7, -1, -1) @@ -832,7 +524,7 @@ def check_ptrack_sanity(self, idx_dict): # corresponding page in old_pages are been dealt with. # We can now safely proceed to comparing old and new pages if idx_dict['new_pages'][ - PageNum] != idx_dict['old_pages'][PageNum]: + PageNum] != idx_dict['old_pages'][PageNum]: # Page has been changed, # meaning that ptrack should be equal to 1 if idx_dict['ptrack'][PageNum] != 1: @@ -878,23 +570,82 @@ def check_ptrack_sanity(self, idx_dict): # ) def get_backup_filelist(self, backup_dir, instance, backup_id): - - filelist_path = os.path.join( - backup_dir, 'backups', - instance, backup_id, 'backup_content.control') - - with open(filelist_path, 'r') as f: - filelist_raw = f.read() - - filelist_splitted = filelist_raw.splitlines() + path = os.path.join('backups', instance, backup_id, 'backup_content.control') + filelist_raw = backup_dir.read_file(path) filelist = {} - for line in filelist_splitted: + for line in io.StringIO(filelist_raw): line = json.loads(line) filelist[line['path']] = line return filelist + def get_backup_listdir(self, backup_dir, instance, backup_id, sub_path): + subpath = os.path.join('backups', instance, backup_id, sub_path) + return backup_dir.list_files(subpath) + + def get_backups_dirs(self, backup_dir, instance): + subpath = os.path.join("backups", instance) + return backup_dir.list_dirs(subpath) + + def read_backup_file(self, backup_dir, instance, backup_id, + sub_path, *, text=False): + subpath = os.path.join('backups', instance, backup_id, sub_path) + return backup_dir.read_file(subpath, text=text) + + def write_backup_file(self, backup_dir, instance, backup_id, + sub_path, content, *, text=False): + subpath = os.path.join('backups', instance, backup_id, sub_path) + return backup_dir.write_file(subpath, content, text=text) + + def corrupt_backup_file(self, backup_dir, instance, backup_id, sub_path, *, + damage: tuple = None, + truncate: int = None, + overwrite=None, + text=False): + subpath = os.path.join('backups', instance, backup_id, sub_path) + if overwrite: + content = overwrite + elif truncate == 0: + content = '' if text else b'' + else: + content = backup_dir.read_file(subpath, text=text) + if damage: + pos, replace = damage + content = content[:pos] + replace + content[pos + len(replace):] + if truncate is not None: + content = content[:truncate] + backup_dir.write_file(subpath, content, text=text) + + def remove_backup_file(self, backup_dir, instance, backup_id, sub_path): + subpath = os.path.join('backups', instance, backup_id, sub_path) + backup_dir.remove_file(subpath) + + def backup_file_exists(self, backup_dir, instance, backup_id, sub_path): + subpath = os.path.join('backups', instance, backup_id, sub_path) + return backup_dir.exists(subpath) + + def remove_backup_config(self, backup_dir, instance): + subpath = os.path.join('backups', instance, 'pg_probackup.conf') + backup_dir.remove_file(subpath) + + @contextlib.contextmanager + def modify_backup_config(self, backup_dir, instance): + path = os.path.join('backups', instance, 'pg_probackup.conf') + control_file = backup_dir.read_file(path) + cf = ProbackupTest.ControlFileContainer(control_file) + yield cf + if control_file != cf.data: + backup_dir.write_file(path, cf.data) + + def remove_one_backup(self, backup_dir, instance, backup_id): + subpath = os.path.join('backups', instance, backup_id) + backup_dir.remove_dir(subpath) + + def remove_one_backup_instance(self, backup_dir, instance): + subpath = os.path.join('backups', instance) + backup_dir.remove_dir(subpath) + # return dict of files from filelist A, # which are not exists in filelist_B def get_backup_filelist_diff(self, filelist_A, filelist_B): @@ -906,6 +657,62 @@ def get_backup_filelist_diff(self, filelist_A, filelist_B): return filelist_diff + def get_instance_wal_list(self, backup_dir, instance): + files = map(str, backup_dir.list_files(os.path.join('wal', instance))) + files = [f for f in files + if not any(x in f for x in ('.backup', '.history', '~tmp'))] + files.sort() + return files + + def read_instance_wal(self, backup_dir, instance, file, decompress=False): + content = backup_dir.read_file(f'wal/{instance}/{file}', text=False) + if decompress: + content = _do_decompress(file, content) + return content + + def write_instance_wal(self, backup_dir, instance, file, data, compress=False): + if compress: + data = _do_compress(file, data) + return backup_dir.write_file(f'wal/{instance}/{file}', data, text=False) + + def corrupt_instance_wal(self, backup_dir, instance, file, pos, damage, decompressed=False): + subpath = f'wal/{instance}/{file}' + content = backup_dir.read_file(subpath, text=False) + if decompressed: + content = _do_decompress(subpath, content) + content = content[:pos] + \ + bytes(d^c for d, c in zip(content[pos:pos+len(damage)], damage)) + \ + content[pos + len(damage):] + if decompressed: + content = _do_compress(subpath, content) + backup_dir.write_file(subpath, content, text=False) + + def remove_instance_wal(self, backup_dir, instance, file): + backup_dir.remove_file(f'wal/{instance}/{file}') + + def instance_wal_exists(self, backup_dir, instance, file): + fl = f'wal/{instance}/{file}' + return backup_dir.exists(fl) + + def wait_instance_wal_exists(self, backup_dir, instance, file, timeout=300): + start = time.time() + fl = f'wal/{instance}/{file}' + while time.time() - start < timeout: + if backup_dir.exists(fl): + break + time.sleep(0.25) + + def wait_server_wal_exists(self, data_dir, wal_dir, file, timeout=300): + start = time.time() + fl = f'{data_dir}/{wal_dir}/{file}' + while time.time() - start < timeout: + if os.path.exists(fl): + return + time.sleep(0.25) + + def remove_instance_waldir(self, backup_dir, instance): + backup_dir.remove_dir(f'wal/{instance}') + # used for partial restore def truncate_every_file_in_dir(self, path): for file in os.listdir(path): @@ -942,487 +749,43 @@ def check_ptrack_clean(self, idx_dict, size): ) ) - def run_pb(self, command, asynchronous=False, gdb=False, old_binary=False, return_id=True, env=None): - if not self.probackup_old_path and old_binary: - print('PGPROBACKUPBIN_OLD is not set') - exit(1) - - if old_binary: - binary_path = self.probackup_old_path - else: - binary_path = self.probackup_path + def read_backup_content_control(self, backup_id, instance_name): + """ + Read the content control file of a backup. + Args: backup_id (str): The ID of the backup. + instance_name (str): The name of the instance + Returns: dict: The parsed JSON content of the backup_content.control file. + Raises: + FileNotFoundError: If the backup content control file does not exist. + json.JSONDecodeError: If the backup content control file is not a valid JSON. + """ + content_control_path = f'{self.backup_dir.path}/backups/{instance_name}/{backup_id}/backup_content.control' - if not env: - env=self.test_env + if not os.path.exists(content_control_path): + raise FileNotFoundError(f"Backup content control file '{content_control_path}' does not exist.") try: - self.cmd = [' '.join(map(str, [binary_path] + command))] - if self.verbose: - print(self.cmd) - if gdb: - return GDBobj([binary_path] + command, self) - if asynchronous: - return subprocess.Popen( - [binary_path] + command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env - ) - else: - self.output = subprocess.check_output( - [binary_path] + command, - stderr=subprocess.STDOUT, - env=env - ).decode('utf-8') - if command[0] == 'backup' and return_id: - # return backup ID - for line in self.output.splitlines(): - if 'INFO: Backup' and 'completed' in line: - return line.split()[2] - else: - return self.output - except subprocess.CalledProcessError as e: - raise ProbackupException(e.output.decode('utf-8').replace("\r",""), - self.cmd) - - def run_binary(self, command, asynchronous=False, env=None): - - if not env: - env = self.test_env - - if self.verbose: - print([' '.join(map(str, command))]) - try: - if asynchronous: - return subprocess.Popen( - command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=env - ) - else: - self.output = subprocess.check_output( - command, - stderr=subprocess.STDOUT, - env=env - ).decode('utf-8') - return self.output - except subprocess.CalledProcessError as e: - raise ProbackupException(e.output.decode('utf-8'), command) - - def init_pb(self, backup_dir, options=[], old_binary=False): - - shutil.rmtree(backup_dir, ignore_errors=True) - - # don`t forget to kill old_binary after remote ssh release - if self.remote and not old_binary: - options = options + [ - '--remote-proto=ssh', - '--remote-host=localhost'] - - return self.run_pb([ - 'init', - '-B', backup_dir - ] + options, - old_binary=old_binary - ) - - def add_instance(self, backup_dir, instance, node, old_binary=False, options=[]): - - cmd = [ - 'add-instance', - '--instance={0}'.format(instance), - '-B', backup_dir, - '-D', node.data_dir - ] - - # don`t forget to kill old_binary after remote ssh release - if self.remote and not old_binary: - options = options + [ - '--remote-proto=ssh', - '--remote-host=localhost'] - - return self.run_pb(cmd + options, old_binary=old_binary) - - def set_config(self, backup_dir, instance, old_binary=False, options=[]): - - cmd = [ - 'set-config', - '--instance={0}'.format(instance), - '-B', backup_dir, - ] - - return self.run_pb(cmd + options, old_binary=old_binary) - - def set_backup(self, backup_dir, instance, backup_id=False, - old_binary=False, options=[]): - - cmd = [ - 'set-backup', - '-B', backup_dir - ] - - if instance: - cmd = cmd + ['--instance={0}'.format(instance)] - - if backup_id: - cmd = cmd + ['-i', backup_id] - - return self.run_pb(cmd + options, old_binary=old_binary) - - def del_instance(self, backup_dir, instance, old_binary=False): - - return self.run_pb([ - 'del-instance', - '--instance={0}'.format(instance), - '-B', backup_dir - ], - old_binary=old_binary - ) + with open(content_control_path) as file: + lines = file.readlines() + content_control_json = [] + for line in lines: + content_control_json.append(json.loads(line)) + return content_control_json + except json.JSONDecodeError as e: + raise json.JSONDecodeError(f"Failed to parse JSON in backup content control file '{content_control_path}'", + e.doc, e.pos) + + def run_pb(self, backup_dir, command, gdb=False, old_binary=False, return_id=True, env=None, + skip_log_directory=False, expect_error=False): + return self.pb.run(command, gdb, old_binary, return_id, env, skip_log_directory, expect_error, use_backup_dir=backup_dir) def clean_pb(self, backup_dir): - shutil.rmtree(backup_dir, ignore_errors=True) - - def backup_node( - self, backup_dir, instance, node, data_dir=False, - backup_type='full', datname=False, options=[], - asynchronous=False, gdb=False, - old_binary=False, return_id=True, no_remote=False, - env=None - ): - if not node and not data_dir: - print('You must provide ether node or data_dir for backup') - exit(1) - - if not datname: - datname = 'postgres' - - cmd_list = [ - 'backup', - '-B', backup_dir, - '--instance={0}'.format(instance), - # "-D", pgdata, - '-p', '%i' % node.port, - '-d', datname - ] - - if data_dir: - cmd_list += ['-D', data_dir] - - # don`t forget to kill old_binary after remote ssh release - if self.remote and not old_binary and not no_remote: - options = options + [ - '--remote-proto=ssh', - '--remote-host=localhost'] - - if backup_type: - cmd_list += ['-b', backup_type] - - if not old_binary: - cmd_list += ['--no-sync'] - - return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary, return_id, env=env) - - def checkdb_node( - self, backup_dir=False, instance=False, data_dir=False, - options=[], asynchronous=False, gdb=False, old_binary=False - ): - - cmd_list = ["checkdb"] - - if backup_dir: - cmd_list += ["-B", backup_dir] - - if instance: - cmd_list += ["--instance={0}".format(instance)] - - if data_dir: - cmd_list += ["-D", data_dir] - - return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary) - - def merge_backup( - self, backup_dir, instance, backup_id, asynchronous=False, - gdb=False, old_binary=False, options=[]): - cmd_list = [ - 'merge', - '-B', backup_dir, - '--instance={0}'.format(instance), - '-i', backup_id - ] - - return self.run_pb(cmd_list + options, asynchronous, gdb, old_binary) - - def restore_node( - self, backup_dir, instance, node=False, - data_dir=None, backup_id=None, old_binary=False, options=[], - gdb=False - ): - - if data_dir is None: - data_dir = node.data_dir - - cmd_list = [ - 'restore', - '-B', backup_dir, - '-D', data_dir, - '--instance={0}'.format(instance) - ] - - # don`t forget to kill old_binary after remote ssh release - if self.remote and not old_binary: - options = options + [ - '--remote-proto=ssh', - '--remote-host=localhost'] - - if backup_id: - cmd_list += ['-i', backup_id] - - if not old_binary: - cmd_list += ['--no-sync'] - - return self.run_pb(cmd_list + options, gdb=gdb, old_binary=old_binary) - - def catchup_node( - self, - backup_mode, source_pgdata, destination_node, - options = [] - ): - - cmd_list = [ - 'catchup', - '--backup-mode={0}'.format(backup_mode), - '--source-pgdata={0}'.format(source_pgdata), - '--destination-pgdata={0}'.format(destination_node.data_dir) - ] - if self.remote: - cmd_list += ['--remote-proto=ssh', '--remote-host=localhost'] - if self.verbose: - cmd_list += [ - '--log-level-file=VERBOSE', - '--log-directory={0}'.format(destination_node.logs_dir) - ] - - return self.run_pb(cmd_list + options) - - def show_pb( - self, backup_dir, instance=None, backup_id=None, - options=[], as_text=False, as_json=True, old_binary=False, - env=None - ): - - backup_list = [] - specific_record = {} - cmd_list = [ - 'show', - '-B', backup_dir, - ] - if instance: - cmd_list += ['--instance={0}'.format(instance)] - - if backup_id: - cmd_list += ['-i', backup_id] - - # AHTUNG, WARNING will break json parsing - if as_json: - cmd_list += ['--format=json', '--log-level-console=error'] - - if as_text: - # You should print it when calling as_text=true - return self.run_pb(cmd_list + options, old_binary=old_binary, env=env) - - # get show result as list of lines - if as_json: - data = json.loads(self.run_pb(cmd_list + options, old_binary=old_binary)) - # print(data) - for instance_data in data: - # find specific instance if requested - if instance and instance_data['instance'] != instance: - continue - - for backup in reversed(instance_data['backups']): - # find specific backup if requested - if backup_id: - if backup['id'] == backup_id: - return backup - else: - backup_list.append(backup) - - if backup_id is not None: - self.assertTrue(False, "Failed to find backup with ID: {0}".format(backup_id)) - - return backup_list - else: - show_splitted = self.run_pb( - cmd_list + options, old_binary=old_binary, env=env).splitlines() - if instance is not None and backup_id is None: - # cut header(ID, Mode, etc) from show as single string - header = show_splitted[1:2][0] - # cut backup records from show as single list - # with string for every backup record - body = show_splitted[3:] - # inverse list so oldest record come first - body = body[::-1] - # split string in list with string for every header element - header_split = re.split(' +', header) - # Remove empty items - for i in header_split: - if i == '': - header_split.remove(i) - continue - header_split = [ - header_element.rstrip() for header_element in header_split - ] - for backup_record in body: - backup_record = backup_record.rstrip() - # split list with str for every backup record element - backup_record_split = re.split(' +', backup_record) - # Remove empty items - for i in backup_record_split: - if i == '': - backup_record_split.remove(i) - if len(header_split) != len(backup_record_split): - print(warning.format( - header=header, body=body, - header_split=header_split, - body_split=backup_record_split) - ) - exit(1) - new_dict = dict(zip(header_split, backup_record_split)) - backup_list.append(new_dict) - return backup_list - else: - # cut out empty lines and lines started with # - # and other garbage then reconstruct it as dictionary - # print show_splitted - sanitized_show = [item for item in show_splitted if item] - sanitized_show = [ - item for item in sanitized_show if not item.startswith('#') - ] - # print sanitized_show - for line in sanitized_show: - name, var = line.partition(' = ')[::2] - var = var.strip('"') - var = var.strip("'") - specific_record[name.strip()] = var - - if not specific_record: - self.assertTrue(False, "Failed to find backup with ID: {0}".format(backup_id)) - - return specific_record - - def show_archive( - self, backup_dir, instance=None, options=[], - as_text=False, as_json=True, old_binary=False, - tli=0 - ): - - cmd_list = [ - 'show', - '--archive', - '-B', backup_dir, - ] - if instance: - cmd_list += ['--instance={0}'.format(instance)] - - # AHTUNG, WARNING will break json parsing - if as_json: - cmd_list += ['--format=json', '--log-level-console=error'] - - if as_text: - # You should print it when calling as_text=true - return self.run_pb(cmd_list + options, old_binary=old_binary) - - if as_json: - if as_text: - data = self.run_pb(cmd_list + options, old_binary=old_binary) - else: - data = json.loads(self.run_pb(cmd_list + options, old_binary=old_binary)) - - if instance: - instance_timelines = None - for instance_name in data: - if instance_name['instance'] == instance: - instance_timelines = instance_name['timelines'] - break - - if tli > 0: - timeline_data = None - for timeline in instance_timelines: - if timeline['tli'] == tli: - return timeline - - return {} - - if instance_timelines: - return instance_timelines - - return data - else: - show_splitted = self.run_pb( - cmd_list + options, old_binary=old_binary).splitlines() - print(show_splitted) - exit(1) - - def validate_pb( - self, backup_dir, instance=None, backup_id=None, - options=[], old_binary=False, gdb=False, asynchronous=False - ): - - cmd_list = [ - 'validate', - '-B', backup_dir - ] - if instance: - cmd_list += ['--instance={0}'.format(instance)] - if backup_id: - cmd_list += ['-i', backup_id] - - return self.run_pb(cmd_list + options, old_binary=old_binary, gdb=gdb, asynchronous=asynchronous) - - def delete_pb( - self, backup_dir, instance, backup_id=None, - options=[], old_binary=False, gdb=False, asynchronous=False): - cmd_list = [ - 'delete', - '-B', backup_dir - ] - - cmd_list += ['--instance={0}'.format(instance)] - if backup_id: - cmd_list += ['-i', backup_id] - - return self.run_pb(cmd_list + options, old_binary=old_binary, gdb=gdb, asynchronous=asynchronous) - - def delete_expired( - self, backup_dir, instance, options=[], old_binary=False): - cmd_list = [ - 'delete', - '-B', backup_dir, - '--instance={0}'.format(instance) - ] - return self.run_pb(cmd_list + options, old_binary=old_binary) - - def show_config(self, backup_dir, instance, old_binary=False): - out_dict = {} - cmd_list = [ - 'show-config', - '-B', backup_dir, - '--instance={0}'.format(instance) - ] - - res = self.run_pb(cmd_list, old_binary=old_binary).splitlines() - for line in res: - if not line.startswith('#'): - name, var = line.partition(' = ')[::2] - out_dict[name] = var - return out_dict + fs_backup_class(backup_dir).cleanup() def get_recovery_conf(self, node): out_dict = {} - if self.get_version(node) >= self.version_to_num('12.0'): + if self.pg_config_version >= self.version_to_num('12.0'): recovery_conf_path = os.path.join(node.data_dir, 'postgresql.auto.conf') with open(recovery_conf_path, 'r') as f: print(f.read()) @@ -1430,7 +793,7 @@ def get_recovery_conf(self, node): recovery_conf_path = os.path.join(node.data_dir, 'recovery.conf') with open( - recovery_conf_path, 'r' + recovery_conf_path, 'r' ) as recovery_conf: for line in recovery_conf: try: @@ -1440,157 +803,41 @@ def get_recovery_conf(self, node): out_dict[key.strip()] = value.strip(" '").replace("'\n", "") return out_dict - def set_archiving( - self, backup_dir, instance, node, replica=False, - overwrite=False, compress=True, old_binary=False, - log_level=False, archive_timeout=False, - custom_archive_command=None): - - # parse postgresql.auto.conf - options = {} - if replica: - options['archive_mode'] = 'always' - options['hot_standby'] = 'on' - else: - options['archive_mode'] = 'on' - - if custom_archive_command is None: - if os.name == 'posix': - options['archive_command'] = '"{0}" archive-push -B {1} --instance={2} '.format( - self.probackup_path, backup_dir, instance) - - elif os.name == 'nt': - options['archive_command'] = '"{0}" archive-push -B {1} --instance={2} '.format( - self.probackup_path.replace("\\","\\\\"), - backup_dir.replace("\\","\\\\"), instance) - - # don`t forget to kill old_binary after remote ssh release - if self.remote and not old_binary: - options['archive_command'] += '--remote-proto=ssh ' - options['archive_command'] += '--remote-host=localhost ' - - if self.archive_compress and compress: - options['archive_command'] += '--compress ' - - if overwrite: - options['archive_command'] += '--overwrite ' - - options['archive_command'] += '--log-level-console=VERBOSE ' - options['archive_command'] += '-j 5 ' - options['archive_command'] += '--batch-size 10 ' - options['archive_command'] += '--no-sync ' - - if archive_timeout: - options['archive_command'] += '--archive-timeout={0} '.format( - archive_timeout) - - if os.name == 'posix': - options['archive_command'] += '--wal-file-path=%p --wal-file-name=%f' - - elif os.name == 'nt': - options['archive_command'] += '--wal-file-path="%p" --wal-file-name="%f"' - - if log_level: - options['archive_command'] += ' --log-level-console={0}'.format(log_level) - options['archive_command'] += ' --log-level-file={0} '.format(log_level) - else: # custom_archive_command is not None - options['archive_command'] = custom_archive_command - - self.set_auto_conf(node, options) - - def get_restore_command(self, backup_dir, instance, node): + def get_restore_command(self, backup_dir, instance): # parse postgresql.auto.conf - restore_command = '' - if os.name == 'posix': - restore_command += '{0} archive-get -B {1} --instance={2} '.format( - self.probackup_path, backup_dir, instance) - - elif os.name == 'nt': - restore_command += '"{0}" archive-get -B {1} --instance={2} '.format( - self.probackup_path.replace("\\","\\\\"), - backup_dir.replace("\\","\\\\"), instance) + restore_command = " ".join([f'"{self.probackup_path}"', + 'archive-get', *backup_dir.pb_args]) + if os.name == 'nt': + restore_command.replace("\\", "\\\\") + restore_command += f' --instance={instance}' # don`t forget to kill old_binary after remote ssh release if self.remote: - restore_command += '--remote-proto=ssh ' - restore_command += '--remote-host=localhost ' + restore_command += ' --remote-proto=ssh' + restore_command += ' --remote-host=localhost' if os.name == 'posix': - restore_command += '--wal-file-path=%p --wal-file-name=%f' + restore_command += ' --wal-file-path=%p --wal-file-name=%f' elif os.name == 'nt': - restore_command += '--wal-file-path="%p" --wal-file-name="%f"' + restore_command += ' --wal-file-path="%p" --wal-file-name="%f"' return restore_command - # rm_options - list of parameter name that should be deleted from current config, - # example: ['wal_keep_segments', 'max_wal_size'] - def set_auto_conf(self, node, options, config='postgresql.auto.conf', rm_options={}): - - # parse postgresql.auto.conf - path = os.path.join(node.data_dir, config) - - with open(path, 'r') as f: - raw_content = f.read() - - current_options = {} - current_directives = [] - for line in raw_content.splitlines(): - - # ignore comments - if line.startswith('#'): - continue - - if line == '': - continue - - if line.startswith('include'): - current_directives.append(line) - continue - - name, var = line.partition('=')[::2] - name = name.strip() - var = var.strip() - var = var.strip('"') - var = var.strip("'") - - # remove options specified in rm_options list - if name in rm_options: - continue - - current_options[name] = var - - for option in options: - current_options[option] = options[option] - - auto_conf = '' - for option in current_options: - auto_conf += "{0} = '{1}'\n".format( - option, current_options[option]) - - for directive in current_directives: - auto_conf += directive + "\n" - - with open(path, 'wt') as f: - f.write(auto_conf) - f.flush() - f.close() - def set_replica( self, master, replica, replica_name='replica', synchronous=False, log_shipping=False - ): + ): - self.set_auto_conf( - replica, + replica.set_auto_conf( options={ 'port': replica.port, 'hot_standby': 'on'}) - if self.get_version(replica) >= self.version_to_num('12.0'): + if self.pg_config_version >= self.version_to_num('12.0'): with open(os.path.join(replica.data_dir, "standby.signal"), 'w') as f: f.flush() f.close() @@ -1598,11 +845,10 @@ def set_replica( config = 'postgresql.auto.conf' if not log_shipping: - self.set_auto_conf( - replica, + replica.set_auto_conf( {'primary_conninfo': 'user={0} port={1} application_name={2} ' - ' sslmode=prefer sslcompression=1'.format( - self.user, master.port, replica_name)}, + ' sslmode=prefer sslcompression=1'.format( + self.username, master.port, replica_name)}, config) else: replica.append_conf('recovery.conf', 'standby_mode = on') @@ -1612,57 +858,61 @@ def set_replica( 'recovery.conf', "primary_conninfo = 'user={0} port={1} application_name={2}" " sslmode=prefer sslcompression=1'".format( - self.user, master.port, replica_name)) + self.username, master.port, replica_name)) if synchronous: - self.set_auto_conf( - master, + master.set_auto_conf( options={ 'synchronous_standby_names': replica_name, 'synchronous_commit': 'remote_apply'}) master.reload() - def change_backup_status(self, backup_dir, instance, backup_id, status): + class ControlFileContainer(object): + __slots__ = ('data',) + + def __init__(self, data): + self.data = data - control_file_path = os.path.join( - backup_dir, 'backups', instance, backup_id, 'backup.control') - - with open(control_file_path, 'r') as f: - actual_control = f.read() - - new_control_file = '' - for line in actual_control.splitlines(): - if line.startswith('status'): - line = 'status = {0}'.format(status) - new_control_file += line - new_control_file += '\n' - - with open(control_file_path, 'wt') as f: - f.write(new_control_file) - f.flush() - f.close() - - with open(control_file_path, 'r') as f: - actual_control = f.read() - - def wrong_wal_clean(self, node, wal_size): - wals_dir = os.path.join(self.backup_dir(node), 'wal') - wals = [ - f for f in os.listdir(wals_dir) if os.path.isfile( - os.path.join(wals_dir, f)) - ] - wals.sort() - file_path = os.path.join(wals_dir, wals[-1]) - if os.path.getsize(file_path) != wal_size: - os.remove(file_path) + @contextlib.contextmanager + def modify_backup_control(self, backup_dir, instance, backup_id): + path = os.path.join('backups', instance, backup_id, 'backup.control') + control_file = backup_dir.read_file(path) + cf = ProbackupTest.ControlFileContainer(control_file) + yield cf + if control_file != cf.data: + backup_dir.write_file(path, cf.data) + + def change_backup_status(self, backup_dir, instance, backup_id, status): + with self.modify_backup_control(backup_dir, instance, backup_id) as cf: + cf.data = re.sub(r'status = \w+', f'status = {status}', cf.data, 1) + + def get_locks(self, backup_dir : TestBackupDir, node : str): + path = "backups/" + node + "/locks" + return backup_dir.list_files(path) + + def read_lock(self, backup_dir : TestBackupDir, node : str, lock : str): + path = "backups/" + node + "/locks/" + lock + return backup_dir.read_file(path, text=False) + + def expire_locks(self, backup_dir : TestBackupDir, node : str, seconds=1): + path = "backups/" + node + "/locks" + now = time.time() + expired = base36enc(int(now) - seconds) + for lock in backup_dir.list_files(path): + base, ts, exclusive = lock.rsplit("_", 2) + lock_expired = "_".join([base, expired, exclusive]) + content = backup_dir.read_file(path+"/"+lock, text = False) + backup_dir.remove_file(path+"/"+lock) + backup_dir.write_file(path+"/"+lock_expired, content, text = False) def guc_wal_segment_size(self, node): var = node.execute( 'postgres', "select setting from pg_settings where name = 'wal_segment_size'" ) - return int(var[0][0]) * self.guc_wal_block_size(node) + print(int(var[0][0])) + return int(var[0][0]) def guc_wal_block_size(self, node): var = node.execute( @@ -1673,19 +923,15 @@ def guc_wal_block_size(self, node): def get_pgpro_edition(self, node): if node.execute( - 'postgres', - "select exists (select 1 from" - " pg_proc where proname = 'pgpro_edition')" + 'postgres', + "select exists (select 1 from" + " pg_proc where proname = 'pgpro_edition')" )[0][0]: var = node.execute('postgres', 'select pgpro_edition()') return str(var[0][0]) else: return False - def get_username(self): - """ Returns current user name """ - return getpass.getuser() - def version_to_num(self, version): if not version: return 0 @@ -1697,100 +943,93 @@ def version_to_num(self, version): num = num * 100 + int(re.sub(r"[^\d]", "", part)) return num - def switch_wal_segment(self, node): + def switch_wal_segment(self, node, sleep_seconds=1, and_tx=False): """ - Execute pg_switch_wal/xlog() in given node + Execute pg_switch_wal() in given node Args: node: an instance of PostgresNode or NodeConnection class """ if isinstance(node, testgres.PostgresNode): - if self.version_to_num( - node.safe_psql('postgres', 'show server_version').decode('utf-8') - ) >= self.version_to_num('10.0'): - node.safe_psql('postgres', 'select pg_switch_wal()') - else: - node.safe_psql('postgres', 'select pg_switch_xlog()') + with node.connect('postgres') as con: + if and_tx: + con.execute('select txid_current()') + lsn = con.execute('select pg_switch_wal()')[0][0] else: - if self.version_to_num( - node.execute('show server_version')[0][0] - ) >= self.version_to_num('10.0'): - node.execute('select pg_switch_wal()') - else: - node.execute('select pg_switch_xlog()') + lsn = node.execute('select pg_switch_wal()')[0][0] - sleep(1) + if sleep_seconds > 0: + time.sleep(sleep_seconds) + return lsn - def wait_until_replica_catch_with_master(self, master, replica): - - version = master.safe_psql( - 'postgres', - 'show server_version').decode('utf-8').rstrip() + @contextlib.contextmanager + def switch_wal_after(self, node, seconds, and_tx=True): + tm = threading.Timer(seconds, self.switch_wal_segment, [node, 0, and_tx]) + tm.start() + try: + yield + finally: + tm.cancel() + tm.join() - if self.version_to_num(version) >= self.version_to_num('10.0'): - master_function = 'pg_catalog.pg_current_wal_lsn()' - replica_function = 'pg_catalog.pg_last_wal_replay_lsn()' - else: - master_function = 'pg_catalog.pg_current_xlog_location()' - replica_function = 'pg_catalog.pg_last_xlog_replay_location()' + def wait_until_replica_catch_with_master(self, master, replica): + master_function = 'pg_catalog.pg_current_wal_insert_lsn()' lsn = master.safe_psql( 'postgres', 'SELECT {0}'.format(master_function)).decode('utf-8').rstrip() # Wait until replica catch up with master + self.wait_until_lsn_replayed(replica, lsn) + return lsn + + def wait_until_lsn_replayed(self, replica, lsn): + replica_function = 'pg_catalog.pg_last_wal_replay_lsn()' replica.poll_query_until( 'postgres', "SELECT '{0}'::pg_lsn <= {1}".format(lsn, replica_function)) - def get_version(self, node): - return self.version_to_num( - testgres.get_pg_config()['VERSION'].split(" ")[1]) - def get_ptrack_version(self, node): version = node.safe_psql( "postgres", "SELECT extversion " - "FROM pg_catalog.pg_extension WHERE extname = 'ptrack'").decode('utf-8').rstrip() + "FROM pg_catalog.pg_extension WHERE extname = 'ptrack'").decode('utf-8').rstrip() return self.version_to_num(version) def get_bin_path(self, binary): return testgres.get_bin_path(binary) - def del_test_dir(self, module_name, fname): + def del_test_dirs(self): """ Del testdir and optimistically try to del module dir""" - - shutil.rmtree( - os.path.join( - self.tmp_path, - module_name, - fname - ), - ignore_errors=True - ) + # Remove FS dependent part first + self.backup_dir.cleanup() + # Remove all the rest + if init_params.delete_logs: + shutil.rmtree(self.test_path, ignore_errors=True) def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): """ return dict with directory content. " " TAKE IT AFTER CHECKPOINT or BACKUP""" - dirs_to_ignore = [ + dirs_to_ignore = { 'pg_xlog', 'pg_wal', 'pg_log', 'pg_stat_tmp', 'pg_subtrans', 'pg_notify' - ] - files_to_ignore = [ + } + files_to_ignore = { 'postmaster.pid', 'postmaster.opts', 'pg_internal.init', 'postgresql.auto.conf', - 'backup_label', 'tablespace_map', 'recovery.conf', + 'backup_label', 'backup_label.old', + 'tablespace_map', 'recovery.conf', 'ptrack_control', 'ptrack_init', 'pg_control', 'probackup_recovery.conf', 'recovery.signal', 'standby.signal', 'ptrack.map', 'ptrack.map.mmap', - 'ptrack.map.tmp' - ] + 'ptrack.map.tmp', 'recovery.done' + } if exclude_dirs: - dirs_to_ignore = dirs_to_ignore + exclude_dirs -# suffixes_to_ignore = ( -# '_ptrack' -# ) + dirs_to_ignore |= set(exclude_dirs) + # suffixes_to_ignore = ( + # '_ptrack' + # ) directory_dict = {} directory_dict['pgdata'] = pgdata directory_dict['files'] = {} @@ -1799,10 +1038,10 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): dirs[:] = [d for d in dirs if d not in dirs_to_ignore] for file in files: if ( - file in files_to_ignore or - (ignore_ptrack and file.endswith('_ptrack')) + file in files_to_ignore or + (ignore_ptrack and file.endswith('_ptrack')) ): - continue + continue file_fullpath = os.path.join(root, file) file_relpath = os.path.relpath(file_fullpath, pgdata) @@ -1812,11 +1051,11 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): # truncate cfm's content's zero tail if file_relpath.endswith('.cfm'): content = f.read() - zero64 = b"\x00"*64 + zero64 = b"\x00" * 64 l = len(content) while l > 64: s = (l - 1) & ~63 - if content[s:l] != zero64[:l-s]: + if content[s:l] != zero64[:l - s]: break l = s content = content[:l] @@ -1824,17 +1063,17 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None): else: digest = hashlib.md5() while True: - b = f.read(64*1024) + b = f.read(64 * 1024) if not b: break digest.update(b) cfile.md5 = digest.hexdigest() # crappy algorithm if cfile.is_datafile: - size_in_pages = os.path.getsize(file_fullpath)/8192 + size_in_pages = os.path.getsize(file_fullpath) / 8192 cfile.md5_per_page = self.get_md5_per_page_for_fork( - file_fullpath, size_in_pages - ) + file_fullpath, size_in_pages + ) for directory in dirs: directory_path = os.path.join(root, directory) @@ -1866,14 +1105,13 @@ def get_known_bugs_comparision_exclusion_dict(self, node): "FROM pg_am, pg_class " "WHERE pg_am.amname = 'spgist' " "AND pg_class.relam = pg_am.oid" - ).decode('utf-8').rstrip().splitlines() + ).decode('utf-8').rstrip().splitlines() for filename in spgist_filelist: comparision_exclusion_dict[filename] = set([0]) return comparision_exclusion_dict - - def compare_pgdata(self, original_pgdata, restored_pgdata, exclusion_dict = dict()): + def compare_pgdata(self, original_pgdata, restored_pgdata, exclusion_dict=dict()): """ return dict with directory content. DO IT BEFORE RECOVERY exclusion_dict is used for exclude files (and it block_no) from comparision @@ -1969,7 +1207,6 @@ def compare_pgdata(self, original_pgdata, restored_pgdata, exclusion_dict = dict os.path.join(restored_pgdata['pgdata'], file) ) - for page in sorted(restored_pages - original_pages): error_message += '\n Extra page {0}\n File: {1}\n'.format( page, @@ -1992,264 +1229,188 @@ def compare_pgdata(self, original_pgdata, restored_pgdata, exclusion_dict = dict restored.md5_per_page[page], os.path.join( restored_pgdata['pgdata'], file) - ) + ) self.assertFalse(fail, error_message) - def gdb_attach(self, pid): - return GDBobj([str(pid)], self, attach=True) - - def _check_gdb_flag_or_skip_test(self): - if not self.gdb: - self.skipTest( - "Specify PGPROBACKUP_GDB and build without " - "optimizations for run this test" - ) - - -class GdbException(Exception): - def __init__(self, message="False"): - self.message = message - - def __str__(self): - return '\n ERROR: {0}\n'.format(repr(self.message)) - - -class GDBobj: - def __init__(self, cmd, env, attach=False): - self.verbose = env.verbose - self.output = '' - - # Check gdb flag is set up - if not env.gdb: - raise GdbException("No `PGPROBACKUP_GDB=on` is set, " - "test should call ProbackupTest::check_gdb_flag_or_skip_test() on its start " - "and be skipped") - # Check gdb presense - try: - gdb_version, _ = subprocess.Popen( - ['gdb', '--version'], - stdout=subprocess.PIPE - ).communicate() - except OSError: - raise GdbException("Couldn't find gdb on the path") - - self.base_cmd = [ - 'gdb', - '--interpreter', - 'mi2', - ] - - if attach: - self.cmd = self.base_cmd + ['--pid'] + cmd - else: - self.cmd = self.base_cmd + ['--args'] + cmd - - # Get version - gdb_version_number = re.search( - br"^GNU gdb [^\d]*(\d+)\.(\d)", - gdb_version) - self.major_version = int(gdb_version_number.group(1)) - self.minor_version = int(gdb_version_number.group(2)) - - if self.verbose: - print([' '.join(map(str, self.cmd))]) - - self.proc = subprocess.Popen( - self.cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - bufsize=0, - text=True, - errors='replace', - ) - self.gdb_pid = self.proc.pid - - while True: - line = self.get_line() - - if 'No such process' in line: - raise GdbException(line) - - if not line.startswith('(gdb)'): - pass - else: - break - - def get_line(self): - line = self.proc.stdout.readline() - self.output += line - return line - - def kill(self): - self.proc.kill() - self.proc.wait() - - def set_breakpoint(self, location): + def compare_instance_dir(self, original_instance, after_backup_instance, exclusion_dict=dict()): + """ + exclusion_dict is used for exclude files (and it block_no) from comparision + it is a dict with relative filenames as keys and set of block numbers as values + """ + fail = False + error_message = 'Instance directory is not equal to original!\n' - result = self._execute('break ' + location) - for line in result: - if line.startswith('~"Breakpoint'): - return + # Compare directories + after_backup = set(after_backup_instance['dirs']) + original_dirs = set(original_instance['dirs']) - elif line.startswith('=breakpoint-created'): - return + for directory in sorted(after_backup - original_dirs): + fail = True + error_message += '\nDirectory was not present' + error_message += ' in original instance: {0}\n'.format(directory) - elif line.startswith('^error'): #or line.startswith('(gdb)'): - break + for directory in sorted(original_dirs - after_backup): + fail = True + error_message += '\nDirectory dissappeared' + error_message += ' in instance after backup: {0}\n'.format(directory) - elif line.startswith('&"break'): - pass + for directory in sorted(original_dirs & after_backup): + original = original_instance['dirs'][directory] + after_backup = after_backup_instance['dirs'][directory] + if original.mode != after_backup.mode: + fail = True + error_message += '\nDir permissions mismatch:\n' + error_message += ' Dir old: {0} Permissions: {1}\n'.format(directory, + original.mode) + error_message += ' Dir new: {0} Permissions: {1}\n'.format(directory, + after_backup.mode) - elif line.startswith('&"Function'): - raise GdbException(line) + after_backup_files = set(after_backup_instance['files']) + original_files = set(original_instance['files']) - elif line.startswith('&"No line'): - raise GdbException(line) + for file in sorted(after_backup_files - original_files): + # File is present in instance after backup + # but not present in original instance + # only backup_label is allowed + fail = True + error_message += '\nFile is not present' + error_message += ' in original instance: {0}\n'.format(file) - elif line.startswith('~"Make breakpoint pending on future shared'): - raise GdbException(line) + for file in sorted(original_files - after_backup_files): + error_message += ( + '\nFile disappearance.\n ' + 'File: {0}\n').format(file) + fail = True - raise GdbException( - 'Failed to set breakpoint.\n Output:\n {0}'.format(result) - ) + for file in sorted(original_files & after_backup_files): + original = original_instance['files'][file] + after_backup = after_backup_instance['files'][file] + if after_backup.mode != original.mode: + fail = True + error_message += '\nFile permissions mismatch:\n' + error_message += ' File_old: {0} Permissions: {1:o}\n'.format(file, + original.mode) + error_message += ' File_new: {0} Permissions: {1:o}\n'.format(file, + after_backup.mode) - def remove_all_breakpoints(self): + if original.md5 != after_backup.md5: + if file not in exclusion_dict: + fail = True + error_message += ( + '\nFile Checksum mismatch.\n' + 'File_old: {0}\nChecksum_old: {1}\n' + 'File_new: {2}\nChecksum_new: {3}\n').format(file, + original.md5, file, after_backup.md5 + ) - result = self._execute('delete') - for line in result: + if not original.is_datafile: + continue - if line.startswith('^done'): - return + original_pages = set(original.md5_per_page) + after_backup_pages = set(after_backup.md5_per_page) - raise GdbException( - 'Failed to remove breakpoints.\n Output:\n {0}'.format(result) - ) + for page in sorted(original_pages - after_backup_pages): + error_message += '\n Page {0} dissappeared.\n File: {1}\n'.format( + page, file) - def run_until_break(self): - result = self._execute('run', False) - for line in result: - if line.startswith('*stopped,reason="breakpoint-hit"'): - return - raise GdbException( - 'Failed to run until breakpoint.\n' - ) - def continue_execution_until_running(self): - result = self._execute('continue') + for page in sorted(after_backup_pages - original_pages): + error_message += '\n Extra page {0}\n File: {1}\n'.format( + page, file) - for line in result: - if line.startswith('*running') or line.startswith('^running'): - return - if line.startswith('*stopped,reason="breakpoint-hit"'): - continue - if line.startswith('*stopped,reason="exited-normally"'): - continue + for page in sorted(original_pages & after_backup_pages): + if file in exclusion_dict and page in exclusion_dict[file]: + continue - raise GdbException( - 'Failed to continue execution until running.\n' - ) + if original.md5_per_page[page] != after_backup.md5_per_page[page]: + fail = True + error_message += ( + '\n Page checksum mismatch: {0}\n ' + ' PAGE Checksum_old: {1}\n ' + ' PAGE Checksum_new: {2}\n ' + ' File: {3}\n' + ).format( + page, + original.md5_per_page[page], + after_backup.md5_per_page[page], + file + ) - def continue_execution_until_exit(self): - result = self._execute('continue', False) + self.assertFalse(fail, error_message) - for line in result: - if line.startswith('*running'): - continue - if line.startswith('*stopped,reason="breakpoint-hit"'): - continue - if ( - line.startswith('*stopped,reason="exited') or - line == '*stopped\n' - ): - return - raise GdbException( - 'Failed to continue execution until exit.\n' - ) + def gdb_attach(self, pid): + return GDBobj([str(pid)], self, attach=True) - def continue_execution_until_error(self): - result = self._execute('continue', False) + def assertMessage(self, actual=None, *, contains=None, regex=None, has_no=None): + if actual is None: + actual = self.output + if self.output and self.output != actual: # Don't want to see this twice + error_message = '\n Unexpected Error Message: `{0}`\n CMD: `{1}`'.format(repr(self.output), + self.cmd) + else: + error_message = '\n Unexpected Error Message. CMD: `{0}`'.format(self.cmd) + if contains: + self.assertIn(contains, actual, error_message) + elif regex: + self.assertRegex(actual, regex, error_message) + elif has_no: + self.assertNotIn(has_no, actual, error_message) - for line in result: - if line.startswith('^error'): - return - if line.startswith('*stopped,reason="exited'): - return - if line.startswith( - '*stopped,reason="signal-received",signal-name="SIGABRT"'): - return +def get_relative_path(run_path, data_dir): + run_path_parts = run_path.split('/') + data_dir_parts = data_dir.split('/') - raise GdbException( - 'Failed to continue execution until error.\n') + # Find index of the first different element in the lists + diff_index = 0 + for i in range(min(len(run_path_parts), len(data_dir_parts))): + if run_path_parts[i] != data_dir_parts[i]: + diff_index = i + break - def continue_execution_until_break(self, ignore_count=0): - if ignore_count > 0: - result = self._execute( - 'continue ' + str(ignore_count), - False - ) - else: - result = self._execute('continue', False) + # Build relative path + relative_path = ['..'] * (len(run_path_parts) - diff_index) + data_dir_parts[diff_index:] - for line in result: - if line.startswith('*stopped,reason="breakpoint-hit"'): - return - if line.startswith('*stopped,reason="exited-normally"'): - break + return '/'.join(relative_path) - raise GdbException( - 'Failed to continue execution until break.\n') - - def stopped_in_breakpoint(self): - while True: - line = self.get_line() - if self.verbose: - print(line) - if line.startswith('*stopped,reason="breakpoint-hit"'): - return True - return False - - def quit(self): - self.proc.terminate() - - # use for breakpoint, run, continue - def _execute(self, cmd, running=True): - output = [] - self.proc.stdin.flush() - self.proc.stdin.write(cmd + '\n') - self.proc.stdin.flush() - sleep(1) - - # look for command we just send - while True: - line = self.get_line() - if self.verbose: - print(repr(line)) - - if cmd not in line: - continue - else: - break - while True: - line = self.get_line() - output += [line] - if self.verbose: - print(repr(line)) - if line.startswith('^done') or line.startswith('*stopped'): - break - if line.startswith('^error'): - break - if running and (line.startswith('*running') or line.startswith('^running')): -# if running and line.startswith('*running'): - break - return output class ContentFile(object): __slots__ = ('is_datafile', 'mode', 'md5', 'md5_per_page') + def __init__(self, is_datafile: bool): self.is_datafile = is_datafile + class ContentDir(object): - __slots__ = ('mode') \ No newline at end of file + __slots__ = ('mode') + +def _lz4_decompress(data): + with lz4.frame.open(io.BytesIO(data), 'rb') as fl: + return fl.read() + +def _lz4_compress(data): + out = io.BytesIO() + with lz4.frame.open(out, 'wb', content_checksum=True) as fl: + fl.write(data) + return out.getvalue() + +def _do_compress(file, data): + if file.endswith('.gz'): + return gzip.compress(data, compresslevel=1) + elif file.endswith('.lz4'): + return _lz4_compress(data) + elif file.endswith('.zst'): + return zstd.compress(data, 1, 1) + else: + return data + +def _do_decompress(file, data): + if file.endswith('.gz'): + return gzip.decompress(data) + elif file.endswith('.lz4'): + return _lz4_decompress(data) + elif file.endswith('.zst'): + return zstd.decompress(data) + else: + return data diff --git a/tests/helpers/state_helper.py b/tests/helpers/state_helper.py new file mode 100644 index 000000000..12552931a --- /dev/null +++ b/tests/helpers/state_helper.py @@ -0,0 +1,25 @@ +import re +from os import path + + +def get_program_version() -> str: + """ + Get pg_probackup version from source file /src/pg_probackup.h + value of PROGRAM_VERSION + The alternative for file /tests/expected/option_version.out + """ + probackup_h_path = '../../src/pg_probackup.h' + probackup_h_full_path = path.join(path.dirname(__file__), probackup_h_path) + define_sub = "#define PROGRAM_VERSION" + try: + with open(probackup_h_full_path, 'r') as probackup_h: + for line in probackup_h: + clean_line = re.sub(' +', ' ', line) # Line without doubled spaces + if define_sub in clean_line: + version = re.findall(r'"([^""]+)"', clean_line)[0] # Get the value between two quotes + return str(version) + raise Exception(f"Couldn't find the line with `{define_sub}` in file `{probackup_h_full_path}` " + f"that contains version between 2 quotes") + except FileNotFoundError: + raise FileNotFoundError( + f"Couldn't get version, check that file `{probackup_h_full_path}` exists and `PROGRAM_VERSION` defined") \ No newline at end of file diff --git a/tests/helpers/validators/show_validator.py b/tests/helpers/validators/show_validator.py new file mode 100644 index 000000000..d7df177a8 --- /dev/null +++ b/tests/helpers/validators/show_validator.py @@ -0,0 +1,141 @@ +import json +from datetime import datetime +from unittest import TestCase + +from ..enums.date_time_enum import DateTimePattern + + +class ShowJsonResultValidator(TestCase): + """ + This class contains all fields from show command result in json format. + It used for more convenient way to set up and validate output results. + + If we want to check the field we should set up it using the appropriate set method + For ex: + my_validator = ShowJsonResultValidator().set_backup_mode("PAGE")\ + .set_status("OK") + + After that we can compare json result from self.pb.show command with this class using `check_show_json` method. + + For informative error output, the validator class is inherited from TestClass. It allows us to use assertEqual + and do not worry about the readability of the error result. + """ + + def __init__(self): + super().__init__() + self.backup_id = None + self.parent_backup_id = None + self.backup_mode = None + self.wal = None + self.compress_alg = None + self.compress_level = None + self.from_replica = None + self.block_size = None + self.xlog_block_size = None + self.checksum_version = None + self.program_version = None + self.server_version = None + self.current_tli = None + self.parent_tli = None + self.start_lsn = None + self.stop_lsn = None + self.start_time = None + self.end_time = None + self.end_validation_time = None + self.recovery_xid = None + self.recovery_time = None + self.data_bytes = None + self.wal_bytes = None + self.uncompressed_bytes = None + self.pgdata_bytes = None + self.primary_conninfo = None + self.status = None + self.content_crc = None + + def check_show_json(self, show_result: json): + # Check equality if the value was set + if self.backup_id: + self.assertEqual(show_result["id"], self.backup_id) + if self.parent_backup_id: + self.assertEqual(show_result["parent-backup-id"], self.parent_backup_id) + if self.backup_mode: + self.assertEqual(show_result["backup-mode"], self.backup_mode) + if self.wal: + self.assertEqual(show_result["wal"], self.wal) + if self.compress_alg: + self.assertEqual(show_result["compress-alg"], self.compress_alg) + if self.compress_level: + self.assertEqual(show_result["compress-level"], self.compress_level) + if self.from_replica: + self.assertEqual(show_result["from-replica"], self.from_replica) + if self.block_size: + self.assertEqual(show_result["block-size"], self.block_size) + if self.xlog_block_size: + self.assertEqual(show_result["xlog-block-size"], self.xlog_block_size) + if self.checksum_version: + self.assertEqual(show_result["checksum-version"], self.checksum_version) + if self.program_version: + self.assertEqual(show_result["program-version"], self.program_version) + if self.server_version: + self.assertEqual(int(show_result["server-version"]), int(self.server_version)) + if self.current_tli: + self.assertEqual(show_result["current-tli"], self.current_tli) + if self.parent_tli: + self.assertEqual(show_result["parent-tli"], self.parent_tli) + if self.start_lsn: + self.assertEqual(show_result["start-lsn"], self.start_lsn) + if self.stop_lsn: + self.assertEqual(show_result["stop-lsn"], self.stop_lsn) + if self.start_time: + self.assertEqual(show_result["start-time"], self.start_time) + if self.end_time: + self.assertEqual(show_result["end-time"], self.end_time) + if self.end_validation_time: + self.assertEqual(show_result["end-validation-time"], self.end_validation_time) + if self.recovery_xid: + self.assertEqual(show_result["recovery-xid"], self.recovery_xid) + if self.recovery_time: + self.assertEqual(show_result["recovery-time"], self.recovery_time) + if self.data_bytes: + self.assertEqual(show_result["data-bytes"], self.data_bytes) + if self.wal_bytes: + self.assertEqual(show_result["wal-bytes"], self.wal_bytes) + if self.uncompressed_bytes: + self.assertEqual(show_result["uncompressed-bytes"], self.uncompressed_bytes) + if self.pgdata_bytes: + self.assertEqual(show_result["pgdata-bytes"], self.pgdata_bytes) + if self.primary_conninfo: + self.assertEqual(show_result["primary-conninfo"], self.primary_conninfo) + if self.status: + self.assertEqual(show_result["status"], self.status) + if self.content_crc: + self.assertEqual(show_result["content-crc"], self.content_crc) + + # Sanity checks + + start_time = self.str_time_to_datetime(show_result["start-time"]) + end_time = self.str_time_to_datetime(show_result["end-time"]) + end_validation_time = self.str_time_to_datetime(show_result["end-validation-time"]) + self.assertLessEqual(start_time, end_time) + self.assertLessEqual(end_time, end_validation_time) + + recovery_time = datetime.strptime(show_result["recovery-time"] + '00', DateTimePattern.Y_m_d_H_M_S_f_z_dash.value) + self.assertLessEqual(start_time, recovery_time) + + data_bytes = show_result["data-bytes"] + self.assertTrue(data_bytes > 0) + + wal_bytes = show_result["wal-bytes"] + self.assertTrue(wal_bytes > 0) + + pgdata_bytes = show_result["pgdata-bytes"] + self.assertTrue(pgdata_bytes > 0) + + @staticmethod + def str_time_to_datetime(time: str): + """ + Convert string time from pg_probackup to datetime format + String '00' was added because '%z' works with 4 digits values (like +0100), but from pg_probackup we get only + 2 digits timezone value (like +01). Because of that we should add additional '00' in the end + """ + return datetime.strptime(time + '00', str(DateTimePattern.Y_m_d_H_M_S_z_dash.value)) diff --git a/tests/incr_restore_test.py b/tests/incr_restore_test.py index f17ee95d1..08bc7b9ad 100644 --- a/tests/incr_restore_test.py +++ b/tests/incr_restore_test.py @@ -1,35 +1,29 @@ import os import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest +from pg_probackup2.gdb import needs_gdb import subprocess -from datetime import datetime -import sys -from time import sleep -from datetime import datetime, timedelta -import hashlib import shutil import json -from testgres import QueryException +from testgres import QueryException, StartNodeException -class IncrRestoreTest(ProbackupTest, unittest.TestCase): +class IncrRestoreTest(ProbackupTest): # @unittest.skip("skip") def test_basic_incr_restore(self): """incremental restore in CHECKSUM mode""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=50) - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -37,7 +31,7 @@ def test_basic_incr_restore(self): pgbench.wait() pgbench.stdout.close() - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -45,7 +39,7 @@ def test_basic_incr_restore(self): pgbench.wait() pgbench.stdout.close() - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -53,7 +47,7 @@ def test_basic_incr_restore(self): pgbench.wait() pgbench.stdout.close() - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id = self.pb.backup_node('node', node, backup_type='page') pgdata = self.pgdata_content(node.data_dir) @@ -65,8 +59,7 @@ def test_basic_incr_restore(self): node.stop() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=["-j", "4", "--incremental-mode=checksum"]) pgdata_restored = self.pgdata_content(node.data_dir) @@ -75,19 +68,17 @@ def test_basic_incr_restore(self): # @unittest.skip("skip") def test_basic_incr_restore_into_missing_directory(self): """""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=10) - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -95,7 +86,7 @@ def test_basic_incr_restore_into_missing_directory(self): pgbench.wait() pgbench.stdout.close() - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -103,14 +94,13 @@ def test_basic_incr_restore_into_missing_directory(self): pgbench.wait() pgbench.stdout.close() - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') pgdata = self.pgdata_content(node.data_dir) node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=["-j", "4", "--incremental-mode=checksum"]) pgdata_restored = self.pgdata_content(node.data_dir) @@ -120,19 +110,17 @@ def test_basic_incr_restore_into_missing_directory(self): def test_checksum_corruption_detection(self): """ """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=10) - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -140,7 +128,7 @@ def test_checksum_corruption_detection(self): pgbench.wait() pgbench.stdout.close() - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -148,7 +136,7 @@ def test_checksum_corruption_detection(self): pgbench.wait() pgbench.stdout.close() - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -156,14 +144,13 @@ def test_checksum_corruption_detection(self): pgbench.wait() pgbench.stdout.close() - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id = self.pb.backup_node('node', node, backup_type='page') pgdata = self.pgdata_content(node.data_dir) node.stop() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=["-j", "4", "--incremental-mode=lsn"]) pgdata_restored = self.pgdata_content(node.data_dir) @@ -173,34 +160,31 @@ def test_checksum_corruption_detection(self): def test_incr_restore_with_tablespace(self): """ """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) tblspace = self.get_tblspace_path(node, 'tblspace') some_directory = self.get_tblspace_path(node, 'some_directory') # stuff new destination with garbage - self.restore_node(backup_dir, 'node', node, data_dir=some_directory) + self.pb.restore_node('node', restore_dir=some_directory) self.create_tblspace_in_node(node, 'tblspace') node.pgbench_init(scale=10, tablespace='tblspace') - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) pgdata = self.pgdata_content(node.data_dir) node.stop() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ "-j", "4", "--incremental-mode=checksum", "--force", "-T{0}={1}".format(tblspace, some_directory)]) @@ -211,27 +195,25 @@ def test_incr_restore_with_tablespace(self): # @unittest.skip("skip") def test_incr_restore_with_tablespace_1(self): """recovery to target timeline""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) tblspace = self.get_tblspace_path(node, 'tblspace') some_directory = self.get_tblspace_path(node, 'some_directory') - self.restore_node(backup_dir, 'node', node, data_dir=some_directory) + self.pb.restore_node('node', restore_dir=some_directory) self.create_tblspace_in_node(node, 'tblspace') node.pgbench_init(scale=10, tablespace='tblspace') - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -239,8 +221,7 @@ def test_incr_restore_with_tablespace_1(self): pgbench.wait() pgbench.stdout.close() - self.backup_node( - backup_dir, 'node', node, backup_type='delta', options=['--stream']) + self.pb.backup_node('node', node, backup_type='delta', options=['--stream']) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -248,15 +229,13 @@ def test_incr_restore_with_tablespace_1(self): pgbench.wait() pgbench.stdout.close() - self.backup_node( - backup_dir, 'node', node, backup_type='delta', options=['--stream']) + self.pb.backup_node('node', node, backup_type='delta', options=['--stream']) pgdata = self.pgdata_content(node.data_dir) node.stop() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=["-j", "4", "--incremental-mode=checksum"]) pgdata_restored = self.pgdata_content(node.data_dir) @@ -268,25 +247,20 @@ def test_incr_restore_with_tablespace_2(self): If "--tablespace-mapping" option is used with incremental restore, then new directory must be empty. """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) - node_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_1')) + node_1 = self.pg_node.make_simple('node_1') # fill node1 with data - out = self.restore_node( - backup_dir, 'node', node, - data_dir=node_1.data_dir, + out = self.pb.restore_node('node', node_1, options=['--incremental-mode=checksum', '--force']) self.assertIn("WARNING: Backup catalog was initialized for system id", out) @@ -299,31 +273,17 @@ def test_incr_restore_with_tablespace_2(self): 'postgres', 'vacuum') - self.backup_node(backup_dir, 'node', node, backup_type='delta', options=['--stream']) + self.pb.backup_node('node', node, backup_type='delta', options=['--stream']) pgdata = self.pgdata_content(node.data_dir) - try: - self.restore_node( - backup_dir, 'node', node, - data_dir=node_1.data_dir, - options=['--incremental-mode=checksum', '-T{0}={1}'.format(tblspace, tblspace)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because remapped directory is not empty.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Remapped tablespace destination is not empty', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node_1, + options=['--incremental-mode=checksum', + '-T{0}={1}'.format(tblspace, tblspace)], + expect_error="because remapped directory is not empty") + self.assertMessage(contains='ERROR: Remapped tablespace destination is not empty') - out = self.restore_node( - backup_dir, 'node', node, - data_dir=node_1.data_dir, + out = self.pb.restore_node('node', node_1, options=[ '--force', '--incremental-mode=checksum', '-T{0}={1}'.format(tblspace, tblspace)]) @@ -335,21 +295,19 @@ def test_incr_restore_with_tablespace_2(self): def test_incr_restore_with_tablespace_3(self): """ """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() self.create_tblspace_in_node(node, 'tblspace1') node.pgbench_init(scale=10, tablespace='tblspace1') # take backup with tblspace1 - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) pgdata = self.pgdata_content(node.data_dir) self.drop_tblspace(node, 'tblspace1') @@ -359,8 +317,7 @@ def test_incr_restore_with_tablespace_3(self): node.stop() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ "-j", "4", "--incremental-mode=checksum"]) @@ -373,63 +330,40 @@ def test_incr_restore_with_tablespace_4(self): """ Check that system ID mismatch is detected, """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() self.create_tblspace_in_node(node, 'tblspace1') node.pgbench_init(scale=10, tablespace='tblspace1') # take backup of node1 with tblspace1 - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) pgdata = self.pgdata_content(node.data_dir) self.drop_tblspace(node, 'tblspace1') node.cleanup() # recreate node - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) node.slow_start() self.create_tblspace_in_node(node, 'tblspace1') node.pgbench_init(scale=10, tablespace='tblspace1') node.stop() - try: - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", - "--incremental-mode=checksum"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because destination directory has wrong system id.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup catalog was initialized for system id', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'ERROR: Incremental restore is not allowed', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node, + options=["-j", "4", "--incremental-mode=checksum"], + expect_error="because destination directory has wrong system id") + self.assertMessage(contains='WARNING: Backup catalog was initialized for system id') + self.assertMessage(contains='ERROR: Incremental restore is not allowed') - out = self.restore_node( - backup_dir, 'node', node, + out = self.pb.restore_node('node', node, options=[ "-j", "4", "--force", "--incremental-mode=checksum"]) @@ -446,30 +380,26 @@ def test_incr_restore_with_tablespace_5(self): with some old content, that belongs to an instance with different system id. """ - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), - set_replication=True, - initdb_params=['--data-checksums']) + node1 = self.pg_node.make_simple('node1', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node1) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node1) node1.slow_start() self.create_tblspace_in_node(node1, 'tblspace') node1.pgbench_init(scale=10, tablespace='tblspace') # take backup of node1 with tblspace - self.backup_node(backup_dir, 'node', node1, options=['--stream']) + self.pb.backup_node('node', node1, options=['--stream']) pgdata = self.pgdata_content(node1.data_dir) node1.stop() # recreate node - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2'), - set_replication=True, - initdb_params=['--data-checksums']) + node2 = self.pg_node.make_simple('node2', + set_replication=True) node2.slow_start() self.create_tblspace_in_node(node2, 'tblspace') @@ -479,8 +409,7 @@ def test_incr_restore_with_tablespace_5(self): tblspc1_path = self.get_tblspace_path(node1, 'tblspace') tblspc2_path = self.get_tblspace_path(node2, 'tblspace') - out = self.restore_node( - backup_dir, 'node', node1, + out = self.pb.restore_node('node', node1, options=[ "-j", "4", "--force", "--incremental-mode=checksum", @@ -499,47 +428,30 @@ def test_incr_restore_with_tablespace_6(self): """ Empty pgdata, not empty tablespace """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() self.create_tblspace_in_node(node, 'tblspace') node.pgbench_init(scale=10, tablespace='tblspace') # take backup of node with tblspace - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) pgdata = self.pgdata_content(node.data_dir) - node.cleanup() + node.stop() + shutil.rmtree(node.data_dir) - try: - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", - "--incremental-mode=checksum"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because there is running postmaster " - "process in destination directory.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: PGDATA is empty, but tablespace destination is not', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node, + options=["-j", "4", "--incremental-mode=checksum"], + expect_error="because there is running postmaster") + self.assertMessage(contains='ERROR: PGDATA is empty, but tablespace destination is not') - out = self.restore_node( - backup_dir, 'node', node, + out = self.pb.restore_node('node', node, options=[ "-j", "4", "--force", "--incremental-mode=checksum"]) @@ -557,46 +469,23 @@ def test_incr_restore_with_tablespace_7(self): Restore backup without tablespace into PGDATA with tablespace. """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # take backup of node with tblspace - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) pgdata = self.pgdata_content(node.data_dir) self.create_tblspace_in_node(node, 'tblspace') node.pgbench_init(scale=5, tablespace='tblspace') node.stop() -# try: -# self.restore_node( -# backup_dir, 'node', node, -# options=[ -# "-j", "4", -# "--incremental-mode=checksum"]) -# # we should die here because exception is what we expect to happen -# self.assertEqual( -# 1, 0, -# "Expecting Error because there is running postmaster " -# "process in destination directory.\n " -# "Output: {0} \n CMD: {1}".format( -# repr(self.output), self.cmd)) -# except ProbackupException as e: -# self.assertIn( -# 'ERROR: PGDATA is empty, but tablespace destination is not', -# e.message, -# '\n Unexpected Error Message: {0}\n CMD: {1}'.format( -# repr(e.message), self.cmd)) - - out = self.restore_node( - backup_dir, 'node', node, + out = self.pb.restore_node('node', node, options=[ "-j", "4", "--incremental-mode=checksum"]) @@ -606,65 +495,29 @@ def test_incr_restore_with_tablespace_7(self): # @unittest.skip("skip") def test_basic_incr_restore_sanity(self): """recovery to target timeline""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) - try: - self.restore_node( - backup_dir, 'node', node, - options=["-j", "4", "--incremental-mode=checksum"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because there is running postmaster " - "process in destination directory.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Postmaster with pid', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'ERROR: Incremental restore is not allowed', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node, + options=["-j", "4", "--incremental-mode=checksum"], + expect_error="because there is running postmaster") + self.assertMessage(contains='WARNING: Postmaster with pid') + self.assertMessage(contains='ERROR: Incremental restore is not allowed') - node_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_1')) + node_1 = self.pg_node.make_simple('node_1') - try: - self.restore_node( - backup_dir, 'node', node_1, data_dir=node_1.data_dir, - options=["-j", "4", "--incremental-mode=checksum"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because destination directory has wrong system id.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup catalog was initialized for system id', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'ERROR: Incremental restore is not allowed', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node_1, + options=["-j", "4", "--incremental-mode=checksum"], + expect_error="because destination directory has wrong system id") + self.assertMessage(contains='WARNING: Backup catalog was initialized for system id') + self.assertMessage(contains='ERROR: Incremental restore is not allowed') # @unittest.skip("skip") def test_incr_checksum_restore(self): @@ -674,24 +527,22 @@ def test_incr_checksum_restore(self): X - is instance, we want to return it to C state. """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', pg_options={'wal_log_hints': 'on'}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=50) - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() @@ -705,17 +556,15 @@ def test_incr_checksum_restore(self): pgbench.wait() node.stop(['-m', 'immediate', '-D', node.data_dir]) - node_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_1')) + node_1 = self.pg_node.make_simple('node_1') node_1.cleanup() - self.restore_node( - backup_dir, 'node', node_1, data_dir=node_1.data_dir, + self.pb.restore_node('node', node_1, options=[ '--recovery-target-action=promote', '--recovery-target-xid={0}'.format(xid)]) - self.set_auto_conf(node_1, {'port': node_1.port}) + node_1.set_auto_conf({'port': node_1.port}) node_1.slow_start() # /-- @@ -725,7 +574,7 @@ def test_incr_checksum_restore(self): # /--C # --A-----B----*----X - self.backup_node(backup_dir, 'node', node_1, + self.pb.backup_node('node', node_1, data_dir=node_1.data_dir, backup_type='page') # /--C------ @@ -733,23 +582,30 @@ def test_incr_checksum_restore(self): pgbench = node_1.pgbench(options=['-T', '50', '-c', '1']) pgbench.wait() + checksums = node_1.pgbench_table_checksums() + # /--C------D # --A-----B----*----X - self.backup_node(backup_dir, 'node', node_1, + self.pb.backup_node('node', node_1, data_dir=node_1.data_dir, backup_type='page') + node_1.stop() + pgdata = self.pgdata_content(node_1.data_dir) - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=["-j", "4", "--incremental-mode=checksum"]) pgdata_restored = self.pgdata_content(node.data_dir) - self.set_auto_conf(node, {'port': node.port}) + node.set_auto_conf({'port': node.port}) node.slow_start() - self.compare_pgdata(pgdata, pgdata_restored) + checksums_restored = node.pgbench_table_checksums() + + if checksums != checksums_restored: + self.compare_pgdata(pgdata, pgdata_restored) + self.assertEqual(checksums, checksums_restored) # @unittest.skip("skip") @@ -760,24 +616,22 @@ def test_incr_lsn_restore(self): X - is instance, we want to return it to C state. """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', pg_options={'wal_log_hints': 'on'}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=50) - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() @@ -791,17 +645,15 @@ def test_incr_lsn_restore(self): pgbench.wait() node.stop(['-m', 'immediate', '-D', node.data_dir]) - node_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_1')) + node_1 = self.pg_node.make_simple('node_1') node_1.cleanup() - self.restore_node( - backup_dir, 'node', node_1, data_dir=node_1.data_dir, + self.pb.restore_node('node', node_1, options=[ '--recovery-target-action=promote', '--recovery-target-xid={0}'.format(xid)]) - self.set_auto_conf(node_1, {'port': node_1.port}) + node_1.set_auto_conf({'port': node_1.port}) node_1.slow_start() # /-- @@ -811,7 +663,7 @@ def test_incr_lsn_restore(self): # /--C # --A-----B----*----X - self.backup_node(backup_dir, 'node', node_1, + self.pb.backup_node('node', node_1, data_dir=node_1.data_dir, backup_type='page') # /--C------ @@ -819,22 +671,29 @@ def test_incr_lsn_restore(self): pgbench = node_1.pgbench(options=['-T', '50', '-c', '1']) pgbench.wait() + checksums = node_1.pgbench_table_checksums() + # /--C------D # --A-----B----*----X - self.backup_node(backup_dir, 'node', node_1, + self.pb.backup_node('node', node_1, data_dir=node_1.data_dir, backup_type='page') + node_1.stop() + pgdata = self.pgdata_content(node_1.data_dir) - self.restore_node( - backup_dir, 'node', node, options=["-j", "4", "--incremental-mode=lsn"]) + self.pb.restore_node('node', node, options=["-j", "4", "--incremental-mode=lsn"]) pgdata_restored = self.pgdata_content(node.data_dir) - self.set_auto_conf(node, {'port': node.port}) + node.set_auto_conf({'port': node.port}) node.slow_start() - self.compare_pgdata(pgdata, pgdata_restored) + checksums_restored = node.pgbench_table_checksums() + + if checksums != checksums_restored: + self.compare_pgdata(pgdata, pgdata_restored) + self.assertEqual(checksums, checksums_restored) # @unittest.skip("skip") def test_incr_lsn_sanity(self): @@ -845,61 +704,45 @@ def test_incr_lsn_sanity(self): X - is instance, we want to return it to state B. fail is expected behaviour in case of lsn restore. """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', pg_options={'wal_log_hints': 'on'}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.pgbench_init(scale=10) - node_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_1')) + node_1 = self.pg_node.make_simple('node_1') node_1.cleanup() - self.restore_node( - backup_dir, 'node', node_1, data_dir=node_1.data_dir) + self.pb.restore_node('node', node=node_1) - self.set_auto_conf(node_1, {'port': node_1.port}) + node_1.set_auto_conf({'port': node_1.port}) node_1.slow_start() pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() - self.backup_node(backup_dir, 'node', node_1, + self.pb.backup_node('node', node_1, data_dir=node_1.data_dir, backup_type='full') pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() - page_id = self.backup_node(backup_dir, 'node', node_1, + page_id = self.pb.backup_node('node', node_1, data_dir=node_1.data_dir, backup_type='page') node.stop() - try: - self.restore_node( - backup_dir, 'node', node, data_dir=node.data_dir, - options=["-j", "4", "--incremental-mode=lsn"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because incremental restore in lsn mode is impossible\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Cannot perform incremental restore of " - "backup chain {0} in 'lsn' mode".format(page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node, + options=["-j", "4", "--incremental-mode=lsn"], + expect_error="because incremental restore in lsn mode is impossible") + self.assertMessage(contains="ERROR: Cannot perform incremental restore of " + "backup chain {0} in 'lsn' mode".format(page_id)) # @unittest.skip("skip") def test_incr_checksum_sanity(self): @@ -909,46 +752,41 @@ def test_incr_checksum_sanity(self): X - is instance, we want to return it to state B. """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.pgbench_init(scale=20) - node_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_1')) + node_1 = self.pg_node.make_simple('node_1') node_1.cleanup() - self.restore_node( - backup_dir, 'node', node_1, data_dir=node_1.data_dir) + self.pb.restore_node('node', node=node_1) - self.set_auto_conf(node_1, {'port': node_1.port}) + node_1.set_auto_conf({'port': node_1.port}) node_1.slow_start() pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() - self.backup_node(backup_dir, 'node', node_1, + self.pb.backup_node('node', node_1, data_dir=node_1.data_dir, backup_type='full') pgbench = node_1.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() - page_id = self.backup_node(backup_dir, 'node', node_1, + page_id = self.pb.backup_node('node', node_1, data_dir=node_1.data_dir, backup_type='page') pgdata = self.pgdata_content(node_1.data_dir) node.stop() - self.restore_node( - backup_dir, 'node', node, data_dir=node.data_dir, + self.pb.restore_node('node', node, options=["-j", "4", "--incremental-mode=checksum"]) pgdata_restored = self.pgdata_content(node.data_dir) @@ -960,24 +798,20 @@ def test_incr_checksum_corruption_detection(self): """ check that corrupted page got detected and replaced """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), -# initdb_params=['--data-checksums'], - pg_options={'wal_log_hints': 'on'}) + node = self.pg_node.make_simple('node', checksum=False, pg_options={'wal_log_hints': 'on'}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.pgbench_init(scale=20) pgbench = node.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() - self.backup_node(backup_dir, 'node', node, + self.pb.backup_node('node', node, data_dir=node.data_dir, backup_type='full') heap_path = node.safe_psql( @@ -987,7 +821,7 @@ def test_incr_checksum_corruption_detection(self): pgbench = node.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() - page_id = self.backup_node(backup_dir, 'node', node, + page_id = self.pb.backup_node('node', node, data_dir=node.data_dir, backup_type='page') pgdata = self.pgdata_content(node.data_dir) @@ -1001,8 +835,7 @@ def test_incr_checksum_corruption_detection(self): f.flush() f.close - self.restore_node( - backup_dir, 'node', node, data_dir=node.data_dir, + self.pb.restore_node('node', node, options=["-j", "4", "--incremental-mode=checksum"]) pgdata_restored = self.pgdata_content(node.data_dir) @@ -1014,24 +847,22 @@ def test_incr_lsn_corruption_detection(self): """ check that corrupted page got detected and replaced """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', pg_options={'wal_log_hints': 'on'}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.pgbench_init(scale=20) pgbench = node.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() - self.backup_node(backup_dir, 'node', node, + self.pb.backup_node('node', node, data_dir=node.data_dir, backup_type='full') heap_path = node.safe_psql( @@ -1041,7 +872,7 @@ def test_incr_lsn_corruption_detection(self): pgbench = node.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() - page_id = self.backup_node(backup_dir, 'node', node, + page_id = self.pb.backup_node('node', node, data_dir=node.data_dir, backup_type='page') pgdata = self.pgdata_content(node.data_dir) @@ -1055,8 +886,7 @@ def test_incr_lsn_corruption_detection(self): f.flush() f.close - self.restore_node( - backup_dir, 'node', node, data_dir=node.data_dir, + self.pb.restore_node('node', node, options=["-j", "4", "--incremental-mode=lsn"]) pgdata_restored = self.pgdata_content(node.data_dir) @@ -1067,15 +897,13 @@ def test_incr_lsn_corruption_detection(self): # @unittest.expectedFailure def test_incr_restore_multiple_external(self): """check that cmdline has priority over config""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() external_dir1 = self.get_tblspace_path(node, 'external_dir1') @@ -1083,35 +911,27 @@ def test_incr_restore_multiple_external(self): # FULL backup node.pgbench_init(scale=20) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type="full", options=["-j", "4"]) # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir1, options=["-j", "4"]) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir2, options=["-j", "4"]) - self.set_config( - backup_dir, 'node', + self.pb.set_config('node', options=['-E{0}{1}{2}'.format( external_dir1, self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2)]) # cmdline option MUST override options in config - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='full', options=["-j", "4"]) pgbench = node.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() # cmdline option MUST override options in config - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='page', options=["-j", "4"]) pgdata = self.pgdata_content( @@ -1122,8 +942,7 @@ def test_incr_restore_multiple_external(self): node.stop() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node=node, options=["-j", "4", '--incremental-mode=checksum']) pgdata_restored = self.pgdata_content( @@ -1134,15 +953,13 @@ def test_incr_restore_multiple_external(self): # @unittest.expectedFailure def test_incr_lsn_restore_multiple_external(self): """check that cmdline has priority over config""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() external_dir1 = self.get_tblspace_path(node, 'external_dir1') @@ -1150,35 +967,27 @@ def test_incr_lsn_restore_multiple_external(self): # FULL backup node.pgbench_init(scale=20) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type="full", options=["-j", "4"]) # fill external directories with data - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir1, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir1, options=["-j", "4"]) - self.restore_node( - backup_dir, 'node', node, - data_dir=external_dir2, options=["-j", "4"]) + self.pb.restore_node('node', restore_dir=external_dir2, options=["-j", "4"]) - self.set_config( - backup_dir, 'node', + self.pb.set_config('node', options=['-E{0}{1}{2}'.format( external_dir1, self.EXTERNAL_DIRECTORY_DELIMITER, external_dir2)]) # cmdline option MUST override options in config - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='full', options=["-j", "4"]) pgbench = node.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() # cmdline option MUST override options in config - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='page', options=["-j", "4"]) pgdata = self.pgdata_content( @@ -1189,8 +998,7 @@ def test_incr_lsn_restore_multiple_external(self): node.stop() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=["-j", "4", '--incremental-mode=lsn']) pgdata_restored = self.pgdata_content( @@ -1202,22 +1010,20 @@ def test_incr_lsn_restore_multiple_external(self): def test_incr_lsn_restore_backward(self): """ """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={'wal_log_hints': 'on', 'hot_standby': 'on'}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL backup node.pgbench_init(scale=2) - full_id = self.backup_node( - backup_dir, 'node', node, + full_checksums = node.pgbench_table_checksums() + full_id = self.pb.backup_node('node', node, backup_type="full", options=["-j", "4"]) full_pgdata = self.pgdata_content(node.data_dir) @@ -1225,8 +1031,8 @@ def test_incr_lsn_restore_backward(self): pgbench = node.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() - page_id = self.backup_node( - backup_dir, 'node', node, + page_checksums = node.pgbench_table_checksums() + page_id = self.pb.backup_node('node', node, backup_type='page', options=["-j", "4"]) page_pgdata = self.pgdata_content(node.data_dir) @@ -1234,8 +1040,8 @@ def test_incr_lsn_restore_backward(self): pgbench = node.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() - delta_id = self.backup_node( - backup_dir, 'node', node, + delta_checksums = node.pgbench_table_checksums() + delta_id = self.pb.backup_node('node', node, backup_type='delta', options=["-j", "4"]) delta_pgdata = self.pgdata_content(node.data_dir) @@ -1245,8 +1051,7 @@ def test_incr_lsn_restore_backward(self): node.stop() - self.restore_node( - backup_dir, 'node', node, backup_id=full_id, + self.pb.restore_node('node', node, backup_id=full_id, options=[ "-j", "4", '--incremental-mode=lsn', @@ -1254,44 +1059,38 @@ def test_incr_lsn_restore_backward(self): '--recovery-target-action=pause']) pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(full_pgdata, pgdata_restored) node.slow_start(replica=True) + checksums_restored = node.pgbench_table_checksums() node.stop() - try: - self.restore_node( - backup_dir, 'node', node, backup_id=page_id, - options=[ - "-j", "4", '--incremental-mode=lsn', - '--recovery-target=immediate', '--recovery-target-action=pause']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because incremental restore in lsn mode is impossible\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "Cannot perform incremental restore of backup chain", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + if full_checksums != checksums_restored: + self.compare_pgdata(full_pgdata, pgdata_restored) + self.assertEqual(full_checksums, checksums_restored) + + self.pb.restore_node('node', node=node, backup_id=page_id, + options=["-j", "4", '--incremental-mode=lsn', + '--recovery-target=immediate', + '--recovery-target-action=pause'], + expect_error="because incremental restore in lsn mode is impossible") + self.assertMessage(contains="Cannot perform incremental restore of backup chain") - self.restore_node( - backup_dir, 'node', node, backup_id=page_id, + self.pb.restore_node('node', node, backup_id=page_id, options=[ "-j", "4", '--incremental-mode=checksum', '--recovery-target=immediate', '--recovery-target-action=pause']) pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(page_pgdata, pgdata_restored) node.slow_start(replica=True) + checksums_restored = node.pgbench_table_checksums() node.stop() - self.restore_node( - backup_dir, 'node', node, backup_id=delta_id, + if page_checksums != checksums_restored: + self.compare_pgdata(page_pgdata, pgdata_restored) + self.assertEqual(page_checksums, checksums_restored) + + self.pb.restore_node('node', node, backup_id=delta_id, options=[ "-j", "4", '--incremental-mode=lsn', @@ -1299,30 +1098,35 @@ def test_incr_lsn_restore_backward(self): '--recovery-target-action=pause']) pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(delta_pgdata, pgdata_restored) + + node.slow_start(replica=True) + checksums_restored = node.pgbench_table_checksums() + node.stop() + + if delta_checksums != checksums_restored: + self.compare_pgdata(delta_pgdata, pgdata_restored) + self.assertEqual(delta_checksums, checksums_restored) # @unittest.skip("skip") # @unittest.expectedFailure def test_incr_checksum_restore_backward(self): """ """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'hot_standby': 'on'}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL backup node.pgbench_init(scale=20) - full_id = self.backup_node( - backup_dir, 'node', node, + full_checksums = node.pgbench_table_checksums() + full_id = self.pb.backup_node('node', node, backup_type="full", options=["-j", "4"]) full_pgdata = self.pgdata_content(node.data_dir) @@ -1330,8 +1134,8 @@ def test_incr_checksum_restore_backward(self): pgbench = node.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() - page_id = self.backup_node( - backup_dir, 'node', node, + page_checksums = node.pgbench_table_checksums() + page_id = self.pb.backup_node('node', node, backup_type='page', options=["-j", "4"]) page_pgdata = self.pgdata_content(node.data_dir) @@ -1339,8 +1143,8 @@ def test_incr_checksum_restore_backward(self): pgbench = node.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() - delta_id = self.backup_node( - backup_dir, 'node', node, + delta_checksums = node.pgbench_table_checksums() + delta_id = self.pb.backup_node('node', node, backup_type='delta', options=["-j", "4"]) delta_pgdata = self.pgdata_content(node.data_dir) @@ -1350,8 +1154,7 @@ def test_incr_checksum_restore_backward(self): node.stop() - self.restore_node( - backup_dir, 'node', node, backup_id=full_id, + self.pb.restore_node('node', node, backup_id=full_id, options=[ "-j", "4", '--incremental-mode=checksum', @@ -1359,13 +1162,16 @@ def test_incr_checksum_restore_backward(self): '--recovery-target-action=pause']) pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(full_pgdata, pgdata_restored) node.slow_start(replica=True) + checksums_restored = node.pgbench_table_checksums() node.stop() - self.restore_node( - backup_dir, 'node', node, backup_id=page_id, + if full_checksums != checksums_restored: + self.compare_pgdata(full_pgdata, pgdata_restored) + self.assertEqual(full_checksums, checksums_restored) + + self.pb.restore_node('node', node, backup_id=page_id, options=[ "-j", "4", '--incremental-mode=checksum', @@ -1373,13 +1179,16 @@ def test_incr_checksum_restore_backward(self): '--recovery-target-action=pause']) pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(page_pgdata, pgdata_restored) node.slow_start(replica=True) + checksums_restored = node.pgbench_table_checksums() node.stop() - self.restore_node( - backup_dir, 'node', node, backup_id=delta_id, + if page_checksums != checksums_restored: + self.compare_pgdata(page_pgdata, pgdata_restored) + self.assertEqual(page_checksums, checksums_restored) + + self.pb.restore_node('node', node, backup_id=delta_id, options=[ "-j", "4", '--incremental-mode=checksum', @@ -1387,37 +1196,36 @@ def test_incr_checksum_restore_backward(self): '--recovery-target-action=pause']) pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(delta_pgdata, pgdata_restored) + + node.slow_start(replica=True) + checksums_restored = node.pgbench_table_checksums() + node.stop() + + if delta_checksums != checksums_restored: + self.compare_pgdata(delta_pgdata, pgdata_restored) + self.assertEqual(delta_checksums, checksums_restored) # @unittest.skip("skip") def test_make_replica_via_incr_checksum_restore(self): """ """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums']) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') + backup_dir = self.backup_dir + master = self.pg_node.make_simple('master', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', master) - self.set_archiving(backup_dir, 'node', master, replica=True) + self.pb.init() + self.pb.add_instance('node', master) + self.pb.set_archiving('node', master, replica=True) master.slow_start() - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() master.pgbench_init(scale=20) - self.backup_node(backup_dir, 'node', master) + self.pb.backup_node('node', master) - self.restore_node( - backup_dir, 'node', replica, options=['-R']) + self.pb.restore_node('node', replica, options=['-R']) # Settings for Replica self.set_replica(master, replica, synchronous=False) @@ -1441,13 +1249,11 @@ def test_make_replica_via_incr_checksum_restore(self): pgbench.wait() # take backup from new master - self.backup_node( - backup_dir, 'node', new_master, + self.pb.backup_node('node', new_master, data_dir=new_master.data_dir, backup_type='page') # restore old master as replica - self.restore_node( - backup_dir, 'node', old_master, data_dir=old_master.data_dir, + self.pb.restore_node('node', old_master, options=['-R', '--incremental-mode=checksum']) self.set_replica(new_master, old_master, synchronous=True) @@ -1461,31 +1267,23 @@ def test_make_replica_via_incr_checksum_restore(self): def test_make_replica_via_incr_lsn_restore(self): """ """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums']) - - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') + backup_dir = self.backup_dir + master = self.pg_node.make_simple('master', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', master) - self.set_archiving(backup_dir, 'node', master, replica=True) + self.pb.init() + self.pb.add_instance('node', master) + self.pb.set_archiving('node', master, replica=True) master.slow_start() - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() master.pgbench_init(scale=20) - self.backup_node(backup_dir, 'node', master) + self.pb.backup_node('node', master) - self.restore_node( - backup_dir, 'node', replica, options=['-R']) + self.pb.restore_node('node', replica, options=['-R']) # Settings for Replica self.set_replica(master, replica, synchronous=False) @@ -1509,13 +1307,11 @@ def test_make_replica_via_incr_lsn_restore(self): pgbench.wait() # take backup from new master - self.backup_node( - backup_dir, 'node', new_master, + self.pb.backup_node('node', new_master, data_dir=new_master.data_dir, backup_type='page') # restore old master as replica - self.restore_node( - backup_dir, 'node', old_master, data_dir=old_master.data_dir, + self.pb.restore_node('node', old_master, options=['-R', '--incremental-mode=lsn']) self.set_replica(new_master, old_master, synchronous=True) @@ -1530,14 +1326,13 @@ def test_make_replica_via_incr_lsn_restore(self): def test_incr_checksum_long_xact(self): """ """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -1558,12 +1353,10 @@ def test_incr_checksum_long_xact(self): con.execute("INSERT INTO t1 values (2)") con2.execute("INSERT INTO t1 values (3)") - full_id = self.backup_node( - backup_dir, 'node', node, + full_id = self.pb.backup_node('node', node, backup_type="full", options=["-j", "4", "--stream"]) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type="delta", options=["-j", "4", "--stream"]) con.commit() @@ -1579,8 +1372,7 @@ def test_incr_checksum_long_xact(self): node.stop() - self.restore_node( - backup_dir, 'node', node, backup_id=full_id, + self.pb.restore_node('node', node, backup_id=full_id, options=["-j", "4", '--incremental-mode=checksum']) node.slow_start() @@ -1593,20 +1385,21 @@ def test_incr_checksum_long_xact(self): # @unittest.skip("skip") # @unittest.expectedFailure - # This test will pass with Enterprise - # because it has checksums enabled by default - @unittest.skipIf(ProbackupTest.enterprise, 'skip') def test_incr_lsn_long_xact_1(self): """ """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + if self.pgpro and not self.shardman: + initdb_params = ['--no-data-checksums'] + else: + initdb_params = [] + node = self.pg_node.make_simple('node', + set_replication=True, + initdb_params=initdb_params, + checksum=False) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -1627,12 +1420,10 @@ def test_incr_lsn_long_xact_1(self): con.execute("INSERT INTO t1 values (2)") con2.execute("INSERT INTO t1 values (3)") - full_id = self.backup_node( - backup_dir, 'node', node, + full_id = self.pb.backup_node('node', node, backup_type="full", options=["-j", "4", "--stream"]) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type="delta", options=["-j", "4", "--stream"]) con.commit() @@ -1649,41 +1440,28 @@ def test_incr_lsn_long_xact_1(self): node.stop() - try: - self.restore_node( - backup_dir, 'node', node, backup_id=full_id, - options=["-j", "4", '--incremental-mode=lsn']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because incremental restore in lsn mode is impossible\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Incremental restore in 'lsn' mode require data_checksums to be " - "enabled in destination data directory", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node, backup_id=full_id, + options=["-j", "4", '--incremental-mode=lsn'], + expect_error="because incremental restore in lsn mode is impossible") + self.assertMessage(contains="ERROR: Incremental restore in 'lsn' mode " + "require data_checksums to be " + "enabled in destination data directory") # @unittest.skip("skip") # @unittest.expectedFailure def test_incr_lsn_long_xact_2(self): """ """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'full_page_writes': 'off', 'wal_log_hints': 'off'}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -1704,12 +1482,10 @@ def test_incr_lsn_long_xact_2(self): con.execute("INSERT INTO t1 values (2)") con2.execute("INSERT INTO t1 values (3)") - full_id = self.backup_node( - backup_dir, 'node', node, + full_id = self.pb.backup_node('node', node, backup_type="full", options=["-j", "4", "--stream"]) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type="delta", options=["-j", "4", "--stream"]) # print(node.safe_psql( @@ -1738,8 +1514,7 @@ def test_incr_lsn_long_xact_2(self): node.stop() - self.restore_node( - backup_dir, 'node', node, backup_id=full_id, + self.pb.restore_node('node', node, backup_id=full_id, options=["-j", "4", '--incremental-mode=lsn']) node.slow_start() @@ -1755,14 +1530,12 @@ def test_incr_lsn_long_xact_2(self): def test_incr_restore_zero_size_file_checksum(self): """ """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() fullpath = os.path.join(node.data_dir, 'simple_file') @@ -1771,8 +1544,7 @@ def test_incr_restore_zero_size_file_checksum(self): f.close # FULL backup - id1 = self.backup_node( - backup_dir, 'node', node, + id1 = self.pb.backup_node('node', node, options=["-j", "4", "--stream"]) pgdata1 = self.pgdata_content(node.data_dir) @@ -1783,37 +1555,32 @@ def test_incr_restore_zero_size_file_checksum(self): f.flush() f.close - id2 = self.backup_node( - backup_dir, 'node', node, + id2 = self.pb.backup_node('node', node, backup_type="delta", options=["-j", "4", "--stream"]) pgdata2 = self.pgdata_content(node.data_dir) with open(fullpath, "w") as f: f.close() - id3 = self.backup_node( - backup_dir, 'node', node, + id3 = self.pb.backup_node('node', node, backup_type="delta", options=["-j", "4", "--stream"]) pgdata3 = self.pgdata_content(node.data_dir) node.stop() - self.restore_node( - backup_dir, 'node', node, backup_id=id1, + self.pb.restore_node('node', node, backup_id=id1, options=["-j", "4", '-I', 'checksum']) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata1, pgdata_restored) - self.restore_node( - backup_dir, 'node', node, backup_id=id2, + self.pb.restore_node('node', node, backup_id=id2, options=["-j", "4", '-I', 'checksum']) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata2, pgdata_restored) - self.restore_node( - backup_dir, 'node', node, backup_id=id3, + self.pb.restore_node('node', node, backup_id=id3, options=["-j", "4", '-I', 'checksum']) pgdata_restored = self.pgdata_content(node.data_dir) @@ -1824,14 +1591,12 @@ def test_incr_restore_zero_size_file_checksum(self): def test_incr_restore_zero_size_file_lsn(self): """ """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() fullpath = os.path.join(node.data_dir, 'simple_file') @@ -1840,8 +1605,7 @@ def test_incr_restore_zero_size_file_lsn(self): f.close # FULL backup - id1 = self.backup_node( - backup_dir, 'node', node, + id1 = self.pb.backup_node('node', node, options=["-j", "4", "--stream"]) pgdata1 = self.pgdata_content(node.data_dir) @@ -1852,23 +1616,20 @@ def test_incr_restore_zero_size_file_lsn(self): f.flush() f.close - id2 = self.backup_node( - backup_dir, 'node', node, + id2 = self.pb.backup_node('node', node, backup_type="delta", options=["-j", "4", "--stream"]) pgdata2 = self.pgdata_content(node.data_dir) with open(fullpath, "w") as f: f.close() - id3 = self.backup_node( - backup_dir, 'node', node, + id3 = self.pb.backup_node('node', node, backup_type="delta", options=["-j", "4", "--stream"]) pgdata3 = self.pgdata_content(node.data_dir) node.stop() - self.restore_node( - backup_dir, 'node', node, backup_id=id1, + self.pb.restore_node('node', node, backup_id=id1, options=["-j", "4", '-I', 'checksum']) pgdata_restored = self.pgdata_content(node.data_dir) @@ -1877,8 +1638,7 @@ def test_incr_restore_zero_size_file_lsn(self): node.slow_start() node.stop() - self.restore_node( - backup_dir, 'node', node, backup_id=id2, + self.pb.restore_node('node', node, backup_id=id2, options=["-j", "4", '-I', 'checksum']) pgdata_restored = self.pgdata_content(node.data_dir) @@ -1887,8 +1647,7 @@ def test_incr_restore_zero_size_file_lsn(self): node.slow_start() node.stop() - self.restore_node( - backup_dir, 'node', node, backup_id=id3, + self.pb.restore_node('node', node, backup_id=id3, options=["-j", "4", '-I', 'checksum']) pgdata_restored = self.pgdata_content(node.data_dir) @@ -1896,14 +1655,12 @@ def test_incr_restore_zero_size_file_lsn(self): def test_incremental_partial_restore_exclude_checksum(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() for i in range(1, 10, 1): @@ -1926,30 +1683,27 @@ def test_incremental_partial_restore_exclude_checksum(self): node.pgbench_init(scale=20) # FULL backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) pgdata = self.pgdata_content(node.data_dir) pgbench = node.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() # PAGE backup - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id = self.pb.backup_node('node', node, backup_type='page') # restore FULL backup into second node2 - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1')) + node1 = self.pg_node.make_simple('node1') node1.cleanup() - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2 = self.pg_node.make_simple('node2') node2.cleanup() # restore some data into node2 - self.restore_node(backup_dir, 'node', node2) + self.pb.restore_node('node', node2) # partial restore backup into node1 - self.restore_node( - backup_dir, 'node', + self.pb.restore_node('node', node1, options=[ "--db-exclude=db1", "--db-exclude=db5"]) @@ -1957,8 +1711,7 @@ def test_incremental_partial_restore_exclude_checksum(self): pgdata1 = self.pgdata_content(node1.data_dir) # partial incremental restore backup into node2 - self.restore_node( - backup_dir, 'node', + self.pb.restore_node('node', node2, options=[ "--db-exclude=db1", "--db-exclude=db5", @@ -1970,7 +1723,7 @@ def test_incremental_partial_restore_exclude_checksum(self): self.compare_pgdata(pgdata1, pgdata2) - self.set_auto_conf(node2, {'port': node2.port}) + node2.set_auto_conf({'port': node2.port}) node2.slow_start() @@ -1999,14 +1752,12 @@ def test_incremental_partial_restore_exclude_checksum(self): def test_incremental_partial_restore_exclude_lsn(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() for i in range(1, 10, 1): @@ -2029,32 +1780,29 @@ def test_incremental_partial_restore_exclude_lsn(self): node.pgbench_init(scale=20) # FULL backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) pgdata = self.pgdata_content(node.data_dir) pgbench = node.pgbench(options=['-T', '10', '-c', '1']) pgbench.wait() # PAGE backup - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id = self.pb.backup_node('node', node, backup_type='page') node.stop() # restore FULL backup into second node2 - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1')) + node1 = self.pg_node.make_simple('node1') node1.cleanup() - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2 = self.pg_node.make_simple('node2') node2.cleanup() # restore some data into node2 - self.restore_node(backup_dir, 'node', node2) + self.pb.restore_node('node', node2) # partial restore backup into node1 - self.restore_node( - backup_dir, 'node', + self.pb.restore_node('node', node1, options=[ "--db-exclude=db1", "--db-exclude=db5"]) @@ -2065,8 +1813,7 @@ def test_incremental_partial_restore_exclude_lsn(self): node2.port = node.port node2.slow_start() node2.stop() - self.restore_node( - backup_dir, 'node', + self.pb.restore_node('node', node2, options=[ "--db-exclude=db1", "--db-exclude=db5", @@ -2078,7 +1825,7 @@ def test_incremental_partial_restore_exclude_lsn(self): self.compare_pgdata(pgdata1, pgdata2) - self.set_auto_conf(node2, {'port': node2.port}) + node2.set_auto_conf({'port': node2.port}) node2.slow_start() @@ -2107,14 +1854,12 @@ def test_incremental_partial_restore_exclude_lsn(self): def test_incremental_partial_restore_exclude_tablespace_checksum(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # cat_version = node.get_control_data()["Catalog version number"] @@ -2151,30 +1896,26 @@ def test_incremental_partial_restore_exclude_tablespace_checksum(self): db_list[line['datname']] = line['oid'] # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) # node1 - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1')) + node1 = self.pg_node.make_simple('node1') node1.cleanup() node1_tablespace = self.get_tblspace_path(node1, 'somedata') # node2 - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2 = self.pg_node.make_simple('node2') node2.cleanup() node2_tablespace = self.get_tblspace_path(node2, 'somedata') # in node2 restore full backup - self.restore_node( - backup_dir, 'node', + self.pb.restore_node('node', node2, options=[ "-T", "{0}={1}".format( node_tablespace, node2_tablespace)]) # partial restore into node1 - self.restore_node( - backup_dir, 'node', + self.pb.restore_node('node', node1, options=[ "--db-exclude=db1", "--db-exclude=db5", @@ -2184,31 +1925,17 @@ def test_incremental_partial_restore_exclude_tablespace_checksum(self): pgdata1 = self.pgdata_content(node1.data_dir) # partial incremental restore into node2 - try: - self.restore_node( - backup_dir, 'node', - node2, options=[ - "-I", "checksum", - "--db-exclude=db1", - "--db-exclude=db5", - "-T", "{0}={1}".format( - node_tablespace, node2_tablespace), - "--destroy-all-other-dbs"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because remapped tablespace contain old data .\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Remapped tablespace destination is not empty:', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.restore_node( - backup_dir, 'node', + self.pb.restore_node('node', node2, + options=["-I", "checksum", + "--db-exclude=db1", + "--db-exclude=db5", + "-T", "{0}={1}".format( + node_tablespace, node2_tablespace), + "--destroy-all-other-dbs",], + expect_error="because remapped tablespace contain old data") + self.assertMessage(contains='ERROR: Remapped tablespace destination is not empty:') + + self.pb.restore_node('node', node2, options=[ "-I", "checksum", "--force", "--db-exclude=db1", @@ -2222,7 +1949,7 @@ def test_incremental_partial_restore_exclude_tablespace_checksum(self): self.compare_pgdata(pgdata1, pgdata2) - self.set_auto_conf(node2, {'port': node2.port}) + node2.set_auto_conf({'port': node2.port}) node2.slow_start() node2.safe_psql( @@ -2252,40 +1979,34 @@ def test_incremental_partial_restore_deny(self): """ Do now allow partial incremental restore into non-empty PGDATA becase we can't limit WAL replay to a single database. + PBCKP-604 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() for i in range(1, 3): node.safe_psql('postgres', f'CREATE database db{i}') # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) pgdata = self.pgdata_content(node.data_dir) - try: - self.restore_node(backup_dir, 'node', node, options=["--db-include=db1", '-I', 'LSN']) - self.fail("incremental partial restore is not allowed") - except ProbackupException as e: - self.assertIn("Incremental restore is not allowed: Postmaster is running.", e.message) + self.pb.restore_node('node', node, options=["--db-include=db1", '-I', 'LSN'], + expect_error="because postmaster is running") + self.assertMessage(contains="Incremental restore is not allowed: Postmaster is running.") node.safe_psql('db2', 'create table x (id int)') node.safe_psql('db2', 'insert into x values (42)') node.stop() - try: - self.restore_node(backup_dir, 'node', node, options=["--db-include=db1", '-I', 'LSN']) - self.fail("because incremental partial restore is not allowed") - except ProbackupException as e: - self.assertIn("Incremental restore is not allowed: Partial incremental restore into non-empty PGDATA is forbidden", e.message) + self.pb.restore_node('node', node, options=["--db-include=db1", '-I', 'LSN'], + expect_error="because incremental partial restore is not allowed") + self.assertMessage(contains="Incremental restore is not allowed: Partial incremental restore into non-empty PGDATA is forbidden") node.slow_start() value = node.execute('db2', 'select * from x')[0][0] @@ -2296,15 +2017,13 @@ def test_deny_incremental_partial_restore_exclude_tablespace_checksum(self): Do now allow partial incremental restore into non-empty PGDATA becase we can't limit WAL replay to a single database. (case of tablespaces) + PBCKP-604 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -2335,67 +2054,58 @@ def test_deny_incremental_partial_restore_exclude_tablespace_checksum(self): db_list[line['datname']] = line['oid'] # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) # node2 - node2 = self.make_simple_node('node2') + node2 = self.pg_node.make_simple('node2') node2.cleanup() node2_tablespace = self.get_tblspace_path(node2, 'somedata') # in node2 restore full backup - self.restore_node( - backup_dir, 'node', + self.pb.restore_node( + 'node', node2, options=[ "-T", f"{node_tablespace}={node2_tablespace}"]) # partial incremental restore into node2 - try: - self.restore_node(backup_dir, 'node', node2, - options=["-I", "checksum", - "--db-exclude=db1", - "--db-exclude=db5", - "-T", f"{node_tablespace}={node2_tablespace}"]) - self.fail("remapped tablespace contain old data") - except ProbackupException as e: - pass - - try: - self.restore_node(backup_dir, 'node', node2, - options=[ - "-I", "checksum", "--force", - "--db-exclude=db1", "--db-exclude=db5", - "-T", f"{node_tablespace}={node2_tablespace}"]) - self.fail("incremental partial restore is not allowed") - except ProbackupException as e: - self.assertIn("Incremental restore is not allowed: Partial incremental restore into non-empty PGDATA is forbidden", e.message) + self.pb.restore_node('node', node2, + options=["-I", "checksum", + "--db-exclude=db1", + "--db-exclude=db5", + "-T", f"{node_tablespace}={node2_tablespace}"], + expect_error="because remapped tablespace contain old data") + + self.pb.restore_node('node', node2, + options=[ + "-I", "checksum", "--force", + "--db-exclude=db1", "--db-exclude=db5", + "-T", f"{node_tablespace}={node2_tablespace}"], + expect_error="because incremental partial restore is not allowed") + self.assertMessage(contains="Incremental restore is not allowed: Partial incremental restore into non-empty PGDATA is forbidden") def test_incremental_pg_filenode_map(self): """ https://github.com/postgrespro/pg_probackup/issues/320 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), - initdb_params=['--data-checksums']) + node1 = self.pg_node.make_simple('node1') node1.cleanup() node.pgbench_init(scale=5) # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) # in node1 restore full backup - self.restore_node(backup_dir, 'node', node1) - self.set_auto_conf(node1, {'port': node1.port}) + self.pb.restore_node('node', node1) + node1.set_auto_conf({'port': node1.port}) node1.slow_start() pgbench = node.pgbench( @@ -2411,14 +2121,14 @@ def test_incremental_pg_filenode_map(self): 'reindex index pg_type_oid_index') # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) node1.stop() # incremental restore into node1 - self.restore_node(backup_dir, 'node', node1, options=["-I", "checksum"]) + self.pb.restore_node('node', node1, options=["-I", "checksum"]) - self.set_auto_conf(node1, {'port': node1.port}) + node1.set_auto_conf({'port': node1.port}) node1.slow_start() node1.safe_psql( @@ -2426,3 +2136,194 @@ def test_incremental_pg_filenode_map(self): 'select 1') # check that MinRecPoint and BackupStartLsn are correctly used in case of --incrementa-lsn + + # @unittest.skip("skip") + @needs_gdb + def test_incr_restore_issue_313(self): + """ + Check that failed incremental restore can be restarted + """ + # fname = self.id().split('.')[3] + + node = self.pg_node.make_simple('node', + set_replication=True) + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + node.pgbench_init(scale = 50) + + full_backup_id = self.pb.backup_node('node', node, backup_type='full') + + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + last_backup_id = self.pb.backup_node('node', node, backup_type='delta') + + pgdata = self.pgdata_content(node.data_dir) + node.cleanup() + + self.pb.restore_node('node', node, backup_id=full_backup_id) + + count = 0 + filelist = self.get_backup_filelist(backup_dir, 'node', last_backup_id) + for file in filelist: + # count only nondata files + if int(filelist[file]['is_datafile']) == 0 and \ + filelist[file]['kind'] != 'dir' and \ + file != 'database_map': + count += 1 + + gdb = self.pb.restore_node('node', node, gdb=True, + backup_id=last_backup_id, options=['--progress', '--incremental-mode=checksum']) + gdb.verbose = False + gdb.set_breakpoint('restore_non_data_file') + gdb.run_until_break() + gdb.continue_execution_until_break(count - 1) + gdb.quit() + + bak_file = os.path.join(node.data_dir, 'global', 'pg_control.pbk.bak') + self.assertTrue( + os.path.exists(bak_file), + "pg_control bak File should not exist: {0}".format(bak_file)) + + try: + node.slow_start() + # we should die here because exception is what we expect to happen + self.assertEqual( + 1, 0, + "Expecting Error because backup is not fully restored") + except StartNodeException as e: + self.assertIn( + 'Cannot start node', + e.message, + '\n Unexpected Error Message: {0}\n CMD: {1}'.format( + repr(e.message), self.cmd)) + + with open(os.path.join(node.logs_dir, 'postgresql.log'), 'r') as f: + self.assertIn( + "postgres: could not find the database system", + f.read()) + self.pb.restore_node('node', node, + backup_id=last_backup_id, options=['--progress', '--incremental-mode=checksum']) + node.slow_start() + + self.compare_pgdata(pgdata, self.pgdata_content(node.data_dir)) + + # @unittest.skip("skip") + def test_skip_pages_at_non_zero_segment_checksum(self): + if self.remote: + self.skipTest("Skipped because this test doesn't work properly in remote mode yet") + + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + # create table of size > 1 GB, so it will have several segments + node.safe_psql( + 'postgres', + "create table t as select i as a, i*2 as b, i*3 as c, i*4 as d, i*5 as e " + "from generate_series(1,20600000) i; " + "CHECKPOINT ") + + filepath = node.safe_psql( + 'postgres', + "SELECT pg_relation_filepath('t')" + ).decode('utf-8').rstrip() + + # segment .1 must exist in order to proceed this test + self.assertTrue(os.path.exists(f'{os.path.join(node.data_dir, filepath)}.1')) + + # do full backup + self.pb.backup_node('node', node) + + node.safe_psql( + 'postgres', + "DELETE FROM t WHERE a < 101; " + "CHECKPOINT") + + # do incremental backup + self.pb.backup_node('node', node, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node.safe_psql( + 'postgres', + "DELETE FROM t WHERE a < 201; " + "CHECKPOINT") + + node.stop() + + self.pb.restore_node('node', node, + options=["-j", "4", "--incremental-mode=checksum", "--log-level-console=INFO"]) + + self.assertNotIn('WARNING: Corruption detected in file', self.output, + 'Incremental restore copied pages from .1 datafile segment that were not changed') + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # @unittest.skip("skip") + def test_skip_pages_at_non_zero_segment_lsn(self): + if self.remote: + self.skipTest("Skipped because this test doesn't work properly in remote mode yet") + + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + # create table of size > 1 GB, so it will have several segments + node.safe_psql( + 'postgres', + "create table t as select i as a, i*2 as b, i*3 as c, i*4 as d, i*5 as e " + "from generate_series(1,20600000) i; " + "CHECKPOINT ") + + filepath = node.safe_psql( + 'postgres', + "SELECT pg_relation_filepath('t')" + ).decode('utf-8').rstrip() + + # segment .1 must exist in order to proceed this test + self.assertTrue(os.path.exists(f'{os.path.join(node.data_dir, filepath)}.1')) + + # do full backup + self.pb.backup_node('node', node) + + node.safe_psql( + 'postgres', + "DELETE FROM t WHERE a < 101; " + "CHECKPOINT") + + # do incremental backup + self.pb.backup_node('node', node, backup_type='page') + + pgdata = self.pgdata_content(node.data_dir) + + node.safe_psql( + 'postgres', + "DELETE FROM t WHERE a < 201; " + "CHECKPOINT") + + node.stop() + + self.pb.restore_node('node', node, + options=["-j", "4", "--incremental-mode=lsn", "--log-level-console=INFO"]) + + self.assertNotIn('WARNING: Corruption detected in file', self.output, + 'Incremental restore copied pages from .1 datafile segment that were not changed') + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) diff --git a/tests/init_test.py b/tests/init_test.py index 4e000c78f..92e684335 100644 --- a/tests/init_test.py +++ b/tests/init_test.py @@ -1,139 +1,218 @@ import os -import unittest -from .helpers.ptrack_helpers import dir_files, ProbackupTest, ProbackupException import shutil +import stat +import unittest + +from .helpers.ptrack_helpers import dir_files, ProbackupTest, fs_backup_class + +DIR_PERMISSION = 0o700 if os.name != 'nt' else 0o777 +CATALOG_DIRS = ['backups', 'wal'] + +class InitTest(ProbackupTest): -class InitTest(ProbackupTest, unittest.TestCase): + def tearDown(self): + super().tearDown() + # Remove some additional backup dirs + if hasattr(self, 'no_access_dir'): + shutil.rmtree(self.no_access_dir, ignore_errors=True) # @unittest.skip("skip") # @unittest.expectedFailure - def test_success(self): + def test_basic_success(self): """Success normal init""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) - self.init_pb(backup_dir) - self.assertEqual( - dir_files(backup_dir), - ['backups', 'wal'] - ) - self.add_instance(backup_dir, 'node', node) - self.assertIn( - "INFO: Instance 'node' successfully deleted", - self.del_instance(backup_dir, 'node'), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(self.output), self.cmd)) + instance_name = 'node' + backup_dir = self.backup_dir + node = self.pg_node.make_simple(instance_name) + self.pb.init() + + if backup_dir.is_file_based: + self.assertEqual( + dir_files(backup_dir), + CATALOG_DIRS + ) + + for subdir in CATALOG_DIRS: + dirname = os.path.join(backup_dir, subdir) + self.assertEqual(DIR_PERMISSION, stat.S_IMODE(os.stat(dirname).st_mode)) + + self.pb.add_instance(instance_name, node) + self.assertMessage(self.pb.del_instance(instance_name), + contains=f"INFO: Instance '{instance_name}' successfully deleted") # Show non-existing instance - try: - self.show_pb(backup_dir, 'node') - self.assertEqual(1, 0, 'Expecting Error due to show of non-existing instance. Output: {0} \n CMD: {1}'.format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Instance 'node' does not exist in this backup catalog", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + error_result = self.pb.show(instance_name, as_text=True, expect_error=True) + self.assertMessage(error_result, + contains=f"ERROR: Instance '{instance_name}' does not exist in this backup catalog") # Delete non-existing instance - try: - self.del_instance(backup_dir, 'node1') - self.assertEqual(1, 0, 'Expecting Error due to delete of non-existing instance. Output: {0} \n CMD: {1}'.format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Instance 'node1' does not exist in this backup catalog", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + error_result = self.pb.del_instance('node1', expect_error=True) + self.assertMessage(error_result, + contains="ERROR: Instance 'node1' does not exist in this backup catalog") # Add instance without pgdata - try: - self.run_pb([ - "add-instance", - "--instance=node1", - "-B", backup_dir - ]) - self.assertEqual(1, 0, 'Expecting Error due to adding instance without pgdata. Output: {0} \n CMD: {1}'.format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: No postgres data directory specified.\n" - "Please specify it either using environment variable PGDATA or\ncommand line option --pgdata (-D)", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(e.message, self.cmd)) + error_result = self.pb.run([ + "add-instance", + "--instance=node1", + ], expect_error=True) + self.assertMessage(error_result, + contains="No postgres data directory specified.\n" + "Please specify it either using environment variable PGDATA or\n" + "command line option --pgdata (-D)") # @unittest.skip("skip") - def test_already_exist(self): + def test_basic_already_exist(self): """Failure with backup catalog already existed""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) - self.init_pb(backup_dir) - try: - self.show_pb(backup_dir, 'node') - self.assertEqual(1, 0, 'Expecting Error due to initialization in non-empty directory. Output: {0} \n CMD: {1}'.format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Instance 'node' does not exist in this backup catalog", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + instance_name = 'node' + self.pg_node.make_simple(instance_name, checksum=False) + self.pb.init() + error_result = self.pb.show(instance_name, expect_error=True) + self.assertMessage(error_result, contains=f"ERROR: Instance '{instance_name}' " + f"does not exist in this backup catalog") # @unittest.skip("skip") - def test_abs_path(self): + def test_basic_abs_path(self): """failure with backup catalog should be given as absolute path""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) - try: - self.run_pb(["init", "-B", os.path.relpath("%s/backup" % node.base_dir, self.dir_path)]) - self.assertEqual(1, 0, 'Expecting Error due to initialization with non-absolute path in --backup-path. Output: {0} \n CMD: {1}'.format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: -B, --backup-path must be an absolute path", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.pg_node.make_simple('node') + error_result = self.pb.run(["init", "-B", "../backups_fake"], expect_error=True, use_backup_dir=None) + self.assertMessage(error_result, regex="backup-path must be an absolute path") # @unittest.skip("skip") # @unittest.expectedFailure - def test_add_instance_idempotence(self): + def test_basic_add_instance_idempotence(self): """ https://github.com/postgrespro/pg_probackup/issues/219 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node')) - self.init_pb(backup_dir) + backup_dir = self.backup_dir + instance_name = 'node' + node = self.pg_node.make_simple(instance_name) + self.pb.init() + + self.pb.add_instance(instance_name, node) + self.remove_one_backup_instance(backup_dir, instance_name) + + self.write_instance_wal(backup_dir, instance_name, '0000', b'') + + error_message = self.pb.add_instance(instance_name, node, expect_error=True) + self.assertMessage(error_message, regex=fr"'{instance_name}'.*WAL.*already exists") + + error_message = self.pb.add_instance(instance_name, node, expect_error=True) + self.assertMessage(error_message, regex=fr"'{instance_name}'.*WAL.*already exists") - self.add_instance(backup_dir, 'node', node) - shutil.rmtree(os.path.join(backup_dir, 'backups', 'node')) + def test_init_backup_catalog_no_access(self): + """ Test pg_probackup init -B backup_dir to a dir with no read access. """ + if not self.backup_dir.is_file_based: + self.skipTest("permission test is not implemented on cloud storage") + backup_dir = self.build_backup_dir('noaccess/backup') + # Store no_access_dir for the teardown in an instance variable + self.no_access_dir = os.path.dirname(backup_dir) + os.makedirs(self.no_access_dir) + os.chmod(self.no_access_dir, stat.S_IREAD) - dir_backups = os.path.join(backup_dir, 'backups', 'node') - dir_wal = os.path.join(backup_dir, 'wal', 'node') + expected = f'ERROR: Cannot open backup catalog directory: Cannot open dir "{backup_dir}": Permission denied' + error_message = self.pb.init(use_backup_dir=backup_dir, expect_error=True) + self.assertMessage(error_message, contains=expected) + def test_init_backup_catalog_no_write(self): + """ Test pg_probackup init -B backup_dir to a dir with no write access. """ + if not self.backup_dir.is_file_based: + self.skipTest("permission test is not implemented on cloud storage") + backup_dir = self.build_backup_dir('noaccess/backup') + # Store no_access_dir for the teardown in an instance variable + self.no_access_dir = os.path.dirname(backup_dir) + os.makedirs(self.no_access_dir) + os.chmod(self.no_access_dir, stat.S_IREAD|stat.S_IEXEC) + + expected = 'ERROR: Can not create backup catalog root directory: Cannot make dir "{0}": Permission denied'.format(backup_dir) + error_message = self.pb.init(use_backup_dir=backup_dir, expect_error=True) + self.assertMessage(error_message, contains=expected) + + def test_init_backup_catalog_no_create(self): + """ Test pg_probackup init -B backup_dir to a dir when backup dir exists but not writeable. """ + if not self.backup_dir.is_file_based: + self.skipTest("permission test is not implemented on cloud storage") + os.makedirs(self.backup_dir) + os.chmod(self.backup_dir, stat.S_IREAD|stat.S_IEXEC) + + expected = 'ERROR: Can not create backup catalog data directory: Cannot make dir "{0}": Permission denied'.format(os.path.join(self.backup_dir, 'backups')) + error_message = self.pb.init(expect_error=True) + self.assertMessage(error_message, contains=expected) + + def test_init_backup_catalog_exists_not_empty(self): + """ Test pg_probackup init -B backup_dir which exists and not empty. """ + backup_dir = self.backup_dir + if backup_dir.is_file_based: + os.makedirs(backup_dir) + backup_dir.write_file('somefile.txt', 'hello') + + error_message = self.pb.init(expect_error=True) + self.assertMessage(error_message, contains=f"ERROR: Backup catalog '{backup_dir}' already exists and is not empty") + + def test_init_add_instance_with_special_symbols(self): + """ Test pg_probackup init -B backup_dir which exists and not empty. """ + backup_dir = self.backup_dir + instance_name = 'instance! -_.*(\')&$@=;:+,?\\{^}%`[\"]<>~#|' + node = self.pg_node.make_simple(instance_name) + self.pb.init() + + error_message = self.pb.add_instance(instance_name, node) + self.assertMessage(error_message, regex=fr"'INFO: Instance {instance_name}' successfully initialized") + + @unittest.skipUnless(fs_backup_class.is_file_based, "AccessPath check is always true on s3") + def test_basic_dry_run_init(self): + """Success init with dry-run option""" + instance_name = 'node' + backup_dir = self.backup_dir + node = self.pg_node.make_simple(instance_name) + self.pb.init(options=['--dry-run']) + #Assertions + self.assertFalse(os.path.exists(backup_dir), + "Directory should not exist: {0}".format(backup_dir)) + + # Check existing backup directory + self.pb.init() + self.pb.init(options=['--dry-run', '--skip-if-exists']) + + # Access check suite if disk mounted as read_only + dir_mode = os.stat(self.test_path).st_mode + os.chmod(self.test_path, 0o500) + + error_message = self.pb.init(options=['--dry-run'], expect_error ='because of changed permissions') try: - self.add_instance(backup_dir, 'node', node) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Instance 'node' WAL archive directory already exists: ", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + self.assertMessage(error_message, contains='ERROR: Check permissions') + finally: + # Cleanup + os.chmod(self.test_path, dir_mode) + + node.cleanup() + + @unittest.skipUnless(fs_backup_class.is_file_based, "AccessPath check is always true on s3") + def test_basic_dry_run_add_instance(self): + """ Check --dry-run option for add_instance command""" + instance_name = 'node' + backup_dir = self.backup_dir + node = self.pg_node.make_simple(instance_name) + self.pb.init() + self.pb.add_instance(instance_name, node, options=['--dry-run']) + # Assetions + self.assertFalse(os.listdir(os.path.join(backup_dir, "wal"))) + self.assertFalse(os.listdir(os.path.join(backup_dir, "backups"))) + # Check existing backup directory + self.pb.add_instance(instance_name, node) + self.pb.add_instance(instance_name, node, options=['--dry-run', '--skip-if-exists']) + + # Check access suite - if disk mounted as read_only + dir_path = os.path.join(self.test_path, 'backup') + dir_mode = os.stat(dir_path).st_mode + os.chmod(dir_path, 0o500) + + error_message = self.pb.add_instance(instance_name, node, options=['--dry-run'], + expect_error ='because of changed permissions') try: - self.add_instance(backup_dir, 'node', node) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Instance 'node' WAL archive directory already exists: ", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + self.assertMessage(error_message, contains='ERROR: Check permissions ') + finally: + # Cleanup + os.chmod(dir_path, dir_mode) + + node.cleanup() diff --git a/tests/locking_test.py b/tests/locking_test.py index 5367c2610..774ced840 100644 --- a/tests/locking_test.py +++ b/tests/locking_test.py @@ -1,35 +1,38 @@ +import re +import time import unittest import os -from time import sleep -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest, fs_backup_class +from pg_probackup2.gdb import needs_gdb -class LockingTest(ProbackupTest, unittest.TestCase): +class LockingTest(ProbackupTest): + + def setUp(self): + super().setUp() # @unittest.skip("skip") # @unittest.expectedFailure + @needs_gdb def test_locking_running_validate_1(self): """ make node, take full backup, stop it in the middle run validate, expect it to successfully executed, concurrent RUNNING backup with pid file and active process is legal """ - self._check_gdb_flag_or_skip_test() - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True) + gdb = self.pb.backup_node('node', node, gdb=True) gdb.set_breakpoint('backup_non_data_file') gdb.run_until_break() @@ -37,310 +40,253 @@ def test_locking_running_validate_1(self): gdb.continue_execution_until_break(20) self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + 'OK', self.pb.show('node')[0]['status']) self.assertEqual( - 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) + 'RUNNING', self.pb.show('node')[1]['status']) + + validate_output = self.pb.validate(options=['--log-level-console=LOG']) - validate_output = self.validate_pb( - backup_dir, options=['--log-level-console=LOG']) + backup_id = self.pb.show('node')[1]['id'] - backup_id = self.show_pb(backup_dir, 'node')[1]['id'] + self.assertIn( + "WARNING: Lock waiting timeout reached. Deleting lock file", + validate_output, + '\n Unexpected Validate Output 1: {0}\n'.format(repr( + validate_output))) self.assertIn( - "is using backup {0}, and is still running".format(backup_id), + "WARNING: Cannot lock backup {0} directory, skip validation".format(backup_id), validate_output, - '\n Unexpected Validate Output: {0}\n'.format(repr(validate_output))) + '\n Unexpected Validate Output 3: {0}\n'.format(repr( + validate_output))) self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + 'OK', self.pb.show('node')[0]['status']) self.assertEqual( - 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) + 'RUNNING', self.pb.show('node')[1]['status']) # Clean after yourself gdb.kill() + @needs_gdb def test_locking_running_validate_2(self): """ make node, take full backup, stop it in the middle, - kill process so no cleanup is done - pid file is in place, - run validate, expect it to not successfully executed, - RUNNING backup with pid file AND without active pid is legal, - but his status must be changed to ERROR and pid file is deleted + kill process so no cleanup is done, then change lock file timestamp + to expired time, run validate, expect it to not successfully + executed, RUNNING backup with expired lock file is legal, but his + status must be changed to ERROR """ - self._check_gdb_flag_or_skip_test() - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True) + gdb = self.pb.backup_node('node', node, gdb=True) gdb.set_breakpoint('backup_non_data_file') gdb.run_until_break() gdb.continue_execution_until_break(20) - gdb._execute('signal SIGKILL') + gdb.signal('SIGKILL') gdb.continue_execution_until_error() self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + 'OK', self.pb.show('node')[0]['status']) self.assertEqual( - 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) - - backup_id = self.show_pb(backup_dir, 'node')[1]['id'] - - try: - self.validate_pb(backup_dir) - self.assertEqual( - 1, 0, - "Expecting Error because RUNNING backup is no longer active.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "which used backup {0} no longer exists".format( - backup_id) in e.message and - "Backup {0} has status RUNNING, change it " - "to ERROR and skip validation".format( - backup_id) in e.message and - "WARNING: Some backups are not valid" in - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + 'RUNNING', self.pb.show('node')[1]['status']) + + backup_id = self.pb.show('node')[1]['id'] + + self.expire_locks(backup_dir, 'node') + + self.pb.validate(options=["--log-level-console=VERBOSE"], + expect_error="because RUNNING backup is no longer active") + self.assertMessage(regex=r"Lock \S* has expired") + self.assertMessage(contains=f"Backup {backup_id} has status RUNNING, change it " + "to ERROR and skip validation") + self.assertMessage(contains="WARNING: Some backups are not valid") self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + 'OK', self.pb.show('node')[0]['status']) self.assertEqual( - 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) + 'ERROR', self.pb.show('node')[1]['status']) # Clean after yourself gdb.kill() + @needs_gdb def test_locking_running_validate_2_specific_id(self): """ make node, take full backup, stop it in the middle, - kill process so no cleanup is done - pid file is in place, - run validate on this specific backup, - expect it to not successfully executed, - RUNNING backup with pid file AND without active pid is legal, - but his status must be changed to ERROR and pid file is deleted + kill process so no cleanup is done, then change lock file timestamp + to expired time, run validate on this specific backup, expect it to + not successfully executed, RUNNING backup with expired lock file is + legal, but his status must be changed to ERROR """ - self._check_gdb_flag_or_skip_test() - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True) + gdb = self.pb.backup_node('node', node, gdb=True) gdb.set_breakpoint('backup_non_data_file') gdb.run_until_break() gdb.continue_execution_until_break(20) - gdb._execute('signal SIGKILL') + gdb.signal('SIGKILL') gdb.continue_execution_until_error() self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + 'OK', self.pb.show('node')[0]['status']) self.assertEqual( - 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) - - backup_id = self.show_pb(backup_dir, 'node')[1]['id'] - - try: - self.validate_pb(backup_dir, 'node', backup_id) - self.assertEqual( - 1, 0, - "Expecting Error because RUNNING backup is no longer active.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "which used backup {0} no longer exists".format( - backup_id) in e.message and - "Backup {0} has status RUNNING, change it " - "to ERROR and skip validation".format( - backup_id) in e.message and - "ERROR: Backup {0} has status: ERROR".format(backup_id) in - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + 'RUNNING', self.pb.show('node')[1]['status']) + + backup_id = self.pb.show('node')[1]['id'] + + self.expire_locks(backup_dir, 'node') + + self.pb.validate('node', backup_id, + options=['--log-level-console=VERBOSE'], + expect_error="because RUNNING backup is no longer active") + self.assertMessage(regex=r"Lock \S* has expired") + self.assertMessage(contains=f"Backup {backup_id} has status RUNNING, change it " + "to ERROR and skip validation") + self.assertMessage(contains=f"ERROR: Backup {backup_id} has status: ERROR") self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + 'OK', self.pb.show('node')[0]['status']) self.assertEqual( - 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) - - try: - self.validate_pb(backup_dir, 'node', backup_id) - self.assertEqual( - 1, 0, - "Expecting Error because backup has status ERROR.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} has status: ERROR".format(backup_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.validate_pb(backup_dir) - self.assertEqual( - 1, 0, - "Expecting Error because backup has status ERROR.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "WARNING: Backup {0} has status ERROR. Skip validation".format( - backup_id) in e.message and - "WARNING: Some backups are not valid" in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + 'ERROR', self.pb.show('node')[1]['status']) + + self.pb.validate('node', backup_id, + expect_error="because backup has status ERROR") + self.assertMessage(contains=f"ERROR: Backup {backup_id} has status: ERROR") + + self.pb.validate(expect_error="because backup has status ERROR") + self.assertMessage(contains=f"WARNING: Backup {backup_id} has status ERROR. Skip validation") + self.assertMessage(contains="WARNING: Some backups are not valid") # Clean after yourself gdb.kill() + @needs_gdb def test_locking_running_3(self): """ - make node, take full backup, stop it in the middle, - terminate process, delete pid file, - run validate, expect it to not successfully executed, - RUNNING backup without pid file AND without active pid is legal, - his status must be changed to ERROR + make node, take full backup, stop it in the middle, kill process so + no cleanup is done, delete lock file to expired time, run validate, + expect it to not successfully executed, RUNNING backup with expired + lock file is legal, but his status must be changed to ERROR """ - self._check_gdb_flag_or_skip_test() - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True) + gdb = self.pb.backup_node('node', node, gdb=True) gdb.set_breakpoint('backup_non_data_file') gdb.run_until_break() gdb.continue_execution_until_break(20) - gdb._execute('signal SIGKILL') + gdb.signal('SIGKILL') gdb.continue_execution_until_error() self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + 'OK', self.pb.show('node')[0]['status']) self.assertEqual( - 'RUNNING', self.show_pb(backup_dir, 'node')[1]['status']) - - backup_id = self.show_pb(backup_dir, 'node')[1]['id'] - - os.remove( - os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid')) - - try: - self.validate_pb(backup_dir) - self.assertEqual( - 1, 0, - "Expecting Error because RUNNING backup is no longer active.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "Backup {0} has status RUNNING, change it " - "to ERROR and skip validation".format( - backup_id) in e.message and - "WARNING: Some backups are not valid" in - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + 'RUNNING', self.pb.show('node')[1]['status']) + + backup_id = self.pb.show('node')[1]['id'] + + # delete lock file + self.expire_locks(backup_dir, 'node') + + self.pb.validate( + expect_error="because RUNNING backup is no longer active") + self.assertMessage(contains=f"Backup {backup_id} has status RUNNING, change it " + "to ERROR and skip validation") + self.assertMessage(contains="WARNING: Some backups are not valid") self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + 'OK', self.pb.show('node')[0]['status']) self.assertEqual( - 'ERROR', self.show_pb(backup_dir, 'node')[1]['status']) + 'ERROR', self.pb.show('node')[1]['status']) # Clean after yourself gdb.kill() + @needs_gdb def test_locking_restore_locked(self): """ make node, take full backup, take two page backups, launch validate on PAGE1 and stop it in the middle, launch restore of PAGE2. - Expect restore to sucseed because read-only locks + Expect restore to succeed because read-only locks do not conflict """ - self._check_gdb_flag_or_skip_test() - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL - full_id = self.backup_node(backup_dir, 'node', node) + full_id = self.pb.backup_node('node', node) # PAGE1 - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id = self.pb.backup_node('node', node, backup_type='page') # PAGE2 - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') - gdb = self.validate_pb( - backup_dir, 'node', backup_id=backup_id, gdb=True) + gdb = self.pb.validate('node', backup_id=backup_id, gdb=True) gdb.set_breakpoint('pgBackupValidate') gdb.run_until_break() node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) # Clean after yourself gdb.kill() + @needs_gdb def test_concurrent_delete_and_restore(self): """ make node, take full backup, take page backup, @@ -349,26 +295,22 @@ def test_concurrent_delete_and_restore(self): Expect restore to fail because validation of intermediate backup is impossible """ - self._check_gdb_flag_or_skip_test() - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) # PAGE1 - restore_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + restore_id = self.pb.backup_node('node', node, backup_type='page') - gdb = self.delete_pb( - backup_dir, 'node', backup_id=backup_id, gdb=True) + gdb = self.pb.delete('node', backup_id=backup_id, gdb=True) # gdb.set_breakpoint('pgFileDelete') gdb.set_breakpoint('delete_backup_files') @@ -376,127 +318,100 @@ def test_concurrent_delete_and_restore(self): node.cleanup() - try: - self.restore_node( - backup_dir, 'node', node, options=['--no-validate']) - self.assertEqual( - 1, 0, - "Expecting Error because restore without whole chain validation " - "is prohibited unless --no-validate provided.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "Backup {0} is used without validation".format( - restore_id) in e.message and - 'is using backup {0}, and is still running'.format( - backup_id) in e.message and - 'ERROR: Cannot lock backup' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node, options=['--no-validate'], + expect_error="because restore without whole chain validation " + "is prohibited unless --no-validate provided.") + self.assertMessage(contains=f"Backup {restore_id} is used without validation") + self.assertMessage(contains='WARNING: Lock waiting timeout reached.') + self.assertMessage(contains=f'Cannot lock backup {backup_id} directory') + self.assertMessage(contains='ERROR: Cannot lock backup') # Clean after yourself gdb.kill() + @needs_gdb def test_locking_concurrent_validate_and_backup(self): """ make node, take full backup, launch validate and stop it in the middle, take page backup. Expect PAGE backup to be successfully executed """ - self._check_gdb_flag_or_skip_test() - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # PAGE2 - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id = self.pb.backup_node('node', node, backup_type='page') - gdb = self.validate_pb( - backup_dir, 'node', backup_id=backup_id, gdb=True) + gdb = self.pb.validate('node', backup_id=backup_id, gdb=True) gdb.set_breakpoint('pgBackupValidate') gdb.run_until_break() # This PAGE backup is expected to be successfull - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') # Clean after yourself gdb.kill() - def test_locking_concurren_restore_and_delete(self): + @needs_gdb + def test_locking_concurrent_restore_and_delete(self): """ make node, take full backup, launch restore and stop it in the middle, delete full backup. Expect it to fail. """ - self._check_gdb_flag_or_skip_test() - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL - full_id = self.backup_node(backup_dir, 'node', node) + full_id = self.pb.backup_node('node', node) node.cleanup() - gdb = self.restore_node(backup_dir, 'node', node, gdb=True) + gdb = self.pb.restore_node('node', node=node, gdb=True) gdb.set_breakpoint('create_data_directories') gdb.run_until_break() - try: - self.delete_pb(backup_dir, 'node', full_id) - self.assertEqual( - 1, 0, - "Expecting Error because backup is locked\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Cannot lock backup {0} directory".format(full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.delete('node', full_id, + expect_error="because backup is locked") + self.assertMessage(contains=f"ERROR: Cannot lock backup {full_id} directory") # Clean after yourself gdb.kill() + @unittest.skipIf(not fs_backup_class.is_file_based, "os.rename is not implemented in a cloud") def test_backup_directory_name(self): """ """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL - full_id_1 = self.backup_node(backup_dir, 'node', node) - page_id_1 = self.backup_node(backup_dir, 'node', node, backup_type='page') + full_id_1 = self.pb.backup_node('node', node) + page_id_1 = self.pb.backup_node('node', node, backup_type='page') - full_id_2 = self.backup_node(backup_dir, 'node', node) - page_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + full_id_2 = self.pb.backup_node('node', node) + page_id_2 = self.pb.backup_node('node', node, backup_type='page') node.cleanup() @@ -505,125 +420,489 @@ def test_backup_directory_name(self): os.rename(old_path, new_path) - # This PAGE backup is expected to be successfull - self.show_pb(backup_dir, 'node', full_id_1) - - self.validate_pb(backup_dir) - self.validate_pb(backup_dir, 'node') - self.validate_pb(backup_dir, 'node', full_id_1) + self.pb.show('node', full_id_1) - self.restore_node(backup_dir, 'node', node, backup_id=full_id_1) + self.pb.validate() + self.pb.validate('node') + self.pb.validate('node', full_id_1) - self.delete_pb(backup_dir, 'node', full_id_1) + self.pb.restore_node('node', node=node, backup_id=full_id_1) - old_path = os.path.join(backup_dir, 'backups', 'node', full_id_2) - new_path = os.path.join(backup_dir, 'backups', 'node', 'hello_kitty') + self.pb.delete('node', full_id_1) - self.set_backup( - backup_dir, 'node', full_id_2, options=['--note=hello']) + self.pb.set_backup('node', full_id_2, options=['--note=hello']) - self.merge_backup(backup_dir, 'node', page_id_2, options=["-j", "4"]) + self.pb.merge_backup('node', page_id_2, options=["-j", "4"]) self.assertNotIn( 'note', - self.show_pb(backup_dir, 'node', page_id_2)) + self.pb.show('node', page_id_2)) # Clean after yourself - def test_empty_lock_file(self): + + @needs_gdb + def test_locks_delete(self): """ - https://github.com/postgrespro/pg_probackup/issues/308 + Make sure that shared and exclusive locks are deleted + after end of pg_probackup operations """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Fill with data - node.pgbench_init(scale=100) + node.pgbench_init(scale=1) # FULL - backup_id = self.backup_node(backup_dir, 'node', node) + gdb = self.pb.backup_node('node', node, gdb=True) - lockfile = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid') - with open(lockfile, "w+") as f: - f.truncate() + gdb.set_breakpoint('do_backup_pg') + gdb.run_until_break() - out = self.validate_pb(backup_dir, 'node', backup_id) + backup_id = self.pb.show('node')[0]['id'] - self.assertIn( - "Waiting 30 seconds on empty exclusive lock for backup", out) - -# lockfile = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid') -# with open(lockfile, "w+") as f: -# f.truncate() -# -# p1 = self.validate_pb(backup_dir, 'node', backup_id, asynchronous=True, -# options=['--log-level-file=LOG', '--log-filename=validate.log']) -# sleep(3) -# p2 = self.delete_pb(backup_dir, 'node', backup_id, asynchronous=True, -# options=['--log-level-file=LOG', '--log-filename=delete.log']) -# -# p1.wait() -# p2.wait() - - def test_shared_lock(self): - """ - Make sure that shared lock leaves no files with pids - """ - self._check_gdb_flag_or_skip_test() + locks = self.get_locks(backup_dir, 'node') + + self.assertEqual(len(locks), 1, + f"There should be just 1 lock, got {locks}") + self.assertTrue(locks[0].startswith(backup_id+"_"), + f"Lock should be for backup {backup_id}, got {locks[0]}") + self.assertTrue(locks[0].endswith("_w"), + f"Lock should be exclusive got {locks[0]}") + + gdb.continue_execution_until_exit() + + locks = self.get_locks(backup_dir, 'node') + self.assertFalse(locks, f"Locks should not exist, got {locks}") + + gdb = self.pb.validate('node', backup_id, gdb=True) + + gdb.set_breakpoint('validate_one_page') + gdb.run_until_break() + + locks = self.get_locks(backup_dir, 'node') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + self.assertEqual(len(locks), 1, + f"There should be just 1 lock, got {locks}") + self.assertTrue(locks[0].startswith(backup_id+"_"), + f"Lock should be for backup {backup_id}, got {locks[0]}") + self.assertTrue(locks[0].endswith("_r"), + f"Lock should be shared got {locks[0]}") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + gdb.continue_execution_until_exit() + + locks = self.get_locks(backup_dir, 'node') + self.assertFalse(locks, f"Locks should not exist, got {locks}") + + + @needs_gdb + def test_concurrent_merge_1(self): + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Fill with data node.pgbench_init(scale=1) - # FULL - backup_id = self.backup_node(backup_dir, 'node', node) + full_id = self.pb.backup_node('node', node, backup_type="full") - lockfile_excl = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup.pid') - lockfile_shr = os.path.join(backup_dir, 'backups', 'node', backup_id, 'backup_ro.pid') + pgbench = node.pgbench(options=['-t', '2000', '-c', '2']) + pgbench.wait() - self.validate_pb(backup_dir, 'node', backup_id) + self.pb.backup_node('node', node, backup_type="page") - self.assertFalse( - os.path.exists(lockfile_excl), - "File should not exist: {0}".format(lockfile_excl)) + pgbench = node.pgbench(options=['-t', '2000', '-c', '2']) + pgbench.wait() - self.assertFalse( - os.path.exists(lockfile_shr), - "File should not exist: {0}".format(lockfile_shr)) - - gdb = self.validate_pb(backup_dir, 'node', backup_id, gdb=True) + prev_id = self.pb.backup_node('node', node, backup_type="page") - gdb.set_breakpoint('validate_one_page') + pgbench = node.pgbench(options=['-t', '2000', '-c', '2']) + pgbench.wait() + + last_id = self.pb.backup_node('node', node, backup_type="page") + + gdb = self.pb.merge_backup('node', prev_id, gdb=True) + gdb.set_breakpoint("merge_chain") gdb.run_until_break() - gdb.kill() - self.assertTrue( - os.path.exists(lockfile_shr), - "File should exist: {0}".format(lockfile_shr)) - - self.validate_pb(backup_dir, 'node', backup_id) + self.pb.merge_backup('node', last_id, + expect_error="because of concurrent merge") + self.assertMessage(contains=f"ERROR: Cannot lock backup {full_id}") + + @needs_gdb + def test_concurrent_merge_2(self): + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=1) + + full_id = self.pb.backup_node('node', node, backup_type="full") + + pgbench = node.pgbench(options=['-t', '2000', '-c', '2']) + pgbench.wait() + + self.pb.backup_node('node', node, backup_type="page") + + pgbench = node.pgbench(options=['-t', '2000', '-c', '2']) + pgbench.wait() + + prev_id = self.pb.backup_node('node', node, backup_type="page") + + pgbench = node.pgbench(options=['-t', '2000', '-c', '2']) + pgbench.wait() + + last_id = self.pb.backup_node('node', node, backup_type="page") + + gdb = self.pb.merge_backup('node', prev_id, gdb=True) + # pthread_create will be called after state changed to merging + gdb.set_breakpoint("merge_files") + gdb.run_until_break() + + print(self.pb.show('node', as_text=True, as_json=False)) + self.assertEqual( + 'MERGING', self.pb.show('node')[0]['status']) + self.assertEqual( + 'MERGING', self.pb.show('node')[-2]['status']) + + self.pb.merge_backup('node', last_id, + expect_error="because of concurrent merge") + self.assertMessage(contains=f"ERROR: Full backup {full_id} has unfinished merge") + + @needs_gdb + def test_concurrent_merge_and_backup_1(self): + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=1) + + full_id = self.pb.backup_node('node', node, backup_type="full") + + pgbench = node.pgbench(options=['-t', '2000', '-c', '2']) + pgbench.wait() + + self.pb.backup_node('node', node, backup_type="page") - self.assertFalse( - os.path.exists(lockfile_excl), - "File should not exist: {0}".format(lockfile_excl)) + pgbench = node.pgbench(options=['-t', '2000', '-c', '2']) + pgbench.wait() - self.assertFalse( - os.path.exists(lockfile_shr), - "File should not exist: {0}".format(lockfile_shr)) + prev_id = self.pb.backup_node('node', node, backup_type="page") + pgbench = node.pgbench(options=['-t', '2000', '-c', '2']) + pgbench.wait() + + self.pb.backup_node('node', node, backup_type="page") + + gdb = self.pb.merge_backup('node', prev_id, gdb=True) + gdb.set_breakpoint("merge_chain") + gdb.run_until_break() + + pgbench = node.pgbench(options=['-t', '2000', '-c', '2']) + pgbench.wait() + + self.pb.backup_node('node', node, backup_type="page") + + @needs_gdb + def test_concurrent_merge_and_backup_2(self): + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=1) + + full_id = self.pb.backup_node('node', node, backup_type="full") + + pgbench = node.pgbench(options=['-t', '2000', '-c', '2']) + pgbench.wait() + + self.pb.backup_node('node', node, backup_type="page") + + pgbench = node.pgbench(options=['-t', '2000', '-c', '2']) + pgbench.wait() + + prev_id = self.pb.backup_node('node', node, backup_type="page") + + pgbench = node.pgbench(options=['-t', '2000', '-c', '2']) + pgbench.wait() + + self.pb.backup_node('node', node, backup_type="page") + + gdb = self.pb.merge_backup('node', prev_id, gdb=True) + # pthread_create will be called after state changed to merging + gdb.set_breakpoint("merge_files") + gdb.run_until_break() + + pgbench = node.pgbench(options=['-t', '2000', '-c', '2']) + pgbench.wait() + + self.pb.backup_node('node', node, backup_type="page", + expect_error="because of concurrent merge") + self.assertMessage( + contains="WARNING: Valid full backup on current timeline 1 " + "is not found, trying to look up on previous timelines") + self.assertMessage( + contains="WARNING: Cannot find valid backup on previous timelines") + self.assertMessage( + contains="ERROR: Create new full backup before an incremental one") + + + @needs_gdb + def test_locks_race_condition(self): + """ + Make locks race condition happen and check that pg_probackup + detected it and retried taking new lock. + + Run full backup. Set breakpoint on create_lock_file function, + stop there. Then run 'pg_probackup delete' command on current full + backup, stop it after taking a lock and before deleting its lock + file. Then continue taking full backup -- it must encounter race + condition because lock file of delete operation appeared between + two checks of /locks directory for concurrent locks + (scan_locks_directory function). + """ + + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + # start full backup + gdb_backup = self.pb.backup_node('node', node, + options=['--log-level-console=LOG'], gdb=True) + + gdb_backup.set_breakpoint('create_lock_file') + gdb_backup.run_until_break() + + backup_id = self.pb.show('node')[0]['id'] + + gdb_delete = self.pb.delete('node', backup_id, + options=['--log-level-console=LOG'], gdb=True) + + gdb_delete.set_breakpoint('create_lock_file') + gdb_delete.run_until_break() + + # we scanned locks directory and found no concurrent locks + # so we proceed to the second scan for race condition check + gdb_backup.set_breakpoint('scan_locks_directory') + gdb_delete.set_breakpoint('scan_locks_directory') + + # we create lock files with no contents + gdb_backup.continue_execution_until_break() + gdb_delete.continue_execution_until_break() + + # check that both exclusive lock files were created with empty contents + locks_list = self.get_locks(backup_dir, 'node') + locks_list_race = locks_list + + self.assertEqual(len(locks_list), 2) + + self.assertFalse(self.read_lock(backup_dir, 'node', locks_list[0])) + self.assertFalse(self.read_lock(backup_dir, 'node', locks_list[1])) + + gdb_backup.set_breakpoint('pioRemove__do') + gdb_delete.set_breakpoint('pioRemove__do') + + # we wait for message about race condition and stop right before we + # delete concurrent lock files to make both processes encounter race + # condition + gdb_backup.continue_execution_until_break() + self.assertIn("Lock race condition detected, taking lock attempt 1 " + "failed", gdb_backup.output) + gdb_delete.continue_execution_until_break() + self.assertIn("Lock race condition detected, taking lock attempt 1 " + "failed", gdb_delete.output) + + # run until next breakpoint ('scan_locks_directory') so old lock + # files will be deleted + gdb_backup.continue_execution_until_break() + gdb_delete.continue_execution_until_break() + + locks_list = self.get_locks(backup_dir, 'node') + self.assertFalse(locks_list) + + # continue backup execution until 'unlock_backup' at-exit util + gdb_backup.remove_all_breakpoints() + gdb_backup.set_breakpoint('unlock_backup') + gdb_backup.continue_execution_until_break() + + locks_list = self.get_locks(backup_dir, 'node') + self.assertTrue(locks_list, + f"Expecting at least 1 lock, got no") + self.assertLessEqual(len(locks_list), 2, + f"Expecting 1 or 2 locks, got {locks_list}") + if len(locks_list) == 2: + id1 = "_".join(locks_list[0].split("_", 2)[:2]) + id2 = "_".join(locks_list[1].split("_", 2)[:2]) + self.assertEqual(id1, id2) + + lock_backup = locks_list[0] + self.assertIn(f"{lock_backup} was taken", + gdb_backup.output) + self.assertFalse(self.read_lock(backup_dir, 'node', lock_backup)) + self.assertNotIn(lock_backup, locks_list_race) + + # make sure that delete command keeps waiting for backup unlocking + gdb_delete.remove_all_breakpoints() + gdb_delete.set_breakpoint('wait_for_conflicting_locks') + gdb_delete.continue_execution_until_break() + + locks_list = self.get_locks(backup_dir, 'node') + self.assertEqual(len(locks_list), 2) + self.assertNotEqual(locks_list[0], locks_list[1]) + lock_delete = (set(locks_list) - {lock_backup}).pop() + + self.assertNotIn(lock_delete, locks_list_race) + self.assertTrue(self.read_lock(backup_dir, 'node', lock_delete)) + + gdb_delete.remove_all_breakpoints() + gdb_delete.set_breakpoint('sleep') + gdb_delete.continue_execution_until_break() + gdb_delete.remove_all_breakpoints() + + self.assertIn(f"Waiting to take lock for backup {backup_id}", + gdb_delete.output) + + # continue all commands + gdb_backup.continue_execution_until_exit() + gdb_delete.continue_execution_until_exit() + + # make sure that locks were deleted + locks_list = self.get_locks(backup_dir, 'node') + self.assertFalse(locks_list) + + + @needs_gdb + def test_expired_locks_delete(self): + """ + check that if locks (shared or exclusive) have timestamp older than + ( - LOCK_LIFETIME) they are deleted by running + pg_probackup process + """ + + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + gdb = self.pb.backup_node('node', node, gdb=True) + + gdb.set_breakpoint('backup_non_data_file') + gdb.run_until_break() + + gdb.continue_execution_until_break(20) + + gdb.signal('SIGKILL') + gdb.continue_execution_until_error() + + self.assertEqual( + 'RUNNING', self.pb.show('node')[0]['status']) + + self.expire_locks(backup_dir, 'node', seconds=3600) + + stale_locks_list = self.get_locks(backup_dir, 'node') + self.assertEqual(len(stale_locks_list), 1) + + backup_id = self.pb.show('node')[0]['id'] + + gdb = self.pb.validate('node', backup_id, gdb=True, + options=['--log-level-console=LOG']) + gdb.set_breakpoint('pgBackupValidate') + gdb.run_until_break() + + self.assertRegex(gdb.output, + r"Expired lock file \S* is older than 180 seconds, deleting") + + new_locks_list = self.get_locks(backup_dir, 'node') + self.assertEqual(len(new_locks_list), 1) + self.assertFalse(set(stale_locks_list) & set(new_locks_list)) + + gdb.continue_execution_until_exit() + + self.assertEqual( + 'ERROR', self.pb.show('node')[0]['status']) + + + @needs_gdb + def test_locks_renovate_time(self): + """ + check that daemon thread renovates locks (shared or exclusive) + timestamps when they are about to expire + """ + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + + node.slow_start() + + gdb = self.pb.backup_node('node', node, gdb=True, options=['-j', '1', + '--write-rate-limit=1', + '--log-level-file=LOG']) + + # we need to stop just main thread + gdb._execute("set pagination off") + gdb._execute("set non-stop on") + gdb.set_breakpoint('do_backup') + gdb.run_until_break() + gdb._execute("set val_LOCK_RENOVATE_TIME=2") + gdb.set_breakpoint('do_backup_pg') + gdb.continue_execution_until_break() + + self.assertEqual( + 'RUNNING', self.pb.show('node')[0]['status']) + + locks_1 = self.get_locks(backup_dir, 'node') + self.assertLessEqual(len(locks_1), 2) + lock_id = '_'.join(locks_1[0].split('_', 2)[:2]) + + for attempt in range(25): + time.sleep(4) + locks_2 = self.get_locks(backup_dir, 'node') + if set(locks_1) != set(locks_2) and len(locks_2) == 2: + new = (set(locks_2) - set(locks_1)).pop() + self.assertTrue(new.startswith(lock_id)) + break + else: + self.fail("locks didn't renovate in 100 seconds") + + + gdb.remove_all_breakpoints() + gdb.continue_execution_until_exit() + + self.assertEqual( + 'OK', self.pb.show('node')[0]['status']) \ No newline at end of file diff --git a/tests/logging_test.py b/tests/logging_test.py index c5cdfa344..85e646c1e 100644 --- a/tests/logging_test.py +++ b/tests/logging_test.py @@ -1,38 +1,33 @@ -import unittest import os -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest +from pg_probackup2.gdb import needs_gdb import datetime -class LogTest(ProbackupTest, unittest.TestCase): +class LogTest(ProbackupTest): # @unittest.skip("skip") # @unittest.expectedFailure # PGPRO-2154 + @needs_gdb def test_log_rotation(self): """ """ - self._check_gdb_flag_or_skip_test() - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - self.set_config( - backup_dir, 'node', + self.pb.set_config('node', options=['--log-rotation-age=1s', '--log-rotation-size=1MB']) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=['--stream', '--log-level-file=verbose']) - gdb = self.backup_node( - backup_dir, 'node', node, + gdb = self.pb.backup_node('node', node, options=['--stream', '--log-level-file=verbose'], gdb=True) gdb.set_breakpoint('open_logfile') @@ -40,22 +35,18 @@ def test_log_rotation(self): gdb.continue_execution_until_exit() def test_log_filename_strftime(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + node = self.pg_node.make_simple('node', + set_replication=True) + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - self.set_config( - backup_dir, 'node', + self.pb.set_config('node', options=['--log-rotation-age=1d']) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=[ '--stream', '--log-level-file=VERBOSE', @@ -63,37 +54,30 @@ def test_log_filename_strftime(self): day_of_week = datetime.datetime.today().strftime("%a") - path = os.path.join( - backup_dir, 'log', 'pg_probackup-{0}.log'.format(day_of_week)) + path = os.path.join(self.pb_log_path, 'pg_probackup-{0}.log'.format(day_of_week)) self.assertTrue(os.path.isfile(path)) def test_truncate_rotation_file(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + node = self.pg_node.make_simple('node', + set_replication=True) + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - self.set_config( - backup_dir, 'node', + self.pb.set_config('node', options=['--log-rotation-age=1d']) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=[ '--stream', '--log-level-file=VERBOSE']) - rotation_file_path = os.path.join( - backup_dir, 'log', 'pg_probackup.log.rotation') + rotation_file_path = os.path.join(self.pb_log_path, 'pg_probackup.log.rotation') - log_file_path = os.path.join( - backup_dir, 'log', 'pg_probackup.log') + log_file_path = os.path.join(self.pb_log_path, 'pg_probackup.log') log_file_size = os.stat(log_file_path).st_size @@ -105,8 +89,7 @@ def test_truncate_rotation_file(self): f.flush() f.close - output = self.backup_node( - backup_dir, 'node', node, + output = self.pb.backup_node('node', node, options=[ '--stream', '--log-level-file=LOG'], @@ -121,8 +104,7 @@ def test_truncate_rotation_file(self): 'WARNING: cannot read creation timestamp from rotation file', output) - output = self.backup_node( - backup_dir, 'node', node, + output = self.pb.backup_node('node', node, options=[ '--stream', '--log-level-file=LOG'], @@ -140,31 +122,25 @@ def test_truncate_rotation_file(self): self.assertTrue(os.path.isfile(rotation_file_path)) def test_unlink_rotation_file(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + node = self.pg_node.make_simple('node', + set_replication=True) + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - self.set_config( - backup_dir, 'node', + self.pb.set_config('node', options=['--log-rotation-age=1d']) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=[ '--stream', '--log-level-file=VERBOSE']) - rotation_file_path = os.path.join( - backup_dir, 'log', 'pg_probackup.log.rotation') + rotation_file_path = os.path.join(self.pb_log_path, 'pg_probackup.log.rotation') - log_file_path = os.path.join( - backup_dir, 'log', 'pg_probackup.log') + log_file_path = os.path.join(self.pb_log_path, 'pg_probackup.log') log_file_size = os.stat(log_file_path).st_size @@ -173,8 +149,7 @@ def test_unlink_rotation_file(self): # unlink .rotation file os.unlink(rotation_file_path) - output = self.backup_node( - backup_dir, 'node', node, + output = self.pb.backup_node('node', node, options=[ '--stream', '--log-level-file=LOG'], @@ -191,8 +166,7 @@ def test_unlink_rotation_file(self): self.assertTrue(os.path.isfile(rotation_file_path)) - output = self.backup_node( - backup_dir, 'node', node, + output = self.pb.backup_node('node', node, options=[ '--stream', '--log-level-file=VERBOSE'], @@ -208,31 +182,24 @@ def test_unlink_rotation_file(self): log_file_size) def test_garbage_in_rotation_file(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - self.set_config( - backup_dir, 'node', + self.pb.set_config('node', options=['--log-rotation-age=1d']) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=[ '--stream', '--log-level-file=VERBOSE']) - rotation_file_path = os.path.join( - backup_dir, 'log', 'pg_probackup.log.rotation') + rotation_file_path = os.path.join(self.pb_log_path, 'pg_probackup.log.rotation') - log_file_path = os.path.join( - backup_dir, 'log', 'pg_probackup.log') + log_file_path = os.path.join(self.pb_log_path, 'pg_probackup.log') log_file_size = os.stat(log_file_path).st_size @@ -241,8 +208,7 @@ def test_garbage_in_rotation_file(self): # mangle .rotation file with open(rotation_file_path, "w+b", 0) as f: f.write(b"blah") - output = self.backup_node( - backup_dir, 'node', node, + output = self.pb.backup_node('node', node, options=[ '--stream', '--log-level-file=LOG'], @@ -263,8 +229,7 @@ def test_garbage_in_rotation_file(self): self.assertTrue(os.path.isfile(rotation_file_path)) - output = self.backup_node( - backup_dir, 'node', node, + output = self.pb.backup_node('node', node, options=[ '--stream', '--log-level-file=LOG'], @@ -280,28 +245,24 @@ def test_garbage_in_rotation_file(self): log_file_size) def test_issue_274(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.backup_node(backup_dir, 'node', node, options=['--stream']) - self.restore_node(backup_dir, 'node', replica) + self.pb.backup_node('node', node, options=['--stream']) + self.pb.restore_node('node', node=replica) # Settings for Replica self.set_replica(node, replica, synchronous=True) - self.set_archiving(backup_dir, 'node', replica, replica=True) - self.set_auto_conf(replica, {'port': replica.port}) + self.pb.set_archiving('node', replica, replica=True) + replica.set_auto_conf({'port': replica.port}) replica.slow_start(replica=True) @@ -311,28 +272,16 @@ def test_issue_274(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,45600) i") - log_dir = os.path.join(backup_dir, "somedir") + log_dir = os.path.join(self.test_path, "somedir") - try: - self.backup_node( - backup_dir, 'node', replica, backup_type='page', + self.pb.backup_node('node', replica, backup_type='page', options=[ '--log-level-console=verbose', '--log-level-file=verbose', '--log-directory={0}'.format(log_dir), '-j1', '--log-filename=somelog.txt', '--archive-timeout=5s', - '--no-validate', '--log-rotation-size=100KB']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of archiving timeout" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: WAL segment', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + '--no-validate', '--log-rotation-size=100KB'], + expect_error="because of archiving timeout") + self.assertMessage(contains='ERROR: WAL segment') log_file_path = os.path.join( log_dir, 'somelog.txt') diff --git a/tests/merge_test.py b/tests/merge_test.py index a9bc6fe68..e67f446cc 100644 --- a/tests/merge_test.py +++ b/tests/merge_test.py @@ -2,34 +2,40 @@ import unittest import os -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +from .compression_test import have_alg +from .helpers.ptrack_helpers import ProbackupTest +from .helpers.ptrack_helpers import fs_backup_class +from .helpers.ptrack_helpers import base36enc, base36dec +from pg_probackup2.gdb import needs_gdb from testgres import QueryException import shutil from datetime import datetime, timedelta import time import subprocess -class MergeTest(ProbackupTest, unittest.TestCase): - def test_basic_merge_full_page(self): +class MergeTest(ProbackupTest): + + def test_basic_merge_full_2page(self): """ - Test MERGE command, it merges FULL backup with target PAGE backups + 1. Full backup -> fill data + 2. First Page backup -> fill data + 3. Second Page backup + 4. Merge 2 "Page" backups + Restore and compare """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") - # Initialize instance and backup directory - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=["--data-checksums"]) + node = self.pg_node.make_simple('node') - self.init_pb(backup_dir) - self.add_instance(backup_dir, "node", node) - self.set_archiving(backup_dir, "node", node) + self.pb.init() + self.pb.add_instance("node", node) + self.pb.set_archiving("node", node) node.slow_start() # Do full backup - self.backup_node(backup_dir, "node", node, options=['--compress']) - show_backup = self.show_pb(backup_dir, "node")[0] + self.pb.backup_node("node", node, options=['--compress']) + show_backup = self.pb.show("node")[0] self.assertEqual(show_backup["status"], "OK") self.assertEqual(show_backup["backup-mode"], "FULL") @@ -42,8 +48,8 @@ def test_basic_merge_full_page(self): conn.commit() # Do first page backup - self.backup_node(backup_dir, "node", node, backup_type="page", options=['--compress']) - show_backup = self.show_pb(backup_dir, "node")[1] + self.pb.backup_node("node", node, backup_type="page", options=['--compress']) + show_backup = self.pb.show("node")[1] # sanity check self.assertEqual(show_backup["status"], "OK") @@ -57,10 +63,9 @@ def test_basic_merge_full_page(self): conn.commit() # Do second page backup - self.backup_node( - backup_dir, "node", node, + self.pb.backup_node("node", node, backup_type="page", options=['--compress']) - show_backup = self.show_pb(backup_dir, "node")[2] + show_backup = self.pb.show("node")[2] page_id = show_backup["id"] if self.paranoia: @@ -71,9 +76,11 @@ def test_basic_merge_full_page(self): self.assertEqual(show_backup["backup-mode"], "PAGE") # Merge all backups - self.merge_backup(backup_dir, "node", page_id, - options=["-j", "4"]) - show_backups = self.show_pb(backup_dir, "node") + output = self.pb.merge_backup("node", page_id, + options=["-j", "4"]) + self.assertNotIn("WARNING", output) + + show_backups = self.pb.show("node") # sanity check self.assertEqual(len(show_backups), 1) @@ -82,7 +89,7 @@ def test_basic_merge_full_page(self): # Drop node and restore it node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) # Check physical correctness if self.paranoia: @@ -96,88 +103,40 @@ def test_basic_merge_full_page(self): count2 = node.execute("postgres", "select count(*) from test") self.assertEqual(count1, count2) - def test_merge_compressed_backups(self): + @unittest.skipIf(not (have_alg('lz4') and have_alg('zstd')), + "pg_probackup is not compiled with lz4 or zstd support") + def test_merge_compressed_delta_page_ptrack(self): """ - Test MERGE command with compressed backups + 1. Full compressed [zlib, 3] backup -> change data + 2. Delta compressed [pglz, 5] -> change data + 3. Page compressed [lz4, 9] -> change data + 4. Ptrack compressed [zstd, default] + 5. Merge all backups in one + Restore and compare """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') # Initialize instance and backup directory - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=["--data-checksums"]) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, "node", node) - self.set_archiving(backup_dir, "node", node) - node.slow_start() - - # Do full compressed backup - self.backup_node(backup_dir, "node", node, options=['--compress']) - show_backup = self.show_pb(backup_dir, "node")[0] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "FULL") - - # Fill with data - with node.connect() as conn: - conn.execute("create table test (id int)") - conn.execute( - "insert into test select i from generate_series(1,10) s(i)") - count1 = conn.execute("select count(*) from test") - conn.commit() - - # Do compressed page backup - self.backup_node( - backup_dir, "node", node, backup_type="page", options=['--compress']) - show_backup = self.show_pb(backup_dir, "node")[1] - page_id = show_backup["id"] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "PAGE") - - # Merge all backups - self.merge_backup(backup_dir, "node", page_id, options=['-j2']) - show_backups = self.show_pb(backup_dir, "node") - - self.assertEqual(len(show_backups), 1) - self.assertEqual(show_backups[0]["status"], "OK") - self.assertEqual(show_backups[0]["backup-mode"], "FULL") + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=True) - # Drop node and restore it - node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance("node", node) + self.pb.set_archiving("node", node) node.slow_start() - # Check restored node - count2 = node.execute("postgres", "select count(*) from test") - self.assertEqual(count1, count2) - - # Clean after yourself - node.cleanup() - - def test_merge_compressed_backups_1(self): - """ - Test MERGE command with compressed backups - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") - - # Initialize instance and backup directory - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=["--data-checksums"]) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, "node", node) - self.set_archiving(backup_dir, "node", node) - node.slow_start() + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # Fill with data node.pgbench_init(scale=10) # Do compressed FULL backup - self.backup_node(backup_dir, "node", node, options=['--compress', '--stream']) - show_backup = self.show_pb(backup_dir, "node")[0] + self.pb.backup_node("node", node, options=['--stream', + '--compress-level', '3', + '--compress-algorithm', 'zlib']) + show_backup = self.pb.show("node")[0] self.assertEqual(show_backup["status"], "OK") self.assertEqual(show_backup["backup-mode"], "FULL") @@ -187,37 +146,56 @@ def test_merge_compressed_backups_1(self): pgbench.wait() # Do compressed DELTA backup - self.backup_node( - backup_dir, "node", node, - backup_type="delta", options=['--compress', '--stream']) + self.pb.backup_node("node", node, + backup_type="delta", options=['--stream', + '--compress-level', '5', + '--compress-algorithm', 'pglz']) # Change data pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() # Do compressed PAGE backup - self.backup_node( - backup_dir, "node", node, backup_type="page", options=['--compress']) + self.pb.backup_node("node", node, backup_type="page", options=['--compress-level', '9', + '--compress-algorithm', 'lz4']) + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do compressed PTRACK backup + self.pb.backup_node("node", node, backup_type='ptrack', options=['--compress-algorithm', 'zstd']) pgdata = self.pgdata_content(node.data_dir) - show_backup = self.show_pb(backup_dir, "node")[2] - page_id = show_backup["id"] + # Check backups + show_backup = self.pb.show("node") + self.assertEqual(show_backup[0]["status"], "OK") + self.assertEqual(show_backup[0]["backup-mode"], "FULL") - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "PAGE") + self.assertEqual(show_backup[1]["status"], "OK") + self.assertEqual(show_backup[1]["backup-mode"], "DELTA") + + self.assertEqual(show_backup[2]["status"], "OK") + self.assertEqual(show_backup[2]["backup-mode"], "PAGE") + + self.assertEqual(show_backup[3]["status"], "OK") + self.assertEqual(show_backup[3]["backup-mode"], "PTRACK") + + ptrack_id = show_backup[3]["id"] # Merge all backups - self.merge_backup(backup_dir, "node", page_id, options=['-j2']) - show_backups = self.show_pb(backup_dir, "node") + self.pb.merge_backup("node", ptrack_id, options=['-j2']) + show_backups = self.pb.show("node") + # Check number of backups and status self.assertEqual(len(show_backups), 1) self.assertEqual(show_backups[0]["status"], "OK") self.assertEqual(show_backups[0]["backup-mode"], "FULL") # Drop node and restore it node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -225,30 +203,38 @@ def test_merge_compressed_backups_1(self): # Clean after yourself node.cleanup() - def test_merge_compressed_and_uncompressed_backups(self): + def test_merge_uncompressed_ptrack_page_delta(self): """ - Test MERGE command with compressed and uncompressed backups + 1. Full uncompressed backup -> change data + 2. uncompressed Ptrack -> change data + 3. uncompressed Page -> change data + 4. uncompressed Delta + 5. Merge all backups in one + Restore and compare """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + + backup_dir = self.backup_dir # Initialize instance and backup directory - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=["--data-checksums"], - ) + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, "node", node) - self.set_archiving(backup_dir, "node", node) + self.pb.init() + self.pb.add_instance("node", node) + self.pb.set_archiving("node", node) node.slow_start() + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + # Fill with data node.pgbench_init(scale=10) - # Do compressed FULL backup - self.backup_node(backup_dir, "node", node, options=[ - '--compress-algorithm=zlib', '--stream']) - show_backup = self.show_pb(backup_dir, "node")[0] + # Do uncompressed FULL backup + self.pb.backup_node("node", node, options=['--stream']) + show_backup = self.pb.show("node")[0] self.assertEqual(show_backup["status"], "OK") self.assertEqual(show_backup["backup-mode"], "FULL") @@ -257,109 +243,54 @@ def test_merge_compressed_and_uncompressed_backups(self): pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() - # Do compressed DELTA backup - self.backup_node( - backup_dir, "node", node, backup_type="delta", - options=['--compress', '--stream']) + # Do uncompressed PTRACK backup + self.pb.backup_node("node", node, + backup_type="ptrack", options=['--stream']) # Change data pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() # Do uncompressed PAGE backup - self.backup_node(backup_dir, "node", node, backup_type="page") - - pgdata = self.pgdata_content(node.data_dir) - - show_backup = self.show_pb(backup_dir, "node")[2] - page_id = show_backup["id"] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "PAGE") - - # Merge all backups - self.merge_backup(backup_dir, "node", page_id, options=['-j2']) - show_backups = self.show_pb(backup_dir, "node") - - self.assertEqual(len(show_backups), 1) - self.assertEqual(show_backups[0]["status"], "OK") - self.assertEqual(show_backups[0]["backup-mode"], "FULL") - - # Drop node and restore it - node.cleanup() - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - - # Clean after yourself - node.cleanup() - - def test_merge_compressed_and_uncompressed_backups_1(self): - """ - Test MERGE command with compressed and uncompressed backups - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") - - # Initialize instance and backup directory - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=["--data-checksums"], - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, "node", node) - self.set_archiving(backup_dir, "node", node) - node.slow_start() - - # Fill with data - node.pgbench_init(scale=5) - - # Do compressed FULL backup - self.backup_node(backup_dir, "node", node, options=[ - '--compress-algorithm=zlib', '--stream']) - show_backup = self.show_pb(backup_dir, "node")[0] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "FULL") + self.pb.backup_node("node", node, backup_type="page") # Change data - pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() # Do uncompressed DELTA backup - self.backup_node( - backup_dir, "node", node, backup_type="delta", - options=['--stream']) + self.pb.backup_node("node", node, backup_type='delta') - # Change data - pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) - pgbench.wait() + pgdata = self.pgdata_content(node.data_dir) - # Do compressed PAGE backup - self.backup_node( - backup_dir, "node", node, backup_type="page", - options=['--compress-algorithm=zlib']) + # Check backups + show_backup = self.pb.show("node") + self.assertEqual(show_backup[0]["status"], "OK") + self.assertEqual(show_backup[0]["backup-mode"], "FULL") - pgdata = self.pgdata_content(node.data_dir) + self.assertEqual(show_backup[1]["status"], "OK") + self.assertEqual(show_backup[1]["backup-mode"], "PTRACK") - show_backup = self.show_pb(backup_dir, "node")[2] - page_id = show_backup["id"] + self.assertEqual(show_backup[2]["status"], "OK") + self.assertEqual(show_backup[2]["backup-mode"], "PAGE") - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "PAGE") + self.assertEqual(show_backup[3]["status"], "OK") + self.assertEqual(show_backup[3]["backup-mode"], "DELTA") + + ptrack_id = show_backup[3]["id"] # Merge all backups - self.merge_backup(backup_dir, "node", page_id) - show_backups = self.show_pb(backup_dir, "node") + self.pb.merge_backup("node", ptrack_id, options=['-j2']) + show_backups = self.pb.show("node") + # Check number of backups and status self.assertEqual(len(show_backups), 1) self.assertEqual(show_backups[0]["status"], "OK") self.assertEqual(show_backups[0]["backup-mode"], "FULL") # Drop node and restore it node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -367,73 +298,6 @@ def test_merge_compressed_and_uncompressed_backups_1(self): # Clean after yourself node.cleanup() - def test_merge_compressed_and_uncompressed_backups_2(self): - """ - Test MERGE command with compressed and uncompressed backups - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, "backup") - - # Initialize instance and backup directory - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=["--data-checksums"], - ) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, "node", node) - self.set_archiving(backup_dir, "node", node) - node.slow_start() - - # Fill with data - node.pgbench_init(scale=20) - - # Do uncompressed FULL backup - self.backup_node(backup_dir, "node", node) - show_backup = self.show_pb(backup_dir, "node")[0] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "FULL") - - # Change data - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # Do compressed DELTA backup - self.backup_node( - backup_dir, "node", node, backup_type="delta", - options=['--compress-algorithm=zlib', '--stream']) - - # Change data - pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) - pgbench.wait() - - # Do uncompressed PAGE backup - self.backup_node( - backup_dir, "node", node, backup_type="page") - - pgdata = self.pgdata_content(node.data_dir) - - show_backup = self.show_pb(backup_dir, "node")[2] - page_id = show_backup["id"] - - self.assertEqual(show_backup["status"], "OK") - self.assertEqual(show_backup["backup-mode"], "PAGE") - - # Merge all backups - self.merge_backup(backup_dir, "node", page_id) - show_backups = self.show_pb(backup_dir, "node") - - self.assertEqual(len(show_backups), 1) - self.assertEqual(show_backups[0]["status"], "OK") - self.assertEqual(show_backups[0]["backup-mode"], "FULL") - - # Drop node and restore it - node.cleanup() - self.restore_node(backup_dir, 'node', node) - - pgdata_restored = self.pgdata_content(node.data_dir) - self.compare_pgdata(pgdata, pgdata_restored) - # @unittest.skip("skip") def test_merge_tablespaces(self): """ @@ -442,15 +306,11 @@ def test_merge_tablespaces(self): tablespace, take page backup, merge it and restore """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], - ) + node = self.pg_node.make_simple('node', set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -461,7 +321,7 @@ def test_merge_tablespaces(self): " from generate_series(0,100) i" ) # FULL backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # Create new tablespace self.create_tblspace_in_node(node, 'somedata1') @@ -485,7 +345,7 @@ def test_merge_tablespaces(self): ) # PAGE backup - backup_id = self.backup_node(backup_dir, 'node', node, backup_type="page") + backup_id = self.pb.backup_node('node', node, backup_type="page") pgdata = self.pgdata_content(node.data_dir) @@ -498,10 +358,9 @@ def test_merge_tablespaces(self): ignore_errors=True) node.cleanup() - self.merge_backup(backup_dir, 'node', backup_id) + self.pb.merge_backup('node', backup_id) - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]) + self.pb.restore_node('node', node, options=["-j", "4"]) pgdata_restored = self.pgdata_content(node.data_dir) @@ -516,21 +375,18 @@ def test_merge_tablespaces_1(self): drop first tablespace and take delta backup, merge it and restore """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], - ) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() self.create_tblspace_in_node(node, 'somedata') # FULL backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.safe_psql( "postgres", "create table t_heap tablespace somedata as select i as id," @@ -549,7 +405,7 @@ def test_merge_tablespaces_1(self): ) # PAGE backup - self.backup_node(backup_dir, 'node', node, backup_type="page") + self.pb.backup_node('node', node, backup_type="page") node.safe_psql( "postgres", @@ -561,8 +417,7 @@ def test_merge_tablespaces_1(self): ) # DELTA backup - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="delta") + backup_id = self.pb.backup_node('node', node, backup_type="delta") pgdata = self.pgdata_content(node.data_dir) @@ -575,10 +430,9 @@ def test_merge_tablespaces_1(self): ignore_errors=True) node.cleanup() - self.merge_backup(backup_dir, 'node', backup_id) + self.pb.merge_backup('node', backup_id) - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=["-j", "4"]) pgdata_restored = self.pgdata_content(node.data_dir) @@ -591,20 +445,17 @@ def test_merge_page_truncate(self): take page backup, merge full and page, restore last page backup and check data correctness """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '300s'}) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node_restored.cleanup() node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -621,7 +472,7 @@ def test_merge_page_truncate(self): "postgres", "vacuum t_heap") - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.safe_psql( "postgres", @@ -630,22 +481,20 @@ def test_merge_page_truncate(self): "postgres", "vacuum t_heap") - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - page_id = self.show_pb(backup_dir, "node")[1]["id"] - self.merge_backup(backup_dir, "node", page_id) + page_id = self.pb.show("node")[1]["id"] + self.pb.merge_backup("node", page_id) - self.validate_pb(backup_dir) + self.pb.validate() old_tablespace = self.get_tblspace_path(node, 'somedata') new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=[ "-j", "4", "-T", "{0}={1}".format(old_tablespace, new_tablespace)]) @@ -655,7 +504,7 @@ def test_merge_page_truncate(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() # Logical comparison @@ -671,20 +520,17 @@ def test_merge_delta_truncate(self): take page backup, merge full and page, restore last page backup and check data correctness """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '300s'}) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node_restored.cleanup() node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -701,7 +547,7 @@ def test_merge_delta_truncate(self): "postgres", "vacuum t_heap") - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.safe_psql( "postgres", @@ -710,22 +556,20 @@ def test_merge_delta_truncate(self): "postgres", "vacuum t_heap") - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - page_id = self.show_pb(backup_dir, "node")[1]["id"] - self.merge_backup(backup_dir, "node", page_id) + page_id = self.pb.show("node")[1]["id"] + self.pb.merge_backup("node", page_id) - self.validate_pb(backup_dir) + self.pb.validate() old_tablespace = self.get_tblspace_path(node, 'somedata') new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=[ "-j", "4", "-T", "{0}={1}".format(old_tablespace, new_tablespace)]) @@ -735,7 +579,7 @@ def test_merge_delta_truncate(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() # Logical comparison @@ -754,16 +598,14 @@ def test_merge_ptrack_truncate(self): if not self.ptrack: self.skipTest('Skipped because ptrack support is disabled') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], ptrack_enable=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -784,7 +626,7 @@ def test_merge_ptrack_truncate(self): "postgres", "vacuum t_heap") - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.safe_psql( "postgres", @@ -794,25 +636,22 @@ def test_merge_ptrack_truncate(self): "postgres", "vacuum t_heap") - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack') + ptrack_id = self.pb.backup_node('node', node, backup_type='ptrack') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - self.merge_backup(backup_dir, "node", page_id) + self.pb.merge_backup("node", ptrack_id) - self.validate_pb(backup_dir) + self.pb.validate() - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() old_tablespace = self.get_tblspace_path(node, 'somedata') new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=[ "-j", "4", "-T", "{0}={1}".format(old_tablespace, new_tablespace)]) @@ -822,7 +661,7 @@ def test_merge_ptrack_truncate(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() # Logical comparison @@ -838,24 +677,21 @@ def test_merge_delta_delete(self): alter tablespace location, take delta backup, merge full and delta, restore database. """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', set_replication=True, pg_options={ 'checkpoint_timeout': '30s', } ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() self.create_tblspace_in_node(node, 'somedata') # FULL backup - self.backup_node(backup_dir, 'node', node, options=["--stream"]) + self.pb.backup_node('node', node, options=["--stream"]) node.safe_psql( "postgres", @@ -875,8 +711,7 @@ def test_merge_delta_delete(self): ) # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='delta', options=["--stream"] ) @@ -884,17 +719,15 @@ def test_merge_delta_delete(self): if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - backup_id = self.show_pb(backup_dir, "node")[1]["id"] - self.merge_backup(backup_dir, "node", backup_id, options=["-j", "4"]) + backup_id = self.pb.show("node")[1]["id"] + self.pb.merge_backup("node", backup_id, options=["-j", "4"]) # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored') + node_restored = self.pg_node.make_simple('node_restored' ) node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=[ "-j", "4", "-T", "{0}={1}".format( @@ -910,30 +743,27 @@ def test_merge_delta_delete(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() # @unittest.skip("skip") + @needs_gdb def test_continue_failed_merge(self): """ Check that failed MERGE can be continued """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join( - self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.safe_psql( "postgres", @@ -943,8 +773,7 @@ def test_continue_failed_merge(self): ) # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta' + self.pb.backup_node('node', node, backup_type='delta' ) node.safe_psql( @@ -958,54 +787,54 @@ def test_continue_failed_merge(self): ) # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta' + self.pb.backup_node('node', node, backup_type='delta' ) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - backup_id = self.show_pb(backup_dir, "node")[2]["id"] + backup_id = self.pb.show("node")[2]["id"] - gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + gdb = self.pb.merge_backup("node", backup_id, gdb=True) gdb.set_breakpoint('backup_non_data_file_internal') gdb.run_until_break() gdb.continue_execution_until_break(5) - gdb._execute('signal SIGKILL') - gdb._execute('detach') + gdb.signal('SIGKILL') + gdb.detach() time.sleep(1) - print(self.show_pb(backup_dir, as_text=True, as_json=False)) + self.expire_locks(backup_dir, 'node') + + print(self.pb.show(as_text=True, as_json=False)) # Try to continue failed MERGE - self.merge_backup(backup_dir, "node", backup_id) + self.pb.merge_backup("node", backup_id) # Drop node and restore it node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) # @unittest.skip("skip") + @needs_gdb def test_continue_failed_merge_with_corrupted_delta_backup(self): """ Fail merge via gdb, corrupt DELTA backup, try to continue merge """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.safe_psql( "postgres", @@ -1018,8 +847,7 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') node.safe_psql( "postgres", @@ -1034,73 +862,53 @@ def test_continue_failed_merge_with_corrupted_delta_backup(self): "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # DELTA BACKUP - backup_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='delta') + backup_id_2 = self.pb.backup_node('node', node, backup_type='delta') - backup_id = self.show_pb(backup_dir, "node")[1]["id"] + backup_id = self.pb.show("node")[1]["id"] # Failed MERGE - gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + gdb = self.pb.merge_backup("node", backup_id, gdb=True) gdb.set_breakpoint('backup_non_data_file_internal') gdb.run_until_break() gdb.continue_execution_until_break(2) - gdb._execute('signal SIGKILL') + gdb.signal('SIGKILL') + + self.expire_locks(backup_dir, 'node') # CORRUPT incremental backup # read block from future # block_size + backup_header = 8200 - file = os.path.join( - backup_dir, 'backups', 'node', - backup_id_2, 'database', new_path) - with open(file, 'rb') as f: - f.seek(8200) - block_1 = f.read(8200) - f.close - + file_content2 = self.read_backup_file(backup_dir, 'node', backup_id_2, + f'database/{new_path}')[:16400] # write block from future - file = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', old_path) - with open(file, 'r+b') as f: - f.seek(8200) - f.write(block_1) - f.close + self.corrupt_backup_file(backup_dir, 'node', backup_id, + f'database/{old_path}', + damage=(8200, file_content2[8200:16400])) # Try to continue failed MERGE - try: - print(self.merge_backup(backup_dir, "node", backup_id)) - self.assertEqual( - 1, 0, - "Expecting Error because of incremental backup corruption.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "ERROR: Backup {0} has status CORRUPT, merge is aborted".format( - backup_id) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.merge_backup("node", backup_id, + expect_error="because of incremental backup corruption") + self.assertMessage(contains=f"ERROR: Backup {backup_id} has status CORRUPT, merge is aborted") + @needs_gdb def test_continue_failed_merge_2(self): """ Check that failed MERGE on delete can be continued """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.safe_psql( "postgres", @@ -1109,8 +917,7 @@ def test_continue_failed_merge_2(self): " from generate_series(0,1000) i") # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') node.safe_psql( "postgres", @@ -1121,52 +928,55 @@ def test_continue_failed_merge_2(self): "vacuum t_heap") # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - backup_id = self.show_pb(backup_dir, "node")[2]["id"] + backup_id = self.pb.show("node")[2]["id"] - gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + gdb = self.pb.merge_backup("node", backup_id, gdb=True) - gdb.set_breakpoint('pgFileDelete') + gdb.set_breakpoint('lock_backup') gdb.run_until_break() gdb._execute('thread apply all bt') + gdb.remove_all_breakpoints() + + gdb.set_breakpoint('pioRemoveDir__do') gdb.continue_execution_until_break(20) gdb._execute('thread apply all bt') - gdb._execute('signal SIGKILL') + gdb.signal('SIGKILL') + + self.expire_locks(backup_dir, 'node') - print(self.show_pb(backup_dir, as_text=True, as_json=False)) + print(self.pb.show(as_text=True, as_json=False)) - backup_id_deleted = self.show_pb(backup_dir, "node")[1]["id"] + backup_id_deleted = self.pb.show("node")[1]["id"] # TODO check that full backup has meta info is equal to DELETTING # Try to continue failed MERGE - self.merge_backup(backup_dir, "node", backup_id) + self.pb.merge_backup("node", backup_id) + @needs_gdb def test_continue_failed_merge_3(self): """ Check that failed MERGE cannot be continued if intermediate backup is missing. """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Create test data @@ -1179,7 +989,7 @@ def test_continue_failed_merge_3(self): ) # FULL backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # CREATE FEW PAGE BACKUP i = 0 @@ -1203,67 +1013,55 @@ def test_continue_failed_merge_3(self): ) # PAGE BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='page' + self.pb.backup_node('node', node, backup_type='page' ) i = i + 1 if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - backup_id_merge = self.show_pb(backup_dir, "node")[2]["id"] - backup_id_delete = self.show_pb(backup_dir, "node")[1]["id"] + backup_id_merge = self.pb.show("node")[2]["id"] + backup_id_delete = self.pb.show("node")[1]["id"] - print(self.show_pb(backup_dir, as_text=True, as_json=False)) + print(self.pb.show(as_text=True, as_json=False)) - gdb = self.merge_backup(backup_dir, "node", backup_id_merge, gdb=True) + gdb = self.pb.merge_backup("node", backup_id_merge, gdb=True) gdb.set_breakpoint('backup_non_data_file_internal') gdb.run_until_break() gdb.continue_execution_until_break(2) - gdb._execute('signal SIGKILL') + gdb.signal('SIGKILL') - print(self.show_pb(backup_dir, as_text=True, as_json=False)) + self.expire_locks(backup_dir, 'node') + + print(self.pb.show(as_text=True, as_json=False)) # print(os.path.join(backup_dir, "backups", "node", backup_id_delete)) # DELETE PAGE1 - shutil.rmtree( - os.path.join(backup_dir, "backups", "node", backup_id_delete)) + self.remove_one_backup(backup_dir, 'node', backup_id_delete) # Try to continue failed MERGE - try: - self.merge_backup(backup_dir, "node", backup_id_merge) - self.assertEqual( - 1, 0, - "Expecting Error because of backup corruption.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "ERROR: Incremental chain is broken, " - "merge is impossible to finish" in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.merge_backup("node", backup_id_merge, + expect_error="because of backup corruption") + self.assertMessage(contains="ERROR: Incremental chain is broken, " + "merge is impossible to finish") def test_merge_different_compression_algo(self): """ Check that backups with different compression algorithms can be merged """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL backup - self.backup_node( - backup_dir, 'node', node, options=['--compress-algorithm=zlib']) + self.pb.backup_node('node', node, options=['--compress-algorithm=zlib']) node.safe_psql( "postgres", @@ -1272,8 +1070,7 @@ def test_merge_different_compression_algo(self): " from generate_series(0,1000) i") # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='delta', options=['--compress-algorithm=pglz']) node.safe_psql( @@ -1285,85 +1082,266 @@ def test_merge_different_compression_algo(self): "vacuum t_heap") # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - backup_id = self.show_pb(backup_dir, "node")[2]["id"] + backup_id = self.pb.show("node")[2]["id"] - self.merge_backup(backup_dir, "node", backup_id) + self.pb.merge_backup("node", backup_id) def test_merge_different_wal_modes(self): """ Check that backups with different wal modes can be merged correctly """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL stream backup - self.backup_node( - backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) # DELTA archive backup - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') + backup_id = self.pb.backup_node('node', node, backup_type='delta') - self.merge_backup(backup_dir, 'node', backup_id=backup_id) + self.pb.merge_backup('node', backup_id=backup_id) self.assertEqual( - 'ARCHIVE', self.show_pb(backup_dir, 'node', backup_id)['wal']) + 'ARCHIVE', self.pb.show('node', backup_id)['wal']) # DELTA stream backup - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, backup_type='delta', options=['--stream']) - self.merge_backup(backup_dir, 'node', backup_id=backup_id) + self.pb.merge_backup('node', backup_id=backup_id) + + self.assertEqual( + 'STREAM', self.pb.show('node', backup_id)['wal']) + + def test_merge_A_B_C_removes_internal_B(self): + """ + check that A->B->C merge removes B merge stub dir. + """ + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True, + initdb_params=['--data-checksums']) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + # FULL + full_id = self.pb.backup_node('node', node, options=['--stream']) + + # DELTA 1 + delta_1_id = self.pb.backup_node('node', node, backup_type='delta') + + # DELTA 2 + delta_2_id = self.pb.backup_node('node', node, backup_type='delta') + + self.pb.merge_backup('node', backup_id=delta_1_id) self.assertEqual( - 'STREAM', self.show_pb(backup_dir, 'node', backup_id)['wal']) + 'ARCHIVE', self.pb.show('node', delta_1_id)['wal']) + self.pb.merge_backup('node', backup_id=delta_2_id) + + backups_dirs_list = self.get_backups_dirs(backup_dir, "node") + + self.assertIn(full_id, backups_dirs_list) + self.assertNotIn(delta_1_id, backups_dirs_list) + self.assertIn(delta_2_id, backups_dirs_list) + + @needs_gdb + def test_merge_A_B_C_broken_on_B_removal(self): + """ + check that A->B->C merge removes B merge stub dir + on second merge try after first merge is killed on B removal. + """ + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True, + initdb_params=['--data-checksums']) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + # FULL + full_id = self.pb.backup_node('node', node, options=['--stream']) + + # DELTA 1 + delta_1_id = self.pb.backup_node('node', node, backup_type='delta') + + # DELTA 2 + delta_2_id = self.pb.backup_node('node', node, backup_type='delta') + + self.pb.merge_backup('node', backup_id=delta_1_id) + + self.assertEqual('ARCHIVE', self.pb.show('node', delta_1_id)['wal']) + + gdb = self.pb.merge_backup('node', backup_id=delta_2_id, gdb=True) + + gdb.set_breakpoint('renameBackupToDir') + gdb.run_until_break() + + gdb.set_breakpoint("pgBackupInit") + # breaks after removing interim B dir, before recreating "C" dir as merged_to + gdb.continue_execution_until_break() + # killing merge on in-critical-section broken inside BACKUP_STATUS_MERGES critical section + gdb.kill() + + self.expire_locks(backup_dir, 'node') + + # rerun merge to C, it should merge fine + self.pb.merge_backup('node', backup_id=delta_2_id)#, gdb=("suspend", 30303)) + backups_dirs_list = self.get_backups_dirs(backup_dir, "node") + + self.assertIn(full_id, backups_dirs_list) + self.assertNotIn(delta_1_id, backups_dirs_list) + self.assertIn(delta_2_id, backups_dirs_list) + + def test_merge_A_B_and_remove_A_removes_B(self): + """ + Check, that after A->B merge and remove of B removes both A and B dirs. + """ + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True, + initdb_params=['--data-checksums']) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + # PRE FULL + pre_full_id = self.pb.backup_node('node', node, options=['--stream']) + + # FULL + full_id = self.pb.backup_node('node', node, options=['--stream']) + + # DELTA 1 + delta_1_id = self.pb.backup_node('node', node, backup_type='delta') + + # DELTA 2 + delta_2_id = self.pb.backup_node('node', node, backup_type='delta') + + # POST FULL + post_full_id = self.pb.backup_node('node', node, options=['--stream']) + + self.pb.merge_backup('node', backup_id=delta_1_id) + + self.assertEqual( + 'ARCHIVE', self.pb.show('node', delta_1_id)['wal']) + + self.pb.delete('node', backup_id=delta_1_id) + + backups_dirs_list = self.get_backups_dirs(backup_dir, "node") + + # these should be deleted obviously + self.assertNotIn(full_id, backups_dirs_list) + self.assertNotIn(delta_2_id, backups_dirs_list) + # these should not be deleted obviously + self.assertIn(pre_full_id, backups_dirs_list) + self.assertIn(post_full_id, backups_dirs_list) + # and actual check for PBCKP-710: deleted symlink directory + self.assertNotIn(delta_1_id, backups_dirs_list) + + + def test_validate_deleted_dirs_merged_from(self): + """ + checks validate fails if we miss full dir + """ + + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True, + initdb_params=['--data-checksums']) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + # FULL 1 + full_1 = self.pb.backup_node('node', node, options=['--stream']) + + # DELTA 1-1 + delta_1_1 = self.pb.backup_node('node', node, backup_type='delta') + + #FULL 2 + full_2 = self.pb.backup_node('node', node, options=['--stream']) + + # DELTA 2-1 + delta_2_1 = self.pb.backup_node('node', node, backup_type='delta') + + # validate is ok + self.pb.merge_backup('node', backup_id=delta_2_1) + self.pb.validate('node') + + # changing DELTA_2_1 backup.control symlink fields + # validate should find problems + with self.modify_backup_control(backup_dir, "node", delta_2_1) as cf: + cf.data += "\nsymlink = " + base36enc(base36dec(delta_2_1)+1) + self.pb.validate('node', expect_error=True) + self.assertMessage(contains="no linked backup") + + with self.modify_backup_control(backup_dir, "node", delta_2_1) as cf: + cf.data = "\n".join(cf.data.splitlines()[:-1]) + + # validate should find previous backup is not FULL + self.remove_one_backup(backup_dir, 'node', full_2) + self.pb.validate('node', expect_error=True) + self.assertMessage(contains="no linked backup") + + # validate should find there's previous FULL backup has bad id or cross references + self.remove_one_backup(backup_dir, 'node', delta_1_1) + with self.modify_backup_control(backup_dir, "node", delta_2_1) as cf: + cf.data += f"\nsymlink = {full_1}" + self.pb.validate('node', expect_error=True) + self.assertMessage(contains="has different 'backup-id'") + + # validate should find there's previous FULL backup has bad id or cross references + self.remove_one_backup(backup_dir, 'node', full_1) + self.pb.validate('node', expect_error=True) + self.assertMessage(contains="no linked backup") + + @needs_gdb def test_crash_after_opening_backup_control_1(self): """ check that crashing after opening backup.control for writing will not result in losing backup metadata """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL stream backup - self.backup_node( - backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) # DELTA archive backup - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') + backup_id = self.pb.backup_node('node', node, backup_type='delta') - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) + print(self.pb.show('node', as_json=False, as_text=True)) - gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + gdb = self.pb.merge_backup("node", backup_id, gdb=True) gdb.set_breakpoint('write_backup_filelist') gdb.run_until_break() @@ -1372,43 +1350,41 @@ def test_crash_after_opening_backup_control_1(self): gdb.set_breakpoint('pgBackupWriteControl') gdb.continue_execution_until_break() - gdb._execute('signal SIGKILL') + gdb.signal('SIGKILL') + + self.expire_locks(backup_dir, 'node') - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) + print(self.pb.show('node', as_json=False, as_text=True)) self.assertEqual( - 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) + 'MERGING', self.pb.show('node')[0]['status']) self.assertEqual( - 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) + 'MERGING', self.pb.show('node')[1]['status']) # @unittest.skip("skip") + @needs_gdb def test_crash_after_opening_backup_control_2(self): """ check that crashing after opening backup_content.control for writing will not result in losing metadata about backup files TODO: rewrite """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Add data node.pgbench_init(scale=3) # FULL backup - full_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) + full_id = self.pb.backup_node('node', node, options=['--stream']) # Change data pgbench = node.pgbench(options=['-T', '20', '-c', '2']) @@ -1425,80 +1401,71 @@ def test_crash_after_opening_backup_control_2(self): 'vacuum pgbench_accounts') # DELTA backup - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') + backup_id = self.pb.backup_node('node', node, backup_type='delta') pgdata = self.pgdata_content(node.data_dir) - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) + print(self.pb.show('node', as_json=False, as_text=True)) - gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + gdb = self.pb.merge_backup("node", backup_id, gdb=True) gdb.set_breakpoint('write_backup_filelist') gdb.run_until_break() # gdb.set_breakpoint('sprintf') # gdb.continue_execution_until_break(1) - gdb._execute('signal SIGKILL') + gdb.signal('SIGKILL') + + self.expire_locks(backup_dir, 'node') - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) + print(self.pb.show('node', as_json=False, as_text=True)) self.assertEqual( - 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) + 'MERGING', self.pb.show('node')[0]['status']) self.assertEqual( - 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) + 'MERGING', self.pb.show('node')[1]['status']) # In to_backup drop file that comes from from_backup # emulate crash during previous merge - file_to_remove = os.path.join( - backup_dir, 'backups', - 'node', full_id, 'database', fsm_path) - - # print(file_to_remove) - - os.remove(file_to_remove) + self.remove_backup_file(backup_dir, 'node', full_id, + f'database/{fsm_path}') # Continue failed merge - self.merge_backup(backup_dir, "node", backup_id) + self.pb.merge_backup("node", backup_id) node.cleanup() # restore merge backup - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") + @needs_gdb def test_losing_file_after_failed_merge(self): """ check that crashing after opening backup_content.control for writing will not result in losing metadata about backup files TODO: rewrite """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Add data node.pgbench_init(scale=1) # FULL backup - full_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) + full_id = self.pb.backup_node('node', node, options=['--stream']) # Change data node.safe_psql( @@ -1516,67 +1483,61 @@ def test_losing_file_after_failed_merge(self): vm_path = path + '_vm' # DELTA backup - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') + backup_id = self.pb.backup_node('node', node, backup_type='delta') pgdata = self.pgdata_content(node.data_dir) - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) + print(self.pb.show('node', as_json=False, as_text=True)) - gdb = self.merge_backup(backup_dir, "node", backup_id, gdb=True) + gdb = self.pb.merge_backup("node", backup_id, gdb=True) gdb.set_breakpoint('write_backup_filelist') gdb.run_until_break() # gdb.set_breakpoint('sprintf') # gdb.continue_execution_until_break(20) - gdb._execute('signal SIGKILL') + gdb.signal('SIGKILL') + + self.expire_locks(backup_dir, 'node') - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) + print(self.pb.show('node', as_json=False, as_text=True)) self.assertEqual( - 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) + 'MERGING', self.pb.show('node')[0]['status']) self.assertEqual( - 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) + 'MERGING', self.pb.show('node')[1]['status']) # In to_backup drop file that comes from from_backup # emulate crash during previous merge - file_to_remove = os.path.join( - backup_dir, 'backups', - 'node', full_id, 'database', vm_path) - - os.remove(file_to_remove) + self.remove_backup_file(backup_dir, 'node', full_id, + f'database/{vm_path}') # Try to continue failed MERGE - self.merge_backup(backup_dir, "node", backup_id) + self.pb.merge_backup("node", backup_id) self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + 'OK', self.pb.show('node')[0]['status']) node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) + @needs_gdb def test_failed_merge_after_delete(self): """ """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # add database @@ -1589,8 +1550,7 @@ def test_failed_merge_after_delete(self): "select oid from pg_database where datname = 'testdb'").decode('utf-8').rstrip() # take FULL backup - full_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) + full_id = self.pb.backup_node('node', node, options=['--stream']) # drop database node.safe_psql( @@ -1598,75 +1558,56 @@ def test_failed_merge_after_delete(self): 'DROP DATABASE testdb') # take PAGE backup - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id = self.pb.backup_node('node', node, backup_type='page') - page_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_2 = self.pb.backup_node('node', node, backup_type='page') - gdb = self.merge_backup( - backup_dir, 'node', page_id, + gdb = self.pb.merge_backup('node', page_id, gdb=True, options=['--log-level-console=verbose']) gdb.set_breakpoint('delete_backup_files') gdb.run_until_break() - gdb.set_breakpoint('pgFileDelete') - gdb.continue_execution_until_break(20) + gdb.set_breakpoint('lock_backup') + gdb.continue_execution_until_break() - gdb._execute('signal SIGKILL') + gdb.signal('SIGKILL') + + self.expire_locks(backup_dir, 'node') # backup half-merged self.assertEqual( - 'MERGED', self.show_pb(backup_dir, 'node')[0]['status']) + 'MERGED', self.pb.show('node')[0]['status']) self.assertEqual( - full_id, self.show_pb(backup_dir, 'node')[0]['id']) - - db_path = os.path.join( - backup_dir, 'backups', 'node', - full_id, 'database', 'base', dboid) + full_id, self.pb.show('node')[0]['id']) - try: - self.merge_backup( - backup_dir, 'node', page_id_2, - options=['--log-level-console=verbose']) - self.assertEqual( - 1, 0, - "Expecting Error because of missing parent.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "ERROR: Full backup {0} has unfinished merge with backup {1}".format( - full_id, page_id) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.merge_backup('node', page_id_2, + options=['--log-level-console=verbose'], + expect_error="because of missing parent") + self.assertMessage(contains=f"ERROR: Full backup {full_id} has " + f"unfinished merge with backup {page_id}") + @needs_gdb def test_failed_merge_after_delete_1(self): """ """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # take FULL backup - full_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) + full_id = self.pb.backup_node('node', node, options=['--stream']) node.pgbench_init(scale=1) - page_1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_1 = self.pb.backup_node('node', node, backup_type='page') # Change PAGE1 backup status to ERROR self.change_backup_status(backup_dir, 'node', page_1, 'ERROR') @@ -1678,14 +1619,12 @@ def test_failed_merge_after_delete_1(self): pgbench.wait() # take PAGE2 backup - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id = self.pb.backup_node('node', node, backup_type='page') # Change PAGE1 backup status to OK self.change_backup_status(backup_dir, 'node', page_1, 'OK') - gdb = self.merge_backup( - backup_dir, 'node', page_id, + gdb = self.pb.merge_backup('node', page_id, gdb=True, options=['--log-level-console=verbose']) gdb.set_breakpoint('delete_backup_files') @@ -1694,105 +1633,80 @@ def test_failed_merge_after_delete_1(self): # gdb.set_breakpoint('parray_bsearch') # gdb.continue_execution_until_break() - gdb.set_breakpoint('pgFileDelete') + gdb.set_breakpoint('lock_backup') gdb.continue_execution_until_break(30) - gdb._execute('signal SIGKILL') + gdb.signal('SIGKILL') + + self.expire_locks(backup_dir, 'node') self.assertEqual( - full_id, self.show_pb(backup_dir, 'node')[0]['id']) + full_id, self.pb.show('node')[0]['id']) # restore node.cleanup() - try: - #self.restore_node(backup_dir, 'node', node, backup_id=page_1) - self.restore_node(backup_dir, 'node', node) - self.assertEqual( - 1, 0, - "Expecting Error because of orphan status.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} is orphan".format(page_1), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node, + expect_error="because of orphan status") + self.assertMessage(contains=f"ERROR: Backup {page_1} is orphan") + @needs_gdb def test_failed_merge_after_delete_2(self): """ """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # take FULL backup - full_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) + full_id = self.pb.backup_node('node', node, options=['--stream']) node.pgbench_init(scale=1) - page_1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_1 = self.pb.backup_node('node', node, backup_type='page') # add data pgbench = node.pgbench(options=['-T', '10', '-c', '2', '--no-vacuum']) pgbench.wait() # take PAGE2 backup - page_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_2 = self.pb.backup_node('node', node, backup_type='page') - gdb = self.merge_backup( - backup_dir, 'node', page_2, gdb=True, + gdb = self.pb.merge_backup('node', page_2, gdb=True, options=['--log-level-console=VERBOSE']) - gdb.set_breakpoint('pgFileDelete') + gdb.set_breakpoint('delete_backup_files') gdb.run_until_break() + gdb.set_breakpoint('lock_backup') gdb.continue_execution_until_break(2) - gdb._execute('signal SIGKILL') + gdb.signal('SIGKILL') - self.delete_pb(backup_dir, 'node', backup_id=page_2) + self.expire_locks(backup_dir, 'node') + + self.pb.delete('node', backup_id=page_2) # rerun merge - try: - #self.restore_node(backup_dir, 'node', node, backup_id=page_1) - self.merge_backup(backup_dir, 'node', page_1) - self.assertEqual( - 1, 0, - "Expecting Error because of backup is missing.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Full backup {0} has unfinished merge " - "with backup {1}".format(full_id, page_2), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.merge_backup('node', page_1, + expect_error="because backup is missing") + self.assertMessage(contains=f"ERROR: Full backup {full_id} has unfinished merge " + f"with backup {page_2}") + @needs_gdb def test_failed_merge_after_delete_3(self): """ """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # add database @@ -1805,8 +1719,7 @@ def test_failed_merge_after_delete_3(self): "select oid from pg_database where datname = 'testdb'").rstrip() # take FULL backup - full_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) + full_id = self.pb.backup_node('node', node, options=['--stream']) # drop database node.safe_psql( @@ -1814,57 +1727,42 @@ def test_failed_merge_after_delete_3(self): 'DROP DATABASE testdb') # take PAGE backup - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id = self.pb.backup_node('node', node, backup_type='page') # create database node.safe_psql( 'postgres', 'create DATABASE testdb') - page_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_2 = self.pb.backup_node('node', node, backup_type='page') - gdb = self.merge_backup( - backup_dir, 'node', page_id, + gdb = self.pb.merge_backup('node', page_id, gdb=True, options=['--log-level-console=verbose']) gdb.set_breakpoint('delete_backup_files') gdb.run_until_break() - gdb.set_breakpoint('pgFileDelete') - gdb.continue_execution_until_break(20) + gdb.set_breakpoint('lock_backup') + gdb.continue_execution_until_break() + + gdb.signal('SIGKILL') - gdb._execute('signal SIGKILL') + self.expire_locks(backup_dir, 'node') # backup half-merged self.assertEqual( - 'MERGED', self.show_pb(backup_dir, 'node')[0]['status']) + 'MERGED', self.pb.show('node')[0]['status']) self.assertEqual( - full_id, self.show_pb(backup_dir, 'node')[0]['id']) - - db_path = os.path.join( - backup_dir, 'backups', 'node', full_id) + full_id, self.pb.show('node')[0]['id']) # FULL backup is missing now - shutil.rmtree(db_path) + self.remove_one_backup(backup_dir, 'node', full_id) - try: - self.merge_backup( - backup_dir, 'node', page_id_2, - options=['--log-level-console=verbose']) - self.assertEqual( - 1, 0, - "Expecting Error because of missing parent.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "ERROR: Failed to find parent full backup for {0}".format( - page_id_2) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.merge_backup('node', page_id_2, + options=['--log-level-console=verbose'], + expect_error="because of missing parent") + self.assertMessage(contains=f"ERROR: Failed to find parent full backup for {page_id_2}") # Skipped, because backups from the future are invalid. # This cause a "ERROR: Can't assign backup_id, there is already a backup in future" @@ -1876,35 +1774,32 @@ def test_merge_backup_from_future(self): take FULL backup, table PAGE backup from future, try to merge page with FULL """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + if not backup_dir.is_file_based: + self.skipTest("test uses rename which is hard for cloud") + + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Take FULL - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.pgbench_init(scale=5) # Take PAGE from future - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id = self.pb.backup_node('node', node, backup_type='page') - with open( - os.path.join( - backup_dir, 'backups', 'node', - backup_id, "backup.control"), "a") as conf: - conf.write("start-time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() + timedelta(days=3))) + with self.modify_backup_control(backup_dir, 'node', backup_id) as cf: + cf.data += "\nstart-time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() + timedelta(days=3)) # rename directory - new_id = self.show_pb(backup_dir, 'node')[1]['id'] + new_id = self.pb.show('node')[1]['id'] os.rename( os.path.join(backup_dir, 'backups', 'node', backup_id), @@ -1913,17 +1808,15 @@ def test_merge_backup_from_future(self): pgbench = node.pgbench(options=['-T', '5', '-c', '1', '--no-vacuum']) pgbench.wait() - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id = self.pb.backup_node('node', node, backup_type='page') pgdata = self.pgdata_content(node.data_dir) result = node.table_checksum("pgbench_accounts") - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', + self.pb.restore_node('node', node_restored, backup_id=backup_id) pgdata_restored = self.pgdata_content(node_restored.data_dir) @@ -1931,15 +1824,12 @@ def test_merge_backup_from_future(self): # check that merged backup has the same state as node_restored.cleanup() - self.merge_backup(backup_dir, 'node', backup_id=backup_id) - self.restore_node( - backup_dir, 'node', + self.pb.merge_backup('node', backup_id=backup_id) + self.pb.restore_node('node', node_restored, backup_id=backup_id) pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.set_auto_conf( - node_restored, - {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() result_new = node_restored.table_checksum("pgbench_accounts") @@ -1950,7 +1840,7 @@ def test_merge_backup_from_future(self): # @unittest.skip("skip") def test_merge_multiple_descendants(self): - """ + r""" PAGEb3 | PAGEa3 PAGEb2 / @@ -1960,26 +1850,23 @@ def test_merge_multiple_descendants(self): FULLb | FULLa """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Take FULL BACKUPs - backup_id_a = self.backup_node(backup_dir, 'node', node) + backup_id_a = self.pb.backup_node('node', node) - backup_id_b = self.backup_node(backup_dir, 'node', node) + backup_id_b = self.pb.backup_node('node', node) # Change FULLb backup status to ERROR self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a1 = self.pb.backup_node('node', node, backup_type='page') # Change FULLb backup status to OK self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') @@ -1991,8 +1878,7 @@ def test_merge_multiple_descendants(self): # FULLb OK # FULLa OK - page_id_b1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b1 = self.pb.backup_node('node', node, backup_type='page') # PAGEb1 OK # PAGEa1 ERROR @@ -2011,8 +1897,7 @@ def test_merge_multiple_descendants(self): # FULLb ERROR # FULLa OK - page_id_a2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a2 = self.pb.backup_node('node', node, backup_type='page') # PAGEa2 OK # PAGEb1 ERROR @@ -2034,8 +1919,7 @@ def test_merge_multiple_descendants(self): # FULLb OK # FULLa ERROR - page_id_b2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b2 = self.pb.backup_node('node', node, backup_type='page') # PAGEb2 OK # PAGEa2 ERROR @@ -2059,8 +1943,7 @@ def test_merge_multiple_descendants(self): # FULLb ERROR # FULLa OK - page_id_a3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a3 = self.pb.backup_node('node', node, backup_type='page') # PAGEa3 OK # PAGEb2 ERROR @@ -2078,8 +1961,7 @@ def test_merge_multiple_descendants(self): self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') self.change_backup_status(backup_dir, 'node', backup_id_a, 'OK') - page_id_b3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b3 = self.pb.backup_node('node', node, backup_type='page') # PAGEb3 OK # PAGEa3 ERROR @@ -2106,32 +1988,20 @@ def test_merge_multiple_descendants(self): # Check that page_id_a3 and page_id_a2 are both direct descendants of page_id_a1 self.assertEqual( - self.show_pb(backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'], + self.pb.show('node', backup_id=page_id_a3)['parent-backup-id'], page_id_a1) self.assertEqual( - self.show_pb(backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'], + self.pb.show('node', backup_id=page_id_a2)['parent-backup-id'], page_id_a1) - self.merge_backup( - backup_dir, 'node', page_id_a2, + self.pb.merge_backup('node', page_id_a2, options=['--merge-expired', '--log-level-console=log']) - try: - self.merge_backup( - backup_dir, 'node', page_id_a3, - options=['--merge-expired', '--log-level-console=log']) - self.assertEqual( - 1, 0, - "Expecting Error because of parent FULL backup is missing.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "ERROR: Failed to find parent full backup for {0}".format( - page_id_a3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.merge_backup('node', page_id_a3, + options=['--merge-expired', '--log-level-console=log'], + expect_error="parent FULL backup is missing") + self.assertMessage(contains=f"ERROR: Failed to find parent full backup for {page_id_a3}") # @unittest.skip("skip") def test_smart_merge(self): @@ -2142,15 +2012,13 @@ def test_smart_merge(self): copied during restore https://github.com/postgrespro/pg_probackup/issues/63 """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # create database @@ -2159,7 +2027,7 @@ def test_smart_merge(self): "CREATE DATABASE testdb") # take FULL backup - full_id = self.backup_node(backup_dir, 'node', node) + full_id = self.pb.backup_node('node', node) # drop database node.safe_psql( @@ -2167,42 +2035,39 @@ def test_smart_merge(self): "DROP DATABASE testdb") # take PAGE backup - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id = self.pb.backup_node('node', node, backup_type='page') # get delta between FULL and PAGE filelists - filelist_full = self.get_backup_filelist( - backup_dir, 'node', full_id) + filelist_full = self.get_backup_filelist(backup_dir, 'node', full_id) - filelist_page = self.get_backup_filelist( - backup_dir, 'node', page_id) + # merge PAGE backup + self.pb.merge_backup('node', page_id, + options=['--log-level-file=VERBOSE']) + + filelist_full_after_merge = self.get_backup_filelist(backup_dir, 'node', full_id) filelist_diff = self.get_backup_filelist_diff( - filelist_full, filelist_page) + filelist_full, filelist_full_after_merge) - # merge PAGE backup - self.merge_backup( - backup_dir, 'node', page_id, - options=['--log-level-file=VERBOSE']) + logfile_content = self.read_pb_log() - logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(logfile, 'r') as f: - logfile_content = f.read() + self.assertTrue(filelist_diff, "There should be deleted files") + for file in filelist_diff: + self.assertIn(file, logfile_content) + + @needs_gdb def test_idempotent_merge(self): """ """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # add database @@ -2211,8 +2076,7 @@ def test_idempotent_merge(self): 'CREATE DATABASE testdb') # take FULL backup - full_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) + full_id = self.pb.backup_node('node', node, options=['--stream']) # create database node.safe_psql( @@ -2220,47 +2084,50 @@ def test_idempotent_merge(self): 'create DATABASE testdb1') # take PAGE backup - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id = self.pb.backup_node('node', node, backup_type='delta') # create database node.safe_psql( 'postgres', 'create DATABASE testdb2') - page_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_2 = self.pb.backup_node('node', node, backup_type='delta') - gdb = self.merge_backup( - backup_dir, 'node', page_id_2, - gdb=True, options=['--log-level-console=verbose']) + gdb = self.pb.merge_backup('node', page_id_2, + gdb=True, options=['--log-level-console=log']) gdb.set_breakpoint('delete_backup_files') gdb.run_until_break() gdb.remove_all_breakpoints() - gdb.set_breakpoint('rename') + gdb.set_breakpoint("renameBackupToDir") gdb.continue_execution_until_break() - gdb.continue_execution_until_break(2) + gdb.set_breakpoint("write_backup") + gdb.continue_execution_until_break() + gdb.set_breakpoint("pgBackupFree") + gdb.continue_execution_until_break() + - gdb._execute('signal SIGKILL') + gdb.signal('SIGKILL') - show_backups = self.show_pb(backup_dir, "node") + self.expire_locks(backup_dir, 'node') + + show_backups = self.pb.show("node") self.assertEqual(len(show_backups), 1) self.assertEqual( - 'MERGED', self.show_pb(backup_dir, 'node')[0]['status']) + 'MERGED', self.pb.show('node')[0]['status']) self.assertEqual( - full_id, self.show_pb(backup_dir, 'node')[0]['id']) + full_id, self.pb.show('node')[0]['id']) - self.merge_backup(backup_dir, 'node', page_id_2) + self.pb.merge_backup('node', page_id_2) self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + 'OK', self.pb.show('node')[0]['status']) self.assertEqual( - page_id_2, self.show_pb(backup_dir, 'node')[0]['id']) + page_id_2, self.pb.show('node')[0]['id']) def test_merge_correct_inheritance(self): """ @@ -2268,15 +2135,13 @@ def test_merge_correct_inheritance(self): 'note' and 'expire-time' are correctly inherited during merge """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # add database @@ -2285,7 +2150,7 @@ def test_merge_correct_inheritance(self): 'CREATE DATABASE testdb') # take FULL backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) # create database node.safe_psql( @@ -2293,25 +2158,23 @@ def test_merge_correct_inheritance(self): 'create DATABASE testdb1') # take PAGE backup - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id = self.pb.backup_node('node', node, backup_type='page') - self.set_backup( - backup_dir, 'node', page_id, options=['--note=hello', '--ttl=20d']) + self.pb.set_backup('node', page_id, options=['--note=hello', '--ttl=20d']) - page_meta = self.show_pb(backup_dir, 'node', page_id) + page_meta = self.pb.show('node', page_id) - self.merge_backup(backup_dir, 'node', page_id) + self.pb.merge_backup('node', page_id) - print(self.show_pb(backup_dir, 'node', page_id)) + print(self.pb.show('node', page_id)) self.assertEqual( page_meta['note'], - self.show_pb(backup_dir, 'node', page_id)['note']) + self.pb.show('node', page_id)['note']) self.assertEqual( page_meta['expire-time'], - self.show_pb(backup_dir, 'node', page_id)['expire-time']) + self.pb.show('node', page_id)['expire-time']) def test_merge_correct_inheritance_1(self): """ @@ -2319,15 +2182,13 @@ def test_merge_correct_inheritance_1(self): 'note' and 'expire-time' are correctly inherited during merge """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # add database @@ -2336,8 +2197,7 @@ def test_merge_correct_inheritance_1(self): 'CREATE DATABASE testdb') # take FULL backup - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=['--stream', '--note=hello', '--ttl=20d']) # create database @@ -2346,18 +2206,17 @@ def test_merge_correct_inheritance_1(self): 'create DATABASE testdb1') # take PAGE backup - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id = self.pb.backup_node('node', node, backup_type='page') - self.merge_backup(backup_dir, 'node', page_id) + self.pb.merge_backup('node', page_id) self.assertNotIn( 'note', - self.show_pb(backup_dir, 'node', page_id)) + self.pb.show('node', page_id)) self.assertNotIn( 'expire-time', - self.show_pb(backup_dir, 'node', page_id)) + self.pb.show('node', page_id)) # @unittest.skip("skip") # @unittest.expectedFailure @@ -2373,15 +2232,12 @@ def test_multi_timeline_merge(self): P must have F as parent """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql("postgres", "create extension pageinspect") @@ -2396,16 +2252,15 @@ def test_multi_timeline_merge(self): "create extension amcheck_next") node.pgbench_init(scale=20) - full_id = self.backup_node(backup_dir, 'node', node) + full_id = self.pb.backup_node('node', node) pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() - self.backup_node(backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') node.cleanup() - self.restore_node( - backup_dir, 'node', node, backup_id=full_id, + self.pb.restore_node('node', node, backup_id=full_id, options=[ '--recovery-target=immediate', '--recovery-target-action=promote']) @@ -2418,8 +2273,7 @@ def test_multi_timeline_merge(self): # create timelines for i in range(2, 7): node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ '--recovery-target=latest', '--recovery-target-action=promote', @@ -2432,23 +2286,22 @@ def test_multi_timeline_merge(self): # create backup at 2, 4 and 6 timeline if i % 2 == 0: - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') - page_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + page_id = self.pb.backup_node('node', node, backup_type='page') pgdata = self.pgdata_content(node.data_dir) - self.merge_backup(backup_dir, 'node', page_id) + self.pb.merge_backup('node', page_id) result = node.table_checksum("pgbench_accounts") - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored) + self.pb.restore_node('node', node=node_restored) pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() result_new = node_restored.table_checksum("pgbench_accounts") @@ -2457,308 +2310,281 @@ def test_multi_timeline_merge(self): self.compare_pgdata(pgdata, pgdata_restored) - self.checkdb_node( - backup_dir, - 'node', + self.pb.checkdb_node( + use_backup_dir=True, + instance='node', options=[ '--amcheck', '-d', 'postgres', '-p', str(node.port)]) - self.checkdb_node( - backup_dir, - 'node', + self.pb.checkdb_node( + use_backup_dir=True, + instance='node', options=[ '--amcheck', '-d', 'postgres', '-p', str(node_restored.port)]) # @unittest.skip("skip") # @unittest.expectedFailure + @needs_gdb def test_merge_page_header_map_retry(self): """ page header map cannot be trusted when running retry """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.pgbench_init(scale=20) - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() - delta_id = self.backup_node( - backup_dir, 'node', node, + delta_id = self.pb.backup_node('node', node, backup_type='delta', options=['--stream']) pgdata = self.pgdata_content(node.data_dir) - gdb = self.merge_backup(backup_dir, 'node', delta_id, gdb=True) + gdb = self.pb.merge_backup('node', delta_id, gdb=True) # our goal here is to get full backup with merged data files, # but with old page header map gdb.set_breakpoint('cleanup_header_map') gdb.run_until_break() - gdb._execute('signal SIGKILL') + gdb.signal('SIGKILL') + + self.expire_locks(backup_dir, 'node') - self.merge_backup(backup_dir, 'node', delta_id) + self.pb.merge_backup('node', delta_id) node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") + @needs_gdb def test_missing_data_file(self): """ """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Add data node.pgbench_init(scale=1) # FULL backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # Change data pgbench = node.pgbench(options=['-T', '5', '-c', '1']) pgbench.wait() # DELTA backup - delta_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + delta_id = self.pb.backup_node('node', node, backup_type='delta') path = node.safe_psql( 'postgres', "select pg_relation_filepath('pgbench_accounts')").decode('utf-8').rstrip() - gdb = self.merge_backup( - backup_dir, "node", delta_id, + gdb = self.pb.merge_backup("node", delta_id, options=['--log-level-file=VERBOSE'], gdb=True) gdb.set_breakpoint('merge_files') gdb.run_until_break() # remove data file in incremental backup - file_to_remove = os.path.join( - backup_dir, 'backups', - 'node', delta_id, 'database', path) - - os.remove(file_to_remove) + self.remove_backup_file(backup_dir, 'node', delta_id, + f'database/{path}') gdb.continue_execution_until_error() - logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(logfile, 'r') as f: - logfile_content = f.read() + logfile_content = self.read_pb_log() - self.assertIn( - 'ERROR: Cannot open backup file "{0}": No such file or directory'.format(file_to_remove), - logfile_content) + if fs_backup_class.is_file_based: + self.assertRegex( + logfile_content, + 'ERROR: Open backup file: Cannot open file "[^"]*{0}": No such file or directory'.format(path)) + else: # suggesting S3 for minio, S3TestBackupDir + regex = 'ERROR: Open backup file: S3 error [0-9a-fA-F]+:[^:]+:/[^\\n].*{0}:NoSuchKey:404: No such file'.format( + path) + self.assertRegex( + logfile_content, + regex) # @unittest.skip("skip") + @needs_gdb def test_missing_non_data_file(self): """ """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # DELTA backup - delta_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + delta_id = self.pb.backup_node('node', node, backup_type='delta') - gdb = self.merge_backup( - backup_dir, "node", delta_id, + gdb = self.pb.merge_backup("node", delta_id, options=['--log-level-file=VERBOSE'], gdb=True) gdb.set_breakpoint('merge_files') gdb.run_until_break() # remove data file in incremental backup - file_to_remove = os.path.join( - backup_dir, 'backups', - 'node', delta_id, 'database', 'backup_label') - - os.remove(file_to_remove) + self.remove_backup_file(backup_dir, 'node', delta_id, + 'database/backup_label') gdb.continue_execution_until_error() - logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(logfile, 'r') as f: - logfile_content = f.read() + logfile_content = self.read_pb_log() - self.assertIn( - 'ERROR: File "{0}" is not found'.format(file_to_remove), - logfile_content) + self.assertRegex( + logfile_content, + 'ERROR: File "[^"]*backup_label" is not found') self.assertIn( 'ERROR: Backup files merging failed', logfile_content) self.assertEqual( - 'MERGING', self.show_pb(backup_dir, 'node')[0]['status']) + 'MERGING', self.pb.show('node')[0]['status']) self.assertEqual( - 'MERGING', self.show_pb(backup_dir, 'node')[1]['status']) + 'MERGING', self.pb.show('node')[1]['status']) # @unittest.skip("skip") + @needs_gdb def test_merge_remote_mode(self): """ """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL backup - full_id = self.backup_node(backup_dir, 'node', node) + full_id = self.pb.backup_node('node', node) # DELTA backup - delta_id = self.backup_node(backup_dir, 'node', node, backup_type='delta') + delta_id = self.pb.backup_node('node', node, backup_type='delta') - self.set_config(backup_dir, 'node', options=['--retention-window=1']) + self.pb.set_config('node', options=['--retention-window=1']) - backups = os.path.join(backup_dir, 'backups', 'node') - with open( - os.path.join( - backups, full_id, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=5))) + with self.modify_backup_control(backup_dir, 'node', full_id) as cf: + cf.data += "\nrecovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=5)) - gdb = self.backup_node( - backup_dir, "node", node, + gdb = self.pb.backup_node("node", node, options=['--log-level-file=VERBOSE', '--merge-expired'], gdb=True) gdb.set_breakpoint('merge_files') gdb.run_until_break() - logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') - - with open(logfile, "w+") as f: - f.truncate() + logfile_content_pre_len = len(self.read_pb_log()) gdb.continue_execution_until_exit() - logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(logfile, 'r') as f: - logfile_content = f.read() + logfile_content = self.read_pb_log()[logfile_content_pre_len:] self.assertNotIn( 'SSH', logfile_content) self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node')[0]['status']) + 'OK', self.pb.show('node')[0]['status']) def test_merge_pg_filenode_map(self): """ https://github.com/postgrespro/pg_probackup/issues/320 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), - initdb_params=['--data-checksums']) + node1 = self.pg_node.make_simple('node1') node1.cleanup() node.pgbench_init(scale=5) # FULL backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT, options=['-T', '10', '-c', '1']) - self.backup_node(backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') node.safe_psql( 'postgres', 'reindex index pg_type_oid_index') - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') + backup_id = self.pb.backup_node('node', node, backup_type='delta') - self.merge_backup(backup_dir, 'node', backup_id) + self.pb.merge_backup('node', backup_id) node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) node.slow_start() node.safe_psql( 'postgres', 'select 1') + @needs_gdb def test_unfinished_merge(self): """ Test when parent has unfinished merge with a different backup. """ - self._check_gdb_flag_or_skip_test() - cases = [('fail_merged', 'write_backup_filelist', ['MERGED', 'MERGING', 'OK']), - ('fail_merging', 'pgBackupWriteControl', ['MERGING', 'OK', 'OK'])] + cases = [('fail_merged', 'delete_backup_files', ['MERGED', 'MERGING', 'OK']), + ('fail_merging', 'create_directories_in_full', ['MERGING', 'MERGING', 'OK'])] for name, terminate_at, states in cases: node_name = 'node_' + name - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, name) - node = self.make_simple_node( + backup_dir = self.backup_dir + self.backup_dir.cleanup() + node = self.pg_node.make_simple( base_dir=os.path.join(self.module_name, self.fname, node_name), - set_replication=True, - initdb_params=['--data-checksums']) + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, node_name, node) - self.set_archiving(backup_dir, node_name, node) + self.pb.init() + self.pb.add_instance(node_name, node) + self.pb.set_archiving(node_name, node) node.slow_start() - full_id=self.backup_node(backup_dir, node_name, node, options=['--stream']) + full_id=self.pb.backup_node(node_name, node, options=['--stream']) - backup_id = self.backup_node(backup_dir, node_name, node, backup_type='delta') - second_backup_id = self.backup_node(backup_dir, node_name, node, backup_type='delta') + backup_id = self.pb.backup_node(node_name, node, backup_type='delta') + second_backup_id = self.pb.backup_node(node_name, node, backup_type='delta') - gdb = self.merge_backup(backup_dir, node_name, backup_id, gdb=True) + gdb = self.pb.merge_backup(node_name, backup_id, gdb=True) gdb.set_breakpoint(terminate_at) gdb.run_until_break() @@ -2766,14 +2592,898 @@ def test_unfinished_merge(self): gdb._execute('signal SIGINT') gdb.continue_execution_until_error() - print(self.show_pb(backup_dir, node_name, as_json=False, as_text=True)) + print(self.pb.show(node_name, as_json=False, as_text=True)) - for expected, real in zip(states, self.show_pb(backup_dir, node_name), strict=True): + backup_infos = self.pb.show(node_name) + self.assertEqual(len(backup_infos), len(states)) + for expected, real in zip(states, backup_infos): self.assertEqual(expected, real['status']) - with self.assertRaisesRegex(ProbackupException, + with self.assertRaisesRegex(Exception, f"Full backup {full_id} has unfinished merge with backup {backup_id}"): - self.merge_backup(backup_dir, node_name, second_backup_id, gdb=False) + self.pb.merge_backup(node_name, second_backup_id, gdb=False) + + @needs_gdb + def test_continue_failed_merge_with_corrupted_full_backup(self): + """ + Fail merge via gdb with corrupted FULL backup + """ + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + node.safe_psql( + "postgres", + "create table t_heap as select i as id," + " md5(i::text) as text, md5(i::text)::tsvector as tsvector" + " from generate_series(0,1000) i") + + old_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + # FULL backup + self.pb.backup_node('node', node) + + node.safe_psql( + "postgres", + "update t_heap set id = 100500") + + node.safe_psql( + "postgres", + "vacuum full t_heap") + + new_path = node.safe_psql( + "postgres", + "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() + + # DELTA BACKUP + backup_id_1 = self.pb.backup_node('node', node, backup_type='delta') + + full_id = self.pb.show("node")[0]["id"] + + # CORRUPT full backup + # read block from future + # block_size + backup_header = 8200 + file_content2 = self.read_backup_file(backup_dir, 'node', backup_id_1, + f'database/{new_path}')[:16400] + # write block from future + self.corrupt_backup_file(backup_dir, 'node', full_id, + f'database/{old_path}', + damage=(8200, file_content2[8200:16400])) + + # Try to continue failed MERGE + self.pb.merge_backup("node", backup_id_1, + expect_error=f"WARNING: Backup {full_id} data files are corrupted") + self.assertMessage(contains=f"ERROR: Backup {full_id} has status CORRUPT, merge is aborted") + + # Check number of backups + show_res = self.pb.show("node") + self.assertEqual(len(show_res), 2) + + @unittest.skipIf(not (have_alg('lz4') and have_alg('zstd')), + "pg_probackup is not compiled with lz4 or zstd support") + def test_merge_compressed_and_uncompressed(self): + """ + 1. Full compressed [zlib, 3] backup -> change data + 2. Delta uncompressed -> change data + 3. Page compressed [lz4, 2] -> change data + 5. Merge all backups in one + Restore and compare + """ + backup_dir = self.backup_dir + + # Initialize instance and backup directory + node = self.pg_node.make_simple('node', set_replication=True) + + self.pb.init() + self.pb.add_instance("node", node) + self.pb.set_archiving("node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=10) + + # Do compressed FULL backup + self.pb.backup_node("node", node, options=['--stream', + '--compress-level', '3', + '--compress-algorithm', 'zlib']) + show_backup = self.pb.show("node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed DELTA backup + self.pb.backup_node("node", node, + backup_type="delta") + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do compressed PAGE backup + self.pb.backup_node("node", node, backup_type="page", options=['--compress-level', '2', + '--compress-algorithm', 'lz4']) + + pgdata = self.pgdata_content(node.data_dir) + + # Check backups + show_backup = self.pb.show("node") + self.assertEqual(len(show_backup), 3) + self.assertEqual(show_backup[0]["status"], "OK") + self.assertEqual(show_backup[0]["backup-mode"], "FULL") + + self.assertEqual(show_backup[1]["status"], "OK") + self.assertEqual(show_backup[1]["backup-mode"], "DELTA") + + self.assertEqual(show_backup[2]["status"], "OK") + self.assertEqual(show_backup[2]["backup-mode"], "PAGE") + + page_id = show_backup[2]["id"] + + # Merge all backups + self.pb.merge_backup("node", page_id, options=['-j5']) + show_backups = self.pb.show("node") + + # Check number of backups and status + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.pb.restore_node('node', node=node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + node.cleanup() + + def test_merge_with_error_backup_in_the_middle(self): + """ + 1. Full uncompressed backup -> change data + 2. Delta with error (stop node) -> change data + 3. Page -> change data + 4. Delta -> change data + 5. Merge all backups in one + Restore and compare + """ + backup_dir = self.backup_dir + + # Initialize instance and backup directory + node = self.pg_node.make_simple('node', set_replication=True) + + self.pb.init() + self.pb.add_instance("node", node) + self.pb.set_archiving("node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=10) + + # Do uncompressed FULL backup + self.pb.backup_node("node", node) + show_backup = self.pb.show("node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + node.stop() + + # Try to create a DELTA backup with disabled archiving (expecting error) + self.pb.backup_node("node", node, backup_type="delta", expect_error=True) + + # Enable archiving + node.slow_start() + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do PAGE backup + self.pb.backup_node("node", node, backup_type="page") + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do DELTA backup + self.pb.backup_node("node", node, backup_type="delta") + + # Check backups + show_backup = self.pb.show("node") + self.assertEqual(len(show_backup), 4) + self.assertEqual(show_backup[0]["status"], "OK") + self.assertEqual(show_backup[0]["backup-mode"], "FULL") + + self.assertEqual(show_backup[1]["status"], "ERROR") + self.assertEqual(show_backup[1]["backup-mode"], "DELTA") + + self.assertEqual(show_backup[2]["status"], "OK") + self.assertEqual(show_backup[2]["backup-mode"], "PAGE") + + self.assertEqual(show_backup[3]["status"], "OK") + self.assertEqual(show_backup[3]["backup-mode"], "DELTA") + + delta_id = show_backup[3]["id"] + + # Merge all backups + self.pb.merge_backup("node", delta_id, options=['-j5']) + show_backup = self.pb.show("node") + + # Check number of backups and status + self.assertEqual(len(show_backup), 2) + self.assertEqual(show_backup[0]["status"], "ERROR") + self.assertEqual(show_backup[0]["backup-mode"], "DELTA") + + self.assertEqual(show_backup[1]["status"], "OK") + self.assertEqual(show_backup[1]["backup-mode"], "FULL") + + def test_merge_with_deleted_backup_in_the_middle(self): + """ + 1. Full uncompressed backup -> change data + 2. Delta uncompressed -> change data + 3. 1 Page uncompressed -> change data + 4. 2 Page uncompressed + 5. Remove 1 Page backup + 5. Merge all backups in one + Restore and compare + """ + backup_dir = self.backup_dir + + # Initialize instance and backup directory + node = self.pg_node.make_simple('node', set_replication=True) + + self.pb.init() + self.pb.add_instance("node", node) + self.pb.set_archiving("node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=10) + + # Do uncompressed FULL backup + self.pb.backup_node("node", node) + show_backup = self.pb.show("node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed DELTA backup + self.pb.backup_node("node", node, + backup_type="delta") + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed 1 PAGE backup + self.pb.backup_node("node", node, backup_type="page") + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed 2 PAGE backup + self.pb.backup_node("node", node, backup_type="page") + + pgdata = self.pgdata_content(node.data_dir) + + # Check backups + show_backup = self.pb.show("node") + self.assertEqual(len(show_backup), 4) + self.assertEqual(show_backup[0]["status"], "OK") + self.assertEqual(show_backup[0]["backup-mode"], "FULL") + + self.assertEqual(show_backup[1]["status"], "OK") + self.assertEqual(show_backup[1]["backup-mode"], "DELTA") + + self.assertEqual(show_backup[2]["status"], "OK") + self.assertEqual(show_backup[2]["backup-mode"], "PAGE") + + self.assertEqual(show_backup[3]["status"], "OK") + self.assertEqual(show_backup[3]["backup-mode"], "PAGE") + + first_page_id = show_backup[2]["id"] + second_page_id = show_backup[3]["id"] + + # Remove backup in the middle + self.remove_one_backup(backup_dir, "node", first_page_id) + + # Merge all backups + error = self.pb.merge_backup("node", second_page_id, options=['-j5'], expect_error=True) + self.assertMessage(error, contains=f"WARNING: Backup {first_page_id} is missing\n" + f"ERROR: Failed to find parent full backup for {second_page_id}") + + def test_merge_with_multiple_full_backups(self): + """ + 1. Full backup -> change data + 2. Delta -> change data + 3. Page -> change data + 4. Full -> change data + 5. Page -> change data + 6. Delta + 7. Merge all backups in one + Restore and compare + """ + # Initialize instance and backup directory + node = self.pg_node.make_simple('node', set_replication=True) + + self.pb.init() + self.pb.add_instance("node", node) + self.pb.set_archiving("node", node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=10) + + # Do uncompressed FULL backup + self.pb.backup_node("node", node) + show_backup = self.pb.show("node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed DELTA backup + self.pb.backup_node("node", node, + backup_type="delta") + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed PAGE backup + self.pb.backup_node("node", node, backup_type="page") + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed FULL backup + self.pb.backup_node("node", node) + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed PAGE backup + self.pb.backup_node("node", node, backup_type="page") + + # Change data + pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) + pgbench.wait() + + # Do uncompressed DELTA backup + self.pb.backup_node("node", node, + backup_type="delta") + + pgdata = self.pgdata_content(node.data_dir) + + # Check backups + show_backup = self.pb.show("node") + self.assertEqual(len(show_backup), 6) + self.assertEqual(show_backup[0]["status"], "OK") + self.assertEqual(show_backup[0]["backup-mode"], "FULL") + + self.assertEqual(show_backup[1]["status"], "OK") + self.assertEqual(show_backup[1]["backup-mode"], "DELTA") + + self.assertEqual(show_backup[2]["status"], "OK") + self.assertEqual(show_backup[2]["backup-mode"], "PAGE") + + self.assertEqual(show_backup[3]["status"], "OK") + self.assertEqual(show_backup[3]["backup-mode"], "FULL") + + self.assertEqual(show_backup[4]["status"], "OK") + self.assertEqual(show_backup[4]["backup-mode"], "PAGE") + + self.assertEqual(show_backup[5]["status"], "OK") + self.assertEqual(show_backup[5]["backup-mode"], "DELTA") + + last_backup_id = show_backup[5]["id"] + + # Merge all backups + self.pb.merge_backup("node", last_backup_id, options=['-j5']) + show_backups = self.pb.show("node") + + # Check number of backups and status + self.assertEqual(len(show_backups), 4) + self.assertEqual(show_backup[0]["status"], "OK") + self.assertEqual(show_backup[0]["backup-mode"], "FULL") + + self.assertEqual(show_backup[1]["status"], "OK") + self.assertEqual(show_backup[1]["backup-mode"], "DELTA") + + self.assertEqual(show_backup[2]["status"], "OK") + self.assertEqual(show_backup[2]["backup-mode"], "PAGE") + + self.assertEqual(show_backup[3]["status"], "OK") + self.assertEqual(show_backup[3]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.pb.restore_node('node', node=node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + # Clean after yourself + node.cleanup() + + def test_merge_with_logical_corruption(self): + """ + 1. Full backup -> change data + 2. Break logic (remove a referenced row from parent table) + 3. Mark foreign key constraint as NOT VALID + 4. Perform PAGE backup + 5. Merge all backups into one + 6. Restore and compare data directories + 7. Validate foreign key constraint to check that the logical corruption is also restored + """ + backup_dir = self.backup_dir + + # Initialize instance and backup directory + node = self.pg_node.make_simple('node', set_replication=True) + + self.pb.init() + self.pb.add_instance("node", node) + self.pb.set_archiving("node", node) + node.slow_start() + + # Create a table and fill + node.safe_psql("postgres", """ + CREATE TABLE parent_table ( + id serial PRIMARY KEY, + name varchar(100) NOT NULL + ); + + CREATE TABLE child_table ( + id serial PRIMARY KEY, + parent_id integer REFERENCES parent_table (id), + value varchar(100) NOT NULL + ); + + INSERT INTO parent_table (name) VALUES ('Parent 1'), ('Parent 2'), ('Parent 3'); + + INSERT INTO child_table (parent_id, value) VALUES (1, 'Child 1.1'), (1, 'Child 1.2'), (2, 'Child 2.1'), + (2, 'Child 2.2'), (3, 'Child 3.1'), (3, 'Child 3.2'); + """) + + # Do Full backup + self.pb.backup_node("node", node) + + # Break logic + node.safe_psql("postgres", """ + ALTER TABLE child_table DROP CONSTRAINT child_table_parent_id_fkey; + DELETE FROM parent_table WHERE id = 2; + ALTER TABLE child_table ADD CONSTRAINT child_table_parent_id_fkey FOREIGN KEY (parent_id) REFERENCES parent_table (id) NOT VALID; + """) + + # Do PAGE backup + self.pb.backup_node("node", node, backup_type="page") + + pgdata = self.pgdata_content(node.data_dir) + + # Check backups + show_backup = self.pb.show("node") + self.assertEqual(len(show_backup), 2) + self.assertEqual(show_backup[0]["status"], "OK") + self.assertEqual(show_backup[0]["backup-mode"], "FULL") + + self.assertEqual(show_backup[1]["status"], "OK") + self.assertEqual(show_backup[1]["backup-mode"], "PAGE") + + page_id = show_backup[1]["id"] + + # Merge all backups + self.pb.merge_backup("node", page_id, options=['-j5']) + show_backups = self.pb.show("node") + + # Check number of backups and status + self.assertEqual(len(show_backups), 1) + self.assertEqual(show_backups[0]["status"], "OK") + self.assertEqual(show_backups[0]["backup-mode"], "FULL") + + # Drop node and restore it + node.cleanup() + self.pb.restore_node('node', node=node) + + pgdata_restored = self.pgdata_content(node.data_dir) + self.compare_pgdata(pgdata, pgdata_restored) + + node.slow_start() + + # Check that logic of restored table also broken + error = node.safe_psql("postgres", """ + ALTER TABLE child_table VALIDATE CONSTRAINT child_table_parent_id_fkey; + """, expect_error=True) + self.assertMessage(error, contains='Key (parent_id)=(2) is not present in table "parent_table"') + + # Clean after yourself + node.cleanup() + + def test_two_merges_1(self): + """ + Test two merges for one full backup. + """ + node = self.pg_node.make_simple('node', + initdb_params=['--data-checksums']) + node.set_auto_conf({'shared_buffers': '2GB', + 'autovacuum': 'off'}) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + # Fill with data + node.pgbench_init(scale=100) + + self.pb.backup_node('node', node, backup_type="full") + + pgbench = node.pgbench(options=['-t', '20000', '-c', '3', '-n']) + pgbench.wait() + + self.pb.backup_node('node', node, backup_type="page") + + pgbench = node.pgbench(options=['-t', '20000', '-c', '3', '-n']) + pgbench.wait() + + prev_id = self.pb.backup_node('node', node, backup_type="page") + + pgbench = node.pgbench(options=['-t', '20000', '-c', '3', '-n']) + pgbench.wait() + + self.pb.backup_node('node', node, backup_type="page") + + pgbench = node.pgbench(options=['-t', '20000', '-c', '3', '-n']) + pgbench.wait() + + last_id = self.pb.backup_node('node', node, backup_type="page") + + checksum = node.pgbench_table_checksums() + node.stop() + + self.pb.merge_backup('node', prev_id) + + self.pb.merge_backup('node', last_id) + + restored = self.pg_node.make_empty('restored') + self.pb.restore_node('node', restored) + restored.set_auto_conf({'port': restored.port}) + restored.slow_start() + + restored_checksum = restored.pgbench_table_checksums() + self.assertEqual(checksum, restored_checksum, + "data are not equal") + + def test_two_merges_2(self): + """ + Test two merges for one full backup with data in tablespace. + """ + node = self.pg_node.make_simple('node', initdb_params=['--data-checksums']) + node.set_auto_conf({'shared_buffers': '2GB', + 'autovacuum': 'off'}) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + self.create_tblspace_in_node(node, 'somedata') + + # Fill with data + node.pgbench_init(scale=100, options=['--tablespace=somedata']) + + self.pb.backup_node('node', node, backup_type="full") + + pgbench = node.pgbench(options=['-t', '20000', '-c', '3', '-n']) + pgbench.wait() + + self.pb.backup_node('node', node, backup_type="page") + + pgbench = node.pgbench(options=['-t', '20000', '-c', '3', '-n']) + pgbench.wait() + + prev_id = self.pb.backup_node('node', node, backup_type="page") + + pgbench = node.pgbench(options=['-t', '20000', '-c', '3', '-n']) + pgbench.wait() + + self.pb.backup_node('node', node, backup_type="page") + + pgbench = node.pgbench(options=['-t', '20000', '-c', '3', '-n']) + pgbench.wait() + + last_id = self.pb.backup_node('node', node, backup_type="page") + + checksum = node.pgbench_table_checksums() + node.stop() + + self.pb.merge_backup('node', prev_id) + + self.pb.merge_backup('node', last_id) + + restored = self.pg_node.make_empty('restored') + node_ts = os.path.join(node.base_dir, 'somedata') + restored_ts = os.path.join(restored.base_dir, 'somedata') + os.mkdir(restored_ts) + self.pb.restore_node('node', restored, + options=['-T', f"{node_ts}={restored_ts}"]) + restored.set_auto_conf({'port': restored.port}) + restored.slow_start() + + restored_checksum = restored.pgbench_table_checksums() + self.assertEqual(checksum, restored_checksum, + "data are not equal") + + @needs_gdb + def test_backup_while_merge(self): + """ + Test backup is not possible while closest full backup is in merge. + (PBCKP-626_) + TODO: fix it if possible. + """ + + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + node.pgbench_init(scale=1) + + self.pb.backup_node('node', node, backup_type="full") + + pgbench = node.pgbench(options=['-t', '100', '-c', '3', '-n']) + pgbench.wait() + + first_page = self.pb.backup_node('node', node, backup_type="page") + + pgbench = node.pgbench(options=['-t', '100', '-c', '3', '-n']) + pgbench.wait() + + self.pb.backup_node('node', node, backup_type="page") + + pgbench = node.pgbench(options=['-t', '100', '-c', '3', '-n']) + pgbench.wait() + + gdb = self.pb.merge_backup('node', first_page, gdb=True) + gdb.set_breakpoint('create_directories_in_full') + gdb.run_until_break() + + self.pb.backup_node('node', node, backup_type="page", + expect_error="just because it goes this way yet.") + + gdb.kill() + +############################################################################ +# dry-run +############################################################################ + + def test_basic_dry_run_merge_full_2page(self): + """ + 1. Full backup -> fill data + 2. First Page backup -> fill data + 3. Second Page backup + 4. Merge 2 "Page" backups with dry-run + Compare instance directory before and after merge + """ + # Initialize instance and backup directory + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance("node", node) + self.pb.set_archiving("node", node) + node.slow_start() + + # Do full backup + self.pb.backup_node("node", node, options=['--compress']) + show_backup = self.pb.show("node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Fill with data + with node.connect() as conn: + conn.execute("create table test (id int);") + conn.execute( + "insert into test select i from generate_series(1,10) s(i);") + conn.commit() + + # Do first page backup + self.pb.backup_node("node", node, backup_type="page", options=['--compress']) + show_backup = self.pb.show("node")[1] + + # sanity check + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Fill with data + with node.connect() as conn: + conn.execute( + "insert into test select i from generate_series(1,10) s(i);") + count1 = conn.execute("select count(*) from test") + conn.commit() + + # take PAGE backup with external directory pointing to a file + external_dir = self.get_tblspace_path(node, 'somedirectory') + os.mkdir(external_dir) + + # Do second page backup + self.pb.backup_node("node", node, + backup_type="page", options=['--compress', '--external-dirs={0}'.format(external_dir)]) + show_backup = self.pb.show("node")[2] + page_id = show_backup["id"] + + # sanity check + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Check data changes absence + instance_before = self.pgdata_content(os.path.join(self.backup_dir, 'backups/node')) + + # Merge all backups + output = self.pb.merge_backup("node", page_id, + options=['--dry-run']) + self.assertNotIn("WARNING", output) + instance_after = self.pgdata_content(os.path.join(self.backup_dir, 'backups/node')) + self.compare_pgdata(instance_before, instance_after) + + show_backups = self.pb.show("node") + node.cleanup() + + @unittest.skipIf(not fs_backup_class.is_file_based, "AccessPath check is always true on s3") + def test_basic_dry_run_check_merge_with_access(self): + """ + Access check suite if disk mounted as read_only + """ + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance("node", node) + self.pb.set_archiving("node", node) + node.slow_start() + + # Do full backup + self.pb.backup_node("node", node, options=['--compress']) + show_backup = self.pb.show("node")[0] + + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "FULL") + + # Fill with data + with node.connect() as conn: + conn.execute("create table test (id int);") + conn.execute( + "insert into test select i from generate_series(1,10) s(i);") + conn.commit() + + # Do first page backup + self.pb.backup_node("node", node, backup_type="page", options=['--compress']) + show_backup = self.pb.show("node")[1] + + # sanity check + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Fill with data + with node.connect() as conn: + conn.execute( + "insert into test select i from generate_series(1,10) s(i);") + count1 = conn.execute("select count(*) from test") + conn.commit() + + # take PAGE backup with external directory pointing to a file + external_dir = self.get_tblspace_path(node, 'somedirectory') + os.mkdir(external_dir) + + # Do second page backup + self.pb.backup_node("node", node, + backup_type="page", options=['--compress', '--external-dirs={0}'.format(external_dir)]) + show_backup = self.pb.show("node")[2] + page_id = show_backup["id"] + + # sanity check + self.assertEqual(show_backup["status"], "OK") + self.assertEqual(show_backup["backup-mode"], "PAGE") + + # Access check + dir_path = os.path.join(self.backup_dir, 'backups/node') + dir_mode = os.stat(dir_path).st_mode + os.chmod(dir_path, 0o500) + + error_message = self.pb.merge_backup("node", page_id, + options=['--dry-run'], expect_error ='because of changed permissions') + try: + self.assertMessage(error_message, contains='ERROR: Check permissions') + finally: + # Cleanup + os.chmod(dir_path, dir_mode) + + dir_path = os.path.join(self.backup_dir, 'backups/node/', page_id) + dir_mode = os.stat(dir_path).st_mode + os.chmod(dir_path, 0o500) + + error_message = self.pb.merge_backup("node", page_id, + options=['--dry-run'], expect_error ='because of changed permissions') + try: + self.assertMessage(error_message, contains='ERROR: Check permissions') + finally: + # Cleanup + os.chmod(dir_path, dir_mode) + + + + +class BenchMerge(ProbackupTest): + + def setUp(self): + super().setUp() + + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() -# 1. Need new test with corrupted FULL backup -# 2. different compression levels + node.execute(""" + do $$ + declare + i int; + begin + for i in 0..2000 loop + execute 'create table x'||i||'(i int primary key, j text);'; + commit; + end loop; + end; + $$; + """) + + start = int(time.time()) + self.pb.backup_node('node', node, + options=['--start-time', str(start)]) + for i in range(50): + start += 1 + self.pb.backup_node('node', node, backup_type='page', + options=['--start-time', str(start)]) + start += 1 + self.backup_id = self.pb.backup_node('node', node, backup_type='page', + options=['--start-time', str(start)]) + + def test_bench_merge_long_chain(self): + """ + test long incremental chain with a lot of tables + """ + + start = time.time() + self.pb.merge_backup('node', self.backup_id) + stop = time.time() + print(f"LASTS FOR {stop - start}") diff --git a/tests/option_test.py b/tests/option_test.py index 66cc13746..89c5c52e0 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -1,235 +1,211 @@ -import unittest import os -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest, fs_backup_class import locale -class OptionTest(ProbackupTest, unittest.TestCase): + +class OptionTest(ProbackupTest): # @unittest.skip("skip") # @unittest.expectedFailure def test_help_1(self): """help options""" - with open(os.path.join(self.dir_path, "expected/option_help.out"), "rb") as help_out: + with open(os.path.join(self.tests_source_path, "expected/option_help.out"), "rb") as help_out: self.assertEqual( - self.run_pb(["--help"]), + self.pb.run(["--help"], use_backup_dir=None), help_out.read().decode("utf-8") ) - # @unittest.skip("skip") - def test_version_2(self): - """help options""" - with open(os.path.join(self.dir_path, "expected/option_version.out"), "rb") as version_out: - self.assertIn( - version_out.read().decode("utf-8").strip(), - self.run_pb(["--version"]) - ) - # @unittest.skip("skip") def test_without_backup_path_3(self): """backup command failure without backup mode option""" - try: - self.run_pb(["backup", "-b", "full"]) - self.assertEqual(1, 0, "Expecting Error because '-B' parameter is not specified.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: No backup catalog path specified.\n' + \ - 'Please specify it either using environment variable BACKUP_PATH or\n' + \ - 'command line option --backup-path (-B)', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.pb.run(["backup", "-b", "full"], + expect_error="because '-B' parameter is not specified", use_backup_dir=None) + self.assertMessage(contains="No backup catalog path specified.\n" + "Please specify it either using environment variable BACKUP_DIR or\n" + "command line option --backup-path (-B)") - # @unittest.skip("skip") def test_options_4(self): """check options test""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node')) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) # backup command failure without instance option - try: - self.run_pb(["backup", "-B", backup_dir, "-D", node.data_dir, "-b", "full"]) - self.assertEqual(1, 0, "Expecting Error because 'instance' parameter is not specified.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Required parameter not specified: --instance', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.pb.run(["backup", "-D", node.data_dir, "-b", "full"], + expect_error="because 'instance' parameter is not specified") + self.assertMessage(contains='ERROR: Required parameter not specified: --instance') # backup command failure without backup mode option - try: - self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-D", node.data_dir]) - self.assertEqual(1, 0, "Expecting Error because '-b' parameter is not specified.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: No backup mode specified.\nPlease specify it either using environment variable BACKUP_MODE or\ncommand line option --backup-mode (-b)', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.pb.run(["backup", "--instance=node", "-D", node.data_dir], + expect_error="Expecting Error because '-b' parameter is not specified") + self.assertMessage(contains="Please specify it either using environment variable BACKUP_MODE or\n" + "command line option --backup-mode (-b)") # backup command failure with invalid backup mode option - try: - self.run_pb(["backup", "-B", backup_dir, "--instance=node", "-b", "bad"]) - self.assertEqual(1, 0, "Expecting Error because backup-mode parameter is invalid.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Invalid backup-mode "bad"', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.pb.run(["backup", "--instance=node", "-b", "bad"], + expect_error="because backup-mode parameter is invalid") + self.assertMessage(contains='ERROR: Invalid backup-mode "bad"') # delete failure without delete options - try: - self.run_pb(["delete", "-B", backup_dir, "--instance=node"]) - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because delete options are omitted.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: You must specify at least one of the delete options: ' - '--delete-expired |--delete-wal |--merge-expired |--status |(-i, --backup-id)', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - + self.pb.run(["delete", "--instance=node"], + expect_error="because delete options are omitted") + self.assertMessage(contains='ERROR: You must specify at least one of the delete options: ' + '--delete-expired |--delete-wal |--merge-expired |--status |(-i, --backup-id)') # delete failure without ID - try: - self.run_pb(["delete", "-B", backup_dir, "--instance=node", '-i']) - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because backup ID is omitted.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "Option '-i' requires an argument", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.pb.run(["delete", "--instance=node", '-i'], + expect_error="because backup ID is omitted") + self.assertMessage(contains="Option '-i' requires an argument") + + #init command with bad option + self.pb.run(["init","--bad"], + expect_error="because unknown option") + self.assertMessage(contains="Unknown option '--bad'") + + # run with bad short option + self.pb.run(["init","-aB"], + expect_error="because unknown option") + self.assertMessage(contains="Unknown option '-aB'") # @unittest.skip("skip") def test_options_5(self): """check options test""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node')) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') - output = self.init_pb(backup_dir) - self.assertIn(f"INFO: Backup catalog '{backup_dir}' successfully initialized", output) + output = self.pb.init() + self.assertIn( + f"INFO: Backup catalog '{backup_dir}' successfully initialized", + output) - self.add_instance(backup_dir, 'node', node) + self.pb.add_instance('node', node) node.slow_start() # syntax error in pg_probackup.conf - conf_file = os.path.join(backup_dir, "backups", "node", "pg_probackup.conf") - with open(conf_file, "a") as conf: - conf.write(" = INFINITE\n") - try: - self.backup_node(backup_dir, 'node', node) - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because of garbage in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Syntax error in " = INFINITE', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - self.clean_pb(backup_dir) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + with self.modify_backup_config(backup_dir, 'node') as cf: + cf.data += " = INFINITE\n" + + self.pb.backup_node('node', node, + expect_error="because of garbage in pg_probackup.conf") + self.assertMessage(regex=r'ERROR: Syntax error .* INFINITE') + + backup_dir.cleanup() + self.pb.init() + self.pb.add_instance('node', node) # invalid value in pg_probackup.conf - with open(conf_file, "a") as conf: - conf.write("BACKUP_MODE=\n") - - try: - self.backup_node(backup_dir, 'node', node, backup_type=None), - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because of invalid backup-mode in pg_probackup.conf.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Invalid option "BACKUP_MODE" in file', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - self.clean_pb(backup_dir) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + with self.modify_backup_config(backup_dir, 'node') as cf: + cf.data += "BACKUP_MODE=\n" + + self.pb.backup_node('node', node, backup_type=None, + expect_error="because of invalid backup-mode in pg_probackup.conf") + self.assertMessage(contains='ERROR: Invalid option "BACKUP_MODE" in file') + + backup_dir.cleanup() + self.pb.init() + self.pb.add_instance('node', node) # Command line parameters should override file values - with open(conf_file, "a") as conf: - conf.write("retention-redundancy=1\n") + with self.modify_backup_config(backup_dir, 'node') as cf: + cf.data += "retention-redundancy=1\n" - self.assertEqual(self.show_config(backup_dir, 'node')['retention-redundancy'], '1') + self.assertEqual(self.pb.show_config('node')['retention-redundancy'], '1') # User cannot send --system-identifier parameter via command line - try: - self.backup_node(backup_dir, 'node', node, options=["--system-identifier", "123"]), - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because option system-identifier cannot be specified in command line.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Option system-identifier cannot be specified in command line', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + self.pb.backup_node('node', node, options=["--system-identifier", "123"], + expect_error="because option system-identifier cannot be specified in command line") + self.assertMessage(contains='ERROR: Option system-identifier cannot be specified in command line') # invalid value in pg_probackup.conf - with open(conf_file, "a") as conf: - conf.write("SMOOTH_CHECKPOINT=FOO\n") - - try: - self.backup_node(backup_dir, 'node', node) - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, "Expecting Error because option -C should be boolean.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Invalid option "SMOOTH_CHECKPOINT" in file', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - self.clean_pb(backup_dir) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + with self.modify_backup_config(backup_dir, 'node') as cf: + cf.data += "SMOOTH_CHECKPOINT=FOO\n" + + self.pb.backup_node('node', node, + expect_error="because option smooth-checkpoint could be specified in command-line only") + self.assertMessage(contains='ERROR: Invalid option "SMOOTH_CHECKPOINT" in file') + + backup_dir.cleanup() + self.pb.init() + self.pb.add_instance('node', node) # invalid option in pg_probackup.conf - with open(conf_file, "a") as conf: - conf.write("TIMELINEID=1\n") - - try: - self.backup_node(backup_dir, 'node', node) - # we should die here because exception is what we expect to happen - self.assertEqual(1, 0, 'Expecting Error because of invalid option "TIMELINEID".\n Output: {0} \n CMD: {1}'.format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Invalid option "TIMELINEID" in file', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) + with self.modify_backup_config(backup_dir, 'node') as cf: + cf.data += "TIMELINEID=1\n" + + self.pb.backup_node('node', node, + expect_error='because of invalid option "TIMELINEID"') + self.assertMessage(contains='ERROR: Invalid option "TIMELINEID" in file') # @unittest.skip("skip") def test_help_6(self): """help options""" if ProbackupTest.enable_nls: - if check_locale('ru_RU.utf-8'): - self.test_env['LC_ALL'] = 'ru_RU.utf-8' - with open(os.path.join(self.dir_path, "expected/option_help_ru.out"), "rb") as help_out: + if check_locale('ru_RU.UTF-8'): + env = self.test_env.copy() + env['LC_CTYPE'] = 'ru_RU.UTF-8' + env['LC_MESSAGES'] = 'ru_RU.UTF-8' + env['LANGUAGE'] = 'ru_RU' + with open(os.path.join(self.tests_source_path, "expected/option_help_ru.out"), "rb") as help_out: self.assertEqual( - self.run_pb(["--help"]), + self.pb.run(["--help"], env=env, use_backup_dir=None), help_out.read().decode("utf-8") ) else: self.skipTest( - "Locale ru_RU.utf-8 doesn't work. You need install ru_RU.utf-8 locale for this test") + "The ru_RU.UTF-8 locale is not available. You may need to install it to run this test.") else: self.skipTest( 'You need configure PostgreSQL with --enabled-nls option for this test') + def test_skip_if_exists(self): + """check options --skip-if-exists""" + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') + + self.pb.init() + self.assertMessage(contains=f"INFO: Backup catalog '{backup_dir}' successfully initialized") + if fs_backup_class.is_file_based: + self.pb.init(expect_error=True) + self.assertMessage(contains=f"ERROR: Backup catalog '{backup_dir}' already exists and is not empty") + + self.pb.init(options=['--skip-if-exists']) + self.assertMessage(contains=f"WARNING: Backup catalog '{backup_dir}' already exists and is not empty, skipping") + self.assertMessage(has_no="successfully initialized") + + self.pb.add_instance('node', node) + self.assertMessage(contains="INFO: Instance 'node' successfully initialized") + self.pb.add_instance('node', node, expect_error=True) + self.assertMessage(contains="ERROR: Instance 'node' backup directory already exists") + + self.pb.add_instance('node', node, options=['--skip-if-exists']) + self.assertMessage(contains=f"WARNING: Instance 'node' backup directory already exists: '{backup_dir}/backups/node'. Skipping") + self.assertMessage(has_no="successfully initialized") + + # @unittest.skip("skip") + def test_options_no_scale_units(self): + """check --no-scale-units option""" + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') + self.pb.init() + self.pb.add_instance('node', node) + + # check that --no-scale-units option works correctly + self.pb.run(["show-config", "-D", node.data_dir, "--instance", "node"]) + self.assertMessage(contains="archive-timeout = 5min") + self.pb.run(["show-config", "-D", node.data_dir, "--instance", "node", "--no-scale-units"]) + self.assertMessage(has_no="archive-timeout = 300s") + self.assertMessage(contains="archive-timeout = 300") + # check that we have now quotes ("") in json output + self.pb.run(["show-config", "--instance", "node", "--no-scale-units", "--format=json"]) + self.assertMessage(contains='"archive-timeout": 300,') + self.assertMessage(contains='"retention-redundancy": 0,') + self.assertMessage(has_no='"archive-timeout": "300",') + + + def check_locale(locale_name): ret=True diff --git a/tests/page_test.py b/tests/page_test.py index 99f3ce992..10959bd8b 100644 --- a/tests/page_test.py +++ b/tests/page_test.py @@ -1,14 +1,9 @@ -import os import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest from testgres import QueryException -from datetime import datetime, timedelta -import subprocess -import gzip -import shutil import time -class PageTest(ProbackupTest, unittest.TestCase): +class PageTest(ProbackupTest): # @unittest.skip("skip") def test_basic_page_vacuum_truncate(self): @@ -18,20 +13,16 @@ def test_basic_page_vacuum_truncate(self): take page backup, take second page backup, restore last page backup and check data correctness """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '300s'}) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node_restored.cleanup() node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -48,7 +39,7 @@ def test_basic_page_vacuum_truncate(self): "postgres", "vacuum t_heap") - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # TODO: make it dynamic node.safe_psql( @@ -58,11 +49,9 @@ def test_basic_page_vacuum_truncate(self): "postgres", "vacuum t_heap") - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -70,8 +59,7 @@ def test_basic_page_vacuum_truncate(self): old_tablespace = self.get_tblspace_path(node, 'somedata') new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=[ "-j", "4", "-T", "{0}={1}".format(old_tablespace, new_tablespace)]) @@ -81,7 +69,7 @@ def test_basic_page_vacuum_truncate(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() # Logical comparison @@ -98,15 +86,12 @@ def test_page_vacuum_truncate_1(self): take page backup, insert some data, take second page backup and check data correctness """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -121,7 +106,7 @@ def test_page_vacuum_truncate_1(self): "postgres", "vacuum t_heap") - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.safe_psql( "postgres", @@ -131,8 +116,7 @@ def test_page_vacuum_truncate_1(self): "postgres", "vacuum t_heap") - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') node.safe_psql( "postgres", @@ -141,22 +125,20 @@ def test_page_vacuum_truncate_1(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,1) i") - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') pgdata = self.pgdata_content(node.data_dir) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored) + self.pb.restore_node('node', node=node_restored) # Physical comparison pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() # @unittest.skip("skip") @@ -166,18 +148,15 @@ def test_page_stream(self): restore them and check data correctness """ self.maxDiff = None - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s'} ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL BACKUP @@ -188,8 +167,7 @@ def test_page_stream(self): "from generate_series(0,100) i") full_result = node.table_checksum("t_heap") - full_backup_id = self.backup_node( - backup_dir, 'node', node, + full_backup_id = self.pb.backup_node('node', node, backup_type='full', options=['--stream']) # PAGE BACKUP @@ -199,8 +177,7 @@ def test_page_stream(self): "md5(i::text)::tsvector as tsvector " "from generate_series(100,200) i") page_result = node.table_checksum("t_heap") - page_backup_id = self.backup_node( - backup_dir, 'node', node, + page_backup_id = self.pb.backup_node('node', node, backup_type='page', options=['--stream', '-j', '4']) if self.paranoia: @@ -210,13 +187,9 @@ def test_page_stream(self): node.cleanup() # Check full backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(full_backup_id), - self.restore_node( - backup_dir, 'node', node, - backup_id=full_backup_id, options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(self.output), self.cmd)) + restore_result = self.pb.restore_node('node', node, + backup_id=full_backup_id, options=["-j", "4"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(full_backup_id)) node.slow_start() full_result_new = node.table_checksum("t_heap") @@ -224,13 +197,9 @@ def test_page_stream(self): node.cleanup() # Check page backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(page_backup_id), - self.restore_node( - backup_dir, 'node', node, - backup_id=page_backup_id, options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(self.output), self.cmd)) + restore_result = self.pb.restore_node('node', node, + backup_id=page_backup_id, options=["-j", "4"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(page_backup_id)) # GET RESTORED PGDATA AND COMPARE if self.paranoia: @@ -249,18 +218,15 @@ def test_page_archive(self): restore them and check data correctness """ self.maxDiff = None - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s'} ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL BACKUP @@ -269,8 +235,7 @@ def test_page_archive(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") full_result = node.table_checksum("t_heap") - full_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='full') + full_backup_id = self.pb.backup_node('node', node, backup_type='full') # PAGE BACKUP node.safe_psql( @@ -279,8 +244,7 @@ def test_page_archive(self): "md5(i::text) as text, md5(i::text)::tsvector as tsvector " "from generate_series(100, 200) i") page_result = node.table_checksum("t_heap") - page_backup_id = self.backup_node( - backup_dir, 'node', node, + page_backup_id = self.pb.backup_node('node', node, backup_type='page', options=["-j", "4"]) if self.paranoia: @@ -290,18 +254,13 @@ def test_page_archive(self): node.cleanup() # Restore and check full backup - self.assertIn("INFO: Restore of backup {0} completed.".format( - full_backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, backup_id=full_backup_id, options=[ "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - + "--recovery-target-action=promote"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(full_backup_id)) node.slow_start() full_result_new = node.table_checksum("t_heap") @@ -309,19 +268,15 @@ def test_page_archive(self): node.cleanup() # Restore and check page backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(page_backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, backup_id=page_backup_id, options=[ "-j", "4", "--immediate", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + "--recovery-target-action=promote"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(page_backup_id)) - # GET RESTORED PGDATA AND COMPARE + # GET RESTORED PGDATA AND COMPARE if self.paranoia: pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -338,20 +293,17 @@ def test_page_multiple_segments(self): Make node, create table with multiple segments, write some data to it, check page and data correctness """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'fsync': 'off', 'shared_buffers': '1GB', 'maintenance_work_mem': '1GB', 'full_page_writes': 'off'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -359,7 +311,7 @@ def test_page_multiple_segments(self): # CREATE TABLE node.pgbench_init(scale=100, options=['--tablespace=somedata']) # FULL BACKUP - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # PGBENCH STUFF pgbench = node.pgbench(options=['-T', '50', '-c', '1', '--no-vacuum']) @@ -368,21 +320,19 @@ def test_page_multiple_segments(self): # GET LOGICAL CONTENT FROM NODE result = node.table_checksum("pgbench_accounts") # PAGE BACKUP - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') # GET PHYSICAL CONTENT FROM NODE pgdata = self.pgdata_content(node.data_dir) # RESTORE NODE - restored_node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) + restored_node = self.pg_node.make_simple('restored_node') restored_node.cleanup() tblspc_path = self.get_tblspace_path(node, 'somedata') tblspc_path_new = self.get_tblspace_path( restored_node, 'somedata_restored') - self.restore_node( - backup_dir, 'node', restored_node, + self.pb.restore_node('node', restored_node, options=[ "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) @@ -391,7 +341,7 @@ def test_page_multiple_segments(self): pgdata_restored = self.pgdata_content(restored_node.data_dir) # START RESTORED NODE - self.set_auto_conf(restored_node, {'port': restored_node.port}) + restored_node.set_auto_conf({'port': restored_node.port}) restored_node.slow_start() result_new = restored_node.table_checksum("pgbench_accounts") @@ -409,23 +359,21 @@ def test_page_delete(self): delete everything from table, vacuum table, take page backup, restore page backup, compare . """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', + set_replication=True, pg_options={ 'checkpoint_timeout': '30s', } ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() self.create_tblspace_in_node(node, 'somedata') # FULL backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.safe_psql( "postgres", "create table t_heap tablespace somedata as select i as id," @@ -441,18 +389,15 @@ def test_page_delete(self): "vacuum t_heap") # PAGE BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=[ "-j", "4", "-T", "{0}={1}".format( @@ -467,7 +412,7 @@ def test_page_delete(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() # @unittest.skip("skip") @@ -477,19 +422,16 @@ def test_page_delete_1(self): delete everything from table, vacuum table, take page backup, restore page backup, compare . """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', } ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -501,7 +443,7 @@ def test_page_delete_1(self): " from generate_series(0,100) i" ) # FULL backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.safe_psql( "postgres", @@ -514,19 +456,16 @@ def test_page_delete_1(self): ) # PAGE BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored') + node_restored = self.pg_node.make_simple('node_restored' ) node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=[ "-j", "4", "-T", "{0}={1}".format( @@ -541,36 +480,31 @@ def test_page_delete_1(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() def test_parallel_pagemap(self): """ Test for parallel WAL segments reading, during which pagemap is built """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - # Initialize instance and backup directory - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', pg_options={ "hot_standby": "on" } ) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), + node_restored = self.pg_node.make_simple('node_restored', ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node_restored.cleanup() - self.set_archiving(backup_dir, 'node', node) + self.pb.set_archiving('node', node) node.slow_start() # Do full backup - self.backup_node(backup_dir, 'node', node) - show_backup = self.show_pb(backup_dir, 'node')[0] + self.pb.backup_node('node', node) + show_backup = self.pb.show('node')[0] self.assertEqual(show_backup['status'], "OK") self.assertEqual(show_backup['backup-mode'], "FULL") @@ -586,9 +520,8 @@ def test_parallel_pagemap(self): count1 = conn.execute("select count(*) from test") # ... and do page backup with parallel pagemap - self.backup_node( - backup_dir, 'node', node, backup_type="page", options=["-j", "4"]) - show_backup = self.show_pb(backup_dir, 'node')[1] + self.pb.backup_node('node', node, backup_type="page", options=["-j", "4"]) + show_backup = self.pb.show('node')[1] self.assertEqual(show_backup['status'], "OK") self.assertEqual(show_backup['backup-mode'], "PAGE") @@ -597,14 +530,14 @@ def test_parallel_pagemap(self): pgdata = self.pgdata_content(node.data_dir) # Restore it - self.restore_node(backup_dir, 'node', node_restored) + self.pb.restore_node('node', node=node_restored) # Physical comparison if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() # Check restored node @@ -620,23 +553,19 @@ def test_parallel_pagemap_1(self): """ Test for parallel WAL segments reading, during which pagemap is built """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - # Initialize instance and backup directory - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', pg_options={} ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Do full backup - self.backup_node(backup_dir, 'node', node) - show_backup = self.show_pb(backup_dir, 'node')[0] + self.pb.backup_node('node', node) + show_backup = self.pb.show('node')[0] self.assertEqual(show_backup['status'], "OK") self.assertEqual(show_backup['backup-mode'], "FULL") @@ -645,22 +574,20 @@ def test_parallel_pagemap_1(self): node.pgbench_init(scale=10) # do page backup in single thread - page_id = self.backup_node( - backup_dir, 'node', node, backup_type="page") + page_id = self.pb.backup_node('node', node, backup_type="page") - self.delete_pb(backup_dir, 'node', page_id) + self.pb.delete('node', page_id) # ... and do page backup with parallel pagemap - self.backup_node( - backup_dir, 'node', node, backup_type="page", options=["-j", "4"]) - show_backup = self.show_pb(backup_dir, 'node')[1] + self.pb.backup_node('node', node, backup_type="page", options=["-j", "4"]) + show_backup = self.pb.show('node')[1] self.assertEqual(show_backup['status'], "OK") self.assertEqual(show_backup['backup-mode'], "PAGE") # Drop node and restore it node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) node.slow_start() # Clean after yourself @@ -675,73 +602,45 @@ def test_page_backup_with_lost_wal_segment(self): run page backup, expecting error because of missing wal segment make sure that backup status is 'ERROR' """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # make some wals node.pgbench_init(scale=3) # delete last wal segment - wals_dir = os.path.join(backup_dir, 'wal', 'node') - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( - wals_dir, f)) and not f.endswith('.backup') and not f.endswith('.part')] - wals = map(str, wals) - file = os.path.join(wals_dir, max(wals)) - os.remove(file) - if self.archive_compress: - file = file[:-3] + walfile = '000000010000000000000004'+self.compress_suffix + self.wait_instance_wal_exists(backup_dir, 'node', walfile) + self.remove_instance_wal(backup_dir, 'node', walfile) # Single-thread PAGE backup - try: - self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.assertEqual( - 1, 0, - "Expecting Error because of wal segment disappearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'Could not read WAL record at' in e.message and - 'is absent' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, backup_type='page', + expect_error="because of wal segment disappearance") + self.assertMessage(contains='Could not read WAL record at') + self.assertMessage(contains='is absent') self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node')[1]['status'], + self.pb.show('node')[1]['status'], 'Backup {0} should have STATUS "ERROR"') # Multi-thread PAGE backup - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='page', - options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of wal segment disappearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'Could not read WAL record at' in e.message and - 'is absent' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, backup_type='page', + options=["-j", "4"], + expect_error="because of wal segment disappearance") + self.assertMessage(contains='Could not read WAL record at') + self.assertMessage(contains='is absent') self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node')[2]['status'], + self.pb.show('node')[2]['status'], 'Backup {0} should have STATUS "ERROR"') # @unittest.skip("skip") @@ -753,102 +652,44 @@ def test_page_backup_with_corrupted_wal_segment(self): run page backup, expecting error because of missing wal segment make sure that backup status is 'ERROR' """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # make some wals node.pgbench_init(scale=10) # delete last wal segment - wals_dir = os.path.join(backup_dir, 'wal', 'node') - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( - wals_dir, f)) and not f.endswith('.backup')] - wals = map(str, wals) - # file = os.path.join(wals_dir, max(wals)) - - if self.archive_compress: - original_file = os.path.join(wals_dir, '000000010000000000000004.gz') - tmp_file = os.path.join(backup_dir, '000000010000000000000004') - - with gzip.open(original_file, 'rb') as f_in, open(tmp_file, 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) - - # drop healthy file - os.remove(original_file) - file = tmp_file - - else: - file = os.path.join(wals_dir, '000000010000000000000004') - - # corrupt file - print(file) - with open(file, "rb+", 0) as f: - f.seek(42) - f.write(b"blah") - f.flush() - f.close - - if self.archive_compress: - # compress corrupted file and replace with it old file - with open(file, 'rb') as f_in, gzip.open(original_file, 'wb', compresslevel=1) as f_out: - shutil.copyfileobj(f_in, f_out) - - file = os.path.join(wals_dir, '000000010000000000000004.gz') - - #if self.archive_compress: - # file = file[:-3] + file = '000000010000000000000004' + self.compress_suffix + self.wait_instance_wal_exists(backup_dir, 'node', file) + self.corrupt_instance_wal(backup_dir, 'node', file, 42, b"blah", + decompressed=self.archive_compress) # Single-thread PAGE backup - try: - self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.assertEqual( - 1, 0, - "Expecting Error because of wal segment disappearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'Could not read WAL record at' in e.message and - 'Possible WAL corruption. Error has occured during reading WAL segment' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, backup_type='page', + expect_error="because of wal segment disappearance") + self.assertMessage(contains='Could not read WAL record at') + self.assertMessage(contains='Possible WAL corruption. Error has occured during reading WAL segment') self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node')[1]['status'], + self.pb.show('node')[1]['status'], 'Backup {0} should have STATUS "ERROR"') # Multi-thread PAGE backup - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of wal segment disappearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'Could not read WAL record at' in e.message and - 'Possible WAL corruption. Error has occured during reading WAL segment "{0}"'.format( - file) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, backup_type='page', + options=["-j", "4"], + expect_error="because of wal segment disappearance") self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node')[2]['status'], + self.pb.show('node')[2]['status'], 'Backup {0} should have STATUS "ERROR"') # @unittest.skip("skip") @@ -862,30 +703,24 @@ def test_page_backup_with_alien_wal_segment(self): expecting error because of alien wal segment make sure that backup status is 'ERROR' """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - alien_node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'alien_node'), - set_replication=True, - initdb_params=['--data-checksums']) + alien_node = self.pg_node.make_simple('alien_node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.add_instance(backup_dir, 'alien_node', alien_node) - self.set_archiving(backup_dir, 'alien_node', alien_node) + self.pb.add_instance('alien_node', alien_node) + self.pb.set_archiving('alien_node', alien_node) alien_node.slow_start() - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - self.backup_node( - backup_dir, 'alien_node', alien_node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) + self.pb.backup_node('alien_node', alien_node, options=['--stream']) # make some wals node.safe_psql( @@ -909,63 +744,38 @@ def test_page_backup_with_alien_wal_segment(self): "from generate_series(0,10000) i;") # copy latest wal segment - wals_dir = os.path.join(backup_dir, 'wal', 'alien_node') - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( - wals_dir, f)) and not f.endswith('.backup')] - wals = map(str, wals) + wals = self.get_instance_wal_list(backup_dir, 'alien_node') filename = max(wals) - file = os.path.join(wals_dir, filename) - file_destination = os.path.join( - os.path.join(backup_dir, 'wal', 'node'), filename) - start = time.time() - while not os.path.exists(file_destination) and time.time() - start < 20: - time.sleep(0.1) - os.remove(file_destination) - os.rename(file, file_destination) + # wait `node` did archived same file + self.wait_instance_wal_exists(backup_dir, 'node', filename) + file_content = self.read_instance_wal(backup_dir, 'alien_node', filename) + self.write_instance_wal(backup_dir, 'node', filename, file_content) # Single-thread PAGE backup - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='page') - self.assertEqual( - 1, 0, - "Expecting Error because of alien wal segment.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'Could not read WAL record at' in e.message and - 'Possible WAL corruption. Error has occured during reading WAL segment' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, backup_type='page', + expect_error="because of alien wal segment") + self.assertMessage(contains='Could not read WAL record at') + self.assertMessage(contains='Possible WAL corruption. Error has occured during reading WAL segment') self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node')[1]['status'], + self.pb.show('node')[1]['status'], 'Backup {0} should have STATUS "ERROR"') # Multi-thread PAGE backup - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of alien wal segment.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn('Could not read WAL record at', e.message) - self.assertIn('WAL file is from different database system: ' - 'WAL file database system identifier is', e.message) - self.assertIn('pg_control database system identifier is', e.message) - self.assertIn('Possible WAL corruption. Error has occured ' - 'during reading WAL segment', e.message) + self.pb.backup_node('node', node, backup_type='page', + options=["-j", "4"], + expect_error="because of alien wal segment") + self.assertMessage(contains='Could not read WAL record at') + self.assertMessage(contains='WAL file is from different database system: ' + 'WAL file database system identifier is') + self.assertMessage(contains='pg_control database system identifier is') + self.assertMessage(contains='Possible WAL corruption. Error has occured ' + 'during reading WAL segment') self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node')[2]['status'], + self.pb.show('node')[2]['status'], 'Backup {0} should have STATUS "ERROR"') # @unittest.skip("skip") @@ -973,17 +783,14 @@ def test_multithread_page_backup_with_toast(self): """ make node, create toast, do multithread PAGE backup """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # make some wals node.safe_psql( @@ -993,8 +800,7 @@ def test_multithread_page_backup_with_toast(self): "from generate_series(0,70) i") # Multi-thread PAGE backup - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='page', options=["-j", "4"]) # @unittest.skip("skip") @@ -1004,20 +810,17 @@ def test_page_create_db(self): restore database and check it presense """ self.maxDiff = None - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'max_wal_size': '10GB', 'checkpoint_timeout': '5min', } ) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL BACKUP @@ -1026,8 +829,7 @@ def test_page_create_db(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - self.backup_node( - backup_dir, 'node', node) + self.pb.backup_node('node', node) # CREATE DATABASE DB1 node.safe_psql("postgres", "create database db1") @@ -1037,18 +839,16 @@ def test_page_create_db(self): "md5(i::text)::tsvector as tsvector from generate_series(0,1000) i") # PAGE BACKUP - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id = self.pb.backup_node('node', node, backup_type='page') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, backup_id=backup_id, options=["-j", "4"]) # COMPARE PHYSICAL CONTENT @@ -1057,7 +857,7 @@ def test_page_create_db(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() node_restored.safe_psql('db1', 'select 1') @@ -1067,15 +867,13 @@ def test_page_create_db(self): node.safe_psql( "postgres", "drop database db1") # SECOND PAGE BACKUP - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id = self.pb.backup_node('node', node, backup_type='page') if self.paranoia: pgdata = self.pgdata_content(node.data_dir) # RESTORE SECOND PAGE BACKUP - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, backup_id=backup_id, options=["-j", "4"] ) @@ -1086,24 +884,11 @@ def test_page_create_db(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() - try: - node_restored.safe_psql('db1', 'select 1') - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because we are connecting to deleted database" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd) - ) - except QueryException as e: - self.assertTrue( - 'FATAL: database "db1" does not exist' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd) - ) + error_result = node_restored.safe_psql('db1', 'select 1', expect_error=True) + self.assertMessage(error_result, contains='FATAL: database "db1" does not exist') # @unittest.skip("skip") # @unittest.expectedFailure @@ -1119,15 +904,12 @@ def test_multi_timeline_page(self): P must have F as parent """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql("postgres", "create extension pageinspect") @@ -1142,16 +924,15 @@ def test_multi_timeline_page(self): "create extension amcheck_next") node.pgbench_init(scale=20) - full_id = self.backup_node(backup_dir, 'node', node) + full_id = self.pb.backup_node('node', node) pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() - self.backup_node(backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') node.cleanup() - self.restore_node( - backup_dir, 'node', node, backup_id=full_id, + self.pb.restore_node('node', node, backup_id=full_id, options=[ '--recovery-target=immediate', '--recovery-target-action=promote']) @@ -1164,8 +945,7 @@ def test_multi_timeline_page(self): # create timelines for i in range(2, 7): node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ '--recovery-target=latest', '--recovery-target-action=promote', @@ -1178,24 +958,22 @@ def test_multi_timeline_page(self): # create backup at 2, 4 and 6 timeline if i % 2 == 0: - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', + page_id = self.pb.backup_node('node', node, backup_type='page', options=['--log-level-file=VERBOSE']) pgdata = self.pgdata_content(node.data_dir) result = node.table_checksum("pgbench_accounts") - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored) + self.pb.restore_node('node', node=node_restored) pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() result_new = node_restored.table_checksum("pgbench_accounts") @@ -1204,21 +982,21 @@ def test_multi_timeline_page(self): self.compare_pgdata(pgdata, pgdata_restored) - self.checkdb_node( - backup_dir, - 'node', + self.pb.checkdb_node( + use_backup_dir=True, + instance='node', options=[ '--amcheck', '-d', 'postgres', '-p', str(node.port)]) - self.checkdb_node( - backup_dir, - 'node', + self.pb.checkdb_node( + use_backup_dir=True, + instance='node', options=[ '--amcheck', '-d', 'postgres', '-p', str(node_restored.port)]) - backup_list = self.show_pb(backup_dir, 'node') + backup_list = self.pb.show('node') self.assertEqual( backup_list[2]['parent-backup-id'], @@ -1251,16 +1029,13 @@ def test_multitimeline_page_1(self): P must have F as parent """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={'wal_log_hints': 'on'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql("postgres", "create extension pageinspect") @@ -1275,21 +1050,20 @@ def test_multitimeline_page_1(self): "create extension amcheck_next") node.pgbench_init(scale=20) - full_id = self.backup_node(backup_dir, 'node', node) + full_id = self.pb.backup_node('node', node) pgbench = node.pgbench(options=['-T', '20', '-c', '1']) pgbench.wait() - page1 = self.backup_node(backup_dir, 'node', node, backup_type='page') + page1 = self.pb.backup_node('node', node, backup_type='page') pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() - page1 = self.backup_node(backup_dir, 'node', node, backup_type='delta') + page1 = self.pb.backup_node('node', node, backup_type='delta') node.cleanup() - self.restore_node( - backup_dir, 'node', node, backup_id=page1, + self.pb.restore_node('node', node, backup_id=page1, options=[ '--recovery-target=immediate', '--recovery-target-action=promote']) @@ -1299,20 +1073,18 @@ def test_multitimeline_page_1(self): pgbench = node.pgbench(options=['-T', '20', '-c', '1', '--no-vacuum']) pgbench.wait() - print(self.backup_node( - backup_dir, 'node', node, backup_type='page', + print(self.pb.backup_node('node', node, backup_type='page', options=['--log-level-console=LOG'], return_id=False)) pgdata = self.pgdata_content(node.data_dir) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored) + self.pb.restore_node('node', node=node_restored) pgdata_restored = self.pgdata_content(node_restored.data_dir) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() self.compare_pgdata(pgdata, pgdata_restored) @@ -1320,18 +1092,16 @@ def test_multitimeline_page_1(self): @unittest.skip("skip") # @unittest.expectedFailure def test_page_pg_resetxlog(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'shared_buffers': '512MB', 'max_wal_size': '3GB'}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Create table @@ -1344,7 +1114,7 @@ def test_page_pg_resetxlog(self): # "from generate_series(0,25600) i") "from generate_series(0,2560) i") - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.safe_psql( 'postgres', @@ -1366,7 +1136,7 @@ def test_page_pg_resetxlog(self): pg_resetxlog_path = self.get_bin_path('pg_resetxlog') wal_dir = 'pg_xlog' - self.run_binary( + self.pb.run_binary( [ pg_resetxlog_path, '-D', @@ -1383,35 +1153,72 @@ def test_page_pg_resetxlog(self): exit(1) # take ptrack backup -# self.backup_node( -# backup_dir, 'node', node, +# self.pb.backup_node( +# 'node', node, # backup_type='page', options=['--stream']) - try: - self.backup_node( - backup_dir, 'node', node, backup_type='page') - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because instance was brutalized by pg_resetxlog" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd) - ) - except ProbackupException as e: - self.assertIn( - 'Insert error message', - e.message, - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(e.message), self.cmd)) + self.pb.backup_node('node', node, backup_type='page', + expect_error="because instance was brutalized by pg_resetxlog") + self.assertMessage(contains='Insert error message') # pgdata = self.pgdata_content(node.data_dir) # -# node_restored = self.make_simple_node( -# base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) +# node_restored = self.pg_node.make_simple('node_restored') # node_restored.cleanup() # -# self.restore_node( -# backup_dir, 'node', node_restored) +# self.pb.restore_node( +# 'node', node_restored) # # pgdata_restored = self.pgdata_content(node_restored.data_dir) # self.compare_pgdata(pgdata, pgdata_restored) + + def test_page_huge_xlog_record(self): + node = self.pg_node.make_simple('node', + set_replication=True, + pg_options={ + 'max_locks_per_transaction': '1000', + 'work_mem': '100MB', + 'temp_buffers': '100MB', + 'wal_buffers': '128MB', + 'wal_level' : 'logical', + }) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + node.pgbench_init(scale=3) + + # Do full backup + self.pb.backup_node('node', node, backup_type='full') + + show_backup = self.pb.show('node')[0] + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "FULL") + + # Originally client had the problem at the transaction that (supposedly) + # deletes a lot of temporary tables (probably it was client disconnect). + # It generated ~40MB COMMIT WAL record. + # + # `pg_logical_emit_message` is much simpler and faster way to generate + # such huge record. + node.safe_psql( + "postgres", + "select pg_logical_emit_message(False, 'z', repeat('o', 60*1000*1000))") + + # Do page backup with 1 thread + backup_id = self.pb.backup_node('node', node, backup_type='page', options=['-j', '1']) + + show_backup = self.pb.show('node')[1] + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") + + self.pb.delete('node', backup_id) + + # Repeat backup with multiple threads + self.pb.backup_node('node', node, backup_type='page', options=['-j', '10']) + + show_backup = self.pb.show('node')[1] + self.assertEqual(show_backup['status'], "OK") + self.assertEqual(show_backup['backup-mode'], "PAGE") diff --git a/tests/pgpro2068_test.py b/tests/pgpro2068_test.py index 04f0eb6fa..e69de29bb 100644 --- a/tests/pgpro2068_test.py +++ b/tests/pgpro2068_test.py @@ -1,171 +0,0 @@ -import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack -from datetime import datetime, timedelta -import subprocess -from time import sleep -import shutil -import signal -from testgres import ProcessType - - -class BugTest(ProbackupTest, unittest.TestCase): - - def test_minrecpoint_on_replica(self): - """ - https://jira.postgrespro.ru/browse/PGPRO-2068 - """ - self._check_gdb_flag_or_skip_test() - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ - # 'checkpoint_timeout': '60min', - 'checkpoint_completion_target': '0.9', - 'bgwriter_delay': '10ms', - 'bgwriter_lru_maxpages': '1000', - 'bgwriter_lru_multiplier': '4.0', - 'max_wal_size': '256MB'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # take full backup and restore it as replica - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - # start replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'node', replica, options=['-R']) - self.set_replica(node, replica) - self.add_instance(backup_dir, 'replica', replica) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - - self.set_auto_conf( - replica, - {'port': replica.port, 'restart_after_crash': 'off'}) - - node.safe_psql( - "postgres", - "CREATE EXTENSION pageinspect") - - replica.slow_start(replica=True) - - # generate some data - node.pgbench_init(scale=10) - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "4", "-T", "20"]) - pgbench.wait() - pgbench.stdout.close() - - # generate some more data and leave it in background - pgbench = node.pgbench( - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - options=["-c", "4", "-j 4", "-T", "100"]) - - # wait for shared buffer on replica to be filled with dirty data - sleep(20) - - # get pids of replica background workers - startup_pid = replica.auxiliary_pids[ProcessType.Startup][0] - checkpointer_pid = replica.auxiliary_pids[ProcessType.Checkpointer][0] - - # break checkpointer on UpdateLastRemovedPtr - gdb_checkpointer = self.gdb_attach(checkpointer_pid) - gdb_checkpointer._execute('handle SIGINT noprint nostop pass') - gdb_checkpointer._execute('handle SIGUSR1 noprint nostop pass') - gdb_checkpointer.set_breakpoint('UpdateLastRemovedPtr') - gdb_checkpointer.continue_execution_until_break() - - # break recovery on UpdateControlFile - gdb_recovery = self.gdb_attach(startup_pid) - gdb_recovery._execute('handle SIGINT noprint nostop pass') - gdb_recovery._execute('handle SIGUSR1 noprint nostop pass') - gdb_recovery.set_breakpoint('UpdateMinRecoveryPoint') - gdb_recovery.continue_execution_until_break() - gdb_recovery.set_breakpoint('UpdateControlFile') - gdb_recovery.continue_execution_until_break() - - # stop data generation - pgbench.wait() - pgbench.stdout.close() - - # kill someone, we need a crash - replica.kill(someone=ProcessType.BackgroundWriter) - gdb_recovery._execute('detach') - gdb_checkpointer._execute('detach') - - # just to be sure - try: - replica.stop(['-m', 'immediate', '-D', replica.data_dir]) - except: - pass - - # MinRecLSN = replica.get_control_data()['Minimum recovery ending location'] - - # Promote replica with 'immediate' target action - if self.get_version(replica) >= self.version_to_num('12.0'): - recovery_config = 'postgresql.auto.conf' - else: - recovery_config = 'recovery.conf' - - replica.append_conf( - recovery_config, "recovery_target = 'immediate'") - replica.append_conf( - recovery_config, "recovery_target_action = 'pause'") - replica.slow_start(replica=True) - - current_xlog_lsn_query = 'SELECT pg_last_wal_replay_lsn() INTO current_xlog_lsn' - if self.get_version(node) < 100000: - current_xlog_lsn_query = 'SELECT min_recovery_end_location INTO current_xlog_lsn FROM pg_control_recovery()' - - script = f''' -DO -$$ -DECLARE - roid oid; - current_xlog_lsn pg_lsn; - pages_from_future RECORD; - found_corruption bool := false; -BEGIN - {current_xlog_lsn_query}; - RAISE NOTICE 'CURRENT LSN: %', current_xlog_lsn; - FOR roid IN select oid from pg_class class where relkind IN ('r', 'i', 't', 'm') and relpersistence = 'p' LOOP - FOR pages_from_future IN - with number_of_blocks as (select blknum from generate_series(0, pg_relation_size(roid) / 8192 -1) as blknum ) - select blknum, lsn, checksum, flags, lower, upper, special, pagesize, version, prune_xid - from number_of_blocks, page_header(get_raw_page(roid::regclass::text, number_of_blocks.blknum::int)) - where lsn > current_xlog_lsn LOOP - RAISE NOTICE 'Found page from future. OID: %, BLKNUM: %, LSN: %', roid, pages_from_future.blknum, pages_from_future.lsn; - found_corruption := true; - END LOOP; - END LOOP; - IF found_corruption THEN - RAISE 'Found Corruption'; - END IF; -END; -$$ LANGUAGE plpgsql; -'''.format(current_xlog_lsn_query=current_xlog_lsn_query) - - # Find blocks from future - replica.safe_psql( - 'postgres', - script) - - # error is expected if version < 10.6 - # gdb_backup.continue_execution_until_exit() - - # do basebackup - - # do pg_probackup, expect error diff --git a/tests/pgpro560_test.py b/tests/pgpro560_test.py index b665fd200..2a9548670 100644 --- a/tests/pgpro560_test.py +++ b/tests/pgpro560_test.py @@ -1,12 +1,12 @@ import os import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest from datetime import datetime, timedelta import subprocess from time import sleep -class CheckSystemID(ProbackupTest, unittest.TestCase): +class CheckSystemID(ProbackupTest): # @unittest.skip("skip") # @unittest.expectedFailure @@ -17,33 +17,20 @@ def test_pgpro560_control_file_loss(self): make backup check that backup failed """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() file = os.path.join(node.base_dir, 'data', 'global', 'pg_control') # Not delete this file permanently os.rename(file, os.path.join(node.base_dir, 'data', 'global', 'pg_control_copy')) - try: - self.backup_node(backup_dir, 'node', node, options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because pg_control was deleted.\n " - "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: Could not open file' in e.message and - 'pg_control' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, options=['--stream'], + expect_error='because pg_control was deleted') + self.assertMessage(regex=r'ERROR: Could not get control file:.*pg_control') # Return this file to avoid Postger fail os.rename(os.path.join(node.base_dir, 'data', 'global', 'pg_control_copy'), file) @@ -55,69 +42,29 @@ def test_pgpro560_systemid_mismatch(self): feed to backup PGDATA from node1 and PGPORT from node2 check that backup failed """ - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), - set_replication=True, - initdb_params=['--data-checksums']) + node1 = self.pg_node.make_simple('node1', + set_replication=True) node1.slow_start() - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2'), - set_replication=True, - initdb_params=['--data-checksums']) + node2 = self.pg_node.make_simple('node2', + set_replication=True) node2.slow_start() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node1', node1) + self.pb.init() + self.pb.add_instance('node1', node1) - try: - self.backup_node(backup_dir, 'node1', node2, options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of SYSTEM ID mismatch.\n " - "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) - except ProbackupException as e: - if self.get_version(node1) > 90600: - self.assertTrue( - 'ERROR: Backup data directory was ' - 'initialized for system id' in e.message and - 'but connected instance system id is' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - else: - self.assertIn( - 'ERROR: System identifier mismatch. ' - 'Connected PostgreSQL instance has system id', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.backup_node('node1', node2, options=['--stream'], + expect_error="because of SYSTEM ID mismatch") + self.assertMessage(contains='ERROR: Backup data directory was ' + 'initialized for system id') + self.assertMessage(contains='but connected instance system id is') sleep(1) - try: - self.backup_node( - backup_dir, 'node1', node2, - data_dir=node1.data_dir, options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of of SYSTEM ID mismatch.\n " - "Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) - except ProbackupException as e: - if self.get_version(node1) > 90600: - self.assertTrue( - 'ERROR: Backup data directory was initialized ' - 'for system id' in e.message and - 'but connected instance system id is' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - else: - self.assertIn( - 'ERROR: System identifier mismatch. ' - 'Connected PostgreSQL instance has system id', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.backup_node('node1', node2, + data_dir=node1.data_dir, options=['--stream'], + expect_error="because of of SYSTEM ID mismatch") + self.assertMessage(contains='ERROR: Backup data directory was ' + 'initialized for system id') + self.assertMessage(contains='but connected instance system id is') diff --git a/tests/pgpro589_test.py b/tests/pgpro589_test.py index 8ce8e1f56..e600f142a 100644 --- a/tests/pgpro589_test.py +++ b/tests/pgpro589_test.py @@ -1,11 +1,11 @@ import os import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack from datetime import datetime, timedelta import subprocess -class ArchiveCheck(ProbackupTest, unittest.TestCase): +class ArchiveCheck(ProbackupTest): def test_pgpro589(self): """ @@ -14,17 +14,15 @@ def test_pgpro589(self): check that backup status equal to ERROR check that no files where copied to backup catalogue """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) # make erroneous archive_command - self.set_auto_conf(node, {'archive_command': 'exit 0'}) + node.set_auto_conf({'archive_command': 'exit 0'}) node.slow_start() node.pgbench_init(scale=5) @@ -40,27 +38,16 @@ def test_pgpro589(self): "select pg_relation_filepath('pgbench_accounts')").rstrip().decode( "utf-8") - try: - self.backup_node( - backup_dir, 'node', node, - options=['--archive-timeout=10']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of missing archive wal " - "segment with start_lsn.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'INFO: Wait for WAL segment' in e.message and - 'ERROR: WAL segment' in e.message and - 'could not be archived in 10 seconds' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, + options=['--archive-timeout=10'], + expect_error="because of missing archive wal segment " + "with start_lsn") + self.assertMessage(contains='INFO: Wait for WAL segment') + self.assertMessage(regex='ERROR: WAL segment .* could not be archived in 10 seconds') - backup_id = self.show_pb(backup_dir, 'node')[0]['id'] + backup_id = self.pb.show('node')[0]['id'] self.assertEqual( - 'ERROR', self.show_pb(backup_dir, 'node', backup_id)['status'], + 'ERROR', self.pb.show('node', backup_id)['status'], 'Backup should have ERROR status') file = os.path.join( backup_dir, 'backups', 'node', diff --git a/tests/ptrack_load_test.py b/tests/ptrack_load_test.py new file mode 100644 index 000000000..9e96ab8b7 --- /dev/null +++ b/tests/ptrack_load_test.py @@ -0,0 +1,150 @@ +import os +from .helpers.ptrack_helpers import ProbackupTest + +PAGE_SIZE = 8192 +ZEROES = b"\x00" * PAGE_SIZE + + +class PtrackLoadTest(ProbackupTest): + def setUp(self): + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') + + def find_zero_pages(self, node, pagemapset, file_path): + """ + Find zero pages in a file using pagemapset. + + Args: + node (Node): The PostgreSQL node instance. + pagemapset (dict): The pagemapset obtained from fetch_ptrack. + file_path (str): Path to the file to analyze. + + Returns: + list: List of missed pages. + """ + missed_pages = [] + + if os.path.isfile(file_path): + rel_path = file_path.replace(f"{node.data_dir}/", "") + with open(file_path, "rb") as f: + bno = 0 + while True: + page_data = f.read(PAGE_SIZE) + if not page_data: + break + + if page_data == ZEROES: + if not self.check_ptrack(pagemapset, rel_path, bno): + print(f"Missed page: {rel_path}|{bno}") + missed_pages.append(f'{rel_path}|{bno}') + else: + print(f"Found page: {rel_path}|{bno}") + + bno += 1 + + return missed_pages + + @staticmethod + def fetch_ptrack(node, lsn): + """ + Fetch pagemapset using ptrack_get_pagemapset function. + + Args: + node (Node): The PostgreSQL node instance. + lsn (str): The LSN (Log Sequence Number). + + Returns: + dict: Dictionary containing pagemapset data. + """ + result_map = {} + ptrack_out = node.execute( + "postgres", + f"select (ptrack_get_pagemapset('{lsn}')).*;") + for row in ptrack_out: + path, pagecount, pagemap = row + result_map[path] = bytearray(pagemap) + return result_map + + def check_ptrack(self, page_map, file, bno): + """ + Check if the given block number has changes in pagemap. + + Args: + page_map (dict): Pagemapset data. + file (str): File name. + bno (int): Block number. + + Returns: + bool: True if changes are detected, False otherwise. + """ + self.assertNotEqual(page_map, {}) + bits = page_map.get(file) + + if bits and bno // 8 < len(bits): + return (bits[bno // 8] & (1 << (bno & 7))) != 0 + else: + return False + + def test_load_ptrack_zero_pages(self): + """ + An error too many clients already for some clients is usual for this test + """ + pg_options = {'max_connections': 1024, + 'ptrack.map_size': 1024, + 'shared_buffers': '8GB', + 'checkpoint_timeout': '1d', + 'synchronous_commit': 'off', + 'fsync': 'off', + 'shared_preload_libraries': 'ptrack', + 'wal_buffers': '128MB', + 'wal_writer_delay': '5s', + 'wal_writer_flush_after': '16MB', + 'commit_delay': 100, + 'checkpoint_flush_after': '2MB', + 'max_wal_size': '10GB', + 'autovacuum': 'off'} + if self.pg_config_version >= 120000: + pg_options['wal_recycle'] = 'off' + + node = self.pg_node.make_simple('node', + set_replication=True, + ptrack_enable=self.ptrack, + ) + node.slow_start() + + start_lsn = node.execute( + "postgres", + "select pg_current_wal_lsn()")[0][0] + + self.pb.init() + self.pb.add_instance('node', node) + + node.execute( + "postgres", + "CREATE EXTENSION ptrack") + + # Initialize and start pgbench + node.pgbench_init(scale=100) + + pgbench = node.pgbench(options=['-T', '20', '-c', '150', '-j', '150']) + pgbench.wait() + + node.execute( + "postgres", + "CHECKPOINT;select txid_current();") + + missed_pages = [] + # Process each file in the data directory + for root, dirs, files in os.walk(node.data_dir): + # Process only the files in the 'global' and 'base' directories + if 'data/global' in root or 'data/base' in root or 'data/pg_tblspc' in root: + for file in files: + if file in ['ptrack.map']: + continue + file_path = os.path.join(root, file) + pagemapset = self.fetch_ptrack(node, start_lsn) + pages = self.find_zero_pages(node, pagemapset, file_path) + if pages: + missed_pages.extend(pages) + # Check that no missed pages + self.assertEqual(missed_pages, []) diff --git a/tests/ptrack_test.py b/tests/ptrack_test.py index 7b5bc416b..29eb2f11a 100644 --- a/tests/ptrack_test.py +++ b/tests/ptrack_test.py @@ -1,38 +1,33 @@ import os import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack -from datetime import datetime, timedelta -import subprocess -from testgres import QueryException, StartNodeException +from .helpers.ptrack_helpers import ProbackupTest, idx_ptrack +from pg_probackup2.gdb import needs_gdb +from testgres import StartNodeException import shutil -import sys from time import sleep from threading import Thread -class PtrackTest(ProbackupTest, unittest.TestCase): +class PtrackTest(ProbackupTest): def setUp(self): - if self.pg_config_version < self.version_to_num('11.0'): - self.skipTest('You need PostgreSQL >= 11 for this test') - self.fname = self.id().split('.')[3] + if not self.ptrack: + self.skipTest('Skipped because ptrack support is disabled') # @unittest.skip("skip") + @needs_gdb def test_drop_rel_during_backup_ptrack(self): """ drop relation during ptrack backup """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) + ptrack_enable=self.ptrack) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -51,11 +46,10 @@ def test_drop_rel_during_backup_ptrack(self): absolute_path = os.path.join(node.data_dir, relative_path) # FULL backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) # PTRACK backup - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + gdb = self.pb.backup_node('node', node, backup_type='ptrack', gdb=True, options=['--log-level-file=LOG']) gdb.set_breakpoint('backup_files') @@ -69,14 +63,13 @@ def test_drop_rel_during_backup_ptrack(self): pgdata = self.pgdata_content(node.data_dir) - with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: - log_content = f.read() - self.assertTrue( + log_content = self.read_pb_log() + self.assertTrue( 'LOG: File not found: "{0}"'.format(absolute_path) in log_content, 'File "{0}" should be deleted but it`s not'.format(absolute_path)) node.cleanup() - self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + self.pb.restore_node('node', node=node, options=["-j", "4"]) # Physical comparison pgdata_restored = self.pgdata_content(node.data_dir) @@ -85,67 +78,52 @@ def test_drop_rel_during_backup_ptrack(self): # @unittest.skip("skip") def test_ptrack_without_full(self): """ptrack backup without validated full backup""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( "postgres", "CREATE EXTENSION ptrack") - try: - self.backup_node(backup_dir, 'node', node, backup_type="ptrack") - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "without valid full backup.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "WARNING: Valid full backup on current timeline 1 is not found" in e.message and - "ERROR: Create new full backup before an incremental one" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, backup_type="ptrack", + expect_error="because page backup should not be " + "possible without valid full backup") + self.assertMessage(contains="WARNING: Valid full backup on current timeline 1 is not found") + self.assertMessage(contains="ERROR: Create new full backup before an incremental one") self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['status'], + self.pb.show('node')[0]['status'], "ERROR") # @unittest.skip("skip") def test_ptrack_threads(self): """ptrack multi thread backup mode""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( "postgres", "CREATE EXTENSION ptrack") - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type="full", options=["-j", "4"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + self.assertEqual(self.pb.show('node')[0]['status'], "OK") - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type="ptrack", options=["-j", "4"]) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + self.assertEqual(self.pb.show('node')[0]['status'], "OK") # @unittest.skip("skip") def test_ptrack_stop_pg(self): @@ -154,15 +132,13 @@ def test_ptrack_stop_pg(self): restart node, check that ptrack backup can be taken """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -172,13 +148,12 @@ def test_ptrack_stop_pg(self): node.pgbench_init(scale=1) # FULL backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) node.stop() node.slow_start() - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='ptrack', options=['--stream']) # @unittest.skip("skip") @@ -187,16 +162,14 @@ def test_ptrack_multi_timeline_backup(self): t2 /------P2 t1 ------F---*-----P1 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -206,7 +179,7 @@ def test_ptrack_multi_timeline_backup(self): node.pgbench_init(scale=5) # FULL backup - full_id = self.backup_node(backup_dir, 'node', node) + full_id = self.pb.backup_node('node', node) pgbench = node.pgbench(options=['-T', '30', '-c', '1', '--no-vacuum']) sleep(15) @@ -216,13 +189,12 @@ def test_ptrack_multi_timeline_backup(self): 'SELECT txid_current()').decode('utf-8').rstrip() pgbench.wait() - self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + self.pb.backup_node('node', node, backup_type='ptrack') node.cleanup() # Restore from full backup to create Timeline 2 - print(self.restore_node( - backup_dir, 'node', node, + print(self.pb.restore_node('node', node, options=[ '--recovery-target-xid={0}'.format(xid), '--recovery-target-action=promote'])) @@ -232,13 +204,13 @@ def test_ptrack_multi_timeline_backup(self): pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() - self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + self.pb.backup_node('node', node, backup_type='ptrack') pgdata = self.pgdata_content(node.data_dir) node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -264,16 +236,13 @@ def test_ptrack_multi_timeline_backup_1(self): t2 /------P2 t1 ---F--------* """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -283,15 +252,15 @@ def test_ptrack_multi_timeline_backup_1(self): node.pgbench_init(scale=5) # FULL backup - full_id = self.backup_node(backup_dir, 'node', node) + full_id = self.pb.backup_node('node', node) pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() - ptrack_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + ptrack_id = self.pb.backup_node('node', node, backup_type='ptrack') node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) node.slow_start() @@ -299,16 +268,16 @@ def test_ptrack_multi_timeline_backup_1(self): pgbench.wait() # delete old PTRACK backup - self.delete_pb(backup_dir, 'node', backup_id=ptrack_id) + self.pb.delete('node', backup_id=ptrack_id) # take new PTRACK backup - self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + self.pb.backup_node('node', node, backup_type='ptrack') pgdata = self.pgdata_content(node.data_dir) node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -330,16 +299,14 @@ def test_ptrack_eat_my_data(self): """ PGPRO-4051 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -348,10 +315,9 @@ def test_ptrack_eat_my_data(self): node.pgbench_init(scale=50) - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') pgbench = node.pgbench(options=['-T', '300', '-c', '1', '--no-vacuum']) @@ -360,12 +326,12 @@ def test_ptrack_eat_my_data(self): sleep(2) - self.backup_node(backup_dir, 'node', node, backup_type='ptrack') + self.pb.backup_node('node', node, backup_type='ptrack') # pgdata = self.pgdata_content(node.data_dir) # # node_restored.cleanup() # -# self.restore_node(backup_dir, 'node', node_restored) +# self.pb.restore_node('node', node=node_restored) # pgdata_restored = self.pgdata_content(node_restored.data_dir) # # self.compare_pgdata(pgdata, pgdata_restored) @@ -378,9 +344,8 @@ def test_ptrack_eat_my_data(self): result = node.table_checksum("pgbench_accounts") node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored) - self.set_auto_conf( - node_restored, {'port': node_restored.port}) + self.pb.restore_node('node', node=node_restored) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() @@ -403,38 +368,34 @@ def test_ptrack_eat_my_data(self): def test_ptrack_simple(self): """make node, make full and ptrack stream backups," " restore them and check data correctness""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( "postgres", "CREATE EXTENSION ptrack") - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) node.safe_psql( "postgres", "create table t_heap as select i" " as id from generate_series(0,1) i") - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + self.pb.backup_node('node', node, backup_type='ptrack', options=['--stream']) node.safe_psql( "postgres", "update t_heap set id = 100500") - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='ptrack', options=['--stream']) if self.paranoia: @@ -442,12 +403,10 @@ def test_ptrack_simple(self): result = node.table_checksum("t_heap") - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, options=["-j", "4"]) + self.pb.restore_node('node', node_restored, options=["-j", "4"]) # Physical comparison if self.paranoia: @@ -455,8 +414,7 @@ def test_ptrack_simple(self): node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) - self.set_auto_conf( - node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() @@ -468,95 +426,22 @@ def test_ptrack_simple(self): # @unittest.skip("skip") def test_ptrack_unprivileged(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - # self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + # self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( "postgres", "CREATE DATABASE backupdb") - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + # PG < 15 + if self.pg_config_version < 150000: node.safe_psql( 'backupdb', "REVOKE ALL ON DATABASE backupdb from PUBLIC; " @@ -642,18 +527,16 @@ def test_ptrack_unprivileged(self): "backupdb", "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup") - if ProbackupTest.enterprise: + if ProbackupTest.pgpro: node.safe_psql( "backupdb", "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_version() TO backup; " 'GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;') - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, datname='backupdb', options=['--stream', "-U", "backup"]) - self.backup_node( - backup_dir, 'node', node, datname='backupdb', + self.pb.backup_node('node', node, datname='backupdb', backup_type='ptrack', options=['--stream', "-U", "backup"]) @@ -661,16 +544,14 @@ def test_ptrack_unprivileged(self): # @unittest.expectedFailure def test_ptrack_enable(self): """make ptrack without full backup, should result in error""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', + set_replication=True, pg_options={ 'checkpoint_timeout': '30s', 'shared_preload_libraries': 'ptrack'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -678,26 +559,10 @@ def test_ptrack_enable(self): "CREATE EXTENSION ptrack") # PTRACK BACKUP - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=["--stream"] - ) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because ptrack disabled.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd - ) - ) - except ProbackupException as e: - self.assertIn( - 'ERROR: Ptrack is disabled\n', - e.message, - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(e.message), self.cmd) - ) + self.pb.backup_node('node', node, backup_type='ptrack', + options=["--stream"], + expect_error="because ptrack disabled") + self.assertMessage(contains='ERROR: Ptrack is disabled') # @unittest.skip("skip") # @unittest.expectedFailure @@ -707,16 +572,14 @@ def test_ptrack_disable(self): enable ptrack, restart postgresql, take ptrack backup which should fail """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], pg_options={'checkpoint_timeout': '30s'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -724,7 +587,7 @@ def test_ptrack_disable(self): "CREATE EXTENSION ptrack") # FULL BACKUP - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) # DISABLE PTRACK node.safe_psql('postgres', "alter system set ptrack.map_size to 0") @@ -738,77 +601,53 @@ def test_ptrack_disable(self): node.slow_start() # PTRACK BACKUP - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=["--stream"] - ) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because ptrack_enable was set to OFF at some" - " point after previous backup.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd - ) - ) - except ProbackupException as e: - self.assertIn( - 'ERROR: LSN from ptrack_control', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd - ) - ) + self.pb.backup_node('node', node, backup_type='ptrack', + options=["--stream"], + expect_error="because ptrack_enable was set to OFF " + "at some point after previous backup") + self.assertMessage(contains='ERROR: LSN from ptrack_control') # @unittest.skip("skip") def test_ptrack_uncommitted_xact(self): """make ptrack backup while there is uncommitted open transaction""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], pg_options={ 'wal_level': 'replica'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( "postgres", "CREATE EXTENSION ptrack") - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) con = node.connect("postgres") con.execute( "create table t_heap as select i" " as id from generate_series(0,1) i") - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + self.pb.backup_node('node', node, backup_type='ptrack', options=['--stream']) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, - node_restored.data_dir, options=["-j", "4"]) + self.pb.restore_node('node', node_restored, options=["-j", "4"]) if self.paranoia: pgdata_restored = self.pgdata_content( node_restored.data_dir, ignore_ptrack=False) - self.set_auto_conf( - node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() @@ -817,20 +656,18 @@ def test_ptrack_uncommitted_xact(self): self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") - def test_ptrack_vacuum_full(self): + @needs_gdb + def test_ptrack_vacuum_full_1(self): """make node, make full and ptrack stream backups, restore them and check data correctness""" - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -839,7 +676,7 @@ def test_ptrack_vacuum_full(self): "postgres", "CREATE EXTENSION ptrack") - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) node.safe_psql( "postgres", @@ -858,33 +695,28 @@ def test_ptrack_vacuum_full(self): target=pg_connect.execute, args=["VACUUM FULL t_heap"]) process.start() - while not gdb.stopped_in_breakpoint: - sleep(1) + gdb.stopped_in_breakpoint() gdb.continue_execution_until_break(20) - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + self.pb.backup_node('node', node, backup_type='ptrack', options=['--stream']) - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + self.pb.backup_node('node', node, backup_type='ptrack', options=['--stream']) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) gdb.remove_all_breakpoints() - gdb._execute('detach') + gdb.detach() process.join() - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() old_tablespace = self.get_tblspace_path(node, 'somedata') new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=["-j", "4", "-T", "{0}={1}".format( old_tablespace, new_tablespace)] ) @@ -895,8 +727,7 @@ def test_ptrack_vacuum_full(self): node_restored.data_dir, ignore_ptrack=False) self.compare_pgdata(pgdata, pgdata_restored) - self.set_auto_conf( - node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() @@ -906,15 +737,13 @@ def test_ptrack_vacuum_truncate(self): delete last 3 pages, vacuum relation, take ptrack backup, take second ptrack backup, restore last ptrack backup and check data correctness""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -935,7 +764,7 @@ def test_ptrack_vacuum_truncate(self): "postgres", "vacuum t_heap") - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) node.safe_psql( "postgres", @@ -945,24 +774,20 @@ def test_ptrack_vacuum_truncate(self): "postgres", "vacuum t_heap") - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + self.pb.backup_node('node', node, backup_type='ptrack', options=['--stream']) - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + self.pb.backup_node('node', node, backup_type='ptrack', options=['--stream']) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() old_tablespace = self.get_tblspace_path(node, 'somedata') new_tablespace = self.get_tblspace_path(node_restored, 'somedata_new') - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=["-j", "4", "-T", "{0}={1}".format( old_tablespace, new_tablespace)] ) @@ -975,28 +800,25 @@ def test_ptrack_vacuum_truncate(self): ) self.compare_pgdata(pgdata, pgdata_restored) - self.set_auto_conf( - node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() # @unittest.skip("skip") + @needs_gdb def test_ptrack_get_block(self): """ make node, make full and ptrack stream backups, restore them and check data correctness """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -1008,9 +830,8 @@ def test_ptrack_get_block(self): "create table t_heap as select i" " as id from generate_series(0,1) i") - self.backup_node(backup_dir, 'node', node, options=['--stream']) - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + self.pb.backup_node('node', node, options=['--stream']) + gdb = self.pb.backup_node('node', node, backup_type='ptrack', options=['--stream'], gdb=True) @@ -1023,8 +844,7 @@ def test_ptrack_get_block(self): gdb.continue_execution_until_exit() - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='ptrack', options=['--stream']) if self.paranoia: @@ -1032,7 +852,7 @@ def test_ptrack_get_block(self): result = node.table_checksum("t_heap") node.cleanup() - self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + self.pb.restore_node('node', node=node, options=["-j", "4"]) # Physical comparison if self.paranoia: @@ -1050,17 +870,15 @@ def test_ptrack_get_block(self): def test_ptrack_stream(self): """make node, make full and ptrack stream backups, restore them and check data correctness""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -1076,8 +894,7 @@ def test_ptrack_stream(self): " as tsvector from generate_series(0,100) i") full_result = node.table_checksum("t_heap") - full_backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) + full_backup_id = self.pb.backup_node('node', node, options=['--stream']) # PTRACK BACKUP node.safe_psql( @@ -1087,8 +904,7 @@ def test_ptrack_stream(self): " from generate_series(100,200) i") ptrack_result = node.table_checksum("t_heap") - ptrack_backup_id = self.backup_node( - backup_dir, 'node', node, + ptrack_backup_id = self.pb.backup_node('node', node, backup_type='ptrack', options=['--stream']) if self.paranoia: @@ -1098,29 +914,20 @@ def test_ptrack_stream(self): node.cleanup() # Restore and check full backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(full_backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, backup_id=full_backup_id, options=["-j", "4"] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd) ) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(full_backup_id)) node.slow_start() full_result_new = node.table_checksum("t_heap") self.assertEqual(full_result, full_result_new) node.cleanup() # Restore and check ptrack backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, backup_id=ptrack_backup_id, options=["-j", "4"] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + ) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(ptrack_backup_id)) if self.paranoia: pgdata_restored = self.pgdata_content( @@ -1135,18 +942,16 @@ def test_ptrack_stream(self): def test_ptrack_archive(self): """make archive node, make full and ptrack backups, check data correctness in restored instance""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -1163,9 +968,8 @@ def test_ptrack_archive(self): " from generate_series(0,100) i") full_result = node.table_checksum("t_heap") - full_backup_id = self.backup_node(backup_dir, 'node', node) - full_target_time = self.show_pb( - backup_dir, 'node', full_backup_id)['recovery-time'] + full_backup_id = self.pb.backup_node('node', node) + full_target_time = self.pb.show('node', full_backup_id)['recovery-time'] # PTRACK BACKUP node.safe_psql( @@ -1176,10 +980,8 @@ def test_ptrack_archive(self): " from generate_series(100,200) i") ptrack_result = node.table_checksum("t_heap") - ptrack_backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack') - ptrack_target_time = self.show_pb( - backup_dir, 'node', ptrack_backup_id)['recovery-time'] + ptrack_backup_id = self.pb.backup_node('node', node, backup_type='ptrack') + ptrack_target_time = self.pb.show('node', ptrack_backup_id)['recovery-time'] if self.paranoia: pgdata = self.pgdata_content(node.data_dir) @@ -1194,18 +996,13 @@ def test_ptrack_archive(self): node.cleanup() # Check full backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(full_backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, backup_id=full_backup_id, options=[ "-j", "4", "--recovery-target-action=promote", - "--time={0}".format(full_target_time)] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd) - ) + "--recovery-target-time={0}".format(full_target_time)] + ) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(full_backup_id)) node.slow_start() full_result_new = node.table_checksum("t_heap") @@ -1213,19 +1010,14 @@ def test_ptrack_archive(self): node.cleanup() # Check ptrack backup - self.assertIn( - "INFO: Restore of backup {0} completed.".format(ptrack_backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, backup_id=ptrack_backup_id, options=[ "-j", "4", - "--time={0}".format(ptrack_target_time), + "--recovery-target-time={0}".format(ptrack_target_time), "--recovery-target-action=promote"] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd) - ) + ) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(ptrack_backup_id)) if self.paranoia: pgdata_restored = self.pgdata_content( @@ -1238,227 +1030,21 @@ def test_ptrack_archive(self): node.cleanup() - @unittest.skip("skip") - def test_ptrack_pgpro417(self): - """ - Make node, take full backup, take ptrack backup, - delete ptrack backup. Try to take ptrack backup, - which should fail. Actual only for PTRACK 1.x - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='full', options=["--stream"]) - - start_lsn_full = self.show_pb( - backup_dir, 'node', backup_id)['start-lsn'] - - # PTRACK BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector " - "from generate_series(100,200) i") - node.table_checksum("t_heap") - backup_id = self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=["--stream"]) - - start_lsn_ptrack = self.show_pb( - backup_dir, 'node', backup_id)['start-lsn'] - - self.delete_pb(backup_dir, 'node', backup_id) - - # SECOND PTRACK BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector " - "from generate_series(200,300) i") - - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=["--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of LSN mismatch from ptrack_control " - "and previous backup start_lsn.\n" - " Output: {0} \n CMD: {1}".format(repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: LSN from ptrack_control' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - @unittest.skip("skip") - def test_page_pgpro417(self): - """ - Make archive node, take full backup, take page backup, - delete page backup. Try to take ptrack backup, which should fail. - Actual only for PTRACK 1.x - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - node.table_checksum("t_heap") - - # PAGE BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector " - "from generate_series(100,200) i") - node.table_checksum("t_heap") - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - self.delete_pb(backup_dir, 'node', backup_id) -# sys.exit(1) - - # PTRACK BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector " - "from generate_series(200,300) i") - - try: - self.backup_node(backup_dir, 'node', node, backup_type='ptrack') - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of LSN mismatch from ptrack_control " - "and previous backup start_lsn.\n " - "Output: {0}\n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: LSN from ptrack_control' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - @unittest.skip("skip") - def test_full_pgpro417(self): - """ - Make node, take two full backups, delete full second backup. - Try to take ptrack backup, which should fail. - Relevant only for PTRACK 1.x - """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'checkpoint_timeout': '30s'}) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # FULL BACKUP - node.safe_psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text," - " md5(i::text)::tsvector as tsvector " - " from generate_series(0,100) i" - ) - node.table_checksum("t_heap") - self.backup_node(backup_dir, 'node', node, options=["--stream"]) - - # SECOND FULL BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text," - " md5(i::text)::tsvector as tsvector" - " from generate_series(100,200) i" - ) - node.table_checksum("t_heap") - backup_id = self.backup_node( - backup_dir, 'node', node, options=["--stream"]) - - self.delete_pb(backup_dir, 'node', backup_id) - - # PTRACK BACKUP - node.safe_psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(i::text)::tsvector as tsvector " - "from generate_series(200,300) i") - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=["--stream"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of LSN mismatch from ptrack_control " - "and previous backup start_lsn.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd) - ) - except ProbackupException as e: - self.assertTrue( - "ERROR: LSN from ptrack_control" in e.message and - "Create new full backup before " - "an incremental one" in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - # @unittest.skip("skip") def test_create_db(self): """ Make node, take full backup, create database db1, take ptrack backup, restore database and check it presense """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], pg_options={ 'max_wal_size': '10GB'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -1472,8 +1058,7 @@ def test_create_db(self): "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") node.table_checksum("t_heap") - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=["--stream"]) # CREATE DATABASE DB1 @@ -1484,20 +1069,17 @@ def test_create_db(self): "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") # PTRACK BACKUP - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, backup_type='ptrack', options=["--stream"]) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, backup_id=backup_id, options=["-j", "4"]) # COMPARE PHYSICAL CONTENT @@ -1507,16 +1089,14 @@ def test_create_db(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - self.set_auto_conf( - node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() # DROP DATABASE DB1 node.safe_psql( "postgres", "drop database db1") # SECOND PTRACK BACKUP - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, backup_type='ptrack', options=["--stream"] ) @@ -1525,8 +1105,7 @@ def test_create_db(self): # RESTORE SECOND PTRACK BACKUP node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, backup_id=backup_id, options=["-j", "4"]) # COMPARE PHYSICAL CONTENT @@ -1536,23 +1115,12 @@ def test_create_db(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - self.set_auto_conf( - node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() - try: - node_restored.safe_psql('db1', 'select 1') - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because we are connecting to deleted database" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except QueryException as e: - self.assertTrue( - 'FATAL: database "db1" does not exist' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + error_result = node_restored.safe_psql('db1', 'select 1', expect_error=True) + + self.assertMessage(error_result, contains='FATAL: database "db1" does not exist') # @unittest.skip("skip") def test_create_db_on_replica(self): @@ -1562,17 +1130,15 @@ def test_create_db_on_replica(self): create database db1, take ptrack backup from replica, restore database and check it presense """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -1585,29 +1151,20 @@ def test_create_db_on_replica(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(i::text)::tsvector as tsvector from generate_series(0,100) i") - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) + self.pb.backup_node('node', node, options=['-j10', '--stream']) - self.restore_node(backup_dir, 'node', replica) + self.pb.restore_node('node', node=replica) # Add replica - self.add_instance(backup_dir, 'replica', replica) + self.pb.add_instance('replica', replica) self.set_replica(node, replica, 'replica', synchronous=True) replica.slow_start(replica=True) - self.backup_node( - backup_dir, 'replica', replica, - options=[ - '-j10', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(node.port), - '--stream' - ] + self.pb.backup_node('replica', replica, + options=['-j10', '--stream'] ) # CREATE DATABASE DB1 @@ -1622,28 +1179,19 @@ def test_create_db_on_replica(self): replica.safe_psql('postgres', 'checkpoint') # PTRACK BACKUP - backup_id = self.backup_node( - backup_dir, 'replica', + backup_id = self.pb.backup_node('replica', replica, backup_type='ptrack', - options=[ - '-j10', - '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(node.port) - ] + options=['-j10', '--stream'] ) if self.paranoia: pgdata = self.pgdata_content(replica.data_dir) # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'replica', node_restored, + self.pb.restore_node('replica', node_restored, backup_id=backup_id, options=["-j", "4"]) # COMPARE PHYSICAL CONTENT @@ -1656,17 +1204,15 @@ def test_create_db_on_replica(self): def test_alter_table_set_tablespace_ptrack(self): """Make node, create tablespace with table, take full backup, alter tablespace location, take ptrack backup, restore database.""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -1681,7 +1227,7 @@ def test_alter_table_set_tablespace_ptrack(self): " md5(i::text) as text, md5(i::text)::tsvector as tsvector" " from generate_series(0,100) i") # FULL backup - self.backup_node(backup_dir, 'node', node, options=["--stream"]) + self.pb.backup_node('node', node, options=["--stream"]) # ALTER TABLESPACE self.create_tblspace_in_node(node, 'somedata_new') @@ -1692,8 +1238,7 @@ def test_alter_table_set_tablespace_ptrack(self): # sys.exit(1) # PTRACK BACKUP #result = node.table_checksum("t_heap") - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='ptrack', options=["--stream"] ) @@ -1703,12 +1248,10 @@ def test_alter_table_set_tablespace_ptrack(self): # node.cleanup() # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, options=[ "-j", "4", "-T", "{0}={1}".format( @@ -1729,8 +1272,7 @@ def test_alter_table_set_tablespace_ptrack(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - self.set_auto_conf( - node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() # result_new = node_restored.table_checksum("t_heap") @@ -1742,17 +1284,15 @@ def test_alter_database_set_tablespace_ptrack(self): """Make node, create tablespace with database," " take full backup, alter tablespace location," " take ptrack backup, restore database.""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -1760,7 +1300,7 @@ def test_alter_database_set_tablespace_ptrack(self): "CREATE EXTENSION ptrack") # FULL BACKUP - self.backup_node(backup_dir, 'node', node, options=["--stream"]) + self.pb.backup_node('node', node, options=["--stream"]) # CREATE TABLESPACE self.create_tblspace_in_node(node, 'somedata') @@ -1771,8 +1311,7 @@ def test_alter_database_set_tablespace_ptrack(self): "alter database postgres set tablespace somedata") # PTRACK BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + self.pb.backup_node('node', node, backup_type='ptrack', options=["--stream"]) if self.paranoia: @@ -1780,11 +1319,9 @@ def test_alter_database_set_tablespace_ptrack(self): node.stop() # RESTORE - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', + self.pb.restore_node('node', node_restored, options=[ "-j", "4", @@ -1808,17 +1345,15 @@ def test_drop_tablespace(self): Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -1835,30 +1370,27 @@ def test_drop_tablespace(self): result = node.table_checksum("t_heap") # FULL BACKUP - self.backup_node(backup_dir, 'node', node, options=["--stream"]) + self.pb.backup_node('node', node, options=["--stream"]) # Move table to tablespace 'somedata' node.safe_psql( "postgres", "alter table t_heap set tablespace somedata") # PTRACK BACKUP - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='ptrack', options=["--stream"]) # Move table back to default tablespace node.safe_psql( "postgres", "alter table t_heap set tablespace pg_default") # SECOND PTRACK BACKUP - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='ptrack', options=["--stream"]) # DROP TABLESPACE 'somedata' node.safe_psql( "postgres", "drop tablespace somedata") # THIRD PTRACK BACKUP - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='ptrack', options=["--stream"]) if self.paranoia: @@ -1868,7 +1400,7 @@ def test_drop_tablespace(self): tblspace = self.get_tblspace_path(node, 'somedata') node.cleanup() shutil.rmtree(tblspace, ignore_errors=True) - self.restore_node(backup_dir, 'node', node, options=["-j", "4"]) + self.pb.restore_node('node', node=node, options=["-j", "4"]) if self.paranoia: pgdata_restored = self.pgdata_content( @@ -1899,17 +1431,15 @@ def test_ptrack_alter_tablespace(self): Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -1927,7 +1457,7 @@ def test_ptrack_alter_tablespace(self): result = node.table_checksum("t_heap") # FULL BACKUP - self.backup_node(backup_dir, 'node', node, options=["--stream"]) + self.pb.backup_node('node', node, options=["--stream"]) # Move table to separate tablespace node.safe_psql( @@ -1937,8 +1467,7 @@ def test_ptrack_alter_tablespace(self): result = node.table_checksum("t_heap") # FIRTS PTRACK BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + self.pb.backup_node('node', node, backup_type='ptrack', options=["--stream"]) # GET PHYSICAL CONTENT FROM NODE @@ -1946,12 +1475,11 @@ def test_ptrack_alter_tablespace(self): pgdata = self.pgdata_content(node.data_dir) # Restore ptrack backup - restored_node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) + restored_node = self.pg_node.make_simple('restored_node') restored_node.cleanup() tblspc_path_new = self.get_tblspace_path( restored_node, 'somedata_restored') - self.restore_node(backup_dir, 'node', restored_node, options=[ + self.pb.restore_node('node', node=restored_node, options=[ "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) # GET PHYSICAL CONTENT FROM RESTORED NODE and COMPARE PHYSICAL CONTENT @@ -1961,8 +1489,7 @@ def test_ptrack_alter_tablespace(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - self.set_auto_conf( - restored_node, {'port': restored_node.port}) + restored_node.set_auto_conf({'port': restored_node.port}) restored_node.slow_start() # COMPARE LOGICAL CONTENT @@ -1976,16 +1503,14 @@ def test_ptrack_alter_tablespace(self): node.safe_psql( "postgres", "alter table t_heap set tablespace pg_default") # SECOND PTRACK BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + self.pb.backup_node('node', node, backup_type='ptrack', options=["--stream"]) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) # Restore second ptrack backup and check table consistency - self.restore_node( - backup_dir, 'node', restored_node, + self.pb.restore_node('node', restored_node, options=[ "-j", "4", "-T", "{0}={1}".format(tblspc_path, tblspc_path_new)]) @@ -1996,8 +1521,7 @@ def test_ptrack_alter_tablespace(self): self.compare_pgdata(pgdata, pgdata_restored) # START RESTORED NODE - self.set_auto_conf( - restored_node, {'port': restored_node.port}) + restored_node.set_auto_conf({'port': restored_node.port}) restored_node.slow_start() result_new = restored_node.table_checksum("t_heap") @@ -2009,17 +1533,15 @@ def test_ptrack_multiple_segments(self): Make node, create table, alter table tablespace, take ptrack backup, move table from tablespace, take ptrack backup """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], pg_options={ 'full_page_writes': 'off'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -2032,7 +1554,7 @@ def test_ptrack_multiple_segments(self): node.pgbench_init(scale=100, options=['--tablespace=somedata']) result = node.table_checksum("pgbench_accounts") # FULL BACKUP - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) # PTRACK STUFF if node.major_version < 11: @@ -2069,23 +1591,20 @@ def test_ptrack_multiple_segments(self): # it`s stupid, because hint`s are ignored by ptrack result = node.table_checksum("pgbench_accounts") # FIRTS PTRACK BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + self.pb.backup_node('node', node, backup_type='ptrack', options=['--stream']) # GET PHYSICAL CONTENT FROM NODE pgdata = self.pgdata_content(node.data_dir) # RESTORE NODE - restored_node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'restored_node')) + restored_node = self.pg_node.make_simple('restored_node') restored_node.cleanup() tblspc_path = self.get_tblspace_path(node, 'somedata') tblspc_path_new = self.get_tblspace_path( restored_node, 'somedata_restored') - self.restore_node( - backup_dir, 'node', restored_node, + self.pb.restore_node('node', restored_node, options=[ "-j", "4", "-T", "{0}={1}".format( tblspc_path, tblspc_path_new)]) @@ -2096,8 +1615,7 @@ def test_ptrack_multiple_segments(self): restored_node.data_dir, ignore_ptrack=False) # START RESTORED NODE - self.set_auto_conf( - restored_node, {'port': restored_node.port}) + restored_node.set_auto_conf({'port': restored_node.port}) restored_node.slow_start() result_new = restored_node.table_checksum("pgbench_accounts") @@ -2108,307 +1626,16 @@ def test_ptrack_multiple_segments(self): if self.paranoia: self.compare_pgdata(pgdata, pgdata_restored) - @unittest.skip("skip") - def test_atexit_fail(self): - """ - Take backups of every available types and check that PTRACK is clean. - Relevant only for PTRACK 1.x - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'max_connections': '15'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - # Take FULL backup to clean every ptrack - self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - try: - self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', - options=["--stream", "-j 30"]) - - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because we are opening too many connections" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd) - ) - except ProbackupException as e: - self.assertIn( - 'setting its status to ERROR', - e.message, - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(e.message), self.cmd) - ) - - self.assertEqual( - node.safe_psql( - "postgres", - "select * from pg_is_in_backup()").rstrip(), - "f") - - @unittest.skip("skip") - # @unittest.expectedFailure - def test_ptrack_clean(self): - """ - Take backups of every available types and check that PTRACK is clean - Relevant only for PTRACK 1.x - """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - node.slow_start() - - self.create_tblspace_in_node(node, 'somedata') - - # Create table and indexes - node.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap tablespace somedata " - "as select i as id, nextval('t_seq') as t_seq, " - "md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - node.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3}) " - "tablespace somedata".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - # Take FULL backup to clean every ptrack - self.backup_node( - backup_dir, 'node', node, - options=['-j10', '--stream']) - node.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get fork size and calculate it in pages - idx_ptrack[i]['size'] = self.get_fork_size(node, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - - # Update everything and vacuum it - node.safe_psql( - 'postgres', - "update t_heap set t_seq = nextval('t_seq'), " - "text = md5(text), " - "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") - node.safe_psql('postgres', 'vacuum t_heap') - - # Take PTRACK backup to clean every ptrack - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', options=['-j10', '--stream']) - - node.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get new size of heap and indexes and calculate it in pages - idx_ptrack[i]['size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - # check that ptrack bits are cleaned - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - - # Update everything and vacuum it - node.safe_psql( - 'postgres', - "update t_heap set t_seq = nextval('t_seq'), " - "text = md5(text), " - "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") - node.safe_psql('postgres', 'vacuum t_heap') - - # Take PAGE backup to clean every ptrack - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=['-j10', '--stream']) - node.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get new size of heap and indexes and calculate it in pages - idx_ptrack[i]['size'] = self.get_fork_size(node, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(node, i) - # # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - node, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - # check that ptrack bits are cleaned - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - - @unittest.skip("skip") - def test_ptrack_clean_replica(self): - """ - Take backups of every available types from - master and check that PTRACK on replica is clean. - Relevant only for PTRACK 1.x - """ - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums'], - pg_options={ - 'archive_timeout': '30s'}) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - master.slow_start() - - self.backup_node(backup_dir, 'master', master, options=['--stream']) - - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) - replica.cleanup() - - self.restore_node(backup_dir, 'master', replica) - - self.add_instance(backup_dir, 'replica', replica) - self.set_replica(master, replica, synchronous=True) - replica.slow_start(replica=True) - - # Create table and indexes - master.safe_psql( - "postgres", - "create extension bloom; create sequence t_seq; " - "create table t_heap as select i as id, " - "nextval('t_seq') as t_seq, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") - for i in idx_ptrack: - if idx_ptrack[i]['type'] != 'heap' and idx_ptrack[i]['type'] != 'seq': - master.safe_psql( - "postgres", - "create index {0} on {1} using {2}({3})".format( - i, idx_ptrack[i]['relation'], - idx_ptrack[i]['type'], - idx_ptrack[i]['column'])) - - # Take FULL backup to clean every ptrack - self.backup_node( - backup_dir, - 'replica', - replica, - options=[ - '-j10', '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) - master.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get fork size and calculate it in pages - idx_ptrack[i]['size'] = self.get_fork_size(replica, i) - # get path to heap and index files - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - - # Update everything and vacuum it - master.safe_psql( - 'postgres', - "update t_heap set t_seq = nextval('t_seq'), " - "text = md5(text), " - "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") - master.safe_psql('postgres', 'vacuum t_heap') - - # Take PTRACK backup to clean every ptrack - backup_id = self.backup_node( - backup_dir, - 'replica', - replica, - backup_type='ptrack', - options=[ - '-j10', '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) - master.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get new size of heap and indexes and calculate it in pages - idx_ptrack[i]['size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - # check that ptrack bits are cleaned - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - - # Update everything and vacuum it - master.safe_psql( - 'postgres', - "update t_heap set t_seq = nextval('t_seq'), text = md5(text), " - "tsvector = md5(repeat(tsvector::text, 10))::tsvector;") - master.safe_psql('postgres', 'vacuum t_heap') - master.safe_psql('postgres', 'checkpoint') - - # Take PAGE backup to clean every ptrack - self.backup_node( - backup_dir, - 'replica', - replica, - backup_type='page', - options=[ - '-j10', '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream']) - master.safe_psql('postgres', 'checkpoint') - - for i in idx_ptrack: - # get new size of heap and indexes and calculate it in pages - idx_ptrack[i]['size'] = self.get_fork_size(replica, i) - # update path to heap and index files in case they`ve changed - idx_ptrack[i]['path'] = self.get_fork_path(replica, i) - # # get ptrack for every idx - idx_ptrack[i]['ptrack'] = self.get_ptrack_bits_per_page_for_fork( - replica, idx_ptrack[i]['path'], [idx_ptrack[i]['size']]) - # check that ptrack bits are cleaned - self.check_ptrack_clean(idx_ptrack[i], idx_ptrack[i]['size']) - # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_cluster_on_btree(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -2447,8 +1674,7 @@ def test_ptrack_cluster_on_btree(self): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) + self.pb.backup_node('node', node, options=['-j10', '--stream']) node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') node.safe_psql('postgres', 'cluster t_heap using t_btree') @@ -2460,15 +1686,13 @@ def test_ptrack_cluster_on_btree(self): # @unittest.skip("skip") def test_ptrack_cluster_on_gist(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -2503,8 +1727,7 @@ def test_ptrack_cluster_on_gist(self): idx_ptrack[i]['old_pages'] = self.get_md5_per_page_for_fork( idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) + self.pb.backup_node('node', node, options=['-j10', '--stream']) node.safe_psql('postgres', 'delete from t_heap where id%2 = 1') node.safe_psql('postgres', 'cluster t_heap using t_gist') @@ -2514,29 +1737,26 @@ def test_ptrack_cluster_on_gist(self): if node.major_version < 11: self.check_ptrack_map_sanity(node, idx_ptrack) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='ptrack', options=['-j10', '--stream']) pgdata = self.pgdata_content(node.data_dir) node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") def test_ptrack_cluster_on_btree_replica(self): - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + master = self.pg_node.make_simple('master', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('master', master) master.slow_start() if master.major_version >= 11: @@ -2544,15 +1764,14 @@ def test_ptrack_cluster_on_btree_replica(self): "postgres", "CREATE EXTENSION ptrack") - self.backup_node(backup_dir, 'master', master, options=['--stream']) + self.pb.backup_node('master', master, options=['--stream']) - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'master', replica) + self.pb.restore_node('master', node=replica) - self.add_instance(backup_dir, 'replica', replica) + self.pb.add_instance('replica', replica) self.set_replica(master, replica, synchronous=True) replica.slow_start(replica=True) @@ -2577,11 +1796,7 @@ def test_ptrack_cluster_on_btree_replica(self): master.safe_psql('postgres', 'vacuum t_heap') master.safe_psql('postgres', 'checkpoint') - self.backup_node( - backup_dir, 'replica', replica, options=[ - '-j10', '--stream', '--master-host=localhost', - '--master-db=postgres', '--master-port={0}'.format( - master.port)]) + self.pb.backup_node('replica', replica, options=['-j10', '--stream']) for i in idx_ptrack: # get size of heap and indexes. size calculated in pages @@ -2604,31 +1819,28 @@ def test_ptrack_cluster_on_btree_replica(self): if master.major_version < 11: self.check_ptrack_map_sanity(replica, idx_ptrack) - self.backup_node( - backup_dir, 'replica', replica, + self.pb.backup_node('replica', replica, backup_type='ptrack', options=['-j10', '--stream']) pgdata = self.pgdata_content(replica.data_dir) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node')) + node = self.pg_node.make_simple('node') node.cleanup() - self.restore_node(backup_dir, 'replica', node) + self.pb.restore_node('replica', node=node) pgdata_restored = self.pgdata_content(replica.data_dir) self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") def test_ptrack_cluster_on_gist_replica(self): - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + master = self.pg_node.make_simple('master', set_replication=True, ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('master', master) master.slow_start() if master.major_version >= 11: @@ -2636,15 +1848,14 @@ def test_ptrack_cluster_on_gist_replica(self): "postgres", "CREATE EXTENSION ptrack") - self.backup_node(backup_dir, 'master', master, options=['--stream']) + self.pb.backup_node('master', master, options=['--stream']) - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'master', replica) + self.pb.restore_node('master', node=replica) - self.add_instance(backup_dir, 'replica', replica) + self.pb.add_instance('replica', replica) self.set_replica(master, replica, 'replica', synchronous=True) replica.slow_start(replica=True) @@ -2673,11 +1884,8 @@ def test_ptrack_cluster_on_gist_replica(self): self.wait_until_replica_catch_with_master(master, replica) replica.safe_psql('postgres', 'checkpoint') - self.backup_node( - backup_dir, 'replica', replica, options=[ - '-j10', '--stream', '--master-host=localhost', - '--master-db=postgres', '--master-port={0}'.format( - master.port)]) + self.pb.backup_node('replica', replica, options=[ + '-j10', '--stream']) for i in idx_ptrack: # get size of heap and indexes. size calculated in pages @@ -2701,18 +1909,16 @@ def test_ptrack_cluster_on_gist_replica(self): replica.safe_psql('postgres', 'CHECKPOINT') self.check_ptrack_map_sanity(replica, idx_ptrack) - self.backup_node( - backup_dir, 'replica', replica, + self.pb.backup_node('replica', replica, backup_type='ptrack', options=['-j10', '--stream']) if self.paranoia: pgdata = self.pgdata_content(replica.data_dir) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node')) + node = self.pg_node.make_simple('node') node.cleanup() - self.restore_node(backup_dir, 'replica', node) + self.pb.restore_node('replica', node) if self.paranoia: pgdata_restored = self.pgdata_content(replica.data_dir) @@ -2722,15 +1928,13 @@ def test_ptrack_cluster_on_gist_replica(self): # @unittest.expectedFailure def test_ptrack_empty(self): """Take backups of every available types and check that PTRACK is clean""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -2748,8 +1952,7 @@ def test_ptrack_empty(self): "tablespace somedata") # Take FULL backup to clean every ptrack - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=['-j10', '--stream']) # Create indexes @@ -2765,23 +1968,20 @@ def test_ptrack_empty(self): node.safe_psql('postgres', 'checkpoint') - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() tblspace1 = self.get_tblspace_path(node, 'somedata') tblspace2 = self.get_tblspace_path(node_restored, 'somedata') # Take PTRACK backup - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='ptrack', + backup_id = self.pb.backup_node('node', node, backup_type='ptrack', options=['-j10', '--stream']) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) - self.restore_node( - backup_dir, 'node', node_restored, + self.pb.restore_node('node', node_restored, backup_id=backup_id, options=[ "-j", "4", @@ -2798,15 +1998,13 @@ def test_ptrack_empty_replica(self): Take backups of every available types from master and check that PTRACK on replica is clean """ - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + master = self.pg_node.make_simple('master', set_replication=True, - initdb_params=['--data-checksums'], ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('master', master) master.slow_start() if master.major_version >= 11: @@ -2814,15 +2012,14 @@ def test_ptrack_empty_replica(self): "postgres", "CREATE EXTENSION ptrack") - self.backup_node(backup_dir, 'master', master, options=['--stream']) + self.pb.backup_node('master', master, options=['--stream']) - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'master', replica) + self.pb.restore_node('master', node=replica) - self.add_instance(backup_dir, 'replica', replica) + self.pb.add_instance('replica', replica) self.set_replica(master, replica, synchronous=True) replica.slow_start(replica=True) @@ -2835,15 +2032,10 @@ def test_ptrack_empty_replica(self): self.wait_until_replica_catch_with_master(master, replica) # Take FULL backup - self.backup_node( - backup_dir, + self.pb.backup_node( 'replica', replica, - options=[ - '-j10', '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + options=['-j10', '--stream']) # Create indexes for i in idx_ptrack: @@ -2858,26 +2050,19 @@ def test_ptrack_empty_replica(self): self.wait_until_replica_catch_with_master(master, replica) # Take PTRACK backup - backup_id = self.backup_node( - backup_dir, + backup_id = self.pb.backup_node( 'replica', replica, backup_type='ptrack', - options=[ - '-j1', '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + options=['-j1', '--stream']) if self.paranoia: pgdata = self.pgdata_content(replica.data_dir) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'replica', node_restored, + self.pb.restore_node('replica', node_restored, backup_id=backup_id, options=["-j", "4"]) if self.paranoia: @@ -2887,15 +2072,13 @@ def test_ptrack_empty_replica(self): # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_truncate(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -2923,8 +2106,7 @@ def test_ptrack_truncate(self): i, idx_ptrack[i]['relation'], idx_ptrack[i]['type'], idx_ptrack[i]['column'])) - self.backup_node( - backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) node.safe_psql('postgres', 'truncate t_heap') node.safe_psql('postgres', 'checkpoint') @@ -2940,8 +2122,7 @@ def test_ptrack_truncate(self): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # Make backup to clean every ptrack - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='ptrack', options=['-j10', '--stream']) pgdata = self.pgdata_content(node.data_dir) @@ -2957,26 +2138,24 @@ def test_ptrack_truncate(self): self.get_tblspace_path(node, 'somedata'), ignore_errors=True) - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") def test_basic_ptrack_truncate_replica(self): - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + master = self.pg_node.make_simple('master', set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], pg_options={ 'max_wal_size': '32MB', 'archive_timeout': '10s', 'checkpoint_timeout': '5min'}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('master', master) master.slow_start() if master.major_version >= 11: @@ -2984,15 +2163,14 @@ def test_basic_ptrack_truncate_replica(self): "postgres", "CREATE EXTENSION ptrack") - self.backup_node(backup_dir, 'master', master, options=['--stream']) + self.pb.backup_node('master', master, options=['--stream']) - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'master', replica) + self.pb.restore_node('master', node=replica) - self.add_instance(backup_dir, 'replica', replica) + self.pb.add_instance('replica', replica) self.set_replica(master, replica, 'replica', synchronous=True) replica.slow_start(replica=True) @@ -3028,14 +2206,8 @@ def test_basic_ptrack_truncate_replica(self): idx_ptrack[i]['path'], idx_ptrack[i]['old_size']) # Make backup to clean every ptrack - self.backup_node( - backup_dir, 'replica', replica, - options=[ - '-j10', - '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + self.pb.backup_node('replica', replica, + options=['-j10', '--stream']) if replica.major_version < 11: for i in idx_ptrack: @@ -3057,29 +2229,22 @@ def test_basic_ptrack_truncate_replica(self): "postgres", "select pg_wal_replay_pause()") - self.backup_node( - backup_dir, 'replica', replica, backup_type='ptrack', - options=[ - '-j10', - '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + self.pb.backup_node('replica', replica, backup_type='ptrack', + options=['-j10', '--stream']) pgdata = self.pgdata_content(replica.data_dir) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node')) + node = self.pg_node.make_simple('node') node.cleanup() - self.restore_node(backup_dir, 'replica', node, data_dir=node.data_dir) + self.pb.restore_node('replica', node) pgdata_restored = self.pgdata_content(node.data_dir) if self.paranoia: self.compare_pgdata(pgdata, pgdata_restored) - self.set_auto_conf(node, {'port': node.port}) + node.set_auto_conf({'port': node.port}) node.slow_start() @@ -3090,15 +2255,13 @@ def test_basic_ptrack_truncate_replica(self): # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -3131,8 +2294,7 @@ def test_ptrack_vacuum(self): node.safe_psql('postgres', 'checkpoint') # Make full backup to clean every ptrack - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) + self.pb.backup_node('node', node, options=['-j10', '--stream']) if node.major_version < 11: for i in idx_ptrack: @@ -3156,8 +2318,7 @@ def test_ptrack_vacuum(self): if node.major_version < 11: self.check_ptrack_map_sanity(node, idx_ptrack) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='ptrack', options=['-j10', '--stream']) pgdata = self.pgdata_content(node.data_dir) @@ -3167,24 +2328,22 @@ def test_ptrack_vacuum(self): self.get_tblspace_path(node, 'somedata'), ignore_errors=True) - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion) # @unittest.skip("skip") def test_ptrack_vacuum_replica(self): - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + master = self.pg_node.make_simple('master', set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30'}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('master', master) master.slow_start() if master.major_version >= 11: @@ -3192,15 +2351,14 @@ def test_ptrack_vacuum_replica(self): "postgres", "CREATE EXTENSION ptrack") - self.backup_node(backup_dir, 'master', master, options=['--stream']) + self.pb.backup_node('master', master, options=['--stream']) - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'master', replica) + self.pb.restore_node('master', node=replica) - self.add_instance(backup_dir, 'replica', replica) + self.pb.add_instance('replica', replica) self.set_replica(master, replica, 'replica', synchronous=True) replica.slow_start(replica=True) @@ -3228,12 +2386,7 @@ def test_ptrack_vacuum_replica(self): replica.safe_psql('postgres', 'checkpoint') # Make FULL backup to clean every ptrack - self.backup_node( - backup_dir, 'replica', replica, options=[ - '-j10', '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream']) + self.pb.backup_node('replica', replica, options=['-j10', '--stream']) if replica.major_version < 11: for i in idx_ptrack: @@ -3261,17 +2414,15 @@ def test_ptrack_vacuum_replica(self): if replica.major_version < 11: self.check_ptrack_map_sanity(master, idx_ptrack) - self.backup_node( - backup_dir, 'replica', replica, + self.pb.backup_node('replica', replica, backup_type='ptrack', options=['-j10', '--stream']) pgdata = self.pgdata_content(replica.data_dir) - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node')) + node = self.pg_node.make_simple('node') node.cleanup() - self.restore_node(backup_dir, 'replica', node, data_dir=node.data_dir) + self.pb.restore_node('replica', node) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -3279,15 +2430,13 @@ def test_ptrack_vacuum_replica(self): # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_bits_frozen(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -3317,8 +2466,7 @@ def test_ptrack_vacuum_bits_frozen(self): comparision_exclusion = self.get_known_bugs_comparision_exclusion_dict(node) node.safe_psql('postgres', 'checkpoint') - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) + self.pb.backup_node('node', node, options=['-j10', '--stream']) node.safe_psql('postgres', 'vacuum freeze t_heap') node.safe_psql('postgres', 'checkpoint') @@ -3337,8 +2485,7 @@ def test_ptrack_vacuum_bits_frozen(self): if node.major_version < 11: self.check_ptrack_map_sanity(node, idx_ptrack) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='ptrack', options=['-j10', '--stream']) pgdata = self.pgdata_content(node.data_dir) @@ -3347,22 +2494,20 @@ def test_ptrack_vacuum_bits_frozen(self): self.get_tblspace_path(node, 'somedata'), ignore_errors=True) - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion) # @unittest.skip("skip") def test_ptrack_vacuum_bits_frozen_replica(self): - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + master = self.pg_node.make_simple('master', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('master', master) master.slow_start() if master.major_version >= 11: @@ -3370,15 +2515,14 @@ def test_ptrack_vacuum_bits_frozen_replica(self): "postgres", "CREATE EXTENSION ptrack") - self.backup_node(backup_dir, 'master', master, options=['--stream']) + self.pb.backup_node('master', master, options=['--stream']) - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'master', replica) + self.pb.restore_node('master', node=replica) - self.add_instance(backup_dir, 'replica', replica) + self.pb.add_instance('replica', replica) self.set_replica(master, replica, synchronous=True) replica.slow_start(replica=True) @@ -3405,14 +2549,8 @@ def test_ptrack_vacuum_bits_frozen_replica(self): replica.safe_psql('postgres', 'checkpoint') # Take backup to clean every ptrack - self.backup_node( - backup_dir, 'replica', replica, - options=[ - '-j10', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream']) + self.pb.backup_node('replica', replica, + options=['-j10', '--stream']) if replica.major_version < 11: for i in idx_ptrack: @@ -3435,14 +2573,13 @@ def test_ptrack_vacuum_bits_frozen_replica(self): if replica.major_version < 11: self.check_ptrack_map_sanity(master, idx_ptrack) - self.backup_node( - backup_dir, 'replica', replica, backup_type='ptrack', + self.pb.backup_node('replica', replica, backup_type='ptrack', options=['-j10', '--stream']) pgdata = self.pgdata_content(replica.data_dir) replica.cleanup() - self.restore_node(backup_dir, 'replica', replica) + self.pb.restore_node('replica', node=replica) pgdata_restored = self.pgdata_content(replica.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -3450,15 +2587,13 @@ def test_ptrack_vacuum_bits_frozen_replica(self): # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_bits_visibility(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -3488,8 +2623,7 @@ def test_ptrack_vacuum_bits_visibility(self): comparision_exclusion = self.get_known_bugs_comparision_exclusion_dict(node) node.safe_psql('postgres', 'checkpoint') - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) + self.pb.backup_node('node', node, options=['-j10', '--stream']) if node.major_version < 11: for i in idx_ptrack: @@ -3508,8 +2642,7 @@ def test_ptrack_vacuum_bits_visibility(self): if node.major_version < 11: self.check_ptrack_map_sanity(node, idx_ptrack) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='ptrack', options=['-j10', '--stream']) pgdata = self.pgdata_content(node.data_dir) @@ -3518,7 +2651,7 @@ def test_ptrack_vacuum_bits_visibility(self): self.get_tblspace_path(node, 'somedata'), ignore_errors=True) - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored, comparision_exclusion) @@ -3526,15 +2659,14 @@ def test_ptrack_vacuum_bits_visibility(self): # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_full_2(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=True, pg_options={ 'wal_log_hints': 'on' }) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -3562,8 +2694,7 @@ def test_ptrack_vacuum_full_2(self): node.safe_psql('postgres', 'vacuum t_heap') node.safe_psql('postgres', 'checkpoint') - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) + self.pb.backup_node('node', node, options=['-j10', '--stream']) if node.major_version < 11: for i in idx_ptrack: @@ -3582,8 +2713,7 @@ def test_ptrack_vacuum_full_2(self): if node.major_version < 11: self.check_ptrack_map_sanity(node, idx_ptrack) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='ptrack', options=['-j10', '--stream']) pgdata = self.pgdata_content(node.data_dir) @@ -3593,7 +2723,7 @@ def test_ptrack_vacuum_full_2(self): self.get_tblspace_path(node, 'somedata'), ignore_errors=True) - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -3601,15 +2731,13 @@ def test_ptrack_vacuum_full_2(self): # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_full_replica(self): - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + master = self.pg_node.make_simple('master', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('master', master) master.slow_start() if master.major_version >= 11: @@ -3617,14 +2745,13 @@ def test_ptrack_vacuum_full_replica(self): "postgres", "CREATE EXTENSION ptrack") - self.backup_node(backup_dir, 'master', master, options=['--stream']) - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + self.pb.backup_node('master', master, options=['--stream']) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'master', replica) + self.pb.restore_node('master', node=replica) - self.add_instance(backup_dir, 'replica', replica) + self.pb.add_instance('replica', replica) self.set_replica(master, replica, 'replica', synchronous=True) replica.slow_start(replica=True) @@ -3654,14 +2781,8 @@ def test_ptrack_vacuum_full_replica(self): replica.safe_psql('postgres', 'checkpoint') # Take FULL backup to clean every ptrack - self.backup_node( - backup_dir, 'replica', replica, - options=[ - '-j10', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port), - '--stream']) + self.pb.backup_node('replica', replica, + options=['-j10', '--stream']) if replica.major_version < 11: for i in idx_ptrack: @@ -3684,14 +2805,13 @@ def test_ptrack_vacuum_full_replica(self): if replica.major_version < 11: self.check_ptrack_map_sanity(master, idx_ptrack) - self.backup_node( - backup_dir, 'replica', replica, + self.pb.backup_node('replica', replica, backup_type='ptrack', options=['-j10', '--stream']) pgdata = self.pgdata_content(replica.data_dir) replica.cleanup() - self.restore_node(backup_dir, 'replica', replica) + self.pb.restore_node('replica', node=replica) pgdata_restored = self.pgdata_content(replica.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -3699,15 +2819,13 @@ def test_ptrack_vacuum_full_replica(self): # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_truncate_2(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -3733,8 +2851,7 @@ def test_ptrack_vacuum_truncate_2(self): node.safe_psql('postgres', 'VACUUM t_heap') - self.backup_node( - backup_dir, 'node', node, options=['-j10', '--stream']) + self.pb.backup_node('node', node, options=['-j10', '--stream']) if node.major_version < 11: for i in idx_ptrack: @@ -3754,17 +2871,15 @@ def test_ptrack_vacuum_truncate_2(self): if node.major_version < 11: self.check_ptrack_map_sanity(node, idx_ptrack) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='ptrack', options=['--stream']) pgdata = self.pgdata_content(node.data_dir) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored) + self.pb.restore_node('node', node_restored) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -3772,15 +2887,13 @@ def test_ptrack_vacuum_truncate_2(self): # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_vacuum_truncate_replica(self): - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + master = self.pg_node.make_simple('master', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('master', master) master.slow_start() if master.major_version >= 11: @@ -3788,15 +2901,14 @@ def test_ptrack_vacuum_truncate_replica(self): "postgres", "CREATE EXTENSION ptrack") - self.backup_node(backup_dir, 'master', master, options=['--stream']) + self.pb.backup_node('master', master, options=['--stream']) - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'master', replica) + self.pb.restore_node('master', node=replica) - self.add_instance(backup_dir, 'replica', replica) + self.pb.add_instance('replica', replica) self.set_replica(master, replica, 'replica', synchronous=True) replica.slow_start(replica=True) @@ -3820,15 +2932,8 @@ def test_ptrack_vacuum_truncate_replica(self): master.safe_psql('postgres', 'checkpoint') # Take FULL backup to clean every ptrack - self.backup_node( - backup_dir, 'replica', replica, - options=[ - '-j10', - '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port) - ] + self.pb.backup_node('replica', replica, + options=['-j10', '--stream'] ) if master.major_version < 11: @@ -3853,8 +2958,7 @@ def test_ptrack_vacuum_truncate_replica(self): if master.major_version < 11: self.check_ptrack_map_sanity(master, idx_ptrack) - self.backup_node( - backup_dir, 'replica', replica, backup_type='ptrack', + self.pb.backup_node('replica', replica, backup_type='ptrack', options=[ '--stream', '--log-level-file=INFO', @@ -3862,11 +2966,10 @@ def test_ptrack_vacuum_truncate_replica(self): pgdata = self.pgdata_content(replica.data_dir) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node(backup_dir, 'replica', node_restored) + self.pb.restore_node('replica', node_restored) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -3877,15 +2980,13 @@ def test_ptrack_recovery(self): Check that ptrack map contain correct bits after recovery. Actual only for PTRACK 1.x """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() self.create_tblspace_in_node(node, 'somedata') @@ -3932,18 +3033,16 @@ def test_ptrack_recovery(self): # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_recovery_1(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], pg_options={ 'shared_buffers': '512MB', 'max_wal_size': '3GB'}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -3960,8 +3059,7 @@ def test_ptrack_recovery_1(self): # "from generate_series(0,25600) i") "from generate_series(0,2560) i") - self.backup_node( - backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) # Create indexes for i in idx_ptrack: @@ -3995,18 +3093,15 @@ def test_ptrack_recovery_1(self): print("Die! Die! Why won't you die?... Why won't you die?") exit(1) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='ptrack', options=['--stream']) pgdata = self.pgdata_content(node.data_dir) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored) + self.pb.restore_node('node', node_restored) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -4014,15 +3109,13 @@ def test_ptrack_recovery_1(self): # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_zero_changes(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -4037,17 +3130,15 @@ def test_ptrack_zero_changes(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,2560) i") - self.backup_node( - backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='ptrack', options=['--stream']) pgdata = self.pgdata_content(node.data_dir) node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -4055,18 +3146,16 @@ def test_ptrack_zero_changes(self): # @unittest.skip("skip") # @unittest.expectedFailure def test_ptrack_pg_resetxlog(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=True, - initdb_params=['--data-checksums'], pg_options={ 'shared_buffers': '512MB', 'max_wal_size': '3GB'}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -4083,8 +3172,7 @@ def test_ptrack_pg_resetxlog(self): # "from generate_series(0,25600) i") "from generate_series(0,2560) i") - self.backup_node( - backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) # Create indexes for i in idx_ptrack: @@ -4121,7 +3209,7 @@ def test_ptrack_pg_resetxlog(self): pg_resetxlog_path = self.get_bin_path('pg_resetxlog') wal_dir = 'pg_xlog' - self.run_binary( + self.pb.run_binary( [ pg_resetxlog_path, '-D', @@ -4138,36 +3226,23 @@ def test_ptrack_pg_resetxlog(self): exit(1) # take ptrack backup -# self.backup_node( -# backup_dir, 'node', node, +# self.pb.backup_node( +# 'node', node, # backup_type='ptrack', options=['--stream']) - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because instance was brutalized by pg_resetxlog" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd) - ) - except ProbackupException as e: - self.assertTrue( - 'ERROR: LSN from ptrack_control ' in e.message and - 'is greater than Start LSN of previous backup' in e.message, - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(e.message), self.cmd)) + self.pb.backup_node('node', node, backup_type='ptrack', + options=['--stream'], + expect_error="because instance was brutalized by pg_resetxlog") + self.assertMessage(regex='ERROR: LSN from ptrack_control .* ' + 'is greater than Start LSN of previous backup') # pgdata = self.pgdata_content(node.data_dir) # -# node_restored = self.make_simple_node( -# base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) +# node_restored = self.pg_node.make_simple('node_restored') # node_restored.cleanup() # -# self.restore_node( -# backup_dir, 'node', node_restored) +# self.pb.restore_node( +# 'node', node_restored) # # pgdata_restored = self.pgdata_content(node_restored.data_dir) # self.compare_pgdata(pgdata, pgdata_restored) @@ -4175,15 +3250,13 @@ def test_ptrack_pg_resetxlog(self): # @unittest.skip("skip") # @unittest.expectedFailure def test_corrupt_ptrack_map(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -4201,8 +3274,7 @@ def test_corrupt_ptrack_map(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,2560) i") - self.backup_node( - backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) node.safe_psql( 'postgres', @@ -4264,25 +3336,13 @@ def test_corrupt_ptrack_map(self): 'FATAL: ptrack init: incorrect checksum of file "{0}"'.format(ptrack_map), log_content) - self.set_auto_conf(node, {'ptrack.map_size': '0'}) + node.set_auto_conf({'ptrack.map_size': '0'}) node.slow_start() - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because instance ptrack is disabled" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Ptrack is disabled', - e.message, - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(e.message), self.cmd)) + self.pb.backup_node('node', node, backup_type='ptrack', + options=['--stream'], + expect_error="because instance ptrack is disabled") + self.assertMessage(contains='ERROR: Ptrack is disabled') node.safe_psql( 'postgres', @@ -4291,28 +3351,15 @@ def test_corrupt_ptrack_map(self): node.stop(['-m', 'immediate', '-D', node.data_dir]) - self.set_auto_conf(node, {'ptrack.map_size': '32', 'shared_preload_libraries': 'ptrack'}) + node.set_auto_conf({'ptrack.map_size': '32', 'shared_preload_libraries': 'ptrack'}) node.slow_start() - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='ptrack', options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because ptrack map is from future" - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: LSN from ptrack_control', - e.message, - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(e.message), self.cmd)) + self.pb.backup_node('node', node, backup_type='ptrack', + options=['--stream'], + expect_error="because ptrack map is from future") + self.assertMessage(contains='ERROR: LSN from ptrack_control') - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='delta', options=['--stream']) node.safe_psql( @@ -4320,15 +3367,14 @@ def test_corrupt_ptrack_map(self): "update t_heap set id = nextval('t_seq'), text = md5(text), " "tsvector = md5(repeat(tsvector::text, 10))::tsvector") - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='ptrack', options=['--stream']) pgdata = self.pgdata_content(node.data_dir) node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -4346,15 +3392,13 @@ def test_horizon_lsn_ptrack(self): self.version_to_num('2.4.15'), 'You need pg_probackup old_binary =< 2.4.15 for this test') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.safe_psql( @@ -4367,16 +3411,16 @@ def test_horizon_lsn_ptrack(self): "You need ptrack >=2.1 for this test") # set map_size to a minimal value - self.set_auto_conf(node, {'ptrack.map_size': '1'}) + node.set_auto_conf({'ptrack.map_size': '1'}) node.restart() node.pgbench_init(scale=100) # FULL backup - full_id = self.backup_node(backup_dir, 'node', node, options=['--stream'], old_binary=True) + full_id = self.pb.backup_node('node', node, options=['--stream'], old_binary=True) # enable archiving so the WAL size to do interfere with data bytes comparison later - self.set_archiving(backup_dir, 'node', node) + self.pb.set_archiving('node', node) node.restart() # change data @@ -4384,14 +3428,13 @@ def test_horizon_lsn_ptrack(self): pgbench.wait() # DELTA is exemplar - delta_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') - delta_bytes = self.show_pb(backup_dir, 'node', backup_id=delta_id)["data-bytes"] - self.delete_pb(backup_dir, 'node', backup_id=delta_id) + delta_id = self.pb.backup_node('node', node, backup_type='delta') + delta_bytes = self.pb.show('node', backup_id=delta_id)["data-bytes"] + self.pb.delete('node', backup_id=delta_id) # PTRACK with current binary - ptrack_id = self.backup_node(backup_dir, 'node', node, backup_type='ptrack') - ptrack_bytes = self.show_pb(backup_dir, 'node', backup_id=ptrack_id)["data-bytes"] + ptrack_id = self.pb.backup_node('node', node, backup_type='ptrack') + ptrack_bytes = self.pb.show('node', backup_id=ptrack_id)["data-bytes"] # make sure that backup size is exactly the same self.assertEqual(delta_bytes, ptrack_bytes) diff --git a/tests/remote_test.py b/tests/remote_test.py index 2d36d7346..519380b3d 100644 --- a/tests/remote_test.py +++ b/tests/remote_test.py @@ -1,43 +1,19 @@ -import unittest -import os -from time import sleep -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException -from .helpers.cfs_helpers import find_by_name +from .helpers.ptrack_helpers import ProbackupTest -class RemoteTest(ProbackupTest, unittest.TestCase): +class RemoteTest(ProbackupTest): # @unittest.skip("skip") # @unittest.expectedFailure def test_remote_sanity(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - output = self.backup_node( - backup_dir, 'node', node, + output = self.pb.backup_node('node', node, options=['--stream'], no_remote=True, return_id=False) self.assertIn('remote: false', output) - - # try: - # self.backup_node( - # backup_dir, 'node', - # node, options=['--remote-proto=ssh', '--stream'], no_remote=True) - # # we should die here because exception is what we expect to happen - # self.assertEqual( - # 1, 0, - # "Expecting Error because remote-host option is missing." - # "\n Output: {0} \n CMD: {1}".format( - # repr(self.output), self.cmd)) - # except ProbackupException as e: - # self.assertIn( - # "Insert correct error", - # e.message, - # "\n Unexpected Error Message: {0}\n CMD: {1}".format( - # repr(e.message), self.cmd)) diff --git a/tests/replica_test.py b/tests/replica_test.py index 17fc5a823..7ff539b34 100644 --- a/tests/replica_test.py +++ b/tests/replica_test.py @@ -1,15 +1,12 @@ import os import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException, idx_ptrack -from datetime import datetime, timedelta -import subprocess -import time -from distutils.dir_util import copy_tree +from .helpers.ptrack_helpers import ProbackupTest +from pg_probackup2.gdb import needs_gdb from testgres import ProcessType from time import sleep -class ReplicaTest(ProbackupTest, unittest.TestCase): +class ReplicaTest(ProbackupTest): # @unittest.skip("skip") # @unittest.expectedFailure @@ -19,35 +16,28 @@ def test_replica_switchover(self): over the course of several switchovers https://www.postgresql.org/message-id/54b059d4-2b48-13a4-6f43-95a087c92367%40postgrespro.ru """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), - set_replication=True, - initdb_params=['--data-checksums']) - - if self.get_version(node1) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') + backup_dir = self.backup_dir + node1 = self.pg_node.make_simple('node1', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node1', node1) + self.pb.init() + self.pb.add_instance('node1', node1) node1.slow_start() # take full backup and restore it - self.backup_node(backup_dir, 'node1', node1, options=['--stream']) - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) + self.pb.backup_node('node1', node1, options=['--stream']) + node2 = self.pg_node.make_simple('node2') node2.cleanup() # create replica - self.restore_node(backup_dir, 'node1', node2) + self.pb.restore_node('node1', node=node2) # setup replica - self.add_instance(backup_dir, 'node2', node2) - self.set_archiving(backup_dir, 'node2', node2, replica=True) + self.pb.add_instance('node2', node2) + self.pb.set_archiving('node2', node2, replica=True) self.set_replica(node1, node2, synchronous=False) - self.set_auto_conf(node2, {'port': node2.port}) + node2.set_auto_conf({'port': node2.port}) node2.slow_start(replica=True) @@ -55,7 +45,7 @@ def test_replica_switchover(self): node1.pgbench_init(scale=5) # take full backup on replica - self.backup_node(backup_dir, 'node2', node2, options=['--stream']) + self.pb.backup_node('node2', node2, options=['--stream']) # first switchover node1.stop() @@ -66,8 +56,7 @@ def test_replica_switchover(self): node1.slow_start(replica=True) # take incremental backup from new master - self.backup_node( - backup_dir, 'node2', node2, + self.pb.backup_node('node2', node2, backup_type='delta', options=['--stream']) # second switchover @@ -81,12 +70,11 @@ def test_replica_switchover(self): node1.pgbench_init(scale=5) # take incremental backup from replica - self.backup_node( - backup_dir, 'node2', node2, + self.pb.backup_node('node2', node2, backup_type='delta', options=['--stream']) # https://github.com/postgrespro/pg_probackup/issues/251 - self.validate_pb(backup_dir) + self.pb.validate() # @unittest.skip("skip") # @unittest.expectedFailure @@ -98,26 +86,19 @@ def test_replica_stream_ptrack_backup(self): if not self.ptrack: self.skipTest('Skipped because ptrack support is disabled') - if self.pg_config_version > self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + backup_dir = self.backup_dir + master = self.pg_node.make_simple('master', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) + self.pb.init() + self.pb.add_instance('master', master) master.slow_start() - if master.major_version >= 12: - master.safe_psql( - "postgres", - "CREATE EXTENSION ptrack") + master.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") # CREATE TABLE master.psql( @@ -128,11 +109,10 @@ def test_replica_stream_ptrack_backup(self): before = master.table_checksum("t_heap") # take full backup and restore it - self.backup_node(backup_dir, 'master', master, options=['--stream']) - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + self.pb.backup_node('master', master, options=['--stream']) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'master', replica) + self.pb.restore_node('master', node=replica) self.set_replica(master, replica) # Check data correctness on replica @@ -149,26 +129,20 @@ def test_replica_stream_ptrack_backup(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(256,512) i") before = master.table_checksum("t_heap") - self.add_instance(backup_dir, 'replica', replica) + self.pb.add_instance('replica', replica) - backup_id = self.backup_node( - backup_dir, 'replica', replica, - options=[ - '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) - self.validate_pb(backup_dir, 'replica') + backup_id = self.pb.backup_node('replica', replica, + options=['--stream']) + self.pb.validate('replica') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + 'OK', self.pb.show('replica', backup_id)['status']) # RESTORE FULL BACKUP TAKEN FROM PREVIOUS STEP - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node')) + node = self.pg_node.make_simple('node') node.cleanup() - self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + self.pb.restore_node('replica', node=node) - self.set_auto_conf(node, {'port': node.port}) + node.set_auto_conf({'port': node.port}) node.slow_start() @@ -187,23 +161,17 @@ def test_replica_stream_ptrack_backup(self): before = master.table_checksum("t_heap") - backup_id = self.backup_node( - backup_dir, 'replica', replica, backup_type='ptrack', - options=[ - '--stream', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) - self.validate_pb(backup_dir, 'replica') + backup_id = self.pb.backup_node('replica', replica, backup_type='ptrack', + options=['--stream']) + self.pb.validate('replica') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + 'OK', self.pb.show('replica', backup_id)['status']) # RESTORE PTRACK BACKUP TAKEN FROM replica node.cleanup() - self.restore_node( - backup_dir, 'replica', data_dir=node.data_dir, backup_id=backup_id) + self.pb.restore_node('replica', node, backup_id=backup_id) - self.set_auto_conf(node, {'port': node.port}) + node.set_auto_conf({'port': node.port}) node.slow_start() @@ -212,143 +180,118 @@ def test_replica_stream_ptrack_backup(self): self.assertEqual(before, after) # @unittest.skip("skip") + @needs_gdb def test_replica_archive_page_backup(self): """ make archive master, take full and page archive backups from master, set replica, make archive backup from replica """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + backup_dir = self.backup_dir + master = self.pg_node.make_simple('master', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '10s', 'checkpoint_timeout': '30s', 'max_wal_size': '32MB'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) + self.pb.init() + self.pb.add_instance('master', master) + self.pb.set_archiving('master', master) master.slow_start() - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.backup_node(backup_dir, 'master', master) + self.pb.backup_node('master', master) - master.psql( - "postgres", - "create table t_heap as select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(0,2560) i") + master.pgbench_init(scale=5) - before = master.table_checksum("t_heap") + before = master.table_checksum("pgbench_accounts") - backup_id = self.backup_node( - backup_dir, 'master', master, backup_type='page') - self.restore_node(backup_dir, 'master', replica) + backup_id = self.pb.backup_node('master', master, backup_type='page') + self.pb.restore_node('master', node=replica) # Settings for Replica - self.add_instance(backup_dir, 'replica', replica) + self.pb.add_instance('replica', replica) self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) + self.pb.set_archiving('replica', replica, replica=True) replica.slow_start(replica=True) # Check data correctness on replica - after = replica.table_checksum("t_heap") + after = replica.table_checksum("pgbench_accounts") self.assertEqual(before, after) # Change data on master, take FULL backup from replica, # restore taken backup and check that restored data # equal to original data - master.psql( - "postgres", - "insert into t_heap select i as id, md5(i::text) as text, " - "md5(repeat(i::text,10))::tsvector as tsvector " - "from generate_series(256,25120) i") + pgbench = master.pgbench(options=['-T', '3', '-c', '2', '--no-vacuum']) + pgbench.wait() - before = master.table_checksum("t_heap") + before = master.table_checksum("pgbench_accounts") self.wait_until_replica_catch_with_master(master, replica) - backup_id = self.backup_node( - backup_dir, 'replica', replica, - options=[ - '--archive-timeout=60', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + backup_id, _ = self.pb.backup_replica_node('replica', replica, + master=master, + options=['--archive-timeout=60']) - self.validate_pb(backup_dir, 'replica') + self.pb.validate('replica') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + 'OK', self.pb.show('replica', backup_id)['status']) # RESTORE FULL BACKUP TAKEN FROM replica - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node')) + node = self.pg_node.make_simple('node') node.cleanup() - self.restore_node(backup_dir, 'replica', data_dir=node.data_dir) + self.pb.restore_node('replica', node=node) - self.set_auto_conf(node, {'port': node.port, 'archive_mode': 'off'}) + node.set_auto_conf({'port': node.port, 'archive_mode': 'off'}) node.slow_start() # CHECK DATA CORRECTNESS - after = node.table_checksum("t_heap") + after = node.table_checksum("pgbench_accounts") self.assertEqual(before, after) node.cleanup() # Change data on master, make PAGE backup from replica, # restore taken backup and check that restored data equal # to original data - master.pgbench_init(scale=5) - pgbench = master.pgbench( - options=['-T', '30', '-c', '2', '--no-vacuum']) + options=['-T', '15', '-c', '1', '--no-vacuum']) - backup_id = self.backup_node( - backup_dir, 'replica', + backup_id, _ = self.pb.backup_replica_node('replica', replica, backup_type='page', - options=[ - '--archive-timeout=60', - '--master-host=localhost', - '--master-db=postgres', - '--master-port={0}'.format(master.port)]) + master=master, + options=['--archive-timeout=60']) pgbench.wait() - self.switch_wal_segment(master) + lsn = self.switch_wal_segment(master) before = master.table_checksum("pgbench_accounts") - self.validate_pb(backup_dir, 'replica') + self.pb.validate('replica') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'replica', backup_id)['status']) + 'OK', self.pb.show('replica', backup_id)['status']) # RESTORE PAGE BACKUP TAKEN FROM replica - self.restore_node( - backup_dir, 'replica', data_dir=node.data_dir, + self.pb.restore_node('replica', node, backup_id=backup_id) - self.set_auto_conf(node, {'port': node.port, 'archive_mode': 'off'}) + node.set_auto_conf({'port': node.port, 'archive_mode': 'off'}) node.slow_start() + self.wait_until_lsn_replayed(node, lsn) + # CHECK DATA CORRECTNESS - after = master.table_checksum("pgbench_accounts") + after = node.table_checksum("pgbench_accounts") self.assertEqual( before, after, 'Restored data is not equal to original') - self.add_instance(backup_dir, 'node', node) - self.backup_node( - backup_dir, 'node', node, options=['--stream']) + self.pb.add_instance('node', node) + self.pb.backup_node('node', node, options=['--stream']) # @unittest.skip("skip") def test_basic_make_replica_via_restore(self): @@ -356,28 +299,21 @@ def test_basic_make_replica_via_restore(self): make archive master, take full and page archive backups from master, set replica, make archive backup from replica """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + backup_dir = self.backup_dir + master = self.pg_node.make_simple('master', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '10s'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) + self.pb.init() + self.pb.add_instance('master', master) + self.pb.set_archiving('master', master) master.slow_start() - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.backup_node(backup_dir, 'master', master) + self.pb.backup_node('master', master) master.psql( "postgres", @@ -387,20 +323,17 @@ def test_basic_make_replica_via_restore(self): before = master.table_checksum("t_heap") - backup_id = self.backup_node( - backup_dir, 'master', master, backup_type='page') - self.restore_node( - backup_dir, 'master', replica, options=['-R']) + backup_id = self.pb.backup_node('master', master, backup_type='page') + self.pb.restore_node('master', replica, options=['-R']) # Settings for Replica - self.add_instance(backup_dir, 'replica', replica) - self.set_archiving(backup_dir, 'replica', replica, replica=True) + self.pb.add_instance('replica', replica) + self.pb.set_archiving('replica', replica, replica=True) self.set_replica(master, replica, synchronous=True) replica.slow_start(replica=True) - self.backup_node( - backup_dir, 'replica', replica, + self.pb.backup_node('replica', replica, options=['--archive-timeout=30s', '--stream']) # @unittest.skip("skip") @@ -410,27 +343,20 @@ def test_take_backup_from_delayed_replica(self): restore full backup as delayed replica, launch pgbench, take FULL, PAGE and DELTA backups from replica """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + backup_dir = self.backup_dir + master = self.pg_node.make_simple('master', set_replication=True, - initdb_params=['--data-checksums'], pg_options={'archive_timeout': '10s'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) + self.pb.init() + self.pb.add_instance('master', master) + self.pb.set_archiving('master', master) master.slow_start() - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.backup_node(backup_dir, 'master', master) + self.pb.backup_node('master', master) master.psql( "postgres", @@ -444,22 +370,20 @@ def test_take_backup_from_delayed_replica(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,165000) i") - self.restore_node( - backup_dir, 'master', replica, options=['-R']) + self.pb.restore_node('master', replica, options=['-R']) # Settings for Replica - self.add_instance(backup_dir, 'replica', replica) - self.set_archiving(backup_dir, 'replica', replica, replica=True) + self.pb.add_instance('replica', replica) + self.pb.set_archiving('replica', replica, replica=True) - self.set_auto_conf(replica, {'port': replica.port}) + replica.set_auto_conf({'port': replica.port}) replica.slow_start(replica=True) self.wait_until_replica_catch_with_master(master, replica) - if self.get_version(master) >= self.version_to_num('12.0'): - self.set_auto_conf( - replica, {'recovery_min_apply_delay': '300s'}) + if self.pg_config_version >= self.version_to_num('12.0'): + replica.set_auto_conf({'recovery_min_apply_delay': '300s'}) else: replica.append_conf( 'recovery.conf', @@ -473,19 +397,16 @@ def test_take_backup_from_delayed_replica(self): pgbench = master.pgbench( options=['-T', '60', '-c', '2', '--no-vacuum']) - self.backup_node( - backup_dir, 'replica', + self.pb.backup_node('replica', replica, options=['--archive-timeout=60s']) - self.backup_node( - backup_dir, 'replica', replica, + self.pb.backup_node('replica', replica, data_dir=replica.data_dir, backup_type='page', options=['--archive-timeout=60s']) sleep(1) - self.backup_node( - backup_dir, 'replica', replica, + self.pb.backup_node('replica', replica, backup_type='delta', options=['--archive-timeout=60s']) pgbench.wait() @@ -493,52 +414,42 @@ def test_take_backup_from_delayed_replica(self): pgbench = master.pgbench( options=['-T', '30', '-c', '2', '--no-vacuum']) - self.backup_node( - backup_dir, 'replica', replica, + self.pb.backup_node('replica', replica, options=['--stream']) - self.backup_node( - backup_dir, 'replica', replica, + self.pb.backup_node('replica', replica, backup_type='page', options=['--stream']) - self.backup_node( - backup_dir, 'replica', replica, + self.pb.backup_node('replica', replica, backup_type='delta', options=['--stream']) pgbench.wait() # @unittest.skip("skip") + @needs_gdb def test_replica_promote(self): """ start backup from replica, during backup promote replica check that backup is failed """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + backup_dir = self.backup_dir + master = self.pg_node.make_simple('master', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '10s', 'checkpoint_timeout': '30s', 'max_wal_size': '32MB'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) + self.pb.init() + self.pb.add_instance('master', master) + self.pb.set_archiving('master', master) master.slow_start() - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.backup_node(backup_dir, 'master', master) + self.pb.backup_node('master', master) master.psql( "postgres", @@ -546,12 +457,11 @@ def test_replica_promote(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,165000) i") - self.restore_node( - backup_dir, 'master', replica, options=['-R']) + self.pb.restore_node('master', replica, options=['-R']) # Settings for Replica - self.add_instance(backup_dir, 'replica', replica) - self.set_archiving(backup_dir, 'replica', replica, replica=True) + self.pb.add_instance('replica', replica) + self.pb.set_archiving('replica', replica, replica=True) self.set_replica( master, replica, replica_name='replica', synchronous=True) @@ -566,8 +476,7 @@ def test_replica_promote(self): self.wait_until_replica_catch_with_master(master, replica) # start backup from replica - gdb = self.backup_node( - backup_dir, 'replica', replica, gdb=True, + gdb = self.pb.backup_node('replica', replica, gdb=True, options=['--log-level-file=verbose']) gdb.set_breakpoint('backup_data_file') @@ -576,16 +485,12 @@ def test_replica_promote(self): replica.promote() - gdb.remove_all_breakpoints() gdb.continue_execution_until_exit() - backup_id = self.show_pb( - backup_dir, 'replica')[0]["id"] + backup_id = self.pb.show('replica')[0]["id"] # read log file content - with open(os.path.join(backup_dir, 'log', 'pg_probackup.log')) as f: - log_content = f.read() - f.close + log_content = self.read_pb_log() self.assertIn( 'ERROR: the standby was promoted during online backup', @@ -597,52 +502,44 @@ def test_replica_promote(self): log_content) # @unittest.skip("skip") + @needs_gdb def test_replica_stop_lsn_null_offset(self): """ """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + self.test_env["PGPROBACKUP_TESTS_SKIP_EMPTY_COMMIT"] = "ON" + backup_dir = self.backup_dir + master = self.pg_node.make_simple('master', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '1h', 'wal_level': 'replica'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', master) - self.set_archiving(backup_dir, 'node', master) + self.pb.init() + self.pb.add_instance('node', master) + self.pb.set_archiving('node', master) master.slow_start() # freeze bgwriter to get rid of RUNNING XACTS records bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] - gdb_checkpointer = self.gdb_attach(bgwriter_pid) + gdb_bgwriter = self.gdb_attach(bgwriter_pid) - self.backup_node(backup_dir, 'node', master) + self.pb.backup_node('node', master) # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'node', replica) + self.pb.restore_node('node', node=replica) # Settings for Replica self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'node', replica, replica=True) + self.pb.set_archiving('node', replica, replica=True) replica.slow_start(replica=True) self.switch_wal_segment(master) self.switch_wal_segment(master) - output = self.backup_node( - backup_dir, 'node', replica, replica.data_dir, + output = self.pb.backup_node('node', replica, replica.data_dir, options=[ '--archive-timeout=30', '--log-level-console=LOG', @@ -650,25 +547,9 @@ def test_replica_stop_lsn_null_offset(self): '--stream'], return_id=False) - self.assertIn( - 'LOG: Invalid offset in stop_lsn value 0/4000000', - output) - - self.assertIn( - 'WARNING: WAL segment 000000010000000000000004 could not be streamed in 30 seconds', - output) - - self.assertIn( - 'WARNING: Failed to get next WAL record after 0/4000000, looking for previous WAL record', - output) - - self.assertIn( - 'LOG: Looking for LSN 0/4000000 in segment: 000000010000000000000003', - output) - self.assertIn( 'has endpoint 0/4000000 which is ' - 'equal or greater than requested LSN 0/4000000', + 'equal or greater than requested LSN', output) self.assertIn( @@ -676,62 +557,48 @@ def test_replica_stop_lsn_null_offset(self): output) # Clean after yourself - gdb_checkpointer.kill() + gdb_bgwriter.detach() # @unittest.skip("skip") + @needs_gdb def test_replica_stop_lsn_null_offset_next_record(self): """ """ - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + self.test_env["PGPROBACKUP_TESTS_SKIP_EMPTY_COMMIT"] = "ON" + backup_dir = self.backup_dir + master = self.pg_node.make_simple('master', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '1h', 'wal_level': 'replica'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) + self.pb.init() + self.pb.add_instance('master', master) + self.pb.set_archiving('master', master) master.slow_start() # freeze bgwriter to get rid of RUNNING XACTS records bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] + gdb_bgwriter = self.gdb_attach(bgwriter_pid) - self.backup_node(backup_dir, 'master', master) + self.pb.backup_node('master', master) # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'master', replica) + self.pb.restore_node('master', node=replica) # Settings for Replica - self.add_instance(backup_dir, 'replica', replica) + self.pb.add_instance('replica', replica) self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - - copy_tree( - os.path.join(backup_dir, 'wal', 'master'), - os.path.join(backup_dir, 'wal', 'replica')) + self.pb.set_archiving('replica', replica, replica=True) replica.slow_start(replica=True) - self.switch_wal_segment(master) - self.switch_wal_segment(master) - # open connection to master conn = master.connect() - gdb = self.backup_node( - backup_dir, 'replica', replica, + gdb = self.pb.backup_node('replica', replica, options=[ '--archive-timeout=40', '--log-level-file=LOG', @@ -740,255 +607,189 @@ def test_replica_stop_lsn_null_offset_next_record(self): gdb=True) # Attention! this breakpoint is set to a probackup internal function, not a postgres core one - gdb.set_breakpoint('pg_stop_backup') + gdb.set_breakpoint('pg_stop_backup_consume') gdb.run_until_break() - gdb.remove_all_breakpoints() - gdb.continue_execution_until_running() - - sleep(5) conn.execute("create table t1()") conn.commit() - while 'RUNNING' in self.show_pb(backup_dir, 'replica')[0]['status']: - sleep(5) + sleep(5) - file = os.path.join(backup_dir, 'log', 'pg_probackup.log') + gdb.continue_execution_until_exit() - with open(file) as f: - log_content = f.read() + log_content = self.read_pb_log() self.assertIn( - 'LOG: Invalid offset in stop_lsn value 0/4000000', + 'has endpoint 0/4000000 which is ' + 'equal or greater than requested LSN', log_content) self.assertIn( - 'LOG: Looking for segment: 000000010000000000000004', + 'LOG: Found prior LSN:', log_content) self.assertIn( - 'LOG: First record in WAL segment "000000010000000000000004": 0/4000028', + 'INFO: backup->stop_lsn 0/4000000', log_content) - self.assertIn( - 'INFO: stop_lsn: 0/4000000', - log_content) + self.assertTrue(self.pb.show('replica')[0]['status'] == 'DONE') - self.assertTrue(self.show_pb(backup_dir, 'replica')[0]['status'] == 'DONE') + gdb_bgwriter.detach() # @unittest.skip("skip") + @needs_gdb def test_archive_replica_null_offset(self): """ """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + backup_dir = self.backup_dir + master = self.pg_node.make_simple('master', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '1h', 'wal_level': 'replica'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', master) - self.set_archiving(backup_dir, 'node', master) + self.pb.init() + self.pb.add_instance('node', master) + self.pb.set_archiving('node', master) master.slow_start() - self.backup_node(backup_dir, 'node', master) + self.pb.backup_node('node', master) # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'node', replica) + self.pb.restore_node('node', node=replica) # Settings for Replica self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'node', replica, replica=True) + self.pb.set_archiving('node', replica, replica=True) # freeze bgwriter to get rid of RUNNING XACTS records bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] - gdb_checkpointer = self.gdb_attach(bgwriter_pid) + gdb_bgwriter = self.gdb_attach(bgwriter_pid) replica.slow_start(replica=True) - self.switch_wal_segment(master) self.switch_wal_segment(master) # take backup from replica - output = self.backup_node( - backup_dir, 'node', replica, replica.data_dir, + _, output = self.pb.backup_replica_node('node', replica, replica.data_dir, + master=master, options=[ - '--archive-timeout=30', - '--log-level-console=LOG', + '--archive-timeout=300', + '--log-level-file=LOG', '--no-validate'], - return_id=False) - - self.assertIn( - 'LOG: Invalid offset in stop_lsn value 0/4000000', - output) - - self.assertIn( - 'WARNING: WAL segment 000000010000000000000004 could not be archived in 30 seconds', - output) - - self.assertIn( - 'WARNING: Failed to get next WAL record after 0/4000000, looking for previous WAL record', - output) + ) - self.assertIn( - 'LOG: Looking for LSN 0/4000000 in segment: 000000010000000000000003', - output) + self.assertRegex( + output, + r'LOG: Looking for LSN 0/[45]000000 in segment: 00000001000000000000000[34]') - self.assertIn( - 'has endpoint 0/4000000 which is ' - 'equal or greater than requested LSN 0/4000000', - output) + self.assertRegex( + output, + r'has endpoint 0/[45]000000 which is ' + r'equal or greater than requested LSN 0/[45]000000') self.assertIn( 'LOG: Found prior LSN:', output) - print(output) + gdb_bgwriter.detach() # @unittest.skip("skip") + @needs_gdb def test_archive_replica_not_null_offset(self): """ """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + backup_dir = self.backup_dir + master = self.pg_node.make_simple('master', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ + 'archive_timeout' : '1h', 'checkpoint_timeout': '1h', 'wal_level': 'replica'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', master) - self.set_archiving(backup_dir, 'node', master) + self.pb.init() + self.pb.add_instance('node', master) + self.pb.set_archiving('node', master) master.slow_start() - self.backup_node(backup_dir, 'node', master) + self.pb.backup_node('node', master) # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'node', replica) + self.pb.restore_node('node', node=replica) # Settings for Replica self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'node', replica, replica=True) + self.pb.set_archiving('node', replica, replica=True) replica.slow_start(replica=True) # take backup from replica - self.backup_node( - backup_dir, 'node', replica, replica.data_dir, + self.pb.backup_replica_node('node', replica, replica.data_dir, + master=master, options=[ - '--archive-timeout=30', - '--log-level-console=LOG', + '--archive-timeout=300', '--no-validate'], - return_id=False) + ) + + master.execute('select txid_current()') + self.wait_until_replica_catch_with_master(master, replica) - try: - self.backup_node( - backup_dir, 'node', replica, replica.data_dir, + output = self.pb.backup_node('node', replica, replica.data_dir, options=[ - '--archive-timeout=30', + '--archive-timeout=10', '--log-level-console=LOG', - '--no-validate']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of archive timeout. " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - # vanilla -- 0/4000060 - # pgproee -- 0/4000078 - self.assertRegex( - e.message, - r'LOG: Looking for LSN (0/4000060|0/4000078) in segment: 000000010000000000000004', - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertRegex( - e.message, - r'INFO: Wait for LSN (0/4000060|0/4000078) in archived WAL segment', - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.assertIn( - 'ERROR: WAL segment 000000010000000000000004 could not be archived in 30 seconds', - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + '--log-level-file=LOG', + '--no-validate'], + expect_error=True) + + self.assertMessage(output, regex=r'LOG: Looking for LSN 0/[45]0000(?!00)[A-F\d]{2} in segment: 0*10*[45]') + + self.assertMessage(output, regex=r'ERROR: WAL segment 0*10*[45] could not be archived in \d+ seconds') # @unittest.skip("skip") + @needs_gdb def test_replica_toast(self): """ make archive master, take full and page archive backups from master, set replica, make archive backup from replica """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + backup_dir = self.backup_dir + master = self.pg_node.make_simple('master', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '1h', 'wal_level': 'replica', 'shared_buffers': '128MB'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) - self.set_archiving(backup_dir, 'master', master) + self.pb.init() + self.pb.add_instance('master', master) + self.pb.set_archiving('master', master) master.slow_start() # freeze bgwriter to get rid of RUNNING XACTS records bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] - gdb_checkpointer = self.gdb_attach(bgwriter_pid) + gdb_bgwriter = self.gdb_attach(bgwriter_pid) - self.backup_node(backup_dir, 'master', master) + self.pb.backup_node('master', master) # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'master', replica) + self.pb.restore_node('master', node=replica) # Settings for Replica - self.add_instance(backup_dir, 'replica', replica) + self.pb.add_instance('replica', replica) self.set_replica(master, replica, synchronous=True) - self.set_archiving(backup_dir, 'replica', replica, replica=True) - - copy_tree( - os.path.join(backup_dir, 'wal', 'master'), - os.path.join(backup_dir, 'wal', 'replica')) + self.pb.set_archiving('replica', replica, replica=True) replica.slow_start(replica=True) - self.switch_wal_segment(master) - self.switch_wal_segment(master) - master.safe_psql( 'postgres', 'CREATE TABLE t1 AS ' @@ -997,8 +798,7 @@ def test_replica_toast(self): self.wait_until_replica_catch_with_master(master, replica) - output = self.backup_node( - backup_dir, 'replica', replica, + output = self.pb.backup_node('replica', replica, options=[ '--archive-timeout=30', '--log-level-console=LOG', @@ -1008,10 +808,6 @@ def test_replica_toast(self): pgdata = self.pgdata_content(replica.data_dir) - self.assertIn( - 'WARNING: Could not read WAL record at', - output) - self.assertIn( 'LOG: Found prior LSN:', output) @@ -1022,7 +818,7 @@ def test_replica_toast(self): replica.cleanup() - self.restore_node(backup_dir, 'replica', replica) + self.pb.restore_node('replica', node=replica) pgdata_restored = self.pgdata_content(replica.data_dir) replica.slow_start() @@ -1036,44 +832,39 @@ def test_replica_toast(self): self.compare_pgdata(pgdata, pgdata_restored) # Clean after yourself - gdb_checkpointer.kill() + gdb_bgwriter.detach() # @unittest.skip("skip") + @needs_gdb def test_start_stop_lsn_in_the_same_segno(self): """ """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + backup_dir = self.backup_dir + master = self.pg_node.make_simple('master', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '1h', 'wal_level': 'replica', 'shared_buffers': '128MB'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) + self.pb.init() + self.pb.add_instance('master', master) master.slow_start() # freeze bgwriter to get rid of RUNNING XACTS records bgwriter_pid = master.auxiliary_pids[ProcessType.BackgroundWriter][0] + gdb_bgwriter = self.gdb_attach(bgwriter_pid) - self.backup_node(backup_dir, 'master', master, options=['--stream']) + self.pb.backup_node('master', master, options=['--stream']) # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'master', replica) + self.pb.restore_node('master', node=replica) # Settings for Replica - self.add_instance(backup_dir, 'replica', replica) + self.pb.add_instance('replica', replica) self.set_replica(master, replica, synchronous=True) replica.slow_start(replica=True) @@ -1095,8 +886,7 @@ def test_start_stop_lsn_in_the_same_segno(self): sleep(60) - self.backup_node( - backup_dir, 'replica', replica, + self.pb.backup_node('replica', replica, options=[ '--archive-timeout=30', '--log-level-console=LOG', @@ -1104,8 +894,7 @@ def test_start_stop_lsn_in_the_same_segno(self): '--stream'], return_id=False) - self.backup_node( - backup_dir, 'replica', replica, + self.pb.backup_node('replica', replica, options=[ '--archive-timeout=30', '--log-level-console=LOG', @@ -1113,36 +902,31 @@ def test_start_stop_lsn_in_the_same_segno(self): '--stream'], return_id=False) + gdb_bgwriter.detach() + @unittest.skip("skip") def test_replica_promote_1(self): """ """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), + backup_dir = self.backup_dir + master = self.pg_node.make_simple('master', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '1h', 'wal_level': 'replica'}) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) + self.pb.init() + self.pb.add_instance('master', master) # set replica True, so archive_mode 'always' is used. - self.set_archiving(backup_dir, 'master', master, replica=True) + self.pb.set_archiving('master', master, replica=True) master.slow_start() - self.backup_node(backup_dir, 'master', master) + self.pb.backup_node('master', master) # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'master', replica) + self.pb.restore_node('master', node=replica) # Settings for Replica self.set_replica(master, replica) @@ -1157,18 +941,14 @@ def test_replica_promote_1(self): self.wait_until_replica_catch_with_master(master, replica) - wal_file = os.path.join( - backup_dir, 'wal', 'master', '000000010000000000000004') - - wal_file_partial = os.path.join( - backup_dir, 'wal', 'master', '000000010000000000000004.partial') + self.assertFalse( + self.instance_wal_exists(backup_dir, master, '000000010000000000000004')) - self.assertFalse(os.path.exists(wal_file)) + wal_file_partial = '000000010000000000000004.partial' replica.promote() - while not os.path.exists(wal_file_partial): - sleep(1) + self.wait_instance_wal_exists(backup_dir, 'master', wal_file_partial) self.switch_wal_segment(master) @@ -1176,41 +956,33 @@ def test_replica_promote_1(self): sleep(70) self.assertTrue( - os.path.exists(wal_file_partial), - "File {0} disappeared".format(wal_file)) - - self.assertTrue( - os.path.exists(wal_file_partial), + self.instance_wal_exists(backup_dir, 'master', wal_file_partial), "File {0} disappeared".format(wal_file_partial)) # @unittest.skip("skip") def test_replica_promote_2(self): """ """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + master = self.pg_node.make_simple('master', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) + self.pb.init() + self.pb.add_instance('master', master) # set replica True, so archive_mode 'always' is used. - self.set_archiving( - backup_dir, 'master', master, replica=True) + self.pb.set_archiving('master', master, replica=True) master.slow_start() - self.backup_node(backup_dir, 'master', master) + self.pb.backup_node('master', master) # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'master', replica) + self.pb.restore_node('master', node=replica) # Settings for Replica self.set_replica(master, replica) - self.set_auto_conf(replica, {'port': replica.port}) + replica.set_auto_conf({'port': replica.port}) replica.slow_start(replica=True) @@ -1224,8 +996,7 @@ def test_replica_promote_2(self): replica.promote() - self.backup_node( - backup_dir, 'master', replica, data_dir=replica.data_dir, + self.pb.backup_node('master', replica, data_dir=replica.data_dir, backup_type='page') # @unittest.skip("skip") @@ -1235,82 +1006,58 @@ def test_replica_promote_archive_delta(self): t2 /-------> t1 --F---D1--D2-- """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), + backup_dir = self.backup_dir + node1 = self.pg_node.make_simple('node1', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', 'archive_timeout': '30s'}) - if self.get_version(node1) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node1) - self.set_config( - backup_dir, 'node', options=['--archive-timeout=60s']) - self.set_archiving(backup_dir, 'node', node1) + self.pb.init() + self.pb.add_instance('node', node1) + self.pb.set_config('node', options=['--archive-timeout=60s']) + self.pb.set_archiving('node', node1) node1.slow_start() - self.backup_node(backup_dir, 'node', node1, options=['--stream']) + self.pb.backup_node('node', node1, options=['--stream']) # Create replica - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2 = self.pg_node.make_simple('node2') node2.cleanup() - self.restore_node(backup_dir, 'node', node2, node2.data_dir) + self.pb.restore_node('node', node=node2) # Settings for Replica self.set_replica(node1, node2) - self.set_auto_conf(node2, {'port': node2.port}) - self.set_archiving(backup_dir, 'node', node2, replica=True) + node2.set_auto_conf({'port': node2.port}) + self.pb.set_archiving('node', node2, replica=True) node2.slow_start(replica=True) - node1.safe_psql( - 'postgres', - 'CREATE TABLE t1 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,20) i') + create_table(node1, 't1') self.wait_until_replica_catch_with_master(node1, node2) - node1.safe_psql( - 'postgres', - 'CREATE TABLE t2 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,20) i') + create_table(node1, 't2') self.wait_until_replica_catch_with_master(node1, node2) # delta backup on replica on timeline 1 - delta1_id = self.backup_node( - backup_dir, 'node', node2, node2.data_dir, + delta1_id = self.pb.backup_node('node', node2, node2.data_dir, 'delta', options=['--stream']) # delta backup on replica on timeline 1 - delta2_id = self.backup_node( - backup_dir, 'node', node2, node2.data_dir, 'delta') + delta2_id = self.pb.backup_node('node', node2, node2.data_dir, 'delta') - self.change_backup_status( - backup_dir, 'node', delta2_id, 'ERROR') + self.change_backup_status(backup_dir, 'node', delta2_id, 'ERROR') # node2 is now master node2.promote() - node2.safe_psql( - 'postgres', - 'CREATE TABLE t3 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,20) i') + create_table(node2, 't3') # node1 is now replica node1.cleanup() # kludge "backup_id=delta1_id" - self.restore_node( - backup_dir, 'node', node1, node1.data_dir, + self.pb.restore_node('node', node1, backup_id=delta1_id, options=[ '--recovery-target-timeline=2', @@ -1318,16 +1065,12 @@ def test_replica_promote_archive_delta(self): # Settings for Replica self.set_replica(node2, node1) - self.set_auto_conf(node1, {'port': node1.port}) - self.set_archiving(backup_dir, 'node', node1, replica=True) + node1.set_auto_conf({'port': node1.port}) + self.pb.set_archiving('node', node1, replica=True) node1.slow_start(replica=True) - node2.safe_psql( - 'postgres', - 'CREATE TABLE t4 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,30) i') + create_table(node2, 't4') self.wait_until_replica_catch_with_master(node2, node1) # node1 is back to be a master @@ -1336,14 +1079,13 @@ def test_replica_promote_archive_delta(self): sleep(5) # delta backup on timeline 3 - self.backup_node( - backup_dir, 'node', node1, node1.data_dir, 'delta', + self.pb.backup_node('node', node1, node1.data_dir, 'delta', options=['--archive-timeout=60']) pgdata = self.pgdata_content(node1.data_dir) node1.cleanup() - self.restore_node(backup_dir, 'node', node1, node1.data_dir) + self.pb.restore_node('node', node=node1) pgdata_restored = self.pgdata_content(node1.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -1355,82 +1097,58 @@ def test_replica_promote_archive_page(self): t2 /-------> t1 --F---P1--P2-- """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), + backup_dir = self.backup_dir + node1 = self.pg_node.make_simple('node1', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'checkpoint_timeout': '30s', 'archive_timeout': '30s'}) - if self.get_version(node1) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node1) - self.set_archiving(backup_dir, 'node', node1) - self.set_config( - backup_dir, 'node', options=['--archive-timeout=60s']) + self.pb.init() + self.pb.add_instance('node', node1) + self.pb.set_archiving('node', node1) + self.pb.set_config('node', options=['--archive-timeout=60s']) node1.slow_start() - self.backup_node(backup_dir, 'node', node1, options=['--stream']) + self.pb.backup_node('node', node1, options=['--stream']) # Create replica - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2 = self.pg_node.make_simple('node2') node2.cleanup() - self.restore_node(backup_dir, 'node', node2, node2.data_dir) + self.pb.restore_node('node', node=node2) # Settings for Replica self.set_replica(node1, node2) - self.set_auto_conf(node2, {'port': node2.port}) - self.set_archiving(backup_dir, 'node', node2, replica=True) + node2.set_auto_conf({'port': node2.port}) + self.pb.set_archiving('node', node2, replica=True) node2.slow_start(replica=True) - node1.safe_psql( - 'postgres', - 'CREATE TABLE t1 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,20) i') + create_table(node1, 't1') self.wait_until_replica_catch_with_master(node1, node2) - node1.safe_psql( - 'postgres', - 'CREATE TABLE t2 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,20) i') + create_table(node1, 't2') self.wait_until_replica_catch_with_master(node1, node2) # page backup on replica on timeline 1 - page1_id = self.backup_node( - backup_dir, 'node', node2, node2.data_dir, + page1_id = self.pb.backup_node('node', node2, node2.data_dir, 'page', options=['--stream']) # page backup on replica on timeline 1 - page2_id = self.backup_node( - backup_dir, 'node', node2, node2.data_dir, 'page') + page2_id = self.pb.backup_node('node', node2, node2.data_dir, 'page') - self.change_backup_status( - backup_dir, 'node', page2_id, 'ERROR') + self.change_backup_status(backup_dir, 'node', page2_id, 'ERROR') # node2 is now master node2.promote() - node2.safe_psql( - 'postgres', - 'CREATE TABLE t3 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,20) i') + create_table(node2, 't3') # node1 is now replica node1.cleanup() # kludge "backup_id=page1_id" - self.restore_node( - backup_dir, 'node', node1, node1.data_dir, + self.pb.restore_node('node', node1, backup_id=page1_id, options=[ '--recovery-target-timeline=2', @@ -1438,16 +1156,12 @@ def test_replica_promote_archive_page(self): # Settings for Replica self.set_replica(node2, node1) - self.set_auto_conf(node1, {'port': node1.port}) - self.set_archiving(backup_dir, 'node', node1, replica=True) + node1.set_auto_conf({'port': node1.port}) + self.pb.set_archiving('node', node1, replica=True) node1.slow_start(replica=True) - node2.safe_psql( - 'postgres', - 'CREATE TABLE t4 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' - 'FROM generate_series(0,30) i') + create_table(node2, 't4') self.wait_until_replica_catch_with_master(node2, node1) # node1 is back to be a master @@ -1456,17 +1170,16 @@ def test_replica_promote_archive_page(self): sleep(5) - # delta3_id = self.backup_node( - # backup_dir, 'node', node2, node2.data_dir, 'delta') + # delta3_id = self.pb.backup_node( + # 'node', node2, node2.data_dir, 'delta') # page backup on timeline 3 - page3_id = self.backup_node( - backup_dir, 'node', node1, node1.data_dir, 'page', + page3_id = self.pb.backup_node('node', node1, node1.data_dir, 'page', options=['--archive-timeout=60']) pgdata = self.pgdata_content(node1.data_dir) node1.cleanup() - self.restore_node(backup_dir, 'node', node1, node1.data_dir) + self.pb.restore_node('node', node=node1) pgdata_restored = self.pgdata_content(node1.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -1475,32 +1188,25 @@ def test_replica_promote_archive_page(self): def test_parent_choosing(self): """ """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - master = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'master'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + master = self.pg_node.make_simple('master', + set_replication=True) - if self.get_version(master) < self.version_to_num('9.6.0'): - self.skipTest( - 'Skipped because backup from replica is not supported in PG 9.5') - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'master', master) + self.pb.init() + self.pb.add_instance('master', master) master.slow_start() - self.backup_node(backup_dir, 'master', master, options=['--stream']) + self.pb.backup_node('master', master, options=['--stream']) # Create replica - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() - self.restore_node(backup_dir, 'master', replica) + self.pb.restore_node('master', node=replica) # Settings for Replica self.set_replica(master, replica) - self.set_auto_conf(replica, {'port': replica.port}) + replica.set_auto_conf({'port': replica.port}) replica.slow_start(replica=True) @@ -1511,10 +1217,9 @@ def test_parent_choosing(self): 'FROM generate_series(0,20) i') self.wait_until_replica_catch_with_master(master, replica) - self.add_instance(backup_dir, 'replica', replica) + self.pb.add_instance('replica', replica) - full_id = self.backup_node( - backup_dir, 'replica', + full_id = self.pb.backup_node('replica', replica, options=['--stream']) master.safe_psql( @@ -1524,82 +1229,64 @@ def test_parent_choosing(self): 'FROM generate_series(0,20) i') self.wait_until_replica_catch_with_master(master, replica) - self.backup_node( - backup_dir, 'replica', replica, + self.pb.backup_node('replica', replica, backup_type='delta', options=['--stream']) replica.promote() # failing, because without archving, it is impossible to # take multi-timeline backup. - self.backup_node( - backup_dir, 'replica', replica, + self.pb.backup_node('replica', replica, backup_type='delta', options=['--stream']) # @unittest.skip("skip") def test_instance_from_the_past(self): """ """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - full_id = self.backup_node(backup_dir, 'node', node, options=['--stream']) + full_id = self.pb.backup_node('node', node, options=['--stream']) node.pgbench_init(scale=10) - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) node.cleanup() - self.restore_node(backup_dir, 'node', node, backup_id=full_id) + self.pb.restore_node('node', node=node, backup_id=full_id) node.slow_start() - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['--stream']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because instance is from the past " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: Current START LSN' in e.message and - 'is lower than START LSN' in e.message and - 'It may indicate that we are trying to backup ' - 'PostgreSQL instance from the past' in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, backup_type='delta', + options=['--stream'], + expect_error="because instance is from the past") + self.assertMessage(regex='ERROR: Current START LSN .* is lower than START LSN') + self.assertMessage(contains='It may indicate that we are trying to backup ' + 'PostgreSQL instance from the past') # @unittest.skip("skip") def test_replica_via_basebackup(self): """ """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={'hot_standby': 'on'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=10) #FULL backup - full_id = self.backup_node(backup_dir, 'node', node) + full_id = self.pb.backup_node('node', node) pgbench = node.pgbench( options=['-T', '10', '-c', '1', '--no-vacuum']) @@ -1607,48 +1294,54 @@ def test_replica_via_basebackup(self): node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=['--recovery-target=latest', '--recovery-target-action=promote']) node.slow_start() # Timeline 2 # Take stream page backup from instance in timeline2 - self.backup_node( - backup_dir, 'node', node, backup_type='full', + self.pb.backup_node('node', node, backup_type='full', options=['--stream', '--log-level-file=verbose']) node.cleanup() # restore stream backup - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) - xlog_dir = 'pg_wal' - if self.get_version(node) < 100000: - xlog_dir = 'pg_xlog' - - filepath = os.path.join(node.data_dir, xlog_dir, "00000002.history") + filepath = os.path.join(node.data_dir, 'pg_wal', "00000002.history") self.assertTrue( os.path.exists(filepath), "History file do not exists: {0}".format(filepath)) node.slow_start() - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() pg_basebackup_path = self.get_bin_path('pg_basebackup') - self.run_binary( + self.pb.run_binary( [ pg_basebackup_path, '-p', str(node.port), '-h', 'localhost', '-R', '-X', 'stream', '-D', node_restored.data_dir ]) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start(replica=True) +def call_repeat(times, func, *args): + for i in range(times): + func(*args) + +def create_table(node, name): + node.safe_psql( + 'postgres', + f"CREATE TABLE {name} AS " + "SELECT i, v as fat_attr " + "FROM generate_series(0,3) i, " + " (SELECT string_agg(md5(j::text), '') as v" + " FROM generate_series(0,500605) as j) v") + # TODO: # null offset STOP LSN and latest record in previous segment is conrecord (manual only) # archiving from promoted delayed replica diff --git a/tests/requirements.txt b/tests/requirements.txt new file mode 100644 index 000000000..30cbcfb8c --- /dev/null +++ b/tests/requirements.txt @@ -0,0 +1,16 @@ +# Testgres can be installed in the following ways: +# 1. From a pip package (recommended) +# testgres==1.8.5 +# 2. From a specific Git branch, tag or commit +# git+https://github.com/postgrespro/testgres.git@ +# 3. From a local directory +# /path/to/local/directory/testgres +testgres==1.10.0 +testgres-pg-probackup2==0.0.2 +allure-pytest +deprecation +minio==7.2.5 +pexpect +pytest==7.4.3 +pytest-xdist +parameterized diff --git a/tests/restore_test.py b/tests/restore_test.py index 67e99515c..798002944 100644 --- a/tests/restore_test.py +++ b/tests/restore_test.py @@ -1,32 +1,32 @@ import os import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest, fs_backup_class +from pg_probackup2.gdb import needs_gdb import subprocess -import sys from time import sleep from datetime import datetime, timedelta, timezone -import hashlib import shutil import json -from shutil import copyfile from testgres import QueryException, StartNodeException -from stat import S_ISDIR +import testgres.utils as testgres_utils +import re -class RestoreTest(ProbackupTest, unittest.TestCase): + +MAGIC_COUNT = 107183 + + +class RestoreTest(ProbackupTest): # @unittest.skip("skip") # @unittest.expectedFailure def test_restore_full_to_latest(self): """recovery to latest from full backup""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=2) @@ -35,22 +35,18 @@ def test_restore_full_to_latest(self): pgbench.wait() pgbench.stdout.close() before = node.table_checksum("pgbench_branches") - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) node.stop() node.cleanup() # 1 - Test recovery from latest - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + restore_result = self.pb.restore_node('node', node, options=["-j", "4"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) # 2 - Test that recovery.conf was created # TODO update test - if self.get_version(node) >= self.version_to_num('12.0'): + if self.pg_config_version >= self.version_to_num('12.0'): recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') with open(recovery_conf, 'r') as f: print(f.read()) @@ -66,39 +62,31 @@ def test_restore_full_to_latest(self): # @unittest.skip("skip") def test_restore_full_page_to_latest(self): """recovery to latest from full + page backups""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=2) - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="page") + backup_id = self.pb.backup_node('node', node, backup_type="page") before = node.table_checksum("pgbench_branches") node.stop() node.cleanup() - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + restore_result = self.pb.restore_node('node', node, options=["-j", "4"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) node.slow_start() @@ -108,33 +96,26 @@ def test_restore_full_page_to_latest(self): # @unittest.skip("skip") def test_restore_to_specific_timeline(self): """recovery to target timeline""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=2) before = node.table_checksum("pgbench_branches") - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) target_tli = int( node.get_control_data()["Latest checkpoint's TimeLineID"]) node.stop() node.cleanup() - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + restore_result = self.pb.restore_node('node', node, options=["-j", "4"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) node.slow_start() pgbench = node.pgbench( @@ -143,24 +124,19 @@ def test_restore_to_specific_timeline(self): pgbench.wait() pgbench.stdout.close() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.stop() node.cleanup() # Correct Backup must be choosen for restore - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, options=[ "-j", "4", "--timeline={0}".format(target_tli)] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + ) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) - recovery_target_timeline = self.get_recovery_conf( - node)["recovery_target_timeline"] + recovery_target_timeline = self.get_recovery_conf(node)["recovery_target_timeline"] self.assertEqual(int(recovery_target_timeline), target_tli) node.slow_start() @@ -170,25 +146,21 @@ def test_restore_to_specific_timeline(self): # @unittest.skip("skip") def test_restore_to_time(self): """recovery to target time""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', pg_options={'TimeZone': 'GMT'}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=2) before = node.table_checksum("pgbench_branches") - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) - target_time = node.execute( - "postgres", "SELECT to_char(now(), 'YYYY-MM-DD HH24:MI:SS+00')" - )[0][0] + node.safe_psql("postgres", "select txid_current()") + target_time = node.safe_psql("postgres", "SELECT current_timestamp").decode('utf-8').strip() pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() @@ -197,17 +169,13 @@ def test_restore_to_time(self): node.stop() node.cleanup() - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, options=[ - "-j", "4", '--time={0}'.format(target_time), + "-j", "4", '--recovery-target-time={0}'.format(target_time), "--recovery-target-action=promote" ] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + ) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) node.slow_start() after = node.table_checksum("pgbench_branches") @@ -216,14 +184,11 @@ def test_restore_to_time(self): # @unittest.skip("skip") def test_restore_to_xid_inclusive(self): """recovery to target xid""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=2) @@ -231,7 +196,7 @@ def test_restore_to_xid_inclusive(self): con.execute("CREATE TABLE tbl0005 (a text)") con.commit() - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -252,17 +217,12 @@ def test_restore_to_xid_inclusive(self): node.stop() node.cleanup() - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, options=[ "-j", "4", '--xid={0}'.format(target_xid), "--recovery-target-action=promote"] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - + ) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) node.slow_start() after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) @@ -272,14 +232,11 @@ def test_restore_to_xid_inclusive(self): # @unittest.skip("skip") def test_restore_to_xid_not_inclusive(self): """recovery with target inclusive false""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=2) @@ -287,7 +244,7 @@ def test_restore_to_xid_not_inclusive(self): con.execute("CREATE TABLE tbl0005 (a text)") con.commit() - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -308,17 +265,13 @@ def test_restore_to_xid_not_inclusive(self): node.stop() node.cleanup() - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, options=[ "-j", "4", '--xid={0}'.format(target_xid), "--inclusive=false", - "--recovery-target-action=promote"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + "--recovery-target-action=promote"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) node.slow_start() after = node.table_checksum("pgbench_branches") @@ -329,17 +282,12 @@ def test_restore_to_xid_not_inclusive(self): # @unittest.skip("skip") def test_restore_to_lsn_inclusive(self): """recovery to target lsn""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - if self.get_version(node) < self.version_to_num('10.0'): - return + node = self.pg_node.make_simple('node') + node.set_auto_conf({"autovacuum": "off"}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=2) @@ -347,7 +295,7 @@ def test_restore_to_lsn_inclusive(self): con.execute("CREATE TABLE tbl0005 (a int)") con.commit() - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -358,13 +306,13 @@ def test_restore_to_lsn_inclusive(self): with node.connect("postgres") as con: con.execute("INSERT INTO tbl0005 VALUES (1)") con.commit() - res = con.execute("SELECT pg_current_wal_lsn()") - con.commit() con.execute("INSERT INTO tbl0005 VALUES (2)") + # With high probability, returned lsn will point at COMMIT start + # If this test still will be flappy, get lsn after commit and add + # one more xlog record (for example, with txid_current()+abort). + res = con.execute("SELECT pg_current_wal_insert_lsn()") con.commit() - xlogid, xrecoff = res[0][0].split('/') - xrecoff = hex(int(xrecoff, 16) + 1)[2:] - target_lsn = "{0}/{1}".format(xlogid, xrecoff) + target_lsn = res[0][0] pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -374,16 +322,12 @@ def test_restore_to_lsn_inclusive(self): node.stop() node.cleanup() - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, options=[ "-j", "4", '--lsn={0}'.format(target_lsn), "--recovery-target-action=promote"] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + ) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) node.slow_start() @@ -395,17 +339,11 @@ def test_restore_to_lsn_inclusive(self): # @unittest.skip("skip") def test_restore_to_lsn_not_inclusive(self): """recovery to target lsn""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - if self.get_version(node) < self.version_to_num('10.0'): - return + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=2) @@ -413,7 +351,7 @@ def test_restore_to_lsn_not_inclusive(self): con.execute("CREATE TABLE tbl0005 (a int)") con.commit() - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -424,13 +362,13 @@ def test_restore_to_lsn_not_inclusive(self): with node.connect("postgres") as con: con.execute("INSERT INTO tbl0005 VALUES (1)") con.commit() - res = con.execute("SELECT pg_current_wal_lsn()") - con.commit() con.execute("INSERT INTO tbl0005 VALUES (2)") + # Returned lsn will certainly point at COMMIT start OR BEFORE IT, + # if some background activity wrote record in between INSERT and + # COMMIT. Any way, test should succeed. + res = con.execute("SELECT pg_current_wal_insert_lsn()") con.commit() - xlogid, xrecoff = res[0][0].split('/') - xrecoff = hex(int(xrecoff, 16) + 1)[2:] - target_lsn = "{0}/{1}".format(xlogid, xrecoff) + target_lsn = res[0][0] pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT) @@ -440,17 +378,13 @@ def test_restore_to_lsn_not_inclusive(self): node.stop() node.cleanup() - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, options=[ "--inclusive=false", "-j", "4", '--lsn={0}'.format(target_lsn), "--recovery-target-action=promote"] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + ) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) node.slow_start() @@ -465,15 +399,12 @@ def test_restore_full_ptrack_archive(self): if not self.ptrack: self.skipTest('Skipped because ptrack support is disabled') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -482,29 +413,23 @@ def test_restore_full_ptrack_archive(self): node.pgbench_init(scale=2) - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="ptrack") + backup_id = self.pb.backup_node('node', node, backup_type="ptrack") before = node.table_checksum("pgbench_branches") node.stop() node.cleanup() - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, - options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - + restore_result = self.pb.restore_node('node', node, + options=["-j", "4"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) node.slow_start() after = node.table_checksum("pgbench_branches") self.assertEqual(before, after) @@ -515,15 +440,12 @@ def test_restore_ptrack(self): if not self.ptrack: self.skipTest('Skipped because ptrack support is disabled') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -532,35 +454,30 @@ def test_restore_ptrack(self): node.pgbench_init(scale=2) - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - self.backup_node(backup_dir, 'node', node, backup_type="ptrack") + self.pb.backup_node('node', node, backup_type="ptrack") pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="ptrack") + backup_id = self.pb.backup_node('node', node, backup_type="ptrack") before = node.table_checksum("pgbench_branches") node.stop() node.cleanup() - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, - options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + restore_result = self.pb.restore_node('node', node, + options=["-j", "4"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) node.slow_start() after = node.table_checksum("pgbench_branches") @@ -572,16 +489,13 @@ def test_restore_full_ptrack_stream(self): if not self.ptrack: self.skipTest('Skipped because ptrack support is disabled') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -590,15 +504,14 @@ def test_restore_full_ptrack_stream(self): node.pgbench_init(scale=2) - self.backup_node(backup_dir, 'node', node, options=["--stream"]) + self.pb.backup_node('node', node, options=["--stream"]) pgbench = node.pgbench( stdout=subprocess.PIPE, stderr=subprocess.STDOUT) pgbench.wait() pgbench.stdout.close() - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, backup_type="ptrack", options=["--stream"]) before = node.table_checksum("pgbench_branches") @@ -606,12 +519,8 @@ def test_restore_full_ptrack_stream(self): node.stop() node.cleanup() - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + restore_result = self.pb.restore_node('node', node, options=["-j", "4"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) node.slow_start() after = node.table_checksum("pgbench_branches") @@ -626,16 +535,13 @@ def test_restore_full_ptrack_under_load(self): if not self.ptrack: self.skipTest('Skipped because ptrack support is disabled') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -644,7 +550,7 @@ def test_restore_full_ptrack_under_load(self): node.pgbench_init(scale=2) - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) pgbench = node.pgbench( stdout=subprocess.PIPE, @@ -652,8 +558,7 @@ def test_restore_full_ptrack_under_load(self): options=["-c", "4", "-T", "8"] ) - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, backup_type="ptrack", options=["--stream"]) pgbench.wait() @@ -668,12 +573,8 @@ def test_restore_full_ptrack_under_load(self): node.stop() node.cleanup() - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + restore_result = self.pb.restore_node('node', node, options=["-j", "4"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) node.slow_start() bbalance = node.execute( @@ -691,16 +592,13 @@ def test_restore_full_under_load_ptrack(self): if not self.ptrack: self.skipTest('Skipped because ptrack support is disabled') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=True, - initdb_params=['--data-checksums']) + ptrack_enable=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -716,13 +614,12 @@ def test_restore_full_under_load_ptrack(self): options=["-c", "4", "-T", "8"] ) - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) pgbench.wait() pgbench.stdout.close() - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, backup_type="ptrack", options=["--stream"]) bbalance = node.execute( @@ -734,14 +631,9 @@ def test_restore_full_under_load_ptrack(self): node.stop() node.cleanup() - # self.wrong_wal_clean(node, wal_segment_size) - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, options=["-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + restore_result = self.pb.restore_node('node', node, options=["-j", "4"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) node.slow_start() bbalance = node.execute( "postgres", "SELECT sum(bbalance) FROM pgbench_branches") @@ -752,14 +644,11 @@ def test_restore_full_under_load_ptrack(self): # @unittest.skip("skip") def test_restore_with_tablespace_mapping_1(self): """recovery using tablespace-mapping option""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Create tablespace @@ -773,45 +662,23 @@ def test_restore_with_tablespace_mapping_1(self): con.execute("INSERT INTO test VALUES (1)") con.commit() - backup_id = self.backup_node(backup_dir, 'node', node) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + backup_id = self.pb.backup_node('node', node) + self.assertEqual(self.pb.show('node')[0]['status'], "OK") # 1 - Try to restore to existing directory node.stop() - try: - self.restore_node(backup_dir, 'node', node) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because restore destination is not empty.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Restore destination is not empty:', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node, + expect_error="because restore destination is not empty") + self.assertMessage(contains='ERROR: Restore destination is not empty:') # 2 - Try to restore to existing tablespace directory tblspc_path_tmp = os.path.join(node.base_dir, "tblspc_tmp") os.rename(tblspc_path, tblspc_path_tmp) - node.cleanup() + shutil.rmtree(node.data_dir) os.rename(tblspc_path_tmp, tblspc_path) - try: - self.restore_node(backup_dir, 'node', node) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because restore tablespace destination is " - "not empty.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Restore tablespace destination is not empty:', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node, + expect_error="because restore tablespace destination is not empty") + self.assertMessage(contains='ERROR: Restore tablespace destination is not empty:') # 3 - Restore using tablespace-mapping to not empty directory tblspc_path_temp = os.path.join(node.base_dir, "tblspc_temp") @@ -819,34 +686,18 @@ def test_restore_with_tablespace_mapping_1(self): with open(os.path.join(tblspc_path_temp, 'file'), 'w+') as f: f.close() - try: - self.restore_node( - backup_dir, 'node', node, - options=["-T", "%s=%s" % (tblspc_path, tblspc_path_temp)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because restore tablespace destination is " - "not empty.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Restore tablespace destination is not empty:', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node, + options=["-T", f"{tblspc_path}={tblspc_path_temp}"], + expect_error="because restore tablespace destination is not empty") + self.assertMessage(contains='ERROR: Restore tablespace destination is not empty:') # 4 - Restore using tablespace-mapping tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, options=[ "-T", "%s=%s" % (tblspc_path, tblspc_path_new)] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + ) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) node.slow_start() @@ -854,29 +705,24 @@ def test_restore_with_tablespace_mapping_1(self): self.assertEqual(result[0][0], 1) # 4 - Restore using tablespace-mapping using page backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) with node.connect("postgres") as con: con.execute("INSERT INTO test VALUES (2)") con.commit() - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="page") + backup_id = self.pb.backup_node('node', node, backup_type="page") - show_pb = self.show_pb(backup_dir, 'node') + show_pb = self.pb.show('node') self.assertEqual(show_pb[1]['status'], "OK") self.assertEqual(show_pb[2]['status'], "OK") node.stop() - node.cleanup() + shutil.rmtree(node.data_dir) tblspc_path_page = os.path.join(node.base_dir, "tblspc_page") - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, options=[ - "-T", "%s=%s" % (tblspc_path_new, tblspc_path_page)]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + "-T", "%s=%s" % (tblspc_path_new, tblspc_path_page)]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) node.slow_start() result = node.execute("postgres", "SELECT id FROM test OFFSET 1") @@ -885,19 +731,16 @@ def test_restore_with_tablespace_mapping_1(self): # @unittest.skip("skip") def test_restore_with_tablespace_mapping_2(self): """recovery using tablespace-mapping option and page backup""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Full backup - self.backup_node(backup_dir, 'node', node) - self.assertEqual(self.show_pb(backup_dir, 'node')[0]['status'], "OK") + self.pb.backup_node('node', node) + self.assertEqual(self.pb.show('node')[0]['status'], "OK") # Create tablespace tblspc_path = os.path.join(node.base_dir, "tblspc") @@ -912,16 +755,16 @@ def test_restore_with_tablespace_mapping_2(self): con.commit() # First page backup - self.backup_node(backup_dir, 'node', node, backup_type="page") - self.assertEqual(self.show_pb(backup_dir, 'node')[1]['status'], "OK") + self.pb.backup_node('node', node, backup_type="page") + self.assertEqual(self.pb.show('node')[1]['status'], "OK") self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['backup-mode'], "PAGE") + self.pb.show('node')[1]['backup-mode'], "PAGE") # Create tablespace table with node.connect("postgres") as con: -# con.connection.autocommit = True -# con.execute("CHECKPOINT") -# con.connection.autocommit = False + con.connection.autocommit = True + con.execute("CHECKPOINT") + con.connection.autocommit = False con.execute("CREATE TABLE tbl1 (a int) TABLESPACE tblspc") con.execute( "INSERT INTO tbl1 SELECT * " @@ -929,25 +772,20 @@ def test_restore_with_tablespace_mapping_2(self): con.commit() # Second page backup - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="page") - self.assertEqual(self.show_pb(backup_dir, 'node')[2]['status'], "OK") + backup_id = self.pb.backup_node('node', node, backup_type="page") + self.assertEqual(self.pb.show('node')[2]['status'], "OK") self.assertEqual( - self.show_pb(backup_dir, 'node')[2]['backup-mode'], "PAGE") + self.pb.show('node')[2]['backup-mode'], "PAGE") node.stop() node.cleanup() tblspc_path_new = os.path.join(node.base_dir, "tblspc_new") - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, options=[ - "-T", "%s=%s" % (tblspc_path, tblspc_path_new)]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + "-T", "%s=%s" % (tblspc_path, tblspc_path_new)]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) node.slow_start() count = node.execute("postgres", "SELECT count(*) FROM tbl") @@ -958,14 +796,12 @@ def test_restore_with_tablespace_mapping_2(self): # @unittest.skip("skip") def test_restore_with_missing_or_corrupted_tablespace_map(self): """restore backup with missing or corrupted tablespace_map""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Create tablespace @@ -973,109 +809,60 @@ def test_restore_with_missing_or_corrupted_tablespace_map(self): node.pgbench_init(scale=1, tablespace='tblspace') # Full backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # Change some data pgbench = node.pgbench(options=['-T', '10', '-c', '1', '--no-vacuum']) pgbench.wait() # Page backup - page_id = self.backup_node(backup_dir, 'node', node, backup_type="page") + page_id = self.pb.backup_node('node', node, backup_type="page") pgdata = self.pgdata_content(node.data_dir) - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2 = self.pg_node.make_simple('node2') node2.cleanup() olddir = self.get_tblspace_path(node, 'tblspace') newdir = self.get_tblspace_path(node2, 'tblspace') # drop tablespace_map - tablespace_map = os.path.join( - backup_dir, 'backups', 'node', - page_id, 'database', 'tablespace_map') - - tablespace_map_tmp = os.path.join( - backup_dir, 'backups', 'node', - page_id, 'database', 'tablespace_map_tmp') - - os.rename(tablespace_map, tablespace_map_tmp) - - try: - self.restore_node( - backup_dir, 'node', node2, - options=["-T", "{0}={1}".format(olddir, newdir)]) - self.assertEqual( - 1, 0, - "Expecting Error because tablespace_map is missing.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Tablespace map is missing: "{0}", ' - 'probably backup {1} is corrupt, validate it'.format( - tablespace_map, page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.restore_node(backup_dir, 'node', node2) - self.assertEqual( - 1, 0, - "Expecting Error because tablespace_map is missing.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Tablespace map is missing: "{0}", ' - 'probably backup {1} is corrupt, validate it'.format( - tablespace_map, page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - copyfile(tablespace_map_tmp, tablespace_map) - - with open(tablespace_map, "a") as f: - f.write("HELLO\n") - - try: - self.restore_node( - backup_dir, 'node', node2, - options=["-T", "{0}={1}".format(olddir, newdir)]) - self.assertEqual( - 1, 0, - "Expecting Error because tablespace_map is corupted.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Invalid CRC of tablespace map file "{0}"'.format(tablespace_map), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.restore_node(backup_dir, 'node', node2) - self.assertEqual( - 1, 0, - "Expecting Error because tablespace_map is corupted.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Invalid CRC of tablespace map file "{0}"'.format(tablespace_map), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # rename it back - os.rename(tablespace_map_tmp, tablespace_map) - - print(self.restore_node( - backup_dir, 'node', node2, + tablespace_map = self.read_backup_file(backup_dir, 'node', page_id, + 'database/tablespace_map', text=True) + self.remove_backup_file(backup_dir, 'node', page_id, 'database/tablespace_map') + + self.pb.restore_node('node', node=node2, + options=["-T", "{0}={1}".format(olddir, newdir)], + expect_error="because tablespace_map is missing") + self.assertMessage(regex= + rf'ERROR: Tablespace map is missing: "[^"]*{page_id}[^"]*tablespace_map", ' + rf'probably backup {page_id} is corrupt, validate it') + + self.pb.restore_node('node', node=node2, + expect_error="because tablespace_map is missing") + self.assertMessage(regex= + rf'ERROR: Tablespace map is missing: "[^"]*{page_id}[^"]*tablespace_map", ' + rf'probably backup {page_id} is corrupt, validate it') + + self.corrupt_backup_file(backup_dir, 'node', page_id, 'database/tablespace_map', + overwrite=tablespace_map + "HELLO\n", text=True) + + self.pb.restore_node('node', node=node2, + options=["-T", f"{olddir}={newdir}"], + expect_error="because tablespace_map is corupted") + self.assertMessage(regex=r'ERROR: Invalid CRC of tablespace map file ' + rf'"[^"]*{page_id}[^"]*tablespace_map"') + + self.pb.restore_node('node', node=node2, + expect_error="because tablespace_map is corupted") + self.assertMessage(regex=r'ERROR: Invalid CRC of tablespace map file ' + rf'"[^"]*{page_id}[^"]*tablespace_map"') + + # write correct back + self.write_backup_file(backup_dir, 'node', page_id, 'database/tablespace_map', + tablespace_map, text=True) + + print(self.pb.restore_node('node', node2, options=["-T", "{0}={1}".format(olddir, newdir)])) pgdata_restored = self.pgdata_content(node2.data_dir) @@ -1087,39 +874,30 @@ def test_archive_node_backup_stream_restore_to_recovery_time(self): make node with archiving, make stream backup, make PITR to Recovery Time """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - backup_id = self.backup_node( - backup_dir, 'node', node, options=["--stream"]) + backup_id = self.pb.backup_node('node', node, options=["--stream"]) node.safe_psql("postgres", "create table t_heap(a int)") node.stop() node.cleanup() - recovery_time = self.show_pb( - backup_dir, 'node', backup_id)['recovery-time'] - - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, - options=[ - "-j", "4", '--time={0}'.format(recovery_time), - "--recovery-target-action=promote" - ] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + recovery_time = self.pb.show('node', backup_id)['recovery-time'] + restore_result = self.pb.restore_node('node', node, + options=[ + "-j", "4", '--time={0}'.format(recovery_time), + "--recovery-target-action=promote" + ] + ) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) node.slow_start() result = node.psql("postgres", 'select * from t_heap') @@ -1132,30 +910,25 @@ def test_archive_node_backup_stream_restore_to_recovery_time(self): make node with archiving, make stream backup, make PITR to Recovery Time """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - backup_id = self.backup_node( - backup_dir, 'node', node, options=["--stream"]) + backup_id = self.pb.backup_node('node', node, options=["--stream"]) node.safe_psql("postgres", "create table t_heap(a int)") node.stop() node.cleanup() - recovery_time = self.show_pb( - backup_dir, 'node', backup_id)['recovery-time'] + recovery_time = self.pb.show('node', backup_id)['recovery-time'] self.assertIn( "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ "-j", "4", '--time={0}'.format(recovery_time), "--recovery-target-action=promote" @@ -1174,38 +947,30 @@ def test_archive_node_backup_stream_pitr(self): """ make node with archiving, make stream backup, create table t_heap, make pitr to Recovery Time, - check that t_heap do not exists + check that t_heap does not exist """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - backup_id = self.backup_node( - backup_dir, 'node', node, options=["--stream"]) + backup_id = self.pb.backup_node('node', node, options=["--stream"]) node.safe_psql("postgres", "create table t_heap(a int)") node.cleanup() - recovery_time = self.show_pb( - backup_dir, 'node', backup_id)['recovery-time'] + recovery_time = self.pb.show('node', backup_id)['recovery-time'] - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, + restore_result = self.pb.restore_node('node', node, options=[ "-j", "4", '--time={0}'.format(recovery_time), "--recovery-target-action=promote" ] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + ) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) node.slow_start() @@ -1218,48 +983,40 @@ def test_archive_node_backup_archive_pitr_2(self): """ make node with archiving, make archive backup, create table t_heap, make pitr to Recovery Time, - check that t_heap do not exists + check that t_heap do not exist """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) if self.paranoia: pgdata = self.pgdata_content(node.data_dir) node.safe_psql("postgres", "create table t_heap(a int)") node.stop() - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - recovery_time = self.show_pb( - backup_dir, 'node', backup_id)['recovery-time'] + recovery_time = self.pb.show('node', backup_id)['recovery-time'] - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node_restored, + resotre_result = self.pb.restore_node('node', node_restored, options=[ "-j", "4", '--time={0}'.format(recovery_time), "--recovery-target-action=promote"] - ), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + ) + self.assertMessage(resotre_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) if self.paranoia: pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - self.set_auto_conf(node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() @@ -1274,17 +1031,15 @@ def test_archive_restore_to_restore_point(self): create table t_heap, make pitr to Recovery Time, check that t_heap do not exists """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.safe_psql( "postgres", @@ -1297,8 +1052,7 @@ def test_archive_restore_to_restore_point(self): "create table t_heap_1 as select generate_series(0,10000)") node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ "--recovery-target-name=savepoint", "--recovery-target-action=promote"]) @@ -1316,17 +1070,15 @@ def test_archive_restore_to_restore_point(self): @unittest.skip("skip") # @unittest.expectedFailure def test_zags_block_corrupt(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) conn = node.connect() with node.connect("postgres") as conn: @@ -1370,37 +1122,29 @@ def test_zags_block_corrupt(self): conn.execute( "insert into tbl select i from generate_series(0,100) as i") - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), - initdb_params=['--data-checksums']) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored) + self.pb.restore_node('node', node_restored) - self.set_auto_conf( - node_restored, - {'archive_mode': 'off', 'hot_standby': 'on', 'port': node_restored.port}) + node_restored.set_auto_conf({'archive_mode': 'off', 'hot_standby': 'on', 'port': node_restored.port}) node_restored.slow_start() @unittest.skip("skip") # @unittest.expectedFailure def test_zags_block_corrupt_1(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', pg_options={ 'full_page_writes': 'on'} ) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.safe_psql('postgres', 'create table tbl(i int)') @@ -1444,20 +1188,15 @@ def test_zags_block_corrupt_1(self): self.switch_wal_segment(node) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored'), - initdb_params=['--data-checksums']) + node_restored = self.pg_node.make_simple('node_restored') pgdata = self.pgdata_content(node.data_dir) node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored) + self.pb.restore_node('node', node_restored) - self.set_auto_conf( - node_restored, - {'archive_mode': 'off', 'hot_standby': 'on', 'port': node_restored.port}) + node_restored.set_auto_conf({'archive_mode': 'off', 'hot_standby': 'on', 'port': node_restored.port}) node_restored.slow_start() @@ -1475,7 +1214,7 @@ def test_zags_block_corrupt_1(self): # pg_xlogdump_path = self.get_bin_path('pg_xlogdump') -# pg_xlogdump = self.run_binary( +# pg_xlogdump = self.pb.run_binary( # [ # pg_xlogdump_path, '-b', # os.path.join(backup_dir, 'wal', 'node', '000000010000000000000003'), @@ -1495,259 +1234,209 @@ def test_restore_chain(self): ERROR delta backups, take valid delta backup, restore must be successfull """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Take FULL - self.backup_node( - backup_dir, 'node', node) + self.pb.backup_node('node', node) # Take DELTA - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') # Take ERROR DELTA - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['-U', 'wrong_name']) - except ProbackupException as e: - pass + self.pb.backup_node('node', node, backup_type='delta', + options=['-U', 'wrong_name'], + expect_error=True) # Take ERROR DELTA - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['-U', 'wrong_name']) - except ProbackupException as e: - pass + self.pb.backup_node('node', node, backup_type='delta', + options=['-U', 'wrong_name'], + expect_error=True) # Take DELTA - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') # Take ERROR DELTA - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['-U', 'wrong_name']) - except ProbackupException as e: - pass + self.pb.backup_node('node', node, backup_type='delta', + options=['-U', 'wrong_name'], + expect_error=True) self.assertEqual( 'OK', - self.show_pb(backup_dir, 'node')[0]['status'], + self.pb.show('node')[0]['status'], 'Backup STATUS should be "OK"') self.assertEqual( 'OK', - self.show_pb(backup_dir, 'node')[1]['status'], + self.pb.show('node')[1]['status'], 'Backup STATUS should be "OK"') self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node')[2]['status'], + self.pb.show('node')[2]['status'], 'Backup STATUS should be "ERROR"') self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node')[3]['status'], + self.pb.show('node')[3]['status'], 'Backup STATUS should be "ERROR"') self.assertEqual( 'OK', - self.show_pb(backup_dir, 'node')[4]['status'], + self.pb.show('node')[4]['status'], 'Backup STATUS should be "OK"') self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node')[5]['status'], + self.pb.show('node')[5]['status'], 'Backup STATUS should be "ERROR"') node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) # @unittest.skip("skip") def test_restore_chain_with_corrupted_backup(self): """more complex test_restore_chain()""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Take FULL - self.backup_node( - backup_dir, 'node', node) + self.pb.backup_node('node', node) # Take DELTA - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') # Take ERROR DELTA - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=['-U', 'wrong_name']) - except ProbackupException as e: - pass + self.pb.backup_node('node', node, backup_type='page', + options=['-U', 'wrong_name'], + expect_error=True) # Take 1 DELTA - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') # Take ERROR DELTA - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['-U', 'wrong_name']) - except ProbackupException as e: - pass + self.pb.backup_node('node', node, backup_type='delta', + options=['-U', 'wrong_name'], + expect_error=True) # Take 2 DELTA - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') # Take ERROR DELTA - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='delta', options=['-U', 'wrong_name']) - except ProbackupException as e: - pass + self.pb.backup_node('node', node, backup_type='delta', + options=['-U', 'wrong_name'], + expect_error=True) # Take 3 DELTA - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') # Corrupted 4 DELTA - corrupt_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') + corrupt_id = self.pb.backup_node('node', node, backup_type='delta') # ORPHAN 5 DELTA - restore_target_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') + restore_target_id = self.pb.backup_node('node', node, backup_type='delta') # ORPHAN 6 DELTA - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') # NEXT FULL BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='full') + self.pb.backup_node('node', node, backup_type='full') # Next Delta - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') # do corrupt 6 DELTA backup - file = os.path.join( - backup_dir, 'backups', 'node', - corrupt_id, 'database', 'global', 'pg_control') - - file_new = os.path.join(backup_dir, 'pg_control') - os.rename(file, file_new) + self.remove_backup_file(backup_dir, 'node', corrupt_id, + 'database/global/pg_control') # RESTORE BACKUP node.cleanup() - try: - self.restore_node( - backup_dir, 'node', node, backup_id=restore_target_id) - self.assertEqual( - 1, 0, - "Expecting Error because restore backup is corrupted.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Backup {0} is orphan'.format(restore_target_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node, backup_id=restore_target_id, + expect_error="because restore backup is corrupted") + self.assertMessage(contains=f'ERROR: Backup {restore_target_id} is orphan') self.assertEqual( 'OK', - self.show_pb(backup_dir, 'node')[0]['status'], + self.pb.show('node')[0]['status'], 'Backup STATUS should be "OK"') self.assertEqual( 'OK', - self.show_pb(backup_dir, 'node')[1]['status'], + self.pb.show('node')[1]['status'], 'Backup STATUS should be "OK"') self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node')[2]['status'], + self.pb.show('node')[2]['status'], 'Backup STATUS should be "ERROR"') self.assertEqual( 'OK', - self.show_pb(backup_dir, 'node')[3]['status'], + self.pb.show('node')[3]['status'], 'Backup STATUS should be "OK"') self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node')[4]['status'], + self.pb.show('node')[4]['status'], 'Backup STATUS should be "ERROR"') self.assertEqual( 'OK', - self.show_pb(backup_dir, 'node')[5]['status'], + self.pb.show('node')[5]['status'], 'Backup STATUS should be "OK"') self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node')[6]['status'], + self.pb.show('node')[6]['status'], 'Backup STATUS should be "ERROR"') self.assertEqual( 'OK', - self.show_pb(backup_dir, 'node')[7]['status'], + self.pb.show('node')[7]['status'], 'Backup STATUS should be "OK"') # corruption victim self.assertEqual( 'CORRUPT', - self.show_pb(backup_dir, 'node')[8]['status'], + self.pb.show('node')[8]['status'], 'Backup STATUS should be "CORRUPT"') # orphaned child self.assertEqual( 'ORPHAN', - self.show_pb(backup_dir, 'node')[9]['status'], + self.pb.show('node')[9]['status'], 'Backup STATUS should be "ORPHAN"') # orphaned child self.assertEqual( 'ORPHAN', - self.show_pb(backup_dir, 'node')[10]['status'], + self.pb.show('node')[10]['status'], 'Backup STATUS should be "ORPHAN"') # next FULL self.assertEqual( 'OK', - self.show_pb(backup_dir, 'node')[11]['status'], + self.pb.show('node')[11]['status'], 'Backup STATUS should be "OK"') # next DELTA self.assertEqual( 'OK', - self.show_pb(backup_dir, 'node')[12]['status'], + self.pb.show('node')[12]['status'], 'Backup STATUS should be "OK"') node.cleanup() @@ -1759,37 +1448,34 @@ def test_restore_chain_with_corrupted_backup(self): @unittest.skip("skip") def test_restore_backup_from_future(self): """more complex test_restore_chain()""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + if not backup_dir.is_file_based: + self.skipTest("test uses 'rename' in backup directory") + + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Take FULL - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.pgbench_init(scale=5) # pgbench = node.pgbench(options=['-T', '20', '-c', '2']) # pgbench.wait() # Take PAGE from future - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id = self.pb.backup_node('node', node, backup_type='page') - with open( - os.path.join( - backup_dir, 'backups', 'node', - backup_id, "backup.control"), "a") as conf: - conf.write("start-time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() + timedelta(days=3))) + with self.modify_backup_control(backup_dir, 'node', backup_id) as cf: + cf.data += "\nstart-time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() + timedelta(days=3)) # rename directory - new_id = self.show_pb(backup_dir, 'node')[1]['id'] + new_id = self.pb.show('node')[1]['id'] os.rename( os.path.join(backup_dir, 'backups', 'node', backup_id), @@ -1798,11 +1484,11 @@ def test_restore_backup_from_future(self): pgbench = node.pgbench(options=['-T', '7', '-c', '1', '--no-vacuum']) pgbench.wait() - backup_id = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id = self.pb.backup_node('node', node, backup_type='page') pgdata = self.pgdata_content(node.data_dir) node.cleanup() - self.restore_node(backup_dir, 'node', node, backup_id=backup_id) + self.pb.restore_node('node', node=node, backup_id=backup_id) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -1813,29 +1499,25 @@ def test_restore_target_immediate_stream(self): correct handling of immediate recovery target for STREAM backups """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # Take FULL - self.backup_node( - backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) # Take delta - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, backup_type='delta', options=['--stream']) pgdata = self.pgdata_content(node.data_dir) # TODO update test - if self.get_version(node) >= self.version_to_num('12.0'): + if self.pg_config_version >= self.version_to_num('12.0'): recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') with open(recovery_conf, 'r') as f: print(f.read()) @@ -1844,8 +1526,7 @@ def test_restore_target_immediate_stream(self): # restore delta backup node.cleanup() - self.restore_node( - backup_dir, 'node', node, options=['--immediate']) + self.pb.restore_node('node', node, options=['--immediate']) self.assertTrue( os.path.isfile(recovery_conf), @@ -1853,8 +1534,7 @@ def test_restore_target_immediate_stream(self): # restore delta backup node.cleanup() - self.restore_node( - backup_dir, 'node', node, options=['--recovery-target=immediate']) + self.pb.restore_node('node', node, options=['--recovery-target=immediate']) self.assertTrue( os.path.isfile(recovery_conf), @@ -1866,30 +1546,26 @@ def test_restore_target_immediate_archive(self): correct handling of immediate recovery target for ARCHIVE backups """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Take FULL - self.backup_node( - backup_dir, 'node', node) + self.pb.backup_node('node', node) # Take delta - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, backup_type='delta') pgdata = self.pgdata_content(node.data_dir) # TODO update test - if self.get_version(node) >= self.version_to_num('12.0'): + if self.pg_config_version >= self.version_to_num('12.0'): recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') with open(recovery_conf, 'r') as f: print(f.read()) @@ -1898,8 +1574,7 @@ def test_restore_target_immediate_archive(self): # restore page backup node.cleanup() - self.restore_node( - backup_dir, 'node', node, options=['--immediate']) + self.pb.restore_node('node', node, options=['--immediate']) # For archive backup with immediate recovery target # recovery.conf is mandatory @@ -1908,42 +1583,41 @@ def test_restore_target_immediate_archive(self): # restore page backup node.cleanup() - self.restore_node( - backup_dir, 'node', node, options=['--recovery-target=immediate']) + self.pb.restore_node('node', node, options=['--recovery-target=immediate']) # For archive backup with immediate recovery target # recovery.conf is mandatory with open(recovery_conf, 'r') as f: self.assertIn("recovery_target = 'immediate'", f.read()) - # @unittest.skip("skip") + # Skipped, because default recovery_target_timeline is 'current' + # Before PBCKP-598 the --recovery-target=latest' option did not work and this test allways passed + @unittest.skip("skip") def test_restore_target_latest_archive(self): """ make sure that recovery_target 'latest' is default recovery target """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Take FULL - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) - if self.get_version(node) >= self.version_to_num('12.0'): + if self.pg_config_version >= self.version_to_num('12.0'): recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') else: recovery_conf = os.path.join(node.data_dir, 'recovery.conf') # restore node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) # hash_1 = hashlib.md5( # open(recovery_conf, 'rb').read()).hexdigest() @@ -1960,7 +1634,7 @@ def test_restore_target_latest_archive(self): content_1 += line node.cleanup() - self.restore_node(backup_dir, 'node', node, options=['--recovery-target=latest']) + self.pb.restore_node('node', node=node, options=['--recovery-target=latest']) # hash_2 = hashlib.md5( # open(recovery_conf, 'rb').read()).hexdigest() @@ -1984,22 +1658,20 @@ def test_restore_target_new_options(self): check that new --recovery-target-* options are working correctly """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Take FULL - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # TODO update test - if self.get_version(node) >= self.version_to_num('12.0'): + if self.pg_config_version >= self.version_to_num('12.0'): recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') with open(recovery_conf, 'r') as f: print(f.read()) @@ -2022,7 +1694,7 @@ def test_restore_target_new_options(self): target_name = 'savepoint' # in python-3.6+ it can be ...now()..astimezone()... - target_time = datetime.utcnow().replace(tzinfo=timezone.utc).astimezone().strftime("%Y-%m-%d %H:%M:%S %z") + target_time = datetime.utcnow().replace(tzinfo=timezone.utc).astimezone().strftime("%Y-%m-%d %H:%M:%S.%f%z") with node.connect("postgres") as con: res = con.execute( "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") @@ -2032,10 +1704,7 @@ def test_restore_target_new_options(self): with node.connect("postgres") as con: con.execute("INSERT INTO tbl0005 VALUES (1)") con.commit() - if self.get_version(node) > self.version_to_num('10.0'): - res = con.execute("SELECT pg_current_wal_lsn()") - else: - res = con.execute("SELECT pg_current_xlog_location()") + res = con.execute("SELECT pg_current_wal_lsn()") con.commit() con.execute("INSERT INTO tbl0005 VALUES (2)") @@ -2046,8 +1715,7 @@ def test_restore_target_new_options(self): # Restore with recovery target time node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ '--recovery-target-time={0}'.format(target_time), "--recovery-target-action=promote", @@ -2073,8 +1741,7 @@ def test_restore_target_new_options(self): # Restore with recovery target xid node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ '--recovery-target-xid={0}'.format(target_xid), "--recovery-target-action=promote", @@ -2100,8 +1767,7 @@ def test_restore_target_new_options(self): # Restore with recovery target name node.cleanup() - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ '--recovery-target-name={0}'.format(target_name), "--recovery-target-action=promote", @@ -2126,33 +1792,31 @@ def test_restore_target_new_options(self): node.slow_start() # Restore with recovery target lsn - if self.get_version(node) >= 100000: - node.cleanup() - self.restore_node( - backup_dir, 'node', node, - options=[ - '--recovery-target-lsn={0}'.format(target_lsn), - "--recovery-target-action=promote", - '--recovery-target-timeline=1', - ]) + node.cleanup() + self.pb.restore_node('node', node, + options=[ + '--recovery-target-lsn={0}'.format(target_lsn), + "--recovery-target-action=promote", + '--recovery-target-timeline=1', + ]) - with open(recovery_conf, 'r') as f: - recovery_conf_content = f.read() + with open(recovery_conf, 'r') as f: + recovery_conf_content = f.read() - self.assertIn( - "recovery_target_lsn = '{0}'".format(target_lsn), - recovery_conf_content) + self.assertIn( + "recovery_target_lsn = '{0}'".format(target_lsn), + recovery_conf_content) - self.assertIn( - "recovery_target_action = 'promote'", - recovery_conf_content) + self.assertIn( + "recovery_target_action = 'promote'", + recovery_conf_content) - self.assertIn( - "recovery_target_timeline = '1'", - recovery_conf_content) + self.assertIn( + "recovery_target_timeline = '1'", + recovery_conf_content) - node.slow_start() + node.slow_start() # @unittest.skip("skip") def test_smart_restore(self): @@ -2163,15 +1827,13 @@ def test_smart_restore(self): copied during restore https://github.com/postgrespro/pg_probackup/issues/63 """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # create database @@ -2180,7 +1842,7 @@ def test_smart_restore(self): "CREATE DATABASE testdb") # take FULL backup - full_id = self.backup_node(backup_dir, 'node', node) + full_id = self.pb.backup_node('node', node) # drop database node.safe_psql( @@ -2188,29 +1850,24 @@ def test_smart_restore(self): "DROP DATABASE testdb") # take PAGE backup - page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id = self.pb.backup_node('node', node, backup_type='page') # restore PAGE backup node.cleanup() - self.restore_node( - backup_dir, 'node', node, backup_id=page_id, + self.pb.restore_node('node', node, backup_id=page_id, options=['--no-validate', '--log-level-file=VERBOSE']) - logfile = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(logfile, 'r') as f: - logfile_content = f.read() + logfile_content = self.read_pb_log() # get delta between FULL and PAGE filelists - filelist_full = self.get_backup_filelist( - backup_dir, 'node', full_id) + filelist_full = self.get_backup_filelist(backup_dir, 'node', full_id) - filelist_page = self.get_backup_filelist( - backup_dir, 'node', page_id) + filelist_page = self.get_backup_filelist(backup_dir, 'node', page_id) filelist_diff = self.get_backup_filelist_diff( filelist_full, filelist_page) + self.assertTrue(filelist_diff, 'There should be deleted files') for file in filelist_diff: self.assertNotIn(file, logfile_content) @@ -2222,61 +1879,52 @@ def test_pg_11_group_access(self): if self.pg_config_version < self.version_to_num('11.0'): self.skipTest('You need PostgreSQL >= 11 for this test') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=[ - '--data-checksums', - '--allow-group-access']) + initdb_params=['--allow-group-access']) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # take FULL backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) pgdata = self.pgdata_content(node.data_dir) # restore backup - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node( - backup_dir, 'node', node_restored) + self.pb.restore_node('node', node_restored) # compare pgdata permissions pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) # @unittest.skip("skip") + @needs_gdb def test_restore_concurrent_drop_table(self): """""" - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=1) # FULL backup - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=['--stream', '--compress']) # DELTA backup - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='delta', + gdb = self.pb.backup_node('node', node, backup_type='delta', options=['--stream', '--compress', '--no-validate'], gdb=True) @@ -2292,14 +1940,12 @@ def test_restore_concurrent_drop_table(self): 'postgres', 'CHECKPOINT') - gdb.remove_all_breakpoints() gdb.continue_execution_until_exit() pgdata = self.pgdata_content(node.data_dir) node.cleanup() - self.restore_node( - backup_dir, 'node', node, options=['--no-validate']) + self.pb.restore_node('node', node, options=['--no-validate']) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -2307,56 +1953,35 @@ def test_restore_concurrent_drop_table(self): # @unittest.skip("skip") def test_lost_non_data_file(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) + backup_id = self.pb.backup_node('node', node, options=['--stream']) - file = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', 'postgresql.auto.conf') - - os.remove(file) + self.remove_backup_file(backup_dir, 'node', backup_id, + 'database/postgresql.auto.conf') node.cleanup() - try: - self.restore_node( - backup_dir, 'node', node, options=['--no-validate']) - self.assertEqual( - 1, 0, - "Expecting Error because of non-data file dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'No such file or directory', e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'ERROR: Backup files restoring failed', e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node, options=['--no-validate'], + expect_error="because of non-data file dissapearance") + self.assertMessage(contains='No such file') + self.assertMessage(contains='ERROR: Backup files restoring failed') def test_partial_restore_exclude(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() for i in range(1, 10, 1): @@ -2377,34 +2002,20 @@ def test_partial_restore_exclude(self): db_list[line['datname']] = line['oid'] # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) pgdata = self.pgdata_content(node.data_dir) # restore FULL backup - node_restored_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) + node_restored_1 = self.pg_node.make_simple('node_restored_1') node_restored_1.cleanup() - try: - self.restore_node( - backup_dir, 'node', - node_restored_1, options=[ - "--db-include=db1", - "--db-exclude=db2"]) - self.assertEqual( - 1, 0, - "Expecting Error because of 'db-exclude' and 'db-include'.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: You cannot specify '--db-include' " - "and '--db-exclude' together", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node_restored_1, + options=["--db-include=db1", "--db-exclude=db2"], + expect_error="because of 'db-exclude' and 'db-include'") + self.assertMessage(contains="ERROR: You cannot specify '--db-include' " + "and '--db-exclude' together") - self.restore_node( - backup_dir, 'node', node_restored_1) + self.pb.restore_node('node', node_restored_1) pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) self.compare_pgdata(pgdata, pgdata_restored_1) @@ -2418,12 +2029,10 @@ def test_partial_restore_exclude(self): self.truncate_every_file_in_dir(db5_path) pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) - node_restored_2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored_2')) + node_restored_2 = self.pg_node.make_simple('node_restored_2') node_restored_2.cleanup() - self.restore_node( - backup_dir, 'node', + self.pb.restore_node('node', node_restored_2, options=[ "--db-exclude=db1", "--db-exclude=db5"]) @@ -2431,7 +2040,7 @@ def test_partial_restore_exclude(self): pgdata_restored_2 = self.pgdata_content(node_restored_2.data_dir) self.compare_pgdata(pgdata_restored_1, pgdata_restored_2) - self.set_auto_conf(node_restored_2, {'port': node_restored_2.port}) + node_restored_2.set_auto_conf({'port': node_restored_2.port}) node_restored_2.slow_start() @@ -2460,14 +2069,12 @@ def test_partial_restore_exclude(self): def test_partial_restore_exclude_tablespace(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() cat_version = node.get_control_data()["Catalog version number"] @@ -2504,18 +2111,16 @@ def test_partial_restore_exclude_tablespace(self): db_list[line['datname']] = line['oid'] # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) pgdata = self.pgdata_content(node.data_dir) # restore FULL backup - node_restored_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) + node_restored_1 = self.pg_node.make_simple('node_restored_1') node_restored_1.cleanup() node1_tablespace = self.get_tblspace_path(node_restored_1, 'somedata') - self.restore_node( - backup_dir, 'node', + self.pb.restore_node('node', node_restored_1, options=[ "-T", "{0}={1}".format( node_tablespace, node1_tablespace)]) @@ -2534,13 +2139,11 @@ def test_partial_restore_exclude_tablespace(self): pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) - node_restored_2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored_2')) + node_restored_2 = self.pg_node.make_simple('node_restored_2') node_restored_2.cleanup() node2_tablespace = self.get_tblspace_path(node_restored_2, 'somedata') - self.restore_node( - backup_dir, 'node', + self.pb.restore_node('node', node_restored_2, options=[ "--db-exclude=db1", "--db-exclude=db5", @@ -2550,7 +2153,7 @@ def test_partial_restore_exclude_tablespace(self): pgdata_restored_2 = self.pgdata_content(node_restored_2.data_dir) self.compare_pgdata(pgdata_restored_1, pgdata_restored_2) - self.set_auto_conf(node_restored_2, {'port': node_restored_2.port}) + node_restored_2.set_auto_conf({'port': node_restored_2.port}) node_restored_2.slow_start() @@ -2580,14 +2183,12 @@ def test_partial_restore_exclude_tablespace(self): def test_partial_restore_include(self): """ """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() for i in range(1, 10, 1): @@ -2608,34 +2209,20 @@ def test_partial_restore_include(self): db_list[line['datname']] = line['oid'] # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) pgdata = self.pgdata_content(node.data_dir) # restore FULL backup - node_restored_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) + node_restored_1 = self.pg_node.make_simple('node_restored_1') node_restored_1.cleanup() - try: - self.restore_node( - backup_dir, 'node', - node_restored_1, options=[ - "--db-include=db1", - "--db-exclude=db2"]) - self.assertEqual( - 1, 0, - "Expecting Error because of 'db-exclude' and 'db-include'.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: You cannot specify '--db-include' " - "and '--db-exclude' together", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node_restored_1, + options=["--db-include=db1", "--db-exclude=db2"], + expect_error="because of 'db-exclude' and 'db-include'") + self.assertMessage(contains="ERROR: You cannot specify '--db-include' " + "and '--db-exclude' together") - self.restore_node( - backup_dir, 'node', node_restored_1) + self.pb.restore_node('node', node_restored_1) pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) self.compare_pgdata(pgdata, pgdata_restored_1) @@ -2651,12 +2238,10 @@ def test_partial_restore_include(self): pgdata_restored_1 = self.pgdata_content(node_restored_1.data_dir) - node_restored_2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored_2')) + node_restored_2 = self.pg_node.make_simple('node_restored_2') node_restored_2.cleanup() - self.restore_node( - backup_dir, 'node', + self.pb.restore_node('node', node_restored_2, options=[ "--db-include=db1", "--db-include=db5", @@ -2665,7 +2250,7 @@ def test_partial_restore_include(self): pgdata_restored_2 = self.pgdata_content(node_restored_2.data_dir) self.compare_pgdata(pgdata_restored_1, pgdata_restored_2) - self.set_auto_conf(node_restored_2, {'port': node_restored_2.port}) + node_restored_2.set_auto_conf({'port': node_restored_2.port}) node_restored_2.slow_start() node_restored_2.safe_psql( @@ -2706,14 +2291,12 @@ def test_partial_restore_backward_compatibility_1(self): if not self.probackup_old_path: self.skipTest("You must specify PGPROBACKUPBIN_OLD" " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) + self.pb.init(old_binary=True) + self.pb.add_instance('node', node, old_binary=True) node.slow_start() @@ -2724,35 +2307,21 @@ def test_partial_restore_backward_compatibility_1(self): 'CREATE database db{0}'.format(i)) # FULL backup with old binary, without partial restore support - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, old_binary=True, options=['--stream']) pgdata = self.pgdata_content(node.data_dir) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - try: - self.restore_node( - backup_dir, 'node', - node_restored, options=[ - "--db-exclude=db5"]) - self.assertEqual( - 1, 0, - "Expecting Error because backup do not support partial restore.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} doesn't contain a database_map, " - "partial restore is impossible".format(backup_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node_restored, + options=["--db-exclude=db5"], + expect_error="because backup do not support partial restore") + self.assertMessage(contains=f"ERROR: Backup {backup_id} doesn't contain " + "a database_map, partial restore is impossible") - self.restore_node(backup_dir, 'node', node_restored) + self.pb.restore_node('node', node=node_restored) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -2774,13 +2343,12 @@ def test_partial_restore_backward_compatibility_1(self): line = json.loads(line) db_list[line['datname']] = line['oid'] - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, backup_type='delta', options=['--stream']) # get etalon node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored) + self.pb.restore_node('node', node=node_restored) self.truncate_every_file_in_dir( os.path.join( node_restored.data_dir, 'base', db_list['db5'])) @@ -2790,12 +2358,10 @@ def test_partial_restore_backward_compatibility_1(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) # get new node - node_restored_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) + node_restored_1 = self.pg_node.make_simple('node_restored_1') node_restored_1.cleanup() - self.restore_node( - backup_dir, 'node', + self.pb.restore_node('node', node_restored_1, options=[ "--db-exclude=db5", "--db-exclude=db14"]) @@ -2811,14 +2377,12 @@ def test_partial_restore_backward_compatibility_merge(self): if not self.probackup_old_path: self.skipTest("You must specify PGPROBACKUPBIN_OLD" " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir, old_binary=True) - self.add_instance(backup_dir, 'node', node, old_binary=True) + self.pb.init(old_binary=True) + self.pb.add_instance('node', node, old_binary=True) node.slow_start() @@ -2829,35 +2393,21 @@ def test_partial_restore_backward_compatibility_merge(self): 'CREATE database db{0}'.format(i)) # FULL backup with old binary, without partial restore support - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, old_binary=True, options=['--stream']) pgdata = self.pgdata_content(node.data_dir) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - try: - self.restore_node( - backup_dir, 'node', - node_restored, options=[ - "--db-exclude=db5"]) - self.assertEqual( - 1, 0, - "Expecting Error because backup do not support partial restore.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} doesn't contain a database_map, " - "partial restore is impossible.".format(backup_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node=node_restored, + options=["--db-exclude=db5"], + expect_error="because backup do not support partial restore") + self.assertMessage(contains=f"ERROR: Backup {backup_id} doesn't contain a database_map, " + "partial restore is impossible.") - self.restore_node(backup_dir, 'node', node_restored) + self.pb.restore_node('node', node=node_restored) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -2879,13 +2429,12 @@ def test_partial_restore_backward_compatibility_merge(self): line = json.loads(line) db_list[line['datname']] = line['oid'] - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, backup_type='delta', options=['--stream']) # get etalon node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored) + self.pb.restore_node('node', node_restored) self.truncate_every_file_in_dir( os.path.join( node_restored.data_dir, 'base', db_list['db5'])) @@ -2895,15 +2444,13 @@ def test_partial_restore_backward_compatibility_merge(self): pgdata_restored = self.pgdata_content(node_restored.data_dir) # get new node - node_restored_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) + node_restored_1 = self.pg_node.make_simple('node_restored_1') node_restored_1.cleanup() # merge - self.merge_backup(backup_dir, 'node', backup_id=backup_id) + self.pb.merge_backup('node', backup_id=backup_id) - self.restore_node( - backup_dir, 'node', + self.pb.restore_node('node', node_restored_1, options=[ "--db-exclude=db5", "--db-exclude=db14"]) @@ -2914,14 +2461,12 @@ def test_partial_restore_backward_compatibility_merge(self): def test_empty_and_mangled_database_map(self): """ """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() @@ -2932,93 +2477,46 @@ def test_empty_and_mangled_database_map(self): 'CREATE database db{0}'.format(i)) # FULL backup with database_map - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) + backup_id = self.pb.backup_node('node', node, options=['--stream']) pgdata = self.pgdata_content(node.data_dir) - # truncate database_map - path = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', 'database_map') - with open(path, "w") as f: - f.close() + self.corrupt_backup_file(backup_dir, 'node', backup_id, + 'database/database_map', truncate=0) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=["--db-include=db1", '--no-validate']) - self.assertEqual( - 1, 0, - "Expecting Error because database_map is empty.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} has empty or mangled database_map, " - "partial restore is impossible".format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=["--db-exclude=db1", '--no-validate']) - self.assertEqual( - 1, 0, - "Expecting Error because database_map is empty.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} has empty or mangled database_map, " - "partial restore is impossible".format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - # mangle database_map - with open(path, "w") as f: - f.write("42") - f.close() - - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=["--db-include=db1", '--no-validate']) - self.assertEqual( - 1, 0, - "Expecting Error because database_map is empty.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Field "dbOid" is not found in the line 42 of ' - 'the file backup_content.control', e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=["--db-exclude=db1", '--no-validate']) - self.assertEqual( - 1, 0, - "Expecting Error because database_map is empty.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Field "dbOid" is not found in the line 42 of ' - 'the file backup_content.control', e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node_restored, + options=["--db-include=db1", '--no-validate'], + expect_error="because database_map is empty") + self.assertMessage(contains=f"ERROR: Backup {backup_id} has empty or " + "mangled database_map, partial restore " + "is impossible") + + self.pb.restore_node('node', node_restored, + options=["--db-exclude=db1", '--no-validate'], + expect_error="because database_map is empty") + self.assertMessage(contains=f"ERROR: Backup {backup_id} has empty or " + "mangled database_map, partial restore " + "is impossible") + + self.corrupt_backup_file(backup_dir, 'node', backup_id, + 'database/database_map', overwrite=b'42') + + self.pb.restore_node('node', node_restored, + options=["--db-include=db1", '--no-validate'], + expect_error="because database_map is corrupted") + self.assertMessage(contains='ERROR: backup_content.control file has ' + 'invalid format in line 42') + + self.pb.restore_node('node', node_restored, + options=["--db-exclude=db1", '--no-validate'], + expect_error="because database_map is corrupted") + self.assertMessage(contains='ERROR: backup_content.control file has ' + 'invalid format in line 42') # check that simple restore is still possible - self.restore_node( - backup_dir, 'node', node_restored, options=['--no-validate']) + self.pb.restore_node('node', node_restored, options=['--no-validate']) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -3026,15 +2524,13 @@ def test_empty_and_mangled_database_map(self): def test_missing_database_map(self): """ """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - ptrack_enable=self.ptrack, - initdb_params=['--data-checksums']) + ptrack_enable=self.ptrack) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() @@ -3048,81 +2544,8 @@ def test_missing_database_map(self): "postgres", "CREATE DATABASE backupdb") - # PG 9.5 - if self.get_version(node) < 90600: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;") - # PG 9.6 - elif self.get_version(node) > 90600 and self.get_version(node) < 100000: - node.safe_psql( - 'backupdb', - "REVOKE ALL ON DATABASE backupdb from PUBLIC; " - "REVOKE ALL ON SCHEMA public from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA public FROM PUBLIC; " - "REVOKE ALL ON SCHEMA pg_catalog from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA pg_catalog FROM PUBLIC; " - "REVOKE ALL ON SCHEMA information_schema from PUBLIC; " - "REVOKE ALL ON ALL TABLES IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL FUNCTIONS IN SCHEMA information_schema FROM PUBLIC; " - "REVOKE ALL ON ALL SEQUENCES IN SCHEMA information_schema FROM PUBLIC; " - "CREATE ROLE backup WITH LOGIN REPLICATION; " - "GRANT CONNECT ON DATABASE backupdb to backup; " - "GRANT USAGE ON SCHEMA pg_catalog TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_proc TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_extension TO backup; " - "GRANT SELECT ON TABLE pg_catalog.pg_database TO backup; " # for partial restore, checkdb and ptrack - "GRANT EXECUTE ON FUNCTION pg_catalog.oideq(oid, oid) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.nameeq(name, name) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.textout(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.timestamptz(timestamp with time zone, integer) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.current_setting(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.set_config(text, text, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_is_in_recovery() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_control_system() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_start_backup(text, boolean, boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_stop_backup(boolean) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_create_restore_point(text) TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_xlog() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.pg_last_xlog_replay_location() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_current_snapshot() TO backup; " - "GRANT EXECUTE ON FUNCTION pg_catalog.txid_snapshot_xmax(txid_snapshot) TO backup;" - ) - # >= 10 && < 15 - elif self.get_version(node) >= 100000 and self.get_version(node) < 150000: + # PG < 15 + if self.pg_config_version >= 100000 and self.pg_config_version < 150000: node.safe_psql( 'backupdb', "REVOKE ALL ON DATABASE backupdb from PUBLIC; " @@ -3207,7 +2630,7 @@ def test_missing_database_map(self): "GRANT USAGE ON SCHEMA ptrack TO backup; " "CREATE EXTENSION ptrack WITH SCHEMA ptrack") - if ProbackupTest.enterprise: + if ProbackupTest.pgpro: node.safe_psql( "backupdb", @@ -3215,53 +2638,29 @@ def test_missing_database_map(self): "GRANT EXECUTE ON FUNCTION pg_catalog.pgpro_edition() TO backup;") # FULL backup without database_map - backup_id = self.backup_node( - backup_dir, 'node', node, datname='backupdb', + backup_id = self.pb.backup_node('node', node, datname='backupdb', options=['--stream', "-U", "backup", '--log-level-file=verbose']) pgdata = self.pgdata_content(node.data_dir) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() # backup has missing database_map and that is legal - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=["--db-exclude=db5", "--db-exclude=db9"]) - self.assertEqual( - 1, 0, - "Expecting Error because user do not have pg_database access.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} doesn't contain a database_map, " - "partial restore is impossible.".format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.restore_node( - backup_dir, 'node', node_restored, - options=["--db-include=db1"]) - self.assertEqual( - 1, 0, - "Expecting Error because user do not have pg_database access.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} doesn't contain a database_map, " - "partial restore is impossible.".format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.restore_node('node', node_restored, + options=["--db-exclude=db5", "--db-exclude=db9"], + expect_error="because user do not have pg_database access") + self.assertMessage(contains=f"ERROR: Backup {backup_id} doesn't contain a database_map, " + "partial restore is impossible.") + + self.pb.restore_node('node', node_restored, + options=["--db-include=db1"], + expect_error="because user do not have pg_database access") + self.assertMessage(contains=f"ERROR: Backup {backup_id} doesn't contain a database_map, " + "partial restore is impossible.") # check that simple restore is still possible - self.restore_node(backup_dir, 'node', node_restored) + self.pb.restore_node('node', node_restored) pgdata_restored = self.pgdata_content(node_restored.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -3280,20 +2679,18 @@ def test_stream_restore_command_option(self): as replica, check that PostgreSQL recovery uses restore_command to obtain WAL from archive. """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={'max_wal_size': '32MB'}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # TODO update test - if self.get_version(node) >= self.version_to_num('12.0'): + if self.pg_config_version >= self.version_to_num('12.0'): recovery_conf = os.path.join(node.data_dir, 'postgresql.auto.conf') with open(recovery_conf, 'r') as f: print(f.read()) @@ -3301,8 +2698,7 @@ def test_stream_restore_command_option(self): recovery_conf = os.path.join(node.data_dir, 'recovery.conf') # Take FULL - self.backup_node( - backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) node.pgbench_init(scale=5) @@ -3314,10 +2710,9 @@ def test_stream_restore_command_option(self): node.cleanup() shutil.rmtree(os.path.join(node.logs_dir)) - restore_cmd = self.get_restore_command(backup_dir, 'node', node) + restore_cmd = self.get_restore_command(backup_dir, 'node') - self.restore_node( - backup_dir, 'node', node, + self.pb.restore_node('node', node, options=[ '--restore-command={0}'.format(restore_cmd)]) @@ -3325,7 +2720,7 @@ def test_stream_restore_command_option(self): os.path.isfile(recovery_conf), "File '{0}' do not exists".format(recovery_conf)) - if self.get_version(node) >= self.version_to_num('12.0'): + if self.pg_config_version >= self.version_to_num('12.0'): recovery_signal = os.path.join(node.data_dir, 'recovery.signal') self.assertTrue( os.path.isfile(recovery_signal), @@ -3347,40 +2742,36 @@ def test_stream_restore_command_option(self): def test_restore_primary_conninfo(self): """ """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # Take FULL - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) node.pgbench_init(scale=1) #primary_conninfo = 'host=192.168.1.50 port=5432 user=foo password=foopass' - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() str_conninfo='host=192.168.1.50 port=5432 user=foo password=foopass' - self.restore_node( - backup_dir, 'node', replica, + self.pb.restore_node('node', replica, options=['-R', '--primary-conninfo={0}'.format(str_conninfo)]) - if self.get_version(node) >= self.version_to_num('12.0'): + if self.pg_config_version >= self.version_to_num('12.0'): standby_signal = os.path.join(replica.data_dir, 'standby.signal') self.assertTrue( os.path.isfile(standby_signal), "File '{0}' do not exists".format(standby_signal)) # TODO update test - if self.get_version(node) >= self.version_to_num('12.0'): + if self.pg_config_version >= self.version_to_num('12.0'): recovery_conf = os.path.join(replica.data_dir, 'postgresql.auto.conf') with open(recovery_conf, 'r') as f: print(f.read()) @@ -3396,36 +2787,32 @@ def test_restore_primary_conninfo(self): def test_restore_primary_slot_info(self): """ """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # Take FULL - self.backup_node(backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) node.pgbench_init(scale=1) - replica = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'replica')) + replica = self.pg_node.make_simple('replica') replica.cleanup() node.safe_psql( "SELECT pg_create_physical_replication_slot('master_slot')") - self.restore_node( - backup_dir, 'node', replica, + self.pb.restore_node('node', replica, options=['-R', '--primary-slot-name=master_slot']) - self.set_auto_conf(replica, {'port': replica.port}) - self.set_auto_conf(replica, {'hot_standby': 'on'}) + replica.set_auto_conf({'port': replica.port}) + replica.set_auto_conf({'hot_standby': 'on'}) - if self.get_version(node) >= self.version_to_num('12.0'): + if self.pg_config_version >= self.version_to_num('12.0'): standby_signal = os.path.join(replica.data_dir, 'standby.signal') self.assertTrue( os.path.isfile(standby_signal), @@ -3437,14 +2824,12 @@ def test_issue_249(self): """ https://github.com/postgrespro/pg_probackup/issues/249 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -3466,24 +2851,20 @@ def test_issue_249(self): 'select * from pgbench_accounts') # FULL backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.safe_psql( 'postgres', 'INSERT INTO pgbench_accounts SELECT * FROM t1') # restore FULL backup - node_restored_1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored_1')) + node_restored_1 = self.pg_node.make_simple('node_restored_1') node_restored_1.cleanup() - self.restore_node( - backup_dir, 'node', + self.pb.restore_node('node', node_restored_1, options=["--db-include=db1"]) - self.set_auto_conf( - node_restored_1, - {'port': node_restored_1.port, 'hot_standby': 'off'}) + node_restored_1.set_auto_conf({'port': node_restored_1.port, 'hot_standby': 'off'}) node_restored_1.slow_start() @@ -3513,18 +2894,16 @@ def test_pg_12_probackup_recovery_conf_compatibility(self): if self.version_to_num(self.old_probackup_version) >= self.version_to_num('2.4.5'): self.assertTrue(False, 'You need pg_probackup < 2.4.5 for this test') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL backup - self.backup_node(backup_dir, 'node', node, old_binary=True) + self.pb.backup_node('node', node, old_binary=True) node.pgbench_init(scale=5) @@ -3541,8 +2920,7 @@ def test_pg_12_probackup_recovery_conf_compatibility(self): node.cleanup() - self.restore_node( - backup_dir, 'node',node, + self.pb.restore_node('node',node, options=[ "--recovery-target-time={0}".format(time), "--recovery-target-action=promote"], @@ -3550,7 +2928,7 @@ def test_pg_12_probackup_recovery_conf_compatibility(self): node.slow_start() - self.backup_node(backup_dir, 'node', node, old_binary=True) + self.pb.backup_node('node', node, old_binary=True) node.pgbench_init(scale=5) @@ -3560,8 +2938,7 @@ def test_pg_12_probackup_recovery_conf_compatibility(self): node.cleanup() - self.restore_node( - backup_dir, 'node',node, + self.pb.restore_node('node',node, options=[ "--recovery-target-xid={0}".format(xid), "--recovery-target-action=promote"]) @@ -3578,29 +2955,26 @@ def test_drop_postgresql_auto_conf(self): if self.pg_config_version < self.version_to_num('12.0'): self.skipTest('You need PostgreSQL >= 12 for this test') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # drop postgresql.auto.conf auto_path = os.path.join(node.data_dir, "postgresql.auto.conf") os.remove(auto_path) - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') node.cleanup() - self.restore_node( - backup_dir, 'node',node, + self.pb.restore_node('node',node, options=[ "--recovery-target=latest", "--recovery-target-action=promote"]) @@ -3619,30 +2993,27 @@ def test_truncate_postgresql_auto_conf(self): if self.pg_config_version < self.version_to_num('12.0'): self.skipTest('You need PostgreSQL >= 12 for this test') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # truncate postgresql.auto.conf auto_path = os.path.join(node.data_dir, "postgresql.auto.conf") with open(auto_path, "w+") as f: f.truncate() - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') node.cleanup() - self.restore_node( - backup_dir, 'node',node, + self.pb.restore_node('node',node, options=[ "--recovery-target=latest", "--recovery-target-action=promote"]) @@ -3651,54 +3022,46 @@ def test_truncate_postgresql_auto_conf(self): self.assertTrue(os.path.exists(auto_path)) # @unittest.skip("skip") + @needs_gdb def test_concurrent_restore(self): """""" - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=1) # FULL backup - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=['--stream', '--compress']) pgbench = node.pgbench(options=['-T', '7', '-c', '1', '--no-vacuum']) pgbench.wait() # DELTA backup - self.backup_node( - backup_dir, 'node', node, backup_type='delta', + self.pb.backup_node('node', node, backup_type='delta', options=['--stream', '--compress', '--no-validate']) pgdata1 = self.pgdata_content(node.data_dir) - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node.cleanup() node_restored.cleanup() - gdb = self.restore_node( - backup_dir, 'node', node, options=['--no-validate'], gdb=True) + gdb = self.pb.restore_node('node', node, options=['--no-validate'], gdb=True) gdb.set_breakpoint('restore_data_file') gdb.run_until_break() - self.restore_node( - backup_dir, 'node', node_restored, options=['--no-validate']) + self.pb.restore_node('node', node_restored, options=['--no-validate']) - gdb.remove_all_breakpoints() gdb.continue_execution_until_exit() pgdata2 = self.pgdata_content(node.data_dir) @@ -3707,44 +3070,241 @@ def test_concurrent_restore(self): self.compare_pgdata(pgdata1, pgdata2) self.compare_pgdata(pgdata2, pgdata3) - # skip this test until https://github.com/postgrespro/pg_probackup/pull/399 - @unittest.skip("skip") + + # @unittest.skip("skip") + def test_restore_with_waldir(self): + """recovery using tablespace-mapping option and page backup""" + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + with node.connect("postgres") as con: + con.execute( + "CREATE TABLE tbl AS SELECT * " + "FROM generate_series(0,3) AS integer") + con.commit() + + # Full backup + backup_id = self.pb.backup_node('node', node) + + node.stop() + node.cleanup() + + # Create waldir + waldir_path = os.path.join(node.base_dir, "waldir") + os.makedirs(waldir_path) + + # Test recovery from latest + restore_result = self.pb.restore_node('node', node, + options=[ + "-X", "%s" % (waldir_path)]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) + node.slow_start() + + count = node.execute("postgres", "SELECT count(*) FROM tbl") + self.assertEqual(count[0][0], 4) + + # check pg_wal is symlink + if node.major_version >= 10: + wal_path=os.path.join(node.data_dir, "pg_wal") + else: + wal_path=os.path.join(node.data_dir, "pg_xlog") + + self.assertEqual(os.path.islink(wal_path), True) + + def test_restore_with_sync(self): + """ + By default our tests use --no-sync to speed up. + This test runs full backup and then `restore' both with fsync enabled. + """ + node = self.pg_node.make_simple('node') + self.pb.init() + self.pb.add_instance('node', node) + node.slow_start() + + node.execute("postgres", "CREATE TABLE tbl AS SELECT i as id FROM generate_series(0,3) AS i") + + backup_id = self.pb.backup_node('node', node, options=["--stream", "-j", "10"], sync=True) + + node.stop() + node.cleanup() + + restore_result = self.pb.restore_node('node', node, options=["-j", "10"], sync=True) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) + + node.slow_start() + + count = node.execute("postgres", "SELECT count(*) FROM tbl") + self.assertEqual(count[0][0], 4) + + def test_restore_target_time(self): + """ + Test that we can restore to the time which we list + as a recovery time for a backup. + """ + node = self.pg_node.make_simple('node') + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + node.safe_psql("postgres", "CREATE TABLE table_1(i int)") + node.safe_psql("postgres", "INSERT INTO table_1 values (42)") + node.safe_psql("postgres", "select pg_create_restore_point('savepoint');") + + backup_id = self.pb.backup_node('node', node) + node.safe_psql("postgres", "select txid_current();") + + node.cleanup() + + backup = self.pb.show('node', backup_id) + target_time = backup['recovery-time'] + + self.pb.restore_node('node', node, options=[f'--recovery-target-time={target_time}', + '--recovery-target-action=promote',]) + + node.slow_start() + + with node.connect("postgres") as con: + res = con.execute("SELECT * from table_1")[0][0] + self.assertEqual(42, res) + + def test_restore_after_failover(self): + """ + PITR: Check that we are able to restore to a correct timeline by replaying + the WALs even though the backup was made on a different timeline. + + Insert some data on master (D0). Take a backup. Failover to replica. + Insert some more data (D1). Record this moment as a PITR target. Insert some more data (D2). + Recover to PITR target. Make sure D1 exists, while D2 does not. + + JIRA: PBCKP-588 + """ + master = self.pg_node.make_simple('master', set_replication=True) + self.pb.init() + self.pb.add_instance('master', master) + # Streaming is not enough. We need full WALs to restore to a point in time later than the backup itself + self.pb.set_archiving('master', master) + master.slow_start() + + self.pb.backup_node('master', master, backup_type='full', options=['--stream']) + + replica = self.pg_node.make_simple('replica') + replica.cleanup() + + master.safe_psql("SELECT pg_create_physical_replication_slot('master_slot')") + + self.pb.restore_node( + 'master', replica, + options=['-R', '--primary-slot-name=master_slot']) + + replica.set_auto_conf({'port': replica.port}) + replica.set_auto_conf({'hot_standby': 'on'}) + + if self.pg_config_version >= self.version_to_num('12.0'): + standby_signal = os.path.join(replica.data_dir, 'standby.signal') + self.assertTrue( + os.path.isfile(standby_signal), + f"File '{standby_signal}' does not exist") + + replica.slow_start(replica=True) + with master.connect("postgres") as con: + master_timeline = con.execute("SELECT timeline_id FROM pg_control_checkpoint()")[0][0] + self.assertNotEqual(master_timeline, 0) + + # Now we have master<=>standby setup. + master.safe_psql("postgres", "CREATE TABLE t1 (a int, b int)") + master.safe_psql("postgres", "INSERT INTO t1 SELECT i/100, i/500 FROM generate_series(1,100000) s(i)") + + # Make a backup on timeline 1 with most of the data missing + self.pb.backup_node('master', master, backup_type='full', options=['--stream']) + + # For debugging purpose it was useful to have an incomplete commit in WAL. Might not be needed anymore + psql_path = testgres_utils.get_bin_path("psql") + os.spawnlp(os.P_NOWAIT, psql_path, psql_path, "-p", str(master.port), "-h", master.host, "-d", "postgres", + "-X", "-A", "-t", "-q", "-c", + "INSERT INTO t1 SELECT i/100, i/500 FROM generate_series(1,1000000) s(i)" + ) + + master.stop(["-m", "immediate"]) + sleep(1) + replica.promote() + + with replica.connect("postgres") as con: + replica_timeline = con.execute("SELECT min_recovery_end_timeline FROM pg_control_recovery()")[0][0] + self.assertNotEqual(master_timeline, replica_timeline) + + # Add some more on timeline 2 + replica.safe_psql("postgres", "CREATE TABLE t2 (a int, b int)") + replica.safe_psql("postgres", f"INSERT INTO t2 SELECT i/100, i/500 FROM generate_series(1,{MAGIC_COUNT}) s(i)") + + # Find out point-in-time where we would like to restore to + with replica.connect("postgres") as con: + restore_time = con.execute("SELECT now(), txid_current();")[0][0] + + # Break MAGIC_COUNT. An insert which should not be restored + replica.safe_psql("postgres", "INSERT INTO t2 SELECT i/100, i/500 FROM generate_series(1,100000) s(i)") + + replica.safe_psql("postgres", "SELECT pg_switch_wal();") + + # Final restore. We expect to find only the data up to {restore_time} and nothing else + node_restored = self.pg_node.make_simple("node_restored") + node_restored.cleanup() + self.pb.restore_node('master', node_restored, options=[ + '--no-validate', + f'--recovery-target-time={restore_time}', + f'--recovery-target-timeline={replica_timeline}', # As per ticket we do not parse WALs. User supplies timeline manually + '--recovery-target-action=promote', + '-j', '4', + ]) + node_restored.set_auto_conf({'port': node_restored.port}) + node_restored.slow_start() + + with node_restored.connect("postgres") as con: + nrows = con.execute("SELECT COUNT(*) from t2")[0][0] + self.assertEqual(MAGIC_COUNT, nrows) + + # @unittest.skip("skip") + @needs_gdb def test_restore_issue_313(self): """ Check that partially restored PostgreSQL instance cannot be started """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) node.cleanup() count = 0 - filelist = self.get_backup_filelist(backup_dir, 'node', backup_id) + filelist = self.get_backup_filelist(backup_dir, 'node', backup_id) for file in filelist: # count only nondata files - if int(filelist[file]['is_datafile']) == 0 and int(filelist[file]['size']) > 0: + if int(filelist[file]['is_datafile']) == 0 and \ + filelist[file]['kind'] != 'dir' and \ + file != 'database_map': count += 1 - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored) + self.pb.restore_node('node', node_restored) - gdb = self.restore_node(backup_dir, 'node', node, gdb=True, options=['--progress']) + gdb = self.pb.restore_node('node', node, gdb=True, options=['--progress']) gdb.verbose = False gdb.set_breakpoint('restore_non_data_file') gdb.run_until_break() - gdb.continue_execution_until_break(count - 2) + gdb.continue_execution_until_break(count - 1) gdb.quit() # emulate the user or HA taking care of PG configuration @@ -3767,54 +3327,254 @@ def test_restore_issue_313(self): '\n Unexpected Error Message: {0}\n CMD: {1}'.format( repr(e.message), self.cmd)) + with open(os.path.join(node.logs_dir, 'postgresql.log'), 'r') as f: + self.assertIn( + "postgres: could not find the database system", + f.read()) + + # @unittest.skip("skip") - def test_restore_with_waldir(self): - """recovery using tablespace-mapping option and page backup""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + def test_restore_to_latest_timeline(self): + """recovery to latest timeline""" + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() + node.pgbench_init(scale=2) - with node.connect("postgres") as con: - con.execute( - "CREATE TABLE tbl AS SELECT * " - "FROM generate_series(0,3) AS integer") - con.commit() - - # Full backup - backup_id = self.backup_node(backup_dir, 'node', node) + before1 = node.table_checksum("pgbench_branches") + backup_id = self.pb.backup_node('node', node) node.stop() node.cleanup() - # Create waldir - waldir_path = os.path.join(node.base_dir, "waldir") - os.makedirs(waldir_path) + restore_result = self.pb.restore_node('node', node, options=["-j", "4"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) - # Test recovery from latest - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id), - self.restore_node( - backup_dir, 'node', node, + node.slow_start() + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + options=['-T', '10', '-c', '2', '--no-vacuum']) + pgbench.wait() + pgbench.stdout.close() + + before2 = node.table_checksum("pgbench_branches") + self.pb.backup_node('node', node) + + node.stop() + node.cleanup() + # restore from first backup + restore_result = self.pb.restore_node('node', node, options=[ - "-X", "%s" % (waldir_path)]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + "-j", "4", "--recovery-target-timeline=latest", "-i", backup_id] + ) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) + # check recovery_target_timeline option in the recovery_conf + recovery_target_timeline = self.get_recovery_conf(node)["recovery_target_timeline"] + self.assertEqual(recovery_target_timeline, "latest") + # check recovery-target=latest option for compatibility with previous versions + node.cleanup() + restore_result = self.pb.restore_node('node', node, + options=[ + "-j", "4", "--recovery-target=latest", "-i", backup_id] + ) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) + # check recovery_target_timeline option in the recovery_conf + recovery_target_timeline = self.get_recovery_conf(node)["recovery_target_timeline"] + self.assertEqual(recovery_target_timeline, "latest") + + # start postgres and promote wal files to latest timeline node.slow_start() - count = node.execute("postgres", "SELECT count(*) FROM tbl") - self.assertEqual(count[0][0], 4) + # check for the latest updates + after = node.table_checksum("pgbench_branches") + self.assertEqual(before2, after) - # check pg_wal is symlink - if node.major_version >= 10: - wal_path=os.path.join(node.data_dir, "pg_wal") - else: - wal_path=os.path.join(node.data_dir, "pg_xlog") + # checking recovery_target_timeline=current is the default option + if self.pg_config_version >= self.version_to_num('12.0'): + node.stop() + node.cleanup() - self.assertEqual(os.path.islink(wal_path), True) + # restore from first backup + restore_result = self.pb.restore_node('node', node, + options=[ + "-j", "4", "-i", backup_id] + ) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) + + # check recovery_target_timeline option in the recovery_conf + recovery_target_timeline = self.get_recovery_conf(node)["recovery_target_timeline"] + self.assertEqual(recovery_target_timeline, "current") + + # start postgres with current timeline + node.slow_start() + + # check for the current updates + after = node.table_checksum("pgbench_branches") + self.assertEqual(before1, after) + +################################################ +# dry-run +############################################### + @unittest.skipUnless(fs_backup_class.is_file_based, "AccessPath check is always true on s3") + def test_basic_dry_run_restore(self): + """recovery dry-run """ + node = self.pg_node.make_simple('node') + + # check external directory with dry_run + external_dir = os.path.join(self.test_path, 'somedirectory') + os.mkdir(external_dir) + + new_external_dir=os.path.join(self.test_path, "restored_external_dir") + # fill external directory with data + f = open(os.path.join(external_dir, "very_important_external_file"), 'x') + f.close() + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + backup_id = self.pb.backup_node('node', node, options=["--external-dirs={0}".format(external_dir)]) + + node.stop() + + # check data absence + restore_dir = os.path.join(self.test_path, "restored_dir") + if fs_backup_class.is_file_based: #AccessPath check is always true on s3 + dir_mode = os.stat(self.test_path).st_mode + os.chmod(self.test_path, 0o500) + + # 1 - Test recovery from latest without permissions + error_message = self.pb.restore_node('node', restore_dir=restore_dir, + options=["-j", "4", + "--external-mapping={0}={1}".format(external_dir, new_external_dir), + "--dry-run"], expect_error ='because of changed permissions') + try: + self.assertMessage(error_message, contains='ERROR: Check permissions') + finally: + # Cleanup + os.chmod(self.test_path, dir_mode) + + instance_before = self.pgdata_content(self.backup_dir) + # 2 - Test recovery from latest + restore_result = self.pb.restore_node('node', restore_dir=restore_dir, + options=["-j", "4", + "--external-mapping={0}={1}".format(external_dir, new_external_dir), + "--dry-run"]) + + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed in dry-run mode".format(backup_id)) + + instance_after = self.pgdata_content(self.backup_dir) + pgdata_after = self.pgdata_content(restore_dir) + + self.compare_instance_dir( + instance_before, + instance_after + ) + + # check external directory absence + self.assertFalse(os.path.exists(new_external_dir)) + + self.assertFalse(os.path.exists(restore_dir)) + + + @unittest.skipUnless(fs_backup_class.is_file_based, "AccessPath check is always true on s3") + def test_basic_dry_run_incremental_restore(self): + """incremental recovery with system_id mismatch and --force flag in --dry-run mode""" + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + node.pgbench_init(scale=2) + pgbench = node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + backup_id = self.pb.backup_node('node', node) + + node.stop() + # check data absence + restore_dir = os.path.join(self.test_path, "restored_dir") + + # 1 - recovery from latest + restore_result = self.pb.restore_node('node', + restore_dir=restore_dir, + options=["-j", "4"]) + self.assertMessage(restore_result, contains="INFO: Restore of backup {0} completed.".format(backup_id)) + + # Make some changes + node.slow_start() + + node.pgbench( + stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + pgbench.wait() + pgbench.stdout.close() + backup_id = self.pb.backup_node('node', node, options=["--stream", + "-b DELTA"]) + node.stop() + + pg_probackup_conf = os.path.join(self.backup_dir, "backups/node/pg_probackup.conf") + + # make system_id mismatch + with open(pg_probackup_conf, 'r') as file: + data = file.read() + + match = re.search(r'(system-identifier)( = )([0-9]+)(\n)', data) + if match: + data = data.replace(match.group(3), '1111111111111111111') + + with open(pg_probackup_conf, 'w') as file: + + file.write(data) + + instance_before = self.pgdata_content(self.backup_dir) + pgdata_before = self.pgdata_content(restore_dir) + if fs_backup_class.is_file_based: #AccessPath check is always true on s3 + # Access check suite if disk mounted as read_only + dir_mode = os.stat(restore_dir).st_mode + os.chmod(restore_dir, 0o500) + + # 2 - incremetal recovery from latest without permissions + try: + error_message = self.pb.restore_node('node', + restore_dir=restore_dir, + options=["-j", "4", + "--dry-run", + "--force", + "-I", "checksum"], expect_error='because of changed permissions') + self.assertMessage(error_message, contains='ERROR: Check permissions') + finally: + # Cleanup + os.chmod(restore_dir, dir_mode) + + self.pb.restore_node('node', + restore_dir=restore_dir, + options=["-j", "4", + "--dry-run", + "--force", + "-I", "checksum"]) + instance_after = self.pgdata_content(self.backup_dir) + pgdata_after = self.pgdata_content(restore_dir) + + self.compare_instance_dir( + instance_before, + instance_after + ) + self.compare_pgdata( + pgdata_before, + pgdata_after + ) + + node.stop() diff --git a/tests/retention_test.py b/tests/retention_test.py index 88432a00f..8a462624f 100644 --- a/tests/retention_test.py +++ b/tests/retention_test.py @@ -1,47 +1,45 @@ -import os import unittest from datetime import datetime, timedelta -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest +from .helpers.ptrack_helpers import fs_backup_class +from pg_probackup2.gdb import needs_gdb +from .helpers.data_helpers import tail_file from time import sleep -from distutils.dir_util import copy_tree +import os.path -class RetentionTest(ProbackupTest, unittest.TestCase): +class RetentionTest(ProbackupTest): # @unittest.skip("skip") # @unittest.expectedFailure def test_retention_redundancy_1(self): """purge backups using redundancy-based retention policy""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.set_config( - backup_dir, 'node', options=['--retention-redundancy=1']) + self.pb.set_config('node', options=['--retention-redundancy=1']) # Make backups to be purged - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type="page") + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type="page") # Make backups to be keeped - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type="page") + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type="page") - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + self.assertEqual(len(self.pb.show('node')), 4) - output_before = self.show_archive(backup_dir, 'node', tli=1) + output_before = self.pb.show_archive('node', tli=1) # Purge backups - self.delete_expired( - backup_dir, 'node', options=['--expired', '--wal']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + self.pb.delete_expired('node', options=['--expired', '--wal']) + self.assertEqual(len(self.pb.show('node')), 2) - output_after = self.show_archive(backup_dir, 'node', tli=1) + output_after = self.pb.show_archive('node', tli=1) self.assertEqual( output_before['max-segno'], @@ -55,183 +53,241 @@ def test_retention_redundancy_1(self): min_wal = output_after['min-segno'] max_wal = output_after['max-segno'] - for wal_name in os.listdir(os.path.join(backup_dir, 'wal', 'node')): - if not wal_name.endswith(".backup"): - - if self.archive_compress: - wal_name = wal_name[-27:] - wal_name = wal_name[:-3] - else: - wal_name = wal_name[-24:] - - self.assertTrue(wal_name >= min_wal) - self.assertTrue(wal_name <= max_wal) + wals = self.get_instance_wal_list(backup_dir, 'node') + for wal_name in wals: + if self.archive_compress and wal_name.endswith(self.compress_suffix): + wal_name = wal_name[:-len(self.compress_suffix)] + self.assertGreaterEqual(wal_name, min_wal) + self.assertLessEqual(wal_name, max_wal) # @unittest.skip("skip") def test_retention_window_2(self): """purge backups using window-based retention policy""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - with open( - os.path.join( - backup_dir, - 'backups', - 'node', - "pg_probackup.conf"), "a") as conf: - conf.write("retention-redundancy = 1\n") - conf.write("retention-window = 1\n") + with self.modify_backup_config(backup_dir, 'node') as cf: + cf.data += "retention-redundancy = 1\n" + cf.data += "retention-window = 1\n" # Make backups to be purged - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type="page") + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type="page") # Make backup to be keeped - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) - backups = os.path.join(backup_dir, 'backups', 'node') days_delta = 5 - for backup in os.listdir(backups): - if backup == 'pg_probackup.conf': - continue - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=days_delta))) - days_delta -= 1 + for backup_id in backup_dir.list_instance_backups('node'): + with self.modify_backup_control(backup_dir, 'node', backup_id) as cf: + cf.data += "\nrecovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=days_delta)) + days_delta -= 1 # Make backup to be keeped - self.backup_node(backup_dir, 'node', node, backup_type="page") + self.pb.backup_node('node', node, backup_type="page") - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + self.assertEqual(len(self.pb.show('node')), 4) # Purge backups - self.delete_expired(backup_dir, 'node', options=['--expired']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + self.pb.delete_expired('node', options=['--expired']) + self.assertEqual(len(self.pb.show('node')), 2) # @unittest.skip("skip") def test_retention_window_3(self): """purge all backups using window-based retention policy""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # take FULL BACKUP - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # Take second FULL BACKUP - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # Take third FULL BACKUP - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup == 'pg_probackup.conf': - continue - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) + for backup in backup_dir.list_instance_backups('node'): + with self.modify_backup_control(backup_dir, 'node', backup) as cf: + cf.data += "\nrecovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3)) # Purge backups - self.delete_expired( - backup_dir, 'node', options=['--retention-window=1', '--expired']) + self.pb.delete_expired('node', options=['--retention-window=1', '--expired']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 0) + self.assertEqual(len(self.pb.show('node')), 0) - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) + print(self.pb.show('node', as_json=False, as_text=True)) # count wal files in ARCHIVE # @unittest.skip("skip") def test_retention_window_4(self): """purge all backups using window-based retention policy""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # take FULL BACKUPs - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) - backup_id_2 = self.backup_node(backup_dir, 'node', node) + backup_id_2 = self.pb.backup_node('node', node) - backup_id_3 = self.backup_node(backup_dir, 'node', node) + backup_id_3 = self.pb.backup_node('node', node) - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup == 'pg_probackup.conf': - continue - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) + for backup in backup_dir.list_instance_backups('node'): + with self.modify_backup_control(backup_dir, 'node', backup) as cf: + cf.data += "\nrecovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3)) - self.delete_pb(backup_dir, 'node', backup_id_2) - self.delete_pb(backup_dir, 'node', backup_id_3) + self.pb.delete('node', backup_id_2) + self.pb.delete('node', backup_id_3) # Purge backups - self.delete_expired( - backup_dir, 'node', + self.pb.delete_expired( + 'node', options=['--retention-window=1', '--expired', '--wal']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 0) + self.assertEqual(len(self.pb.show('node')), 0) - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) + print(self.pb.show('node', as_json=False, as_text=True)) # count wal files in ARCHIVE - wals_dir = os.path.join(backup_dir, 'wal', 'node') - # n_wals = len(os.listdir(wals_dir)) + wals = self.get_instance_wal_list(backup_dir, 'node') + self.assertFalse(wals) + + @unittest.skipIf(not fs_backup_class.is_file_based, "Locks are not implemented in cloud") + @needs_gdb + def test_concurrent_retention_1(self): + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + with self.modify_backup_config(backup_dir, 'node') as cf: + cf.data += "retention-redundancy = 1\n" + cf.data += "retention-window = 2\n" + + # Fill with data + node.pgbench_init(scale=1) + + full_id = self.pb.backup_node('node', node, backup_type="full") + + pgbench = node.pgbench(options=['-t', '20', '-c', '2']) + pgbench.wait() + + self.pb.backup_node('node', node, backup_type="page") + + pgbench = node.pgbench(options=['-t', '20', '-c', '2']) + pgbench.wait() + + prev_id = self.pb.backup_node('node', node, backup_type="page") + + pgbench = node.pgbench(options=['-t', '20', '-c', '2']) + pgbench.wait() + + last_id = self.pb.backup_node('node', node, backup_type="page") + + days_delta = 4 + for backup_id in backup_dir.list_instance_backups('node'): + with self.modify_backup_control(backup_dir, 'node', backup_id) as cf: + cf.data += "\nrecovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=days_delta)) + days_delta -= 1 + + gdb = self.pb.backup_node('node', node, gdb=True, + options=['--merge-expired']) + gdb.set_breakpoint("merge_chain") + gdb.run_until_break() + + self.pb.backup_node('node', node, + options=['--merge-expired'], + expect_error="because of concurrent merge") + self.assertMessage(contains=f"ERROR: Cannot lock backup {full_id}") + + @needs_gdb + def test_concurrent_retention_2(self): + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + with self.modify_backup_config(backup_dir, 'node') as cf: + cf.data += "retention-redundancy = 1\n" + cf.data += "retention-window = 2\n" + + # Fill with data + node.pgbench_init(scale=1) + + full_id = self.pb.backup_node('node', node, backup_type="full") + + pgbench = node.pgbench(options=['-t', '20', '-c', '2']) + pgbench.wait() + + self.pb.backup_node('node', node, backup_type="page") + + pgbench = node.pgbench(options=['-t', '20', '-c', '2']) + pgbench.wait() + + prev_id = self.pb.backup_node('node', node, backup_type="page") + + pgbench = node.pgbench(options=['-t', '20', '-c', '2']) + pgbench.wait() + + last_id = self.pb.backup_node('node', node, backup_type="page") - # self.assertTrue(n_wals > 0) + days_delta = 4 + for backup_id in backup_dir.list_instance_backups('node'): + with self.modify_backup_control(backup_dir, 'node', backup_id) as cf: + cf.data += "\nrecovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=days_delta)) + days_delta -= 1 - # self.delete_expired( - # backup_dir, 'node', - # options=['--retention-window=1', '--expired', '--wal']) + gdb = self.pb.backup_node('node', node, gdb=True, + options=['--merge-expired']) + gdb.set_breakpoint("merge_files") + gdb.run_until_break() - # count again - n_wals = len(os.listdir(wals_dir)) - self.assertTrue(n_wals == 0) + out = self.pb.backup_node('node', node, + options=['--merge-expired'],return_id=False) + #expect_error="because of concurrent merge") + self.assertMessage(out, contains=f"WARNING: Backup {full_id} is not in stable state") + self.assertMessage(out, contains=f"There are no backups to merge by retention policy") # @unittest.skip("skip") def test_window_expire_interleaved_incremental_chains(self): """complicated case of interleaved backup chains""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # take FULL BACKUPs - backup_id_a = self.backup_node(backup_dir, 'node', node) - backup_id_b = self.backup_node(backup_dir, 'node', node) + backup_id_a = self.pb.backup_node('node', node) + backup_id_b = self.pb.backup_node('node', node) # Change FULLb backup status to ERROR self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') @@ -240,8 +296,7 @@ def test_window_expire_interleaved_incremental_chains(self): # FULLa OK # Take PAGEa1 backup - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a1 = self.pb.backup_node('node', node, backup_type='page') # PAGEa1 OK # FULLb ERROR @@ -258,8 +313,7 @@ def test_window_expire_interleaved_incremental_chains(self): # FULLb OK # FULLa ERROR - page_id_b1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b1 = self.pb.backup_node('node', node, backup_type='page') # PAGEb1 OK # PAGEa1 ERROR @@ -280,8 +334,7 @@ def test_window_expire_interleaved_incremental_chains(self): # FULLb ERROR # FULLa OK - page_id_a2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a2 = self.pb.backup_node('node', node, backup_type='page') # PAGEa2 OK # PAGEb1 ERROR @@ -303,8 +356,7 @@ def test_window_expire_interleaved_incremental_chains(self): # FULLb OK # FULLa ERROR - page_id_b2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b2 = self.pb.backup_node('node', node, backup_type='page') # Change PAGEa2 and FULla to OK self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') @@ -318,40 +370,35 @@ def test_window_expire_interleaved_incremental_chains(self): # FULLa OK # Purge backups - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup not in [page_id_a2, page_id_b2, 'pg_probackup.conf']: - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) - - self.delete_expired( - backup_dir, 'node', + for backup in backup_dir.list_instance_backups('node'): + if backup in [page_id_a2, page_id_b2]: + continue + with self.modify_backup_control(backup_dir, 'node', backup) as cf: + cf.data += "\nrecovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3)) + + self.pb.delete_expired( + 'node', options=['--retention-window=1', '--expired']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 6) + self.assertEqual(len(self.pb.show('node')), 6) - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) + print(self.pb.show('node', as_json=False, as_text=True)) # @unittest.skip("skip") def test_redundancy_expire_interleaved_incremental_chains(self): """complicated case of interleaved backup chains""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # take FULL BACKUPs - backup_id_a = self.backup_node(backup_dir, 'node', node) - backup_id_b = self.backup_node(backup_dir, 'node', node) + backup_id_a = self.pb.backup_node('node', node) + backup_id_b = self.pb.backup_node('node', node) # Change FULL B backup status to ERROR self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') @@ -359,8 +406,7 @@ def test_redundancy_expire_interleaved_incremental_chains(self): # FULLb ERROR # FULLa OK # Take PAGEa1 backup - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a1 = self.pb.backup_node('node', node, backup_type='page') # PAGEa1 OK # FULLb ERROR @@ -377,8 +423,7 @@ def test_redundancy_expire_interleaved_incremental_chains(self): # FULLb OK # FULLa ERROR - page_id_b1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b1 = self.pb.backup_node('node', node, backup_type='page') # PAGEb1 OK # PAGEa1 ERROR @@ -398,8 +443,7 @@ def test_redundancy_expire_interleaved_incremental_chains(self): # PAGEa1 OK # FULLb ERROR # FULLa OK - page_id_a2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a2 = self.pb.backup_node('node', node, backup_type='page') # PAGEa2 OK # PAGEb1 ERROR @@ -420,7 +464,7 @@ def test_redundancy_expire_interleaved_incremental_chains(self): # PAGEa1 OK # FULLb OK # FULLa ERROR - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') # Change PAGEa2 and FULLa status to OK self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') @@ -433,31 +477,28 @@ def test_redundancy_expire_interleaved_incremental_chains(self): # FULLb OK # FULLa OK - self.delete_expired( - backup_dir, 'node', + self.pb.delete_expired( + 'node', options=['--retention-redundancy=1', '--expired']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) + self.assertEqual(len(self.pb.show('node')), 3) - print(self.show_pb( - backup_dir, 'node', as_json=False, as_text=True)) + print(self.pb.show('node', as_json=False, as_text=True)) # @unittest.skip("skip") def test_window_merge_interleaved_incremental_chains(self): """complicated case of interleaved backup chains""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Take FULL BACKUPs - backup_id_a = self.backup_node(backup_dir, 'node', node) - backup_id_b = self.backup_node(backup_dir, 'node', node) + backup_id_a = self.pb.backup_node('node', node) + backup_id_b = self.pb.backup_node('node', node) # Change FULLb backup status to ERROR self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') @@ -466,8 +507,7 @@ def test_window_merge_interleaved_incremental_chains(self): # FULLa OK # Take PAGEa1 backup - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a1 = self.pb.backup_node('node', node, backup_type='page') # PAGEa1 OK # FULLb ERROR @@ -483,8 +523,7 @@ def test_window_merge_interleaved_incremental_chains(self): # FULLb OK # FULLa OK - page_id_b1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b1 = self.pb.backup_node('node', node, backup_type='page') # PAGEb1 OK # PAGEa1 ERROR @@ -504,8 +543,7 @@ def test_window_merge_interleaved_incremental_chains(self): # FULLb ERROR # FULLa OK - page_id_a2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a2 = self.pb.backup_node('node', node, backup_type='page') # PAGEa2 OK # PAGEb1 ERROR @@ -527,8 +565,7 @@ def test_window_merge_interleaved_incremental_chains(self): # FULLb OK # FULLa ERROR - page_id_b2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b2 = self.pb.backup_node('node', node, backup_type='page') # Change PAGEa2 and FULLa to OK self.change_backup_status(backup_dir, 'node', page_id_a2, 'OK') @@ -542,17 +579,15 @@ def test_window_merge_interleaved_incremental_chains(self): # FULLa OK # Purge backups - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup not in [page_id_a2, page_id_b2, 'pg_probackup.conf']: - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) - - output = self.delete_expired( - backup_dir, 'node', + for backup in backup_dir.list_instance_backups('node'): + if backup in [page_id_a2, page_id_b2]: + continue + with self.modify_backup_control(backup_dir, 'node', backup) as cf: + cf.data += "\nrecovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3)) + + output = self.pb.delete_expired( + 'node', options=['--retention-window=1', '--expired', '--merge-expired']) self.assertIn( @@ -573,7 +608,7 @@ def test_window_merge_interleaved_incremental_chains(self): "Rename merged full backup {0} to {1}".format( backup_id_b, page_id_b2), output) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + self.assertEqual(len(self.pb.show('node')), 2) # @unittest.skip("skip") def test_window_merge_interleaved_incremental_chains_1(self): @@ -585,32 +620,29 @@ def test_window_merge_interleaved_incremental_chains_1(self): FULLb FULLa """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=5) # Take FULL BACKUPs - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) pgbench = node.pgbench(options=['-t', '20', '-c', '1']) pgbench.wait() - backup_id_b = self.backup_node(backup_dir, 'node', node) + backup_id_b = self.pb.backup_node('node', node) pgbench = node.pgbench(options=['-t', '20', '-c', '1']) pgbench.wait() # Change FULL B backup status to ERROR self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a1 = self.pb.backup_node('node', node, backup_type='page') pgdata_a1 = self.pgdata_content(node.data_dir) @@ -629,20 +661,17 @@ def test_window_merge_interleaved_incremental_chains_1(self): # PAGEa1 ERROR # FULLb OK # FULLa OK - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') pgbench = node.pgbench(options=['-t', '20', '-c', '1']) pgbench.wait() - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') pgbench = node.pgbench(options=['-t', '20', '-c', '1']) pgbench.wait() - page_id_b3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b3 = self.pb.backup_node('node', node, backup_type='page') pgdata_b3 = self.pgdata_content(node.data_dir) pgbench = node.pgbench(options=['-t', '20', '-c', '1']) @@ -666,56 +695,52 @@ def test_window_merge_interleaved_incremental_chains_1(self): # FULLa OK # Purge backups - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup in [page_id_a1, page_id_b3, 'pg_probackup.conf']: + for backup_id in backup_dir.list_instance_backups('node'): + if backup_id in [page_id_a1, page_id_b3]: continue - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) + with self.modify_backup_control(backup_dir, 'node', backup_id) as cf: + cf.data += "\nrecovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3)) - self.delete_expired( - backup_dir, 'node', + self.pb.delete_expired( + 'node', options=['--retention-window=1', '--expired', '--merge-expired']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + self.assertEqual(len(self.pb.show('node')), 2) self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['id'], + self.pb.show('node')[1]['id'], page_id_b3) self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['id'], + self.pb.show('node')[0]['id'], page_id_a1) self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['backup-mode'], + self.pb.show('node')[1]['backup-mode'], 'FULL') self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['backup-mode'], + self.pb.show('node')[0]['backup-mode'], 'FULL') node.cleanup() # Data correctness of PAGEa3 - self.restore_node(backup_dir, 'node', node, backup_id=page_id_a1) + self.pb.restore_node('node', node, backup_id=page_id_a1) pgdata_restored_a1 = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata_a1, pgdata_restored_a1) node.cleanup() # Data correctness of PAGEb3 - self.restore_node(backup_dir, 'node', node, backup_id=page_id_b3) + self.pb.restore_node('node', node, backup_id=page_id_b3) pgdata_restored_b3 = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata_b3, pgdata_restored_b3) - # @unittest.skip("skip") def test_basic_window_merge_multiple_descendants(self): - """ + r""" PAGEb3 | PAGEa3 -----------------------------retention window @@ -726,32 +751,29 @@ def test_basic_window_merge_multiple_descendants(self): FULLb | FULLa """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=3) # Take FULL BACKUPs - backup_id_a = self.backup_node(backup_dir, 'node', node) + backup_id_a = self.pb.backup_node('node', node) # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) # pgbench.wait() - backup_id_b = self.backup_node(backup_dir, 'node', node) + backup_id_b = self.pb.backup_node('node', node) # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) # pgbench.wait() # Change FULLb backup status to ERROR self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a1 = self.pb.backup_node('node', node, backup_type='page') # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) # pgbench.wait() @@ -766,8 +788,7 @@ def test_basic_window_merge_multiple_descendants(self): # FULLb OK # FULLa OK - page_id_b1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b1 = self.pb.backup_node('node', node, backup_type='page') # PAGEb1 OK # PAGEa1 ERROR @@ -789,8 +810,7 @@ def test_basic_window_merge_multiple_descendants(self): # FULLb ERROR # FULLa OK - page_id_a2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a2 = self.pb.backup_node('node', node, backup_type='page') # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) # pgbench.wait() @@ -815,8 +835,7 @@ def test_basic_window_merge_multiple_descendants(self): # FULLb OK # FULLa ERROR - page_id_b2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b2 = self.pb.backup_node('node', node, backup_type='page') # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) # pgbench.wait() @@ -843,8 +862,7 @@ def test_basic_window_merge_multiple_descendants(self): # FULLb ERROR # FULLa OK - page_id_a3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a3 = self.pb.backup_node('node', node, backup_type='page') # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) # pgbench.wait() @@ -864,8 +882,7 @@ def test_basic_window_merge_multiple_descendants(self): self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - page_id_b3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b3 = self.pb.backup_node('node', node, backup_type='page') # PAGEb3 OK # PAGEa3 ERROR @@ -892,34 +909,28 @@ def test_basic_window_merge_multiple_descendants(self): # Check that page_id_a3 and page_id_a2 are both direct descendants of page_id_a1 self.assertEqual( - self.show_pb( - backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'], + self.pb.show('node', backup_id=page_id_a3)['parent-backup-id'], page_id_a1) self.assertEqual( - self.show_pb( - backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'], + self.pb.show('node', backup_id=page_id_a2)['parent-backup-id'], page_id_a1) # Purge backups - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup in [page_id_a3, page_id_b3, 'pg_probackup.conf']: + for backup in backup_dir.list_instance_backups('node'): + if backup in [page_id_a3, page_id_b3]: continue + with self.modify_backup_control(backup_dir, 'node', backup) as cf: + cf.data += "\nrecovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3)) - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) - - output = self.delete_expired( - backup_dir, 'node', + output = self.pb.delete_expired( + 'node', options=[ '--retention-window=1', '--delete-expired', '--merge-expired', '--log-level-console=log']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + self.assertEqual(len(self.pb.show('node')), 2) # Merging chain A self.assertIn( @@ -954,24 +965,24 @@ def test_basic_window_merge_multiple_descendants(self): "Delete: {0}".format(page_id_a2), output) self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['id'], + self.pb.show('node')[1]['id'], page_id_b3) self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['id'], + self.pb.show('node')[0]['id'], page_id_a3) self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['backup-mode'], + self.pb.show('node')[1]['backup-mode'], 'FULL') self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['backup-mode'], + self.pb.show('node')[0]['backup-mode'], 'FULL') # @unittest.skip("skip") def test_basic_window_merge_multiple_descendants_1(self): - """ + r""" PAGEb3 | PAGEa3 -----------------------------retention window @@ -982,32 +993,29 @@ def test_basic_window_merge_multiple_descendants_1(self): FULLb | FULLa """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=3) # Take FULL BACKUPs - backup_id_a = self.backup_node(backup_dir, 'node', node) + backup_id_a = self.pb.backup_node('node', node) # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) # pgbench.wait() - backup_id_b = self.backup_node(backup_dir, 'node', node) + backup_id_b = self.pb.backup_node('node', node) # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) # pgbench.wait() # Change FULLb backup status to ERROR self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') - page_id_a1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a1 = self.pb.backup_node('node', node, backup_type='page') # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) # pgbench.wait() @@ -1022,8 +1030,7 @@ def test_basic_window_merge_multiple_descendants_1(self): # FULLb OK # FULLa OK - page_id_b1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b1 = self.pb.backup_node('node', node, backup_type='page') # PAGEb1 OK # PAGEa1 ERROR @@ -1045,8 +1052,7 @@ def test_basic_window_merge_multiple_descendants_1(self): # FULLb ERROR # FULLa OK - page_id_a2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a2 = self.pb.backup_node('node', node, backup_type='page') # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) # pgbench.wait() @@ -1071,8 +1077,7 @@ def test_basic_window_merge_multiple_descendants_1(self): # FULLb OK # FULLa ERROR - page_id_b2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b2 = self.pb.backup_node('node', node, backup_type='page') # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) # pgbench.wait() @@ -1099,8 +1104,7 @@ def test_basic_window_merge_multiple_descendants_1(self): # FULLb ERROR # FULLa OK - page_id_a3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_a3 = self.pb.backup_node('node', node, backup_type='page') # pgbench = node.pgbench(options=['-T', '10', '-c', '2']) # pgbench.wait() @@ -1120,8 +1124,7 @@ def test_basic_window_merge_multiple_descendants_1(self): self.change_backup_status(backup_dir, 'node', page_id_b1, 'OK') self.change_backup_status(backup_dir, 'node', backup_id_b, 'OK') - page_id_b3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + page_id_b3 = self.pb.backup_node('node', node, backup_type='page') # PAGEb3 OK # PAGEa3 ERROR @@ -1148,34 +1151,28 @@ def test_basic_window_merge_multiple_descendants_1(self): # Check that page_id_a3 and page_id_a2 are both direct descendants of page_id_a1 self.assertEqual( - self.show_pb( - backup_dir, 'node', backup_id=page_id_a3)['parent-backup-id'], + self.pb.show('node', backup_id=page_id_a3)['parent-backup-id'], page_id_a1) self.assertEqual( - self.show_pb( - backup_dir, 'node', backup_id=page_id_a2)['parent-backup-id'], + self.pb.show('node', backup_id=page_id_a2)['parent-backup-id'], page_id_a1) # Purge backups - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup in [page_id_a3, page_id_b3, 'pg_probackup.conf']: + for backup in backup_dir.list_instance_backups('node'): + if backup in [page_id_a3, page_id_b3]: continue + with self.modify_backup_control(backup_dir, 'node', backup) as cf: + cf.data += "\nrecovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3)) - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) - - output = self.delete_expired( - backup_dir, 'node', + output = self.pb.delete_expired( + 'node', options=[ '--retention-window=1', '--merge-expired', '--log-level-console=log']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) + self.assertEqual(len(self.pb.show('node')), 3) # Merging chain A self.assertIn( @@ -1202,36 +1199,35 @@ def test_basic_window_merge_multiple_descendants_1(self): backup_id_b, page_id_b3), output) self.assertEqual( - self.show_pb(backup_dir, 'node')[2]['id'], + self.pb.show('node')[2]['id'], page_id_b3) self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['id'], + self.pb.show('node')[1]['id'], page_id_a3) self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['id'], + self.pb.show('node')[0]['id'], page_id_a2) self.assertEqual( - self.show_pb(backup_dir, 'node')[2]['backup-mode'], + self.pb.show('node')[2]['backup-mode'], 'FULL') self.assertEqual( - self.show_pb(backup_dir, 'node')[1]['backup-mode'], + self.pb.show('node')[1]['backup-mode'], 'FULL') self.assertEqual( - self.show_pb(backup_dir, 'node')[0]['backup-mode'], + self.pb.show('node')[0]['backup-mode'], 'PAGE') - output = self.delete_expired( - backup_dir, 'node', + output = self.pb.delete_expired( + 'node', options=[ '--retention-window=1', '--delete-expired', '--log-level-console=log']) - # @unittest.skip("skip") def test_window_chains(self): """ PAGE @@ -1243,77 +1239,65 @@ def test_window_chains(self): PAGE FULL """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=3) # Chain A - self.backup_node(backup_dir, 'node', node) - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type='page') - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') # Chain B - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) pgbench = node.pgbench(options=['-T', '10', '-c', '2']) pgbench.wait() - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') pgbench = node.pgbench(options=['-T', '10', '-c', '2']) pgbench.wait() - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') pgbench = node.pgbench(options=['-T', '10', '-c', '2']) pgbench.wait() - page_id_b3 = self.backup_node( - backup_dir, 'node', node, backup_type='delta') + page_id_b3 = self.pb.backup_node('node', node, backup_type='delta') pgdata = self.pgdata_content(node.data_dir) # Purge backups - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup in [page_id_b3, 'pg_probackup.conf']: + for backup in backup_dir.list_instance_backups('node'): + if backup in [page_id_b3]: continue + with self.modify_backup_control(backup_dir, 'node', backup) as cf: + cf.data += "\nrecovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3)) - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) - - self.delete_expired( - backup_dir, 'node', + self.pb.delete_expired( + 'node', options=[ '--retention-window=1', '--expired', '--merge-expired', '--log-level-console=log']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 1) + self.assertEqual(len(self.pb.show('node')), 1) node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) - # @unittest.skip("skip") def test_window_chains_1(self): """ PAGE @@ -1325,59 +1309,48 @@ def test_window_chains_1(self): PAGE FULL """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=3) # Chain A - self.backup_node(backup_dir, 'node', node) - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type='page') - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') # Chain B - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') - page_id_b3 = self.backup_node( - backup_dir, 'node', node, backup_type='delta') + page_id_b3 = self.pb.backup_node('node', node, backup_type='delta') self.pgdata_content(node.data_dir) # Purge backups - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup in [page_id_b3, 'pg_probackup.conf']: + for backup in backup_dir.list_instance_backups('node'): + if backup in [page_id_b3]: continue + with self.modify_backup_control(backup_dir, 'node', backup) as cf: + cf.data += "\nrecovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3)) - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) - - output = self.delete_expired( - backup_dir, 'node', + output = self.pb.delete_expired( + 'node', options=[ '--retention-window=1', '--merge-expired', '--log-level-console=log']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + self.assertEqual(len(self.pb.show('node')), 4) self.assertIn( "There are no backups to delete by retention policy", @@ -1387,13 +1360,13 @@ def test_window_chains_1(self): "Retention merging finished", output) - output = self.delete_expired( - backup_dir, 'node', + output = self.pb.delete_expired( + 'node', options=[ '--retention-window=1', '--expired', '--log-level-console=log']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 1) + self.assertEqual(len(self.pb.show('node')), 1) self.assertIn( "There are no backups to merge by retention policy", @@ -1415,28 +1388,25 @@ def test_window_error_backups(self): FULL -------redundancy """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Take FULL BACKUPs - self.backup_node(backup_dir, 'node', node) - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type='page') - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') # Change FULLb backup status to ERROR # self.change_backup_status(backup_dir, 'node', backup_id_b, 'ERROR') # @unittest.skip("skip") + @needs_gdb def test_window_error_backups_1(self): """ DELTA @@ -1444,45 +1414,40 @@ def test_window_error_backups_1(self): FULL -------window """ - self._check_gdb_flag_or_skip_test() - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Take FULL BACKUP - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # Take PAGE BACKUP - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='page', gdb=True) + gdb = self.pb.backup_node('node', node, backup_type='page', gdb=True) # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() - gdb.remove_all_breakpoints() - gdb._execute('signal SIGINT') + gdb.signal('SIGINT') gdb.continue_execution_until_error() - self.show_pb(backup_dir, 'node')[1]['id'] + self.pb.show('node')[1]['id'] # Take DELTA backup - self.backup_node( - backup_dir, 'node', node, backup_type='delta', + self.pb.backup_node('node', node, backup_type='delta', options=['--retention-window=2', '--delete-expired']) # Take FULL BACKUP - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + self.assertEqual(len(self.pb.show('node')), 4) # @unittest.skip("skip") + @needs_gdb def test_window_error_backups_2(self): """ DELTA @@ -1490,281 +1455,220 @@ def test_window_error_backups_2(self): FULL -------window """ - self._check_gdb_flag_or_skip_test() - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Take FULL BACKUP - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # Take PAGE BACKUP - gdb = self.backup_node( - backup_dir, 'node', node, backup_type='page', gdb=True) + gdb = self.pb.backup_node('node', node, backup_type='page', gdb=True) # Attention! this breakpoint has been set on internal probackup function, not on a postgres core one gdb.set_breakpoint('pg_stop_backup') gdb.run_until_break() - gdb._execute('signal SIGKILL') + gdb.signal('SIGKILL') gdb.continue_execution_until_error() - self.show_pb(backup_dir, 'node')[1]['id'] - - if self.get_version(node) < 90600: - node.safe_psql( - 'postgres', - 'SELECT pg_catalog.pg_stop_backup()') + self.pb.show('node')[1]['id'] # Take DELTA backup - self.backup_node( - backup_dir, 'node', node, backup_type='delta', + self.pb.backup_node('node', node, backup_type='delta', options=['--retention-window=2', '--delete-expired']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 3) + self.assertEqual(len(self.pb.show('node')), 3) + @needs_gdb def test_retention_redundancy_overlapping_chains(self): """""" - self._check_gdb_flag_or_skip_test() - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - if self.get_version(node) < 90600: - self.skipTest('Skipped because ptrack support is disabled') - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.set_config( - backup_dir, 'node', options=['--retention-redundancy=1']) + self.pb.set_config('node', options=['--retention-redundancy=1']) # Make backups to be purged - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type="page") + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type="page") # Make backups to be keeped - gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb = self.pb.backup_node('node', node, gdb=True) gdb.set_breakpoint('backup_files') gdb.run_until_break() sleep(1) - self.backup_node(backup_dir, 'node', node, backup_type="page") + self.pb.backup_node('node', node, backup_type="page") - gdb.remove_all_breakpoints() gdb.continue_execution_until_exit() - self.backup_node(backup_dir, 'node', node, backup_type="page") + self.pb.backup_node('node', node, backup_type="page") # Purge backups - self.delete_expired( - backup_dir, 'node', options=['--expired', '--wal']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + self.pb.delete_expired( + 'node', options=['--expired', '--wal']) + self.assertEqual(len(self.pb.show('node')), 2) - self.validate_pb(backup_dir, 'node') + self.pb.validate('node') + @needs_gdb def test_retention_redundancy_overlapping_chains_1(self): """""" - self._check_gdb_flag_or_skip_test() - - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - if self.get_version(node) < 90600: - self.skipTest('Skipped because ptrack support is disabled') + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.set_config( - backup_dir, 'node', options=['--retention-redundancy=1']) + self.pb.set_config('node', options=['--retention-redundancy=1']) # Make backups to be purged - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type="page") + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type="page") # Make backups to be keeped - gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb = self.pb.backup_node('node', node, gdb=True) gdb.set_breakpoint('backup_files') gdb.run_until_break() sleep(1) - self.backup_node(backup_dir, 'node', node, backup_type="page") + self.pb.backup_node('node', node, backup_type="page") - gdb.remove_all_breakpoints() gdb.continue_execution_until_exit() - self.backup_node(backup_dir, 'node', node, backup_type="page") + self.pb.backup_node('node', node, backup_type="page") # Purge backups - self.delete_expired( - backup_dir, 'node', options=['--expired', '--wal']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + self.pb.delete_expired( + 'node', options=['--expired', '--wal']) + self.assertEqual(len(self.pb.show('node')), 2) - self.validate_pb(backup_dir, 'node') + self.pb.validate('node') def test_wal_purge_victim(self): """ https://github.com/postgrespro/pg_probackup/issues/103 """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # Make ERROR incremental backup - try: - self.backup_node(backup_dir, 'node', node, backup_type='page') - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "without valid full backup.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "WARNING: Valid full backup on current timeline 1 is not found" in e.message and - "ERROR: Create new full backup before an incremental one" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - page_id = self.show_pb(backup_dir, 'node')[0]['id'] + self.pb.backup_node('node', node, backup_type='page', + expect_error="because page backup should not be " + "possible without valid full backup") + self.assertMessage(contains="WARNING: Valid full backup on current timeline 1 is not found") + self.assertMessage(contains="ERROR: Create new full backup before an incremental one") + + page_id = self.pb.show('node')[0]['id'] sleep(1) # Make FULL backup - full_id = self.backup_node(backup_dir, 'node', node, options=['--delete-wal']) - - try: - self.validate_pb(backup_dir, 'node') - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "without valid full backup.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "INFO: Backup {0} WAL segments are valid".format(full_id), - e.message) - self.assertIn( - "WARNING: Backup {0} has missing parent 0".format(page_id), - e.message) + full_id = self.pb.backup_node('node', node, options=['--delete-wal']) + + self.pb.validate('node', + expect_error="because page backup should not be " + "possible without valid full backup") + self.assertMessage(contains=f"INFO: Backup {full_id} WAL segments are valid") + self.assertMessage(contains=f"WARNING: Backup {page_id} has missing parent 0") # @unittest.skip("skip") + @needs_gdb def test_failed_merge_redundancy_retention(self): """ Check that retention purge works correctly with MERGING backups """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join( - self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL1 backup - full_id = self.backup_node(backup_dir, 'node', node) + full_id = self.pb.backup_node('node', node) # DELTA BACKUP - delta_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') + delta_id = self.pb.backup_node('node', node, backup_type='delta') # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') # FULL2 backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') # FULL3 backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') # DELTA BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') - self.set_config( - backup_dir, 'node', options=['--retention-redundancy=2']) + self.pb.set_config('node', options=['--retention-redundancy=2']) - self.set_config( - backup_dir, 'node', options=['--retention-window=2']) + self.pb.set_config('node', options=['--retention-window=2']) # create pair of MERGING backup as a result of failed merge - gdb = self.merge_backup( - backup_dir, 'node', delta_id, gdb=True) + gdb = self.pb.merge_backup('node', delta_id, gdb=True) gdb.set_breakpoint('backup_non_data_file') gdb.run_until_break() gdb.continue_execution_until_break(2) - gdb._execute('signal SIGKILL') + gdb.signal('SIGKILL') # "expire" first full backup - backups = os.path.join(backup_dir, 'backups', 'node') - with open( - os.path.join( - backups, full_id, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) + with self.modify_backup_control(backup_dir, 'node', full_id) as cf: + cf.data += "\nrecovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3)) # run retention merge - self.delete_expired( - backup_dir, 'node', options=['--delete-expired']) + self.pb.delete_expired( + 'node', options=['--delete-expired']) self.assertEqual( 'MERGING', - self.show_pb(backup_dir, 'node', full_id)['status'], + self.pb.show('node', full_id)['status'], 'Backup STATUS should be "MERGING"') self.assertEqual( 'MERGING', - self.show_pb(backup_dir, 'node', delta_id)['status'], + self.pb.show('node', delta_id)['status'], 'Backup STATUS should be "MERGING"') - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 10) + self.assertEqual(len(self.pb.show('node')), 10) def test_wal_depth_1(self): """ @@ -1774,31 +1678,28 @@ def test_wal_depth_1(self): wal-depth=2 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={ 'archive_timeout': '30s', 'checkpoint_timeout': '30s'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) - self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) + self.pb.set_config('node', options=['--archive-timeout=60s']) node.slow_start() # FULL node.pgbench_init(scale=1) - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # PAGE node.pgbench_init(scale=1) - B2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + B2 = self.pb.backup_node('node', node, backup_type='page') # generate_some more data node.pgbench_init(scale=1) @@ -1809,22 +1710,18 @@ def test_wal_depth_1(self): node.pgbench_init(scale=1) - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') node.pgbench_init(scale=1) - self.backup_node( - backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') # Timeline 2 - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - output = self.restore_node( - backup_dir, 'node', node_restored, + output = self.pb.restore_node('node', node_restored, options=[ '--recovery-target-xid={0}'.format(target_xid), '--recovery-target-action=promote']) @@ -1833,7 +1730,7 @@ def test_wal_depth_1(self): 'Restore of backup {0} completed'.format(B2), output) - self.set_auto_conf(node_restored, options={'port': node_restored.port}) + node_restored.set_auto_conf(options={'port': node_restored.port}) node_restored.slow_start() @@ -1848,8 +1745,7 @@ def test_wal_depth_1(self): # Timeline 3 node_restored.cleanup() - output = self.restore_node( - backup_dir, 'node', node_restored, + output = self.pb.restore_node('node', node_restored, options=[ '--recovery-target-xid={0}'.format(target_xid), '--recovery-target-timeline=2', @@ -1859,24 +1755,23 @@ def test_wal_depth_1(self): 'Restore of backup {0} completed'.format(B2), output) - self.set_auto_conf(node_restored, options={'port': node_restored.port}) + node_restored.set_auto_conf(options={'port': node_restored.port}) node_restored.slow_start() node_restored.pgbench_init(scale=1) - self.backup_node( - backup_dir, 'node', node_restored, data_dir=node_restored.data_dir) + self.pb.backup_node('node', node_restored, data_dir=node_restored.data_dir) node.pgbench_init(scale=1) - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) - lsn = self.show_archive(backup_dir, 'node', tli=2)['switchpoint'] + lsn = self.pb.show_archive('node', tli=2)['switchpoint'] - self.validate_pb( - backup_dir, 'node', backup_id=B2, + self.pb.validate( + 'node', backup_id=B2, options=['--recovery-target-lsn={0}'.format(lsn)]) - self.validate_pb(backup_dir, 'node') + self.pb.validate('node') def test_wal_purge(self): """ @@ -1898,28 +1793,25 @@ def test_wal_purge(self): wal-depth=2 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_config('node', options=['--archive-timeout=60s']) node.slow_start() # STREAM FULL - stream_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) + stream_id = self.pb.backup_node('node', node, options=['--stream']) node.stop() - self.set_archiving(backup_dir, 'node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL - B1 = self.backup_node(backup_dir, 'node', node) + B1 = self.pb.backup_node('node', node) node.pgbench_init(scale=1) target_xid = node.safe_psql( @@ -1928,20 +1820,18 @@ def test_wal_purge(self): node.pgbench_init(scale=5) # B2 FULL on TLI1 - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.pgbench_init(scale=4) - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.pgbench_init(scale=4) - self.delete_pb(backup_dir, 'node', options=['--delete-wal']) + self.pb.delete('node', options=['--delete-wal']) # TLI 2 - node_tli2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_tli2')) + node_tli2 = self.pg_node.make_simple('node_tli2') node_tli2.cleanup() - output = self.restore_node( - backup_dir, 'node', node_tli2, + output = self.pb.restore_node('node', node_tli2, options=[ '--recovery-target-xid={0}'.format(target_xid), '--recovery-target-timeline=1', @@ -1951,7 +1841,7 @@ def test_wal_purge(self): 'INFO: Restore of backup {0} completed'.format(B1), output) - self.set_auto_conf(node_tli2, options={'port': node_tli2.port}) + node_tli2.set_auto_conf(options={'port': node_tli2.port}) node_tli2.slow_start() node_tli2.pgbench_init(scale=4) @@ -1960,23 +1850,19 @@ def test_wal_purge(self): "select txid_current()").decode('utf-8').rstrip() node_tli2.pgbench_init(scale=1) - self.backup_node( - backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) + self.pb.backup_node('node', node_tli2, data_dir=node_tli2.data_dir) node_tli2.pgbench_init(scale=3) - self.backup_node( - backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) + self.pb.backup_node('node', node_tli2, data_dir=node_tli2.data_dir) node_tli2.pgbench_init(scale=1) node_tli2.cleanup() # TLI3 - node_tli3 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_tli3')) + node_tli3 = self.pg_node.make_simple('node_tli3') node_tli3.cleanup() # Note, that successful validation here is a happy coincidence - output = self.restore_node( - backup_dir, 'node', node_tli3, + output = self.pb.restore_node('node', node_tli3, options=[ '--recovery-target-xid={0}'.format(target_xid), '--recovery-target-timeline=2', @@ -1985,56 +1871,57 @@ def test_wal_purge(self): self.assertIn( 'INFO: Restore of backup {0} completed'.format(B1), output) - self.set_auto_conf(node_tli3, options={'port': node_tli3.port}) + node_tli3.set_auto_conf(options={'port': node_tli3.port}) node_tli3.slow_start() node_tli3.pgbench_init(scale=5) node_tli3.cleanup() # TLI4 - node_tli4 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_tli4')) + node_tli4 = self.pg_node.make_simple('node_tli4') node_tli4.cleanup() - self.restore_node( - backup_dir, 'node', node_tli4, backup_id=stream_id, + self.pb.restore_node('node', node_tli4, backup_id=stream_id, options=[ '--recovery-target=immediate', '--recovery-target-action=promote']) - self.set_auto_conf(node_tli4, options={'port': node_tli4.port}) - self.set_archiving(backup_dir, 'node', node_tli4) + node_tli4.set_auto_conf(options={'port': node_tli4.port}) + self.pb.set_archiving('node', node_tli4) node_tli4.slow_start() node_tli4.pgbench_init(scale=5) - self.backup_node( - backup_dir, 'node', node_tli4, data_dir=node_tli4.data_dir) + self.pb.backup_node('node', node_tli4, data_dir=node_tli4.data_dir) node_tli4.pgbench_init(scale=5) node_tli4.cleanup() # TLI5 - node_tli5 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_tli5')) + node_tli5 = self.pg_node.make_simple('node_tli5') node_tli5.cleanup() - self.restore_node( - backup_dir, 'node', node_tli5, backup_id=stream_id, + self.pb.restore_node('node', node_tli5, backup_id=stream_id, options=[ '--recovery-target=immediate', '--recovery-target-action=promote']) - self.set_auto_conf(node_tli5, options={'port': node_tli5.port}) - self.set_archiving(backup_dir, 'node', node_tli5) + node_tli5.set_auto_conf(options={'port': node_tli5.port}) + self.pb.set_archiving('node', node_tli5) node_tli5.slow_start() node_tli5.pgbench_init(scale=10) # delete '.history' file of TLI4 - os.remove(os.path.join(backup_dir, 'wal', 'node', '00000004.history')) + self.remove_instance_wal(backup_dir, 'node', '00000004.history') # delete '.history' file of TLI5 - os.remove(os.path.join(backup_dir, 'wal', 'node', '00000005.history')) + self.wait_instance_wal_exists(backup_dir, 'node', '00000005.history') + self.remove_instance_wal(backup_dir, 'node', '00000005.history') + + tailer = tail_file(os.path.join(node_tli5.logs_dir, 'postgresql.log')) + tailer.wait(contains='LOG: pushing file "000000050000000000000007') + tailer.wait_archive_push_completed() + del tailer + node_tli5.stop() - output = self.delete_pb( - backup_dir, 'node', + output = self.pb.delete('node', options=[ '--delete-wal', '--dry-run', '--log-level-console=verbose']) @@ -2048,11 +1935,11 @@ def test_wal_purge(self): 'INFO: On timeline 5 all files can be removed', output) - show_tli1_before = self.show_archive(backup_dir, 'node', tli=1) - show_tli2_before = self.show_archive(backup_dir, 'node', tli=2) - show_tli3_before = self.show_archive(backup_dir, 'node', tli=3) - show_tli4_before = self.show_archive(backup_dir, 'node', tli=4) - show_tli5_before = self.show_archive(backup_dir, 'node', tli=5) + show_tli1_before = self.pb.show_archive('node', tli=1) + show_tli2_before = self.pb.show_archive('node', tli=2) + show_tli3_before = self.pb.show_archive('node', tli=3) + show_tli4_before = self.pb.show_archive('node', tli=4) + show_tli5_before = self.pb.show_archive('node', tli=5) self.assertTrue(show_tli1_before) self.assertTrue(show_tli2_before) @@ -2060,8 +1947,7 @@ def test_wal_purge(self): self.assertTrue(show_tli4_before) self.assertTrue(show_tli5_before) - output = self.delete_pb( - backup_dir, 'node', + output = self.pb.delete('node', options=['--delete-wal', '--log-level-console=verbose']) self.assertIn( @@ -2073,11 +1959,11 @@ def test_wal_purge(self): 'INFO: On timeline 5 all files will be removed', output) - show_tli1_after = self.show_archive(backup_dir, 'node', tli=1) - show_tli2_after = self.show_archive(backup_dir, 'node', tli=2) - show_tli3_after = self.show_archive(backup_dir, 'node', tli=3) - show_tli4_after = self.show_archive(backup_dir, 'node', tli=4) - show_tli5_after = self.show_archive(backup_dir, 'node', tli=5) + show_tli1_after = self.pb.show_archive('node', tli=1) + show_tli2_after = self.pb.show_archive('node', tli=2) + show_tli3_after = self.pb.show_archive('node', tli=3) + show_tli4_after = self.pb.show_archive('node', tli=4) + show_tli5_after = self.pb.show_archive('node', tli=5) self.assertEqual(show_tli1_before, show_tli1_after) self.assertEqual(show_tli2_before, show_tli2_after) @@ -2095,7 +1981,7 @@ def test_wal_purge(self): self.assertFalse(show_tli5_after) - self.validate_pb(backup_dir, 'node') + self.pb.validate('node') def test_wal_depth_2(self): """ @@ -2118,28 +2004,25 @@ def test_wal_depth_2(self): wal-depth=2 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_config('node', options=['--archive-timeout=60s']) node.slow_start() # STREAM FULL - stream_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) + stream_id = self.pb.backup_node('node', node, options=['--stream']) node.stop() - self.set_archiving(backup_dir, 'node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL - B1 = self.backup_node(backup_dir, 'node', node) + B1 = self.pb.backup_node('node', node) node.pgbench_init(scale=1) target_xid = node.safe_psql( @@ -2148,18 +2031,16 @@ def test_wal_depth_2(self): node.pgbench_init(scale=5) # B2 FULL on TLI1 - B2 = self.backup_node(backup_dir, 'node', node) + B2 = self.pb.backup_node('node', node) node.pgbench_init(scale=4) - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.pgbench_init(scale=4) # TLI 2 - node_tli2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_tli2')) + node_tli2 = self.pg_node.make_simple('node_tli2') node_tli2.cleanup() - output = self.restore_node( - backup_dir, 'node', node_tli2, + output = self.pb.restore_node('node', node_tli2, options=[ '--recovery-target-xid={0}'.format(target_xid), '--recovery-target-timeline=1', @@ -2169,7 +2050,7 @@ def test_wal_depth_2(self): 'INFO: Restore of backup {0} completed'.format(B1), output) - self.set_auto_conf(node_tli2, options={'port': node_tli2.port}) + node_tli2.set_auto_conf(options={'port': node_tli2.port}) node_tli2.slow_start() node_tli2.pgbench_init(scale=4) @@ -2178,23 +2059,19 @@ def test_wal_depth_2(self): "select txid_current()").decode('utf-8').rstrip() node_tli2.pgbench_init(scale=1) - B4 = self.backup_node( - backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) + B4 = self.pb.backup_node('node', node_tli2, data_dir=node_tli2.data_dir) node_tli2.pgbench_init(scale=3) - self.backup_node( - backup_dir, 'node', node_tli2, data_dir=node_tli2.data_dir) + self.pb.backup_node('node', node_tli2, data_dir=node_tli2.data_dir) node_tli2.pgbench_init(scale=1) node_tli2.cleanup() # TLI3 - node_tli3 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_tli3')) + node_tli3 = self.pg_node.make_simple('node_tli3') node_tli3.cleanup() # Note, that successful validation here is a happy coincidence - output = self.restore_node( - backup_dir, 'node', node_tli3, + output = self.pb.restore_node('node', node_tli3, options=[ '--recovery-target-xid={0}'.format(target_xid), '--recovery-target-timeline=2', @@ -2203,61 +2080,56 @@ def test_wal_depth_2(self): self.assertIn( 'INFO: Restore of backup {0} completed'.format(B1), output) - self.set_auto_conf(node_tli3, options={'port': node_tli3.port}) + node_tli3.set_auto_conf(options={'port': node_tli3.port}) node_tli3.slow_start() node_tli3.pgbench_init(scale=5) node_tli3.cleanup() # TLI4 - node_tli4 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_tli4')) + node_tli4 = self.pg_node.make_simple('node_tli4') node_tli4.cleanup() - self.restore_node( - backup_dir, 'node', node_tli4, backup_id=stream_id, + self.pb.restore_node('node', node_tli4, backup_id=stream_id, options=[ '--recovery-target=immediate', '--recovery-target-action=promote']) - self.set_auto_conf(node_tli4, options={'port': node_tli4.port}) - self.set_archiving(backup_dir, 'node', node_tli4) + node_tli4.set_auto_conf(options={'port': node_tli4.port}) + self.pb.set_archiving('node', node_tli4) node_tli4.slow_start() node_tli4.pgbench_init(scale=5) - self.backup_node( - backup_dir, 'node', node_tli4, data_dir=node_tli4.data_dir) + self.pb.backup_node('node', node_tli4, data_dir=node_tli4.data_dir) node_tli4.pgbench_init(scale=5) node_tli4.cleanup() # TLI5 - node_tli5 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_tli5')) + node_tli5 = self.pg_node.make_simple('node_tli5') node_tli5.cleanup() - self.restore_node( - backup_dir, 'node', node_tli5, backup_id=stream_id, + self.pb.restore_node('node', node_tli5, backup_id=stream_id, options=[ '--recovery-target=immediate', '--recovery-target-action=promote']) - self.set_auto_conf(node_tli5, options={'port': node_tli5.port}) - self.set_archiving(backup_dir, 'node', node_tli5) + node_tli5.set_auto_conf(options={'port': node_tli5.port}) + self.pb.set_archiving('node', node_tli5) node_tli5.slow_start() node_tli5.pgbench_init(scale=10) # delete '.history' file of TLI4 - os.remove(os.path.join(backup_dir, 'wal', 'node', '00000004.history')) + self.remove_instance_wal(backup_dir, 'node', '00000004.history') # delete '.history' file of TLI5 - os.remove(os.path.join(backup_dir, 'wal', 'node', '00000005.history')) + self.wait_instance_wal_exists(backup_dir, 'node', '00000005.history') + self.remove_instance_wal(backup_dir, 'node', '00000005.history') - output = self.delete_pb( - backup_dir, 'node', + output = self.pb.delete('node', options=[ '--delete-wal', '--dry-run', '--wal-depth=2', '--log-level-console=verbose']) - start_lsn_B2 = self.show_pb(backup_dir, 'node', B2)['start-lsn'] + start_lsn_B2 = self.pb.show('node', B2)['start-lsn'] self.assertIn( 'On timeline 1 WAL is protected from purge at {0}'.format(start_lsn_B2), output) @@ -2267,7 +2139,7 @@ def test_wal_depth_2(self): 'purge WAL interval between 000000010000000000000004 ' 'and 000000010000000000000005 on timeline 1'.format(B1), output) - start_lsn_B4 = self.show_pb(backup_dir, 'node', B4)['start-lsn'] + start_lsn_B4 = self.pb.show('node', B4)['start-lsn'] self.assertIn( 'On timeline 2 WAL is protected from purge at {0}'.format(start_lsn_B4), output) @@ -2282,11 +2154,11 @@ def test_wal_depth_2(self): 'from purge WAL interval between 000000010000000000000004 and ' '000000010000000000000006 on timeline 1', output) - show_tli1_before = self.show_archive(backup_dir, 'node', tli=1) - show_tli2_before = self.show_archive(backup_dir, 'node', tli=2) - show_tli3_before = self.show_archive(backup_dir, 'node', tli=3) - show_tli4_before = self.show_archive(backup_dir, 'node', tli=4) - show_tli5_before = self.show_archive(backup_dir, 'node', tli=5) + show_tli1_before = self.pb.show_archive('node', tli=1) + show_tli2_before = self.pb.show_archive('node', tli=2) + show_tli3_before = self.pb.show_archive('node', tli=3) + show_tli4_before = self.pb.show_archive('node', tli=4) + show_tli5_before = self.pb.show_archive('node', tli=5) self.assertTrue(show_tli1_before) self.assertTrue(show_tli2_before) @@ -2296,17 +2168,16 @@ def test_wal_depth_2(self): sleep(5) - output = self.delete_pb( - backup_dir, 'node', + output = self.pb.delete('node', options=['--delete-wal', '--wal-depth=2', '--log-level-console=verbose']) # print(output) - show_tli1_after = self.show_archive(backup_dir, 'node', tli=1) - show_tli2_after = self.show_archive(backup_dir, 'node', tli=2) - show_tli3_after = self.show_archive(backup_dir, 'node', tli=3) - show_tli4_after = self.show_archive(backup_dir, 'node', tli=4) - show_tli5_after = self.show_archive(backup_dir, 'node', tli=5) + show_tli1_after = self.pb.show_archive('node', tli=1) + show_tli2_after = self.pb.show_archive('node', tli=2) + show_tli3_after = self.pb.show_archive('node', tli=3) + show_tli4_after = self.pb.show_archive('node', tli=4) + show_tli5_after = self.pb.show_archive('node', tli=5) self.assertNotEqual(show_tli1_before, show_tli1_after) self.assertNotEqual(show_tli2_before, show_tli2_after) @@ -2349,7 +2220,7 @@ def test_wal_depth_2(self): show_tli2_after['lost-segments'][0]['end-segno'], '00000002000000000000000A') - self.validate_pb(backup_dir, 'node') + self.pb.validate('node') def test_basic_wal_depth(self): """ @@ -2360,46 +2231,40 @@ def test_basic_wal_depth(self): wal-depth=1 """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_config(backup_dir, 'node', options=['--archive-timeout=60s']) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_config('node', options=['--archive-timeout=60s']) + self.pb.set_archiving('node', node) node.slow_start() # FULL node.pgbench_init(scale=1) - B1 = self.backup_node(backup_dir, 'node', node) + B1 = self.pb.backup_node('node', node) # B2 pgbench = node.pgbench(options=['-T', '10', '-c', '2']) pgbench.wait() - B2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + B2 = self.pb.backup_node('node', node, backup_type='page') # B3 pgbench = node.pgbench(options=['-T', '10', '-c', '2']) pgbench.wait() - B3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + B3 = self.pb.backup_node('node', node, backup_type='page') # B4 pgbench = node.pgbench(options=['-T', '10', '-c', '2']) pgbench.wait() - B4 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + B4 = self.pb.backup_node('node', node, backup_type='page') # B5 pgbench = node.pgbench(options=['-T', '10', '-c', '2']) pgbench.wait() - B5 = self.backup_node( - backup_dir, 'node', node, backup_type='page', + B5 = self.pb.backup_node('node', node, backup_type='page', options=['--wal-depth=1', '--delete-wal']) pgbench = node.pgbench(options=['-T', '10', '-c', '2']) @@ -2414,17 +2279,16 @@ def test_basic_wal_depth(self): pgbench = node.pgbench(options=['-T', '10', '-c', '2']) pgbench.wait() - tli1 = self.show_archive(backup_dir, 'node', tli=1) + tli1 = self.pb.show_archive('node', tli=1, + options=['--log-level-file=VERBOSE']) # check that there are 4 lost_segments intervals self.assertEqual(len(tli1['lost-segments']), 4) - output = self.validate_pb( - backup_dir, 'node', B5, + output = self.pb.validate( + 'node', B5, options=['--recovery-target-xid={0}'.format(target_xid)]) - print(output) - self.assertIn( 'INFO: Backup validation completed successfully on time', output) @@ -2434,96 +2298,85 @@ def test_basic_wal_depth(self): output) for backup_id in [B1, B2, B3, B4]: - try: - self.validate_pb( - backup_dir, 'node', backup_id, - options=['--recovery-target-xid={0}'.format(target_xid)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because page backup should not be possible " - "without valid full backup.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Not enough WAL records to xid {0}".format(target_xid), - e.message) - - self.validate_pb(backup_dir, 'node') + self.pb.validate('node', backup_id, + options=['--recovery-target-xid', target_xid], + expect_error="because page backup should not be " + "possible without valid full backup") + self.assertMessage(contains=f"ERROR: Not enough WAL records to xid {target_xid}") + + self.pb.validate('node') + @needs_gdb def test_concurrent_running_full_backup(self): """ https://github.com/postgrespro/pg_probackup/issues/328 """ - self._check_gdb_flag_or_skip_test() - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) - gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb = self.pb.backup_node('node', node, gdb=True) gdb.set_breakpoint('backup_data_file') gdb.run_until_break() gdb.kill() self.assertTrue( - self.show_pb(backup_dir, 'node')[0]['status'], + self.pb.show('node')[0]['status'], 'RUNNING') - self.backup_node( - backup_dir, 'node', node, backup_type='delta', + self.pb.backup_node('node', node, backup_type='delta', options=['--retention-redundancy=2', '--delete-expired']) self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['status'], + self.pb.show('node')[1]['status'], 'RUNNING') - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) - gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb = self.pb.backup_node('node', node, gdb=True) gdb.set_breakpoint('backup_data_file') gdb.run_until_break() gdb.kill() - gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb = self.pb.backup_node('node', node, gdb=True) gdb.set_breakpoint('backup_data_file') gdb.run_until_break() gdb.kill() - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) - gdb = self.backup_node(backup_dir, 'node', node, gdb=True) + gdb = self.pb.backup_node('node', node, gdb=True) gdb.set_breakpoint('backup_data_file') gdb.run_until_break() gdb.kill() - self.backup_node( - backup_dir, 'node', node, backup_type='delta', + self.expire_locks(backup_dir, 'node') + + self.pb.backup_node('node', node, backup_type='delta', options=['--retention-redundancy=2', '--delete-expired'], return_id=False) self.assertTrue( - self.show_pb(backup_dir, 'node')[0]['status'], + self.pb.show('node')[0]['status'], 'OK') self.assertTrue( - self.show_pb(backup_dir, 'node')[1]['status'], + self.pb.show('node')[1]['status'], 'RUNNING') self.assertTrue( - self.show_pb(backup_dir, 'node')[2]['status'], + self.pb.show('node')[2]['status'], 'OK') self.assertEqual( - len(self.show_pb(backup_dir, 'node')), + len(self.pb.show('node')), 6) diff --git a/tests/set_backup_test.py b/tests/set_backup_test.py index 31334cfba..0d149ce63 100644 --- a/tests/set_backup_test.py +++ b/tests/set_backup_test.py @@ -1,118 +1,75 @@ import unittest import subprocess import os -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException + +from .helpers.data_helpers import tail_file +from .helpers.ptrack_helpers import ProbackupTest from sys import exit from datetime import datetime, timedelta +from .helpers.enums.date_time_enum import DateTimePattern - -class SetBackupTest(ProbackupTest, unittest.TestCase): +class SetBackupTest(ProbackupTest): # @unittest.expectedFailure # @unittest.skip("skip") def test_set_backup_sanity(self): """general sanity for set-backup command""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) + backup_id = self.pb.backup_node('node', node, options=['--stream']) - recovery_time = self.show_pb( - backup_dir, 'node', backup_id=backup_id)['recovery-time'] + recovery_time = self.pb.show('node', backup_id=backup_id)['recovery-time'] + # Remove microseconds + recovery_time = datetime.strptime(recovery_time + '00', DateTimePattern.Y_m_d_H_M_S_f_z_dash.value) + recovery_time = recovery_time.strftime(DateTimePattern.Y_m_d_H_M_S_z_dash.value) expire_time_1 = "{:%Y-%m-%d %H:%M:%S}".format( datetime.now() + timedelta(days=5)) - try: - self.set_backup(backup_dir, False, options=['--ttl=30d']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of missing instance. " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Required parameter not specified: --instance', - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - try: - self.set_backup( - backup_dir, 'node', - options=[ - "--ttl=30d", - "--expire-time='{0}'".format(expire_time_1)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because options cannot be mixed. " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: You cannot specify '--expire-time' " - "and '--ttl' options together", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - try: - self.set_backup(backup_dir, 'node', options=["--ttl=30d"]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because of missing backup_id. " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: You must specify parameter (-i, --backup-id) " - "for 'set-backup' command", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) - - self.set_backup( - backup_dir, 'node', backup_id, options=["--ttl=30d"]) - - actual_expire_time = self.show_pb( - backup_dir, 'node', backup_id=backup_id)['expire-time'] + self.pb.set_backup(False, options=['--ttl=30d'], + expect_error="because of missing instance") + self.assertMessage(contains='ERROR: Required parameter not specified: --instance') + + self.pb.set_backup('node', + options=["--ttl=30d", f"--expire-time='{expire_time_1}'"], + expect_error="because options cannot be mixed") + self.assertMessage(contains="ERROR: You cannot specify '--expire-time' " + "and '--ttl' options together") + + self.pb.set_backup('node', options=["--ttl=30d"], + expect_error="because of missing backup_id") + self.assertMessage(contains="ERROR: You must specify parameter (-i, " + "--backup-id) for 'set-backup' command") + + self.pb.set_backup('node', backup_id, options=["--ttl=30d"]) + + actual_expire_time = self.pb.show('node', backup_id=backup_id)['expire-time'] self.assertNotEqual(expire_time_1, actual_expire_time) expire_time_2 = "{:%Y-%m-%d %H:%M:%S}".format( datetime.now() + timedelta(days=6)) - self.set_backup( - backup_dir, 'node', backup_id, + self.pb.set_backup('node', backup_id, options=["--expire-time={0}".format(expire_time_2)]) - actual_expire_time = self.show_pb( - backup_dir, 'node', backup_id=backup_id)['expire-time'] + actual_expire_time = self.pb.show('node', backup_id=backup_id)['expire-time'] self.assertIn(expire_time_2, actual_expire_time) # unpin backup - self.set_backup( - backup_dir, 'node', backup_id, options=["--ttl=0"]) + self.pb.set_backup('node', backup_id, options=["--ttl=0"]) - attr_list = self.show_pb( - backup_dir, 'node', backup_id=backup_id) + attr_list = self.pb.show('node', backup_id=backup_id) self.assertNotIn('expire-time', attr_list) - self.set_backup( - backup_dir, 'node', backup_id, options=["--expire-time={0}".format(recovery_time)]) + self.pb.set_backup('node', backup_id, options=["--expire-time={0}".format(recovery_time)]) # parse string to datetime object #new_expire_time = datetime.strptime(new_expire_time, '%Y-%m-%d %H:%M:%S%z') @@ -121,42 +78,31 @@ def test_set_backup_sanity(self): # @unittest.expectedFailure def test_retention_redundancy_pinning(self): """""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() + node = self.pg_node.make_simple('node') - with open(os.path.join( - backup_dir, 'backups', 'node', - "pg_probackup.conf"), "a") as conf: - conf.write("retention-redundancy = 1\n") + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() - self.set_config( - backup_dir, 'node', options=['--retention-redundancy=1']) + self.pb.set_config('node', options=['--retention-redundancy=1']) # Make backups to be purged - full_id = self.backup_node(backup_dir, 'node', node) - page_id = self.backup_node( - backup_dir, 'node', node, backup_type="page") + full_id = self.pb.backup_node('node', node) + page_id = self.pb.backup_node('node', node, backup_type="page") # Make backups to be keeped - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type="page") + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type="page") - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + self.assertEqual(len(self.pb.show('node')), 4) - self.set_backup( - backup_dir, 'node', page_id, options=['--ttl=5d']) + self.pb.set_backup('node', page_id, options=['--ttl=5d']) # Purge backups - log = self.delete_expired( - backup_dir, 'node', + log = self.pb.delete_expired( + 'node', options=['--delete-expired', '--log-level-console=LOG']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 4) + self.assertEqual(len(self.pb.show('node')), 4) self.assertIn('Time Window: 0d/5d', log) self.assertIn( @@ -170,53 +116,42 @@ def test_retention_redundancy_pinning(self): # @unittest.skip("skip") def test_retention_window_pinning(self): """purge all backups using window-based retention policy""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # take FULL BACKUP - backup_id_1 = self.backup_node(backup_dir, 'node', node) - page1 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_1 = self.pb.backup_node('node', node) + page1 = self.pb.backup_node('node', node, backup_type='page') # Take second FULL BACKUP - backup_id_2 = self.backup_node(backup_dir, 'node', node) - page2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_2 = self.pb.backup_node('node', node) + page2 = self.pb.backup_node('node', node, backup_type='page') # Take third FULL BACKUP - backup_id_3 = self.backup_node(backup_dir, 'node', node) - page2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') - - backups = os.path.join(backup_dir, 'backups', 'node') - for backup in os.listdir(backups): - if backup == 'pg_probackup.conf': - continue - with open( - os.path.join( - backups, backup, "backup.control"), "a") as conf: - conf.write("recovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( - datetime.now() - timedelta(days=3))) - - self.set_backup( - backup_dir, 'node', page1, options=['--ttl=30d']) + backup_id_3 = self.pb.backup_node('node', node) + page2 = self.pb.backup_node('node', node, backup_type='page') + + for backup in backup_dir.list_instance_backups('node'): + with self.modify_backup_control(backup_dir, 'node', backup) as cf: + cf.data += "\nrecovery_time='{:%Y-%m-%d %H:%M:%S}'\n".format( + datetime.now() - timedelta(days=3)) + + self.pb.set_backup('node', page1, options=['--ttl=30d']) # Purge backups - out = self.delete_expired( - backup_dir, 'node', + out = self.pb.delete_expired( + 'node', options=[ '--log-level-console=LOG', '--retention-window=1', '--delete-expired']) - self.assertEqual(len(self.show_pb(backup_dir, 'node')), 2) + self.assertEqual(len(self.pb.show('node')), 2) self.assertIn( 'LOG: Backup {0} is pinned until'.format(page1), out) @@ -237,26 +172,21 @@ def test_wal_retention_and_pinning(self): B1 B2---P---B3---> """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # take FULL BACKUP - self.backup_node( - backup_dir, 'node', node, options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) node.pgbench_init(scale=1) # Take PAGE BACKUP - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, backup_type='page', options=['--stream']) node.pgbench_init(scale=1) @@ -264,8 +194,7 @@ def test_wal_retention_and_pinning(self): # Take DELTA BACKUP and pin it expire_time = "{:%Y-%m-%d %H:%M:%S}".format( datetime.now() + timedelta(days=6)) - backup_id_pinned = self.backup_node( - backup_dir, 'node', node, + backup_id_pinned = self.pb.backup_node('node', node, backup_type='delta', options=[ '--stream', @@ -274,14 +203,16 @@ def test_wal_retention_and_pinning(self): node.pgbench_init(scale=1) # Take second PAGE BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta', options=['--stream']) + self.pb.backup_node('node', node, backup_type='delta', options=['--stream']) node.pgbench_init(scale=1) + tailer = tail_file(os.path.join(node.logs_dir, 'postgresql.log')) + tailer.wait(contains='LOG: pushing file "000000010000000000000004"') + # Purge backups - out = self.delete_expired( - backup_dir, 'node', + out = self.pb.delete_expired( + 'node', options=[ '--log-level-console=LOG', '--delete-wal', '--wal-depth=2']) @@ -292,15 +223,13 @@ def test_wal_retention_and_pinning(self): 'purpose of WAL retention'.format(backup_id_pinned), out) - for instance in self.show_archive(backup_dir): + for instance in self.pb.show_archive(): timelines = instance['timelines'] - - # sanity - for timeline in timelines: - self.assertEqual( - timeline['min-segno'], - '000000010000000000000004') - self.assertEqual(timeline['status'], 'OK') + for timeline in timelines: + self.assertEqual( + timeline['min-segno'], + '000000010000000000000004') + self.assertEqual(timeline['status'], 'OK') # @unittest.skip("skip") def test_wal_retention_and_pinning_1(self): @@ -313,35 +242,37 @@ def test_wal_retention_and_pinning_1(self): P---B1---> """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() expire_time = "{:%Y-%m-%d %H:%M:%S}".format( datetime.now() + timedelta(days=6)) # take FULL BACKUP - backup_id_pinned = self.backup_node( - backup_dir, 'node', node, + backup_id_pinned = self.pb.backup_node('node', node, options=['--expire-time={0}'.format(expire_time)]) node.pgbench_init(scale=2) # Take second PAGE BACKUP - self.backup_node( - backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') node.pgbench_init(scale=2) + self.wait_instance_wal_exists(backup_dir, 'node', + "000000010000000000000001.gz") + + tailer = tail_file(os.path.join(node.logs_dir, 'postgresql.log')) + tailer.wait(contains='LOG: pushing file "000000010000000000000002"') + # Purge backups - out = self.delete_expired( - backup_dir, 'node', + out = self.pb.delete_expired( + 'node', options=[ '--log-level-console=verbose', '--delete-wal', '--wal-depth=2']) @@ -352,60 +283,51 @@ def test_wal_retention_and_pinning_1(self): 'purpose of WAL retention'.format(backup_id_pinned), out) - for instance in self.show_archive(backup_dir): + for instance in self.pb.show_archive(): timelines = instance['timelines'] + for timeline in timelines: + self.assertEqual( + timeline['min-segno'], + '000000010000000000000002') + self.assertEqual(timeline['status'], 'OK') - # sanity - for timeline in timelines: - self.assertEqual( - timeline['min-segno'], - '000000010000000000000002') - self.assertEqual(timeline['status'], 'OK') - - self.validate_pb(backup_dir) + self.pb.validate() # @unittest.skip("skip") def test_add_note_newlines(self): """""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # FULL - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, options=['--stream', '--note={0}'.format('hello\nhello')]) - backup_meta = self.show_pb(backup_dir, 'node', backup_id) + backup_meta = self.pb.show('node', backup_id) self.assertEqual(backup_meta['note'], "hello") - self.set_backup(backup_dir, 'node', backup_id, options=['--note=hello\nhello']) + self.pb.set_backup('node', backup_id, options=['--note=hello\nhello']) - backup_meta = self.show_pb(backup_dir, 'node', backup_id) + backup_meta = self.pb.show('node', backup_id) self.assertEqual(backup_meta['note'], "hello") - self.set_backup(backup_dir, 'node', backup_id, options=['--note=none']) + self.pb.set_backup('node', backup_id, options=['--note=none']) - backup_meta = self.show_pb(backup_dir, 'node', backup_id) + backup_meta = self.pb.show('node', backup_id) self.assertNotIn('note', backup_meta) # @unittest.skip("skip") def test_add_big_note(self): """""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # note = node.safe_psql( @@ -417,46 +339,30 @@ def test_add_big_note(self): "SELECT repeat('hello', 210)").rstrip() # FULL - try: - self.backup_node( - backup_dir, 'node', node, - options=['--stream', '--note={0}'.format(note)]) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because note is too large " - "\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup note cannot exceed 1024 bytes", - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + self.pb.backup_node('node', node, + options=['--stream', '--note', note], + expect_error="because note is too large") + self.assertMessage(contains="ERROR: Backup note cannot exceed 1024 bytes") note = node.safe_psql( "postgres", "SELECT repeat('hello', 200)").decode('utf-8').rstrip() - backup_id = self.backup_node( - backup_dir, 'node', node, + backup_id = self.pb.backup_node('node', node, options=['--stream', '--note={0}'.format(note)]) - backup_meta = self.show_pb(backup_dir, 'node', backup_id) + backup_meta = self.pb.show('node', backup_id) self.assertEqual(backup_meta['note'], note) # @unittest.skip("skip") def test_add_big_note_1(self): """""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() note = node.safe_psql( @@ -464,13 +370,56 @@ def test_add_big_note_1(self): "SELECT repeat('q', 1024)").decode('utf-8').rstrip() # FULL - backup_id = self.backup_node(backup_dir, 'node', node, options=['--stream']) + backup_id = self.pb.backup_node('node', node, options=['--stream']) - self.set_backup( - backup_dir, 'node', backup_id, + self.pb.set_backup('node', backup_id, options=['--note={0}'.format(note)]) - backup_meta = self.show_pb(backup_dir, 'node', backup_id) + backup_meta = self.pb.show('node', backup_id) print(backup_meta) self.assertEqual(backup_meta['note'], note) + +#################################################################### +# dry-run +#################################################################### + + def test_basic_dry_run_set_backup(self): + """""" + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) + node.slow_start() + + note = node.safe_psql( + "postgres", + "SELECT repeat('q', 1024)").decode('utf-8').rstrip() + + backup_id = self.pb.backup_node('node', node, options=['--stream']) + + expire_time = "{:%Y-%m-%d %H:%M:%S}".format( + datetime.now() + timedelta(days=6)) + + self.pb.set_backup('node', backup_id, + options=['--expire-time={}'.format(expire_time), + '--dry-run', + '--note={0}'.format(note)]) + + backup_meta = self.pb.show('node', backup_id) + + print(backup_meta) + self.assertFalse(any('expire-time' in d for d in backup_meta)) + self.assertFalse(any('note' in d for d in backup_meta)) + + self.pb.set_backup('node', backup_id, + options=['--ttl=30d', + '--dry-run', + '--note={0}'.format(note)]) + + backup_meta = self.pb.show('node', backup_id) + + print(backup_meta) + self.assertFalse(any('ttl' in d for d in backup_meta)) + self.assertFalse(any('note' in d for d in backup_meta)) diff --git a/tests/show_test.py b/tests/show_test.py index c4b96499d..ae4fd2822 100644 --- a/tests/show_test.py +++ b/tests/show_test.py @@ -1,117 +1,106 @@ -import os +import copy import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +import os + +from .compression_test import have_alg +from .helpers.ptrack_helpers import ProbackupTest, fs_backup_class +from .helpers.state_helper import get_program_version +from .helpers.validators.show_validator import ShowJsonResultValidator -class ShowTest(ProbackupTest, unittest.TestCase): +class ShowTest(ProbackupTest): # @unittest.skip("skip") # @unittest.expectedFailure def test_show_1(self): """Status DONE and OK""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() self.assertEqual( - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=["--log-level-console=off"]), None ) - self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) + self.assertIn("OK", self.pb.show('node', as_text=True)) # @unittest.skip("skip") # @unittest.expectedFailure def test_show_json(self): """Status DONE and OK""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() self.assertEqual( - self.backup_node( - backup_dir, 'node', node, + self.pb.backup_node('node', node, options=["--log-level-console=off"]), None ) - self.backup_node(backup_dir, 'node', node) - self.assertIn("OK", self.show_pb(backup_dir, 'node', as_text=True)) + self.pb.backup_node('node', node) + self.assertIn("OK", self.pb.show('node', as_text=True)) # @unittest.skip("skip") def test_corrupt_2(self): """Status CORRUPT""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) # delete file which belong to backup - file = os.path.join( - backup_dir, "backups", "node", - backup_id, "database", "postgresql.conf") - os.remove(file) - - try: - self.validate_pb(backup_dir, 'node', backup_id) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because backup corrupted." - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd - ) - ) - except ProbackupException as e: - self.assertIn( - 'data files are corrupted', - e.message, - '\n Unexpected Error Message: {0}\n' - ' CMD: {1}'.format(repr(e.message), self.cmd) - ) - self.assertIn("CORRUPT", self.show_pb(backup_dir, as_text=True)) + self.remove_backup_file(backup_dir, 'node', backup_id, "database/postgresql.conf") + + error_result = self.pb.validate('node', backup_id, expect_error=True) + + self.assertMessage(error_result, contains='data files are corrupted') + self.assertIn("CORRUPT", self.pb.show(as_text=True)) + + def test_failed_backup_status(self): + """Status ERROR - showing recovery-time for faield backup""" + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + self.pb.backup_node('node', node, backup_type='delta', expect_error=True) + + show_res = self.pb.show('node', as_text=True, as_json=False) + self.assertIn("ERROR", show_res) + self.assertIn("Recovery Time", show_res) + self.assertNotIn("---- DELTA", show_res) # @unittest.skip("skip") def test_no_control_file(self): """backup.control doesn't exist""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) # delete backup.control file - file = os.path.join( - backup_dir, "backups", "node", - backup_id, "backup.control") - os.remove(file) + self.remove_backup_file(backup_dir, "node", backup_id, "backup.control") - output = self.show_pb(backup_dir, 'node', as_text=True, as_json=False) + output = self.pb.show('node', as_text=True, as_json=False) self.assertIn( 'Control file', @@ -124,26 +113,20 @@ def test_no_control_file(self): # @unittest.skip("skip") def test_empty_control_file(self): """backup.control is empty""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) # truncate backup.control file - file = os.path.join( - backup_dir, "backups", "node", - backup_id, "backup.control") - fd = open(file, 'w') - fd.close() + with self.modify_backup_control(self.backup_dir, 'node', backup_id) as cf: + cf.data = '' - output = self.show_pb(backup_dir, 'node', as_text=True, as_json=False) + output = self.pb.show('node', as_text=True, as_json=False) self.assertIn( 'Control file', @@ -157,29 +140,22 @@ def test_empty_control_file(self): # @unittest.expectedFailure def test_corrupt_control_file(self): """backup.control contains invalid option""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) # corrupt backup.control file - file = os.path.join( - backup_dir, "backups", "node", - backup_id, "backup.control") - fd = open(file, 'a') - fd.write("statuss = OK") - fd.close() + with self.modify_backup_control(self.backup_dir, 'node', backup_id) as cf: + cf.data += "\nstatuss = OK" self.assertIn( 'WARNING: Invalid option "statuss" in file', - self.show_pb(backup_dir, 'node', as_json=False, as_text=True)) + self.pb.show('node', as_json=False, as_text=True)) # @unittest.skip("skip") # @unittest.expectedFailure @@ -188,88 +164,65 @@ def test_corrupt_correctness(self): if not self.remote: self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=1) # FULL - backup_local_id = self.backup_node( - backup_dir, 'node', node, no_remote=True) - - output_local = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_local_id) + backup_local_id = self.pb.backup_node('node', node, no_remote=True) - backup_remote_id = self.backup_node(backup_dir, 'node', node) - - output_remote = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + backup_remote_id = self.pb.backup_node('node', node) # check correctness - self.assertEqual( - output_local['data-bytes'], - output_remote['data-bytes']) - - self.assertEqual( - output_local['uncompressed-bytes'], - output_remote['uncompressed-bytes']) + self.check_backup_size_in_show(backup_local_id, backup_remote_id, 'node', compressed=True) + self.check_backup_size_in_show(backup_local_id, backup_remote_id, 'node', compressed=False) # DELTA - backup_local_id = self.backup_node( - backup_dir, 'node', node, + backup_local_id = self.pb.backup_node('node', node, backup_type='delta', no_remote=True) - output_local = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_local_id) - self.delete_pb(backup_dir, 'node', backup_local_id) + output_local = self.pb.show('node', as_json=False, backup_id=backup_local_id) + self.pb.delete('node', backup_local_id) - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') + backup_remote_id = self.pb.backup_node('node', node, backup_type='delta') - output_remote = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_remote_id) - self.delete_pb(backup_dir, 'node', backup_remote_id) + output_remote = self.pb.show('node', as_json=False, backup_id=backup_remote_id) + self.pb.delete('node', backup_remote_id) # check correctness - self.assertEqual( - output_local['data-bytes'], - output_remote['data-bytes']) + self.assertAlmostEqual( + int(output_local['data-bytes']), + int(output_remote['data-bytes']), delta=2) - self.assertEqual( - output_local['uncompressed-bytes'], - output_remote['uncompressed-bytes']) + self.assertAlmostEqual( + int(output_local['uncompressed-bytes']), + int(output_remote['uncompressed-bytes']), delta=2) # PAGE - backup_local_id = self.backup_node( - backup_dir, 'node', node, + backup_local_id = self.pb.backup_node('node', node, backup_type='page', no_remote=True) - output_local = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_local_id) - self.delete_pb(backup_dir, 'node', backup_local_id) + output_local = self.pb.show('node', as_json=False, backup_id=backup_local_id) + self.pb.delete('node', backup_local_id) - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_remote_id = self.pb.backup_node('node', node, backup_type='page') - output_remote = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_remote_id) - self.delete_pb(backup_dir, 'node', backup_remote_id) + output_remote = self.pb.show('node', as_json=False, backup_id=backup_remote_id) + self.pb.delete('node', backup_remote_id) # check correctness - self.assertEqual( - output_local['data-bytes'], - output_remote['data-bytes']) + self.assertAlmostEqual( + int(output_local['data-bytes']), + int(output_remote['data-bytes']), delta=2) - self.assertEqual( - output_local['uncompressed-bytes'], - output_remote['uncompressed-bytes']) + self.assertAlmostEqual( + int(output_local['uncompressed-bytes']), + int(output_remote['uncompressed-bytes']), delta=2) # @unittest.skip("skip") # @unittest.expectedFailure @@ -278,92 +231,78 @@ def test_corrupt_correctness_1(self): if not self.remote: self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() + # stabilize test + # there were situation that due to internal wal segment switches + # backup_label differed in size: + # - first page backup had 0/E000028 location, and + # - second page backup - 0/10000028 + # Stabilize by adding more segments therefore it is always long + for i in range(8): + self.switch_wal_segment(node) + node.pgbench_init(scale=1) # FULL - backup_local_id = self.backup_node( - backup_dir, 'node', node, no_remote=True) - - output_local = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_local_id) + backup_local_id = self.pb.backup_node('node', node, no_remote=True) - backup_remote_id = self.backup_node(backup_dir, 'node', node) - - output_remote = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_remote_id) + backup_remote_id = self.pb.backup_node('node', node) # check correctness - self.assertEqual( - output_local['data-bytes'], - output_remote['data-bytes']) - - self.assertEqual( - output_local['uncompressed-bytes'], - output_remote['uncompressed-bytes']) + self.check_backup_size_in_show(backup_local_id, backup_remote_id, 'node', compressed=True) + self.check_backup_size_in_show(backup_local_id, backup_remote_id, 'node', compressed=False) # change data pgbench = node.pgbench(options=['-T', '10', '--no-vacuum']) pgbench.wait() # DELTA - backup_local_id = self.backup_node( - backup_dir, 'node', node, + backup_local_id = self.pb.backup_node('node', node, backup_type='delta', no_remote=True) - output_local = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_local_id) - self.delete_pb(backup_dir, 'node', backup_local_id) + output_local = self.pb.show('node', as_json=False, backup_id=backup_local_id) + self.pb.delete('node', backup_local_id) - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta') + backup_remote_id = self.pb.backup_node('node', node, backup_type='delta') - output_remote = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_remote_id) - self.delete_pb(backup_dir, 'node', backup_remote_id) + output_remote = self.pb.show('node', as_json=False, backup_id=backup_remote_id) + self.pb.delete('node', backup_remote_id) # check correctness - self.assertEqual( - output_local['data-bytes'], - output_remote['data-bytes']) + self.assertAlmostEqual( + int(output_local['data-bytes']), + int(output_remote['data-bytes']), delta=2) - self.assertEqual( - output_local['uncompressed-bytes'], - output_remote['uncompressed-bytes']) + self.assertAlmostEqual( + int(output_local['uncompressed-bytes']), + int(output_remote['uncompressed-bytes']), delta=2) # PAGE - backup_local_id = self.backup_node( - backup_dir, 'node', node, + backup_local_id = self.pb.backup_node('node', node, backup_type='page', no_remote=True) - output_local = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_local_id) - self.delete_pb(backup_dir, 'node', backup_local_id) + output_local = self.pb.show('node', as_json=False, backup_id=backup_local_id) + self.pb.delete('node', backup_local_id) - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_remote_id = self.pb.backup_node('node', node, backup_type='page') - output_remote = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_remote_id) - self.delete_pb(backup_dir, 'node', backup_remote_id) + output_remote = self.pb.show('node', as_json=False, backup_id=backup_remote_id) + self.pb.delete('node', backup_remote_id) # check correctness - self.assertEqual( - output_local['data-bytes'], - output_remote['data-bytes']) + self.assertAlmostEqual( + int(output_local['data-bytes']), + int(output_remote['data-bytes']), delta=2) - self.assertEqual( - output_local['uncompressed-bytes'], - output_remote['uncompressed-bytes']) + self.assertAlmostEqual( + int(output_local['uncompressed-bytes']), + int(output_remote['uncompressed-bytes']), delta=2) # @unittest.skip("skip") # @unittest.expectedFailure @@ -372,138 +311,305 @@ def test_corrupt_correctness_2(self): if not self.remote: self.skipTest("You must enable PGPROBACKUP_SSH_REMOTE" " for run this test") - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() + # stabilize test + # there were situation that due to internal wal segment switches + # backup_label differed in size: + # - first page backup had 0/E000028 location, and + # - second page backup - 0/10000028 + # Stabilize by adding more segments therefore it is always long + for i in range(8): + self.switch_wal_segment(node) + node.pgbench_init(scale=1) # FULL - backup_local_id = self.backup_node( - backup_dir, 'node', node, + backup_local_id = self.pb.backup_node('node', node, options=['--compress'], no_remote=True) - output_local = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_local_id) - if self.remote: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, options=['--compress']) + backup_remote_id = self.pb.backup_node('node', node, options=['--compress']) else: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, + backup_remote_id = self.pb.backup_node('node', node, options=['--remote-proto=ssh', '--remote-host=localhost', '--compress']) - output_remote = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_remote_id) - # check correctness - self.assertEqual( - output_local['data-bytes'], - output_remote['data-bytes']) - - self.assertEqual( - output_local['uncompressed-bytes'], - output_remote['uncompressed-bytes']) + self.check_backup_size_in_show(backup_local_id, backup_remote_id, 'node', compressed=True) + self.check_backup_size_in_show(backup_local_id, backup_remote_id, 'node', compressed=False) # change data pgbench = node.pgbench(options=['-T', '10', '--no-vacuum']) pgbench.wait() # DELTA - backup_local_id = self.backup_node( - backup_dir, 'node', node, + backup_local_id = self.pb.backup_node('node', node, backup_type='delta', options=['--compress'], no_remote=True) - output_local = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_local_id) - self.delete_pb(backup_dir, 'node', backup_local_id) + output_local = self.pb.show('node', as_json=False, backup_id=backup_local_id) + self.pb.delete('node', backup_local_id) if self.remote: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', options=['--compress']) + backup_remote_id = self.pb.backup_node('node', node, backup_type='delta', options=['--compress']) else: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='delta', + backup_remote_id = self.pb.backup_node('node', node, backup_type='delta', options=['--remote-proto=ssh', '--remote-host=localhost', '--compress']) - output_remote = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_remote_id) - self.delete_pb(backup_dir, 'node', backup_remote_id) + output_remote = self.pb.show('node', as_json=False, backup_id=backup_remote_id) + self.pb.delete('node', backup_remote_id) # check correctness - self.assertEqual( - output_local['data-bytes'], - output_remote['data-bytes']) + self.assertAlmostEqual( + int(output_local['data-bytes']), + int(output_remote['data-bytes']), delta=2) - self.assertEqual( - output_local['uncompressed-bytes'], - output_remote['uncompressed-bytes']) + self.assertAlmostEqual( + int(output_local['uncompressed-bytes']), + int(output_remote['uncompressed-bytes']), delta=2) # PAGE - backup_local_id = self.backup_node( - backup_dir, 'node', node, + backup_local_id = self.pb.backup_node('node', node, backup_type='page', options=['--compress'], no_remote=True) - output_local = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_local_id) - self.delete_pb(backup_dir, 'node', backup_local_id) + output_local = self.pb.show('node', as_json=False, backup_id=backup_local_id) + self.pb.delete('node', backup_local_id) if self.remote: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', options=['--compress']) + backup_remote_id = self.pb.backup_node('node', node, backup_type='page', options=['--compress']) else: - backup_remote_id = self.backup_node( - backup_dir, 'node', node, backup_type='page', + backup_remote_id = self.pb.backup_node('node', node, backup_type='page', options=['--remote-proto=ssh', '--remote-host=localhost', '--compress']) - output_remote = self.show_pb( - backup_dir, 'node', as_json=False, backup_id=backup_remote_id) - self.delete_pb(backup_dir, 'node', backup_remote_id) + output_remote = self.pb.show('node', as_json=False, backup_id=backup_remote_id) + self.pb.delete('node', backup_remote_id) # check correctness - self.assertEqual( - output_local['data-bytes'], - output_remote['data-bytes']) + self.assertAlmostEqual( + int(output_local['data-bytes']), + int(output_remote['data-bytes']), delta=2) - self.assertEqual( - output_local['uncompressed-bytes'], - output_remote['uncompressed-bytes']) + self.assertAlmostEqual( + int(output_local['uncompressed-bytes']), + int(output_remote['uncompressed-bytes']), delta=2) # @unittest.skip("skip") # @unittest.expectedFailure def test_color_with_no_terminal(self): """backup.control contains invalid option""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums'], + node = self.pg_node.make_simple('node', pg_options={'autovacuum': 'off'}) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() node.pgbench_init(scale=1) # FULL - try: - self.backup_node( - backup_dir, 'node', node, options=['--archive-timeout=1s']) - # we should die here because exception is what we expect to happen - self.assertEqual( - 1, 0, - "Expecting Error because archiving is disabled\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertNotIn( - '[0m', e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + error_result = self.pb.backup_node('node', node, options=['--archive-timeout=1s'], expect_error=True) + self.assertNotIn('[0m', error_result) + + @unittest.skipIf(not (have_alg('lz4') and have_alg('zstd')), + "pg_probackup is not compiled with lz4 or zstd support") + def test_show_command_as_text(self): + instance_name = 'node' + node = self.pg_node.make_simple( + base_dir=instance_name) + + self.pb.init() + self.pb.add_instance(instance_name, node) + self.pb.set_archiving(instance_name, node) + node.slow_start() + + self.pb.backup_node(instance_name, node, backup_type="full", options=['--compress-level', '1', + '--compress-algorithm', 'pglz']) + + self.pb.backup_node(instance_name, node, backup_type="delta", options=['--compress-level', '3', + '--compress-algorithm', 'lz4']) + + self.pb.backup_node(instance_name, node, backup_type="page", options=['--compress-level', '9', + '--compress-algorithm', 'zstd']) + self.pb.backup_node(instance_name, node, backup_type="page") + + show_backups = self.pb.show(instance_name, as_text=True, as_json=False) + self.assertIn(" FULL ARCHIVE ", show_backups) # Mode, Wal mode + self.assertIn(" DELTA ARCHIVE ", show_backups) # Mode, Wal mode + self.assertIn(" PAGE ARCHIVE ", show_backups) # Mode, Wal mode + self.assertIn(" pglz ", show_backups) + self.assertIn(" lz4 ", show_backups) + self.assertIn(" zstd ", show_backups) + self.assertIn(" none ", show_backups) + self.assertIn(" OK ", show_backups) # Status + + def test_show_command_as_json(self): + instance_name = 'node' + node = self.pg_node.make_simple( + base_dir=instance_name) + + self.pb.init() + self.pb.add_instance(instance_name, node) + self.pb.set_archiving(instance_name, node) + node.slow_start() + + pg_version = int(self.pg_config_version/10000) + + full_backup_id = self.pb.backup_node(instance_name, node, backup_type="full") + + delta_backup_id = self.pb.backup_node(instance_name, node, backup_type="delta") + + page_backup_id = self.pb.backup_node(instance_name, node, backup_type="page") + + show_backups = self.pb.show(instance_name, as_text=False, as_json=True) + + common_show_result = ShowJsonResultValidator() + common_show_result.wal = "ARCHIVE" + common_show_result.compress_alg = "none" + common_show_result.compress_level = 1 + common_show_result.from_replica = "false" + common_show_result.block_size = 8192 + common_show_result.xlog_block_size = 8192 + common_show_result.checksum_version = 1 + common_show_result.program_version = get_program_version() + common_show_result.server_version = pg_version + common_show_result.status = "OK" + + full_show_result = copy.deepcopy(common_show_result) + full_show_result.backup_mode = "FULL" + full_show_result.set_backup_id = full_backup_id + + delta_show_result = copy.deepcopy(common_show_result) + delta_show_result.backup_mode = "DELTA" + delta_show_result.backup_id = delta_backup_id + delta_show_result.parent_backup_id = full_backup_id + + page_show_result = copy.deepcopy(common_show_result) + page_show_result.backup_mode = "PAGE" + page_show_result.backup_id = page_backup_id + page_show_result.parent_backup_id = delta_backup_id + + full_show_result.check_show_json(show_backups[0]) + delta_show_result.check_show_json(show_backups[1]) + page_show_result.check_show_json(show_backups[2]) + + def test_tablespace_print_issue_431(self): + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + # Create tablespace + tblspc_path = os.path.join(node.base_dir, "tblspc") + os.makedirs(tblspc_path) + with node.connect("postgres") as con: + con.connection.autocommit = True + con.execute("CREATE TABLESPACE tblspc LOCATION '%s'" % tblspc_path) + con.connection.autocommit = False + con.execute("CREATE TABLE test (id int) TABLESPACE tblspc") + con.execute("INSERT INTO test VALUES (1)") + con.commit() + + full_backup_id = self.pb.backup_node('node', node) + self.assertIn("OK", self.pb.show('node', as_text=True)) + # Check that tablespace info exists. JSON + self.assertIn("tablespace_map", self.pb.show('node', as_text=True)) + self.assertIn("oid", self.pb.show('node', as_text=True)) + self.assertIn("path", self.pb.show('node', as_text=True)) + self.assertIn(tblspc_path, self.pb.show('node', as_text=True)) + # Check that tablespace info exists. PLAIN + self.assertIn("tablespace_map", self.pb.show('node', backup_id=full_backup_id, as_text=True, as_json=False)) + self.assertIn(tblspc_path, self.pb.show('node', backup_id=full_backup_id, as_text=True, as_json=False)) + # Check that tablespace info NOT exists if backup id not provided. PLAIN + self.assertNotIn("tablespace_map", self.pb.show('node', as_text=True, as_json=False)) + + def test_show_hidden_merged_dirs_as_json(self): + instance_name = 'node' + node = self.pg_node.make_simple( + base_dir=instance_name) + + self.pb.init() + self.pb.add_instance(instance_name, node) + self.pb.set_archiving(instance_name, node) + node.slow_start() + + pg_version = int(self.pg_config_version/10000) + + full_backup_id = self.pb.backup_node(instance_name, node, backup_type="full") + + delta_backup_id = self.pb.backup_node(instance_name, node, backup_type="delta") + + page_backup_id = self.pb.backup_node(instance_name, node, backup_type="page") + + self.pb.merge_backup(instance_name, delta_backup_id) + show_backups = self.pb.show(instance_name, as_text=False, as_json=True, options=["--show-symlinks"]) + + self.assertEqual(show_backups[0]['backup-mode'], "FULL") + self.assertEqual(show_backups[0]['id'], delta_backup_id) + self.assertEqual(show_backups[0]['id'], show_backups[1]['id']) + self.assertEqual(show_backups[0]['dir'], full_backup_id) + + self.assertEqual(show_backups[1]['status'], "SYMLINK") + self.assertEqual(show_backups[1]['id'], show_backups[0]['id']) + self.assertEqual(show_backups[1]['symlink'], full_backup_id) + + + def test_show_hidden_merged_dirs_as_plain(self): + instance_name = 'node' + node = self.pg_node.make_simple( + base_dir=instance_name) + + self.pb.init() + self.pb.add_instance(instance_name, node) + self.pb.set_archiving(instance_name, node) + node.slow_start() + + pg_version = int(self.pg_config_version/10000) + + full_backup_id = self.pb.backup_node(instance_name, node, backup_type="full") + + delta_backup_id = self.pb.backup_node(instance_name, node, backup_type="delta") + + page_backup_id = self.pb.backup_node(instance_name, node, backup_type="page") + + self.pb.merge_backup(instance_name, delta_backup_id) + show_backups = self.pb.show(instance_name, as_text=True, as_json=False, options=["--show-symlinks"]) + + self.assertIn(" PAGE ARCHIVE ", show_backups) # Mode, Wal mode + self.assertIn(" FULL ARCHIVE ", show_backups) # Mode, Wal mode + + def get_backup_label_size(self, backup_id, instance_name): + """Get backup_label size from file backup_content.control""" + content_control_json = self.read_backup_content_control(backup_id, instance_name) + for item in content_control_json: + if item.get('path') == 'backup_label': + return item['size'] + + def check_backup_size_in_show(self, first_backup_id, second_backup_id, instance_name, compressed=True): + """Use show command to check backup size. If we have difference, + try to compare size without backuo_label file""" + first_out = self.pb.show('node', as_json=False, backup_id=first_backup_id) + + second_out = self.pb.show('node', as_json=False, backup_id=second_backup_id) + + # check correctness + if compressed: + first_size = first_out['data-bytes'] + second_size = second_out['data-bytes'] + else: + first_size = first_out['uncompressed-bytes'] + second_size = second_out['uncompressed-bytes'] + if fs_backup_class.is_file_based: + local_label_size = self.get_backup_label_size(first_backup_id, instance_name) + remote_label_size = self.get_backup_label_size(second_backup_id, instance_name) + # If we have difference in full size check without backup_label file + self.assertTrue(first_size == second_size or + first_size - local_label_size == second_size - remote_label_size) + self.assertAlmostEqual(int(local_label_size), int(remote_label_size), delta=2) + else: + self.assertAlmostEqual(int(first_size), int(second_size), delta=2) diff --git a/tests/test_utils/__init__.py b/tests/test_utils/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/test_utils/config_provider.py b/tests/test_utils/config_provider.py new file mode 100644 index 000000000..76184d4e4 --- /dev/null +++ b/tests/test_utils/config_provider.py @@ -0,0 +1,8 @@ +import configparser + + +def read_config(s3_config_file): + config = configparser.ConfigParser() + config.read_string('[fake-section]\n' + open(s3_config_file).read()) + + return config['fake-section'] diff --git a/tests/test_utils/s3_backup.py b/tests/test_utils/s3_backup.py new file mode 100644 index 000000000..c951e38bf --- /dev/null +++ b/tests/test_utils/s3_backup.py @@ -0,0 +1,208 @@ +import os +import io +import sys + +import minio +from minio import Minio +from minio.deleteobjects import DeleteObject +import urllib3 +from pg_probackup2.storage.fs_backup import TestBackupDir +from pg_probackup2.init_helpers import init_params +from . import config_provider + +root = os.path.realpath(os.path.join(os.path.dirname(__file__), '../..')) +if root not in sys.path: + sys.path.append(root) + +status_forcelist = [413, # RequestBodyTooLarge + 429, # TooManyRequests + 500, # InternalError + 503, # ServerBusy + ] + +DEFAULT_CONF_FILE = 's3/tests/s3.conf' + + +class S3TestBackupDir(TestBackupDir): + is_file_based = False + + def __init__(self, *, rel_path, backup): + self.access_key = None + self.secret_key = None + self.s3_type = None + self.tmp_path = None + self.host = None + self.port = None + self.bucket_name = None + self.region = None + self.bucket = None + self.path_suffix = None + self.https = None + self.s3_config_file = None + self.ca_certificate = None + + self.set_s3_config_file() + self.setup_s3_env() + + path = "pg_probackup" + if self.path_suffix: + path += "_" + self.path_suffix + if self.tmp_path == '' or os.path.isabs(self.tmp_path): + self.path = f"{path}{self.tmp_path}/{rel_path}/{backup}" + else: + self.path = f"{path}/{self.tmp_path}/{rel_path}/{backup}" + + secure: bool = False + self.versioning: bool = False + if self.https in ['ON', 'HTTPS']: + secure = True + if self.https and self.ca_certificate: + http_client = urllib3.PoolManager(cert_reqs='CERT_REQUIRED', + ca_certs=self.ca_certificate, + retries=urllib3.Retry(total=5, + backoff_factor=1, + status_forcelist=status_forcelist)) + else: + http_client = urllib3.PoolManager(retries=urllib3.Retry(total=5, + backoff_factor=1, + status_forcelist=status_forcelist)) + + self.conn = Minio(self.host + ":" + self.port, secure=secure, access_key=self.access_key, + secret_key=self.secret_key, http_client=http_client) + if not self.conn.bucket_exists(self.bucket): + raise Exception(f"Test bucket {self.bucket} does not exist.") + + try: + config = self.conn.get_bucket_versioning(self.bucket) + if config.status.lower() == "enabled" or config.status.lower() == "suspended": + self.versioning = True + else: + self.versioning = False + except Exception as e: + if "NotImplemented" in repr(e): + self.versioning = False + else: + raise e + self.pb_args = ('-B', '/' + self.path, f'--s3={init_params.s3_type}') + if self.s3_config_file: + self.pb_args += (f'--s3-config-file={self.s3_config_file}',) + return + + def setup_s3_env(self, s3_config=None): + self.tmp_path = os.environ.get('PGPROBACKUP_TMP_DIR', default='') + self.host = os.environ.get('PG_PROBACKUP_S3_HOST', default='') + + # If environment variables are not setup, use from config + if self.s3_config_file or s3_config: + minio_config = config_provider.read_config(self.s3_config_file or s3_config) + self.access_key = minio_config['access-key'] + self.secret_key = minio_config['secret-key'] + self.host = minio_config['s3-host'] + self.port = minio_config['s3-port'] + self.bucket = minio_config['s3-bucket'] + self.region = minio_config['s3-region'] + self.https = minio_config['s3-secure'] + init_params.s3_type = 'minio' + else: + self.access_key = os.environ.get('PG_PROBACKUP_S3_ACCESS_KEY') + self.secret_key = os.environ.get('PG_PROBACKUP_S3_SECRET_ACCESS_KEY') + self.host = os.environ.get('PG_PROBACKUP_S3_HOST') + self.port = os.environ.get('PG_PROBACKUP_S3_PORT') + self.bucket = os.environ.get('PG_PROBACKUP_S3_BUCKET_NAME') + self.region = os.environ.get('PG_PROBACKUP_S3_REGION') + self.https = os.environ.get('PG_PROBACKUP_S3_HTTPS') + self.ca_certificate = os.environ.get('PG_PROBACKUP_S3_CA_CERTIFICATE') + init_params.s3_type = os.environ.get('PG_PROBACKUP_S3_TEST') + + # multi-url case + # remove all urls from string except the first one + if ';' in self.host: + self.host = self.host[:self.host.find(';')] + if ':' in self.host: # also change port if it was overridden in multihost string + self.port = self.host[self.host.find(':') + 1:] + self.host = self.host[:self.host.find(':')] + + def set_s3_config_file(self): + s3_config = os.environ.get('PG_PROBACKUP_S3_CONFIG_FILE') + if s3_config is not None and s3_config.strip().lower() == "true": + self.s3_config_file = DEFAULT_CONF_FILE + else: + self.s3_config_file = s3_config + + def list_instance_backups(self, instance): + full_path = os.path.join(self.path, 'backups', instance) + candidates = self.conn.list_objects(self.bucket, prefix=full_path, recursive=True) + return [os.path.basename(os.path.dirname(x.object_name)) + for x in candidates if x.object_name.endswith('backup.control')] + + def list_files(self, sub_dir, recursive=False): + full_path = os.path.join(self.path, sub_dir) + # Need '/' in the end to find inside the folder + full_path_dir = full_path if full_path[-1] == '/' else full_path + '/' + object_list = self.conn.list_objects(self.bucket, prefix=full_path_dir, recursive=recursive) + return [obj.object_name.replace(full_path_dir, '', 1) + for obj in object_list + if not obj.is_dir] + + def list_dirs(self, sub_dir): + full_path = os.path.join(self.path, sub_dir) + # Need '/' in the end to find inside the folder + full_path_dir = full_path if full_path[-1] == '/' else full_path + '/' + object_list = self.conn.list_objects(self.bucket, prefix=full_path_dir, recursive=False) + return [obj.object_name.replace(full_path_dir, '', 1).rstrip('\\/') + for obj in object_list + if obj.is_dir] + + def read_file(self, sub_path, *, text=True): + full_path = os.path.join(self.path, sub_path) + bytes = self.conn.get_object(self.bucket, full_path).read() + if not text: + return bytes + return bytes.decode('utf-8') + + def write_file(self, sub_path, data, *, text=True): + full_path = os.path.join(self.path, sub_path) + if text: + data = data.encode('utf-8') + self.conn.put_object(self.bucket, full_path, io.BytesIO(data), length=len(data)) + + def cleanup(self, dir=''): + self.remove_dir(dir) + + def remove_file(self, sub_path): + full_path = os.path.join(self.path, sub_path) + self.conn.remove_object(self.bucket, full_path) + + def remove_dir(self, sub_path): + if sub_path: + full_path = os.path.join(self.path, sub_path) + else: + full_path = self.path + objs = self.conn.list_objects(self.bucket, prefix=full_path, recursive=True, + include_version=self.versioning) + delobjs = (DeleteObject(o.object_name, o.version_id) for o in objs) + errs = list(self.conn.remove_objects(self.bucket, delobjs)) + if errs: + strerrs = "; ".join(str(err) for err in errs) + raise Exception("There were errors: {0}".format(strerrs)) + + def exists(self, sub_path): + full_path = os.path.join(self.path, sub_path) + try: + self.conn.stat_object(self.bucket, full_path) + return True + except minio.error.S3Error as s3err: + if s3err.code == 'NoSuchKey': + return False + raise s3err + except Exception as err: + raise err + + def __str__(self): + return '/' + self.path + + def __repr__(self): + return "S3TestBackupDir" + str(self.path) + + def __fspath__(self): + return self.path diff --git a/tests/time_consuming_test.py b/tests/time_consuming_test.py index c0038c085..3da2208db 100644 --- a/tests/time_consuming_test.py +++ b/tests/time_consuming_test.py @@ -5,7 +5,7 @@ from time import sleep -class TimeConsumingTests(ProbackupTest, unittest.TestCase): +class TimeConsumingTests(ProbackupTest): def test_pbckp150(self): """ https://jira.postgrespro.ru/browse/PBCKP-150 @@ -19,11 +19,9 @@ def test_pbckp150(self): if not self.ptrack: self.skipTest('Skipped because ptrack support is disabled') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), + node = self.pg_node.make_simple('node', set_replication=True, ptrack_enable=self.ptrack, - initdb_params=['--data-checksums'], pg_options={ 'max_connections': 100, 'log_statement': 'none', @@ -32,14 +30,13 @@ def test_pbckp150(self): 'ptrack.map_size': 1}) if node.major_version >= 13: - self.set_auto_conf(node, {'wal_keep_size': '16000MB'}) + node.set_auto_conf({'wal_keep_size': '16000MB'}) else: - self.set_auto_conf(node, {'wal_keep_segments': '1000'}) + node.set_auto_conf({'wal_keep_segments': '1000'}) # init probackup and add an instance - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) # run the node and init ptrack node.slow_start() @@ -48,8 +45,8 @@ def test_pbckp150(self): node.pgbench_init(scale=5) # FULL backup followed by PTRACK backup - self.backup_node(backup_dir, 'node', node, options=['--stream']) - self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream']) + self.pb.backup_node('node', node, options=['--stream']) + self.pb.backup_node('node', node, backup_type='ptrack', options=['--stream']) # run ordinary pgbench scenario to imitate some activity and another pgbench for vacuuming in parallel nBenchDuration = 30 @@ -61,7 +58,7 @@ def test_pbckp150(self): # several PTRACK backups for i in range(nBenchDuration): print("[{}] backing up PTRACK diff...".format(i+1)) - self.backup_node(backup_dir, 'node', node, backup_type='ptrack', options=['--stream', '--log-level-console', 'VERBOSE']) + self.pb.backup_node('node', node, backup_type='ptrack', options=['--stream', '--log-level-console', 'VERBOSE']) sleep(0.1) # if the activity pgbench has finished, stop backing up if pgbench.poll() is not None: @@ -72,6 +69,6 @@ def test_pbckp150(self): pgbench.wait() pgbenchval.wait() - backups = self.show_pb(backup_dir, 'node') + backups = self.pb.show('node') for b in backups: self.assertEqual("OK", b['status']) diff --git a/tests/time_stamp_test.py b/tests/time_stamp_test.py index 170c62cd4..7398556f9 100644 --- a/tests/time_stamp_test.py +++ b/tests/time_stamp_test.py @@ -1,96 +1,78 @@ import os import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest import subprocess from time import sleep -class TimeStamp(ProbackupTest, unittest.TestCase): +class TimeStamp(ProbackupTest): def test_start_time_format(self): """Test backup ID changing after start-time editing in backup.control. We should convert local time in UTC format""" # Create simple node - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), - set_replication=True, - initdb_params=['--data-checksums']) - + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + backup_dir = self.backup_dir + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.start() - backup_id = self.backup_node(backup_dir, 'node', node, options=['--stream', '-j 2']) - show_backup = self.show_pb(backup_dir, 'node') - - i = 0 - while i < 2: - with open(os.path.join(backup_dir, "backups", "node", backup_id, "backup.control"), "r+") as f: - output = "" - for line in f: - if line.startswith('start-time') is True: - if i == 0: - output = output + str(line[:-5])+'+00\''+'\n' - else: - output = output + str(line[:-5]) + '\'' + '\n' + backup_id = self.pb.backup_node('node', node, options=['--stream', '-j 2']) + show_backup = self.pb.show('node') + + for i in range(2): + with self.modify_backup_control(backup_dir, 'node', backup_id) as cf: + lines = cf.data.splitlines(keepends=True) + for j, line in enumerate(lines): + if not line.startswith('start-time'): + continue + if i == 0: + lines[j] = line[:-5] + "+00'\n" else: - output = output + str(line) - f.close() - - with open(os.path.join(backup_dir, "backups", "node", backup_id, "backup.control"), "w") as fw: - fw.write(output) - fw.flush() - show_backup = show_backup + self.show_pb(backup_dir, 'node') - i += 1 + lines[j] = line[:-5] + "'\n" + cf.data = "".join(lines) + show_backup = show_backup + self.pb.show('node') print(show_backup[1]['id']) print(show_backup[2]['id']) self.assertTrue(show_backup[1]['id'] == show_backup[2]['id'], "ERROR: Localtime format using instead of UTC") - output = self.show_pb(backup_dir, as_json=False, as_text=True) + output = self.pb.show(as_json=False, as_text=True) self.assertNotIn("backup ID in control file", output) node.stop() def test_server_date_style(self): """Issue #112""" - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), + node = self.pg_node.make_simple('node', set_replication=True, - initdb_params=['--data-checksums'], pg_options={"datestyle": "GERMAN, DMY"}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.start() - self.backup_node( - backup_dir, 'node', node, options=['--stream', '-j 2']) + self.pb.backup_node('node', node, options=['--stream', '-j 2']) def test_handling_of_TZ_env_variable(self): """Issue #284""" - node = self.make_simple_node( - base_dir="{0}/{1}/node".format(self.module_name, self.fname), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.start() my_env = os.environ.copy() my_env["TZ"] = "America/Detroit" - self.backup_node( - backup_dir, 'node', node, options=['--stream', '-j 2'], env=my_env) + self.pb.backup_node('node', node, options=['--stream', '-j 2'], env=my_env) - output = self.show_pb(backup_dir, 'node', as_json=False, as_text=True, env=my_env) + output = self.pb.show('node', as_json=False, as_text=True, env=my_env) self.assertNotIn("backup ID in control file", output) @@ -98,14 +80,11 @@ def test_handling_of_TZ_env_variable(self): # @unittest.expectedFailure def test_dst_timezone_handling(self): """for manual testing""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() print(subprocess.Popen( @@ -124,7 +103,7 @@ def test_dst_timezone_handling(self): stderr=subprocess.PIPE).communicate() # FULL - output = self.backup_node(backup_dir, 'node', node, return_id=False) + output = self.pb.backup_node('node', node, return_id=False) self.assertNotIn("backup ID in control file", output) # move to dst @@ -134,8 +113,7 @@ def test_dst_timezone_handling(self): stderr=subprocess.PIPE).communicate() # DELTA - output = self.backup_node( - backup_dir, 'node', node, backup_type='delta', return_id=False) + output = self.pb.backup_node('node', node, backup_type='delta', return_id=False) self.assertNotIn("backup ID in control file", output) subprocess.Popen( @@ -144,9 +122,9 @@ def test_dst_timezone_handling(self): stderr=subprocess.PIPE).communicate() # DELTA - self.backup_node(backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') - output = self.show_pb(backup_dir, as_json=False, as_text=True) + output = self.pb.show(as_json=False, as_text=True) self.assertNotIn("backup ID in control file", output) subprocess.Popen( @@ -156,9 +134,9 @@ def test_dst_timezone_handling(self): sleep(10) - self.backup_node(backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') - output = self.show_pb(backup_dir, as_json=False, as_text=True) + output = self.pb.show(as_json=False, as_text=True) self.assertNotIn("backup ID in control file", output) subprocess.Popen( @@ -169,14 +147,11 @@ def test_dst_timezone_handling(self): @unittest.skip("skip") def test_dst_timezone_handling_backward_compatibilty(self): """for manual testing""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() subprocess.Popen( @@ -195,7 +170,7 @@ def test_dst_timezone_handling_backward_compatibilty(self): stderr=subprocess.PIPE).communicate() # FULL - self.backup_node(backup_dir, 'node', node, old_binary=True, return_id=False) + self.pb.backup_node('node', node, old_binary=True, return_id=False) # move to dst subprocess.Popen( @@ -204,8 +179,7 @@ def test_dst_timezone_handling_backward_compatibilty(self): stderr=subprocess.PIPE).communicate() # DELTA - output = self.backup_node( - backup_dir, 'node', node, backup_type='delta', old_binary=True, return_id=False) + output = self.pb.backup_node('node', node, backup_type='delta', old_binary=True, return_id=False) subprocess.Popen( ['sudo', 'timedatectl', 'set-time', '2020-12-01 12:00:00'], @@ -213,9 +187,9 @@ def test_dst_timezone_handling_backward_compatibilty(self): stderr=subprocess.PIPE).communicate() # DELTA - self.backup_node(backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') - output = self.show_pb(backup_dir, as_json=False, as_text=True) + output = self.pb.show(as_json=False, as_text=True) self.assertNotIn("backup ID in control file", output) subprocess.Popen( @@ -225,9 +199,9 @@ def test_dst_timezone_handling_backward_compatibilty(self): sleep(10) - self.backup_node(backup_dir, 'node', node, backup_type='delta') + self.pb.backup_node('node', node, backup_type='delta') - output = self.show_pb(backup_dir, as_json=False, as_text=True) + output = self.pb.show(as_json=False, as_text=True) self.assertNotIn("backup ID in control file", output) subprocess.Popen( diff --git a/tests/validate_test.py b/tests/validate_test.py index 4ff44941f..3b97171d4 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -1,15 +1,17 @@ import os -import unittest -from .helpers.ptrack_helpers import ProbackupTest, ProbackupException +from .helpers.ptrack_helpers import ProbackupTest +from pg_probackup2.gdb import needs_gdb from datetime import datetime, timedelta from pathlib import Path -import subprocess -from sys import exit import time import hashlib -class ValidateTest(ProbackupTest, unittest.TestCase): +class ValidateTest(ProbackupTest): + + def setUp(self): + super().setUp() + self.test_env["PGPROBACKUP_TESTS_SKIP_HIDDEN"] = "ON" # @unittest.skip("skip") # @unittest.expectedFailure @@ -17,14 +19,11 @@ def test_basic_validate_nullified_heap_page_backup(self): """ make node with nullified heap block """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=3) @@ -43,26 +42,27 @@ def test_basic_validate_nullified_heap_page_backup(self): f.seek(8192) f.write(b"\x00"*8192) f.flush() - f.close - self.backup_node( - backup_dir, 'node', node, options=['--log-level-file=verbose']) + self.pb.backup_node( + 'node', node, options=['--log-level-file=verbose']) pgdata = self.pgdata_content(node.data_dir) + log_content = self.read_pb_log() + self.assertIn( + 'File: {0} blknum 1, empty zeroed page'.format(file_path), + log_content, + 'Failed to detect nullified block') if not self.remote: - log_file_path = os.path.join(backup_dir, "log", "pg_probackup.log") - with open(log_file_path) as f: - log_content = f.read() - self.assertIn( + self.assertIn( 'File: "{0}" blknum 1, empty page'.format(Path(file).as_posix()), log_content, 'Failed to detect nullified block') - self.validate_pb(backup_dir, options=["-j", "4"]) + self.pb.validate(options=["-j", "4"]) node.cleanup() - self.restore_node(backup_dir, 'node', node) + self.pb.restore_node('node', node=node) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -74,14 +74,12 @@ def test_validate_wal_unreal_values(self): make node with archiving, make archive backup validate to both real and unreal values """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + instance_name = 'node' + node = self.pg_node.make_simple(instance_name) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance(instance_name, node) + self.pb.set_archiving(instance_name, node) node.slow_start() node.pgbench_init(scale=3) @@ -89,60 +87,34 @@ def test_validate_wal_unreal_values(self): con.execute("CREATE TABLE tbl0005 (a text)") con.commit() - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node(instance_name, node) node.pgbench_init(scale=3) - target_time = self.show_pb( - backup_dir, 'node', backup_id)['recovery-time'] + target_time = self.pb.show( + instance_name, backup_id)['recovery-time'] after_backup_time = datetime.now().replace(second=0, microsecond=0) # Validate to real time - self.assertIn( - "INFO: Backup validation completed successfully", - self.validate_pb( - backup_dir, 'node', - options=["--time={0}".format(target_time), "-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + validate_result = self.pb.validate(instance_name, options=[f"--recovery-target-time={target_time}", "-j", "4"]) + self.assertMessage(validate_result, contains="INFO: Backup validation completed successfully") # Validate to unreal time unreal_time_1 = after_backup_time - timedelta(days=2) - try: - self.validate_pb( - backup_dir, 'node', options=["--time={0}".format( - unreal_time_1), "-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of validation to unreal time.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Backup satisfying target options is not found', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + error_result = self.pb.validate(instance_name, + options=[f"--time={unreal_time_1}", "-j", "4"], + expect_error=True) + + self.assertMessage(error_result, contains='ERROR: Backup satisfying target options is not found') # Validate to unreal time #2 unreal_time_2 = after_backup_time + timedelta(days=2) - try: - self.validate_pb( - backup_dir, 'node', - options=["--time={0}".format(unreal_time_2), "-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of validation to unreal time.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: Not enough WAL records to time' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + error_result = self.pb.validate(instance_name, + options=["--time={0}".format(unreal_time_2), "-j", "4"], + expect_error=True) + self.assertMessage(error_result, contains='ERROR: Not enough WAL records to time') # Validate to real xid - target_xid = None with node.connect("postgres") as con: res = con.execute( "INSERT INTO tbl0005 VALUES ('inserted') RETURNING (xmin)") @@ -151,59 +123,25 @@ def test_validate_wal_unreal_values(self): self.switch_wal_segment(node) time.sleep(5) - self.assertIn( - "INFO: Backup validation completed successfully", - self.validate_pb( - backup_dir, 'node', options=["--xid={0}".format(target_xid), - "-j", "4"]), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + output = self.pb.validate(instance_name, + options=["--xid={0}".format(target_xid), "-j", "4"]) + self.assertMessage(output, contains="INFO: Backup validation completed successfully") # Validate to unreal xid unreal_xid = int(target_xid) + 1000 - try: - self.validate_pb( - backup_dir, 'node', options=["--xid={0}".format(unreal_xid), - "-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of validation to unreal xid.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: Not enough WAL records to xid' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + error_result = self.pb.validate(instance_name, + options=["--xid={0}".format(unreal_xid), "-j", "4"], + expect_error=True) + self.assertMessage(error_result, contains='ERROR: Not enough WAL records to xid') # Validate with backup ID - output = self.validate_pb(backup_dir, 'node', backup_id, + output = self.pb.validate(instance_name, backup_id, options=["-j", "4"]) - self.assertIn( - "INFO: Validating backup {0}".format(backup_id), - output, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - self.assertIn( - "INFO: Backup {0} data files are valid".format(backup_id), - output, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - self.assertIn( - "INFO: Backup {0} WAL segments are valid".format(backup_id), - output, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - self.assertIn( - "INFO: Backup {0} is valid".format(backup_id), - output, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) - self.assertIn( - "INFO: Validate of backup {0} completed".format(backup_id), - output, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + self.assertMessage(output, contains=f"INFO: Validating backup {backup_id}") + self.assertMessage(output, contains=f"INFO: Backup {backup_id} data files are valid") + self.assertMessage(output, contains=f"INFO: Backup {backup_id} WAL segments are valid") + self.assertMessage(output, contains=f"INFO: Backup {backup_id} is valid") + self.assertMessage(output, contains=f"INFO: Validate of backup {backup_id} completed") # @unittest.skip("skip") def test_basic_validate_corrupted_intermediate_backup(self): @@ -213,18 +151,17 @@ def test_basic_validate_corrupted_intermediate_backup(self): run validate on PAGE1, expect PAGE1 to gain status CORRUPT and PAGE2 gain status ORPHAN """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + instance_name = 'node' + node = self.pg_node.make_simple(instance_name) + - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance(instance_name, node) + self.pb.set_archiving(instance_name, node) node.slow_start() # FULL - backup_id_1 = self.backup_node(backup_dir, 'node', node) + backup_id_full = self.pb.backup_node(instance_name, node) node.safe_psql( "postgres", @@ -235,8 +172,8 @@ def test_basic_validate_corrupted_intermediate_backup(self): "postgres", "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # PAGE1 - backup_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_page = self.pb.backup_node( + instance_name, node, backup_type='page') node.safe_psql( "postgres", @@ -244,47 +181,25 @@ def test_basic_validate_corrupted_intermediate_backup(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(10000,20000) i") # PAGE2 - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_page_2 = self.pb.backup_node( + instance_name, node, backup_type='page') # Corrupt some file - file = os.path.join( - backup_dir, 'backups', 'node', - backup_id_2, 'database', file_path) - with open(file, "r+b", 0) as f: - f.seek(42) - f.write(b"blah") - f.flush() - f.close + self.corrupt_backup_file(self.backup_dir, instance_name, backup_id_page, + f'database/{file_path}', damage=(42, b"blah")) # Simple validate - try: - self.validate_pb( - backup_dir, 'node', backup_id=backup_id_2, options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data files corruption.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'INFO: Validating parents for backup {0}'.format( - backup_id_2) in e.message and - 'ERROR: Backup {0} is corrupt'.format( - backup_id_2) in e.message and - 'WARNING: Backup {0} data files are corrupted'.format( - backup_id_2) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + error_result = self.pb.validate(instance_name, backup_id=backup_id_page, + options=["-j", "4"], expect_error=True) + self.assertMessage(error_result, contains=f'INFO: Validating parents for backup {backup_id_page}') + self.assertMessage(error_result, contains=f'ERROR: Backup {backup_id_page} is corrupt') + self.assertMessage(error_result, contains=f'WARNING: Backup {backup_id_page} data files are corrupted') - self.assertEqual( - 'CORRUPT', - self.show_pb(backup_dir, 'node', backup_id_2)['status'], - 'Backup STATUS should be "CORRUPT"') - self.assertEqual( - 'ORPHAN', - self.show_pb(backup_dir, 'node', backup_id_3)['status'], - 'Backup STATUS should be "ORPHAN"') + page_backup_status = self.pb.show(instance_name, backup_id_page)['status'] + self.assertEqual('CORRUPT', page_backup_status, 'Backup STATUS should be "CORRUPT"') + + second_page_backup_status = self.pb.show(instance_name, backup_id_page_2)['status'] + self.assertEqual('ORPHAN', second_page_backup_status, 'Backup STATUS should be "ORPHAN"') # @unittest.skip("skip") def test_validate_corrupted_intermediate_backups(self): @@ -294,14 +209,13 @@ def test_validate_corrupted_intermediate_backups(self): expect FULL and PAGE1 to gain status CORRUPT and PAGE2 gain status ORPHAN """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + instance_name = 'node' + node = self.pg_node.make_simple(instance_name) + - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance(instance_name, node) + self.pb.set_archiving(instance_name, node) node.slow_start() node.safe_psql( @@ -313,7 +227,7 @@ def test_validate_corrupted_intermediate_backups(self): "postgres", "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # FULL - backup_id_1 = self.backup_node(backup_dir, 'node', node) + backup_id_1 = self.pb.backup_node(instance_name, node) node.safe_psql( "postgres", @@ -324,8 +238,8 @@ def test_validate_corrupted_intermediate_backups(self): "postgres", "select pg_relation_filepath('t_heap_1')").decode('utf-8').rstrip() # PAGE1 - backup_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_2 = self.pb.backup_node( + instance_name, node, backup_type='page') node.safe_psql( "postgres", @@ -333,74 +247,39 @@ def test_validate_corrupted_intermediate_backups(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(20000,30000) i") # PAGE2 - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_3 = self.pb.backup_node( + instance_name, node, backup_type='page') # Corrupt some file in FULL backup - file_full = os.path.join( - backup_dir, 'backups', 'node', - backup_id_1, 'database', file_path_t_heap) - with open(file_full, "rb+", 0) as f: - f.seek(84) - f.write(b"blah") - f.flush() - f.close + self.corrupt_backup_file(self.backup_dir, instance_name, backup_id_1, + f'database/{file_path_t_heap}', + damage=(84, b"blah")) # Corrupt some file in PAGE1 backup - file_page1 = os.path.join( - backup_dir, 'backups', 'node', - backup_id_2, 'database', file_path_t_heap_1) - with open(file_page1, "rb+", 0) as f: - f.seek(42) - f.write(b"blah") - f.flush() - f.close + self.corrupt_backup_file(self.backup_dir, instance_name, backup_id_2, + f'database/{file_path_t_heap_1}', + damage=(42, b"blah")) # Validate PAGE1 - try: - self.validate_pb( - backup_dir, 'node', backup_id=backup_id_2, options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data files corruption.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'INFO: Validating parents for backup {0}'.format( - backup_id_2) in e.message, - '\n Unexpected Error Message: {0}\n ' - 'CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_1) in e.message and - 'WARNING: Invalid CRC of backup file' in e.message and - 'WARNING: Backup {0} data files are corrupted'.format( - backup_id_1) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Backup {0} is orphaned because his parent'.format( - backup_id_2) in e.message and - 'WARNING: Backup {0} is orphaned because his parent'.format( - backup_id_3) in e.message and - 'ERROR: Backup {0} is orphan.'.format( - backup_id_2) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual( - 'CORRUPT', - self.show_pb(backup_dir, 'node', backup_id_1)['status'], + error_result = self.pb.validate(instance_name, backup_id=backup_id_2, + options=["-j", "4"], expect_error=True) + self.assertMessage(error_result, contains=f'INFO: Validating parents for backup {backup_id_2}') + self.assertMessage(error_result, contains=f'INFO: Validating backup {backup_id_1}') + self.assertMessage(error_result, contains=f'WARNING: Invalid CRC of backup file') + self.assertMessage(error_result, contains=f'WARNING: Backup {backup_id_1} data files are corrupted') + self.assertMessage(error_result, contains=f'WARNING: Backup {backup_id_2} is orphaned because his parent') + self.assertMessage(error_result, contains=f'WARNING: Backup {backup_id_3} is orphaned because his parent') + self.assertMessage(error_result, contains=f'ERROR: Backup {backup_id_2} is orphan.') + self.assertEqual('CORRUPT', + self.pb.show(instance_name, backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') self.assertEqual( 'ORPHAN', - self.show_pb(backup_dir, 'node', backup_id_2)['status'], + self.pb.show(instance_name, backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') self.assertEqual( 'ORPHAN', - self.show_pb(backup_dir, 'node', backup_id_3)['status'], + self.pb.show(instance_name, backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') # @unittest.skip("skip") @@ -412,79 +291,49 @@ def test_validate_specific_error_intermediate_backups(self): purpose of this test is to be sure that not only CORRUPT backup descendants can be orphanized """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + instance_name = 'node' + node = self.pg_node.make_simple(instance_name) + - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance(instance_name, node) + self.pb.set_archiving(instance_name, node) node.slow_start() # FULL - backup_id_1 = self.backup_node(backup_dir, 'node', node) + backup_id_1 = self.pb.backup_node(instance_name, node) # PAGE1 - backup_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_2 = self.pb.backup_node( + instance_name, node, backup_type='page') # PAGE2 - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_3 = self.pb.backup_node( + instance_name, node, backup_type='page') # Change FULL backup status to ERROR - control_path = os.path.join( - backup_dir, 'backups', 'node', backup_id_1, 'backup.control') - - with open(control_path, 'r') as f: - actual_control = f.read() - - new_control_file = '' - for line in actual_control.splitlines(): - new_control_file += line.replace( - 'status = OK', 'status = ERROR') - new_control_file += '\n' - - with open(control_path, 'wt') as f: - f.write(new_control_file) - f.flush() - f.close() + self.change_backup_status(self.backup_dir, instance_name, backup_id_1, 'ERROR') # Validate PAGE1 - try: - self.validate_pb( - backup_dir, 'node', backup_id=backup_id_2, options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because backup has status ERROR.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'WARNING: Backup {0} is orphaned because ' - 'his parent {1} has status: ERROR'.format( - backup_id_2, backup_id_1) in e.message and - 'INFO: Validating parents for backup {0}'.format( - backup_id_2) in e.message and - 'WARNING: Backup {0} has status ERROR. Skip validation.'.format( - backup_id_1) and - 'ERROR: Backup {0} is orphan.'.format(backup_id_2) in e.message, - '\n Unexpected Error Message: {0}\n ' - 'CMD: {1}'.format( - repr(e.message), self.cmd)) + error_message = self.pb.validate(instance_name, backup_id=backup_id_2, options=["-j", "4"], + expect_error=True) + self.assertMessage(error_message, contains=f'WARNING: Backup {backup_id_2} is orphaned because his parent {backup_id_1} has status: ERROR') + self.assertMessage(error_message, contains=f'INFO: Validating parents for backup {backup_id_2}') + self.assertMessage(error_message, contains=f'WARNING: Backup {backup_id_1} has status ERROR. Skip validation.') + self.assertMessage(error_message, contains=f'ERROR: Backup {backup_id_2} is orphan.') + self.assertMessage(error_message, contains=f'ERROR: Backup {backup_id_2} is orphan.') self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node', backup_id_1)['status'], + self.pb.show(instance_name, backup_id_1)['status'], 'Backup STATUS should be "ERROR"') self.assertEqual( 'ORPHAN', - self.show_pb(backup_dir, 'node', backup_id_2)['status'], + self.pb.show(instance_name, backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') self.assertEqual( 'ORPHAN', - self.show_pb(backup_dir, 'node', backup_id_3)['status'], + self.pb.show(instance_name, backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') # @unittest.skip("skip") @@ -496,75 +345,45 @@ def test_validate_error_intermediate_backups(self): purpose of this test is to be sure that not only CORRUPT backup descendants can be orphanized """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + instance_name = 'node' + node = self.pg_node.make_simple(instance_name) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + + self.pb.init() + self.pb.add_instance(instance_name, node) + self.pb.set_archiving(instance_name, node) node.slow_start() # FULL - backup_id_1 = self.backup_node(backup_dir, 'node', node) + backup_id_1 = self.pb.backup_node(instance_name, node) # PAGE1 - backup_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_2 = self.pb.backup_node( + instance_name, node, backup_type='page') # PAGE2 - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_3 = self.pb.backup_node( + instance_name, node, backup_type='page') # Change FULL backup status to ERROR - control_path = os.path.join( - backup_dir, 'backups', 'node', backup_id_1, 'backup.control') - - with open(control_path, 'r') as f: - actual_control = f.read() - - new_control_file = '' - for line in actual_control.splitlines(): - new_control_file += line.replace( - 'status = OK', 'status = ERROR') - new_control_file += '\n' - - with open(control_path, 'wt') as f: - f.write(new_control_file) - f.flush() - f.close() + self.change_backup_status(self.backup_dir, instance_name, backup_id_1, 'ERROR') # Validate instance - try: - self.validate_pb(backup_dir, options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because backup has status ERROR.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "WARNING: Backup {0} is orphaned because " - "his parent {1} has status: ERROR".format( - backup_id_2, backup_id_1) in e.message and - 'WARNING: Backup {0} has status ERROR. Skip validation'.format( - backup_id_1) in e.message and - "WARNING: Some backups are not valid" in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + error_message = self.pb.validate(options=["-j", "4"], expect_error=True) + self.assertMessage(error_message, contains=f'WARNING: Backup {backup_id_2} is orphaned because his parent {backup_id_1} has status: ERROR') + self.assertMessage(error_message, contains=f'WARNING: Backup {backup_id_1} has status ERROR. Skip validation') self.assertEqual( 'ERROR', - self.show_pb(backup_dir, 'node', backup_id_1)['status'], + self.pb.show(instance_name, backup_id_1)['status'], 'Backup STATUS should be "ERROR"') self.assertEqual( 'ORPHAN', - self.show_pb(backup_dir, 'node', backup_id_2)['status'], + self.pb.show(instance_name, backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') self.assertEqual( 'ORPHAN', - self.show_pb(backup_dir, 'node', backup_id_3)['status'], + self.pb.show(instance_name, backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') # @unittest.skip("skip") @@ -575,18 +394,16 @@ def test_validate_corrupted_intermediate_backups_1(self): expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 to gain status ORPHAN """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') + - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL1 - backup_id_1 = self.backup_node(backup_dir, 'node', node) + backup_id_1 = self.pb.backup_node('node', node) # PAGE1 node.safe_psql( @@ -594,8 +411,8 @@ def test_validate_corrupted_intermediate_backups_1(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,10000) i") - backup_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_2 = self.pb.backup_node( + 'node', node, backup_type='page') # PAGE2 node.safe_psql( @@ -606,8 +423,8 @@ def test_validate_corrupted_intermediate_backups_1(self): file_page_2 = node.safe_psql( "postgres", "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_3 = self.pb.backup_node( + 'node', node, backup_type='page') # PAGE3 node.safe_psql( @@ -615,8 +432,8 @@ def test_validate_corrupted_intermediate_backups_1(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(10000,20000) i") - backup_id_4 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_4 = self.pb.backup_node( + 'node', node, backup_type='page') # PAGE4 node.safe_psql( @@ -624,8 +441,8 @@ def test_validate_corrupted_intermediate_backups_1(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(20000,30000) i") - backup_id_5 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_5 = self.pb.backup_node( + 'node', node, backup_type='page') # PAGE5 node.safe_psql( @@ -636,8 +453,8 @@ def test_validate_corrupted_intermediate_backups_1(self): file_page_5 = node.safe_psql( "postgres", "select pg_relation_filepath('t_heap1')").decode('utf-8').rstrip() - backup_id_6 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_6 = self.pb.backup_node( + 'node', node, backup_type='page') # PAGE6 node.safe_psql( @@ -645,119 +462,64 @@ def test_validate_corrupted_intermediate_backups_1(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(30000,40000) i") - backup_id_7 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_7 = self.pb.backup_node( + 'node', node, backup_type='page') # FULL2 - backup_id_8 = self.backup_node(backup_dir, 'node', node) + backup_id_8 = self.pb.backup_node('node', node) # Corrupt some file in PAGE2 and PAGE5 backups - file_page1 = os.path.join( - backup_dir, 'backups', 'node', backup_id_3, 'database', file_page_2) - with open(file_page1, "rb+", 0) as f: - f.seek(84) - f.write(b"blah") - f.flush() - f.close + self.corrupt_backup_file(self.backup_dir, 'node', backup_id_3, + f'database/{file_page_2}', + damage=(84, b"blah")) - file_page4 = os.path.join( - backup_dir, 'backups', 'node', backup_id_6, 'database', file_page_5) - with open(file_page4, "rb+", 0) as f: - f.seek(42) - f.write(b"blah") - f.flush() - f.close + self.corrupt_backup_file(self.backup_dir, 'node', backup_id_6, + f'database/{file_page_5}', + damage=(42, b"blah")) # Validate PAGE3 - try: - self.validate_pb( - backup_dir, 'node', - backup_id=backup_id_4, options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data files corruption.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'INFO: Validating parents for backup {0}'.format( - backup_id_4) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_1) in e.message and - 'INFO: Backup {0} data files are valid'.format( - backup_id_1) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_2) in e.message and - 'INFO: Backup {0} data files are valid'.format( - backup_id_2) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_3) in e.message and - 'WARNING: Invalid CRC of backup file' in e.message and - 'WARNING: Backup {0} data files are corrupted'.format( - backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Backup {0} is orphaned because ' - 'his parent {1} has status: CORRUPT'.format( - backup_id_4, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Backup {0} is orphaned because ' - 'his parent {1} has status: CORRUPT'.format( - backup_id_5, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Backup {0} is orphaned because ' - 'his parent {1} has status: CORRUPT'.format( - backup_id_6, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Backup {0} is orphaned because ' - 'his parent {1} has status: CORRUPT'.format( - backup_id_7, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'ERROR: Backup {0} is orphan'.format(backup_id_4) in e.message, - '\n Unexpected Error Message: {0}\n ' - 'CMD: {1}'.format(repr(e.message), self.cmd)) + self.pb.validate('node', + backup_id=backup_id_4, options=["-j", "4"], + expect_error="because of data files corruption") + + self.assertMessage(contains=f'INFO: Validating parents for backup {backup_id_4}') + self.assertMessage(contains=f'INFO: Validating backup {backup_id_1}') + self.assertMessage(contains=f'INFO: Backup {backup_id_1} data files are valid') + self.assertMessage(contains=f'INFO: Backup {backup_id_1} data files are valid') + self.assertMessage(contains=f'INFO: Validating backup {backup_id_2}') + self.assertMessage(contains=f'INFO: Backup {backup_id_2} data files are valid') + self.assertMessage(contains=f'INFO: Validating backup {backup_id_3}') + self.assertMessage(contains=f'WARNING: Invalid CRC of backup file') + self.assertMessage(contains=f'WARNING: Backup {backup_id_3} data files are corrupted') + self.assertMessage(contains=f'WARNING: Backup {backup_id_4} is orphaned because his parent {backup_id_3} has status: CORRUPT') + self.assertMessage(contains=f'WARNING: Backup {backup_id_5} is orphaned because his parent {backup_id_3} has status: CORRUPT') + self.assertMessage(contains=f'WARNING: Backup {backup_id_6} is orphaned because his parent {backup_id_3} has status: CORRUPT') + self.assertMessage(contains=f'WARNING: Backup {backup_id_7} is orphaned because his parent {backup_id_3} has status: CORRUPT') + self.assertMessage(contains=f'ERROR: Backup {backup_id_4} is orphan') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'OK', self.pb.show('node', backup_id_1)['status'], 'Backup STATUS should be "OK"') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'OK', self.pb.show('node', backup_id_2)['status'], 'Backup STATUS should be "OK"') self.assertEqual( - 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'CORRUPT', self.pb.show('node', backup_id_3)['status'], 'Backup STATUS should be "CORRUPT"') self.assertEqual( - 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'ORPHAN', self.pb.show('node', backup_id_4)['status'], 'Backup STATUS should be "ORPHAN"') self.assertEqual( - 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'ORPHAN', self.pb.show('node', backup_id_5)['status'], 'Backup STATUS should be "ORPHAN"') self.assertEqual( - 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_6)['status'], + 'ORPHAN', self.pb.show('node', backup_id_6)['status'], 'Backup STATUS should be "ORPHAN"') self.assertEqual( - 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'], + 'ORPHAN', self.pb.show('node', backup_id_7)['status'], 'Backup STATUS should be "ORPHAN"') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], + 'OK', self.pb.show('node', backup_id_8)['status'], 'Backup STATUS should be "OK"') # @unittest.skip("skip") @@ -768,18 +530,16 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): expect PAGE1 to gain status CORRUPT, PAGE2, PAGE3, PAGE4 and PAGE5 to gain status ORPHAN """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL1 - backup_id_1 = self.backup_node(backup_dir, 'node', node) + backup_id_1 = self.pb.backup_node('node', node) # PAGE1 node.safe_psql( @@ -787,8 +547,8 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): "create table t_heap as select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,10000) i") - backup_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_2 = self.pb.backup_node( + 'node', node, backup_type='page') # PAGE2 node.safe_psql( @@ -799,8 +559,8 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): file_page_2 = node.safe_psql( "postgres", "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_3 = self.pb.backup_node( + 'node', node, backup_type='page') # PAGE3 node.safe_psql( @@ -808,8 +568,8 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(10000,20000) i") - backup_id_4 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_4 = self.pb.backup_node( + 'node', node, backup_type='page') # PAGE4 node.safe_psql( @@ -824,8 +584,8 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(30001, 30001) i RETURNING (xmin)").decode('utf-8').rstrip() - backup_id_5 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_5 = self.pb.backup_node( + 'node', node, backup_type='page') # PAGE5 node.safe_psql( @@ -836,8 +596,8 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): file_page_5 = node.safe_psql( "postgres", "select pg_relation_filepath('t_heap1')").decode('utf-8').rstrip() - backup_id_6 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_6 = self.pb.backup_node( + 'node', node, backup_type='page') # PAGE6 node.safe_psql( @@ -845,108 +605,51 @@ def test_validate_specific_target_corrupted_intermediate_backups(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(30000,40000) i") - backup_id_7 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_7 = self.pb.backup_node( + 'node', node, backup_type='page') # FULL2 - backup_id_8 = self.backup_node(backup_dir, 'node', node) + backup_id_8 = self.pb.backup_node('node', node) # Corrupt some file in PAGE2 and PAGE5 backups - file_page1 = os.path.join( - backup_dir, 'backups', 'node', - backup_id_3, 'database', file_page_2) - with open(file_page1, "rb+", 0) as f: - f.seek(84) - f.write(b"blah") - f.flush() - f.close - - file_page4 = os.path.join( - backup_dir, 'backups', 'node', - backup_id_6, 'database', file_page_5) - with open(file_page4, "rb+", 0) as f: - f.seek(42) - f.write(b"blah") - f.flush() - f.close + self.corrupt_backup_file(self.backup_dir, 'node', backup_id_3, + f'database/{file_page_2}', + damage=(84, b"blah")) + self.corrupt_backup_file(self.backup_dir, 'node', backup_id_6, + f'database/{file_page_5}', + damage=(42, b"blah")) # Validate PAGE3 - try: - self.validate_pb( - backup_dir, 'node', - options=[ - '-i', backup_id_4, '--xid={0}'.format(target_xid), "-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data files corruption.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'INFO: Validating parents for backup {0}'.format( - backup_id_4) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_1) in e.message and - 'INFO: Backup {0} data files are valid'.format( - backup_id_1) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_2) in e.message and - 'INFO: Backup {0} data files are valid'.format( - backup_id_2) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_3) in e.message and - 'WARNING: Invalid CRC of backup file' in e.message and - 'WARNING: Backup {0} data files are corrupted'.format( - backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Backup {0} is orphaned because his ' - 'parent {1} has status: CORRUPT'.format( - backup_id_4, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Backup {0} is orphaned because his ' - 'parent {1} has status: CORRUPT'.format( - backup_id_5, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Backup {0} is orphaned because his ' - 'parent {1} has status: CORRUPT'.format( - backup_id_6, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Backup {0} is orphaned because his ' - 'parent {1} has status: CORRUPT'.format( - backup_id_7, backup_id_3) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'ERROR: Backup {0} is orphan'.format( - backup_id_4) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "OK"') - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "OK"') - self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "CORRUPT"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_6)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_7)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_8)['status'], 'Backup STATUS should be "OK"') + self.pb.validate('node', + options=['-i', backup_id_4, '--xid', target_xid, "-j", "4"], + expect_error="because of data files corruption") + + self.assertMessage(contains=f'INFO: Validating parents for backup {backup_id_4}') + self.assertMessage(contains=f'INFO: Validating backup {backup_id_1}') + self.assertMessage(contains=f'INFO: Backup {backup_id_1} data files are valid') + self.assertMessage(contains=f'INFO: Validating backup {backup_id_2}') + self.assertMessage(contains=f'INFO: Backup {backup_id_2} data files are valid') + self.assertMessage(contains=f'INFO: Validating backup {backup_id_3}') + self.assertMessage(contains='WARNING: Invalid CRC of backup file') + self.assertMessage(contains=f'WARNING: Backup {backup_id_3} data files are corrupted') + self.assertMessage(contains=f'WARNING: Backup {backup_id_4} is orphaned because his ' + f'parent {backup_id_3} has status: CORRUPT') + self.assertMessage(contains=f'WARNING: Backup {backup_id_5} is orphaned because his ' + f'parent {backup_id_3} has status: CORRUPT') + self.assertMessage(contains=f'WARNING: Backup {backup_id_6} is orphaned because his ' + f'parent {backup_id_3} has status: CORRUPT') + self.assertMessage(contains=f'WARNING: Backup {backup_id_7} is orphaned because his ' + f'parent {backup_id_3} has status: CORRUPT') + self.assertMessage(contains=f'ERROR: Backup {backup_id_4} is orphan') + + self.assertEqual('OK', self.pb.show('node', backup_id_1)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.pb.show('node', backup_id_2)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('CORRUPT', self.pb.show('node', backup_id_3)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.pb.show('node', backup_id_4)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.pb.show('node', backup_id_5)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.pb.show('node', backup_id_6)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.pb.show('node', backup_id_7)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.pb.show('node', backup_id_8)['status'], 'Backup STATUS should be "OK"') # @unittest.skip("skip") def test_validate_instance_with_several_corrupt_backups(self): @@ -956,25 +659,23 @@ def test_validate_instance_with_several_corrupt_backups(self): expect FULL1 to gain status CORRUPT, PAGE1_1 to gain status ORPHAN FULL2 to gain status CORRUPT, PAGE2_1 to gain status ORPHAN """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( "postgres", "create table t_heap as select generate_series(0,1) i") # FULL1 - backup_id_1 = self.backup_node( - backup_dir, 'node', node, options=['--no-validate']) + backup_id_1 = self.pb.backup_node( + 'node', node, options=['--no-validate']) # FULL2 - backup_id_2 = self.backup_node(backup_dir, 'node', node) + backup_id_2 = self.pb.backup_node('node', node) rel_path = node.safe_psql( "postgres", "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() @@ -983,94 +684,78 @@ def test_validate_instance_with_several_corrupt_backups(self): "postgres", "insert into t_heap values(2)") - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_3 = self.pb.backup_node( + 'node', node, backup_type='page') # FULL3 - backup_id_4 = self.backup_node(backup_dir, 'node', node) + backup_id_4 = self.pb.backup_node('node', node) node.safe_psql( "postgres", "insert into t_heap values(3)") - backup_id_5 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_5 = self.pb.backup_node( + 'node', node, backup_type='page') # FULL4 - backup_id_6 = self.backup_node( - backup_dir, 'node', node, options=['--no-validate']) + backup_id_6 = self.pb.backup_node( + 'node', node, options=['--no-validate']) # Corrupt some files in FULL2 and FULL3 backup - os.remove(os.path.join( - backup_dir, 'backups', 'node', backup_id_2, - 'database', rel_path)) - os.remove(os.path.join( - backup_dir, 'backups', 'node', backup_id_4, - 'database', rel_path)) + self.remove_backup_file(self.backup_dir, 'node', backup_id_2, + f'database/{rel_path}') + self.remove_backup_file(self.backup_dir, 'node', backup_id_4, + f'database/{rel_path}') # Validate Instance - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4", "--log-level-file=LOG"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data files corruption.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "INFO: Validate backups of the instance 'node'" in e.message, - "\n Unexpected Error Message: {0}\n " - "CMD: {1}".format(repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Some backups are not valid' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.validate('node', options=["-j", "4", "--log-level-file=LOG"], + expect_error="because of data files corruption") + self.assertMessage(contains="INFO: Validate backups of the instance 'node'") + self.assertMessage(contains='WARNING: Some backups are not valid') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'OK', self.pb.show('node', backup_id_1)['status'], 'Backup STATUS should be "OK"') self.assertEqual( - 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'CORRUPT', self.pb.show('node', backup_id_2)['status'], 'Backup STATUS should be "CORRUPT"') self.assertEqual( - 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'ORPHAN', self.pb.show('node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') self.assertEqual( - 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'CORRUPT', self.pb.show('node', backup_id_4)['status'], 'Backup STATUS should be "CORRUPT"') self.assertEqual( - 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'ORPHAN', self.pb.show('node', backup_id_5)['status'], 'Backup STATUS should be "ORPHAN"') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_6)['status'], + 'OK', self.pb.show('node', backup_id_6)['status'], 'Backup STATUS should be "OK"') # @unittest.skip("skip") + @needs_gdb def test_validate_instance_with_several_corrupt_backups_interrupt(self): """ check that interrupt during validation is handled correctly """ - self._check_gdb_flag_or_skip_test() - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') + - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( "postgres", "create table t_heap as select generate_series(0,1) i") # FULL1 - backup_id_1 = self.backup_node( - backup_dir, 'node', node, options=['--no-validate']) + backup_id_1 = self.pb.backup_node( + 'node', node, options=['--no-validate']) # FULL2 - backup_id_2 = self.backup_node(backup_dir, 'node', node) + backup_id_2 = self.pb.backup_node('node', node) rel_path = node.safe_psql( "postgres", "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() @@ -1079,65 +764,60 @@ def test_validate_instance_with_several_corrupt_backups_interrupt(self): "postgres", "insert into t_heap values(2)") - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_3 = self.pb.backup_node( + 'node', node, backup_type='page') # FULL3 - backup_id_4 = self.backup_node(backup_dir, 'node', node) + backup_id_4 = self.pb.backup_node('node', node) node.safe_psql( "postgres", "insert into t_heap values(3)") - backup_id_5 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_5 = self.pb.backup_node( + 'node', node, backup_type='page') # FULL4 - backup_id_6 = self.backup_node( - backup_dir, 'node', node, options=['--no-validate']) + backup_id_6 = self.pb.backup_node( + 'node', node, options=['--no-validate']) # Corrupt some files in FULL2 and FULL3 backup - os.remove(os.path.join( - backup_dir, 'backups', 'node', backup_id_1, - 'database', rel_path)) - os.remove(os.path.join( - backup_dir, 'backups', 'node', backup_id_3, - 'database', rel_path)) + self.remove_backup_file(self.backup_dir, 'node', backup_id_1, + f'database/{rel_path}') + self.remove_backup_file(self.backup_dir, 'node', backup_id_3, + f'database/{rel_path}') # Validate Instance - gdb = self.validate_pb( - backup_dir, 'node', options=["-j", "4", "--log-level-file=LOG"], gdb=True) + gdb = self.pb.validate( + 'node', options=["-j", "4", "--log-level-file=LOG"], gdb=True) gdb.set_breakpoint('validate_file_pages') gdb.run_until_break() gdb.continue_execution_until_break() - gdb.remove_all_breakpoints() - gdb._execute('signal SIGINT') + gdb.signal('SIGINT') gdb.continue_execution_until_error() self.assertEqual( - 'DONE', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'DONE', self.pb.show('node', backup_id_1)['status'], 'Backup STATUS should be "OK"') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'OK', self.pb.show('node', backup_id_2)['status'], 'Backup STATUS should be "OK"') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'OK', self.pb.show('node', backup_id_3)['status'], 'Backup STATUS should be "CORRUPT"') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'OK', self.pb.show('node', backup_id_4)['status'], 'Backup STATUS should be "ORPHAN"') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'OK', self.pb.show('node', backup_id_5)['status'], 'Backup STATUS should be "OK"') self.assertEqual( - 'DONE', self.show_pb(backup_dir, 'node', backup_id_6)['status'], + 'DONE', self.pb.show('node', backup_id_6)['status'], 'Backup STATUS should be "OK"') - log_file = os.path.join(backup_dir, 'log', 'pg_probackup.log') - with open(log_file, 'r') as f: - log_content = f.read() - self.assertNotIn( + log_content = self.read_pb_log() + self.assertNotIn( 'Interrupted while locking backup', log_content) # @unittest.skip("skip") @@ -1147,14 +827,12 @@ def test_validate_instance_with_corrupted_page(self): corrupt file in PAGE1 backup and run validate on instance, expect PAGE1 to gain status CORRUPT, PAGE2 to gain status ORPHAN """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -1163,7 +841,7 @@ def test_validate_instance_with_corrupted_page(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,10000) i") # FULL1 - backup_id_1 = self.backup_node(backup_dir, 'node', node) + backup_id_1 = self.pb.backup_node('node', node) node.safe_psql( "postgres", @@ -1174,8 +852,8 @@ def test_validate_instance_with_corrupted_page(self): "postgres", "select pg_relation_filepath('t_heap1')").decode('utf-8').rstrip() # PAGE1 - backup_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_2 = self.pb.backup_node( + 'node', node, backup_type='page') node.safe_psql( "postgres", @@ -1183,104 +861,59 @@ def test_validate_instance_with_corrupted_page(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(20000,30000) i") # PAGE2 - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_3 = self.pb.backup_node( + 'node', node, backup_type='page') # FULL1 - backup_id_4 = self.backup_node( - backup_dir, 'node', node) + backup_id_4 = self.pb.backup_node( + 'node', node) # PAGE3 - backup_id_5 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_5 = self.pb.backup_node( + 'node', node, backup_type='page') # Corrupt some file in FULL backup - file_full = os.path.join( - backup_dir, 'backups', 'node', backup_id_2, - 'database', file_path_t_heap1) - with open(file_full, "rb+", 0) as f: - f.seek(84) - f.write(b"blah") - f.flush() - f.close + self.corrupt_backup_file(self.backup_dir, 'node', backup_id_2, + f'database/{file_path_t_heap1}', + damage=(84, b"blah")) # Validate Instance - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data files corruption.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "INFO: Validate backups of the instance 'node'" in e.message, - "\n Unexpected Error Message: {0}\n " - "CMD: {1}".format(repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_5) in e.message and - 'INFO: Backup {0} data files are valid'.format( - backup_id_5) in e.message and - 'INFO: Backup {0} WAL segments are valid'.format( - backup_id_5) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_4) in e.message and - 'INFO: Backup {0} data files are valid'.format( - backup_id_4) in e.message and - 'INFO: Backup {0} WAL segments are valid'.format( - backup_id_4) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_3) in e.message and - 'INFO: Backup {0} data files are valid'.format( - backup_id_3) in e.message and - 'INFO: Backup {0} WAL segments are valid'.format( - backup_id_3) in e.message and - 'WARNING: Backup {0} is orphaned because ' - 'his parent {1} has status: CORRUPT'.format( - backup_id_3, backup_id_2) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_2) in e.message and - 'WARNING: Invalid CRC of backup file' in e.message and - 'WARNING: Backup {0} data files are corrupted'.format( - backup_id_2) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'INFO: Validating backup {0}'.format( - backup_id_1) in e.message and - 'INFO: Backup {0} data files are valid'.format( - backup_id_1) in e.message and - 'INFO: Backup {0} WAL segments are valid'.format( - backup_id_1) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertTrue( - 'WARNING: Some backups are not valid' in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.validate('node', options=["-j", "4"], + expect_error="because of data files corruption") + self.assertMessage(contains="INFO: Validate backups of the instance 'node'") + self.assertMessage(contains=f'INFO: Validating backup {backup_id_5}') + self.assertMessage(contains=f'INFO: Backup {backup_id_5} data files are valid') + self.assertMessage(contains=f'INFO: Backup {backup_id_5} WAL segments are valid') + self.assertMessage(contains=f'INFO: Validating backup {backup_id_4}') + self.assertMessage(contains=f'INFO: Backup {backup_id_4} data files are valid') + self.assertMessage(contains=f'INFO: Backup {backup_id_4} WAL segments are valid') + self.assertMessage(contains=f'INFO: Validating backup {backup_id_3}') + self.assertMessage(contains=f'INFO: Backup {backup_id_3} data files are valid') + self.assertMessage(contains=f'INFO: Backup {backup_id_3} WAL segments are valid') + + self.assertMessage(contains=f'WARNING: Backup {backup_id_3} is orphaned because ' + f'his parent {backup_id_2} has status: CORRUPT') + self.assertMessage(contains=f'INFO: Validating backup {backup_id_2}') + self.assertMessage(contains='WARNING: Invalid CRC of backup file') + self.assertMessage(contains=f'WARNING: Backup {backup_id_2} data files are corrupted') + + self.assertMessage(contains=f'INFO: Validating backup {backup_id_1}') + self.assertMessage(contains=f'INFO: Backup {backup_id_1} data files are valid') + self.assertMessage(contains=f'INFO: Backup {backup_id_1} WAL segments are valid') + self.assertMessage(contains='WARNING: Some backups are not valid') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_1)['status'], + 'OK', self.pb.show('node', backup_id_1)['status'], 'Backup STATUS should be "OK"') self.assertEqual( - 'CORRUPT', self.show_pb(backup_dir, 'node', backup_id_2)['status'], + 'CORRUPT', self.pb.show('node', backup_id_2)['status'], 'Backup STATUS should be "CORRUPT"') self.assertEqual( - 'ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], + 'ORPHAN', self.pb.show('node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], + 'OK', self.pb.show('node', backup_id_4)['status'], 'Backup STATUS should be "OK"') self.assertEqual( - 'OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], + 'OK', self.pb.show('node', backup_id_5)['status'], 'Backup STATUS should be "OK"') # @unittest.skip("skip") @@ -1289,13 +922,12 @@ def test_validate_instance_with_corrupted_full_and_try_restore(self): corrupt file in FULL backup and run validate on instance, expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN, try to restore backup with --no-validation option""" - node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') + - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -1307,7 +939,7 @@ def test_validate_instance_with_corrupted_full_and_try_restore(self): "postgres", "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # FULL1 - backup_id_1 = self.backup_node(backup_dir, 'node', node) + backup_id_1 = self.pb.backup_node('node', node) node.safe_psql( "postgres", @@ -1315,7 +947,7 @@ def test_validate_instance_with_corrupted_full_and_try_restore(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,10000) i") # PAGE1 - backup_id_2 = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id_2 = self.pb.backup_node('node', node, backup_type='page') # PAGE2 node.safe_psql( @@ -1323,10 +955,10 @@ def test_validate_instance_with_corrupted_full_and_try_restore(self): "insert into t_heap select i as id, md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(20000,30000) i") - backup_id_3 = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id_3 = self.pb.backup_node('node', node, backup_type='page') # FULL1 - backup_id_4 = self.backup_node(backup_dir, 'node', node) + backup_id_4 = self.pb.backup_node('node', node) # PAGE3 node.safe_psql( @@ -1334,59 +966,44 @@ def test_validate_instance_with_corrupted_full_and_try_restore(self): "insert into t_heap select i as id, " "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(30000,40000) i") - backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id_5 = self.pb.backup_node('node', node, backup_type='page') # Corrupt some file in FULL backup - file_full = os.path.join( - backup_dir, 'backups', 'node', - backup_id_1, 'database', file_path_t_heap) - with open(file_full, "rb+", 0) as f: - f.seek(84) - f.write(b"blah") - f.flush() - f.close + self.corrupt_backup_file(self.backup_dir, 'node', backup_id_1, + f'database/{file_path_t_heap}', + damage=(84, b"blah")) # Validate Instance - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4"]) - self.assertEqual(1, 0, "Expecting Error because of data files corruption.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'INFO: Validating backup {0}'.format(backup_id_1) in e.message - and "INFO: Validate backups of the instance 'node'" in e.message - and 'WARNING: Invalid CRC of backup file' in e.message - and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"') - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') + self.pb.validate('node', options=["-j", "4"], + expect_error="because of data files corruption") + self.assertMessage(contains=f'INFO: Validating backup {backup_id_1}') + self.assertMessage(contains="INFO: Validate backups of the instance 'node'") + self.assertMessage(contains='WARNING: Invalid CRC of backup file') + self.assertMessage(contains=f'WARNING: Backup {backup_id_1} data files are corrupted') + + self.assertEqual('CORRUPT', self.pb.show('node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.pb.show('node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.pb.show('node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.pb.show('node', backup_id_4)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.pb.show('node', backup_id_5)['status'], 'Backup STATUS should be "OK"') node.cleanup() - restore_out = self.restore_node( - backup_dir, 'node', node, + restore_out = self.pb.restore_node( + 'node', node, options=["--no-validate"]) - self.assertIn( - "INFO: Restore of backup {0} completed.".format(backup_id_5), - restore_out, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(self.output), self.cmd)) + self.assertMessage(restore_out, contains="INFO: Restore of backup {0} completed.".format(backup_id_5)) # @unittest.skip("skip") def test_validate_instance_with_corrupted_full(self): """make archive node, take FULL, PAGE1, PAGE2, FULL2, PAGE3 backups, corrupt file in FULL backup and run validate on instance, expect FULL to gain status CORRUPT, PAGE1 and PAGE2 to gain status ORPHAN""" - node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') + - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.safe_psql( @@ -1399,7 +1016,7 @@ def test_validate_instance_with_corrupted_full(self): "postgres", "select pg_relation_filepath('t_heap')").decode('utf-8').rstrip() # FULL1 - backup_id_1 = self.backup_node(backup_dir, 'node', node) + backup_id_1 = self.pb.backup_node('node', node) node.safe_psql( "postgres", @@ -1408,8 +1025,8 @@ def test_validate_instance_with_corrupted_full(self): "from generate_series(0,10000) i") # PAGE1 - backup_id_2 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_2 = self.pb.backup_node( + 'node', node, backup_type='page') # PAGE2 node.safe_psql( @@ -1418,12 +1035,12 @@ def test_validate_instance_with_corrupted_full(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(20000,30000) i") - backup_id_3 = self.backup_node( - backup_dir, 'node', node, backup_type='page') + backup_id_3 = self.pb.backup_node( + 'node', node, backup_type='page') # FULL1 - backup_id_4 = self.backup_node( - backup_dir, 'node', node) + backup_id_4 = self.pb.backup_node( + 'node', node) # PAGE3 node.safe_psql( @@ -1431,115 +1048,84 @@ def test_validate_instance_with_corrupted_full(self): "insert into t_heap select i as id, " "md5(i::text) as text, md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(30000,40000) i") - backup_id_5 = self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id_5 = self.pb.backup_node('node', node, backup_type='page') # Corrupt some file in FULL backup - file_full = os.path.join( - backup_dir, 'backups', 'node', - backup_id_1, 'database', file_path_t_heap) - with open(file_full, "rb+", 0) as f: - f.seek(84) - f.write(b"blah") - f.flush() - f.close + self.corrupt_backup_file(self.backup_dir, 'node', backup_id_1, + f'database/{file_path_t_heap}', + damage=(84, b"blah")) # Validate Instance - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data files corruption.\n " - "Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'INFO: Validating backup {0}'.format(backup_id_1) in e.message - and "INFO: Validate backups of the instance 'node'" in e.message - and 'WARNING: Invalid CRC of backup file' in e.message - and 'WARNING: Backup {0} data files are corrupted'.format(backup_id_1) in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format(repr(e.message), self.cmd)) - - self.assertEqual('CORRUPT', self.show_pb(backup_dir, 'node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('ORPHAN', self.show_pb(backup_dir, 'node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_4)['status'], 'Backup STATUS should be "OK"') - self.assertEqual('OK', self.show_pb(backup_dir, 'node', backup_id_5)['status'], 'Backup STATUS should be "OK"') + self.pb.validate('node', options=["-j", "4"], + expect_error="because of data files corruption") + self.assertMessage(contains=f'INFO: Validating backup {backup_id_1}') + self.assertMessage(contains="INFO: Validate backups of the instance 'node'") + self.assertMessage(contains='WARNING: Invalid CRC of backup file') + self.assertMessage(contains=f'WARNING: Backup {backup_id_1} data files are corrupted') + + self.assertEqual('CORRUPT', self.pb.show('node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') + self.assertEqual('ORPHAN', self.pb.show('node', backup_id_2)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('ORPHAN', self.pb.show('node', backup_id_3)['status'], 'Backup STATUS should be "ORPHAN"') + self.assertEqual('OK', self.pb.show('node', backup_id_4)['status'], 'Backup STATUS should be "OK"') + self.assertEqual('OK', self.pb.show('node', backup_id_5)['status'], 'Backup STATUS should be "OK"') # @unittest.skip("skip") def test_validate_corrupt_wal_1(self): """make archive node, take FULL1, PAGE1,PAGE2,FULL2,PAGE3,PAGE4 backups, corrupt all wal files, run validate, expect errors""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - backup_id_1 = self.backup_node(backup_dir, 'node', node) + backup_id_1 = self.pb.backup_node('node', node) with node.connect("postgres") as con: con.execute("CREATE TABLE tbl0005 (a text)") con.commit() - backup_id_2 = self.backup_node(backup_dir, 'node', node) + backup_id_2 = self.pb.backup_node('node', node) # Corrupt WAL - wals_dir = os.path.join(backup_dir, 'wal', 'node') - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] - wals.sort() - for wal in wals: - with open(os.path.join(wals_dir, wal), "rb+", 0) as f: - f.seek(42) - f.write(b"blablablaadssaaaaaaaaaaaaaaa") - f.flush() - f.close + bla = b"blablablaadssaaaaaaaaaaaaaaa" + for wal in self.get_instance_wal_list(self.backup_dir, 'node'): + self.corrupt_instance_wal(self.backup_dir, 'node', wal, 42, bla) # Simple validate - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of wal segments corruption.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'WARNING: Backup' in e.message and - 'WAL segments are corrupted' in e.message and - "WARNING: There are not enough WAL " - "records to consistenly restore backup" in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.validate('node', options=["-j", "4"], + expect_error="because of wal segments corruption") + self.assertMessage(contains='WARNING: Backup') + self.assertMessage(contains='WAL segments are corrupted') + self.assertMessage(contains="WARNING: There are not enough WAL " + "records to consistenly restore backup") self.assertEqual( 'CORRUPT', - self.show_pb(backup_dir, 'node', backup_id_1)['status'], + self.pb.show('node', backup_id_1)['status'], 'Backup STATUS should be "CORRUPT"') self.assertEqual( 'CORRUPT', - self.show_pb(backup_dir, 'node', backup_id_2)['status'], + self.pb.show('node', backup_id_2)['status'], 'Backup STATUS should be "CORRUPT"') # @unittest.skip("skip") def test_validate_corrupt_wal_2(self): """make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors""" - node = self.make_simple_node(base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() with node.connect("postgres") as con: con.execute("CREATE TABLE tbl0005 (a text)") con.commit() - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) target_xid = None with node.connect("postgres") as con: res = con.execute( @@ -1548,41 +1134,22 @@ def test_validate_corrupt_wal_2(self): target_xid = res[0][0] # Corrupt WAL - wals_dir = os.path.join(backup_dir, 'wal', 'node') - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] - wals.sort() - for wal in wals: - with open(os.path.join(wals_dir, wal), "rb+", 0) as f: - f.seek(128) - f.write(b"blablablaadssaaaaaaaaaaaaaaa") - f.flush() - f.close + bla = b"blablablaadssaaaaaaaaaaaaaaa" + for wal in self.get_instance_wal_list(self.backup_dir, 'node'): + self.corrupt_instance_wal(self.backup_dir, 'node', wal, 128, bla) # Validate to xid - try: - self.validate_pb( - backup_dir, - 'node', - backup_id, - options=[ - "--xid={0}".format(target_xid), "-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of wal segments corruption.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'WARNING: Backup' in e.message and - 'WAL segments are corrupted' in e.message and - "WARNING: There are not enough WAL " - "records to consistenly restore backup" in e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.validate('node', backup_id, + options=[f"--xid={target_xid}", "-j", "4"], + expect_error="because of wal segments corruption") + self.assertMessage(contains='WARNING: Backup') + self.assertMessage(contains='WAL segments are corrupted') + self.assertMessage(contains="WARNING: There are not enough WAL " + "records to consistenly restore backup") self.assertEqual( 'CORRUPT', - self.show_pb(backup_dir, 'node', backup_id)['status'], + self.pb.show('node', backup_id)['status'], 'Backup STATUS should be "CORRUPT"') # @unittest.skip("skip") @@ -1592,71 +1159,39 @@ def test_validate_wal_lost_segment_1(self): run validate, expecting error because of missing wal segment make sure that backup status is 'CORRUPT' """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') + - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() node.pgbench_init(scale=3) - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) # Delete wal segment - wals_dir = os.path.join(backup_dir, 'wal', 'node') - wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join(wals_dir, f)) and not f.endswith('.backup')] - wals.sort() - file = os.path.join(backup_dir, 'wal', 'node', wals[-1]) - os.remove(file) - - # cut out '.gz' - if self.archive_compress: - file = file[:-3] - - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of wal segment disappearance.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - "is absent" in e.message and - "WARNING: There are not enough WAL records to consistenly " - "restore backup {0}".format(backup_id) in e.message and - "WARNING: Backup {0} WAL segments are corrupted".format( - backup_id) in e.message and - "WARNING: Some backups are not valid" in e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + wals = self.get_instance_wal_list(self.backup_dir, 'node') + self.remove_instance_wal(self.backup_dir, 'node', max(wals)) + + self.pb.validate('node', options=["-j", "4"], + expect_error="because of wal segment disappearance") + self.assertMessage(contains="is absent") + self.assertMessage(contains="WARNING: There are not enough WAL records to consistenly " + f"restore backup {backup_id}") + self.assertMessage(contains=f"WARNING: Backup {backup_id} WAL segments are corrupted") self.assertEqual( 'CORRUPT', - self.show_pb(backup_dir, 'node', backup_id)['status'], + self.pb.show('node', backup_id)['status'], 'Backup {0} should have STATUS "CORRUPT"') # Run validate again - try: - self.validate_pb(backup_dir, 'node', backup_id, options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of backup corruption.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - 'INFO: Revalidating backup {0}'.format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'ERROR: Backup {0} is corrupt.'.format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.validate('node', backup_id, options=["-j", "4"], + expect_error="because of backup corruption") + self.assertMessage(contains=f'INFO: Revalidating backup {backup_id}') + self.assertMessage(contains=f'ERROR: Backup {backup_id} is corrupt.') # @unittest.skip("skip") def test_validate_corrupt_wal_between_backups(self): @@ -1664,17 +1199,15 @@ def test_validate_corrupt_wal_between_backups(self): make archive node, make full backup, corrupt all wal files, run validate to real xid, expect errors """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node') + - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) # make some wals node.pgbench_init(scale=3) @@ -1689,62 +1222,42 @@ def test_validate_corrupt_wal_between_backups(self): con.commit() target_xid = res[0][0] - if self.get_version(node) < self.version_to_num('10.0'): - walfile = node.safe_psql( - 'postgres', - 'select pg_xlogfile_name(pg_current_xlog_location())').decode('utf-8').rstrip() - else: - walfile = node.safe_psql( - 'postgres', - 'select pg_walfile_name(pg_current_wal_lsn())').decode('utf-8').rstrip() + walfile = node.safe_psql( + 'postgres', + 'select pg_walfile_name(pg_current_wal_lsn())').decode('utf-8').rstrip() - if self.archive_compress: - walfile = walfile + '.gz' + walfile = walfile + self.compress_suffix self.switch_wal_segment(node) # generate some wals node.pgbench_init(scale=3) - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) # Corrupt WAL - wals_dir = os.path.join(backup_dir, 'wal', 'node') - with open(os.path.join(wals_dir, walfile), "rb+", 0) as f: - f.seek(9000) - f.write(b"b") - f.flush() - f.close + self.corrupt_instance_wal(self.backup_dir, 'node', walfile, 9000, b"b") # Validate to xid - try: - self.validate_pb( - backup_dir, - 'node', - backup_id, - options=[ - "--xid={0}".format(target_xid), "-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of wal segments corruption.\n" - " Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertTrue( - 'ERROR: Not enough WAL records to xid' in e.message and - 'WARNING: Recovery can be done up to time' in e.message and - "ERROR: Not enough WAL records to xid {0}\n".format( - target_xid), - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.validate('node', backup_id, + options=[f"--xid={target_xid}", "-j", "4"], + expect_error="because of wal segments corruption") + self.assertMessage(contains='ERROR: Not enough WAL records to xid') + self.assertMessage(contains='WARNING: Recovery can be done up to time') + self.assertMessage(contains=f"ERROR: Not enough WAL records to xid {target_xid}") + + # Validate whole WAL Archive. It shouldn't be error, only warning in LOG. [PBCKP-55] + self.pb.validate('node', + options=[f"--wal", "-j", "4"], expect_error=True) + self.assertMessage(contains='ERROR: WAL archive check error') self.assertEqual( 'OK', - self.show_pb(backup_dir, 'node')[0]['status'], + self.pb.show('node')[0]['status'], 'Backup STATUS should be "OK"') self.assertEqual( 'OK', - self.show_pb(backup_dir, 'node')[1]['status'], + self.pb.show('node')[1]['status'], 'Backup STATUS should be "OK"') # @unittest.skip("skip") @@ -1753,36 +1266,24 @@ def test_pgpro702_688(self): make node without archiving, make stream backup, get Recovery Time, validate to Recovery Time """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) + - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - backup_id = self.backup_node( - backup_dir, 'node', node, options=["--stream"]) - recovery_time = self.show_pb( - backup_dir, 'node', backup_id=backup_id)['recovery-time'] - - try: - self.validate_pb( - backup_dir, 'node', - options=["--time={0}".format(recovery_time), "-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of wal segment disappearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WAL archive is empty. You cannot restore backup to a ' - 'recovery target without WAL archive', e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + backup_id = self.pb.backup_node( + 'node', node, options=["--stream"]) + recovery_time = self.pb.show( + 'node', backup_id=backup_id)['recovery-time'] + + self.pb.validate('node', + options=[f"--time={recovery_time}", "-j", "4"], + expect_error="because of wal segment disappearance") + self.assertMessage(contains='WAL archive is empty. You cannot restore backup to a ' + 'recovery target without WAL archive') # @unittest.skip("skip") def test_pgpro688(self): @@ -1790,23 +1291,21 @@ def test_pgpro688(self): make node with archiving, make backup, get Recovery Time, validate to Recovery Time. Waiting PGPRO-688. RESOLVED """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) + - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - backup_id = self.backup_node(backup_dir, 'node', node) - recovery_time = self.show_pb( - backup_dir, 'node', backup_id)['recovery-time'] + backup_id = self.pb.backup_node('node', node) + recovery_time = self.pb.show( + 'node', backup_id)['recovery-time'] - self.validate_pb( - backup_dir, 'node', options=["--time={0}".format(recovery_time), + self.pb.validate( + 'node', options=["--time={0}".format(recovery_time), "-j", "4"]) # @unittest.skip("skip") @@ -1816,22 +1315,17 @@ def test_pgpro561(self): make node with archiving, make stream backup, restore it to node1, check that archiving is not successful on node1 """ - node1 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node1'), - set_replication=True, - initdb_params=['--data-checksums']) + node1 = self.pg_node.make_simple('node1', set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node1', node1) - self.set_archiving(backup_dir, 'node1', node1) + + self.pb.init() + self.pb.add_instance('node1', node1) + self.pb.set_archiving('node1', node1) node1.slow_start() - backup_id = self.backup_node( - backup_dir, 'node1', node1, options=["--stream"]) + backup_id = self.pb.backup_node('node1', node1, options=["--stream"]) - node2 = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node2')) + node2 = self.pg_node.make_simple('node2') node2.cleanup() node1.psql( @@ -1840,18 +1334,16 @@ def test_pgpro561(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,256) i") - self.backup_node( - backup_dir, 'node1', node1, + self.pb.backup_node( + 'node1', node1, backup_type='page', options=["--stream"]) - self.restore_node(backup_dir, 'node1', data_dir=node2.data_dir) + self.pb.restore_node('node1', node=node2) - self.set_auto_conf( - node2, {'port': node2.port, 'archive_mode': 'off'}) + node2.set_auto_conf({'port': node2.port, 'archive_mode': 'off'}) node2.slow_start() - self.set_auto_conf( - node2, {'archive_mode': 'on'}) + node2.set_auto_conf({'archive_mode': 'on'}) node2.stop() node2.slow_start() @@ -1885,7 +1377,7 @@ def test_pgpro561(self): self.switch_wal_segment(node1) -# wals_dir = os.path.join(backup_dir, 'wal', 'node1') +# wals_dir = os.path.join(self.backup_dir, 'wal', 'node1') # wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( # wals_dir, f)) and not f.endswith('.backup') and not f.endswith('.part')] # wals = map(str, wals) @@ -1893,7 +1385,7 @@ def test_pgpro561(self): self.switch_wal_segment(node2) -# wals_dir = os.path.join(backup_dir, 'wal', 'node1') +# wals_dir = os.path.join(self.backup_dir, 'wal', 'node1') # wals = [f for f in os.listdir(wals_dir) if os.path.isfile(os.path.join( # wals_dir, f)) and not f.endswith('.backup') and not f.endswith('.part')] # wals = map(str, wals) @@ -1923,112 +1415,80 @@ def test_validate_corrupted_full(self): remove corruption and run valudate again, check that second full backup and his page backups are OK """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums'], - pg_options={ + node = self.pg_node.make_simple('node', + set_replication=True, + pg_options={ 'checkpoint_timeout': '30'}) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') - backup_id = self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') + backup_id = self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') node.safe_psql( "postgres", "alter system set archive_command = 'false'") node.reload() - try: - self.backup_node( - backup_dir, 'node', node, - backup_type='page', options=['--archive-timeout=1s']) - self.assertEqual( - 1, 0, - "Expecting Error because of data file dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - pass + self.pb.backup_node('node', node, backup_type='page', + options=['--archive-timeout=1s'], + expect_error="because of data file dissapearance") self.assertTrue( - self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') - self.set_archiving(backup_dir, 'node', node) + self.pb.show('node')[6]['status'] == 'ERROR') + self.pb.set_archiving('node', node) node.reload() - self.backup_node(backup_dir, 'node', node, backup_type='page') - - file = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', 'postgresql.auto.conf') - - file_new = os.path.join(backup_dir, 'postgresql.auto.conf') - os.rename(file, file_new) - - try: - self.validate_pb(backup_dir, options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data file dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'Validating backup {0}'.format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} data files are corrupted'.format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Some backups are not valid'.format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') + self.pb.backup_node('node', node, backup_type='page') + + auto_conf = self.read_backup_file(self.backup_dir, 'node', backup_id, + 'database/postgresql.auto.conf') + self.remove_backup_file(self.backup_dir, 'node', backup_id, + 'database/postgresql.auto.conf') + + self.pb.validate(options=["-j", "4"], + expect_error="because of data file dissapearance") + self.assertMessage(contains=f'Validating backup {backup_id}') + self.assertMessage(contains=f'WARNING: Backup {backup_id} data files are corrupted') + self.assertMessage(contains='WARNING: Some backups are not valid') + + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') self.assertTrue( - self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') + self.pb.show('node')[3]['status'] == 'CORRUPT') self.assertTrue( - self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.pb.show('node')[4]['status'] == 'ORPHAN') self.assertTrue( - self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.pb.show('node')[5]['status'] == 'ORPHAN') self.assertTrue( - self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') + self.pb.show('node')[6]['status'] == 'ERROR') self.assertTrue( - self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') + self.pb.show('node')[7]['status'] == 'ORPHAN') - os.rename(file_new, file) - try: - self.validate_pb(backup_dir, options=["-j", "4"]) - except ProbackupException as e: - self.assertIn( - 'WARNING: Some backups are not valid'.format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') + self.write_backup_file(self.backup_dir, 'node', backup_id, + 'database/postgresql.auto.conf', auto_conf) + + self.pb.validate(options=["-j", "4"], + expect_error=True) + self.assertMessage(contains='WARNING: Some backups are not valid') + + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[3]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[4]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[5]['status'] == 'OK') self.assertTrue( - self.show_pb(backup_dir, 'node')[6]['status'] == 'ERROR') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') + self.pb.show('node')[6]['status'] == 'ERROR') + self.assertTrue(self.pb.show('node')[7]['status'] == 'OK') # @unittest.skip("skip") def test_validate_corrupted_full_1(self): @@ -2043,90 +1503,61 @@ def test_validate_corrupted_full_1(self): second page should be CORRUPT third page should be ORPHAN """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) + - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - backup_id = self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - backup_id_page = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - file = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', 'postgresql.auto.conf') - - file_new = os.path.join(backup_dir, 'postgresql.auto.conf') - os.rename(file, file_new) - - try: - self.validate_pb(backup_dir, options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data file dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'Validating backup {0}'.format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} data files are corrupted'.format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Some backups are not valid'.format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'CORRUPT') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - os.rename(file_new, file) - - file = os.path.join( - backup_dir, 'backups', 'node', - backup_id_page, 'database', 'backup_label') - - file_new = os.path.join(backup_dir, 'backup_label') - os.rename(file, file_new) - - try: - self.validate_pb(backup_dir, options=["-j", "4"]) - except ProbackupException as e: - self.assertIn( - 'WARNING: Some backups are not valid'.format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'CORRUPT') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') + + backup_id = self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type='page') + backup_id_page = self.pb.backup_node( + 'node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') + + auto_conf = self.read_backup_file(self.backup_dir, 'node', backup_id, + 'database/postgresql.auto.conf') + self.remove_backup_file(self.backup_dir, 'node', backup_id, + 'database/postgresql.auto.conf') + + self.pb.validate(options=["-j", "4"], + expect_error="because of data file dissapearance") + self.assertMessage(contains=f'Validating backup {backup_id}') + self.assertMessage(contains=f'WARNING: Backup {backup_id} data files are corrupted') + self.assertMessage(contains='WARNING: Some backups are not valid') + + self.assertTrue(self.pb.show('node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[4]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[3]['status'] == 'CORRUPT') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') + + self.write_backup_file(self.backup_dir, 'node', backup_id, + 'database/postgresql.auto.conf', auto_conf) + + self.remove_backup_file(self.backup_dir, 'node', backup_id_page, + 'database/backup_label') + + self.pb.validate(options=["-j", "4"], + expect_error=True) + self.assertMessage(contains='WARNING: Some backups are not valid') + + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[3]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[4]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[5]['status'] == 'CORRUPT') + self.assertTrue(self.pb.show('node')[6]['status'] == 'ORPHAN') # @unittest.skip("skip") def test_validate_corrupted_full_2(self): @@ -2150,341 +1581,124 @@ def test_validate_corrupted_full_2(self): remove corruption from PAGE2_2 and run validate on PAGE2_4 """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - corrupt_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - validate_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - file = os.path.join( - backup_dir, 'backups', 'node', - corrupt_id, 'database', 'backup_label') - - file_new = os.path.join(backup_dir, 'backup_label') - os.rename(file, file_new) - - try: - self.validate_pb(backup_dir, 'node', validate_id, - options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data file dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'INFO: Validating parents for backup {0}'.format(validate_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'INFO: Validating backup {0}'.format( - self.show_pb(backup_dir, 'node')[2]['id']), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'INFO: Validating backup {0}'.format( - self.show_pb(backup_dir, 'node')[3]['id']), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'INFO: Validating backup {0}'.format( - corrupt_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} data files are corrupted'.format( - corrupt_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'CORRUPT') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type='page') + + backup_id_3 = self.pb.backup_node('node', node) + backup_id_4 = self.pb.backup_node('node', node, backup_type='page') + corrupt_id = self.pb.backup_node( + 'node', node, backup_type='page') + backup_id_6 = self.pb.backup_node('node', node, backup_type='page') + validate_id = self.pb.backup_node( + 'node', node, backup_type='page') + backup_id_8 = self.pb.backup_node('node', node, backup_type='page') + + backup_label = self.read_backup_file(self.backup_dir, 'node', corrupt_id, + 'database/backup_label') + self.remove_backup_file(self.backup_dir, 'node', corrupt_id, + 'database/backup_label') + + self.pb.validate('node', validate_id, options=["-j", "4"], + expect_error="because of data file dissapearance") + self.assertMessage(contains=f'INFO: Validating parents for backup {validate_id}') + self.assertMessage(contains=f'INFO: Validating backup {backup_id_3}') + self.assertMessage(contains=f'INFO: Validating backup {backup_id_4}') + self.assertMessage(contains=f'INFO: Validating backup {corrupt_id}') + self.assertMessage(contains=f'WARNING: Backup {corrupt_id} data files are corrupted') + + self.assertTrue(self.pb.show('node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[4]['status'] == 'CORRUPT') + self.assertTrue(self.pb.show('node')[3]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') # THIS IS GOLD!!!! - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data file dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'Backup {0} data files are valid'.format( - self.show_pb(backup_dir, 'node')[9]['id']), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'Backup {0} data files are valid'.format( - self.show_pb(backup_dir, 'node')[8]['id']), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[7]['id'], corrupt_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[6]['id'], corrupt_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[5]['id'], corrupt_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'INFO: Revalidating backup {0}'.format( - corrupt_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Some backups are not valid', e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'CORRUPT') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + backup_id_9 = self.pb.backup_node('node', node, backup_type='page') + backup_id_a = self.pb.backup_node('node', node, backup_type='page') + + self.pb.validate('node', options=["-j", "4"], + expect_error="because of data file dissapearance") + self.assertMessage(contains=f'Backup {backup_id_a} data files are valid') + self.assertMessage(contains=f'Backup {backup_id_9} data files are valid') + self.assertMessage(regex=f'WARNING: Backup {backup_id_8} .* parent {corrupt_id} .* CORRUPT') + self.assertMessage(regex=f'WARNING: Backup {validate_id} .* parent {corrupt_id} .* CORRUPT') + self.assertMessage(regex=f'WARNING: Backup {backup_id_6} .* parent {corrupt_id} .* CORRUPT') + self.assertMessage(contains=f'INFO: Revalidating backup {corrupt_id}') + self.assertMessage(contains='WARNING: Some backups are not valid') + + self.assertTrue(self.pb.show('node')[9]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[8]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[4]['status'] == 'CORRUPT') + self.assertTrue(self.pb.show('node')[3]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') # revalidate again - try: - self.validate_pb(backup_dir, 'node', validate_id, - options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data file dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} has status: ORPHAN'.format(validate_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[7]['id'], corrupt_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[6]['id'], corrupt_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[5]['id'], corrupt_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'INFO: Validating parents for backup {0}'.format( - validate_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'INFO: Validating backup {0}'.format( - self.show_pb(backup_dir, 'node')[2]['id']), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'INFO: Validating backup {0}'.format( - self.show_pb(backup_dir, 'node')[3]['id']), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'INFO: Revalidating backup {0}'.format( - corrupt_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Backup {0} data files are corrupted'.format( - corrupt_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'ERROR: Backup {0} is orphan.'.format( - validate_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.validate('node', validate_id, options=["-j", "4"], + expect_error="because of data file dissapearance") + self.assertMessage(contains=f'WARNING: Backup {validate_id} has status: ORPHAN') + self.assertMessage(contains=f'Backup {backup_id_8} has parent {corrupt_id} with status: CORRUPT') + self.assertMessage(contains=f'Backup {validate_id} has parent {corrupt_id} with status: CORRUPT') + self.assertMessage(contains=f'Backup {backup_id_6} has parent {corrupt_id} with status: CORRUPT') + self.assertMessage(contains=f'INFO: Validating parents for backup {validate_id}') + self.assertMessage(contains=f'INFO: Validating backup {backup_id_3}') + self.assertMessage(contains=f'INFO: Validating backup {backup_id_4}') + self.assertMessage(contains=f'INFO: Revalidating backup {corrupt_id}') + self.assertMessage(contains=f'WARNING: Backup {corrupt_id} data files are corrupted') + self.assertMessage(contains=f'ERROR: Backup {validate_id} is orphan.') # Fix CORRUPT - os.rename(file_new, file) - - output = self.validate_pb(backup_dir, 'node', validate_id, - options=["-j", "4"]) - - self.assertIn( - 'WARNING: Backup {0} has status: ORPHAN'.format(validate_id), - output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[7]['id'], corrupt_id), - output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[6]['id'], corrupt_id), - output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[5]['id'], corrupt_id), - output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'INFO: Validating parents for backup {0}'.format( - validate_id), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'INFO: Validating backup {0}'.format( - self.show_pb(backup_dir, 'node')[2]['id']), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'INFO: Validating backup {0}'.format( - self.show_pb(backup_dir, 'node')[3]['id']), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'INFO: Revalidating backup {0}'.format( - corrupt_id), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'Backup {0} data files are valid'.format( - corrupt_id), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'INFO: Revalidating backup {0}'.format( - self.show_pb(backup_dir, 'node')[5]['id']), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'Backup {0} data files are valid'.format( - self.show_pb(backup_dir, 'node')[5]['id']), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'INFO: Revalidating backup {0}'.format( - validate_id), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'Backup {0} data files are valid'.format( - validate_id), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'INFO: Backup {0} WAL segments are valid'.format( - validate_id), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'INFO: Backup {0} is valid.'.format( - validate_id), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'INFO: Validate of backup {0} completed.'.format( - validate_id), output, - '\n Unexpected Output Message: {0}\n'.format( - repr(output))) + self.write_backup_file(self.backup_dir, 'node', corrupt_id, + 'database/backup_label', backup_label) + + self.pb.validate('node', validate_id, options=["-j", "4"]) + + self.assertMessage(contains=f'WARNING: Backup {validate_id} has status: ORPHAN') + self.assertMessage(contains=f'Backup {backup_id_8} has parent {corrupt_id} with status: CORRUPT') + self.assertMessage(contains=f'Backup {validate_id} has parent {corrupt_id} with status: CORRUPT') + self.assertMessage(contains=f'Backup {backup_id_6} has parent {corrupt_id} with status: CORRUPT') + self.assertMessage(contains=f'INFO: Validating parents for backup {validate_id}') + self.assertMessage(contains=f'INFO: Validating backup {backup_id_3}') + self.assertMessage(contains=f'INFO: Validating backup {backup_id_4}') + self.assertMessage(contains=f'INFO: Revalidating backup {corrupt_id}') + self.assertMessage(contains=f'Backup {corrupt_id} data files are valid') + self.assertMessage(contains=f'INFO: Revalidating backup {backup_id_6}') + self.assertMessage(contains=f'Backup {backup_id_6} data files are valid') + self.assertMessage(contains=f'INFO: Revalidating backup {validate_id}') + self.assertMessage(contains=f'Backup {validate_id} data files are valid') + self.assertMessage(contains=f'INFO: Backup {validate_id} WAL segments are valid') + self.assertMessage(contains=f'INFO: Backup {validate_id} is valid.') + self.assertMessage(contains=f'INFO: Validate of backup {validate_id} completed.') # Now we have two perfectly valid backup chains based on FULL2 - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[9]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[8]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[6]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[5]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[4]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[3]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') # @unittest.skip("skip") def test_validate_corrupted_full_missing(self): @@ -2498,235 +1712,131 @@ def test_validate_corrupted_full_missing(self): second full backup and his firts page backups are OK, third page should be ORPHAN """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - backup_id = self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - backup_id_page = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - file = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', 'postgresql.auto.conf') - - file_new = os.path.join(backup_dir, 'postgresql.auto.conf') - os.rename(file, file_new) - - try: - self.validate_pb(backup_dir, options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of data file dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'Validating backup {0}'.format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} data files are corrupted'.format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} has status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[5]['id'], backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'CORRUPT') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') + + backup_id = self.pb.backup_node('node', node) + backup_id_6 = self.pb.backup_node('node', node, backup_type='page') + backup_id_page = self.pb.backup_node('node', node, backup_type='page') + backup_id_8 = self.pb.backup_node('node', node, backup_type='page') + backup_id_9 = self.pb.backup_node('node', node, backup_type='page') + + auto_conf = self.read_backup_file(self.backup_dir, 'node', backup_id, + 'database/postgresql.auto.conf') + self.remove_backup_file(self.backup_dir, 'node', backup_id, + 'database/postgresql.auto.conf') + + self.pb.validate(options=["-j", "4"], + expect_error="because of data file dissapearance") + self.assertMessage(contains=f'Validating backup {backup_id}') + self.assertMessage(contains=f'WARNING: Backup {backup_id} data files are corrupted') + self.assertMessage(contains=f'WARNING: Backup {backup_id_6} is orphaned because his parent {backup_id} has status: CORRUPT') + + self.assertTrue(self.pb.show('node')[8]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[4]['status'] == 'CORRUPT') + self.assertTrue(self.pb.show('node')[3]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') # Full backup is fixed - os.rename(file_new, file) + self.write_backup_file(self.backup_dir, 'node', backup_id, + 'database/postgresql.auto.conf', auto_conf) # break PAGE - old_directory = os.path.join( - backup_dir, 'backups', 'node', backup_id_page) - new_directory = os.path.join(backup_dir, backup_id_page) - os.rename(old_directory, new_directory) - - try: - self.validate_pb(backup_dir, options=["-j", "4"]) - except ProbackupException as e: - self.assertIn( - 'WARNING: Some backups are not valid', e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[7]['id'], - backup_id_page), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[6]['id'], - backup_id_page), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Backup {0} has parent {1} with status: CORRUPT'.format( - self.show_pb(backup_dir, 'node')[5]['id'], backup_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + self.change_backup_status(self.backup_dir, 'node', backup_id_page, + 'THIS_BACKUP_IS_HIDDEN_FOR_TESTS') + + self.pb.validate(options=["-j", "4"], + expect_error="because backup in chain is removed") + self.assertMessage(contains='WARNING: Some backups are not valid') + self.assertMessage(regex=fr'WARNING: Backup {backup_id_9} (.*(missing|parent {backup_id_page})){{2}}') + self.assertMessage(regex=fr'WARNING: Backup {backup_id_8} (.*(missing|parent {backup_id_page})){{2}}') + self.assertMessage(contains=f'INFO: Backup {backup_id_6} WAL segments are valid') + + self.assertTrue(self.pb.show('node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[6]['status'] == 'ORPHAN') # missing backup is here - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[5]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[4]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[3]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') # validate should be idempotent - user running validate # second time must be provided with ID of missing backup - try: - self.validate_pb(backup_dir, options=["-j", "4"]) - except ProbackupException as e: - self.assertIn( - 'WARNING: Some backups are not valid', e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[7]['id'], - backup_id_page), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.validate(options=["-j", "4"], + expect_error=True) + self.assertMessage(contains='WARNING: Some backups are not valid') + self.assertMessage(regex=fr'WARNING: Backup {backup_id_9} (.*(missing|parent {backup_id_page})){{2}}') + self.assertMessage(regex=fr'WARNING: Backup {backup_id_8} (.*(missing|parent {backup_id_page})){{2}}') - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[6]['id'], - backup_id_page), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[6]['status'] == 'ORPHAN') # missing backup is here - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[5]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[4]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[3]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') # fix missing PAGE backup - os.rename(new_directory, old_directory) + self.change_backup_status(self.backup_dir, 'node', backup_id_page, 'ORPHAN') # exit(1) - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - output = self.validate_pb(backup_dir, options=["-j", "4"]) - - self.assertIn( - 'INFO: All backups are valid', - output, - '\n Unexpected Error Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'WARNING: Backup {0} has parent {1} with status: ORPHAN'.format( - self.show_pb(backup_dir, 'node')[8]['id'], - self.show_pb(backup_dir, 'node')[6]['id']), - output, - '\n Unexpected Error Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'WARNING: Backup {0} has parent {1} with status: ORPHAN'.format( - self.show_pb(backup_dir, 'node')[7]['id'], - self.show_pb(backup_dir, 'node')[6]['id']), - output, - '\n Unexpected Error Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'Revalidating backup {0}'.format( - self.show_pb(backup_dir, 'node')[6]['id']), - output, - '\n Unexpected Error Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'Revalidating backup {0}'.format( - self.show_pb(backup_dir, 'node')[7]['id']), - output, - '\n Unexpected Error Message: {0}\n'.format( - repr(output))) - - self.assertIn( - 'Revalidating backup {0}'.format( - self.show_pb(backup_dir, 'node')[8]['id']), - output, - '\n Unexpected Error Message: {0}\n'.format( - repr(output))) - - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[8]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[5]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[4]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[3]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') + + self.pb.validate(options=["-j", "4"]) + + self.assertMessage(contains='INFO: All backups are valid') + self.assertMessage(contains=f'Revalidating backup {backup_id_page}') + self.assertMessage(contains=f'Revalidating backup {backup_id_8}') + self.assertMessage(contains=f'Revalidating backup {backup_id_9}') + + self.assertTrue(self.pb.show('node')[8]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[7]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[6]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[5]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[4]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[3]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') def test_file_size_corruption_no_validate(self): - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - # initdb_params=['--data-checksums'], - ) + node = self.pg_node.make_simple('node', checksum=False) + - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() @@ -2746,32 +1856,23 @@ def test_file_size_corruption_no_validate(self): "postgres", "select pg_relation_size('t_heap')") - backup_id = self.backup_node( - backup_dir, 'node', node, backup_type="full", - options=["-j", "4"], asynchronous=False, gdb=False) + backup_id = self.pb.backup_node( + 'node', node, backup_type="full", + options=["-j", "4"], gdb=False) node.stop() node.cleanup() # Let`s do file corruption - with open( - os.path.join( - backup_dir, "backups", 'node', backup_id, - "database", heap_path), "rb+", 0) as f: - f.truncate(int(heap_size) - 4096) - f.flush() - f.close + self.corrupt_backup_file(self.backup_dir, 'node', backup_id, + os.path.join("database", heap_path), + truncate=(int(heap_size) - 4096)) node.cleanup() - try: - self.restore_node( - backup_dir, 'node', node, - options=["--no-validate"]) - except ProbackupException as e: - self.assertTrue( - "ERROR: Backup files restoring failed" in e.message, - repr(e.message)) + self.pb.restore_node('node', node=node, options=["--no-validate"], + expect_error=True) + self.assertMessage(contains="ERROR: Backup files restoring failed") # @unittest.skip("skip") def test_validate_specific_backup_with_missing_backup(self): @@ -2789,128 +1890,86 @@ def test_validate_specific_backup_with_missing_backup(self): PAGE1_1 FULL1 """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) + - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # CHAIN1 - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') # CHAIN2 - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - missing_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - validate_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type='page') + missing_id = self.pb.backup_node( + 'node', node, backup_type='page') + backup_id_6 = self.pb.backup_node('node', node, backup_type='page') + validate_id = self.pb.backup_node( + 'node', node, backup_type='page') + backup_id_8 = self.pb.backup_node('node', node, backup_type='page') # CHAIN3 - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - old_directory = os.path.join(backup_dir, 'backups', 'node', missing_id) - new_directory = os.path.join(backup_dir, missing_id) - - os.rename(old_directory, new_directory) - - try: - self.validate_pb(backup_dir, 'node', validate_id, - options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of backup dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[7]['id'], missing_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[6]['id'], missing_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[5]['id'], missing_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') + + self.change_backup_status(self.backup_dir, 'node', missing_id, + "THIS_BACKUP_IS_HIDDEN_FOR_TESTS") + + self.pb.validate('node', validate_id, options=["-j", "4"], + expect_error="because of backup dissapearance") + self.assertMessage(contains=f'WARNING: Backup {backup_id_8} is orphaned ' + f'because his parent {missing_id} is missing') + self.assertMessage(contains=f'WARNING: Backup {validate_id} is orphaned ' + f'because his parent {missing_id} is missing') + self.assertMessage(contains=f'WARNING: Backup {backup_id_6} is orphaned ' + f'because his parent {missing_id} is missing') + + self.assertTrue(self.pb.show('node')[10]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[9]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[8]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[5]['status'] == 'ORPHAN') # missing backup - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') - - try: - self.validate_pb(backup_dir, 'node', validate_id, - options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of backup dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[7]['id'], missing_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[6]['id'], missing_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[5]['id'], missing_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - os.rename(new_directory, old_directory) + self.assertTrue(self.pb.show('node')[4]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[3]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') + + self.pb.validate('node', validate_id, options=["-j", "4"], + expect_error="because of backup dissapearance") + self.assertMessage(contains=f'WARNING: Backup {backup_id_8} has missing ' + f'parent {missing_id}') + self.assertMessage(contains=f'WARNING: Backup {validate_id} has missing ' + f'parent {missing_id}') + self.assertMessage(contains=f'WARNING: Backup {backup_id_6} has missing ' + f'parent {missing_id}') + + self.change_backup_status(self.backup_dir, 'node', missing_id, "OK") # Revalidate backup chain - self.validate_pb(backup_dir, 'node', validate_id, options=["-j", "4"]) - - self.assertTrue(self.show_pb(backup_dir, 'node')[11]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.pb.validate('node', validate_id, options=["-j", "4"]) + + self.assertTrue(self.pb.show('node')[11]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[10]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[9]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[8]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[7]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[6]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[5]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[4]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[3]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') # @unittest.skip("skip") def test_validate_specific_backup_with_missing_backup_1(self): @@ -2928,106 +1987,80 @@ def test_validate_specific_backup_with_missing_backup_1(self): PAGE1_1 FULL1 """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) + - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # CHAIN1 - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') # CHAIN2 - missing_full_id = self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - missing_page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - validate_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') + missing_full_id = self.pb.backup_node('node', node) + backup_id_5 = self.pb.backup_node('node', node, backup_type='page') + missing_page_id = self.pb.backup_node( + 'node', node, backup_type='page') + backup_id_7 = self.pb.backup_node('node', node, backup_type='page') + validate_id = self.pb.backup_node( + 'node', node, backup_type='page') + backup_id_9 = self.pb.backup_node('node', node, backup_type='page') # CHAIN3 - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - page_old_directory = os.path.join( - backup_dir, 'backups', 'node', missing_page_id) - page_new_directory = os.path.join(backup_dir, missing_page_id) - os.rename(page_old_directory, page_new_directory) - - full_old_directory = os.path.join( - backup_dir, 'backups', 'node', missing_full_id) - full_new_directory = os.path.join(backup_dir, missing_full_id) - os.rename(full_old_directory, full_new_directory) - - try: - self.validate_pb(backup_dir, 'node', validate_id, - options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of backup dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[6]['id'], missing_page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[5]['id'], missing_page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[4]['id'], missing_page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') + + self.change_backup_status(self.backup_dir, 'node', missing_page_id, + "THIS_BACKUP_IS_HIDDEN_FOR_TESTS") + self.change_backup_status(self.backup_dir, 'node', missing_full_id, + "THIS_BACKUP_IS_HIDDEN_FOR_TESTS") + + self.pb.validate('node', validate_id, options=["-j", "4"], + expect_error="because of backup dissapearance") + self.assertMessage(contains=f'WARNING: Backup {backup_id_9} is orphaned ' + f'because his parent {missing_page_id} is missing') + self.assertMessage(contains=f'WARNING: Backup {validate_id} is orphaned ' + f'because his parent {missing_page_id} is missing') + self.assertMessage(contains=f'WARNING: Backup {backup_id_7} is orphaned ' + f'because his parent {missing_page_id} is missing') + + self.assertTrue(self.pb.show('node')[9]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[8]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[7]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[4]['status'] == 'ORPHAN') # PAGE2_1 - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') # <- SHit + self.assertTrue(self.pb.show('node')[3]['status'] == 'OK') # <- SHit # FULL2 - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') - os.rename(page_new_directory, page_old_directory) - os.rename(full_new_directory, full_old_directory) + self.change_backup_status(self.backup_dir, 'node', missing_page_id, "OK") + self.change_backup_status(self.backup_dir, 'node', missing_full_id, "OK") # Revalidate backup chain - self.validate_pb(backup_dir, 'node', validate_id, options=["-j", "4"]) - - self.assertTrue(self.show_pb(backup_dir, 'node')[11]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'ORPHAN') # <- Fail - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.pb.validate('node', validate_id, options=["-j", "4"]) + + self.assertTrue(self.pb.show('node')[11]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[10]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[9]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[8]['status'] == 'ORPHAN') # <- Fail + self.assertTrue(self.pb.show('node')[7]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[6]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[5]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[4]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[3]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') # @unittest.skip("skip") def test_validate_with_missing_backup_1(self): @@ -3045,174 +2078,112 @@ def test_validate_with_missing_backup_1(self): PAGE1_1 FULL1 """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # CHAIN1 - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') # CHAIN2 - missing_full_id = self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - missing_page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - validate_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') + missing_full_id = self.pb.backup_node('node', node) + backup_id_5 = self.pb.backup_node('node', node, backup_type='page') + missing_page_id = self.pb.backup_node( + 'node', node, backup_type='page') + backup_id_7 = self.pb.backup_node('node', node, backup_type='page') + validate_id = self.pb.backup_node( + 'node', node, backup_type='page') + backup_id_9 = self.pb.backup_node('node', node, backup_type='page') # CHAIN3 - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') # Break PAGE - page_old_directory = os.path.join( - backup_dir, 'backups', 'node', missing_page_id) - page_new_directory = os.path.join(backup_dir, missing_page_id) - os.rename(page_old_directory, page_new_directory) + self.change_backup_status(self.backup_dir, 'node', missing_page_id, + 'THIS_BACKUP_IS_HIDDEN_FOR_TESTS') # Break FULL - full_old_directory = os.path.join( - backup_dir, 'backups', 'node', missing_full_id) - full_new_directory = os.path.join(backup_dir, missing_full_id) - os.rename(full_old_directory, full_new_directory) - - try: - self.validate_pb(backup_dir, 'node', validate_id, - options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of backup dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[6]['id'], missing_page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[5]['id'], missing_page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[4]['id'], missing_page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.change_backup_status(self.backup_dir, 'node', missing_full_id, + 'THIS_BACKUP_IS_HIDDEN_FOR_TESTS') + + self.pb.validate('node', validate_id, options=["-j", "4"], + expect_error="because of backup dissapearance") + self.assertMessage(contains=f'WARNING: Backup {backup_id_9} is orphaned ' + f'because his parent {missing_page_id} is missing') + self.assertMessage(contains=f'WARNING: Backup {validate_id} is orphaned ' + f'because his parent {missing_page_id} is missing') + self.assertMessage(contains=f'WARNING: Backup {backup_id_7} is orphaned ' + f'because his parent {missing_page_id} is missing') + + self.assertTrue(self.pb.show('node')[9]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[8]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[7]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[4]['status'] == 'ORPHAN') # PAGE2_2 is missing - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[3]['status'] == 'OK') # FULL1 - is missing - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') - os.rename(page_new_directory, page_old_directory) + self.change_backup_status(self.backup_dir, 'node', missing_page_id, 'OK') # Revalidate backup chain - try: - self.validate_pb(backup_dir, 'node', validate_id, - options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of backup dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} has status: ORPHAN'.format( - validate_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[7]['id'], - missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[6]['id'], - missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[5]['id'], - missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[4]['id'], - missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[3]['id'], - missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'ORPHAN') + self.pb.validate('node', validate_id, options=["-j", "4"], + expect_error="because of backup dissapearance") + self.assertMessage(contains=f'WARNING: Backup {validate_id} has status: ORPHAN') + self.assertMessage(contains=f'WARNING: Backup {backup_id_9} has missing ' + f'parent {missing_full_id}') + self.assertMessage(contains=f'WARNING: Backup {validate_id} has missing ' + f'parent {missing_full_id}') + self.assertMessage(contains=f'WARNING: Backup {backup_id_7} has missing ' + f'parent {missing_full_id}') + self.assertMessage(contains=f'WARNING: Backup {missing_page_id} is orphaned ' + f'because his parent {missing_full_id} is missing') + self.assertMessage(contains=f'WARNING: Backup {backup_id_5} is orphaned ' + f'because his parent {missing_full_id} is missing') + + self.assertTrue(self.pb.show('node')[10]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[9]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[8]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[4]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[3]['status'] == 'ORPHAN') # FULL1 - is missing - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') - os.rename(full_new_directory, full_old_directory) + self.change_backup_status(self.backup_dir, 'node', missing_full_id, 'OK') # Revalidate chain - self.validate_pb(backup_dir, 'node', validate_id, options=["-j", "4"]) - - self.assertTrue(self.show_pb(backup_dir, 'node')[11]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.pb.validate('node', validate_id, options=["-j", "4"]) + + self.assertTrue(self.pb.show('node')[11]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[10]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[9]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[8]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[7]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[6]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[5]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[4]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[3]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') # @unittest.skip("skip") def test_validate_with_missing_backup_2(self): @@ -3230,185 +2201,123 @@ def test_validate_with_missing_backup_2(self): PAGE1_1 FULL1 """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) + - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # CHAIN1 - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') # CHAIN2 - missing_full_id = self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - missing_page_id = self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node( - backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') + missing_full_id = self.pb.backup_node('node', node) + backup_id_5 = self.pb.backup_node('node', node, backup_type='page') + missing_page_id = self.pb.backup_node( + 'node', node, backup_type='page') + backup_id_7 = self.pb.backup_node('node', node, backup_type='page') + backup_id_8 = self.pb.backup_node('node', node, backup_type='page') + backup_id_9 = self.pb.backup_node('node', node, backup_type='page') # CHAIN3 - self.backup_node(backup_dir, 'node', node) - self.backup_node(backup_dir, 'node', node, backup_type='page') - self.backup_node(backup_dir, 'node', node, backup_type='page') - - page_old_directory = os.path.join(backup_dir, 'backups', 'node', missing_page_id) - page_new_directory = os.path.join(backup_dir, missing_page_id) - os.rename(page_old_directory, page_new_directory) - - full_old_directory = os.path.join(backup_dir, 'backups', 'node', missing_full_id) - full_new_directory = os.path.join(backup_dir, missing_full_id) - os.rename(full_old_directory, full_new_directory) - - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of backup dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[6]['id'], missing_page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[5]['id'], missing_page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[4]['id'], missing_page_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[3]['id'], missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') + self.pb.backup_node('node', node) + self.pb.backup_node('node', node, backup_type='page') + self.pb.backup_node('node', node, backup_type='page') + + self.change_backup_status(self.backup_dir, 'node', missing_page_id, + 'THIS_BACKUP_IS_HIDDEN_FOR_TESTS') + self.change_backup_status(self.backup_dir, 'node', missing_full_id, + 'THIS_BACKUP_IS_HIDDEN_FOR_TESTS') + + self.pb.validate('node', options=["-j", "4"], + expect_error="because of backup dissapearance") + self.assertMessage(contains=f'WARNING: Backup {backup_id_9} is orphaned ' + f'because his parent {missing_page_id} is missing') + self.assertMessage(contains=f'WARNING: Backup {backup_id_8} is orphaned ' + f'because his parent {missing_page_id} is missing') + self.assertMessage(contains=f'WARNING: Backup {backup_id_7} is orphaned ' + f'because his parent {missing_page_id} is missing') + self.assertMessage(contains=f'WARNING: Backup {backup_id_5} is orphaned ' + f'because his parent {missing_full_id} is missing') + + self.assertTrue(self.pb.show('node')[9]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[8]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[7]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[4]['status'] == 'ORPHAN') # PAGE2_2 is missing - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[3]['status'] == 'ORPHAN') # FULL1 - is missing - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') - os.rename(page_new_directory, page_old_directory) + self.change_backup_status(self.backup_dir, 'node', missing_page_id, 'OK') # Revalidate backup chain - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of backup dissapearance.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[7]['id'], missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[6]['id'], missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[5]['id'], missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} is orphaned because his parent {1} is missing'.format( - self.show_pb(backup_dir, 'node')[4]['id'], missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - self.assertIn( - 'WARNING: Backup {0} has missing parent {1}'.format( - self.show_pb(backup_dir, 'node')[3]['id'], missing_full_id), - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - self.assertTrue(self.show_pb(backup_dir, 'node')[10]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[9]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[8]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[7]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[6]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[5]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[4]['status'] == 'ORPHAN') - self.assertTrue(self.show_pb(backup_dir, 'node')[3]['status'] == 'ORPHAN') + self.pb.validate('node', options=["-j", "4"], + expect_error="because of backup dissapearance") + self.assertMessage(regex=fr'WARNING: Backup {backup_id_9} (.*(missing|parent {missing_full_id})){{2}}') + self.assertMessage(regex=fr'WARNING: Backup {backup_id_8} (.*(missing|parent {missing_full_id})){{2}}') + self.assertMessage(regex=fr'WARNING: Backup {backup_id_7} (.*(missing|parent {missing_full_id})){{2}}') + self.assertMessage(regex=fr'WARNING: Backup {missing_page_id} (.*(missing|parent {missing_full_id})){{2}}') + self.assertMessage(regex=fr'WARNING: Backup {backup_id_5} (.*(missing|parent {missing_full_id})){{2}}') + + self.assertTrue(self.pb.show('node')[10]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[9]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[8]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[7]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[6]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[5]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[4]['status'] == 'ORPHAN') + self.assertTrue(self.pb.show('node')[3]['status'] == 'ORPHAN') # FULL1 - is missing - self.assertTrue(self.show_pb(backup_dir, 'node')[2]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[1]['status'] == 'OK') - self.assertTrue(self.show_pb(backup_dir, 'node')[0]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[2]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[1]['status'] == 'OK') + self.assertTrue(self.pb.show('node')[0]['status'] == 'OK') # @unittest.skip("skip") def test_corrupt_pg_control_via_resetxlog(self): """ PGPRO-2096 """ - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + + if not self.backup_dir.is_file_based: + self.skipTest('tests uses pg_resetxlog on backup') + + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) - if self.get_version(node) < 100000: - pg_resetxlog_path = self.get_bin_path('pg_resetxlog') - wal_dir = 'pg_xlog' - else: - pg_resetxlog_path = self.get_bin_path('pg_resetwal') - wal_dir = 'pg_wal' + pg_resetxlog_path = self.get_bin_path('pg_resetwal') + wal_dir = 'pg_wal' os.mkdir( os.path.join( - backup_dir, 'backups', 'node', backup_id, 'database', wal_dir, 'archive_status')) + self.backup_dir, 'backups', 'node', backup_id, 'database', wal_dir, 'archive_status')) pg_control_path = os.path.join( - backup_dir, 'backups', 'node', + self.backup_dir, 'backups', 'node', backup_id, 'database', 'global', 'pg_control') md5_before = hashlib.md5( open(pg_control_path, 'rb').read()).hexdigest() - self.run_binary( + self.pb.run_binary( [ pg_resetxlog_path, '-D', - os.path.join(backup_dir, 'backups', 'node', backup_id, 'database'), + os.path.join(self.backup_dir, 'backups', 'node', backup_id, 'database'), '-o 42', '-f' ], @@ -3422,54 +2331,37 @@ def test_corrupt_pg_control_via_resetxlog(self): md5_before, md5_after)) # Validate backup - try: - self.validate_pb(backup_dir, 'node', options=["-j", "4"]) - self.assertEqual( - 1, 0, - "Expecting Error because of pg_control change.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'data files are corrupted', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.validate('node', options=["-j", "4"], + expect_error="because of pg_control change") + self.assertMessage(contains='data files are corrupted') - # @unittest.skip("skip") + @needs_gdb def test_validation_after_backup(self): """""" - self._check_gdb_flag_or_skip_test() - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) + node = self.pg_node.make_simple('node', + set_replication=True) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() # FULL backup - gdb = self.backup_node( - backup_dir, 'node', node, gdb=True, options=['--stream']) + gdb = self.pb.backup_node( + 'node', node, gdb=True, options=['--stream']) gdb.set_breakpoint('pgBackupValidate') gdb.run_until_break() - backup_id = self.show_pb(backup_dir, 'node')[0]['id'] + backup_id = self.pb.show('node')[0]['id'] - file = os.path.join( - backup_dir, "backups", "node", backup_id, - "database", "postgresql.conf") - os.remove(file) + self.remove_backup_file(self.backup_dir, 'node', backup_id, + 'database/postgresql.conf') gdb.continue_execution_until_exit() self.assertEqual( 'CORRUPT', - self.show_pb(backup_dir, 'node', backup_id)['status'], + self.pb.show('node', backup_id)['status'], 'Backup STATUS should be "ERROR"') # @unittest.expectedFailure @@ -3478,14 +2370,12 @@ def test_validate_corrupt_tablespace_map(self): """ Check that corruption in tablespace_map is detected """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() self.create_tblspace_in_node(node, 'external_dir') @@ -3495,54 +2385,32 @@ def test_validate_corrupt_tablespace_map(self): 'CREATE TABLE t_heap(a int) TABLESPACE "external_dir"') # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - tablespace_map = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', 'tablespace_map') + backup_id = self.pb.backup_node( + 'node', node, options=['--stream']) # Corrupt tablespace_map file in FULL backup - with open(tablespace_map, "rb+", 0) as f: - f.seek(84) - f.write(b"blah") - f.flush() - f.close - - try: - self.validate_pb(backup_dir, 'node', backup_id=backup_id) - self.assertEqual( - 1, 0, - "Expecting Error because tablespace_map is corrupted.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'WARNING: Invalid CRC of backup file', - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.corrupt_backup_file(self.backup_dir, 'node', backup_id, + 'database/tablespace_map', damage=(84,b"blah")) + + self.pb.validate('node', backup_id=backup_id, + expect_error="because tablespace_map is corrupted") + self.assertMessage(contains='WARNING: Invalid CRC of backup file') - #TODO fix the test - @unittest.expectedFailure - # @unittest.skip("skip") def test_validate_target_lsn(self): """ - Check validation to specific LSN + Check validation to specific LSN from "forked" backup """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() # FULL backup - self.backup_node(backup_dir, 'node', node) + self.pb.backup_node('node', node) node.safe_psql( "postgres", @@ -3550,45 +2418,38 @@ def test_validate_target_lsn(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,10000) i") - node_restored = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node_restored')) + node_restored = self.pg_node.make_simple('node_restored') node_restored.cleanup() - self.restore_node(backup_dir, 'node', node_restored) + self.pb.restore_node('node', node_restored) - self.set_auto_conf( - node_restored, {'port': node_restored.port}) + node_restored.set_auto_conf({'port': node_restored.port}) node_restored.slow_start() self.switch_wal_segment(node) - backup_id = self.backup_node( - backup_dir, 'node', node_restored, + self.pb.backup_node( + 'node', node_restored, data_dir=node_restored.data_dir) - target_lsn = self.show_pb(backup_dir, 'node')[1]['stop-lsn'] + target_lsn = self.pb.show('node')[1]['stop-lsn'] - self.delete_pb(backup_dir, 'node', backup_id) - - self.validate_pb( - backup_dir, 'node', + self.pb.validate( + 'node', options=[ '--recovery-target-timeline=2', '--recovery-target-lsn={0}'.format(target_lsn)]) - @unittest.skip("skip") def test_partial_validate_empty_and_mangled_database_map(self): """ """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() @@ -3599,65 +2460,35 @@ def test_partial_validate_empty_and_mangled_database_map(self): 'CREATE database db{0}'.format(i)) # FULL backup with database_map - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - pgdata = self.pgdata_content(node.data_dir) + backup_id = self.pb.backup_node( + 'node', node, options=['--stream']) # truncate database_map - path = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', 'database_map') - with open(path, "w") as f: - f.close() - - try: - self.validate_pb( - backup_dir, 'node', - options=["--db-include=db1"]) - self.assertEqual( - 1, 0, - "Expecting Error because database_map is empty.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "WARNING: Backup {0} data files are corrupted".format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.corrupt_backup_file(self.backup_dir, 'node', backup_id, + 'database/database_map', truncate=0) + + self.pb.validate('node', backup_id, + options=["--db-include=db1"], + expect_error="because database_map is empty") + self.assertMessage(contains=f"WARNING: Backup {backup_id} data files are corrupted") # mangle database_map - with open(path, "w") as f: - f.write("42") - f.close() - - try: - self.validate_pb( - backup_dir, 'node', - options=["--db-include=db1"]) - self.assertEqual( - 1, 0, - "Expecting Error because database_map is empty.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "WARNING: Backup {0} data files are corrupted".format( - backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.corrupt_backup_file(self.backup_dir, 'node', backup_id, + 'database/database_map', overwrite=b'42') + + self.pb.validate('node', backup_id, + options=["--db-include=db1"], + expect_error="because database_map is mangled") + self.assertMessage(contains=f"WARNING: Backup {backup_id} data files are corrupted") - @unittest.skip("skip") def test_partial_validate_exclude(self): """""" - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() for i in range(1, 10, 1): @@ -3666,67 +2497,37 @@ def test_partial_validate_exclude(self): 'CREATE database db{0}'.format(i)) # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) - - try: - self.validate_pb( - backup_dir, 'node', - options=[ - "--db-include=db1", - "--db-exclude=db2"]) - self.assertEqual( - 1, 0, - "Expecting Error because of 'db-exclude' and 'db-include'.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: You cannot specify '--db-include' " - "and '--db-exclude' together", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.validate_pb( - backup_dir, 'node', - options=[ - "--db-exclude=db1", - "--db-exclude=db5", - "--log-level-console=verbose"]) - self.assertEqual( - 1, 0, - "Expecting Error because of missing backup ID.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: You must specify parameter (-i, --backup-id) for partial validation", - e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - output = self.validate_pb( - backup_dir, 'node', backup_id, + backup_id = self.pb.backup_node('node', node) + + self.pb.validate('node', + options=["--db-include=db1", "--db-exclude=db2"], + expect_error="because of 'db-exclude' and 'db-include'") + self.assertMessage(contains="ERROR: You cannot specify '--db-include' " + "and '--db-exclude' together") + + self.pb.validate('node', + options=[ + "--db-exclude=db1", + "--db-exclude=db5", + "--log-level-console=verbose"], + expect_error="because of missing backup ID") + self.assertMessage(contains="ERROR: You must specify parameter (-i, --backup-id) for partial validation") + + self.pb.validate( + 'node', backup_id, options=[ "--db-exclude=db1", - "--db-exclude=db5", - "--log-level-console=verbose"]) + "--db-exclude=db5"]) - self.assertIn( - "VERBOSE: Skip file validation due to partial restore", output) - - @unittest.skip("skip") def test_partial_validate_include(self): """ """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) + node = self.pg_node.make_simple('node') + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) node.slow_start() for i in range(1, 10, 1): @@ -3735,62 +2536,37 @@ def test_partial_validate_include(self): 'CREATE database db{0}'.format(i)) # FULL backup - backup_id = self.backup_node(backup_dir, 'node', node) + backup_id = self.pb.backup_node('node', node) - try: - self.validate_pb( - backup_dir, 'node', - options=[ - "--db-include=db1", - "--db-exclude=db2"]) - self.assertEqual( - 1, 0, - "Expecting Error because of 'db-exclude' and 'db-include'.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: You cannot specify '--db-include' " - "and '--db-exclude' together", e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.validate('node', + options=["--db-include=db1", "--db-exclude=db2"], + expect_error="because of 'db-exclude' and 'db-include'") + self.assertMessage(contains="ERROR: You cannot specify '--db-include' " + "and '--db-exclude' together") - output = self.validate_pb( - backup_dir, 'node', backup_id, + self.pb.validate( + 'node', backup_id, options=[ "--db-include=db1", "--db-include=db5", - "--db-include=postgres", - "--log-level-console=verbose"]) + "--db-include=postgres"]) - self.assertIn( - "VERBOSE: Skip file validation due to partial restore", output) - - output = self.validate_pb( - backup_dir, 'node', backup_id, - options=["--log-level-console=verbose"]) - - self.assertNotIn( - "VERBOSE: Skip file validation due to partial restore", output) + self.pb.validate( + 'node', backup_id, + options=[]) # @unittest.skip("skip") def test_not_validate_diffenent_pg_version(self): """Do not validate backup, if binary is compiled with different PG version""" - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - initdb_params=['--data-checksums']) - - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) - self.set_archiving(backup_dir, 'node', node) - node.slow_start() + node = self.pg_node.make_simple('node') - backup_id = self.backup_node(backup_dir, 'node', node) - control_file = os.path.join( - backup_dir, "backups", "node", backup_id, - "backup.control") + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + backup_id = self.pb.backup_node('node', node) pg_version = node.major_version @@ -3799,29 +2575,15 @@ def test_not_validate_diffenent_pg_version(self): fake_new_pg_version = pg_version + 1 - with open(control_file, 'r') as f: - data = f.read(); - - data = data.replace( - "server-version = {0}".format(str(pg_version)), - "server-version = {0}".format(str(fake_new_pg_version))) + with self.modify_backup_control(self.backup_dir, 'node', backup_id) as cf: + cf.data = cf.data.replace( + "server-version = {0}".format(str(pg_version)), + "server-version = {0}".format(str(fake_new_pg_version)) + ) - with open(control_file, 'w') as f: - f.write(data); - - try: - self.validate_pb(backup_dir) - self.assertEqual( - 1, 0, - "Expecting Error because validation is forbidden if server version of backup " - "is different from the server version of pg_probackup.\n Output: {0} \n CMD: {1}".format( - repr(self.output), self.cmd)) - except ProbackupException as e: - self.assertIn( - "ERROR: Backup {0} has server version".format(backup_id), - e.message, - "\n Unexpected Error Message: {0}\n CMD: {1}".format( - repr(e.message), self.cmd)) + self.pb.validate(expect_error="because validation is forbidden if server version of backup " + "is different from the server version of pg_probackup.") + self.assertMessage(contains=f"ERROR: Backup {backup_id} has server version") # @unittest.expectedFailure # @unittest.skip("skip") @@ -3829,60 +2591,42 @@ def test_validate_corrupt_page_header_map(self): """ Check that corruption in page_header_map is detected """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - ok_1 = self.backup_node(backup_dir, 'node', node, options=['--stream']) + ok_1 = self.pb.backup_node('node', node, options=['--stream']) # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) - - ok_2 = self.backup_node(backup_dir, 'node', node, options=['--stream']) + backup_id = self.pb.backup_node( + 'node', node, options=['--stream']) - page_header_map = os.path.join( - backup_dir, 'backups', 'node', backup_id, 'page_header_map') - - # Corrupt tablespace_map file in FULL backup - with open(page_header_map, "rb+", 0) as f: - f.seek(42) - f.write(b"blah") - f.flush() + ok_2 = self.pb.backup_node('node', node, options=['--stream']) - with self.assertRaises(ProbackupException) as cm: - self.validate_pb(backup_dir, 'node', backup_id=backup_id) + self.corrupt_backup_file(self.backup_dir, 'node', backup_id, + 'page_header_map', damage=(42, b"blah")) - e = cm.exception - self.assertRegex( - cm.exception.message, - r'WARNING: An error occured during metadata decompression for file "[\w/]+": (data|buffer) error', - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.validate('node', backup_id=backup_id, + expect_error="because page_header_map is corrupted") - self.assertIn("Backup {0} is corrupt".format(backup_id), e.message) + self.assertMessage(regex= + r'WARNING: An error occured during metadata decompression for file "[\w/]+": (data|buffer) error') - with self.assertRaises(ProbackupException) as cm: - self.validate_pb(backup_dir) + self.assertMessage(contains=f"Backup {backup_id} is corrupt") - e = cm.exception - self.assertRegex( - e.message, - r'WARNING: An error occured during metadata decompression for file "[\w/]+": (data|buffer) error', - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) + self.pb.validate(expect_error="because page_header_map is corrupted") - self.assertIn("INFO: Backup {0} data files are valid".format(ok_1), e.message) - self.assertIn("WARNING: Backup {0} data files are corrupted".format(backup_id), e.message) - self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) + self.assertMessage(regex= + r'WARNING: An error occured during metadata decompression for file "[\w/]+": (data|buffer) error') - self.assertIn("WARNING: Some backups are not valid", e.message) + self.assertMessage(contains=f"INFO: Backup {ok_1} data files are valid") + self.assertMessage(contains=f"WARNING: Backup {backup_id} data files are corrupted") + self.assertMessage(contains=f"INFO: Backup {ok_2} data files are valid") + self.assertMessage(contains="WARNING: Some backups are not valid") # @unittest.expectedFailure # @unittest.skip("skip") @@ -3890,58 +2634,34 @@ def test_validate_truncated_page_header_map(self): """ Check that corruption in page_header_map is detected """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - ok_1 = self.backup_node(backup_dir, 'node', node, options=['--stream']) + ok_1 = self.pb.backup_node('node', node, options=['--stream']) # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) + backup_id = self.pb.backup_node( + 'node', node, options=['--stream']) - ok_2 = self.backup_node(backup_dir, 'node', node, options=['--stream']) + ok_2 = self.pb.backup_node('node', node, options=['--stream']) - page_header_map = os.path.join( - backup_dir, 'backups', 'node', backup_id, 'page_header_map') + self.corrupt_backup_file(self.backup_dir, 'node', backup_id, + 'page_header_map', truncate=121) - # truncate page_header_map file - with open(page_header_map, "rb+", 0) as f: - f.truncate(121) - f.flush() - f.close - - try: - self.validate_pb(backup_dir, 'node', backup_id=backup_id) - self.assertEqual( - 1, 0, - "Expecting Error because page_header is corrupted.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Backup {0} is corrupt'.format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.validate_pb(backup_dir) - self.assertEqual( - 1, 0, - "Expecting Error because page_header is corrupted.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn("INFO: Backup {0} data files are valid".format(ok_1), e.message) - self.assertIn("WARNING: Backup {0} data files are corrupted".format(backup_id), e.message) - self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) - self.assertIn("WARNING: Some backups are not valid", e.message) + self.pb.validate('node', backup_id=backup_id, + expect_error="because page_header_map is corrupted") + self.assertMessage(contains=f'ERROR: Backup {backup_id} is corrupt') + + self.pb.validate(expect_error="because page_header_map is corrupted") + self.assertMessage(contains=f"INFO: Backup {ok_1} data files are valid") + self.assertMessage(contains=f"WARNING: Backup {backup_id} data files are corrupted") + self.assertMessage(contains=f"INFO: Backup {ok_2} data files are valid") + self.assertMessage(contains="WARNING: Some backups are not valid") # @unittest.expectedFailure # @unittest.skip("skip") @@ -3949,55 +2669,34 @@ def test_validate_missing_page_header_map(self): """ Check that corruption in page_header_map is detected """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() - ok_1 = self.backup_node(backup_dir, 'node', node, options=['--stream']) + ok_1 = self.pb.backup_node('node', node, options=['--stream']) # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) + backup_id = self.pb.backup_node( + 'node', node, options=['--stream']) - ok_2 = self.backup_node(backup_dir, 'node', node, options=['--stream']) + ok_2 = self.pb.backup_node('node', node, options=['--stream']) - page_header_map = os.path.join( - backup_dir, 'backups', 'node', backup_id, 'page_header_map') + self.remove_backup_file(self.backup_dir, 'node', backup_id, + 'page_header_map') - # unlink page_header_map file - os.remove(page_header_map) + self.pb.validate('node', backup_id=backup_id, + expect_error="because page_header_map is missing") + self.assertMessage(contains=f'ERROR: Backup {backup_id} is corrupt') - try: - self.validate_pb(backup_dir, 'node', backup_id=backup_id) - self.assertEqual( - 1, 0, - "Expecting Error because page_header is corrupted.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn( - 'ERROR: Backup {0} is corrupt'.format(backup_id), e.message, - '\n Unexpected Error Message: {0}\n CMD: {1}'.format( - repr(e.message), self.cmd)) - - try: - self.validate_pb(backup_dir) - self.assertEqual( - 1, 0, - "Expecting Error because page_header is corrupted.\n " - "Output: {0} \n CMD: {1}".format( - self.output, self.cmd)) - except ProbackupException as e: - self.assertIn("INFO: Backup {0} data files are valid".format(ok_1), e.message) - self.assertIn("WARNING: Backup {0} data files are corrupted".format(backup_id), e.message) - self.assertIn("INFO: Backup {0} data files are valid".format(ok_2), e.message) - self.assertIn("WARNING: Some backups are not valid", e.message) + self.pb.validate(expect_error="because page_header_map is missing") + self.assertMessage(contains=f"INFO: Backup {ok_1} data files are valid") + self.assertMessage(contains=f"WARNING: Backup {backup_id} data files are corrupted") + self.assertMessage(contains=f"INFO: Backup {ok_2} data files are valid") + self.assertMessage(contains="WARNING: Some backups are not valid") # @unittest.expectedFailure # @unittest.skip("skip") @@ -4005,14 +2704,12 @@ def test_no_validate_tablespace_map(self): """ Check that --no-validate is propagated to tablespace_map """ - backup_dir = os.path.join(self.tmp_path, self.module_name, self.fname, 'backup') - node = self.make_simple_node( - base_dir=os.path.join(self.module_name, self.fname, 'node'), - set_replication=True, - initdb_params=['--data-checksums']) - self.init_pb(backup_dir) - self.add_instance(backup_dir, 'node', node) + node = self.pg_node.make_simple('node', + set_replication=True) + + self.pb.init() + self.pb.add_instance('node', node) node.slow_start() self.create_tblspace_in_node(node, 'external_dir') @@ -4028,23 +2725,19 @@ def test_no_validate_tablespace_map(self): "select oid from pg_tablespace where spcname = 'external_dir'").decode('utf-8').rstrip() # FULL backup - backup_id = self.backup_node( - backup_dir, 'node', node, options=['--stream']) + backup_id = self.pb.backup_node( + 'node', node, options=['--stream']) pgdata = self.pgdata_content(node.data_dir) - tablespace_map = os.path.join( - backup_dir, 'backups', 'node', - backup_id, 'database', 'tablespace_map') - - # overwrite tablespace_map file - with open(tablespace_map, "w") as f: - f.write("{0} {1}".format(oid, tblspace_new)) - f.close + self.corrupt_backup_file(self.backup_dir, 'node', backup_id, + 'database/tablespace_map', + overwrite="{0} {1}".format(oid, tblspace_new), + text=True) node.cleanup() - self.restore_node(backup_dir, 'node', node, options=['--no-validate']) + self.pb.restore_node('node', node, options=['--no-validate']) pgdata_restored = self.pgdata_content(node.data_dir) self.compare_pgdata(pgdata, pgdata_restored) @@ -4060,6 +2753,23 @@ def test_no_validate_tablespace_map(self): tblspace_new, "Symlink '{0}' do not points to '{1}'".format(tablespace_link, tblspace_new)) + def test_custom_wal_segsize(self): + """ + Check that we can validate a specific instance or a whole catalog + having a custom wal segment size. + """ + node = self.pg_node.make_simple('node', + initdb_params=['--wal-segsize=64'], + pg_options={'min_wal_size': '128MB'}) + self.pb.init() + self.pb.add_instance('node', node) + node.slow_start() + + self.pb.backup_node('node', node, options=['--stream']) + + self.pb.validate('node') + self.pb.validate() + # validate empty backup list # page from future during validate # page from future during backup From 09da114fe231f4745eb6771a2216ae8c1df1d341 Mon Sep 17 00:00:00 2001 From: vshepard Date: Fri, 3 May 2024 15:37:54 +0200 Subject: [PATCH 2/4] Add s3 tests - fix json parsing in show --- tests/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 30cbcfb8c..32910a3b6 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -6,7 +6,7 @@ # 3. From a local directory # /path/to/local/directory/testgres testgres==1.10.0 -testgres-pg-probackup2==0.0.2 +git+https://github.com/postgrespro/testgres.git@fix-json-parse-in-show#egg=testgres_pg_probackup2&subdirectory=testgres/plugins/pg_probackup2 allure-pytest deprecation minio==7.2.5 From 4239e7bc17cb227b391171e07c6f5e093a41205a Mon Sep 17 00:00:00 2001 From: vshepard Date: Wed, 4 Sep 2024 10:01:31 +0200 Subject: [PATCH 3/4] tests 2.8.2 --- .env | 26 + src/archive.c | 1837 -------------- src/backup.c | 2619 -------------------- src/catalog.c | 3181 ------------------------ src/catchup.c | 1130 --------- src/checkdb.c | 779 ------ src/configure.c | 806 ------- src/data.c | 2524 ------------------- src/datapagemap.c | 113 - src/datapagemap.h | 34 - src/delete.c | 1130 --------- src/dir.c | 1870 --------------- src/fetch.c | 108 - src/help.c | 1146 --------- src/init.c | 127 - src/merge.c | 1441 ----------- src/parsexlog.c | 1972 --------------- src/pg_probackup.c | 1213 ---------- src/pg_probackup.h | 1367 ----------- src/pg_probackup_state.h | 31 - src/ptrack.c | 309 --- src/restore.c | 2253 ----------------- src/show.c | 1176 --------- src/stream.c | 779 ------ src/util.c | 583 ----- src/utils/configuration.c | 1602 ------------- src/utils/configuration.h | 134 -- src/utils/file.c | 3999 ------------------------------- src/utils/file.h | 154 -- src/utils/json.c | 164 -- src/utils/json.h | 33 - src/utils/logger.c | 980 -------- src/utils/logger.h | 69 - src/utils/parray.c | 246 -- src/utils/parray.h | 41 - src/utils/pgut.c | 1375 ----------- src/utils/pgut.h | 128 - src/utils/remote.c | 379 --- src/utils/remote.h | 24 - src/utils/thread.c | 113 - src/utils/thread.h | 41 - src/validate.c | 752 ------ tests/archive_test.py | 1 + tests/auth_test.py | 356 ++- tests/backup_test.py | 14 +- tests/catchup_test.py | 43 +- tests/delete_test.py | 7 +- tests/helpers/ptrack_helpers.py | 7 +- tests/logging_test.py | 6 +- tests/option_test.py | 2 +- tests/page_test.py | 22 +- tests/pbckp1242_test.py | 662 +++++ tests/replica_test.py | 4 +- tests/requirements.txt | 4 +- tests/s3_auth_test.py | 20 + tests/time_consuming_test.py | 2 + tests/validate_test.py | 3 + 57 files changed, 1135 insertions(+), 38806 deletions(-) create mode 100644 .env delete mode 100644 src/archive.c delete mode 100644 src/backup.c delete mode 100644 src/catalog.c delete mode 100644 src/catchup.c delete mode 100644 src/checkdb.c delete mode 100644 src/configure.c delete mode 100644 src/data.c delete mode 100644 src/datapagemap.c delete mode 100644 src/datapagemap.h delete mode 100644 src/delete.c delete mode 100644 src/dir.c delete mode 100644 src/fetch.c delete mode 100644 src/help.c delete mode 100644 src/init.c delete mode 100644 src/merge.c delete mode 100644 src/parsexlog.c delete mode 100644 src/pg_probackup.c delete mode 100644 src/pg_probackup.h delete mode 100644 src/pg_probackup_state.h delete mode 100644 src/ptrack.c delete mode 100644 src/restore.c delete mode 100644 src/show.c delete mode 100644 src/stream.c delete mode 100644 src/util.c delete mode 100644 src/utils/configuration.c delete mode 100644 src/utils/configuration.h delete mode 100644 src/utils/file.c delete mode 100644 src/utils/file.h delete mode 100644 src/utils/json.c delete mode 100644 src/utils/json.h delete mode 100644 src/utils/logger.c delete mode 100644 src/utils/logger.h delete mode 100644 src/utils/parray.c delete mode 100644 src/utils/parray.h delete mode 100644 src/utils/pgut.c delete mode 100644 src/utils/pgut.h delete mode 100644 src/utils/remote.c delete mode 100644 src/utils/remote.h delete mode 100644 src/utils/thread.c delete mode 100644 src/utils/thread.h delete mode 100644 src/validate.c create mode 100644 tests/pbckp1242_test.py create mode 100644 tests/s3_auth_test.py diff --git a/.env b/.env new file mode 100644 index 000000000..54cf812bb --- /dev/null +++ b/.env @@ -0,0 +1,26 @@ +TARGET_OS=ubuntu +TARGET_OS_VERSION=22.04 +PG_PRODUCT=enterprise +PG_REPO=postgrespro +PG_SRCDIR=./postgrespro +PG_PRODUCT_SUFFIX=-ent +PG_VERSION=15 +PG_VERSION_SUFFIX=-15 +PTRACK=ON +PG_PROBACKUP_PTRACK=ON +PGPROBACKUPBIN=/home/vshepard/pbckp/ent-15/bin/pg_probackup +PG_CONFIG=/home/vshepard/pbckp/ent-15/bin/pg_config +PGPROBACKUPBIN3=/home/vshepard/workspace/work/pg_probackup/dev-ee-probackup/pg_probackup3/builddir/src/pg_probackup3 +LANG=C.UTF-8 +LC_ALL=C + +PG_PROBACKUP_S3_HOST=10.5.52.86 +PG_PROBACKUP_S3_PORT=9000 +PG_PROBACKUP_S3_REGION=us-east-1 +PG_PROBACKUP_S3_BUCKET_NAME=test1 +PG_PROBACKUP_S3_ACCESS_KEY=minioadmin +PG_PROBACKUP_S3_SECRET_ACCESS_KEY=minioadmin +PG_PROBACKUP_S3_HTTPS=OFF +PG_PROBACKUP_S3_TEST=minio +PG_PROBACKUP_S3_BUFFER_SIZE=64 +PG_PROBACKUP_S3_RETRIES=10 diff --git a/src/archive.c b/src/archive.c deleted file mode 100644 index 7d753c8b3..000000000 --- a/src/archive.c +++ /dev/null @@ -1,1837 +0,0 @@ -/*------------------------------------------------------------------------- - * - * archive.c: - pg_probackup specific archive commands for archive backups. - * - * - * Portions Copyright (c) 2018-2022, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include -#include "pg_probackup.h" -#include "utils/thread.h" -#include "instr_time.h" - -static void *push_files(void *arg); -static void *get_files(void *arg); -static bool get_wal_file(const char *filename, const char *from_path, const char *to_path, - bool prefetch_mode); -static int get_wal_file_internal(const char *from_path, const char *to_path, FILE *out, - bool is_decompress); -#ifdef HAVE_LIBZ -static const char *get_gz_error(gzFile gzf, int errnum); -#endif -//static void copy_file_attributes(const char *from_path, -// fio_location from_location, -// const char *to_path, fio_location to_location, -// bool unlink_on_error); - -static bool next_wal_segment_exists(TimeLineID tli, XLogSegNo segno, const char *prefetch_dir, uint32 wal_seg_size); -static uint32 run_wal_prefetch(const char *prefetch_dir, const char *archive_dir, TimeLineID tli, - XLogSegNo first_segno, int num_threads, bool inclusive, int batch_size, - uint32 wal_seg_size); -static bool wal_satisfy_from_prefetch(TimeLineID tli, XLogSegNo segno, const char *wal_file_name, - const char *prefetch_dir, const char *absolute_wal_file_path, - uint32 wal_seg_size, bool parse_wal); - -static uint32 maintain_prefetch(const char *prefetch_dir, XLogSegNo first_segno, uint32 wal_seg_size); - -static bool prefetch_stop = false; -static uint32 xlog_seg_size; - -typedef struct -{ - const char *first_filename; - const char *pg_xlog_dir; - const char *archive_dir; - const char *archive_status_dir; - bool overwrite; - bool compress; - bool no_sync; - bool no_ready_rename; - uint32 archive_timeout; - - CompressAlg compress_alg; - int compress_level; - int thread_num; - - parray *files; - - uint32 n_pushed; - uint32 n_skipped; - - /* - * Return value from the thread. - * 0 means there is no error, - * 1 - there is an error. - * 2 - no error, but nothing to push - */ - int ret; -} archive_push_arg; - -typedef struct -{ - const char *prefetch_dir; - const char *archive_dir; - int thread_num; - parray *files; - uint32 n_fetched; -} archive_get_arg; - -typedef struct WALSegno -{ - char name[MAXFNAMELEN]; - volatile pg_atomic_flag lock; - volatile pg_atomic_uint32 done; - struct WALSegno* prev; -} WALSegno; - -static int push_file_internal_uncompressed(WALSegno *wal_file_name, const char *pg_xlog_dir, - const char *archive_dir, bool overwrite, bool no_sync, - uint32 archive_timeout); -#ifdef HAVE_LIBZ -static int push_file_internal_gz(WALSegno *wal_file_name, const char *pg_xlog_dir, - const char *archive_dir, bool overwrite, bool no_sync, - int compress_level, uint32 archive_timeout); -#endif - -static int push_file(WALSegno *xlogfile, const char *archive_status_dir, - const char *pg_xlog_dir, const char *archive_dir, - bool overwrite, bool no_sync, uint32 archive_timeout, - bool no_ready_rename, bool is_compress, - int compress_level); - -static parray *setup_push_filelist(const char *archive_status_dir, - const char *first_file, int batch_size); - -/* - * At this point, we already done one roundtrip to archive server - * to get instance config. - * - * pg_probackup specific archive command for archive backups - * set archive_command to - * 'pg_probackup archive-push -B /home/anastasia/backup --wal-file-name %f', - * to move backups into arclog_path. - * Where archlog_path is $BACKUP_PATH/wal/instance_name - */ -void -do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *pg_xlog_dir, - char *wal_file_name, int batch_size, bool overwrite, - bool no_sync, bool no_ready_rename) -{ - uint64 i; - /* usually instance pgdata/pg_wal/archive_status, empty if no_ready_rename or batch_size == 1 */ - char archive_status_dir[MAXPGPATH] = ""; - bool is_compress = false; - - /* arrays with meta info for multi threaded backup */ - pthread_t *threads; - archive_push_arg *threads_args; - bool push_isok = true; - - /* reporting */ - uint32 n_total_pushed = 0; - uint32 n_total_skipped = 0; - uint32 n_total_failed = 0; - instr_time start_time, end_time; - double push_time; - char pretty_time_str[20]; - - /* files to push in multi-thread mode */ - parray *batch_files = NULL; - int n_threads; - - if (!no_ready_rename || batch_size > 1) - join_path_components(archive_status_dir, pg_xlog_dir, "archive_status"); - -#ifdef HAVE_LIBZ - if (instance->compress_alg == ZLIB_COMPRESS) - is_compress = true; -#endif - - /* Setup filelist and locks */ - batch_files = setup_push_filelist(archive_status_dir, wal_file_name, batch_size); - - n_threads = num_threads; - if (num_threads > parray_num(batch_files)) - n_threads = parray_num(batch_files); - - elog(INFO, "pg_probackup archive-push WAL file: %s, " - "threads: %i/%i, batch: %lu/%i, compression: %s", - wal_file_name, n_threads, num_threads, - parray_num(batch_files), batch_size, - is_compress ? "zlib" : "none"); - - num_threads = n_threads; - - /* Single-thread push - * We don`t want to start multi-thread push, if number of threads in equal to 1, - * or the number of files ready to push is small. - * Multithreading in remote mode isn`t cheap, - * establishing ssh connection can take 100-200ms, so running and terminating - * one thread using generic multithread approach can take - * almost as much time as copying itself. - * TODO: maybe we should be more conservative and force single thread - * push if batch_files array is small. - */ - if (num_threads == 1 || (parray_num(batch_files) == 1)) - { - INSTR_TIME_SET_CURRENT(start_time); - for (i = 0; i < parray_num(batch_files); i++) - { - int rc; - WALSegno *xlogfile = (WALSegno *) parray_get(batch_files, i); - bool first_wal = strcmp(xlogfile->name, wal_file_name) == 0; - - rc = push_file(xlogfile, first_wal ? NULL : archive_status_dir, - pg_xlog_dir, instanceState->instance_wal_subdir_path, - overwrite, no_sync, - instance->archive_timeout, - no_ready_rename || first_wal, - is_compress && IsXLogFileName(xlogfile->name) ? true : false, - instance->compress_level); - if (rc == 0) - n_total_pushed++; - else - n_total_skipped++; - } - - push_isok = true; - goto push_done; - } - - /* init thread args with its own segno */ - threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - threads_args = (archive_push_arg *) palloc(sizeof(archive_push_arg) * num_threads); - - for (i = 0; i < num_threads; i++) - { - archive_push_arg *arg = &(threads_args[i]); - - arg->first_filename = wal_file_name; - arg->archive_dir = instanceState->instance_wal_subdir_path; - arg->pg_xlog_dir = pg_xlog_dir; - arg->archive_status_dir = (!no_ready_rename || batch_size > 1) ? archive_status_dir : NULL; - arg->overwrite = overwrite; - arg->compress = is_compress; - arg->no_sync = no_sync; - arg->no_ready_rename = no_ready_rename; - arg->archive_timeout = instance->archive_timeout; - - arg->compress_alg = instance->compress_alg; - arg->compress_level = instance->compress_level; - - arg->files = batch_files; - arg->n_pushed = 0; - arg->n_skipped = 0; - - arg->thread_num = i+1; - /* By default there are some error */ - arg->ret = 1; - } - - /* Run threads */ - INSTR_TIME_SET_CURRENT(start_time); - for (i = 0; i < num_threads; i++) - { - archive_push_arg *arg = &(threads_args[i]); - pthread_create(&threads[i], NULL, push_files, arg); - } - - /* Wait threads */ - for (i = 0; i < num_threads; i++) - { - pthread_join(threads[i], NULL); - if (threads_args[i].ret == 1) - { - push_isok = false; - n_total_failed++; - } - - n_total_pushed += threads_args[i].n_pushed; - n_total_skipped += threads_args[i].n_skipped; - } - - /* Note, that we are leaking memory here, - * because pushing into archive is a very - * time-sensitive operation, so we skip freeing stuff. - */ - -push_done: - fio_disconnect(); - /* calculate elapsed time */ - INSTR_TIME_SET_CURRENT(end_time); - INSTR_TIME_SUBTRACT(end_time, start_time); - push_time = INSTR_TIME_GET_DOUBLE(end_time); - pretty_time_interval(push_time, pretty_time_str, 20); - - if (push_isok) - /* report number of files pushed into archive */ - elog(INFO, "pg_probackup archive-push completed successfully, " - "pushed: %u, skipped: %u, time elapsed: %s", - n_total_pushed, n_total_skipped, pretty_time_str); - else - elog(ERROR, "pg_probackup archive-push failed, " - "pushed: %i, skipped: %u, failed: %u, time elapsed: %s", - n_total_pushed, n_total_skipped, n_total_failed, - pretty_time_str); -} - -/* ------------- INTERNAL FUNCTIONS ---------- */ -/* - * Copy files from pg_wal to archive catalog with possible compression. - */ -static void * -push_files(void *arg) -{ - int i; - int rc; - archive_push_arg *args = (archive_push_arg *) arg; - - my_thread_num = args->thread_num; - - for (i = 0; i < parray_num(args->files); i++) - { - bool no_ready_rename = args->no_ready_rename; - WALSegno *xlogfile = (WALSegno *) parray_get(args->files, i); - - if (!pg_atomic_test_set_flag(&xlogfile->lock)) - continue; - - /* Do not rename ready file of the first file, - * we do this to avoid flooding PostgreSQL log with - * warnings about ready file been missing. - */ - if (strcmp(args->first_filename, xlogfile->name) == 0) - no_ready_rename = true; - - rc = push_file(xlogfile, args->archive_status_dir, - args->pg_xlog_dir, args->archive_dir, - args->overwrite, args->no_sync, - args->archive_timeout, no_ready_rename, - /* do not compress .backup, .partial and .history files */ - args->compress && IsXLogFileName(xlogfile->name) ? true : false, - args->compress_level); - - if (rc == 0) - args->n_pushed++; - else - args->n_skipped++; - } - - /* close ssh connection */ - fio_disconnect(); - - args->ret = 0; - return NULL; -} - -int -push_file(WALSegno *xlogfile, const char *archive_status_dir, - const char *pg_xlog_dir, const char *archive_dir, - bool overwrite, bool no_sync, uint32 archive_timeout, - bool no_ready_rename, bool is_compress, - int compress_level) -{ - int rc; - - elog(LOG, "pushing file \"%s\"", xlogfile->name); - - /* If compression is not required, then just copy it as is */ - if (!is_compress) - rc = push_file_internal_uncompressed(xlogfile, pg_xlog_dir, - archive_dir, overwrite, no_sync, - archive_timeout); -#ifdef HAVE_LIBZ - else - rc = push_file_internal_gz(xlogfile, pg_xlog_dir, archive_dir, - overwrite, no_sync, compress_level, - archive_timeout); -#endif - - pg_atomic_write_u32(&xlogfile->done, 1); - - /* take '--no-ready-rename' flag into account */ - if (!no_ready_rename && archive_status_dir != NULL) - { - char wal_file_dummy[MAXPGPATH]; - char wal_file_ready[MAXPGPATH]; - char wal_file_done[MAXPGPATH]; - - join_path_components(wal_file_dummy, archive_status_dir, xlogfile->name); - snprintf(wal_file_ready, MAXPGPATH, "%s.%s", wal_file_dummy, "ready"); - snprintf(wal_file_done, MAXPGPATH, "%s.%s", wal_file_dummy, "done"); - - canonicalize_path(wal_file_ready); - canonicalize_path(wal_file_done); - /* It is ok to rename status file in archive_status directory */ - elog(LOG, "Rename \"%s\" to \"%s\"", wal_file_ready, wal_file_done); - - /* do not error out, if rename failed */ - if (fio_rename(wal_file_ready, wal_file_done, FIO_DB_HOST) < 0) - elog(WARNING, "Cannot rename ready file \"%s\" to \"%s\": %s", - wal_file_ready, wal_file_done, strerror(errno)); - } - - return rc; -} - -/* - * Copy non WAL file, such as .backup or .history file, into WAL archive. - * Such files are not compressed. - * Returns: - * 0 - file was successfully pushed - * 1 - push was skipped because file already exists in the archive and - * has the same checksum - */ -int -push_file_internal_uncompressed(WALSegno *wal_file, const char *pg_xlog_dir, - const char *archive_dir, bool overwrite, bool no_sync, - uint32 archive_timeout) -{ - FILE *in = NULL; - int out = -1; - char *buf = pgut_malloc(OUT_BUF_SIZE); /* 1MB buffer */ - const char *wal_file_name = wal_file->name; - char from_fullpath[MAXPGPATH]; - char to_fullpath[MAXPGPATH]; - /* partial handling */ - struct stat st; - char to_fullpath_part[MAXPGPATH]; - int partial_try_count = 0; - int partial_file_size = 0; - bool partial_is_stale = true; - /* remote agent error message */ - char *errmsg = NULL; - - /* from path */ - join_path_components(from_fullpath, pg_xlog_dir, wal_file_name); - canonicalize_path(from_fullpath); - /* to path */ - join_path_components(to_fullpath, archive_dir, wal_file_name); - canonicalize_path(to_fullpath); - - /* Open source file for read */ - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - { - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Cannot open source file \"%s\": %s", from_fullpath, strerror(errno)); - } - - /* disable stdio buffering for input file */ - setvbuf(in, NULL, _IONBF, BUFSIZ); - - /* open destination partial file for write */ - snprintf(to_fullpath_part, sizeof(to_fullpath_part), "%s.part", to_fullpath); - - /* Grab lock by creating temp file in exclusive mode */ - out = fio_open(to_fullpath_part, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); - if (out < 0) - { - if (errno != EEXIST) - { - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Failed to open temp WAL file \"%s\": %s", - to_fullpath_part, strerror(errno)); - } - /* Already existing destination temp file is not an error condition */ - } - else - goto part_opened; - - /* - * Partial file already exists, it could have happened due to: - * 1. failed archive-push - * 2. concurrent archiving - * - * For ARCHIVE_TIMEOUT period we will try to create partial file - * and look for the size of already existing partial file, to - * determine if it is changing or not. - * If after ARCHIVE_TIMEOUT we still failed to create partial - * file, we will make a decision about discarding - * already existing partial file. - */ - - while (partial_try_count < archive_timeout) - { - if (fio_stat(to_fullpath_part, &st, false, FIO_BACKUP_HOST) < 0) - { - if (errno == ENOENT) - { - //part file is gone, lets try to grab it - out = fio_open(to_fullpath_part, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); - if (out < 0) - { - if (errno != EEXIST) - { - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Failed to open temp WAL file \"%s\": %s", - to_fullpath_part, strerror(errno)); - } - } - else - /* Successfully created partial file */ - break; - } - else - { - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Cannot stat temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); - } - } - - /* first round */ - if (!partial_try_count) - { - elog(LOG, "Temp WAL file already exists, waiting on it %u seconds: \"%s\"", - archive_timeout, to_fullpath_part); - partial_file_size = st.st_size; - } - - /* file size is changing */ - if (st.st_size > partial_file_size) - partial_is_stale = false; - - sleep(1); - partial_try_count++; - } - /* The possible exit conditions: - * 1. File is grabbed - * 2. File is not grabbed, and it is not stale - * 2. File is not grabbed, and it is stale. - */ - - /* - * If temp file was not grabbed for ARCHIVE_TIMEOUT and temp file is not stale, - * then exit with error. - */ - if (out < 0) - { - if (!partial_is_stale) - { - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Failed to open temp WAL file \"%s\" in %i seconds", - to_fullpath_part, archive_timeout); - } - - /* Partial segment is considered stale, so reuse it */ - elog(LOG, "Reusing stale temp WAL file \"%s\"", to_fullpath_part); - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - - out = fio_open(to_fullpath_part, O_RDWR | O_CREAT | O_EXCL | PG_BINARY, FIO_BACKUP_HOST); - if (out < 0) - { - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Cannot open temp WAL file \"%s\": %s", to_fullpath_part, strerror(errno)); - } - } - -part_opened: - elog(LOG, "Temp WAL file successfully created: \"%s\"", to_fullpath_part); - /* Check if possible to skip copying */ - if (fileExists(to_fullpath, FIO_BACKUP_HOST)) - { - pg_crc32 crc32_src; - pg_crc32 crc32_dst; - - crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, false); - crc32_dst = fio_get_crc32(to_fullpath, FIO_BACKUP_HOST, false, false); - - if (crc32_src == crc32_dst) - { - elog(LOG, "WAL file already exists in archive with the same " - "checksum, skip pushing: \"%s\"", from_fullpath); - /* cleanup */ - fclose(in); - fio_close(out); - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - return 1; - } - else - { - if (overwrite) - elog(LOG, "WAL file already exists in archive with " - "different checksum, overwriting: \"%s\"", to_fullpath); - else - { - /* Overwriting is forbidden, - * so we must unlink partial file and exit with error. - */ - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "WAL file already exists in archive with " - "different checksum: \"%s\"", to_fullpath); - } - } - } - - /* copy content */ - errno = 0; - for (;;) - { - size_t read_len = 0; - - read_len = fread(buf, 1, OUT_BUF_SIZE, in); - - if (ferror(in)) - { - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Cannot read source file \"%s\": %s", - from_fullpath, strerror(errno)); - } - - if (read_len > 0 && fio_write_async(out, buf, read_len) != read_len) - { - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Cannot write to destination temp file \"%s\": %s", - to_fullpath_part, strerror(errno)); - } - - if (feof(in)) - break; - } - - /* close source file */ - fclose(in); - - /* Writing is asynchronous in case of push in remote mode, so check agent status */ - if (fio_check_error_fd(out, &errmsg)) - { - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Cannot write to the remote file \"%s\": %s", - to_fullpath_part, errmsg); - } - - if (wal_file->prev != NULL) - { - while (!pg_atomic_read_u32(&wal_file->prev->done)) - { - if (thread_interrupted || interrupted) - { - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Terminated while waiting for prev file"); - } - usleep(250); - } - } - - /* close temp file */ - if (fio_close(out) != 0) - { - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Cannot close temp WAL file \"%s\": %s", - to_fullpath_part, strerror(errno)); - } - - /* sync temp file to disk */ - if (!no_sync) - { - if (fio_sync(to_fullpath_part, FIO_BACKUP_HOST) != 0) - { - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Failed to sync file \"%s\": %s", - to_fullpath_part, strerror(errno)); - } - } - - elog(LOG, "Rename \"%s\" to \"%s\"", to_fullpath_part, to_fullpath); - - //copy_file_attributes(from_path, FIO_DB_HOST, to_path_temp, FIO_BACKUP_HOST, true); - - /* Rename temp file to destination file */ - if (fio_rename(to_fullpath_part, to_fullpath, FIO_BACKUP_HOST) < 0) - { - fio_unlink(to_fullpath_part, FIO_BACKUP_HOST); - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", - to_fullpath_part, to_fullpath, strerror(errno)); - } - - pg_free(buf); - return 0; -} - -#ifdef HAVE_LIBZ -/* - * Push WAL segment into archive and apply streaming compression to it. - * Returns: - * 0 - file was successfully pushed - * 1 - push was skipped because file already exists in the archive and - * has the same checksum - */ -int -push_file_internal_gz(WALSegno *wal_file, const char *pg_xlog_dir, - const char *archive_dir, bool overwrite, bool no_sync, - int compress_level, uint32 archive_timeout) -{ - FILE *in = NULL; - gzFile out = NULL; - char *buf = pgut_malloc(OUT_BUF_SIZE); - const char *wal_file_name = wal_file->name; - char from_fullpath[MAXPGPATH]; - char to_fullpath[MAXPGPATH]; - char to_fullpath_gz[MAXPGPATH]; - - /* partial handling */ - struct stat st; - - char to_fullpath_gz_part[MAXPGPATH]; - int partial_try_count = 0; - int partial_file_size = 0; - bool partial_is_stale = true; - /* remote agent errormsg */ - char *errmsg = NULL; - - /* from path */ - join_path_components(from_fullpath, pg_xlog_dir, wal_file_name); - canonicalize_path(from_fullpath); - /* to path */ - join_path_components(to_fullpath, archive_dir, wal_file_name); - canonicalize_path(to_fullpath); - - /* destination file with .gz suffix */ - snprintf(to_fullpath_gz, sizeof(to_fullpath_gz), "%s.gz", to_fullpath); - /* destination temp file */ - snprintf(to_fullpath_gz_part, sizeof(to_fullpath_gz_part), "%s.part", to_fullpath_gz); - - /* Open source file for read */ - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - { - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Cannot open source WAL file \"%s\": %s", - from_fullpath, strerror(errno)); - } - - /* disable stdio buffering for input file */ - setvbuf(in, NULL, _IONBF, BUFSIZ); - - /* Grab lock by creating temp file in exclusive mode */ - out = fio_gzopen(to_fullpath_gz_part, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); - if (out == NULL) - { - if (errno != EEXIST) - { - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Cannot open temp WAL file \"%s\": %s", - to_fullpath_gz_part, strerror(errno)); - } - /* Already existing destination temp file is not an error condition */ - } - else - goto part_opened; - - /* - * Partial file already exists, it could have happened due to: - * 1. failed archive-push - * 2. concurrent archiving - * - * For ARCHIVE_TIMEOUT period we will try to create partial file - * and look for the size of already existing partial file, to - * determine if it is changing or not. - * If after ARCHIVE_TIMEOUT we still failed to create partial - * file, we will make a decision about discarding - * already existing partial file. - */ - - while (partial_try_count < archive_timeout) - { - if (fio_stat(to_fullpath_gz_part, &st, false, FIO_BACKUP_HOST) < 0) - { - if (errno == ENOENT) - { - //part file is gone, lets try to grab it - out = fio_gzopen(to_fullpath_gz_part, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); - if (out == NULL) - { - if (errno != EEXIST) - { - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Failed to open temp WAL file \"%s\": %s", - to_fullpath_gz_part, strerror(errno)); - } - } - else - /* Successfully created partial file */ - break; - } - else - { - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Cannot stat temp WAL file \"%s\": %s", - to_fullpath_gz_part, strerror(errno)); - } - } - - /* first round */ - if (!partial_try_count) - { - elog(LOG, "Temp WAL file already exists, waiting on it %u seconds: \"%s\"", - archive_timeout, to_fullpath_gz_part); - partial_file_size = st.st_size; - } - - /* file size is changing */ - if (st.st_size > partial_file_size) - partial_is_stale = false; - - sleep(1); - partial_try_count++; - } - /* The possible exit conditions: - * 1. File is grabbed - * 2. File is not grabbed, and it is not stale - * 2. File is not grabbed, and it is stale. - */ - - /* - * If temp file was not grabbed for ARCHIVE_TIMEOUT and temp file is not stale, - * then exit with error. - */ - if (out == NULL) - { - if (!partial_is_stale) - { - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Failed to open temp WAL file \"%s\" in %i seconds", - to_fullpath_gz_part, archive_timeout); - } - - /* Partial segment is considered stale, so reuse it */ - elog(LOG, "Reusing stale temp WAL file \"%s\"", to_fullpath_gz_part); - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - - out = fio_gzopen(to_fullpath_gz_part, PG_BINARY_W, compress_level, FIO_BACKUP_HOST); - if (out == NULL) - { - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Cannot open temp WAL file \"%s\": %s", - to_fullpath_gz_part, strerror(errno)); - } - } - -part_opened: - elog(LOG, "Temp WAL file successfully created: \"%s\"", to_fullpath_gz_part); - /* Check if possible to skip copying, - */ - if (fileExists(to_fullpath_gz, FIO_BACKUP_HOST)) - { - pg_crc32 crc32_src; - pg_crc32 crc32_dst; - - crc32_src = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, false); - crc32_dst = fio_get_crc32(to_fullpath_gz, FIO_BACKUP_HOST, true, false); - - if (crc32_src == crc32_dst) - { - elog(LOG, "WAL file already exists in archive with the same " - "checksum, skip pushing: \"%s\"", from_fullpath); - /* cleanup */ - fclose(in); - fio_gzclose(out); - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - return 1; - } - else - { - if (overwrite) - elog(LOG, "WAL file already exists in archive with " - "different checksum, overwriting: \"%s\"", to_fullpath_gz); - else - { - /* Overwriting is forbidden, - * so we must unlink partial file and exit with error. - */ - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "WAL file already exists in archive with " - "different checksum: \"%s\"", to_fullpath_gz); - } - } - } - - /* copy content */ - /* TODO: move to separate function */ - for (;;) - { - size_t read_len = 0; - - read_len = fread(buf, 1, OUT_BUF_SIZE, in); - - if (ferror(in)) - { - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Cannot read from source file \"%s\": %s", - from_fullpath, strerror(errno)); - } - - if (read_len > 0 && fio_gzwrite(out, buf, read_len) != read_len) - { - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Cannot write to compressed temp WAL file \"%s\": %s", - to_fullpath_gz_part, get_gz_error(out, errno)); - } - - if (feof(in)) - break; - } - - /* close source file */ - fclose(in); - - /* Writing is asynchronous in case of push in remote mode, so check agent status */ - if (fio_check_error_fd_gz(out, &errmsg)) - { - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Cannot write to the remote compressed file \"%s\": %s", - to_fullpath_gz_part, errmsg); - } - - if (wal_file->prev != NULL) - { - while (!pg_atomic_read_u32(&wal_file->prev->done)) - { - if (thread_interrupted || interrupted) - { - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Terminated while waiting for prev file"); - } - usleep(250); - } - } - - /* close temp file, TODO: make it synchronous */ - if (fio_gzclose(out) != 0) - { - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Cannot close compressed temp WAL file \"%s\": %s", - to_fullpath_gz_part, strerror(errno)); - } - - /* sync temp file to disk */ - if (!no_sync) - { - if (fio_sync(to_fullpath_gz_part, FIO_BACKUP_HOST) != 0) - { - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Failed to sync file \"%s\": %s", - to_fullpath_gz_part, strerror(errno)); - } - } - - elog(LOG, "Rename \"%s\" to \"%s\"", - to_fullpath_gz_part, to_fullpath_gz); - - //copy_file_attributes(from_path, FIO_DB_HOST, to_path_temp, FIO_BACKUP_HOST, true); - - /* Rename temp file to destination file */ - if (fio_rename(to_fullpath_gz_part, to_fullpath_gz, FIO_BACKUP_HOST) < 0) - { - fio_unlink(to_fullpath_gz_part, FIO_BACKUP_HOST); - pg_atomic_write_u32(&wal_file->done, 1); - elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", - to_fullpath_gz_part, to_fullpath_gz, strerror(errno)); - } - - pg_free(buf); - - return 0; -} -#endif - -#ifdef HAVE_LIBZ -/* - * Show error during work with compressed file - */ -static const char * -get_gz_error(gzFile gzf, int errnum) -{ - int gz_errnum; - const char *errmsg; - - errmsg = fio_gzerror(gzf, &gz_errnum); - if (gz_errnum == Z_ERRNO) - return strerror(errnum); - else - return errmsg; -} -#endif - -/* Copy file attributes */ -//static void -//copy_file_attributes(const char *from_path, fio_location from_location, -// const char *to_path, fio_location to_location, -// bool unlink_on_error) -//{ -// struct stat st; -// -// if (fio_stat(from_path, &st, true, from_location) == -1) -// { -// if (unlink_on_error) -// fio_unlink(to_path, to_location); -// elog(ERROR, "Cannot stat file \"%s\": %s", -// from_path, strerror(errno)); -// } -// -// if (fio_chmod(to_path, st.st_mode, to_location) == -1) -// { -// if (unlink_on_error) -// fio_unlink(to_path, to_location); -// elog(ERROR, "Cannot change mode of file \"%s\": %s", -// to_path, strerror(errno)); -// } -//} - -static int -walSegnoCompareName(const void *f1, const void *f2) -{ - WALSegno *w1 = *(WALSegno**)f1; - WALSegno *w2 = *(WALSegno**)f2; - - return strcmp(w1->name, w2->name); -} - -/* Look for files with '.ready' suffix in archive_status directory - * and pack such files into batch sized array. - */ -parray * -setup_push_filelist(const char *archive_status_dir, const char *first_file, - int batch_size) -{ - WALSegno *xlogfile = NULL; - parray *status_files = NULL; - parray *batch_files = parray_new(); - size_t i; - - /* guarantee that first filename is in batch list */ - xlogfile = palloc0(sizeof(WALSegno)); - pg_atomic_init_flag(&xlogfile->lock); - pg_atomic_init_u32(&xlogfile->done, 0); - snprintf(xlogfile->name, MAXFNAMELEN, "%s", first_file); - parray_append(batch_files, xlogfile); - - if (batch_size < 2) - return batch_files; - - /* get list of files from archive_status */ - status_files = parray_new(); - dir_list_file(status_files, archive_status_dir, false, false, false, false, true, 0, FIO_DB_HOST); - parray_qsort(status_files, pgFileCompareName); - - for (i = 0; i < parray_num(status_files); i++) - { - int result = 0; - char filename[MAXFNAMELEN]; - char suffix[MAXFNAMELEN]; - pgFile *file = (pgFile *) parray_get(status_files, i); - - result = sscanf(file->name, "%[^.]%s", (char *) &filename, (char *) &suffix); - - if (result != 2) - continue; - - if (strcmp(suffix, ".ready") != 0) - continue; - - /* first filename already in batch list */ - if (strcmp(filename, first_file) == 0) - continue; - - xlogfile = palloc0(sizeof(WALSegno)); - pg_atomic_init_flag(&xlogfile->lock); - pg_atomic_init_u32(&xlogfile->done, 0); - - snprintf(xlogfile->name, MAXFNAMELEN, "%s", filename); - parray_append(batch_files, xlogfile); - - if (parray_num(batch_files) >= batch_size) - break; - } - - parray_qsort(batch_files, walSegnoCompareName); - for (i = 1; i < parray_num(batch_files); i++) - { - xlogfile = (WALSegno*) parray_get(batch_files, i); - xlogfile->prev = (WALSegno*) parray_get(batch_files, i-1); - } - - /* cleanup */ - parray_walk(status_files, pgFileFree); - parray_free(status_files); - - return batch_files; -} - -/* - * pg_probackup specific restore command. - * Move files from arclog_path to pgdata/wal_file_path. - * - * The problem with archive-get: we must be very careful about - * erroring out, because postgres will interpretent our negative exit code - * as the fact, that requested file is missing and may take irreversible actions. - * So if file copying has failed we must retry several times before bailing out. - * - * TODO: add support of -D option. - * TOTHINK: what can be done about ssh connection been broken? - * TOTHINk: do we need our own rmtree function ? - * TOTHINk: so sort of async prefetch ? - - */ -void -do_archive_get(InstanceState *instanceState, InstanceConfig *instance, const char *prefetch_dir_arg, - char *wal_file_path, char *wal_file_name, int batch_size, - bool validate_wal) -{ - int fail_count = 0; - char backup_wal_file_path[MAXPGPATH]; - char absolute_wal_file_path[MAXPGPATH]; - char current_dir[MAXPGPATH]; - char prefetch_dir[MAXPGPATH]; - char pg_xlog_dir[MAXPGPATH]; - char prefetched_file[MAXPGPATH]; - - /* reporting */ - uint32 n_fetched = 0; - int n_actual_threads = num_threads; - uint32 n_files_in_prefetch = 0; - - /* time reporting */ - instr_time start_time, end_time; - double get_time; - char pretty_time_str[20]; - - if (wal_file_name == NULL) - elog(ERROR, "Required parameter not specified: --wal-file-name %%f"); - - if (wal_file_path == NULL) - elog(ERROR, "Required parameter not specified: --wal_file_path %%p"); - - if (!getcwd(current_dir, sizeof(current_dir))) - elog(ERROR, "getcwd() error"); - - /* path to PGDATA/pg_wal directory */ - join_path_components(pg_xlog_dir, current_dir, XLOGDIR); - - /* destination full filepath, usually it is PGDATA/pg_wal/RECOVERYXLOG */ - join_path_components(absolute_wal_file_path, current_dir, wal_file_path); - - /* full filepath to WAL file in archive directory. - * $BACKUP_PATH/wal/instance_name/000000010000000000000001 */ - join_path_components(backup_wal_file_path, instanceState->instance_wal_subdir_path, wal_file_name); - - INSTR_TIME_SET_CURRENT(start_time); - if (num_threads > batch_size) - n_actual_threads = batch_size; - elog(INFO, "pg_probackup archive-get WAL file: %s, remote: %s, threads: %i/%i, batch: %i", - wal_file_name, IsSshProtocol() ? "ssh" : "none", n_actual_threads, num_threads, batch_size); - - num_threads = n_actual_threads; - - elog(VERBOSE, "Obtaining XLOG_SEG_SIZE from pg_control file"); - instance->xlog_seg_size = get_xlog_seg_size(current_dir); - - /* Prefetch optimization kicks in only if simple XLOG segments is requested - * and batching is enabled. - * - * We check that file do exists in prefetch directory, then we validate it and - * rename to destination path. - * If file do not exists, then we run prefetch and rename it. - */ - if (IsXLogFileName(wal_file_name) && batch_size > 1) - { - XLogSegNo segno; - TimeLineID tli; - - GetXLogFromFileName(wal_file_name, &tli, &segno, instance->xlog_seg_size); - - if (prefetch_dir_arg) - /* use provided prefetch directory */ - snprintf(prefetch_dir, sizeof(prefetch_dir), "%s", prefetch_dir_arg); - else - /* use default path */ - join_path_components(prefetch_dir, pg_xlog_dir, "pbk_prefetch"); - - /* Construct path to WAL file in prefetch directory. - * current_dir/pg_wal/pbk_prefech/000000010000000000000001 - */ - join_path_components(prefetched_file, prefetch_dir, wal_file_name); - - /* check if file is available in prefetch directory */ - if (access(prefetched_file, F_OK) == 0) - { - /* Prefetched WAL segment is available, before using it, we must validate it. - * But for validation to work properly(because of contrecord), we must be sure - * that next WAL segment is also available in prefetch directory. - * If next segment do not exists in prefetch directory, we must provide it from - * archive. If it is NOT available in the archive, then file in prefetch directory - * cannot be trusted. In this case we discard all prefetched files and - * copy requested file directly from archive. - */ - if (!next_wal_segment_exists(tli, segno, prefetch_dir, instance->xlog_seg_size)) - n_fetched = run_wal_prefetch(prefetch_dir, instanceState->instance_wal_subdir_path, - tli, segno, num_threads, false, batch_size, - instance->xlog_seg_size); - - n_files_in_prefetch = maintain_prefetch(prefetch_dir, segno, instance->xlog_seg_size); - - if (wal_satisfy_from_prefetch(tli, segno, wal_file_name, prefetch_dir, - absolute_wal_file_path, instance->xlog_seg_size, - validate_wal)) - { - n_files_in_prefetch--; - elog(INFO, "pg_probackup archive-get used prefetched WAL segment %s, prefetch state: %u/%u", - wal_file_name, n_files_in_prefetch, batch_size); - goto get_done; - } - else - { - /* discard prefetch */ -// n_fetched = 0; - pgut_rmtree(prefetch_dir, false, false); - } - } - else - { - /* Do prefetch maintenance here */ - - mkdir(prefetch_dir, DIR_PERMISSION); /* In case prefetch directory do not exists yet */ - - /* We`ve failed to satisfy current request from prefetch directory, - * therefore we can discard its content, since it may be corrupted or - * contain stale files. - * - * UPDATE: we should not discard prefetch easily, because failing to satisfy - * request for WAL may come from this recovery behavior: - * https://www.postgresql.org/message-id/flat/16159-f5a34a3a04dc67e0%40postgresql.org - */ -// rmtree(prefetch_dir, false); - - /* prefetch files */ - n_fetched = run_wal_prefetch(prefetch_dir, instanceState->instance_wal_subdir_path, - tli, segno, num_threads, true, batch_size, - instance->xlog_seg_size); - - n_files_in_prefetch = maintain_prefetch(prefetch_dir, segno, instance->xlog_seg_size); - - if (wal_satisfy_from_prefetch(tli, segno, wal_file_name, prefetch_dir, absolute_wal_file_path, - instance->xlog_seg_size, validate_wal)) - { - n_files_in_prefetch--; - elog(INFO, "pg_probackup archive-get copied WAL file %s, prefetch state: %u/%u", - wal_file_name, n_files_in_prefetch, batch_size); - goto get_done; - } -// else -// { -// /* yet again failed to satisfy request from prefetch */ -// n_fetched = 0; -// rmtree(prefetch_dir, false); -// } - } - } - - /* we use it to extend partial file later */ - xlog_seg_size = instance->xlog_seg_size; - - /* Either prefetch didn`t cut it, or batch mode is disabled or - * the requested file is not WAL segment. - * Copy file from the archive directly. - * Retry several times before bailing out. - * - * TODO: - * files copied from archive directly are not validated, which is not ok. - * TOTHINK: - * Current WAL validation cannot be applied to partial files. - */ - - while (fail_count < 3) - { - if (get_wal_file(wal_file_name, backup_wal_file_path, absolute_wal_file_path, false)) - { - fail_count = 0; - elog(LOG, "pg_probackup archive-get copied WAL file %s", wal_file_name); - n_fetched++; - break; - } - else - fail_count++; - - elog(LOG, "Failed to get WAL file %s, retry %i/3", wal_file_name, fail_count); - } - - /* TODO/TOTHINK: - * If requested file is corrupted, we have no way to warn PostgreSQL about it. - * We either can: - * 1. feed to recovery and let PostgreSQL sort it out. Currently we do this. - * 2. error out. - * - * Also note, that we can detect corruption only if prefetch mode is used. - * TODO: if corruption or network problem encountered, kill yourself - * with SIGTERN to prevent recovery from starting up database. - */ - -get_done: - INSTR_TIME_SET_CURRENT(end_time); - INSTR_TIME_SUBTRACT(end_time, start_time); - get_time = INSTR_TIME_GET_DOUBLE(end_time); - pretty_time_interval(get_time, pretty_time_str, 20); - - if (fail_count == 0) - elog(INFO, "pg_probackup archive-get completed successfully, fetched: %i/%i, time elapsed: %s", - n_fetched, batch_size, pretty_time_str); - else - elog(ERROR, "pg_probackup archive-get failed to deliver WAL file: %s, time elapsed: %s", - wal_file_name, pretty_time_str); -} - -/* - * Copy batch_size of regular WAL segments into prefetch directory, - * starting with first_file. - * - * inclusive - should we copy first_file or not. - */ -uint32 run_wal_prefetch(const char *prefetch_dir, const char *archive_dir, - TimeLineID tli, XLogSegNo first_segno, int num_threads, - bool inclusive, int batch_size, uint32 wal_seg_size) -{ - int i; - XLogSegNo segno; - parray *batch_files = parray_new(); - int n_total_fetched = 0; - - if (!inclusive) - first_segno++; - - for (segno = first_segno; segno < (first_segno + batch_size); segno++) - { - WALSegno *xlogfile = palloc(sizeof(WALSegno)); - pg_atomic_init_flag(&xlogfile->lock); - - /* construct filename for WAL segment */ - GetXLogFileName(xlogfile->name, tli, segno, wal_seg_size); - - parray_append(batch_files, xlogfile); - - } - - /* copy segments */ - if (num_threads == 1) - { - for (i = 0; i < parray_num(batch_files); i++) - { - char to_fullpath[MAXPGPATH]; - char from_fullpath[MAXPGPATH]; - WALSegno *xlogfile = (WALSegno *) parray_get(batch_files, i); - - join_path_components(to_fullpath, prefetch_dir, xlogfile->name); - join_path_components(from_fullpath, archive_dir, xlogfile->name); - - /* It is ok, maybe requested batch is greater than the number of available - * files in the archive - */ - if (!get_wal_file(xlogfile->name, from_fullpath, to_fullpath, true)) - { - elog(LOG, "Thread [%d]: Failed to prefetch WAL segment %s", 0, xlogfile->name); - break; - } - - n_total_fetched++; - } - } - else - { - /* arrays with meta info for multi threaded archive-get */ - pthread_t *threads; - archive_get_arg *threads_args; - - /* init thread args */ - threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - threads_args = (archive_get_arg *) palloc(sizeof(archive_get_arg) * num_threads); - - for (i = 0; i < num_threads; i++) - { - archive_get_arg *arg = &(threads_args[i]); - - arg->prefetch_dir = prefetch_dir; - arg->archive_dir = archive_dir; - - arg->thread_num = i+1; - arg->files = batch_files; - arg->n_fetched = 0; - } - - /* Run threads */ - for (i = 0; i < num_threads; i++) - { - archive_get_arg *arg = &(threads_args[i]); - pthread_create(&threads[i], NULL, get_files, arg); - } - - /* Wait threads */ - for (i = 0; i < num_threads; i++) - { - pthread_join(threads[i], NULL); - n_total_fetched += threads_args[i].n_fetched; - } - } - /* TODO: free batch_files */ - return n_total_fetched; -} - -/* - * Copy files from archive catalog to pg_wal. - */ -static void * -get_files(void *arg) -{ - int i; - char to_fullpath[MAXPGPATH]; - char from_fullpath[MAXPGPATH]; - archive_get_arg *args = (archive_get_arg *) arg; - - my_thread_num = args->thread_num; - - for (i = 0; i < parray_num(args->files); i++) - { - WALSegno *xlogfile = (WALSegno *) parray_get(args->files, i); - - if (prefetch_stop) - break; - - if (!pg_atomic_test_set_flag(&xlogfile->lock)) - continue; - - join_path_components(from_fullpath, args->archive_dir, xlogfile->name); - join_path_components(to_fullpath, args->prefetch_dir, xlogfile->name); - - if (!get_wal_file(xlogfile->name, from_fullpath, to_fullpath, true)) - { - /* It is ok, maybe requested batch is greater than the number of available - * files in the archive - */ - elog(LOG, "Failed to prefetch WAL segment %s", xlogfile->name); - prefetch_stop = true; - break; - } - - args->n_fetched++; - } - - /* close ssh connection */ - fio_disconnect(); - - return NULL; -} - -/* - * Copy WAL segment from archive catalog to pgdata with possible decompression. - * When running in prefetch mode, we should not error out. - */ -bool -get_wal_file(const char *filename, const char *from_fullpath, - const char *to_fullpath, bool prefetch_mode) -{ - int rc = FILE_MISSING; - FILE *out; - char from_fullpath_gz[MAXPGPATH]; - bool src_partial = false; - - snprintf(from_fullpath_gz, sizeof(from_fullpath_gz), "%s.gz", from_fullpath); - - /* open destination file */ - out = fopen(to_fullpath, PG_BINARY_W); - if (!out) - { - elog(WARNING, "Failed to open file '%s': %s", - to_fullpath, strerror(errno)); - return false; - } - - if (chmod(to_fullpath, FILE_PERMISSION) == -1) - { - elog(WARNING, "Cannot change mode of file '%s': %s", - to_fullpath, strerror(errno)); - fclose(out); - unlink(to_fullpath); - return false; - } - - /* disable buffering for output file */ - setvbuf(out, NULL, _IONBF, BUFSIZ); - - /* In prefetch mode, we do look only for full WAL segments - * In non-prefetch mode, do look up '.partial' and '.gz.partial' - * segments. - */ - if (fio_is_remote(FIO_BACKUP_HOST)) - { - char *errmsg = NULL; - /* get file via ssh */ -#ifdef HAVE_LIBZ - /* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */ - if (IsXLogFileName(filename)) - rc = fio_send_file_gz(from_fullpath_gz, out, &errmsg); - if (rc == FILE_MISSING) -#endif - /* ... failing that, use uncompressed */ - rc = fio_send_file(from_fullpath, out, false, NULL, &errmsg); - - /* When not in prefetch mode, try to use partial file */ - if (rc == FILE_MISSING && !prefetch_mode && IsXLogFileName(filename)) - { - char from_partial[MAXPGPATH]; - -#ifdef HAVE_LIBZ - /* '.gz.partial' goes first ... */ - snprintf(from_partial, sizeof(from_partial), "%s.gz.partial", from_fullpath); - rc = fio_send_file_gz(from_partial, out, &errmsg); - if (rc == FILE_MISSING) -#endif - { - /* ... failing that, use '.partial' */ - snprintf(from_partial, sizeof(from_partial), "%s.partial", from_fullpath); - rc = fio_send_file(from_partial, out, false, NULL, &errmsg); - } - - if (rc == SEND_OK) - src_partial = true; - } - - if (rc == WRITE_FAILED) - elog(WARNING, "Cannot write to file '%s': %s", - to_fullpath, strerror(errno)); - - if (errmsg) - elog(WARNING, "%s", errmsg); - - pg_free(errmsg); - } - else - { - /* get file locally */ -#ifdef HAVE_LIBZ - /* If requested file is regular WAL segment, then try to open it with '.gz' suffix... */ - if (IsXLogFileName(filename)) - rc = get_wal_file_internal(from_fullpath_gz, to_fullpath, out, true); - if (rc == FILE_MISSING) -#endif - /* ... failing that, use uncompressed */ - rc = get_wal_file_internal(from_fullpath, to_fullpath, out, false); - - /* When not in prefetch mode, try to use partial file */ - if (rc == FILE_MISSING && !prefetch_mode && IsXLogFileName(filename)) - { - char from_partial[MAXPGPATH]; - -#ifdef HAVE_LIBZ - /* '.gz.partial' goes first ... */ - snprintf(from_partial, sizeof(from_partial), "%s.gz.partial", from_fullpath); - rc = get_wal_file_internal(from_partial, to_fullpath, out, true); - if (rc == FILE_MISSING) -#endif - { - /* ... failing that, use '.partial' */ - snprintf(from_partial, sizeof(from_partial), "%s.partial", from_fullpath); - rc = get_wal_file_internal(from_partial, to_fullpath, out, false); - } - - if (rc == SEND_OK) - src_partial = true; - } - } - - if (!prefetch_mode && (rc == FILE_MISSING)) - elog(LOG, "Target WAL file is missing: %s", filename); - - if (rc < 0) - { - fclose(out); - unlink(to_fullpath); - return false; - } - - /* If partial file was used as source, then it is very likely that destination - * file is not equal to XLOG_SEG_SIZE - that is the way pg_receivexlog works. - * We must manually extent it up to XLOG_SEG_SIZE. - */ - if (src_partial) - { - - if (fflush(out) != 0) - { - elog(WARNING, "Cannot flush file \"%s\": %s", to_fullpath, strerror(errno)); - fclose(out); - unlink(to_fullpath); - return false; - } - - if (ftruncate(fileno(out), xlog_seg_size) != 0) - { - elog(WARNING, "Cannot extend file \"%s\": %s", to_fullpath, strerror(errno)); - fclose(out); - unlink(to_fullpath); - return false; - } - } - - if (fclose(out) != 0) - { - elog(WARNING, "Cannot close file '%s': %s", to_fullpath, strerror(errno)); - unlink(to_fullpath); - return false; - } - - elog(LOG, "WAL file successfully %s: %s", - prefetch_mode ? "prefetched" : "copied", filename); - return true; -} - -/* - * Copy WAL segment with possible decompression from local archive. - * Return codes: - * FILE_MISSING (-1) - * OPEN_FAILED (-2) - * READ_FAILED (-3) - * WRITE_FAILED (-4) - * ZLIB_ERROR (-5) - */ -int -get_wal_file_internal(const char *from_path, const char *to_path, FILE *out, - bool is_decompress) -{ -#ifdef HAVE_LIBZ - gzFile gz_in = NULL; -#endif - FILE *in = NULL; - char *buf = pgut_malloc(OUT_BUF_SIZE); /* 1MB buffer */ - int exit_code = 0; - - elog(LOG, "Attempting to %s WAL file '%s'", - is_decompress ? "open compressed" : "open", from_path); - - /* open source file for read */ - if (!is_decompress) - { - in = fopen(from_path, PG_BINARY_R); - if (in == NULL) - { - if (errno == ENOENT) - exit_code = FILE_MISSING; - else - { - elog(WARNING, "Cannot open source WAL file \"%s\": %s", - from_path, strerror(errno)); - exit_code = OPEN_FAILED; - } - goto cleanup; - } - - /* disable stdio buffering */ - setvbuf(out, NULL, _IONBF, BUFSIZ); - } -#ifdef HAVE_LIBZ - else - { - gz_in = gzopen(from_path, PG_BINARY_R); - if (gz_in == NULL) - { - if (errno == ENOENT) - exit_code = FILE_MISSING; - else - { - elog(WARNING, "Cannot open compressed WAL file \"%s\": %s", - from_path, strerror(errno)); - exit_code = OPEN_FAILED; - } - - goto cleanup; - } - } -#endif - - /* copy content */ - for (;;) - { - int read_len = 0; - -#ifdef HAVE_LIBZ - if (is_decompress) - { - read_len = gzread(gz_in, buf, OUT_BUF_SIZE); - - if (read_len <= 0) - { - if (gzeof(gz_in)) - break; - else - { - elog(WARNING, "Cannot read compressed WAL file \"%s\": %s", - from_path, get_gz_error(gz_in, errno)); - exit_code = READ_FAILED; - break; - } - } - } - else -#endif - { - read_len = fread(buf, 1, OUT_BUF_SIZE, in); - - if (ferror(in)) - { - elog(WARNING, "Cannot read source WAL file \"%s\": %s", - from_path, strerror(errno)); - exit_code = READ_FAILED; - break; - } - - if (read_len == 0 && feof(in)) - break; - } - - if (read_len > 0) - { - if (fwrite(buf, 1, read_len, out) != read_len) - { - elog(WARNING, "Cannot write to WAL file '%s': %s", - to_path, strerror(errno)); - exit_code = WRITE_FAILED; - break; - } - } - } - -cleanup: -#ifdef HAVE_LIBZ - if (gz_in) - gzclose(gz_in); -#endif - if (in) - fclose(in); - - pg_free(buf); - return exit_code; -} - -bool next_wal_segment_exists(TimeLineID tli, XLogSegNo segno, const char *prefetch_dir, uint32 wal_seg_size) -{ - char next_wal_filename[MAXFNAMELEN]; - char next_wal_fullpath[MAXPGPATH]; - - GetXLogFileName(next_wal_filename, tli, segno + 1, wal_seg_size); - - join_path_components(next_wal_fullpath, prefetch_dir, next_wal_filename); - - if (access(next_wal_fullpath, F_OK) == 0) - return true; - - return false; -} - -/* Try to use content of prefetch directory to satisfy request for WAL segment - * If file is found, then validate it and rename. - * If requested file do not exists or validation has failed, then - * caller must copy WAL file directly from archive. - */ -bool wal_satisfy_from_prefetch(TimeLineID tli, XLogSegNo segno, const char *wal_file_name, - const char *prefetch_dir, const char *absolute_wal_file_path, - uint32 wal_seg_size, bool parse_wal) -{ - char prefetched_file[MAXPGPATH]; - - join_path_components(prefetched_file, prefetch_dir, wal_file_name); - - /* If prefetched file do not exists, then nothing can be done */ - if (access(prefetched_file, F_OK) != 0) - return false; - - /* If the next WAL segment do not exists in prefetch directory, - * then current segment cannot be validated, therefore cannot be used - * to satisfy recovery request. - */ - if (parse_wal && !next_wal_segment_exists(tli, segno, prefetch_dir, wal_seg_size)) - return false; - - if (parse_wal && !validate_wal_segment(tli, segno, prefetch_dir, wal_seg_size)) - { - /* prefetched WAL segment is not looking good */ - elog(LOG, "Prefetched WAL segment %s is invalid, cannot use it", wal_file_name); - unlink(prefetched_file); - return false; - } - - /* file is available in prefetch directory */ - if (rename(prefetched_file, absolute_wal_file_path) == 0) - return true; - else - { - elog(WARNING, "Cannot rename file '%s' to '%s': %s", - prefetched_file, absolute_wal_file_path, strerror(errno)); - unlink(prefetched_file); - } - - return false; -} - -/* - * Maintain prefetch directory: drop redundant files - * Return number of files in prefetch directory. - */ -uint32 maintain_prefetch(const char *prefetch_dir, XLogSegNo first_segno, uint32 wal_seg_size) -{ - DIR *dir; - struct dirent *dir_ent; - uint32 n_files = 0; - - XLogSegNo segno; - TimeLineID tli; - - char fullpath[MAXPGPATH]; - - dir = opendir(prefetch_dir); - if (dir == NULL) - { - if (errno != ENOENT) - elog(WARNING, "Cannot open directory \"%s\": %s", prefetch_dir, strerror(errno)); - - return n_files; - } - - while ((dir_ent = readdir(dir))) - { - /* Skip entries point current dir or parent dir */ - if (strcmp(dir_ent->d_name, ".") == 0 || - strcmp(dir_ent->d_name, "..") == 0) - continue; - - if (IsXLogFileName(dir_ent->d_name)) - { - - GetXLogFromFileName(dir_ent->d_name, &tli, &segno, wal_seg_size); - - /* potentially useful segment, keep it */ - if (segno >= first_segno) - { - n_files++; - continue; - } - } - - join_path_components(fullpath, prefetch_dir, dir_ent->d_name); - unlink(fullpath); - } - - closedir(dir); - - return n_files; -} diff --git a/src/backup.c b/src/backup.c deleted file mode 100644 index 41f035a86..000000000 --- a/src/backup.c +++ /dev/null @@ -1,2619 +0,0 @@ -/*------------------------------------------------------------------------- - * - * backup.c: backup DB cluster, archived WAL - * - * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2022, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -#if PG_VERSION_NUM < 110000 -#include "catalog/catalog.h" -#endif -#if PG_VERSION_NUM < 120000 -#include "access/transam.h" -#endif -#include "catalog/pg_tablespace.h" -#include "pgtar.h" -#include "streamutil.h" - -#include -#include -#include - -#include "utils/thread.h" -#include "utils/file.h" - -//const char *progname = "pg_probackup"; - -/* list of files contained in backup */ -parray *backup_files_list = NULL; - -/* We need critical section for datapagemap_add() in case of using threads */ -static pthread_mutex_t backup_pagemap_mutex = PTHREAD_MUTEX_INITIALIZER; - -// TODO: move to PGnodeInfo -bool exclusive_backup = false; - -/* Is pg_start_backup() was executed */ -bool backup_in_progress = false; - -/* - * Backup routines - */ -static void backup_cleanup(bool fatal, void *userdata); - -static void *backup_files(void *arg); - -static void do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, - PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs); - -static void pg_switch_wal(PGconn *conn); - -static void pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startbackup_conn, PGNodeInfo *nodeInfo); - -static void check_external_for_tablespaces(parray *external_list, - PGconn *backup_conn); -static parray *get_database_map(PGconn *pg_startbackup_conn); - -/* pgpro specific functions */ -static bool pgpro_support(PGconn *conn); - -/* Check functions */ -static bool pg_is_checksum_enabled(PGconn *conn); -static bool pg_is_in_recovery(PGconn *conn); -static bool pg_is_superuser(PGconn *conn); -static void check_server_version(PGconn *conn, PGNodeInfo *nodeInfo); -static void confirm_block_size(PGconn *conn, const char *name, int blcksz); -static void rewind_and_mark_cfs_datafiles(parray *files, const char *root, char *relative, size_t i); -static bool remove_excluded_files_criterion(void *value, void *exclude_args); -static void backup_cfs_segment(int i, pgFile *file, backup_files_arg *arguments); -static void process_file(int i, pgFile *file, backup_files_arg *arguments); - -static StopBackupCallbackParams stop_callback_params; - -static void -backup_stopbackup_callback(bool fatal, void *userdata) -{ - StopBackupCallbackParams *st = (StopBackupCallbackParams *) userdata; - /* - * If backup is in progress, notify stop of backup to PostgreSQL - */ - if (backup_in_progress) - { - elog(WARNING, "A backup is in progress, stopping it."); - /* don't care about stop_lsn in case of error */ - pg_stop_backup_send(st->conn, st->server_version, current.from_replica, exclusive_backup, NULL); - } -} - -/* - * Take a backup of a single postgresql instance. - * Move files from 'pgdata' to a subdirectory in backup catalog. - */ -static void -do_backup_pg(InstanceState *instanceState, PGconn *backup_conn, - PGNodeInfo *nodeInfo, bool no_sync, bool backup_logs) -{ - int i; - char external_prefix[MAXPGPATH]; /* Temp value. Used as template */ - char label[1024]; - XLogRecPtr prev_backup_start_lsn = InvalidXLogRecPtr; - - /* arrays with meta info for multi threaded backup */ - pthread_t *threads; - backup_files_arg *threads_args; - bool backup_isok = true; - - pgBackup *prev_backup = NULL; - parray *prev_backup_filelist = NULL; - parray *backup_list = NULL; - parray *external_dirs = NULL; - parray *database_map = NULL; - - /* used for multitimeline incremental backup */ - parray *tli_list = NULL; - - /* for fancy reporting */ - time_t start_time, end_time; - char pretty_time[20]; - char pretty_bytes[20]; - - elog(INFO, "Database backup start"); - if(current.external_dir_str) - { - external_dirs = make_external_directory_list(current.external_dir_str, - false); - check_external_for_tablespaces(external_dirs, backup_conn); - } - - /* notify start of backup to PostgreSQL server */ - time2iso(label, lengthof(label), current.start_time, false); - strncat(label, " with pg_probackup", lengthof(label) - - strlen(" with pg_probackup")); - - /* Call pg_start_backup function in PostgreSQL connect */ - pg_start_backup(label, smooth_checkpoint, ¤t, nodeInfo, backup_conn); - - /* Obtain current timeline */ -#if PG_VERSION_NUM >= 90600 - current.tli = get_current_timeline(backup_conn); -#else - current.tli = get_current_timeline_from_control(instance_config.pgdata, FIO_DB_HOST, false); -#endif - - /* - * In incremental backup mode ensure that already-validated - * backup on current timeline exists and get its filelist. - */ - if (current.backup_mode == BACKUP_MODE_DIFF_PAGE || - current.backup_mode == BACKUP_MODE_DIFF_PTRACK || - current.backup_mode == BACKUP_MODE_DIFF_DELTA) - { - /* get list of backups already taken */ - backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); - - prev_backup = catalog_get_last_data_backup(backup_list, current.tli, current.start_time); - if (prev_backup == NULL) - { - /* try to setup multi-timeline backup chain */ - elog(WARNING, "Valid full backup on current timeline %u is not found, " - "trying to look up on previous timelines", - current.tli); - - tli_list = get_history_streaming(&instance_config.conn_opt, current.tli, backup_list); - if (!tli_list) - { - elog(WARNING, "Failed to obtain current timeline history file via replication protocol"); - /* fallback to using archive */ - tli_list = catalog_get_timelines(instanceState, &instance_config); - } - - if (parray_num(tli_list) == 0) - elog(WARNING, "Cannot find valid backup on previous timelines, " - "WAL archive is not available"); - else - { - prev_backup = get_multi_timeline_parent(backup_list, tli_list, current.tli, - current.start_time, &instance_config); - - if (prev_backup == NULL) - elog(WARNING, "Cannot find valid backup on previous timelines"); - } - - /* failed to find suitable parent, error out */ - if (!prev_backup) - elog(ERROR, "Create new full backup before an incremental one"); - } - } - - if (prev_backup) - { - if (parse_program_version(prev_backup->program_version) > parse_program_version(PROGRAM_VERSION)) - elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " - "pg_probackup do not guarantee to be forward compatible. " - "Please upgrade pg_probackup binary.", - PROGRAM_VERSION, backup_id_of(prev_backup), prev_backup->program_version); - - elog(INFO, "Parent backup: %s", backup_id_of(prev_backup)); - - /* Files of previous backup needed by DELTA backup */ - prev_backup_filelist = get_backup_filelist(prev_backup, true); - - /* If lsn is not NULL, only pages with higher lsn will be copied. */ - prev_backup_start_lsn = prev_backup->start_lsn; - current.parent_backup = prev_backup->start_time; - - write_backup(¤t, true); - } - - /* - * It`s illegal to take PTRACK backup if LSN from ptrack_control() is not - * equal to start_lsn of previous backup. - */ - if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) - { - XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(backup_conn, nodeInfo); - - // new ptrack (>=2.0) is more robust and checks Start LSN - if (ptrack_lsn > prev_backup->start_lsn || ptrack_lsn == InvalidXLogRecPtr) - { - elog(ERROR, "LSN from ptrack_control %X/%X is greater than Start LSN of previous backup %X/%X.\n" - "Create new full backup before an incremental one.", - (uint32) (ptrack_lsn >> 32), (uint32) (ptrack_lsn), - (uint32) (prev_backup->start_lsn >> 32), - (uint32) (prev_backup->start_lsn)); - } - } - - /* For incremental backup check that start_lsn is not from the past - * Though it will not save us if PostgreSQL instance is actually - * restored STREAM backup. - */ - if (current.backup_mode != BACKUP_MODE_FULL && - prev_backup->start_lsn > current.start_lsn) - elog(ERROR, "Current START LSN %X/%X is lower than START LSN %X/%X of previous backup %s. " - "It may indicate that we are trying to backup PostgreSQL instance from the past.", - (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), - (uint32) (prev_backup->start_lsn >> 32), (uint32) (prev_backup->start_lsn), - backup_id_of(prev_backup)); - - /* Update running backup meta with START LSN */ - write_backup(¤t, true); - - /* In PAGE mode or in ARCHIVE wal-mode wait for current segment */ - if (current.backup_mode == BACKUP_MODE_DIFF_PAGE || !current.stream) - { - /* Check that archive_dir can be reached */ - if (fio_access(instanceState->instance_wal_subdir_path, F_OK, FIO_BACKUP_HOST) != 0) - elog(ERROR, "WAL archive directory is not accessible \"%s\": %s", - instanceState->instance_wal_subdir_path, strerror(errno)); - - /* - * Do not wait start_lsn for stream backup. - * Because WAL streaming will start after pg_start_backup() in stream - * mode. - */ - wait_wal_lsn(instanceState->instance_wal_subdir_path, current.start_lsn, true, current.tli, false, true, ERROR, false); - } - - /* start stream replication */ - if (current.stream) - { - char stream_xlog_path[MAXPGPATH]; - - join_path_components(stream_xlog_path, current.database_dir, PG_XLOG_DIR); - fio_mkdir(stream_xlog_path, DIR_PERMISSION, FIO_BACKUP_HOST); - - start_WAL_streaming(backup_conn, stream_xlog_path, &instance_config.conn_opt, - current.start_lsn, current.tli, true); - - /* Make sure that WAL streaming is working - * PAGE backup in stream mode is waited twice, first for - * segment in WAL archive and then for streamed segment - */ - wait_wal_lsn(stream_xlog_path, current.start_lsn, true, current.tli, false, true, ERROR, true); - } - - /* initialize backup's file list */ - backup_files_list = parray_new(); - join_path_components(external_prefix, current.root_dir, EXTERNAL_DIR); - - /* list files with the logical path. omit $PGDATA */ - fio_list_dir(backup_files_list, instance_config.pgdata, - true, true, false, backup_logs, true, 0); - - /* - * Get database_map (name to oid) for use in partial restore feature. - * It's possible that we fail and database_map will be NULL. - */ - database_map = get_database_map(backup_conn); - - /* - * Append to backup list all files and directories - * from external directory option - */ - if (external_dirs) - { - for (i = 0; i < parray_num(external_dirs); i++) - { - /* External dirs numeration starts with 1. - * 0 value is not external dir */ - if (fio_is_remote(FIO_DB_HOST)) - fio_list_dir(backup_files_list, parray_get(external_dirs, i), - false, true, false, false, true, i+1); - else - dir_list_file(backup_files_list, parray_get(external_dirs, i), - false, true, false, false, true, i+1, FIO_LOCAL_HOST); - } - } - - /* close ssh session in main thread */ - fio_disconnect(); - - /* Sanity check for backup_files_list, thank you, Windows: - * https://github.com/postgrespro/pg_probackup/issues/48 - */ - - if (parray_num(backup_files_list) < 100) - elog(ERROR, "PGDATA is almost empty. Either it was concurrently deleted or " - "pg_probackup do not possess sufficient permissions to list PGDATA content"); - - current.pgdata_bytes += calculate_datasize_of_filelist(backup_files_list); - pretty_size(current.pgdata_bytes, pretty_bytes, lengthof(pretty_bytes)); - elog(INFO, "PGDATA size: %s", pretty_bytes); - - /* - * Sort pathname ascending. It is necessary to create intermediate - * directories sequentially. - * - * For example: - * 1 - create 'base' - * 2 - create 'base/1' - * - * Sorted array is used at least in parse_filelist_filenames(), - * extractPageMap(), make_pagemap_from_ptrack(). - */ - parray_qsort(backup_files_list, pgFileCompareRelPathWithExternal); - - /* Extract information about files in backup_list parsing their names:*/ - parse_filelist_filenames(backup_files_list, instance_config.pgdata); - - elog(INFO, "Current Start LSN: %X/%X, TLI: %X", - (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), - current.tli); - if (current.backup_mode != BACKUP_MODE_FULL) - elog(INFO, "Parent Start LSN: %X/%X, TLI: %X", - (uint32) (prev_backup->start_lsn >> 32), (uint32) (prev_backup->start_lsn), - prev_backup->tli); - - /* - * Build page mapping in incremental mode. - */ - - if (current.backup_mode == BACKUP_MODE_DIFF_PAGE || - current.backup_mode == BACKUP_MODE_DIFF_PTRACK) - { - bool pagemap_isok = true; - - time(&start_time); - elog(INFO, "Extracting pagemap of changed blocks"); - - if (current.backup_mode == BACKUP_MODE_DIFF_PAGE) - { - /* - * Build the page map. Obtain information about changed pages - * reading WAL segments present in archives up to the point - * where this backup has started. - */ - pagemap_isok = extractPageMap(instanceState->instance_wal_subdir_path, - instance_config.xlog_seg_size, - prev_backup->start_lsn, prev_backup->tli, - current.start_lsn, current.tli, tli_list); - } - else if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) - { - /* - * Build the page map from ptrack information. - */ - make_pagemap_from_ptrack_2(backup_files_list, backup_conn, - nodeInfo->ptrack_schema, - nodeInfo->ptrack_version_num, - prev_backup_start_lsn); - } - - time(&end_time); - - /* TODO: add ms precision */ - if (pagemap_isok) - elog(INFO, "Pagemap successfully extracted, time elapsed: %.0f sec", - difftime(end_time, start_time)); - else - elog(ERROR, "Pagemap extraction failed, time elasped: %.0f sec", - difftime(end_time, start_time)); - } - - /* - * Make directories before backup - */ - for (i = 0; i < parray_num(backup_files_list); i++) - { - pgFile *file = (pgFile *) parray_get(backup_files_list, i); - - /* if the entry was a directory, create it in the backup */ - if (S_ISDIR(file->mode)) - { - char dirpath[MAXPGPATH]; - - if (file->external_dir_num) - { - char temp[MAXPGPATH]; - snprintf(temp, MAXPGPATH, "%s%d", external_prefix, - file->external_dir_num); - join_path_components(dirpath, temp, file->rel_path); - } - else - join_path_components(dirpath, current.database_dir, file->rel_path); - - elog(LOG, "Create directory '%s'", dirpath); - fio_mkdir(dirpath, DIR_PERMISSION, FIO_BACKUP_HOST); - } - - } - - /* setup thread locks */ - pfilearray_clear_locks(backup_files_list); - - /* Sort by size for load balancing */ - parray_qsort(backup_files_list, pgFileCompareSize); - /* Sort the array for binary search */ - if (prev_backup_filelist) - parray_qsort(prev_backup_filelist, pgFileCompareRelPathWithExternal); - - /* write initial backup_content.control file and update backup.control */ - write_backup_filelist(¤t, backup_files_list, - instance_config.pgdata, external_dirs, true); - write_backup(¤t, true); - - /* Init backup page header map */ - init_header_map(¤t); - - /* init thread args with own file lists */ - threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - threads_args = (backup_files_arg *) palloc(sizeof(backup_files_arg)*num_threads); - - for (i = 0; i < num_threads; i++) - { - backup_files_arg *arg = &(threads_args[i]); - - arg->nodeInfo = nodeInfo; - arg->from_root = instance_config.pgdata; - arg->to_root = current.database_dir; - arg->external_prefix = external_prefix; - arg->external_dirs = external_dirs; - arg->files_list = backup_files_list; - arg->prev_filelist = prev_backup_filelist; - arg->prev_start_lsn = prev_backup_start_lsn; - arg->hdr_map = &(current.hdr_map); - arg->thread_num = i+1; - /* By default there are some error */ - arg->ret = 1; - } - - /* Run threads */ - thread_interrupted = false; - elog(INFO, "Start transferring data files"); - time(&start_time); - for (i = 0; i < num_threads; i++) - { - backup_files_arg *arg = &(threads_args[i]); - - elog(VERBOSE, "Start thread num: %i", i); - pthread_create(&threads[i], NULL, backup_files, arg); - } - - /* Wait threads */ - for (i = 0; i < num_threads; i++) - { - pthread_join(threads[i], NULL); - if (threads_args[i].ret == 1) - backup_isok = false; - } - - time(&end_time); - pretty_time_interval(difftime(end_time, start_time), - pretty_time, lengthof(pretty_time)); - if (backup_isok) - elog(INFO, "Data files are transferred, time elapsed: %s", - pretty_time); - else - elog(ERROR, "Data files transferring failed, time elapsed: %s", - pretty_time); - - /* clean previous backup file list */ - if (prev_backup_filelist) - { - parray_walk(prev_backup_filelist, pgFileFree); - parray_free(prev_backup_filelist); - } - - /* Notify end of backup */ - pg_stop_backup(instanceState, ¤t, backup_conn, nodeInfo); - - /* In case of backup from replica >= 9.6 we must fix minRecPoint, - * First we must find pg_control in backup_files_list. - */ - if (current.from_replica && !exclusive_backup) - { - pgFile *pg_control = NULL; - - for (i = 0; i < parray_num(backup_files_list); i++) - { - pgFile *tmp_file = (pgFile *) parray_get(backup_files_list, i); - - if (tmp_file->external_dir_num == 0 && - (strcmp(tmp_file->rel_path, XLOG_CONTROL_FILE) == 0)) - { - pg_control = tmp_file; - break; - } - } - - if (!pg_control) - elog(ERROR, "Failed to find file \"%s\" in backup filelist.", - XLOG_CONTROL_FILE); - - set_min_recovery_point(pg_control, current.database_dir, current.stop_lsn); - } - - /* close and sync page header map */ - if (current.hdr_map.fp) - { - cleanup_header_map(&(current.hdr_map)); - - if (fio_sync(current.hdr_map.path, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Cannot sync file \"%s\": %s", current.hdr_map.path, strerror(errno)); - } - - /* close ssh session in main thread */ - fio_disconnect(); - - /* - * Add archived xlog files into the list of files of this backup - * NOTHING TO DO HERE - */ - - /* write database map to file and add it to control file */ - if (database_map) - { - write_database_map(¤t, database_map, backup_files_list); - /* cleanup */ - parray_walk(database_map, db_map_entry_free); - parray_free(database_map); - } - - /* Print the list of files to backup catalog */ - write_backup_filelist(¤t, backup_files_list, instance_config.pgdata, - external_dirs, true); - /* update backup control file to update size info */ - write_backup(¤t, true); - - /* Sync all copied files unless '--no-sync' flag is used */ - if (no_sync) - elog(WARNING, "Backup files are not synced to disk"); - else - { - elog(INFO, "Syncing backup files to disk"); - time(&start_time); - - for (i = 0; i < parray_num(backup_files_list); i++) - { - char to_fullpath[MAXPGPATH]; - pgFile *file = (pgFile *) parray_get(backup_files_list, i); - - /* TODO: sync directory ? */ - if (S_ISDIR(file->mode)) - continue; - - if (file->write_size <= 0) - continue; - - /* construct fullpath */ - if (file->external_dir_num == 0) - join_path_components(to_fullpath, current.database_dir, file->rel_path); - else - { - char external_dst[MAXPGPATH]; - - makeExternalDirPathByNum(external_dst, external_prefix, - file->external_dir_num); - join_path_components(to_fullpath, external_dst, file->rel_path); - } - - if (fio_sync(to_fullpath, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Cannot sync file \"%s\": %s", to_fullpath, strerror(errno)); - } - - time(&end_time); - pretty_time_interval(difftime(end_time, start_time), - pretty_time, lengthof(pretty_time)); - elog(INFO, "Backup files are synced, time elapsed: %s", pretty_time); - } - - /* be paranoid about instance been from the past */ - if (current.backup_mode != BACKUP_MODE_FULL && - current.stop_lsn < prev_backup->stop_lsn) - elog(ERROR, "Current backup STOP LSN %X/%X is lower than STOP LSN %X/%X of previous backup %s. " - "It may indicate that we are trying to backup PostgreSQL instance from the past.", - (uint32) (current.stop_lsn >> 32), (uint32) (current.stop_lsn), - (uint32) (prev_backup->stop_lsn >> 32), (uint32) (prev_backup->stop_lsn), - backup_id_of(prev_backup)); - - /* clean external directories list */ - if (external_dirs) - free_dir_list(external_dirs); - - /* Cleanup */ - if (backup_list) - { - parray_walk(backup_list, pgBackupFree); - parray_free(backup_list); - } - - if (tli_list) - { - parray_walk(tli_list, timelineInfoFree); - parray_free(tli_list); - } - - parray_walk(backup_files_list, pgFileFree); - parray_free(backup_files_list); - backup_files_list = NULL; -} - -/* - * Common code for CHECKDB and BACKUP commands. - * Ensure that we're able to connect to the instance - * check compatibility and fill basic info. - * For checkdb launched in amcheck mode with pgdata validation - * do not check system ID, it gives user an opportunity to - * check remote PostgreSQL instance. - * Also checking system ID in this case serves no purpose, because - * all work is done by server. - * - * Returns established connection - */ -PGconn * -pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo) -{ - PGconn *cur_conn; - - /* Create connection for PostgreSQL */ - cur_conn = pgut_connect(conn_opt.pghost, conn_opt.pgport, - conn_opt.pgdatabase, - conn_opt.pguser); - - current.primary_conninfo = pgut_get_conninfo_string(cur_conn); - - /* Confirm data block size and xlog block size are compatible */ - confirm_block_size(cur_conn, "block_size", BLCKSZ); - confirm_block_size(cur_conn, "wal_block_size", XLOG_BLCKSZ); - nodeInfo->block_size = BLCKSZ; - nodeInfo->wal_block_size = XLOG_BLCKSZ; - nodeInfo->is_superuser = pg_is_superuser(cur_conn); - nodeInfo->pgpro_support = pgpro_support(cur_conn); - - current.from_replica = pg_is_in_recovery(cur_conn); - - /* Confirm that this server version is supported */ - check_server_version(cur_conn, nodeInfo); - - if (pg_is_checksum_enabled(cur_conn)) - current.checksum_version = 1; - else - current.checksum_version = 0; - - nodeInfo->checksum_version = current.checksum_version; - - if (current.checksum_version) - elog(INFO, "This PostgreSQL instance was initialized with data block checksums. " - "Data block corruption will be detected"); - else - elog(WARNING, "This PostgreSQL instance was initialized without data block checksums. " - "pg_probackup have no way to detect data block corruption without them. " - "Reinitialize PGDATA with option '--data-checksums'."); - - if (nodeInfo->is_superuser) - elog(WARNING, "Current PostgreSQL role is superuser. " - "It is not recommended to run pg_probackup under superuser."); - - strlcpy(current.server_version, nodeInfo->server_version_str, - sizeof(current.server_version)); - - return cur_conn; -} - -/* - * Entry point of pg_probackup BACKUP subcommand. - * - * if start_time == INVALID_BACKUP_ID then we can generate backup_id - */ -int -do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, - bool no_validate, bool no_sync, bool backup_logs, time_t start_time) -{ - PGconn *backup_conn = NULL; - PGNodeInfo nodeInfo; - time_t latest_backup_id = INVALID_BACKUP_ID; - char pretty_bytes[20]; - - if (!instance_config.pgdata) - elog(ERROR, "No postgres data directory specified.\n" - "Please specify it either using environment variable PGDATA or\n" - "command line option --pgdata (-D)"); - - /* Initialize PGInfonode */ - pgNodeInit(&nodeInfo); - - /* Save list of external directories */ - if (instance_config.external_dir_str && - (pg_strcasecmp(instance_config.external_dir_str, "none") != 0)) - current.external_dir_str = instance_config.external_dir_str; - - /* Find latest backup_id */ - { - parray *backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); - - if (parray_num(backup_list) > 0) - latest_backup_id = ((pgBackup *)parray_get(backup_list, 0))->backup_id; - - parray_walk(backup_list, pgBackupFree); - parray_free(backup_list); - } - - /* Try to pick backup_id and create backup directory with BACKUP_CONTROL_FILE */ - if (start_time != INVALID_BACKUP_ID) - { - /* If user already choosed backup_id for us, then try to use it. */ - if (start_time <= latest_backup_id) - /* don't care about freeing base36enc_dup memory, we exit anyway */ - elog(ERROR, "Can't assign backup_id from requested start_time (%s), " - "this time must be later that backup %s", - base36enc(start_time), base36enc(latest_backup_id)); - - current.backup_id = start_time; - pgBackupInitDir(¤t, instanceState->instance_backup_subdir_path); - } - else - { - /* We can generate our own unique backup_id - * Sometimes (when we try to backup twice in one second) - * backup_id will be duplicated -> try more times. - */ - int attempts = 10; - - if (time(NULL) < latest_backup_id) - elog(ERROR, "Can't assign backup_id, there is already a backup in future (%s)", - base36enc(latest_backup_id)); - - do - { - current.backup_id = time(NULL); - pgBackupInitDir(¤t, instanceState->instance_backup_subdir_path); - if (current.backup_id == INVALID_BACKUP_ID) - sleep(1); - } - while (current.backup_id == INVALID_BACKUP_ID && attempts-- > 0); - } - - /* If creation of backup dir was unsuccessful, there will be WARNINGS in logs already */ - if (current.backup_id == INVALID_BACKUP_ID) - elog(ERROR, "Can't create backup directory"); - - /* Update backup status and other metainfo. */ - current.status = BACKUP_STATUS_RUNNING; - /* XXX BACKUP_ID change it when backup_id wouldn't match start_time */ - current.start_time = current.backup_id; - - strlcpy(current.program_version, PROGRAM_VERSION, - sizeof(current.program_version)); - - current.compress_alg = instance_config.compress_alg; - current.compress_level = instance_config.compress_level; - - elog(INFO, "Backup start, pg_probackup version: %s, instance: %s, backup ID: %s, backup mode: %s, " - "wal mode: %s, remote: %s, compress-algorithm: %s, compress-level: %i", - PROGRAM_VERSION, instanceState->instance_name, backup_id_of(¤t), pgBackupGetBackupMode(¤t, false), - current.stream ? "STREAM" : "ARCHIVE", IsSshProtocol() ? "true" : "false", - deparse_compress_alg(current.compress_alg), current.compress_level); - - if (!lock_backup(¤t, true, true)) - elog(ERROR, "Cannot lock backup %s directory", - backup_id_of(¤t)); - write_backup(¤t, true); - - /* set the error processing function for the backup process */ - pgut_atexit_push(backup_cleanup, NULL); - - elog(LOG, "Backup destination is initialized"); - - /* - * setup backup_conn, do some compatibility checks and - * fill basic info about instance - */ - backup_conn = pgdata_basic_setup(instance_config.conn_opt, &nodeInfo); - - if (current.from_replica) - elog(INFO, "Backup %s is going to be taken from standby", backup_id_of(¤t)); - - /* TODO, print PostgreSQL full version */ - //elog(INFO, "PostgreSQL version: %s", nodeInfo.server_version_str); - - /* - * Ensure that backup directory was initialized for the same PostgreSQL - * instance we opened connection to. And that target backup database PGDATA - * belogns to the same instance. - */ - check_system_identifiers(backup_conn, instance_config.pgdata); - - /* below perform checks specific for backup command */ -#if PG_VERSION_NUM >= 110000 - if (!RetrieveWalSegSize(backup_conn)) - elog(ERROR, "Failed to retrieve wal_segment_size"); -#endif - - get_ptrack_version(backup_conn, &nodeInfo); - // elog(WARNING, "ptrack_version_num %d", ptrack_version_num); - - if (nodeInfo.ptrack_version_num > 0) - nodeInfo.is_ptrack_enabled = pg_is_ptrack_enabled(backup_conn, nodeInfo.ptrack_version_num); - - if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) - { - /* ptrack_version_num < 2.0 was already checked in get_ptrack_version() */ - if (nodeInfo.ptrack_version_num == 0) - elog(ERROR, "This PostgreSQL instance does not support ptrack"); - else - { - if (!nodeInfo.is_ptrack_enabled) - elog(ERROR, "Ptrack is disabled"); - } - } - - if (current.from_replica && exclusive_backup) - /* Check master connection options */ - if (instance_config.master_conn_opt.pghost == NULL) - elog(ERROR, "Options for connection to master must be provided to perform backup from replica"); - - /* add note to backup if requested */ - if (set_backup_params && set_backup_params->note) - add_note(¤t, set_backup_params->note); - - /* backup data */ - do_backup_pg(instanceState, backup_conn, &nodeInfo, no_sync, backup_logs); - pgut_atexit_pop(backup_cleanup, NULL); - - /* compute size of wal files of this backup stored in the archive */ - if (!current.stream) - { - XLogSegNo start_segno; - XLogSegNo stop_segno; - - GetXLogSegNo(current.start_lsn, start_segno, instance_config.xlog_seg_size); - GetXLogSegNo(current.stop_lsn, stop_segno, instance_config.xlog_seg_size); - current.wal_bytes = (stop_segno - start_segno) * instance_config.xlog_seg_size; - - /* - * If start_lsn and stop_lsn are located in the same segment, then - * set wal_bytes to the size of 1 segment. - */ - if (current.wal_bytes <= 0) - current.wal_bytes = instance_config.xlog_seg_size; - } - - /* Backup is done. Update backup status */ - current.end_time = time(NULL); - current.status = BACKUP_STATUS_DONE; - write_backup(¤t, true); - - /* Pin backup if requested */ - if (set_backup_params && - (set_backup_params->ttl > 0 || - set_backup_params->expire_time > 0)) - { - pin_backup(¤t, set_backup_params); - } - - if (!no_validate) - pgBackupValidate(¤t, NULL); - - /* Notify user about backup size */ - if (current.stream) - pretty_size(current.data_bytes + current.wal_bytes, pretty_bytes, lengthof(pretty_bytes)); - else - pretty_size(current.data_bytes, pretty_bytes, lengthof(pretty_bytes)); - elog(INFO, "Backup %s resident size: %s", backup_id_of(¤t), pretty_bytes); - - if (current.status == BACKUP_STATUS_OK || - current.status == BACKUP_STATUS_DONE) - elog(INFO, "Backup %s completed", backup_id_of(¤t)); - else - elog(ERROR, "Backup %s failed", backup_id_of(¤t)); - - /* - * After successful backup completion remove backups - * which are expired according to retention policies - */ - if (delete_expired || merge_expired || delete_wal) - do_retention(instanceState, no_validate, no_sync); - - return 0; -} - -/* - * Confirm that this server version is supported - */ -static void -check_server_version(PGconn *conn, PGNodeInfo *nodeInfo) -{ - PGresult *res = NULL; - - /* confirm server version */ - nodeInfo->server_version = PQserverVersion(conn); - - if (nodeInfo->server_version == 0) - elog(ERROR, "Unknown server version %d", nodeInfo->server_version); - - if (nodeInfo->server_version < 100000) - sprintf(nodeInfo->server_version_str, "%d.%d", - nodeInfo->server_version / 10000, - (nodeInfo->server_version / 100) % 100); - else - sprintf(nodeInfo->server_version_str, "%d", - nodeInfo->server_version / 10000); - - if (nodeInfo->server_version < 90500) - elog(ERROR, - "Server version is %s, must be %s or higher", - nodeInfo->server_version_str, "9.5"); - - if (current.from_replica && nodeInfo->server_version < 90600) - elog(ERROR, - "Server version is %s, must be %s or higher for backup from replica", - nodeInfo->server_version_str, "9.6"); - - if (nodeInfo->pgpro_support) - res = pgut_execute(conn, "SELECT pg_catalog.pgpro_edition()", 0, NULL); - - /* - * Check major version of connected PostgreSQL and major version of - * compiled PostgreSQL. - */ -#ifdef PGPRO_VERSION - if (!res) - { - /* It seems we connected to PostgreSQL (not Postgres Pro) */ - if(strcmp(PGPRO_EDITION, "1C") != 0) - { - elog(ERROR, "%s was built with Postgres Pro %s %s, " - "but connection is made with PostgreSQL %s", - PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str); - } - /* We have PostgresPro for 1C and connect to PostgreSQL or PostgresPro for 1C - * Check the major version - */ - if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0) - elog(ERROR, "%s was built with PostgrePro %s %s, but connection is made with %s", - PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, nodeInfo->server_version_str); - } - else - { - if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0 && - strcmp(PQgetvalue(res, 0, 0), PGPRO_EDITION) != 0) - elog(ERROR, "%s was built with Postgres Pro %s %s, " - "but connection is made with Postgres Pro %s %s", - PROGRAM_NAME, PG_MAJORVERSION, PGPRO_EDITION, - nodeInfo->server_version_str, PQgetvalue(res, 0, 0)); - } -#else - if (res) - /* It seems we connected to Postgres Pro (not PostgreSQL) */ - elog(ERROR, "%s was built with PostgreSQL %s, " - "but connection is made with Postgres Pro %s %s", - PROGRAM_NAME, PG_MAJORVERSION, - nodeInfo->server_version_str, PQgetvalue(res, 0, 0)); - else - { - if (strcmp(nodeInfo->server_version_str, PG_MAJORVERSION) != 0) - elog(ERROR, "%s was built with PostgreSQL %s, but connection is made with %s", - PROGRAM_NAME, PG_MAJORVERSION, nodeInfo->server_version_str); - } -#endif - - if (res) - PQclear(res); - - /* Do exclusive backup only for PostgreSQL 9.5 */ - exclusive_backup = nodeInfo->server_version < 90600; -} - -/* - * Ensure that backup directory was initialized for the same PostgreSQL - * instance we opened connection to. And that target backup database PGDATA - * belogns to the same instance. - * All system identifiers must be equal. - */ -void -check_system_identifiers(PGconn *conn, const char *pgdata) -{ - uint64 system_id_conn; - uint64 system_id_pgdata; - - system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST, false); - system_id_conn = get_remote_system_identifier(conn); - - /* for checkdb check only system_id_pgdata and system_id_conn */ - if (current.backup_mode == BACKUP_MODE_INVALID) - { - if (system_id_conn != system_id_pgdata) - { - elog(ERROR, "Data directory initialized with system id " UINT64_FORMAT ", " - "but connected instance system id is " UINT64_FORMAT, - system_id_pgdata, system_id_conn); - } - return; - } - - if (system_id_conn != instance_config.system_identifier) - elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT ", " - "but connected instance system id is " UINT64_FORMAT, - instance_config.system_identifier, system_id_conn); - - if (system_id_pgdata != instance_config.system_identifier) - elog(ERROR, "Backup data directory was initialized for system id " UINT64_FORMAT ", " - "but target backup directory system id is " UINT64_FORMAT, - instance_config.system_identifier, system_id_pgdata); -} - -/* - * Ensure that target backup database is initialized with - * compatible settings. Currently check BLCKSZ and XLOG_BLCKSZ. - */ -static void -confirm_block_size(PGconn *conn, const char *name, int blcksz) -{ - PGresult *res; - char *endp; - int block_size; - - res = pgut_execute(conn, "SELECT pg_catalog.current_setting($1)", 1, &name); - if (PQntuples(res) != 1 || PQnfields(res) != 1) - elog(ERROR, "Cannot get %s: %s", name, PQerrorMessage(conn)); - - block_size = strtol(PQgetvalue(res, 0, 0), &endp, 10); - if ((endp && *endp) || block_size != blcksz) - elog(ERROR, - "%s(%d) is not compatible(%d expected)", - name, block_size, blcksz); - - PQclear(res); -} - -/* - * Notify start of backup to PostgreSQL server. - */ -void -pg_start_backup(const char *label, bool smooth, pgBackup *backup, - PGNodeInfo *nodeInfo, PGconn *conn) -{ - PGresult *res; - const char *params[2]; - uint32 lsn_hi; - uint32 lsn_lo; - params[0] = label; - -#if PG_VERSION_NUM >= 150000 - elog(INFO, "wait for pg_backup_start()"); -#else - elog(INFO, "wait for pg_start_backup()"); -#endif - - /* 2nd argument is 'fast'*/ - params[1] = smooth ? "false" : "true"; - res = pgut_execute(conn, -#if PG_VERSION_NUM >= 150000 - "SELECT pg_catalog.pg_backup_start($1, $2)", -#else - "SELECT pg_catalog.pg_start_backup($1, $2, false)", -#endif - 2, - params); - - /* - * Set flag that pg_start_backup() was called. If an error will happen it - * is necessary to call pg_stop_backup() in backup_cleanup(). - */ - backup_in_progress = true; - stop_callback_params.conn = conn; - stop_callback_params.server_version = nodeInfo->server_version; - pgut_atexit_push(backup_stopbackup_callback, &stop_callback_params); - - /* Extract timeline and LSN from results of pg_start_backup() */ - XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); - /* Calculate LSN */ - backup->start_lsn = ((uint64) lsn_hi )<< 32 | lsn_lo; - - PQclear(res); - - if ((!backup->stream || backup->backup_mode == BACKUP_MODE_DIFF_PAGE) && - !backup->from_replica && - !(nodeInfo->server_version < 90600 && - !nodeInfo->is_superuser)) - /* - * Switch to a new WAL segment. It is necessary to get archived WAL - * segment, which includes start LSN of current backup. - * Don`t do this for replica backups and for PG 9.5 if pguser is not superuser - * (because in 9.5 only superuser can switch WAL) - */ - pg_switch_wal(conn); -} - -/* - * Switch to a new WAL segment. It should be called only for master. - * For PG 9.5 it should be called only if pguser is superuser. - */ -void -pg_switch_wal(PGconn *conn) -{ - PGresult *res; - - pg_silent_client_messages(conn); - -#if PG_VERSION_NUM >= 100000 - res = pgut_execute(conn, "SELECT pg_catalog.pg_switch_wal()", 0, NULL); -#else - res = pgut_execute(conn, "SELECT pg_catalog.pg_switch_xlog()", 0, NULL); -#endif - - PQclear(res); -} - -/* - * Check if the instance is PostgresPro fork. - */ -static bool -pgpro_support(PGconn *conn) -{ - PGresult *res; - - res = pgut_execute(conn, - "SELECT proname FROM pg_catalog.pg_proc WHERE proname='pgpro_edition'::name AND pronamespace='pg_catalog'::regnamespace::oid", - 0, NULL); - - if (PQresultStatus(res) == PGRES_TUPLES_OK && - (PQntuples(res) == 1) && - (strcmp(PQgetvalue(res, 0, 0), "pgpro_edition") == 0)) - { - PQclear(res); - return true; - } - - PQclear(res); - return false; -} - -/* - * Fill 'datname to Oid' map - * - * This function can fail to get the map for legal reasons, e.g. missing - * permissions on pg_database during `backup`. - * As long as user do not use partial restore feature it`s fine. - * - * To avoid breaking a backward compatibility don't throw an ERROR, - * throw a warning instead of an error and return NULL. - * Caller is responsible for checking the result. - */ -parray * -get_database_map(PGconn *conn) -{ - PGresult *res; - parray *database_map = NULL; - int i; - - /* - * Do not include template0 and template1 to the map - * as default databases that must always be restored. - */ - res = pgut_execute_extended(conn, - "SELECT oid, datname FROM pg_catalog.pg_database " - "WHERE datname NOT IN ('template1'::name, 'template0'::name)", - 0, NULL, true, true); - - /* Don't error out, simply return NULL. See comment above. */ - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - PQclear(res); - elog(WARNING, "Failed to get database map: %s", - PQerrorMessage(conn)); - - return NULL; - } - - /* Construct database map */ - for (i = 0; i < PQntuples(res); i++) - { - char *datname = NULL; - db_map_entry *db_entry = (db_map_entry *) pgut_malloc(sizeof(db_map_entry)); - - /* get Oid */ - db_entry->dbOid = atoll(PQgetvalue(res, i, 0)); - - /* get datname */ - datname = PQgetvalue(res, i, 1); - db_entry->datname = pgut_malloc(strlen(datname) + 1); - strcpy(db_entry->datname, datname); - - if (database_map == NULL) - database_map = parray_new(); - - parray_append(database_map, db_entry); - } - - return database_map; -} - -/* Check if ptrack is enabled in target instance */ -static bool -pg_is_checksum_enabled(PGconn *conn) -{ - PGresult *res_db; - - res_db = pgut_execute(conn, "SHOW data_checksums", 0, NULL); - - if (strcmp(PQgetvalue(res_db, 0, 0), "on") == 0) - { - PQclear(res_db); - return true; - } - PQclear(res_db); - return false; -} - -/* Check if target instance is replica */ -static bool -pg_is_in_recovery(PGconn *conn) -{ - PGresult *res_db; - - res_db = pgut_execute(conn, "SELECT pg_catalog.pg_is_in_recovery()", 0, NULL); - - if (PQgetvalue(res_db, 0, 0)[0] == 't') - { - PQclear(res_db); - return true; - } - PQclear(res_db); - return false; -} - - -/* Check if current PostgreSQL role is superuser */ -static bool -pg_is_superuser(PGconn *conn) -{ - PGresult *res; - - res = pgut_execute(conn, "SELECT pg_catalog.current_setting('is_superuser')", 0, NULL); - - if (strcmp(PQgetvalue(res, 0, 0), "on") == 0) - { - PQclear(res); - return true; - } - PQclear(res); - return false; -} - -/* - * Wait for target LSN or WAL segment, containing target LSN. - * - * Depending on value of flag in_stream_dir wait for target LSN to archived or - * streamed in 'archive_dir' or 'pg_wal' directory. - * - * If flag 'is_start_lsn' is set then issue warning for first-time users. - * If flag 'in_prev_segment' is set, look for LSN in previous segment, - * with EndRecPtr >= Target LSN. It should be used only for solving - * invalid XRecOff problem. - * If flag 'segment_only' is set, then, instead of waiting for LSN, wait for segment, - * containing that LSN. - * If flags 'in_prev_segment' and 'segment_only' are both set, then wait for - * previous segment. - * - * Flag 'in_stream_dir' determine whether we looking for WAL in 'pg_wal' directory or - * in archive. Do note, that we cannot rely sorely on global variable 'stream_wal' (current.stream) because, - * for example, PAGE backup must(!) look for start_lsn in archive regardless of wal_mode. - * - * 'timeout_elevel' determine the elevel for timeout elog message. If elevel lighter than - * ERROR is used, then return InvalidXLogRecPtr. TODO: return something more concrete, for example 1. - * - * Returns target LSN if such is found, failing that returns LSN of record prior to target LSN. - * Returns InvalidXLogRecPtr if 'segment_only' flag is used. - */ -XLogRecPtr -wait_wal_lsn(const char *wal_segment_dir, XLogRecPtr target_lsn, bool is_start_lsn, TimeLineID tli, - bool in_prev_segment, bool segment_only, - int timeout_elevel, bool in_stream_dir) -{ - XLogSegNo targetSegNo; - char wal_segment_path[MAXPGPATH], - wal_segment[MAXFNAMELEN]; - bool file_exists = false; - uint32 try_count = 0, - timeout; - char *wal_delivery_str = in_stream_dir ? "streamed":"archived"; - -#ifdef HAVE_LIBZ - char gz_wal_segment_path[MAXPGPATH]; -#endif - - /* Compute the name of the WAL file containing requested LSN */ - GetXLogSegNo(target_lsn, targetSegNo, instance_config.xlog_seg_size); - if (in_prev_segment) - targetSegNo--; - GetXLogFileName(wal_segment, tli, targetSegNo, - instance_config.xlog_seg_size); - - join_path_components(wal_segment_path, wal_segment_dir, wal_segment); - /* - * In pg_start_backup we wait for 'target_lsn' in 'pg_wal' directory if it is - * stream and non-page backup. Page backup needs archived WAL files, so we - * wait for 'target_lsn' in archive 'wal' directory for page backups. - * - * In pg_stop_backup it depends only on stream_wal. - */ - - /* TODO: remove this in 3.0 (it is a cludge against some old bug with archive_timeout) */ - if (instance_config.archive_timeout > 0) - timeout = instance_config.archive_timeout; - else - timeout = ARCHIVE_TIMEOUT_DEFAULT; - - if (segment_only) - elog(LOG, "Looking for segment: %s", wal_segment); - else - elog(LOG, "Looking for LSN %X/%X in segment: %s", - (uint32) (target_lsn >> 32), (uint32) target_lsn, wal_segment); - -#ifdef HAVE_LIBZ - snprintf(gz_wal_segment_path, sizeof(gz_wal_segment_path), "%s.gz", - wal_segment_path); -#endif - - /* Wait until target LSN is archived or streamed */ - while (true) - { - if (!file_exists) - { - file_exists = fileExists(wal_segment_path, FIO_BACKUP_HOST); - - /* Try to find compressed WAL file */ - if (!file_exists) - { -#ifdef HAVE_LIBZ - file_exists = fileExists(gz_wal_segment_path, FIO_BACKUP_HOST); - if (file_exists) - elog(LOG, "Found compressed WAL segment: %s", wal_segment_path); -#endif - } - else - elog(LOG, "Found WAL segment: %s", wal_segment_path); - } - - if (file_exists) - { - /* Do not check for target LSN */ - if (segment_only) - return InvalidXLogRecPtr; - - /* - * A WAL segment found. Look for target LSN in it. - */ - if (!XRecOffIsNull(target_lsn) && - wal_contains_lsn(wal_segment_dir, target_lsn, tli, - instance_config.xlog_seg_size)) - /* Target LSN was found */ - { - elog(LOG, "Found LSN: %X/%X", (uint32) (target_lsn >> 32), (uint32) target_lsn); - return target_lsn; - } - - /* - * If we failed to get target LSN in a reasonable time, try - * to get LSN of last valid record prior to the target LSN. But only - * in case of a backup from a replica. - * Note, that with NullXRecOff target_lsn we do not wait - * for 'timeout / 2' seconds before going for previous record, - * because such LSN cannot be delivered at all. - * - * There are two cases for this: - * 1. Replica returned readpoint LSN which just do not exists. We want to look - * for previous record in the same(!) WAL segment which endpoint points to this LSN. - * 2. Replica returened endpoint LSN with NullXRecOff. We want to look - * for previous record which endpoint points greater or equal LSN in previous WAL segment. - */ - if (current.from_replica && - (XRecOffIsNull(target_lsn) || try_count > timeout / 2)) - { - XLogRecPtr res; - - res = get_prior_record_lsn(wal_segment_dir, current.start_lsn, target_lsn, tli, - in_prev_segment, instance_config.xlog_seg_size); - - if (!XLogRecPtrIsInvalid(res)) - { - /* LSN of the prior record was found */ - elog(LOG, "Found prior LSN: %X/%X", - (uint32) (res >> 32), (uint32) res); - return res; - } - } - } - - sleep(1); - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during waiting for WAL %s", in_stream_dir ? "streaming" : "archiving"); - try_count++; - - /* Inform user if WAL segment is absent in first attempt */ - if (try_count == 1) - { - if (segment_only) - elog(INFO, "Wait for WAL segment %s to be %s", - wal_segment_path, wal_delivery_str); - else - elog(INFO, "Wait for LSN %X/%X in %s WAL segment %s", - (uint32) (target_lsn >> 32), (uint32) target_lsn, - wal_delivery_str, wal_segment_path); - } - - if (!current.stream && is_start_lsn && try_count == 30) - elog(WARNING, "By default pg_probackup assumes that WAL delivery method to be ARCHIVE. " - "If continuous archiving is not set up, use '--stream' option to make autonomous backup. " - "Otherwise check that continuous archiving works correctly."); - - if (timeout > 0 && try_count > timeout) - { - if (file_exists) - elog(timeout_elevel, "WAL segment %s was %s, " - "but target LSN %X/%X could not be %s in %d seconds", - wal_segment, wal_delivery_str, - (uint32) (target_lsn >> 32), (uint32) target_lsn, - wal_delivery_str, timeout); - /* If WAL segment doesn't exist or we wait for previous segment */ - else - elog(timeout_elevel, - "WAL segment %s could not be %s in %d seconds", - wal_segment, wal_delivery_str, timeout); - - return InvalidXLogRecPtr; - } - } -} - -/* - * Check stop_lsn (returned from pg_stop_backup()) and update backup->stop_lsn - */ -void -wait_wal_and_calculate_stop_lsn(const char *xlog_path, XLogRecPtr stop_lsn, pgBackup *backup) -{ - bool stop_lsn_exists = false; - - /* It is ok for replica to return invalid STOP LSN - * UPD: Apparently it is ok even for a master. - */ - if (!XRecOffIsValid(stop_lsn)) - { - XLogSegNo segno = 0; - XLogRecPtr lsn_tmp = InvalidXLogRecPtr; - - /* - * Even though the value is invalid, it's expected postgres behaviour - * and we're trying to fix it below. - */ - elog(LOG, "Invalid offset in stop_lsn value %X/%X, trying to fix", - (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); - - /* - * Note: even with gdb it is very hard to produce automated tests for - * contrecord + invalid LSN, so emulate it for manual testing. - */ - //lsn = lsn - XLOG_SEG_SIZE; - //elog(WARNING, "New Invalid stop_backup_lsn value %X/%X", - // (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); - - GetXLogSegNo(stop_lsn, segno, instance_config.xlog_seg_size); - - /* - * Note, that there is no guarantee that corresponding WAL file even exists. - * Replica may return LSN from future and keep staying in present. - * Or it can return invalid LSN. - * - * That's bad, since we want to get real LSN to save it in backup label file - * and to use it in WAL validation. - * - * So we try to do the following: - * 1. Wait 'archive_timeout' seconds for segment containing stop_lsn and - * look for the first valid record in it. - * It solves the problem of occasional invalid LSN on write-busy system. - * 2. Failing that, look for record in previous segment with endpoint - * equal or greater than stop_lsn. It may(!) solve the problem of invalid LSN - * on write-idle system. If that fails too, error out. - */ - - /* stop_lsn is pointing to a 0 byte of xlog segment */ - if (stop_lsn % instance_config.xlog_seg_size == 0) - { - /* Wait for segment with current stop_lsn, it is ok for it to never arrive */ - wait_wal_lsn(xlog_path, stop_lsn, false, backup->tli, - false, true, WARNING, backup->stream); - - /* Get the first record in segment with current stop_lsn */ - lsn_tmp = get_first_record_lsn(xlog_path, segno, backup->tli, - instance_config.xlog_seg_size, - instance_config.archive_timeout); - - /* Check that returned LSN is valid and greater than stop_lsn */ - if (XLogRecPtrIsInvalid(lsn_tmp) || - !XRecOffIsValid(lsn_tmp) || - lsn_tmp < stop_lsn) - { - /* Backup from master should error out here */ - if (!backup->from_replica) - elog(ERROR, "Failed to get next WAL record after %X/%X", - (uint32) (stop_lsn >> 32), - (uint32) (stop_lsn)); - - /* No luck, falling back to looking up for previous record */ - elog(WARNING, "Failed to get next WAL record after %X/%X, " - "looking for previous WAL record", - (uint32) (stop_lsn >> 32), - (uint32) (stop_lsn)); - - /* Despite looking for previous record there is not guarantee of success - * because previous record can be the contrecord. - */ - lsn_tmp = wait_wal_lsn(xlog_path, stop_lsn, false, backup->tli, - true, false, ERROR, backup->stream); - - /* sanity */ - if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp)) - elog(ERROR, "Failed to get WAL record prior to %X/%X", - (uint32) (stop_lsn >> 32), - (uint32) (stop_lsn)); - } - } - /* stop lsn is aligned to xlog block size, just find next lsn */ - else if (stop_lsn % XLOG_BLCKSZ == 0) - { - /* Wait for segment with current stop_lsn */ - wait_wal_lsn(xlog_path, stop_lsn, false, backup->tli, - false, true, ERROR, backup->stream); - - /* Get the next closest record in segment with current stop_lsn */ - lsn_tmp = get_next_record_lsn(xlog_path, segno, backup->tli, - instance_config.xlog_seg_size, - instance_config.archive_timeout, - stop_lsn); - - /* sanity */ - if (!XRecOffIsValid(lsn_tmp) || XLogRecPtrIsInvalid(lsn_tmp)) - elog(ERROR, "Failed to get WAL record next to %X/%X", - (uint32) (stop_lsn >> 32), - (uint32) (stop_lsn)); - } - /* PostgreSQL returned something very illegal as STOP_LSN, error out */ - else - elog(ERROR, "Invalid stop_backup_lsn value %X/%X", - (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); - - /* Setting stop_backup_lsn will set stop point for streaming */ - stop_backup_lsn = lsn_tmp; - stop_lsn_exists = true; - } - - elog(INFO, "stop_lsn: %X/%X", - (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); - - /* - * Wait for stop_lsn to be archived or streamed. - * If replica returned valid STOP_LSN of not actually existing record, - * look for previous record with endpoint >= STOP_LSN. - */ - if (!stop_lsn_exists) - stop_backup_lsn = wait_wal_lsn(xlog_path, stop_lsn, false, backup->tli, - false, false, ERROR, backup->stream); - - backup->stop_lsn = stop_backup_lsn; -} - -/* Remove annoying NOTICE messages generated by backend */ -void -pg_silent_client_messages(PGconn *conn) -{ - PGresult *res; - res = pgut_execute(conn, "SET client_min_messages = warning;", - 0, NULL); - PQclear(res); -} - -void -pg_create_restore_point(PGconn *conn, time_t backup_start_time) -{ - PGresult *res; - const char *params[1]; - char name[1024]; - - snprintf(name, lengthof(name), "pg_probackup, backup_id %s", - base36enc(backup_start_time)); - params[0] = name; - - res = pgut_execute(conn, "SELECT pg_catalog.pg_create_restore_point($1)", - 1, params); - PQclear(res); -} - -void -pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica, bool is_exclusive, char **query_text) -{ - static const char - stop_exlusive_backup_query[] = - /* - * Stop the non-exclusive backup. Besides stop_lsn it returns from - * pg_stop_backup(false) copy of the backup label and tablespace map - * so they can be written to disk by the caller. - * TODO, question: add NULLs as backup_label and tablespace_map? - */ - "SELECT" - " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," - " current_timestamp(0)::timestamptz," - " pg_catalog.pg_stop_backup() as lsn", - stop_backup_on_master_query[] = - "SELECT" - " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," - " current_timestamp(0)::timestamptz," - " lsn," - " labelfile," - " spcmapfile" - " FROM pg_catalog.pg_stop_backup(false, false)", - stop_backup_on_master_before10_query[] = - "SELECT" - " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," - " current_timestamp(0)::timestamptz," - " lsn," - " labelfile," - " spcmapfile" - " FROM pg_catalog.pg_stop_backup(false)", - stop_backup_on_master_after15_query[] = - "SELECT" - " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," - " current_timestamp(0)::timestamptz," - " lsn," - " labelfile," - " spcmapfile" - " FROM pg_catalog.pg_backup_stop(false)", - /* - * In case of backup from replica >= 9.6 we do not trust minRecPoint - * and stop_backup LSN, so we use latest replayed LSN as STOP LSN. - */ - stop_backup_on_replica_query[] = - "SELECT" - " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," - " current_timestamp(0)::timestamptz," - " pg_catalog.pg_last_wal_replay_lsn()," - " labelfile," - " spcmapfile" - " FROM pg_catalog.pg_stop_backup(false, false)", - stop_backup_on_replica_before10_query[] = - "SELECT" - " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," - " current_timestamp(0)::timestamptz," - " pg_catalog.pg_last_xlog_replay_location()," - " labelfile," - " spcmapfile" - " FROM pg_catalog.pg_stop_backup(false)", - stop_backup_on_replica_after15_query[] = - "SELECT" - " pg_catalog.txid_snapshot_xmax(pg_catalog.txid_current_snapshot())," - " current_timestamp(0)::timestamptz," - " pg_catalog.pg_last_wal_replay_lsn()," - " labelfile," - " spcmapfile" - " FROM pg_catalog.pg_backup_stop(false)"; - - const char * const stop_backup_query = - is_exclusive ? - stop_exlusive_backup_query : - server_version >= 150000 ? - (is_started_on_replica ? - stop_backup_on_replica_after15_query : - stop_backup_on_master_after15_query - ) : - (server_version >= 100000 ? - (is_started_on_replica ? - stop_backup_on_replica_query : - stop_backup_on_master_query - ) : - (is_started_on_replica ? - stop_backup_on_replica_before10_query : - stop_backup_on_master_before10_query - ) - ); - bool sent = false; - - /* Make proper timestamp format for parse_time(recovery_time) */ - pgut_execute(conn, "SET datestyle = 'ISO, DMY';", 0, NULL); - // TODO: check result - - /* - * send pg_stop_backup asynchronously because we could came - * here from backup_cleanup() after some error caused by - * postgres archive_command problem and in this case we will - * wait for pg_stop_backup() forever. - */ - sent = pgut_send(conn, stop_backup_query, 0, NULL, WARNING); - if (!sent) -#if PG_VERSION_NUM >= 150000 - elog(ERROR, "Failed to send pg_backup_stop query"); -#else - elog(ERROR, "Failed to send pg_stop_backup query"); -#endif - - /* After we have sent pg_stop_backup, we don't need this callback anymore */ - pgut_atexit_pop(backup_stopbackup_callback, &stop_callback_params); - - if (query_text) - *query_text = pgut_strdup(stop_backup_query); -} - -/* - * pg_stop_backup_consume -- get 'pg_stop_backup' query results - * side effects: - * - allocates memory for tablespace_map and backup_label contents, so it must freed by caller (if its not null) - * parameters: - * - - */ -void -pg_stop_backup_consume(PGconn *conn, int server_version, - bool is_exclusive, uint32 timeout, const char *query_text, - PGStopBackupResult *result) -{ - PGresult *query_result; - uint32 pg_stop_backup_timeout = 0; - enum stop_backup_query_result_column_numbers { - recovery_xid_colno = 0, - recovery_time_colno, - lsn_colno, - backup_label_colno, - tablespace_map_colno - }; - - /* and now wait */ - while (1) - { - if (!PQconsumeInput(conn)) - elog(ERROR, "pg_stop backup() failed: %s", - PQerrorMessage(conn)); - - if (PQisBusy(conn)) - { - pg_stop_backup_timeout++; - sleep(1); - - if (interrupted) - { - pgut_cancel(conn); -#if PG_VERSION_NUM >= 150000 - elog(ERROR, "Interrupted during waiting for pg_backup_stop"); -#else - elog(ERROR, "Interrupted during waiting for pg_stop_backup"); -#endif - } - - if (pg_stop_backup_timeout == 1) - elog(INFO, "wait for pg_stop_backup()"); - - /* - * If postgres haven't answered in archive_timeout seconds, - * send an interrupt. - */ - if (pg_stop_backup_timeout > timeout) - { - pgut_cancel(conn); -#if PG_VERSION_NUM >= 150000 - elog(ERROR, "pg_backup_stop doesn't answer in %d seconds, cancel it", timeout); -#else - elog(ERROR, "pg_stop_backup doesn't answer in %d seconds, cancel it", timeout); -#endif - } - } - else - { - query_result = PQgetResult(conn); - break; - } - } - - /* Check successfull execution of pg_stop_backup() */ - if (!query_result) -#if PG_VERSION_NUM >= 150000 - elog(ERROR, "pg_backup_stop() failed"); -#else - elog(ERROR, "pg_stop_backup() failed"); -#endif - else - { - switch (PQresultStatus(query_result)) - { - /* - * We should expect only PGRES_TUPLES_OK since pg_stop_backup - * returns tuples. - */ - case PGRES_TUPLES_OK: - break; - default: - elog(ERROR, "Query failed: %s query was: %s", - PQerrorMessage(conn), query_text); - } - backup_in_progress = false; - elog(INFO, "pg_stop backup() successfully executed"); - } - - /* get results and fill result structure */ - /* get&check recovery_xid */ - if (sscanf(PQgetvalue(query_result, 0, recovery_xid_colno), XID_FMT, &result->snapshot_xid) != 1) - elog(ERROR, - "Result of txid_snapshot_xmax() is invalid: %s", - PQgetvalue(query_result, 0, recovery_xid_colno)); - - /* get&check recovery_time */ - if (!parse_time(PQgetvalue(query_result, 0, recovery_time_colno), &result->invocation_time, true)) - elog(ERROR, - "Result of current_timestamp is invalid: %s", - PQgetvalue(query_result, 0, recovery_time_colno)); - - /* get stop_backup_lsn */ - { - uint32 lsn_hi; - uint32 lsn_lo; - -// char *target_lsn = "2/F578A000"; -// XLogDataFromLSN(target_lsn, &lsn_hi, &lsn_lo); - - /* Extract timeline and LSN from results of pg_stop_backup() */ - XLogDataFromLSN(PQgetvalue(query_result, 0, lsn_colno), &lsn_hi, &lsn_lo); - /* Calculate LSN */ - result->lsn = ((uint64) lsn_hi) << 32 | lsn_lo; - } - - /* get backup_label_content */ - result->backup_label_content = NULL; - // if (!PQgetisnull(query_result, 0, backup_label_colno)) - if (!is_exclusive) - { - result->backup_label_content_len = PQgetlength(query_result, 0, backup_label_colno); - if (result->backup_label_content_len > 0) - result->backup_label_content = pgut_strndup(PQgetvalue(query_result, 0, backup_label_colno), - result->backup_label_content_len); - } else { - result->backup_label_content_len = 0; - } - - /* get tablespace_map_content */ - result->tablespace_map_content = NULL; - // if (!PQgetisnull(query_result, 0, tablespace_map_colno)) - if (!is_exclusive) - { - result->tablespace_map_content_len = PQgetlength(query_result, 0, tablespace_map_colno); - if (result->tablespace_map_content_len > 0) - result->tablespace_map_content = pgut_strndup(PQgetvalue(query_result, 0, tablespace_map_colno), - result->tablespace_map_content_len); - } else { - result->tablespace_map_content_len = 0; - } -} - -/* - * helper routine used to write backup_label and tablespace_map in pg_stop_backup() - */ -void -pg_stop_backup_write_file_helper(const char *path, const char *filename, const char *error_msg_filename, - const void *data, size_t len, parray *file_list) -{ - FILE *fp; - pgFile *file; - char full_filename[MAXPGPATH]; - - join_path_components(full_filename, path, filename); - fp = fio_fopen(full_filename, PG_BINARY_W, FIO_BACKUP_HOST); - if (fp == NULL) - elog(ERROR, "Can't open %s file \"%s\": %s", - error_msg_filename, full_filename, strerror(errno)); - - if (fio_fwrite(fp, data, len) != len || - fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "Can't write %s file \"%s\": %s", - error_msg_filename, full_filename, strerror(errno)); - - /* - * It's vital to check if files_list is initialized, - * because we could get here because the backup was interrupted - */ - if (file_list) - { - file = pgFileNew(full_filename, filename, true, 0, - FIO_BACKUP_HOST); - - if (S_ISREG(file->mode)) - { - file->crc = pgFileGetCRC(full_filename, true, false); - - file->write_size = file->size; - file->uncompressed_size = file->size; - } - parray_append(file_list, file); - } -} - -/* - * Notify end of backup to PostgreSQL server. - */ -static void -pg_stop_backup(InstanceState *instanceState, pgBackup *backup, PGconn *pg_startbackup_conn, - PGNodeInfo *nodeInfo) -{ - PGStopBackupResult stop_backup_result; - char *xlog_path, stream_xlog_path[MAXPGPATH]; - /* kludge against some old bug in archive_timeout. TODO: remove in 3.0.0 */ - int timeout = (instance_config.archive_timeout > 0) ? - instance_config.archive_timeout : ARCHIVE_TIMEOUT_DEFAULT; - char *query_text = NULL; - - /* Remove it ? */ - if (!backup_in_progress) - elog(ERROR, "Backup is not in progress"); - - pg_silent_client_messages(pg_startbackup_conn); - - /* Create restore point - * Only if backup is from master. - * For PG 9.5 create restore point only if pguser is superuser. - */ - if (!backup->from_replica && - !(nodeInfo->server_version < 90600 && - !nodeInfo->is_superuser)) //TODO: check correctness - pg_create_restore_point(pg_startbackup_conn, backup->start_time); - - /* Execute pg_stop_backup using PostgreSQL connection */ - pg_stop_backup_send(pg_startbackup_conn, nodeInfo->server_version, backup->from_replica, exclusive_backup, &query_text); - - /* - * Wait for the result of pg_stop_backup(), but no longer than - * archive_timeout seconds - */ - pg_stop_backup_consume(pg_startbackup_conn, nodeInfo->server_version, exclusive_backup, timeout, query_text, &stop_backup_result); - - if (backup->stream) - { - join_path_components(stream_xlog_path, backup->database_dir, PG_XLOG_DIR); - xlog_path = stream_xlog_path; - } - else - xlog_path = instanceState->instance_wal_subdir_path; - - wait_wal_and_calculate_stop_lsn(xlog_path, stop_backup_result.lsn, backup); - - /* Write backup_label and tablespace_map */ - if (!exclusive_backup) - { - Assert(stop_backup_result.backup_label_content != NULL); - - /* Write backup_label */ - pg_stop_backup_write_file_helper(backup->database_dir, PG_BACKUP_LABEL_FILE, "backup label", - stop_backup_result.backup_label_content, stop_backup_result.backup_label_content_len, - backup_files_list); - free(stop_backup_result.backup_label_content); - stop_backup_result.backup_label_content = NULL; - stop_backup_result.backup_label_content_len = 0; - - /* Write tablespace_map */ - if (stop_backup_result.tablespace_map_content != NULL) - { - pg_stop_backup_write_file_helper(backup->database_dir, PG_TABLESPACE_MAP_FILE, "tablespace map", - stop_backup_result.tablespace_map_content, stop_backup_result.tablespace_map_content_len, - backup_files_list); - free(stop_backup_result.tablespace_map_content); - stop_backup_result.tablespace_map_content = NULL; - stop_backup_result.tablespace_map_content_len = 0; - } - } - - if (backup->stream) - { - /* This function will also add list of xlog files - * to the passed filelist */ - if(wait_WAL_streaming_end(backup_files_list)) - elog(ERROR, "WAL streaming failed"); - } - - backup->recovery_xid = stop_backup_result.snapshot_xid; - - elog(INFO, "Getting the Recovery Time from WAL"); - - /* iterate over WAL from stop_backup lsn to start_backup lsn */ - if (!read_recovery_info(xlog_path, backup->tli, - instance_config.xlog_seg_size, - backup->start_lsn, backup->stop_lsn, - &backup->recovery_time)) - { - elog(INFO, "Failed to find Recovery Time in WAL, forced to trust current_timestamp"); - backup->recovery_time = stop_backup_result.invocation_time; - } - - /* Cleanup */ - pg_free(query_text); -} - -/* - * Notify end of backup to server when "backup_label" is in the root directory - * of the DB cluster. - * Also update backup status to ERROR when the backup is not finished. - */ -static void -backup_cleanup(bool fatal, void *userdata) -{ - /* - * Update status of backup in BACKUP_CONTROL_FILE to ERROR. - * end_time != 0 means backup finished - */ - if (current.status == BACKUP_STATUS_RUNNING && current.end_time == 0) - { - elog(WARNING, "Backup %s is running, setting its status to ERROR", - backup_id_of(¤t)); - current.end_time = time(NULL); - current.status = BACKUP_STATUS_ERROR; - write_backup(¤t, true); - } -} - -/* - * Take a backup of the PGDATA at a file level. - * Copy all directories and files listed in backup_files_list. - * If the file is 'datafile' (regular relation's main fork), read it page by page, - * verify checksum and copy. - * In incremental backup mode, copy only files or datafiles' pages changed after - * previous backup. - */ -static void * -backup_files(void *arg) -{ - int i; - static time_t prev_time; - - backup_files_arg *arguments = (backup_files_arg *) arg; - int n_backup_files_list = parray_num(arguments->files_list); - - prev_time = current.start_time; - - /* backup a file */ - for (i = 0; i < n_backup_files_list; i++) - { - pgFile *file = (pgFile *) parray_get(arguments->files_list, i); - - /* We have already copied all directories */ - if (S_ISDIR(file->mode)) - continue; - - if (arguments->thread_num == 1) - { - /* update backup_content.control every 60 seconds */ - if ((difftime(time(NULL), prev_time)) > 60) - { - write_backup_filelist(¤t, arguments->files_list, arguments->from_root, - arguments->external_dirs, false); - /* update backup control file to update size info */ - write_backup(¤t, true); - - prev_time = time(NULL); - } - } - - if (file->skip_cfs_nested) - continue; - - if (!pg_atomic_test_set_flag(&file->lock)) - continue; - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during backup"); - - elog(progress ? INFO : LOG, "Progress: (%d/%d). Process file \"%s\"", - i + 1, n_backup_files_list, file->rel_path); - - if (file->is_cfs) - { - backup_cfs_segment(i, file, arguments); - } - else - { - process_file(i, file, arguments); - } - } - - /* ssh connection to longer needed */ - fio_disconnect(); - - /* Data files transferring is successful */ - arguments->ret = 0; - - return NULL; -} - -static void -process_file(int i, pgFile *file, backup_files_arg *arguments) -{ - char from_fullpath[MAXPGPATH]; - char to_fullpath[MAXPGPATH]; - pgFile *prev_file = NULL; - - elog(progress ? INFO : LOG, "Progress: (%d/%zu). Process file \"%s\"", - i + 1, parray_num(arguments->files_list), file->rel_path); - - /* Handle zero sized files */ - if (file->size == 0) - { - file->write_size = 0; - return; - } - - /* construct from_fullpath & to_fullpath */ - if (file->external_dir_num == 0) - { - join_path_components(from_fullpath, arguments->from_root, file->rel_path); - join_path_components(to_fullpath, arguments->to_root, file->rel_path); - } - else - { - char external_dst[MAXPGPATH]; - char *external_path = parray_get(arguments->external_dirs, - file->external_dir_num - 1); - - makeExternalDirPathByNum(external_dst, - arguments->external_prefix, - file->external_dir_num); - - join_path_components(to_fullpath, external_dst, file->rel_path); - join_path_components(from_fullpath, external_path, file->rel_path); - } - - /* Encountered some strange beast */ - if (!S_ISREG(file->mode)) - { - elog(WARNING, "Unexpected type %d of file \"%s\", skipping", - file->mode, from_fullpath); - return; - } - - /* Check that file exist in previous backup */ - if (current.backup_mode != BACKUP_MODE_FULL) - { - pgFile **prevFileTmp = NULL; - prevFileTmp = (pgFile **) parray_bsearch(arguments->prev_filelist, - file, pgFileCompareRelPathWithExternal); - if (prevFileTmp) - { - /* File exists in previous backup */ - file->exists_in_prev = true; - prev_file = *prevFileTmp; - } - } - - /* backup file */ - if (file->is_datafile && !file->is_cfs) - { - backup_data_file(file, from_fullpath, to_fullpath, - arguments->prev_start_lsn, - current.backup_mode, - instance_config.compress_alg, - instance_config.compress_level, - arguments->nodeInfo->checksum_version, - arguments->hdr_map, false); - } - else - { - backup_non_data_file(file, prev_file, from_fullpath, to_fullpath, - current.backup_mode, current.parent_backup, true); - } - - if (file->write_size == FILE_NOT_FOUND) - return; - - if (file->write_size == BYTES_INVALID) - { - elog(LOG, "Skipping the unchanged file: \"%s\"", from_fullpath); - return; - } - - elog(LOG, "File \"%s\". Copied "INT64_FORMAT " bytes", - from_fullpath, file->write_size); - -} - -static void -backup_cfs_segment(int i, pgFile *file, backup_files_arg *arguments) { - pgFile *data_file = file; - pgFile *cfm_file = NULL; - pgFile *data_bck_file = NULL; - pgFile *cfm_bck_file = NULL; - - while (data_file->cfs_chain) - { - data_file = data_file->cfs_chain; - if (data_file->forkName == cfm) - cfm_file = data_file; - if (data_file->forkName == cfs_bck) - data_bck_file = data_file; - if (data_file->forkName == cfm_bck) - cfm_bck_file = data_file; - } - data_file = file; - if (data_file->relOid >= FirstNormalObjectId && cfm_file == NULL) - { - elog(ERROR, "'CFS' file '%s' have to have '%s.cfm' companion file", - data_file->rel_path, data_file->name); - } - - elog(LOG, "backup CFS segment %s, data_file=%s, cfm_file=%s, data_bck_file=%s, cfm_bck_file=%s", - data_file->name, data_file->name, cfm_file->name, data_bck_file == NULL? "NULL": data_bck_file->name, cfm_bck_file == NULL? "NULL": cfm_bck_file->name); - - /* storing cfs segment. processing corner case [PBCKP-287] stage 1. - * - when we do have data_bck_file we should skip both data_bck_file and cfm_bck_file if exists. - * they are removed by cfs_recover() during postgres start. - */ - if (data_bck_file) - { - if (cfm_bck_file) - cfm_bck_file->write_size = FILE_NOT_FOUND; - data_bck_file->write_size = FILE_NOT_FOUND; - } - /* else we store cfm_bck_file. processing corner case [PBCKP-287] stage 2. - * - when we do have cfm_bck_file only we should store it. - * it will replace cfm_file after postgres start. - */ - else if (cfm_bck_file) - process_file(i, cfm_bck_file, arguments); - - /* storing cfs segment in order cfm_file -> datafile to guarantee their consistency */ - /* cfm_file could be NULL for system tables. But we don't clear is_cfs flag - * for compatibility with older pg_probackup. */ - if (cfm_file) - process_file(i, cfm_file, arguments); - process_file(i, data_file, arguments); - elog(LOG, "Backup CFS segment %s done", data_file->name); -} - -/* - * Extract information about files in backup_list parsing their names: - * - remove temp tables from the list - * - remove unlogged tables from the list (leave the _init fork) - * - set flags for database directories - * - set flags for datafiles - */ -void -parse_filelist_filenames(parray *files, const char *root) -{ - size_t i = 0; - Oid unlogged_file_reloid = 0; - - while (i < parray_num(files)) - { - pgFile *file = (pgFile *) parray_get(files, i); - int sscanf_result; - - if (S_ISREG(file->mode) && - path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path)) - { - /* - * Found file in pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY - * Legal only in case of 'pg_compression' - */ - if (strcmp(file->name, "pg_compression") == 0) - { - /* processing potential cfs tablespace */ - Oid tblspcOid; - Oid dbOid; - char tmp_rel_path[MAXPGPATH]; - /* - * Check that pg_compression is located under - * TABLESPACE_VERSION_DIRECTORY - */ - sscanf_result = sscanf(file->rel_path, PG_TBLSPC_DIR "/%u/%s/%u", - &tblspcOid, tmp_rel_path, &dbOid); - - /* Yes, it is */ - if (sscanf_result == 2 && - strncmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY, - strlen(TABLESPACE_VERSION_DIRECTORY)) == 0) { - /* rewind index to the beginning of cfs tablespace */ - rewind_and_mark_cfs_datafiles(files, root, file->rel_path, i); - } - } - } - - if (S_ISREG(file->mode) && file->tblspcOid != 0 && - file->name && file->name[0]) - { - if (file->forkName == init) - { - /* - * Do not backup files of unlogged relations. - * scan filelist backward and exclude these files. - */ - int unlogged_file_num = i - 1; - pgFile *unlogged_file = (pgFile *) parray_get(files, - unlogged_file_num); - - unlogged_file_reloid = file->relOid; - - while (unlogged_file_num >= 0 && - (unlogged_file_reloid != 0) && - (unlogged_file->relOid == unlogged_file_reloid)) - { - /* flagged to remove from list on stage 2 */ - unlogged_file->remove_from_list = true; - - unlogged_file_num--; - - unlogged_file = (pgFile *) parray_get(files, - unlogged_file_num); - } - } - } - - i++; - } - - /* stage 2. clean up from temporary tables */ - parray_remove_if(files, remove_excluded_files_criterion, NULL, pgFileFree); -} - -static bool -remove_excluded_files_criterion(void *value, void *exclude_args) { - pgFile *file = (pgFile*)value; - return file->remove_from_list; -} - -static uint32 -hash_rel_seg(pgFile* file) -{ - uint32 hash = hash_mix32_2(file->relOid, file->segno); - return hash_mix32_2(hash, 0xcf5); -} - -/* If file is equal to pg_compression, then we consider this tablespace as - * cfs-compressed and should mark every file in this tablespace as cfs-file - * Setting is_cfs is done via going back through 'files' set every file - * that contain cfs_tablespace in his path as 'is_cfs' - * Goings back through array 'files' is valid option possible because of current - * sort rules: - * tblspcOid/TABLESPACE_VERSION_DIRECTORY - * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid - * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid/1 - * tblspcOid/TABLESPACE_VERSION_DIRECTORY/dboid/1.cfm - * tblspcOid/TABLESPACE_VERSION_DIRECTORY/pg_compression - * - * @returns index of first tablespace entry, i.e tblspcOid/TABLESPACE_VERSION_DIRECTORY - */ -static void -rewind_and_mark_cfs_datafiles(parray *files, const char *root, char *relative, size_t i) -{ - int len; - int p; - int j; - pgFile *prev_file; - pgFile *tmp_file; - char *cfs_tblspc_path; - uint32 h; - - /* hash table for cfm files */ -#define HASHN 128 - parray *hashtab[HASHN] = {NULL}; - parray *bucket; - for (p = 0; p < HASHN; p++) - hashtab[p] = parray_new(); - - - cfs_tblspc_path = strdup(relative); - if(!cfs_tblspc_path) - elog(ERROR, "Out of memory"); - len = strlen("/pg_compression"); - cfs_tblspc_path[strlen(cfs_tblspc_path) - len] = 0; - elog(LOG, "CFS DIRECTORY %s, pg_compression path: %s", cfs_tblspc_path, relative); - - for (p = (int) i; p >= 0; p--) - { - prev_file = (pgFile *) parray_get(files, (size_t) p); - - elog(LOG, "Checking file in cfs tablespace %s", prev_file->rel_path); - - if (strstr(prev_file->rel_path, cfs_tblspc_path) == NULL) - { - elog(LOG, "Breaking on %s", prev_file->rel_path); - break; - } - - if (!S_ISREG(prev_file->mode)) - continue; - - h = hash_rel_seg(prev_file); - bucket = hashtab[h % HASHN]; - - if (prev_file->forkName == cfm || prev_file->forkName == cfm_bck || - prev_file->forkName == cfs_bck) - { - prev_file->skip_cfs_nested = true; - parray_append(bucket, prev_file); - } - else if (prev_file->is_datafile && prev_file->forkName == none) - { - elog(LOG, "Processing 'cfs' file %s", prev_file->rel_path); - /* have to mark as is_cfs even for system-tables for compatibility - * with older pg_probackup */ - prev_file->is_cfs = true; - prev_file->cfs_chain = NULL; - for (j = 0; j < parray_num(bucket); j++) - { - tmp_file = parray_get(bucket, j); - elog(LOG, "Linking 'cfs' file '%s' to '%s'", - tmp_file->rel_path, prev_file->rel_path); - if (tmp_file->relOid == prev_file->relOid && - tmp_file->segno == prev_file->segno) - { - tmp_file->cfs_chain = prev_file->cfs_chain; - prev_file->cfs_chain = tmp_file; - parray_remove(bucket, j); - j--; - } - } - } - } - - for (p = 0; p < HASHN; p++) - { - bucket = hashtab[p]; - for (j = 0; j < parray_num(bucket); j++) - { - tmp_file = parray_get(bucket, j); - elog(WARNING, "Orphaned cfs related file '%s'", tmp_file->rel_path); - } - parray_free(bucket); - hashtab[p] = NULL; - } -#undef HASHN - free(cfs_tblspc_path); -} - -/* - * Find pgfile by given rnode in the backup_files_list - * and add given blkno to its pagemap. - */ -void -process_block_change(ForkNumber forknum, RelFileNode rnode, BlockNumber blkno) -{ -// char *path; - char *rel_path; - BlockNumber blkno_inseg; - int segno; - pgFile **file_item; - pgFile f; - - segno = blkno / RELSEG_SIZE; - blkno_inseg = blkno % RELSEG_SIZE; - - rel_path = relpathperm(rnode, forknum); - if (segno > 0) - f.rel_path = psprintf("%s.%u", rel_path, segno); - else - f.rel_path = rel_path; - - f.external_dir_num = 0; - - /* backup_files_list should be sorted before */ - file_item = (pgFile **) parray_bsearch(backup_files_list, &f, - pgFileCompareRelPathWithExternal); - - /* - * If we don't have any record of this file in the file map, it means - * that it's a relation that did not have much activity since the last - * backup. We can safely ignore it. If it is a new relation file, the - * backup would simply copy it as-is. - */ - if (file_item) - { - /* We need critical section only we use more than one threads */ - if (num_threads > 1) - pthread_lock(&backup_pagemap_mutex); - - datapagemap_add(&(*file_item)->pagemap, blkno_inseg); - - if (num_threads > 1) - pthread_mutex_unlock(&backup_pagemap_mutex); - } - - if (segno > 0) - pg_free(f.rel_path); - pg_free(rel_path); - -} - -void -check_external_for_tablespaces(parray *external_list, PGconn *backup_conn) -{ - PGresult *res; - int i = 0; - int j = 0; - char *tablespace_path = NULL; - char *query = "SELECT pg_catalog.pg_tablespace_location(oid) " - "FROM pg_catalog.pg_tablespace " - "WHERE pg_catalog.pg_tablespace_location(oid) <> '';"; - - res = pgut_execute(backup_conn, query, 0, NULL); - - /* Check successfull execution of query */ - if (!res) - elog(ERROR, "Failed to get list of tablespaces"); - - for (i = 0; i < res->ntups; i++) - { - tablespace_path = PQgetvalue(res, i, 0); - Assert (strlen(tablespace_path) > 0); - - canonicalize_path(tablespace_path); - - for (j = 0; j < parray_num(external_list); j++) - { - char *external_path = parray_get(external_list, j); - - if (path_is_prefix_of_path(external_path, tablespace_path)) - elog(ERROR, "External directory path (-E option) \"%s\" " - "contains tablespace \"%s\"", - external_path, tablespace_path); - if (path_is_prefix_of_path(tablespace_path, external_path)) - elog(WARNING, "External directory path (-E option) \"%s\" " - "is in tablespace directory \"%s\"", - tablespace_path, external_path); - } - } - PQclear(res); - - /* Check that external directories do not overlap */ - if (parray_num(external_list) < 2) - return; - - for (i = 0; i < parray_num(external_list); i++) - { - char *external_path = parray_get(external_list, i); - - for (j = 0; j < parray_num(external_list); j++) - { - char *tmp_external_path = parray_get(external_list, j); - - /* skip yourself */ - if (j == i) - continue; - - if (path_is_prefix_of_path(external_path, tmp_external_path)) - elog(ERROR, "External directory path (-E option) \"%s\" " - "contain another external directory \"%s\"", - external_path, tmp_external_path); - - } - } -} - -/* - * Calculate pgdata_bytes - * accepts (parray *) of (pgFile *) - */ -int64 -calculate_datasize_of_filelist(parray *filelist) -{ - int64 bytes = 0; - int i; - - /* parray_num don't check for NULL */ - if (filelist == NULL) - return 0; - - for (i = 0; i < parray_num(filelist); i++) - { - pgFile *file = (pgFile *) parray_get(filelist, i); - - if (file->external_dir_num != 0 || file->excluded) - continue; - - if (S_ISDIR(file->mode)) - { - // TODO is a dir always 4K? - bytes += 4096; - continue; - } - - bytes += file->size; - } - return bytes; -} diff --git a/src/catalog.c b/src/catalog.c deleted file mode 100644 index b29090789..000000000 --- a/src/catalog.c +++ /dev/null @@ -1,3181 +0,0 @@ -/*------------------------------------------------------------------------- - * - * catalog.c: backup catalog operation - * - * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" -#include "access/timeline.h" - -#include -#include -#include -#include - -#include "utils/file.h" -#include "utils/configuration.h" - -static pgBackup* get_closest_backup(timelineInfo *tlinfo); -static pgBackup* get_oldest_backup(timelineInfo *tlinfo); -static const char *backupModes[] = {"", "PAGE", "PTRACK", "DELTA", "FULL"}; -static pgBackup *readBackupControlFile(const char *path); -static int create_backup_dir(pgBackup *backup, const char *backup_instance_path); - -static bool backup_lock_exit_hook_registered = false; -static parray *locks = NULL; - -static int grab_excl_lock_file(const char *backup_dir, const char *backup_id, bool strict); -static int grab_shared_lock_file(pgBackup *backup); -static int wait_shared_owners(pgBackup *backup); - - -static void unlink_lock_atexit(bool fatal, void *userdata); -static void unlock_backup(const char *backup_dir, const char *backup_id, bool exclusive); -static void release_excl_lock_file(const char *backup_dir); -static void release_shared_lock_file(const char *backup_dir); - -#define LOCK_OK 0 -#define LOCK_FAIL_TIMEOUT 1 -#define LOCK_FAIL_ENOSPC 2 -#define LOCK_FAIL_EROFS 3 - -typedef struct LockInfo -{ - char backup_id[10]; - char backup_dir[MAXPGPATH]; - bool exclusive; -} LockInfo; - -timelineInfo * -timelineInfoNew(TimeLineID tli) -{ - timelineInfo *tlinfo = (timelineInfo *) pgut_malloc(sizeof(timelineInfo)); - MemSet(tlinfo, 0, sizeof(timelineInfo)); - tlinfo->tli = tli; - tlinfo->switchpoint = InvalidXLogRecPtr; - tlinfo->parent_link = NULL; - tlinfo->xlog_filelist = parray_new(); - tlinfo->anchor_lsn = InvalidXLogRecPtr; - tlinfo->anchor_tli = 0; - tlinfo->n_xlog_files = 0; - return tlinfo; -} - -/* free timelineInfo object */ -void -timelineInfoFree(void *tliInfo) -{ - timelineInfo *tli = (timelineInfo *) tliInfo; - - parray_walk(tli->xlog_filelist, pgFileFree); - parray_free(tli->xlog_filelist); - - if (tli->backups) - { - /* backups themselves should freed separately */ -// parray_walk(tli->backups, pgBackupFree); - parray_free(tli->backups); - } - - pfree(tliInfo); -} - -/* Iterate over locked backups and unlock them */ -void -unlink_lock_atexit(bool unused_fatal, void *unused_userdata) -{ - int i; - - if (locks == NULL) - return; - - for (i = 0; i < parray_num(locks); i++) - { - LockInfo *lock = (LockInfo *) parray_get(locks, i); - unlock_backup(lock->backup_dir, lock->backup_id, lock->exclusive); - } - - parray_walk(locks, pg_free); - parray_free(locks); - locks = NULL; -} - -/* - * Read backup meta information from BACKUP_CONTROL_FILE. - * If no backup matches, return NULL. - */ -pgBackup * -read_backup(const char *root_dir) -{ - char conf_path[MAXPGPATH]; - - join_path_components(conf_path, root_dir, BACKUP_CONTROL_FILE); - - return readBackupControlFile(conf_path); -} - -/* - * Save the backup status into BACKUP_CONTROL_FILE. - * - * We need to reread the backup using its ID and save it changing only its - * status. - */ -void -write_backup_status(pgBackup *backup, BackupStatus status, - bool strict) -{ - pgBackup *tmp; - - tmp = read_backup(backup->root_dir); - if (!tmp) - { - /* - * Silently exit the function, since read_backup already logged the - * warning message. - */ - return; - } - - /* overwrite control file only if status has changed */ - if (tmp->status == status) - { - pgBackupFree(tmp); - return; - } - - backup->status = status; - tmp->status = backup->status; - tmp->root_dir = pgut_strdup(backup->root_dir); - - /* lock backup in exclusive mode */ - if (!lock_backup(tmp, strict, true)) - elog(ERROR, "Cannot lock backup %s directory", backup_id_of(backup)); - - write_backup(tmp, strict); - - pgBackupFree(tmp); -} - -/* - * Lock backup in either exclusive or shared mode. - * "strict" flag allows to ignore "out of space" errors and should be - * used only by DELETE command to free disk space on filled up - * filesystem. - * - * Only read only tasks (validate, restore) are allowed to take shared locks. - * Changing backup metadata must be done with exclusive lock. - * - * Only one process can hold exclusive lock at any time. - * Exlusive lock - PID of process, holding the lock - is placed in - * lock file: BACKUP_LOCK_FILE. - * - * Multiple proccess are allowed to take shared locks simultaneously. - * Shared locks - PIDs of proccesses, holding the lock - are placed in - * separate lock file: BACKUP_RO_LOCK_FILE. - * When taking shared lock, a brief exclusive lock is taken. - * - * -> exclusive -> grab exclusive lock file and wait until all shared lockers are gone, return - * -> shared -> grab exclusive lock file, grab shared lock file, release exclusive lock file, return - * - * TODO: lock-timeout as parameter - */ -bool -lock_backup(pgBackup *backup, bool strict, bool exclusive) -{ - int rc; - char lock_file[MAXPGPATH]; - bool enospc_detected = false; - LockInfo *lock = NULL; - - join_path_components(lock_file, backup->root_dir, BACKUP_LOCK_FILE); - - rc = grab_excl_lock_file(backup->root_dir, backup_id_of(backup), strict); - - if (rc == LOCK_FAIL_TIMEOUT) - return false; - else if (rc == LOCK_FAIL_ENOSPC) - { - /* - * If we failed to take exclusive lock due to ENOSPC, - * then in lax mode treat such condition as if lock was taken. - */ - - enospc_detected = true; - if (strict) - return false; - } - else if (rc == LOCK_FAIL_EROFS) - { - /* - * If we failed to take exclusive lock due to EROFS, - * then in shared mode treat such condition as if lock was taken. - */ - return !exclusive; - } - - /* - * We have exclusive lock, now there are following scenarios: - * - * 1. If we are for exlusive lock, then we must open the shared lock file - * and check if any of the processes listed there are still alive. - * If some processes are alive and are not going away in lock_timeout, - * then return false. - * - * 2. If we are here for non-exlusive lock, then write the pid - * into shared lock file and release the exclusive lock. - */ - - if (exclusive) - rc = wait_shared_owners(backup); - else - rc = grab_shared_lock_file(backup); - - if (rc != 0) - { - /* - * Failed to grab shared lock or (in case of exclusive mode) shared lock owners - * are not going away in time, release the exclusive lock file and return in shame. - */ - release_excl_lock_file(backup->root_dir); - return false; - } - - if (!exclusive) - { - /* Shared lock file is grabbed, now we can release exclusive lock file */ - release_excl_lock_file(backup->root_dir); - } - - if (exclusive && !strict && enospc_detected) - { - /* We are in lax exclusive mode and EONSPC was encountered: - * once again try to grab exclusive lock file, - * because there is a chance that release of shared lock file in wait_shared_owners may have - * freed some space on filesystem, thanks to unlinking of BACKUP_RO_LOCK_FILE. - * If somebody concurrently acquired exclusive lock file first, then we should give up. - */ - if (grab_excl_lock_file(backup->root_dir, backup_id_of(backup), strict) == LOCK_FAIL_TIMEOUT) - return false; - - return true; - } - - /* - * Arrange the unlocking at proc_exit. - */ - if (!backup_lock_exit_hook_registered) - { - pgut_atexit_push(unlink_lock_atexit, NULL); - backup_lock_exit_hook_registered = true; - } - - /* save lock metadata for later unlocking */ - lock = pgut_malloc(sizeof(LockInfo)); - snprintf(lock->backup_id, 10, "%s", backup_id_of(backup)); - snprintf(lock->backup_dir, MAXPGPATH, "%s", backup->root_dir); - lock->exclusive = exclusive; - - /* Use parray for lock release */ - if (locks == NULL) - locks = parray_new(); - parray_append(locks, lock); - - return true; -} - -/* - * Lock backup in exclusive mode - * Result codes: - * LOCK_OK Success - * LOCK_FAIL_TIMEOUT Failed to acquire lock in lock_timeout time - * LOCK_FAIL_ENOSPC Failed to acquire lock due to ENOSPC - * LOCK_FAIL_EROFS Failed to acquire lock due to EROFS - */ -int -grab_excl_lock_file(const char *root_dir, const char *backup_id, bool strict) -{ - char lock_file[MAXPGPATH]; - int fd = 0; - char buffer[256]; - int ntries = LOCK_TIMEOUT; - int empty_tries = LOCK_STALE_TIMEOUT; - int len; - int encoded_pid; - - join_path_components(lock_file, root_dir, BACKUP_LOCK_FILE); - - /* - * We need a loop here because of race conditions. But don't loop forever - * (for example, a non-writable $backup_instance_path directory might cause a failure - * that won't go away). - */ - do - { - FILE *fp_out = NULL; - - if (interrupted) - elog(ERROR, "Interrupted while locking backup %s", backup_id); - - /* - * Try to create the lock file --- O_EXCL makes this atomic. - * - * Think not to make the file protection weaker than 0600. See - * comments below. - */ - fd = fio_open(lock_file, O_RDWR | O_CREAT | O_EXCL, FIO_BACKUP_HOST); - if (fd >= 0) - break; /* Success; exit the retry loop */ - - /* read-only fs is a special case */ - if (errno == EROFS) - { - elog(WARNING, "Could not create lock file \"%s\": %s", - lock_file, strerror(errno)); - return LOCK_FAIL_EROFS; - } - - /* - * Couldn't create the pid file. Probably it already exists. - * If file already exists or we have some permission problem (???), - * then retry; - */ -// if ((errno != EEXIST && errno != EACCES)) - if (errno != EEXIST) - elog(ERROR, "Could not create lock file \"%s\": %s", - lock_file, strerror(errno)); - - /* - * Read the file to get the old owner's PID. Note race condition - * here: file might have been deleted since we tried to create it. - */ - - fp_out = fopen(lock_file, "r"); - if (fp_out == NULL) - { - if (errno == ENOENT) - continue; /* race condition; try again */ - elog(ERROR, "Cannot open lock file \"%s\": %s", lock_file, strerror(errno)); - } - - len = fread(buffer, 1, sizeof(buffer) - 1, fp_out); - if (ferror(fp_out)) - elog(ERROR, "Cannot read from lock file: \"%s\"", lock_file); - fclose(fp_out); - - /* - * There are several possible reasons for lock file - * to be empty: - * - system crash - * - process crash - * - race between writer and reader - * - * Consider empty file to be stale after LOCK_STALE_TIMEOUT attempts. - * - * TODO: alternatively we can write into temp file (lock_file_%pid), - * rename it and then re-read lock file to make sure, - * that we are successfully acquired the lock. - */ - if (len == 0) - { - if (empty_tries == 0) - { - elog(WARNING, "Lock file \"%s\" is empty", lock_file); - goto grab_lock; - } - - if ((empty_tries % LOG_FREQ) == 0) - elog(WARNING, "Waiting %u seconds on empty exclusive lock for backup %s", - empty_tries, backup_id); - - sleep(1); - /* - * waiting on empty lock file should not affect - * the timer for concurrent lockers (ntries). - */ - empty_tries--; - ntries++; - continue; - } - - encoded_pid = atoi(buffer); - - if (encoded_pid <= 0) - { - elog(WARNING, "Bogus data in lock file \"%s\": \"%s\"", - lock_file, buffer); - goto grab_lock; - } - - /* - * Check to see if the other process still exists - * Normally kill() will fail with ESRCH if the given PID doesn't - * exist. - */ - if (encoded_pid == my_pid) - return LOCK_OK; - - if (kill(encoded_pid, 0) == 0) - { - /* complain every fifth interval */ - if ((ntries % LOG_FREQ) == 0) - { - elog(WARNING, "Process %d is using backup %s, and is still running", - encoded_pid, backup_id); - - elog(WARNING, "Waiting %u seconds on exclusive lock for backup %s", - ntries, backup_id); - } - - sleep(1); - - /* try again */ - continue; - } - else - { - if (errno == ESRCH) - elog(WARNING, "Process %d which used backup %s no longer exists", - encoded_pid, backup_id); - else - elog(ERROR, "Failed to send signal 0 to a process %d: %s", - encoded_pid, strerror(errno)); - } - -grab_lock: - /* - * Looks like nobody's home. Unlink the file and try again to create - * it. Need a loop because of possible race condition against other - * would-be creators. - */ - if (fio_unlink(lock_file, FIO_BACKUP_HOST) < 0) - { - if (errno == ENOENT) - continue; /* race condition, again */ - elog(ERROR, "Could not remove old lock file \"%s\": %s", - lock_file, strerror(errno)); - } - - } while (ntries--); - - /* Failed to acquire exclusive lock in time */ - if (fd <= 0) - return LOCK_FAIL_TIMEOUT; - - /* - * Successfully created the file, now fill it. - */ - snprintf(buffer, sizeof(buffer), "%d\n", my_pid); - - errno = 0; - if (fio_write(fd, buffer, strlen(buffer)) != strlen(buffer)) - { - int save_errno = errno; - - fio_close(fd); - fio_unlink(lock_file, FIO_BACKUP_HOST); - - /* In lax mode if we failed to grab lock because of 'out of space error', - * then treat backup as locked. - * Only delete command should be run in lax mode. - */ - if (!strict && save_errno == ENOSPC) - return LOCK_FAIL_ENOSPC; - else - elog(ERROR, "Could not write lock file \"%s\": %s", - lock_file, strerror(save_errno)); - } - - if (fio_flush(fd) != 0) - { - int save_errno = errno; - - fio_close(fd); - fio_unlink(lock_file, FIO_BACKUP_HOST); - - /* In lax mode if we failed to grab lock because of 'out of space error', - * then treat backup as locked. - * Only delete command should be run in lax mode. - */ - if (!strict && save_errno == ENOSPC) - return LOCK_FAIL_ENOSPC; - else - elog(ERROR, "Could not flush lock file \"%s\": %s", - lock_file, strerror(save_errno)); - } - - if (fio_close(fd) != 0) - { - int save_errno = errno; - - fio_unlink(lock_file, FIO_BACKUP_HOST); - - if (!strict && errno == ENOSPC) - return LOCK_FAIL_ENOSPC; - else - elog(ERROR, "Could not close lock file \"%s\": %s", - lock_file, strerror(save_errno)); - } - -// elog(LOG, "Acquired exclusive lock for backup %s after %ds", -// backup_id_of(backup), -// LOCK_TIMEOUT - ntries + LOCK_STALE_TIMEOUT - empty_tries); - - return LOCK_OK; -} - -/* Wait until all shared lock owners are gone - * 0 - successs - * 1 - fail - */ -int -wait_shared_owners(pgBackup *backup) -{ - FILE *fp = NULL; - char buffer[256]; - pid_t encoded_pid = 0; - int ntries = LOCK_TIMEOUT; - char lock_file[MAXPGPATH]; - - join_path_components(lock_file, backup->root_dir, BACKUP_RO_LOCK_FILE); - - fp = fopen(lock_file, "r"); - if (fp == NULL && errno != ENOENT) - elog(ERROR, "Cannot open lock file \"%s\": %s", lock_file, strerror(errno)); - - /* iterate over pids in lock file */ - while (fp && fgets(buffer, sizeof(buffer), fp)) - { - encoded_pid = atoi(buffer); - if (encoded_pid <= 0) - { - elog(WARNING, "Bogus data in lock file \"%s\": \"%s\"", lock_file, buffer); - continue; - } - - /* wait until shared lock owners go away */ - do - { - if (interrupted) - elog(ERROR, "Interrupted while locking backup %s", - backup_id_of(backup)); - - if (encoded_pid == my_pid) - break; - - /* check if lock owner is still alive */ - if (kill(encoded_pid, 0) == 0) - { - /* complain from time to time */ - if ((ntries % LOG_FREQ) == 0) - { - elog(WARNING, "Process %d is using backup %s in shared mode, and is still running", - encoded_pid, backup_id_of(backup)); - - elog(WARNING, "Waiting %u seconds on lock for backup %s", ntries, - backup_id_of(backup)); - } - - sleep(1); - - /* try again */ - continue; - } - else if (errno != ESRCH) - elog(ERROR, "Failed to send signal 0 to a process %d: %s", - encoded_pid, strerror(errno)); - - /* locker is dead */ - break; - - } while (ntries--); - } - - if (fp && ferror(fp)) - elog(ERROR, "Cannot read from lock file: \"%s\"", lock_file); - - if (fp) - fclose(fp); - - /* some shared owners are still alive */ - if (ntries <= 0) - { - elog(WARNING, "Cannot to lock backup %s in exclusive mode, because process %u owns shared lock", - backup_id_of(backup), encoded_pid); - return 1; - } - - /* unlink shared lock file */ - fio_unlink(lock_file, FIO_BACKUP_HOST); - return 0; -} - -/* - * Lock backup in shared mode - * 0 - successs - * 1 - fail - */ -int -grab_shared_lock_file(pgBackup *backup) -{ - FILE *fp_in = NULL; - FILE *fp_out = NULL; - char buf_in[256]; - pid_t encoded_pid; - char lock_file[MAXPGPATH]; - - char buffer[8192]; /*TODO: should be enough, but maybe malloc+realloc is better ? */ - char lock_file_tmp[MAXPGPATH]; - int buffer_len = 0; - - join_path_components(lock_file, backup->root_dir, BACKUP_RO_LOCK_FILE); - snprintf(lock_file_tmp, MAXPGPATH, "%s%s", lock_file, "tmp"); - - /* open already existing lock files */ - fp_in = fopen(lock_file, "r"); - if (fp_in == NULL && errno != ENOENT) - elog(ERROR, "Cannot open lock file \"%s\": %s", lock_file, strerror(errno)); - - /* read PIDs of owners */ - while (fp_in && fgets(buf_in, sizeof(buf_in), fp_in)) - { - encoded_pid = atoi(buf_in); - if (encoded_pid <= 0) - { - elog(WARNING, "Bogus data in lock file \"%s\": \"%s\"", lock_file, buf_in); - continue; - } - - if (encoded_pid == my_pid) - continue; - - if (kill(encoded_pid, 0) == 0) - { - /* - * Somebody is still using this backup in shared mode, - * copy this pid into a new file. - */ - buffer_len += snprintf(buffer+buffer_len, 4096, "%u\n", encoded_pid); - } - else if (errno != ESRCH) - elog(ERROR, "Failed to send signal 0 to a process %d: %s", - encoded_pid, strerror(errno)); - } - - if (fp_in) - { - if (ferror(fp_in)) - elog(ERROR, "Cannot read from lock file: \"%s\"", lock_file); - fclose(fp_in); - } - - fp_out = fopen(lock_file_tmp, "w"); - if (fp_out == NULL) - { - if (errno == EROFS) - return 0; - - elog(ERROR, "Cannot open temp lock file \"%s\": %s", lock_file_tmp, strerror(errno)); - } - - /* add my own pid */ - buffer_len += snprintf(buffer+buffer_len, sizeof(buffer), "%u\n", my_pid); - - /* write out the collected PIDs to temp lock file */ - fwrite(buffer, 1, buffer_len, fp_out); - - if (ferror(fp_out)) - elog(ERROR, "Cannot write to lock file: \"%s\"", lock_file_tmp); - - if (fclose(fp_out) != 0) - elog(ERROR, "Cannot close temp lock file \"%s\": %s", lock_file_tmp, strerror(errno)); - - if (rename(lock_file_tmp, lock_file) < 0) - elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", - lock_file_tmp, lock_file, strerror(errno)); - - return 0; -} - -void -unlock_backup(const char *backup_dir, const char *backup_id, bool exclusive) -{ - if (exclusive) - { - release_excl_lock_file(backup_dir); - return; - } - - /* To remove shared lock, we must briefly obtain exclusive lock, ... */ - if (grab_excl_lock_file(backup_dir, backup_id, false) != LOCK_OK) - /* ... if it's not possible then leave shared lock */ - return; - - release_shared_lock_file(backup_dir); - release_excl_lock_file(backup_dir); -} - -void -release_excl_lock_file(const char *backup_dir) -{ - char lock_file[MAXPGPATH]; - - join_path_components(lock_file, backup_dir, BACKUP_LOCK_FILE); - - /* TODO Sanity check: maybe we should check, that pid in lock file is my_pid */ - - /* unlink pid file */ - fio_unlink(lock_file, FIO_BACKUP_HOST); -} - -void -release_shared_lock_file(const char *backup_dir) -{ - FILE *fp_in = NULL; - FILE *fp_out = NULL; - char buf_in[256]; - pid_t encoded_pid; - char lock_file[MAXPGPATH]; - - char buffer[8192]; /*TODO: should be enough, but maybe malloc+realloc is better ? */ - char lock_file_tmp[MAXPGPATH]; - int buffer_len = 0; - - join_path_components(lock_file, backup_dir, BACKUP_RO_LOCK_FILE); - snprintf(lock_file_tmp, MAXPGPATH, "%s%s", lock_file, "tmp"); - - /* open lock file */ - fp_in = fopen(lock_file, "r"); - if (fp_in == NULL) - { - if (errno == ENOENT) - return; - else - elog(ERROR, "Cannot open lock file \"%s\": %s", lock_file, strerror(errno)); - } - - /* read PIDs of owners */ - while (fgets(buf_in, sizeof(buf_in), fp_in)) - { - encoded_pid = atoi(buf_in); - - if (encoded_pid <= 0) - { - elog(WARNING, "Bogus data in lock file \"%s\": \"%s\"", lock_file, buf_in); - continue; - } - - /* remove my pid */ - if (encoded_pid == my_pid) - continue; - - if (kill(encoded_pid, 0) == 0) - { - /* - * Somebody is still using this backup in shared mode, - * copy this pid into a new file. - */ - buffer_len += snprintf(buffer+buffer_len, 4096, "%u\n", encoded_pid); - } - else if (errno != ESRCH) - elog(ERROR, "Failed to send signal 0 to a process %d: %s", - encoded_pid, strerror(errno)); - } - - if (ferror(fp_in)) - elog(ERROR, "Cannot read from lock file: \"%s\"", lock_file); - fclose(fp_in); - - /* if there is no active pid left, then there is nothing to do */ - if (buffer_len == 0) - { - fio_unlink(lock_file, FIO_BACKUP_HOST); - return; - } - - fp_out = fopen(lock_file_tmp, "w"); - if (fp_out == NULL) - elog(ERROR, "Cannot open temp lock file \"%s\": %s", lock_file_tmp, strerror(errno)); - - /* write out the collected PIDs to temp lock file */ - fwrite(buffer, 1, buffer_len, fp_out); - - if (ferror(fp_out)) - elog(ERROR, "Cannot write to lock file: \"%s\"", lock_file_tmp); - - if (fclose(fp_out) != 0) - elog(ERROR, "Cannot close temp lock file \"%s\": %s", lock_file_tmp, strerror(errno)); - - if (rename(lock_file_tmp, lock_file) < 0) - elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", - lock_file_tmp, lock_file, strerror(errno)); - - return; -} - -/* - * Get backup_mode in string representation. - */ -const char * -pgBackupGetBackupMode(pgBackup *backup, bool show_color) -{ - if (show_color) - { - /* color the Backup mode */ - char *mode = pgut_malloc(24); /* leaking memory here */ - - if (backup->backup_mode == BACKUP_MODE_FULL) - snprintf(mode, 24, "%s%s%s", TC_GREEN_BOLD, backupModes[backup->backup_mode], TC_RESET); - else - snprintf(mode, 24, "%s%s%s", TC_BLUE_BOLD, backupModes[backup->backup_mode], TC_RESET); - - return mode; - } - else - return backupModes[backup->backup_mode]; -} - -static bool -IsDir(const char *dirpath, const char *entry, fio_location location) -{ - char path[MAXPGPATH]; - struct stat st; - - join_path_components(path, dirpath, entry); - - return fio_stat(path, &st, false, location) == 0 && S_ISDIR(st.st_mode); -} - -/* - * Create list of instances in given backup catalog. - * - * Returns parray of InstanceState structures. - */ -parray * -catalog_get_instance_list(CatalogState *catalogState) -{ - DIR *dir; - struct dirent *dent; - parray *instances; - - instances = parray_new(); - - /* open directory and list contents */ - dir = opendir(catalogState->backup_subdir_path); - if (dir == NULL) - elog(ERROR, "Cannot open directory \"%s\": %s", - catalogState->backup_subdir_path, strerror(errno)); - - while (errno = 0, (dent = readdir(dir)) != NULL) - { - char child[MAXPGPATH]; - struct stat st; - InstanceState *instanceState = NULL; - - /* skip entries point current dir or parent dir */ - if (strcmp(dent->d_name, ".") == 0 || - strcmp(dent->d_name, "..") == 0) - continue; - - join_path_components(child, catalogState->backup_subdir_path, dent->d_name); - - if (lstat(child, &st) == -1) - elog(ERROR, "Cannot stat file \"%s\": %s", - child, strerror(errno)); - - if (!S_ISDIR(st.st_mode)) - continue; - - instanceState = pgut_new(InstanceState); - - strlcpy(instanceState->instance_name, dent->d_name, MAXPGPATH); - join_path_components(instanceState->instance_backup_subdir_path, - catalogState->backup_subdir_path, instanceState->instance_name); - join_path_components(instanceState->instance_wal_subdir_path, - catalogState->wal_subdir_path, instanceState->instance_name); - join_path_components(instanceState->instance_config_path, - instanceState->instance_backup_subdir_path, BACKUP_CATALOG_CONF_FILE); - - instanceState->config = readInstanceConfigFile(instanceState); - parray_append(instances, instanceState); - } - - /* TODO 3.0: switch to ERROR */ - if (parray_num(instances) == 0) - elog(WARNING, "This backup catalog contains no backup instances. Backup instance can be added via 'add-instance' command."); - - if (errno) - elog(ERROR, "Cannot read directory \"%s\": %s", - catalogState->backup_subdir_path, strerror(errno)); - - if (closedir(dir)) - elog(ERROR, "Cannot close directory \"%s\": %s", - catalogState->backup_subdir_path, strerror(errno)); - - return instances; -} - -/* - * Create list of backups. - * If 'requested_backup_id' is INVALID_BACKUP_ID, return list of all backups. - * The list is sorted in order of descending start time. - * If valid backup id is passed only matching backup will be added to the list. - */ -parray * -catalog_get_backup_list(InstanceState *instanceState, time_t requested_backup_id) -{ - DIR *data_dir = NULL; - struct dirent *data_ent = NULL; - parray *backups = NULL; - int i; - - /* open backup instance backups directory */ - data_dir = fio_opendir(instanceState->instance_backup_subdir_path, FIO_BACKUP_HOST); - if (data_dir == NULL) - { - elog(WARNING, "cannot open directory \"%s\": %s", instanceState->instance_backup_subdir_path, - strerror(errno)); - goto err_proc; - } - - /* scan the directory and list backups */ - backups = parray_new(); - for (; (data_ent = fio_readdir(data_dir)) != NULL; errno = 0) - { - char backup_conf_path[MAXPGPATH]; - char data_path[MAXPGPATH]; - pgBackup *backup = NULL; - - /* skip not-directory entries and hidden entries */ - if (!IsDir(instanceState->instance_backup_subdir_path, data_ent->d_name, FIO_BACKUP_HOST) - || data_ent->d_name[0] == '.') - continue; - - /* open subdirectory of specific backup */ - join_path_components(data_path, instanceState->instance_backup_subdir_path, data_ent->d_name); - - /* read backup information from BACKUP_CONTROL_FILE */ - join_path_components(backup_conf_path, data_path, BACKUP_CONTROL_FILE); - backup = readBackupControlFile(backup_conf_path); - - if (!backup) - { - backup = pgut_new0(pgBackup); - pgBackupInit(backup); - backup->start_time = base36dec(data_ent->d_name); - /* XXX BACKUP_ID change it when backup_id wouldn't match start_time */ - Assert(backup->backup_id == 0 || backup->backup_id == backup->start_time); - backup->backup_id = backup->start_time; - } - else if (strcmp(backup_id_of(backup), data_ent->d_name) != 0) - { - /* TODO there is no such guarantees */ - elog(WARNING, "backup ID in control file \"%s\" doesn't match name of the backup folder \"%s\"", - backup_id_of(backup), backup_conf_path); - } - - backup->root_dir = pgut_strdup(data_path); - - backup->database_dir = pgut_malloc(MAXPGPATH); - join_path_components(backup->database_dir, backup->root_dir, DATABASE_DIR); - - /* Initialize page header map */ - init_header_map(backup); - - /* TODO: save encoded backup id */ - if (requested_backup_id != INVALID_BACKUP_ID - && requested_backup_id != backup->start_time) - { - pgBackupFree(backup); - continue; - } - parray_append(backups, backup); - } - - if (errno) - { - elog(WARNING, "Cannot read backup root directory \"%s\": %s", - instanceState->instance_backup_subdir_path, strerror(errno)); - goto err_proc; - } - - fio_closedir(data_dir); - data_dir = NULL; - - parray_qsort(backups, pgBackupCompareIdDesc); - - /* Link incremental backups with their ancestors.*/ - for (i = 0; i < parray_num(backups); i++) - { - pgBackup *curr = parray_get(backups, i); - pgBackup **ancestor; - pgBackup key = {0}; - - if (curr->backup_mode == BACKUP_MODE_FULL) - continue; - - key.start_time = curr->parent_backup; - ancestor = (pgBackup **) parray_bsearch(backups, &key, - pgBackupCompareIdDesc); - if (ancestor) - curr->parent_backup_link = *ancestor; - } - - return backups; - -err_proc: - if (data_dir) - fio_closedir(data_dir); - if (backups) - parray_walk(backups, pgBackupFree); - parray_free(backups); - - elog(ERROR, "Failed to get backup list"); - - return NULL; -} - -/* - * Get list of files in the backup from the DATABASE_FILE_LIST. - */ -parray * -get_backup_filelist(pgBackup *backup, bool strict) -{ - parray *files = NULL; - char backup_filelist_path[MAXPGPATH]; - FILE *fp; - char buf[BLCKSZ]; - char stdio_buf[STDIO_BUFSIZE]; - pg_crc32 content_crc = 0; - - join_path_components(backup_filelist_path, backup->root_dir, DATABASE_FILE_LIST); - - fp = fio_open_stream(backup_filelist_path, FIO_BACKUP_HOST); - if (fp == NULL) - elog(ERROR, "Cannot open \"%s\": %s", backup_filelist_path, strerror(errno)); - - /* enable stdio buffering for local file */ - if (!fio_is_remote(FIO_BACKUP_HOST)) - setvbuf(fp, stdio_buf, _IOFBF, STDIO_BUFSIZE); - - files = parray_new(); - - INIT_FILE_CRC32(true, content_crc); - - while (fgets(buf, lengthof(buf), fp)) - { - char path[MAXPGPATH]; - char linked[MAXPGPATH]; - char compress_alg_string[MAXPGPATH]; - int64 write_size, - uncompressed_size, - mode, /* bit length of mode_t depends on platforms */ - is_datafile, - is_cfs, - external_dir_num, - crc, - segno, - n_blocks, - n_headers, - dbOid, /* used for partial restore */ - hdr_crc, - hdr_off, - hdr_size; - pgFile *file; - - COMP_FILE_CRC32(true, content_crc, buf, strlen(buf)); - - get_control_value_str(buf, "path", path, sizeof(path),true); - get_control_value_int64(buf, "size", &write_size, true); - get_control_value_int64(buf, "mode", &mode, true); - get_control_value_int64(buf, "is_datafile", &is_datafile, true); - get_control_value_int64(buf, "is_cfs", &is_cfs, false); - get_control_value_int64(buf, "crc", &crc, true); - get_control_value_str(buf, "compress_alg", compress_alg_string, sizeof(compress_alg_string), false); - get_control_value_int64(buf, "external_dir_num", &external_dir_num, false); - get_control_value_int64(buf, "dbOid", &dbOid, false); - - file = pgFileInit(path); - file->write_size = (int64) write_size; - file->mode = (mode_t) mode; - file->is_datafile = is_datafile ? true : false; - file->is_cfs = is_cfs ? true : false; - file->crc = (pg_crc32) crc; - file->compress_alg = parse_compress_alg(compress_alg_string); - file->external_dir_num = external_dir_num; - file->dbOid = dbOid ? dbOid : 0; - - /* - * Optional fields - */ - if (get_control_value_str(buf, "linked", linked, sizeof(linked), false) && linked[0]) - { - file->linked = pgut_strdup(linked); - canonicalize_path(file->linked); - } - - if (get_control_value_int64(buf, "segno", &segno, false)) - file->segno = (int) segno; - - if (get_control_value_int64(buf, "n_blocks", &n_blocks, false)) - file->n_blocks = (int) n_blocks; - - if (get_control_value_int64(buf, "n_headers", &n_headers, false)) - file->n_headers = (int) n_headers; - - if (get_control_value_int64(buf, "hdr_crc", &hdr_crc, false)) - file->hdr_crc = (pg_crc32) hdr_crc; - - if (get_control_value_int64(buf, "hdr_off", &hdr_off, false)) - file->hdr_off = hdr_off; - - if (get_control_value_int64(buf, "hdr_size", &hdr_size, false)) - file->hdr_size = (int) hdr_size; - - if (get_control_value_int64(buf, "full_size", &uncompressed_size, false)) - file->uncompressed_size = uncompressed_size; - else - file->uncompressed_size = write_size; - if (!file->is_datafile || file->is_cfs) - file->size = file->uncompressed_size; - - if (file->external_dir_num == 0 && - (file->dbOid != 0 || - path_is_prefix_of_path("global", file->rel_path)) && - S_ISREG(file->mode)) - { - bool is_datafile = file->is_datafile; - set_forkname(file); - if (is_datafile != file->is_datafile) - { - if (is_datafile) - elog(WARNING, "File '%s' was stored as datafile, but looks like it is not", - file->rel_path); - else - elog(WARNING, "File '%s' was stored as non-datafile, but looks like it is", - file->rel_path); - /* Lets fail in tests */ - Assert(file->is_datafile == file->is_datafile); - file->is_datafile = is_datafile; - } - } - - parray_append(files, file); - } - - FIN_FILE_CRC32(true, content_crc); - - if (ferror(fp)) - elog(ERROR, "Failed to read from file: \"%s\"", backup_filelist_path); - - fio_close_stream(fp); - - if (backup->content_crc != 0 && - backup->content_crc != content_crc) - { - elog(WARNING, "Invalid CRC of backup control file '%s': %u. Expected: %u", - backup_filelist_path, content_crc, backup->content_crc); - parray_free(files); - files = NULL; - - } - - /* redundant sanity? */ - if (!files) - elog(strict ? ERROR : WARNING, "Failed to get file list for backup %s", backup_id_of(backup)); - - return files; -} - -/* - * Lock list of backups. Function goes in backward direction. - */ -void -catalog_lock_backup_list(parray *backup_list, int from_idx, int to_idx, bool strict, bool exclusive) -{ - int start_idx, - end_idx; - int i; - - if (parray_num(backup_list) == 0) - return; - - start_idx = Max(from_idx, to_idx); - end_idx = Min(from_idx, to_idx); - - for (i = start_idx; i >= end_idx; i--) - { - pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - if (!lock_backup(backup, strict, exclusive)) - elog(ERROR, "Cannot lock backup %s directory", - backup_id_of(backup)); - } -} - -/* - * Find the latest valid child of latest valid FULL backup on given timeline - */ -pgBackup * -catalog_get_last_data_backup(parray *backup_list, TimeLineID tli, time_t current_start_time) -{ - int i; - pgBackup *full_backup = NULL; - pgBackup *tmp_backup = NULL; - - /* backup_list is sorted in order of descending ID */ - for (i = 0; i < parray_num(backup_list); i++) - { - pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - - if ((backup->backup_mode == BACKUP_MODE_FULL && - (backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE)) && backup->tli == tli) - { - full_backup = backup; - break; - } - } - - /* Failed to find valid FULL backup to fulfill ancestor role */ - if (!full_backup) - return NULL; - - elog(LOG, "Latest valid FULL backup: %s", - backup_id_of(full_backup)); - - /* FULL backup is found, lets find his latest child */ - for (i = 0; i < parray_num(backup_list); i++) - { - pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - - /* only valid descendants are acceptable for evaluation */ - if ((backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE)) - { - switch (scan_parent_chain(backup, &tmp_backup)) - { - /* broken chain */ - case ChainIsBroken: - elog(WARNING, "Backup %s has missing parent: %s. Cannot be a parent", - backup_id_of(backup), base36enc(tmp_backup->parent_backup)); - continue; - - /* chain is intact, but at least one parent is invalid */ - case ChainIsInvalid: - elog(WARNING, "Backup %s has invalid parent: %s. Cannot be a parent", - backup_id_of(backup), backup_id_of(tmp_backup)); - continue; - - /* chain is ok */ - case ChainIsOk: - /* Yes, we could call is_parent() earlier - after choosing the ancestor, - * but this way we have an opportunity to detect and report all possible - * anomalies. - */ - if (is_parent(full_backup->start_time, backup, true)) - return backup; - } - } - /* skip yourself */ - else if (backup->start_time == current_start_time) - continue; - else - { - elog(WARNING, "Backup %s has status: %s. Cannot be a parent.", - backup_id_of(backup), status2str(backup->status)); - } - } - - return NULL; -} - -/* - * For multi-timeline chain, look up suitable parent for incremental backup. - * Multi-timeline chain has full backup and one or more descendants located - * on different timelines. - */ -pgBackup * -get_multi_timeline_parent(parray *backup_list, parray *tli_list, - TimeLineID current_tli, time_t current_start_time, - InstanceConfig *instance) -{ - int i; - timelineInfo *my_tlinfo = NULL; - timelineInfo *tmp_tlinfo = NULL; - pgBackup *ancestor_backup = NULL; - - /* there are no timelines in the archive */ - if (parray_num(tli_list) == 0) - return NULL; - - /* look for current timelineInfo */ - for (i = 0; i < parray_num(tli_list); i++) - { - timelineInfo *tlinfo = (timelineInfo *) parray_get(tli_list, i); - - if (tlinfo->tli == current_tli) - { - my_tlinfo = tlinfo; - break; - } - } - - if (my_tlinfo == NULL) - return NULL; - - /* Locate tlinfo of suitable full backup. - * Consider this example: - * t3 s2-------X <-! We are here - * / - * t2 s1----D---*----E---> - * / - * t1--A--B--*---C-------> - * - * A, E - full backups - * B, C, D - incremental backups - * - * We must find A. - */ - tmp_tlinfo = my_tlinfo; - while (tmp_tlinfo->parent_link) - { - /* if timeline has backups, iterate over them */ - if (tmp_tlinfo->parent_link->backups) - { - for (i = 0; i < parray_num(tmp_tlinfo->parent_link->backups); i++) - { - pgBackup *backup = (pgBackup *) parray_get(tmp_tlinfo->parent_link->backups, i); - - if (backup->backup_mode == BACKUP_MODE_FULL && - (backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE) && - backup->stop_lsn <= tmp_tlinfo->switchpoint) - { - ancestor_backup = backup; - break; - } - } - } - - if (ancestor_backup) - break; - - tmp_tlinfo = tmp_tlinfo->parent_link; - } - - /* failed to find valid FULL backup on parent timelines */ - if (!ancestor_backup) - return NULL; - else - elog(LOG, "Latest valid full backup: %s, tli: %i", - backup_id_of(ancestor_backup), ancestor_backup->tli); - - /* At this point we found suitable full backup, - * now we must find his latest child, suitable to be - * parent of current incremental backup. - * Consider this example: - * t3 s2-------X <-! We are here - * / - * t2 s1----D---*----E---> - * / - * t1--A--B--*---C-------> - * - * A, E - full backups - * B, C, D - incremental backups - * - * We found A, now we must find D. - */ - - /* Optimistically, look on current timeline for valid incremental backup, child of ancestor */ - if (my_tlinfo->backups) - { - /* backups are sorted in descending order and we need latest valid */ - for (i = 0; i < parray_num(my_tlinfo->backups); i++) - { - pgBackup *tmp_backup = NULL; - pgBackup *backup = (pgBackup *) parray_get(my_tlinfo->backups, i); - - /* found suitable parent */ - if (scan_parent_chain(backup, &tmp_backup) == ChainIsOk && - is_parent(ancestor_backup->start_time, backup, false)) - return backup; - } - } - - /* Iterate over parent timelines and look for a valid backup, child of ancestor */ - tmp_tlinfo = my_tlinfo; - while (tmp_tlinfo->parent_link) - { - - /* if timeline has backups, iterate over them */ - if (tmp_tlinfo->parent_link->backups) - { - for (i = 0; i < parray_num(tmp_tlinfo->parent_link->backups); i++) - { - pgBackup *tmp_backup = NULL; - pgBackup *backup = (pgBackup *) parray_get(tmp_tlinfo->parent_link->backups, i); - - /* We are not interested in backups - * located outside of our timeline history - */ - if (backup->stop_lsn > tmp_tlinfo->switchpoint) - continue; - - if (scan_parent_chain(backup, &tmp_backup) == ChainIsOk && - is_parent(ancestor_backup->start_time, backup, true)) - return backup; - } - } - - tmp_tlinfo = tmp_tlinfo->parent_link; - } - - return NULL; -} - -/* - * Create backup directory in $BACKUP_PATH - * (with proposed backup->backup_id) - * and initialize this directory. - * If creation of directory fails, then - * backup_id will be cleared (set to INVALID_BACKUP_ID). - * It is possible to get diffrent values in - * pgBackup.start_time and pgBackup.backup_id. - * It may be ok or maybe not, so it's up to the caller - * to fix it or let it be. - */ - -void -pgBackupInitDir(pgBackup *backup, const char *backup_instance_path) -{ - int i; - char temp[MAXPGPATH]; - parray *subdirs; - - /* Try to create backup directory at first */ - if (create_backup_dir(backup, backup_instance_path) != 0) - { - /* Clear backup_id as indication of error */ - reset_backup_id(backup); - return; - } - - subdirs = parray_new(); - parray_append(subdirs, pg_strdup(DATABASE_DIR)); - - /* Add external dirs containers */ - if (backup->external_dir_str) - { - parray *external_list; - - external_list = make_external_directory_list(backup->external_dir_str, - false); - for (i = 0; i < parray_num(external_list); i++) - { - /* Numeration of externaldirs starts with 1 */ - makeExternalDirPathByNum(temp, EXTERNAL_DIR, i+1); - parray_append(subdirs, pg_strdup(temp)); - } - free_dir_list(external_list); - } - - backup->database_dir = pgut_malloc(MAXPGPATH); - join_path_components(backup->database_dir, backup->root_dir, DATABASE_DIR); - - /* block header map */ - init_header_map(backup); - - /* create directories for actual backup files */ - for (i = 0; i < parray_num(subdirs); i++) - { - join_path_components(temp, backup->root_dir, parray_get(subdirs, i)); - fio_mkdir(temp, DIR_PERMISSION, FIO_BACKUP_HOST); - } - - free_dir_list(subdirs); -} - -/* - * Create root directory for backup, - * update pgBackup.root_dir if directory creation was a success - * Return values (same as dir_create_dir()): - * 0 - ok - * -1 - error (warning message already emitted) - */ -int -create_backup_dir(pgBackup *backup, const char *backup_instance_path) -{ - int rc; - char path[MAXPGPATH]; - - join_path_components(path, backup_instance_path, backup_id_of(backup)); - - /* TODO: add wrapper for remote mode */ - rc = dir_create_dir(path, DIR_PERMISSION, true); - - if (rc == 0) - backup->root_dir = pgut_strdup(path); - else - elog(WARNING, "Cannot create directory \"%s\": %s", path, strerror(errno)); - return rc; -} - -/* - * Create list of timelines. - * TODO: '.partial' and '.part' segno information should be added to tlinfo. - */ -parray * -catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance) -{ - int i,j,k; - parray *xlog_files_list = parray_new(); - parray *timelineinfos; - parray *backups; - timelineInfo *tlinfo; - - /* for fancy reporting */ - char begin_segno_str[MAXFNAMELEN]; - char end_segno_str[MAXFNAMELEN]; - - /* read all xlog files that belong to this archive */ - dir_list_file(xlog_files_list, instanceState->instance_wal_subdir_path, - false, true, false, false, true, 0, FIO_BACKUP_HOST); - parray_qsort(xlog_files_list, pgFileCompareName); - - timelineinfos = parray_new(); - tlinfo = NULL; - - /* walk through files and collect info about timelines */ - for (i = 0; i < parray_num(xlog_files_list); i++) - { - pgFile *file = (pgFile *) parray_get(xlog_files_list, i); - TimeLineID tli; - parray *timelines; - xlogFile *wal_file = NULL; - - /* - * Regular WAL file. - * IsXLogFileName() cannot be used here - */ - if (strspn(file->name, "0123456789ABCDEF") == XLOG_FNAME_LEN) - { - int result = 0; - uint32 log, seg; - XLogSegNo segno = 0; - char suffix[MAXFNAMELEN]; - - result = sscanf(file->name, "%08X%08X%08X.%s", - &tli, &log, &seg, (char *) &suffix); - - /* sanity */ - if (result < 3) - { - elog(WARNING, "unexpected WAL file name \"%s\"", file->name); - continue; - } - - /* get segno from log */ - GetXLogSegNoFromScrath(segno, log, seg, instance->xlog_seg_size); - - /* regular WAL file with suffix */ - if (result == 4) - { - /* backup history file. Currently we don't use them */ - if (IsBackupHistoryFileName(file->name)) - { - elog(VERBOSE, "backup history file \"%s\"", file->name); - - if (!tlinfo || tlinfo->tli != tli) - { - tlinfo = timelineInfoNew(tli); - parray_append(timelineinfos, tlinfo); - } - - /* append file to xlog file list */ - wal_file = palloc(sizeof(xlogFile)); - wal_file->file = *file; - wal_file->segno = segno; - wal_file->type = BACKUP_HISTORY_FILE; - wal_file->keep = false; - parray_append(tlinfo->xlog_filelist, wal_file); - continue; - } - /* partial WAL segment */ - else if (IsPartialXLogFileName(file->name) || - IsPartialCompressXLogFileName(file->name)) - { - elog(VERBOSE, "partial WAL file \"%s\"", file->name); - - if (!tlinfo || tlinfo->tli != tli) - { - tlinfo = timelineInfoNew(tli); - parray_append(timelineinfos, tlinfo); - } - - /* append file to xlog file list */ - wal_file = palloc(sizeof(xlogFile)); - wal_file->file = *file; - wal_file->segno = segno; - wal_file->type = PARTIAL_SEGMENT; - wal_file->keep = false; - parray_append(tlinfo->xlog_filelist, wal_file); - continue; - } - /* temp WAL segment */ - else if (IsTempXLogFileName(file->name) || - IsTempCompressXLogFileName(file->name) || - IsTempPartialXLogFileName(file->name)) - { - elog(VERBOSE, "temp WAL file \"%s\"", file->name); - - if (!tlinfo || tlinfo->tli != tli) - { - tlinfo = timelineInfoNew(tli); - parray_append(timelineinfos, tlinfo); - } - - /* append file to xlog file list */ - wal_file = palloc(sizeof(xlogFile)); - wal_file->file = *file; - wal_file->segno = segno; - wal_file->type = TEMP_SEGMENT; - wal_file->keep = false; - parray_append(tlinfo->xlog_filelist, wal_file); - continue; - } - /* we only expect compressed wal files with .gz suffix */ - else if (strcmp(suffix, "gz") != 0) - { - elog(WARNING, "unexpected WAL file name \"%s\"", file->name); - continue; - } - } - - /* new file belongs to new timeline */ - if (!tlinfo || tlinfo->tli != tli) - { - tlinfo = timelineInfoNew(tli); - parray_append(timelineinfos, tlinfo); - } - /* - * As it is impossible to detect if segments before segno are lost, - * or just do not exist, do not report them as lost. - */ - else if (tlinfo->n_xlog_files != 0) - { - /* check, if segments are consequent */ - XLogSegNo expected_segno = tlinfo->end_segno + 1; - - /* - * Some segments are missing. remember them in lost_segments to report. - * Normally we expect that segment numbers form an increasing sequence, - * though it's legal to find two files with equal segno in case there - * are both compressed and non-compessed versions. For example - * 000000010000000000000002 and 000000010000000000000002.gz - * - */ - if (segno != expected_segno && segno != tlinfo->end_segno) - { - xlogInterval *interval = palloc(sizeof(xlogInterval));; - interval->begin_segno = expected_segno; - interval->end_segno = segno - 1; - - if (tlinfo->lost_segments == NULL) - tlinfo->lost_segments = parray_new(); - - parray_append(tlinfo->lost_segments, interval); - } - } - - if (tlinfo->begin_segno == 0) - tlinfo->begin_segno = segno; - - /* this file is the last for this timeline so far */ - tlinfo->end_segno = segno; - /* update counters */ - tlinfo->n_xlog_files++; - tlinfo->size += file->size; - - /* append file to xlog file list */ - wal_file = palloc(sizeof(xlogFile)); - wal_file->file = *file; - wal_file->segno = segno; - wal_file->type = SEGMENT; - wal_file->keep = false; - parray_append(tlinfo->xlog_filelist, wal_file); - } - /* timeline history file */ - else if (IsTLHistoryFileName(file->name)) - { - TimeLineHistoryEntry *tln; - - sscanf(file->name, "%08X.history", &tli); - timelines = read_timeline_history(instanceState->instance_wal_subdir_path, tli, true); - - /* History file is empty or corrupted, disregard it */ - if (!timelines) - continue; - - if (!tlinfo || tlinfo->tli != tli) - { - tlinfo = timelineInfoNew(tli); - parray_append(timelineinfos, tlinfo); - /* - * 1 is the latest timeline in the timelines list. - * 0 - is our timeline, which is of no interest here - */ - tln = (TimeLineHistoryEntry *) parray_get(timelines, 1); - tlinfo->switchpoint = tln->end; - tlinfo->parent_tli = tln->tli; - - /* find parent timeline to link it with this one */ - for (j = 0; j < parray_num(timelineinfos); j++) - { - timelineInfo *cur = (timelineInfo *) parray_get(timelineinfos, j); - if (cur->tli == tlinfo->parent_tli) - { - tlinfo->parent_link = cur; - break; - } - } - } - - parray_walk(timelines, pfree); - parray_free(timelines); - } - else - elog(WARNING, "unexpected WAL file name \"%s\"", file->name); - } - - /* save information about backups belonging to each timeline */ - backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); - - for (i = 0; i < parray_num(timelineinfos); i++) - { - timelineInfo *tlinfo = parray_get(timelineinfos, i); - for (j = 0; j < parray_num(backups); j++) - { - pgBackup *backup = parray_get(backups, j); - if (tlinfo->tli == backup->tli) - { - if (tlinfo->backups == NULL) - tlinfo->backups = parray_new(); - - parray_append(tlinfo->backups, backup); - } - } - } - - /* determine oldest backup and closest backup for every timeline */ - for (i = 0; i < parray_num(timelineinfos); i++) - { - timelineInfo *tlinfo = parray_get(timelineinfos, i); - - tlinfo->oldest_backup = get_oldest_backup(tlinfo); - tlinfo->closest_backup = get_closest_backup(tlinfo); - } - - /* determine which WAL segments must be kept because of wal retention */ - if (instance->wal_depth <= 0) - return timelineinfos; - - /* - * WAL retention for now is fairly simple. - * User can set only one parameter - 'wal-depth'. - * It determines how many latest valid(!) backups on timeline - * must have an ability to perform PITR: - * Consider the example: - * - * ---B1-------B2-------B3-------B4--------> WAL timeline1 - * - * If 'wal-depth' is set to 2, then WAL purge should produce the following result: - * - * B1 B2 B3-------B4--------> WAL timeline1 - * - * Only valid backup can satisfy 'wal-depth' condition, so if B3 is not OK or DONE, - * then WAL purge should produce the following result: - * B1 B2-------B3-------B4--------> WAL timeline1 - * - * Complicated cases, such as branched timelines are taken into account. - * wal-depth is applied to each timeline independently: - * - * |---------> WAL timeline2 - * ---B1---|---B2-------B3-------B4--------> WAL timeline1 - * - * after WAL purge with wal-depth=2: - * - * |---------> WAL timeline2 - * B1---| B2 B3-------B4--------> WAL timeline1 - * - * In this example WAL retention prevents purge of WAL required by tli2 - * to stay reachable from backup B on tli1. - * - * To protect WAL from purge we try to set 'anchor_lsn' and 'anchor_tli' in every timeline. - * They are usually comes from 'start-lsn' and 'tli' attributes of backup - * calculated by 'wal-depth' parameter. - * With 'wal-depth=2' anchor_backup in tli1 is B3. - - * If timeline has not enough valid backups to satisfy 'wal-depth' condition, - * then 'anchor_lsn' and 'anchor_tli' taken from from 'start-lsn' and 'tli - * attribute of closest_backup. - * The interval of WAL starting from closest_backup to switchpoint is - * saved into 'keep_segments' attribute. - * If there is several intermediate timelines between timeline and its closest_backup - * then on every intermediate timeline WAL interval between switchpoint - * and starting segment is placed in 'keep_segments' attributes: - * - * |---------> WAL timeline3 - * |------| B5-----B6--> WAL timeline2 - * B1---| B2 B3-------B4------------> WAL timeline1 - * - * On timeline where closest_backup is located the WAL interval between - * closest_backup and switchpoint is placed into 'keep_segments'. - * If timeline has no 'closest_backup', then 'wal-depth' rules cannot be applied - * to this timeline and its WAL must be purged by following the basic rules of WAL purging. - * - * Third part is handling of ARCHIVE backups. - * If B1 and B2 have ARCHIVE wal-mode, then we must preserve WAL intervals - * between start_lsn and stop_lsn for each of them in 'keep_segments'. - */ - - /* determine anchor_lsn and keep_segments for every timeline */ - for (i = 0; i < parray_num(timelineinfos); i++) - { - int count = 0; - timelineInfo *tlinfo = parray_get(timelineinfos, i); - - /* - * Iterate backward on backups belonging to this timeline to find - * anchor_backup. NOTE Here we rely on the fact that backups list - * is ordered by start_lsn DESC. - */ - if (tlinfo->backups) - { - for (j = 0; j < parray_num(tlinfo->backups); j++) - { - pgBackup *backup = parray_get(tlinfo->backups, j); - - /* sanity */ - if (XLogRecPtrIsInvalid(backup->start_lsn) || - backup->tli <= 0) - continue; - - /* skip invalid backups */ - if (backup->status != BACKUP_STATUS_OK && - backup->status != BACKUP_STATUS_DONE) - continue; - - /* - * Pinned backups should be ignored for the - * purpose of retention fulfillment, so skip them. - */ - if (backup->expire_time > 0 && - backup->expire_time > current_time) - { - elog(LOG, "Pinned backup %s is ignored for the " - "purpose of WAL retention", - backup_id_of(backup)); - continue; - } - - count++; - - if (count == instance->wal_depth) - { - elog(LOG, "On timeline %i WAL is protected from purge at %X/%X", - tlinfo->tli, - (uint32) (backup->start_lsn >> 32), - (uint32) (backup->start_lsn)); - - tlinfo->anchor_lsn = backup->start_lsn; - tlinfo->anchor_tli = backup->tli; - break; - } - } - } - - /* - * Failed to find anchor backup for this timeline. - * We cannot just thrown it to the wolves, because by - * doing that we will violate our own guarantees. - * So check the existence of closest_backup for - * this timeline. If there is one, then - * set the 'anchor_lsn' and 'anchor_tli' to closest_backup - * 'start-lsn' and 'tli' respectively. - * |-------------B5----------> WAL timeline3 - * |-----|-------------------------> WAL timeline2 - * B1 B2---| B3 B4-------B6-----> WAL timeline1 - * - * wal-depth=2 - * - * If number of valid backups on timelines is less than 'wal-depth' - * then timeline must(!) stay reachable via parent timelines if any. - * If closest_backup is not available, then general WAL purge rules - * are applied. - */ - if (XLogRecPtrIsInvalid(tlinfo->anchor_lsn)) - { - /* - * Failed to find anchor_lsn in our own timeline. - * Consider the case: - * -------------------------------------> tli5 - * ----------------------------B4-------> tli4 - * S3`--------------> tli3 - * S1`------------S3---B3-------B6-> tli2 - * B1---S1-------------B2--------B5-----> tli1 - * - * B* - backups - * S* - switchpoints - * wal-depth=2 - * - * Expected result: - * TLI5 will be purged entirely - * B4-------> tli4 - * S2`--------------> tli3 - * S1`------------S2 B3-------B6-> tli2 - * B1---S1 B2--------B5-----> tli1 - */ - pgBackup *closest_backup = NULL; - xlogInterval *interval = NULL; - TimeLineID tli = 0; - /* check if tli has closest_backup */ - if (!tlinfo->closest_backup) - /* timeline has no closest_backup, wal retention cannot be - * applied to this timeline. - * Timeline will be purged up to oldest_backup if any or - * purge entirely if there is none. - * In example above: tli5 and tli4. - */ - continue; - - /* sanity for closest_backup */ - if (XLogRecPtrIsInvalid(tlinfo->closest_backup->start_lsn) || - tlinfo->closest_backup->tli <= 0) - continue; - - /* - * Set anchor_lsn and anchor_tli to protect whole timeline from purge - * In the example above: tli3. - */ - tlinfo->anchor_lsn = tlinfo->closest_backup->start_lsn; - tlinfo->anchor_tli = tlinfo->closest_backup->tli; - - /* closest backup may be located not in parent timeline */ - closest_backup = tlinfo->closest_backup; - - tli = tlinfo->tli; - - /* - * Iterate over parent timeline chain and - * look for timeline where closest_backup belong - */ - while (tlinfo->parent_link) - { - /* In case of intermediate timeline save to keep_segments - * begin_segno and switchpoint segment. - * In case of final timelines save to keep_segments - * closest_backup start_lsn segment and switchpoint segment. - */ - XLogRecPtr switchpoint = tlinfo->switchpoint; - - tlinfo = tlinfo->parent_link; - - if (tlinfo->keep_segments == NULL) - tlinfo->keep_segments = parray_new(); - - /* in any case, switchpoint segment must be added to interval */ - interval = palloc(sizeof(xlogInterval)); - GetXLogSegNo(switchpoint, interval->end_segno, instance->xlog_seg_size); - - /* Save [S1`, S2] to keep_segments */ - if (tlinfo->tli != closest_backup->tli) - interval->begin_segno = tlinfo->begin_segno; - /* Save [B1, S1] to keep_segments */ - else - GetXLogSegNo(closest_backup->start_lsn, interval->begin_segno, instance->xlog_seg_size); - - /* - * TODO: check, maybe this interval is already here or - * covered by other larger interval. - */ - - GetXLogFileName(begin_segno_str, tlinfo->tli, interval->begin_segno, instance->xlog_seg_size); - GetXLogFileName(end_segno_str, tlinfo->tli, interval->end_segno, instance->xlog_seg_size); - - elog(LOG, "Timeline %i to stay reachable from timeline %i " - "protect from purge WAL interval between " - "%s and %s on timeline %i", - tli, closest_backup->tli, begin_segno_str, - end_segno_str, tlinfo->tli); - - parray_append(tlinfo->keep_segments, interval); - continue; - } - continue; - } - - /* Iterate over backups left */ - for (j = count; j < parray_num(tlinfo->backups); j++) - { - XLogSegNo segno = 0; - xlogInterval *interval = NULL; - pgBackup *backup = parray_get(tlinfo->backups, j); - - /* - * We must calculate keep_segments intervals for ARCHIVE backups - * with start_lsn less than anchor_lsn. - */ - - /* STREAM backups cannot contribute to keep_segments */ - if (backup->stream) - continue; - - /* sanity */ - if (XLogRecPtrIsInvalid(backup->start_lsn) || - backup->tli <= 0) - continue; - - /* no point in clogging keep_segments by backups protected by anchor_lsn */ - if (backup->start_lsn >= tlinfo->anchor_lsn) - continue; - - /* append interval to keep_segments */ - interval = palloc(sizeof(xlogInterval)); - GetXLogSegNo(backup->start_lsn, segno, instance->xlog_seg_size); - interval->begin_segno = segno; - GetXLogSegNo(backup->stop_lsn, segno, instance->xlog_seg_size); - - /* - * On replica it is possible to get STOP_LSN pointing to contrecord, - * so set end_segno to the next segment after STOP_LSN just to be safe. - */ - if (backup->from_replica) - interval->end_segno = segno + 1; - else - interval->end_segno = segno; - - GetXLogFileName(begin_segno_str, tlinfo->tli, interval->begin_segno, instance->xlog_seg_size); - GetXLogFileName(end_segno_str, tlinfo->tli, interval->end_segno, instance->xlog_seg_size); - - elog(LOG, "Archive backup %s to stay consistent " - "protect from purge WAL interval " - "between %s and %s on timeline %i", - backup_id_of(backup), - begin_segno_str, end_segno_str, backup->tli); - - if (tlinfo->keep_segments == NULL) - tlinfo->keep_segments = parray_new(); - - parray_append(tlinfo->keep_segments, interval); - } - } - - /* - * Protect WAL segments from deletion by setting 'keep' flag. - * We must keep all WAL segments after anchor_lsn (including), and also segments - * required by ARCHIVE backups for consistency - WAL between [start_lsn, stop_lsn]. - */ - for (i = 0; i < parray_num(timelineinfos); i++) - { - XLogSegNo anchor_segno = 0; - timelineInfo *tlinfo = parray_get(timelineinfos, i); - - /* - * At this point invalid anchor_lsn can be only in one case: - * timeline is going to be purged by regular WAL purge rules. - */ - if (XLogRecPtrIsInvalid(tlinfo->anchor_lsn)) - continue; - - /* - * anchor_lsn is located in another timeline, it means that the timeline - * will be protected from purge entirely. - */ - if (tlinfo->anchor_tli > 0 && tlinfo->anchor_tli != tlinfo->tli) - continue; - - GetXLogSegNo(tlinfo->anchor_lsn, anchor_segno, instance->xlog_seg_size); - - for (j = 0; j < parray_num(tlinfo->xlog_filelist); j++) - { - xlogFile *wal_file = (xlogFile *) parray_get(tlinfo->xlog_filelist, j); - - if (wal_file->segno >= anchor_segno) - { - wal_file->keep = true; - continue; - } - - /* no keep segments */ - if (!tlinfo->keep_segments) - continue; - - /* Protect segments belonging to one of the keep invervals */ - for (k = 0; k < parray_num(tlinfo->keep_segments); k++) - { - xlogInterval *keep_segments = (xlogInterval *) parray_get(tlinfo->keep_segments, k); - - if ((wal_file->segno >= keep_segments->begin_segno) && - wal_file->segno <= keep_segments->end_segno) - { - wal_file->keep = true; - break; - } - } - } - } - - return timelineinfos; -} - -/* - * Iterate over parent timelines and look for valid backup - * closest to given timeline switchpoint. - * - * If such backup doesn't exist, it means that - * timeline is unreachable. Return NULL. - */ -pgBackup* -get_closest_backup(timelineInfo *tlinfo) -{ - pgBackup *closest_backup = NULL; - int i; - - /* - * Iterate over backups belonging to parent timelines - * and look for candidates. - */ - while (tlinfo->parent_link && !closest_backup) - { - parray *backup_list = tlinfo->parent_link->backups; - if (backup_list != NULL) - { - for (i = 0; i < parray_num(backup_list); i++) - { - pgBackup *backup = parray_get(backup_list, i); - - /* - * Only valid backups made before switchpoint - * should be considered. - */ - if (!XLogRecPtrIsInvalid(backup->stop_lsn) && - XRecOffIsValid(backup->stop_lsn) && - backup->stop_lsn <= tlinfo->switchpoint && - (backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE)) - { - /* Check if backup is closer to switchpoint than current candidate */ - if (!closest_backup || backup->stop_lsn > closest_backup->stop_lsn) - closest_backup = backup; - } - } - } - - /* Continue with parent */ - tlinfo = tlinfo->parent_link; - } - - return closest_backup; -} - -/* - * Find oldest backup in given timeline - * to determine what WAL segments of this timeline - * are reachable from backups belonging to it. - * - * If such backup doesn't exist, it means that - * there is no backups on this timeline. Return NULL. - */ -pgBackup* -get_oldest_backup(timelineInfo *tlinfo) -{ - pgBackup *oldest_backup = NULL; - int i; - parray *backup_list = tlinfo->backups; - - if (backup_list != NULL) - { - for (i = 0; i < parray_num(backup_list); i++) - { - pgBackup *backup = parray_get(backup_list, i); - - /* Backups with invalid START LSN can be safely skipped */ - if (XLogRecPtrIsInvalid(backup->start_lsn) || - !XRecOffIsValid(backup->start_lsn)) - continue; - - /* - * Check if backup is older than current candidate. - * Here we use start_lsn for comparison, because backup that - * started earlier needs more WAL. - */ - if (!oldest_backup || backup->start_lsn < oldest_backup->start_lsn) - oldest_backup = backup; - } - } - - return oldest_backup; -} - -/* - * Overwrite backup metadata. - */ -void -do_set_backup(InstanceState *instanceState, time_t backup_id, - pgSetBackupParams *set_backup_params) -{ - pgBackup *target_backup = NULL; - parray *backup_list = NULL; - - if (!set_backup_params) - elog(ERROR, "Nothing to set by 'set-backup' command"); - - backup_list = catalog_get_backup_list(instanceState, backup_id); - if (parray_num(backup_list) != 1) - elog(ERROR, "Failed to find backup %s", base36enc(backup_id)); - - target_backup = (pgBackup *) parray_get(backup_list, 0); - - /* Pin or unpin backup if requested */ - if (set_backup_params->ttl >= 0 || set_backup_params->expire_time > 0) - pin_backup(target_backup, set_backup_params); - - if (set_backup_params->note) - add_note(target_backup, set_backup_params->note); - /* Cleanup */ - if (backup_list) - { - parray_walk(backup_list, pgBackupFree); - parray_free(backup_list); - } -} - -/* - * Set 'expire-time' attribute based on set_backup_params, or unpin backup - * if ttl is equal to zero. - */ -void -pin_backup(pgBackup *target_backup, pgSetBackupParams *set_backup_params) -{ - - /* sanity, backup must have positive recovery-time */ - if (target_backup->recovery_time <= 0) - elog(ERROR, "Failed to set 'expire-time' for backup %s: invalid 'recovery-time'", - backup_id_of(target_backup)); - - /* Pin comes from ttl */ - if (set_backup_params->ttl > 0) - target_backup->expire_time = target_backup->recovery_time + set_backup_params->ttl; - /* Unpin backup */ - else if (set_backup_params->ttl == 0) - { - /* If backup was not pinned in the first place, - * then there is nothing to unpin. - */ - if (target_backup->expire_time == 0) - { - elog(WARNING, "Backup %s is not pinned, nothing to unpin", - backup_id_of(target_backup)); - return; - } - target_backup->expire_time = 0; - } - /* Pin comes from expire-time */ - else if (set_backup_params->expire_time > 0) - target_backup->expire_time = set_backup_params->expire_time; - else - /* nothing to do */ - return; - - /* Update backup.control */ - write_backup(target_backup, true); - - if (set_backup_params->ttl > 0 || set_backup_params->expire_time > 0) - { - char expire_timestamp[100]; - - time2iso(expire_timestamp, lengthof(expire_timestamp), target_backup->expire_time, false); - elog(INFO, "Backup %s is pinned until '%s'", backup_id_of(target_backup), - expire_timestamp); - } - else - elog(INFO, "Backup %s is unpinned", backup_id_of(target_backup)); - - return; -} - -/* - * Add note to backup metadata or unset already existing note. - * It is a job of the caller to make sure that note is not NULL. - */ -void -add_note(pgBackup *target_backup, char *note) -{ - - char *note_string; - char *p; - - /* unset note */ - if (pg_strcasecmp(note, "none") == 0) - { - target_backup->note = NULL; - elog(INFO, "Removing note from backup %s", - backup_id_of(target_backup)); - } - else - { - /* Currently we do not allow string with newlines as note, - * because it will break parsing of backup.control. - * So if user provides string like this "aaa\nbbbbb", - * we save only "aaa" - * Example: tests.set_backup.SetBackupTest.test_add_note_newlines - */ - p = strchr(note, '\n'); - note_string = pgut_strndup(note, p ? (p-note) : MAX_NOTE_SIZE); - - target_backup->note = note_string; - elog(INFO, "Adding note to backup %s: '%s'", - backup_id_of(target_backup), target_backup->note); - } - - /* Update backup.control */ - write_backup(target_backup, true); -} - -/* - * Write information about backup.in to stream "out". - */ -void -pgBackupWriteControl(FILE *out, pgBackup *backup, bool utc) -{ - char timestamp[100]; - - fio_fprintf(out, "#Configuration\n"); - fio_fprintf(out, "backup-mode = %s\n", pgBackupGetBackupMode(backup, false)); - fio_fprintf(out, "stream = %s\n", backup->stream ? "true" : "false"); - fio_fprintf(out, "compress-alg = %s\n", - deparse_compress_alg(backup->compress_alg)); - fio_fprintf(out, "compress-level = %d\n", backup->compress_level); - fio_fprintf(out, "from-replica = %s\n", backup->from_replica ? "true" : "false"); - - fio_fprintf(out, "\n#Compatibility\n"); - fio_fprintf(out, "block-size = %u\n", backup->block_size); - fio_fprintf(out, "xlog-block-size = %u\n", backup->wal_block_size); - fio_fprintf(out, "checksum-version = %u\n", backup->checksum_version); - if (backup->program_version[0] != '\0') - fio_fprintf(out, "program-version = %s\n", backup->program_version); - if (backup->server_version[0] != '\0') - fio_fprintf(out, "server-version = %s\n", backup->server_version); - - fio_fprintf(out, "\n#Result backup info\n"); - fio_fprintf(out, "timelineid = %d\n", backup->tli); - /* LSN returned by pg_start_backup */ - fio_fprintf(out, "start-lsn = %X/%X\n", - (uint32) (backup->start_lsn >> 32), - (uint32) backup->start_lsn); - /* LSN returned by pg_stop_backup */ - fio_fprintf(out, "stop-lsn = %X/%X\n", - (uint32) (backup->stop_lsn >> 32), - (uint32) backup->stop_lsn); - - time2iso(timestamp, lengthof(timestamp), backup->start_time, utc); - fio_fprintf(out, "start-time = '%s'\n", timestamp); - if (backup->merge_time > 0) - { - time2iso(timestamp, lengthof(timestamp), backup->merge_time, utc); - fio_fprintf(out, "merge-time = '%s'\n", timestamp); - } - if (backup->end_time > 0) - { - time2iso(timestamp, lengthof(timestamp), backup->end_time, utc); - fio_fprintf(out, "end-time = '%s'\n", timestamp); - } - fio_fprintf(out, "recovery-xid = " XID_FMT "\n", backup->recovery_xid); - if (backup->recovery_time > 0) - { - time2iso(timestamp, lengthof(timestamp), backup->recovery_time, utc); - fio_fprintf(out, "recovery-time = '%s'\n", timestamp); - } - if (backup->expire_time > 0) - { - time2iso(timestamp, lengthof(timestamp), backup->expire_time, utc); - fio_fprintf(out, "expire-time = '%s'\n", timestamp); - } - - if (backup->merge_dest_backup != 0) - fio_fprintf(out, "merge-dest-id = '%s'\n", base36enc(backup->merge_dest_backup)); - - /* - * Size of PGDATA directory. The size does not include size of related - * WAL segments in archive 'wal' directory. - */ - if (backup->data_bytes != BYTES_INVALID) - fio_fprintf(out, "data-bytes = " INT64_FORMAT "\n", backup->data_bytes); - - if (backup->wal_bytes != BYTES_INVALID) - fio_fprintf(out, "wal-bytes = " INT64_FORMAT "\n", backup->wal_bytes); - - if (backup->uncompressed_bytes >= 0) - fio_fprintf(out, "uncompressed-bytes = " INT64_FORMAT "\n", backup->uncompressed_bytes); - - if (backup->pgdata_bytes >= 0) - fio_fprintf(out, "pgdata-bytes = " INT64_FORMAT "\n", backup->pgdata_bytes); - - fio_fprintf(out, "status = %s\n", status2str(backup->status)); - - /* 'parent_backup' is set if it is incremental backup */ - if (backup->parent_backup != 0) - fio_fprintf(out, "parent-backup-id = '%s'\n", base36enc(backup->parent_backup)); - - /* print connection info except password */ - if (backup->primary_conninfo) - fio_fprintf(out, "primary_conninfo = '%s'\n", backup->primary_conninfo); - - /* print external directories list */ - if (backup->external_dir_str) - fio_fprintf(out, "external-dirs = '%s'\n", backup->external_dir_str); - - if (backup->note) - fio_fprintf(out, "note = '%s'\n", backup->note); - - if (backup->content_crc != 0) - fio_fprintf(out, "content-crc = %u\n", backup->content_crc); - -} - -/* - * Save the backup content into BACKUP_CONTROL_FILE. - * Flag strict allows to ignore "out of space" error - * when attempting to lock backup. Only delete is allowed - * to use this functionality. - */ -void -write_backup(pgBackup *backup, bool strict) -{ - FILE *fp = NULL; - char path[MAXPGPATH]; - char path_temp[MAXPGPATH]; - char buf[8192]; - - join_path_components(path, backup->root_dir, BACKUP_CONTROL_FILE); - snprintf(path_temp, sizeof(path_temp), "%s.tmp", path); - - fp = fopen(path_temp, PG_BINARY_W); - if (fp == NULL) - elog(ERROR, "Cannot open control file \"%s\": %s", - path_temp, strerror(errno)); - - if (chmod(path_temp, FILE_PERMISSION) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", path_temp, - strerror(errno)); - - setvbuf(fp, buf, _IOFBF, sizeof(buf)); - - pgBackupWriteControl(fp, backup, true); - - /* Ignore 'out of space' error in lax mode */ - if (fflush(fp) != 0) - { - int elevel = ERROR; - int save_errno = errno; - - if (!strict && (errno == ENOSPC)) - elevel = WARNING; - - elog(elevel, "Cannot flush control file \"%s\": %s", - path_temp, strerror(save_errno)); - - if (!strict && (save_errno == ENOSPC)) - { - fclose(fp); - fio_unlink(path_temp, FIO_BACKUP_HOST); - return; - } - } - - if (fclose(fp) != 0) - elog(ERROR, "Cannot close control file \"%s\": %s", - path_temp, strerror(errno)); - - if (fio_sync(path_temp, FIO_BACKUP_HOST) < 0) - elog(ERROR, "Cannot sync control file \"%s\": %s", - path_temp, strerror(errno)); - - if (rename(path_temp, path) < 0) - elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", - path_temp, path, strerror(errno)); -} - -/* - * Output the list of files to backup catalog DATABASE_FILE_LIST - */ -void -write_backup_filelist(pgBackup *backup, parray *files, const char *root, - parray *external_list, bool sync) -{ - FILE *out; - char control_path[MAXPGPATH]; - char control_path_temp[MAXPGPATH]; - size_t i = 0; - #define BUFFERSZ (1024*1024) - char *buf; - int64 backup_size_on_disk = 0; - int64 uncompressed_size_on_disk = 0; - int64 wal_size_on_disk = 0; - - join_path_components(control_path, backup->root_dir, DATABASE_FILE_LIST); - snprintf(control_path_temp, sizeof(control_path_temp), "%s.tmp", control_path); - - out = fopen(control_path_temp, PG_BINARY_W); - if (out == NULL) - elog(ERROR, "Cannot open file list \"%s\": %s", control_path_temp, - strerror(errno)); - - if (chmod(control_path_temp, FILE_PERMISSION) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", control_path_temp, - strerror(errno)); - - buf = pgut_malloc(BUFFERSZ); - setvbuf(out, buf, _IOFBF, BUFFERSZ); - - if (sync) - INIT_FILE_CRC32(true, backup->content_crc); - - /* print each file in the list */ - for (i = 0; i < parray_num(files); i++) - { - int len = 0; - char line[BLCKSZ]; - pgFile *file = (pgFile *) parray_get(files, i); - - /* Ignore disappeared file */ - if (file->write_size == FILE_NOT_FOUND) - continue; - - if (S_ISDIR(file->mode)) - { - backup_size_on_disk += 4096; - uncompressed_size_on_disk += 4096; - } - - /* Count the amount of the data actually copied */ - if (S_ISREG(file->mode) && file->write_size > 0) - { - /* - * Size of WAL files in 'pg_wal' is counted separately - * TODO: in 3.0 add attribute is_walfile - */ - if (IsXLogFileName(file->name) && file->external_dir_num == 0) - wal_size_on_disk += file->write_size; - else - { - backup_size_on_disk += file->write_size; - uncompressed_size_on_disk += file->uncompressed_size; - } - } - - len = sprintf(line, "{\"path\":\"%s\", \"size\":\"" INT64_FORMAT "\", " - "\"mode\":\"%u\", \"is_datafile\":\"%u\", " - "\"is_cfs\":\"%u\", \"crc\":\"%u\", " - "\"compress_alg\":\"%s\", \"external_dir_num\":\"%d\", " - "\"dbOid\":\"%u\"", - file->rel_path, file->write_size, file->mode, - file->is_datafile ? 1 : 0, - file->is_cfs ? 1 : 0, - file->crc, - deparse_compress_alg(file->compress_alg), - file->external_dir_num, - file->dbOid); - - if (file->uncompressed_size != 0 && - file->uncompressed_size != file->write_size) - len += sprintf(line+len, ",\"full_size\":\"" INT64_FORMAT "\"", - file->uncompressed_size); - - if (file->is_datafile) - len += sprintf(line+len, ",\"segno\":\"%d\"", file->segno); - - if (file->linked) - len += sprintf(line+len, ",\"linked\":\"%s\"", file->linked); - - if (file->n_blocks > 0) - len += sprintf(line+len, ",\"n_blocks\":\"%i\"", file->n_blocks); - - if (file->n_headers > 0) - { - len += sprintf(line+len, ",\"n_headers\":\"%i\"", file->n_headers); - len += sprintf(line+len, ",\"hdr_crc\":\"%u\"", file->hdr_crc); - len += sprintf(line+len, ",\"hdr_off\":\"%llu\"", file->hdr_off); - len += sprintf(line+len, ",\"hdr_size\":\"%i\"", file->hdr_size); - } - - sprintf(line+len, "}\n"); - - if (sync) - COMP_FILE_CRC32(true, backup->content_crc, line, strlen(line)); - - fprintf(out, "%s", line); - } - - if (sync) - FIN_FILE_CRC32(true, backup->content_crc); - - if (fflush(out) != 0) - elog(ERROR, "Cannot flush file list \"%s\": %s", - control_path_temp, strerror(errno)); - - if (sync && fsync(fileno(out)) < 0) - elog(ERROR, "Cannot sync file list \"%s\": %s", - control_path_temp, strerror(errno)); - - if (fclose(out) != 0) - elog(ERROR, "Cannot close file list \"%s\": %s", - control_path_temp, strerror(errno)); - - if (rename(control_path_temp, control_path) < 0) - elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", - control_path_temp, control_path, strerror(errno)); - - /* use extra variable to avoid reset of previous data_bytes value in case of error */ - backup->data_bytes = backup_size_on_disk; - backup->uncompressed_bytes = uncompressed_size_on_disk; - - if (backup->stream) - backup->wal_bytes = wal_size_on_disk; - - free(buf); -} - -/* - * Read BACKUP_CONTROL_FILE and create pgBackup. - * - Comment starts with ';'. - * - Do not care section. - */ -static pgBackup * -readBackupControlFile(const char *path) -{ - pgBackup *backup = pgut_new0(pgBackup); - char *backup_mode = NULL; - char *start_lsn = NULL; - char *stop_lsn = NULL; - char *status = NULL; - char *parent_backup = NULL; - char *merge_dest_backup = NULL; - char *program_version = NULL; - char *server_version = NULL; - char *compress_alg = NULL; - int parsed_options; - - ConfigOption options[] = - { - {'s', 0, "backup-mode", &backup_mode, SOURCE_FILE_STRICT}, - {'u', 0, "timelineid", &backup->tli, SOURCE_FILE_STRICT}, - {'s', 0, "start-lsn", &start_lsn, SOURCE_FILE_STRICT}, - {'s', 0, "stop-lsn", &stop_lsn, SOURCE_FILE_STRICT}, - {'t', 0, "start-time", &backup->start_time, SOURCE_FILE_STRICT}, - {'t', 0, "merge-time", &backup->merge_time, SOURCE_FILE_STRICT}, - {'t', 0, "end-time", &backup->end_time, SOURCE_FILE_STRICT}, - {'U', 0, "recovery-xid", &backup->recovery_xid, SOURCE_FILE_STRICT}, - {'t', 0, "recovery-time", &backup->recovery_time, SOURCE_FILE_STRICT}, - {'t', 0, "expire-time", &backup->expire_time, SOURCE_FILE_STRICT}, - {'I', 0, "data-bytes", &backup->data_bytes, SOURCE_FILE_STRICT}, - {'I', 0, "wal-bytes", &backup->wal_bytes, SOURCE_FILE_STRICT}, - {'I', 0, "uncompressed-bytes", &backup->uncompressed_bytes, SOURCE_FILE_STRICT}, - {'I', 0, "pgdata-bytes", &backup->pgdata_bytes, SOURCE_FILE_STRICT}, - {'u', 0, "block-size", &backup->block_size, SOURCE_FILE_STRICT}, - {'u', 0, "xlog-block-size", &backup->wal_block_size, SOURCE_FILE_STRICT}, - {'u', 0, "checksum-version", &backup->checksum_version, SOURCE_FILE_STRICT}, - {'s', 0, "program-version", &program_version, SOURCE_FILE_STRICT}, - {'s', 0, "server-version", &server_version, SOURCE_FILE_STRICT}, - {'b', 0, "stream", &backup->stream, SOURCE_FILE_STRICT}, - {'s', 0, "status", &status, SOURCE_FILE_STRICT}, - {'s', 0, "parent-backup-id", &parent_backup, SOURCE_FILE_STRICT}, - {'s', 0, "merge-dest-id", &merge_dest_backup, SOURCE_FILE_STRICT}, - {'s', 0, "compress-alg", &compress_alg, SOURCE_FILE_STRICT}, - {'u', 0, "compress-level", &backup->compress_level, SOURCE_FILE_STRICT}, - {'b', 0, "from-replica", &backup->from_replica, SOURCE_FILE_STRICT}, - {'s', 0, "primary-conninfo", &backup->primary_conninfo, SOURCE_FILE_STRICT}, - {'s', 0, "external-dirs", &backup->external_dir_str, SOURCE_FILE_STRICT}, - {'s', 0, "note", &backup->note, SOURCE_FILE_STRICT}, - {'u', 0, "content-crc", &backup->content_crc, SOURCE_FILE_STRICT}, - {0} - }; - - pgBackupInit(backup); - if (fio_access(path, F_OK, FIO_BACKUP_HOST) != 0) - { - elog(WARNING, "Control file \"%s\" doesn't exist", path); - pgBackupFree(backup); - return NULL; - } - - parsed_options = config_read_opt(path, options, WARNING, true, true); - - if (parsed_options == 0) - { - elog(WARNING, "Control file \"%s\" is empty", path); - pgBackupFree(backup); - return NULL; - } - - if (backup->start_time == 0) - { - elog(WARNING, "Invalid ID/start-time, control file \"%s\" is corrupted", path); - pgBackupFree(backup); - return NULL; - } - /* XXX BACKUP_ID change it when backup_id wouldn't match start_time */ - Assert(backup->backup_id == 0 || backup->backup_id == backup->start_time); - backup->backup_id = backup->start_time; - - if (backup_mode) - { - backup->backup_mode = parse_backup_mode(backup_mode); - free(backup_mode); - } - - if (start_lsn) - { - uint32 xlogid; - uint32 xrecoff; - - if (sscanf(start_lsn, "%X/%X", &xlogid, &xrecoff) == 2) - backup->start_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; - else - elog(WARNING, "Invalid START_LSN \"%s\"", start_lsn); - free(start_lsn); - } - - if (stop_lsn) - { - uint32 xlogid; - uint32 xrecoff; - - if (sscanf(stop_lsn, "%X/%X", &xlogid, &xrecoff) == 2) - backup->stop_lsn = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; - else - elog(WARNING, "Invalid STOP_LSN \"%s\"", stop_lsn); - free(stop_lsn); - } - - if (status) - { - if (strcmp(status, "OK") == 0) - backup->status = BACKUP_STATUS_OK; - else if (strcmp(status, "ERROR") == 0) - backup->status = BACKUP_STATUS_ERROR; - else if (strcmp(status, "RUNNING") == 0) - backup->status = BACKUP_STATUS_RUNNING; - else if (strcmp(status, "MERGING") == 0) - backup->status = BACKUP_STATUS_MERGING; - else if (strcmp(status, "MERGED") == 0) - backup->status = BACKUP_STATUS_MERGED; - else if (strcmp(status, "DELETING") == 0) - backup->status = BACKUP_STATUS_DELETING; - else if (strcmp(status, "DELETED") == 0) - backup->status = BACKUP_STATUS_DELETED; - else if (strcmp(status, "DONE") == 0) - backup->status = BACKUP_STATUS_DONE; - else if (strcmp(status, "ORPHAN") == 0) - backup->status = BACKUP_STATUS_ORPHAN; - else if (strcmp(status, "CORRUPT") == 0) - backup->status = BACKUP_STATUS_CORRUPT; - else - elog(WARNING, "Invalid STATUS \"%s\"", status); - free(status); - } - - if (parent_backup) - { - backup->parent_backup = base36dec(parent_backup); - free(parent_backup); - } - - if (merge_dest_backup) - { - backup->merge_dest_backup = base36dec(merge_dest_backup); - free(merge_dest_backup); - } - - if (program_version) - { - strlcpy(backup->program_version, program_version, - sizeof(backup->program_version)); - pfree(program_version); - } - - if (server_version) - { - strlcpy(backup->server_version, server_version, - sizeof(backup->server_version)); - pfree(server_version); - } - - if (compress_alg) - backup->compress_alg = parse_compress_alg(compress_alg); - - return backup; -} - -BackupMode -parse_backup_mode(const char *value) -{ - const char *v = value; - size_t len; - - /* Skip all spaces detected */ - while (IsSpace(*v)) - v++; - len = strlen(v); - - if (len > 0 && pg_strncasecmp("full", v, len) == 0) - return BACKUP_MODE_FULL; - else if (len > 0 && pg_strncasecmp("page", v, len) == 0) - return BACKUP_MODE_DIFF_PAGE; - else if (len > 0 && pg_strncasecmp("ptrack", v, len) == 0) - return BACKUP_MODE_DIFF_PTRACK; - else if (len > 0 && pg_strncasecmp("delta", v, len) == 0) - return BACKUP_MODE_DIFF_DELTA; - - /* Backup mode is invalid, so leave with an error */ - elog(ERROR, "Invalid backup-mode \"%s\"", value); - return BACKUP_MODE_INVALID; -} - -const char * -deparse_backup_mode(BackupMode mode) -{ - switch (mode) - { - case BACKUP_MODE_FULL: - return "full"; - case BACKUP_MODE_DIFF_PAGE: - return "page"; - case BACKUP_MODE_DIFF_PTRACK: - return "ptrack"; - case BACKUP_MODE_DIFF_DELTA: - return "delta"; - case BACKUP_MODE_INVALID: - return "invalid"; - } - - return NULL; -} - -CompressAlg -parse_compress_alg(const char *arg) -{ - size_t len; - - /* Skip all spaces detected */ - while (isspace((unsigned char)*arg)) - arg++; - len = strlen(arg); - - if (len == 0) - elog(ERROR, "Compress algorithm is empty"); - - if (pg_strncasecmp("zlib", arg, len) == 0) - return ZLIB_COMPRESS; - else if (pg_strncasecmp("pglz", arg, len) == 0) - return PGLZ_COMPRESS; - else if (pg_strncasecmp("none", arg, len) == 0) - return NONE_COMPRESS; - else - elog(ERROR, "Invalid compress algorithm value \"%s\"", arg); - - return NOT_DEFINED_COMPRESS; -} - -const char* -deparse_compress_alg(int alg) -{ - switch (alg) - { - case NONE_COMPRESS: - case NOT_DEFINED_COMPRESS: - return "none"; - case ZLIB_COMPRESS: - return "zlib"; - case PGLZ_COMPRESS: - return "pglz"; - } - - return NULL; -} - -/* - * Fill PGNodeInfo struct with default values. - */ -void -pgNodeInit(PGNodeInfo *node) -{ - node->block_size = 0; - node->wal_block_size = 0; - node->checksum_version = 0; - - node->is_superuser = false; - node->pgpro_support = false; - - node->server_version = 0; - node->server_version_str[0] = '\0'; - - node->ptrack_version_num = 0; - node->is_ptrack_enabled = false; - node->ptrack_schema = NULL; -} - -/* - * Fill pgBackup struct with default values. - */ -void -pgBackupInit(pgBackup *backup) -{ - backup->backup_id = INVALID_BACKUP_ID; - backup->backup_mode = BACKUP_MODE_INVALID; - backup->status = BACKUP_STATUS_INVALID; - backup->tli = 0; - backup->start_lsn = 0; - backup->stop_lsn = 0; - backup->start_time = (time_t) 0; - backup->merge_time = (time_t) 0; - backup->end_time = (time_t) 0; - backup->recovery_xid = 0; - backup->recovery_time = (time_t) 0; - backup->expire_time = (time_t) 0; - - backup->data_bytes = BYTES_INVALID; - backup->wal_bytes = BYTES_INVALID; - backup->uncompressed_bytes = 0; - backup->pgdata_bytes = 0; - - backup->compress_alg = COMPRESS_ALG_DEFAULT; - backup->compress_level = COMPRESS_LEVEL_DEFAULT; - - backup->block_size = BLCKSZ; - backup->wal_block_size = XLOG_BLCKSZ; - backup->checksum_version = 0; - - backup->stream = false; - backup->from_replica = false; - backup->parent_backup = INVALID_BACKUP_ID; - backup->merge_dest_backup = INVALID_BACKUP_ID; - backup->parent_backup_link = NULL; - backup->primary_conninfo = NULL; - backup->program_version[0] = '\0'; - backup->server_version[0] = '\0'; - backup->external_dir_str = NULL; - backup->root_dir = NULL; - backup->database_dir = NULL; - backup->files = NULL; - backup->note = NULL; - backup->content_crc = 0; -} - -/* free pgBackup object */ -void -pgBackupFree(void *backup) -{ - pgBackup *b = (pgBackup *) backup; - - pg_free(b->primary_conninfo); - pg_free(b->external_dir_str); - pg_free(b->root_dir); - pg_free(b->database_dir); - pg_free(b->note); - pg_free(backup); -} - -/* Compare two pgBackup with their IDs (start time) in ascending order */ -int -pgBackupCompareId(const void *l, const void *r) -{ - pgBackup *lp = *(pgBackup **)l; - pgBackup *rp = *(pgBackup **)r; - - if (lp->start_time > rp->start_time) - return 1; - else if (lp->start_time < rp->start_time) - return -1; - else - return 0; -} - -/* Compare two pgBackup with their IDs in descending order */ -int -pgBackupCompareIdDesc(const void *l, const void *r) -{ - return -pgBackupCompareId(l, r); -} - -/* - * Check if multiple backups consider target backup to be their direct parent - */ -bool -is_prolific(parray *backup_list, pgBackup *target_backup) -{ - int i; - int child_counter = 0; - - for (i = 0; i < parray_num(backup_list); i++) - { - pgBackup *tmp_backup = (pgBackup *) parray_get(backup_list, i); - - /* consider only OK and DONE backups */ - if (tmp_backup->parent_backup == target_backup->start_time && - (tmp_backup->status == BACKUP_STATUS_OK || - tmp_backup->status == BACKUP_STATUS_DONE)) - { - child_counter++; - if (child_counter > 1) - return true; - } - } - - return false; -} - -/* - * Find parent base FULL backup for current backup using parent_backup_link - */ -pgBackup* -find_parent_full_backup(pgBackup *current_backup) -{ - pgBackup *base_full_backup = NULL; - base_full_backup = current_backup; - - /* sanity */ - if (!current_backup) - elog(ERROR, "Target backup cannot be NULL"); - - while (base_full_backup->parent_backup_link != NULL) - { - base_full_backup = base_full_backup->parent_backup_link; - } - - if (base_full_backup->backup_mode != BACKUP_MODE_FULL) - { - if (base_full_backup->parent_backup) - elog(WARNING, "Backup %s is missing", - base36enc(base_full_backup->parent_backup)); - else - elog(WARNING, "Failed to find parent FULL backup for %s", - backup_id_of(current_backup)); - return NULL; - } - - return base_full_backup; -} - -/* - * Iterate over parent chain and look for any problems. - * Return 0 if chain is broken. - * result_backup must contain oldest existing backup after missing backup. - * we have no way to know if there are multiple missing backups. - * Return 1 if chain is intact, but at least one backup is !OK. - * result_backup must contain oldest !OK backup. - * Return 2 if chain is intact and all backups are OK. - * result_backup must contain FULL backup on which chain is based. - */ -int -scan_parent_chain(pgBackup *current_backup, pgBackup **result_backup) -{ - pgBackup *target_backup = NULL; - pgBackup *invalid_backup = NULL; - - if (!current_backup) - elog(ERROR, "Target backup cannot be NULL"); - - target_backup = current_backup; - - while (target_backup->parent_backup_link) - { - if (target_backup->status != BACKUP_STATUS_OK && - target_backup->status != BACKUP_STATUS_DONE) - /* oldest invalid backup in parent chain */ - invalid_backup = target_backup; - - - target_backup = target_backup->parent_backup_link; - } - - /* Previous loop will skip FULL backup because his parent_backup_link is NULL */ - if (target_backup->backup_mode == BACKUP_MODE_FULL && - (target_backup->status != BACKUP_STATUS_OK && - target_backup->status != BACKUP_STATUS_DONE)) - { - invalid_backup = target_backup; - } - - /* found chain end and oldest backup is not FULL */ - if (target_backup->backup_mode != BACKUP_MODE_FULL) - { - /* Set oldest child backup in chain */ - *result_backup = target_backup; - return ChainIsBroken; - } - - /* chain is ok, but some backups are invalid */ - if (invalid_backup) - { - *result_backup = invalid_backup; - return ChainIsInvalid; - } - - *result_backup = target_backup; - return ChainIsOk; -} - -/* - * Determine if child_backup descend from parent_backup - * This check DO NOT(!!!) guarantee that parent chain is intact, - * because parent_backup can be missing. - * If inclusive is true, then child_backup counts as a child of himself - * if parent_backup_time is start_time of child_backup. - */ -bool -is_parent(time_t parent_backup_time, pgBackup *child_backup, bool inclusive) -{ - if (!child_backup) - elog(ERROR, "Target backup cannot be NULL"); - - if (inclusive && child_backup->start_time == parent_backup_time) - return true; - - while (child_backup->parent_backup_link && - child_backup->parent_backup != parent_backup_time) - { - child_backup = child_backup->parent_backup_link; - } - - if (child_backup->parent_backup == parent_backup_time) - return true; - - //if (inclusive && child_backup->start_time == parent_backup_time) - // return true; - - return false; -} - -/* On backup_list lookup children of target_backup and append them to append_list */ -void -append_children(parray *backup_list, pgBackup *target_backup, parray *append_list) -{ - int i; - - for (i = 0; i < parray_num(backup_list); i++) - { - pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - - /* check if backup is descendant of target backup */ - if (is_parent(target_backup->start_time, backup, false)) - { - /* if backup is already in the list, then skip it */ - if (!parray_contains(append_list, backup)) - parray_append(append_list, backup); - } - } -} diff --git a/src/catchup.c b/src/catchup.c deleted file mode 100644 index 427542dda..000000000 --- a/src/catchup.c +++ /dev/null @@ -1,1130 +0,0 @@ -/*------------------------------------------------------------------------- - * - * catchup.c: sync DB cluster - * - * Copyright (c) 2021-2022, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -#if PG_VERSION_NUM < 110000 -#include "catalog/catalog.h" -#endif -#include "catalog/pg_tablespace.h" -#include "access/timeline.h" -#include "pgtar.h" -#include "streamutil.h" - -#include -#include -#include - -#include "utils/thread.h" -#include "utils/file.h" - -/* - * Catchup routines - */ -static PGconn *catchup_init_state(PGNodeInfo *source_node_info, const char *source_pgdata, const char *dest_pgdata); -static void catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, const char *source_pgdata, - const char *dest_pgdata); -static void catchup_check_tablespaces_existance_in_tbsmapping(PGconn *conn); -static parray* catchup_get_tli_history(ConnectionOptions *conn_opt, TimeLineID tli); - -//REVIEW I'd also suggest to wrap all these fields into some CatchupState, but it isn't urgent. -//REVIEW_ANSWER what for? -/* - * Prepare for work: fill some globals, open connection to source database - */ -static PGconn * -catchup_init_state(PGNodeInfo *source_node_info, const char *source_pgdata, const char *dest_pgdata) -{ - PGconn *source_conn; - - /* Initialize PGInfonode */ - pgNodeInit(source_node_info); - - /* Get WAL segments size and system ID of source PG instance */ - instance_config.xlog_seg_size = get_xlog_seg_size(source_pgdata); - instance_config.system_identifier = get_system_identifier(source_pgdata, FIO_DB_HOST, false); - current.start_time = time(NULL); - - strlcpy(current.program_version, PROGRAM_VERSION, sizeof(current.program_version)); - - /* Do some compatibility checks and fill basic info about PG instance */ - source_conn = pgdata_basic_setup(instance_config.conn_opt, source_node_info); - -#if PG_VERSION_NUM >= 110000 - if (!RetrieveWalSegSize(source_conn)) - elog(ERROR, "Failed to retrieve wal_segment_size"); -#endif - - get_ptrack_version(source_conn, source_node_info); - if (source_node_info->ptrack_version_num > 0) - source_node_info->is_ptrack_enabled = pg_is_ptrack_enabled(source_conn, source_node_info->ptrack_version_num); - - /* Obtain current timeline */ -#if PG_VERSION_NUM >= 90600 - current.tli = get_current_timeline(source_conn); -#else - instance_config.pgdata = source_pgdata; - current.tli = get_current_timeline_from_control(source_pgdata, FIO_DB_HOST, false); -#endif - - elog(INFO, "Catchup start, pg_probackup version: %s, " - "PostgreSQL version: %s, " - "remote: %s, source-pgdata: %s, destination-pgdata: %s", - PROGRAM_VERSION, source_node_info->server_version_str, - IsSshProtocol() ? "true" : "false", - source_pgdata, dest_pgdata); - - if (current.from_replica) - elog(INFO, "Running catchup from standby"); - - return source_conn; -} - -/* - * Check that catchup can be performed on source and dest - * this function is for checks, that can be performed without modification of data on disk - */ -static void -catchup_preflight_checks(PGNodeInfo *source_node_info, PGconn *source_conn, - const char *source_pgdata, const char *dest_pgdata) -{ - /* TODO - * gsmol - fallback to FULL mode if dest PGDATA is empty - * kulaginm -- I think this is a harmful feature. If user requested an incremental catchup, then - * he expects that this will be done quickly and efficiently. If, for example, he made a mistake - * with dest_dir, then he will receive a second full copy instead of an error message, and I think - * that in some cases he would prefer the error. - * I propose in future versions to offer a backup_mode auto, in which we will look to the dest_dir - * and decide which of the modes will be the most effective. - * I.e.: - * if(requested_backup_mode == BACKUP_MODE_DIFF_AUTO) - * { - * if(dest_pgdata_is_empty) - * backup_mode = BACKUP_MODE_FULL; - * else - * if(ptrack supported and applicable) - * backup_mode = BACKUP_MODE_DIFF_PTRACK; - * else - * backup_mode = BACKUP_MODE_DIFF_DELTA; - * } - */ - - if (dir_is_empty(dest_pgdata, FIO_LOCAL_HOST)) - { - if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK || - current.backup_mode == BACKUP_MODE_DIFF_DELTA) - elog(ERROR, "\"%s\" is empty, but incremental catchup mode requested.", - dest_pgdata); - } - else /* dest dir not empty */ - { - if (current.backup_mode == BACKUP_MODE_FULL) - elog(ERROR, "Can't perform full catchup into non-empty directory \"%s\".", - dest_pgdata); - } - - /* check that postmaster is not running in destination */ - if (current.backup_mode != BACKUP_MODE_FULL) - { - pid_t pid; - pid = fio_check_postmaster(dest_pgdata, FIO_LOCAL_HOST); - if (pid == 1) /* postmaster.pid is mangled */ - { - char pid_filename[MAXPGPATH]; - join_path_components(pid_filename, dest_pgdata, "postmaster.pid"); - elog(ERROR, "Pid file \"%s\" is mangled, cannot determine whether postmaster is running or not", - pid_filename); - } - else if (pid > 1) /* postmaster is up */ - { - elog(ERROR, "Postmaster with pid %u is running in destination directory \"%s\"", - pid, dest_pgdata); - } - } - - /* check backup_label absence in dest */ - if (current.backup_mode != BACKUP_MODE_FULL) - { - char backup_label_filename[MAXPGPATH]; - - join_path_components(backup_label_filename, dest_pgdata, PG_BACKUP_LABEL_FILE); - if (fio_access(backup_label_filename, F_OK, FIO_LOCAL_HOST) == 0) - elog(ERROR, "Destination directory contains \"" PG_BACKUP_LABEL_FILE "\" file"); - } - - /* Check that connected PG instance, source and destination PGDATA are the same */ - { - uint64 source_conn_id, source_id, dest_id; - - source_conn_id = get_remote_system_identifier(source_conn); - source_id = get_system_identifier(source_pgdata, FIO_DB_HOST, false); /* same as instance_config.system_identifier */ - - if (source_conn_id != source_id) - elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu", - source_conn_id, source_pgdata, source_id); - - if (current.backup_mode != BACKUP_MODE_FULL) - { - dest_id = get_system_identifier(dest_pgdata, FIO_LOCAL_HOST, false); - if (source_conn_id != dest_id) - elog(ERROR, "Database identifiers mismatch: we connected to DB id %lu, but in \"%s\" we found id %lu", - source_conn_id, dest_pgdata, dest_id); - } - } - - /* check PTRACK version */ - if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) - { - if (source_node_info->ptrack_version_num == 0) - elog(ERROR, "This PostgreSQL instance does not support ptrack"); - else if (source_node_info->ptrack_version_num < 200) - elog(ERROR, "Ptrack extension is too old.\n" - "Upgrade ptrack to version >= 2"); - else if (!source_node_info->is_ptrack_enabled) - elog(ERROR, "Ptrack is disabled"); - } - - if (current.from_replica && exclusive_backup) - elog(ERROR, "Catchup from standby is only available for PostgreSQL >= 9.6"); - - /* check that we don't overwrite tablespace in source pgdata */ - catchup_check_tablespaces_existance_in_tbsmapping(source_conn); - - /* check timelines */ - if (current.backup_mode != BACKUP_MODE_FULL) - { - RedoParams dest_redo = { 0, InvalidXLogRecPtr, 0 }; - - /* fill dest_redo.lsn and dest_redo.tli */ - get_redo(dest_pgdata, FIO_LOCAL_HOST, &dest_redo); - elog(LOG, "source.tli = %X, dest_redo.lsn = %X/%X, dest_redo.tli = %X", - current.tli, (uint32) (dest_redo.lsn >> 32), (uint32) dest_redo.lsn, dest_redo.tli); - - if (current.tli != 1) - { - parray *source_timelines; /* parray* of TimeLineHistoryEntry* */ - source_timelines = catchup_get_tli_history(&instance_config.conn_opt, current.tli); - - if (source_timelines == NULL) - elog(ERROR, "Cannot get source timeline history"); - - if (!satisfy_timeline(source_timelines, dest_redo.tli, dest_redo.lsn)) - elog(ERROR, "Destination is not in source timeline history"); - - parray_walk(source_timelines, pfree); - parray_free(source_timelines); - } - else /* special case -- no history files in source */ - { - if (dest_redo.tli != 1) - elog(ERROR, "Source is behind destination in timeline history"); - } - } -} - -/* - * Check that all tablespaces exists in tablespace mapping (--tablespace-mapping option) - * Check that all local mapped directories is empty if it is local FULL catchup - * Emit fatal error if that (not existent in map or not empty) tablespace found - */ -static void -catchup_check_tablespaces_existance_in_tbsmapping(PGconn *conn) -{ - PGresult *res; - int i; - char *tablespace_path = NULL; - const char *linked_path = NULL; - char *query = "SELECT pg_catalog.pg_tablespace_location(oid) " - "FROM pg_catalog.pg_tablespace " - "WHERE pg_catalog.pg_tablespace_location(oid) <> '';"; - - res = pgut_execute(conn, query, 0, NULL); - - if (!res) - elog(ERROR, "Failed to get list of tablespaces"); - - for (i = 0; i < res->ntups; i++) - { - tablespace_path = PQgetvalue(res, i, 0); - Assert (strlen(tablespace_path) > 0); - - canonicalize_path(tablespace_path); - linked_path = get_tablespace_mapping(tablespace_path); - - if (strcmp(tablespace_path, linked_path) == 0) - /* same result -> not found in mapping */ - { - if (!fio_is_remote(FIO_DB_HOST)) - elog(ERROR, "Local catchup executed, but source database contains " - "tablespace (\"%s\"), that is not listed in the map", tablespace_path); - else - elog(WARNING, "Remote catchup executed and source database contains " - "tablespace (\"%s\"), that is not listed in the map", tablespace_path); - } - - if (!is_absolute_path(linked_path)) - elog(ERROR, "Tablespace directory path must be an absolute path: \"%s\"", - linked_path); - - if (current.backup_mode == BACKUP_MODE_FULL - && !dir_is_empty(linked_path, FIO_LOCAL_HOST)) - elog(ERROR, "Target mapped tablespace directory (\"%s\") is not empty in FULL catchup", - linked_path); - } - PQclear(res); -} - -/* - * Get timeline history via replication connection - * returns parray* of TimeLineHistoryEntry* - */ -static parray* -catchup_get_tli_history(ConnectionOptions *conn_opt, TimeLineID tli) -{ - PGresult *res; - PGconn *conn; - char *history; - char query[128]; - parray *result = NULL; - TimeLineHistoryEntry *entry = NULL; - - snprintf(query, sizeof(query), "TIMELINE_HISTORY %u", tli); - - /* - * Connect in replication mode to the server. - */ - conn = pgut_connect_replication(conn_opt->pghost, - conn_opt->pgport, - conn_opt->pgdatabase, - conn_opt->pguser, - false); - - if (!conn) - return NULL; - - res = PQexec(conn, query); - PQfinish(conn); - - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - elog(WARNING, "Could not send replication command \"%s\": %s", - query, PQresultErrorMessage(res)); - PQclear(res); - return NULL; - } - - /* - * The response to TIMELINE_HISTORY is a single row result set - * with two fields: filename and content - */ - if (PQnfields(res) != 2 || PQntuples(res) != 1) - { - elog(ERROR, "Unexpected response to TIMELINE_HISTORY command: " - "got %d rows and %d fields, expected %d rows and %d fields", - PQntuples(res), PQnfields(res), 1, 2); - PQclear(res); - return NULL; - } - - history = pgut_strdup(PQgetvalue(res, 0, 1)); - result = parse_tli_history_buffer(history, tli); - - /* some cleanup */ - pg_free(history); - PQclear(res); - - /* append last timeline entry (as read_timeline_history() do) */ - entry = pgut_new(TimeLineHistoryEntry); - entry->tli = tli; - entry->end = InvalidXLogRecPtr; - parray_insert(result, 0, entry); - - return result; -} - -/* - * catchup multithreaded copy rountine and helper structure and function - */ - -/* parameters for catchup_thread_runner() passed from catchup_multithreaded_copy() */ -typedef struct -{ - PGNodeInfo *nodeInfo; - const char *from_root; - const char *to_root; - parray *source_filelist; - parray *dest_filelist; - XLogRecPtr sync_lsn; - BackupMode backup_mode; - int thread_num; - size_t transfered_bytes; - bool completed; -} catchup_thread_runner_arg; - -/* Catchup file copier executed in separate thread */ -static void * -catchup_thread_runner(void *arg) -{ - int i; - char from_fullpath[MAXPGPATH]; - char to_fullpath[MAXPGPATH]; - - catchup_thread_runner_arg *arguments = (catchup_thread_runner_arg *) arg; - int n_files = parray_num(arguments->source_filelist); - - /* catchup a file */ - for (i = 0; i < n_files; i++) - { - pgFile *file = (pgFile *) parray_get(arguments->source_filelist, i); - pgFile *dest_file = NULL; - - /* We have already copied all directories */ - if (S_ISDIR(file->mode)) - continue; - - if (file->excluded) - continue; - - if (!pg_atomic_test_set_flag(&file->lock)) - continue; - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during catchup"); - - elog(progress ? INFO : LOG, "Progress: (%d/%d). Process file \"%s\"", - i + 1, n_files, file->rel_path); - - /* construct destination filepath */ - Assert(file->external_dir_num == 0); - join_path_components(from_fullpath, arguments->from_root, file->rel_path); - join_path_components(to_fullpath, arguments->to_root, file->rel_path); - - /* Encountered some strange beast */ - if (!S_ISREG(file->mode)) - elog(WARNING, "Unexpected type %d of file \"%s\", skipping", - file->mode, from_fullpath); - - /* Check that file exist in dest pgdata */ - if (arguments->backup_mode != BACKUP_MODE_FULL) - { - pgFile **dest_file_tmp = NULL; - dest_file_tmp = (pgFile **) parray_bsearch(arguments->dest_filelist, - file, pgFileCompareRelPathWithExternal); - if (dest_file_tmp) - { - /* File exists in destination PGDATA */ - file->exists_in_prev = true; - dest_file = *dest_file_tmp; - } - } - - /* Do actual work */ - if (file->is_datafile && !file->is_cfs) - { - catchup_data_file(file, from_fullpath, to_fullpath, - arguments->sync_lsn, - arguments->backup_mode, - arguments->nodeInfo->checksum_version, - dest_file != NULL ? dest_file->size : 0); - } - else - { - backup_non_data_file(file, dest_file, from_fullpath, to_fullpath, - arguments->backup_mode, current.parent_backup, true); - } - - /* file went missing during catchup */ - if (file->write_size == FILE_NOT_FOUND) - continue; - - if (file->write_size == BYTES_INVALID) - { - elog(LOG, "Skipping the unchanged file: \"%s\", read %li bytes", from_fullpath, file->read_size); - continue; - } - - arguments->transfered_bytes += file->write_size; - elog(LOG, "File \"%s\". Copied "INT64_FORMAT " bytes", - from_fullpath, file->write_size); - } - - /* ssh connection to longer needed */ - fio_disconnect(); - - /* Data files transferring is successful */ - arguments->completed = true; - - return NULL; -} - -/* - * main multithreaded copier - * returns size of transfered data file - * or -1 in case of error - */ -static ssize_t -catchup_multithreaded_copy(int num_threads, - PGNodeInfo *source_node_info, - const char *source_pgdata_path, - const char *dest_pgdata_path, - parray *source_filelist, - parray *dest_filelist, - XLogRecPtr sync_lsn, - BackupMode backup_mode) -{ - /* arrays with meta info for multi threaded catchup */ - catchup_thread_runner_arg *threads_args; - pthread_t *threads; - - bool all_threads_successful = true; - ssize_t transfered_bytes_result = 0; - int i; - - /* init thread args */ - threads_args = (catchup_thread_runner_arg *) palloc(sizeof(catchup_thread_runner_arg) * num_threads); - for (i = 0; i < num_threads; i++) - threads_args[i] = (catchup_thread_runner_arg){ - .nodeInfo = source_node_info, - .from_root = source_pgdata_path, - .to_root = dest_pgdata_path, - .source_filelist = source_filelist, - .dest_filelist = dest_filelist, - .sync_lsn = sync_lsn, - .backup_mode = backup_mode, - .thread_num = i + 1, - .transfered_bytes = 0, - .completed = false, - }; - - /* Run threads */ - thread_interrupted = false; - threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - if (!dry_run) - { - for (i = 0; i < num_threads; i++) - { - elog(VERBOSE, "Start thread num: %i", i); - pthread_create(&threads[i], NULL, &catchup_thread_runner, &(threads_args[i])); - } - } - - /* Wait threads */ - for (i = 0; i < num_threads; i++) - { - if (!dry_run) - pthread_join(threads[i], NULL); - all_threads_successful &= threads_args[i].completed; - transfered_bytes_result += threads_args[i].transfered_bytes; - } - - free(threads); - free(threads_args); - return all_threads_successful ? transfered_bytes_result : -1; -} - -/* - * Sync every file in destination directory to disk - */ -static void -catchup_sync_destination_files(const char* pgdata_path, fio_location location, parray *filelist, pgFile *pg_control_file) -{ - char fullpath[MAXPGPATH]; - time_t start_time, end_time; - char pretty_time[20]; - int i; - - elog(INFO, "Syncing copied files to disk"); - time(&start_time); - - for (i = 0; i < parray_num(filelist); i++) - { - pgFile *file = (pgFile *) parray_get(filelist, i); - - /* TODO: sync directory ? - * - at first glance we can rely on fs journaling, - * which is enabled by default on most platforms - * - but PG itself is not relying on fs, its durable_sync - * includes directory sync - */ - if (S_ISDIR(file->mode) || file->excluded) - continue; - - Assert(file->external_dir_num == 0); - join_path_components(fullpath, pgdata_path, file->rel_path); - if (fio_sync(fullpath, location) != 0) - elog(ERROR, "Cannot sync file \"%s\": %s", fullpath, strerror(errno)); - } - - /* - * sync pg_control file - */ - join_path_components(fullpath, pgdata_path, pg_control_file->rel_path); - if (fio_sync(fullpath, location) != 0) - elog(ERROR, "Cannot sync file \"%s\": %s", fullpath, strerror(errno)); - - time(&end_time); - pretty_time_interval(difftime(end_time, start_time), - pretty_time, lengthof(pretty_time)); - elog(INFO, "Files are synced, time elapsed: %s", pretty_time); -} - -/* - * Filter filelist helper function (used to process --exclude-path's) - * filelist -- parray of pgFile *, can't be NULL - * exclude_absolute_paths_list -- sorted parray of char * (absolute paths, starting with '/'), can be NULL - * exclude_relative_paths_list -- sorted parray of char * (relative paths), can be NULL - * logging_string -- helper parameter, used for generating verbose log messages ("Source" or "Destination") - */ -static void -filter_filelist(parray *filelist, const char *pgdata, - parray *exclude_absolute_paths_list, parray *exclude_relative_paths_list, - const char *logging_string) -{ - int i; - - if (exclude_absolute_paths_list == NULL && exclude_relative_paths_list == NULL) - return; - - for (i = 0; i < parray_num(filelist); ++i) - { - char full_path[MAXPGPATH]; - pgFile *file = (pgFile *) parray_get(filelist, i); - join_path_components(full_path, pgdata, file->rel_path); - - if ( - (exclude_absolute_paths_list != NULL - && parray_bsearch(exclude_absolute_paths_list, full_path, pgPrefixCompareString)!= NULL - ) || ( - exclude_relative_paths_list != NULL - && parray_bsearch(exclude_relative_paths_list, file->rel_path, pgPrefixCompareString)!= NULL) - ) - { - elog(INFO, "%s file \"%s\" excluded with --exclude-path option", logging_string, full_path); - file->excluded = true; - } - } -} - -/* - * Entry point of pg_probackup CATCHUP subcommand. - * exclude_*_paths_list are parray's of char * - */ -int -do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, bool sync_dest_files, - parray *exclude_absolute_paths_list, parray *exclude_relative_paths_list) -{ - PGconn *source_conn = NULL; - PGNodeInfo source_node_info; - bool backup_logs = false; - parray *source_filelist = NULL; - pgFile *source_pg_control_file = NULL; - parray *dest_filelist = NULL; - char dest_xlog_path[MAXPGPATH]; - - RedoParams dest_redo = { 0, InvalidXLogRecPtr, 0 }; - PGStopBackupResult stop_backup_result; - bool catchup_isok = true; - - int i; - - /* for fancy reporting */ - time_t start_time, end_time; - ssize_t transfered_datafiles_bytes = 0; - ssize_t transfered_walfiles_bytes = 0; - char pretty_source_bytes[20]; - - source_conn = catchup_init_state(&source_node_info, source_pgdata, dest_pgdata); - catchup_preflight_checks(&source_node_info, source_conn, source_pgdata, dest_pgdata); - - /* we need to sort --exclude_path's for future searching */ - if (exclude_absolute_paths_list != NULL) - parray_qsort(exclude_absolute_paths_list, pgCompareString); - if (exclude_relative_paths_list != NULL) - parray_qsort(exclude_relative_paths_list, pgCompareString); - - elog(INFO, "Database catchup start"); - - if (current.backup_mode != BACKUP_MODE_FULL) - { - dest_filelist = parray_new(); - dir_list_file(dest_filelist, dest_pgdata, - true, true, false, backup_logs, true, 0, FIO_LOCAL_HOST); - filter_filelist(dest_filelist, dest_pgdata, exclude_absolute_paths_list, exclude_relative_paths_list, "Destination"); - - // fill dest_redo.lsn and dest_redo.tli - get_redo(dest_pgdata, FIO_LOCAL_HOST, &dest_redo); - elog(INFO, "syncLSN = %X/%X", (uint32) (dest_redo.lsn >> 32), (uint32) dest_redo.lsn); - - /* - * Future improvement to catch partial catchup: - * 1. rename dest pg_control into something like pg_control.pbk - * (so user can't start partial catchup'ed instance from this point) - * 2. try to read by get_redo() pg_control and pg_control.pbk (to detect partial catchup) - * 3. at the end (after copy of correct pg_control), remove pg_control.pbk - */ - } - - /* - * Make sure that sync point is withing ptrack tracking range - * TODO: move to separate function to use in both backup.c and catchup.c - */ - if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) - { - XLogRecPtr ptrack_lsn = get_last_ptrack_lsn(source_conn, &source_node_info); - - if (ptrack_lsn > dest_redo.lsn || ptrack_lsn == InvalidXLogRecPtr) - elog(ERROR, "LSN from ptrack_control in source %X/%X is greater than checkpoint LSN in destination %X/%X.\n" - "You can perform only FULL catchup.", - (uint32) (ptrack_lsn >> 32), (uint32) (ptrack_lsn), - (uint32) (dest_redo.lsn >> 32), - (uint32) (dest_redo.lsn)); - } - - { - char label[1024]; - /* notify start of backup to PostgreSQL server */ - time2iso(label, lengthof(label), current.start_time, false); - strncat(label, " with pg_probackup", lengthof(label) - - strlen(" with pg_probackup")); - - /* Call pg_start_backup function in PostgreSQL connect */ - pg_start_backup(label, smooth_checkpoint, ¤t, &source_node_info, source_conn); - elog(INFO, "pg_start_backup START LSN %X/%X", (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn)); - } - - /* Sanity: source cluster must be "in future" relatively to dest cluster */ - if (current.backup_mode != BACKUP_MODE_FULL && - dest_redo.lsn > current.start_lsn) - elog(ERROR, "Current START LSN %X/%X is lower than SYNC LSN %X/%X, " - "it may indicate that we are trying to catchup with PostgreSQL instance from the past", - (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), - (uint32) (dest_redo.lsn >> 32), (uint32) (dest_redo.lsn)); - - /* Start stream replication */ - join_path_components(dest_xlog_path, dest_pgdata, PG_XLOG_DIR); - if (!dry_run) - { - fio_mkdir(dest_xlog_path, DIR_PERMISSION, FIO_LOCAL_HOST); - start_WAL_streaming(source_conn, dest_xlog_path, &instance_config.conn_opt, - current.start_lsn, current.tli, false); - } - else - elog(INFO, "WAL streaming skipping with --dry-run option"); - - source_filelist = parray_new(); - - /* list files with the logical path. omit $PGDATA */ - if (fio_is_remote(FIO_DB_HOST)) - fio_list_dir(source_filelist, source_pgdata, - true, true, false, backup_logs, true, 0); - else - dir_list_file(source_filelist, source_pgdata, - true, true, false, backup_logs, true, 0, FIO_LOCAL_HOST); - - //REVIEW FIXME. Let's fix that before release. - // TODO what if wal is not a dir (symlink to a dir)? - // - Currently backup/restore transform pg_wal symlink to directory - // so the problem is not only with catchup. - // if we want to make it right - we must provide the way - // for symlink remapping during restore and catchup. - // By default everything must be left as it is. - - /* close ssh session in main thread */ - fio_disconnect(); - - /* - * Sort pathname ascending. It is necessary to create intermediate - * directories sequentially. - * - * For example: - * 1 - create 'base' - * 2 - create 'base/1' - * - * Sorted array is used at least in parse_filelist_filenames(), - * extractPageMap(), make_pagemap_from_ptrack(). - */ - parray_qsort(source_filelist, pgFileCompareRelPathWithExternal); - - //REVIEW Do we want to do similar calculation for dest? - //REVIEW_ANSWER what for? - { - ssize_t source_bytes = 0; - char pretty_bytes[20]; - - source_bytes += calculate_datasize_of_filelist(source_filelist); - - /* Extract information about files in source_filelist parsing their names:*/ - parse_filelist_filenames(source_filelist, source_pgdata); - filter_filelist(source_filelist, source_pgdata, exclude_absolute_paths_list, exclude_relative_paths_list, "Source"); - - current.pgdata_bytes += calculate_datasize_of_filelist(source_filelist); - - pretty_size(current.pgdata_bytes, pretty_source_bytes, lengthof(pretty_source_bytes)); - pretty_size(source_bytes - current.pgdata_bytes, pretty_bytes, lengthof(pretty_bytes)); - elog(INFO, "Source PGDATA size: %s (excluded %s)", pretty_source_bytes, pretty_bytes); - } - - elog(INFO, "Start LSN (source): %X/%X, TLI: %X", - (uint32) (current.start_lsn >> 32), (uint32) (current.start_lsn), - current.tli); - if (current.backup_mode != BACKUP_MODE_FULL) - elog(INFO, "LSN in destination: %X/%X, TLI: %X", - (uint32) (dest_redo.lsn >> 32), (uint32) (dest_redo.lsn), - dest_redo.tli); - - /* Build page mapping in PTRACK mode */ - if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) - { - time(&start_time); - elog(INFO, "Extracting pagemap of changed blocks"); - - /* Build the page map from ptrack information */ - make_pagemap_from_ptrack_2(source_filelist, source_conn, - source_node_info.ptrack_schema, - source_node_info.ptrack_version_num, - dest_redo.lsn); - time(&end_time); - elog(INFO, "Pagemap successfully extracted, time elapsed: %.0f sec", - difftime(end_time, start_time)); - } - - /* - * Make directories before catchup - */ - /* - * We iterate over source_filelist and for every directory with parent 'pg_tblspc' - * we must lookup this directory name in tablespace map. - * If we got a match, we treat this directory as tablespace. - * It means that we create directory specified in tablespace map and - * original directory created as symlink to it. - */ - for (i = 0; i < parray_num(source_filelist); i++) - { - pgFile *file = (pgFile *) parray_get(source_filelist, i); - char parent_dir[MAXPGPATH]; - - if (!S_ISDIR(file->mode) || file->excluded) - continue; - - /* - * check if it is fake "directory" and is a tablespace link - * this is because we passed the follow_symlink when building the list - */ - /* get parent dir of rel_path */ - strncpy(parent_dir, file->rel_path, MAXPGPATH); - get_parent_directory(parent_dir); - - /* check if directory is actually link to tablespace */ - if (strcmp(parent_dir, PG_TBLSPC_DIR) != 0) - { - /* if the entry is a regular directory, create it in the destination */ - char dirpath[MAXPGPATH]; - - join_path_components(dirpath, dest_pgdata, file->rel_path); - - elog(LOG, "Create directory '%s'", dirpath); - if (!dry_run) - fio_mkdir(dirpath, DIR_PERMISSION, FIO_LOCAL_HOST); - } - else - { - /* this directory located in pg_tblspc */ - const char *linked_path = NULL; - char to_path[MAXPGPATH]; - - // TODO perform additional check that this is actually symlink? - { /* get full symlink path and map this path to new location */ - char source_full_path[MAXPGPATH]; - char symlink_content[MAXPGPATH]; - join_path_components(source_full_path, source_pgdata, file->rel_path); - fio_readlink(source_full_path, symlink_content, sizeof(symlink_content), FIO_DB_HOST); - /* we checked that mapping exists in preflight_checks for local catchup */ - linked_path = get_tablespace_mapping(symlink_content); - elog(INFO, "Map tablespace full_path: \"%s\" old_symlink_content: \"%s\" new_symlink_content: \"%s\"\n", - source_full_path, - symlink_content, - linked_path); - } - - if (!is_absolute_path(linked_path)) - elog(ERROR, "Tablespace directory path must be an absolute path: %s\n", - linked_path); - - join_path_components(to_path, dest_pgdata, file->rel_path); - - elog(INFO, "Create directory \"%s\" and symbolic link \"%s\"", - linked_path, to_path); - - if (!dry_run) - { - /* create tablespace directory */ - if (fio_mkdir(linked_path, file->mode, FIO_LOCAL_HOST) != 0) - elog(ERROR, "Could not create tablespace directory \"%s\": %s", - linked_path, strerror(errno)); - - /* create link to linked_path */ - if (fio_symlink(linked_path, to_path, true, FIO_LOCAL_HOST) < 0) - elog(ERROR, "Could not create symbolic link \"%s\" -> \"%s\": %s", - linked_path, to_path, strerror(errno)); - } - } - } - - /* - * find pg_control file (in already sorted source_filelist) - * and exclude it from list for future special processing - */ - { - int control_file_elem_index; - pgFile search_key; - MemSet(&search_key, 0, sizeof(pgFile)); - /* pgFileCompareRelPathWithExternal uses only .rel_path and .external_dir_num for comparision */ - search_key.rel_path = XLOG_CONTROL_FILE; - search_key.external_dir_num = 0; - control_file_elem_index = parray_bsearch_index(source_filelist, &search_key, pgFileCompareRelPathWithExternal); - if(control_file_elem_index < 0) - elog(ERROR, "\"%s\" not found in \"%s\"\n", XLOG_CONTROL_FILE, source_pgdata); - source_pg_control_file = parray_remove(source_filelist, control_file_elem_index); - } - - /* TODO before public release: must be more careful with pg_control. - * when running catchup or incremental restore - * cluster is actually in two states - * simultaneously - old and new, so - * it must contain both pg_control files - * describing those states: global/pg_control_old, global/pg_control_new - * 1. This approach will provide us with means of - * robust detection of previos failures and thus correct operation retrying (or forbidding). - * 2. We will have the ability of preventing instance from starting - * in the middle of our operations. - */ - - /* - * remove absent source files in dest (dropped tables, etc...) - * note: global/pg_control will also be deleted here - * mark dest files (that excluded with source --exclude-path) also for exclusion - */ - if (current.backup_mode != BACKUP_MODE_FULL) - { - elog(INFO, "Removing redundant files in destination directory"); - parray_qsort(dest_filelist, pgFileCompareRelPathWithExternalDesc); - for (i = 0; i < parray_num(dest_filelist); i++) - { - bool redundant = true; - pgFile *file = (pgFile *) parray_get(dest_filelist, i); - pgFile **src_file = NULL; - - //TODO optimize it and use some merge-like algorithm - //instead of bsearch for each file. - src_file = (pgFile **) parray_bsearch(source_filelist, file, pgFileCompareRelPathWithExternal); - - if (src_file!= NULL && !(*src_file)->excluded && file->excluded) - (*src_file)->excluded = true; - - if (src_file!= NULL || file->excluded) - redundant = false; - - /* pg_filenode.map are always copied, because it's crc cannot be trusted */ - Assert(file->external_dir_num == 0); - if (pg_strcasecmp(file->name, RELMAPPER_FILENAME) == 0) - redundant = true; - - /* if file does not exists in destination list, then we can safely unlink it */ - if (redundant) - { - char fullpath[MAXPGPATH]; - - join_path_components(fullpath, dest_pgdata, file->rel_path); - if (!dry_run) - { - fio_delete(file->mode, fullpath, FIO_LOCAL_HOST); - } - elog(LOG, "Deleted file \"%s\"", fullpath); - - /* shrink dest pgdata list */ - pgFileFree(file); - parray_remove(dest_filelist, i); - i--; - } - } - } - - /* clear file locks */ - pfilearray_clear_locks(source_filelist); - - /* Sort by size for load balancing */ - parray_qsort(source_filelist, pgFileCompareSizeDesc); - - /* Sort the array for binary search */ - if (dest_filelist) - parray_qsort(dest_filelist, pgFileCompareRelPathWithExternal); - - /* run copy threads */ - elog(INFO, "Start transferring data files"); - time(&start_time); - transfered_datafiles_bytes = catchup_multithreaded_copy(num_threads, &source_node_info, - source_pgdata, dest_pgdata, - source_filelist, dest_filelist, - dest_redo.lsn, current.backup_mode); - catchup_isok = transfered_datafiles_bytes != -1; - - /* at last copy control file */ - if (catchup_isok && !dry_run) - { - char from_fullpath[MAXPGPATH]; - char to_fullpath[MAXPGPATH]; - join_path_components(from_fullpath, source_pgdata, source_pg_control_file->rel_path); - join_path_components(to_fullpath, dest_pgdata, source_pg_control_file->rel_path); - copy_pgcontrol_file(from_fullpath, FIO_DB_HOST, - to_fullpath, FIO_LOCAL_HOST, source_pg_control_file); - transfered_datafiles_bytes += source_pg_control_file->size; - } - - if (!catchup_isok && !dry_run) - { - char pretty_time[20]; - char pretty_transfered_data_bytes[20]; - - time(&end_time); - pretty_time_interval(difftime(end_time, start_time), - pretty_time, lengthof(pretty_time)); - pretty_size(transfered_datafiles_bytes, pretty_transfered_data_bytes, lengthof(pretty_transfered_data_bytes)); - - elog(ERROR, "Catchup failed. Transfered: %s, time elapsed: %s", - pretty_transfered_data_bytes, pretty_time); - } - - /* Notify end of backup */ - { - //REVIEW Is it relevant to catchup? I suppose it isn't, since catchup is a new code. - //If we do need it, please write a comment explaining that. - /* kludge against some old bug in archive_timeout. TODO: remove in 3.0.0 */ - int timeout = (instance_config.archive_timeout > 0) ? - instance_config.archive_timeout : ARCHIVE_TIMEOUT_DEFAULT; - char *stop_backup_query_text = NULL; - - pg_silent_client_messages(source_conn); - - /* Execute pg_stop_backup using PostgreSQL connection */ - pg_stop_backup_send(source_conn, source_node_info.server_version, current.from_replica, exclusive_backup, &stop_backup_query_text); - - /* - * Wait for the result of pg_stop_backup(), but no longer than - * archive_timeout seconds - */ - pg_stop_backup_consume(source_conn, source_node_info.server_version, exclusive_backup, timeout, stop_backup_query_text, &stop_backup_result); - - /* Cleanup */ - pg_free(stop_backup_query_text); - } - - if (!dry_run) - wait_wal_and_calculate_stop_lsn(dest_xlog_path, stop_backup_result.lsn, ¤t); - -#if PG_VERSION_NUM >= 90600 - /* Write backup_label */ - Assert(stop_backup_result.backup_label_content != NULL); - if (!dry_run) - { - pg_stop_backup_write_file_helper(dest_pgdata, PG_BACKUP_LABEL_FILE, "backup label", - stop_backup_result.backup_label_content, stop_backup_result.backup_label_content_len, - NULL); - } - free(stop_backup_result.backup_label_content); - stop_backup_result.backup_label_content = NULL; - stop_backup_result.backup_label_content_len = 0; - - /* tablespace_map */ - if (stop_backup_result.tablespace_map_content != NULL) - { - // TODO what if tablespace is created during catchup? - /* Because we have already created symlinks in pg_tblspc earlier, - * we do not need to write the tablespace_map file. - * So this call is unnecessary: - * pg_stop_backup_write_file_helper(dest_pgdata, PG_TABLESPACE_MAP_FILE, "tablespace map", - * stop_backup_result.tablespace_map_content, stop_backup_result.tablespace_map_content_len, - * NULL); - */ - free(stop_backup_result.tablespace_map_content); - stop_backup_result.tablespace_map_content = NULL; - stop_backup_result.tablespace_map_content_len = 0; - } -#endif - - /* wait for end of wal streaming and calculate wal size transfered */ - if (!dry_run) - { - parray *wal_files_list = NULL; - wal_files_list = parray_new(); - - if (wait_WAL_streaming_end(wal_files_list)) - elog(ERROR, "WAL streaming failed"); - - for (i = 0; i < parray_num(wal_files_list); i++) - { - pgFile *file = (pgFile *) parray_get(wal_files_list, i); - transfered_walfiles_bytes += file->size; - } - - parray_walk(wal_files_list, pgFileFree); - parray_free(wal_files_list); - wal_files_list = NULL; - } - - /* - * In case of backup from replica >= 9.6 we must fix minRecPoint - */ - if (current.from_replica && !exclusive_backup) - { - set_min_recovery_point(source_pg_control_file, dest_pgdata, current.stop_lsn); - } - - /* close ssh session in main thread */ - fio_disconnect(); - - /* fancy reporting */ - { - char pretty_transfered_data_bytes[20]; - char pretty_transfered_wal_bytes[20]; - char pretty_time[20]; - - time(&end_time); - pretty_time_interval(difftime(end_time, start_time), - pretty_time, lengthof(pretty_time)); - pretty_size(transfered_datafiles_bytes, pretty_transfered_data_bytes, lengthof(pretty_transfered_data_bytes)); - pretty_size(transfered_walfiles_bytes, pretty_transfered_wal_bytes, lengthof(pretty_transfered_wal_bytes)); - - elog(INFO, "Databases synchronized. Transfered datafiles size: %s, transfered wal size: %s, time elapsed: %s", - pretty_transfered_data_bytes, pretty_transfered_wal_bytes, pretty_time); - - if (current.backup_mode != BACKUP_MODE_FULL) - elog(INFO, "Catchup incremental ratio (less is better): %.f%% (%s/%s)", - ((float) transfered_datafiles_bytes / current.pgdata_bytes) * 100, - pretty_transfered_data_bytes, pretty_source_bytes); - } - - /* Sync all copied files unless '--no-sync' flag is used */ - if (sync_dest_files && !dry_run) - catchup_sync_destination_files(dest_pgdata, FIO_LOCAL_HOST, source_filelist, source_pg_control_file); - else - elog(WARNING, "Files are not synced to disk"); - - /* Cleanup */ - if (dest_filelist && !dry_run) - { - parray_walk(dest_filelist, pgFileFree); - } - parray_free(dest_filelist); - parray_walk(source_filelist, pgFileFree); - parray_free(source_filelist); - pgFileFree(source_pg_control_file); - - return 0; -} diff --git a/src/checkdb.c b/src/checkdb.c deleted file mode 100644 index 2a7d4e9eb..000000000 --- a/src/checkdb.c +++ /dev/null @@ -1,779 +0,0 @@ -/*------------------------------------------------------------------------- - * - * src/checkdb.c - * pg_probackup checkdb subcommand - * - * It allows to validate all data files located in PGDATA - * via block checksums matching and page header sanity checks. - * Optionally all indexes in all databases in PostgreSQL - * instance can be logically verified using extensions - * amcheck or amcheck_next. - * - * Portions Copyright (c) 2019-2019, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -#include -#include -#include - -#include "utils/thread.h" -#include "utils/file.h" - - -typedef struct -{ - /* list of files to validate */ - parray *files_list; - /* if page checksums are enabled in this postgres instance? */ - uint32 checksum_version; - /* - * conn and cancel_conn - * to use in check_data_file - * to connect to postgres if we've failed to validate page - * and want to read it via buffer cache to ensure - */ - ConnectionArgs conn_arg; - /* number of thread for debugging */ - int thread_num; - /* pgdata path */ - const char *from_root; - /* - * Return value from the thread: - * 0 everything is ok - * 1 thread errored during execution, e.g. interruption (default value) - * 2 corruption is definitely(!) found - */ - int ret; -} check_files_arg; - - -typedef struct -{ - /* list of indexes to amcheck */ - parray *index_list; - /* - * credentials to connect to postgres instance - * used for compatibility checks of blocksize, - * server version and so on - */ - ConnectionOptions conn_opt; - /* - * conn and cancel_conn - * to use in threads to connect to databases - */ - ConnectionArgs conn_arg; - /* number of thread for debugging */ - int thread_num; - /* - * Return value from the thread: - * 0 everything is ok - * 1 thread errored during execution, e.g. interruption (default value) - * 2 corruption is definitely(!) found - */ - int ret; -} check_indexes_arg; - -typedef struct pg_indexEntry -{ - Oid indexrelid; - char *name; - char *namespace; - bool heapallindexed_is_supported; - bool checkunique_is_supported; - /* schema where amcheck extension is located */ - char *amcheck_nspname; - /* lock for synchronization of parallel threads */ - volatile pg_atomic_flag lock; -} pg_indexEntry; - -static void -pg_indexEntry_free(void *index) -{ - pg_indexEntry *index_ptr; - - if (index == NULL) - return; - - index_ptr = (pg_indexEntry *) index; - - if (index_ptr->name) - free(index_ptr->name); - if (index_ptr->name) - free(index_ptr->namespace); - if (index_ptr->amcheck_nspname) - free(index_ptr->amcheck_nspname); - - free(index_ptr); -} - - -static void *check_files(void *arg); -static void do_block_validation(char *pgdata, uint32 checksum_version); - -static void *check_indexes(void *arg); -static parray* get_index_list(const char *dbname, bool first_db_with_amcheck, - PGconn *db_conn); -static bool amcheck_one_index(check_indexes_arg *arguments, - pg_indexEntry *ind); -static void do_amcheck(ConnectionOptions conn_opt, PGconn *conn); - -/* - * Check files in PGDATA. - * Read all files listed in files_list. - * If the file is 'datafile' (regular relation's main fork), read it page by page, - * verify checksum and copy. - */ -static void * -check_files(void *arg) -{ - int i; - check_files_arg *arguments = (check_files_arg *) arg; - int n_files_list = 0; - char from_fullpath[MAXPGPATH]; - - if (arguments->files_list) - n_files_list = parray_num(arguments->files_list); - - /* check a file */ - for (i = 0; i < n_files_list; i++) - { - pgFile *file = (pgFile *) parray_get(arguments->files_list, i); - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during checkdb"); - - /* No need to check directories */ - if (S_ISDIR(file->mode)) - continue; - - if (!pg_atomic_test_set_flag(&file->lock)) - continue; - - join_path_components(from_fullpath, arguments->from_root, file->rel_path); - - elog(VERBOSE, "Checking file: \"%s\" ", from_fullpath); - - if (progress) - elog(INFO, "Progress: (%d/%d). Process file \"%s\"", - i + 1, n_files_list, from_fullpath); - - if (S_ISREG(file->mode)) - { - /* check only uncompressed by cfs datafiles */ - if (file->is_datafile && !file->is_cfs) - { - /* - * TODO deep inside check_data_file - * uses global variables to set connections. - * Need refactoring. - */ - if (!check_data_file(&(arguments->conn_arg), - file, from_fullpath, - arguments->checksum_version)) - arguments->ret = 2; /* corruption found */ - } - } - else - elog(WARNING, "unexpected file type %d", file->mode); - } - - /* Ret values: - * 0 everything is ok - * 1 thread errored during execution, e.g. interruption (default value) - * 2 corruption is definitely(!) found - */ - if (arguments->ret == 1) - arguments->ret = 0; - - return NULL; -} - -/* collect list of files and run threads to check files in the instance */ -static void -do_block_validation(char *pgdata, uint32 checksum_version) -{ - int i; - /* arrays with meta info for multi threaded check */ - pthread_t *threads; - check_files_arg *threads_args; - bool check_isok = true; - parray *files_list = NULL; - - /* initialize file list */ - files_list = parray_new(); - - /* list files with the logical path. omit $PGDATA */ - dir_list_file(files_list, pgdata, true, true, - false, false, true, 0, FIO_DB_HOST); - - /* - * Sort pathname ascending. - * - * For example: - * 1 - create 'base' - * 2 - create 'base/1' - */ - parray_qsort(files_list, pgFileCompareRelPathWithExternal); - /* Extract information about files in pgdata parsing their names:*/ - parse_filelist_filenames(files_list, pgdata); - - /* setup threads */ - for (i = 0; i < parray_num(files_list); i++) - { - pgFile *file = (pgFile *) parray_get(files_list, i); - pg_atomic_init_flag(&file->lock); - } - - /* Sort by size for load balancing */ - parray_qsort(files_list, pgFileCompareSize); - - /* init thread args with own file lists */ - threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - threads_args = (check_files_arg *) palloc(sizeof(check_files_arg)*num_threads); - - for (i = 0; i < num_threads; i++) - { - check_files_arg *arg = &(threads_args[i]); - - arg->files_list = files_list; - arg->checksum_version = checksum_version; - arg->from_root = pgdata; - - arg->conn_arg.conn = NULL; - arg->conn_arg.cancel_conn = NULL; - - arg->thread_num = i + 1; - /* By default there is some error */ - arg->ret = 1; - } - - elog(INFO, "Start checking data files"); - - /* Run threads */ - for (i = 0; i < num_threads; i++) - { - check_files_arg *arg = &(threads_args[i]); - - elog(VERBOSE, "Start thread num: %i", i); - - pthread_create(&threads[i], NULL, check_files, arg); - } - - /* Wait threads */ - for (i = 0; i < num_threads; i++) - { - pthread_join(threads[i], NULL); - if (threads_args[i].ret > 0) - check_isok = false; - } - - /* cleanup */ - if (files_list) - { - parray_walk(files_list, pgFileFree); - parray_free(files_list); - files_list = NULL; - } - - if (check_isok) - elog(INFO, "Data files are valid"); - else - elog(ERROR, "Checkdb failed"); -} - -/* Check indexes with amcheck */ -static void * -check_indexes(void *arg) -{ - int i; - check_indexes_arg *arguments = (check_indexes_arg *) arg; - int n_indexes = 0; - my_thread_num = arguments->thread_num; - - if (arguments->index_list) - n_indexes = parray_num(arguments->index_list); - - for (i = 0; i < n_indexes; i++) - { - pg_indexEntry *ind = (pg_indexEntry *) parray_get(arguments->index_list, i); - - if (!pg_atomic_test_set_flag(&ind->lock)) - continue; - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "Thread [%d]: interrupted during checkdb --amcheck", - arguments->thread_num); - - if (progress) - elog(INFO, "Thread [%d]. Progress: (%d/%d). Amchecking index '%s.%s'", - arguments->thread_num, i + 1, n_indexes, - ind->namespace, ind->name); - - if (arguments->conn_arg.conn == NULL) - { - - arguments->conn_arg.conn = pgut_connect(arguments->conn_opt.pghost, - arguments->conn_opt.pgport, - arguments->conn_opt.pgdatabase, - arguments->conn_opt.pguser); - arguments->conn_arg.cancel_conn = PQgetCancel(arguments->conn_arg.conn); - } - - /* remember that we have a failed check */ - if (!amcheck_one_index(arguments, ind)) - arguments->ret = 2; /* corruption found */ - } - - /* Close connection. */ - if (arguments->conn_arg.conn) - pgut_disconnect(arguments->conn_arg.conn); - - /* Ret values: - * 0 everything is ok - * 1 thread errored during execution, e.g. interruption (default value) - * 2 corruption is definitely(!) found - */ - if (arguments->ret == 1) - arguments->ret = 0; - - return NULL; -} - -/* Get index list for given database */ -static parray* -get_index_list(const char *dbname, bool first_db_with_amcheck, - PGconn *db_conn) -{ - PGresult *res; - char *amcheck_nspname = NULL; - char *amcheck_extname = NULL; - char *amcheck_extversion = NULL; - int i; - bool heapallindexed_is_supported = false; - bool checkunique_is_supported = false; - parray *index_list = NULL; - - /* Check amcheck extension version */ - res = pgut_execute(db_conn, "SELECT " - "extname, nspname, extversion " - "FROM pg_catalog.pg_namespace n " - "JOIN pg_catalog.pg_extension e " - "ON n.oid=e.extnamespace " - "WHERE e.extname IN ('amcheck'::name, 'amcheck_next'::name) " - "ORDER BY extversion DESC " - "LIMIT 1", - 0, NULL); - - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - PQclear(res); - elog(ERROR, "Cannot check if amcheck is installed in database %s: %s", - dbname, PQerrorMessage(db_conn)); - } - - if (PQntuples(res) < 1) - { - elog(WARNING, "Extension 'amcheck' or 'amcheck_next' are " - "not installed in database %s", dbname); - return NULL; - } - - amcheck_extname = pgut_malloc(strlen(PQgetvalue(res, 0, 0)) + 1); - strcpy(amcheck_extname, PQgetvalue(res, 0, 0)); - amcheck_nspname = pgut_malloc(strlen(PQgetvalue(res, 0, 1)) + 1); - strcpy(amcheck_nspname, PQgetvalue(res, 0, 1)); - amcheck_extversion = pgut_malloc(strlen(PQgetvalue(res, 0, 2)) + 1); - strcpy(amcheck_extversion, PQgetvalue(res, 0, 2)); - PQclear(res); - - /* heapallindexed_is_supported is database specific */ - /* TODO this is wrong check, heapallindexed supported also in 1.1.1, 1.2 and 1.2.1... */ - if (strcmp(amcheck_extversion, "1.0") != 0 && - strcmp(amcheck_extversion, "1") != 0) - heapallindexed_is_supported = true; - - elog(INFO, "Amchecking database '%s' using extension '%s' " - "version %s from schema '%s'", - dbname, amcheck_extname, - amcheck_extversion, amcheck_nspname); - - if (!heapallindexed_is_supported && heapallindexed) - elog(WARNING, "Extension '%s' version %s in schema '%s'" - "do not support 'heapallindexed' option", - amcheck_extname, amcheck_extversion, - amcheck_nspname); - -#ifndef PGPRO_EE - /* - * Will support when the vanilla patch will commited https://commitfest.postgresql.org/32/2976/ - */ - checkunique_is_supported = false; -#else - /* - * Check bt_index_check function signature to determine support of checkunique parameter - * This can't be exactly checked by checking extension version, - * For example, 1.1.1 and 1.2.1 supports this parameter, but 1.2 doesn't (PGPROEE-12.4.1) - */ - res = pgut_execute(db_conn, "SELECT " - " oid " - "FROM pg_catalog.pg_proc " - "WHERE " - " pronamespace = $1::regnamespace " - "AND proname = 'bt_index_check' " - "AND 'checkunique' = ANY(proargnames) " - "AND (pg_catalog.string_to_array(proargtypes::text, ' ')::regtype[])[pg_catalog.array_position(proargnames, 'checkunique')] = 'bool'::regtype", - 1, (const char **) &amcheck_nspname); - - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - PQclear(res); - elog(ERROR, "Cannot check 'checkunique' option is supported in bt_index_check function %s: %s", - dbname, PQerrorMessage(db_conn)); - } - - checkunique_is_supported = PQntuples(res) >= 1; - PQclear(res); -#endif - - if (!checkunique_is_supported && checkunique) - elog(WARNING, "Extension '%s' version %s in schema '%s' " - "do not support 'checkunique' parameter", - amcheck_extname, amcheck_extversion, - amcheck_nspname); - - /* - * In order to avoid duplicates, select global indexes - * (tablespace pg_global with oid 1664) only once. - * - * select only persistent btree indexes. - */ - if (first_db_with_amcheck) - { - - res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname, nmspc.nspname " - "FROM pg_catalog.pg_index idx " - "LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid " - "LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid " - "LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid " - "WHERE am.amname='btree' " - "AND cls.relpersistence != 't' " - "AND cls.relkind != 'I' " - "ORDER BY nmspc.nspname DESC", - 0, NULL); - } - else - { - - res = pgut_execute(db_conn, "SELECT cls.oid, cls.relname, nmspc.nspname " - "FROM pg_catalog.pg_index idx " - "LEFT JOIN pg_catalog.pg_class cls ON idx.indexrelid=cls.oid " - "LEFT JOIN pg_catalog.pg_namespace nmspc ON cls.relnamespace=nmspc.oid " - "LEFT JOIN pg_catalog.pg_am am ON cls.relam=am.oid " - "WHERE am.amname='btree' " - "AND cls.relpersistence != 't' " - "AND cls.relkind != 'I' " - "AND (cls.reltablespace IN " - "(SELECT oid from pg_catalog.pg_tablespace where spcname <> 'pg_global') " - "OR cls.reltablespace = 0) " - "ORDER BY nmspc.nspname DESC", - 0, NULL); - } - - /* add info needed to check indexes into index_list */ - for (i = 0; i < PQntuples(res); i++) - { - pg_indexEntry *ind = (pg_indexEntry *) pgut_malloc(sizeof(pg_indexEntry)); - char *name = NULL; - char *namespace = NULL; - - /* index oid */ - ind->indexrelid = atoll(PQgetvalue(res, i, 0)); - - /* index relname */ - name = PQgetvalue(res, i, 1); - ind->name = pgut_malloc(strlen(name) + 1); - strcpy(ind->name, name); /* enough buffer size guaranteed */ - - /* index namespace */ - namespace = PQgetvalue(res, i, 2); - ind->namespace = pgut_malloc(strlen(namespace) + 1); - strcpy(ind->namespace, namespace); /* enough buffer size guaranteed */ - - ind->heapallindexed_is_supported = heapallindexed_is_supported; - ind->checkunique_is_supported = checkunique_is_supported; - ind->amcheck_nspname = pgut_malloc(strlen(amcheck_nspname) + 1); - strcpy(ind->amcheck_nspname, amcheck_nspname); - pg_atomic_clear_flag(&ind->lock); - - if (index_list == NULL) - index_list = parray_new(); - - parray_append(index_list, ind); - } - - PQclear(res); - free(amcheck_extversion); - free(amcheck_nspname); - free(amcheck_extname); - - return index_list; -} - -/* check one index. Return true if everything is ok, false otherwise. */ -static bool -amcheck_one_index(check_indexes_arg *arguments, - pg_indexEntry *ind) -{ - PGresult *res; - char *params[3]; - static const char *queries[] = { - "SELECT %s.bt_index_check(index => $1)", - "SELECT %s.bt_index_check(index => $1, heapallindexed => $2)", - "SELECT %s.bt_index_check(index => $1, heapallindexed => $2, checkunique => $3)", - }; - int params_count; - char *query = NULL; - - if (interrupted) - elog(ERROR, "Interrupted"); - -#define INDEXRELID 0 -#define HEAPALLINDEXED 1 -#define CHECKUNIQUE 2 - /* first argument is index oid */ - params[INDEXRELID] = palloc(64); - sprintf(params[INDEXRELID], "%u", ind->indexrelid); - /* second argument is heapallindexed */ - params[HEAPALLINDEXED] = heapallindexed ? "true" : "false"; - /* third optional argument is checkunique */ - params[CHECKUNIQUE] = checkunique ? "true" : "false"; -#undef CHECKUNIQUE -#undef HEAPALLINDEXED - - params_count = ind->checkunique_is_supported ? - 3 : - ( ind->heapallindexed_is_supported ? 2 : 1 ); - - /* - * Prepare query text with schema name - * +1 for \0 and -2 for %s - */ - query = palloc(strlen(ind->amcheck_nspname) + strlen(queries[params_count - 1]) + 1 - 2); - sprintf(query, queries[params_count - 1], ind->amcheck_nspname); - - res = pgut_execute_parallel(arguments->conn_arg.conn, - arguments->conn_arg.cancel_conn, - query, params_count, (const char **)params, true, true, true); - - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - elog(WARNING, "Thread [%d]. Amcheck failed in database '%s' for index: '%s.%s': %s", - arguments->thread_num, arguments->conn_opt.pgdatabase, - ind->namespace, ind->name, PQresultErrorMessage(res)); - - pfree(params[INDEXRELID]); - pfree(query); - PQclear(res); - return false; - } - else - elog(LOG, "Thread [%d]. Amcheck succeeded in database '%s' for index: '%s.%s'", - arguments->thread_num, - arguments->conn_opt.pgdatabase, ind->namespace, ind->name); - - pfree(params[INDEXRELID]); -#undef INDEXRELID - pfree(query); - PQclear(res); - return true; -} - -/* - * Entry point of checkdb --amcheck. - * - * Connect to all databases in the cluster - * and get list of persistent indexes, - * then run parallel threads to perform bt_index_check() - * for all indexes from the list. - * - * If amcheck extension is not installed in the database, - * skip this database and report it via warning message. - */ -static void -do_amcheck(ConnectionOptions conn_opt, PGconn *conn) -{ - int i; - /* arrays with meta info for multi threaded amcheck */ - pthread_t *threads; - check_indexes_arg *threads_args; - bool check_isok = true; - PGresult *res_db; - int n_databases = 0; - bool first_db_with_amcheck = true; - bool db_skipped = false; - - elog(INFO, "Start amchecking PostgreSQL instance"); - - res_db = pgut_execute(conn, - "SELECT datname, oid, dattablespace " - "FROM pg_catalog.pg_database " - "WHERE datname NOT IN ('template0'::name, 'template1'::name)", - 0, NULL); - - /* we don't need this connection anymore */ - if (conn) - pgut_disconnect(conn); - - n_databases = PQntuples(res_db); - - /* For each database check indexes. In parallel. */ - for(i = 0; i < n_databases; i++) - { - int j; - const char *dbname; - PGconn *db_conn = NULL; - parray *index_list = NULL; - - dbname = PQgetvalue(res_db, i, 0); - db_conn = pgut_connect(conn_opt.pghost, conn_opt.pgport, - dbname, conn_opt.pguser); - - index_list = get_index_list(dbname, first_db_with_amcheck, - db_conn); - - /* we don't need this connection anymore */ - if (db_conn) - pgut_disconnect(db_conn); - - if (index_list == NULL) - { - db_skipped = true; - continue; - } - - first_db_with_amcheck = false; - - /* init thread args with own index lists */ - threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - threads_args = (check_indexes_arg *) palloc(sizeof(check_indexes_arg)*num_threads); - - for (j = 0; j < num_threads; j++) - { - check_indexes_arg *arg = &(threads_args[j]); - - arg->index_list = index_list; - arg->conn_arg.conn = NULL; - arg->conn_arg.cancel_conn = NULL; - - arg->conn_opt.pghost = conn_opt.pghost; - arg->conn_opt.pgport = conn_opt.pgport; - arg->conn_opt.pgdatabase = dbname; - arg->conn_opt.pguser = conn_opt.pguser; - - arg->thread_num = j + 1; - /* By default there are some error */ - arg->ret = 1; - } - - /* Run threads */ - for (j = 0; j < num_threads; j++) - { - check_indexes_arg *arg = &(threads_args[j]); - elog(VERBOSE, "Start thread num: %i", j); - pthread_create(&threads[j], NULL, check_indexes, arg); - } - - /* Wait threads */ - for (j = 0; j < num_threads; j++) - { - pthread_join(threads[j], NULL); - if (threads_args[j].ret > 0) - check_isok = false; - } - - if (check_isok) - elog(INFO, "Amcheck succeeded for database '%s'", dbname); - else - elog(WARNING, "Amcheck failed for database '%s'", dbname); - - parray_walk(index_list, pg_indexEntry_free); - parray_free(index_list); - - if (interrupted) - break; - } - - /* cleanup */ - PQclear(res_db); - - /* Inform user about amcheck results */ - if (interrupted) - elog(ERROR, "checkdb --amcheck is interrupted."); - - if (check_isok) - { - elog(INFO, "checkdb --amcheck finished successfully. " - "All checked indexes are valid."); - - if (db_skipped) - elog(ERROR, "Some databases were not amchecked."); - else - elog(INFO, "All databases were amchecked."); - } - else - elog(ERROR, "checkdb --amcheck finished with failure. " - "Not all checked indexes are valid. %s", - db_skipped?"Some databases were not amchecked.": - "All databases were amchecked."); -} - -/* Entry point of pg_probackup CHECKDB subcommand */ -void -do_checkdb(bool need_amcheck, - ConnectionOptions conn_opt, char *pgdata) -{ - PGNodeInfo nodeInfo; - PGconn *cur_conn; - - /* Initialize PGInfonode */ - pgNodeInit(&nodeInfo); - - if (skip_block_validation && !need_amcheck) - elog(ERROR, "Option '--skip-block-validation' must be used with '--amcheck' option"); - - if (!skip_block_validation) - { - if (!pgdata) - elog(ERROR, "Required parameter not specified: PGDATA " - "(-D, --pgdata)"); - - /* get node info */ - cur_conn = pgdata_basic_setup(conn_opt, &nodeInfo); - - /* ensure that conn credentials and pgdata are consistent */ - check_system_identifiers(cur_conn, pgdata); - - /* - * we don't need this connection anymore. - * block validation can last long time, - * so we don't hold the connection open, - * rather open new connection for amcheck - */ - if (cur_conn) - pgut_disconnect(cur_conn); - - do_block_validation(pgdata, nodeInfo.checksum_version); - } - - if (need_amcheck) - { - cur_conn = pgdata_basic_setup(conn_opt, &nodeInfo); - do_amcheck(conn_opt, cur_conn); - } -} diff --git a/src/configure.c b/src/configure.c deleted file mode 100644 index f7befb0c5..000000000 --- a/src/configure.c +++ /dev/null @@ -1,806 +0,0 @@ -/*------------------------------------------------------------------------- - * - * configure.c: - manage backup catalog. - * - * Copyright (c) 2017-2019, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -#include - -#include "utils/configuration.h" -#include "utils/json.h" - - -static void assign_log_level_console(ConfigOption *opt, const char *arg); -static void assign_log_level_file(ConfigOption *opt, const char *arg); -static void assign_log_format_console(ConfigOption *opt, const char *arg); -static void assign_log_format_file(ConfigOption *opt, const char *arg); -static void assign_compress_alg(ConfigOption *opt, const char *arg); - -static char *get_log_level_console(ConfigOption *opt); -static char *get_log_level_file(ConfigOption *opt); -static char *get_log_format_console(ConfigOption *opt); -static char *get_log_format_file(ConfigOption *opt); -static char *get_compress_alg(ConfigOption *opt); - -static void show_configure_start(void); -static void show_configure_end(void); - -static void show_configure_plain(ConfigOption *opt); -static void show_configure_json(ConfigOption *opt); - -#define RETENTION_REDUNDANCY_DEFAULT 0 -#define RETENTION_WINDOW_DEFAULT 0 - -#define OPTION_INSTANCE_GROUP "Backup instance information" -#define OPTION_CONN_GROUP "Connection parameters" -#define OPTION_REPLICA_GROUP "Replica parameters" -#define OPTION_ARCHIVE_GROUP "Archive parameters" -#define OPTION_LOG_GROUP "Logging parameters" -#define OPTION_RETENTION_GROUP "Retention parameters" -#define OPTION_COMPRESS_GROUP "Compression parameters" -#define OPTION_REMOTE_GROUP "Remote access parameters" - -/* - * Short name should be non-printable ASCII character. - */ -ConfigOption instance_options[] = -{ - /* Instance options */ - { - 's', 'D', "pgdata", - &instance_config.pgdata, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_INSTANCE_GROUP, 0, option_get_value - }, - { - 'U', 200, "system-identifier", - &instance_config.system_identifier, SOURCE_FILE_STRICT, 0, - OPTION_INSTANCE_GROUP, 0, option_get_value - }, -#if PG_VERSION_NUM >= 110000 - { - 'u', 201, "xlog-seg-size", - &instance_config.xlog_seg_size, SOURCE_FILE_STRICT, 0, - OPTION_INSTANCE_GROUP, 0, option_get_value - }, -#endif - { - 's', 'E', "external-dirs", - &instance_config.external_dir_str, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_INSTANCE_GROUP, 0, option_get_value - }, - /* Connection options */ - { - 's', 'd', "pgdatabase", - &instance_config.conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_CONN_GROUP, 0, option_get_value - }, - { - 's', 'h', "pghost", - &instance_config.conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_CONN_GROUP, 0, option_get_value - }, - { - 's', 'p', "pgport", - &instance_config.conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_CONN_GROUP, 0, option_get_value - }, - { - 's', 'U', "pguser", - &instance_config.conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_CONN_GROUP, 0, option_get_value - }, - /* Replica options */ - { - 's', 202, "master-db", - &instance_config.master_conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, 0, option_get_value - }, - { - 's', 203, "master-host", - &instance_config.master_conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, 0, option_get_value - }, - { - 's', 204, "master-port", - &instance_config.master_conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, 0, option_get_value - }, - { - 's', 205, "master-user", - &instance_config.master_conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, 0, option_get_value - }, - { - 'u', 206, "replica-timeout", - &instance_config.replica_timeout, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, OPTION_UNIT_S, option_get_value - }, - /* Archive options */ - { - 'u', 207, "archive-timeout", - &instance_config.archive_timeout, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_ARCHIVE_GROUP, OPTION_UNIT_S, option_get_value - }, - { - 's', 208, "archive-host", - &instance_config.archive.host, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_ARCHIVE_GROUP, 0, option_get_value - }, - { - 's', 209, "archive-port", - &instance_config.archive.port, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_ARCHIVE_GROUP, 0, option_get_value - }, - { - 's', 210, "archive-user", - &instance_config.archive.user, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_ARCHIVE_GROUP, 0, option_get_value - }, - { - 's', 211, "restore-command", - &instance_config.restore_command, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_ARCHIVE_GROUP, 0, option_get_value - }, - /* Logging options */ - { - 'f', 212, "log-level-console", - assign_log_level_console, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_LOG_GROUP, 0, get_log_level_console - }, - { - 'f', 213, "log-level-file", - assign_log_level_file, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_LOG_GROUP, 0, get_log_level_file - }, - { - 'f', 214, "log-format-console", - assign_log_format_console, SOURCE_CMD_STRICT, SOURCE_DEFAULT, - OPTION_LOG_GROUP, 0, get_log_format_console - }, - { - 'f', 215, "log-format-file", - assign_log_format_file, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_LOG_GROUP, 0, get_log_format_file - }, - { - 's', 216, "log-filename", - &instance_config.logger.log_filename, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_LOG_GROUP, 0, option_get_value - }, - { - 's', 217, "error-log-filename", - &instance_config.logger.error_log_filename, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_LOG_GROUP, 0, option_get_value - }, - { - 's', 218, "log-directory", - &instance_config.logger.log_directory, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_LOG_GROUP, 0, option_get_value - }, - { - 'U', 219, "log-rotation-size", - &instance_config.logger.log_rotation_size, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_LOG_GROUP, OPTION_UNIT_KB, option_get_value - }, - { - 'U', 220, "log-rotation-age", - &instance_config.logger.log_rotation_age, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_LOG_GROUP, OPTION_UNIT_MS, option_get_value - }, - /* Retention options */ - { - 'u', 221, "retention-redundancy", - &instance_config.retention_redundancy, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_RETENTION_GROUP, 0, option_get_value - }, - { - 'u', 222, "retention-window", - &instance_config.retention_window, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_RETENTION_GROUP, 0, option_get_value - }, - { - 'u', 223, "wal-depth", - &instance_config.wal_depth, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_RETENTION_GROUP, 0, option_get_value - }, - /* Compression options */ - { - 'f', 224, "compress-algorithm", - assign_compress_alg, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_COMPRESS_GROUP, 0, get_compress_alg - }, - { - 'u', 225, "compress-level", - &instance_config.compress_level, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_COMPRESS_GROUP, 0, option_get_value - }, - /* Remote backup options */ - { - 's', 226, "remote-proto", - &instance_config.remote.proto, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REMOTE_GROUP, 0, option_get_value - }, - { - 's', 227, "remote-host", - &instance_config.remote.host, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REMOTE_GROUP, 0, option_get_value - }, - { - 's', 228, "remote-port", - &instance_config.remote.port, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REMOTE_GROUP, 0, option_get_value - }, - { - 's', 229, "remote-path", - &instance_config.remote.path, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REMOTE_GROUP, 0, option_get_value - }, - { - 's', 230, "remote-user", - &instance_config.remote.user, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REMOTE_GROUP, 0, option_get_value - }, - { - 's', 231, "ssh-options", - &instance_config.remote.ssh_options, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REMOTE_GROUP, 0, option_get_value - }, - { - 's', 232, "ssh-config", - &instance_config.remote.ssh_config, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REMOTE_GROUP, 0, option_get_value - }, - { 0 } -}; - -/* An instance configuration with default options */ -InstanceConfig instance_config; - -static PQExpBufferData show_buf; -static int32 json_level = 0; -static const char *current_group = NULL; - -/* - * Show configure options including default values. - */ -void -do_show_config(void) -{ - int i; - - show_configure_start(); - - for (i = 0; instance_options[i].type; i++) - { - if (show_format == SHOW_PLAIN) - show_configure_plain(&instance_options[i]); - else - show_configure_json(&instance_options[i]); - } - - show_configure_end(); -} - -/* - * Save configure options into BACKUP_CATALOG_CONF_FILE. Do not save default - * values into the file. - */ -void -do_set_config(InstanceState *instanceState, bool missing_ok) -{ - char path_temp[MAXPGPATH]; - FILE *fp; - int i; - - snprintf(path_temp, sizeof(path_temp), "%s.tmp", instanceState->instance_config_path); - - if (!missing_ok && !fileExists(instanceState->instance_config_path, FIO_LOCAL_HOST)) - elog(ERROR, "Configuration file \"%s\" doesn't exist", instanceState->instance_config_path); - - fp = fopen(path_temp, "wt"); - if (fp == NULL) - elog(ERROR, "Cannot create configuration file \"%s\": %s", - BACKUP_CATALOG_CONF_FILE, strerror(errno)); - - current_group = NULL; - - for (i = 0; instance_options[i].type; i++) - { - int rc = 0; - ConfigOption *opt = &instance_options[i]; - char *value; - - /* Save only options from command line */ - if (opt->source != SOURCE_CMD && - /* ...or options from the previous configure file */ - opt->source != SOURCE_FILE && opt->source != SOURCE_FILE_STRICT) - continue; - - value = opt->get_value(opt); - if (value == NULL) - continue; - - if (current_group == NULL || strcmp(opt->group, current_group) != 0) - { - current_group = opt->group; - fprintf(fp, "# %s\n", current_group); - } - - if (strchr(value, ' ')) - rc = fprintf(fp, "%s = '%s'\n", opt->lname, value); - else - rc = fprintf(fp, "%s = %s\n", opt->lname, value); - - if (rc < 0) - elog(ERROR, "Cannot write to configuration file: \"%s\"", path_temp); - - pfree(value); - } - - if (ferror(fp) || fflush(fp)) - elog(ERROR, "Cannot write to configuration file: \"%s\"", path_temp); - - if (fclose(fp)) - elog(ERROR, "Cannot close configuration file: \"%s\"", path_temp); - - if (fio_sync(path_temp, FIO_LOCAL_HOST) != 0) - elog(ERROR, "Failed to sync temp configuration file \"%s\": %s", - path_temp, strerror(errno)); - - if (rename(path_temp, instanceState->instance_config_path) < 0) - { - int errno_temp = errno; - unlink(path_temp); - elog(ERROR, "Cannot rename configuration file \"%s\" to \"%s\": %s", - path_temp, instanceState->instance_config_path, strerror(errno_temp)); - } -} - -void -init_config(InstanceConfig *config, const char *instance_name) -{ - MemSet(config, 0, sizeof(InstanceConfig)); - - /* - * Starting from PostgreSQL 11 WAL segment size may vary. Prior to - * PostgreSQL 10 xlog_seg_size is equal to XLOG_SEG_SIZE. - */ -#if PG_VERSION_NUM >= 110000 - config->xlog_seg_size = 0; -#else - config->xlog_seg_size = XLOG_SEG_SIZE; -#endif - - config->replica_timeout = REPLICA_TIMEOUT_DEFAULT; - - config->archive_timeout = ARCHIVE_TIMEOUT_DEFAULT; - - /* Copy logger defaults */ - config->logger = logger_config; - - config->retention_redundancy = RETENTION_REDUNDANCY_DEFAULT; - config->retention_window = RETENTION_WINDOW_DEFAULT; - config->wal_depth = 0; - - config->compress_alg = COMPRESS_ALG_DEFAULT; - config->compress_level = COMPRESS_LEVEL_DEFAULT; - - config->remote.proto = (char*)"ssh"; -} - -/* - * read instance config from file - */ -InstanceConfig * -readInstanceConfigFile(InstanceState *instanceState) -{ - InstanceConfig *instance = pgut_new(InstanceConfig); - char *log_level_console = NULL; - char *log_level_file = NULL; - char *log_format_console = NULL; - char *log_format_file = NULL; - char *compress_alg = NULL; - int parsed_options; - - ConfigOption instance_options[] = - { - /* Instance options */ - { - 's', 'D', "pgdata", - &instance->pgdata, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_INSTANCE_GROUP, 0, option_get_value - }, - { - 'U', 200, "system-identifier", - &instance->system_identifier, SOURCE_FILE_STRICT, 0, - OPTION_INSTANCE_GROUP, 0, option_get_value - }, - #if PG_VERSION_NUM >= 110000 - { - 'u', 201, "xlog-seg-size", - &instance->xlog_seg_size, SOURCE_FILE_STRICT, 0, - OPTION_INSTANCE_GROUP, 0, option_get_value - }, - #endif - { - 's', 'E', "external-dirs", - &instance->external_dir_str, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_INSTANCE_GROUP, 0, option_get_value - }, - /* Connection options */ - { - 's', 'd', "pgdatabase", - &instance->conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_CONN_GROUP, 0, option_get_value - }, - { - 's', 'h', "pghost", - &instance->conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_CONN_GROUP, 0, option_get_value - }, - { - 's', 'p', "pgport", - &instance->conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_CONN_GROUP, 0, option_get_value - }, - { - 's', 'U', "pguser", - &instance->conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_CONN_GROUP, 0, option_get_value - }, - /* Replica options */ - { - 's', 202, "master-db", - &instance->master_conn_opt.pgdatabase, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, 0, option_get_value - }, - { - 's', 203, "master-host", - &instance->master_conn_opt.pghost, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, 0, option_get_value - }, - { - 's', 204, "master-port", - &instance->master_conn_opt.pgport, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, 0, option_get_value - }, - { - 's', 205, "master-user", - &instance->master_conn_opt.pguser, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, 0, option_get_value - }, - { - 'u', 206, "replica-timeout", - &instance->replica_timeout, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REPLICA_GROUP, OPTION_UNIT_S, option_get_value - }, - /* Archive options */ - { - 'u', 207, "archive-timeout", - &instance->archive_timeout, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_ARCHIVE_GROUP, OPTION_UNIT_S, option_get_value - }, - { - 's', 208, "archive-host", - &instance_config.archive.host, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_ARCHIVE_GROUP, 0, option_get_value - }, - { - 's', 209, "archive-port", - &instance_config.archive.port, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_ARCHIVE_GROUP, 0, option_get_value - }, - { - 's', 210, "archive-user", - &instance_config.archive.user, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_ARCHIVE_GROUP, 0, option_get_value - }, - { - 's', 211, "restore-command", - &instance->restore_command, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_ARCHIVE_GROUP, 0, option_get_value - }, - - /* Instance options */ - { - 's', 'D', "pgdata", - &instance->pgdata, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_INSTANCE_GROUP, 0, option_get_value - }, - - /* Logging options */ - { - 's', 212, "log-level-console", - &log_level_console, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_LOG_GROUP, 0, option_get_value - }, - { - 's', 213, "log-level-file", - &log_level_file, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_LOG_GROUP, 0, option_get_value - }, - { - 's', 214, "log-format-console", - &log_format_console, SOURCE_CMD_STRICT, SOURCE_DEFAULT, - OPTION_LOG_GROUP, 0, option_get_value - }, - { - 's', 215, "log-format-file", - &log_format_file, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_LOG_GROUP, 0, option_get_value - }, - { - 's', 216, "log-filename", - &instance->logger.log_filename, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_LOG_GROUP, 0, option_get_value - }, - { - 's', 217, "error-log-filename", - &instance->logger.error_log_filename, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_LOG_GROUP, 0, option_get_value - }, - { - 's', 218, "log-directory", - &instance->logger.log_directory, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_LOG_GROUP, 0, option_get_value - }, - { - 'U', 219, "log-rotation-size", - &instance->logger.log_rotation_size, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_LOG_GROUP, OPTION_UNIT_KB, option_get_value - }, - { - 'U', 220, "log-rotation-age", - &instance->logger.log_rotation_age, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_LOG_GROUP, OPTION_UNIT_MS, option_get_value - }, - /* Retention options */ - { - 'u', 221, "retention-redundancy", - &instance->retention_redundancy, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_RETENTION_GROUP, 0, option_get_value - }, - { - 'u', 222, "retention-window", - &instance->retention_window, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_RETENTION_GROUP, 0, option_get_value - }, - { - 'u', 223, "wal-depth", - &instance->wal_depth, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_RETENTION_GROUP, 0, option_get_value - }, - /* Compression options */ - { - 's', 224, "compress-algorithm", - &compress_alg, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_LOG_GROUP, 0, option_get_value - }, - { - 'u', 225, "compress-level", - &instance->compress_level, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_COMPRESS_GROUP, 0, option_get_value - }, - /* Remote backup options */ - { - 's', 226, "remote-proto", - &instance->remote.proto, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REMOTE_GROUP, 0, option_get_value - }, - { - 's', 227, "remote-host", - &instance->remote.host, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REMOTE_GROUP, 0, option_get_value - }, - { - 's', 228, "remote-port", - &instance->remote.port, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REMOTE_GROUP, 0, option_get_value - }, - { - 's', 229, "remote-path", - &instance->remote.path, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REMOTE_GROUP, 0, option_get_value - }, - { - 's', 230, "remote-user", - &instance->remote.user, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REMOTE_GROUP, 0, option_get_value - }, - { - 's', 231, "ssh-options", - &instance->remote.ssh_options, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REMOTE_GROUP, 0, option_get_value - }, - { - 's', 232, "ssh-config", - &instance->remote.ssh_config, SOURCE_CMD, SOURCE_DEFAULT, - OPTION_REMOTE_GROUP, 0, option_get_value - }, - { 0 } - }; - - - init_config(instance, instanceState->instance_name); - - if (fio_access(instanceState->instance_config_path, F_OK, FIO_BACKUP_HOST) != 0) - { - elog(WARNING, "Control file \"%s\" doesn't exist", instanceState->instance_config_path); - pfree(instance); - return NULL; - } - - parsed_options = config_read_opt(instanceState->instance_config_path, - instance_options, WARNING, true, true); - - if (parsed_options == 0) - { - elog(WARNING, "Control file \"%s\" is empty", instanceState->instance_config_path); - pfree(instance); - return NULL; - } - - if (log_level_console) - instance->logger.log_level_console = parse_log_level(log_level_console); - - if (log_level_file) - instance->logger.log_level_file = parse_log_level(log_level_file); - - if (log_format_console) - instance->logger.log_format_console = parse_log_format(log_format_console); - - if (log_format_file) - instance->logger.log_format_file = parse_log_format(log_format_file); - - if (compress_alg) - instance->compress_alg = parse_compress_alg(compress_alg); - -#if PG_VERSION_NUM >= 110000 - /* If for some reason xlog-seg-size is missing, then set it to 16MB */ - if (!instance->xlog_seg_size) - instance->xlog_seg_size = DEFAULT_XLOG_SEG_SIZE; -#endif - - return instance; -} - -static void -assign_log_level_console(ConfigOption *opt, const char *arg) -{ - instance_config.logger.log_level_console = parse_log_level(arg); -} - -static void -assign_log_level_file(ConfigOption *opt, const char *arg) -{ - instance_config.logger.log_level_file = parse_log_level(arg); -} - -static void -assign_log_format_console(ConfigOption *opt, const char *arg) -{ - instance_config.logger.log_format_console = parse_log_format(arg); -} - -static void -assign_log_format_file(ConfigOption *opt, const char *arg) -{ - instance_config.logger.log_format_file = parse_log_format(arg); -} - -static void -assign_compress_alg(ConfigOption *opt, const char *arg) -{ - instance_config.compress_alg = parse_compress_alg(arg); -} - -static char * -get_log_level_console(ConfigOption *opt) -{ - return pstrdup(deparse_log_level(instance_config.logger.log_level_console)); -} - -static char * -get_log_level_file(ConfigOption *opt) -{ - return pstrdup(deparse_log_level(instance_config.logger.log_level_file)); -} - -static char * -get_log_format_console(ConfigOption *opt) -{ - return pstrdup(deparse_log_format(instance_config.logger.log_format_console)); -} - -static char * -get_log_format_file(ConfigOption *opt) -{ - return pstrdup(deparse_log_format(instance_config.logger.log_format_file)); -} - -static char * -get_compress_alg(ConfigOption *opt) -{ - return pstrdup(deparse_compress_alg(instance_config.compress_alg)); -} - -/* - * Initialize configure visualization. - */ -static void -show_configure_start(void) -{ - initPQExpBuffer(&show_buf); - - if (show_format == SHOW_PLAIN) - current_group = NULL; - else - { - json_level = 0; - json_add(&show_buf, JT_BEGIN_OBJECT, &json_level); - } -} - -/* - * Finalize configure visualization. - */ -static void -show_configure_end(void) -{ - if (show_format == SHOW_PLAIN) - current_group = NULL; - else - { - json_add(&show_buf, JT_END_OBJECT, &json_level); - appendPQExpBufferChar(&show_buf, '\n'); - } - - fputs(show_buf.data, stdout); - termPQExpBuffer(&show_buf); -} - -/* - * Plain output. - */ - -static void -show_configure_plain(ConfigOption *opt) -{ - char *value; - - value = opt->get_value(opt); - if (value == NULL) - return; - - if (current_group == NULL || strcmp(opt->group, current_group) != 0) - { - current_group = opt->group; - appendPQExpBuffer(&show_buf, "# %s\n", current_group); - } - - appendPQExpBuffer(&show_buf, "%s = %s\n", opt->lname, value); - pfree(value); -} - -/* - * Json output. - */ - -static void -show_configure_json(ConfigOption *opt) -{ - char *value; - - value = opt->get_value(opt); - if (value == NULL) - return; - - json_add_value(&show_buf, opt->lname, value, json_level, - true); - pfree(value); -} diff --git a/src/data.c b/src/data.c deleted file mode 100644 index a287218ea..000000000 --- a/src/data.c +++ /dev/null @@ -1,2524 +0,0 @@ -/*------------------------------------------------------------------------- - * - * data.c: utils to parse and backup data pages - * - * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2022, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -#include "storage/checksum.h" -#include "storage/checksum_impl.h" -#include -#include "utils/file.h" - -#include -#include - -#ifdef HAVE_LIBZ -#include -#endif - -#include "utils/thread.h" - -/* Union to ease operations on relation pages */ -typedef struct DataPage -{ - BackupPageHeader bph; - char data[BLCKSZ]; -} DataPage; - -static bool get_page_header(FILE *in, const char *fullpath, BackupPageHeader *bph, - pg_crc32 *crc, bool use_crc32c); - -#ifdef HAVE_LIBZ -/* Implementation of zlib compression method */ -static int32 -zlib_compress(void *dst, size_t dst_size, void const *src, size_t src_size, - int level) -{ - uLongf compressed_size = dst_size; - int rc = compress2(dst, &compressed_size, src, src_size, - level); - - return rc == Z_OK ? compressed_size : rc; -} - -/* Implementation of zlib compression method */ -static int32 -zlib_decompress(void *dst, size_t dst_size, void const *src, size_t src_size) -{ - uLongf dest_len = dst_size; - int rc = uncompress(dst, &dest_len, src, src_size); - - return rc == Z_OK ? dest_len : rc; -} -#endif - -/* - * Compresses source into dest using algorithm. Returns the number of bytes - * written in the destination buffer, or -1 if compression fails. - */ -int32 -do_compress(void *dst, size_t dst_size, void const *src, size_t src_size, - CompressAlg alg, int level, const char **errormsg) -{ - switch (alg) - { - case NONE_COMPRESS: - case NOT_DEFINED_COMPRESS: - return -1; -#ifdef HAVE_LIBZ - case ZLIB_COMPRESS: - { - int32 ret; - ret = zlib_compress(dst, dst_size, src, src_size, level); - if (ret < Z_OK && errormsg) - *errormsg = zError(ret); - return ret; - } -#endif - case PGLZ_COMPRESS: - return pglz_compress(src, src_size, dst, PGLZ_strategy_always); - } - - return -1; -} - -/* - * Decompresses source into dest using algorithm. Returns the number of bytes - * decompressed in the destination buffer, or -1 if decompression fails. - */ -int32 -do_decompress(void *dst, size_t dst_size, void const *src, size_t src_size, - CompressAlg alg, const char **errormsg) -{ - switch (alg) - { - case NONE_COMPRESS: - case NOT_DEFINED_COMPRESS: - if (errormsg) - *errormsg = "Invalid compression algorithm"; - return -1; -#ifdef HAVE_LIBZ - case ZLIB_COMPRESS: - { - int32 ret; - ret = zlib_decompress(dst, dst_size, src, src_size); - if (ret < Z_OK && errormsg) - *errormsg = zError(ret); - return ret; - } -#endif - case PGLZ_COMPRESS: - -#if PG_VERSION_NUM >= 120000 - return pglz_decompress(src, src_size, dst, dst_size, true); -#else - return pglz_decompress(src, src_size, dst, dst_size); -#endif - } - - return -1; -} - -#define ZLIB_MAGIC 0x78 - -/* - * Before version 2.0.23 there was a bug in pro_backup that pages which compressed - * size is exactly the same as original size are not treated as compressed. - * This check tries to detect and decompress such pages. - * There is no 100% criteria to determine whether page is compressed or not. - * But at least we will do this check only for pages which will no pass validation step. - */ -static bool -page_may_be_compressed(Page page, CompressAlg alg, uint32 backup_version) -{ - PageHeader phdr; - - phdr = (PageHeader) page; - - /* First check if page header is valid (it seems to be fast enough check) */ - if (!(PageGetPageSize(page) == BLCKSZ && - // PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && - (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && - phdr->pd_lower >= SizeOfPageHeaderData && - phdr->pd_lower <= phdr->pd_upper && - phdr->pd_upper <= phdr->pd_special && - phdr->pd_special <= BLCKSZ && - phdr->pd_special == MAXALIGN(phdr->pd_special))) - { - /* ... end only if it is invalid, then do more checks */ - if (backup_version >= 20023) - { - /* Versions 2.0.23 and higher don't have such bug */ - return false; - } -#ifdef HAVE_LIBZ - /* For zlib we can check page magic: - * https://stackoverflow.com/questions/9050260/what-does-a-zlib-header-look-like - */ - if (alg == ZLIB_COMPRESS && *(char *)page != ZLIB_MAGIC) - { - return false; - } -#endif - /* otherwise let's try to decompress the page */ - return true; - } - return false; -} - -/* Verify page's header */ -bool -parse_page(Page page, XLogRecPtr *lsn) -{ - PageHeader phdr = (PageHeader) page; - - /* Get lsn from page header */ - *lsn = PageXLogRecPtrGet(phdr->pd_lsn); - - if (PageGetPageSize(page) == BLCKSZ && - // PageGetPageLayoutVersion(phdr) == PG_PAGE_LAYOUT_VERSION && - (phdr->pd_flags & ~PD_VALID_FLAG_BITS) == 0 && - phdr->pd_lower >= SizeOfPageHeaderData && - phdr->pd_lower <= phdr->pd_upper && - phdr->pd_upper <= phdr->pd_special && - phdr->pd_special <= BLCKSZ && - phdr->pd_special == MAXALIGN(phdr->pd_special)) - return true; - - return false; -} - -/* We know that header is invalid, store specific - * details in errormsg. - */ -void -get_header_errormsg(Page page, char **errormsg) -{ - PageHeader phdr = (PageHeader) page; - *errormsg = pgut_malloc(ERRMSG_MAX_LEN); - - if (PageGetPageSize(page) != BLCKSZ) - snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " - "page size %lu is not equal to block size %u", - PageGetPageSize(page), BLCKSZ); - - else if (phdr->pd_lower < SizeOfPageHeaderData) - snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " - "pd_lower %i is less than page header size %lu", - phdr->pd_lower, SizeOfPageHeaderData); - - else if (phdr->pd_lower > phdr->pd_upper) - snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " - "pd_lower %u is greater than pd_upper %u", - phdr->pd_lower, phdr->pd_upper); - - else if (phdr->pd_upper > phdr->pd_special) - snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " - "pd_upper %u is greater than pd_special %u", - phdr->pd_upper, phdr->pd_special); - - else if (phdr->pd_special > BLCKSZ) - snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " - "pd_special %u is greater than block size %u", - phdr->pd_special, BLCKSZ); - - else if (phdr->pd_special != MAXALIGN(phdr->pd_special)) - snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " - "pd_special %i is misaligned, expected %lu", - phdr->pd_special, MAXALIGN(phdr->pd_special)); - - else if (phdr->pd_flags & ~PD_VALID_FLAG_BITS) - snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid, " - "pd_flags mask contain illegal bits"); - - else - snprintf(*errormsg, ERRMSG_MAX_LEN, "page header invalid"); -} - -/* We know that checksumms are mismatched, store specific - * details in errormsg. - */ -void -get_checksum_errormsg(Page page, char **errormsg, BlockNumber absolute_blkno) -{ - PageHeader phdr = (PageHeader) page; - *errormsg = pgut_malloc(ERRMSG_MAX_LEN); - - snprintf(*errormsg, ERRMSG_MAX_LEN, - "page verification failed, " - "calculated checksum %u but expected %u", - phdr->pd_checksum, - pg_checksum_page(page, absolute_blkno)); -} - -/* - * Retrieves a page taking the backup mode into account - * and writes it into argument "page". Argument "page" - * should be a pointer to allocated BLCKSZ of bytes. - * - * Prints appropriate warnings/errors/etc into log. - * Returns: - * PageIsOk(0) if page was successfully retrieved - * PageIsTruncated(-1) if the page was truncated - * SkipCurrentPage(-2) if we need to skip this page, - * only used for DELTA and PTRACK backup - * PageIsCorrupted(-3) if the page checksum mismatch - * or header corruption, - * only used for checkdb - * TODO: probably we should always - * return it to the caller - */ -static int32 -prepare_page(pgFile *file, XLogRecPtr prev_backup_start_lsn, - BlockNumber blknum, FILE *in, - BackupMode backup_mode, - Page page, bool strict, - uint32 checksum_version, - const char *from_fullpath, - PageState *page_st) -{ - int try_again = PAGE_READ_ATTEMPTS; - bool page_is_valid = false; - BlockNumber absolute_blknum = file->segno * RELSEG_SIZE + blknum; - int rc = 0; - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during page reading"); - - /* - * Read the page and verify its header and checksum. - * Under high write load it's possible that we've read partly - * flushed page, so try several times before throwing an error. - */ - while (!page_is_valid && try_again--) - { - /* read the block */ - int read_len = fio_pread(in, page, blknum * BLCKSZ); - - /* The block could have been truncated. It is fine. */ - if (read_len == 0) - { - elog(VERBOSE, "Cannot read block %u of \"%s\": " - "block truncated", blknum, from_fullpath); - return PageIsTruncated; - } - else if (read_len < 0) - elog(ERROR, "Cannot read block %u of \"%s\": %s", - blknum, from_fullpath, strerror(errno)); - else if (read_len != BLCKSZ) - elog(WARNING, "Cannot read block %u of \"%s\": " - "read %i of %d, try again", - blknum, from_fullpath, read_len, BLCKSZ); - else - { - /* We have BLCKSZ of raw data, validate it */ - rc = validate_one_page(page, absolute_blknum, - InvalidXLogRecPtr, page_st, - checksum_version); - switch (rc) - { - case PAGE_IS_ZEROED: - elog(VERBOSE, "File: \"%s\" blknum %u, empty page", from_fullpath, blknum); - return PageIsOk; - - case PAGE_IS_VALID: - /* in DELTA or PTRACK modes we must compare lsn */ - if (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) - page_is_valid = true; - else - return PageIsOk; - break; - - case PAGE_HEADER_IS_INVALID: - elog(VERBOSE, "File: \"%s\" blknum %u have wrong page header, try again", - from_fullpath, blknum); - break; - - case PAGE_CHECKSUM_MISMATCH: - elog(VERBOSE, "File: \"%s\" blknum %u have wrong checksum, try again", - from_fullpath, blknum); - break; - default: - Assert(false); - } - } - /* avoid re-reading once buffered data, flushing on further attempts, see PBCKP-150 */ - fflush(in); - } - - /* - * If page is not valid after PAGE_READ_ATTEMPTS attempts to read it - * throw an error. - */ - if (!page_is_valid) - { - int elevel = ERROR; - char *errormsg = NULL; - - /* Get the details of corruption */ - if (rc == PAGE_HEADER_IS_INVALID) - get_header_errormsg(page, &errormsg); - else if (rc == PAGE_CHECKSUM_MISMATCH) - get_checksum_errormsg(page, &errormsg, - file->segno * RELSEG_SIZE + blknum); - - /* Error out in case of merge or backup without ptrack support; - * issue warning in case of checkdb or backup with ptrack support - */ - if (!strict) - elevel = WARNING; - - if (errormsg) - elog(elevel, "Corruption detected in file \"%s\", block %u: %s", - from_fullpath, blknum, errormsg); - else - elog(elevel, "Corruption detected in file \"%s\", block %u", - from_fullpath, blknum); - - pg_free(errormsg); - return PageIsCorrupted; - } - - /* Checkdb not going futher */ - if (!strict) - return PageIsOk; - - /* - * Skip page if page lsn is less than START_LSN of parent backup. - * Nullified pages must be copied by DELTA backup, just to be safe. - */ - if ((backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && - file->exists_in_prev && - page_st->lsn > 0 && - page_st->lsn < prev_backup_start_lsn) - { - elog(VERBOSE, "Skipping blknum %u in file: \"%s\", file->exists_in_prev: %s, page_st->lsn: %X/%X, prev_backup_start_lsn: %X/%X", - blknum, from_fullpath, - file->exists_in_prev ? "true" : "false", - (uint32) (page_st->lsn >> 32), (uint32) page_st->lsn, - (uint32) (prev_backup_start_lsn >> 32), (uint32) prev_backup_start_lsn); - return SkipCurrentPage; - } - - return PageIsOk; -} - -/* split this function in two: compress() and backup() */ -static int -compress_and_backup_page(pgFile *file, BlockNumber blknum, - FILE *in, FILE *out, pg_crc32 *crc, - int page_state, Page page, - CompressAlg calg, int clevel, - const char *from_fullpath, const char *to_fullpath) -{ - int compressed_size = 0; - size_t write_buffer_size = 0; - char write_buffer[BLCKSZ*2]; /* compressed page may require more space than uncompressed */ - BackupPageHeader* bph = (BackupPageHeader*)write_buffer; - const char *errormsg = NULL; - - /* Compress the page */ - compressed_size = do_compress(write_buffer + sizeof(BackupPageHeader), - sizeof(write_buffer) - sizeof(BackupPageHeader), - page, BLCKSZ, calg, clevel, - &errormsg); - /* Something went wrong and errormsg was assigned, throw a warning */ - if (compressed_size < 0 && errormsg != NULL) - elog(WARNING, "An error occured during compressing block %u of file \"%s\": %s", - blknum, from_fullpath, errormsg); - - file->compress_alg = calg; /* TODO: wtf? why here? */ - - /* compression didn`t worked */ - if (compressed_size <= 0 || compressed_size >= BLCKSZ) - { - /* Do not compress page */ - memcpy(write_buffer + sizeof(BackupPageHeader), page, BLCKSZ); - compressed_size = BLCKSZ; - } - bph->block = blknum; - bph->compressed_size = compressed_size; - write_buffer_size = compressed_size + sizeof(BackupPageHeader); - - /* Update CRC */ - COMP_FILE_CRC32(true, *crc, write_buffer, write_buffer_size); - - /* write data page */ - if (fio_fwrite(out, write_buffer, write_buffer_size) != write_buffer_size) - elog(ERROR, "File: \"%s\", cannot write at block %u: %s", - to_fullpath, blknum, strerror(errno)); - - file->write_size += write_buffer_size; - file->uncompressed_size += BLCKSZ; - - return compressed_size; -} - -/* Write page as-is. TODO: make it fastpath option in compress_and_backup_page() */ -static int -write_page(pgFile *file, FILE *out, Page page) -{ - /* write data page */ - if (fio_fwrite(out, page, BLCKSZ) != BLCKSZ) - return -1; - - file->write_size += BLCKSZ; - file->uncompressed_size += BLCKSZ; - - return BLCKSZ; -} - -/* - * Backup data file in the from_root directory to the to_root directory with - * same relative path. If prev_backup_start_lsn is not NULL, only pages with - * higher lsn will be copied. - * Not just copy file, but read it block by block (use bitmap in case of - * incremental backup), validate checksum, optionally compress and write to - * backup with special header. - */ -void -backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, - XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, - CompressAlg calg, int clevel, uint32 checksum_version, - HeaderMap *hdr_map, bool is_merge) -{ - int rc; - bool use_pagemap; - char *errmsg = NULL; - BlockNumber err_blknum = 0; - /* page headers */ - BackupPageHeader2 *headers = NULL; - - /* sanity */ - if (file->size % BLCKSZ != 0) - elog(WARNING, "File: \"%s\", invalid file size %zu", from_fullpath, file->size); - - /* - * Compute expected number of blocks in the file. - * NOTE This is a normal situation, if the file size has changed - * since the moment we computed it. - */ - file->n_blocks = file->size/BLCKSZ; - - /* - * Skip unchanged file only if it exists in previous backup. - * This way we can correctly handle null-sized files which are - * not tracked by pagemap and thus always marked as unchanged. - */ - if ((backup_mode == BACKUP_MODE_DIFF_PAGE || - backup_mode == BACKUP_MODE_DIFF_PTRACK) && - file->pagemap.bitmapsize == PageBitmapIsEmpty && - file->exists_in_prev && !file->pagemap_isabsent) - { - /* - * There are no changed blocks since last backup. We want to make - * incremental backup, so we should exit. - */ - file->write_size = BYTES_INVALID; - return; - } - - /* reset size summary */ - file->read_size = 0; - file->write_size = 0; - file->uncompressed_size = 0; - INIT_FILE_CRC32(true, file->crc); - - /* - * Read each page, verify checksum and write it to backup. - * If page map is empty or file is not present in previous backup - * backup all pages of the relation. - * - * In PTRACK 1.x there was a problem - * of data files with missing _ptrack map. - * Such files should be fully copied. - */ - - if (file->pagemap.bitmapsize == PageBitmapIsEmpty || - file->pagemap_isabsent || !file->exists_in_prev || - !file->pagemap.bitmap) - use_pagemap = false; - else - use_pagemap = true; - - /* Remote mode */ - if (fio_is_remote(FIO_DB_HOST)) - { - rc = fio_send_pages(to_fullpath, from_fullpath, file, - /* send prev backup START_LSN */ - (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && - file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, - calg, clevel, checksum_version, - /* send pagemap if any */ - use_pagemap, - /* variables for error reporting */ - &err_blknum, &errmsg, &headers); - } - else - { - /* TODO: stop handling errors internally */ - rc = send_pages(to_fullpath, from_fullpath, file, - /* send prev backup START_LSN */ - (backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && - file->exists_in_prev ? prev_backup_start_lsn : InvalidXLogRecPtr, - calg, clevel, checksum_version, use_pagemap, - &headers, backup_mode); - } - - /* check for errors */ - if (rc == FILE_MISSING) - { - elog(is_merge ? ERROR : LOG, "File not found: \"%s\"", from_fullpath); - file->write_size = FILE_NOT_FOUND; - goto cleanup; - } - - else if (rc == WRITE_FAILED) - elog(ERROR, "Cannot write block %u of \"%s\": %s", - err_blknum, to_fullpath, strerror(errno)); - - else if (rc == PAGE_CORRUPTION) - { - if (errmsg) - elog(ERROR, "Corruption detected in file \"%s\", block %u: %s", - from_fullpath, err_blknum, errmsg); - else - elog(ERROR, "Corruption detected in file \"%s\", block %u", - from_fullpath, err_blknum); - } - /* OPEN_FAILED and READ_FAILED */ - else if (rc == OPEN_FAILED) - { - if (errmsg) - elog(ERROR, "%s", errmsg); - else - elog(ERROR, "Cannot open file \"%s\"", from_fullpath); - } - else if (rc == READ_FAILED) - { - if (errmsg) - elog(ERROR, "%s", errmsg); - else - elog(ERROR, "Cannot read file \"%s\"", from_fullpath); - } - - file->read_size = rc * BLCKSZ; - - /* refresh n_blocks for FULL and DELTA */ - if (backup_mode == BACKUP_MODE_FULL || - backup_mode == BACKUP_MODE_DIFF_DELTA) - file->n_blocks = file->read_size / BLCKSZ; - - /* Determine that file didn`t changed in case of incremental backup */ - if (backup_mode != BACKUP_MODE_FULL && - file->exists_in_prev && - file->write_size == 0 && - file->n_blocks > 0) - { - file->write_size = BYTES_INVALID; - } - -cleanup: - - /* finish CRC calculation */ - FIN_FILE_CRC32(true, file->crc); - - /* dump page headers */ - write_page_headers(headers, file, hdr_map, is_merge); - - pg_free(errmsg); - pg_free(file->pagemap.bitmap); - pg_free(headers); -} - -/* - * Catchup data file in the from_root directory to the to_root directory with - * same relative path. If sync_lsn is not NULL, only pages with equal or - * higher lsn will be copied. - * Not just copy file, but read it block by block (use bitmap in case of - * incremental catchup), validate page checksum. - */ -void -catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, - XLogRecPtr sync_lsn, BackupMode backup_mode, - uint32 checksum_version, size_t prev_size) -{ - int rc; - bool use_pagemap; - char *errmsg = NULL; - BlockNumber err_blknum = 0; - - /* - * Compute expected number of blocks in the file. - * NOTE This is a normal situation, if the file size has changed - * since the moment we computed it. - */ - file->n_blocks = file->size/BLCKSZ; - - /* - * Skip unchanged file only if it exists in destination directory. - * This way we can correctly handle null-sized files which are - * not tracked by pagemap and thus always marked as unchanged. - */ - if (backup_mode == BACKUP_MODE_DIFF_PTRACK && - file->pagemap.bitmapsize == PageBitmapIsEmpty && - file->exists_in_prev && file->size == prev_size && !file->pagemap_isabsent) - { - /* - * There are none changed pages. - */ - file->write_size = BYTES_INVALID; - return; - } - - /* reset size summary */ - file->read_size = 0; - file->write_size = 0; - file->uncompressed_size = 0; - - /* - * If page map is empty or file is not present in destination directory, - * then copy backup all pages of the relation. - */ - - if (file->pagemap.bitmapsize == PageBitmapIsEmpty || - file->pagemap_isabsent || !file->exists_in_prev || - !file->pagemap.bitmap) - use_pagemap = false; - else - use_pagemap = true; - - if (use_pagemap) - elog(LOG, "Using pagemap for file \"%s\"", file->rel_path); - - /* Remote mode */ - if (fio_is_remote(FIO_DB_HOST)) - { - rc = fio_copy_pages(to_fullpath, from_fullpath, file, - /* send prev backup START_LSN */ - ((backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && - file->exists_in_prev) ? sync_lsn : InvalidXLogRecPtr, - NONE_COMPRESS, 1, checksum_version, - /* send pagemap if any */ - use_pagemap, - /* variables for error reporting */ - &err_blknum, &errmsg); - } - else - { - /* TODO: stop handling errors internally */ - rc = copy_pages(to_fullpath, from_fullpath, file, - /* send prev backup START_LSN */ - ((backup_mode == BACKUP_MODE_DIFF_DELTA || backup_mode == BACKUP_MODE_DIFF_PTRACK) && - file->exists_in_prev) ? sync_lsn : InvalidXLogRecPtr, - checksum_version, use_pagemap, backup_mode); - } - - /* check for errors */ - if (rc == FILE_MISSING) - { - elog(LOG, "File not found: \"%s\"", from_fullpath); - file->write_size = FILE_NOT_FOUND; - goto cleanup; - } - - else if (rc == WRITE_FAILED) - elog(ERROR, "Cannot write block %u of \"%s\": %s", - err_blknum, to_fullpath, strerror(errno)); - - else if (rc == PAGE_CORRUPTION) - { - if (errmsg) - elog(ERROR, "Corruption detected in file \"%s\", block %u: %s", - from_fullpath, err_blknum, errmsg); - else - elog(ERROR, "Corruption detected in file \"%s\", block %u", - from_fullpath, err_blknum); - } - /* OPEN_FAILED and READ_FAILED */ - else if (rc == OPEN_FAILED) - { - if (errmsg) - elog(ERROR, "%s", errmsg); - else - elog(ERROR, "Cannot open file \"%s\"", from_fullpath); - } - else if (rc == READ_FAILED) - { - if (errmsg) - elog(ERROR, "%s", errmsg); - else - elog(ERROR, "Cannot read file \"%s\"", from_fullpath); - } - - file->read_size = rc * BLCKSZ; - - /* Determine that file didn`t changed in case of incremental catchup */ - if (backup_mode != BACKUP_MODE_FULL && - file->exists_in_prev && - file->write_size == 0 && - file->n_blocks > 0) - { - file->write_size = BYTES_INVALID; - } - -cleanup: - pg_free(errmsg); - pg_free(file->pagemap.bitmap); -} - -/* - * Backup non data file - * We do not apply compression to this file. - * If file exists in previous backup, then compare checksums - * and make a decision about copying or skiping the file. - */ -void -backup_non_data_file(pgFile *file, pgFile *prev_file, - const char *from_fullpath, const char *to_fullpath, - BackupMode backup_mode, time_t parent_backup_time, - bool missing_ok) -{ - /* special treatment for global/pg_control */ - if (file->external_dir_num == 0 && strcmp(file->rel_path, XLOG_CONTROL_FILE) == 0) - { - copy_pgcontrol_file(from_fullpath, FIO_DB_HOST, - to_fullpath, FIO_BACKUP_HOST, file); - return; - } - - /* - * If non-data file exists in previous backup - * and its mtime is less than parent backup start time ... */ - if ((pg_strcasecmp(file->name, RELMAPPER_FILENAME) != 0) && - (prev_file && file->exists_in_prev && - file->size == prev_file->size && - file->mtime <= parent_backup_time)) - { - /* - * file could be deleted under our feets. - * But then backup_non_data_file_internal will handle it safely - */ - if (file->forkName != cfm) - file->crc = fio_get_crc32(from_fullpath, FIO_DB_HOST, false, true); - else - file->crc = fio_get_crc32_truncated(from_fullpath, FIO_DB_HOST, true); - - /* ...and checksum is the same... */ - if (EQ_TRADITIONAL_CRC32(file->crc, prev_file->crc)) - { - file->write_size = BYTES_INVALID; - /* get full size from previous backup for unchanged file */ - file->uncompressed_size = prev_file->uncompressed_size; - return; /* ...skip copying file. */ - } - } - - backup_non_data_file_internal(from_fullpath, FIO_DB_HOST, - to_fullpath, file, missing_ok); -} - -/* - * Iterate over parent backup chain and lookup given destination file in - * filelist of every chain member starting with FULL backup. - * Apply changed blocks to destination file from every backup in parent chain. - */ -size_t -restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, - const char *to_fullpath, bool use_bitmap, PageState *checksum_map, - XLogRecPtr shift_lsn, datapagemap_t *lsn_map, bool use_headers) -{ - size_t total_write_len = 0; - char *in_buf = pgut_malloc(STDIO_BUFSIZE); - int backup_seq = 0; - - /* - * FULL -> INCR -> DEST - * 2 1 0 - * Restore of backups of older versions cannot be optimized with bitmap - * because of n_blocks - */ - if (use_bitmap) - /* start with dest backup */ - backup_seq = 0; - else - /* start with full backup */ - backup_seq = parray_num(parent_chain) - 1; - -// for (i = parray_num(parent_chain) - 1; i >= 0; i--) -// for (i = 0; i < parray_num(parent_chain); i++) - while (backup_seq >= 0 && backup_seq < parray_num(parent_chain)) - { - char from_root[MAXPGPATH]; - char from_fullpath[MAXPGPATH]; - FILE *in = NULL; - - pgFile **res_file = NULL; - pgFile *tmp_file = NULL; - - /* page headers */ - BackupPageHeader2 *headers = NULL; - - pgBackup *backup = (pgBackup *) parray_get(parent_chain, backup_seq); - - if (use_bitmap) - backup_seq++; - else - backup_seq--; - - /* lookup file in intermediate backup */ - res_file = parray_bsearch(backup->files, dest_file, pgFileCompareRelPathWithExternal); - tmp_file = (res_file) ? *res_file : NULL; - - /* Destination file is not exists yet at this moment */ - if (tmp_file == NULL) - continue; - - /* - * Skip file if it haven't changed since previous backup - * and thus was not backed up. - */ - if (tmp_file->write_size == BYTES_INVALID) - continue; - - /* If file was truncated in intermediate backup, - * it is ok not to truncate it now, because old blocks will be - * overwritten by new blocks from next backup. - */ - if (tmp_file->write_size == 0) - continue; - - /* - * At this point we are sure, that something is going to be copied - * Open source file. - */ - join_path_components(from_root, backup->root_dir, DATABASE_DIR); - join_path_components(from_fullpath, from_root, tmp_file->rel_path); - - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, - strerror(errno)); - - /* set stdio buffering for input data file */ - setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); - - /* get headers for this file */ - if (use_headers && tmp_file->n_headers > 0) - headers = get_data_file_headers(&(backup->hdr_map), tmp_file, - parse_program_version(backup->program_version), - true); - - if (use_headers && !headers && tmp_file->n_headers > 0) - elog(ERROR, "Failed to get page headers for file \"%s\"", from_fullpath); - - /* - * Restore the file. - * Datafiles are backed up block by block and every block - * have BackupPageHeader with meta information, so we cannot just - * copy the file from backup. - */ - total_write_len += restore_data_file_internal(in, out, tmp_file, - parse_program_version(backup->program_version), - from_fullpath, to_fullpath, dest_file->n_blocks, - use_bitmap ? &(dest_file)->pagemap : NULL, - checksum_map, backup->checksum_version, - /* shiftmap can be used only if backup state precedes the shift */ - backup->stop_lsn <= shift_lsn ? lsn_map : NULL, - headers); - - if (fclose(in) != 0) - elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, - strerror(errno)); - - pg_free(headers); - -// datapagemap_print_debug(&(dest_file)->pagemap); - } - pg_free(in_buf); - - return total_write_len; -} - -/* Restore block from "in" file to "out" file. - * If "nblocks" is greater than zero, then skip restoring blocks, - * whose position if greater than "nblocks". - * If map is NULL, then page bitmap cannot be used for restore optimization - * Page bitmap optimize restore of incremental chains, consisting of more than one - * backup. We restoring from newest to oldest and page, once restored, marked in map. - * When the same page, but in older backup, encountered, we check the map, if it is - * marked as already restored, then page is skipped. - */ -size_t -restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, - const char *from_fullpath, const char *to_fullpath, int nblocks, - datapagemap_t *map, PageState *checksum_map, int checksum_version, - datapagemap_t *lsn_map, BackupPageHeader2 *headers) -{ - BlockNumber blknum = 0; - int n_hdr = -1; - size_t write_len = 0; - off_t cur_pos_out = 0; - off_t cur_pos_in = 0; - - /* should not be possible */ - Assert(!(backup_version >= 20400 && file->n_headers <= 0)); - - /* - * We rely on stdio buffering of input and output. - * For buffering to be efficient, we try to minimize the - * number of lseek syscalls, because it forces buffer flush. - * For that, we track current write position in - * output file and issue fseek only when offset of block to be - * written not equal to current write position, which happens - * a lot when blocks from incremental backup are restored, - * but should never happen in case of blocks from FULL backup. - */ - if (fio_fseek(out, cur_pos_out) < 0) - elog(ERROR, "Cannot seek block %u of \"%s\": %s", - blknum, to_fullpath, strerror(errno)); - - for (;;) - { - off_t write_pos; - size_t len; - size_t read_len; - DataPage page; - int32 compressed_size = 0; - bool is_compressed = false; - - /* incremental restore vars */ - uint16 page_crc = 0; - XLogRecPtr page_lsn = InvalidXLogRecPtr; - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during data file restore"); - - /* newer backups have headers in separate storage */ - if (headers) - { - n_hdr++; - if (n_hdr >= file->n_headers) - break; - - blknum = headers[n_hdr].block; - page_lsn = headers[n_hdr].lsn; - page_crc = headers[n_hdr].checksum; - /* calculate payload size by comparing current and next page positions, - * page header is not included */ - compressed_size = headers[n_hdr+1].pos - headers[n_hdr].pos - sizeof(BackupPageHeader); - - Assert(compressed_size > 0); - Assert(compressed_size <= BLCKSZ); - - read_len = compressed_size + sizeof(BackupPageHeader); - } - else - { - /* We get into this function either when restoring old backup - * or when merging something. Align read_len only when restoring - * or merging old backups. - */ - if (get_page_header(in, from_fullpath, &(page).bph, NULL, false)) - { - cur_pos_in += sizeof(BackupPageHeader); - - /* backward compatibility kludge TODO: remove in 3.0 */ - blknum = page.bph.block; - compressed_size = page.bph.compressed_size; - - /* this has a potential to backfire when retrying merge of old backups, - * so we just forbid the retrying of failed merges between versions >= 2.4.0 and - * version < 2.4.0 - */ - if (backup_version >= 20400) - read_len = compressed_size; - else - /* For some unknown and possibly dump reason I/O operations - * in versions < 2.4.0 were always aligned to 8 bytes. - * Now we have to deal with backward compatibility. - */ - read_len = MAXALIGN(compressed_size); - } - else - break; - } - - /* - * Backward compatibility kludge: in the good old days - * n_blocks attribute was available only in DELTA backups. - * File truncate in PAGE and PTRACK happened on the fly when - * special value PageIsTruncated is encountered. - * It was inefficient. - * - * Nowadays every backup type has n_blocks, so instead of - * writing and then truncating redundant data, writing - * is not happening in the first place. - * TODO: remove in 3.0.0 - */ - if (compressed_size == PageIsTruncated) - { - /* - * Block header contains information that this block was truncated. - * We need to truncate file to this length. - */ - - elog(VERBOSE, "Truncate file \"%s\" to block %u", to_fullpath, blknum); - - /* To correctly truncate file, we must first flush STDIO buffers */ - if (fio_fflush(out) != 0) - elog(ERROR, "Cannot flush file \"%s\": %s", to_fullpath, strerror(errno)); - - /* Set position to the start of file */ - if (fio_fseek(out, 0) < 0) - elog(ERROR, "Cannot seek to the start of file \"%s\": %s", to_fullpath, strerror(errno)); - - if (fio_ftruncate(out, blknum * BLCKSZ) != 0) - elog(ERROR, "Cannot truncate file \"%s\": %s", to_fullpath, strerror(errno)); - - break; - } - - Assert(compressed_size > 0); - Assert(compressed_size <= BLCKSZ); - - /* no point in writing redundant data */ - if (nblocks > 0 && blknum >= nblocks) - break; - - if (compressed_size > BLCKSZ) - elog(ERROR, "Size of a blknum %i exceed BLCKSZ: %i", blknum, compressed_size); - - /* Incremental restore in LSN mode */ - if (map && lsn_map && datapagemap_is_set(lsn_map, blknum)) - datapagemap_add(map, blknum); - - if (map && checksum_map && checksum_map[blknum].checksum != 0) - { - //elog(INFO, "HDR CRC: %u, MAP CRC: %u", page_crc, checksum_map[blknum].checksum); - /* - * The heart of incremental restore in CHECKSUM mode - * If page in backup has the same checksum and lsn as - * page in backup, then page can be skipped. - */ - if (page_crc == checksum_map[blknum].checksum && - page_lsn == checksum_map[blknum].lsn) - { - datapagemap_add(map, blknum); - } - } - - /* if this page is marked as already restored, then skip it */ - if (map && datapagemap_is_set(map, blknum)) - { - /* Backward compatibility kludge TODO: remove in 3.0 - * go to the next page. - */ - if (!headers && fseek(in, read_len, SEEK_CUR) != 0) - elog(ERROR, "Cannot seek block %u of \"%s\": %s", - blknum, from_fullpath, strerror(errno)); - continue; - } - - if (headers && - cur_pos_in != headers[n_hdr].pos) - { - if (fseek(in, headers[n_hdr].pos, SEEK_SET) != 0) - elog(ERROR, "Cannot seek to offset %u of \"%s\": %s", - headers[n_hdr].pos, from_fullpath, strerror(errno)); - - cur_pos_in = headers[n_hdr].pos; - } - - /* read a page from file */ - if (headers) - len = fread(&page, 1, read_len, in); - else - len = fread(page.data, 1, read_len, in); - - if (len != read_len) - elog(ERROR, "Cannot read block %u file \"%s\": %s", - blknum, from_fullpath, strerror(errno)); - - cur_pos_in += read_len; - - /* - * if page size is smaller than BLCKSZ, decompress the page. - * BUGFIX for versions < 2.0.23: if page size is equal to BLCKSZ. - * we have to check, whether it is compressed or not using - * page_may_be_compressed() function. - */ - if (compressed_size != BLCKSZ - || page_may_be_compressed(page.data, file->compress_alg, backup_version)) - { - is_compressed = true; - } - - /* - * Seek and write the restored page. - * When restoring file from FULL backup, pages are written sequentially, - * so there is no need to issue fseek for every page. - */ - write_pos = blknum * BLCKSZ; - - if (cur_pos_out != write_pos) - { - if (fio_fseek(out, write_pos) < 0) - elog(ERROR, "Cannot seek block %u of \"%s\": %s", - blknum, to_fullpath, strerror(errno)); - - cur_pos_out = write_pos; - } - - /* - * If page is compressed and restore is in remote mode, - * send compressed page to the remote side. - */ - if (is_compressed) - { - ssize_t rc; - rc = fio_fwrite_async_compressed(out, page.data, compressed_size, file->compress_alg); - - if (!fio_is_remote_file(out) && rc != BLCKSZ) - elog(ERROR, "Cannot write block %u of \"%s\": %s, size: %u", - blknum, to_fullpath, strerror(errno), compressed_size); - } - else - { - if (fio_fwrite_async(out, page.data, BLCKSZ) != BLCKSZ) - elog(ERROR, "Cannot write block %u of \"%s\": %s", - blknum, to_fullpath, strerror(errno)); - } - - write_len += BLCKSZ; - cur_pos_out += BLCKSZ; /* update current write position */ - - /* Mark page as restored to avoid reading this page when restoring parent backups */ - if (map) - datapagemap_add(map, blknum); - } - - elog(LOG, "Copied file \"%s\": %lu bytes", from_fullpath, write_len); - return write_len; -} - -/* - * Copy file to backup. - * We do not apply compression to these files, because - * it is either small control file or already compressed cfs file. - */ -void -restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, - const char *from_fullpath, const char *to_fullpath) -{ - size_t read_len = 0; - char *buf = pgut_malloc(STDIO_BUFSIZE); /* 64kB buffer */ - - /* copy content */ - for (;;) - { - read_len = 0; - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during non-data file restore"); - - read_len = fread(buf, 1, STDIO_BUFSIZE, in); - - if (ferror(in)) - elog(ERROR, "Cannot read backup file \"%s\": %s", - from_fullpath, strerror(errno)); - - if (read_len > 0) - { - if (fio_fwrite_async(out, buf, read_len) != read_len) - elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, - strerror(errno)); - } - - if (feof(in)) - break; - } - - pg_free(buf); - - elog(LOG, "Copied file \"%s\": %lu bytes", from_fullpath, file->write_size); -} - -size_t -restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, - pgFile *dest_file, FILE *out, const char *to_fullpath, - bool already_exists) -{ - char from_root[MAXPGPATH]; - char from_fullpath[MAXPGPATH]; - FILE *in = NULL; - - pgFile *tmp_file = NULL; - pgBackup *tmp_backup = NULL; - - /* Check if full copy of destination file is available in destination backup */ - if (dest_file->write_size > 0) - { - tmp_file = dest_file; - tmp_backup = dest_backup; - } - else - { - /* - * Iterate over parent chain starting from direct parent of destination - * backup to oldest backup in chain, and look for the first - * full copy of destination file. - * Full copy is latest possible destination file with size equal or - * greater than zero. - */ - tmp_backup = dest_backup->parent_backup_link; - while (tmp_backup) - { - pgFile **res_file = NULL; - - /* lookup file in intermediate backup */ - res_file = parray_bsearch(tmp_backup->files, dest_file, pgFileCompareRelPathWithExternal); - tmp_file = (res_file) ? *res_file : NULL; - - /* - * It should not be possible not to find destination file in intermediate - * backup, without encountering full copy first. - */ - if (!tmp_file) - { - elog(ERROR, "Failed to locate non-data file \"%s\" in backup %s", - dest_file->rel_path, backup_id_of(tmp_backup)); - continue; - } - - /* Full copy is found and it is null sized, nothing to do here */ - if (tmp_file->write_size == 0) - { - /* In case of incremental restore truncate file just to be safe */ - if (already_exists && fio_ftruncate(out, 0)) - elog(ERROR, "Cannot truncate file \"%s\": %s", - to_fullpath, strerror(errno)); - return 0; - } - - /* Full copy is found */ - if (tmp_file->write_size > 0) - break; - - tmp_backup = tmp_backup->parent_backup_link; - } - } - - /* sanity */ - if (!tmp_backup) - elog(ERROR, "Failed to locate a backup containing full copy of non-data file \"%s\"", - to_fullpath); - - if (!tmp_file) - elog(ERROR, "Failed to locate a full copy of non-data file \"%s\"", to_fullpath); - - if (tmp_file->write_size <= 0) - elog(ERROR, "Full copy of non-data file has invalid size: %li. " - "Metadata corruption in backup %s in file: \"%s\"", - tmp_file->write_size, backup_id_of(tmp_backup), - to_fullpath); - - /* incremental restore */ - if (already_exists) - { - /* compare checksums of already existing file and backup file */ - pg_crc32 file_crc; - if (tmp_file->forkName == cfm && - tmp_file->uncompressed_size > tmp_file->write_size) - file_crc = fio_get_crc32_truncated(to_fullpath, FIO_DB_HOST, false); - else - file_crc = fio_get_crc32(to_fullpath, FIO_DB_HOST, false, false); - - if (file_crc == tmp_file->crc) - { - elog(LOG, "Already existing non-data file \"%s\" has the same checksum, skip restore", - to_fullpath); - return 0; - } - - /* Checksum mismatch, truncate file and overwrite it */ - if (fio_ftruncate(out, 0)) - elog(ERROR, "Cannot truncate file \"%s\": %s", - to_fullpath, strerror(errno)); - } - - if (tmp_file->external_dir_num == 0) - join_path_components(from_root, tmp_backup->root_dir, DATABASE_DIR); - else - { - char external_prefix[MAXPGPATH]; - - join_path_components(external_prefix, tmp_backup->root_dir, EXTERNAL_DIR); - makeExternalDirPathByNum(from_root, external_prefix, tmp_file->external_dir_num); - } - - join_path_components(from_fullpath, from_root, dest_file->rel_path); - - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - elog(ERROR, "Cannot open backup file \"%s\": %s", from_fullpath, - strerror(errno)); - - /* disable stdio buffering for non-data files */ - setvbuf(in, NULL, _IONBF, BUFSIZ); - - /* do actual work */ - restore_non_data_file_internal(in, out, tmp_file, from_fullpath, to_fullpath); - - if (fclose(in) != 0) - elog(ERROR, "Cannot close file \"%s\": %s", from_fullpath, - strerror(errno)); - - return tmp_file->write_size; -} - -/* - * Copy file to backup. - * We do not apply compression to these files, because - * it is either small control file or already compressed cfs file. - * TODO: optimize remote copying - */ -void -backup_non_data_file_internal(const char *from_fullpath, - fio_location from_location, - const char *to_fullpath, pgFile *file, - bool missing_ok) -{ - FILE *out = NULL; - char *errmsg = NULL; - int rc; - bool cut_zero_tail; - - cut_zero_tail = file->forkName == cfm; - - INIT_FILE_CRC32(true, file->crc); - - /* reset size summary */ - file->read_size = 0; - file->write_size = 0; - file->uncompressed_size = 0; - - /* open backup file for write */ - out = fopen(to_fullpath, PG_BINARY_W); - if (out == NULL) - elog(ERROR, "Cannot open destination file \"%s\": %s", - to_fullpath, strerror(errno)); - - /* update file permission */ - if (chmod(to_fullpath, file->mode) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, - strerror(errno)); - - /* backup remote file */ - if (fio_is_remote(FIO_DB_HOST)) - rc = fio_send_file(from_fullpath, out, cut_zero_tail, file, &errmsg); - else - rc = fio_send_file_local(from_fullpath, out, cut_zero_tail, file, &errmsg); - - /* handle errors */ - if (rc == FILE_MISSING) - { - /* maybe deleted, it's not error in case of backup */ - if (missing_ok) - { - elog(LOG, "File \"%s\" is not found", from_fullpath); - file->write_size = FILE_NOT_FOUND; - goto cleanup; - } - else - elog(ERROR, "File \"%s\" is not found", from_fullpath); - } - else if (rc == WRITE_FAILED) - elog(ERROR, "Cannot write to \"%s\": %s", to_fullpath, strerror(errno)); - else if (rc != SEND_OK) - { - if (errmsg) - elog(ERROR, "%s", errmsg); - else - elog(ERROR, "Cannot access remote file \"%s\"", from_fullpath); - } - - file->uncompressed_size = file->read_size; - -cleanup: - if (errmsg != NULL) - pg_free(errmsg); - - /* finish CRC calculation and store into pgFile */ - FIN_FILE_CRC32(true, file->crc); - - if (out && fclose(out)) - elog(ERROR, "Cannot close the file \"%s\": %s", to_fullpath, strerror(errno)); -} - -/* - * Create empty file, used for partial restore - */ -bool -create_empty_file(fio_location from_location, const char *to_root, - fio_location to_location, pgFile *file) -{ - char to_path[MAXPGPATH]; - FILE *out; - - /* open file for write */ - join_path_components(to_path, to_root, file->rel_path); - out = fio_fopen(to_path, PG_BINARY_W, to_location); - - if (out == NULL) - elog(ERROR, "Cannot open destination file \"%s\": %s", - to_path, strerror(errno)); - - /* update file permission */ - if (fio_chmod(to_path, file->mode, to_location) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", to_path, - strerror(errno)); - - if (fio_fclose(out)) - elog(ERROR, "Cannot close \"%s\": %s", to_path, strerror(errno)); - - return true; -} - -/* - * Validate given page. - * This function is expected to be executed multiple times, - * so avoid using elog within it. - * lsn from page is assigned to page_lsn pointer. - * TODO: switch to enum for return codes. - */ -int -validate_one_page(Page page, BlockNumber absolute_blkno, - XLogRecPtr stop_lsn, PageState *page_st, - uint32 checksum_version) -{ - page_st->lsn = InvalidXLogRecPtr; - page_st->checksum = 0; - - /* new level of paranoia */ - if (page == NULL) - return PAGE_IS_NOT_FOUND; - - /* check that page header is ok */ - if (!parse_page(page, &(page_st)->lsn)) - { - int i; - /* Check if the page is zeroed. */ - for (i = 0; i < BLCKSZ && page[i] == 0; i++); - - /* Page is zeroed. No need to verify checksums */ - if (i == BLCKSZ) - return PAGE_IS_ZEROED; - - /* Page does not looking good */ - return PAGE_HEADER_IS_INVALID; - } - - /* Verify checksum */ - page_st->checksum = pg_checksum_page(page, absolute_blkno); - - if (checksum_version) - { - /* Checksums are enabled, so check them. */ - if (page_st->checksum != ((PageHeader) page)->pd_checksum) - return PAGE_CHECKSUM_MISMATCH; - } - - /* At this point page header is sane, if checksums are enabled - the`re ok. - * Check that page is not from future. - * Note, this check should be used only by validate command. - */ - if (stop_lsn > 0) - { - /* Get lsn from page header. Ensure that page is from our time. */ - if (page_st->lsn > stop_lsn) - return PAGE_LSN_FROM_FUTURE; - } - - return PAGE_IS_VALID; -} - -/* - * Validate pages of datafile in PGDATA one by one. - * - * returns true if the file is valid - * also returns true if the file was not found - */ -bool -check_data_file(ConnectionArgs *arguments, pgFile *file, - const char *from_fullpath, uint32 checksum_version) -{ - FILE *in; - BlockNumber blknum = 0; - BlockNumber nblocks = 0; - int page_state; - char curr_page[BLCKSZ]; - bool is_valid = true; - - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - { - /* - * If file is not found, this is not en error. - * It could have been deleted by concurrent postgres transaction. - */ - if (errno == ENOENT) - { - elog(LOG, "File \"%s\" is not found", from_fullpath); - return true; - } - - elog(WARNING, "Cannot open file \"%s\": %s", - from_fullpath, strerror(errno)); - return false; - } - - if (file->size % BLCKSZ != 0) - elog(WARNING, "File: \"%s\", invalid file size %zu", from_fullpath, file->size); - - /* - * Compute expected number of blocks in the file. - * NOTE This is a normal situation, if the file size has changed - * since the moment we computed it. - */ - nblocks = file->size/BLCKSZ; - - for (blknum = 0; blknum < nblocks; blknum++) - { - PageState page_st; - page_state = prepare_page(file, InvalidXLogRecPtr, - blknum, in, BACKUP_MODE_FULL, - curr_page, false, checksum_version, - from_fullpath, &page_st); - - if (page_state == PageIsTruncated) - break; - - if (page_state == PageIsCorrupted) - { - /* Page is corrupted, no need to elog about it, - * prepare_page() already done that - */ - is_valid = false; - continue; - } - } - - fclose(in); - return is_valid; -} - -/* Valiate pages of datafile in backup one by one */ -bool -validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, - uint32 checksum_version, uint32 backup_version, HeaderMap *hdr_map) -{ - size_t read_len = 0; - bool is_valid = true; - FILE *in; - pg_crc32 crc; - bool use_crc32c = backup_version <= 20021 || backup_version >= 20025; - BackupPageHeader2 *headers = NULL; - int n_hdr = -1; - off_t cur_pos_in = 0; - - elog(LOG, "Validate relation blocks for file \"%s\"", fullpath); - - /* should not be possible */ - Assert(!(backup_version >= 20400 && file->n_headers <= 0)); - - in = fopen(fullpath, PG_BINARY_R); - if (in == NULL) - elog(ERROR, "Cannot open file \"%s\": %s", - fullpath, strerror(errno)); - - headers = get_data_file_headers(hdr_map, file, backup_version, false); - - if (!headers && file->n_headers > 0) - { - elog(WARNING, "Cannot get page headers for file \"%s\"", fullpath); - return false; - } - - /* calc CRC of backup file */ - INIT_FILE_CRC32(use_crc32c, crc); - - /* read and validate pages one by one */ - while (true) - { - int rc = 0; - size_t len = 0; - DataPage compressed_page; /* used as read buffer */ - int compressed_size = 0; - DataPage page; - BlockNumber blknum = 0; - PageState page_st; - - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during data file validation"); - - /* newer backups have page headers in separate storage */ - if (headers) - { - n_hdr++; - if (n_hdr >= file->n_headers) - break; - - blknum = headers[n_hdr].block; - /* calculate payload size by comparing current and next page positions, - * page header is not included. - */ - compressed_size = headers[n_hdr+1].pos - headers[n_hdr].pos - sizeof(BackupPageHeader); - - Assert(compressed_size > 0); - Assert(compressed_size <= BLCKSZ); - - read_len = sizeof(BackupPageHeader) + compressed_size; - - if (cur_pos_in != headers[n_hdr].pos) - { - if (fio_fseek(in, headers[n_hdr].pos) < 0) - elog(ERROR, "Cannot seek block %u of \"%s\": %s", - blknum, fullpath, strerror(errno)); - else - elog(VERBOSE, "Seek to %u", headers[n_hdr].pos); - - cur_pos_in = headers[n_hdr].pos; - } - } - /* old backups rely on header located directly in data file */ - else - { - if (get_page_header(in, fullpath, &(compressed_page).bph, &crc, use_crc32c)) - { - /* Backward compatibility kludge, TODO: remove in 3.0 - * for some reason we padded compressed pages in old versions - */ - blknum = compressed_page.bph.block; - compressed_size = compressed_page.bph.compressed_size; - read_len = MAXALIGN(compressed_size); - } - else - break; - } - - /* backward compatibility kludge TODO: remove in 3.0 */ - if (compressed_size == PageIsTruncated) - { - elog(VERBOSE, "Block %u of \"%s\" is truncated", - blknum, fullpath); - continue; - } - - Assert(compressed_size <= BLCKSZ); - Assert(compressed_size > 0); - - if (headers) - len = fread(&compressed_page, 1, read_len, in); - else - len = fread(compressed_page.data, 1, read_len, in); - - if (len != read_len) - { - elog(WARNING, "Cannot read block %u file \"%s\": %s", - blknum, fullpath, strerror(errno)); - return false; - } - - /* update current position */ - cur_pos_in += read_len; - - if (headers) - COMP_FILE_CRC32(use_crc32c, crc, &compressed_page, read_len); - else - COMP_FILE_CRC32(use_crc32c, crc, compressed_page.data, read_len); - - if (compressed_size != BLCKSZ - || page_may_be_compressed(compressed_page.data, file->compress_alg, - backup_version)) - { - int32 uncompressed_size = 0; - const char *errormsg = NULL; - - uncompressed_size = do_decompress(page.data, BLCKSZ, - compressed_page.data, - compressed_size, - file->compress_alg, - &errormsg); - if (uncompressed_size < 0 && errormsg != NULL) - { - elog(WARNING, "An error occured during decompressing block %u of file \"%s\": %s", - blknum, fullpath, errormsg); - return false; - } - - if (uncompressed_size != BLCKSZ) - { - if (compressed_size == BLCKSZ) - { - is_valid = false; - continue; - } - elog(WARNING, "Page %u of file \"%s\" uncompressed to %d bytes. != BLCKSZ", - blknum, fullpath, uncompressed_size); - return false; - } - - rc = validate_one_page(page.data, - file->segno * RELSEG_SIZE + blknum, - stop_lsn, &page_st, checksum_version); - } - else - rc = validate_one_page(compressed_page.data, - file->segno * RELSEG_SIZE + blknum, - stop_lsn, &page_st, checksum_version); - - switch (rc) - { - case PAGE_IS_NOT_FOUND: - elog(VERBOSE, "File \"%s\", block %u, page is NULL", file->rel_path, blknum); - break; - case PAGE_IS_ZEROED: - elog(VERBOSE, "File: %s blknum %u, empty zeroed page", file->rel_path, blknum); - break; - case PAGE_HEADER_IS_INVALID: - elog(WARNING, "Page header is looking insane: %s, block %i", file->rel_path, blknum); - is_valid = false; - break; - case PAGE_CHECKSUM_MISMATCH: - elog(WARNING, "File: %s blknum %u have wrong checksum: %u", file->rel_path, blknum, page_st.checksum); - is_valid = false; - break; - case PAGE_LSN_FROM_FUTURE: - elog(WARNING, "File: %s, block %u, checksum is %s. " - "Page is from future: pageLSN %X/%X stopLSN %X/%X", - file->rel_path, blknum, - checksum_version ? "correct" : "not enabled", - (uint32) (page_st.lsn >> 32), (uint32) page_st.lsn, - (uint32) (stop_lsn >> 32), (uint32) stop_lsn); - break; - } - } - - FIN_FILE_CRC32(use_crc32c, crc); - fclose(in); - - if (crc != file->crc) - { - elog(WARNING, "Invalid CRC of backup file \"%s\": %X. Expected %X", - fullpath, crc, file->crc); - is_valid = false; - } - - pg_free(headers); - - return is_valid; -} - -/* read local data file and construct map with block checksums */ -PageState* -get_checksum_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno) -{ - PageState *checksum_map = NULL; - FILE *in = NULL; - BlockNumber blknum = 0; - char read_buffer[BLCKSZ]; - char in_buf[STDIO_BUFSIZE]; - - /* open file */ - in = fopen(fullpath, "r+b"); - if (!in) - elog(ERROR, "Cannot open source file \"%s\": %s", fullpath, strerror(errno)); - - /* truncate up to blocks */ - if (ftruncate(fileno(in), n_blocks * BLCKSZ) != 0) - elog(ERROR, "Cannot truncate file to blknum %u \"%s\": %s", - n_blocks, fullpath, strerror(errno)); - - setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); - - /* initialize array of checksums */ - checksum_map = pgut_malloc(n_blocks * sizeof(PageState)); - memset(checksum_map, 0, n_blocks * sizeof(PageState)); - - for (blknum = 0; blknum < n_blocks; blknum++) - { - size_t read_len = fread(read_buffer, 1, BLCKSZ, in); - PageState page_st; - - /* report error */ - if (ferror(in)) - elog(ERROR, "Cannot read block %u of \"%s\": %s", - blknum, fullpath, strerror(errno)); - - if (read_len == BLCKSZ) - { - int rc = validate_one_page(read_buffer, segmentno + blknum, - dest_stop_lsn, &page_st, - checksum_version); - - if (rc == PAGE_IS_VALID) - { -// if (checksum_version) -// checksum_map[blknum].checksum = ((PageHeader) read_buffer)->pd_checksum; -// else -// checksum_map[blknum].checksum = page_st.checksum; - checksum_map[blknum].checksum = page_st.checksum; - checksum_map[blknum].lsn = page_st.lsn; - } - } - else - elog(ERROR, "Failed to read blknum %u from file \"%s\"", blknum, fullpath); - - if (feof(in)) - break; - - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during page reading"); - } - - if (in) - fclose(in); - - return checksum_map; -} - -/* return bitmap of valid blocks, bitmap is empty, then NULL is returned */ -datapagemap_t * -get_lsn_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno) -{ - FILE *in = NULL; - BlockNumber blknum = 0; - char read_buffer[BLCKSZ]; - char in_buf[STDIO_BUFSIZE]; - datapagemap_t *lsn_map = NULL; - - Assert(shift_lsn > 0); - - /* open file */ - in = fopen(fullpath, "r+b"); - if (!in) - elog(ERROR, "Cannot open source file \"%s\": %s", fullpath, strerror(errno)); - - /* truncate up to blocks */ - if (ftruncate(fileno(in), n_blocks * BLCKSZ) != 0) - elog(ERROR, "Cannot truncate file to blknum %u \"%s\": %s", - n_blocks, fullpath, strerror(errno)); - - setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); - - lsn_map = pgut_malloc(sizeof(datapagemap_t)); - memset(lsn_map, 0, sizeof(datapagemap_t)); - - for (blknum = 0; blknum < n_blocks; blknum++) - { - size_t read_len = fread(read_buffer, 1, BLCKSZ, in); - PageState page_st; - - /* report error */ - if (ferror(in)) - elog(ERROR, "Cannot read block %u of \"%s\": %s", - blknum, fullpath, strerror(errno)); - - if (read_len == BLCKSZ) - { - int rc = validate_one_page(read_buffer, segmentno + blknum, - shift_lsn, &page_st, checksum_version); - - if (rc == PAGE_IS_VALID) - datapagemap_add(lsn_map, blknum); - } - else - elog(ERROR, "Cannot read block %u from file \"%s\": %s", - blknum, fullpath, strerror(errno)); - - if (feof(in)) - break; - - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during page reading"); - } - - if (in) - fclose(in); - - if (lsn_map->bitmapsize == 0) - { - pg_free(lsn_map); - lsn_map = NULL; - } - - return lsn_map; -} - -/* Every page in data file contains BackupPageHeader, extract it */ -bool -get_page_header(FILE *in, const char *fullpath, BackupPageHeader* bph, - pg_crc32 *crc, bool use_crc32c) -{ - /* read BackupPageHeader */ - size_t read_len = fread(bph, 1, sizeof(BackupPageHeader), in); - - if (ferror(in)) - elog(ERROR, "Cannot read file \"%s\": %s", - fullpath, strerror(errno)); - - if (read_len != sizeof(BackupPageHeader)) - { - if (read_len == 0 && feof(in)) - return false; /* EOF found */ - else if (read_len != 0 && feof(in)) - elog(ERROR, - "Odd size page found at offset %ld of \"%s\"", - ftello(in), fullpath); - else - elog(ERROR, "Cannot read header at offset %ld of \"%s\": %s", - ftello(in), fullpath, strerror(errno)); - } - - /* In older versions < 2.4.0, when crc for file was calculated, header was - * not included in crc calculations. Now it is. And now we have - * the problem of backward compatibility for backups of old versions - */ - if (crc) - COMP_FILE_CRC32(use_crc32c, *crc, bph, read_len); - - if (bph->block == 0 && bph->compressed_size == 0) - elog(ERROR, "Empty block in file \"%s\"", fullpath); - - Assert(bph->compressed_size != 0); - return true; -} - -/* Open local backup file for writing, set permissions and buffering */ -FILE* -open_local_file_rw(const char *to_fullpath, char **out_buf, uint32 buf_size) -{ - FILE *out = NULL; - /* open backup file for write */ - out = fopen(to_fullpath, PG_BINARY_W); - if (out == NULL) - elog(ERROR, "Cannot open backup file \"%s\": %s", - to_fullpath, strerror(errno)); - - /* update file permission */ - if (chmod(to_fullpath, FILE_PERMISSION) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, - strerror(errno)); - - /* enable stdio buffering for output file */ - *out_buf = pgut_malloc(buf_size); - setvbuf(out, *out_buf, _IOFBF, buf_size); - - return out; -} - -/* backup local file */ -int -send_pages(const char *to_fullpath, const char *from_fullpath, - pgFile *file, XLogRecPtr prev_backup_start_lsn, CompressAlg calg, int clevel, - uint32 checksum_version, bool use_pagemap, BackupPageHeader2 **headers, - BackupMode backup_mode) -{ - FILE *in = NULL; - FILE *out = NULL; - off_t cur_pos_out = 0; - char curr_page[BLCKSZ]; - int n_blocks_read = 0; - BlockNumber blknum = 0; - datapagemap_iterator_t *iter = NULL; - int compressed_size = 0; - BackupPageHeader2 *header = NULL; - parray *harray = NULL; - - /* stdio buffers */ - char *in_buf = NULL; - char *out_buf = NULL; - - /* open source file for read */ - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - { - /* - * If file is not found, this is not en error. - * It could have been deleted by concurrent postgres transaction. - */ - if (errno == ENOENT) - return FILE_MISSING; - - elog(ERROR, "Cannot open file \"%s\": %s", from_fullpath, strerror(errno)); - } - - /* - * Enable stdio buffering for local input file, - * unless the pagemap is involved, which - * imply a lot of random access. - */ - - if (use_pagemap) - { - iter = datapagemap_iterate(&file->pagemap); - datapagemap_next(iter, &blknum); /* set first block */ - - setvbuf(in, NULL, _IONBF, BUFSIZ); - } - else - { - in_buf = pgut_malloc(STDIO_BUFSIZE); - setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); - } - - harray = parray_new(); - - while (blknum < file->n_blocks) - { - PageState page_st; - int rc = prepare_page(file, prev_backup_start_lsn, - blknum, in, backup_mode, curr_page, - true, checksum_version, - from_fullpath, &page_st); - - if (rc == PageIsTruncated) - break; - - else if (rc == PageIsOk) - { - /* lazily open backup file (useful for s3) */ - if (!out) - out = open_local_file_rw(to_fullpath, &out_buf, STDIO_BUFSIZE); - - header = pgut_new0(BackupPageHeader2); - *header = (BackupPageHeader2){ - .block = blknum, - .pos = cur_pos_out, - .lsn = page_st.lsn, - .checksum = page_st.checksum, - }; - - parray_append(harray, header); - - compressed_size = compress_and_backup_page(file, blknum, in, out, &(file->crc), - rc, curr_page, calg, clevel, - from_fullpath, to_fullpath); - cur_pos_out += compressed_size + sizeof(BackupPageHeader); - } - - n_blocks_read++; - - /* next block */ - if (use_pagemap) - { - /* exit if pagemap is exhausted */ - if (!datapagemap_next(iter, &blknum)) - break; - } - else - blknum++; - } - - /* - * Add dummy header, so we can later extract the length of last header - * as difference between their offsets. - */ - if (parray_num(harray) > 0) - { - size_t hdr_num = parray_num(harray); - size_t i; - - file->n_headers = (int) hdr_num; /* is it valid? */ - *headers = (BackupPageHeader2 *) pgut_malloc0((hdr_num + 1) * sizeof(BackupPageHeader2)); - for (i = 0; i < hdr_num; i++) - { - header = (BackupPageHeader2 *)parray_get(harray, i); - (*headers)[i] = *header; - pg_free(header); - } - (*headers)[hdr_num] = (BackupPageHeader2){.pos=cur_pos_out}; - } - parray_free(harray); - - /* cleanup */ - if (in && fclose(in)) - elog(ERROR, "Cannot close the source file \"%s\": %s", - to_fullpath, strerror(errno)); - - /* close local output file */ - if (out && fclose(out)) - elog(ERROR, "Cannot close the backup file \"%s\": %s", - to_fullpath, strerror(errno)); - - pg_free(iter); - pg_free(in_buf); - pg_free(out_buf); - - return n_blocks_read; -} - -/* - * Copy local data file just as send_pages but without attaching additional header and compression - */ -int -copy_pages(const char *to_fullpath, const char *from_fullpath, - pgFile *file, XLogRecPtr sync_lsn, - uint32 checksum_version, bool use_pagemap, - BackupMode backup_mode) -{ - FILE *in = NULL; - FILE *out = NULL; - char curr_page[BLCKSZ]; - int n_blocks_read = 0; - BlockNumber blknum = 0; - datapagemap_iterator_t *iter = NULL; - - /* stdio buffers */ - char *in_buf = NULL; - char *out_buf = NULL; - - /* open source file for read */ - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - { - /* - * If file is not found, this is not en error. - * It could have been deleted by concurrent postgres transaction. - */ - if (errno == ENOENT) - return FILE_MISSING; - - elog(ERROR, "Cannot open file \"%s\": %s", from_fullpath, strerror(errno)); - } - - /* - * Enable stdio buffering for local input file, - * unless the pagemap is involved, which - * imply a lot of random access. - */ - - if (use_pagemap) - { - iter = datapagemap_iterate(&file->pagemap); - datapagemap_next(iter, &blknum); /* set first block */ - - setvbuf(in, NULL, _IONBF, BUFSIZ); - } - else - { - in_buf = pgut_malloc(STDIO_BUFSIZE); - setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); - } - - out = fio_fopen(to_fullpath, PG_BINARY_R "+", FIO_BACKUP_HOST); - if (out == NULL) - elog(ERROR, "Cannot open destination file \"%s\": %s", - to_fullpath, strerror(errno)); - - /* update file permission */ - if (chmod(to_fullpath, file->mode) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, - strerror(errno)); - - /* Enable buffering for output file */ - out_buf = pgut_malloc(STDIO_BUFSIZE); - setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); - - while (blknum < file->n_blocks) - { - PageState page_st; - int rc = prepare_page(file, sync_lsn, - blknum, in, backup_mode, curr_page, - true, checksum_version, - from_fullpath, &page_st); - if (rc == PageIsTruncated) - break; - - else if (rc == PageIsOk) - { - if (fseek(out, blknum * BLCKSZ, SEEK_SET) != 0) - elog(ERROR, "Cannot seek to position %u in destination file \"%s\": %s", - blknum * BLCKSZ, to_fullpath, strerror(errno)); - - if (write_page(file, out, curr_page) != BLCKSZ) - elog(ERROR, "File: \"%s\", cannot write at block %u: %s", - to_fullpath, blknum, strerror(errno)); - } - - n_blocks_read++; - - /* next block */ - if (use_pagemap) - { - /* exit if pagemap is exhausted */ - if (!datapagemap_next(iter, &blknum)) - break; - } - else - blknum++; - } - - /* truncate output file if required */ - if (fseek(out, 0, SEEK_END) != 0) - elog(ERROR, "Cannot seek to end of file position in destination file \"%s\": %s", - to_fullpath, strerror(errno)); - { - long pos = ftell(out); - - if (pos < 0) - elog(ERROR, "Cannot get position in destination file \"%s\": %s", - to_fullpath, strerror(errno)); - - if (pos != file->size) - { - if (fflush(out) != 0) - elog(ERROR, "Cannot flush destination file \"%s\": %s", - to_fullpath, strerror(errno)); - - if (ftruncate(fileno(out), file->size) == -1) - elog(ERROR, "Cannot ftruncate file \"%s\" to size %lu: %s", - to_fullpath, file->size, strerror(errno)); - } - } - - /* cleanup */ - if (fclose(in)) - elog(ERROR, "Cannot close the source file \"%s\": %s", - to_fullpath, strerror(errno)); - - /* close output file */ - if (fclose(out)) - elog(ERROR, "Cannot close the destination file \"%s\": %s", - to_fullpath, strerror(errno)); - - pg_free(iter); - pg_free(in_buf); - pg_free(out_buf); - - return n_blocks_read; -} - -/* - * Attempt to open header file, read content and return as - * array of headers. - * TODO: some access optimizations would be great here: - * less fseeks, buffering, descriptor sharing, etc. - */ -BackupPageHeader2* -get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, bool strict) -{ - bool success = false; - FILE *in = NULL; - size_t read_len = 0; - pg_crc32 hdr_crc; - BackupPageHeader2 *headers = NULL; - /* header decompression */ - int z_len = 0; - char *zheaders = NULL; - const char *errormsg = NULL; - - if (backup_version < 20400) - return NULL; - - if (file->n_headers <= 0) - return NULL; - - /* TODO: consider to make this descriptor thread-specific */ - in = fopen(hdr_map->path, PG_BINARY_R); - - if (!in) - { - elog(strict ? ERROR : WARNING, "Cannot open header file \"%s\": %s", hdr_map->path, strerror(errno)); - return NULL; - } - /* disable buffering for header file */ - setvbuf(in, NULL, _IONBF, 0); - - if (fseeko(in, file->hdr_off, SEEK_SET)) - { - elog(strict ? ERROR : WARNING, "Cannot seek to position %llu in page header map \"%s\": %s", - file->hdr_off, hdr_map->path, strerror(errno)); - goto cleanup; - } - - /* - * The actual number of headers in header file is n+1, last one is a dummy header, - * used for calculation of read_len for actual last header. - */ - read_len = (file->n_headers+1) * sizeof(BackupPageHeader2); - - /* allocate memory for compressed headers */ - zheaders = pgut_malloc(file->hdr_size); - memset(zheaders, 0, file->hdr_size); - - if (fread(zheaders, 1, file->hdr_size, in) != file->hdr_size) - { - elog(strict ? ERROR : WARNING, "Cannot read header file at offset: %llu len: %i \"%s\": %s", - file->hdr_off, file->hdr_size, hdr_map->path, strerror(errno)); - goto cleanup; - } - - /* allocate memory for uncompressed headers */ - headers = pgut_malloc(read_len); - memset(headers, 0, read_len); - - z_len = do_decompress(headers, read_len, zheaders, file->hdr_size, - ZLIB_COMPRESS, &errormsg); - if (z_len <= 0) - { - if (errormsg) - elog(strict ? ERROR : WARNING, "An error occured during metadata decompression for file \"%s\": %s", - file->rel_path, errormsg); - else - elog(strict ? ERROR : WARNING, "An error occured during metadata decompression for file \"%s\": %i", - file->rel_path, z_len); - - goto cleanup; - } - - /* validate checksum */ - INIT_FILE_CRC32(true, hdr_crc); - COMP_FILE_CRC32(true, hdr_crc, headers, read_len); - FIN_FILE_CRC32(true, hdr_crc); - - if (hdr_crc != file->hdr_crc) - { - elog(strict ? ERROR : WARNING, "Header map for file \"%s\" crc mismatch \"%s\" " - "offset: %llu, len: %lu, current: %u, expected: %u", - file->rel_path, hdr_map->path, file->hdr_off, read_len, hdr_crc, file->hdr_crc); - goto cleanup; - } - - success = true; - -cleanup: - - pg_free(zheaders); - if (in && fclose(in)) - elog(ERROR, "Cannot close file \"%s\"", hdr_map->path); - - if (!success) - { - pg_free(headers); - headers = NULL; - } - - return headers; -} - -/* write headers of all blocks belonging to file to header map and - * save its offset and size */ -void -write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, bool is_merge) -{ - size_t read_len = 0; - char *map_path = NULL; - /* header compression */ - int z_len = 0; - char *zheaders = NULL; - const char *errormsg = NULL; - - if (file->n_headers <= 0) - return; - - /* when running merge we must write headers into temp map */ - map_path = (is_merge) ? hdr_map->path_tmp : hdr_map->path; - read_len = (file->n_headers + 1) * sizeof(BackupPageHeader2); - - /* calculate checksums */ - INIT_FILE_CRC32(true, file->hdr_crc); - COMP_FILE_CRC32(true, file->hdr_crc, headers, read_len); - FIN_FILE_CRC32(true, file->hdr_crc); - - zheaders = pgut_malloc(read_len * 2); - memset(zheaders, 0, read_len * 2); - - /* compress headers */ - z_len = do_compress(zheaders, read_len * 2, headers, - read_len, ZLIB_COMPRESS, 1, &errormsg); - - /* writing to header map must be serialized */ - pthread_lock(&(hdr_map->mutex)); /* what if we crash while trying to obtain mutex? */ - - if (!hdr_map->fp) - { - elog(LOG, "Creating page header map \"%s\"", map_path); - - hdr_map->fp = fopen(map_path, PG_BINARY_A); - if (hdr_map->fp == NULL) - elog(ERROR, "Cannot open header file \"%s\": %s", - map_path, strerror(errno)); - - /* enable buffering for header file */ - hdr_map->buf = pgut_malloc(LARGE_CHUNK_SIZE); - setvbuf(hdr_map->fp, hdr_map->buf, _IOFBF, LARGE_CHUNK_SIZE); - - /* update file permission */ - if (chmod(map_path, FILE_PERMISSION) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", map_path, - strerror(errno)); - - file->hdr_off = 0; - } - else - file->hdr_off = hdr_map->offset; - - if (z_len <= 0) - { - if (errormsg) - elog(ERROR, "An error occured during compressing metadata for file \"%s\": %s", - file->rel_path, errormsg); - else - elog(ERROR, "An error occured during compressing metadata for file \"%s\": %i", - file->rel_path, z_len); - } - - elog(VERBOSE, "Writing headers for file \"%s\" offset: %llu, len: %i, crc: %u", - file->rel_path, file->hdr_off, z_len, file->hdr_crc); - - if (fwrite(zheaders, 1, z_len, hdr_map->fp) != z_len) - elog(ERROR, "Cannot write to file \"%s\": %s", map_path, strerror(errno)); - - file->hdr_size = z_len; /* save the length of compressed headers */ - hdr_map->offset += z_len; /* update current offset in map */ - - /* End critical section */ - pthread_mutex_unlock(&(hdr_map->mutex)); - - pg_free(zheaders); -} - -void -init_header_map(pgBackup *backup) -{ - backup->hdr_map.fp = NULL; - backup->hdr_map.buf = NULL; - join_path_components(backup->hdr_map.path, backup->root_dir, HEADER_MAP); - join_path_components(backup->hdr_map.path_tmp, backup->root_dir, HEADER_MAP_TMP); - backup->hdr_map.mutex = (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER; -} - -void -cleanup_header_map(HeaderMap *hdr_map) -{ - /* cleanup descriptor */ - if (hdr_map->fp && fclose(hdr_map->fp)) - elog(ERROR, "Cannot close file \"%s\"", hdr_map->path); - hdr_map->fp = NULL; - hdr_map->offset = 0; - pg_free(hdr_map->buf); - hdr_map->buf = NULL; -} diff --git a/src/datapagemap.c b/src/datapagemap.c deleted file mode 100644 index 7e4202a72..000000000 --- a/src/datapagemap.c +++ /dev/null @@ -1,113 +0,0 @@ -/*------------------------------------------------------------------------- - * - * datapagemap.c - * A data structure for keeping track of data pages that have changed. - * - * This is a fairly simple bitmap. - * - * Copyright (c) 2013-2019, PostgreSQL Global Development Group - * - *------------------------------------------------------------------------- - */ - -#include "postgres_fe.h" - -#include "datapagemap.h" - -struct datapagemap_iterator -{ - datapagemap_t *map; - BlockNumber nextblkno; -}; - -/***** - * Public functions - */ - -/* - * Add a block to the bitmap. - */ -void -datapagemap_add(datapagemap_t *map, BlockNumber blkno) -{ - int offset; - int bitno; - int oldsize = map->bitmapsize; - - offset = blkno / 8; - bitno = blkno % 8; - - /* enlarge or create bitmap if needed */ - if (oldsize <= offset) - { - int newsize; - - /* - * The minimum to hold the new bit is offset + 1. But add some - * headroom, so that we don't need to repeatedly enlarge the bitmap in - * the common case that blocks are modified in order, from beginning - * of a relation to the end. - */ - newsize = (oldsize == 0) ? 16 : oldsize; - while (newsize <= offset) { - newsize <<= 1; - } - - map->bitmap = pg_realloc(map->bitmap, newsize); - - /* zero out the newly allocated region */ - memset(&map->bitmap[oldsize], 0, newsize - oldsize); - - map->bitmapsize = newsize; - } - - /* Set the bit */ - map->bitmap[offset] |= (1 << bitno); -} - -/* - * Start iterating through all entries in the page map. - * - * After datapagemap_iterate, call datapagemap_next to return the entries, - * until it returns false. After you're done, use pg_free() to destroy the - * iterator. - */ -datapagemap_iterator_t * -datapagemap_iterate(datapagemap_t *map) -{ - datapagemap_iterator_t *iter; - - iter = pg_malloc(sizeof(datapagemap_iterator_t)); - iter->map = map; - iter->nextblkno = 0; - - return iter; -} - -bool -datapagemap_next(datapagemap_iterator_t *iter, BlockNumber *blkno) -{ - datapagemap_t *map = iter->map; - - for (;;) - { - BlockNumber blk = iter->nextblkno; - int nextoff = blk / 8; - int bitno = blk % 8; - - if (nextoff >= map->bitmapsize) - break; - - iter->nextblkno++; - - if (map->bitmap[nextoff] & (1 << bitno)) - { - *blkno = blk; - return true; - } - } - - /* no more set bits in this bitmap. */ - return false; -} - diff --git a/src/datapagemap.h b/src/datapagemap.h deleted file mode 100644 index 6ad7a6204..000000000 --- a/src/datapagemap.h +++ /dev/null @@ -1,34 +0,0 @@ -/*------------------------------------------------------------------------- - * - * datapagemap.h - * - * Copyright (c) 2013-2019, PostgreSQL Global Development Group - * - *------------------------------------------------------------------------- - */ -#ifndef DATAPAGEMAP_H -#define DATAPAGEMAP_H - -#if PG_VERSION_NUM < 160000 -#include "storage/relfilenode.h" -#else -#include "storage/relfilelocator.h" -#define RelFileNode RelFileLocator -#endif -#include "storage/block.h" - - -struct datapagemap -{ - char *bitmap; - int bitmapsize; -}; - -typedef struct datapagemap datapagemap_t; -typedef struct datapagemap_iterator datapagemap_iterator_t; - -extern void datapagemap_add(datapagemap_t *map, BlockNumber blkno); -extern datapagemap_iterator_t *datapagemap_iterate(datapagemap_t *map); -extern bool datapagemap_next(datapagemap_iterator_t *iter, BlockNumber *blkno); - -#endif /* DATAPAGEMAP_H */ diff --git a/src/delete.c b/src/delete.c deleted file mode 100644 index f48ecc95f..000000000 --- a/src/delete.c +++ /dev/null @@ -1,1130 +0,0 @@ -/*------------------------------------------------------------------------- - * - * delete.c: delete backup files. - * - * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -#include -#include -#include - -static void delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timelineInfo *tli, - uint32 xlog_seg_size, bool dry_run); -static void do_retention_internal(parray *backup_list, parray *to_keep_list, - parray *to_purge_list); -static void do_retention_merge(InstanceState *instanceState, parray *backup_list, - parray *to_keep_list, parray *to_purge_list, - bool no_validate, bool no_sync); -static void do_retention_purge(parray *to_keep_list, parray *to_purge_list); -static void do_retention_wal(InstanceState *instanceState, bool dry_run); - -// TODO: more useful messages for dry run. -static bool backup_deleted = false; /* At least one backup was deleted */ -static bool backup_merged = false; /* At least one merge was enacted */ -static bool wal_deleted = false; /* At least one WAL segments was deleted */ - -void -do_delete(InstanceState *instanceState, time_t backup_id) -{ - int i; - parray *backup_list, - *delete_list; - pgBackup *target_backup = NULL; - int64 size_to_delete = 0; - char size_to_delete_pretty[20]; - - /* Get complete list of backups */ - backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); - - delete_list = parray_new(); - - /* Find backup to be deleted and make increment backups array to be deleted */ - for (i = 0; i < parray_num(backup_list); i++) - { - pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - - if (backup->start_time == backup_id) - { - target_backup = backup; - break; - } - } - - /* sanity */ - if (!target_backup) - elog(ERROR, "Failed to find backup %s, cannot delete", base36enc(backup_id)); - - /* form delete list */ - for (i = 0; i < parray_num(backup_list); i++) - { - pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - - /* check if backup is descendant of delete target */ - if (is_parent(target_backup->start_time, backup, true)) - { - parray_append(delete_list, backup); - - elog(LOG, "Backup %s %s be deleted", - backup_id_of(backup), dry_run? "can":"will"); - - size_to_delete += backup->data_bytes; - if (backup->stream) - size_to_delete += backup->wal_bytes; - } - } - - /* Report the resident size to delete */ - if (size_to_delete >= 0) - { - pretty_size(size_to_delete, size_to_delete_pretty, lengthof(size_to_delete_pretty)); - elog(INFO, "Resident data size to free by delete of backup %s : %s", - backup_id_of(target_backup), size_to_delete_pretty); - } - - if (!dry_run) - { - /* Lock marked for delete backups */ - catalog_lock_backup_list(delete_list, parray_num(delete_list) - 1, 0, false, true); - - /* Delete backups from the end of list */ - for (i = (int) parray_num(delete_list) - 1; i >= 0; i--) - { - pgBackup *backup = (pgBackup *) parray_get(delete_list, (size_t) i); - - if (interrupted) - elog(ERROR, "interrupted during delete backup"); - - delete_backup_files(backup); - } - } - - /* Clean WAL segments */ - if (delete_wal) - do_retention_wal(instanceState, dry_run); - - /* cleanup */ - parray_free(delete_list); - parray_walk(backup_list, pgBackupFree); - parray_free(backup_list); -} - -/* - * Merge and purge backups by retention policy. Retention policy is configured by - * retention_redundancy and retention_window variables. - * - * Invalid backups handled in Oracle style, so invalid backups are ignored - * for the purpose of retention fulfillment, - * i.e. CORRUPT full backup do not taken in account when determine - * which FULL backup should be keeped for redundancy obligation(only valid do), - * but if invalid backup is not guarded by retention - it is removed - */ -void do_retention(InstanceState *instanceState, bool no_validate, bool no_sync) -{ - parray *backup_list = NULL; - parray *to_keep_list = parray_new(); - parray *to_purge_list = parray_new(); - - bool retention_is_set = false; /* At least one retention policy is set */ - bool backup_list_is_empty = false; - - backup_deleted = false; - backup_merged = false; - - /* For now retention is possible only locally */ - MyLocation = FIO_LOCAL_HOST; - - /* Get a complete list of backups. */ - backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); - - if (parray_num(backup_list) == 0) - backup_list_is_empty = true; - - if (delete_expired || merge_expired) - { - if (instance_config.retention_redundancy > 0) - elog(LOG, "REDUNDANCY=%u", instance_config.retention_redundancy); - if (instance_config.retention_window > 0) - elog(LOG, "WINDOW=%u", instance_config.retention_window); - - if (instance_config.retention_redundancy == 0 && - instance_config.retention_window == 0) - { - /* Retention is disabled but we still can cleanup wal */ - elog(WARNING, "Retention policy is not set"); - if (!delete_wal) - { - parray_walk(backup_list, pgBackupFree); - parray_free(backup_list); - parray_free(to_keep_list); - parray_free(to_purge_list); - return; - } - } - else - /* At least one retention policy is active */ - retention_is_set = true; - } - - if (retention_is_set && backup_list_is_empty) - elog(WARNING, "Backup list is empty, retention purge and merge are problematic"); - - /* Populate purge and keep lists, and show retention state messages */ - if (retention_is_set && !backup_list_is_empty) - do_retention_internal(backup_list, to_keep_list, to_purge_list); - - if (merge_expired && !dry_run && !backup_list_is_empty) - do_retention_merge(instanceState, backup_list, to_keep_list, to_purge_list, no_validate, no_sync); - - if (delete_expired && !dry_run && !backup_list_is_empty) - do_retention_purge(to_keep_list, to_purge_list); - - /* TODO: some sort of dry run for delete_wal */ - if (delete_wal) - do_retention_wal(instanceState, dry_run); - - /* TODO: consider dry-run flag */ - - if (!backup_merged) - elog(INFO, "There are no backups to merge by retention policy"); - - if (backup_deleted) - elog(INFO, "Purging finished"); - else - elog(INFO, "There are no backups to delete by retention policy"); - - if (!wal_deleted) - elog(INFO, "There is no WAL to purge by retention policy"); - - /* Cleanup */ - parray_walk(backup_list, pgBackupFree); - parray_free(backup_list); - parray_free(to_keep_list); - parray_free(to_purge_list); -} - -/* Evaluate every backup by retention policies and populate purge and keep lists. - * Also for every backup print its status ('Active' or 'Expired') according - * to active retention policies. - */ -static void -do_retention_internal(parray *backup_list, parray *to_keep_list, parray *to_purge_list) -{ - int i; - - parray *redundancy_full_backup_list = NULL; - - /* For retention calculation */ - uint32 n_full_backups = 0; - int cur_full_backup_num = 0; - time_t days_threshold = 0; - - /* For fancy reporting */ - uint32 actual_window = 0; - - /* Calculate n_full_backups and days_threshold */ - if (instance_config.retention_redundancy > 0) - { - for (i = 0; i < parray_num(backup_list); i++) - { - pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - - if (backup->backup_mode == BACKUP_MODE_FULL) - { - /* Add every FULL backup that satisfy Redundancy policy to separate list */ - if (n_full_backups < instance_config.retention_redundancy) - { - if (!redundancy_full_backup_list) - redundancy_full_backup_list = parray_new(); - - parray_append(redundancy_full_backup_list, backup); - } - - /* Consider only valid FULL backups for Redundancy fulfillment */ - if (backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE) - { - n_full_backups++; - } - } - } - /* Sort list of full backups to keep */ - if (redundancy_full_backup_list) - parray_qsort(redundancy_full_backup_list, pgBackupCompareIdDesc); - } - - if (instance_config.retention_window > 0) - { - days_threshold = current_time - - (instance_config.retention_window * 60 * 60 * 24); - } - - elog(INFO, "Evaluate backups by retention"); - for (i = (int) parray_num(backup_list) - 1; i >= 0; i--) - { - - bool redundancy_keep = false; - time_t backup_time = 0; - pgBackup *backup = (pgBackup *) parray_get(backup_list, (size_t) i); - - /* check if backup`s FULL ancestor is in redundancy list */ - if (redundancy_full_backup_list) - { - pgBackup *full_backup = find_parent_full_backup(backup); - - if (full_backup && parray_bsearch(redundancy_full_backup_list, - full_backup, - pgBackupCompareIdDesc)) - redundancy_keep = true; - } - - /* Remember the serial number of latest valid FULL backup */ - if (backup->backup_mode == BACKUP_MODE_FULL && - (backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE)) - { - cur_full_backup_num++; - } - - /* Invalid and running backups most likely to have recovery_time == 0, - * so in this case use start_time instead. - */ - if (backup->recovery_time) - backup_time = backup->recovery_time; - else - backup_time = backup->start_time; - - /* Check if backup in needed by retention policy */ - if ((days_threshold == 0 || (days_threshold > backup_time)) && - (instance_config.retention_redundancy == 0 || !redundancy_keep)) - { - /* This backup is not guarded by retention - * - * Redundancy = 1 - * FULL CORRUPT in retention (not count toward redundancy limit) - * FULL in retention - * ------retention redundancy ------- - * PAGE3 in retention - * ------retention window ----------- - * PAGE2 out of retention - * PAGE1 out of retention - * FULL out of retention <- We are here - * FULL CORRUPT out of retention - */ - - /* Save backup from purge if backup is pinned and - * expire date is not yet due. - */ - if ((backup->expire_time > 0) && - (backup->expire_time > current_time)) - { - char expire_timestamp[100]; - time2iso(expire_timestamp, lengthof(expire_timestamp), backup->expire_time, false); - - elog(LOG, "Backup %s is pinned until '%s', retain", - backup_id_of(backup), expire_timestamp); - continue; - } - - /* Add backup to purge_list */ - elog(VERBOSE, "Mark backup %s for purge.", backup_id_of(backup)); - parray_append(to_purge_list, backup); - continue; - } - } - - /* sort keep_list and purge list */ - parray_qsort(to_keep_list, pgBackupCompareIdDesc); - parray_qsort(to_purge_list, pgBackupCompareIdDesc); - - /* FULL - * PAGE - * PAGE <- Only such backups must go into keep list - ---------retention window ---- - * PAGE - * FULL - * PAGE - * FULL - */ - - for (i = 0; i < parray_num(backup_list); i++) - { - pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - - /* Do not keep invalid backups by retention - * Turns out it was not a very good idea - [Issue #114] - */ - //if (backup->status != BACKUP_STATUS_OK && - // backup->status != BACKUP_STATUS_DONE) - // continue; - - /* only incremental backups should be in keep list */ - if (backup->backup_mode == BACKUP_MODE_FULL) - continue; - - /* orphan backup cannot be in keep list */ - if (!backup->parent_backup_link) - continue; - - /* skip if backup already in purge list */ - if (parray_bsearch(to_purge_list, backup, pgBackupCompareIdDesc)) - continue; - - /* if parent in purge_list, add backup to keep list */ - if (parray_bsearch(to_purge_list, - backup->parent_backup_link, - pgBackupCompareIdDesc)) - { - /* make keep list a bit more compact */ - parray_append(to_keep_list, backup); - continue; - } - } - - /* Message about retention state of backups - * TODO: message is ugly, rewrite it to something like show table in stdout. - */ - - cur_full_backup_num = 1; - for (i = 0; i < parray_num(backup_list); i++) - { - char *action = "Active"; - uint32 pinning_window = 0; - - pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - - if (parray_bsearch(to_purge_list, backup, pgBackupCompareIdDesc)) - action = "Expired"; - - if (backup->recovery_time == 0) - actual_window = 0; - else - actual_window = (current_time - backup->recovery_time)/(3600 * 24); - - /* For pinned backups show expire date */ - if (backup->expire_time > 0 && backup->expire_time > backup->recovery_time) - pinning_window = (backup->expire_time - backup->recovery_time)/(3600 * 24); - - /* TODO: add ancestor(chain full backup) ID */ - elog(INFO, "Backup %s, mode: %s, status: %s. Redundancy: %i/%i, Time Window: %ud/%ud. %s", - backup_id_of(backup), - pgBackupGetBackupMode(backup, false), - status2str(backup->status), - cur_full_backup_num, - instance_config.retention_redundancy, - actual_window, - pinning_window ? pinning_window : instance_config.retention_window, - action); - - /* Only valid full backups are count to something */ - if (backup->backup_mode == BACKUP_MODE_FULL && - (backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE)) - cur_full_backup_num++; - } -} - -/* Merge partially expired incremental chains */ -static void -do_retention_merge(InstanceState *instanceState, parray *backup_list, - parray *to_keep_list, parray *to_purge_list, - bool no_validate, bool no_sync) -{ - int i; - int j; - - /* IMPORTANT: we can merge to only those FULL backup, that is NOT - * guarded by retention and final target of such merge must be - * an incremental backup that is guarded by retention !!! - * - * PAGE4 E - * PAGE3 D - --------retention window --- - * PAGE2 C - * PAGE1 B - * FULL A - * - * after retention merge: - * PAGE4 E - * FULL D - */ - - /* Merging happens here */ - for (i = 0; i < parray_num(to_keep_list); i++) - { - pgBackup *full_backup = NULL; - parray *merge_list = NULL; - - pgBackup *keep_backup = (pgBackup *) parray_get(to_keep_list, i); - - /* keep list may shrink during merge */ - if (!keep_backup) - continue; - - elog(INFO, "Consider backup %s for merge", backup_id_of(keep_backup)); - - /* Got valid incremental backup, find its FULL ancestor */ - full_backup = find_parent_full_backup(keep_backup); - - /* Failed to find parent */ - if (!full_backup) - { - elog(WARNING, "Failed to find FULL parent for %s", backup_id_of(keep_backup)); - continue; - } - - /* Check that ancestor is in purge_list */ - if (!parray_bsearch(to_purge_list, - full_backup, - pgBackupCompareIdDesc)) - { - elog(WARNING, "Skip backup %s for merging, " - "because his FULL parent is not marked for purge", backup_id_of(keep_backup)); - continue; - } - - /* FULL backup in purge list, thanks to compacting of keep_list current backup is - * final target for merge, but there could be intermediate incremental - * backups from purge_list. - */ - - elog(INFO, "Merge incremental chain between full backup %s and backup %s", - backup_id_of(full_backup), - backup_id_of(keep_backup)); - - merge_list = parray_new(); - - /* Form up a merge list */ - while (keep_backup->parent_backup_link) - { - parray_append(merge_list, keep_backup); - keep_backup = keep_backup->parent_backup_link; - } - - /* sanity */ - if (!merge_list) - continue; - - /* sanity */ - if (parray_num(merge_list) == 0) - { - parray_free(merge_list); - continue; - } - - /* In the end add FULL backup for easy locking */ - parray_append(merge_list, full_backup); - - /* Remove FULL backup from purge list */ - parray_rm(to_purge_list, full_backup, pgBackupCompareId); - - /* Lock merge chain */ - catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0, true, true); - - /* Consider this extreme case */ - // PAGEa1 PAGEb1 both valid - // \ / - // FULL - - /* Check that FULL backup do not has multiple descendants - * full_backup always point to current full_backup after merge - */ -// if (is_prolific(backup_list, full_backup)) -// { -// elog(WARNING, "Backup %s has multiple valid descendants. " -// "Automatic merge is not possible.", backup_id_of(full_backup)); -// } - - /* Merge list example: - * 0 PAGE3 - * 1 PAGE2 - * 2 PAGE1 - * 3 FULL - * - * Merge incremental chain from PAGE3 into FULL. - */ - keep_backup = parray_get(merge_list, 0); - merge_chain(instanceState, merge_list, full_backup, keep_backup, no_validate, no_sync); - backup_merged = true; - - for (j = parray_num(merge_list) - 2; j >= 0; j--) - { - pgBackup *tmp_backup = (pgBackup *) parray_get(merge_list, j); - - /* Try to remove merged incremental backup from both keep and purge lists */ - parray_rm(to_purge_list, tmp_backup, pgBackupCompareId); - for (i = 0; i < parray_num(to_keep_list); i++) - if (parray_get(to_keep_list, i) == tmp_backup) - { - parray_set(to_keep_list, i, NULL); - break; - } - } - if (!no_validate) - pgBackupValidate(full_backup, NULL); - if (full_backup->status == BACKUP_STATUS_CORRUPT) - elog(ERROR, "Merging of backup %s failed", backup_id_of(full_backup)); - - /* Cleanup */ - parray_free(merge_list); - } - - elog(INFO, "Retention merging finished"); - -} - -/* Purge expired backups */ -static void -do_retention_purge(parray *to_keep_list, parray *to_purge_list) -{ - int i; - int j; - - /* Remove backups by retention policy. Retention policy is configured by - * retention_redundancy and retention_window - * Remove only backups, that do not have children guarded by retention - * - * TODO: We do not consider the situation if child is marked for purge - * but parent isn`t. Maybe something bad happened with time on server? - */ - - for (j = 0; j < parray_num(to_purge_list); j++) - { - bool purge = true; - - pgBackup *delete_backup = (pgBackup *) parray_get(to_purge_list, j); - - elog(LOG, "Consider backup %s for purge", - backup_id_of(delete_backup)); - - /* Evaluate marked for delete backup against every backup in keep list. - * If marked for delete backup is recognized as parent of one of those, - * then this backup should not be deleted. - */ - for (i = 0; i < parray_num(to_keep_list); i++) - { - pgBackup *keep_backup = (pgBackup *) parray_get(to_keep_list, i); - - /* item could have been nullified in merge */ - if (!keep_backup) - continue; - - /* Full backup cannot be a descendant */ - if (keep_backup->backup_mode == BACKUP_MODE_FULL) - continue; - - elog(LOG, "Check if backup %s is parent of backup %s", - backup_id_of(delete_backup), - backup_id_of(keep_backup)); - - if (is_parent(delete_backup->start_time, keep_backup, true)) - { - - /* We must not delete this backup, evict it from purge list */ - elog(LOG, "Retain backup %s because his " - "descendant %s is guarded by retention", - backup_id_of(delete_backup), - backup_id_of(keep_backup)); - - purge = false; - break; - } - } - - /* Retain backup */ - if (!purge) - continue; - - /* Actual purge */ - if (!lock_backup(delete_backup, false, true)) - { - /* If the backup still is used, do not interrupt and go to the next */ - elog(WARNING, "Cannot lock backup %s directory, skip purging", - backup_id_of(delete_backup)); - continue; - } - - /* Delete backup and update status to DELETED */ - delete_backup_files(delete_backup); - backup_deleted = true; - - } -} - -/* - * Purge WAL - * Iterate over timelines - * Look for WAL segment not reachable from existing backups - * and delete them. - */ -static void -do_retention_wal(InstanceState *instanceState, bool dry_run) -{ - parray *tli_list; - int i; - - //TODO check that instanceState is not NULL - tli_list = catalog_get_timelines(instanceState, &instance_config); - - for (i = 0; i < parray_num(tli_list); i++) - { - timelineInfo *tlinfo = (timelineInfo *) parray_get(tli_list, i); - - /* - * Empty timeline (only mentioned in timeline history file) - * has nothing to cleanup. - */ - if (tlinfo->n_xlog_files == 0 && parray_num(tlinfo->xlog_filelist) == 0) - continue; - - /* - * If closest backup exists, then timeline is reachable from - * at least one backup and no file should be removed. - * Unless wal-depth is enabled. - */ - if ((tlinfo->closest_backup) && instance_config.wal_depth == 0) - continue; - - /* WAL retention keeps this timeline from purge */ - if (tlinfo->anchor_tli > 0 && tlinfo->anchor_tli != tlinfo->tli) - continue; - - /* - * Purge all WAL segments before START LSN of oldest backup. - * If timeline doesn't have a backup, then whole timeline - * can be safely purged. - * Note, that oldest_backup is not necessarily valid here, - * but still we keep wal for it. - * If wal-depth is enabled then use anchor_lsn instead - * of oldest_backup. - */ - if (tlinfo->oldest_backup) - { - if (!(XLogRecPtrIsInvalid(tlinfo->anchor_lsn))) - { - delete_walfiles_in_tli(instanceState, tlinfo->anchor_lsn, - tlinfo, instance_config.xlog_seg_size, dry_run); - } - else - { - delete_walfiles_in_tli(instanceState, tlinfo->oldest_backup->start_lsn, - tlinfo, instance_config.xlog_seg_size, dry_run); - } - } - else - { - if (!(XLogRecPtrIsInvalid(tlinfo->anchor_lsn))) - delete_walfiles_in_tli(instanceState, tlinfo->anchor_lsn, - tlinfo, instance_config.xlog_seg_size, dry_run); - else - delete_walfiles_in_tli(instanceState, InvalidXLogRecPtr, - tlinfo, instance_config.xlog_seg_size, dry_run); - } - } -} - -/* - * Delete backup files of the backup and update the status of the backup to - * BACKUP_STATUS_DELETED. - * TODO: delete files on multiple threads - */ -void -delete_backup_files(pgBackup *backup) -{ - size_t i; - char timestamp[100]; - parray *files; - size_t num_files; - char full_path[MAXPGPATH]; - - /* - * If the backup was deleted already, there is nothing to do. - */ - if (backup->status == BACKUP_STATUS_DELETED) - { - elog(WARNING, "Backup %s already deleted", - backup_id_of(backup)); - return; - } - - if (backup->recovery_time) - time2iso(timestamp, lengthof(timestamp), backup->recovery_time, false); - else - time2iso(timestamp, lengthof(timestamp), backup->start_time, false); - - elog(INFO, "Delete: %s %s", - backup_id_of(backup), timestamp); - - /* - * Update STATUS to BACKUP_STATUS_DELETING in preparation for the case which - * the error occurs before deleting all backup files. - */ - write_backup_status(backup, BACKUP_STATUS_DELETING, false); - - /* list files to be deleted */ - files = parray_new(); - dir_list_file(files, backup->root_dir, false, false, true, false, false, 0, FIO_BACKUP_HOST); - - /* delete leaf node first */ - parray_qsort(files, pgFileCompareRelPathWithExternalDesc); - num_files = parray_num(files); - for (i = 0; i < num_files; i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - - join_path_components(full_path, backup->root_dir, file->rel_path); - - if (interrupted) - elog(ERROR, "interrupted during delete backup"); - - if (progress) - elog(INFO, "Progress: (%zd/%zd). Delete file \"%s\"", - i + 1, num_files, full_path); - - pgFileDelete(file->mode, full_path); - } - - parray_walk(files, pgFileFree); - parray_free(files); - backup->status = BACKUP_STATUS_DELETED; - - return; -} - -/* - * Purge WAL archive. One timeline at a time. - * If 'keep_lsn' is InvalidXLogRecPtr, then whole timeline can be purged - * If 'keep_lsn' is valid LSN, then every lesser segment can be purged. - * If 'dry_run' is set, then don`t actually delete anything. - * - * Case 1: - * archive is not empty, 'keep_lsn' is valid and we can delete something. - * Case 2: - * archive is not empty, 'keep_lsn' is valid and prevening us from deleting anything. - * Case 3: - * archive is not empty, 'keep_lsn' is invalid, drop all WAL files in archive, - * belonging to the timeline. - * Case 4: - * archive is empty, 'keep_lsn' is valid, assume corruption of WAL archive. - * Case 5: - * archive is empty, 'keep_lsn' is invalid, drop backup history files - * and partial WAL segments in archive. - * - * Q: Maybe we should stop treating partial WAL segments as second-class citizens? - */ -static void -delete_walfiles_in_tli(InstanceState *instanceState, XLogRecPtr keep_lsn, timelineInfo *tlinfo, - uint32 xlog_seg_size, bool dry_run) -{ - XLogSegNo FirstToDeleteSegNo; - XLogSegNo OldestToKeepSegNo = 0; - char first_to_del_str[MAXFNAMELEN]; - char oldest_to_keep_str[MAXFNAMELEN]; - int i; - size_t wal_size_logical = 0; - size_t wal_size_actual = 0; - char wal_pretty_size[20]; - bool purge_all = false; - - - /* Timeline is completely empty */ - if (parray_num(tlinfo->xlog_filelist) == 0) - { - elog(INFO, "Timeline %i is empty, nothing to remove", tlinfo->tli); - return; - } - - if (XLogRecPtrIsInvalid(keep_lsn)) - { - /* Drop all files in timeline */ - elog(INFO, "On timeline %i all files %s be removed", - tlinfo->tli, dry_run?"can":"will"); - FirstToDeleteSegNo = tlinfo->begin_segno; - OldestToKeepSegNo = tlinfo->end_segno; - purge_all = true; - } - else - { - /* Drop all segments between begin_segno and segment with keep_lsn (excluding) */ - FirstToDeleteSegNo = tlinfo->begin_segno; - GetXLogSegNo(keep_lsn, OldestToKeepSegNo, xlog_seg_size); - } - - if (OldestToKeepSegNo > 0 && OldestToKeepSegNo > FirstToDeleteSegNo) - { - /* translate segno number into human readable format */ - GetXLogFileName(first_to_del_str, tlinfo->tli, FirstToDeleteSegNo, xlog_seg_size); - GetXLogFileName(oldest_to_keep_str, tlinfo->tli, OldestToKeepSegNo, xlog_seg_size); - - elog(INFO, "On timeline %i WAL segments between %s and %s %s be removed", - tlinfo->tli, first_to_del_str, - oldest_to_keep_str, dry_run?"can":"will"); - } - - /* sanity */ - if (OldestToKeepSegNo > FirstToDeleteSegNo) - { - wal_size_logical = (OldestToKeepSegNo - FirstToDeleteSegNo) * xlog_seg_size; - - /* In case of 'purge all' scenario OldestToKeepSegNo will be deleted too */ - if (purge_all) - wal_size_logical += xlog_seg_size; - } - else if (OldestToKeepSegNo < FirstToDeleteSegNo) - { - /* It is actually possible for OldestToKeepSegNo to be less than FirstToDeleteSegNo - * in case of : - * 1. WAL archive corruption. - * 2. There is no actual WAL archive to speak of and - * 'keep_lsn' is coming from STREAM backup. - */ - - if (FirstToDeleteSegNo > 0 && OldestToKeepSegNo > 0) - { - GetXLogFileName(first_to_del_str, tlinfo->tli, FirstToDeleteSegNo, xlog_seg_size); - GetXLogFileName(oldest_to_keep_str, tlinfo->tli, OldestToKeepSegNo, xlog_seg_size); - - elog(LOG, "On timeline %i first segment %s is greater than oldest segment to keep %s", - tlinfo->tli, first_to_del_str, oldest_to_keep_str); - } - } - else if (OldestToKeepSegNo == FirstToDeleteSegNo && !purge_all) - { - /* 'Nothing to delete' scenario because of 'keep_lsn' - * with possible exception of partial and backup history files. - */ - elog(INFO, "Nothing to remove on timeline %i", tlinfo->tli); - } - - /* Report the logical size to delete */ - if (wal_size_logical > 0) - { - pretty_size(wal_size_logical, wal_pretty_size, lengthof(wal_pretty_size)); - elog(INFO, "Logical WAL size to remove on timeline %i : %s", - tlinfo->tli, wal_pretty_size); - } - - /* Calculate the actual size to delete */ - for (i = 0; i < parray_num(tlinfo->xlog_filelist); i++) - { - xlogFile *wal_file = (xlogFile *) parray_get(tlinfo->xlog_filelist, i); - - if (purge_all || wal_file->segno < OldestToKeepSegNo) - wal_size_actual += wal_file->file.size; - } - - /* Report the actual size to delete */ - if (wal_size_actual > 0) - { - pretty_size(wal_size_actual, wal_pretty_size, lengthof(wal_pretty_size)); - elog(INFO, "Resident WAL size to free on timeline %i : %s", - tlinfo->tli, wal_pretty_size); - } - - if (dry_run) - return; - - for (i = 0; i < parray_num(tlinfo->xlog_filelist); i++) - { - xlogFile *wal_file = (xlogFile *) parray_get(tlinfo->xlog_filelist, i); - - if (interrupted) - elog(ERROR, "interrupted during WAL archive purge"); - - /* Any segment equal or greater than EndSegNo must be kept - * unless it`s a 'purge all' scenario. - */ - if (purge_all || wal_file->segno < OldestToKeepSegNo) - { - char wal_fullpath[MAXPGPATH]; - - join_path_components(wal_fullpath, instanceState->instance_wal_subdir_path, wal_file->file.name); - - /* save segment from purging */ - if (wal_file->keep) - { - elog(VERBOSE, "Retain WAL segment \"%s\"", wal_fullpath); - continue; - } - - /* unlink segment */ - if (fio_unlink(wal_fullpath, FIO_BACKUP_HOST) < 0) - { - /* Missing file is not considered as error condition */ - if (errno != ENOENT) - elog(ERROR, "Could not remove file \"%s\": %s", - wal_fullpath, strerror(errno)); - } - else - { - if (wal_file->type == SEGMENT) - elog(VERBOSE, "Removed WAL segment \"%s\"", wal_fullpath); - else if (wal_file->type == TEMP_SEGMENT) - elog(VERBOSE, "Removed temp WAL segment \"%s\"", wal_fullpath); - else if (wal_file->type == PARTIAL_SEGMENT) - elog(VERBOSE, "Removed partial WAL segment \"%s\"", wal_fullpath); - else if (wal_file->type == BACKUP_HISTORY_FILE) - elog(VERBOSE, "Removed backup history file \"%s\"", wal_fullpath); - } - - wal_deleted = true; - } - } -} - - -/* Delete all backup files and wal files of given instance. */ -int -do_delete_instance(InstanceState *instanceState) -{ - parray *backup_list; - int i; - - /* Delete all backups. */ - backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); - - catalog_lock_backup_list(backup_list, 0, parray_num(backup_list) - 1, true, true); - - for (i = 0; i < parray_num(backup_list); i++) - { - pgBackup *backup = (pgBackup *) parray_get(backup_list, i); - delete_backup_files(backup); - } - - /* Cleanup */ - parray_walk(backup_list, pgBackupFree); - parray_free(backup_list); - - /* Delete all wal files. */ - pgut_rmtree(instanceState->instance_wal_subdir_path, false, true); - - /* Delete backup instance config file */ - if (remove(instanceState->instance_config_path)) - { - elog(ERROR, "Can't remove \"%s\": %s", instanceState->instance_config_path, - strerror(errno)); - } - - /* Delete instance root directories */ - if (rmdir(instanceState->instance_backup_subdir_path) != 0) - elog(ERROR, "Can't remove \"%s\": %s", instanceState->instance_backup_subdir_path, - strerror(errno)); - - if (rmdir(instanceState->instance_wal_subdir_path) != 0) - elog(ERROR, "Can't remove \"%s\": %s", instanceState->instance_wal_subdir_path, - strerror(errno)); - - elog(INFO, "Instance '%s' successfully deleted", instanceState->instance_name); - return 0; -} - -/* Delete all backups of given status in instance */ -void -do_delete_status(InstanceState *instanceState, InstanceConfig *instance_config, const char *status) -{ - int i; - parray *backup_list, *delete_list; - const char *pretty_status; - int n_deleted = 0, n_found = 0; - int64 size_to_delete = 0; - char size_to_delete_pretty[20]; - pgBackup *backup; - - BackupStatus status_for_delete = str2status(status); - delete_list = parray_new(); - - if (status_for_delete == BACKUP_STATUS_INVALID) - elog(ERROR, "Unknown value for '--status' option: '%s'", status); - - /* - * User may have provided status string in lower case, but - * we should print backup statuses consistently with show command, - * so convert it. - */ - pretty_status = status2str(status_for_delete); - - backup_list = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); - - if (parray_num(backup_list) == 0) - { - elog(WARNING, "Instance '%s' has no backups", instanceState->instance_name); - parray_free(delete_list); - parray_free(backup_list); - return; - } - - if (dry_run) - elog(INFO, "Deleting all backups with status '%s' in dry run mode", pretty_status); - else - elog(INFO, "Deleting all backups with status '%s'", pretty_status); - - /* Selects backups with specified status and their children into delete_list array. */ - for (i = 0; i < parray_num(backup_list); i++) - { - backup = (pgBackup *) parray_get(backup_list, i); - - if (backup->status == status_for_delete) - { - n_found++; - - /* incremental backup can be already in delete_list due to append_children() */ - if (parray_contains(delete_list, backup)) - continue; - parray_append(delete_list, backup); - - append_children(backup_list, backup, delete_list); - } - } - - parray_qsort(delete_list, pgBackupCompareIdDesc); - - /* delete and calculate free size from delete_list */ - for (i = 0; i < parray_num(delete_list); i++) - { - backup = (pgBackup *)parray_get(delete_list, i); - - elog(INFO, "Backup %s with status %s %s be deleted", - backup_id_of(backup), status2str(backup->status), dry_run ? "can" : "will"); - - size_to_delete += backup->data_bytes; - if (backup->stream) - size_to_delete += backup->wal_bytes; - - if (!dry_run && lock_backup(backup, false, true)) - delete_backup_files(backup); - - n_deleted++; - } - - /* Inform about data size to free */ - if (size_to_delete >= 0) - { - pretty_size(size_to_delete, size_to_delete_pretty, lengthof(size_to_delete_pretty)); - elog(INFO, "Resident data size to free by delete of %i backups: %s", - n_deleted, size_to_delete_pretty); - } - - /* delete selected backups */ - if (!dry_run && n_deleted > 0) - elog(INFO, "Successfully deleted %i %s from instance '%s'", - n_deleted, n_deleted == 1 ? "backup" : "backups", - instanceState->instance_name); - - - if (n_found == 0) - elog(WARNING, "Instance '%s' has no backups with status '%s'", - instanceState->instance_name, pretty_status); - - // we don`t do WAL purge here, because it is impossible to correctly handle - // dry-run case. - - /* Cleanup */ - parray_free(delete_list); - parray_walk(backup_list, pgBackupFree); - parray_free(backup_list); -} diff --git a/src/dir.c b/src/dir.c deleted file mode 100644 index 353ed2d43..000000000 --- a/src/dir.c +++ /dev/null @@ -1,1870 +0,0 @@ -/*------------------------------------------------------------------------- - * - * dir.c: directory operation utility. - * - * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2022, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include -#include "pg_probackup.h" -#include "utils/file.h" - - -#if PG_VERSION_NUM < 110000 -#include "catalog/catalog.h" -#endif -#include "catalog/pg_tablespace.h" - -#include -#include -#include - -#include "utils/configuration.h" - -/* - * The contents of these directories are removed or recreated during server - * start so they are not included in backups. The directories themselves are - * kept and included as empty to preserve access permissions. - */ -static const char *pgdata_exclude_dir[] = -{ - PG_XLOG_DIR, - /* - * Skip temporary statistics files. PG_STAT_TMP_DIR must be skipped even - * when stats_temp_directory is set because PGSS_TEXT_FILE is always created - * there. - */ - "pg_stat_tmp", - "pgsql_tmp", - - /* - * It is generally not useful to backup the contents of this directory even - * if the intention is to restore to another master. See backup.sgml for a - * more detailed description. - */ - "pg_replslot", - - /* Contents removed on startup, see dsm_cleanup_for_mmap(). */ - "pg_dynshmem", - - /* Contents removed on startup, see AsyncShmemInit(). */ - "pg_notify", - - /* - * Old contents are loaded for possible debugging but are not required for - * normal operation, see OldSerXidInit(). - */ - "pg_serial", - - /* Contents removed on startup, see DeleteAllExportedSnapshotFiles(). */ - "pg_snapshots", - - /* Contents zeroed on startup, see StartupSUBTRANS(). */ - "pg_subtrans", - - /* end of list */ - NULL, /* pg_log will be set later */ - NULL -}; - -static char *pgdata_exclude_files[] = -{ - /* Skip auto conf temporary file. */ - "postgresql.auto.conf.tmp", - - /* Skip current log file temporary file */ - "current_logfiles.tmp", - "recovery.conf", - "postmaster.pid", - "postmaster.opts", - "probackup_recovery.conf", - "recovery.signal", - "standby.signal", - NULL -}; - -static char *pgdata_exclude_files_non_exclusive[] = -{ - /*skip in non-exclusive backup */ - "backup_label", - "tablespace_map", - NULL -}; - -/* Tablespace mapping structures */ - -typedef struct TablespaceListCell -{ - struct TablespaceListCell *next; - char old_dir[MAXPGPATH]; - char new_dir[MAXPGPATH]; -} TablespaceListCell; - -typedef struct TablespaceList -{ - TablespaceListCell *head; - TablespaceListCell *tail; -} TablespaceList; - -typedef struct TablespaceCreatedListCell -{ - struct TablespaceCreatedListCell *next; - char link_name[MAXPGPATH]; - char linked_dir[MAXPGPATH]; -} TablespaceCreatedListCell; - -typedef struct TablespaceCreatedList -{ - TablespaceCreatedListCell *head; - TablespaceCreatedListCell *tail; -} TablespaceCreatedList; - -static char dir_check_file(pgFile *file, bool backup_logs); - -static void dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, - bool exclude, bool follow_symlink, bool backup_logs, - bool skip_hidden, int external_dir_num, fio_location location); -static void opt_path_map(ConfigOption *opt, const char *arg, - TablespaceList *list, const char *type); -static void cleanup_tablespace(const char *path); - -static void control_string_bad_format(const char* str); - - -/* Tablespace mapping */ -static TablespaceList tablespace_dirs = {NULL, NULL}; -/* Extra directories mapping */ -static TablespaceList external_remap_list = {NULL, NULL}; - -/* - * Create directory, also create parent directories if necessary. - * In strict mode treat already existing directory as error. - * Return values: - * 0 - ok - * -1 - error (check errno) - */ -int -dir_create_dir(const char *dir, mode_t mode, bool strict) -{ - char parent[MAXPGPATH]; - - strlcpy(parent, dir, MAXPGPATH); - get_parent_directory(parent); - - /* Create parent first */ - if (strlen(parent) > 0 && access(parent, F_OK) == -1) - dir_create_dir(parent, mode, false); - - /* Create directory */ - if (mkdir(dir, mode) == -1) - { - if (errno == EEXIST && !strict) /* already exist */ - return 0; - return -1; - } - - return 0; -} - -pgFile * -pgFileNew(const char *path, const char *rel_path, bool follow_symlink, - int external_dir_num, fio_location location) -{ - struct stat st; - pgFile *file; - - /* stat the file */ - if (fio_stat(path, &st, follow_symlink, location) < 0) - { - /* file not found is not an error case */ - if (errno == ENOENT) - return NULL; - elog(ERROR, "Cannot stat file \"%s\": %s", path, - strerror(errno)); - } - - file = pgFileInit(rel_path); - file->size = st.st_size; - file->mode = st.st_mode; - file->mtime = st.st_mtime; - file->external_dir_num = external_dir_num; - - return file; -} - -pgFile * -pgFileInit(const char *rel_path) -{ - pgFile *file; - char *file_name = NULL; - - file = (pgFile *) pgut_malloc(sizeof(pgFile)); - MemSet(file, 0, sizeof(pgFile)); - - file->rel_path = pgut_strdup(rel_path); - canonicalize_path(file->rel_path); - - /* Get file name from the path */ - file_name = last_dir_separator(file->rel_path); - - if (file_name == NULL) - file->name = file->rel_path; - else - { - file_name++; - file->name = file_name; - } - - /* Number of blocks readed during backup */ - file->n_blocks = BLOCKNUM_INVALID; - - /* Number of blocks backed up during backup */ - file->n_headers = 0; - - // May be add? - // pg_atomic_clear_flag(file->lock); - file->excluded = false; - return file; -} - -/* - * Delete file pointed by the pgFile. - * If the pgFile points directory, the directory must be empty. - */ -void -pgFileDelete(mode_t mode, const char *full_path) -{ - if (S_ISDIR(mode)) - { - if (rmdir(full_path) == -1) - { - if (errno == ENOENT) - return; - else if (errno == ENOTDIR) /* could be symbolic link */ - goto delete_file; - - elog(ERROR, "Cannot remove directory \"%s\": %s", - full_path, strerror(errno)); - } - return; - } - -delete_file: - if (remove(full_path) == -1) - { - if (errno == ENOENT) - return; - elog(ERROR, "Cannot remove file \"%s\": %s", full_path, - strerror(errno)); - } -} - -void -pgFileFree(void *file) -{ - pgFile *file_ptr; - - if (file == NULL) - return; - - file_ptr = (pgFile *) file; - - pfree(file_ptr->linked); - pfree(file_ptr->rel_path); - - pfree(file); -} - -/* Compare two pgFile with their path in ascending order of ASCII code. */ -int -pgFileMapComparePath(const void *f1, const void *f2) -{ - page_map_entry *f1p = *(page_map_entry **)f1; - page_map_entry *f2p = *(page_map_entry **)f2; - - return strcmp(f1p->path, f2p->path); -} - -/* Compare two pgFile with their name in ascending order of ASCII code. */ -int -pgFileCompareName(const void *f1, const void *f2) -{ - pgFile *f1p = *(pgFile **)f1; - pgFile *f2p = *(pgFile **)f2; - - return strcmp(f1p->name, f2p->name); -} - -/* Compare pgFile->name with string in ascending order of ASCII code. */ -int -pgFileCompareNameWithString(const void *f1, const void *f2) -{ - pgFile *f1p = *(pgFile **)f1; - char *f2s = *(char **)f2; - - return strcmp(f1p->name, f2s); -} - -/* Compare pgFile->rel_path with string in ascending order of ASCII code. */ -int -pgFileCompareRelPathWithString(const void *f1, const void *f2) -{ - pgFile *f1p = *(pgFile **)f1; - char *f2s = *(char **)f2; - - return strcmp(f1p->rel_path, f2s); -} - -/* - * Compare two pgFile with their relative path and external_dir_num in ascending - * order of ASСII code. - */ -int -pgFileCompareRelPathWithExternal(const void *f1, const void *f2) -{ - pgFile *f1p = *(pgFile **)f1; - pgFile *f2p = *(pgFile **)f2; - int res; - - res = strcmp(f1p->rel_path, f2p->rel_path); - if (res == 0) - { - if (f1p->external_dir_num > f2p->external_dir_num) - return 1; - else if (f1p->external_dir_num < f2p->external_dir_num) - return -1; - else - return 0; - } - return res; -} - -/* - * Compare two pgFile with their rel_path and external_dir_num - * in descending order of ASCII code. - */ -int -pgFileCompareRelPathWithExternalDesc(const void *f1, const void *f2) -{ - return -pgFileCompareRelPathWithExternal(f1, f2); -} - -/* Compare two pgFile with their linked directory path. */ -int -pgFileCompareLinked(const void *f1, const void *f2) -{ - pgFile *f1p = *(pgFile **)f1; - pgFile *f2p = *(pgFile **)f2; - - return strcmp(f1p->linked, f2p->linked); -} - -/* Compare two pgFile with their size */ -int -pgFileCompareSize(const void *f1, const void *f2) -{ - pgFile *f1p = *(pgFile **)f1; - pgFile *f2p = *(pgFile **)f2; - - if (f1p->size > f2p->size) - return 1; - else if (f1p->size < f2p->size) - return -1; - else - return 0; -} - -/* Compare two pgFile with their size in descending order */ -int -pgFileCompareSizeDesc(const void *f1, const void *f2) -{ - return -1 * pgFileCompareSize(f1, f2); -} - -int -pgCompareString(const void *str1, const void *str2) -{ - return strcmp(*(char **) str1, *(char **) str2); -} - -/* - * From bsearch(3): "The compar routine is expected to have two argu‐ - * ments which point to the key object and to an array member, in that order" - * But in practice this is opposite, so we took strlen from second string (search key) - * This is checked by tests.catchup.CatchupTest.test_catchup_with_exclude_path - */ -int -pgPrefixCompareString(const void *str1, const void *str2) -{ - const char *s1 = *(char **) str1; - const char *s2 = *(char **) str2; - return strncmp(s1, s2, strlen(s2)); -} - -/* Compare two Oids */ -int -pgCompareOid(const void *f1, const void *f2) -{ - Oid *v1 = *(Oid **) f1; - Oid *v2 = *(Oid **) f2; - - if (*v1 > *v2) - return 1; - else if (*v1 < *v2) - return -1; - else - return 0;} - - -void -db_map_entry_free(void *entry) -{ - db_map_entry *m = (db_map_entry *) entry; - - free(m->datname); - free(entry); -} - -/* - * List files, symbolic links and directories in the directory "root" and add - * pgFile objects to "files". We add "root" to "files" if add_root is true. - * - * When follow_symlink is true, symbolic link is ignored and only file or - * directory linked to will be listed. - * - * TODO: make it strictly local - */ -void -dir_list_file(parray *files, const char *root, bool exclude, bool follow_symlink, - bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num, - fio_location location) -{ - pgFile *file; - - file = pgFileNew(root, "", follow_symlink, external_dir_num, location); - if (file == NULL) - { - /* For external directory this is not ok */ - if (external_dir_num > 0) - elog(ERROR, "External directory is not found: \"%s\"", root); - else - return; - } - - if (!S_ISDIR(file->mode)) - { - if (external_dir_num > 0) - elog(ERROR, " --external-dirs option \"%s\": directory or symbolic link expected", - root); - else - elog(WARNING, "Skip \"%s\": unexpected file format", root); - return; - } - if (add_root) - parray_append(files, file); - - dir_list_file_internal(files, file, root, exclude, follow_symlink, - backup_logs, skip_hidden, external_dir_num, location); - - if (!add_root) - pgFileFree(file); -} - -#define CHECK_FALSE 0 -#define CHECK_TRUE 1 -#define CHECK_EXCLUDE_FALSE 2 - -/* - * Check file or directory. - * - * Check for exclude. - * Extract information about the file parsing its name. - * Skip files: - * - skip temp tables files - * - skip unlogged tables files - * Skip recursive tablespace content - * Set flags for: - * - database directories - * - datafiles - */ -static char -dir_check_file(pgFile *file, bool backup_logs) -{ - int i; - int sscanf_res; - bool in_tablespace = false; - - in_tablespace = path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path); - - /* Check if we need to exclude file by name */ - if (S_ISREG(file->mode)) - { - if (!exclusive_backup) - { - for (i = 0; pgdata_exclude_files_non_exclusive[i]; i++) - if (strcmp(file->rel_path, - pgdata_exclude_files_non_exclusive[i]) == 0) - { - /* Skip */ - elog(LOG, "Excluding file: %s", file->name); - return CHECK_FALSE; - } - } - - for (i = 0; pgdata_exclude_files[i]; i++) - if (strcmp(file->rel_path, pgdata_exclude_files[i]) == 0) - { - /* Skip */ - elog(LOG, "Excluding file: %s", file->name); - return CHECK_FALSE; - } - } - /* - * If the directory name is in the exclude list, do not list the - * contents. - */ - else if (S_ISDIR(file->mode) && !in_tablespace && file->external_dir_num == 0) - { - /* - * If the item in the exclude list starts with '/', compare to - * the absolute path of the directory. Otherwise compare to the - * directory name portion. - */ - for (i = 0; pgdata_exclude_dir[i]; i++) - { - /* exclude by dirname */ - if (strcmp(file->name, pgdata_exclude_dir[i]) == 0) - { - elog(LOG, "Excluding directory content: %s", file->rel_path); - return CHECK_EXCLUDE_FALSE; - } - } - - if (!backup_logs) - { - if (strcmp(file->rel_path, PG_LOG_DIR) == 0) - { - /* Skip */ - elog(LOG, "Excluding directory content: %s", file->rel_path); - return CHECK_EXCLUDE_FALSE; - } - } - } - - /* - * Do not copy tablespaces twice. It may happen if the tablespace is located - * inside the PGDATA. - */ - if (S_ISDIR(file->mode) && - strcmp(file->name, TABLESPACE_VERSION_DIRECTORY) == 0) - { - Oid tblspcOid; - char tmp_rel_path[MAXPGPATH]; - - /* - * Valid path for the tablespace is - * pg_tblspc/tblsOid/TABLESPACE_VERSION_DIRECTORY - */ - if (!path_is_prefix_of_path(PG_TBLSPC_DIR, file->rel_path)) - return CHECK_FALSE; - sscanf_res = sscanf(file->rel_path, PG_TBLSPC_DIR "/%u/%s", - &tblspcOid, tmp_rel_path); - if (sscanf_res == 0) - return CHECK_FALSE; - } - - if (in_tablespace) - { - char tmp_rel_path[MAXPGPATH]; - - sscanf_res = sscanf(file->rel_path, PG_TBLSPC_DIR "/%u/%[^/]/%u/", - &(file->tblspcOid), tmp_rel_path, - &(file->dbOid)); - - /* - * We should skip other files and directories rather than - * TABLESPACE_VERSION_DIRECTORY, if this is recursive tablespace. - */ - if (sscanf_res == 2 && strcmp(tmp_rel_path, TABLESPACE_VERSION_DIRECTORY) != 0) - return CHECK_FALSE; - } - else if (path_is_prefix_of_path("global", file->rel_path)) - { - file->tblspcOid = GLOBALTABLESPACE_OID; - } - else if (path_is_prefix_of_path("base", file->rel_path)) - { - file->tblspcOid = DEFAULTTABLESPACE_OID; - - sscanf(file->rel_path, "base/%u/", &(file->dbOid)); - } - - /* Do not backup ptrack_init files */ - if (S_ISREG(file->mode) && strcmp(file->name, "ptrack_init") == 0) - return CHECK_FALSE; - - /* - * Check files located inside database directories including directory - * 'global' - */ - if (S_ISREG(file->mode) && file->tblspcOid != 0 && - file->name && file->name[0]) - { - if (strcmp(file->name, "pg_internal.init") == 0) - return CHECK_FALSE; - /* Do not backup ptrack2.x temp map files */ -// else if (strcmp(file->name, "ptrack.map") == 0) -// return CHECK_FALSE; - else if (strcmp(file->name, "ptrack.map.mmap") == 0) - return CHECK_FALSE; - else if (strcmp(file->name, "ptrack.map.tmp") == 0) - return CHECK_FALSE; - /* Do not backup temp files */ - else if (file->name[0] == 't' && isdigit(file->name[1])) - return CHECK_FALSE; - else if (isdigit(file->name[0])) - { - set_forkname(file); - - if (file->forkName == ptrack) /* Compatibility with left-overs from ptrack1 */ - return CHECK_FALSE; - } - } - - return CHECK_TRUE; -} - -/* - * List files in parent->path directory. If "exclude" is true do not add into - * "files" files from pgdata_exclude_files and directories from - * pgdata_exclude_dir. - * - * TODO: should we check for interrupt here ? - */ -static void -dir_list_file_internal(parray *files, pgFile *parent, const char *parent_dir, - bool exclude, bool follow_symlink, bool backup_logs, - bool skip_hidden, int external_dir_num, fio_location location) -{ - DIR *dir; - struct dirent *dent; - - if (!S_ISDIR(parent->mode)) - elog(ERROR, "\"%s\" is not a directory", parent_dir); - - /* Open directory and list contents */ - dir = fio_opendir(parent_dir, location); - if (dir == NULL) - { - if (errno == ENOENT) - { - /* Maybe the directory was removed */ - return; - } - elog(ERROR, "Cannot open directory \"%s\": %s", - parent_dir, strerror(errno)); - } - - errno = 0; - while ((dent = fio_readdir(dir))) - { - pgFile *file; - char child[MAXPGPATH]; - char rel_child[MAXPGPATH]; - char check_res; - - join_path_components(child, parent_dir, dent->d_name); - join_path_components(rel_child, parent->rel_path, dent->d_name); - - file = pgFileNew(child, rel_child, follow_symlink, external_dir_num, - location); - if (file == NULL) - continue; - - /* Skip entries point current dir or parent dir */ - if (S_ISDIR(file->mode) && - (strcmp(dent->d_name, ".") == 0 || strcmp(dent->d_name, "..") == 0)) - { - pgFileFree(file); - continue; - } - - /* skip hidden files and directories */ - if (skip_hidden && file->name[0] == '.') - { - elog(WARNING, "Skip hidden file: '%s'", child); - pgFileFree(file); - continue; - } - - /* - * Add only files, directories and links. Skip sockets and other - * unexpected file formats. - */ - if (!S_ISDIR(file->mode) && !S_ISREG(file->mode)) - { - elog(WARNING, "Skip '%s': unexpected file format", child); - pgFileFree(file); - continue; - } - - if (exclude) - { - check_res = dir_check_file(file, backup_logs); - if (check_res == CHECK_FALSE) - { - /* Skip */ - pgFileFree(file); - continue; - } - else if (check_res == CHECK_EXCLUDE_FALSE) - { - /* We add the directory itself which content was excluded */ - parray_append(files, file); - continue; - } - } - - parray_append(files, file); - - /* - * If the entry is a directory call dir_list_file_internal() - * recursively. - */ - if (S_ISDIR(file->mode)) - dir_list_file_internal(files, file, child, exclude, follow_symlink, - backup_logs, skip_hidden, external_dir_num, location); - } - - if (errno && errno != ENOENT) - { - int errno_tmp = errno; - fio_closedir(dir); - elog(ERROR, "Cannot read directory \"%s\": %s", - parent_dir, strerror(errno_tmp)); - } - fio_closedir(dir); -} - -/* - * Retrieve tablespace path, either relocated or original depending on whether - * -T was passed or not. - * - * Copy of function get_tablespace_mapping() from pg_basebackup.c. - */ -const char * -get_tablespace_mapping(const char *dir) -{ - TablespaceListCell *cell; - - for (cell = tablespace_dirs.head; cell; cell = cell->next) - if (strcmp(dir, cell->old_dir) == 0) - return cell->new_dir; - - return dir; -} - -/* - * Split argument into old_dir and new_dir and append to mapping - * list. - * - * Copy of function tablespace_list_append() from pg_basebackup.c. - */ -static void -opt_path_map(ConfigOption *opt, const char *arg, TablespaceList *list, - const char *type) -{ - TablespaceListCell *cell = pgut_new(TablespaceListCell); - char *dst; - char *dst_ptr; - const char *arg_ptr; - - memset(cell, 0, sizeof(TablespaceListCell)); - dst_ptr = dst = cell->old_dir; - for (arg_ptr = arg; *arg_ptr; arg_ptr++) - { - if (dst_ptr - dst >= MAXPGPATH) - elog(ERROR, "Directory name too long"); - - if (*arg_ptr == '\\' && *(arg_ptr + 1) == '=') - ; /* skip backslash escaping = */ - else if (*arg_ptr == '=' && (arg_ptr == arg || *(arg_ptr - 1) != '\\')) - { - if (*cell->new_dir) - elog(ERROR, "Multiple \"=\" signs in %s mapping\n", type); - else - dst = dst_ptr = cell->new_dir; - } - else - *dst_ptr++ = *arg_ptr; - } - - if (!*cell->old_dir || !*cell->new_dir) - elog(ERROR, "Invalid %s mapping format \"%s\", " - "must be \"OLDDIR=NEWDIR\"", type, arg); - canonicalize_path(cell->old_dir); - canonicalize_path(cell->new_dir); - - /* - * This check isn't absolutely necessary. But all tablespaces are created - * with absolute directories, so specifying a non-absolute path here would - * just never match, possibly confusing users. It's also good to be - * consistent with the new_dir check. - */ - if (!is_absolute_path(cell->old_dir)) - elog(ERROR, "Old directory is not an absolute path in %s mapping: %s\n", - type, cell->old_dir); - - if (!is_absolute_path(cell->new_dir)) - elog(ERROR, "New directory is not an absolute path in %s mapping: %s\n", - type, cell->new_dir); - - if (list->tail) - list->tail->next = cell; - else - list->head = cell; - list->tail = cell; -} - -/* Parse tablespace mapping */ -void -opt_tablespace_map(ConfigOption *opt, const char *arg) -{ - opt_path_map(opt, arg, &tablespace_dirs, "tablespace"); -} - -/* Parse external directories mapping */ -void -opt_externaldir_map(ConfigOption *opt, const char *arg) -{ - opt_path_map(opt, arg, &external_remap_list, "external directory"); -} - -/* - * Create directories from **dest_files** in **data_dir**. - * - * If **extract_tablespaces** is true then try to extract tablespace data - * directories into their initial path using tablespace_map file. - * Use **backup_dir** for tablespace_map extracting. - * - * Enforce permissions from backup_content.control. The only - * problem now is with PGDATA itself. - * TODO: we must preserve PGDATA permissions somewhere. Is it actually a problem? - * Shouldn`t starting postgres force correct permissions on PGDATA? - * - * TODO: symlink handling. If user located symlink in PG_TBLSPC_DIR, it will - * be restored as directory. - */ -void -create_data_directories(parray *dest_files, const char *data_dir, const char *backup_dir, - bool extract_tablespaces, bool incremental, fio_location location, - const char* waldir_path) -{ - int i; - parray *links = NULL; - mode_t pg_tablespace_mode = DIR_PERMISSION; - char to_path[MAXPGPATH]; - - if (waldir_path && !dir_is_empty(waldir_path, location)) - { - elog(ERROR, "WAL directory location is not empty: \"%s\"", waldir_path); - } - - - /* get tablespace map */ - if (extract_tablespaces) - { - links = parray_new(); - read_tablespace_map(links, backup_dir); - /* Sort links by a link name */ - parray_qsort(links, pgFileCompareName); - } - - /* - * We have no idea about tablespace permission - * For PG < 11 we can just force default permissions. - */ -#if PG_VERSION_NUM >= 110000 - if (links) - { - /* For PG>=11 we use temp kludge: trust permissions on 'pg_tblspc' - * and force them on every tablespace. - * TODO: remove kludge and ask data_directory_mode - * at the start of backup. - */ - for (i = 0; i < parray_num(dest_files); i++) - { - pgFile *file = (pgFile *) parray_get(dest_files, i); - - if (!S_ISDIR(file->mode)) - continue; - - /* skip external directory content */ - if (file->external_dir_num != 0) - continue; - - /* look for 'pg_tblspc' directory */ - if (strcmp(file->rel_path, PG_TBLSPC_DIR) == 0) - { - pg_tablespace_mode = file->mode; - break; - } - } - } -#endif - - /* - * We iterate over dest_files and for every directory with parent 'pg_tblspc' - * we must lookup this directory name in tablespace map. - * If we got a match, we treat this directory as tablespace. - * It means that we create directory specified in tablespace_map and - * original directory created as symlink to it. - */ - - elog(LOG, "Restore directories and symlinks..."); - - /* create directories */ - for (i = 0; i < parray_num(dest_files); i++) - { - char parent_dir[MAXPGPATH]; - pgFile *dir = (pgFile *) parray_get(dest_files, i); - - if (!S_ISDIR(dir->mode)) - continue; - - /* skip external directory content */ - if (dir->external_dir_num != 0) - continue; - /* Create WAL directory and symlink if waldir_path is setting */ - if (waldir_path && strcmp(dir->rel_path, PG_XLOG_DIR) == 0) { - /* get full path to PG_XLOG_DIR */ - - join_path_components(to_path, data_dir, PG_XLOG_DIR); - - elog(VERBOSE, "Create directory \"%s\" and symbolic link \"%s\"", - waldir_path, to_path); - - /* create tablespace directory from waldir_path*/ - fio_mkdir(waldir_path, pg_tablespace_mode, location); - - /* create link to linked_path */ - if (fio_symlink(waldir_path, to_path, incremental, location) < 0) - elog(ERROR, "Could not create symbolic link \"%s\": %s", - to_path, strerror(errno)); - - continue; - - - } - - /* tablespace_map exists */ - if (links) - { - /* get parent dir of rel_path */ - strlcpy(parent_dir, dir->rel_path, MAXPGPATH); - get_parent_directory(parent_dir); - - /* check if directory is actually link to tablespace */ - if (strcmp(parent_dir, PG_TBLSPC_DIR) == 0) - { - /* this directory located in pg_tblspc - * check it against tablespace map - */ - pgFile **link = (pgFile **) parray_bsearch(links, dir, pgFileCompareName); - - /* got match */ - if (link) - { - const char *linked_path = get_tablespace_mapping((*link)->linked); - - if (!is_absolute_path(linked_path)) - elog(ERROR, "Tablespace directory path must be an absolute path: %s\n", - linked_path); - - join_path_components(to_path, data_dir, dir->rel_path); - - elog(LOG, "Create directory \"%s\" and symbolic link \"%s\"", - linked_path, to_path); - - /* create tablespace directory */ - fio_mkdir(linked_path, pg_tablespace_mode, location); - - /* create link to linked_path */ - if (fio_symlink(linked_path, to_path, incremental, location) < 0) - elog(ERROR, "Could not create symbolic link \"%s\": %s", - to_path, strerror(errno)); - - continue; - } - } - } - - /* This is not symlink, create directory */ - elog(LOG, "Create directory \"%s\"", dir->rel_path); - - join_path_components(to_path, data_dir, dir->rel_path); - - // TODO check exit code - fio_mkdir(to_path, dir->mode, location); - } - - if (extract_tablespaces) - { - parray_walk(links, pgFileFree); - parray_free(links); - } -} - -/* - * Read names of symbolic names of tablespaces with links to directories from - * tablespace_map or tablespace_map.txt. - */ -void -read_tablespace_map(parray *links, const char *backup_dir) -{ - FILE *fp; - char db_path[MAXPGPATH], - map_path[MAXPGPATH]; - char buf[MAXPGPATH * 2]; - - join_path_components(db_path, backup_dir, DATABASE_DIR); - join_path_components(map_path, db_path, PG_TABLESPACE_MAP_FILE); - - fp = fio_open_stream(map_path, FIO_BACKUP_HOST); - if (fp == NULL) - elog(ERROR, "Cannot open tablespace map file \"%s\": %s", map_path, strerror(errno)); - - while (fgets(buf, lengthof(buf), fp)) - { - char link_name[MAXPGPATH]; - char *path; - int n = 0; - pgFile *file; - int i = 0; - - if (sscanf(buf, "%s %n", link_name, &n) != 1) - elog(ERROR, "Invalid format found in \"%s\"", map_path); - - path = buf + n; - - /* Remove newline character at the end of string if any */ - i = strcspn(path, "\n"); - if (strlen(path) > i) - path[i] = '\0'; - - file = pgut_new(pgFile); - memset(file, 0, sizeof(pgFile)); - - /* follow the convention for pgFileFree */ - file->name = pgut_strdup(link_name); - file->linked = pgut_strdup(path); - canonicalize_path(file->linked); - - parray_append(links, file); - } - - if (ferror(fp)) - elog(ERROR, "Failed to read from file: \"%s\"", map_path); - - fio_close_stream(fp); -} - -/* - * Check that all tablespace mapping entries have correct linked directory - * paths. Linked directories must be empty or do not exist, unless - * we are running incremental restore, then linked directories can be nonempty. - * - * If tablespace-mapping option is supplied, all OLDDIR entries must have - * entries in tablespace_map file. - * - * When running incremental restore with tablespace remapping, then - * new tablespace directory MUST be empty, because there is no way - * we can be sure, that files laying there belong to our instance. - * But "force" flag allows to ignore this condition, by wiping out - * the current content on the directory. - * - * Exit codes: - * 1. backup has no tablespaces - * 2. backup has tablespaces and they are empty - * 3. backup has tablespaces and some of them are not empty - */ -int -check_tablespace_mapping(pgBackup *backup, bool incremental, bool force, bool pgdata_is_empty, bool no_validate) -{ - parray *links = parray_new(); - size_t i; - TablespaceListCell *cell; - pgFile *tmp_file = pgut_new(pgFile); - bool tblspaces_are_empty = true; - - elog(LOG, "Checking tablespace directories of backup %s", - backup_id_of(backup)); - - /* validate tablespace map, - * if there are no tablespaces, then there is nothing left to do - */ - if (!validate_tablespace_map(backup, no_validate)) - { - /* - * Sanity check - * If there is no tablespaces in backup, - * then using the '--tablespace-mapping' option is a mistake. - */ - if (tablespace_dirs.head != NULL) - elog(ERROR, "Backup %s has no tablespaceses, nothing to remap " - "via \"--tablespace-mapping\" option", backup_id_of(backup)); - return NoTblspc; - } - - read_tablespace_map(links, backup->root_dir); - /* Sort links by the path of a linked file*/ - parray_qsort(links, pgFileCompareLinked); - - /* 1 - each OLDDIR must have an entry in tablespace_map file (links) */ - for (cell = tablespace_dirs.head; cell; cell = cell->next) - { - tmp_file->linked = cell->old_dir; - - if (parray_bsearch(links, tmp_file, pgFileCompareLinked) == NULL) - elog(ERROR, "--tablespace-mapping option's old directory " - "doesn't have an entry in tablespace_map file: \"%s\"", - cell->old_dir); - } - - /* - * There is difference between incremental restore of already existing - * tablespaceses and remapped tablespaceses. - * Former are allowed to be not empty, because we treat them like an - * extension of PGDATA. - * The latter are not, unless "--force" flag is used. - * in which case the remapped directory is nuked - just to be safe, - * because it is hard to be sure that there are no some tricky corner - * cases of pages from different systems having the same crc. - * This is a strict approach. - * - * Why can`t we not nuke it and just let it roll ? - * What if user just wants to rerun failed restore with the same - * parameters? Nuking is bad for this case. - * - * Consider the example of existing PGDATA: - * .... - * pg_tablespace - * 100500-> /somedirectory - * .... - * - * We want to remap it during restore like that: - * .... - * pg_tablespace - * 100500-> /somedirectory1 - * .... - * - * Usually it is required for "/somedirectory1" to be empty, but - * in case of incremental restore with 'force' flag, which required - * of us to drop already existing content of "/somedirectory1". - * - * TODO: Ideally in case of incremental restore we must also - * drop the "/somedirectory" directory first, but currently - * we don`t do that. - */ - - /* 2 - all linked directories must be empty */ - for (i = 0; i < parray_num(links); i++) - { - pgFile *link = (pgFile *) parray_get(links, i); - const char *linked_path = link->linked; - bool remapped = false; - - for (cell = tablespace_dirs.head; cell; cell = cell->next) - { - if (strcmp(link->linked, cell->old_dir) == 0) - { - linked_path = cell->new_dir; - remapped = true; - break; - } - } - - if (remapped) - elog(INFO, "Tablespace %s will be remapped from \"%s\" to \"%s\"", link->name, cell->old_dir, cell->new_dir); - else - elog(INFO, "Tablespace %s will be restored using old path \"%s\"", link->name, linked_path); - - if (!is_absolute_path(linked_path)) - elog(ERROR, "Tablespace directory path must be an absolute path: %s\n", - linked_path); - - if (!dir_is_empty(linked_path, FIO_DB_HOST)) - { - - if (!incremental) - elog(ERROR, "Restore tablespace destination is not empty: \"%s\"", linked_path); - - else if (remapped && !force) - elog(ERROR, "Remapped tablespace destination is not empty: \"%s\". " - "Use \"--force\" flag if you want to automatically clean up the " - "content of new tablespace destination", - linked_path); - - else if (pgdata_is_empty && !force) - elog(ERROR, "PGDATA is empty, but tablespace destination is not: \"%s\". " - "Use \"--force\" flag is you want to automatically clean up the " - "content of tablespace destination", - linked_path); - - /* - * TODO: compile the list of tblspc Oids to delete later, - * similar to what we do with database_map. - */ - else if (force && (pgdata_is_empty || remapped)) - { - elog(WARNING, "Cleaning up the content of %s directory: \"%s\"", - remapped ? "remapped tablespace" : "tablespace", linked_path); - cleanup_tablespace(linked_path); - continue; - } - - tblspaces_are_empty = false; - } - } - - free(tmp_file); - parray_walk(links, pgFileFree); - parray_free(links); - - if (tblspaces_are_empty) - return EmptyTblspc; - - return NotEmptyTblspc; -} - -/* TODO: Make it consistent with check_tablespace_mapping */ -void -check_external_dir_mapping(pgBackup *backup, bool incremental) -{ - TablespaceListCell *cell; - parray *external_dirs_to_restore; - int i; - - elog(LOG, "check external directories of backup %s", - backup_id_of(backup)); - - if (!backup->external_dir_str) - { - if (external_remap_list.head) - elog(ERROR, "--external-mapping option's old directory doesn't " - "have an entry in list of external directories of current " - "backup: \"%s\"", external_remap_list.head->old_dir); - return; - } - - external_dirs_to_restore = make_external_directory_list( - backup->external_dir_str, - false); - /* 1 - each OLDDIR must have an entry in external_dirs_to_restore */ - for (cell = external_remap_list.head; cell; cell = cell->next) - { - bool found = false; - - for (i = 0; i < parray_num(external_dirs_to_restore); i++) - { - char *external_dir = parray_get(external_dirs_to_restore, i); - - if (strcmp(cell->old_dir, external_dir) == 0) - { - /* Swap new dir name with old one, it is used by 2-nd step */ - parray_set(external_dirs_to_restore, i, - pgut_strdup(cell->new_dir)); - pfree(external_dir); - - found = true; - break; - } - } - if (!found) - elog(ERROR, "--external-mapping option's old directory doesn't " - "have an entry in list of external directories of current " - "backup: \"%s\"", cell->old_dir); - } - - /* 2 - all linked directories must be empty */ - for (i = 0; i < parray_num(external_dirs_to_restore); i++) - { - char *external_dir = (char *) parray_get(external_dirs_to_restore, - i); - - if (!incremental && !dir_is_empty(external_dir, FIO_DB_HOST)) - elog(ERROR, "External directory is not empty: \"%s\"", - external_dir); - } - - free_dir_list(external_dirs_to_restore); -} - -char * -get_external_remap(char *current_dir) -{ - TablespaceListCell *cell; - - for (cell = external_remap_list.head; cell; cell = cell->next) - { - char *old_dir = cell->old_dir; - - if (strcmp(old_dir, current_dir) == 0) - return cell->new_dir; - } - return current_dir; -} - -/* Parsing states for get_control_value_str() */ -#define CONTROL_WAIT_NAME 1 -#define CONTROL_INNAME 2 -#define CONTROL_WAIT_COLON 3 -#define CONTROL_WAIT_VALUE 4 -#define CONTROL_INVALUE 5 -#define CONTROL_WAIT_NEXT_NAME 6 - -/* - * Get value from json-like line "str" of backup_content.control file. - * - * The line has the following format: - * {"name1":"value1", "name2":"value2"} - * - * The value will be returned in "value_int64" as int64. - * - * Returns true if the value was found in the line and parsed. - */ -bool -get_control_value_int64(const char *str, const char *name, int64 *value_int64, bool is_mandatory) -{ - - char buf_int64[32]; - - assert(value_int64); - - /* Set default value */ - *value_int64 = 0; - - if (!get_control_value_str(str, name, buf_int64, sizeof(buf_int64), is_mandatory)) - return false; - - if (!parse_int64(buf_int64, value_int64, 0)) - { - /* We assume that too big value is -1 */ - if (errno == ERANGE) - *value_int64 = BYTES_INVALID; - else - control_string_bad_format(str); - return false; - } - - return true; -} - -/* - * Get value from json-like line "str" of backup_content.control file. - * - * The line has the following format: - * {"name1":"value1", "name2":"value2"} - * - * The value will be returned to "value_str" as string. - * - * Returns true if the value was found in the line. - */ - -bool -get_control_value_str(const char *str, const char *name, - char *value_str, size_t value_str_size, bool is_mandatory) -{ - int state = CONTROL_WAIT_NAME; - char *name_ptr = (char *) name; - char *buf = (char *) str; - char *const value_str_start = value_str; - - assert(value_str); - assert(value_str_size > 0); - - /* Set default value */ - *value_str = '\0'; - - while (*buf) - { - switch (state) - { - case CONTROL_WAIT_NAME: - if (*buf == '"') - state = CONTROL_INNAME; - else if (IsAlpha(*buf)) - control_string_bad_format(str); - break; - case CONTROL_INNAME: - /* Found target field. Parse value. */ - if (*buf == '"') - state = CONTROL_WAIT_COLON; - /* Check next field */ - else if (*buf != *name_ptr) - { - name_ptr = (char *) name; - state = CONTROL_WAIT_NEXT_NAME; - } - else - name_ptr++; - break; - case CONTROL_WAIT_COLON: - if (*buf == ':') - state = CONTROL_WAIT_VALUE; - else if (!IsSpace(*buf)) - control_string_bad_format(str); - break; - case CONTROL_WAIT_VALUE: - if (*buf == '"') - { - state = CONTROL_INVALUE; - } - else if (IsAlpha(*buf)) - control_string_bad_format(str); - break; - case CONTROL_INVALUE: - /* Value was parsed, exit */ - if (*buf == '"') - { - *value_str = '\0'; - return true; - } - else - { - /* verify if value_str not exceeds value_str_size limits */ - if (value_str - value_str_start >= value_str_size - 1) { - elog(ERROR, "Field \"%s\" is out of range in the line %s of the file %s", - name, str, DATABASE_FILE_LIST); - } - *value_str = *buf; - value_str++; - } - break; - case CONTROL_WAIT_NEXT_NAME: - if (*buf == ',') - state = CONTROL_WAIT_NAME; - break; - default: - /* Should not happen */ - break; - } - - buf++; - } - - /* There is no close quotes */ - if (state == CONTROL_INNAME || state == CONTROL_INVALUE) - control_string_bad_format(str); - - /* Did not find target field */ - if (is_mandatory) - elog(ERROR, "Field \"%s\" is not found in the line %s of the file %s", - name, str, DATABASE_FILE_LIST); - return false; -} - -static void -control_string_bad_format(const char* str) -{ - elog(ERROR, "%s file has invalid format in line %s", - DATABASE_FILE_LIST, str); -} - -/* - * Check if directory empty. - */ -bool -dir_is_empty(const char *path, fio_location location) -{ - DIR *dir; - struct dirent *dir_ent; - - dir = fio_opendir(path, location); - if (dir == NULL) - { - /* Directory in path doesn't exist */ - if (errno == ENOENT) - return true; - elog(ERROR, "Cannot open directory \"%s\": %s", path, strerror(errno)); - } - - errno = 0; - while ((dir_ent = fio_readdir(dir))) - { - /* Skip entries point current dir or parent dir */ - if (strcmp(dir_ent->d_name, ".") == 0 || - strcmp(dir_ent->d_name, "..") == 0) - continue; - - /* Directory is not empty */ - fio_closedir(dir); - return false; - } - if (errno) - elog(ERROR, "Cannot read directory \"%s\": %s", path, strerror(errno)); - - fio_closedir(dir); - - return true; -} - -/* - * Return true if the path is a existing regular file. - */ -bool -fileExists(const char *path, fio_location location) -{ - struct stat buf; - - if (fio_stat(path, &buf, true, location) == -1 && errno == ENOENT) - return false; - else if (!S_ISREG(buf.st_mode)) - return false; - else - return true; -} - -size_t -pgFileSize(const char *path) -{ - struct stat buf; - - if (stat(path, &buf) == -1) - elog(ERROR, "Cannot stat file \"%s\": %s", path, strerror(errno)); - - return buf.st_size; -} - -/* - * Construct parray containing remapped external directories paths - * from string like /path1:/path2 - */ -parray * -make_external_directory_list(const char *colon_separated_dirs, bool remap) -{ - char *p; - parray *list = parray_new(); - char *tmp = pg_strdup(colon_separated_dirs); - -#ifndef WIN32 -#define EXTERNAL_DIRECTORY_DELIMITER ":" -#else -#define EXTERNAL_DIRECTORY_DELIMITER ";" -#endif - - p = strtok(tmp, EXTERNAL_DIRECTORY_DELIMITER); - while(p!=NULL) - { - char *external_path = pg_strdup(p); - - canonicalize_path(external_path); - if (is_absolute_path(external_path)) - { - if (remap) - { - char *full_path = get_external_remap(external_path); - - if (full_path != external_path) - { - full_path = pg_strdup(full_path); - pfree(external_path); - external_path = full_path; - } - } - parray_append(list, external_path); - } - else - elog(ERROR, "External directory \"%s\" is not an absolute path", - external_path); - - p = strtok(NULL, EXTERNAL_DIRECTORY_DELIMITER); - } - pfree(tmp); - parray_qsort(list, pgCompareString); - return list; -} - -/* Free memory of parray containing strings */ -void -free_dir_list(parray *list) -{ - parray_walk(list, pfree); - parray_free(list); -} - -/* Append to string "path_prefix" int "dir_num" */ -void -makeExternalDirPathByNum(char *ret_path, const char *path_prefix, const int dir_num) -{ - sprintf(ret_path, "%s%d", path_prefix, dir_num); -} - -/* Check if "dir" presents in "dirs_list" */ -bool -backup_contains_external(const char *dir, parray *dirs_list) -{ - void *search_result; - - if (!dirs_list) /* There is no external dirs in backup */ - return false; - search_result = parray_bsearch(dirs_list, dir, pgCompareString); - return search_result != NULL; -} - -/* - * Print database_map - */ -void -print_database_map(FILE *out, parray *database_map) -{ - int i; - - for (i = 0; i < parray_num(database_map); i++) - { - db_map_entry *db_entry = (db_map_entry *) parray_get(database_map, i); - - fio_fprintf(out, "{\"dbOid\":\"%u\", \"datname\":\"%s\"}\n", - db_entry->dbOid, db_entry->datname); - } - -} - -/* - * Create file 'database_map' and add its meta to backup_files_list - * NULL check for database_map must be done by the caller. - */ -void -write_database_map(pgBackup *backup, parray *database_map, parray *backup_files_list) -{ - FILE *fp; - pgFile *file; - char database_dir[MAXPGPATH]; - char database_map_path[MAXPGPATH]; - - join_path_components(database_dir, backup->root_dir, DATABASE_DIR); - join_path_components(database_map_path, database_dir, DATABASE_MAP); - - fp = fio_fopen(database_map_path, PG_BINARY_W, FIO_BACKUP_HOST); - if (fp == NULL) - elog(ERROR, "Cannot open database map \"%s\": %s", database_map_path, - strerror(errno)); - - print_database_map(fp, database_map); - if (fio_fflush(fp) || fio_fclose(fp)) - { - fio_unlink(database_map_path, FIO_BACKUP_HOST); - elog(ERROR, "Cannot write database map \"%s\": %s", - database_map_path, strerror(errno)); - } - - /* Add metadata to backup_content.control */ - file = pgFileNew(database_map_path, DATABASE_MAP, true, 0, - FIO_BACKUP_HOST); - file->crc = pgFileGetCRC(database_map_path, true, false); - file->write_size = file->size; - file->uncompressed_size = file->size; - - parray_append(backup_files_list, file); -} - -/* - * read database map, return NULL if database_map in empty or missing - */ -parray * -read_database_map(pgBackup *backup) -{ - FILE *fp; - parray *database_map; - char buf[MAXPGPATH]; - char path[MAXPGPATH]; - char database_map_path[MAXPGPATH]; - - join_path_components(path, backup->root_dir, DATABASE_DIR); - join_path_components(database_map_path, path, DATABASE_MAP); - - fp = fio_open_stream(database_map_path, FIO_BACKUP_HOST); - if (fp == NULL) - { - /* It is NOT ok for database_map to be missing at this point, so - * we should error here. - * It`s a job of the caller to error if database_map is not empty. - */ - elog(ERROR, "Cannot open \"%s\": %s", database_map_path, strerror(errno)); - } - - database_map = parray_new(); - - while (fgets(buf, lengthof(buf), fp)) - { - char datname[MAXPGPATH]; - int64 dbOid; - - db_map_entry *db_entry = (db_map_entry *) pgut_malloc(sizeof(db_map_entry)); - - get_control_value_int64(buf, "dbOid", &dbOid, true); - get_control_value_str(buf, "datname", datname, sizeof(datname), true); - - db_entry->dbOid = dbOid; - db_entry->datname = pgut_strdup(datname); - - parray_append(database_map, db_entry); - } - - if (ferror(fp)) - elog(ERROR, "Failed to read from file: \"%s\"", database_map_path); - - fio_close_stream(fp); - - /* Return NULL if file is empty */ - if (parray_num(database_map) == 0) - { - parray_free(database_map); - return NULL; - } - - return database_map; -} - -/* - * Use it to cleanup tablespaces - * TODO: Current algorihtm is not very efficient in remote mode, - * due to round-trip to delete every file. - */ -void -cleanup_tablespace(const char *path) -{ - int i; - char fullpath[MAXPGPATH]; - parray *files = parray_new(); - - fio_list_dir(files, path, false, false, false, false, false, 0); - - /* delete leaf node first */ - parray_qsort(files, pgFileCompareRelPathWithExternalDesc); - - for (i = 0; i < parray_num(files); i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - - join_path_components(fullpath, path, file->rel_path); - - fio_delete(file->mode, fullpath, FIO_DB_HOST); - elog(LOG, "Deleted file \"%s\"", fullpath); - } - - parray_walk(files, pgFileFree); - parray_free(files); -} - -/* - * Clear the synchronisation locks in a parray of (pgFile *)'s - */ -void -pfilearray_clear_locks(parray *file_list) -{ - int i; - for (i = 0; i < parray_num(file_list); i++) - { - pgFile *file = (pgFile *) parray_get(file_list, i); - pg_atomic_clear_flag(&file->lock); - } -} - -static inline bool -is_forkname(char *name, size_t *pos, const char *forkname) -{ - size_t fnlen = strlen(forkname); - if (strncmp(name + *pos, forkname, fnlen) != 0) - return false; - *pos += fnlen; - return true; -} - -#define OIDCHARS 10 -#define MAXSEGNO (((uint64_t)1<<32)/RELSEG_SIZE-1) -#define SEGNOCHARS 5 /* when BLCKSZ == (1<<15) */ - -/* Set forkName if possible */ -bool -set_forkname(pgFile *file) -{ - size_t i = 0; - uint64_t oid = 0; /* use 64bit to not check for overflow in a loop */ - uint64_t segno = 0; - - /* pretend it is not relation file */ - file->relOid = 0; - file->forkName = none; - file->is_datafile = false; - - for (i = 0; isdigit(file->name[i]); i++) - { - if (i == 0 && file->name[i] == '0') - return false; - oid = oid * 10 + file->name[i] - '0'; - } - if (i == 0 || i > OIDCHARS || oid > UINT32_MAX) - return false; - - /* usual fork name */ - /* /^\d+_(vm|fsm|init|ptrack)$/ */ - if (is_forkname(file->name, &i, "_vm")) - file->forkName = vm; - else if (is_forkname(file->name, &i, "_fsm")) - file->forkName = fsm; - else if (is_forkname(file->name, &i, "_init")) - file->forkName = init; - else if (is_forkname(file->name, &i, "_ptrack")) - file->forkName = ptrack; - - /* segment number */ - /* /^\d+(_(vm|fsm|init|ptrack))?\.\d+$/ */ - if (file->name[i] == '.' && isdigit(file->name[i+1])) - { - size_t start = i+1; - for (i++; isdigit(file->name[i]); i++) - { - if (i == start && file->name[i] == '0') - return false; - segno = segno * 10 + file->name[i] - '0'; - } - if (i - start > SEGNOCHARS || segno > MAXSEGNO) - return false; - } - - /* CFS family fork names */ - if (file->forkName == none && - is_forkname(file->name, &i, ".cfm.bck")) - { - /* /^\d+(\.\d+)?\.cfm\.bck$/ */ - file->forkName = cfm_bck; - } - if (file->forkName == none && - is_forkname(file->name, &i, ".bck")) - { - /* /^\d+(\.\d+)?\.bck$/ */ - file->forkName = cfs_bck; - } - if (file->forkName == none && - is_forkname(file->name, &i, ".cfm")) - { - /* /^\d+(\.\d+)?.cfm$/ */ - file->forkName = cfm; - } - - /* If there are excess characters, it is not relation file */ - if (file->name[i] != 0) - { - file->forkName = none; - return false; - } - - file->relOid = oid; - file->segno = segno; - file->is_datafile = file->forkName == none; - return true; -} diff --git a/src/fetch.c b/src/fetch.c deleted file mode 100644 index 5401d815e..000000000 --- a/src/fetch.c +++ /dev/null @@ -1,108 +0,0 @@ -/*------------------------------------------------------------------------- - * - * fetch.c - * Functions for fetching files from PostgreSQL data directory - * - * Portions Copyright (c) 1996-2018, PostgreSQL Global Development Group - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -#include -#include - -/* - * Read a file into memory. The file to be read is /. - * The file contents are returned in a malloc'd buffer, and *filesize - * is set to the length of the file. - * - * The returned buffer is always zero-terminated; the size of the returned - * buffer is actually *filesize + 1. That's handy when reading a text file. - * This function can be used to read binary files as well, you can just - * ignore the zero-terminator in that case. - * - */ -char * -slurpFile(const char *datadir, const char *path, size_t *filesize, bool safe, fio_location location) -{ - int fd; - char *buffer; - struct stat statbuf; - char fullpath[MAXPGPATH]; - int len; - - join_path_components(fullpath, datadir, path); - - if ((fd = fio_open(fullpath, O_RDONLY | PG_BINARY, location)) == -1) - { - if (safe) - return NULL; - else - elog(ERROR, "Could not open file \"%s\" for reading: %s", - fullpath, strerror(errno)); - } - - if (fio_stat(fullpath, &statbuf, true, location) < 0) - { - if (safe) - return NULL; - else - elog(ERROR, "Could not stat file \"%s\": %s", - fullpath, strerror(errno)); - } - - len = statbuf.st_size; - buffer = pg_malloc(len + 1); - - if (fio_read(fd, buffer, len) != len) - { - if (safe) - return NULL; - else - elog(ERROR, "Could not read file \"%s\": %s\n", - fullpath, strerror(errno)); - } - - fio_close(fd); - - /* Zero-terminate the buffer. */ - buffer[len] = '\0'; - - if (filesize) - *filesize = len; - return buffer; -} - -/* - * Receive a single file as a malloc'd buffer. - */ -char * -fetchFile(PGconn *conn, const char *filename, size_t *filesize) -{ - PGresult *res; - char *result; - const char *params[1]; - int len; - - params[0] = filename; - res = pgut_execute_extended(conn, "SELECT pg_catalog.pg_read_binary_file($1)", - 1, params, false, false); - - /* sanity check the result set */ - if (PQntuples(res) != 1 || PQgetisnull(res, 0, 0)) - elog(ERROR, "Unexpected result set while fetching remote file \"%s\"", - filename); - - /* Read result to local variables */ - len = PQgetlength(res, 0, 0); - result = pg_malloc(len + 1); - memcpy(result, PQgetvalue(res, 0, 0), len); - result[len] = '\0'; - - PQclear(res); - *filesize = len; - - return result; -} diff --git a/src/help.c b/src/help.c deleted file mode 100644 index 954ba6416..000000000 --- a/src/help.c +++ /dev/null @@ -1,1146 +0,0 @@ -/*------------------------------------------------------------------------- - * - * help.c - * - * Copyright (c) 2017-2021, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include -#include "pg_probackup.h" - -static void help_nocmd(void); -static void help_internal(void); -static void help_init(void); -static void help_backup(void); -static void help_restore(void); -static void help_validate(void); -static void help_show(void); -static void help_delete(void); -static void help_merge(void); -static void help_set_backup(void); -static void help_set_config(void); -static void help_show_config(void); -static void help_add_instance(void); -static void help_del_instance(void); -static void help_archive_push(void); -static void help_archive_get(void); -static void help_checkdb(void); -static void help_help(void); -static void help_version(void); -static void help_catchup(void); - -void -help_print_version(void) -{ -#ifdef PGPRO_VERSION - fprintf(stdout, "%s %s (Postgres Pro %s %s)\n", - PROGRAM_NAME, PROGRAM_VERSION, - PGPRO_VERSION, PGPRO_EDITION); -#else - fprintf(stdout, "%s %s (PostgreSQL %s)\n", - PROGRAM_NAME, PROGRAM_VERSION, PG_VERSION); -#endif -} - -void -help_command(ProbackupSubcmd const subcmd) -{ - typedef void (* help_function_ptr)(void); - /* Order is important, keep it in sync with utils/configuration.h:enum ProbackupSubcmd declaration */ - static help_function_ptr const help_functions[] = - { - &help_nocmd, - &help_init, - &help_add_instance, - &help_del_instance, - &help_archive_push, - &help_archive_get, - &help_backup, - &help_restore, - &help_validate, - &help_delete, - &help_merge, - &help_show, - &help_set_config, - &help_set_backup, - &help_show_config, - &help_checkdb, - &help_internal, // SSH_CMD - &help_internal, // AGENT_CMD - &help_help, - &help_version, - &help_catchup, - }; - - Assert((int)subcmd < sizeof(help_functions) / sizeof(help_functions[0])); - help_functions[(int)subcmd](); -} - -void -help_pg_probackup(void) -{ - printf(_("\n%s - utility to manage backup/recovery of PostgreSQL database.\n"), PROGRAM_NAME); - - printf(_("\n %s help [COMMAND]\n"), PROGRAM_NAME); - - printf(_("\n %s version\n"), PROGRAM_NAME); - - printf(_("\n %s init -B backup-path\n"), PROGRAM_NAME); - - printf(_("\n %s set-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-path]\n")); - printf(_(" [--external-dirs=external-directories-paths]\n")); - printf(_(" [--log-level-console=log-level-console]\n")); - printf(_(" [--log-level-file=log-level-file]\n")); - printf(_(" [--log-format-file=log-format-file]\n")); - printf(_(" [--log-filename=log-filename]\n")); - printf(_(" [--error-log-filename=error-log-filename]\n")); - printf(_(" [--log-directory=log-directory]\n")); - printf(_(" [--log-rotation-size=log-rotation-size]\n")); - printf(_(" [--log-rotation-age=log-rotation-age]\n")); - printf(_(" [--retention-redundancy=retention-redundancy]\n")); - printf(_(" [--retention-window=retention-window]\n")); - printf(_(" [--wal-depth=wal-depth]\n")); - printf(_(" [--compress-algorithm=compress-algorithm]\n")); - printf(_(" [--compress-level=compress-level]\n")); - printf(_(" [--archive-timeout=timeout]\n")); - printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); - printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); - printf(_(" [--ssh-options]\n")); - printf(_(" [--restore-command=cmdline] [--archive-host=destination]\n")); - printf(_(" [--archive-port=port] [--archive-user=username]\n")); - printf(_(" [--help]\n")); - - printf(_("\n %s set-backup -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" -i backup-id [--ttl=interval] [--expire-time=timestamp]\n")); - printf(_(" [--note=text]\n")); - printf(_(" [--help]\n")); - - printf(_("\n %s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [--format=format]\n")); - printf(_(" [--help]\n")); - - printf(_("\n %s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-path] [-C]\n")); - printf(_(" [--stream [-S slot-name] [--temp-slot]]\n")); - printf(_(" [--backup-pg-log] [-j num-threads] [--progress]\n")); - printf(_(" [--no-validate] [--skip-block-validation]\n")); - printf(_(" [--external-dirs=external-directories-paths]\n")); - printf(_(" [--no-sync]\n")); - printf(_(" [--log-level-console=log-level-console]\n")); - printf(_(" [--log-level-file=log-level-file]\n")); - printf(_(" [--log-format-console=log-format-console]\n")); - printf(_(" [--log-format-file=log-format-file]\n")); - printf(_(" [--log-filename=log-filename]\n")); - printf(_(" [--error-log-filename=error-log-filename]\n")); - printf(_(" [--log-directory=log-directory]\n")); - printf(_(" [--log-rotation-size=log-rotation-size]\n")); - printf(_(" [--log-rotation-age=log-rotation-age] [--no-color]\n")); - printf(_(" [--delete-expired] [--delete-wal] [--merge-expired]\n")); - printf(_(" [--retention-redundancy=retention-redundancy]\n")); - printf(_(" [--retention-window=retention-window]\n")); - printf(_(" [--wal-depth=wal-depth]\n")); - printf(_(" [--compress]\n")); - printf(_(" [--compress-algorithm=compress-algorithm]\n")); - printf(_(" [--compress-level=compress-level]\n")); - printf(_(" [--archive-timeout=archive-timeout]\n")); - printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); - printf(_(" [-w --no-password] [-W --password]\n")); - printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); - printf(_(" [--ssh-options]\n")); - printf(_(" [--ttl=interval] [--expire-time=timestamp] [--note=text]\n")); - printf(_(" [--help]\n")); - - - printf(_("\n %s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads]\n")); - printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); - printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); - printf(_(" [--recovery-target-timeline=timeline]\n")); - printf(_(" [--recovery-target=immediate|latest]\n")); - printf(_(" [--recovery-target-name=target-name]\n")); - printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); - printf(_(" [--restore-command=cmdline]\n")); - printf(_(" [-R | --restore-as-replica] [--force]\n")); - printf(_(" [--primary-conninfo=primary_conninfo]\n")); - printf(_(" [-S | --primary-slot-name=slotname]\n")); - printf(_(" [--no-validate] [--skip-block-validation]\n")); - printf(_(" [-T OLDDIR=NEWDIR] [--progress]\n")); - printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); - printf(_(" [--skip-external-dirs] [--no-sync]\n")); - printf(_(" [-X WALDIR | --waldir=WALDIR]\n")); - printf(_(" [-I | --incremental-mode=none|checksum|lsn]\n")); - printf(_(" [--db-include | --db-exclude]\n")); - printf(_(" [--destroy-all-other-dbs]\n")); - printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); - printf(_(" [--ssh-options]\n")); - printf(_(" [--archive-host=hostname]\n")); - printf(_(" [--archive-port=port] [--archive-user=username]\n")); - printf(_(" [--help]\n")); - - printf(_("\n %s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); - printf(_(" [-i backup-id] [--progress] [-j num-threads]\n")); - printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); - printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); - printf(_(" [--recovery-target-timeline=timeline]\n")); - printf(_(" [--recovery-target-name=target-name]\n")); - printf(_(" [--skip-block-validation]\n")); - printf(_(" [--help]\n")); - - printf(_("\n %s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-path] [--progress] [-j num-threads]\n")); - printf(_(" [--amcheck] [--skip-block-validation]\n")); - printf(_(" [--heapallindexed] [--checkunique]\n")); - printf(_(" [--help]\n")); - - printf(_("\n %s show -B backup-path\n"), PROGRAM_NAME); - printf(_(" [--instance=instance_name [-i backup-id]]\n")); - printf(_(" [--format=format] [--archive]\n")); - printf(_(" [--no-color] [--help]\n")); - - printf(_("\n %s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [-j num-threads] [--progress]\n")); - printf(_(" [--retention-redundancy=retention-redundancy]\n")); - printf(_(" [--retention-window=retention-window]\n")); - printf(_(" [--wal-depth=wal-depth]\n")); - printf(_(" [-i backup-id | --delete-expired | --merge-expired | --status=backup_status]\n")); - printf(_(" [--delete-wal]\n")); - printf(_(" [--dry-run] [--no-validate] [--no-sync]\n")); - printf(_(" [--help]\n")); - - printf(_("\n %s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" -i backup-id [--progress] [-j num-threads]\n")); - printf(_(" [--no-validate] [--no-sync]\n")); - printf(_(" [--help]\n")); - - printf(_("\n %s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); - printf(_(" --instance=instance_name\n")); - printf(_(" [--external-dirs=external-directories-paths]\n")); - printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); - printf(_(" [--ssh-options]\n")); - printf(_(" [--help]\n")); - - printf(_("\n %s del-instance -B backup-path\n"), PROGRAM_NAME); - printf(_(" --instance=instance_name\n")); - printf(_(" [--help]\n")); - - printf(_("\n %s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" --wal-file-name=wal-file-name\n")); - printf(_(" [--wal-file-path=wal-file-path]\n")); - printf(_(" [-j num-threads] [--batch-size=batch_size]\n")); - printf(_(" [--archive-timeout=timeout]\n")); - printf(_(" [--no-ready-rename] [--no-sync]\n")); - printf(_(" [--overwrite] [--compress]\n")); - printf(_(" [--compress-algorithm=compress-algorithm]\n")); - printf(_(" [--compress-level=compress-level]\n")); - printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); - printf(_(" [--ssh-options]\n")); - printf(_(" [--help]\n")); - - printf(_("\n %s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" --wal-file-path=wal-file-path\n")); - printf(_(" --wal-file-name=wal-file-name\n")); - printf(_(" [-j num-threads] [--batch-size=batch_size]\n")); - printf(_(" [--no-validate-wal]\n")); - printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); - printf(_(" [--ssh-options]\n")); - printf(_(" [--help]\n")); - - printf(_("\n %s catchup -b catchup-mode\n"), PROGRAM_NAME); - printf(_(" --source-pgdata=path_to_pgdata_on_remote_server\n")); - printf(_(" --destination-pgdata=path_to_local_dir\n")); - printf(_(" [--stream [-S slot-name] [--temp-slot | --perm-slot]]\n")); - printf(_(" [-j num-threads]\n")); - printf(_(" [-T OLDDIR=NEWDIR]\n")); - printf(_(" [--exclude-path=path_prefix]\n")); - printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); - printf(_(" [-w --no-password] [-W --password]\n")); - printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); - printf(_(" [--ssh-options]\n")); - printf(_(" [--dry-run]\n")); - printf(_(" [--help]\n")); - - if ((PROGRAM_URL || PROGRAM_EMAIL)) - { - printf("\n"); - if (PROGRAM_URL) - printf(_("Read the website for details <%s>.\n"), PROGRAM_URL); - if (PROGRAM_EMAIL) - printf(_("Report bugs to <%s>.\n"), PROGRAM_EMAIL); - } -} - -static void -help_nocmd(void) -{ - printf(_("\nUnknown command. Try pg_probackup help\n\n")); -} - -static void -help_internal(void) -{ - printf(_("\nThis command is intended for internal use\n\n")); -} - -static void -help_init(void) -{ - printf(_("\n%s init -B backup-path\n\n"), PROGRAM_NAME); - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n\n")); -} - -static void -help_backup(void) -{ - printf(_("\n%s backup -B backup-path -b backup-mode --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-path] [-C]\n")); - printf(_(" [--stream [-S slot-name] [--temp-slot]]\n")); - printf(_(" [--backup-pg-log] [-j num-threads] [--progress]\n")); - printf(_(" [--no-validate] [--skip-block-validation]\n")); - printf(_(" [-E external-directories-paths]\n")); - printf(_(" [--no-sync]\n")); - printf(_(" [--log-level-console=log-level-console]\n")); - printf(_(" [--log-level-file=log-level-file]\n")); - printf(_(" [--log-format-console=log-format-console]\n")); - printf(_(" [--log-format-file=log-format-file]\n")); - printf(_(" [--log-filename=log-filename]\n")); - printf(_(" [--error-log-filename=error-log-filename]\n")); - printf(_(" [--log-directory=log-directory]\n")); - printf(_(" [--log-rotation-size=log-rotation-size]\n")); - printf(_(" [--log-rotation-age=log-rotation-age] [--no-color]\n")); - printf(_(" [--delete-expired] [--delete-wal] [--merge-expired]\n")); - printf(_(" [--retention-redundancy=retention-redundancy]\n")); - printf(_(" [--retention-window=retention-window]\n")); - printf(_(" [--wal-depth=wal-depth]\n")); - printf(_(" [--compress]\n")); - printf(_(" [--compress-algorithm=compress-algorithm]\n")); - printf(_(" [--compress-level=compress-level]\n")); - printf(_(" [--archive-timeout=archive-timeout]\n")); - printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); - printf(_(" [-w --no-password] [-W --password]\n")); - printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); - printf(_(" [--ssh-options]\n")); - printf(_(" [--ttl=interval] [--expire-time=timestamp] [--note=text]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" -b, --backup-mode=backup-mode backup mode=FULL|PAGE|DELTA|PTRACK\n")); - printf(_(" --instance=instance_name name of the instance\n")); - printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); - printf(_(" -C, --smooth-checkpoint do smooth checkpoint before backup\n")); - printf(_(" --stream stream the transaction log and include it in the backup\n")); - printf(_(" -S, --slot=SLOTNAME replication slot to use\n")); - printf(_(" --temp-slot use temporary replication slot\n")); - printf(_(" --backup-pg-log backup of '%s' directory\n"), PG_LOG_DIR); - printf(_(" -j, --threads=NUM number of parallel threads\n")); - printf(_(" --progress show progress\n")); - printf(_(" --no-validate disable validation after backup\n")); - printf(_(" --skip-block-validation set to validate only file-level checksum\n")); - printf(_(" -E --external-dirs=external-directories-paths\n")); - printf(_(" backup some directories not from pgdata \n")); - printf(_(" (example: --external-dirs=/tmp/dir1:/tmp/dir2)\n")); - printf(_(" --no-sync do not sync backed up files to disk\n")); - printf(_(" --note=text add note to backup\n")); - printf(_(" (example: --note='backup before app update to v13.1')\n")); - - printf(_("\n Logging options:\n")); - printf(_(" --log-level-console=log-level-console\n")); - printf(_(" level for console logging (default: info)\n")); - printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); - printf(_(" --log-level-file=log-level-file\n")); - printf(_(" level for file logging (default: off)\n")); - printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); - printf(_(" --log-format-console=log-format-console\n")); - printf(_(" defines the format of the console log (default: plain)\n")); - printf(_(" available options: 'plain', 'json'\n")); - printf(_(" --log-format-file=log-format-file\n")); - printf(_(" defines the format of log files (default: plain)\n")); - printf(_(" available options: 'plain', 'json'\n")); - printf(_(" --log-filename=log-filename\n")); - printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); - printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); - printf(_(" --error-log-filename=error-log-filename\n")); - printf(_(" filename for error logging (default: none)\n")); - printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); - printf(_(" --log-rotation-size=log-rotation-size\n")); - printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); - printf(_(" --log-rotation-age=log-rotation-age\n")); - printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); - printf(_(" --no-color disable the coloring of error and warning console messages\n")); - - printf(_("\n Retention options:\n")); - printf(_(" --delete-expired delete backups expired according to current\n")); - printf(_(" retention policy after successful backup completion\n")); - printf(_(" --merge-expired merge backups expired according to current\n")); - printf(_(" retention policy after successful backup completion\n")); - printf(_(" --delete-wal remove redundant files in WAL archive\n")); - printf(_(" --retention-redundancy=retention-redundancy\n")); - printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); - printf(_(" --retention-window=retention-window\n")); - printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); - printf(_(" --wal-depth=wal-depth number of latest valid backups per timeline that must\n")); - printf(_(" retain the ability to perform PITR; 0 disables; (default: 0)\n")); - printf(_(" --dry-run perform a trial run without any changes\n")); - - printf(_("\n Pinning options:\n")); - printf(_(" --ttl=interval pin backup for specified amount of time; 0 unpin\n")); - printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: s)\n")); - printf(_(" (example: --ttl=20d)\n")); - printf(_(" --expire-time=time pin backup until specified time stamp\n")); - printf(_(" (example: --expire-time='2024-01-01 00:00:00+03')\n")); - - printf(_("\n Compression options:\n")); - printf(_(" --compress alias for --compress-algorithm='zlib' and --compress-level=1\n")); - printf(_(" --compress-algorithm=compress-algorithm\n")); - printf(_(" available options: 'zlib', 'pglz', 'none' (default: none)\n")); - printf(_(" --compress-level=compress-level\n")); - printf(_(" level of compression [0-9] (default: 1)\n")); - - printf(_("\n Archive options:\n")); - printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); - - printf(_("\n Connection options:\n")); - printf(_(" -U, --pguser=USERNAME user name to connect as (default: current local user)\n")); - printf(_(" -d, --pgdatabase=DBNAME database to connect (default: username)\n")); - printf(_(" -h, --pghost=HOSTNAME database server host or socket directory(default: 'local socket')\n")); - printf(_(" -p, --pgport=PORT database server port (default: 5432)\n")); - printf(_(" -w, --no-password never prompt for password\n")); - printf(_(" -W, --password force password prompt\n")); - - printf(_("\n Remote options:\n")); - printf(_(" --remote-proto=protocol remote protocol to use\n")); - printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); - printf(_(" --remote-host=destination remote host address or hostname\n")); - printf(_(" --remote-port=port remote host port (default: 22)\n")); - printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); - printf(_(" (default: current binary path)\n")); - printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); - printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); - printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n")); - - printf(_("\n Replica options:\n")); - printf(_(" --master-user=user_name user name to connect to master (deprecated)\n")); - printf(_(" --master-db=db_name database to connect to master (deprecated)\n")); - printf(_(" --master-host=host_name database server host of master (deprecated)\n")); - printf(_(" --master-port=port database server port of master (deprecated)\n")); - printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (deprecated)\n\n")); -} - -static void -help_restore(void) -{ - printf(_("\n%s restore -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-path] [-i backup-id] [-j num-threads]\n")); - printf(_(" [--progress] [--force] [--no-sync]\n")); - printf(_(" [--no-validate] [--skip-block-validation]\n")); - printf(_(" [-T OLDDIR=NEWDIR]\n")); - printf(_(" [--external-mapping=OLDDIR=NEWDIR]\n")); - printf(_(" [--skip-external-dirs]\n")); - printf(_(" [-X WALDIR | --waldir=WALDIR]\n")); - printf(_(" [-I | --incremental-mode=none|checksum|lsn]\n")); - printf(_(" [--db-include dbname | --db-exclude dbname]\n")); - printf(_(" [--destroy-all-other-dbs]\n")); - printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); - printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); - printf(_(" [--recovery-target-timeline=timeline]\n")); - printf(_(" [--recovery-target=immediate|latest]\n")); - printf(_(" [--recovery-target-name=target-name]\n")); - printf(_(" [--recovery-target-action=pause|promote|shutdown]\n")); - printf(_(" [--restore-command=cmdline]\n")); - printf(_(" [-R | --restore-as-replica]\n")); - printf(_(" [--primary-conninfo=primary_conninfo]\n")); - printf(_(" [-S | --primary-slot-name=slotname]\n")); - printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); - printf(_(" [--ssh-options]\n")); - printf(_(" [--archive-host=hostname] [--archive-port=port]\n")); - printf(_(" [--archive-user=username]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); - - printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); - printf(_(" -i, --backup-id=backup-id backup to restore\n")); - printf(_(" -j, --threads=NUM number of parallel threads\n")); - - printf(_(" --progress show progress\n")); - printf(_(" --force ignore invalid status of the restored backup\n")); - printf(_(" --no-sync do not sync restored files to disk\n")); - printf(_(" --no-validate disable backup validation during restore\n")); - printf(_(" --skip-block-validation set to validate only file-level checksum\n")); - - printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); - printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); - printf(_(" --external-mapping=OLDDIR=NEWDIR\n")); - printf(_(" relocate the external directory from OLDDIR to NEWDIR\n")); - printf(_(" --skip-external-dirs do not restore all external directories\n")); - - - printf(_(" -X, --waldir=WALDIR location for the write-ahead log directory\n")); - - - printf(_("\n Incremental restore options:\n")); - printf(_(" -I, --incremental-mode=none|checksum|lsn\n")); - printf(_(" reuse valid pages available in PGDATA if they have not changed\n")); - printf(_(" (default: none)\n")); - - printf(_("\n Partial restore options:\n")); - printf(_(" --db-include dbname restore only specified databases\n")); - printf(_(" --db-exclude dbname do not restore specified databases\n")); - printf(_(" --destroy-all-other-dbs\n")); - printf(_(" allows to do partial restore that is prohibited by default,\n")); - printf(_(" because it might remove all other databases.\n")); - - printf(_("\n Recovery options:\n")); - printf(_(" --recovery-target-time=time time stamp up to which recovery will proceed\n")); - printf(_(" --recovery-target-xid=xid transaction ID up to which recovery will proceed\n")); - printf(_(" --recovery-target-lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); - printf(_(" --recovery-target-inclusive=boolean\n")); - printf(_(" whether we stop just after the recovery target\n")); - printf(_(" --recovery-target-timeline=timeline\n")); - printf(_(" recovering into a particular timeline\n")); - printf(_(" --recovery-target=immediate|latest\n")); - printf(_(" end recovery as soon as a consistent state is reached or as late as possible\n")); - printf(_(" --recovery-target-name=target-name\n")); - printf(_(" the named restore point to which recovery will proceed\n")); - printf(_(" --recovery-target-action=pause|promote|shutdown\n")); - printf(_(" action the server should take once the recovery target is reached\n")); - printf(_(" (default: pause)\n")); - printf(_(" --restore-command=cmdline command to use as 'restore_command' in recovery.conf; 'none' disables\n")); - - printf(_("\n Standby options:\n")); - printf(_(" -R, --restore-as-replica write a minimal recovery.conf in the output directory\n")); - printf(_(" to ease setting up a standby server\n")); - printf(_(" --primary-conninfo=primary_conninfo\n")); - printf(_(" connection string to be used for establishing connection\n")); - printf(_(" with the primary server\n")); - printf(_(" -S, --primary-slot-name=slotname replication slot to be used for WAL streaming from the primary server\n")); - - printf(_("\n Logging options:\n")); - printf(_(" --log-level-console=log-level-console\n")); - printf(_(" level for console logging (default: info)\n")); - printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); - printf(_(" --log-level-file=log-level-file\n")); - printf(_(" level for file logging (default: off)\n")); - printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); - printf(_(" --log-format-console=log-format-console\n")); - printf(_(" defines the format of the console log (default: plain)\n")); - printf(_(" available options: 'plain', 'json'\n")); - printf(_(" --log-format-file=log-format-file\n")); - printf(_(" defines the format of log files (default: plain)\n")); - printf(_(" available options: 'plain', 'json'\n")); - printf(_(" --log-filename=log-filename\n")); - printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); - printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); - printf(_(" --error-log-filename=error-log-filename\n")); - printf(_(" filename for error logging (default: none)\n")); - printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); - printf(_(" --log-rotation-size=log-rotation-size\n")); - printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); - printf(_(" --log-rotation-age=log-rotation-age\n")); - printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); - printf(_(" --no-color disable the coloring of error and warning console messages\n")); - - printf(_("\n Remote options:\n")); - printf(_(" --remote-proto=protocol remote protocol to use\n")); - printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); - printf(_(" --remote-host=destination remote host address or hostname\n")); - printf(_(" --remote-port=port remote host port (default: 22)\n")); - printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); - printf(_(" (default: current binary path)\n")); - printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); - printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); - printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n")); - - printf(_("\n Remote WAL archive options:\n")); - printf(_(" --archive-host=destination address or hostname for ssh connection to archive host\n")); - printf(_(" --archive-port=port port for ssh connection to archive host (default: 22)\n")); - printf(_(" --archive-user=username user name for ssh connection to archive host (default: PostgreSQL user)\n\n")); -} - -static void -help_validate(void) -{ - printf(_("\n%s validate -B backup-path [--instance=instance_name]\n"), PROGRAM_NAME); - printf(_(" [-i backup-id] [--progress] [-j num-threads]\n")); - printf(_(" [--recovery-target-time=time|--recovery-target-xid=xid\n")); - printf(_(" |--recovery-target-lsn=lsn [--recovery-target-inclusive=boolean]]\n")); - printf(_(" [--recovery-target-timeline=timeline]\n")); - printf(_(" [--recovery-target-name=target-name]\n")); - printf(_(" [--skip-block-validation]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); - printf(_(" -i, --backup-id=backup-id backup to validate\n")); - - printf(_(" --progress show progress\n")); - printf(_(" -j, --threads=NUM number of parallel threads\n")); - printf(_(" --recovery-target-time=time time stamp up to which recovery will proceed\n")); - printf(_(" --recovery-target-xid=xid transaction ID up to which recovery will proceed\n")); - printf(_(" --recovery-target-lsn=lsn LSN of the write-ahead log location up to which recovery will proceed\n")); - printf(_(" --recovery-target-inclusive=boolean\n")); - printf(_(" whether we stop just after the recovery target\n")); - printf(_(" --recovery-target-timeline=timeline\n")); - printf(_(" recovering into a particular timeline\n")); - printf(_(" --recovery-target-name=target-name\n")); - printf(_(" the named restore point to which recovery will proceed\n")); - printf(_(" --skip-block-validation set to validate only file-level checksum\n")); - - printf(_("\n Logging options:\n")); - printf(_(" --log-level-console=log-level-console\n")); - printf(_(" level for console logging (default: info)\n")); - printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); - printf(_(" --log-level-file=log-level-file\n")); - printf(_(" level for file logging (default: off)\n")); - printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); - printf(_(" --log-format-console=log-format-console\n")); - printf(_(" defines the format of the console log (default: plain)\n")); - printf(_(" available options: 'plain', 'json'\n")); - printf(_(" --log-format-file=log-format-file\n")); - printf(_(" defines the format of log files (default: plain)\n")); - printf(_(" available options: 'plain', 'json'\n")); - printf(_(" --log-filename=log-filename\n")); - printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); - printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); - printf(_(" --error-log-filename=error-log-filename\n")); - printf(_(" filename for error logging (default: none)\n")); - printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); - printf(_(" --log-rotation-size=log-rotation-size\n")); - printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); - printf(_(" --log-rotation-age=log-rotation-age\n")); - printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); - printf(_(" --no-color disable the coloring of error and warning console messages\n\n")); -} - -static void -help_checkdb(void) -{ - printf(_("\n%s checkdb [-B backup-path] [--instance=instance_name]\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-path] [-j num-threads] [--progress]\n")); - printf(_(" [--amcheck] [--skip-block-validation]\n")); - printf(_(" [--heapallindexed] [--checkunique]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); - printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); - - printf(_(" --progress show progress\n")); - printf(_(" -j, --threads=NUM number of parallel threads\n")); - printf(_(" --skip-block-validation skip file-level checking\n")); - printf(_(" can be used only with '--amcheck' option\n")); - printf(_(" --amcheck in addition to file-level block checking\n")); - printf(_(" check btree indexes via function 'bt_index_check()'\n")); - printf(_(" using 'amcheck' or 'amcheck_next' extensions\n")); - printf(_(" --heapallindexed also check that heap is indexed\n")); - printf(_(" can be used only with '--amcheck' option\n")); - printf(_(" --checkunique also check unique constraints\n")); - printf(_(" can be used only with '--amcheck' option\n")); - - printf(_("\n Logging options:\n")); - printf(_(" --log-level-console=log-level-console\n")); - printf(_(" level for console logging (default: info)\n")); - printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); - printf(_(" --log-level-file=log-level-file\n")); - printf(_(" level for file logging (default: off)\n")); - printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); - printf(_(" --log-format-console=log-format-console\n")); - printf(_(" defines the format of the console log (default: plain)\n")); - printf(_(" available options: 'plain', 'json'\n")); - printf(_(" --log-format-file=log-format-file\n")); - printf(_(" defines the format of log files (default: plain)\n")); - printf(_(" available options: 'plain', 'json'\n")); - printf(_(" --log-filename=log-filename\n")); - printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); - printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log\n")); - printf(_(" --error-log-filename=error-log-filename\n")); - printf(_(" filename for error logging (default: none)\n")); - printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); - printf(_(" --log-rotation-size=log-rotation-size\n")); - printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); - printf(_(" --log-rotation-age=log-rotation-age\n")); - printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); - printf(_(" --no-color disable the coloring of error and warning console messages\n")); - - printf(_("\n Connection options:\n")); - printf(_(" -U, --pguser=USERNAME user name to connect as (default: current local user)\n")); - printf(_(" -d, --pgdatabase=DBNAME database to connect (default: username)\n")); - printf(_(" -h, --pghost=HOSTNAME database server host or socket directory(default: 'local socket')\n")); - printf(_(" -p, --pgport=PORT database server port (default: 5432)\n")); - printf(_(" -w, --no-password never prompt for password\n")); - printf(_(" -W, --password force password prompt\n\n")); -} - -static void -help_show(void) -{ - printf(_("\n%s show -B backup-path\n"), PROGRAM_NAME); - printf(_(" [--instance=instance_name [-i backup-id]]\n")); - printf(_(" [--format=format] [--archive]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name show info about specific instance\n")); - printf(_(" -i, --backup-id=backup-id show info about specific backups\n")); - printf(_(" --archive show WAL archive information\n")); - printf(_(" --format=format show format=PLAIN|JSON\n")); - printf(_(" --no-color disable the coloring for plain format\n\n")); -} - -static void -help_delete(void) -{ - printf(_("\n%s delete -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [-i backup-id | --delete-expired | --merge-expired] [--delete-wal]\n")); - printf(_(" [-j num-threads] [--progress]\n")); - printf(_(" [--retention-redundancy=retention-redundancy]\n")); - printf(_(" [--retention-window=retention-window]\n")); - printf(_(" [--wal-depth=wal-depth]\n")); - printf(_(" [--no-validate] [--no-sync]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); - printf(_(" -i, --backup-id=backup-id backup to delete\n")); - printf(_(" -j, --threads=NUM number of parallel threads\n")); - printf(_(" --progress show progress\n")); - printf(_(" --no-validate disable validation during retention merge\n")); - printf(_(" --no-sync do not sync merged files to disk\n")); - - printf(_("\n Retention options:\n")); - printf(_(" --delete-expired delete backups expired according to current\n")); - printf(_(" retention policy\n")); - printf(_(" --merge-expired merge backups expired according to current\n")); - printf(_(" retention policy\n")); - printf(_(" --delete-wal remove redundant files in WAL archive\n")); - printf(_(" --retention-redundancy=retention-redundancy\n")); - printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); - printf(_(" --retention-window=retention-window\n")); - printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); - printf(_(" --wal-depth=wal-depth number of latest valid backups per timeline that must\n")); - printf(_(" retain the ability to perform PITR; 0 disables; (default: 0)\n")); - printf(_(" --dry-run perform a trial run without any changes\n")); - printf(_(" --status=backup_status delete all backups with specified status\n")); - - printf(_("\n Logging options:\n")); - printf(_(" --log-level-console=log-level-console\n")); - printf(_(" level for console logging (default: info)\n")); - printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); - printf(_(" --log-level-file=log-level-file\n")); - printf(_(" level for file logging (default: off)\n")); - printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); - printf(_(" --log-format-console=log-format-console\n")); - printf(_(" defines the format of the console log (default: plain)\n")); - printf(_(" available options: 'plain', 'json'\n")); - printf(_(" --log-format-file=log-format-file\n")); - printf(_(" defines the format of log files (default: plain)\n")); - printf(_(" available options: 'plain', 'json'\n")); - printf(_(" --log-filename=log-filename\n")); - printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); - printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); - printf(_(" --error-log-filename=error-log-filename\n")); - printf(_(" filename for error logging (default: none)\n")); - printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); - printf(_(" --log-rotation-size=log-rotation-size\n")); - printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); - printf(_(" --log-rotation-age=log-rotation-age\n")); - printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); - printf(_(" --no-color disable the coloring of error and warning console messages\n\n")); -} - -static void -help_merge(void) -{ - printf(_("\n%s merge -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" -i backup-id [-j num-threads] [--progress]\n")); - printf(_(" [--no-validate] [--no-sync]\n")); - printf(_(" [--log-level-console=log-level-console]\n")); - printf(_(" [--log-level-file=log-level-file]\n")); - printf(_(" [--log-format-console=log-format-console]\n")); - printf(_(" [--log-format-file=log-format-file]\n")); - printf(_(" [--log-filename=log-filename]\n")); - printf(_(" [--error-log-filename=error-log-filename]\n")); - printf(_(" [--log-directory=log-directory]\n")); - printf(_(" [--log-rotation-size=log-rotation-size]\n")); - printf(_(" [--log-rotation-age=log-rotation-age]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); - printf(_(" -i, --backup-id=backup-id backup to merge\n")); - - printf(_(" -j, --threads=NUM number of parallel threads\n")); - printf(_(" --progress show progress\n")); - printf(_(" --no-validate disable validation during retention merge\n")); - printf(_(" --no-sync do not sync merged files to disk\n")); - - printf(_("\n Logging options:\n")); - printf(_(" --log-level-console=log-level-console\n")); - printf(_(" level for console logging (default: info)\n")); - printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); - printf(_(" --log-level-file=log-level-file\n")); - printf(_(" level for file logging (default: off)\n")); - printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); - printf(_(" --log-format-console=log-format-console\n")); - printf(_(" defines the format of the console log (default: plain)\n")); - printf(_(" available options: 'plain', 'json'\n")); - printf(_(" --log-format-file=log-format-file\n")); - printf(_(" defines the format of log files (default: plain)\n")); - printf(_(" available options: 'plain', 'json'\n")); - printf(_(" --log-filename=log-filename\n")); - printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); - printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); - printf(_(" --error-log-filename=error-log-filename\n")); - printf(_(" filename for error logging (default: none)\n")); - printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); - printf(_(" --log-rotation-size=log-rotation-size\n")); - printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); - printf(_(" --log-rotation-age=log-rotation-age\n")); - printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); - printf(_(" --no-color disable the coloring of error and warning console messages\n\n")); -} - -static void -help_set_backup(void) -{ - printf(_("\n%s set-backup -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" -i backup-id\n")); - printf(_(" [--ttl=interval] [--expire-time=time] [--note=text]\n\n")); - - printf(_(" --ttl=interval pin backup for specified amount of time; 0 unpin\n")); - printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: s)\n")); - printf(_(" (example: --ttl=20d)\n")); - printf(_(" --expire-time=time pin backup until specified time stamp\n")); - printf(_(" (example: --expire-time='2024-01-01 00:00:00+03')\n")); - printf(_(" --note=text add note to backup; 'none' to remove note\n")); - printf(_(" (example: --note='backup before app update to v13.1')\n")); -} - -static void -help_set_config(void) -{ - printf(_("\n%s set-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [-D pgdata-path]\n")); - printf(_(" [-E external-directories-paths]\n")); - printf(_(" [--restore-command=cmdline]\n")); - printf(_(" [--log-level-console=log-level-console]\n")); - printf(_(" [--log-level-file=log-level-file]\n")); - printf(_(" [--log-format-file=log-format-file]\n")); - printf(_(" [--log-filename=log-filename]\n")); - printf(_(" [--error-log-filename=error-log-filename]\n")); - printf(_(" [--log-directory=log-directory]\n")); - printf(_(" [--log-rotation-size=log-rotation-size]\n")); - printf(_(" [--log-rotation-age=log-rotation-age]\n")); - printf(_(" [--retention-redundancy=retention-redundancy]\n")); - printf(_(" [--retention-window=retention-window]\n")); - printf(_(" [--wal-depth=wal-depth]\n")); - printf(_(" [--compress-algorithm=compress-algorithm]\n")); - printf(_(" [--compress-level=compress-level]\n")); - printf(_(" [--archive-timeout=timeout]\n")); - printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); - printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); - printf(_(" [--ssh-options]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); - printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); - printf(_(" -E --external-dirs=external-directories-paths\n")); - printf(_(" backup some directories not from pgdata \n")); - printf(_(" (example: --external-dirs=/tmp/dir1:/tmp/dir2)\n")); - printf(_(" --restore-command=cmdline command to use as 'restore_command' in recovery.conf; 'none' disables\n")); - - printf(_("\n Logging options:\n")); - printf(_(" --log-level-console=log-level-console\n")); - printf(_(" level for console logging (default: info)\n")); - printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); - printf(_(" --log-level-file=log-level-file\n")); - printf(_(" level for file logging (default: off)\n")); - printf(_(" available options: 'off', 'error', 'warning', 'info', 'log', 'verbose'\n")); - printf(_(" --log-format-file=log-format-file\n")); - printf(_(" defines the format of log files (default: plain)\n")); - printf(_(" available options: 'plain', 'json'\n")); - printf(_(" --log-filename=log-filename\n")); - printf(_(" filename for file logging (default: 'pg_probackup.log')\n")); - printf(_(" support strftime format (example: pg_probackup-%%Y-%%m-%%d_%%H%%M%%S.log)\n")); - printf(_(" --error-log-filename=error-log-filename\n")); - printf(_(" filename for error logging (default: none)\n")); - printf(_(" --log-directory=log-directory\n")); - printf(_(" directory for file logging (default: BACKUP_PATH/log)\n")); - printf(_(" --log-rotation-size=log-rotation-size\n")); - printf(_(" rotate logfile if its size exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'kB', 'MB', 'GB', 'TB' (default: kB)\n")); - printf(_(" --log-rotation-age=log-rotation-age\n")); - printf(_(" rotate logfile if its age exceeds this value; 0 disables; (default: 0)\n")); - printf(_(" available units: 'ms', 's', 'min', 'h', 'd' (default: min)\n")); - - printf(_("\n Retention options:\n")); - printf(_(" --retention-redundancy=retention-redundancy\n")); - printf(_(" number of full backups to keep; 0 disables; (default: 0)\n")); - printf(_(" --retention-window=retention-window\n")); - printf(_(" number of days of recoverability; 0 disables; (default: 0)\n")); - printf(_(" --wal-depth=wal-depth number of latest valid backups with ability to perform\n")); - printf(_(" the point in time recovery; disables; (default: 0)\n")); - - printf(_("\n Compression options:\n")); - printf(_(" --compress alias for --compress-algorithm='zlib' and --compress-level=1\n")); - printf(_(" --compress-algorithm=compress-algorithm\n")); - printf(_(" available options: 'zlib','pglz','none' (default: 'none')\n")); - printf(_(" --compress-level=compress-level\n")); - printf(_(" level of compression [0-9] (default: 1)\n")); - - printf(_("\n Archive options:\n")); - printf(_(" --archive-timeout=timeout wait timeout for WAL segment archiving (default: 5min)\n")); - - printf(_("\n Connection options:\n")); - printf(_(" -U, --pguser=USERNAME user name to connect as (default: current local user)\n")); - printf(_(" -d, --pgdatabase=DBNAME database to connect (default: username)\n")); - printf(_(" -h, --pghost=HOSTNAME database server host or socket directory(default: 'local socket')\n")); - printf(_(" -p, --pgport=PORT database server port (default: 5432)\n")); - - printf(_("\n Remote options:\n")); - printf(_(" --remote-proto=protocol remote protocol to use\n")); - printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); - printf(_(" --remote-host=destination remote host address or hostname\n")); - printf(_(" --remote-port=port remote host port (default: 22)\n")); - printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); - printf(_(" (default: current binary path)\n")); - printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); - printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); - printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n")); - - printf(_("\n Remote WAL archive options:\n")); - printf(_(" --archive-host=destination address or hostname for ssh connection to archive host\n")); - printf(_(" --archive-port=port port for ssh connection to archive host (default: 22)\n")); - printf(_(" --archive-user=username user name for ssh connection to archive host (default: PostgreSQL user)\n")); - - printf(_("\n Replica options:\n")); - printf(_(" --master-user=user_name user name to connect to master (deprecated)\n")); - printf(_(" --master-db=db_name database to connect to master (deprecated)\n")); - printf(_(" --master-host=host_name database server host of master (deprecated)\n")); - printf(_(" --master-port=port database server port of master (deprecated)\n")); - printf(_(" --replica-timeout=timeout wait timeout for WAL segment streaming through replication (deprecated)\n\n")); -} - -static void -help_show_config(void) -{ - printf(_("\n%s show-config -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" [--format=format]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance\n")); - printf(_(" --format=format show format=PLAIN|JSON\n\n")); -} - -static void -help_add_instance(void) -{ - printf(_("\n%s add-instance -B backup-path -D pgdata-path\n"), PROGRAM_NAME); - printf(_(" --instance=instance_name\n")); - printf(_(" [-E external-directory-path]\n")); - printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); - printf(_(" [--ssh-options]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" -D, --pgdata=pgdata-path location of the database storage area\n")); - printf(_(" --instance=instance_name name of the new instance\n")); - - printf(_(" -E --external-dirs=external-directories-paths\n")); - printf(_(" backup some directories not from pgdata \n")); - printf(_(" (example: --external-dirs=/tmp/dir1:/tmp/dir2)\n")); - printf(_("\n Remote options:\n")); - printf(_(" --remote-proto=protocol remote protocol to use\n")); - printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); - printf(_(" --remote-host=destination remote host address or hostname\n")); - printf(_(" --remote-port=port remote host port (default: 22)\n")); - printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); - printf(_(" (default: current binary path)\n")); - printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); - printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); - printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n")); -} - -static void -help_del_instance(void) -{ - printf(_("\n%s del-instance -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance to delete\n\n")); -} - -static void -help_archive_push(void) -{ - printf(_("\n%s archive-push -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" --wal-file-name=wal-file-name\n")); - printf(_(" [--wal-file-path=wal-file-path]\n")); - printf(_(" [-j num-threads] [--batch-size=batch_size]\n")); - printf(_(" [--archive-timeout=timeout]\n")); - printf(_(" [--no-ready-rename] [--no-sync]\n")); - printf(_(" [--overwrite] [--compress]\n")); - printf(_(" [--compress-algorithm=compress-algorithm]\n")); - printf(_(" [--compress-level=compress-level]\n")); - printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); - printf(_(" [--ssh-options]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance to delete\n")); - printf(_(" --wal-file-name=wal-file-name\n")); - printf(_(" name of the file to copy into WAL archive\n")); - printf(_(" --wal-file-path=wal-file-path\n")); - printf(_(" relative destination path of the WAL archive\n")); - printf(_(" -j, --threads=NUM number of parallel threads\n")); - printf(_(" --batch-size=NUM number of files to be copied\n")); - printf(_(" --archive-timeout=timeout wait timeout before discarding stale temp file(default: 5min)\n")); - printf(_(" --no-ready-rename do not rename '.ready' files in 'archive_status' directory\n")); - printf(_(" --no-sync do not sync WAL file to disk\n")); - printf(_(" --overwrite overwrite archived WAL file\n")); - - printf(_("\n Compression options:\n")); - printf(_(" --compress alias for --compress-algorithm='zlib' and --compress-level=1\n")); - printf(_(" --compress-algorithm=compress-algorithm\n")); - printf(_(" available options: 'zlib','pglz','none' (default: 'none')\n")); - printf(_(" --compress-level=compress-level\n")); - printf(_(" level of compression [0-9] (default: 1)\n")); - - printf(_("\n Remote options:\n")); - printf(_(" --remote-proto=protocol remote protocol to use\n")); - printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); - printf(_(" --remote-host=hostname remote host address or hostname\n")); - printf(_(" --remote-port=port remote host port (default: 22)\n")); - printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); - printf(_(" (default: current binary path)\n")); - printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); - printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); - printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n")); -} - -static void -help_archive_get(void) -{ - printf(_("\n%s archive-get -B backup-path --instance=instance_name\n"), PROGRAM_NAME); - printf(_(" --wal-file-name=wal-file-name\n")); - printf(_(" [--wal-file-path=wal-file-path]\n")); - printf(_(" [-j num-threads] [--batch-size=batch_size]\n")); - printf(_(" [--no-validate-wal]\n")); - printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); - printf(_(" [--ssh-options]\n\n")); - - printf(_(" -B, --backup-path=backup-path location of the backup storage area\n")); - printf(_(" --instance=instance_name name of the instance to delete\n")); - printf(_(" --wal-file-path=wal-file-path\n")); - printf(_(" relative destination path name of the WAL file on the server\n")); - printf(_(" --wal-file-name=wal-file-name\n")); - printf(_(" name of the WAL file to retrieve from the archive\n")); - printf(_(" -j, --threads=NUM number of parallel threads\n")); - printf(_(" --batch-size=NUM number of files to be prefetched\n")); - printf(_(" --prefetch-dir=path location of the store area for prefetched WAL files\n")); - printf(_(" --no-validate-wal skip validation of prefetched WAL file before using it\n")); - - printf(_("\n Remote options:\n")); - printf(_(" --remote-proto=protocol remote protocol to use\n")); - printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); - printf(_(" --remote-host=hostname remote host address or hostname\n")); - printf(_(" --remote-port=port remote host port (default: 22)\n")); - printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); - printf(_(" (default: current binary path)\n")); - printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); - printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); - printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n")); -} - -static void -help_help(void) -{ - printf(_("\n%s help [command]\n"), PROGRAM_NAME); - printf(_("%s command --help\n\n"), PROGRAM_NAME); -} - -static void -help_version(void) -{ - printf(_("\n%s version\n"), PROGRAM_NAME); - printf(_("%s --version\n\n"), PROGRAM_NAME); -} - -static void -help_catchup(void) -{ - printf(_("\n%s catchup -b catchup-mode\n"), PROGRAM_NAME); - printf(_(" --source-pgdata=path_to_pgdata_on_remote_server\n")); - printf(_(" --destination-pgdata=path_to_local_dir\n")); - printf(_(" [--stream [-S slot-name]] [--temp-slot | --perm-slot]\n")); - printf(_(" [-j num-threads]\n")); - printf(_(" [-T OLDDIR=NEWDIR]\n")); - printf(_(" [--exclude-path=path_prefix]\n")); - printf(_(" [-d dbname] [-h host] [-p port] [-U username]\n")); - printf(_(" [-w --no-password] [-W --password]\n")); - printf(_(" [--remote-proto] [--remote-host]\n")); - printf(_(" [--remote-port] [--remote-path] [--remote-user]\n")); - printf(_(" [--ssh-options]\n")); - printf(_(" [--dry-run]\n")); - printf(_(" [--help]\n\n")); - - printf(_(" -b, --backup-mode=catchup-mode catchup mode=FULL|DELTA|PTRACK\n")); - printf(_(" --stream stream the transaction log (only supported mode)\n")); - printf(_(" -S, --slot=SLOTNAME replication slot to use\n")); - printf(_(" --temp-slot use temporary replication slot\n")); - printf(_(" -P --perm-slot create permanent replication slot\n")); - - printf(_(" -j, --threads=NUM number of parallel threads\n")); - - printf(_(" -T, --tablespace-mapping=OLDDIR=NEWDIR\n")); - printf(_(" relocate the tablespace from directory OLDDIR to NEWDIR\n")); - printf(_(" -x, --exclude-path=path_prefix files with path_prefix (relative to pgdata) will be\n")); - printf(_(" excluded from catchup (can be used multiple times)\n")); - printf(_(" Dangerous option! Use at your own risk!\n")); - - printf(_("\n Connection options:\n")); - printf(_(" -U, --pguser=USERNAME user name to connect as (default: current local user)\n")); - printf(_(" -d, --pgdatabase=DBNAME database to connect (default: username)\n")); - printf(_(" -h, --pghost=HOSTNAME database server host or socket directory(default: 'local socket')\n")); - printf(_(" -p, --pgport=PORT database server port (default: 5432)\n")); - printf(_(" -w, --no-password never prompt for password\n")); - printf(_(" -W, --password force password prompt\n\n")); - - printf(_("\n Remote options:\n")); - printf(_(" --remote-proto=protocol remote protocol to use\n")); - printf(_(" available options: 'ssh', 'none' (default: ssh)\n")); - printf(_(" --remote-host=hostname remote host address or hostname\n")); - printf(_(" --remote-port=port remote host port (default: 22)\n")); - printf(_(" --remote-path=path path to directory with pg_probackup binary on remote host\n")); - printf(_(" (default: current binary path)\n")); - printf(_(" --remote-user=username user name for ssh connection (default: current user)\n")); - printf(_(" --ssh-options=ssh_options additional ssh options (default: none)\n")); - printf(_(" (example: --ssh-options='-c cipher_spec -F configfile')\n\n")); - - printf(_(" --dry-run perform a trial run without any changes\n\n")); -} diff --git a/src/init.c b/src/init.c deleted file mode 100644 index 837e2bad0..000000000 --- a/src/init.c +++ /dev/null @@ -1,127 +0,0 @@ -/*------------------------------------------------------------------------- - * - * init.c: - initialize backup catalog. - * - * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -#include -#include - -/* - * Initialize backup catalog. - */ -int -do_init(CatalogState *catalogState) -{ - int results; - - results = pg_check_dir(catalogState->catalog_path); - - if (results == 4) /* exists and not empty*/ - elog(ERROR, "The backup catalog already exists and is not empty"); - else if (results == -1) /*trouble accessing directory*/ - { - int errno_tmp = errno; - elog(ERROR, "Cannot open backup catalog directory \"%s\": %s", - catalogState->catalog_path, strerror(errno_tmp)); - } - - /* create backup catalog root directory */ - dir_create_dir(catalogState->catalog_path, DIR_PERMISSION, false); - - /* create backup catalog data directory */ - dir_create_dir(catalogState->backup_subdir_path, DIR_PERMISSION, false); - - /* create backup catalog wal directory */ - dir_create_dir(catalogState->wal_subdir_path, DIR_PERMISSION, false); - - elog(INFO, "Backup catalog '%s' successfully initialized", catalogState->catalog_path); - return 0; -} - -int -do_add_instance(InstanceState *instanceState, InstanceConfig *instance) -{ - struct stat st; - CatalogState *catalogState = instanceState->catalog_state; - - /* PGDATA is always required */ - if (instance->pgdata == NULL) - elog(ERROR, "No postgres data directory specified.\n" - "Please specify it either using environment variable PGDATA or\n" - "command line option --pgdata (-D)"); - - /* Read system_identifier from PGDATA */ - instance->system_identifier = get_system_identifier(instance->pgdata, FIO_DB_HOST, false); - /* Starting from PostgreSQL 11 read WAL segment size from PGDATA */ - instance->xlog_seg_size = get_xlog_seg_size(instance->pgdata); - - /* Ensure that all root directories already exist */ - /* TODO maybe call do_init() here instead of error?*/ - if (access(catalogState->catalog_path, F_OK) != 0) - elog(ERROR, "Directory does not exist: '%s'", catalogState->catalog_path); - - if (access(catalogState->backup_subdir_path, F_OK) != 0) - elog(ERROR, "Directory does not exist: '%s'", catalogState->backup_subdir_path); - - if (access(catalogState->wal_subdir_path, F_OK) != 0) - elog(ERROR, "Directory does not exist: '%s'", catalogState->wal_subdir_path); - - if (stat(instanceState->instance_backup_subdir_path, &st) == 0 && S_ISDIR(st.st_mode)) - elog(ERROR, "Instance '%s' backup directory already exists: '%s'", - instanceState->instance_name, instanceState->instance_backup_subdir_path); - - /* - * Create directory for wal files of this specific instance. - * Existence check is extra paranoid because if we don't have such a - * directory in data dir, we shouldn't have it in wal as well. - */ - if (stat(instanceState->instance_wal_subdir_path, &st) == 0 && S_ISDIR(st.st_mode)) - elog(ERROR, "Instance '%s' WAL archive directory already exists: '%s'", - instanceState->instance_name, instanceState->instance_wal_subdir_path); - - /* Create directory for data files of this specific instance */ - dir_create_dir(instanceState->instance_backup_subdir_path, DIR_PERMISSION, false); - dir_create_dir(instanceState->instance_wal_subdir_path, DIR_PERMISSION, false); - - /* - * Write initial configuration file. - * system-identifier, xlog-seg-size and pgdata are set in init subcommand - * and will never be updated. - * - * We need to manually set options source to save them to the configuration - * file. - */ - config_set_opt(instance_options, &instance->system_identifier, - SOURCE_FILE); - config_set_opt(instance_options, &instance->xlog_seg_size, - SOURCE_FILE); - - /* Kludge: do not save remote options into config */ - config_set_opt(instance_options, &instance_config.remote.host, - SOURCE_DEFAULT); - config_set_opt(instance_options, &instance_config.remote.proto, - SOURCE_DEFAULT); - config_set_opt(instance_options, &instance_config.remote.port, - SOURCE_DEFAULT); - config_set_opt(instance_options, &instance_config.remote.path, - SOURCE_DEFAULT); - config_set_opt(instance_options, &instance_config.remote.user, - SOURCE_DEFAULT); - config_set_opt(instance_options, &instance_config.remote.ssh_options, - SOURCE_DEFAULT); - config_set_opt(instance_options, &instance_config.remote.ssh_config, - SOURCE_DEFAULT); - - /* pgdata was set through command line */ - do_set_config(instanceState, true); - - elog(INFO, "Instance '%s' successfully initialized", instanceState->instance_name); - return 0; -} diff --git a/src/merge.c b/src/merge.c deleted file mode 100644 index e8f926795..000000000 --- a/src/merge.c +++ /dev/null @@ -1,1441 +0,0 @@ -/*------------------------------------------------------------------------- - * - * merge.c: merge FULL and incremental backups - * - * Copyright (c) 2018-2022, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -#include -#include - -#include "utils/thread.h" - -typedef struct -{ - parray *merge_filelist; - parray *parent_chain; - - pgBackup *dest_backup; - pgBackup *full_backup; - - const char *full_database_dir; - const char *full_external_prefix; - -// size_t in_place_merge_bytes; - bool compression_match; - bool program_version_match; - bool use_bitmap; - bool is_retry; - bool no_sync; - - /* - * Return value from the thread. - * 0 means there is no error, 1 - there is an error. - */ - int ret; -} merge_files_arg; - - -static void *merge_files(void *arg); -static void -reorder_external_dirs(pgBackup *to_backup, parray *to_external, - parray *from_external); -static int -get_external_index(const char *key, const parray *list); - -static void -merge_data_file(parray *parent_chain, pgBackup *full_backup, - pgBackup *dest_backup, pgFile *dest_file, - pgFile *tmp_file, const char *to_root, bool use_bitmap, - bool is_retry, bool no_sync); - -static void -merge_non_data_file(parray *parent_chain, pgBackup *full_backup, - pgBackup *dest_backup, pgFile *dest_file, - pgFile *tmp_file, const char *full_database_dir, - const char *full_external_prefix, bool no_sync); - -static bool is_forward_compatible(parray *parent_chain); - -/* - * Implementation of MERGE command. - * - * - Find target and its parent full backup - * - Merge data files of target, parent and and intermediate backups - * - Remove unnecessary files, which doesn't exist in the target backup anymore - */ -void -do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool no_sync) -{ - parray *backups; - parray *merge_list = parray_new(); - pgBackup *dest_backup = NULL; - pgBackup *dest_backup_tmp = NULL; - pgBackup *full_backup = NULL; - int i; - - if (backup_id == INVALID_BACKUP_ID) - elog(ERROR, "Required parameter is not specified: --backup-id"); - - if (instanceState == NULL) - elog(ERROR, "Required parameter is not specified: --instance"); - - elog(INFO, "Merge started"); - - /* Get list of all backups sorted in order of descending start time */ - backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); - - /* Find destination backup first */ - for (i = 0; i < parray_num(backups); i++) - { - pgBackup *backup = (pgBackup *) parray_get(backups, i); - - /* found target */ - if (backup->start_time == backup_id) - { - /* sanity */ - if (backup->status != BACKUP_STATUS_OK && - backup->status != BACKUP_STATUS_DONE && - /* It is possible that previous merging was interrupted */ - backup->status != BACKUP_STATUS_MERGING && - backup->status != BACKUP_STATUS_MERGED && - backup->status != BACKUP_STATUS_DELETING) - elog(ERROR, "Backup %s has status: %s", - backup_id_of(backup), status2str(backup->status)); - - dest_backup = backup; - break; - } - } - - /* - * Handle the case of crash right after deletion of the target - * incremental backup. We still can recover from this. - * Iterate over backups and look for the FULL backup with - * MERGED status, that has merge-target-id eqial to backup_id. - */ - if (dest_backup == NULL) - { - for (i = 0; i < parray_num(backups); i++) - { - pgBackup *backup = (pgBackup *) parray_get(backups, i); - - if (backup->status == BACKUP_STATUS_MERGED && - backup->merge_dest_backup == backup_id) - { - dest_backup = backup; - break; - } - } - } - - if (dest_backup == NULL) - elog(ERROR, "Target backup %s was not found", base36enc(backup_id)); - - /* It is possible to use FULL backup as target backup for merge. - * There are two possible cases: - * 1. The user want to merge FULL backup with closest incremental backup. - * In this case we must find suitable destination backup and merge them. - * - * 2. Previous merge has failed after destination backup was deleted, - * but before FULL backup was renamed: - * Example A: - * PAGE2_1 OK - * FULL2 OK - * PAGE1_1 MISSING/DELETING <- - * FULL1 MERGED/MERGING - */ - if (dest_backup->backup_mode == BACKUP_MODE_FULL) - { - full_backup = dest_backup; - dest_backup = NULL; - elog(INFO, "Merge target backup %s is full backup", - backup_id_of(full_backup)); - - /* sanity */ - if (full_backup->status == BACKUP_STATUS_DELETING) - elog(ERROR, "Backup %s has status: %s", - backup_id_of(full_backup), - status2str(full_backup->status)); - - /* Case #1 */ - if (full_backup->status == BACKUP_STATUS_OK || - full_backup->status == BACKUP_STATUS_DONE) - { - /* Check the case of FULL backup having more than one direct children */ - if (is_prolific(backups, full_backup)) - elog(ERROR, "Merge target is full backup and has multiple direct children, " - "you must specify child backup id you want to merge with"); - - elog(INFO, "Looking for closest incremental backup to merge with"); - - /* Look for closest child backup */ - for (i = 0; i < parray_num(backups); i++) - { - pgBackup *backup = (pgBackup *) parray_get(backups, i); - - /* skip unsuitable candidates */ - if (backup->status != BACKUP_STATUS_OK && - backup->status != BACKUP_STATUS_DONE) - continue; - - if (backup->parent_backup == full_backup->start_time) - { - dest_backup = backup; - break; - } - } - - /* sanity */ - if (dest_backup == NULL) - elog(ERROR, "Failed to find merge candidate, " - "backup %s has no valid children", - backup_id_of(full_backup)); - - } - /* Case #2 */ - else if (full_backup->status == BACKUP_STATUS_MERGING) - { - /* - * MERGING - merge was ongoing at the moment of crash. - * We must find destination backup and rerun merge. - * If destination backup is missing, then merge must be aborted, - * there is no recovery from this situation. - */ - - if (full_backup->merge_dest_backup == INVALID_BACKUP_ID) - elog(ERROR, "Failed to determine merge destination backup"); - - /* look up destination backup */ - for (i = 0; i < parray_num(backups); i++) - { - pgBackup *backup = (pgBackup *) parray_get(backups, i); - - if (backup->start_time == full_backup->merge_dest_backup) - { - dest_backup = backup; - break; - } - } - if (!dest_backup) - { - elog(ERROR, "Full backup %s has unfinished merge with missing backup %s", - backup_id_of(full_backup), - base36enc(full_backup->merge_dest_backup)); - } - } - else if (full_backup->status == BACKUP_STATUS_MERGED) - { - /* - * MERGED - merge crashed after files were transfered, but - * before rename could take place. - * If destination backup is missing, this is ok. - * If destination backup is present, then it should be deleted. - * After that FULL backup must acquire destination backup ID. - */ - - /* destination backup may or may not exists */ - for (i = 0; i < parray_num(backups); i++) - { - pgBackup *backup = (pgBackup *) parray_get(backups, i); - - if (backup->start_time == full_backup->merge_dest_backup) - { - dest_backup = backup; - break; - } - } - if (!dest_backup) - { - elog(WARNING, "Full backup %s has unfinished merge with missing backup %s", - backup_id_of(full_backup), - base36enc(full_backup->merge_dest_backup)); - } - } - else - elog(ERROR, "Backup %s has status: %s", - backup_id_of(full_backup), - status2str(full_backup->status)); - } - else - { - /* - * Legal Case #1: - * PAGE2 OK <- target - * PAGE1 OK - * FULL OK - * Legal Case #2: - * PAGE2 MERGING <- target - * PAGE1 MERGING - * FULL MERGING - * Legal Case #3: - * PAGE2 MERGING <- target - * PAGE1 DELETING - * FULL MERGED - * Legal Case #4: - * PAGE2 MERGING <- target - * PAGE1 missing - * FULL MERGED - * Legal Case #5: - * PAGE2 DELETING <- target - * FULL MERGED - * Legal Case #6: - * PAGE2 MERGING <- target - * PAGE1 missing - * FULL MERGED - * Illegal Case #7: - * PAGE2 MERGING <- target - * PAGE1 missing - * FULL MERGING - */ - - if (dest_backup->status == BACKUP_STATUS_MERGING || - dest_backup->status == BACKUP_STATUS_DELETING) - elog(WARNING, "Rerun unfinished merge for backup %s", - backup_id_of(dest_backup)); - - /* First we should try to find parent FULL backup */ - full_backup = find_parent_full_backup(dest_backup); - - /* Chain is broken, one or more member of parent chain is missing */ - if (full_backup == NULL) - { - /* It is the legal state of affairs in Case #4, but - * only for MERGING incremental target backup and only - * if FULL backup has MERGED status. - */ - if (dest_backup->status != BACKUP_STATUS_MERGING) - elog(ERROR, "Failed to find parent full backup for %s", - backup_id_of(dest_backup)); - - /* Find FULL backup that has unfinished merge with dest backup */ - for (i = 0; i < parray_num(backups); i++) - { - pgBackup *backup = (pgBackup *) parray_get(backups, i); - - if (backup->merge_dest_backup == dest_backup->start_time) - { - full_backup = backup; - break; - } - } - - if (!full_backup) - elog(ERROR, "Failed to find full backup that has unfinished merge" - "with backup %s, cannot rerun merge", - backup_id_of(dest_backup)); - - if (full_backup->status == BACKUP_STATUS_MERGED) - elog(WARNING, "Incremental chain is broken, try to recover unfinished merge"); - else - elog(ERROR, "Incremental chain is broken, merge is impossible to finish"); - } - else - { - if ((full_backup->status == BACKUP_STATUS_MERGED || - full_backup->status == BACKUP_STATUS_MERGING) && - dest_backup->start_time != full_backup->merge_dest_backup) - { - elog(ERROR, "Full backup %s has unfinished merge with backup %s", - backup_id_of(full_backup), - base36enc(full_backup->merge_dest_backup)); - } - - } - } - - /* sanity */ - if (full_backup == NULL) - elog(ERROR, "Parent full backup for the given backup %s was not found", - base36enc(backup_id)); - - /* At this point NULL as dest_backup is allowed only in case of full backup - * having status MERGED */ - if (dest_backup == NULL && full_backup->status != BACKUP_STATUS_MERGED) - elog(ERROR, "Cannot run merge for full backup %s", - backup_id_of(full_backup)); - - /* sanity */ - if (full_backup->status != BACKUP_STATUS_OK && - full_backup->status != BACKUP_STATUS_DONE && - /* It is possible that previous merging was interrupted */ - full_backup->status != BACKUP_STATUS_MERGED && - full_backup->status != BACKUP_STATUS_MERGING) - elog(ERROR, "Backup %s has status: %s", - backup_id_of(full_backup), status2str(full_backup->status)); - - /* Form merge list */ - dest_backup_tmp = dest_backup; - - /* While loop below may looks strange, it is done so on purpose - * to handle both whole and broken incremental chains. - */ - while (dest_backup_tmp) - { - /* sanity */ - if (dest_backup_tmp->status != BACKUP_STATUS_OK && - dest_backup_tmp->status != BACKUP_STATUS_DONE && - /* It is possible that previous merging was interrupted */ - dest_backup_tmp->status != BACKUP_STATUS_MERGING && - dest_backup_tmp->status != BACKUP_STATUS_MERGED && - dest_backup_tmp->status != BACKUP_STATUS_DELETING) - elog(ERROR, "Backup %s has status: %s", - backup_id_of(dest_backup_tmp), - status2str(dest_backup_tmp->status)); - - if (dest_backup_tmp->backup_mode == BACKUP_MODE_FULL) - break; - - parray_append(merge_list, dest_backup_tmp); - dest_backup_tmp = dest_backup_tmp->parent_backup_link; - } - - /* Add FULL backup */ - parray_append(merge_list, full_backup); - - /* Lock merge chain */ - catalog_lock_backup_list(merge_list, parray_num(merge_list) - 1, 0, true, true); - - /* do actual merge */ - merge_chain(instanceState, merge_list, full_backup, dest_backup, no_validate, no_sync); - - if (!no_validate) - pgBackupValidate(full_backup, NULL); - if (full_backup->status == BACKUP_STATUS_CORRUPT) - elog(ERROR, "Merging of backup %s failed", base36enc(backup_id)); - - /* cleanup */ - parray_walk(backups, pgBackupFree); - parray_free(backups); - parray_free(merge_list); - - elog(INFO, "Merge of backup %s completed", base36enc(backup_id)); -} - -/* - * Merge backup chain. - * dest_backup - incremental backup. - * parent_chain - array of backups starting with dest_backup and - * ending with full_backup. - * - * Copy backup files from incremental backups from parent_chain into - * full backup directory. - * Remove unnecessary directories and files from full backup directory. - * Update metadata of full backup to represent destination backup. - * - * TODO: stop relying on caller to provide valid parent_chain, make sure - * that chain is ok. - */ -void -merge_chain(InstanceState *instanceState, - parray *parent_chain, pgBackup *full_backup, pgBackup *dest_backup, - bool no_validate, bool no_sync) -{ - int i; - char full_external_prefix[MAXPGPATH]; - char full_database_dir[MAXPGPATH]; - parray *full_externals = NULL, - *dest_externals = NULL; - - parray *result_filelist = NULL; - bool use_bitmap = true; - bool is_retry = false; -// size_t total_in_place_merge_bytes = 0; - - pthread_t *threads = NULL; - merge_files_arg *threads_args = NULL; - time_t merge_time; - bool merge_isok = true; - /* for fancy reporting */ - time_t end_time; - char pretty_time[20]; - /* in-place merge flags */ - bool compression_match = false; - bool program_version_match = false; - /* It's redundant to check block checksumms during merge */ - skip_block_validation = true; - - /* Handle corner cases of missing destination backup */ - if (dest_backup == NULL && - full_backup->status == BACKUP_STATUS_MERGED) - goto merge_rename; - - if (!dest_backup) - elog(ERROR, "Destination backup is missing, cannot continue merge"); - - if (dest_backup->status == BACKUP_STATUS_MERGING || - full_backup->status == BACKUP_STATUS_MERGING || - full_backup->status == BACKUP_STATUS_MERGED) - { - is_retry = true; - elog(INFO, "Retry failed merge of backup %s with parent chain", backup_id_of(dest_backup)); - } - else - elog(INFO, "Merging backup %s with parent chain", backup_id_of(dest_backup)); - - /* sanity */ - if (full_backup->merge_dest_backup != INVALID_BACKUP_ID && - full_backup->merge_dest_backup != dest_backup->start_time) - { - elog(ERROR, "Cannot run merge for %s, because full backup %s has " - "unfinished merge with backup %s", - backup_id_of(dest_backup), - backup_id_of(full_backup), - base36enc(full_backup->merge_dest_backup)); - } - - /* - * Previous merging was interrupted during deleting source backup. It is - * safe just to delete it again. - */ - if (full_backup->status == BACKUP_STATUS_MERGED) - goto merge_delete; - - /* Forward compatibility is not supported */ - for (i = parray_num(parent_chain) - 1; i >= 0; i--) - { - pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - - if (parse_program_version(backup->program_version) > - parse_program_version(PROGRAM_VERSION)) - { - elog(ERROR, "Backup %s has been produced by pg_probackup version %s, " - "but current program version is %s. Forward compatibility " - "is not supported.", - backup_id_of(backup), - backup->program_version, - PROGRAM_VERSION); - } - } - - /* If destination backup compression algorithm differs from - * full backup compression algorithm, then in-place merge is - * not possible. - */ - if (full_backup->compress_alg == dest_backup->compress_alg) - compression_match = true; - else - elog(WARNING, "In-place merge is disabled because of compression " - "algorithms mismatch"); - - /* - * If current program version differs from destination backup version, - * then in-place merge is not possible. - */ - program_version_match = is_forward_compatible(parent_chain); - - /* Forbid merge retry for failed merges between 2.4.0 and any - * older version. Several format changes makes it impossible - * to determine the exact format any speific file is got. - */ - if (is_retry && - parse_program_version(dest_backup->program_version) >= 20400 && - parse_program_version(full_backup->program_version) < 20400) - { - elog(ERROR, "Retry of failed merge for backups with different between minor " - "versions is forbidden to avoid data corruption because of storage format " - "changes introduced in 2.4.0 version, please take a new full backup"); - } - - /* - * Validate or revalidate all members of parent chain - * with sole exception of FULL backup. If it has MERGING status - * then it isn't valid backup until merging is finished. - */ - if (!no_validate) - { - elog(INFO, "Validate parent chain for backup %s", - backup_id_of(dest_backup)); - - for (i = parray_num(parent_chain) - 1; i >= 0; i--) - { - pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - - /* FULL backup is not to be validated if its status is MERGING */ - if (backup->backup_mode == BACKUP_MODE_FULL && - backup->status == BACKUP_STATUS_MERGING) - { - continue; - } - - pgBackupValidate(backup, NULL); - - if (backup->status != BACKUP_STATUS_OK) - elog(ERROR, "Backup %s has status %s, merge is aborted", - backup_id_of(backup), status2str(backup->status)); - } - } - - /* - * Get backup files. - */ - for (i = parray_num(parent_chain) - 1; i >= 0; i--) - { - pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - - backup->files = get_backup_filelist(backup, true); - parray_qsort(backup->files, pgFileCompareRelPathWithExternal); - - /* Set MERGING status for every member of the chain */ - if (backup->backup_mode == BACKUP_MODE_FULL) - { - /* In case of FULL backup also remember backup_id of - * of destination backup we are merging with, so - * we can safely allow rerun merge in case of failure. - */ - backup->merge_dest_backup = dest_backup->start_time; - backup->status = BACKUP_STATUS_MERGING; - write_backup(backup, true); - } - else - write_backup_status(backup, BACKUP_STATUS_MERGING, true); - } - - /* Construct path to database dir: /backup_dir/instance_name/FULL/database */ - join_path_components(full_database_dir, full_backup->root_dir, DATABASE_DIR); - /* Construct path to external dir: /backup_dir/instance_name/FULL/external */ - join_path_components(full_external_prefix, full_backup->root_dir, EXTERNAL_DIR); - - /* Create directories */ - create_data_directories(dest_backup->files, full_database_dir, - dest_backup->root_dir, false, false, FIO_BACKUP_HOST, NULL); - - /* External directories stuff */ - if (dest_backup->external_dir_str) - dest_externals = make_external_directory_list(dest_backup->external_dir_str, false); - if (full_backup->external_dir_str) - full_externals = make_external_directory_list(full_backup->external_dir_str, false); - /* - * Rename external directories in FULL backup (if exists) - * according to numeration of external dirs in destionation backup. - */ - if (full_externals && dest_externals) - reorder_external_dirs(full_backup, full_externals, dest_externals); - - /* bitmap optimization rely on n_blocks, which is generally available since 2.3.0 */ - if (parse_program_version(dest_backup->program_version) < 20300) - use_bitmap = false; - - /* Setup threads */ - for (i = 0; i < parray_num(dest_backup->files); i++) - { - pgFile *file = (pgFile *) parray_get(dest_backup->files, i); - - /* if the entry was an external directory, create it in the backup */ - if (file->external_dir_num && S_ISDIR(file->mode)) - { - char dirpath[MAXPGPATH]; - char new_container[MAXPGPATH]; - - makeExternalDirPathByNum(new_container, full_external_prefix, - file->external_dir_num); - join_path_components(dirpath, new_container, file->rel_path); - dir_create_dir(dirpath, DIR_PERMISSION, false); - } - - pg_atomic_init_flag(&file->lock); - } - - threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - threads_args = (merge_files_arg *) palloc(sizeof(merge_files_arg) * num_threads); - - thread_interrupted = false; - merge_time = time(NULL); - elog(INFO, "Start merging backup files"); - for (i = 0; i < num_threads; i++) - { - merge_files_arg *arg = &(threads_args[i]); - arg->merge_filelist = parray_new(); - arg->parent_chain = parent_chain; - arg->dest_backup = dest_backup; - arg->full_backup = full_backup; - arg->full_database_dir = full_database_dir; - arg->full_external_prefix = full_external_prefix; - - arg->compression_match = compression_match; - arg->program_version_match = program_version_match; - arg->use_bitmap = use_bitmap; - arg->is_retry = is_retry; - arg->no_sync = no_sync; - /* By default there are some error */ - arg->ret = 1; - - elog(VERBOSE, "Start thread: %d", i); - - pthread_create(&threads[i], NULL, merge_files, arg); - } - - /* Wait threads */ - result_filelist = parray_new(); - for (i = 0; i < num_threads; i++) - { - pthread_join(threads[i], NULL); - if (threads_args[i].ret == 1) - merge_isok = false; - - /* Compile final filelist */ - parray_concat(result_filelist, threads_args[i].merge_filelist); - - /* cleanup */ - parray_free(threads_args[i].merge_filelist); - //total_in_place_merge_bytes += threads_args[i].in_place_merge_bytes; - } - - time(&end_time); - pretty_time_interval(difftime(end_time, merge_time), - pretty_time, lengthof(pretty_time)); - - if (merge_isok) - elog(INFO, "Backup files are successfully merged, time elapsed: %s", - pretty_time); - else - elog(ERROR, "Backup files merging failed, time elapsed: %s", - pretty_time); - - /* If temp header map is open, then close it and make rename */ - if (full_backup->hdr_map.fp) - { - cleanup_header_map(&(full_backup->hdr_map)); - - /* sync new header map to disk */ - if (fio_sync(full_backup->hdr_map.path_tmp, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Cannot sync temp header map \"%s\": %s", - full_backup->hdr_map.path_tmp, strerror(errno)); - - /* Replace old header map with new one */ - if (rename(full_backup->hdr_map.path_tmp, full_backup->hdr_map.path)) - elog(ERROR, "Could not rename file \"%s\" to \"%s\": %s", - full_backup->hdr_map.path_tmp, full_backup->hdr_map.path, strerror(errno)); - } - - /* Close page header maps */ - for (i = parray_num(parent_chain) - 1; i >= 0; i--) - { - pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - cleanup_header_map(&(backup->hdr_map)); - } - - /* - * Update FULL backup metadata. - * We cannot set backup status to OK just yet, - * because it still has old start_time. - */ - strlcpy(full_backup->program_version, PROGRAM_VERSION, - sizeof(full_backup->program_version)); - full_backup->parent_backup = INVALID_BACKUP_ID; - full_backup->start_lsn = dest_backup->start_lsn; - full_backup->stop_lsn = dest_backup->stop_lsn; - full_backup->recovery_time = dest_backup->recovery_time; - full_backup->recovery_xid = dest_backup->recovery_xid; - full_backup->tli = dest_backup->tli; - full_backup->from_replica = dest_backup->from_replica; - - pfree(full_backup->external_dir_str); - full_backup->external_dir_str = pgut_strdup(dest_backup->external_dir_str); - pfree(full_backup->primary_conninfo); - full_backup->primary_conninfo = pgut_strdup(dest_backup->primary_conninfo); - - full_backup->merge_time = merge_time; - full_backup->end_time = time(NULL); - - full_backup->compress_alg = dest_backup->compress_alg; - full_backup->compress_level = dest_backup->compress_level; - - /* If incremental backup is pinned, - * then result FULL backup must also be pinned. - * And reverse, if FULL backup was pinned and dest was not, - * then pinning is no more. - */ - full_backup->expire_time = dest_backup->expire_time; - - pg_free(full_backup->note); - full_backup->note = NULL; - - if (dest_backup->note) - full_backup->note = pgut_strdup(dest_backup->note); - - /* FULL backup must inherit wal mode. */ - full_backup->stream = dest_backup->stream; - - /* ARCHIVE backup must inherit wal_bytes too. - * STREAM backup will have its wal_bytes calculated by - * write_backup_filelist(). - */ - if (!dest_backup->stream) - full_backup->wal_bytes = dest_backup->wal_bytes; - - parray_qsort(result_filelist, pgFileCompareRelPathWithExternal); - - write_backup_filelist(full_backup, result_filelist, full_database_dir, NULL, true); - write_backup(full_backup, true); - - /* Delete FULL backup files, that do not exists in destination backup - * Both arrays must be sorted in in reversed order to delete from leaf - */ - parray_qsort(dest_backup->files, pgFileCompareRelPathWithExternalDesc); - parray_qsort(full_backup->files, pgFileCompareRelPathWithExternalDesc); - for (i = 0; i < parray_num(full_backup->files); i++) - { - pgFile *full_file = (pgFile *) parray_get(full_backup->files, i); - - if (full_file->external_dir_num && full_externals) - { - char *dir_name = parray_get(full_externals, full_file->external_dir_num - 1); - if (backup_contains_external(dir_name, full_externals)) - /* Dir already removed*/ - continue; - } - - if (parray_bsearch(dest_backup->files, full_file, pgFileCompareRelPathWithExternalDesc) == NULL) - { - char full_file_path[MAXPGPATH]; - - /* We need full path, file object has relative path */ - join_path_components(full_file_path, full_database_dir, full_file->rel_path); - - pgFileDelete(full_file->mode, full_file_path); - elog(LOG, "Deleted \"%s\"", full_file_path); - } - } - - /* Critical section starts. - * Change status of FULL backup. - * Files are merged into FULL backup. It is time to remove incremental chain. - */ - full_backup->status = BACKUP_STATUS_MERGED; - write_backup(full_backup, true); - -merge_delete: - for (i = parray_num(parent_chain) - 2; i >= 0; i--) - { - pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - delete_backup_files(backup); - } - - /* - * PAGE2 DELETED - * PAGE1 DELETED - * FULL MERGED - * If we crash now, automatic rerun of failed merge is still possible: - * The user should start merge with full backup ID as an argument to option '-i'. - */ - -merge_rename: - /* - * Rename FULL backup directory to destination backup directory. - */ - if (dest_backup) - { - elog(LOG, "Rename %s to %s", full_backup->root_dir, dest_backup->root_dir); - if (rename(full_backup->root_dir, dest_backup->root_dir) == -1) - elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", - full_backup->root_dir, dest_backup->root_dir, strerror(errno)); - - /* update root_dir after rename */ - pg_free(full_backup->root_dir); - full_backup->root_dir = pgut_strdup(dest_backup->root_dir); - } - else - { - /* Ugly */ - char destination_path[MAXPGPATH]; - - join_path_components(destination_path, instanceState->instance_backup_subdir_path, - base36enc(full_backup->merge_dest_backup)); - - elog(LOG, "Rename %s to %s", full_backup->root_dir, destination_path); - if (rename(full_backup->root_dir, destination_path) == -1) - elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", - full_backup->root_dir, destination_path, strerror(errno)); - - /* update root_dir after rename */ - pg_free(full_backup->root_dir); - full_backup->root_dir = pgut_strdup(destination_path); - } - - /* Reinit path to database_dir */ - join_path_components(full_backup->database_dir, full_backup->root_dir, DATABASE_DIR); - - /* If we crash here, it will produce full backup in MERGED - * status, located in directory with wrong backup id. - * It should not be a problem. - */ - - /* - * Merging finished, now we can safely update ID of the FULL backup - */ - elog(INFO, "Rename merged full backup %s to %s", - backup_id_of(full_backup), - base36enc(full_backup->merge_dest_backup)); - - full_backup->status = BACKUP_STATUS_OK; - full_backup->start_time = full_backup->merge_dest_backup; - /* XXX BACKUP_ID change it when backup_id wouldn't match start_time */ - full_backup->backup_id = full_backup->start_time; - full_backup->merge_dest_backup = INVALID_BACKUP_ID; - write_backup(full_backup, true); - /* Critical section end */ - - /* Cleanup */ - if (threads) - { - pfree(threads_args); - pfree(threads); - } - - if (result_filelist) - { - parray_walk(result_filelist, pgFileFree); - parray_free(result_filelist); - } - - if (dest_externals != NULL) - free_dir_list(dest_externals); - - if (full_externals != NULL) - free_dir_list(full_externals); - - for (i = parray_num(parent_chain) - 1; i >= 0; i--) - { - pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - - if (backup->files) - { - parray_walk(backup->files, pgFileFree); - parray_free(backup->files); - } - } -} - -/* - * Thread worker of merge_chain(). - */ -static void * -merge_files(void *arg) -{ - int i; - merge_files_arg *arguments = (merge_files_arg *) arg; - size_t n_files = parray_num(arguments->dest_backup->files); - - for (i = 0; i < n_files; i++) - { - pgFile *dest_file = (pgFile *) parray_get(arguments->dest_backup->files, i); - pgFile *tmp_file; - bool in_place = false; /* keep file as it is */ - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during merge"); - - if (!pg_atomic_test_set_flag(&dest_file->lock)) - continue; - - tmp_file = pgFileInit(dest_file->rel_path); - tmp_file->mode = dest_file->mode; - tmp_file->is_datafile = dest_file->is_datafile; - tmp_file->is_cfs = dest_file->is_cfs; - tmp_file->external_dir_num = dest_file->external_dir_num; - tmp_file->dbOid = dest_file->dbOid; - - /* Directories were created before */ - if (S_ISDIR(dest_file->mode)) - goto done; - - elog(progress ? INFO : LOG, "Progress: (%d/%lu). Merging file \"%s\"", - i + 1, n_files, dest_file->rel_path); - - if (dest_file->is_datafile && !dest_file->is_cfs) - tmp_file->segno = dest_file->segno; - - // If destination file is 0 sized, then go for the next - if (dest_file->write_size == 0) - { - if (!dest_file->is_datafile || dest_file->is_cfs) - tmp_file->crc = dest_file->crc; - - tmp_file->write_size = 0; - goto done; - } - - /* - * If file didn`t changed over the course of all incremental chain, - * then do in-place merge, unless destination backup has - * different compression algorithm. - * In-place merge is also impossible, if program version of destination - * backup differs from PROGRAM_VERSION - */ - if (arguments->program_version_match && arguments->compression_match && - !arguments->is_retry) - { - /* - * Case 1: - * in this case in place merge is possible: - * 0 PAGE; file, size BYTES_INVALID - * 1 PAGE; file, size BYTES_INVALID - * 2 FULL; file, size 100500 - * - * Case 2: - * in this case in place merge is possible: - * 0 PAGE; file, size 0 - * 1 PAGE; file, size 0 - * 2 FULL; file, size 100500 - * - * Case 3: - * in this case in place merge is impossible: - * 0 PAGE; file, size BYTES_INVALID - * 1 PAGE; file, size 100501 - * 2 FULL; file, size 100500 - * - * Case 4 (good candidate for future optimization): - * in this case in place merge is impossible: - * 0 PAGE; file, size BYTES_INVALID - * 1 PAGE; file, size 100501 - * 2 FULL; file, not exists yet - */ - - in_place = true; - - for (i = parray_num(arguments->parent_chain) - 1; i >= 0; i--) - { - pgFile **res_file = NULL; - pgFile *file = NULL; - - pgBackup *backup = (pgBackup *) parray_get(arguments->parent_chain, i); - - /* lookup file in intermediate backup */ - res_file = parray_bsearch(backup->files, dest_file, pgFileCompareRelPathWithExternal); - file = (res_file) ? *res_file : NULL; - - /* Destination file is not exists yet, - * in-place merge is impossible - */ - if (file == NULL) - { - in_place = false; - break; - } - - /* Skip file from FULL backup */ - if (backup->backup_mode == BACKUP_MODE_FULL) - continue; - - if (file->write_size != BYTES_INVALID) - { - in_place = false; - break; - } - } - } - - /* - * In-place merge means that file in FULL backup stays as it is, - * no additional actions are required. - * page header map cannot be trusted when retrying, so no - * in place merge for retry. - */ - if (in_place) - { - pgFile **res_file = NULL; - pgFile *file = NULL; - res_file = parray_bsearch(arguments->full_backup->files, dest_file, - pgFileCompareRelPathWithExternal); - file = (res_file) ? *res_file : NULL; - - /* If file didn`t changed in any way, then in-place merge is possible */ - if (file && - file->n_blocks == dest_file->n_blocks) - { - BackupPageHeader2 *headers = NULL; - - elog(LOG, "The file didn`t changed since FULL backup, skip merge: \"%s\"", - file->rel_path); - - tmp_file->crc = file->crc; - tmp_file->write_size = file->write_size; - - if (dest_file->is_datafile && !dest_file->is_cfs) - { - tmp_file->n_blocks = file->n_blocks; - tmp_file->compress_alg = file->compress_alg; - tmp_file->uncompressed_size = file->n_blocks * BLCKSZ; - - tmp_file->n_headers = file->n_headers; - tmp_file->hdr_crc = file->hdr_crc; - } - else - tmp_file->uncompressed_size = file->uncompressed_size; - - /* Copy header metadata from old map into a new one */ - tmp_file->n_headers = file->n_headers; - headers = get_data_file_headers(&(arguments->full_backup->hdr_map), file, - parse_program_version(arguments->full_backup->program_version), - true); - - /* sanity */ - if (!headers && file->n_headers > 0) - elog(ERROR, "Failed to get headers for file \"%s\"", file->rel_path); - - write_page_headers(headers, tmp_file, &(arguments->full_backup->hdr_map), true); - pg_free(headers); - - //TODO: report in_place merge bytes. - goto done; - } - } - - if (dest_file->is_datafile && !dest_file->is_cfs) - merge_data_file(arguments->parent_chain, - arguments->full_backup, - arguments->dest_backup, - dest_file, tmp_file, - arguments->full_database_dir, - arguments->use_bitmap, - arguments->is_retry, - arguments->no_sync); - else - merge_non_data_file(arguments->parent_chain, - arguments->full_backup, - arguments->dest_backup, - dest_file, tmp_file, - arguments->full_database_dir, - arguments->full_external_prefix, - arguments->no_sync); - -done: - parray_append(arguments->merge_filelist, tmp_file); - } - - /* Data files merging is successful */ - arguments->ret = 0; - - return NULL; -} - -/* Recursively delete a directory and its contents */ -static void -remove_dir_with_files(const char *path) -{ - parray *files = parray_new(); - int i; - char full_path[MAXPGPATH]; - - dir_list_file(files, path, false, false, true, false, false, 0, FIO_LOCAL_HOST); - parray_qsort(files, pgFileCompareRelPathWithExternalDesc); - for (i = 0; i < parray_num(files); i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - - join_path_components(full_path, path, file->rel_path); - - pgFileDelete(file->mode, full_path); - elog(LOG, "Deleted \"%s\"", full_path); - } - - /* cleanup */ - parray_walk(files, pgFileFree); - parray_free(files); -} - -/* Get index of external directory */ -static int -get_external_index(const char *key, const parray *list) -{ - int i; - - if (!list) /* Nowhere to search */ - return -1; - for (i = 0; i < parray_num(list); i++) - { - if (strcmp(key, parray_get(list, i)) == 0) - return i + 1; - } - return -1; -} - -/* Rename directories in to_backup according to order in from_external */ -static void -reorder_external_dirs(pgBackup *to_backup, parray *to_external, - parray *from_external) -{ - char externaldir_template[MAXPGPATH]; - int i; - - join_path_components(externaldir_template, to_backup->root_dir, EXTERNAL_DIR); - for (i = 0; i < parray_num(to_external); i++) - { - int from_num = get_external_index(parray_get(to_external, i), - from_external); - if (from_num == -1) - { - char old_path[MAXPGPATH]; - makeExternalDirPathByNum(old_path, externaldir_template, i + 1); - remove_dir_with_files(old_path); - } - else if (from_num != i + 1) - { - char old_path[MAXPGPATH]; - char new_path[MAXPGPATH]; - makeExternalDirPathByNum(old_path, externaldir_template, i + 1); - makeExternalDirPathByNum(new_path, externaldir_template, from_num); - elog(LOG, "Rename %s to %s", old_path, new_path); - if (rename (old_path, new_path) == -1) - elog(ERROR, "Could not rename directory \"%s\" to \"%s\": %s", - old_path, new_path, strerror(errno)); - } - } -} - -/* Merge is usually happens as usual backup/restore via temp files, unless - * file didn`t changed since FULL backup AND full a dest backup have the - * same compression algorithm. In this case file can be left as it is. - */ -void -merge_data_file(parray *parent_chain, pgBackup *full_backup, - pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file, - const char *full_database_dir, bool use_bitmap, bool is_retry, - bool no_sync) -{ - FILE *out = NULL; - char *buffer = pgut_malloc(STDIO_BUFSIZE); - char to_fullpath[MAXPGPATH]; - char to_fullpath_tmp1[MAXPGPATH]; /* used for restore */ - char to_fullpath_tmp2[MAXPGPATH]; /* used for backup */ - - /* The next possible optimization is copying "as is" the file - * from intermediate incremental backup, that didn`t changed in - * subsequent incremental backups. TODO. - */ - - /* set fullpath of destination file and temp files */ - join_path_components(to_fullpath, full_database_dir, tmp_file->rel_path); - snprintf(to_fullpath_tmp1, MAXPGPATH, "%s_tmp1", to_fullpath); - snprintf(to_fullpath_tmp2, MAXPGPATH, "%s_tmp2", to_fullpath); - - /* open temp file */ - out = fopen(to_fullpath_tmp1, PG_BINARY_W); - if (out == NULL) - elog(ERROR, "Cannot open merge target file \"%s\": %s", - to_fullpath_tmp1, strerror(errno)); - setvbuf(out, buffer, _IOFBF, STDIO_BUFSIZE); - - /* restore file into temp file */ - tmp_file->size = restore_data_file(parent_chain, dest_file, out, to_fullpath_tmp1, - use_bitmap, NULL, InvalidXLogRecPtr, NULL, - /* when retrying merge header map cannot be trusted */ - is_retry ? false : true); - if (fclose(out) != 0) - elog(ERROR, "Cannot close file \"%s\": %s", - to_fullpath_tmp1, strerror(errno)); - - pg_free(buffer); - - /* tmp_file->size is greedy, even if there is single 8KB block in file, - * that was overwritten twice during restore_data_file, we would assume that its size is - * 16KB. - * TODO: maybe we should just trust dest_file->n_blocks? - * No, we can`t, because current binary can be used to merge - * 2 backups of old versions, where n_blocks is missing. - */ - - backup_data_file(tmp_file, to_fullpath_tmp1, to_fullpath_tmp2, - InvalidXLogRecPtr, BACKUP_MODE_FULL, - dest_backup->compress_alg, dest_backup->compress_level, - dest_backup->checksum_version, - &(full_backup->hdr_map), true); - - /* drop restored temp file */ - if (unlink(to_fullpath_tmp1) == -1) - elog(ERROR, "Cannot remove file \"%s\": %s", to_fullpath_tmp1, - strerror(errno)); - - /* - * In old (=<2.2.7) versions of pg_probackup n_blocks attribute of files - * in PAGE and PTRACK wasn`t filled. - */ - //Assert(tmp_file->n_blocks == dest_file->n_blocks); - - /* Backward compatibility kludge: - * When merging old backups, it is possible that - * to_fullpath_tmp2 size will be 0, and so it will be - * truncated in backup_data_file(). - * TODO: remove in 3.0.0 - */ - if (tmp_file->write_size == 0) - return; - - /* sync second temp file to disk */ - if (!no_sync && fio_sync(to_fullpath_tmp2, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Cannot sync merge temp file \"%s\": %s", - to_fullpath_tmp2, strerror(errno)); - - /* Do atomic rename from second temp file to destination file */ - if (rename(to_fullpath_tmp2, to_fullpath) == -1) - elog(ERROR, "Could not rename file \"%s\" to \"%s\": %s", - to_fullpath_tmp2, to_fullpath, strerror(errno)); - - /* drop temp file */ - unlink(to_fullpath_tmp1); -} - -/* - * For every destionation file lookup the newest file in chain and - * copy it. - * Additional pain is external directories. - */ -void -merge_non_data_file(parray *parent_chain, pgBackup *full_backup, - pgBackup *dest_backup, pgFile *dest_file, pgFile *tmp_file, - const char *full_database_dir, const char *to_external_prefix, - bool no_sync) -{ - int i; - char to_fullpath[MAXPGPATH]; - char to_fullpath_tmp[MAXPGPATH]; /* used for backup */ - char from_fullpath[MAXPGPATH]; - pgBackup *from_backup = NULL; - pgFile *from_file = NULL; - - /* We need to make full path to destination file */ - if (dest_file->external_dir_num) - { - char temp[MAXPGPATH]; - makeExternalDirPathByNum(temp, to_external_prefix, - dest_file->external_dir_num); - join_path_components(to_fullpath, temp, dest_file->rel_path); - } - else - join_path_components(to_fullpath, full_database_dir, dest_file->rel_path); - - snprintf(to_fullpath_tmp, MAXPGPATH, "%s_tmp", to_fullpath); - - /* - * Iterate over parent chain starting from direct parent of destination - * backup to oldest backup in chain, and look for the first - * full copy of destination file. - * Full copy is latest possible destination file with size equal(!) - * or greater than zero. - */ - for (i = 0; i < parray_num(parent_chain); i++) - { - pgFile **res_file = NULL; - from_backup = (pgBackup *) parray_get(parent_chain, i); - - /* lookup file in intermediate backup */ - res_file = parray_bsearch(from_backup->files, dest_file, pgFileCompareRelPathWithExternal); - from_file = (res_file) ? *res_file : NULL; - - /* - * It should not be possible not to find source file in intermediate - * backup, without encountering full copy first. - */ - if (!from_file) - { - elog(ERROR, "Failed to locate non-data file \"%s\" in backup %s", - dest_file->rel_path, backup_id_of(from_backup)); - continue; - } - - if (from_file->write_size > 0) - break; - } - - /* sanity */ - if (!from_backup) - elog(ERROR, "Failed to found a backup containing full copy of non-data file \"%s\"", - dest_file->rel_path); - - if (!from_file) - elog(ERROR, "Failed to locate a full copy of non-data file \"%s\"", dest_file->rel_path); - - /* set path to source file */ - if (from_file->external_dir_num) - { - char temp[MAXPGPATH]; - char external_prefix[MAXPGPATH]; - - join_path_components(external_prefix, from_backup->root_dir, EXTERNAL_DIR); - makeExternalDirPathByNum(temp, external_prefix, dest_file->external_dir_num); - - join_path_components(from_fullpath, temp, from_file->rel_path); - } - else - { - char backup_database_dir[MAXPGPATH]; - join_path_components(backup_database_dir, from_backup->root_dir, DATABASE_DIR); - join_path_components(from_fullpath, backup_database_dir, from_file->rel_path); - } - - /* Copy file to FULL backup directory into temp file */ - backup_non_data_file(tmp_file, NULL, from_fullpath, - to_fullpath_tmp, BACKUP_MODE_FULL, 0, false); - - /* sync temp file to disk */ - if (!no_sync && fio_sync(to_fullpath_tmp, FIO_BACKUP_HOST) != 0) - elog(ERROR, "Cannot sync merge temp file \"%s\": %s", - to_fullpath_tmp, strerror(errno)); - - /* Do atomic rename from second temp file to destination file */ - if (rename(to_fullpath_tmp, to_fullpath) == -1) - elog(ERROR, "Could not rename file \"%s\" to \"%s\": %s", - to_fullpath_tmp, to_fullpath, strerror(errno)); - -} - -/* - * If file format in incremental chain is compatible - * with current storage format. - * If not, then in-place merge is not possible. - * - * Consider the following examples: - * STORAGE_FORMAT_VERSION = 2.4.4 - * 2.3.3 \ - * 2.3.4 \ disable in-place merge, because - * 2.4.1 / current STORAGE_FORMAT_VERSION > 2.3.3 - * 2.4.3 / - * - * 2.4.4 \ enable in_place merge, because - * 2.4.5 / current STORAGE_FORMAT_VERSION == 2.4.4 - * - * 2.4.5 \ enable in_place merge, because - * 2.4.6 / current STORAGE_FORMAT_VERSION < 2.4.5 - * - */ -bool -is_forward_compatible(parray *parent_chain) -{ - int i; - pgBackup *oldest_ver_backup = NULL; - uint32 oldest_ver_in_chain = parse_program_version(PROGRAM_VERSION); - - for (i = 0; i < parray_num(parent_chain); i++) - { - pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - uint32 current_version = parse_program_version(backup->program_version); - - if (!oldest_ver_backup) - oldest_ver_backup = backup; - - if (current_version < oldest_ver_in_chain) - { - oldest_ver_in_chain = current_version; - oldest_ver_backup = backup; - } - } - - if (oldest_ver_in_chain < parse_program_version(STORAGE_FORMAT_VERSION)) - { - elog(WARNING, "In-place merge is disabled because of storage format incompatibility. " - "Backup %s storage format version: %s, " - "current storage format version: %s", - backup_id_of(oldest_ver_backup), - oldest_ver_backup->program_version, - STORAGE_FORMAT_VERSION); - return false; - } - - return true; -} diff --git a/src/parsexlog.c b/src/parsexlog.c deleted file mode 100644 index 7c4b5b349..000000000 --- a/src/parsexlog.c +++ /dev/null @@ -1,1972 +0,0 @@ -/*------------------------------------------------------------------------- - * - * parsexlog.c - * Functions for reading Write-Ahead-Log - * - * Portions Copyright (c) 1996-2016, PostgreSQL Global Development Group - * Portions Copyright (c) 1994, Regents of the University of California - * Portions Copyright (c) 2015-2019, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -#include "access/transam.h" -#include "catalog/pg_control.h" -#include "commands/dbcommands_xlog.h" -#include "catalog/storage_xlog.h" - -#ifdef HAVE_LIBZ -#include -#endif - -#include "utils/thread.h" -#include -#include - -/* - * RmgrNames is an array of resource manager names, to make error messages - * a bit nicer. - */ -#if PG_VERSION_NUM >= 150000 -#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask,decode) \ - name, -#elif PG_VERSION_NUM >= 100000 -#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup,mask) \ - name, -#else -#define PG_RMGR(symname,name,redo,desc,identify,startup,cleanup) \ - name, -#endif - -static const char *RmgrNames[RM_MAX_ID + 1] = { -#include "access/rmgrlist.h" -}; - -/* some from access/xact.h */ -/* - * XLOG allows to store some information in high 4 bits of log record xl_info - * field. We use 3 for the opcode, and one about an optional flag variable. - */ -#define XLOG_XACT_COMMIT 0x00 -#define XLOG_XACT_PREPARE 0x10 -#define XLOG_XACT_ABORT 0x20 -#define XLOG_XACT_COMMIT_PREPARED 0x30 -#define XLOG_XACT_ABORT_PREPARED 0x40 -#define XLOG_XACT_ASSIGNMENT 0x50 -/* free opcode 0x60 */ -/* free opcode 0x70 */ - -/* mask for filtering opcodes out of xl_info */ -#define XLOG_XACT_OPMASK 0x70 - -typedef struct xl_xact_commit -{ - TimestampTz xact_time; /* time of commit */ - - /* xl_xact_xinfo follows if XLOG_XACT_HAS_INFO */ - /* xl_xact_dbinfo follows if XINFO_HAS_DBINFO */ - /* xl_xact_subxacts follows if XINFO_HAS_SUBXACT */ - /* xl_xact_relfilenodes follows if XINFO_HAS_RELFILENODES */ - /* xl_xact_invals follows if XINFO_HAS_INVALS */ - /* xl_xact_twophase follows if XINFO_HAS_TWOPHASE */ - /* xl_xact_origin follows if XINFO_HAS_ORIGIN, stored unaligned! */ -} xl_xact_commit; - -typedef struct xl_xact_abort -{ - TimestampTz xact_time; /* time of abort */ - - /* xl_xact_xinfo follows if XLOG_XACT_HAS_INFO */ - /* No db_info required */ - /* xl_xact_subxacts follows if HAS_SUBXACT */ - /* xl_xact_relfilenodes follows if HAS_RELFILENODES */ - /* No invalidation messages needed. */ - /* xl_xact_twophase follows if XINFO_HAS_TWOPHASE */ -} xl_xact_abort; - -/* - * XLogRecTarget allows to track the last recovery targets. Currently used only - * within validate_wal(). - */ -typedef struct XLogRecTarget -{ - TimestampTz rec_time; - TransactionId rec_xid; - XLogRecPtr rec_lsn; -} XLogRecTarget; - -typedef struct XLogReaderData -{ - int thread_num; - TimeLineID tli; - - XLogRecTarget cur_rec; - XLogSegNo xlogsegno; - bool xlogexists; - - char page_buf[XLOG_BLCKSZ]; - uint32 prev_page_off; - - bool need_switch; - - int xlogfile; - char xlogpath[MAXPGPATH]; - -#ifdef HAVE_LIBZ - gzFile gz_xlogfile; - char gz_xlogpath[MAXPGPATH]; -#endif -} XLogReaderData; - -/* Function to process a WAL record */ -typedef void (*xlog_record_function) (XLogReaderState *record, - XLogReaderData *reader_data, - bool *stop_reading); - -/* An argument for a thread function */ -typedef struct -{ - XLogReaderData reader_data; - - xlog_record_function process_record; - - XLogRecPtr startpoint; - XLogRecPtr endpoint; - XLogSegNo endSegNo; - - /* - * The thread got the recovery target. - */ - bool got_target; - - /* Should we read record, located at endpoint position */ - bool inclusive_endpoint; - - /* - * Return value from the thread. - * 0 means there is no error, 1 - there is an error. - */ - int ret; -} xlog_thread_arg; - -static XLogRecord* WalReadRecord(XLogReaderState *xlogreader, XLogRecPtr startpoint, char **errormsg); -static XLogReaderState* WalReaderAllocate(uint32 wal_seg_size, XLogReaderData *reader_data); - -static int SimpleXLogPageRead(XLogReaderState *xlogreader, - XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf -#if PG_VERSION_NUM < 130000 - ,TimeLineID *pageTLI -#endif - ); -static XLogReaderState *InitXLogPageRead(XLogReaderData *reader_data, - const char *archivedir, - TimeLineID tli, uint32 segment_size, - bool manual_switch, - bool consistent_read, - bool allocate_reader); -static bool RunXLogThreads(const char *archivedir, - time_t target_time, TransactionId target_xid, - XLogRecPtr target_lsn, - TimeLineID tli, uint32 segment_size, - XLogRecPtr startpoint, XLogRecPtr endpoint, - bool consistent_read, - xlog_record_function process_record, - XLogRecTarget *last_rec, - bool inclusive_endpoint); -//static XLogReaderState *InitXLogThreadRead(xlog_thread_arg *arg); -static bool SwitchThreadToNextWal(XLogReaderState *xlogreader, - xlog_thread_arg *arg); -static bool XLogWaitForConsistency(XLogReaderState *xlogreader); -static void *XLogThreadWorker(void *arg); -static void CleanupXLogPageRead(XLogReaderState *xlogreader); -static void PrintXLogCorruptionMsg(XLogReaderData *reader_data, int elevel); - -static void extractPageInfo(XLogReaderState *record, - XLogReaderData *reader_data, bool *stop_reading); -static void validateXLogRecord(XLogReaderState *record, - XLogReaderData *reader_data, bool *stop_reading); -static bool getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime); - -static XLogSegNo segno_start = 0; -/* Segment number where target record is located */ -static XLogSegNo segno_target = 0; -/* Next segment number to read by a thread */ -static XLogSegNo segno_next = 0; -/* Number of segments already read by threads */ -static uint32 segnum_read = 0; -/* Number of detected corrupted or absent segments */ -static uint32 segnum_corrupted = 0; -static pthread_mutex_t wal_segment_mutex = PTHREAD_MUTEX_INITIALIZER; - -/* copied from timestamp.c */ -static pg_time_t -timestamptz_to_time_t(TimestampTz t) -{ - pg_time_t result; - -#ifdef HAVE_INT64_TIMESTAMP - result = (pg_time_t) (t / USECS_PER_SEC + - ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY)); -#else - result = (pg_time_t) (t + - ((POSTGRES_EPOCH_JDATE - UNIX_EPOCH_JDATE) * SECS_PER_DAY)); -#endif - return result; -} - -static const char *wal_archivedir = NULL; -static uint32 wal_seg_size = 0; -/* - * If true a wal reader thread switches to the next segment using - * segno_next. - */ -static bool wal_manual_switch = false; -/* - * If true a wal reader thread waits for other threads if the thread met absent - * wal segment. - */ -static bool wal_consistent_read = false; - -/* - * Variables used within validate_wal() and validateXLogRecord() to stop workers - */ -static time_t wal_target_time = 0; -static TransactionId wal_target_xid = InvalidTransactionId; -static XLogRecPtr wal_target_lsn = InvalidXLogRecPtr; - -/* - * Read WAL from the archive directory, from 'startpoint' to 'endpoint' on the - * given timeline. Collect data blocks touched by the WAL records into a page map. - * - * Pagemap extracting is processed using threads. Each thread reads single WAL - * file. - */ -bool -extractPageMap(const char *archivedir, uint32 wal_seg_size, - XLogRecPtr startpoint, TimeLineID start_tli, - XLogRecPtr endpoint, TimeLineID end_tli, - parray *tli_list) -{ - bool extract_isok = false; - - if (start_tli == end_tli) - /* easy case */ - extract_isok = RunXLogThreads(archivedir, 0, InvalidTransactionId, - InvalidXLogRecPtr, end_tli, wal_seg_size, - startpoint, endpoint, false, extractPageInfo, - NULL, true); - else - { - /* We have to process WAL located on several different xlog intervals, - * located on different timelines. - * - * Consider this example: - * t3 C-----X - * / - * t1 -A----*-------> - * - * A - prev backup START_LSN - * B - switchpoint for t2, available as t2->switchpoint - * C - switch for t3, available as t3->switchpoint - * X - current backup START_LSN - * - * Intervals to be parsed: - * - [A,B) on t1 - * - [B,C) on t2 - * - [C,X] on t3 - */ - int i; - parray *interval_list = parray_new(); - timelineInfo *end_tlinfo = NULL; - timelineInfo *tmp_tlinfo = NULL; - XLogRecPtr prev_switchpoint = InvalidXLogRecPtr; - - /* We must find TLI information about final timeline (t3 in example) */ - for (i = 0; i < parray_num(tli_list); i++) - { - tmp_tlinfo = parray_get(tli_list, i); - - if (tmp_tlinfo->tli == end_tli) - { - end_tlinfo = tmp_tlinfo; - break; - } - } - - /* Iterate over timelines backward, - * starting with end_tli and ending with start_tli. - * For every timeline calculate LSN-interval that must be parsed. - */ - - tmp_tlinfo = end_tlinfo; - while (tmp_tlinfo) - { - lsnInterval *wal_interval = pgut_malloc(sizeof(lsnInterval)); - wal_interval->tli = tmp_tlinfo->tli; - - if (tmp_tlinfo->tli == end_tli) - { - wal_interval->begin_lsn = tmp_tlinfo->switchpoint; - wal_interval->end_lsn = endpoint; - } - else if (tmp_tlinfo->tli == start_tli) - { - wal_interval->begin_lsn = startpoint; - wal_interval->end_lsn = prev_switchpoint; - } - else - { - wal_interval->begin_lsn = tmp_tlinfo->switchpoint; - wal_interval->end_lsn = prev_switchpoint; - } - - parray_append(interval_list, wal_interval); - - if (tmp_tlinfo->tli == start_tli) - break; - - prev_switchpoint = tmp_tlinfo->switchpoint; - tmp_tlinfo = tmp_tlinfo->parent_link; - } - - for (i = parray_num(interval_list) - 1; i >= 0; i--) - { - bool inclusive_endpoint; - lsnInterval *tmp_interval = (lsnInterval *) parray_get(interval_list, i); - - /* In case of replica promotion, endpoints of intermediate - * timelines can be unreachable. - */ - inclusive_endpoint = false; - - /* ... but not the end timeline */ - if (tmp_interval->tli == end_tli) - inclusive_endpoint = true; - - extract_isok = RunXLogThreads(archivedir, 0, InvalidTransactionId, - InvalidXLogRecPtr, tmp_interval->tli, wal_seg_size, - tmp_interval->begin_lsn, tmp_interval->end_lsn, - false, extractPageInfo, NULL, inclusive_endpoint); - if (!extract_isok) - break; - - pg_free(tmp_interval); - } - pg_free(interval_list); - } - - return extract_isok; -} - -/* - * Ensure that the backup has all wal files needed for recovery to consistent - * state. - * - * WAL records reading is processed using threads. Each thread reads single WAL - * file. - */ -static void -validate_backup_wal_from_start_to_stop(pgBackup *backup, - const char *archivedir, TimeLineID tli, - uint32 xlog_seg_size) -{ - bool got_endpoint; - - got_endpoint = RunXLogThreads(archivedir, 0, InvalidTransactionId, - InvalidXLogRecPtr, tli, xlog_seg_size, - backup->start_lsn, backup->stop_lsn, - false, NULL, NULL, true); - - if (!got_endpoint) - { - /* - * If we don't have WAL between start_lsn and stop_lsn, - * the backup is definitely corrupted. Update its status. - */ - write_backup_status(backup, BACKUP_STATUS_CORRUPT, true); - - elog(WARNING, "There are not enough WAL records to consistenly restore " - "backup %s from START LSN: %X/%X to STOP LSN: %X/%X", - backup_id_of(backup), - (uint32) (backup->start_lsn >> 32), - (uint32) (backup->start_lsn), - (uint32) (backup->stop_lsn >> 32), - (uint32) (backup->stop_lsn)); - } -} - -/* - * Ensure that the backup has all wal files needed for recovery to consistent - * state. And check if we have in archive all files needed to restore the backup - * up to the given recovery target. - */ -void -validate_wal(pgBackup *backup, const char *archivedir, - time_t target_time, TransactionId target_xid, - XLogRecPtr target_lsn, TimeLineID tli, uint32 wal_seg_size) -{ - XLogRecTarget last_rec; - char last_timestamp[100], - target_timestamp[100]; - bool all_wal = false; - - if (!XRecOffIsValid(backup->start_lsn)) - elog(ERROR, "Invalid start_lsn value %X/%X of backup %s", - (uint32) (backup->start_lsn >> 32), (uint32) (backup->start_lsn), - backup_id_of(backup)); - - if (!XRecOffIsValid(backup->stop_lsn)) - elog(ERROR, "Invalid stop_lsn value %X/%X of backup %s", - (uint32) (backup->stop_lsn >> 32), (uint32) (backup->stop_lsn), - backup_id_of(backup)); - - /* - * Check that the backup has all wal files needed - * for recovery to consistent state. - */ - if (backup->stream) - { - char backup_database_dir[MAXPGPATH]; - char backup_xlog_path[MAXPGPATH]; - - join_path_components(backup_database_dir, backup->root_dir, DATABASE_DIR); - join_path_components(backup_xlog_path, backup_database_dir, PG_XLOG_DIR); - - validate_backup_wal_from_start_to_stop(backup, backup_xlog_path, tli, - wal_seg_size); - } - else - validate_backup_wal_from_start_to_stop(backup, (char *) archivedir, tli, - wal_seg_size); - - if (backup->status == BACKUP_STATUS_CORRUPT) - { - elog(WARNING, "Backup %s WAL segments are corrupted", backup_id_of(backup)); - return; - } - /* - * If recovery target is provided check that we can restore backup to a - * recovery target time or xid. - */ - if (!TransactionIdIsValid(target_xid) && target_time == 0 && - !XRecOffIsValid(target_lsn)) - { - /* Recovery target is not given so exit */ - elog(INFO, "Backup %s WAL segments are valid", backup_id_of(backup)); - return; - } - - /* - * If recovery target is provided, ensure that archive files exist in - * archive directory. - */ - if (dir_is_empty(archivedir, FIO_LOCAL_HOST)) - elog(ERROR, "WAL archive is empty. You cannot restore backup to a recovery target without WAL archive."); - - /* - * Check if we have in archive all files needed to restore backup - * up to the given recovery target. - * In any case we cannot restore to the point before stop_lsn. - */ - - /* We can restore at least up to the backup end */ - last_rec.rec_time = 0; - last_rec.rec_xid = backup->recovery_xid; - last_rec.rec_lsn = backup->stop_lsn; - - time2iso(last_timestamp, lengthof(last_timestamp), backup->recovery_time, false); - - if ((TransactionIdIsValid(target_xid) && target_xid == last_rec.rec_xid) - || (target_time != 0 && backup->recovery_time >= target_time) - || (XRecOffIsValid(target_lsn) && last_rec.rec_lsn >= target_lsn)) - all_wal = true; - - all_wal = all_wal || - RunXLogThreads(archivedir, target_time, target_xid, target_lsn, - tli, wal_seg_size, backup->stop_lsn, - InvalidXLogRecPtr, true, validateXLogRecord, &last_rec, true); - if (last_rec.rec_time > 0) - time2iso(last_timestamp, lengthof(last_timestamp), - timestamptz_to_time_t(last_rec.rec_time), false); - - /* There are all needed WAL records */ - if (all_wal) - elog(INFO, "Backup validation completed successfully on time %s, xid " XID_FMT " and LSN %X/%X", - last_timestamp, last_rec.rec_xid, - (uint32) (last_rec.rec_lsn >> 32), (uint32) last_rec.rec_lsn); - /* Some needed WAL records are absent */ - else - { - elog(WARNING, "Recovery can be done up to time %s, xid " XID_FMT " and LSN %X/%X", - last_timestamp, last_rec.rec_xid, - (uint32) (last_rec.rec_lsn >> 32), (uint32) last_rec.rec_lsn); - - if (target_time > 0) - time2iso(target_timestamp, lengthof(target_timestamp), target_time, false); - if (TransactionIdIsValid(target_xid) && target_time != 0) - elog(ERROR, "Not enough WAL records to time %s and xid " XID_FMT, - target_timestamp, target_xid); - else if (TransactionIdIsValid(target_xid)) - elog(ERROR, "Not enough WAL records to xid " XID_FMT, - target_xid); - else if (target_time != 0) - elog(ERROR, "Not enough WAL records to time %s", - target_timestamp); - else if (XRecOffIsValid(target_lsn)) - elog(ERROR, "Not enough WAL records to lsn %X/%X", - (uint32) (target_lsn >> 32), (uint32) (target_lsn)); - } -} - -/* - * Read from archived WAL segments latest recovery time and xid. All necessary - * segments present at archive folder. We waited **stop_lsn** in - * pg_stop_backup(). - */ -bool -read_recovery_info(const char *archivedir, TimeLineID tli, uint32 wal_seg_size, - XLogRecPtr start_lsn, XLogRecPtr stop_lsn, - time_t *recovery_time) -{ - XLogRecPtr startpoint = stop_lsn; - XLogReaderState *xlogreader; - XLogReaderData reader_data; - bool res; - - if (!XRecOffIsValid(start_lsn)) - elog(ERROR, "Invalid start_lsn value %X/%X", - (uint32) (start_lsn >> 32), (uint32) (start_lsn)); - - if (!XRecOffIsValid(stop_lsn)) - elog(ERROR, "Invalid stop_lsn value %X/%X", - (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); - - xlogreader = InitXLogPageRead(&reader_data, archivedir, tli, wal_seg_size, - false, true, true); - - /* Read records from stop_lsn down to start_lsn */ - do - { - XLogRecord *record; - TimestampTz last_time = 0; - char *errormsg; - -#if PG_VERSION_NUM >= 130000 - if (XLogRecPtrIsInvalid(startpoint)) - startpoint = SizeOfXLogShortPHD; - XLogBeginRead(xlogreader, startpoint); -#endif - - record = WalReadRecord(xlogreader, startpoint, &errormsg); - if (record == NULL) - { - XLogRecPtr errptr; - - errptr = startpoint ? startpoint : xlogreader->EndRecPtr; - - if (errormsg) - elog(ERROR, "Could not read WAL record at %X/%X: %s", - (uint32) (errptr >> 32), (uint32) (errptr), - errormsg); - else - elog(ERROR, "Could not read WAL record at %X/%X", - (uint32) (errptr >> 32), (uint32) (errptr)); - } - - /* Read previous record */ - startpoint = record->xl_prev; - - if (getRecordTimestamp(xlogreader, &last_time)) - { - *recovery_time = timestamptz_to_time_t(last_time); - - /* Found timestamp in WAL record 'record' */ - res = true; - goto cleanup; - } - } while (startpoint >= start_lsn); - - /* Didn't find timestamp from WAL records between start_lsn and stop_lsn */ - res = false; - -cleanup: - CleanupXLogPageRead(xlogreader); - XLogReaderFree(xlogreader); - - return res; -} - -/* - * Check if there is a WAL segment file in 'archivedir' which contains - * 'target_lsn'. - */ -bool -wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, - TimeLineID target_tli, uint32 wal_seg_size) -{ - XLogReaderState *xlogreader; - XLogReaderData reader_data; - char *errormsg; - bool res; - - if (!XRecOffIsValid(target_lsn)) - elog(ERROR, "Invalid target_lsn value %X/%X", - (uint32) (target_lsn >> 32), (uint32) (target_lsn)); - - xlogreader = InitXLogPageRead(&reader_data, archivedir, target_tli, - wal_seg_size, false, false, true); - - if (xlogreader == NULL) - elog(ERROR, "Out of memory"); - - xlogreader->system_identifier = instance_config.system_identifier; - -#if PG_VERSION_NUM >= 130000 - if (XLogRecPtrIsInvalid(target_lsn)) - target_lsn = SizeOfXLogShortPHD; - XLogBeginRead(xlogreader, target_lsn); -#endif - - res = WalReadRecord(xlogreader, target_lsn, &errormsg) != NULL; - /* Didn't find 'target_lsn' and there is no error, return false */ - - if (errormsg) - elog(WARNING, "Could not read WAL record at %X/%X: %s", - (uint32) (target_lsn >> 32), (uint32) (target_lsn), errormsg); - - CleanupXLogPageRead(xlogreader); - XLogReaderFree(xlogreader); - - return res; -} - -/* - * Get LSN of a first record within the WAL segment with number 'segno'. - */ -XLogRecPtr -get_first_record_lsn(const char *archivedir, XLogSegNo segno, - TimeLineID tli, uint32 wal_seg_size, int timeout) -{ - XLogReaderState *xlogreader; - XLogReaderData reader_data; - XLogRecPtr record = InvalidXLogRecPtr; - XLogRecPtr startpoint; - char wal_segment[MAXFNAMELEN]; - int attempts = 0; - - if (segno <= 1) - elog(ERROR, "Invalid WAL segment number " UINT64_FORMAT, segno); - - GetXLogFileName(wal_segment, tli, segno, instance_config.xlog_seg_size); - - xlogreader = InitXLogPageRead(&reader_data, archivedir, tli, wal_seg_size, - false, false, true); - if (xlogreader == NULL) - elog(ERROR, "Out of memory"); - xlogreader->system_identifier = instance_config.system_identifier; - - /* Set startpoint to 0 in segno */ - GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); - -#if PG_VERSION_NUM >= 130000 - if (XLogRecPtrIsInvalid(startpoint)) - startpoint = SizeOfXLogShortPHD; - XLogBeginRead(xlogreader, startpoint); -#endif - - while (attempts <= timeout) - { - record = XLogFindNextRecord(xlogreader, startpoint); - - if (XLogRecPtrIsInvalid(record)) - record = InvalidXLogRecPtr; - else - { - elog(LOG, "First record in WAL segment \"%s\": %X/%X", wal_segment, - (uint32) (record >> 32), (uint32) (record)); - break; - } - - attempts++; - sleep(1); - } - - /* cleanup */ - CleanupXLogPageRead(xlogreader); - XLogReaderFree(xlogreader); - - return record; -} - - -/* - * Get LSN of the record next after target lsn. - */ -XLogRecPtr -get_next_record_lsn(const char *archivedir, XLogSegNo segno, - TimeLineID tli, uint32 wal_seg_size, int timeout, - XLogRecPtr target) -{ - XLogReaderState *xlogreader; - XLogReaderData reader_data; - XLogRecPtr startpoint, found; - XLogRecPtr res = InvalidXLogRecPtr; - char wal_segment[MAXFNAMELEN]; - int attempts = 0; - - if (segno <= 1) - elog(ERROR, "Invalid WAL segment number " UINT64_FORMAT, segno); - - GetXLogFileName(wal_segment, tli, segno, instance_config.xlog_seg_size); - - xlogreader = InitXLogPageRead(&reader_data, archivedir, tli, wal_seg_size, - false, false, true); - if (xlogreader == NULL) - elog(ERROR, "Out of memory"); - xlogreader->system_identifier = instance_config.system_identifier; - - /* Set startpoint to 0 in segno */ - GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); - -#if PG_VERSION_NUM >= 130000 - if (XLogRecPtrIsInvalid(startpoint)) - startpoint = SizeOfXLogShortPHD; - XLogBeginRead(xlogreader, startpoint); -#endif - - found = XLogFindNextRecord(xlogreader, startpoint); - - if (XLogRecPtrIsInvalid(found)) - { - if (xlogreader->errormsg_buf[0] != '\0') - elog(WARNING, "Could not read WAL record at %X/%X: %s", - (uint32) (startpoint >> 32), (uint32) (startpoint), - xlogreader->errormsg_buf); - else - elog(WARNING, "Could not read WAL record at %X/%X", - (uint32) (startpoint >> 32), (uint32) (startpoint)); - PrintXLogCorruptionMsg(&reader_data, ERROR); - } - startpoint = found; - - while (attempts <= timeout) - { - XLogRecord *record; - char *errormsg; - - if (interrupted) - elog(ERROR, "Interrupted during WAL reading"); - - record = WalReadRecord(xlogreader, startpoint, &errormsg); - - if (record == NULL) - { - XLogRecPtr errptr; - - errptr = XLogRecPtrIsInvalid(startpoint) ? xlogreader->EndRecPtr : - startpoint; - - if (errormsg) - elog(WARNING, "Could not read WAL record at %X/%X: %s", - (uint32) (errptr >> 32), (uint32) (errptr), - errormsg); - else - elog(WARNING, "Could not read WAL record at %X/%X", - (uint32) (errptr >> 32), (uint32) (errptr)); - PrintXLogCorruptionMsg(&reader_data, ERROR); - } - - if (xlogreader->ReadRecPtr >= target) - { - elog(LOG, "Record %X/%X is next after target LSN %X/%X", - (uint32) (xlogreader->ReadRecPtr >> 32), (uint32) (xlogreader->ReadRecPtr), - (uint32) (target >> 32), (uint32) (target)); - res = xlogreader->ReadRecPtr; - break; - } - else - startpoint = InvalidXLogRecPtr; - } - - /* cleanup */ - CleanupXLogPageRead(xlogreader); - XLogReaderFree(xlogreader); - - return res; -} - - -/* - * Get LSN of a record prior to target_lsn. - * If 'start_lsn' is in the segment with number 'segno' then start from 'start_lsn', - * otherwise start from offset 0 within the segment. - * - * Returns LSN of a record which EndRecPtr is greater or equal to target_lsn. - * If 'seek_prev_segment' is true, then look for prior record in prior WAL segment. - * - * it's unclear that "last" in "last_wal_lsn" refers to the - * "closest to stop_lsn backward or forward, depending on seek_prev_segment setting". - */ -XLogRecPtr -get_prior_record_lsn(const char *archivedir, XLogRecPtr start_lsn, - XLogRecPtr stop_lsn, TimeLineID tli, bool seek_prev_segment, - uint32 wal_seg_size) -{ - XLogReaderState *xlogreader; - XLogReaderData reader_data; - XLogRecPtr startpoint; - XLogSegNo start_segno; - XLogSegNo segno; - XLogRecPtr res = InvalidXLogRecPtr; - - GetXLogSegNo(stop_lsn, segno, wal_seg_size); - - if (segno <= 1) - elog(ERROR, "Invalid WAL segment number " UINT64_FORMAT, segno); - - if (seek_prev_segment) - segno = segno - 1; - - xlogreader = InitXLogPageRead(&reader_data, archivedir, tli, wal_seg_size, - false, false, true); - - if (xlogreader == NULL) - elog(ERROR, "Out of memory"); - - xlogreader->system_identifier = instance_config.system_identifier; - - /* - * Calculate startpoint. Decide: we should use 'start_lsn' or offset 0. - */ - GetXLogSegNo(start_lsn, start_segno, wal_seg_size); - if (start_segno == segno) - { - startpoint = start_lsn; -#if PG_VERSION_NUM >= 130000 - if (XLogRecPtrIsInvalid(startpoint)) - startpoint = SizeOfXLogShortPHD; - XLogBeginRead(xlogreader, startpoint); -#endif - } - else - { - XLogRecPtr found; - - GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); - -#if PG_VERSION_NUM >= 130000 - if (XLogRecPtrIsInvalid(startpoint)) - startpoint = SizeOfXLogShortPHD; - XLogBeginRead(xlogreader, startpoint); -#endif - - found = XLogFindNextRecord(xlogreader, startpoint); - - if (XLogRecPtrIsInvalid(found)) - { - if (xlogreader->errormsg_buf[0] != '\0') - elog(WARNING, "Could not read WAL record at %X/%X: %s", - (uint32) (startpoint >> 32), (uint32) (startpoint), - xlogreader->errormsg_buf); - else - elog(WARNING, "Could not read WAL record at %X/%X", - (uint32) (startpoint >> 32), (uint32) (startpoint)); - PrintXLogCorruptionMsg(&reader_data, ERROR); - } - startpoint = found; - } - - while (true) - { - XLogRecord *record; - char *errormsg; - - if (interrupted) - elog(ERROR, "Interrupted during WAL reading"); - - record = WalReadRecord(xlogreader, startpoint, &errormsg); - if (record == NULL) - { - XLogRecPtr errptr; - - errptr = XLogRecPtrIsInvalid(startpoint) ? xlogreader->EndRecPtr : - startpoint; - - if (errormsg) - elog(WARNING, "Could not read WAL record at %X/%X: %s", - (uint32) (errptr >> 32), (uint32) (errptr), - errormsg); - else - elog(WARNING, "Could not read WAL record at %X/%X", - (uint32) (errptr >> 32), (uint32) (errptr)); - PrintXLogCorruptionMsg(&reader_data, ERROR); - } - - if (xlogreader->EndRecPtr >= stop_lsn) - { - elog(LOG, "Record %X/%X has endpoint %X/%X which is equal or greater than requested LSN %X/%X", - (uint32) (xlogreader->ReadRecPtr >> 32), (uint32) (xlogreader->ReadRecPtr), - (uint32) (xlogreader->EndRecPtr >> 32), (uint32) (xlogreader->EndRecPtr), - (uint32) (stop_lsn >> 32), (uint32) (stop_lsn)); - res = xlogreader->ReadRecPtr; - break; - } - - /* continue reading at next record */ - startpoint = InvalidXLogRecPtr; - } - - CleanupXLogPageRead(xlogreader); - XLogReaderFree(xlogreader); - - return res; -} - -#ifdef HAVE_LIBZ -/* - * Show error during work with compressed file - */ -static const char * -get_gz_error(gzFile gzf) -{ - int errnum; - const char *errmsg; - - errmsg = fio_gzerror(gzf, &errnum); - if (errnum == Z_ERRNO) - return strerror(errno); - else - return errmsg; -} -#endif - -/* XLogreader callback function, to read a WAL page */ -static int -SimpleXLogPageRead(XLogReaderState *xlogreader, XLogRecPtr targetPagePtr, - int reqLen, XLogRecPtr targetRecPtr, char *readBuf -#if PG_VERSION_NUM < 130000 - ,TimeLineID *pageTLI -#endif - ) -{ - XLogReaderData *reader_data; - uint32 targetPageOff; - - reader_data = (XLogReaderData *) xlogreader->private_data; - targetPageOff = targetPagePtr % wal_seg_size; - - if (interrupted || thread_interrupted) - elog(ERROR, "Thread [%d]: Interrupted during WAL reading", - reader_data->thread_num); - - /* - * See if we need to switch to a new segment because the requested record - * is not in the currently open one. - */ - if (!IsInXLogSeg(targetPagePtr, reader_data->xlogsegno, wal_seg_size)) - { - elog(VERBOSE, "Thread [%d]: Need to switch to the next WAL segment, page LSN %X/%X, record being read LSN %X/%X", - reader_data->thread_num, - (uint32) (targetPagePtr >> 32), (uint32) (targetPagePtr), - (uint32) (xlogreader->currRecPtr >> 32), - (uint32) (xlogreader->currRecPtr )); - - /* - * If the last record on the page is not complete, - * we must continue reading pages in the same thread - */ - if (!XLogRecPtrIsInvalid(xlogreader->currRecPtr) && - xlogreader->currRecPtr < targetPagePtr) - { - CleanupXLogPageRead(xlogreader); - - /* - * Switch to the next WAL segment after reading contrecord. - */ - if (wal_manual_switch) - reader_data->need_switch = true; - } - else - { - CleanupXLogPageRead(xlogreader); - /* - * Do not switch to next WAL segment in this function. It is - * manually switched by a thread routine. - */ - if (wal_manual_switch) - { - reader_data->need_switch = true; - return -1; - } - } - } - - GetXLogSegNo(targetPagePtr, reader_data->xlogsegno, wal_seg_size); - - /* Try to switch to the next WAL segment */ - if (!reader_data->xlogexists) - { - char xlogfname[MAXFNAMELEN]; - char partial_file[MAXPGPATH]; - - GetXLogFileName(xlogfname, reader_data->tli, reader_data->xlogsegno, wal_seg_size); - - join_path_components(reader_data->xlogpath, wal_archivedir, xlogfname); - snprintf(reader_data->gz_xlogpath, MAXPGPATH, "%s.gz", reader_data->xlogpath); - - /* We fall back to using .partial segment in case if we are running - * multi-timeline incremental backup right after standby promotion. - * TODO: it should be explicitly enabled. - */ - snprintf(partial_file, MAXPGPATH, "%s.partial", reader_data->xlogpath); - - /* If segment do not exists, but the same - * segment with '.partial' suffix does, use it instead */ - if (!fileExists(reader_data->xlogpath, FIO_LOCAL_HOST) && - fileExists(partial_file, FIO_LOCAL_HOST)) - { - snprintf(reader_data->xlogpath, MAXPGPATH, "%s", partial_file); - } - - if (fileExists(reader_data->xlogpath, FIO_LOCAL_HOST)) - { - elog(LOG, "Thread [%d]: Opening WAL segment \"%s\"", - reader_data->thread_num, reader_data->xlogpath); - - reader_data->xlogexists = true; - reader_data->xlogfile = fio_open(reader_data->xlogpath, - O_RDONLY | PG_BINARY, FIO_LOCAL_HOST); - - if (reader_data->xlogfile < 0) - { - elog(WARNING, "Thread [%d]: Could not open WAL segment \"%s\": %s", - reader_data->thread_num, reader_data->xlogpath, - strerror(errno)); - return -1; - } - } -#ifdef HAVE_LIBZ - /* Try to open compressed WAL segment */ - else if (fileExists(reader_data->gz_xlogpath, FIO_LOCAL_HOST)) - { - elog(LOG, "Thread [%d]: Opening compressed WAL segment \"%s\"", - reader_data->thread_num, reader_data->gz_xlogpath); - - reader_data->xlogexists = true; - reader_data->gz_xlogfile = fio_gzopen(reader_data->gz_xlogpath, - "rb", -1, FIO_LOCAL_HOST); - if (reader_data->gz_xlogfile == NULL) - { - elog(WARNING, "Thread [%d]: Could not open compressed WAL segment \"%s\": %s", - reader_data->thread_num, reader_data->gz_xlogpath, - strerror(errno)); - return -1; - } - } -#endif - /* Exit without error if WAL segment doesn't exist */ - if (!reader_data->xlogexists) - return -1; - } - - /* - * At this point, we have the right segment open. - */ - Assert(reader_data->xlogexists); - - /* - * Do not read same page read earlier from the file, read it from the buffer - */ - if (reader_data->prev_page_off != 0 && - reader_data->prev_page_off == targetPageOff) - { - memcpy(readBuf, reader_data->page_buf, XLOG_BLCKSZ); -#if PG_VERSION_NUM < 130000 - *pageTLI = reader_data->tli; -#endif - return XLOG_BLCKSZ; - } - - /* Read the requested page */ - if (reader_data->xlogfile != -1) - { - if (fio_seek(reader_data->xlogfile, (off_t) targetPageOff) < 0) - { - elog(WARNING, "Thread [%d]: Could not seek in WAL segment \"%s\": %s", - reader_data->thread_num, reader_data->xlogpath, strerror(errno)); - return -1; - } - - if (fio_read(reader_data->xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) - { - elog(WARNING, "Thread [%d]: Could not read from WAL segment \"%s\": %s", - reader_data->thread_num, reader_data->xlogpath, strerror(errno)); - return -1; - } - } -#ifdef HAVE_LIBZ - else - { - if (fio_gzseek(reader_data->gz_xlogfile, (z_off_t) targetPageOff, SEEK_SET) == -1) - { - elog(WARNING, "Thread [%d]: Could not seek in compressed WAL segment \"%s\": %s", - reader_data->thread_num, reader_data->gz_xlogpath, - get_gz_error(reader_data->gz_xlogfile)); - return -1; - } - - if (fio_gzread(reader_data->gz_xlogfile, readBuf, XLOG_BLCKSZ) != XLOG_BLCKSZ) - { - elog(WARNING, "Thread [%d]: Could not read from compressed WAL segment \"%s\": %s", - reader_data->thread_num, reader_data->gz_xlogpath, - get_gz_error(reader_data->gz_xlogfile)); - return -1; - } - } -#endif - - memcpy(reader_data->page_buf, readBuf, XLOG_BLCKSZ); - reader_data->prev_page_off = targetPageOff; -#if PG_VERSION_NUM < 130000 - *pageTLI = reader_data->tli; -#endif - return XLOG_BLCKSZ; -} - -/* - * Initialize WAL segments reading. - */ -static XLogReaderState * -InitXLogPageRead(XLogReaderData *reader_data, const char *archivedir, - TimeLineID tli, uint32 segment_size, bool manual_switch, - bool consistent_read, bool allocate_reader) -{ - XLogReaderState *xlogreader = NULL; - - wal_archivedir = archivedir; - wal_seg_size = segment_size; - wal_manual_switch = manual_switch; - wal_consistent_read = consistent_read; - - MemSet(reader_data, 0, sizeof(XLogReaderData)); - reader_data->tli = tli; - reader_data->xlogfile = -1; - - if (allocate_reader) - { - xlogreader = WalReaderAllocate(wal_seg_size, reader_data); - if (xlogreader == NULL) - elog(ERROR, "Out of memory"); - xlogreader->system_identifier = instance_config.system_identifier; - } - - return xlogreader; -} - -/* - * Comparison function to sort xlog_thread_arg array. - */ -static int -xlog_thread_arg_comp(const void *a1, const void *a2) -{ - const xlog_thread_arg *arg1 = a1; - const xlog_thread_arg *arg2 = a2; - - return arg1->reader_data.xlogsegno - arg2->reader_data.xlogsegno; -} - -/* - * Run WAL processing routines using threads. Start from startpoint up to - * endpoint. It is possible to send zero endpoint, threads will read WAL - * infinitely in this case. - */ -static bool -RunXLogThreads(const char *archivedir, time_t target_time, - TransactionId target_xid, XLogRecPtr target_lsn, TimeLineID tli, - uint32 segment_size, XLogRecPtr startpoint, XLogRecPtr endpoint, - bool consistent_read, xlog_record_function process_record, - XLogRecTarget *last_rec, bool inclusive_endpoint) -{ - pthread_t *threads; - xlog_thread_arg *thread_args; - int i; - int threads_need = 0; - XLogSegNo endSegNo = 0; - bool result = true; - - if (!XRecOffIsValid(startpoint) && !XRecOffIsNull(startpoint)) - elog(ERROR, "Invalid startpoint value %X/%X", - (uint32) (startpoint >> 32), (uint32) (startpoint)); - - if (process_record) - elog(LOG, "Extracting pagemap from tli %i on range from %X/%X to %X/%X", - tli, - (uint32) (startpoint >> 32), (uint32) (startpoint), - (uint32) (endpoint >> 32), (uint32) (endpoint)); - - if (!XLogRecPtrIsInvalid(endpoint)) - { -// if (XRecOffIsNull(endpoint) && !inclusive_endpoint) - if (XRecOffIsNull(endpoint)) - { - GetXLogSegNo(endpoint, endSegNo, segment_size); - endSegNo--; - } - else if (!XRecOffIsValid(endpoint)) - { - elog(ERROR, "Invalid endpoint value %X/%X", - (uint32) (endpoint >> 32), (uint32) (endpoint)); - } - else - GetXLogSegNo(endpoint, endSegNo, segment_size); - } - - /* Initialize static variables for workers */ - wal_target_time = target_time; - wal_target_xid = target_xid; - wal_target_lsn = target_lsn; - - GetXLogSegNo(startpoint, segno_start, segment_size); - segno_target = 0; - GetXLogSegNo(startpoint, segno_next, segment_size); - segnum_read = 0; - segnum_corrupted = 0; - - threads = (pthread_t *) pgut_malloc(sizeof(pthread_t) * num_threads); - thread_args = (xlog_thread_arg *) pgut_malloc(sizeof(xlog_thread_arg) * num_threads); - - /* - * Initialize thread args. - * - * Each thread works with its own WAL segment and we need to adjust - * startpoint value for each thread. - */ - for (i = 0; i < num_threads; i++) - { - xlog_thread_arg *arg = &thread_args[i]; - - InitXLogPageRead(&arg->reader_data, archivedir, tli, segment_size, true, - consistent_read, false); - arg->reader_data.xlogsegno = segno_next; - arg->reader_data.thread_num = i + 1; - arg->process_record = process_record; - arg->startpoint = startpoint; - arg->endpoint = endpoint; - arg->endSegNo = endSegNo; - arg->inclusive_endpoint = inclusive_endpoint; - arg->got_target = false; - /* By default there is some error */ - arg->ret = 1; - - threads_need++; - segno_next++; - /* - * If we need to read less WAL segments than num_threads, create less - * threads. - */ - if (endSegNo != 0 && segno_next > endSegNo) - break; - GetXLogRecPtr(segno_next, 0, segment_size, startpoint); - } - - /* Run threads */ - thread_interrupted = false; - for (i = 0; i < threads_need; i++) - { - elog(VERBOSE, "Start WAL reader thread: %d", i + 1); - pthread_create(&threads[i], NULL, XLogThreadWorker, &thread_args[i]); - } - - /* Wait for threads */ - for (i = 0; i < threads_need; i++) - { - pthread_join(threads[i], NULL); - if (thread_args[i].ret == 1) - result = false; - } - thread_interrupted = false; - -// TODO: we must detect difference between actual error (failed to read WAL) and interrupt signal -// if (interrupted) -// elog(ERROR, "Interrupted during WAL parsing"); - - /* Release threads here, use thread_args only below */ - pfree(threads); - threads = NULL; - - if (last_rec) - { - /* - * We need to sort xlog_thread_arg array by xlogsegno to return latest - * possible record up to which restore is possible. We need to sort to - * detect failed thread between start segment and target segment. - * - * Loop stops on first failed thread. - */ - if (threads_need > 1) - qsort((void *) thread_args, threads_need, sizeof(xlog_thread_arg), - xlog_thread_arg_comp); - - for (i = 0; i < threads_need; i++) - { - XLogRecTarget *cur_rec; - - cur_rec = &thread_args[i].reader_data.cur_rec; - /* - * If we got the target return minimum possible record. - */ - if (segno_target > 0) - { - if (thread_args[i].got_target && - thread_args[i].reader_data.xlogsegno == segno_target) - { - *last_rec = *cur_rec; - break; - } - } - /* - * Else return maximum possible record up to which restore is - * possible. - */ - else if (last_rec->rec_lsn < cur_rec->rec_lsn) - *last_rec = *cur_rec; - - /* - * We reached failed thread, so stop here. We cannot use following - * WAL records after failed segment. - */ - if (thread_args[i].ret != 0) - break; - } - } - - pfree(thread_args); - - return result; -} - -/* - * WAL reader worker. - */ -void * -XLogThreadWorker(void *arg) -{ - xlog_thread_arg *thread_arg = (xlog_thread_arg *) arg; - XLogReaderData *reader_data = &thread_arg->reader_data; - XLogReaderState *xlogreader; - XLogSegNo nextSegNo = 0; - XLogRecPtr found; - uint32 prev_page_off = 0; - bool need_read = true; - - xlogreader = WalReaderAllocate(wal_seg_size, reader_data); - - if (xlogreader == NULL) - elog(ERROR, "Thread [%d]: out of memory", reader_data->thread_num); - xlogreader->system_identifier = instance_config.system_identifier; - -#if PG_VERSION_NUM >= 130000 - if (XLogRecPtrIsInvalid(thread_arg->startpoint)) - thread_arg->startpoint = SizeOfXLogShortPHD; - XLogBeginRead(xlogreader, thread_arg->startpoint); -#endif - - found = XLogFindNextRecord(xlogreader, thread_arg->startpoint); - - /* - * We get invalid WAL record pointer usually when WAL segment is absent or - * is corrupted. - */ - if (XLogRecPtrIsInvalid(found)) - { - if (wal_consistent_read && XLogWaitForConsistency(xlogreader)) - need_read = false; - else - { - if (xlogreader->errormsg_buf[0] != '\0') - elog(WARNING, "Thread [%d]: Could not read WAL record at %X/%X: %s", - reader_data->thread_num, - (uint32) (thread_arg->startpoint >> 32), - (uint32) (thread_arg->startpoint), - xlogreader->errormsg_buf); - else - elog(WARNING, "Thread [%d]: Could not read WAL record at %X/%X", - reader_data->thread_num, - (uint32) (thread_arg->startpoint >> 32), - (uint32) (thread_arg->startpoint)); - PrintXLogCorruptionMsg(reader_data, ERROR); - } - } - - thread_arg->startpoint = found; - - elog(VERBOSE, "Thread [%d]: Starting LSN: %X/%X", - reader_data->thread_num, - (uint32) (thread_arg->startpoint >> 32), - (uint32) (thread_arg->startpoint)); - - while (need_read) - { - XLogRecord *record; - char *errormsg; - bool stop_reading = false; - - if (interrupted || thread_interrupted) - elog(ERROR, "Thread [%d]: Interrupted during WAL reading", - reader_data->thread_num); - - /* - * We need to switch to the next WAL segment after reading previous - * record. It may happen if we read contrecord. - */ - if (reader_data->need_switch && - !SwitchThreadToNextWal(xlogreader, thread_arg)) - break; - - record = WalReadRecord(xlogreader, thread_arg->startpoint, &errormsg); - - if (record == NULL) - { - XLogRecPtr errptr; - - /* - * There is no record, try to switch to the next WAL segment. - * Usually SimpleXLogPageRead() does it by itself. But here we need - * to do it manually to support threads. - */ -#if PG_VERSION_NUM >= 150000 - if (reader_data->need_switch && ( - errormsg == NULL || - /* - * Pg15 now informs if "contrecord" is missing. - * TODO: probably we should abort reading logs at this moment. - * But we continue as we did with bug present in Pg < 15. - */ - !XLogRecPtrIsInvalid(xlogreader->abortedRecPtr))) -#else - if (reader_data->need_switch && errormsg == NULL) -#endif - { - if (SwitchThreadToNextWal(xlogreader, thread_arg)) - continue; - else - break; - } - - /* - * XLogWaitForConsistency() is normally used only with threads. - * Call it here for just in case. - */ - if (wal_consistent_read && XLogWaitForConsistency(xlogreader)) - break; - else if (wal_consistent_read) - { - XLogSegNo segno_report; - - pthread_lock(&wal_segment_mutex); - segno_report = segno_start + segnum_read; - pthread_mutex_unlock(&wal_segment_mutex); - - /* - * Report error message if this is the first corrupted WAL. - */ - if (reader_data->xlogsegno > segno_report) - return NULL; /* otherwise just stop the thread */ - } - - errptr = thread_arg->startpoint ? - thread_arg->startpoint : xlogreader->EndRecPtr; - - if (errormsg) - elog(WARNING, "Thread [%d]: Could not read WAL record at %X/%X: %s", - reader_data->thread_num, - (uint32) (errptr >> 32), (uint32) (errptr), - errormsg); - else - elog(WARNING, "Thread [%d]: Could not read WAL record at %X/%X", - reader_data->thread_num, - (uint32) (errptr >> 32), (uint32) (errptr)); - - /* In we failed to read record located at endpoint position, - * and endpoint is not inclusive, do not consider this as an error. - */ - if (!thread_arg->inclusive_endpoint && - errptr == thread_arg->endpoint) - { - elog(LOG, "Thread [%d]: Endpoint %X/%X is not inclusive, switch to the next timeline", - reader_data->thread_num, - (uint32) (thread_arg->endpoint >> 32), (uint32) (thread_arg->endpoint)); - break; - } - - /* - * If we don't have all WAL files from prev backup start_lsn to current - * start_lsn, we won't be able to build page map and PAGE backup will - * be incorrect. Stop it and throw an error. - */ - PrintXLogCorruptionMsg(reader_data, ERROR); - } - - getRecordTimestamp(xlogreader, &reader_data->cur_rec.rec_time); - if (TransactionIdIsValid(XLogRecGetXid(xlogreader))) - reader_data->cur_rec.rec_xid = XLogRecGetXid(xlogreader); - reader_data->cur_rec.rec_lsn = xlogreader->ReadRecPtr; - - if (thread_arg->process_record) - thread_arg->process_record(xlogreader, reader_data, &stop_reading); - if (stop_reading) - { - thread_arg->got_target = true; - - pthread_lock(&wal_segment_mutex); - /* We should store least target segment number */ - if (segno_target == 0 || segno_target > reader_data->xlogsegno) - segno_target = reader_data->xlogsegno; - pthread_mutex_unlock(&wal_segment_mutex); - - break; - } - - /* - * Check if other thread got the target segment. Check it not very - * often, only every WAL page. - */ - if (wal_consistent_read && prev_page_off != 0 && - prev_page_off != reader_data->prev_page_off) - { - XLogSegNo segno; - - pthread_lock(&wal_segment_mutex); - segno = segno_target; - pthread_mutex_unlock(&wal_segment_mutex); - - if (segno != 0 && segno < reader_data->xlogsegno) - break; - } - prev_page_off = reader_data->prev_page_off; - - /* continue reading at next record */ - thread_arg->startpoint = InvalidXLogRecPtr; - - GetXLogSegNo(xlogreader->EndRecPtr, nextSegNo, wal_seg_size); - - if (thread_arg->endSegNo != 0 && - !XLogRecPtrIsInvalid(thread_arg->endpoint) && - /* - * Consider thread_arg->endSegNo and thread_arg->endpoint only if - * they are valid. - */ - xlogreader->ReadRecPtr >= thread_arg->endpoint && - nextSegNo >= thread_arg->endSegNo) - break; - } - - CleanupXLogPageRead(xlogreader); - XLogReaderFree(xlogreader); - - /* Extracting is successful */ - thread_arg->ret = 0; - return NULL; -} - -/* - * Do manual switch to the next WAL segment. - * - * Returns false if the reader reaches the end of a WAL segment list. - */ -static bool -SwitchThreadToNextWal(XLogReaderState *xlogreader, xlog_thread_arg *arg) -{ - XLogReaderData *reader_data; - XLogRecPtr found; - - reader_data = (XLogReaderData *) xlogreader->private_data; - reader_data->need_switch = false; - - /* Critical section */ - pthread_lock(&wal_segment_mutex); - Assert(segno_next); - reader_data->xlogsegno = segno_next; - segnum_read++; - segno_next++; - pthread_mutex_unlock(&wal_segment_mutex); - - /* We've reached the end */ - if (arg->endSegNo != 0 && reader_data->xlogsegno > arg->endSegNo) - return false; - - /* Adjust next record position */ - GetXLogRecPtr(reader_data->xlogsegno, 0, wal_seg_size, arg->startpoint); - /* We need to close previously opened file if it wasn't closed earlier */ - CleanupXLogPageRead(xlogreader); - /* Skip over the page header and contrecord if any */ - found = XLogFindNextRecord(xlogreader, arg->startpoint); - - /* - * We get invalid WAL record pointer usually when WAL segment is - * absent or is corrupted. - */ - if (XLogRecPtrIsInvalid(found)) - { - /* - * Check if we need to stop reading. We stop if other thread found a - * target segment. - */ - if (wal_consistent_read && XLogWaitForConsistency(xlogreader)) - return false; - else if (wal_consistent_read) - { - XLogSegNo segno_report; - - pthread_lock(&wal_segment_mutex); - segno_report = segno_start + segnum_read; - pthread_mutex_unlock(&wal_segment_mutex); - - /* - * Report error message if this is the first corrupted WAL. - */ - if (reader_data->xlogsegno > segno_report) - return false; /* otherwise just stop the thread */ - } - - elog(WARNING, "Thread [%d]: Could not read WAL record at %X/%X", - reader_data->thread_num, - (uint32) (arg->startpoint >> 32), (uint32) (arg->startpoint)); - PrintXLogCorruptionMsg(reader_data, ERROR); - } - arg->startpoint = found; - - elog(VERBOSE, "Thread [%d]: Switched to LSN %X/%X", - reader_data->thread_num, - (uint32) (arg->startpoint >> 32), (uint32) (arg->startpoint)); - - return true; -} - -/* - * Wait for other threads since the current thread couldn't read its segment. - * We need to decide is it fail or not. - * - * Returns true if there is no failure and previous target segment was found. - * Otherwise return false. - */ -static bool -XLogWaitForConsistency(XLogReaderState *xlogreader) -{ - uint32 segnum_need; - XLogReaderData *reader_data =(XLogReaderData *) xlogreader->private_data; - bool log_message = true; - - segnum_need = reader_data->xlogsegno - segno_start; - while (true) - { - uint32 segnum_current_read; - XLogSegNo segno; - - if (log_message) - { - char xlogfname[MAXFNAMELEN]; - - GetXLogFileName(xlogfname, reader_data->tli, reader_data->xlogsegno, - wal_seg_size); - - elog(VERBOSE, "Thread [%d]: Possible WAL corruption in %s. Wait for other threads to decide is this a failure", - reader_data->thread_num, xlogfname); - log_message = false; - } - - if (interrupted || thread_interrupted) - elog(ERROR, "Thread [%d]: Interrupted during WAL reading", - reader_data->thread_num); - - pthread_lock(&wal_segment_mutex); - segnum_current_read = segnum_read + segnum_corrupted; - segno = segno_target; - pthread_mutex_unlock(&wal_segment_mutex); - - /* Other threads read all previous segments and didn't find target */ - if (segnum_need <= segnum_current_read) - { - /* Mark current segment as corrupted */ - pthread_lock(&wal_segment_mutex); - segnum_corrupted++; - pthread_mutex_unlock(&wal_segment_mutex); - return false; - } - - if (segno != 0 && segno < reader_data->xlogsegno) - return true; - - pg_usleep(500000L); /* 500 ms */ - } - - /* We shouldn't reach it */ - return false; -} - -/* - * Cleanup after WAL segment reading. - */ -static void -CleanupXLogPageRead(XLogReaderState *xlogreader) -{ - XLogReaderData *reader_data; - - reader_data = (XLogReaderData *) xlogreader->private_data; - if (reader_data->xlogfile >= 0) - { - fio_close(reader_data->xlogfile); - reader_data->xlogfile = -1; - } -#ifdef HAVE_LIBZ - else if (reader_data->gz_xlogfile != NULL) - { - fio_gzclose(reader_data->gz_xlogfile); - reader_data->gz_xlogfile = NULL; - } -#endif - reader_data->prev_page_off = 0; - reader_data->xlogexists = false; -} - -static void -PrintXLogCorruptionMsg(XLogReaderData *reader_data, int elevel) -{ - if (reader_data->xlogpath[0] != 0) - { - /* - * XLOG reader couldn't read WAL segment. - * We throw a WARNING here to be able to update backup status. - */ - if (!reader_data->xlogexists) - elog(elevel, "Thread [%d]: WAL segment \"%s\" is absent", - reader_data->thread_num, reader_data->xlogpath); - else if (reader_data->xlogfile != -1) - elog(elevel, "Thread [%d]: Possible WAL corruption. " - "Error has occured during reading WAL segment \"%s\"", - reader_data->thread_num, reader_data->xlogpath); -#ifdef HAVE_LIBZ - else if (reader_data->gz_xlogfile != NULL) - elog(elevel, "Thread [%d]: Possible WAL corruption. " - "Error has occured during reading WAL segment \"%s\"", - reader_data->thread_num, reader_data->gz_xlogpath); -#endif - } - else - { - /* Cannot tell what happened specifically */ - elog(elevel, "Thread [%d]: An error occured during WAL reading", - reader_data->thread_num); - } -} - -/* - * Extract information about blocks modified in this record. - */ -static void -extractPageInfo(XLogReaderState *record, XLogReaderData *reader_data, - bool *stop_reading) -{ - uint8 block_id; - RmgrId rmid = XLogRecGetRmid(record); - uint8 info = XLogRecGetInfo(record); - uint8 rminfo = info & ~XLR_INFO_MASK; - - /* Is this a special record type that I recognize? */ - - if (rmid == RM_DBASE_ID -#if PG_VERSION_NUM >= 150000 - && (rminfo == XLOG_DBASE_CREATE_WAL_LOG || rminfo == XLOG_DBASE_CREATE_FILE_COPY)) -#else - && rminfo == XLOG_DBASE_CREATE) -#endif - { - /* - * New databases can be safely ignored. They would be completely - * copied if found. - */ - } - else if (rmid == RM_DBASE_ID && rminfo == XLOG_DBASE_DROP) - { - /* - * An existing database was dropped. It is fine to ignore that - * they will be removed appropriately. - */ - } - else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_CREATE) - { - /* - * We can safely ignore these. The file will be removed when - * combining the backups in the case of differential on. - */ - } - else if (rmid == RM_SMGR_ID && rminfo == XLOG_SMGR_TRUNCATE) - { - /* - * We can safely ignore these. When we compare the sizes later on, - * we'll notice that they differ, and copy the missing tail from - * source system. - */ - } - else if (rmid == RM_XACT_ID && - ((rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_COMMIT || - (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_COMMIT_PREPARED || - (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_ABORT || - (rminfo & XLOG_XACT_OPMASK) == XLOG_XACT_ABORT_PREPARED)) - { - /* - * These records can include "dropped rels". We can safely ignore - * them, we will see that they are missing and copy them from the - * source. - */ - } - else if (info & XLR_SPECIAL_REL_UPDATE) - { - /* - * This record type modifies a relation file in some special way, but - * we don't recognize the type. That's bad - we don't know how to - * track that change. - */ - elog(ERROR, "WAL record modifies a relation, but record type is not recognized\n" - "lsn: %X/%X, rmgr: %s, info: %02X", - (uint32) (record->ReadRecPtr >> 32), (uint32) (record->ReadRecPtr), - RmgrNames[rmid], info); - } - -#if PG_VERSION_NUM >= 150000 - for (block_id = 0; block_id <= record->record->max_block_id; block_id++) -#else - for (block_id = 0; block_id <= record->max_block_id; block_id++) -#endif - { - RelFileNode rnode; - ForkNumber forknum; - BlockNumber blkno; - -#if PG_VERSION_NUM >= 150000 - if (!XLogRecGetBlockTagExtended(record, block_id, &rnode, &forknum, &blkno, NULL)) -#else - if (!XLogRecGetBlockTag(record, block_id, &rnode, &forknum, &blkno)) -#endif - continue; - - /* We only care about the main fork; others are copied as is */ - if (forknum != MAIN_FORKNUM) - continue; - - process_block_change(forknum, rnode, blkno); - } -} - -/* - * Check the current read WAL record during validation. - */ -static void -validateXLogRecord(XLogReaderState *record, XLogReaderData *reader_data, - bool *stop_reading) -{ - /* Check target xid */ - if (TransactionIdIsValid(wal_target_xid) && - wal_target_xid == reader_data->cur_rec.rec_xid) - *stop_reading = true; - /* Check target time */ - else if (wal_target_time != 0 && - timestamptz_to_time_t(reader_data->cur_rec.rec_time) >= wal_target_time) - *stop_reading = true; - /* Check target lsn */ - else if (XRecOffIsValid(wal_target_lsn) && - reader_data->cur_rec.rec_lsn >= wal_target_lsn) - *stop_reading = true; -} - -/* - * Extract timestamp from WAL record. - * - * If the record contains a timestamp, returns true, and saves the timestamp - * in *recordXtime. If the record type has no timestamp, returns false. - * Currently, only transaction commit/abort records and restore points contain - * timestamps. - */ -static bool -getRecordTimestamp(XLogReaderState *record, TimestampTz *recordXtime) -{ - uint8 info = XLogRecGetInfo(record) & ~XLR_INFO_MASK; - uint8 xact_info = info & XLOG_XACT_OPMASK; - uint8 rmid = XLogRecGetRmid(record); - - if (rmid == RM_XLOG_ID && info == XLOG_RESTORE_POINT) - { - *recordXtime = ((xl_restore_point *) XLogRecGetData(record))->rp_time; - return true; - } - else if (rmid == RM_XACT_ID && (xact_info == XLOG_XACT_COMMIT || - xact_info == XLOG_XACT_COMMIT_PREPARED)) - { - *recordXtime = ((xl_xact_commit *) XLogRecGetData(record))->xact_time; - return true; - } - else if (rmid == RM_XACT_ID && (xact_info == XLOG_XACT_ABORT || - xact_info == XLOG_XACT_ABORT_PREPARED)) - { - *recordXtime = ((xl_xact_abort *) XLogRecGetData(record))->xact_time; - return true; - } - - return false; -} - -bool validate_wal_segment(TimeLineID tli, XLogSegNo segno, const char *prefetch_dir, uint32 wal_seg_size) -{ - XLogRecPtr startpoint; - XLogRecPtr endpoint; - - bool rc; - int tmp_num_threads = num_threads; - num_threads = 1; - - /* calculate startpoint and endpoint */ - GetXLogRecPtr(segno, 0, wal_seg_size, startpoint); - GetXLogRecPtr(segno+1, 0, wal_seg_size, endpoint); - - /* disable multi-threading */ - num_threads = 1; - - rc = RunXLogThreads(prefetch_dir, 0, InvalidTransactionId, - InvalidXLogRecPtr, tli, wal_seg_size, - startpoint, endpoint, false, NULL, NULL, true); - - num_threads = tmp_num_threads; - - return rc; -} - -static XLogRecord* WalReadRecord(XLogReaderState *xlogreader, XLogRecPtr startpoint, char **errormsg) -{ - -#if PG_VERSION_NUM >= 130000 - return XLogReadRecord(xlogreader, errormsg); -#else - return XLogReadRecord(xlogreader, startpoint, errormsg); -#endif - -} - -static XLogReaderState* WalReaderAllocate(uint32 wal_seg_size, XLogReaderData *reader_data) -{ - -#if PG_VERSION_NUM >= 130000 - return XLogReaderAllocate(wal_seg_size, NULL, - XL_ROUTINE(.page_read = &SimpleXLogPageRead), - reader_data); -#elif PG_VERSION_NUM >= 110000 - return XLogReaderAllocate(wal_seg_size, &SimpleXLogPageRead, - reader_data); -#else - return XLogReaderAllocate(&SimpleXLogPageRead, reader_data); -#endif -} diff --git a/src/pg_probackup.c b/src/pg_probackup.c deleted file mode 100644 index 30b4212b4..000000000 --- a/src/pg_probackup.c +++ /dev/null @@ -1,1213 +0,0 @@ -/*------------------------------------------------------------------------- - * - * pg_probackup.c: Backup/Recovery manager for PostgreSQL. - * - * This is an entry point for the program. - * Parse command name and it's options, verify them and call a - * do_***() function that implements the command. - * - * Avoid using global variables in the code. - * Pass all needed information as funciton arguments: - * - - * - * TODO (see pg_probackup_state.h): - * - * Functions that work with a backup catalog accept catalogState, - * which currently only contains pathes to backup catalog subdirectories - * + function specific options. - * - * Functions that work with an instance accept instanceState argument, which - * includes catalogState, instance_name, - * info about pgdata associated with the instance (see pgState), - * various instance config options, and list of backups belonging to the instance. - * + function specific options. - * - * Functions that work with multiple backups in the catalog - * accept instanceState and info needed to determine the range of backups to handle. - * + function specific options. - * - * Functions that work with a single backup accept backupState argument, - * which includes link to the instanceState, backup_id and backup-specific info. - * + function specific options. - * - * Functions that work with a postgreSQL instance (i.e. checkdb) accept pgState, - * which includes info about pgdata directory and connection. - * - * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2021, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" -#include "pg_probackup_state.h" - -#include "pg_getopt.h" -#include "streamutil.h" -#include "utils/file.h" - -#include - -#include "utils/configuration.h" -#include "utils/thread.h" -#include - -const char *PROGRAM_NAME = NULL; /* PROGRAM_NAME_FULL without .exe suffix - * if any */ -const char *PROGRAM_NAME_FULL = NULL; -const char *PROGRAM_FULL_PATH = NULL; -const char *PROGRAM_URL = "https://github.com/postgrespro/pg_probackup"; -const char *PROGRAM_EMAIL = "https://github.com/postgrespro/pg_probackup/issues"; - -/* ================ catalogState =========== */ -/* directory options */ -/* TODO make it local variable, pass as an argument to all commands that need it. */ -static char *backup_path = NULL; - -static CatalogState *catalogState = NULL; -/* ================ catalogState (END) =========== */ - -/* common options */ -int num_threads = 1; -bool stream_wal = false; -bool no_color = false; -bool show_color = true; -bool is_archive_cmd = false; -pid_t my_pid = 0; -__thread int my_thread_num = 1; -bool progress = false; -bool no_sync = false; -time_t start_time = INVALID_BACKUP_ID; -#if PG_VERSION_NUM >= 100000 -char *replication_slot = NULL; -bool temp_slot = false; -#endif -bool perm_slot = false; - -/* backup options */ -bool backup_logs = false; -bool smooth_checkpoint; -bool remote_agent = false; -static char *backup_note = NULL; -/* catchup options */ -static char *catchup_source_pgdata = NULL; -static char *catchup_destination_pgdata = NULL; -/* restore options */ -static char *target_time = NULL; -static char *target_xid = NULL; -static char *target_lsn = NULL; -static char *target_inclusive = NULL; -static TimeLineID target_tli; -static char *target_stop; -static bool target_immediate; -static char *target_name = NULL; -static char *target_action = NULL; - -static char *primary_conninfo = NULL; - -static pgRecoveryTarget *recovery_target_options = NULL; -static pgRestoreParams *restore_params = NULL; - -time_t current_time = 0; -static bool restore_as_replica = false; -bool no_validate = false; -IncrRestoreMode incremental_mode = INCR_NONE; - -bool skip_block_validation = false; -bool skip_external_dirs = false; - -/* array for datnames, provided via db-include and db-exclude */ -static parray *datname_exclude_list = NULL; -static parray *datname_include_list = NULL; -/* arrays for --exclude-path's */ -static parray *exclude_absolute_paths_list = NULL; -static parray *exclude_relative_paths_list = NULL; -static char* gl_waldir_path = NULL; -static bool allow_partial_incremental = false; - -/* checkdb options */ -bool need_amcheck = false; -bool heapallindexed = false; -bool checkunique = false; -bool amcheck_parent = false; - -/* delete options */ -bool delete_wal = false; -bool delete_expired = false; -bool merge_expired = false; -bool force = false; -bool dry_run = false; -static char *delete_status = NULL; -/* compression options */ -static bool compress_shortcut = false; - -/* ================ instanceState =========== */ -static char *instance_name; - -static InstanceState *instanceState = NULL; - -/* ================ instanceState (END) =========== */ - -/* archive push options */ -int batch_size = 1; -static char *wal_file_path; -static char *wal_file_name; -static bool file_overwrite = false; -static bool no_ready_rename = false; -static char archive_push_xlog_dir[MAXPGPATH] = ""; - -/* archive get options */ -static char *prefetch_dir; -bool no_validate_wal = false; - -/* show options */ -ShowFormat show_format = SHOW_PLAIN; -bool show_archive = false; - -/* set-backup options */ -int64 ttl = -1; -static char *expire_time_string = NULL; -static pgSetBackupParams *set_backup_params = NULL; - -/* ================ backupState =========== */ -static char *backup_id_string = NULL; -pgBackup current; -/* ================ backupState (END) =========== */ - -static bool help_opt = false; - -static void opt_incr_restore_mode(ConfigOption *opt, const char *arg); -static void opt_backup_mode(ConfigOption *opt, const char *arg); -static void opt_show_format(ConfigOption *opt, const char *arg); - -static void compress_init(ProbackupSubcmd const subcmd); - -static void opt_datname_exclude_list(ConfigOption *opt, const char *arg); -static void opt_datname_include_list(ConfigOption *opt, const char *arg); -static void opt_exclude_path(ConfigOption *opt, const char *arg); - -/* - * Short name should be non-printable ASCII character. - * Use values between 128 and 255. - */ -static ConfigOption cmd_options[] = -{ - /* directory options */ - { 'b', 130, "help", &help_opt, SOURCE_CMD_STRICT }, - { 's', 'B', "backup-path", &backup_path, SOURCE_CMD_STRICT }, - /* common options */ - { 'u', 'j', "threads", &num_threads, SOURCE_CMD_STRICT }, - { 'b', 131, "stream", &stream_wal, SOURCE_CMD_STRICT }, - { 'b', 132, "progress", &progress, SOURCE_CMD_STRICT }, - { 's', 'i', "backup-id", &backup_id_string, SOURCE_CMD_STRICT }, - { 'b', 133, "no-sync", &no_sync, SOURCE_CMD_STRICT }, - { 'b', 134, "no-color", &no_color, SOURCE_CMD_STRICT }, - /* backup options */ - { 'b', 180, "backup-pg-log", &backup_logs, SOURCE_CMD_STRICT }, - { 'f', 'b', "backup-mode", opt_backup_mode, SOURCE_CMD_STRICT }, - { 'b', 'C', "smooth-checkpoint", &smooth_checkpoint, SOURCE_CMD_STRICT }, - { 's', 'S', "slot", &replication_slot, SOURCE_CMD_STRICT }, -#if PG_VERSION_NUM >= 100000 - { 'b', 181, "temp-slot", &temp_slot, SOURCE_CMD_STRICT }, -#endif - { 'b', 'P', "perm-slot", &perm_slot, SOURCE_CMD_STRICT }, - { 'b', 182, "delete-wal", &delete_wal, SOURCE_CMD_STRICT }, - { 'b', 183, "delete-expired", &delete_expired, SOURCE_CMD_STRICT }, - { 'b', 184, "merge-expired", &merge_expired, SOURCE_CMD_STRICT }, - { 'b', 185, "dry-run", &dry_run, SOURCE_CMD_STRICT }, - { 's', 238, "note", &backup_note, SOURCE_CMD_STRICT }, - { 'U', 241, "start-time", &start_time, SOURCE_CMD_STRICT }, - /* catchup options */ - { 's', 239, "source-pgdata", &catchup_source_pgdata, SOURCE_CMD_STRICT }, - { 's', 240, "destination-pgdata", &catchup_destination_pgdata, SOURCE_CMD_STRICT }, - { 'f', 'x', "exclude-path", opt_exclude_path, SOURCE_CMD_STRICT }, - /* restore options */ - { 's', 136, "recovery-target-time", &target_time, SOURCE_CMD_STRICT }, - { 's', 137, "recovery-target-xid", &target_xid, SOURCE_CMD_STRICT }, - { 's', 144, "recovery-target-lsn", &target_lsn, SOURCE_CMD_STRICT }, - { 's', 138, "recovery-target-inclusive", &target_inclusive, SOURCE_CMD_STRICT }, - { 'u', 139, "recovery-target-timeline", &target_tli, SOURCE_CMD_STRICT }, - { 's', 157, "recovery-target", &target_stop, SOURCE_CMD_STRICT }, - { 'f', 'T', "tablespace-mapping", opt_tablespace_map, SOURCE_CMD_STRICT }, - { 'f', 155, "external-mapping", opt_externaldir_map, SOURCE_CMD_STRICT }, - { 's', 141, "recovery-target-name", &target_name, SOURCE_CMD_STRICT }, - { 's', 142, "recovery-target-action", &target_action, SOURCE_CMD_STRICT }, - { 'b', 143, "no-validate", &no_validate, SOURCE_CMD_STRICT }, - { 'b', 154, "skip-block-validation", &skip_block_validation, SOURCE_CMD_STRICT }, - { 'b', 156, "skip-external-dirs", &skip_external_dirs, SOURCE_CMD_STRICT }, - { 'f', 158, "db-include", opt_datname_include_list, SOURCE_CMD_STRICT }, - { 'f', 159, "db-exclude", opt_datname_exclude_list, SOURCE_CMD_STRICT }, - { 'b', 'R', "restore-as-replica", &restore_as_replica, SOURCE_CMD_STRICT }, - { 's', 160, "primary-conninfo", &primary_conninfo, SOURCE_CMD_STRICT }, - { 's', 'S', "primary-slot-name",&replication_slot, SOURCE_CMD_STRICT }, - { 'f', 'I', "incremental-mode", opt_incr_restore_mode, SOURCE_CMD_STRICT }, - { 's', 'X', "waldir", &gl_waldir_path, SOURCE_CMD_STRICT }, - { 'b', 242, "destroy-all-other-dbs", &allow_partial_incremental, SOURCE_CMD_STRICT }, - /* checkdb options */ - { 'b', 195, "amcheck", &need_amcheck, SOURCE_CMD_STRICT }, - { 'b', 196, "heapallindexed", &heapallindexed, SOURCE_CMD_STRICT }, - { 'b', 198, "checkunique", &checkunique, SOURCE_CMD_STRICT }, - { 'b', 197, "parent", &amcheck_parent, SOURCE_CMD_STRICT }, - /* delete options */ - { 'b', 145, "wal", &delete_wal, SOURCE_CMD_STRICT }, - { 'b', 146, "expired", &delete_expired, SOURCE_CMD_STRICT }, - { 's', 172, "status", &delete_status, SOURCE_CMD_STRICT }, - - /* TODO not implemented yet */ - { 'b', 147, "force", &force, SOURCE_CMD_STRICT }, - /* compression options */ - { 'b', 148, "compress", &compress_shortcut, SOURCE_CMD_STRICT }, - /* connection options */ - { 'B', 'w', "no-password", &prompt_password, SOURCE_CMD_STRICT }, - { 'b', 'W', "password", &force_password, SOURCE_CMD_STRICT }, - /* other options */ - { 's', 149, "instance", &instance_name, SOURCE_CMD_STRICT }, - /* archive-push options */ - { 's', 150, "wal-file-path", &wal_file_path, SOURCE_CMD_STRICT }, - { 's', 151, "wal-file-name", &wal_file_name, SOURCE_CMD_STRICT }, - { 'b', 152, "overwrite", &file_overwrite, SOURCE_CMD_STRICT }, - { 'b', 153, "no-ready-rename", &no_ready_rename, SOURCE_CMD_STRICT }, - { 'i', 162, "batch-size", &batch_size, SOURCE_CMD_STRICT }, - /* archive-get options */ - { 's', 163, "prefetch-dir", &prefetch_dir, SOURCE_CMD_STRICT }, - { 'b', 164, "no-validate-wal", &no_validate_wal, SOURCE_CMD_STRICT }, - /* show options */ - { 'f', 165, "format", opt_show_format, SOURCE_CMD_STRICT }, - { 'b', 166, "archive", &show_archive, SOURCE_CMD_STRICT }, - /* set-backup options */ - { 'I', 170, "ttl", &ttl, SOURCE_CMD_STRICT, SOURCE_DEFAULT, 0, OPTION_UNIT_S, option_get_value}, - { 's', 171, "expire-time", &expire_time_string, SOURCE_CMD_STRICT }, - - /* options for backward compatibility - * TODO: remove in 3.0.0 - */ - { 's', 136, "time", &target_time, SOURCE_CMD_STRICT }, - { 's', 137, "xid", &target_xid, SOURCE_CMD_STRICT }, - { 's', 138, "inclusive", &target_inclusive, SOURCE_CMD_STRICT }, - { 'u', 139, "timeline", &target_tli, SOURCE_CMD_STRICT }, - { 's', 144, "lsn", &target_lsn, SOURCE_CMD_STRICT }, - { 'b', 140, "immediate", &target_immediate, SOURCE_CMD_STRICT }, - - { 0 } -}; - -/* - * Entry point of pg_probackup command. - */ -int -main(int argc, char *argv[]) -{ - char *command = NULL; - ProbackupSubcmd backup_subcmd = NO_CMD; - - PROGRAM_NAME_FULL = argv[0]; - - /* Check terminal presense and initialize ANSI escape codes for Windows */ - init_console(); - - /* Initialize current backup */ - pgBackupInit(¤t); - - /* Initialize current instance configuration */ - //TODO get git of this global variable craziness - init_config(&instance_config, instance_name); - - PROGRAM_NAME = get_progname(argv[0]); - set_pglocale_pgservice(argv[0], PG_TEXTDOMAIN("pg_probackup")); - PROGRAM_FULL_PATH = palloc0(MAXPGPATH); - - // Setting C locale for numeric values in order to impose dot-based floating-point representation - memorize_environment_locale(); - setlocale(LC_NUMERIC, "C"); - - /* Get current time */ - current_time = time(NULL); - - my_pid = getpid(); - //set_pglocale_pgservice(argv[0], "pgscripts"); - -#if PG_VERSION_NUM >= 110000 - /* - * Reset WAL segment size, we will retreive it using RetrieveWalSegSize() - * later. - */ - WalSegSz = 0; -#endif - - /* - * Save main thread's tid. It is used call exit() in case of errors. - */ - main_tid = pthread_self(); - - /* Parse subcommands and non-subcommand options */ - if (argc > 1) - { - backup_subcmd = parse_subcmd(argv[1]); - switch(backup_subcmd) - { - case SSH_CMD: -#ifdef WIN32 - launch_ssh(argv); - break; -#else - elog(ERROR, "\"ssh\" command implemented only for Windows"); - break; -#endif - case AGENT_CMD: - /* 'No forward compatibility' sanity: - * /old/binary -> ssh execute -> /newer/binary agent version_num - * If we are executed as an agent for older binary, then exit with error - */ - if (argc > 2) - elog(ERROR, "Version mismatch, pg_probackup binary with version '%s' " - "is launched as an agent for pg_probackup binary with version '%s'", - PROGRAM_VERSION, argv[2]); - remote_agent = true; - fio_communicate(STDIN_FILENO, STDOUT_FILENO); - return 0; - case HELP_CMD: - if (argc > 2) - { - /* 'pg_probackup help command' style */ - help_command(parse_subcmd(argv[2])); - exit(0); - } - else - { - help_pg_probackup(); - exit(0); - } - break; - case VERSION_CMD: - help_print_version(); - exit(0); - case NO_CMD: - elog(ERROR, "Unknown subcommand \"%s\"", argv[1]); - default: - /* Silence compiler warnings */ - break; - } - } - else - elog(ERROR, "No subcommand specified. Please run with \"help\" argument to see possible subcommands."); - - /* - * Make command string before getopt_long() will call. It permutes the - * content of argv. - */ - /* TODO why do we do that only for some commands? */ - if (backup_subcmd == BACKUP_CMD || - backup_subcmd == RESTORE_CMD || - backup_subcmd == VALIDATE_CMD || - backup_subcmd == DELETE_CMD || - backup_subcmd == MERGE_CMD || - backup_subcmd == SET_CONFIG_CMD || - backup_subcmd == SET_BACKUP_CMD) - { - int i, - len = 0, - allocated = 0; - - allocated = sizeof(char) * MAXPGPATH; - command = (char *) palloc(allocated); - - for (i = 0; i < argc; i++) - { - int arglen = strlen(argv[i]); - - if (arglen + len > allocated) - { - allocated *= 2; - command = repalloc(command, allocated); - } - - strncpy(command + len, argv[i], arglen); - len += arglen; - command[len++] = ' '; - } - - command[len] = '\0'; - } - - optind += 1; - /* Parse command line only arguments */ - config_get_opt(argc, argv, cmd_options, instance_options); - - if (backup_subcmd == SET_CONFIG_CMD) - { - int i; - for (i = 0; i < argc; i++) - { - if (strncmp("--log-format-console", argv[i], strlen("--log-format-console")) == 0) - { - elog(ERROR, "Option 'log-format-console' set only from terminal\n"); - } - } - } - - pgut_init(); - - if (no_color) - show_color = false; - - if (help_opt) - { - /* 'pg_probackup command --help' style */ - help_command(backup_subcmd); - exit(0); - } - - /* set location based on cmdline options only */ - setMyLocation(backup_subcmd); - - /* ===== catalogState ======*/ - if (backup_path == NULL) - { - /* - * If command line argument is not set, try to read BACKUP_PATH - * from environment variable - */ - backup_path = getenv("BACKUP_PATH"); - } - - if (backup_path != NULL) - { - canonicalize_path(backup_path); - - /* Ensure that backup_path is an absolute path */ - if (!is_absolute_path(backup_path)) - elog(ERROR, "-B, --backup-path must be an absolute path"); - - catalogState = pgut_new(CatalogState); - strncpy(catalogState->catalog_path, backup_path, MAXPGPATH); - join_path_components(catalogState->backup_subdir_path, - catalogState->catalog_path, BACKUPS_DIR); - join_path_components(catalogState->wal_subdir_path, - catalogState->catalog_path, WAL_SUBDIR); - } - - /* backup_path is required for all pg_probackup commands except help, version, checkdb and catchup */ - if (backup_path == NULL && - backup_subcmd != CHECKDB_CMD && - backup_subcmd != HELP_CMD && - backup_subcmd != VERSION_CMD && - backup_subcmd != CATCHUP_CMD) - elog(ERROR, - "No backup catalog path specified.\n" - "Please specify it either using environment variable BACKUP_PATH or\n" - "command line option --backup-path (-B)"); - - /* ===== catalogState (END) ======*/ - - /* ===== instanceState ======*/ - - /* - * Option --instance is required for all commands except - * init, show, checkdb, validate and catchup - */ - if (instance_name == NULL) - { - if (backup_subcmd != INIT_CMD && backup_subcmd != SHOW_CMD && - backup_subcmd != VALIDATE_CMD && backup_subcmd != CHECKDB_CMD && backup_subcmd != CATCHUP_CMD) - elog(ERROR, "Required parameter not specified: --instance"); - } - else - { - instanceState = pgut_new(InstanceState); - instanceState->catalog_state = catalogState; - - strncpy(instanceState->instance_name, instance_name, MAXPGPATH); - join_path_components(instanceState->instance_backup_subdir_path, - catalogState->backup_subdir_path, instanceState->instance_name); - join_path_components(instanceState->instance_wal_subdir_path, - catalogState->wal_subdir_path, instanceState->instance_name); - join_path_components(instanceState->instance_config_path, - instanceState->instance_backup_subdir_path, BACKUP_CATALOG_CONF_FILE); - - } - /* ===== instanceState (END) ======*/ - - /* - * If --instance option was passed, construct paths for backup data and - * xlog files of this backup instance. - */ - if ((backup_path != NULL) && instance_name) - { - /* - * Ensure that requested backup instance exists. - * for all commands except init, which doesn't take this parameter, - * add-instance, which creates new instance, - * and archive-get, which just do not require it at this point - */ - if (backup_subcmd != INIT_CMD && backup_subcmd != ADD_INSTANCE_CMD && - backup_subcmd != ARCHIVE_GET_CMD) - { - struct stat st; - - if (fio_stat(instanceState->instance_backup_subdir_path, - &st, true, FIO_BACKUP_HOST) != 0) - { - elog(WARNING, "Failed to access directory \"%s\": %s", - instanceState->instance_backup_subdir_path, strerror(errno)); - - // TODO: redundant message, should we get rid of it? - elog(ERROR, "Instance '%s' does not exist in this backup catalog", - instance_name); - } - else - { - /* Ensure that backup_path is a path to a directory */ - if (!S_ISDIR(st.st_mode)) - elog(ERROR, "-B, --backup-path must be a path to directory"); - } - } - } - - /* - * We read options from command line, now we need to read them from - * configuration file since we got backup path and instance name. - * For some commands an instance option isn't required, see above. - */ - if (instance_name) - { - /* Read environment variables */ - config_get_opt_env(instance_options); - - /* Read options from configuration file */ - if (backup_subcmd != ADD_INSTANCE_CMD && - backup_subcmd != ARCHIVE_GET_CMD) - { - if (backup_subcmd == CHECKDB_CMD) - config_read_opt(instanceState->instance_config_path, instance_options, ERROR, true, true); - else - config_read_opt(instanceState->instance_config_path, instance_options, ERROR, true, false); - - /* - * We can determine our location only after reading the configuration file, - * unless we are running arcive-push/archive-get - they are allowed to trust - * cmdline only. - */ - setMyLocation(backup_subcmd); - } - } - else if (backup_subcmd == CATCHUP_CMD) - { - config_get_opt_env(instance_options); - } - - /* - * Disable logging into file for archive-push and archive-get. - * Note, that we should NOT use fio_is_remote() here, - * because it will launch ssh connection and we do not - * want it, because it will kill archive-get prefetch - * performance. - * - * TODO: make logging into file possible via ssh - */ - if (fio_is_remote_simple(FIO_BACKUP_HOST) && - (backup_subcmd == ARCHIVE_GET_CMD || - backup_subcmd == ARCHIVE_PUSH_CMD)) - { - instance_config.logger.log_level_file = LOG_OFF; - is_archive_cmd = true; - } - - - /* Just read environment variables */ - if (backup_path == NULL && backup_subcmd == CHECKDB_CMD) - config_get_opt_env(instance_options); - - /* Sanity for checkdb, if backup_dir is provided but pgdata and instance are not */ - if (backup_subcmd == CHECKDB_CMD && - backup_path != NULL && - instance_name == NULL && - instance_config.pgdata == NULL) - elog(ERROR, "Required parameter not specified: --instance"); - - /* Check checkdb command options consistency */ - if (backup_subcmd == CHECKDB_CMD && - !need_amcheck) - { - if (heapallindexed) - elog(ERROR, "--heapallindexed can only be used with --amcheck option"); - if (checkunique) - elog(ERROR, "--checkunique can only be used with --amcheck option"); - } - - /* Usually checkdb for file logging requires log_directory - * to be specified explicitly, but if backup_dir and instance name are provided, - * checkdb can use the usual default values or values from config - */ - if (backup_subcmd == CHECKDB_CMD && - (instance_config.logger.log_level_file != LOG_OFF && - instance_config.logger.log_directory == NULL) && - (!instance_config.pgdata || !instance_name)) - elog(ERROR, "Cannot save checkdb logs to a file. " - "You must specify --log-directory option when running checkdb with " - "--log-level-file option enabled."); - - if (backup_subcmd == CATCHUP_CMD && - instance_config.logger.log_level_file != LOG_OFF && - instance_config.logger.log_directory == NULL) - elog(ERROR, "Cannot save catchup logs to a file. " - "You must specify --log-directory option when running catchup with " - "--log-level-file option enabled."); - - /* Initialize logger */ - init_logger(backup_path, &instance_config.logger); - - /* command was initialized for a few commands */ - if (command) - { - elog_file(INFO, "command: %s", command); - - pfree(command); - command = NULL; - } - - /* For archive-push and archive-get skip full path lookup */ - if ((backup_subcmd != ARCHIVE_GET_CMD && - backup_subcmd != ARCHIVE_PUSH_CMD) && - (find_my_exec(argv[0],(char *) PROGRAM_FULL_PATH) < 0)) - { - PROGRAM_FULL_PATH = NULL; - elog(WARNING, "%s: could not find a full path to executable", PROGRAM_NAME); - } - - /* - * We have read pgdata path from command line or from configuration file. - * Ensure that pgdata is an absolute path. - */ - if (instance_config.pgdata != NULL) - canonicalize_path(instance_config.pgdata); - if (instance_config.pgdata != NULL && - (backup_subcmd != ARCHIVE_GET_CMD && backup_subcmd != CATCHUP_CMD) && - !is_absolute_path(instance_config.pgdata)) - elog(ERROR, "-D, --pgdata must be an absolute path"); - -#if PG_VERSION_NUM >= 110000 - /* Check xlog-seg-size option */ - if (instance_name && - backup_subcmd != INIT_CMD && - backup_subcmd != ADD_INSTANCE_CMD && backup_subcmd != SET_CONFIG_CMD && - !IsValidWalSegSize(instance_config.xlog_seg_size)) - { - /* If we are working with instance of PG<11 using PG11 binary, - * then xlog_seg_size is equal to zero. Manually set it to 16MB. - */ - if (instance_config.xlog_seg_size == 0) - instance_config.xlog_seg_size = DEFAULT_XLOG_SEG_SIZE; - else - elog(ERROR, "Invalid WAL segment size %u", instance_config.xlog_seg_size); - } -#endif - - /* Sanity check of --backup-id option */ - if (backup_id_string != NULL) - { - if (backup_subcmd != RESTORE_CMD && - backup_subcmd != VALIDATE_CMD && - backup_subcmd != DELETE_CMD && - backup_subcmd != MERGE_CMD && - backup_subcmd != SET_BACKUP_CMD && - backup_subcmd != SHOW_CMD) - elog(ERROR, "Cannot use -i (--backup-id) option together with the \"%s\" command", - get_subcmd_name(backup_subcmd)); - - current.backup_id = base36dec(backup_id_string); - if (current.backup_id == 0) - elog(ERROR, "Invalid backup-id \"%s\"", backup_id_string); - } - - if (!instance_config.conn_opt.pghost && instance_config.remote.host) - instance_config.conn_opt.pghost = instance_config.remote.host; - - /* Setup stream options. They are used in streamutil.c. */ - if (instance_config.conn_opt.pghost != NULL) - dbhost = pstrdup(instance_config.conn_opt.pghost); - if (instance_config.conn_opt.pgport != NULL) - dbport = pstrdup(instance_config.conn_opt.pgport); - if (instance_config.conn_opt.pguser != NULL) - dbuser = pstrdup(instance_config.conn_opt.pguser); - - if (backup_subcmd == VALIDATE_CMD || backup_subcmd == RESTORE_CMD) - { - /* - * Parse all recovery target options into recovery_target_options - * structure. - */ - recovery_target_options = - parseRecoveryTargetOptions(target_time, target_xid, - target_inclusive, target_tli, target_lsn, - (target_stop != NULL) ? target_stop : - (target_immediate) ? "immediate" : NULL, - target_name, target_action); - - if (force && backup_subcmd != RESTORE_CMD) - elog(ERROR, "You cannot specify \"--force\" flag with the \"%s\" command", - get_subcmd_name(backup_subcmd)); - - if (force) - no_validate = true; - - /* keep all params in one structure */ - restore_params = pgut_new(pgRestoreParams); - restore_params->is_restore = (backup_subcmd == RESTORE_CMD); - restore_params->force = force; - restore_params->no_validate = no_validate; - restore_params->restore_as_replica = restore_as_replica; - restore_params->recovery_settings_mode = DEFAULT; - - restore_params->primary_slot_name = replication_slot; - restore_params->skip_block_validation = skip_block_validation; - restore_params->skip_external_dirs = skip_external_dirs; - restore_params->partial_db_list = NULL; - restore_params->partial_restore_type = NONE; - restore_params->primary_conninfo = primary_conninfo; - restore_params->incremental_mode = incremental_mode; - restore_params->allow_partial_incremental = allow_partial_incremental; - - /* handle partial restore parameters */ - if (datname_exclude_list && datname_include_list) - elog(ERROR, "You cannot specify '--db-include' and '--db-exclude' together"); - - if (datname_exclude_list) - { - restore_params->partial_restore_type = EXCLUDE; - restore_params->partial_db_list = datname_exclude_list; - } - else if (datname_include_list) - { - restore_params->partial_restore_type = INCLUDE; - restore_params->partial_db_list = datname_include_list; - } - - if (gl_waldir_path) - { - /* clean up xlog directory name, check it's absolute */ - canonicalize_path(gl_waldir_path); - if (!is_absolute_path(gl_waldir_path)) - { - elog(ERROR, "WAL directory location must be an absolute path"); - } - if (strlen(gl_waldir_path) > MAXPGPATH) - elog(ERROR, "Value specified to --waldir is too long"); - - } - restore_params->waldir = gl_waldir_path; - - } - - /* - * Parse set-backup options into set_backup_params structure. - */ - if (backup_subcmd == SET_BACKUP_CMD || backup_subcmd == BACKUP_CMD) - { - time_t expire_time = 0; - - if (expire_time_string && ttl >= 0) - elog(ERROR, "You cannot specify '--expire-time' and '--ttl' options together"); - - /* Parse string to seconds */ - if (expire_time_string) - { - if (!parse_time(expire_time_string, &expire_time, false)) - elog(ERROR, "Invalid value for '--expire-time' option: '%s'", - expire_time_string); - } - - if (expire_time > 0 || ttl >= 0 || backup_note) - { - set_backup_params = pgut_new(pgSetBackupParams); - set_backup_params->ttl = ttl; - set_backup_params->expire_time = expire_time; - set_backup_params->note = backup_note; - - if (backup_note && strlen(backup_note) > MAX_NOTE_SIZE) - elog(ERROR, "Backup note cannot exceed %u bytes", MAX_NOTE_SIZE); - } - } - - /* checking required options */ - if (backup_subcmd == CATCHUP_CMD) - { - if (catchup_source_pgdata == NULL) - elog(ERROR, "You must specify \"--source-pgdata\" option with the \"%s\" command", get_subcmd_name(backup_subcmd)); - if (catchup_destination_pgdata == NULL) - elog(ERROR, "You must specify \"--destination-pgdata\" option with the \"%s\" command", get_subcmd_name(backup_subcmd)); - if (current.backup_mode == BACKUP_MODE_INVALID) - elog(ERROR, "No backup mode specified.\n" - "Please specify it either using environment variable BACKUP_MODE or\n" - "command line option --backup-mode (-b)"); - if (current.backup_mode != BACKUP_MODE_FULL && current.backup_mode != BACKUP_MODE_DIFF_PTRACK && current.backup_mode != BACKUP_MODE_DIFF_DELTA) - elog(ERROR, "Only \"FULL\", \"PTRACK\" and \"DELTA\" modes are supported with the \"%s\" command", get_subcmd_name(backup_subcmd)); - if (!stream_wal) - elog(INFO, "--stream is required, forcing stream mode"); - current.stream = stream_wal = true; - if (instance_config.external_dir_str) - elog(ERROR, "External directories not supported fom \"%s\" command", get_subcmd_name(backup_subcmd)); - // TODO check instance_config.conn_opt - } - - /* sanity */ - if (backup_subcmd == VALIDATE_CMD && restore_params->no_validate) - elog(ERROR, "You cannot specify \"--no-validate\" option with the \"%s\" command", - get_subcmd_name(backup_subcmd)); - - if (backup_subcmd == ARCHIVE_PUSH_CMD) - { - /* Check archive-push parameters and construct archive_push_xlog_dir - * - * There are 4 cases: - * 1. no --wal-file-path specified -- use cwd, ./PG_XLOG_DIR for wal files - * (and ./PG_XLOG_DIR/archive_status for .done files inside do_archive_push()) - * in this case we can use batches and threads - * 2. --wal-file-path is specified and it is the same dir as stored in pg_probackup.conf (instance_config.pgdata) - * in this case we can use this path, as well as batches and thread - * 3. --wal-file-path is specified and it isn't same dir as stored in pg_probackup.conf but control file present with correct system_id - * in this case we can use this path, as well as batches and thread - * (replica for example, see test_archive_push_sanity) - * 4. --wal-file-path is specified and it is different from instance_config.pgdata and no control file found - * disable optimizations and work with user specified path - */ - bool check_system_id = true; - uint64 system_id; - char current_dir[MAXPGPATH]; - - if (wal_file_name == NULL) - elog(ERROR, "Required parameter is not specified: --wal-file-name %%f"); - - if (instance_config.pgdata == NULL) - elog(ERROR, "Cannot read pg_probackup.conf for this instance"); - - /* TODO may be remove in preference of checking inside compress_init()? */ - if (instance_config.compress_alg == PGLZ_COMPRESS) - elog(ERROR, "Cannot use pglz for WAL compression"); - - if (!getcwd(current_dir, sizeof(current_dir))) - elog(ERROR, "getcwd() error"); - - if (wal_file_path == NULL) - { - /* 1st case */ - system_id = get_system_identifier(current_dir, FIO_DB_HOST, false); - join_path_components(archive_push_xlog_dir, current_dir, XLOGDIR); - } - else - { - /* - * Usually we get something like - * wal_file_path = "pg_wal/0000000100000000000000A1" - * wal_file_name = "0000000100000000000000A1" - * instance_config.pgdata = "/pgdata/.../node/data" - * We need to strip wal_file_name from wal_file_path, add XLOGDIR to instance_config.pgdata - * and compare this directories. - * Note, that pg_wal can be symlink (see test_waldir_outside_pgdata_archiving) - */ - char *stripped_wal_file_path = pgut_str_strip_trailing_filename(wal_file_path, wal_file_name); - join_path_components(archive_push_xlog_dir, instance_config.pgdata, XLOGDIR); - if (fio_is_same_file(stripped_wal_file_path, archive_push_xlog_dir, true, FIO_DB_HOST)) - { - /* 2nd case */ - system_id = get_system_identifier(instance_config.pgdata, FIO_DB_HOST, false); - /* archive_push_xlog_dir already have right value */ - } - else - { - if (strlen(stripped_wal_file_path) < MAXPGPATH) - strncpy(archive_push_xlog_dir, stripped_wal_file_path, MAXPGPATH); - else - elog(ERROR, "Value specified to --wal_file_path is too long"); - - system_id = get_system_identifier(current_dir, FIO_DB_HOST, true); - /* 3rd case if control file present -- i.e. system_id != 0 */ - - if (system_id == 0) - { - /* 4th case */ - check_system_id = false; - - if (batch_size > 1 || num_threads > 1 || !no_ready_rename) - { - elog(WARNING, "Supplied --wal_file_path is outside pgdata, force safe values for options: --batch-size=1 -j 1 --no-ready-rename"); - batch_size = 1; - num_threads = 1; - no_ready_rename = true; - } - } - } - pfree(stripped_wal_file_path); - } - - if (check_system_id && system_id != instance_config.system_identifier) - elog(ERROR, "Refuse to push WAL segment %s into archive. Instance parameters mismatch." - "Instance '%s' should have SYSTEM_ID = " UINT64_FORMAT " instead of " UINT64_FORMAT, - wal_file_name, instanceState->instance_name, instance_config.system_identifier, system_id); - } - -#if PG_VERSION_NUM >= 100000 - if (temp_slot && perm_slot) - elog(ERROR, "You cannot specify \"--perm-slot\" option with the \"--temp-slot\" option"); - - /* if slot name was not provided for temp slot, use default slot name */ - if (!replication_slot && temp_slot) - replication_slot = DEFAULT_TEMP_SLOT_NAME; -#endif - if (!replication_slot && perm_slot) - replication_slot = DEFAULT_PERMANENT_SLOT_NAME; - - if (num_threads < 1) - num_threads = 1; - - if (batch_size < 1) - batch_size = 1; - - compress_init(backup_subcmd); - - /* do actual operation */ - switch (backup_subcmd) - { - case ARCHIVE_PUSH_CMD: - do_archive_push(instanceState, &instance_config, archive_push_xlog_dir, wal_file_name, - batch_size, file_overwrite, no_sync, no_ready_rename); - break; - case ARCHIVE_GET_CMD: - do_archive_get(instanceState, &instance_config, prefetch_dir, - wal_file_path, wal_file_name, batch_size, !no_validate_wal); - break; - case ADD_INSTANCE_CMD: - return do_add_instance(instanceState, &instance_config); - case DELETE_INSTANCE_CMD: - return do_delete_instance(instanceState); - case INIT_CMD: - return do_init(catalogState); - case BACKUP_CMD: - { - current.stream = stream_wal; - if (start_time != INVALID_BACKUP_ID) - elog(WARNING, "Please do not use the --start-time option to start backup. " - "This is a service option required to work with other extensions. " - "We do not guarantee future support for this flag."); - - - /* sanity */ - if (current.backup_mode == BACKUP_MODE_INVALID) - elog(ERROR, "No backup mode specified.\n" - "Please specify it either using environment variable BACKUP_MODE or\n" - "command line option --backup-mode (-b)"); - - return do_backup(instanceState, set_backup_params, - no_validate, no_sync, backup_logs, start_time); - } - case CATCHUP_CMD: - return do_catchup(catchup_source_pgdata, catchup_destination_pgdata, num_threads, !no_sync, - exclude_absolute_paths_list, exclude_relative_paths_list); - case RESTORE_CMD: - return do_restore_or_validate(instanceState, current.backup_id, - recovery_target_options, - restore_params, no_sync); - case VALIDATE_CMD: - if (current.backup_id == 0 && target_time == 0 && target_xid == 0 && !target_lsn) - { - /* sanity */ - if (datname_exclude_list || datname_include_list) - elog(ERROR, "You must specify parameter (-i, --backup-id) for partial validation"); - - return do_validate_all(catalogState, instanceState); - } - else - /* PITR validation and, optionally, partial validation */ - return do_restore_or_validate(instanceState, current.backup_id, - recovery_target_options, - restore_params, - no_sync); - case SHOW_CMD: - return do_show(catalogState, instanceState, current.backup_id, show_archive); - case DELETE_CMD: - - if (delete_expired && backup_id_string) - elog(ERROR, "You cannot specify --delete-expired and (-i, --backup-id) options together"); - if (merge_expired && backup_id_string) - elog(ERROR, "You cannot specify --merge-expired and (-i, --backup-id) options together"); - if (delete_status && backup_id_string) - elog(ERROR, "You cannot specify --status and (-i, --backup-id) options together"); - if (!delete_expired && !merge_expired && !delete_wal && delete_status == NULL && !backup_id_string) - elog(ERROR, "You must specify at least one of the delete options: " - "--delete-expired |--delete-wal |--merge-expired |--status |(-i, --backup-id)"); - if (!backup_id_string) - { - if (delete_status) - do_delete_status(instanceState, &instance_config, delete_status); - else - do_retention(instanceState, no_validate, no_sync); - } - else - do_delete(instanceState, current.backup_id); - break; - case MERGE_CMD: - do_merge(instanceState, current.backup_id, no_validate, no_sync); - break; - case SHOW_CONFIG_CMD: - do_show_config(); - break; - case SET_CONFIG_CMD: - do_set_config(instanceState, false); - break; - case SET_BACKUP_CMD: - if (!backup_id_string) - elog(ERROR, "You must specify parameter (-i, --backup-id) for 'set-backup' command"); - do_set_backup(instanceState, current.backup_id, set_backup_params); - break; - case CHECKDB_CMD: - do_checkdb(need_amcheck, - instance_config.conn_opt, instance_config.pgdata); - break; - case NO_CMD: - /* Should not happen */ - elog(ERROR, "Unknown subcommand"); - case SSH_CMD: - case AGENT_CMD: - /* Может перейти на использование какого-нибудь do_agent() для однобразия? */ - case HELP_CMD: - case VERSION_CMD: - /* Silence compiler warnings, these already handled earlier */ - break; - } - - free_environment_locale(); - - return 0; -} - -static void -opt_incr_restore_mode(ConfigOption *opt, const char *arg) -{ - if (pg_strcasecmp(arg, "none") == 0) - { - incremental_mode = INCR_NONE; - return; - } - else if (pg_strcasecmp(arg, "checksum") == 0) - { - incremental_mode = INCR_CHECKSUM; - return; - } - else if (pg_strcasecmp(arg, "lsn") == 0) - { - incremental_mode = INCR_LSN; - return; - } - - /* Backup mode is invalid, so leave with an error */ - elog(ERROR, "Invalid value for '--incremental-mode' option: '%s'", arg); -} - -static void -opt_backup_mode(ConfigOption *opt, const char *arg) -{ - current.backup_mode = parse_backup_mode(arg); -} - -static void -opt_show_format(ConfigOption *opt, const char *arg) -{ - const char *v = arg; - size_t len; - - /* Skip all spaces detected */ - while (IsSpace(*v)) - v++; - len = strlen(v); - - if (len > 0) - { - if (pg_strncasecmp("plain", v, len) == 0) - show_format = SHOW_PLAIN; - else if (pg_strncasecmp("json", v, len) == 0) - show_format = SHOW_JSON; - else - elog(ERROR, "Invalid show format \"%s\"", arg); - } - else - elog(ERROR, "Invalid show format \"%s\"", arg); -} - -/* - * Initialize compress and sanity checks for compress. - */ -static void -compress_init(ProbackupSubcmd const subcmd) -{ - /* Default algorithm is zlib */ - if (compress_shortcut) - instance_config.compress_alg = ZLIB_COMPRESS; - - if (subcmd != SET_CONFIG_CMD) - { - if (instance_config.compress_level != COMPRESS_LEVEL_DEFAULT - && instance_config.compress_alg == NOT_DEFINED_COMPRESS) - elog(ERROR, "Cannot specify compress-level option alone without " - "compress-algorithm option"); - } - - if (instance_config.compress_level < 0 || instance_config.compress_level > 9) - elog(ERROR, "--compress-level value must be in the range from 0 to 9"); - - if (instance_config.compress_alg == ZLIB_COMPRESS && instance_config.compress_level == 0) - elog(WARNING, "Compression level 0 will lead to data bloat!"); - - if (subcmd == BACKUP_CMD || subcmd == ARCHIVE_PUSH_CMD) - { -#ifndef HAVE_LIBZ - if (instance_config.compress_alg == ZLIB_COMPRESS) - elog(ERROR, "This build does not support zlib compression"); - else -#endif - if (instance_config.compress_alg == PGLZ_COMPRESS && num_threads > 1) - elog(ERROR, "Multithread backup does not support pglz compression"); - } -} - -static void -opt_parser_add_to_parray_helper(parray **list, const char *str) -{ - char *elem = NULL; - - if (*list == NULL) - *list = parray_new(); - - elem = pgut_malloc(strlen(str) + 1); - strcpy(elem, str); - - parray_append(*list, elem); -} - -/* Construct array of datnames, provided by user via db-exclude option */ -void -opt_datname_exclude_list(ConfigOption *opt, const char *arg) -{ - /* TODO add sanity for database name */ - opt_parser_add_to_parray_helper(&datname_exclude_list, arg); -} - -/* Construct array of datnames, provided by user via db-include option */ -void -opt_datname_include_list(ConfigOption *opt, const char *arg) -{ - if (strcmp(arg, "template0") == 0 || - strcmp(arg, "template1") == 0) - elog(ERROR, "Databases 'template0' and 'template1' cannot be used for partial restore or validation"); - - opt_parser_add_to_parray_helper(&datname_include_list, arg); -} - -/* Parse --exclude-path option */ -void -opt_exclude_path(ConfigOption *opt, const char *arg) -{ - if (is_absolute_path(arg)) - opt_parser_add_to_parray_helper(&exclude_absolute_paths_list, arg); - else - opt_parser_add_to_parray_helper(&exclude_relative_paths_list, arg); -} diff --git a/src/pg_probackup.h b/src/pg_probackup.h deleted file mode 100644 index 48b9bf884..000000000 --- a/src/pg_probackup.h +++ /dev/null @@ -1,1367 +0,0 @@ -/*------------------------------------------------------------------------- - * - * pg_probackup.h: Backup/Recovery manager for PostgreSQL. - * - * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2022, Postgres Professional - * - *------------------------------------------------------------------------- - */ -#ifndef PG_PROBACKUP_H -#define PG_PROBACKUP_H - - -#include "postgres_fe.h" -#include "libpq-fe.h" -#include "libpq-int.h" - -#include "access/xlog_internal.h" -#include "utils/pg_crc.h" -#include "catalog/pg_control.h" - -#if PG_VERSION_NUM >= 120000 -#include "common/logging.h" -#endif - -#ifdef FRONTEND -#undef FRONTEND -#include -#define FRONTEND -#else -#include -#endif - -#include "utils/configuration.h" -#include "utils/logger.h" -#include "utils/remote.h" -#include "utils/parray.h" -#include "utils/pgut.h" -#include "utils/file.h" - -#include "datapagemap.h" -#include "utils/thread.h" - -#include "pg_probackup_state.h" - - -#ifdef WIN32 -#define __thread __declspec(thread) -#else -#include -#endif - -#if PG_VERSION_NUM >= 150000 -// _() is explicitly undefined in libpq-int.h -// https://github.com/postgres/postgres/commit/28ec316787674dd74d00b296724a009b6edc2fb0 -#define _(s) gettext(s) -#endif - -/* Wrap the code that we're going to delete after refactoring in this define*/ -#define REFACTORE_ME - -/* pgut client variables and full path */ -extern const char *PROGRAM_NAME; -extern const char *PROGRAM_NAME_FULL; -extern const char *PROGRAM_FULL_PATH; -extern const char *PROGRAM_URL; -extern const char *PROGRAM_EMAIL; - -/* Directory/File names */ -#define DATABASE_DIR "database" -#define BACKUPS_DIR "backups" -#define WAL_SUBDIR "wal" -#if PG_VERSION_NUM >= 100000 -#define PG_XLOG_DIR "pg_wal" -#define PG_LOG_DIR "log" -#else -#define PG_XLOG_DIR "pg_xlog" -#define PG_LOG_DIR "pg_log" -#endif -#define PG_TBLSPC_DIR "pg_tblspc" -#define PG_GLOBAL_DIR "global" -#define BACKUP_CONTROL_FILE "backup.control" -#define BACKUP_CATALOG_CONF_FILE "pg_probackup.conf" -#define BACKUP_LOCK_FILE "backup.pid" -#define BACKUP_RO_LOCK_FILE "backup_ro.pid" -#define DATABASE_FILE_LIST "backup_content.control" -#define PG_BACKUP_LABEL_FILE "backup_label" -#define PG_TABLESPACE_MAP_FILE "tablespace_map" -#define RELMAPPER_FILENAME "pg_filenode.map" -#define EXTERNAL_DIR "external_directories/externaldir" -#define DATABASE_MAP "database_map" -#define HEADER_MAP "page_header_map" -#define HEADER_MAP_TMP "page_header_map_tmp" - -/* default replication slot names */ -#define DEFAULT_TEMP_SLOT_NAME "pg_probackup_slot"; -#define DEFAULT_PERMANENT_SLOT_NAME "pg_probackup_perm_slot"; - -/* Timeout defaults */ -#define ARCHIVE_TIMEOUT_DEFAULT 300 -#define REPLICA_TIMEOUT_DEFAULT 300 -#define LOCK_TIMEOUT 60 -#define LOCK_STALE_TIMEOUT 30 -#define LOG_FREQ 10 - -/* Directory/File permission */ -#define DIR_PERMISSION (0700) -#define FILE_PERMISSION (0600) - -/* 64-bit xid support for PGPRO_EE */ -#ifndef PGPRO_EE -#define XID_FMT "%u" -#elif !defined(XID_FMT) -#define XID_FMT UINT64_FORMAT -#endif - -#ifndef STDIN_FILENO -#define STDIN_FILENO 0 -#define STDOUT_FILENO 1 -#endif - -/* stdio buffer size */ -#define STDIO_BUFSIZE 65536 - -#define ERRMSG_MAX_LEN 2048 -#define CHUNK_SIZE (128 * 1024) -#define LARGE_CHUNK_SIZE (4 * 1024 * 1024) -#define OUT_BUF_SIZE (512 * 1024) - -/* retry attempts */ -#define PAGE_READ_ATTEMPTS 300 - -/* max size of note, that can be added to backup */ -#define MAX_NOTE_SIZE 1024 - -/* Check if an XLogRecPtr value is pointed to 0 offset */ -#define XRecOffIsNull(xlrp) \ - ((xlrp) % XLOG_BLCKSZ == 0) - -/* log(2**64) / log(36) = 12.38 => max 13 char + '\0' */ -#define base36bufsize 14 - -/* Text Coloring macro */ -#define TC_LEN 11 -#define TC_RED "\033[0;31m" -#define TC_RED_BOLD "\033[1;31m" -#define TC_BLUE "\033[0;34m" -#define TC_BLUE_BOLD "\033[1;34m" -#define TC_GREEN "\033[0;32m" -#define TC_GREEN_BOLD "\033[1;32m" -#define TC_YELLOW "\033[0;33m" -#define TC_YELLOW_BOLD "\033[1;33m" -#define TC_MAGENTA "\033[0;35m" -#define TC_MAGENTA_BOLD "\033[1;35m" -#define TC_CYAN "\033[0;36m" -#define TC_CYAN_BOLD "\033[1;36m" -#define TC_RESET "\033[0m" - -typedef struct RedoParams -{ - TimeLineID tli; - XLogRecPtr lsn; - uint32 checksum_version; -} RedoParams; - -typedef struct PageState -{ - uint16 checksum; - XLogRecPtr lsn; -} PageState; - -typedef struct db_map_entry -{ - Oid dbOid; - char *datname; -} db_map_entry; - -/* State of pgdata in the context of its compatibility for incremental restore */ -typedef enum DestDirIncrCompatibility -{ - POSTMASTER_IS_RUNNING, - SYSTEM_ID_MISMATCH, - BACKUP_LABEL_EXISTS, - PARTIAL_INCREMENTAL_FORBIDDEN, - DEST_IS_NOT_OK, - DEST_OK -} DestDirIncrCompatibility; - -typedef enum IncrRestoreMode -{ - INCR_NONE, - INCR_CHECKSUM, - INCR_LSN -} IncrRestoreMode; - -typedef enum PartialRestoreType -{ - NONE, - INCLUDE, - EXCLUDE, -} PartialRestoreType; - -typedef enum RecoverySettingsMode -{ - DEFAULT, /* not set */ - DONTWRITE, /* explicitly forbid to update recovery settings */ - //TODO Should we always clean/preserve old recovery settings, - // or make it configurable? - PITR_REQUESTED, /* can be set based on other parameters - * if not explicitly forbidden */ -} RecoverySettingsMode; - -typedef enum CompressAlg -{ - NOT_DEFINED_COMPRESS = 0, - NONE_COMPRESS, - PGLZ_COMPRESS, - ZLIB_COMPRESS, -} CompressAlg; - -typedef enum ForkName -{ - none, - vm, - fsm, - cfm, - init, - ptrack, - cfs_bck, - cfm_bck -} ForkName; - -#define INIT_FILE_CRC32(use_crc32c, crc) \ -do { \ - if (use_crc32c) \ - INIT_CRC32C(crc); \ - else \ - INIT_TRADITIONAL_CRC32(crc); \ -} while (0) -#define COMP_FILE_CRC32(use_crc32c, crc, data, len) \ -do { \ - if (use_crc32c) \ - COMP_CRC32C((crc), (data), (len)); \ - else \ - COMP_TRADITIONAL_CRC32(crc, data, len); \ -} while (0) -#define FIN_FILE_CRC32(use_crc32c, crc) \ -do { \ - if (use_crc32c) \ - FIN_CRC32C(crc); \ - else \ - FIN_TRADITIONAL_CRC32(crc); \ -} while (0) - -#define pg_off_t unsigned long long - - -/* Information about single file (or dir) in backup */ -typedef struct pgFile -{ - char *name; /* file or directory name */ - mode_t mode; /* protection (file type and permission) */ - size_t size; /* size of the file */ - time_t mtime; /* file st_mtime attribute, can be used only - during backup */ - size_t read_size; /* size of the portion read (if only some pages are - backed up, it's different from size) */ - int64 write_size; /* size of the backed-up file. BYTES_INVALID means - that the file existed but was not backed up - because not modified since last backup. */ - size_t uncompressed_size; /* size of the backed-up file before compression - * and adding block headers. - */ - /* we need int64 here to store '-1' value */ - pg_crc32 crc; /* CRC value of the file, regular file only */ - char *rel_path; /* relative path of the file */ - char *linked; /* path of the linked file */ - bool is_datafile; /* true if the file is PostgreSQL data file */ - Oid tblspcOid; /* tblspcOid extracted from path, if applicable */ - Oid dbOid; /* dbOid extracted from path, if applicable */ - Oid relOid; /* relOid extracted from path, if applicable */ - ForkName forkName; /* forkName extracted from path, if applicable */ - int segno; /* Segment number for ptrack */ - int n_blocks; /* number of blocks in the data file in data directory */ - bool is_cfs; /* Flag to distinguish files compressed by CFS*/ - struct pgFile *cfs_chain; /* linked list of CFS segment's cfm, bck, cfm_bck related files */ - int external_dir_num; /* Number of external directory. 0 if not external */ - bool exists_in_prev; /* Mark files, both data and regular, that exists in previous backup */ - CompressAlg compress_alg; /* compression algorithm applied to the file */ - volatile pg_atomic_flag lock;/* lock for synchronization of parallel threads */ - datapagemap_t pagemap; /* bitmap of pages updated since previous backup - may take up to 16kB per file */ - bool pagemap_isabsent; /* Used to mark files with unknown state of pagemap, - * i.e. datafiles without _ptrack */ - /* Coordinates in header map */ - int n_headers; /* number of blocks in the data file in backup */ - pg_crc32 hdr_crc; /* CRC value of header file: name_hdr */ - pg_off_t hdr_off; /* offset in header map */ - int hdr_size; /* length of headers */ - bool excluded; /* excluded via --exclude-path option */ - bool skip_cfs_nested; /* mark to skip in processing treads as nested to cfs_chain */ - bool remove_from_list; /* tmp flag to clean up files list from temp and unlogged tables */ -} pgFile; - -typedef struct page_map_entry -{ - const char *path; /* file or directory name */ - char *pagemap; - size_t pagemapsize; -} page_map_entry; - -/* Special values of datapagemap_t bitmapsize */ -#define PageBitmapIsEmpty 0 /* Used to mark unchanged datafiles */ - -/* Return codes for check_tablespace_mapping */ -#define NoTblspc 0 -#define EmptyTblspc 1 -#define NotEmptyTblspc 2 - -/* Current state of backup */ -typedef enum BackupStatus -{ - BACKUP_STATUS_INVALID, /* the pgBackup is invalid */ - BACKUP_STATUS_OK, /* completed backup */ - BACKUP_STATUS_ERROR, /* aborted because of unexpected error */ - BACKUP_STATUS_RUNNING, /* running backup */ - BACKUP_STATUS_MERGING, /* merging backups */ - BACKUP_STATUS_MERGED, /* backup has been successfully merged and now awaits - * the assignment of new start_time */ - BACKUP_STATUS_DELETING, /* data files are being deleted */ - BACKUP_STATUS_DELETED, /* data files have been deleted */ - BACKUP_STATUS_DONE, /* completed but not validated yet */ - BACKUP_STATUS_ORPHAN, /* backup validity is unknown but at least one parent backup is corrupted */ - BACKUP_STATUS_CORRUPT /* files are corrupted, not available */ -} BackupStatus; - -typedef enum BackupMode -{ - BACKUP_MODE_INVALID = 0, - BACKUP_MODE_DIFF_PAGE, /* incremental page backup */ - BACKUP_MODE_DIFF_PTRACK, /* incremental page backup with ptrack system */ - BACKUP_MODE_DIFF_DELTA, /* incremental page backup with lsn comparison */ - BACKUP_MODE_FULL /* full backup */ -} BackupMode; - -typedef enum ShowFormat -{ - SHOW_PLAIN, - SHOW_JSON -} ShowFormat; - - -/* special values of pgBackup fields */ -#define INVALID_BACKUP_ID 0 /* backup ID is not provided by user */ -#define BYTES_INVALID (-1) /* file didn`t changed since previous backup, DELTA backup do not rely on it */ -#define FILE_NOT_FOUND (-2) /* file disappeared during backup */ -#define BLOCKNUM_INVALID (-1) -#define PROGRAM_VERSION "2.5.13" - -/* update when remote agent API or behaviour changes */ -#define AGENT_PROTOCOL_VERSION 20509 -#define AGENT_PROTOCOL_VERSION_STR "2.5.9" - -/* update only when changing storage format */ -#define STORAGE_FORMAT_VERSION "2.4.4" - -typedef struct ConnectionOptions -{ - const char *pgdatabase; - const char *pghost; - const char *pgport; - const char *pguser; -} ConnectionOptions; - -typedef struct ConnectionArgs -{ - PGconn *conn; - PGcancel *cancel_conn; -} ConnectionArgs; - -/* Store values for --remote-* option for 'restore_command' constructor */ -typedef struct ArchiveOptions -{ - const char *host; - const char *port; - const char *user; -} ArchiveOptions; - -/* - * An instance configuration. It can be stored in a configuration file or passed - * from command line. - */ -typedef struct InstanceConfig -{ - uint64 system_identifier; - uint32 xlog_seg_size; - - char *pgdata; - char *external_dir_str; - - ConnectionOptions conn_opt; - ConnectionOptions master_conn_opt; - - uint32 replica_timeout; //Deprecated. Not used anywhere - - /* Wait timeout for WAL segment archiving */ - uint32 archive_timeout; - - /* cmdline to be used as restore_command */ - char *restore_command; - - /* Logger parameters */ - LoggerConfig logger; - - /* Remote access parameters */ - RemoteConfig remote; - - /* Retention options. 0 disables the option. */ - uint32 retention_redundancy; - uint32 retention_window; - uint32 wal_depth; - - CompressAlg compress_alg; - int compress_level; - - /* Archive description */ - ArchiveOptions archive; -} InstanceConfig; - -extern ConfigOption instance_options[]; -extern InstanceConfig instance_config; -extern time_t current_time; - -typedef struct PGNodeInfo -{ - uint32 block_size; - uint32 wal_block_size; - uint32 checksum_version; - bool is_superuser; - bool pgpro_support; - - int server_version; - char server_version_str[100]; - - int ptrack_version_num; - bool is_ptrack_enabled; - const char *ptrack_schema; /* used only for ptrack 2.x */ - -} PGNodeInfo; - -/* structure used for access to block header map */ -typedef struct HeaderMap -{ - char path[MAXPGPATH]; - char path_tmp[MAXPGPATH]; /* used only in merge */ - FILE *fp; /* used only for writing */ - char *buf; /* buffer */ - pg_off_t offset; /* current position in fp */ - pthread_mutex_t mutex; - -} HeaderMap; - -typedef struct pgBackup pgBackup; - -/* Information about single backup stored in backup.conf */ -struct pgBackup -{ - BackupMode backup_mode; /* Mode - one of BACKUP_MODE_xxx above*/ - time_t backup_id; /* Identifier of the backup. - * By default it's the same as start_time - * but can be increased if same backup_id - * already exists. It can be also set by - * start_time parameter */ - BackupStatus status; /* Status - one of BACKUP_STATUS_xxx above*/ - TimeLineID tli; /* timeline of start and stop backup lsns */ - XLogRecPtr start_lsn; /* backup's starting transaction log location */ - XLogRecPtr stop_lsn; /* backup's finishing transaction log location */ - time_t start_time; /* UTC time of backup creation */ - time_t merge_dest_backup; /* start_time of incremental backup with - * which this backup is merging with. - * Only available for FULL backups - * with MERGING or MERGED statuses */ - time_t merge_time; /* the moment when merge was started or 0 */ - time_t end_time; /* the moment when backup was finished, or the moment - * when we realized that backup is broken */ - time_t recovery_time; /* Earliest moment for which you can restore - * the state of the database cluster using - * this backup */ - time_t expire_time; /* Backup expiration date */ - TransactionId recovery_xid; /* Earliest xid for which you can restore - * the state of the database cluster using - * this backup */ - /* - * Amount of raw data. For a full backup, this is the total amount of - * data while for a differential backup this is just the difference - * of data taken. - * BYTES_INVALID means nothing was backed up. - */ - int64 data_bytes; - /* Size of WAL files needed to replay on top of this - * backup to reach the consistency. - */ - int64 wal_bytes; - /* Size of data files before applying compression and block header, - * WAL files are not included. - */ - int64 uncompressed_bytes; - - /* Size of data files in PGDATA at the moment of backup. */ - int64 pgdata_bytes; - - CompressAlg compress_alg; - int compress_level; - - /* Fields needed for compatibility check */ - uint32 block_size; - uint32 wal_block_size; - uint32 checksum_version; - char program_version[100]; - char server_version[100]; - - bool stream; /* Was this backup taken in stream mode? - * i.e. does it include all needed WAL files? */ - bool from_replica; /* Was this backup taken from replica */ - time_t parent_backup; /* Identifier of the previous backup. - * Which is basic backup for this - * incremental backup. */ - pgBackup *parent_backup_link; - char *primary_conninfo; /* Connection parameters of the backup - * in the format suitable for recovery.conf */ - char *external_dir_str; /* List of external directories, - * separated by ':' */ - char *root_dir; /* Full path for root backup directory: - backup_path/instance_name/backup_id */ - char *database_dir; /* Full path to directory with data files: - backup_path/instance_name/backup_id/database */ - parray *files; /* list of files belonging to this backup - * must be populated explicitly */ - char *note; - - pg_crc32 content_crc; - - /* map used for access to page headers */ - HeaderMap hdr_map; - - char backup_id_encoded[base36bufsize]; -}; - -/* Recovery target for restore and validate subcommands */ -typedef struct pgRecoveryTarget -{ - time_t target_time; - /* add one more field in order to avoid deparsing target_time back */ - const char *time_string; - TransactionId target_xid; - /* add one more field in order to avoid deparsing target_xid back */ - const char *xid_string; - XLogRecPtr target_lsn; - /* add one more field in order to avoid deparsing target_lsn back */ - const char *lsn_string; - TimeLineID target_tli; - bool target_inclusive; - bool inclusive_specified; - const char *target_stop; - const char *target_name; - const char *target_action; -} pgRecoveryTarget; - -/* Options needed for restore and validate commands */ -typedef struct pgRestoreParams -{ - bool force; - bool is_restore; - bool no_validate; - bool restore_as_replica; - //TODO maybe somehow add restore_as_replica as one of RecoverySettingsModes - RecoverySettingsMode recovery_settings_mode; - bool skip_external_dirs; - bool skip_block_validation; //Start using it - const char *restore_command; - const char *primary_slot_name; - const char *primary_conninfo; - - /* options for incremental restore */ - IncrRestoreMode incremental_mode; - XLogRecPtr shift_lsn; - - /* options for partial restore */ - PartialRestoreType partial_restore_type; - parray *partial_db_list; - bool allow_partial_incremental; - - char* waldir; -} pgRestoreParams; - -/* Options needed for set-backup command */ -typedef struct pgSetBackupParams -{ - int64 ttl; /* amount of time backup must be pinned - * -1 - do nothing - * 0 - disable pinning - */ - time_t expire_time; /* Point in time until backup - * must be pinned. - */ - char *note; -} pgSetBackupParams; - -typedef struct -{ - PGNodeInfo *nodeInfo; - - const char *from_root; - const char *to_root; - const char *external_prefix; - - parray *files_list; - parray *prev_filelist; - parray *external_dirs; - XLogRecPtr prev_start_lsn; - - int thread_num; - HeaderMap *hdr_map; - - /* - * Return value from the thread. - * 0 means there is no error, 1 - there is an error. - */ - int ret; -} backup_files_arg; - -typedef struct timelineInfo timelineInfo; - -/* struct to collect info about timelines in WAL archive */ -struct timelineInfo { - - TimeLineID tli; /* this timeline */ - TimeLineID parent_tli; /* parent timeline. 0 if none */ - timelineInfo *parent_link; /* link to parent timeline */ - XLogRecPtr switchpoint; /* if this timeline has a parent, then - * switchpoint contains switchpoint LSN, - * otherwise 0 */ - XLogSegNo begin_segno; /* first present segment in this timeline */ - XLogSegNo end_segno; /* last present segment in this timeline */ - size_t n_xlog_files; /* number of segments (only really existing) - * does not include lost segments */ - size_t size; /* space on disk taken by regular WAL files */ - parray *backups; /* array of pgBackup sturctures with info - * about backups belonging to this timeline */ - parray *xlog_filelist; /* array of ordinary WAL segments, '.partial' - * and '.backup' files belonging to this timeline */ - parray *lost_segments; /* array of intervals of lost segments */ - parray *keep_segments; /* array of intervals of segments used by WAL retention */ - pgBackup *closest_backup; /* link to valid backup, closest to timeline */ - pgBackup *oldest_backup; /* link to oldest backup on timeline */ - XLogRecPtr anchor_lsn; /* LSN belonging to the oldest segno to keep for 'wal-depth' */ - TimeLineID anchor_tli; /* timeline of anchor_lsn */ -}; - -typedef struct xlogInterval -{ - XLogSegNo begin_segno; - XLogSegNo end_segno; -} xlogInterval; - -typedef struct lsnInterval -{ - TimeLineID tli; - XLogRecPtr begin_lsn; - XLogRecPtr end_lsn; -} lsnInterval; - -typedef enum xlogFileType -{ - SEGMENT, - TEMP_SEGMENT, - PARTIAL_SEGMENT, - BACKUP_HISTORY_FILE -} xlogFileType; - -typedef struct xlogFile -{ - pgFile file; - XLogSegNo segno; - xlogFileType type; - bool keep; /* Used to prevent removal of WAL segments - * required by ARCHIVE backups. */ -} xlogFile; - - -/* - * When copying datafiles to backup we validate and compress them block - * by block. Thus special header is required for each data block. - */ -typedef struct BackupPageHeader -{ - BlockNumber block; /* block number */ - int32 compressed_size; -} BackupPageHeader; - -/* 4MB for 1GB file */ -typedef struct BackupPageHeader2 -{ - XLogRecPtr lsn; - int32 block; /* block number */ - int32 pos; /* position in backup file */ - uint16 checksum; -} BackupPageHeader2; - -typedef struct StopBackupCallbackParams -{ - PGconn *conn; - int server_version; -} StopBackupCallbackParams; - -/* Special value for compressed_size field */ -#define PageIsOk 0 -#define SkipCurrentPage -1 -#define PageIsTruncated -2 -#define PageIsCorrupted -3 /* used by checkdb */ - -/* - * Return timeline, xlog ID and record offset from an LSN of the type - * 0/B000188, usual result from pg_stop_backup() and friends. - */ -#define XLogDataFromLSN(data, xlogid, xrecoff) \ - sscanf(data, "%X/%X", xlogid, xrecoff) - -#define IsCompressedXLogFileName(fname) \ - (strlen(fname) == XLOG_FNAME_LEN + strlen(".gz") && \ - strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ - strcmp((fname) + XLOG_FNAME_LEN, ".gz") == 0) - -#if PG_VERSION_NUM >= 110000 - -#define WalSegmentOffset(xlogptr, wal_segsz_bytes) \ - XLogSegmentOffset(xlogptr, wal_segsz_bytes) -#define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ - XLByteToSeg(xlrp, logSegNo, wal_segsz_bytes) -#define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \ - XLogSegNoOffsetToRecPtr(segno, offset, wal_segsz_bytes, dest) -#define GetXLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \ - XLogFileName(fname, tli, logSegNo, wal_segsz_bytes) -#define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \ - XLByteInSeg(xlrp, logSegNo, wal_segsz_bytes) -#define GetXLogSegName(fname, logSegNo, wal_segsz_bytes) \ - snprintf(fname, 20, "%08X%08X", \ - (uint32) ((logSegNo) / XLogSegmentsPerXLogId(wal_segsz_bytes)), \ - (uint32) ((logSegNo) % XLogSegmentsPerXLogId(wal_segsz_bytes))) - -#define GetXLogSegNoFromScrath(logSegNo, log, seg, wal_segsz_bytes) \ - logSegNo = (uint64) log * XLogSegmentsPerXLogId(wal_segsz_bytes) + seg - -#define GetXLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes) \ - XLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes) -#else -#define WalSegmentOffset(xlogptr, wal_segsz_bytes) \ - ((xlogptr) & ((XLogSegSize) - 1)) -#define GetXLogSegNo(xlrp, logSegNo, wal_segsz_bytes) \ - XLByteToSeg(xlrp, logSegNo) -#define GetXLogRecPtr(segno, offset, wal_segsz_bytes, dest) \ - XLogSegNoOffsetToRecPtr(segno, offset, dest) -#define GetXLogFileName(fname, tli, logSegNo, wal_segsz_bytes) \ - XLogFileName(fname, tli, logSegNo) -#define IsInXLogSeg(xlrp, logSegNo, wal_segsz_bytes) \ - XLByteInSeg(xlrp, logSegNo) -#define GetXLogSegName(fname, logSegNo, wal_segsz_bytes) \ - snprintf(fname, 20, "%08X%08X",\ - (uint32) ((logSegNo) / XLogSegmentsPerXLogId), \ - (uint32) ((logSegNo) % XLogSegmentsPerXLogId)) - -#define GetXLogSegNoFromScrath(logSegNo, log, seg, wal_segsz_bytes) \ - logSegNo = (uint64) log * XLogSegmentsPerXLogId + seg - -#define GetXLogFromFileName(fname, tli, logSegNo, wal_segsz_bytes) \ - XLogFromFileName(fname, tli, logSegNo) -#endif - -#define IsPartialCompressXLogFileName(fname) \ - (strlen(fname) == XLOG_FNAME_LEN + strlen(".gz.partial") && \ - strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ - strcmp((fname) + XLOG_FNAME_LEN, ".gz.partial") == 0) - -#define IsTempXLogFileName(fname) \ - (strlen(fname) == XLOG_FNAME_LEN + strlen(".part") && \ - strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ - strcmp((fname) + XLOG_FNAME_LEN, ".part") == 0) - -#define IsTempPartialXLogFileName(fname) \ - (strlen(fname) == XLOG_FNAME_LEN + strlen(".partial.part") && \ - strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ - strcmp((fname) + XLOG_FNAME_LEN, ".partial.part") == 0) - -#define IsTempCompressXLogFileName(fname) \ - (strlen(fname) == XLOG_FNAME_LEN + strlen(".gz.part") && \ - strspn(fname, "0123456789ABCDEF") == XLOG_FNAME_LEN && \ - strcmp((fname) + XLOG_FNAME_LEN, ".gz.part") == 0) - -#define IsSshProtocol() (instance_config.remote.host && strcmp(instance_config.remote.proto, "ssh") == 0) - -/* common options */ -extern pid_t my_pid; -extern __thread int my_thread_num; -extern int num_threads; -extern bool stream_wal; -extern bool show_color; -extern bool progress; -extern bool is_archive_cmd; /* true for archive-{get,push} */ -/* In pre-10 'replication_slot' is defined in receivelog.h */ -extern char *replication_slot; -#if PG_VERSION_NUM >= 100000 -extern bool temp_slot; -#endif -extern bool perm_slot; - -/* backup options */ -extern bool smooth_checkpoint; - -/* remote probackup options */ -extern bool remote_agent; - -extern bool exclusive_backup; - -/* delete options */ -extern bool delete_wal; -extern bool delete_expired; -extern bool merge_expired; -extern bool dry_run; - -/* ===== instanceState ===== */ - -typedef struct InstanceState -{ - /* catalog, this instance belongs to */ - CatalogState *catalog_state; - - char instance_name[MAXPGPATH]; //previously global var instance_name - /* $BACKUP_PATH/backups/instance_name */ - char instance_backup_subdir_path[MAXPGPATH]; - - /* $BACKUP_PATH/backups/instance_name/BACKUP_CATALOG_CONF_FILE */ - char instance_config_path[MAXPGPATH]; - - /* $BACKUP_PATH/backups/instance_name */ - char instance_wal_subdir_path[MAXPGPATH]; // previously global var arclog_path - - /* TODO: Make it more specific */ - PGconn *conn; - - - //TODO split into some more meaningdul parts - InstanceConfig *config; -} InstanceState; - -/* ===== instanceState (END) ===== */ - -/* show options */ -extern ShowFormat show_format; - -/* checkdb options */ -extern bool heapallindexed; -extern bool checkunique; -extern bool skip_block_validation; - -/* current settings */ -extern pgBackup current; - -/* argv of the process */ -extern char** commands_args; - -/* in backup.c */ -extern int do_backup(InstanceState *instanceState, pgSetBackupParams *set_backup_params, - bool no_validate, bool no_sync, bool backup_logs, time_t start_time); -extern void do_checkdb(bool need_amcheck, ConnectionOptions conn_opt, - char *pgdata); -extern BackupMode parse_backup_mode(const char *value); -extern const char *deparse_backup_mode(BackupMode mode); -extern void process_block_change(ForkNumber forknum, RelFileNode rnode, - BlockNumber blkno); - -/* in catchup.c */ -extern int do_catchup(const char *source_pgdata, const char *dest_pgdata, int num_threads, bool sync_dest_files, - parray *exclude_absolute_paths_list, parray *exclude_relative_paths_list); - -/* in restore.c */ -extern int do_restore_or_validate(InstanceState *instanceState, - time_t target_backup_id, - pgRecoveryTarget *rt, - pgRestoreParams *params, - bool no_sync); -extern bool satisfy_timeline(const parray *timelines, TimeLineID tli, XLogRecPtr lsn); -extern bool satisfy_recovery_target(const pgBackup *backup, - const pgRecoveryTarget *rt); -extern pgRecoveryTarget *parseRecoveryTargetOptions( - const char *target_time, const char *target_xid, - const char *target_inclusive, TimeLineID target_tli, const char* target_lsn, - const char *target_stop, const char *target_name, - const char *target_action); - -extern parray *get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, - PartialRestoreType partial_restore_type); - -extern const char* backup_id_of(pgBackup *backup); -extern void reset_backup_id(pgBackup *backup); - -extern parray *get_backup_filelist(pgBackup *backup, bool strict); -extern parray *read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict); -extern bool tliIsPartOfHistory(const parray *timelines, TimeLineID tli); -extern DestDirIncrCompatibility check_incremental_compatibility(const char *pgdata, uint64 system_identifier, - IncrRestoreMode incremental_mode, - parray *partial_db_list, - bool allow_partial_incremental); - -/* in remote.c */ -extern void check_remote_agent_compatibility(int agent_version, - char *compatibility_str, size_t compatibility_str_max_size); -extern size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_buf_size); - -/* in merge.c */ -extern void do_merge(InstanceState *instanceState, time_t backup_id, bool no_validate, bool no_sync); -extern void merge_backups(pgBackup *backup, pgBackup *next_backup); -extern void merge_chain(InstanceState *instanceState, parray *parent_chain, - pgBackup *full_backup, pgBackup *dest_backup, - bool no_validate, bool no_sync); - -extern parray *read_database_map(pgBackup *backup); - -/* in init.c */ -extern int do_init(CatalogState *catalogState); -extern int do_add_instance(InstanceState *instanceState, InstanceConfig *instance); - -/* in archive.c */ -extern void do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *pg_xlog_dir, - char *wal_file_name, int batch_size, bool overwrite, - bool no_sync, bool no_ready_rename); -extern void do_archive_get(InstanceState *instanceState, InstanceConfig *instance, const char *prefetch_dir_arg, char *wal_file_path, - char *wal_file_name, int batch_size, bool validate_wal); - -/* in configure.c */ -extern void do_show_config(void); -extern void do_set_config(InstanceState *instanceState, bool missing_ok); -extern void init_config(InstanceConfig *config, const char *instance_name); -extern InstanceConfig *readInstanceConfigFile(InstanceState *instanceState); - -/* in show.c */ -extern int do_show(CatalogState *catalogState, InstanceState *instanceState, - time_t requested_backup_id, bool show_archive); -extern void memorize_environment_locale(void); -extern void free_environment_locale(void); - -/* in delete.c */ -extern void do_delete(InstanceState *instanceState, time_t backup_id); -extern void delete_backup_files(pgBackup *backup); -extern void do_retention(InstanceState *instanceState, bool no_validate, bool no_sync); -extern int do_delete_instance(InstanceState *instanceState); -extern void do_delete_status(InstanceState *instanceState, - InstanceConfig *instance_config, const char *status); - -/* in fetch.c */ -extern char *slurpFile(const char *datadir, - const char *path, - size_t *filesize, - bool safe, - fio_location location); -extern char *fetchFile(PGconn *conn, const char *filename, size_t *filesize); - -/* in help.c */ -extern void help_print_version(void); -extern void help_pg_probackup(void); -extern void help_command(ProbackupSubcmd const subcmd); - -/* in validate.c */ -extern void pgBackupValidate(pgBackup* backup, pgRestoreParams *params); -extern int do_validate_all(CatalogState *catalogState, InstanceState *instanceState); -extern int validate_one_page(Page page, BlockNumber absolute_blkno, - XLogRecPtr stop_lsn, PageState *page_st, - uint32 checksum_version); -extern bool validate_tablespace_map(pgBackup *backup, bool no_validate); - -extern parray* get_history_streaming(ConnectionOptions *conn_opt, TimeLineID tli, parray *backup_list); - -/* return codes for validate_one_page */ -/* TODO: use enum */ -#define PAGE_IS_VALID (-1) -#define PAGE_IS_NOT_FOUND (-2) -#define PAGE_IS_ZEROED (-3) -#define PAGE_HEADER_IS_INVALID (-4) -#define PAGE_CHECKSUM_MISMATCH (-5) -#define PAGE_LSN_FROM_FUTURE (-6) - -/* in catalog.c */ -extern pgBackup *read_backup(const char *root_dir); -extern void write_backup(pgBackup *backup, bool strict); -extern void write_backup_status(pgBackup *backup, BackupStatus status, - bool strict); -extern void write_backup_data_bytes(pgBackup *backup); -extern bool lock_backup(pgBackup *backup, bool strict, bool exclusive); - -extern const char *pgBackupGetBackupMode(pgBackup *backup, bool show_color); -extern void pgBackupGetBackupModeColor(pgBackup *backup, char *mode); - -extern parray *catalog_get_instance_list(CatalogState *catalogState); - -extern parray *catalog_get_backup_list(InstanceState *instanceState, time_t requested_backup_id); -extern void catalog_lock_backup_list(parray *backup_list, int from_idx, - int to_idx, bool strict, bool exclusive); -extern pgBackup *catalog_get_last_data_backup(parray *backup_list, - TimeLineID tli, - time_t current_start_time); -extern pgBackup *get_multi_timeline_parent(parray *backup_list, parray *tli_list, - TimeLineID current_tli, time_t current_start_time, - InstanceConfig *instance); -extern timelineInfo *timelineInfoNew(TimeLineID tli); -extern void timelineInfoFree(void *tliInfo); -extern parray *catalog_get_timelines(InstanceState *instanceState, InstanceConfig *instance); -extern void do_set_backup(InstanceState *instanceState, time_t backup_id, - pgSetBackupParams *set_backup_params); -extern void pin_backup(pgBackup *target_backup, - pgSetBackupParams *set_backup_params); -extern void add_note(pgBackup *target_backup, char *note); -extern void pgBackupWriteControl(FILE *out, pgBackup *backup, bool utc); -extern void write_backup_filelist(pgBackup *backup, parray *files, - const char *root, parray *external_list, bool sync); - - -extern void pgBackupInitDir(pgBackup *backup, const char *backup_instance_path); -extern void pgNodeInit(PGNodeInfo *node); -extern void pgBackupInit(pgBackup *backup); -extern void pgBackupFree(void *backup); -extern int pgBackupCompareId(const void *f1, const void *f2); -extern int pgBackupCompareIdDesc(const void *f1, const void *f2); -extern int pgBackupCompareIdEqual(const void *l, const void *r); - -extern pgBackup* find_parent_full_backup(pgBackup *current_backup); -extern int scan_parent_chain(pgBackup *current_backup, pgBackup **result_backup); -/* return codes for scan_parent_chain */ -#define ChainIsBroken 0 -#define ChainIsInvalid 1 -#define ChainIsOk 2 - -extern bool is_parent(time_t parent_backup_time, pgBackup *child_backup, bool inclusive); -extern bool is_prolific(parray *backup_list, pgBackup *target_backup); -extern void append_children(parray *backup_list, pgBackup *target_backup, parray *append_list); -extern bool launch_agent(void); -extern void launch_ssh(char* argv[]); -extern void wait_ssh(void); - -#define COMPRESS_ALG_DEFAULT NOT_DEFINED_COMPRESS -#define COMPRESS_LEVEL_DEFAULT 1 - -extern CompressAlg parse_compress_alg(const char *arg); -extern const char* deparse_compress_alg(int alg); - -/* in dir.c */ -extern bool get_control_value_int64(const char *str, const char *name, int64 *value_int64, bool is_mandatory); -extern bool get_control_value_str(const char *str, const char *name, - char *value_str, size_t value_str_size, bool is_mandatory); -extern void dir_list_file(parray *files, const char *root, bool exclude, - bool follow_symlink, bool add_root, bool backup_logs, - bool skip_hidden, int external_dir_num, fio_location location); - -extern const char *get_tablespace_mapping(const char *dir); -extern void create_data_directories(parray *dest_files, - const char *data_dir, - const char *backup_dir, - bool extract_tablespaces, - bool incremental, - fio_location location, - const char *waldir_path); - -extern void read_tablespace_map(parray *links, const char *backup_dir); -extern void opt_tablespace_map(ConfigOption *opt, const char *arg); -extern void opt_externaldir_map(ConfigOption *opt, const char *arg); -extern int check_tablespace_mapping(pgBackup *backup, bool incremental, bool force, bool pgdata_is_empty, bool no_validate); -extern void check_external_dir_mapping(pgBackup *backup, bool incremental); -extern char *get_external_remap(char *current_dir); - -extern void print_database_map(FILE *out, parray *database_list); -extern void write_database_map(pgBackup *backup, parray *database_list, - parray *backup_file_list); -extern void db_map_entry_free(void *map); - -extern void print_file_list(FILE *out, const parray *files, const char *root, - const char *external_prefix, parray *external_list); -extern parray *make_external_directory_list(const char *colon_separated_dirs, - bool remap); -extern void free_dir_list(parray *list); -extern void makeExternalDirPathByNum(char *ret_path, const char *pattern_path, - const int dir_num); -extern bool backup_contains_external(const char *dir, parray *dirs_list); - -extern int dir_create_dir(const char *path, mode_t mode, bool strict); -extern bool dir_is_empty(const char *path, fio_location location); - -extern bool fileExists(const char *path, fio_location location); -extern size_t pgFileSize(const char *path); - -extern pgFile *pgFileNew(const char *path, const char *rel_path, - bool follow_symlink, int external_dir_num, - fio_location location); -extern pgFile *pgFileInit(const char *rel_path); -extern void pgFileDelete(mode_t mode, const char *full_path); -extern void fio_pgFileDelete(pgFile *file, const char *full_path); - -extern void pgFileFree(void *file); - -extern pg_crc32 pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok); -extern pg_crc32 pgFileGetCRCTruncated(const char *file_path, bool use_crc32c, bool missing_ok); -extern pg_crc32 pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok); - -extern int pgFileMapComparePath(const void *f1, const void *f2); -extern int pgFileCompareName(const void *f1, const void *f2); -extern int pgFileCompareNameWithString(const void *f1, const void *f2); -extern int pgFileCompareRelPathWithString(const void *f1, const void *f2); -extern int pgFileCompareRelPathWithExternal(const void *f1, const void *f2); -extern int pgFileCompareRelPathWithExternalDesc(const void *f1, const void *f2); -extern int pgFileCompareLinked(const void *f1, const void *f2); -extern int pgFileCompareSize(const void *f1, const void *f2); -extern int pgFileCompareSizeDesc(const void *f1, const void *f2); -extern int pgCompareString(const void *str1, const void *str2); -extern int pgPrefixCompareString(const void *str1, const void *str2); -extern int pgCompareOid(const void *f1, const void *f2); -extern void pfilearray_clear_locks(parray *file_list); -extern bool set_forkname(pgFile *file); - -/* in data.c */ -extern bool check_data_file(ConnectionArgs *arguments, pgFile *file, - const char *from_fullpath, uint32 checksum_version); - - -extern void catchup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, - XLogRecPtr sync_lsn, BackupMode backup_mode, - uint32 checksum_version, size_t prev_size); -extern void backup_data_file(pgFile *file, const char *from_fullpath, const char *to_fullpath, - XLogRecPtr prev_backup_start_lsn, BackupMode backup_mode, - CompressAlg calg, int clevel, uint32 checksum_version, - HeaderMap *hdr_map, bool missing_ok); -extern void backup_non_data_file(pgFile *file, pgFile *prev_file, - const char *from_fullpath, const char *to_fullpath, - BackupMode backup_mode, time_t parent_backup_time, - bool missing_ok); -extern void backup_non_data_file_internal(const char *from_fullpath, - fio_location from_location, - const char *to_fullpath, pgFile *file, - bool missing_ok); - -extern size_t restore_data_file(parray *parent_chain, pgFile *dest_file, FILE *out, - const char *to_fullpath, bool use_bitmap, PageState *checksum_map, - XLogRecPtr shift_lsn, datapagemap_t *lsn_map, bool use_headers); -extern size_t restore_data_file_internal(FILE *in, FILE *out, pgFile *file, uint32 backup_version, - const char *from_fullpath, const char *to_fullpath, int nblocks, - datapagemap_t *map, PageState *checksum_map, int checksum_version, - datapagemap_t *lsn_map, BackupPageHeader2 *headers); -extern size_t restore_non_data_file(parray *parent_chain, pgBackup *dest_backup, - pgFile *dest_file, FILE *out, const char *to_fullpath, - bool already_exists); -extern void restore_non_data_file_internal(FILE *in, FILE *out, pgFile *file, - const char *from_fullpath, const char *to_fullpath); -extern bool create_empty_file(fio_location from_location, const char *to_root, - fio_location to_location, pgFile *file); - -extern PageState *get_checksum_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr dest_stop_lsn, BlockNumber segmentno); -extern datapagemap_t *get_lsn_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno); -extern bool validate_file_pages(pgFile *file, const char *fullpath, XLogRecPtr stop_lsn, - uint32 checksum_version, uint32 backup_version, HeaderMap *hdr_map); - -extern BackupPageHeader2* get_data_file_headers(HeaderMap *hdr_map, pgFile *file, uint32 backup_version, bool strict); -extern void write_page_headers(BackupPageHeader2 *headers, pgFile *file, HeaderMap *hdr_map, bool is_merge); -extern void init_header_map(pgBackup *backup); -extern void cleanup_header_map(HeaderMap *hdr_map); -/* parsexlog.c */ -extern bool extractPageMap(const char *archivedir, uint32 wal_seg_size, - XLogRecPtr startpoint, TimeLineID start_tli, - XLogRecPtr endpoint, TimeLineID end_tli, - parray *tli_list); -extern void validate_wal(pgBackup *backup, const char *archivedir, - time_t target_time, TransactionId target_xid, - XLogRecPtr target_lsn, TimeLineID tli, - uint32 seg_size); -extern bool validate_wal_segment(TimeLineID tli, XLogSegNo segno, - const char *prefetch_dir, uint32 wal_seg_size); -extern bool read_recovery_info(const char *archivedir, TimeLineID tli, - uint32 seg_size, - XLogRecPtr start_lsn, XLogRecPtr stop_lsn, - time_t *recovery_time); -extern bool wal_contains_lsn(const char *archivedir, XLogRecPtr target_lsn, - TimeLineID target_tli, uint32 seg_size); -extern XLogRecPtr get_prior_record_lsn(const char *archivedir, XLogRecPtr start_lsn, - XLogRecPtr stop_lsn, TimeLineID tli, - bool seek_prev_segment, uint32 seg_size); - -extern XLogRecPtr get_first_record_lsn(const char *archivedir, XLogRecPtr start_lsn, - TimeLineID tli, uint32 wal_seg_size, int timeout); -extern XLogRecPtr get_next_record_lsn(const char *archivedir, XLogSegNo segno, TimeLineID tli, - uint32 wal_seg_size, int timeout, XLogRecPtr target); - -/* in util.c */ -extern TimeLineID get_current_timeline(PGconn *conn); -extern TimeLineID get_current_timeline_from_control(const char *pgdata_path, fio_location location, bool safe); -extern XLogRecPtr get_checkpoint_location(PGconn *conn); -extern uint64 get_system_identifier(const char *pgdata_path, fio_location location, bool safe); -extern uint64 get_remote_system_identifier(PGconn *conn); -extern uint32 get_data_checksum_version(bool safe); -extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path); -extern uint32 get_xlog_seg_size(const char *pgdata_path); -extern void get_redo(const char *pgdata_path, fio_location pgdata_location, RedoParams *redo); -extern void set_min_recovery_point(pgFile *file, const char *backup_path, - XLogRecPtr stop_backup_lsn); -extern void copy_pgcontrol_file(const char *from_fullpath, fio_location from_location, - const char *to_fullpath, fio_location to_location, pgFile *file); - -extern void time2iso(char *buf, size_t len, time_t time, bool utc); -extern const char *status2str(BackupStatus status); -const char *status2str_color(BackupStatus status); -extern BackupStatus str2status(const char *status); -extern const char *base36enc_to(long unsigned int value, char buf[ARG_SIZE_HINT base36bufsize]); -/* Abuse C99 Compound Literal's lifetime */ -#define base36enc(value) (base36enc_to((value), (char[base36bufsize]){0})) -extern long unsigned int base36dec(const char *text); -extern uint32 parse_server_version(const char *server_version_str); -extern uint32 parse_program_version(const char *program_version); -extern bool parse_page(Page page, XLogRecPtr *lsn); -extern int32 do_compress(void* dst, size_t dst_size, void const* src, size_t src_size, - CompressAlg alg, int level, const char **errormsg); -extern int32 do_decompress(void* dst, size_t dst_size, void const* src, size_t src_size, - CompressAlg alg, const char **errormsg); - -extern void pretty_size(int64 size, char *buf, size_t len); -extern void pretty_time_interval(double time, char *buf, size_t len); - -extern PGconn *pgdata_basic_setup(ConnectionOptions conn_opt, PGNodeInfo *nodeInfo); -extern void check_system_identifiers(PGconn *conn, const char *pgdata); -extern void parse_filelist_filenames(parray *files, const char *root); - -/* in ptrack.c */ -extern void make_pagemap_from_ptrack_2(parray* files, PGconn* backup_conn, - const char *ptrack_schema, - int ptrack_version_num, - XLogRecPtr lsn); -extern void get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo); -extern bool pg_is_ptrack_enabled(PGconn *backup_conn, int ptrack_version_num); - -extern XLogRecPtr get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo); -extern parray * pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, - int ptrack_version_num, XLogRecPtr lsn); - -/* open local file to writing */ -extern FILE* open_local_file_rw(const char *to_fullpath, char **out_buf, uint32 buf_size); - -extern int send_pages(const char *to_fullpath, const char *from_fullpath, - pgFile *file, XLogRecPtr prev_backup_start_lsn, CompressAlg calg, int clevel, - uint32 checksum_version, bool use_pagemap, BackupPageHeader2 **headers, - BackupMode backup_mode); -extern int copy_pages(const char *to_fullpath, const char *from_fullpath, - pgFile *file, XLogRecPtr prev_backup_start_lsn, - uint32 checksum_version, bool use_pagemap, - BackupMode backup_mode); - -/* FIO */ -extern void setMyLocation(ProbackupSubcmd const subcmd); -extern void fio_delete(mode_t mode, const char *fullpath, fio_location location); -extern int fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, - XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, - bool use_pagemap, BlockNumber *err_blknum, char **errormsg, - BackupPageHeader2 **headers); -extern int fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, - XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, - bool use_pagemap, BlockNumber *err_blknum, char **errormsg); -/* return codes for fio_send_pages */ -extern int fio_send_file_gz(const char *from_fullpath, FILE* out, char **errormsg); -extern int fio_send_file(const char *from_fullpath, FILE* out, bool cut_zero_tail, - pgFile *file, char **errormsg); -extern int fio_send_file_local(const char *from_fullpath, FILE* out, bool cut_zero_tail, - pgFile *file, char **errormsg); - -extern void fio_list_dir(parray *files, const char *root, bool exclude, bool follow_symlink, - bool add_root, bool backup_logs, bool skip_hidden, int external_dir_num); - -extern bool pgut_rmtree(const char *path, bool rmtopdir, bool strict); - -extern void pgut_setenv(const char *key, const char *val); -extern void pgut_unsetenv(const char *key); - -extern PageState *fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, - XLogRecPtr dest_stop_lsn, BlockNumber segmentno, fio_location location); - -extern datapagemap_t *fio_get_lsn_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr horizonLsn, BlockNumber segmentno, - fio_location location); -extern pid_t fio_check_postmaster(const char *pgdata, fio_location location); - -extern int32 fio_decompress(void* dst, void const* src, size_t size, int compress_alg, char **errormsg); - -/* return codes for fio_send_pages() and fio_send_file() */ -#define SEND_OK (0) -#define FILE_MISSING (-1) -#define OPEN_FAILED (-2) -#define READ_FAILED (-3) -#define WRITE_FAILED (-4) -#define ZLIB_ERROR (-5) -#define REMOTE_ERROR (-6) -#define PAGE_CORRUPTION (-8) - -/* Check if specified location is local for current node */ -extern bool fio_is_remote(fio_location location); -extern bool fio_is_remote_simple(fio_location location); - -extern void get_header_errormsg(Page page, char **errormsg); -extern void get_checksum_errormsg(Page page, char **errormsg, - BlockNumber absolute_blkno); - -extern bool -datapagemap_is_set(datapagemap_t *map, BlockNumber blkno); - -extern void -datapagemap_print_debug(datapagemap_t *map); - -/* in stream.c */ -extern XLogRecPtr stop_backup_lsn; -extern void start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, - ConnectionOptions *conn_opt, - XLogRecPtr startpos, TimeLineID starttli, - bool is_backup); -extern int wait_WAL_streaming_end(parray *backup_files_list); -extern parray* parse_tli_history_buffer(char *history, TimeLineID tli); - -/* external variables and functions, implemented in backup.c */ -typedef struct PGStopBackupResult -{ - /* - * We will use values of snapshot_xid and invocation_time if there are - * no transactions between start_lsn and stop_lsn. - */ - TransactionId snapshot_xid; - time_t invocation_time; - /* - * Fields that store pg_catalog.pg_stop_backup() result - */ - XLogRecPtr lsn; - size_t backup_label_content_len; - char *backup_label_content; - size_t tablespace_map_content_len; - char *tablespace_map_content; -} PGStopBackupResult; - -extern bool backup_in_progress; -extern parray *backup_files_list; - -extern void pg_start_backup(const char *label, bool smooth, pgBackup *backup, - PGNodeInfo *nodeInfo, PGconn *conn); -extern void pg_silent_client_messages(PGconn *conn); -extern void pg_create_restore_point(PGconn *conn, time_t backup_start_time); -extern void pg_stop_backup_send(PGconn *conn, int server_version, bool is_started_on_replica, bool is_exclusive, char **query_text); -extern void pg_stop_backup_consume(PGconn *conn, int server_version, - bool is_exclusive, uint32 timeout, const char *query_text, - PGStopBackupResult *result); -extern void pg_stop_backup_write_file_helper(const char *path, const char *filename, const char *error_msg_filename, - const void *data, size_t len, parray *file_list); -extern XLogRecPtr wait_wal_lsn(const char *wal_segment_dir, XLogRecPtr lsn, bool is_start_lsn, TimeLineID tli, - bool in_prev_segment, bool segment_only, - int timeout_elevel, bool in_stream_dir); -extern void wait_wal_and_calculate_stop_lsn(const char *xlog_path, XLogRecPtr stop_lsn, pgBackup *backup); -extern int64 calculate_datasize_of_filelist(parray *filelist); - -#endif /* PG_PROBACKUP_H */ diff --git a/src/pg_probackup_state.h b/src/pg_probackup_state.h deleted file mode 100644 index 56d852537..000000000 --- a/src/pg_probackup_state.h +++ /dev/null @@ -1,31 +0,0 @@ -/*------------------------------------------------------------------------- - * - * pg_probackup_state.h: Definitions of internal pg_probackup states - * - * Portions Copyright (c) 2021, Postgres Professional - * - *------------------------------------------------------------------------- - */ -#ifndef PG_PROBACKUP_STATE_H -#define PG_PROBACKUP_STATE_H - -/* ====== CatalogState ======= */ - -typedef struct CatalogState -{ - /* $BACKUP_PATH */ - char catalog_path[MAXPGPATH]; //previously global var backup_path - /* $BACKUP_PATH/backups */ - char backup_subdir_path[MAXPGPATH]; - /* $BACKUP_PATH/wal */ - char wal_subdir_path[MAXPGPATH]; // previously global var arclog_path -} CatalogState; - -/* ====== CatalogState (END) ======= */ - - -/* ===== instanceState ===== */ - -/* ===== instanceState (END) ===== */ - -#endif /* PG_PROBACKUP_STATE_H */ diff --git a/src/ptrack.c b/src/ptrack.c deleted file mode 100644 index d27629e45..000000000 --- a/src/ptrack.c +++ /dev/null @@ -1,309 +0,0 @@ -/*------------------------------------------------------------------------- - * - * ptrack.c: support functions for ptrack backups - * - * Copyright (c) 2021 Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -#if PG_VERSION_NUM < 110000 -#include "catalog/catalog.h" -#endif -#include "catalog/pg_tablespace.h" - -/* - * Macro needed to parse ptrack. - * NOTE Keep those values synchronized with definitions in ptrack.h - */ -#define PTRACK_BITS_PER_HEAPBLOCK 1 -#define HEAPBLOCKS_PER_BYTE (BITS_PER_BYTE / PTRACK_BITS_PER_HEAPBLOCK) - -/* - * Parse a string like "2.1" into int - * result: int by formula major_number * 100 + minor_number - * or -1 if string cannot be parsed - */ -static int -ptrack_parse_version_string(const char *version_str) -{ - int ma, mi; - int sscanf_readed_count; - if (sscanf(version_str, "%u.%2u%n", &ma, &mi, &sscanf_readed_count) != 2) - return -1; - if (sscanf_readed_count != strlen(version_str)) - return -1; - return ma * 100 + mi; -} - -/* Check if the instance supports compatible version of ptrack, - * fill-in version number if it does. - * Also for ptrack 2.x save schema namespace. - */ -void -get_ptrack_version(PGconn *backup_conn, PGNodeInfo *nodeInfo) -{ - PGresult *res_db; - char *ptrack_version_str; - int ptrack_version_num; - - res_db = pgut_execute(backup_conn, - "SELECT extnamespace::regnamespace, extversion " - "FROM pg_catalog.pg_extension WHERE extname = 'ptrack'::name", - 0, NULL); - - if (PQntuples(res_db) > 0) - { - /* ptrack 2.x is supported, save schema name and version */ - nodeInfo->ptrack_schema = pgut_strdup(PQgetvalue(res_db, 0, 0)); - - if (nodeInfo->ptrack_schema == NULL) - elog(ERROR, "Failed to obtain schema name of ptrack extension"); - - ptrack_version_str = PQgetvalue(res_db, 0, 1); - } - else - { - /* ptrack 1.x is supported, save version */ - PQclear(res_db); - res_db = pgut_execute(backup_conn, - "SELECT proname FROM pg_catalog.pg_proc WHERE proname='ptrack_version'::name", - 0, NULL); - - if (PQntuples(res_db) == 0) - { - /* ptrack is not supported */ - PQclear(res_db); - return; - } - - /* - * it's ok not to have permission to call this old function in PGPRO-11 version (ok_error = true) - * see deprication notice https://postgrespro.com/docs/postgrespro/11/release-pro-11-9-1 - */ - res_db = pgut_execute_extended(backup_conn, - "SELECT pg_catalog.ptrack_version()", - 0, NULL, true, true); - if (PQntuples(res_db) == 0) - { - PQclear(res_db); - elog(WARNING, "Can't call pg_catalog.ptrack_version(), it is assumed that there is no ptrack extension installed."); - return; - } - ptrack_version_str = PQgetvalue(res_db, 0, 0); - } - - ptrack_version_num = ptrack_parse_version_string(ptrack_version_str); - if (ptrack_version_num == -1) - /* leave default nodeInfo->ptrack_version_num = 0 from pgNodeInit() */ - elog(WARNING, "Cannot parse ptrack version string \"%s\"", - ptrack_version_str); - else - nodeInfo->ptrack_version_num = ptrack_version_num; - - /* ptrack 1.X is buggy, so fall back to DELTA backup strategy for safety */ - if (nodeInfo->ptrack_version_num < 200) - { - if (current.backup_mode == BACKUP_MODE_DIFF_PTRACK) - { - elog(WARNING, "Update your ptrack to the version 2.1 or upper. Current version is %s. " - "Fall back to DELTA backup.", - ptrack_version_str); - current.backup_mode = BACKUP_MODE_DIFF_DELTA; - } - } - - PQclear(res_db); -} - -/* - * Check if ptrack is enabled in target instance - */ -bool -pg_is_ptrack_enabled(PGconn *backup_conn, int ptrack_version_num) -{ - PGresult *res_db; - bool result = false; - - if (ptrack_version_num > 200) - { - res_db = pgut_execute(backup_conn, "SHOW ptrack.map_size", 0, NULL); - result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0 && - strcmp(PQgetvalue(res_db, 0, 0), "-1") != 0; - PQclear(res_db); - } - else if (ptrack_version_num == 200) - { - res_db = pgut_execute(backup_conn, "SHOW ptrack_map_size", 0, NULL); - result = strcmp(PQgetvalue(res_db, 0, 0), "0") != 0; - PQclear(res_db); - } - else - { - result = false; - } - - return result; -} - -/* - * Get lsn of the moment when ptrack was enabled the last time. - */ -XLogRecPtr -get_last_ptrack_lsn(PGconn *backup_conn, PGNodeInfo *nodeInfo) - -{ - PGresult *res; - uint32 lsn_hi; - uint32 lsn_lo; - XLogRecPtr lsn; - - char query[128]; - - if (nodeInfo->ptrack_version_num == 200) - sprintf(query, "SELECT %s.pg_ptrack_control_lsn()", nodeInfo->ptrack_schema); - else - sprintf(query, "SELECT %s.ptrack_init_lsn()", nodeInfo->ptrack_schema); - - res = pgut_execute(backup_conn, query, 0, NULL); - - /* Extract timeline and LSN from results of pg_start_backup() */ - XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); - /* Calculate LSN */ - lsn = ((uint64) lsn_hi) << 32 | lsn_lo; - - PQclear(res); - return lsn; -} - -/* ---------------------------- - * Ptrack 2.* support functions - * ---------------------------- - */ - -/* - * Fetch a list of changed files with their ptrack maps. - */ -parray * -pg_ptrack_get_pagemapset(PGconn *backup_conn, const char *ptrack_schema, - int ptrack_version_num, XLogRecPtr lsn) -{ - PGresult *res; - char lsn_buf[17 + 1]; - char *params[1]; - parray *pagemapset = NULL; - int i; - char query[512]; - - snprintf(lsn_buf, sizeof lsn_buf, "%X/%X", (uint32) (lsn >> 32), (uint32) lsn); - params[0] = pstrdup(lsn_buf); - - if (!ptrack_schema) - elog(ERROR, "Schema name of ptrack extension is missing"); - - if (ptrack_version_num == 200) - sprintf(query, "SELECT path, pagemap FROM %s.pg_ptrack_get_pagemapset($1) ORDER BY 1", - ptrack_schema); - else - sprintf(query, "SELECT path, pagemap FROM %s.ptrack_get_pagemapset($1) ORDER BY 1", - ptrack_schema); - - res = pgut_execute(backup_conn, query, 1, (const char **) params); - pfree(params[0]); - - if (PQnfields(res) != 2) - elog(ERROR, "Cannot get ptrack pagemapset"); - - /* sanity ? */ - - /* Construct database map */ - for (i = 0; i < PQntuples(res); i++) - { - page_map_entry *pm_entry = (page_map_entry *) pgut_malloc(sizeof(page_map_entry)); - - /* get path */ - pm_entry->path = pgut_strdup(PQgetvalue(res, i, 0)); - - /* get bytea */ - pm_entry->pagemap = (char *) PQunescapeBytea((unsigned char *) PQgetvalue(res, i, 1), - &pm_entry->pagemapsize); - - if (pagemapset == NULL) - pagemapset = parray_new(); - - parray_append(pagemapset, pm_entry); - } - - PQclear(res); - - return pagemapset; -} - -/* - * Given a list of files in the instance to backup, build a pagemap for each - * data file that has ptrack. Result is saved in the pagemap field of pgFile. - * - * We fetch a list of changed files with their ptrack maps. After that files - * are merged with their bitmaps. File without bitmap is treated as unchanged. - */ -void -make_pagemap_from_ptrack_2(parray *files, - PGconn *backup_conn, - const char *ptrack_schema, - int ptrack_version_num, - XLogRecPtr lsn) -{ - parray *filemaps; - int file_i = 0; - page_map_entry *dummy_map = NULL; - - /* Receive all available ptrack bitmaps at once */ - filemaps = pg_ptrack_get_pagemapset(backup_conn, ptrack_schema, - ptrack_version_num, lsn); - - if (filemaps != NULL) - parray_qsort(filemaps, pgFileMapComparePath); - else - return; - - dummy_map = (page_map_entry *) pgut_malloc(sizeof(page_map_entry)); - - /* Iterate over files and look for corresponding pagemap if any */ - for (file_i = 0; file_i < parray_num(files); file_i++) - { - pgFile *file = (pgFile *) parray_get(files, file_i); - page_map_entry **res_map = NULL; - page_map_entry *map = NULL; - - /* - * For now nondata files are not entitled to have pagemap - * TODO It's possible to use ptrack for incremental backup of - * relation forks. Not implemented yet. - */ - if (!file->is_datafile || file->is_cfs) - continue; - - /* Consider only files from PGDATA (this check is probably redundant) */ - if (file->external_dir_num != 0) - continue; - - if (filemaps) - { - dummy_map->path = file->rel_path; - res_map = parray_bsearch(filemaps, dummy_map, pgFileMapComparePath); - map = (res_map) ? *res_map : NULL; - } - - /* Found map */ - if (map) - { - elog(VERBOSE, "Using ptrack pagemap for file \"%s\"", file->rel_path); - file->pagemap.bitmapsize = map->pagemapsize; - file->pagemap.bitmap = map->pagemap; - } - } - - free(dummy_map); -} diff --git a/src/restore.c b/src/restore.c deleted file mode 100644 index 5b1585024..000000000 --- a/src/restore.c +++ /dev/null @@ -1,2253 +0,0 @@ -/*------------------------------------------------------------------------- - * - * restore.c: restore DB cluster and archived WAL. - * - * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2022, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -#include "access/timeline.h" - -#include -#include - -#include "utils/thread.h" - -typedef struct -{ - parray *pgdata_files; - parray *dest_files; - pgBackup *dest_backup; - parray *dest_external_dirs; - parray *parent_chain; - parray *dbOid_exclude_list; - bool skip_external_dirs; - const char *to_root; - size_t restored_bytes; - bool use_bitmap; - IncrRestoreMode incremental_mode; - XLogRecPtr shift_lsn; /* used only in LSN incremental_mode */ - - /* - * Return value from the thread. - * 0 means there is no error, 1 - there is an error. - */ - int ret; -} restore_files_arg; - - -static void -print_recovery_settings(InstanceState *instanceState, FILE *fp, pgBackup *backup, - pgRestoreParams *params, pgRecoveryTarget *rt); -static void -print_standby_settings_common(FILE *fp, pgBackup *backup, pgRestoreParams *params); - -#if PG_VERSION_NUM >= 120000 -static void -update_recovery_options(InstanceState *instanceState, pgBackup *backup, - pgRestoreParams *params, pgRecoveryTarget *rt); -#else -static void -update_recovery_options_before_v12(InstanceState *instanceState, pgBackup *backup, - pgRestoreParams *params, pgRecoveryTarget *rt); -#endif - -static void create_recovery_conf(InstanceState *instanceState, time_t backup_id, - pgRecoveryTarget *rt, - pgBackup *backup, - pgRestoreParams *params); -static void *restore_files(void *arg); -static void set_orphan_status(parray *backups, pgBackup *parent_backup); - -static void restore_chain(pgBackup *dest_backup, parray *parent_chain, - parray *dbOid_exclude_list, pgRestoreParams *params, - const char *pgdata_path, bool no_sync, bool cleanup_pgdata, - bool backup_has_tblspc); - -/* - * Iterate over backup list to find all ancestors of the broken parent_backup - * and update their status to BACKUP_STATUS_ORPHAN - */ -static void -set_orphan_status(parray *backups, pgBackup *parent_backup) -{ - /* chain is intact, but at least one parent is invalid */ - int j; - - for (j = 0; j < parray_num(backups); j++) - { - - pgBackup *backup = (pgBackup *) parray_get(backups, j); - - if (is_parent(parent_backup->start_time, backup, false)) - { - if (backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE) - { - write_backup_status(backup, BACKUP_STATUS_ORPHAN, true); - - elog(WARNING, - "Backup %s is orphaned because his parent %s has status: %s", - backup_id_of(backup), - backup_id_of(parent_backup), - status2str(parent_backup->status)); - } - else - { - elog(WARNING, "Backup %s has parent %s with status: %s", - backup_id_of(backup), - backup_id_of(parent_backup), - status2str(parent_backup->status)); - } - } - } -} - -/* - * Entry point of pg_probackup RESTORE and VALIDATE subcommands. - */ -int -do_restore_or_validate(InstanceState *instanceState, time_t target_backup_id, pgRecoveryTarget *rt, - pgRestoreParams *params, bool no_sync) -{ - int i = 0; - int j = 0; - parray *backups = NULL; - pgBackup *tmp_backup = NULL; - pgBackup *current_backup = NULL; - pgBackup *dest_backup = NULL; - pgBackup *base_full_backup = NULL; - pgBackup *corrupted_backup = NULL; - char *action = params->is_restore ? "Restore":"Validate"; - parray *parent_chain = NULL; - parray *dbOid_exclude_list = NULL; - bool pgdata_is_empty = true; - bool cleanup_pgdata = false; - bool backup_has_tblspc = true; /* backup contain tablespace */ - XLogRecPtr shift_lsn = InvalidXLogRecPtr; - - if (instanceState == NULL) - elog(ERROR, "Required parameter not specified: --instance"); - - if (params->is_restore) - { - if (instance_config.pgdata == NULL) - elog(ERROR, "No postgres data directory specified.\n" - "Please specify it either using environment variable PGDATA or\n" - "command line option --pgdata (-D)"); - - /* Check if restore destination empty */ - if (!dir_is_empty(instance_config.pgdata, FIO_DB_HOST)) - { - /* if destination directory is empty, then incremental restore may be disabled */ - pgdata_is_empty = false; - - /* Check that remote system is NOT running and systemd id is the same as ours */ - if (params->incremental_mode != INCR_NONE) - { - DestDirIncrCompatibility rc; - const char *message = NULL; - bool ok_to_go = true; - - elog(INFO, "Running incremental restore into nonempty directory: \"%s\"", - instance_config.pgdata); - - rc = check_incremental_compatibility(instance_config.pgdata, - instance_config.system_identifier, - params->incremental_mode, - params->partial_db_list, - params->allow_partial_incremental); - if (rc == POSTMASTER_IS_RUNNING) - { - /* Even with force flag it is unwise to run - * incremental restore over running instance - */ - message = "Postmaster is running."; - ok_to_go = false; - } - else if (rc == SYSTEM_ID_MISMATCH) - { - /* - * In force mode it is possible to ignore system id mismatch - * by just wiping clean the destination directory. - */ - if (params->incremental_mode != INCR_NONE && params->force) - cleanup_pgdata = true; - else - { - message = "System ID mismatch."; - ok_to_go = false; - } - } - else if (rc == BACKUP_LABEL_EXISTS) - { - /* - * A big no-no for lsn-based incremental restore - * If there is backup label in PGDATA, then this cluster was probably - * restored from backup, but not started yet. Which means that values - * in pg_control are not synchronized with PGDATA and so we cannot use - * incremental restore in LSN mode, because it is relying on pg_control - * to calculate switchpoint. - */ - if (params->incremental_mode == INCR_LSN) - { - message = "Backup label exists. Cannot use incremental restore in LSN mode."; - ok_to_go = false; - } - } - else if (rc == DEST_IS_NOT_OK) - { - /* - * Something else is wrong. For example, postmaster.pid is mangled, - * so we cannot be sure that postmaster is running or not. - * It is better to just error out. - */ - message = "We cannot be sure about the database state."; - ok_to_go = false; - } else if (rc == PARTIAL_INCREMENTAL_FORBIDDEN) - { - message = "Partial incremental restore into non-empty PGDATA is forbidden."; - ok_to_go = false; - } - - if (!ok_to_go) - elog(ERROR, "Incremental restore is not allowed: %s", message); - } - else - elog(ERROR, "Restore destination is not empty: \"%s\"", - instance_config.pgdata); - } - } - - elog(LOG, "%s begin.", action); - - /* Get list of all backups sorted in order of descending start time */ - backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); - - /* Find backup range we should restore or validate. */ - while ((i < parray_num(backups)) && !dest_backup) - { - current_backup = (pgBackup *) parray_get(backups, i); - i++; - - /* Skip all backups which started after target backup */ - if (target_backup_id && current_backup->start_time > target_backup_id) - continue; - - /* - * [PGPRO-1164] If BACKUP_ID is not provided for restore command, - * we must find the first valid(!) backup. - - * If target_backup_id is not provided, we can be sure that - * PITR for restore or validate is requested. - * So we can assume that user is more interested in recovery to specific point - * in time and NOT interested in revalidation of invalid backups. - * So based on that assumptions we should choose only OK and DONE backups - * as candidates for validate and restore. - */ - - if (target_backup_id == INVALID_BACKUP_ID && - (current_backup->status != BACKUP_STATUS_OK && - current_backup->status != BACKUP_STATUS_DONE)) - { - elog(WARNING, "Skipping backup %s, because it has non-valid status: %s", - backup_id_of(current_backup), status2str(current_backup->status)); - continue; - } - - /* - * We found target backup. Check its status and - * ensure that it satisfies recovery target. - */ - if ((target_backup_id == current_backup->start_time - || target_backup_id == INVALID_BACKUP_ID)) - { - - /* backup is not ok, - * but in case of CORRUPT or ORPHAN revalidation is possible - * unless --no-validate is used, - * in other cases throw an error. - */ - // 1. validate - // 2. validate -i INVALID_ID <- allowed revalidate - // 3. restore -i INVALID_ID <- allowed revalidate and restore - // 4. restore <- impossible - // 5. restore --no-validate <- forbidden - if (current_backup->status != BACKUP_STATUS_OK && - current_backup->status != BACKUP_STATUS_DONE) - { - if ((current_backup->status == BACKUP_STATUS_ORPHAN || - current_backup->status == BACKUP_STATUS_CORRUPT || - current_backup->status == BACKUP_STATUS_RUNNING) - && (!params->no_validate || params->force)) - elog(WARNING, "Backup %s has status: %s", - backup_id_of(current_backup), status2str(current_backup->status)); - else - elog(ERROR, "Backup %s has status: %s", - backup_id_of(current_backup), status2str(current_backup->status)); - } - - if (rt->target_tli) - { - parray *timelines; - - // elog(LOG, "target timeline ID = %u", rt->target_tli); - /* Read timeline history files from archives */ - timelines = read_timeline_history(instanceState->instance_wal_subdir_path, - rt->target_tli, true); - - if (!timelines) - elog(ERROR, "Failed to get history file for target timeline %i", rt->target_tli); - - if (!satisfy_timeline(timelines, current_backup->tli, current_backup->stop_lsn)) - { - if (target_backup_id != INVALID_BACKUP_ID) - elog(ERROR, "Target backup %s does not satisfy target timeline", - base36enc(target_backup_id)); - else - /* Try to find another backup that satisfies target timeline */ - continue; - } - - parray_walk(timelines, pfree); - parray_free(timelines); - } - - if (!satisfy_recovery_target(current_backup, rt)) - { - if (target_backup_id != INVALID_BACKUP_ID) - elog(ERROR, "Requested backup %s does not satisfy restore options", - base36enc(target_backup_id)); - else - /* Try to find another backup that satisfies target options */ - continue; - } - - /* - * Backup is fine and satisfies all recovery options. - * Save it as dest_backup - */ - dest_backup = current_backup; - } - } - - /* TODO: Show latest possible target */ - if (dest_backup == NULL) - { - /* Failed to find target backup */ - if (target_backup_id) - elog(ERROR, "Requested backup %s is not found.", base36enc(target_backup_id)); - else - elog(ERROR, "Backup satisfying target options is not found."); - /* TODO: check if user asked PITR or just restore of latest backup */ - } - - /* If we already found dest_backup, look for full backup. */ - if (dest_backup->backup_mode == BACKUP_MODE_FULL) - base_full_backup = dest_backup; - else - { - int result; - - result = scan_parent_chain(dest_backup, &tmp_backup); - - if (result == ChainIsBroken) - { - /* chain is broken, determine missing backup ID - * and orphinize all his descendants - */ - const char *missing_backup_id; - time_t missing_backup_start_time; - - missing_backup_start_time = tmp_backup->parent_backup; - missing_backup_id = base36enc(tmp_backup->parent_backup); - - for (j = 0; j < parray_num(backups); j++) - { - pgBackup *backup = (pgBackup *) parray_get(backups, j); - - /* use parent backup start_time because he is missing - * and we must orphinize his descendants - */ - if (is_parent(missing_backup_start_time, backup, false)) - { - if (backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE) - { - write_backup_status(backup, BACKUP_STATUS_ORPHAN, true); - - elog(WARNING, "Backup %s is orphaned because his parent %s is missing", - backup_id_of(backup), missing_backup_id); - } - else - { - elog(WARNING, "Backup %s has missing parent %s", - backup_id_of(backup), missing_backup_id); - } - } - } - /* No point in doing futher */ - elog(ERROR, "%s of backup %s failed.", action, backup_id_of(dest_backup)); - } - else if (result == ChainIsInvalid) - { - /* chain is intact, but at least one parent is invalid */ - set_orphan_status(backups, tmp_backup); - tmp_backup = find_parent_full_backup(dest_backup); - - /* sanity */ - if (!tmp_backup) - elog(ERROR, "Parent full backup for the given backup %s was not found", - backup_id_of(dest_backup)); - } - - /* We have found full backup */ - base_full_backup = tmp_backup; - } - - if (base_full_backup == NULL) - elog(ERROR, "Full backup satisfying target options is not found."); - - /* - * Ensure that directories provided in tablespace mapping are valid - * i.e. empty or not exist. - */ - if (params->is_restore) - { - int rc = check_tablespace_mapping(dest_backup, - params->incremental_mode != INCR_NONE, params->force, - pgdata_is_empty, params->no_validate); - - /* backup contain no tablespaces */ - if (rc == NoTblspc) - backup_has_tblspc = false; - - if (params->incremental_mode != INCR_NONE && !cleanup_pgdata && pgdata_is_empty && (rc != NotEmptyTblspc)) - { - elog(INFO, "Destination directory and tablespace directories are empty, " - "disable incremental restore"); - params->incremental_mode = INCR_NONE; - } - - /* no point in checking external directories if their restore is not requested */ - //TODO: - // - make check_external_dir_mapping more like check_tablespace_mapping - // - honor force flag in case of incremental restore just like check_tablespace_mapping - if (!params->skip_external_dirs) - check_external_dir_mapping(dest_backup, params->incremental_mode != INCR_NONE); - } - - /* At this point we are sure that parent chain is whole - * so we can build separate array, containing all needed backups, - * to simplify validation and restore - */ - parent_chain = parray_new(); - - /* Take every backup that is a child of base_backup AND parent of dest_backup - * including base_backup and dest_backup - */ - - tmp_backup = dest_backup; - while (tmp_backup) - { - parray_append(parent_chain, tmp_backup); - tmp_backup = tmp_backup->parent_backup_link; - } - - /* - * Determine the shift-LSN - * Consider the example A: - * - * - * /----D----------F-> - * -A--B---C---*-------X-----> - * - * [A,F] - incremental chain - * X - the state of pgdata - * F - destination backup - * * - switch point - * - * When running incremental restore in 'lsn' mode, we get a bitmap of pages, - * whose LSN is less than shift-LSN (backup C stop_lsn). - * So when restoring file, we can skip restore of pages coming from - * A, B and C. - * Pages from D and F cannot be skipped due to incremental restore. - * - * Consider the example B: - * - * - * /----------X----> - * ----*---A---B---C--> - * - * [A,C] - incremental chain - * X - the state of pgdata - * C - destination backup - * * - switch point - * - * Incremental restore in shift mode IS NOT POSSIBLE in this case. - * We must be able to differentiate the scenario A and scenario B. - * - */ - if (params->is_restore && params->incremental_mode == INCR_LSN) - { - RedoParams redo; - parray *timelines = NULL; - get_redo(instance_config.pgdata, FIO_DB_HOST, &redo); - - if (redo.checksum_version == 0) - elog(ERROR, "Incremental restore in 'lsn' mode require " - "data_checksums to be enabled in destination data directory"); - - timelines = read_timeline_history(instanceState->instance_wal_subdir_path, - redo.tli, false); - - if (!timelines) - elog(WARNING, "Failed to get history for redo timeline %i, " - "multi-timeline incremental restore in 'lsn' mode is impossible", redo.tli); - - tmp_backup = dest_backup; - - while (tmp_backup) - { - /* Candidate, whose stop_lsn if less than shift LSN, is found */ - if (tmp_backup->stop_lsn < redo.lsn) - { - /* if candidate timeline is the same as redo TLI, - * then we are good to go. - */ - if (redo.tli == tmp_backup->tli) - { - elog(INFO, "Backup %s is chosen as shiftpoint, its Stop LSN will be used as shift LSN", - backup_id_of(tmp_backup)); - - shift_lsn = tmp_backup->stop_lsn; - break; - } - - if (!timelines) - { - elog(WARNING, "Redo timeline %i differs from target timeline %i, " - "in this case, to safely run incremental restore in 'lsn' mode, " - "the history file for timeline %i is mandatory", - redo.tli, tmp_backup->tli, redo.tli); - break; - } - - /* check whether the candidate tli is a part of redo TLI history */ - if (tliIsPartOfHistory(timelines, tmp_backup->tli)) - { - shift_lsn = tmp_backup->stop_lsn; - break; - } - else - elog(INFO, "Backup %s cannot be a shiftpoint, " - "because its tli %i is not in history of redo timeline %i", - backup_id_of(tmp_backup), tmp_backup->tli, redo.tli); - } - - tmp_backup = tmp_backup->parent_backup_link; - } - - if (XLogRecPtrIsInvalid(shift_lsn)) - elog(ERROR, "Cannot perform incremental restore of backup chain %s in 'lsn' mode, " - "because destination directory redo point %X/%X on tli %i is out of reach", - backup_id_of(dest_backup), - (uint32) (redo.lsn >> 32), (uint32) redo.lsn, redo.tli); - else - elog(INFO, "Destination directory redo point %X/%X on tli %i is " - "within reach of backup %s with Stop LSN %X/%X on tli %i", - (uint32) (redo.lsn >> 32), (uint32) redo.lsn, redo.tli, - backup_id_of(tmp_backup), - (uint32) (tmp_backup->stop_lsn >> 32), (uint32) tmp_backup->stop_lsn, - tmp_backup->tli); - - elog(INFO, "shift LSN: %X/%X", - (uint32) (shift_lsn >> 32), (uint32) shift_lsn); - - } - params->shift_lsn = shift_lsn; - - /* for validation or restore with enabled validation */ - if (!params->is_restore || !params->no_validate) - { - if (dest_backup->backup_mode != BACKUP_MODE_FULL) - elog(INFO, "Validating parents for backup %s", backup_id_of(dest_backup)); - - /* - * Validate backups from base_full_backup to dest_backup. - */ - for (i = parray_num(parent_chain) - 1; i >= 0; i--) - { - tmp_backup = (pgBackup *) parray_get(parent_chain, i); - - /* lock every backup in chain in read-only mode */ - if (!lock_backup(tmp_backup, true, false)) - { - elog(ERROR, "Cannot lock backup %s directory", - backup_id_of(tmp_backup)); - } - - /* validate datafiles only */ - pgBackupValidate(tmp_backup, params); - - /* After pgBackupValidate() only following backup - * states are possible: ERROR, RUNNING, CORRUPT and OK. - * Validate WAL only for OK, because there is no point - * in WAL validation for corrupted, errored or running backups. - */ - if (tmp_backup->status != BACKUP_STATUS_OK) - { - corrupted_backup = tmp_backup; - break; - } - /* We do not validate WAL files of intermediate backups - * It`s done to speed up restore - */ - } - - /* There is no point in wal validation of corrupted backups */ - // TODO: there should be a way for a user to request only(!) WAL validation - if (!corrupted_backup) - { - /* - * Validate corresponding WAL files. - * We pass base_full_backup timeline as last argument to this function, - * because it's needed to form the name of xlog file. - */ - validate_wal(dest_backup, instanceState->instance_wal_subdir_path, rt->target_time, - rt->target_xid, rt->target_lsn, - dest_backup->tli, instance_config.xlog_seg_size); - } - /* Orphanize every OK descendant of corrupted backup */ - else - set_orphan_status(backups, corrupted_backup); - } - - /* - * If dest backup is corrupted or was orphaned in previous check - * produce corresponding error message - */ - if (dest_backup->status == BACKUP_STATUS_OK || - dest_backup->status == BACKUP_STATUS_DONE) - { - if (params->no_validate) - elog(WARNING, "Backup %s is used without validation.", backup_id_of(dest_backup)); - else - elog(INFO, "Backup %s is valid.", backup_id_of(dest_backup)); - } - else if (dest_backup->status == BACKUP_STATUS_CORRUPT) - { - if (params->force) - elog(WARNING, "Backup %s is corrupt.", backup_id_of(dest_backup)); - else - elog(ERROR, "Backup %s is corrupt.", backup_id_of(dest_backup)); - } - else if (dest_backup->status == BACKUP_STATUS_ORPHAN) - { - if (params->force) - elog(WARNING, "Backup %s is orphan.", backup_id_of(dest_backup)); - else - elog(ERROR, "Backup %s is orphan.", backup_id_of(dest_backup)); - } - else - elog(ERROR, "Backup %s has status: %s", - backup_id_of(dest_backup), status2str(dest_backup->status)); - - /* We ensured that all backups are valid, now restore if required - */ - if (params->is_restore) - { - /* - * Get a list of dbOids to skip if user requested the partial restore. - * It is important that we do this after(!) validation so - * database_map can be trusted. - * NOTE: database_map could be missing for legal reasons, e.g. missing - * permissions on pg_database during `backup` and, as long as user - * do not request partial restore, it`s OK. - * - * If partial restore is requested and database map doesn't exist, - * throw an error. - */ - if (params->partial_db_list) - dbOid_exclude_list = get_dbOid_exclude_list(dest_backup, params->partial_db_list, - params->partial_restore_type); - - if (rt->lsn_string && - parse_server_version(dest_backup->server_version) < 100000) - elog(ERROR, "Backup %s was created for version %s which doesn't support recovery_target_lsn", - backup_id_of(dest_backup), - dest_backup->server_version); - - restore_chain(dest_backup, parent_chain, dbOid_exclude_list, params, - instance_config.pgdata, no_sync, cleanup_pgdata, backup_has_tblspc); - - //TODO rename and update comment - /* Create recovery.conf with given recovery target parameters */ - create_recovery_conf(instanceState, target_backup_id, rt, dest_backup, params); - } - - /* ssh connection to longer needed */ - fio_disconnect(); - - elog(INFO, "%s of backup %s completed.", - action, backup_id_of(dest_backup)); - - /* cleanup */ - parray_walk(backups, pgBackupFree); - parray_free(backups); - parray_free(parent_chain); - - return 0; -} - -/* - * Restore backup chain. - * Flag 'cleanup_pgdata' demands the removing of already existing content in PGDATA. - */ -void -restore_chain(pgBackup *dest_backup, parray *parent_chain, - parray *dbOid_exclude_list, pgRestoreParams *params, - const char *pgdata_path, bool no_sync, bool cleanup_pgdata, - bool backup_has_tblspc) -{ - int i; - char timestamp[100]; - parray *pgdata_files = NULL; - parray *dest_files = NULL; - parray *external_dirs = NULL; - /* arrays with meta info for multi threaded backup */ - pthread_t *threads; - restore_files_arg *threads_args; - bool restore_isok = true; - bool use_bitmap = true; - - /* fancy reporting */ - char pretty_dest_bytes[20]; - char pretty_total_bytes[20]; - size_t dest_bytes = 0; - size_t total_bytes = 0; - char pretty_time[20]; - time_t start_time, end_time; - - /* Preparations for actual restoring */ - time2iso(timestamp, lengthof(timestamp), dest_backup->start_time, false); - elog(INFO, "Restoring the database from backup at %s", timestamp); - - dest_files = get_backup_filelist(dest_backup, true); - - /* Lock backup chain and make sanity checks */ - for (i = parray_num(parent_chain) - 1; i >= 0; i--) - { - pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - - if (!lock_backup(backup, true, false)) - elog(ERROR, "Cannot lock backup %s", backup_id_of(backup)); - - if (backup->status != BACKUP_STATUS_OK && - backup->status != BACKUP_STATUS_DONE) - { - if (params->force) - elog(WARNING, "Backup %s is not valid, restore is forced", - backup_id_of(backup)); - else - elog(ERROR, "Backup %s cannot be restored because it is not valid", - backup_id_of(backup)); - } - - /* confirm block size compatibility */ - if (backup->block_size != BLCKSZ) - elog(ERROR, - "BLCKSZ(%d) is not compatible(%d expected)", - backup->block_size, BLCKSZ); - - if (backup->wal_block_size != XLOG_BLCKSZ) - elog(ERROR, - "XLOG_BLCKSZ(%d) is not compatible(%d expected)", - backup->wal_block_size, XLOG_BLCKSZ); - - /* populate backup filelist */ - if (backup->start_time != dest_backup->start_time) - backup->files = get_backup_filelist(backup, true); - else - backup->files = dest_files; - - /* - * this sorting is important, because we rely on it to find - * destination file in intermediate backups file lists - * using bsearch. - */ - parray_qsort(backup->files, pgFileCompareRelPathWithExternal); - } - - /* If dest backup version is older than 2.4.0, then bitmap optimization - * is impossible to use, because bitmap restore rely on pgFile.n_blocks, - * which is not always available in old backups. - */ - if (parse_program_version(dest_backup->program_version) < 20400) - { - use_bitmap = false; - - if (params->incremental_mode != INCR_NONE) - elog(ERROR, "Incremental restore is not possible for backups older than 2.3.0 version"); - } - - /* There is no point in bitmap restore, when restoring a single FULL backup, - * unless we are running incremental-lsn restore, then bitmap is mandatory. - */ - if (use_bitmap && parray_num(parent_chain) == 1) - { - if (params->incremental_mode == INCR_NONE) - use_bitmap = false; - else - use_bitmap = true; - } - - /* - * Restore dest_backup internal directories. - */ - create_data_directories(dest_files, instance_config.pgdata, - dest_backup->root_dir, backup_has_tblspc, - params->incremental_mode != INCR_NONE, - FIO_DB_HOST, params->waldir); - - /* - * Restore dest_backup external directories. - */ - if (dest_backup->external_dir_str && !params->skip_external_dirs) - { - external_dirs = make_external_directory_list(dest_backup->external_dir_str, true); - - if (!external_dirs) - elog(ERROR, "Failed to get a list of external directories"); - - if (parray_num(external_dirs) > 0) - elog(LOG, "Restore external directories"); - - for (i = 0; i < parray_num(external_dirs); i++) - fio_mkdir(parray_get(external_dirs, i), - DIR_PERMISSION, FIO_DB_HOST); - } - - /* - * Setup directory structure for external directories - */ - for (i = 0; i < parray_num(dest_files); i++) - { - pgFile *file = (pgFile *) parray_get(dest_files, i); - - if (S_ISDIR(file->mode)) - total_bytes += 4096; - - if (!params->skip_external_dirs && - file->external_dir_num && S_ISDIR(file->mode)) - { - char *external_path; - char dirpath[MAXPGPATH]; - - if (parray_num(external_dirs) < file->external_dir_num - 1) - elog(ERROR, "Inconsistent external directory backup metadata"); - - external_path = parray_get(external_dirs, file->external_dir_num - 1); - join_path_components(dirpath, external_path, file->rel_path); - - elog(LOG, "Create external directory \"%s\"", dirpath); - fio_mkdir(dirpath, file->mode, FIO_DB_HOST); - } - } - - /* setup threads */ - pfilearray_clear_locks(dest_files); - - /* Get list of files in destination directory and remove redundant files */ - if (params->incremental_mode != INCR_NONE || cleanup_pgdata) - { - pgdata_files = parray_new(); - - elog(INFO, "Extracting the content of destination directory for incremental restore"); - - time(&start_time); - fio_list_dir(pgdata_files, pgdata_path, false, true, false, false, true, 0); - - /* - * TODO: - * 1. Currently we are cleaning the tablespaces in check_tablespace_mapping and PGDATA here. - * It would be great to do all this work in one place. - * - * 2. In case of tablespace remapping we do not cleanup the old tablespace path, - * it is just left as it is. - * Lookup tests.incr_restore.IncrRestoreTest.test_incr_restore_with_tablespace_5 - */ - - /* get external dirs content */ - if (external_dirs) - { - for (i = 0; i < parray_num(external_dirs); i++) - { - char *external_path = parray_get(external_dirs, i); - parray *external_files = parray_new(); - - fio_list_dir(external_files, external_path, - false, true, false, false, true, i+1); - - parray_concat(pgdata_files, external_files); - parray_free(external_files); - } - } - - parray_qsort(pgdata_files, pgFileCompareRelPathWithExternalDesc); - - time(&end_time); - pretty_time_interval(difftime(end_time, start_time), - pretty_time, lengthof(pretty_time)); - - elog(INFO, "Destination directory content extracted, time elapsed: %s", - pretty_time); - - elog(INFO, "Removing redundant files in destination directory"); - time(&start_time); - for (i = 0; i < parray_num(pgdata_files); i++) - { - bool redundant = true; - pgFile *file = (pgFile *) parray_get(pgdata_files, i); - - if (parray_bsearch(dest_backup->files, file, pgFileCompareRelPathWithExternal)) - redundant = false; - - /* pg_filenode.map are always restored, because it's crc cannot be trusted */ - if (file->external_dir_num == 0 && - pg_strcasecmp(file->name, RELMAPPER_FILENAME) == 0) - redundant = true; - - /* do not delete the useful internal directories */ - if (S_ISDIR(file->mode) && !redundant) - continue; - - /* if file does not exists in destination list, then we can safely unlink it */ - if (cleanup_pgdata || redundant) - { - char fullpath[MAXPGPATH]; - - join_path_components(fullpath, pgdata_path, file->rel_path); - - fio_delete(file->mode, fullpath, FIO_DB_HOST); - elog(LOG, "Deleted file \"%s\"", fullpath); - - /* shrink pgdata list */ - pgFileFree(file); - parray_remove(pgdata_files, i); - i--; - } - } - - if (cleanup_pgdata) - { - /* Destination PGDATA and tablespaces were cleaned up, so it's the regular restore from this point */ - params->incremental_mode = INCR_NONE; - parray_free(pgdata_files); - pgdata_files = NULL; - } - - time(&end_time); - pretty_time_interval(difftime(end_time, start_time), - pretty_time, lengthof(pretty_time)); - - /* At this point PDATA do not contain files, that do not exists in dest backup file list */ - elog(INFO, "Redundant files are removed, time elapsed: %s", pretty_time); - } - - /* - * Close ssh connection belonging to the main thread - * to avoid the possibility of been killed for idleness - */ - fio_disconnect(); - - threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - threads_args = (restore_files_arg *) palloc(sizeof(restore_files_arg) * - num_threads); - if (dest_backup->stream) - dest_bytes = dest_backup->pgdata_bytes + dest_backup->wal_bytes; - else - dest_bytes = dest_backup->pgdata_bytes; - - pretty_size(dest_bytes, pretty_dest_bytes, lengthof(pretty_dest_bytes)); - elog(INFO, "Start restoring backup files. PGDATA size: %s", pretty_dest_bytes); - time(&start_time); - thread_interrupted = false; - - /* Restore files into target directory */ - for (i = 0; i < num_threads; i++) - { - restore_files_arg *arg = &(threads_args[i]); - - arg->dest_files = dest_files; - arg->pgdata_files = pgdata_files; - arg->dest_backup = dest_backup; - arg->dest_external_dirs = external_dirs; - arg->parent_chain = parent_chain; - arg->dbOid_exclude_list = dbOid_exclude_list; - arg->skip_external_dirs = params->skip_external_dirs; - arg->to_root = pgdata_path; - arg->use_bitmap = use_bitmap; - arg->incremental_mode = params->incremental_mode; - arg->shift_lsn = params->shift_lsn; - threads_args[i].restored_bytes = 0; - /* By default there are some error */ - threads_args[i].ret = 1; - - /* Useless message TODO: rewrite */ - elog(LOG, "Start thread %i", i + 1); - - pthread_create(&threads[i], NULL, restore_files, arg); - } - - /* Wait theads */ - for (i = 0; i < num_threads; i++) - { - pthread_join(threads[i], NULL); - if (threads_args[i].ret == 1) - restore_isok = false; - - total_bytes += threads_args[i].restored_bytes; - } - - time(&end_time); - pretty_time_interval(difftime(end_time, start_time), - pretty_time, lengthof(pretty_time)); - pretty_size(total_bytes, pretty_total_bytes, lengthof(pretty_total_bytes)); - - if (restore_isok) - { - elog(INFO, "Backup files are restored. Transfered bytes: %s, time elapsed: %s", - pretty_total_bytes, pretty_time); - - elog(INFO, "Restore incremental ratio (less is better): %.f%% (%s/%s)", - ((float) total_bytes / dest_bytes) * 100, - pretty_total_bytes, pretty_dest_bytes); - } - else - elog(ERROR, "Backup files restoring failed. Transfered bytes: %s, time elapsed: %s", - pretty_total_bytes, pretty_time); - - /* Close page header maps */ - for (i = parray_num(parent_chain) - 1; i >= 0; i--) - { - pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - cleanup_header_map(&(backup->hdr_map)); - } - - if (no_sync) - elog(WARNING, "Restored files are not synced to disk"); - else - { - elog(INFO, "Syncing restored files to disk"); - time(&start_time); - - for (i = 0; i < parray_num(dest_files); i++) - { - char to_fullpath[MAXPGPATH]; - pgFile *dest_file = (pgFile *) parray_get(dest_files, i); - - if (S_ISDIR(dest_file->mode)) - continue; - - /* skip external files if ordered to do so */ - if (dest_file->external_dir_num > 0 && - params->skip_external_dirs) - continue; - - /* construct fullpath */ - if (dest_file->external_dir_num == 0) - { - if (strcmp(PG_TABLESPACE_MAP_FILE, dest_file->rel_path) == 0) - continue; - if (strcmp(DATABASE_MAP, dest_file->rel_path) == 0) - continue; - join_path_components(to_fullpath, pgdata_path, dest_file->rel_path); - } - else - { - char *external_path = parray_get(external_dirs, dest_file->external_dir_num - 1); - join_path_components(to_fullpath, external_path, dest_file->rel_path); - } - - /* TODO: write test for case: file to be synced is missing */ - if (fio_sync(to_fullpath, FIO_DB_HOST) != 0) - elog(ERROR, "Failed to sync file \"%s\": %s", to_fullpath, strerror(errno)); - } - - time(&end_time); - pretty_time_interval(difftime(end_time, start_time), - pretty_time, lengthof(pretty_time)); - elog(INFO, "Restored backup files are synced, time elapsed: %s", pretty_time); - } - - /* cleanup */ - pfree(threads); - pfree(threads_args); - - if (external_dirs != NULL) - free_dir_list(external_dirs); - - if (pgdata_files) - { - parray_walk(pgdata_files, pgFileFree); - parray_free(pgdata_files); - } - - for (i = parray_num(parent_chain) - 1; i >= 0; i--) - { - pgBackup *backup = (pgBackup *) parray_get(parent_chain, i); - - parray_walk(backup->files, pgFileFree); - parray_free(backup->files); - } -} - -/* - * Restore files into $PGDATA. - */ -static void * -restore_files(void *arg) -{ - int i; - uint64 n_files; - char to_fullpath[MAXPGPATH]; - FILE *out = NULL; - char *out_buf = pgut_malloc(STDIO_BUFSIZE); - - restore_files_arg *arguments = (restore_files_arg *) arg; - - n_files = (unsigned long) parray_num(arguments->dest_files); - - for (i = 0; i < parray_num(arguments->dest_files); i++) - { - bool already_exists = false; - PageState *checksum_map = NULL; /* it should take ~1.5MB at most */ - datapagemap_t *lsn_map = NULL; /* it should take 16kB at most */ - char *errmsg = NULL; /* remote agent error message */ - pgFile *dest_file = (pgFile *) parray_get(arguments->dest_files, i); - - /* Directories were created before */ - if (S_ISDIR(dest_file->mode)) - continue; - - if (!pg_atomic_test_set_flag(&dest_file->lock)) - continue; - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during restore"); - - elog(progress ? INFO : LOG, "Progress: (%d/%lu). Restore file \"%s\"", - i + 1, n_files, dest_file->rel_path); - - /* Only files from pgdata can be skipped by partial restore */ - if (arguments->dbOid_exclude_list && dest_file->external_dir_num == 0) - { - /* Check if the file belongs to the database we exclude */ - if (parray_bsearch(arguments->dbOid_exclude_list, - &dest_file->dbOid, pgCompareOid)) - { - /* - * We cannot simply skip the file, because it may lead to - * failure during WAL redo; hence, create empty file. - */ - create_empty_file(FIO_BACKUP_HOST, - arguments->to_root, FIO_DB_HOST, dest_file); - - elog(LOG, "Skip file due to partial restore: \"%s\"", - dest_file->rel_path); - continue; - } - } - - /* Do not restore tablespace_map file */ - if ((dest_file->external_dir_num == 0) && - strcmp(PG_TABLESPACE_MAP_FILE, dest_file->rel_path) == 0) - { - elog(LOG, "Skip tablespace_map"); - continue; - } - - /* Do not restore database_map file */ - if ((dest_file->external_dir_num == 0) && - strcmp(DATABASE_MAP, dest_file->rel_path) == 0) - { - elog(LOG, "Skip database_map"); - continue; - } - - /* Do no restore external directory file if a user doesn't want */ - if (arguments->skip_external_dirs && dest_file->external_dir_num > 0) - continue; - - /* set fullpath of destination file */ - if (dest_file->external_dir_num == 0) - join_path_components(to_fullpath, arguments->to_root, dest_file->rel_path); - else - { - char *external_path = parray_get(arguments->dest_external_dirs, - dest_file->external_dir_num - 1); - join_path_components(to_fullpath, external_path, dest_file->rel_path); - } - - if (arguments->incremental_mode != INCR_NONE && - parray_bsearch(arguments->pgdata_files, dest_file, pgFileCompareRelPathWithExternalDesc)) - { - already_exists = true; - } - - /* - * Handle incremental restore case for data files. - * If file is already exists in pgdata, then - * we scan it block by block and get - * array of checksums for every page. - */ - if (already_exists && - dest_file->is_datafile && !dest_file->is_cfs && - dest_file->n_blocks > 0) - { - if (arguments->incremental_mode == INCR_LSN) - { - lsn_map = fio_get_lsn_map(to_fullpath, arguments->dest_backup->checksum_version, - dest_file->n_blocks, arguments->shift_lsn, - dest_file->segno * RELSEG_SIZE, FIO_DB_HOST); - } - else if (arguments->incremental_mode == INCR_CHECKSUM) - { - checksum_map = fio_get_checksum_map(to_fullpath, arguments->dest_backup->checksum_version, - dest_file->n_blocks, arguments->dest_backup->stop_lsn, - dest_file->segno * RELSEG_SIZE, FIO_DB_HOST); - } - } - - /* - * Open dest file and truncate it to zero, if destination - * file already exists and dest file size is zero, or - * if file do not exist - */ - if ((already_exists && dest_file->write_size == 0) || !already_exists) - out = fio_fopen(to_fullpath, PG_BINARY_W, FIO_DB_HOST); - /* - * If file already exists and dest size is not zero, - * then open it for reading and writing. - */ - else - out = fio_fopen(to_fullpath, PG_BINARY_R "+", FIO_DB_HOST); - - if (out == NULL) - elog(ERROR, "Cannot open restore target file \"%s\": %s", - to_fullpath, strerror(errno)); - - /* update file permission */ - if (fio_chmod(to_fullpath, dest_file->mode, FIO_DB_HOST) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, - strerror(errno)); - - if (!dest_file->is_datafile || dest_file->is_cfs) - elog(LOG, "Restoring non-data file: \"%s\"", to_fullpath); - else - elog(LOG, "Restoring data file: \"%s\"", to_fullpath); - - // If destination file is 0 sized, then just close it and go for the next - if (dest_file->write_size == 0) - goto done; - - /* Restore destination file */ - if (dest_file->is_datafile && !dest_file->is_cfs) - { - /* enable stdio buffering for local destination data file */ - if (!fio_is_remote_file(out)) - setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); - /* Destination file is data file */ - arguments->restored_bytes += restore_data_file(arguments->parent_chain, - dest_file, out, to_fullpath, - arguments->use_bitmap, checksum_map, - arguments->shift_lsn, lsn_map, true); - } - else - { - /* disable stdio buffering for local destination non-data file */ - if (!fio_is_remote_file(out)) - setvbuf(out, NULL, _IONBF, BUFSIZ); - /* Destination file is non-data file */ - arguments->restored_bytes += restore_non_data_file(arguments->parent_chain, - arguments->dest_backup, dest_file, out, to_fullpath, - already_exists); - } - -done: - /* Writing is asynchronous in case of restore in remote mode, so check the agent status */ - if (fio_check_error_file(out, &errmsg)) - elog(ERROR, "Cannot write to the remote file \"%s\": %s", to_fullpath, errmsg); - - /* close file */ - if (fio_fclose(out) != 0) - elog(ERROR, "Cannot close file \"%s\": %s", to_fullpath, - strerror(errno)); - - /* free pagemap used for restore optimization */ - pg_free(dest_file->pagemap.bitmap); - - if (lsn_map) - pg_free(lsn_map->bitmap); - - pg_free(lsn_map); - pg_free(checksum_map); - } - - free(out_buf); - - /* ssh connection to longer needed */ - fio_disconnect(); - - /* Data files restoring is successful */ - arguments->ret = 0; - - return NULL; -} - -/* - * Create recovery.conf (postgresql.auto.conf in case of PG12) - * with given recovery target parameters - */ -static void -create_recovery_conf(InstanceState *instanceState, time_t backup_id, - pgRecoveryTarget *rt, - pgBackup *backup, - pgRestoreParams *params) -{ - bool target_latest; - bool target_immediate; - bool restore_command_provided = false; - - if (instance_config.restore_command && - (pg_strcasecmp(instance_config.restore_command, "none") != 0)) - { - restore_command_provided = true; - } - - /* restore-target='latest' support */ - target_latest = rt->target_stop != NULL && - strcmp(rt->target_stop, "latest") == 0; - - target_immediate = rt->target_stop != NULL && - strcmp(rt->target_stop, "immediate") == 0; - - /* - * Note that setting restore_command alone interpreted - * as PITR with target - "until all available WAL is replayed". - * We do this because of the following case: - * The user is restoring STREAM backup as replica but - * also relies on WAL archive to catch-up with master. - * If restore_command is provided, then it should be - * added to recovery config. - * In this scenario, "would be" replica will replay - * all WAL segments available in WAL archive, after that - * it will try to connect to master via repprotocol. - * - * The risk is obvious, what if masters current state is - * in "the past" relatively to latest state in the archive? - * We will get a replica that is "in the future" to the master. - * We accept this risk because its probability is low. - */ - if (!backup->stream || rt->time_string || - rt->xid_string || rt->lsn_string || rt->target_name || - target_immediate || target_latest || restore_command_provided) - params->recovery_settings_mode = PITR_REQUESTED; - - elog(LOG, "----------------------------------------"); - -#if PG_VERSION_NUM >= 120000 - update_recovery_options(instanceState, backup, params, rt); -#else - update_recovery_options_before_v12(instanceState, backup, params, rt); -#endif -} - - -/* TODO get rid of using global variables: instance_config */ -static void -print_recovery_settings(InstanceState *instanceState, FILE *fp, pgBackup *backup, - pgRestoreParams *params, pgRecoveryTarget *rt) -{ - char restore_command_guc[16384]; - fio_fprintf(fp, "## recovery settings\n"); - - /* If restore_command is provided, use it. Otherwise construct it from scratch. */ - if (instance_config.restore_command && - (pg_strcasecmp(instance_config.restore_command, "none") != 0)) - sprintf(restore_command_guc, "%s", instance_config.restore_command); - else - { - /* default cmdline, ok for local restore */ - sprintf(restore_command_guc, "\"%s\" archive-get -B \"%s\" --instance \"%s\" " - "--wal-file-path=%%p --wal-file-name=%%f", - PROGRAM_FULL_PATH ? PROGRAM_FULL_PATH : PROGRAM_NAME, - /* TODO What is going on here? Why do we use catalog path as wal-file-path? */ - instanceState->catalog_state->catalog_path, instanceState->instance_name); - - /* append --remote-* parameters provided via --archive-* settings */ - if (instance_config.archive.host) - { - strcat(restore_command_guc, " --remote-host="); - strcat(restore_command_guc, instance_config.archive.host); - } - - if (instance_config.archive.port) - { - strcat(restore_command_guc, " --remote-port="); - strcat(restore_command_guc, instance_config.archive.port); - } - - if (instance_config.archive.user) - { - strcat(restore_command_guc, " --remote-user="); - strcat(restore_command_guc, instance_config.archive.user); - } - } - - /* - * We've already checked that only one of the four following mutually - * exclusive options is specified, so the order of calls is insignificant. - */ - if (rt->target_name) - fio_fprintf(fp, "recovery_target_name = '%s'\n", rt->target_name); - - if (rt->time_string) - fio_fprintf(fp, "recovery_target_time = '%s'\n", rt->time_string); - - if (rt->xid_string) - fio_fprintf(fp, "recovery_target_xid = '%s'\n", rt->xid_string); - - if (rt->lsn_string) - fio_fprintf(fp, "recovery_target_lsn = '%s'\n", rt->lsn_string); - - if (rt->target_stop && (strcmp(rt->target_stop, "immediate") == 0)) - fio_fprintf(fp, "recovery_target = '%s'\n", rt->target_stop); - - if (rt->inclusive_specified) - fio_fprintf(fp, "recovery_target_inclusive = '%s'\n", - rt->target_inclusive ? "true" : "false"); - - if (rt->target_tli) - fio_fprintf(fp, "recovery_target_timeline = '%u'\n", rt->target_tli); - else - { -#if PG_VERSION_NUM >= 120000 - - /* - * In PG12 default recovery target timeline was changed to 'latest', which - * is extremely risky. Explicitly preserve old behavior of recovering to current - * timneline for PG12. - */ - fio_fprintf(fp, "recovery_target_timeline = 'current'\n"); -#endif - } - - if (rt->target_action) - fio_fprintf(fp, "recovery_target_action = '%s'\n", rt->target_action); - else - /* default recovery_target_action is 'pause' */ - fio_fprintf(fp, "recovery_target_action = '%s'\n", "pause"); - - elog(LOG, "Setting restore_command to '%s'", restore_command_guc); - fio_fprintf(fp, "restore_command = '%s'\n", restore_command_guc); -} - -static void -print_standby_settings_common(FILE *fp, pgBackup *backup, pgRestoreParams *params) -{ - fio_fprintf(fp, "\n## standby settings\n"); - if (params->primary_conninfo) - fio_fprintf(fp, "primary_conninfo = '%s'\n", params->primary_conninfo); - else if (backup->primary_conninfo) - fio_fprintf(fp, "primary_conninfo = '%s'\n", backup->primary_conninfo); - - if (params->primary_slot_name != NULL) - fio_fprintf(fp, "primary_slot_name = '%s'\n", params->primary_slot_name); -} - -#if PG_VERSION_NUM < 120000 -static void -update_recovery_options_before_v12(InstanceState *instanceState, pgBackup *backup, - pgRestoreParams *params, pgRecoveryTarget *rt) -{ - FILE *fp; - char path[MAXPGPATH]; - - /* - * If PITR is not requested and instance is not restored as replica, - * then recovery.conf should not be created. - */ - if (params->recovery_settings_mode != PITR_REQUESTED && - !params->restore_as_replica) - { - return; - } - - elog(LOG, "update recovery settings in recovery.conf"); - join_path_components(path, instance_config.pgdata, "recovery.conf"); - - fp = fio_fopen(path, "w", FIO_DB_HOST); - if (fp == NULL) - elog(ERROR, "Cannot open file \"%s\": %s", path, - strerror(errno)); - - if (fio_chmod(path, FILE_PERMISSION, FIO_DB_HOST) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", path, strerror(errno)); - - fio_fprintf(fp, "# recovery.conf generated by pg_probackup %s\n", - PROGRAM_VERSION); - - if (params->recovery_settings_mode == PITR_REQUESTED) - print_recovery_settings(instanceState, fp, backup, params, rt); - - if (params->restore_as_replica) - { - print_standby_settings_common(fp, backup, params); - fio_fprintf(fp, "standby_mode = 'on'\n"); - } - - if (fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "Cannot write file \"%s\": %s", path, - strerror(errno)); -} -#endif - -/* - * Read postgresql.auto.conf, clean old recovery options, - * to avoid unexpected intersections. - * Write recovery options for this backup. - */ -#if PG_VERSION_NUM >= 120000 -static void -update_recovery_options(InstanceState *instanceState, pgBackup *backup, - pgRestoreParams *params, pgRecoveryTarget *rt) - -{ - char postgres_auto_path[MAXPGPATH]; - char postgres_auto_path_tmp[MAXPGPATH]; - char path[MAXPGPATH]; - FILE *fp = NULL; - FILE *fp_tmp = NULL; - struct stat st; - char current_time_str[100]; - /* postgresql.auto.conf parsing */ - char line[16384] = "\0"; - char *buf = NULL; - int buf_len = 0; - int buf_len_max = 16384; - - elog(LOG, "update recovery settings in postgresql.auto.conf"); - - time2iso(current_time_str, lengthof(current_time_str), current_time, false); - - join_path_components(postgres_auto_path, instance_config.pgdata, "postgresql.auto.conf"); - - if (fio_stat(postgres_auto_path, &st, false, FIO_DB_HOST) < 0) - { - /* file not found is not an error case */ - if (errno != ENOENT) - elog(ERROR, "Cannot stat file \"%s\": %s", postgres_auto_path, - strerror(errno)); - st.st_size = 0; - } - - /* Kludge for 0-sized postgresql.auto.conf file. TODO: make something more intelligent */ - if (st.st_size > 0) - { - fp = fio_open_stream(postgres_auto_path, FIO_DB_HOST); - if (fp == NULL) - elog(ERROR, "Cannot open \"%s\": %s", postgres_auto_path, strerror(errno)); - } - - sprintf(postgres_auto_path_tmp, "%s.tmp", postgres_auto_path); - fp_tmp = fio_fopen(postgres_auto_path_tmp, "w", FIO_DB_HOST); - if (fp_tmp == NULL) - elog(ERROR, "Cannot open \"%s\": %s", postgres_auto_path_tmp, strerror(errno)); - - while (fp && fgets(line, lengthof(line), fp)) - { - /* ignore "include 'probackup_recovery.conf'" directive */ - if (strstr(line, "include") && - strstr(line, "probackup_recovery.conf")) - { - continue; - } - - /* ignore already existing recovery options */ - if (strstr(line, "restore_command") || - strstr(line, "recovery_target")) - { - continue; - } - - if (!buf) - buf = pgut_malloc(buf_len_max); - - /* avoid buffer overflow */ - if ((buf_len + strlen(line)) >= buf_len_max) - { - buf_len_max += (buf_len + strlen(line)) *2; - buf = pgut_realloc(buf, buf_len_max); - } - - buf_len += snprintf(buf+buf_len, sizeof(line), "%s", line); - } - - /* close input postgresql.auto.conf */ - if (fp) - fio_close_stream(fp); - - /* Write data to postgresql.auto.conf.tmp */ - if (buf_len > 0 && - (fio_fwrite(fp_tmp, buf, buf_len) != buf_len)) - elog(ERROR, "Cannot write to \"%s\": %s", - postgres_auto_path_tmp, strerror(errno)); - - if (fio_fflush(fp_tmp) != 0 || - fio_fclose(fp_tmp)) - elog(ERROR, "Cannot write file \"%s\": %s", postgres_auto_path_tmp, - strerror(errno)); - pg_free(buf); - - if (fio_rename(postgres_auto_path_tmp, postgres_auto_path, FIO_DB_HOST) < 0) - elog(ERROR, "Cannot rename file \"%s\" to \"%s\": %s", - postgres_auto_path_tmp, postgres_auto_path, strerror(errno)); - - if (fio_chmod(postgres_auto_path, FILE_PERMISSION, FIO_DB_HOST) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", postgres_auto_path, strerror(errno)); - - if (params) - { - fp = fio_fopen(postgres_auto_path, "a", FIO_DB_HOST); - if (fp == NULL) - elog(ERROR, "Cannot open file \"%s\": %s", postgres_auto_path, - strerror(errno)); - - fio_fprintf(fp, "\n# recovery settings added by pg_probackup restore of backup %s at '%s'\n", - backup_id_of(backup), current_time_str); - - if (params->recovery_settings_mode == PITR_REQUESTED) - print_recovery_settings(instanceState, fp, backup, params, rt); - - if (params->restore_as_replica) - print_standby_settings_common(fp, backup, params); - - if (fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "Cannot write file \"%s\": %s", postgres_auto_path, - strerror(errno)); - - /* - * Create "recovery.signal" to mark this recovery as PITR for PostgreSQL. - * In older versions presense of recovery.conf alone was enough. - * To keep behaviour consistent with older versions, - * we are forced to create "recovery.signal" - * even when only restore_command is provided. - * Presense of "recovery.signal" by itself determine only - * one thing: do PostgreSQL must switch to a new timeline - * after successfull recovery or not? - */ - if (params->recovery_settings_mode == PITR_REQUESTED) - { - elog(LOG, "creating recovery.signal file"); - join_path_components(path, instance_config.pgdata, "recovery.signal"); - - fp = fio_fopen(path, PG_BINARY_W, FIO_DB_HOST); - if (fp == NULL) - elog(ERROR, "Cannot open file \"%s\": %s", path, - strerror(errno)); - - if (fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "Cannot write file \"%s\": %s", path, - strerror(errno)); - } - - if (params->restore_as_replica) - { - elog(LOG, "creating standby.signal file"); - join_path_components(path, instance_config.pgdata, "standby.signal"); - - fp = fio_fopen(path, PG_BINARY_W, FIO_DB_HOST); - if (fp == NULL) - elog(ERROR, "Cannot open file \"%s\": %s", path, - strerror(errno)); - - if (fio_fflush(fp) != 0 || - fio_fclose(fp)) - elog(ERROR, "Cannot write file \"%s\": %s", path, - strerror(errno)); - } - } -} -#endif - -/* - * Try to read a timeline's history file. - * - * If successful, return the list of component TLIs (the ancestor - * timelines followed by target timeline). If we cannot find the history file, - * assume that the timeline has no parents, and return a list of just the - * specified timeline ID. - * based on readTimeLineHistory() in timeline.c - */ -parray * -read_timeline_history(const char *arclog_path, TimeLineID targetTLI, bool strict) -{ - parray *result; - char path[MAXPGPATH]; - char fline[MAXPGPATH]; - FILE *fd = NULL; - TimeLineHistoryEntry *entry; - TimeLineHistoryEntry *last_timeline = NULL; - - /* Look for timeline history file in archlog_path */ - snprintf(path, lengthof(path), "%s/%08X.history", arclog_path, - targetTLI); - - /* Timeline 1 does not have a history file */ - if (targetTLI != 1) - { - fd = fopen(path, "rt"); - if (fd == NULL) - { - if (errno != ENOENT) - elog(ERROR, "Could not open file \"%s\": %s", path, - strerror(errno)); - - /* There is no history file for target timeline */ - if (strict) - elog(ERROR, "Recovery target timeline %u does not exist", - targetTLI); - else - return NULL; - } - } - - result = parray_new(); - - /* - * Parse the file... - */ - while (fd && fgets(fline, sizeof(fline), fd) != NULL) - { - char *ptr; - TimeLineID tli; - uint32 switchpoint_hi; - uint32 switchpoint_lo; - int nfields; - - for (ptr = fline; *ptr; ptr++) - { - if (!isspace((unsigned char) *ptr)) - break; - } - if (*ptr == '\0' || *ptr == '#') - continue; - - nfields = sscanf(fline, "%u\t%X/%X", &tli, &switchpoint_hi, &switchpoint_lo); - - if (nfields < 1) - { - /* expect a numeric timeline ID as first field of line */ - elog(ERROR, - "Syntax error in history file: %s. Expected a numeric timeline ID.", - fline); - } - if (nfields != 3) - elog(ERROR, - "Syntax error in history file: %s. Expected a transaction log switchpoint location.", - fline); - - if (last_timeline && tli <= last_timeline->tli) - elog(ERROR, - "Timeline IDs must be in increasing sequence."); - - entry = pgut_new(TimeLineHistoryEntry); - entry->tli = tli; - entry->end = ((uint64) switchpoint_hi << 32) | switchpoint_lo; - - last_timeline = entry; - /* Build list with newest item first */ - parray_insert(result, 0, entry); - - /* we ignore the remainder of each line */ - } - - if (fd && (ferror(fd))) - elog(ERROR, "Failed to read from file: \"%s\"", path); - - if (fd) - fclose(fd); - - if (last_timeline && targetTLI <= last_timeline->tli) - elog(ERROR, "Timeline IDs must be less than child timeline's ID."); - - /* History file is empty or corrupted */ - if (parray_num(result) == 0 && targetTLI != 1) - { - elog(WARNING, "History file is corrupted or missing: \"%s\"", path); - pg_free(result); - return NULL; - } - - /* append target timeline */ - entry = pgut_new(TimeLineHistoryEntry); - entry->tli = targetTLI; - /* LSN in target timeline is valid */ - entry->end = InvalidXLogRecPtr; - parray_insert(result, 0, entry); - - return result; -} - -/* TODO: do not ignore timelines. What if requested target located in different timeline? */ -bool -satisfy_recovery_target(const pgBackup *backup, const pgRecoveryTarget *rt) -{ - if (rt->xid_string) - return backup->recovery_xid <= rt->target_xid; - - if (rt->time_string) - return backup->recovery_time <= rt->target_time; - - if (rt->lsn_string) - return backup->stop_lsn <= rt->target_lsn; - - return true; -} - -/* TODO description */ -bool -satisfy_timeline(const parray *timelines, TimeLineID tli, XLogRecPtr lsn) -{ - int i; - - elog(VERBOSE, "satisfy_timeline() checking: tli = %X, lsn = %X/%X", - tli, (uint32) (lsn >> 32), (uint32) lsn); - for (i = 0; i < parray_num(timelines); i++) - { - TimeLineHistoryEntry *timeline; - - timeline = (TimeLineHistoryEntry *) parray_get(timelines, i); - elog(VERBOSE, "satisfy_timeline() check %i entry: timeline->tli = %X, timeline->end = %X/%X", - i, timeline->tli, (uint32) (timeline->end >> 32), (uint32) timeline->end); - if (tli == timeline->tli && - (XLogRecPtrIsInvalid(timeline->end) || - lsn <= timeline->end)) - return true; - } - return false; -} - -/* timelines represents a history of one particular timeline, - * we must determine whether a target tli is part of that history. - * - * /--------* - * ---------*--------------> - */ -bool -tliIsPartOfHistory(const parray *timelines, TimeLineID tli) -{ - int i; - - for (i = 0; i < parray_num(timelines); i++) - { - TimeLineHistoryEntry *timeline = (TimeLineHistoryEntry *) parray_get(timelines, i); - - if (tli == timeline->tli) - return true; - } - - return false; -} -/* - * Get recovery options in the string format, parse them - * and fill up the pgRecoveryTarget structure. - */ -pgRecoveryTarget * -parseRecoveryTargetOptions(const char *target_time, - const char *target_xid, - const char *target_inclusive, - TimeLineID target_tli, - const char *target_lsn, - const char *target_stop, - const char *target_name, - const char *target_action) -{ - bool dummy_bool; - /* - * count the number of the mutually exclusive options which may specify - * recovery target. If final value > 1, throw an error. - */ - int recovery_target_specified = 0; - pgRecoveryTarget *rt = pgut_new(pgRecoveryTarget); - - /* fill all options with default values */ - MemSet(rt, 0, sizeof(pgRecoveryTarget)); - - /* parse given options */ - if (target_time) - { - time_t dummy_time; - - recovery_target_specified++; - rt->time_string = target_time; - - if (parse_time(target_time, &dummy_time, false)) - rt->target_time = dummy_time; - else - elog(ERROR, "Invalid value for '--recovery-target-time' option '%s'", - target_time); - } - - if (target_xid) - { - TransactionId dummy_xid; - - recovery_target_specified++; - rt->xid_string = target_xid; - -#ifdef PGPRO_EE - if (parse_uint64(target_xid, &dummy_xid, 0)) -#else - if (parse_uint32(target_xid, &dummy_xid, 0)) -#endif - rt->target_xid = dummy_xid; - else - elog(ERROR, "Invalid value for '--recovery-target-xid' option '%s'", - target_xid); - } - - if (target_lsn) - { - XLogRecPtr dummy_lsn; - - recovery_target_specified++; - rt->lsn_string = target_lsn; - if (parse_lsn(target_lsn, &dummy_lsn)) - rt->target_lsn = dummy_lsn; - else - elog(ERROR, "Invalid value of '--recovery-target-lsn' option '%s'", - target_lsn); - } - - if (target_inclusive) - { - rt->inclusive_specified = true; - if (parse_bool(target_inclusive, &dummy_bool)) - rt->target_inclusive = dummy_bool; - else - elog(ERROR, "Invalid value for '--recovery-target-inclusive' option '%s'", - target_inclusive); - } - - rt->target_tli = target_tli; - if (target_stop) - { - if ((strcmp(target_stop, "immediate") != 0) - && (strcmp(target_stop, "latest") != 0)) - elog(ERROR, "Invalid value for '--recovery-target' option '%s'", - target_stop); - - recovery_target_specified++; - rt->target_stop = target_stop; - } - - if (target_name) - { - recovery_target_specified++; - rt->target_name = target_name; - } - - if (target_action) - { - if ((strcmp(target_action, "pause") != 0) - && (strcmp(target_action, "promote") != 0) - && (strcmp(target_action, "shutdown") != 0)) - elog(ERROR, "Invalid value for '--recovery-target-action' option '%s'", - target_action); - - rt->target_action = target_action; - } - - /* More than one mutually exclusive option was defined. */ - if (recovery_target_specified > 1) - elog(ERROR, "At most one of '--recovery-target', '--recovery-target-name', " - "'--recovery-target-time', '--recovery-target-xid' or " - "'--recovery-target-lsn' options can be specified"); - - /* - * If none of the options is defined, '--recovery-target-inclusive' option - * is meaningless. - */ - if (!(rt->xid_string || rt->time_string || rt->lsn_string) && - rt->target_inclusive) - elog(ERROR, "The '--recovery-target-inclusive' option can be applied only when " - "either of '--recovery-target-time', '--recovery-target-xid' or " - "'--recovery-target-lsn' options is specified"); - - /* If none of the options is defined, '--recovery-target-action' is meaningless */ - if (rt->target_action && recovery_target_specified == 0) - elog(ERROR, "The '--recovery-target-action' option can be applied only when " - "either of '--recovery-target', '--recovery-target-time', '--recovery-target-xid', " - "'--recovery-target-lsn' or '--recovery-target-name' options is specified"); - - /* TODO: sanity for recovery-target-timeline */ - - return rt; -} - -/* - * Return array of dbOids of databases that should not be restored - * Regardless of what option user used, db-include or db-exclude, - * we always convert it into exclude_list. - */ -parray * -get_dbOid_exclude_list(pgBackup *backup, parray *datname_list, - PartialRestoreType partial_restore_type) -{ - int i; - int j; -// pg_crc32 crc; - parray *database_map = NULL; - parray *dbOid_exclude_list = NULL; - pgFile *database_map_file = NULL; - char path[MAXPGPATH]; - char database_map_path[MAXPGPATH]; - parray *files = NULL; - - files = get_backup_filelist(backup, true); - - /* look for 'database_map' file in backup_content.control */ - for (i = 0; i < parray_num(files); i++) - { - pgFile *file = (pgFile *) parray_get(files, i); - - if ((file->external_dir_num == 0) && - strcmp(DATABASE_MAP, file->name) == 0) - { - database_map_file = file; - break; - } - } - - if (!database_map_file) - elog(ERROR, "Backup %s doesn't contain a database_map, partial restore is impossible.", - backup_id_of(backup)); - - join_path_components(path, backup->root_dir, DATABASE_DIR); - join_path_components(database_map_path, path, DATABASE_MAP); - - /* check database_map CRC */ -// crc = pgFileGetCRC(database_map_path, true, true, NULL, FIO_LOCAL_HOST); -// -// if (crc != database_map_file->crc) -// elog(ERROR, "Invalid CRC of backup file \"%s\" : %X. Expected %X", -// database_map_file->path, crc, database_map_file->crc); - - /* get database_map from file */ - database_map = read_database_map(backup); - - /* partial restore requested but database_map is missing */ - if (!database_map) - elog(ERROR, "Backup %s has empty or mangled database_map, partial restore is impossible.", - backup_id_of(backup)); - - /* - * So we have a list of datnames and a database_map for it. - * We must construct a list of dbOids to exclude. - */ - if (partial_restore_type == INCLUDE) - { - /* For 'include', keep dbOid of every datname NOT specified by user */ - for (i = 0; i < parray_num(datname_list); i++) - { - bool found_match = false; - char *datname = (char *) parray_get(datname_list, i); - - for (j = 0; j < parray_num(database_map); j++) - { - db_map_entry *db_entry = (db_map_entry *) parray_get(database_map, j); - - /* got a match */ - if (strcmp(db_entry->datname, datname) == 0) - { - found_match = true; - /* for db-include we must exclude db_entry from database_map */ - parray_remove(database_map, j); - j--; - } - } - /* If specified datname is not found in database_map, error out */ - if (!found_match) - elog(ERROR, "Failed to find a database '%s' in database_map of backup %s", - datname, backup_id_of(backup)); - } - - /* At this moment only databases to exclude are left in the map */ - for (j = 0; j < parray_num(database_map); j++) - { - db_map_entry *db_entry = (db_map_entry *) parray_get(database_map, j); - - if (!dbOid_exclude_list) - dbOid_exclude_list = parray_new(); - parray_append(dbOid_exclude_list, &db_entry->dbOid); - } - } - else if (partial_restore_type == EXCLUDE) - { - /* For exclude, job is easier - find dbOid for every specified datname */ - for (i = 0; i < parray_num(datname_list); i++) - { - bool found_match = false; - char *datname = (char *) parray_get(datname_list, i); - - for (j = 0; j < parray_num(database_map); j++) - { - db_map_entry *db_entry = (db_map_entry *) parray_get(database_map, j); - - /* got a match */ - if (strcmp(db_entry->datname, datname) == 0) - { - found_match = true; - /* for db-exclude we must add dbOid to exclude list */ - if (!dbOid_exclude_list) - dbOid_exclude_list = parray_new(); - parray_append(dbOid_exclude_list, &db_entry->dbOid); - } - } - /* If specified datname is not found in database_map, error out */ - if (!found_match) - elog(ERROR, "Failed to find a database '%s' in database_map of backup %s", - datname, backup_id_of(backup)); - } - } - - /* extra sanity: ensure that list is not empty */ - if (!dbOid_exclude_list || parray_num(dbOid_exclude_list) < 1) - elog(ERROR, "Failed to find a match in database_map of backup %s for partial restore", - backup_id_of(backup)); - - /* clean backup filelist */ - if (files) - { - parray_walk(files, pgFileFree); - parray_free(files); - } - - /* sort dbOid array in ASC order */ - parray_qsort(dbOid_exclude_list, pgCompareOid); - - return dbOid_exclude_list; -} - -/* Check that instance is suitable for incremental restore - * Depending on type of incremental restore requirements are differs. - * - * TODO: add PG_CONTROL_IS_MISSING - */ -DestDirIncrCompatibility -check_incremental_compatibility(const char *pgdata, uint64 system_identifier, - IncrRestoreMode incremental_mode, - parray *partial_db_list, - bool allow_partial_incremental) -{ - uint64 system_id_pgdata; - bool system_id_match = false; - bool success = true; - bool postmaster_is_up = false; - bool backup_label_exists = false; - pid_t pid; - char backup_label[MAXPGPATH]; - - /* check postmaster pid */ - pid = fio_check_postmaster(pgdata, FIO_DB_HOST); - - if (pid == 1) /* postmaster.pid is mangled */ - { - char pid_file[MAXPGPATH]; - - join_path_components(pid_file, pgdata, "postmaster.pid"); - elog(WARNING, "Pid file \"%s\" is mangled, cannot determine whether postmaster is running or not", - pid_file); - success = false; - } - else if (pid > 1) /* postmaster is up */ - { - elog(WARNING, "Postmaster with pid %u is running in destination directory \"%s\"", - pid, pgdata); - success = false; - postmaster_is_up = true; - } - - /* check that PG_VERSION is the same */ - - /* slurp pg_control and check that system ID is the same - * check that instance is not running - * if lsn_based, check that there is no backup_label files is around AND - * get redo point lsn from destination pg_control. - - * It is really important to be sure that pg_control is in cohesion with - * data files content, because based on pg_control information we will - * choose a backup suitable for lsn based incremental restore. - */ - elog(LOG, "Trying to read pg_control file in destination directory"); - - system_id_pgdata = get_system_identifier(pgdata, FIO_DB_HOST, false); - - if (system_id_pgdata == instance_config.system_identifier) - system_id_match = true; - else - elog(WARNING, "Backup catalog was initialized for system id %lu, " - "but destination directory system id is %lu", - system_identifier, system_id_pgdata); - - /* - * TODO: maybe there should be some other signs, pointing to pg_control - * desynchronization with cluster state. - */ - if (incremental_mode == INCR_LSN) - { - join_path_components(backup_label, pgdata, "backup_label"); - if (fio_access(backup_label, F_OK, FIO_DB_HOST) == 0) - { - elog(WARNING, "Destination directory contains \"backup_control\" file. " - "This does NOT mean that you should delete this file and retry, only that " - "incremental restore in 'lsn' mode may produce incorrect result, when applied " - "to cluster with pg_control not synchronized with cluster state." - "Consider to use incremental restore in 'checksum' mode"); - success = false; - backup_label_exists = true; - } - } - - if (postmaster_is_up) - return POSTMASTER_IS_RUNNING; - - /* PG_CONTROL MISSING */ - - /* PG_CONTROL unreadable */ - - if (!system_id_match) - return SYSTEM_ID_MISMATCH; - - if (backup_label_exists) - return BACKUP_LABEL_EXISTS; - - if (partial_db_list && !allow_partial_incremental) - return PARTIAL_INCREMENTAL_FORBIDDEN; - /* some other error condition */ - if (!success) - return DEST_IS_NOT_OK; - - return DEST_OK; -} diff --git a/src/show.c b/src/show.c deleted file mode 100644 index 86a122698..000000000 --- a/src/show.c +++ /dev/null @@ -1,1176 +0,0 @@ -/*------------------------------------------------------------------------- - * - * show.c: show backup information. - * - * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2022, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -#include -#include -#include -#include - -#include "utils/json.h" - -#define half_rounded(x) (((x) + ((x) < 0 ? 0 : 1)) / 2) - -/* struct to align fields printed in plain format */ -typedef struct ShowBackendRow -{ - const char *instance; - const char *version; - char backup_id[20]; - char recovery_time[100]; - const char *mode; - const char *wal_mode; - char tli[20]; - char duration[20]; - char data_bytes[20]; - char wal_bytes[20]; - char zratio[20]; - char start_lsn[20]; - char stop_lsn[20]; - const char *status; -} ShowBackendRow; - -/* struct to align fields printed in plain format */ -typedef struct ShowArchiveRow -{ - char tli[20]; - char parent_tli[20]; - char switchpoint[20]; - char min_segno[MAXFNAMELEN]; - char max_segno[MAXFNAMELEN]; - char n_segments[20]; - char size[20]; - char zratio[20]; - const char *status; - char n_backups[20]; -} ShowArchiveRow; - -static void show_instance_start(void); -static void show_instance_end(void); -static void show_instance(InstanceState *instanceState, time_t requested_backup_id, bool show_name); -static void print_backup_json_object(PQExpBuffer buf, pgBackup *backup); -static int show_backup(InstanceState *instanceState, time_t requested_backup_id); - -static void show_instance_plain(const char *instance_name, parray *backup_list, bool show_name); -static void show_instance_json(const char *instance_name, parray *backup_list); - -static void show_instance_archive(InstanceState *instanceState, InstanceConfig *instance); -static void show_archive_plain(const char *instance_name, uint32 xlog_seg_size, - parray *timelines_list, bool show_name); -static void show_archive_json(const char *instance_name, uint32 xlog_seg_size, - parray *tli_list); - -static PQExpBufferData show_buf; -static bool first_instance = true; -static int32 json_level = 0; - -static const char* lc_env_locale; -typedef enum { - LOCALE_C, // Used for formatting output to unify the dot-based floating point representation - LOCALE_ENV // Default environment locale -} output_numeric_locale; - -#ifdef HAVE_USELOCALE -static locale_t env_locale, c_locale; -#endif -void memorize_environment_locale() { - lc_env_locale = (const char *)getenv("LC_NUMERIC"); - lc_env_locale = lc_env_locale != NULL ? lc_env_locale : "C"; -#ifdef HAVE_USELOCALE - env_locale = newlocale(LC_NUMERIC_MASK, lc_env_locale, (locale_t)0); - c_locale = newlocale(LC_NUMERIC_MASK, "C", (locale_t)0); -#else -#ifdef HAVE__CONFIGTHREADLOCALE - _configthreadlocale(_ENABLE_PER_THREAD_LOCALE); -#endif -#endif -} - -void free_environment_locale() { -#ifdef HAVE_USELOCALE - freelocale(env_locale); - freelocale(c_locale); -#endif -} - -static void set_output_numeric_locale(output_numeric_locale loc) { -#ifdef HAVE_USELOCALE - uselocale(loc == LOCALE_C ? c_locale : env_locale); -#else - setlocale(LC_NUMERIC, loc == LOCALE_C ? "C" : lc_env_locale); -#endif -} - -/* - * Entry point of pg_probackup SHOW subcommand. - */ -int -do_show(CatalogState *catalogState, InstanceState *instanceState, - time_t requested_backup_id, bool show_archive) -{ - int i; - - if (instanceState == NULL && - requested_backup_id != INVALID_BACKUP_ID) - elog(ERROR, "You must specify --instance to use (-i, --backup-id) option"); - - if (show_archive && - requested_backup_id != INVALID_BACKUP_ID) - elog(ERROR, "You cannot specify --archive and (-i, --backup-id) options together"); - - /* - * if instance is not specified, - * show information about all instances in this backup catalog - */ - if (instanceState == NULL) - { - parray *instances = catalog_get_instance_list(catalogState); - - show_instance_start(); - for (i = 0; i < parray_num(instances); i++) - { - instanceState = parray_get(instances, i); - - if (interrupted) - elog(ERROR, "Interrupted during show"); - - if (show_archive) - show_instance_archive(instanceState, instanceState->config); - else - show_instance(instanceState, INVALID_BACKUP_ID, true); - } - show_instance_end(); - - return 0; - } - /* always use */ - else if (show_format == SHOW_JSON || - requested_backup_id == INVALID_BACKUP_ID) - { - show_instance_start(); - - if (show_archive) - { - InstanceConfig *instance = readInstanceConfigFile(instanceState); - show_instance_archive(instanceState, instance); - } - else - show_instance(instanceState, requested_backup_id, false); - - show_instance_end(); - - return 0; - } - else - { - if (show_archive) - { - InstanceConfig *instance = readInstanceConfigFile(instanceState); - show_instance_archive(instanceState, instance); - } - else - show_backup(instanceState, requested_backup_id); - - return 0; - } -} - -void -pretty_size(int64 size, char *buf, size_t len) -{ - int64 limit = 10 * 1024; - int64 limit2 = limit * 2 - 1; - - /* minus means the size is invalid */ -// if (size < 0) -// { -// strncpy(buf, "----", len); -// return; -// } - - if (size <= 0) - { - strncpy(buf, "0", len); - return; - } - - if (size < limit) - snprintf(buf, len, "%dB", (int) size); - else - { - size >>= 9; - if (size < limit2) - snprintf(buf, len, "%dkB", (int) half_rounded(size)); - else - { - size >>= 10; - if (size < limit2) - snprintf(buf, len, "%dMB", (int) half_rounded(size)); - else - { - size >>= 10; - if (size < limit2) - snprintf(buf, len, "%dGB", (int) half_rounded(size)); - else - { - size >>= 10; - snprintf(buf, len, "%dTB", (int) half_rounded(size)); - } - } - } - } -} - -void -pretty_time_interval(double time, char *buf, size_t len) -{ - int num_seconds = 0; - int milliseconds = 0; - int seconds = 0; - int minutes = 0; - int hours = 0; - int days = 0; - - num_seconds = (int) time; - - if (time <= 0) - { - strncpy(buf, "0", len); - return; - } - - days = num_seconds / (24 * 3600); - num_seconds %= (24 * 3600); - - hours = num_seconds / 3600; - num_seconds %= 3600; - - minutes = num_seconds / 60; - num_seconds %= 60; - - seconds = num_seconds; - milliseconds = (int)((time - (int) time) * 1000.0); - - if (days > 0) - { - snprintf(buf, len, "%dd:%dh", days, hours); - return; - } - - if (hours > 0) - { - snprintf(buf, len, "%dh:%dm", hours, minutes); - return; - } - - if (minutes > 0) - { - snprintf(buf, len, "%dm:%ds", minutes, seconds); - return; - } - - if (seconds > 0) - { - if (milliseconds > 0) - snprintf(buf, len, "%ds:%dms", seconds, milliseconds); - else - snprintf(buf, len, "%ds", seconds); - return; - } - - snprintf(buf, len, "%dms", milliseconds); - return; -} - -/* - * Initialize instance visualization. - */ -static void -show_instance_start(void) -{ - initPQExpBuffer(&show_buf); - - if (show_format == SHOW_PLAIN) - return; - - first_instance = true; - json_level = 0; - - appendPQExpBufferChar(&show_buf, '['); - json_level++; -} - -/* - * Finalize instance visualization. - */ -static void -show_instance_end(void) -{ - if (show_format == SHOW_JSON) - appendPQExpBufferStr(&show_buf, "\n]\n"); - - fputs(show_buf.data, stdout); - termPQExpBuffer(&show_buf); -} - -/* - * Show brief meta information about all backups in the backup instance. - */ -static void -show_instance(InstanceState *instanceState, time_t requested_backup_id, bool show_name) -{ - parray *backup_list; - - backup_list = catalog_get_backup_list(instanceState, requested_backup_id); - - if (show_format == SHOW_PLAIN) - show_instance_plain(instanceState->instance_name, backup_list, show_name); - else if (show_format == SHOW_JSON) - show_instance_json(instanceState->instance_name, backup_list); - else - elog(ERROR, "Invalid show format %d", (int) show_format); - - /* cleanup */ - parray_walk(backup_list, pgBackupFree); - parray_free(backup_list); -} - -/* helper routine to print backup info as json object */ -static void -print_backup_json_object(PQExpBuffer buf, pgBackup *backup) -{ - TimeLineID parent_tli = 0; - char timestamp[100] = "----"; - char lsn[20]; - - json_add(buf, JT_BEGIN_OBJECT, &json_level); - - json_add_value(buf, "id", backup_id_of(backup), json_level, - true); - - if (backup->parent_backup != 0) - json_add_value(buf, "parent-backup-id", - base36enc(backup->parent_backup), json_level, true); - - json_add_value(buf, "backup-mode", pgBackupGetBackupMode(backup, false), - json_level, true); - - json_add_value(buf, "wal", backup->stream ? "STREAM": "ARCHIVE", - json_level, true); - - json_add_value(buf, "compress-alg", - deparse_compress_alg(backup->compress_alg), json_level, - true); - - json_add_key(buf, "compress-level", json_level); - appendPQExpBuffer(buf, "%d", backup->compress_level); - - json_add_value(buf, "from-replica", - backup->from_replica ? "true" : "false", json_level, - true); - - json_add_key(buf, "block-size", json_level); - appendPQExpBuffer(buf, "%u", backup->block_size); - - json_add_key(buf, "xlog-block-size", json_level); - appendPQExpBuffer(buf, "%u", backup->wal_block_size); - - json_add_key(buf, "checksum-version", json_level); - appendPQExpBuffer(buf, "%u", backup->checksum_version); - - json_add_value(buf, "program-version", backup->program_version, - json_level, true); - json_add_value(buf, "server-version", backup->server_version, - json_level, true); - - json_add_key(buf, "current-tli", json_level); - appendPQExpBuffer(buf, "%d", backup->tli); - - json_add_key(buf, "parent-tli", json_level); - - /* Only incremental backup can have Parent TLI */ - if (backup->parent_backup_link) - parent_tli = backup->parent_backup_link->tli; - - appendPQExpBuffer(buf, "%u", parent_tli); - - snprintf(lsn, lengthof(lsn), "%X/%X", - (uint32) (backup->start_lsn >> 32), (uint32) backup->start_lsn); - json_add_value(buf, "start-lsn", lsn, json_level, true); - - snprintf(lsn, lengthof(lsn), "%X/%X", - (uint32) (backup->stop_lsn >> 32), (uint32) backup->stop_lsn); - json_add_value(buf, "stop-lsn", lsn, json_level, true); - - time2iso(timestamp, lengthof(timestamp), backup->start_time, false); - json_add_value(buf, "start-time", timestamp, json_level, true); - - if (backup->end_time) - { - time2iso(timestamp, lengthof(timestamp), backup->end_time, false); - json_add_value(buf, "end-time", timestamp, json_level, true); - } - - json_add_key(buf, "recovery-xid", json_level); - appendPQExpBuffer(buf, XID_FMT, backup->recovery_xid); - - if (backup->recovery_time > 0) - { - time2iso(timestamp, lengthof(timestamp), backup->recovery_time, false); - json_add_value(buf, "recovery-time", timestamp, json_level, true); - } - - if (backup->expire_time > 0) - { - time2iso(timestamp, lengthof(timestamp), backup->expire_time, false); - json_add_value(buf, "expire-time", timestamp, json_level, true); - } - - if (backup->data_bytes != BYTES_INVALID) - { - json_add_key(buf, "data-bytes", json_level); - appendPQExpBuffer(buf, INT64_FORMAT, backup->data_bytes); - } - - if (backup->wal_bytes != BYTES_INVALID) - { - json_add_key(buf, "wal-bytes", json_level); - appendPQExpBuffer(buf, INT64_FORMAT, backup->wal_bytes); - } - - if (backup->uncompressed_bytes >= 0) - { - json_add_key(buf, "uncompressed-bytes", json_level); - appendPQExpBuffer(buf, INT64_FORMAT, backup->uncompressed_bytes); - } - - if (backup->pgdata_bytes >= 0) - { - json_add_key(buf, "pgdata-bytes", json_level); - appendPQExpBuffer(buf, INT64_FORMAT, backup->pgdata_bytes); - } - - if (backup->primary_conninfo) - json_add_value(buf, "primary_conninfo", backup->primary_conninfo, - json_level, true); - - if (backup->external_dir_str) - json_add_value(buf, "external-dirs", backup->external_dir_str, - json_level, true); - - json_add_value(buf, "status", status2str(backup->status), json_level, - true); - - if (backup->note) - json_add_value(buf, "note", backup->note, - json_level, true); - - if (backup->content_crc != 0) - { - json_add_key(buf, "content-crc", json_level); - appendPQExpBuffer(buf, "%u", backup->content_crc); - } - - json_add(buf, JT_END_OBJECT, &json_level); -} - -/* - * Show detailed meta information about specified backup. - */ -static int -show_backup(InstanceState *instanceState, time_t requested_backup_id) -{ - int i; - pgBackup *backup = NULL; - parray *backups; - - //TODO pass requested_backup_id to the function - backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); - - /* Find requested backup */ - for (i = 0; i < parray_num(backups); i++) - { - pgBackup *tmp_backup = (pgBackup *) parray_get(backups, i); - - /* found target */ - if (tmp_backup->start_time == requested_backup_id) - { - backup = tmp_backup; - break; - } - } - - if (backup == NULL) - { - // TODO for 3.0: we should ERROR out here. - elog(INFO, "Requested backup \"%s\" is not found.", - /* We do not need free base36enc's result, we exit anyway */ - base36enc(requested_backup_id)); - parray_walk(backups, pgBackupFree); - parray_free(backups); - /* This is not error */ - return 0; - } - - if (show_format == SHOW_PLAIN) - pgBackupWriteControl(stdout, backup, false); - else - elog(ERROR, "Invalid show format %d", (int) show_format); - - /* cleanup */ - parray_walk(backups, pgBackupFree); - parray_free(backups); - - return 0; -} - -/* - * Show instance backups in plain format. - */ -static void -show_instance_plain(const char *instance_name, parray *backup_list, bool show_name) -{ -#define SHOW_FIELDS_COUNT 14 - int i; - const char *names[SHOW_FIELDS_COUNT] = - { "Instance", "Version", "ID", "Recovery Time", - "Mode", "WAL Mode", "TLI", "Time", "Data", "WAL", - "Zratio", "Start LSN", "Stop LSN", "Status" }; - const char *field_formats[SHOW_FIELDS_COUNT] = - { " %-*s ", " %-*s ", " %-*s ", " %-*s ", - " %-*s ", " %-*s ", " %-*s ", " %*s ", " %*s ", " %*s ", - " %*s ", " %-*s ", " %-*s ", " %-*s "}; - uint32 widths[SHOW_FIELDS_COUNT]; - uint32 widths_sum = 0; - ShowBackendRow *rows; - TimeLineID parent_tli = 0; - - // Since we've been printing a table, set LC_NUMERIC to its default environment value - set_output_numeric_locale(LOCALE_ENV); - - for (i = 0; i < SHOW_FIELDS_COUNT; i++) - widths[i] = strlen(names[i]); - - rows = (ShowBackendRow *) palloc(parray_num(backup_list) * - sizeof(ShowBackendRow)); - - /* - * Fill row values and calculate maximum width of each field. - */ - for (i = 0; i < parray_num(backup_list); i++) - { - pgBackup *backup = parray_get(backup_list, i); - ShowBackendRow *row = &rows[i]; - int cur = 0; - float zratio = 1; - - /* Instance */ - row->instance = instance_name; - widths[cur] = Max(widths[cur], strlen(row->instance)); - cur++; - - /* Version */ - row->version = backup->server_version[0] ? - backup->server_version : "----"; - widths[cur] = Max(widths[cur], strlen(row->version)); - cur++; - - /* ID */ - snprintf(row->backup_id, lengthof(row->backup_id), "%s", - backup_id_of(backup)); - widths[cur] = Max(widths[cur], strlen(row->backup_id)); - cur++; - - /* Recovery Time */ - if (backup->recovery_time != (time_t) 0) - time2iso(row->recovery_time, lengthof(row->recovery_time), - backup->recovery_time, false); - else - strlcpy(row->recovery_time, "----", sizeof(row->recovery_time)); - widths[cur] = Max(widths[cur], strlen(row->recovery_time)); - cur++; - - /* Mode */ - row->mode = pgBackupGetBackupMode(backup, show_color); - widths[cur] = Max(widths[cur], strlen(row->mode) - (show_color ? TC_LEN : 0)); - cur++; - - /* WAL mode*/ - row->wal_mode = backup->stream ? "STREAM": "ARCHIVE"; - widths[cur] = Max(widths[cur], strlen(row->wal_mode)); - cur++; - - /* Current/Parent TLI */ - if (backup->parent_backup_link != NULL) - parent_tli = backup->parent_backup_link->tli; - - snprintf(row->tli, lengthof(row->tli), "%u/%u", - backup->tli, - backup->backup_mode == BACKUP_MODE_FULL ? 0 : parent_tli); - widths[cur] = Max(widths[cur], strlen(row->tli)); - cur++; - - /* Time */ - if (backup->status == BACKUP_STATUS_RUNNING) - pretty_time_interval(difftime(current_time, backup->start_time), - row->duration, lengthof(row->duration)); - else if (backup->merge_time != (time_t) 0) - pretty_time_interval(difftime(backup->end_time, backup->merge_time), - row->duration, lengthof(row->duration)); - else if (backup->end_time != (time_t) 0) - pretty_time_interval(difftime(backup->end_time, backup->start_time), - row->duration, lengthof(row->duration)); - else - strlcpy(row->duration, "----", sizeof(row->duration)); - widths[cur] = Max(widths[cur], strlen(row->duration)); - cur++; - - /* Data */ - pretty_size(backup->data_bytes, row->data_bytes, - lengthof(row->data_bytes)); - widths[cur] = Max(widths[cur], strlen(row->data_bytes)); - cur++; - - /* WAL */ - pretty_size(backup->wal_bytes, row->wal_bytes, - lengthof(row->wal_bytes)); - widths[cur] = Max(widths[cur], strlen(row->wal_bytes)); - cur++; - - /* Zratio (compression ratio) */ - if (backup->uncompressed_bytes != BYTES_INVALID && - (backup->uncompressed_bytes > 0 && backup->data_bytes > 0)) - { - zratio = (float)backup->uncompressed_bytes / (backup->data_bytes); - snprintf(row->zratio, lengthof(row->zratio), "%.2f", zratio); - } - else - snprintf(row->zratio, lengthof(row->zratio), "%.2f", zratio); - - widths[cur] = Max(widths[cur], strlen(row->zratio)); - cur++; - - /* Start LSN */ - snprintf(row->start_lsn, lengthof(row->start_lsn), "%X/%X", - (uint32) (backup->start_lsn >> 32), - (uint32) backup->start_lsn); - widths[cur] = Max(widths[cur], strlen(row->start_lsn)); - cur++; - - /* Stop LSN */ - snprintf(row->stop_lsn, lengthof(row->stop_lsn), "%X/%X", - (uint32) (backup->stop_lsn >> 32), - (uint32) backup->stop_lsn); - widths[cur] = Max(widths[cur], strlen(row->stop_lsn)); - cur++; - - /* Status */ - row->status = show_color ? status2str_color(backup->status) : status2str(backup->status); - widths[cur] = Max(widths[cur], strlen(row->status) - (show_color ? TC_LEN : 0)); - - } - - for (i = 0; i < SHOW_FIELDS_COUNT; i++) - widths_sum += widths[i] + 2 /* two space */; - - if (show_name) - appendPQExpBuffer(&show_buf, "\nBACKUP INSTANCE '%s'\n", instance_name); - - /* - * Print header. - */ - for (i = 0; i < widths_sum; i++) - appendPQExpBufferChar(&show_buf, '='); - appendPQExpBufferChar(&show_buf, '\n'); - - for (i = 0; i < SHOW_FIELDS_COUNT; i++) - { - appendPQExpBuffer(&show_buf, field_formats[i], widths[i], names[i]); - } - appendPQExpBufferChar(&show_buf, '\n'); - - for (i = 0; i < widths_sum; i++) - appendPQExpBufferChar(&show_buf, '='); - appendPQExpBufferChar(&show_buf, '\n'); - - /* - * Print values. - */ - for (i = 0; i < parray_num(backup_list); i++) - { - ShowBackendRow *row = &rows[i]; - int cur = 0; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->instance); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->version); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->backup_id); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->recovery_time); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur] + (show_color ? TC_LEN : 0), - row->mode); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->wal_mode); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->tli); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->duration); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->data_bytes); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->wal_bytes); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->zratio); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->start_lsn); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->stop_lsn); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur] + (show_color ? TC_LEN : 0), - row->status); - cur++; - - appendPQExpBufferChar(&show_buf, '\n'); - } - - pfree(rows); - // Restore the C locale - set_output_numeric_locale(LOCALE_C); -} - -/* - * Show instance backups in json format. - */ -static void -show_instance_json(const char *instance_name, parray *backup_list) -{ - int i; - PQExpBuffer buf = &show_buf; - - if (!first_instance) - appendPQExpBufferChar(buf, ','); - - /* Begin of instance object */ - json_add(buf, JT_BEGIN_OBJECT, &json_level); - - json_add_value(buf, "instance", instance_name, json_level, true); - json_add_key(buf, "backups", json_level); - - /* - * List backups. - */ - json_add(buf, JT_BEGIN_ARRAY, &json_level); - - for (i = 0; i < parray_num(backup_list); i++) - { - pgBackup *backup = parray_get(backup_list, i); - - if (i != 0) - appendPQExpBufferChar(buf, ','); - - print_backup_json_object(buf, backup); - } - - /* End of backups */ - json_add(buf, JT_END_ARRAY, &json_level); - - /* End of instance object */ - json_add(buf, JT_END_OBJECT, &json_level); - - first_instance = false; -} - -/* - * show information about WAL archive of the instance - */ -static void -show_instance_archive(InstanceState *instanceState, InstanceConfig *instance) -{ - parray *timelineinfos; - - timelineinfos = catalog_get_timelines(instanceState, instance); - - if (show_format == SHOW_PLAIN) - show_archive_plain(instanceState->instance_name, instance->xlog_seg_size, timelineinfos, true); - else if (show_format == SHOW_JSON) - show_archive_json(instanceState->instance_name, instance->xlog_seg_size, timelineinfos); - else - elog(ERROR, "Invalid show format %d", (int) show_format); -} - -static void -show_archive_plain(const char *instance_name, uint32 xlog_seg_size, - parray *tli_list, bool show_name) -{ - char segno_tmp[MAXFNAMELEN]; - parray *actual_tli_list = parray_new(); -#define SHOW_ARCHIVE_FIELDS_COUNT 10 - int i; - const char *names[SHOW_ARCHIVE_FIELDS_COUNT] = - { "TLI", "Parent TLI", "Switchpoint", - "Min Segno", "Max Segno", "N segments", "Size", "Zratio", "N backups", "Status"}; - const char *field_formats[SHOW_ARCHIVE_FIELDS_COUNT] = - { " %-*s ", " %-*s ", " %-*s ", " %-*s ", - " %-*s ", " %-*s ", " %-*s ", " %-*s ", " %-*s ", " %-*s "}; - uint32 widths[SHOW_ARCHIVE_FIELDS_COUNT]; - uint32 widths_sum = 0; - ShowArchiveRow *rows; - - // Since we've been printing a table, set LC_NUMERIC to its default environment value - set_output_numeric_locale(LOCALE_ENV); - - for (i = 0; i < SHOW_ARCHIVE_FIELDS_COUNT; i++) - widths[i] = strlen(names[i]); - - /* Ignore empty timelines */ - for (i = 0; i < parray_num(tli_list); i++) - { - timelineInfo *tlinfo = (timelineInfo *) parray_get(tli_list, i); - - if (tlinfo->n_xlog_files > 0) - parray_append(actual_tli_list, tlinfo); - } - - rows = (ShowArchiveRow *) palloc0(parray_num(actual_tli_list) * - sizeof(ShowArchiveRow)); - - /* - * Fill row values and calculate maximum width of each field. - */ - for (i = 0; i < parray_num(actual_tli_list); i++) - { - timelineInfo *tlinfo = (timelineInfo *) parray_get(actual_tli_list, i); - ShowArchiveRow *row = &rows[i]; - int cur = 0; - float zratio = 0; - - /* TLI */ - snprintf(row->tli, lengthof(row->tli), "%u", - tlinfo->tli); - widths[cur] = Max(widths[cur], strlen(row->tli)); - cur++; - - /* Parent TLI */ - snprintf(row->parent_tli, lengthof(row->parent_tli), "%u", - tlinfo->parent_tli); - widths[cur] = Max(widths[cur], strlen(row->parent_tli)); - cur++; - - /* Switchpoint LSN */ - snprintf(row->switchpoint, lengthof(row->switchpoint), "%X/%X", - (uint32) (tlinfo->switchpoint >> 32), - (uint32) tlinfo->switchpoint); - widths[cur] = Max(widths[cur], strlen(row->switchpoint)); - cur++; - - /* Min Segno */ - GetXLogFileName(segno_tmp, tlinfo->tli, tlinfo->begin_segno, xlog_seg_size); - snprintf(row->min_segno, lengthof(row->min_segno), "%s",segno_tmp); - - widths[cur] = Max(widths[cur], strlen(row->min_segno)); - cur++; - - /* Max Segno */ - GetXLogFileName(segno_tmp, tlinfo->tli, tlinfo->end_segno, xlog_seg_size); - snprintf(row->max_segno, lengthof(row->max_segno), "%s", segno_tmp); - - widths[cur] = Max(widths[cur], strlen(row->max_segno)); - cur++; - - /* N files */ - snprintf(row->n_segments, lengthof(row->n_segments), "%lu", - tlinfo->n_xlog_files); - widths[cur] = Max(widths[cur], strlen(row->n_segments)); - cur++; - - /* Size */ - pretty_size(tlinfo->size, row->size, - lengthof(row->size)); - widths[cur] = Max(widths[cur], strlen(row->size)); - cur++; - - /* Zratio (compression ratio) */ - if (tlinfo->size != 0) - zratio = ((float)xlog_seg_size*tlinfo->n_xlog_files) / tlinfo->size; - - snprintf(row->zratio, lengthof(row->n_segments), "%.2f", zratio); - widths[cur] = Max(widths[cur], strlen(row->zratio)); - cur++; - - /* N backups */ - snprintf(row->n_backups, lengthof(row->n_backups), "%lu", - tlinfo->backups?parray_num(tlinfo->backups):0); - widths[cur] = Max(widths[cur], strlen(row->n_backups)); - cur++; - - /* Status */ - if (tlinfo->lost_segments == NULL) - row->status = "OK"; - else - row->status = "DEGRADED"; - widths[cur] = Max(widths[cur], strlen(row->status)); - cur++; - } - - for (i = 0; i < SHOW_ARCHIVE_FIELDS_COUNT; i++) - widths_sum += widths[i] + 2 /* two space */; - - if (show_name) - appendPQExpBuffer(&show_buf, "\nARCHIVE INSTANCE '%s'\n", instance_name); - - /* - * Print header. - */ - for (i = 0; i < widths_sum; i++) - appendPQExpBufferChar(&show_buf, '='); - appendPQExpBufferChar(&show_buf, '\n'); - - for (i = 0; i < SHOW_ARCHIVE_FIELDS_COUNT; i++) - { - appendPQExpBuffer(&show_buf, field_formats[i], widths[i], names[i]); - } - appendPQExpBufferChar(&show_buf, '\n'); - - for (i = 0; i < widths_sum; i++) - appendPQExpBufferChar(&show_buf, '='); - appendPQExpBufferChar(&show_buf, '\n'); - - /* - * Print values. - */ - for (i = parray_num(actual_tli_list) - 1; i >= 0; i--) - { - ShowArchiveRow *row = &rows[i]; - int cur = 0; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->tli); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->parent_tli); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->switchpoint); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->min_segno); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->max_segno); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->n_segments); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->size); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->zratio); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->n_backups); - cur++; - - appendPQExpBuffer(&show_buf, field_formats[cur], widths[cur], - row->status); - cur++; - appendPQExpBufferChar(&show_buf, '\n'); - } - - pfree(rows); - // Restore the C locale - set_output_numeric_locale(LOCALE_C); - //TODO: free timelines -} - -static void -show_archive_json(const char *instance_name, uint32 xlog_seg_size, - parray *tli_list) -{ - int i,j; - PQExpBuffer buf = &show_buf; - parray *actual_tli_list = parray_new(); - char segno_tmp[MAXFNAMELEN]; - - if (!first_instance) - appendPQExpBufferChar(buf, ','); - - /* Begin of instance object */ - json_add(buf, JT_BEGIN_OBJECT, &json_level); - - json_add_value(buf, "instance", instance_name, json_level, true); - json_add_key(buf, "timelines", json_level); - - /* Ignore empty timelines */ - - for (i = 0; i < parray_num(tli_list); i++) - { - timelineInfo *tlinfo = (timelineInfo *) parray_get(tli_list, i); - - if (tlinfo->n_xlog_files > 0) - parray_append(actual_tli_list, tlinfo); - } - - /* - * List timelines. - */ - json_add(buf, JT_BEGIN_ARRAY, &json_level); - - for (i = parray_num(actual_tli_list) - 1; i >= 0; i--) - { - timelineInfo *tlinfo = (timelineInfo *) parray_get(actual_tli_list, i); - char tmp_buf[MAXFNAMELEN]; - float zratio = 0; - - if (i != (parray_num(actual_tli_list) - 1)) - appendPQExpBufferChar(buf, ','); - - json_add(buf, JT_BEGIN_OBJECT, &json_level); - - json_add_key(buf, "tli", json_level); - appendPQExpBuffer(buf, "%u", tlinfo->tli); - - json_add_key(buf, "parent-tli", json_level); - appendPQExpBuffer(buf, "%u", tlinfo->parent_tli); - - snprintf(tmp_buf, lengthof(tmp_buf), "%X/%X", - (uint32) (tlinfo->switchpoint >> 32), (uint32) tlinfo->switchpoint); - json_add_value(buf, "switchpoint", tmp_buf, json_level, true); - - GetXLogFileName(segno_tmp, tlinfo->tli, tlinfo->begin_segno, xlog_seg_size); - snprintf(tmp_buf, lengthof(tmp_buf), "%s", segno_tmp); - json_add_value(buf, "min-segno", tmp_buf, json_level, true); - - GetXLogFileName(segno_tmp, tlinfo->tli, tlinfo->end_segno, xlog_seg_size); - snprintf(tmp_buf, lengthof(tmp_buf), "%s", segno_tmp); - json_add_value(buf, "max-segno", tmp_buf, json_level, true); - - json_add_key(buf, "n-segments", json_level); - appendPQExpBuffer(buf, "%lu", tlinfo->n_xlog_files); - - json_add_key(buf, "size", json_level); - appendPQExpBuffer(buf, "%lu", tlinfo->size); - - json_add_key(buf, "zratio", json_level); - - if (tlinfo->size != 0) - zratio = ((float) xlog_seg_size * tlinfo->n_xlog_files) / tlinfo->size; - appendPQExpBuffer(buf, "%.2f", zratio); - - if (tlinfo->closest_backup != NULL) - snprintf(tmp_buf, lengthof(tmp_buf), "%s", - backup_id_of(tlinfo->closest_backup)); - else - snprintf(tmp_buf, lengthof(tmp_buf), "%s", ""); - - json_add_value(buf, "closest-backup-id", tmp_buf, json_level, true); - - if (tlinfo->lost_segments == NULL) - json_add_value(buf, "status", "OK", json_level, true); - else - json_add_value(buf, "status", "DEGRADED", json_level, true); - - json_add_key(buf, "lost-segments", json_level); - - if (tlinfo->lost_segments != NULL) - { - json_add(buf, JT_BEGIN_ARRAY, &json_level); - - for (j = 0; j < parray_num(tlinfo->lost_segments); j++) - { - xlogInterval *lost_segments = (xlogInterval *) parray_get(tlinfo->lost_segments, j); - - if (j != 0) - appendPQExpBufferChar(buf, ','); - - json_add(buf, JT_BEGIN_OBJECT, &json_level); - - GetXLogFileName(segno_tmp, tlinfo->tli, lost_segments->begin_segno, xlog_seg_size); - snprintf(tmp_buf, lengthof(tmp_buf), "%s", segno_tmp); - json_add_value(buf, "begin-segno", tmp_buf, json_level, true); - - GetXLogFileName(segno_tmp, tlinfo->tli, lost_segments->end_segno, xlog_seg_size); - snprintf(tmp_buf, lengthof(tmp_buf), "%s", segno_tmp); - json_add_value(buf, "end-segno", tmp_buf, json_level, true); - json_add(buf, JT_END_OBJECT, &json_level); - } - json_add(buf, JT_END_ARRAY, &json_level); - } - else - appendPQExpBuffer(buf, "[]"); - - json_add_key(buf, "backups", json_level); - - if (tlinfo->backups != NULL) - { - json_add(buf, JT_BEGIN_ARRAY, &json_level); - for (j = 0; j < parray_num(tlinfo->backups); j++) - { - pgBackup *backup = parray_get(tlinfo->backups, j); - - if (j != 0) - appendPQExpBufferChar(buf, ','); - - print_backup_json_object(buf, backup); - } - - json_add(buf, JT_END_ARRAY, &json_level); - } - else - appendPQExpBuffer(buf, "[]"); - - /* End of timeline */ - json_add(buf, JT_END_OBJECT, &json_level); - } - - /* End of timelines object */ - json_add(buf, JT_END_ARRAY, &json_level); - - /* End of instance object */ - json_add(buf, JT_END_OBJECT, &json_level); - - first_instance = false; -} diff --git a/src/stream.c b/src/stream.c deleted file mode 100644 index 77453e997..000000000 --- a/src/stream.c +++ /dev/null @@ -1,779 +0,0 @@ -/*------------------------------------------------------------------------- - * - * stream.c: pg_probackup specific code for WAL streaming - * - * Portions Copyright (c) 2015-2020, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" -#include "receivelog.h" -#include "streamutil.h" -#include "access/timeline.h" - -#include -#include - -/* - * global variable needed by ReceiveXlogStream() - * - * standby_message_timeout controls how often we send a message - * back to the primary letting it know our progress, in milliseconds. - * - * in pg_probackup we use a default setting = 10 sec - */ -static int standby_message_timeout = 10 * 1000; - -/* stop_backup_lsn is set by pg_stop_backup() to stop streaming */ -XLogRecPtr stop_backup_lsn = InvalidXLogRecPtr; -static XLogRecPtr stop_stream_lsn = InvalidXLogRecPtr; - -/* - * How long we should wait for streaming end in seconds. - * Retrieved as checkpoint_timeout + checkpoint_timeout * 0.1 - */ -static uint32 stream_stop_timeout = 0; -/* Time in which we started to wait for streaming end */ -static time_t stream_stop_begin = 0; - -/* - * We need to wait end of WAL streaming before execute pg_stop_backup(). - */ -typedef struct -{ - char basedir[MAXPGPATH]; - PGconn *conn; - - /* - * Return value from the thread. - * 0 means there is no error, 1 - there is an error. - */ - int ret; - - XLogRecPtr startpos; - TimeLineID starttli; -} StreamThreadArg; - -static pthread_t stream_thread; -static StreamThreadArg stream_thread_arg = {"", NULL, 1}; - -static parray *xlog_files_list = NULL; -static bool do_crc = true; - -static void IdentifySystem(StreamThreadArg *stream_thread_arg); -static int checkpoint_timeout(PGconn *backup_conn); -static void *StreamLog(void *arg); -static bool stop_streaming(XLogRecPtr xlogpos, uint32 timeline, - bool segment_finished); -static void add_walsegment_to_filelist(parray *filelist, uint32 timeline, - XLogRecPtr xlogpos, char *basedir, - uint32 xlog_seg_size); -static void add_history_file_to_filelist(parray *filelist, uint32 timeline, - char *basedir); - -/* - * Run IDENTIFY_SYSTEM through a given connection and - * check system identifier and timeline are matching - */ -static void -IdentifySystem(StreamThreadArg *stream_thread_arg) -{ - PGresult *res; - - uint64 stream_conn_sysidentifier = 0; - char *stream_conn_sysidentifier_str; - TimeLineID stream_conn_tli = 0; - - if (!CheckServerVersionForStreaming(stream_thread_arg->conn)) - { - PQfinish(stream_thread_arg->conn); - /* - * Error message already written in CheckServerVersionForStreaming(). - * There's no hope of recovering from a version mismatch, so don't - * retry. - */ - elog(ERROR, "Cannot continue backup because stream connect has failed."); - } - - /* - * Identify server, obtain server system identifier and timeline - */ - res = pgut_execute(stream_thread_arg->conn, "IDENTIFY_SYSTEM", 0, NULL); - - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - elog(WARNING,"Could not send replication command \"%s\": %s", - "IDENTIFY_SYSTEM", PQerrorMessage(stream_thread_arg->conn)); - PQfinish(stream_thread_arg->conn); - elog(ERROR, "Cannot continue backup because stream connect has failed."); - } - - stream_conn_sysidentifier_str = PQgetvalue(res, 0, 0); - stream_conn_tli = atoll(PQgetvalue(res, 0, 1)); - - /* Additional sanity, primary for PG 9.5, - * where system id can be obtained only via "IDENTIFY SYSTEM" - */ - if (!parse_uint64(stream_conn_sysidentifier_str, &stream_conn_sysidentifier, 0)) - elog(ERROR, "%s is not system_identifier", stream_conn_sysidentifier_str); - - if (stream_conn_sysidentifier != instance_config.system_identifier) - elog(ERROR, "System identifier mismatch. Connected PostgreSQL instance has system id: " - "" UINT64_FORMAT ". Expected: " UINT64_FORMAT ".", - stream_conn_sysidentifier, instance_config.system_identifier); - - if (stream_conn_tli != current.tli) - elog(ERROR, "Timeline identifier mismatch. " - "Connected PostgreSQL instance has timeline id: %X. Expected: %X.", - stream_conn_tli, current.tli); - - PQclear(res); -} - -/* - * Retrieve checkpoint_timeout GUC value in seconds. - */ -static int -checkpoint_timeout(PGconn *backup_conn) -{ - PGresult *res; - const char *val; - const char *hintmsg; - int val_int; - - res = pgut_execute(backup_conn, "show checkpoint_timeout", 0, NULL); - val = PQgetvalue(res, 0, 0); - - if (!parse_int(val, &val_int, OPTION_UNIT_S, &hintmsg)) - { - PQclear(res); - if (hintmsg) - elog(ERROR, "Invalid value of checkout_timeout %s: %s", val, - hintmsg); - else - elog(ERROR, "Invalid value of checkout_timeout %s", val); - } - - PQclear(res); - - return val_int; -} - -/* - * CreateReplicationSlot_compat() -- wrapper for CreateReplicationSlot() used in StreamLog() - * src/bin/pg_basebackup/streamutil.c - * CreateReplicationSlot() has different signatures on different PG versions: - * PG 15 - * bool - * CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin, - * bool is_temporary, bool is_physical, bool reserve_wal, - * bool slot_exists_ok, bool two_phase) - * PG 11-14 - * bool - * CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin, - * bool is_temporary, bool is_physical, bool reserve_wal, - * bool slot_exists_ok) - * PG 9.5-10 - * CreateReplicationSlot(PGconn *conn, const char *slot_name, const char *plugin, - * bool is_physical, bool slot_exists_ok) - * NOTE: PG 9.6 and 10 support reserve_wal in - * pg_catalog.pg_create_physical_replication_slot(slot_name name [, immediately_reserve boolean]) - * and - * CREATE_REPLICATION_SLOT slot_name { PHYSICAL [ RESERVE_WAL ] | LOGICAL output_plugin } - * replication protocol command, but CreateReplicationSlot() C function doesn't - */ -static bool -CreateReplicationSlot_compat(PGconn *conn, const char *slot_name, const char *plugin, - bool is_temporary, bool is_physical, - bool slot_exists_ok) -{ -#if PG_VERSION_NUM >= 150000 - return CreateReplicationSlot(conn, slot_name, plugin, is_temporary, is_physical, - /* reserve_wal = */ true, slot_exists_ok, /* two_phase = */ false); -#elif PG_VERSION_NUM >= 110000 - return CreateReplicationSlot(conn, slot_name, plugin, is_temporary, is_physical, - /* reserve_wal = */ true, slot_exists_ok); -#elif PG_VERSION_NUM >= 100000 - /* - * PG-10 doesn't support creating temp_slot by calling CreateReplicationSlot(), but - * it will be created by setting StreamCtl.temp_slot later in StreamLog() - */ - if (!is_temporary) - return CreateReplicationSlot(conn, slot_name, plugin, /*is_temporary,*/ is_physical, /*reserve_wal,*/ slot_exists_ok); - else - return true; -#else - /* these parameters not supported in PG < 10 */ - Assert(!is_temporary); - return CreateReplicationSlot(conn, slot_name, plugin, /*is_temporary,*/ is_physical, /*reserve_wal,*/ slot_exists_ok); -#endif -} - -/* - * Start the log streaming - */ -static void * -StreamLog(void *arg) -{ - StreamThreadArg *stream_arg = (StreamThreadArg *) arg; - - /* - * Always start streaming at the beginning of a segment - */ - stream_arg->startpos -= stream_arg->startpos % instance_config.xlog_seg_size; - - xlog_files_list = parray_new(); - - /* Initialize timeout */ - stream_stop_begin = 0; - - /* Create repslot */ -#if PG_VERSION_NUM >= 100000 - if (temp_slot || perm_slot) - if (!CreateReplicationSlot_compat(stream_arg->conn, replication_slot, NULL, temp_slot, true, false)) -#else - if (perm_slot) - if (!CreateReplicationSlot_compat(stream_arg->conn, replication_slot, NULL, false, true, false)) -#endif - { - interrupted = true; - elog(ERROR, "Couldn't create physical replication slot %s", replication_slot); - } - - /* - * Start the replication - */ - if (replication_slot) - elog(LOG, "started streaming WAL at %X/%X (timeline %u) using%s slot %s", - (uint32) (stream_arg->startpos >> 32), (uint32) stream_arg->startpos, - stream_arg->starttli, -#if PG_VERSION_NUM >= 100000 - temp_slot ? " temporary" : "", -#else - "", -#endif - replication_slot); - else - elog(LOG, "started streaming WAL at %X/%X (timeline %u)", - (uint32) (stream_arg->startpos >> 32), (uint32) stream_arg->startpos, - stream_arg->starttli); - -#if PG_VERSION_NUM >= 90600 - { - StreamCtl ctl; - - MemSet(&ctl, 0, sizeof(ctl)); - - ctl.startpos = stream_arg->startpos; - ctl.timeline = stream_arg->starttli; - ctl.sysidentifier = NULL; - ctl.stream_stop = stop_streaming; - ctl.standby_message_timeout = standby_message_timeout; - ctl.partial_suffix = NULL; - ctl.synchronous = false; - ctl.mark_done = false; - -#if PG_VERSION_NUM >= 100000 -#if PG_VERSION_NUM >= 150000 - ctl.walmethod = CreateWalDirectoryMethod( - stream_arg->basedir, - PG_COMPRESSION_NONE, - 0, - false); -#else /* PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 150000 */ - ctl.walmethod = CreateWalDirectoryMethod( - stream_arg->basedir, -// (instance_config.compress_alg == NONE_COMPRESS) ? 0 : instance_config.compress_level, - 0, - false); -#endif /* PG_VERSION_NUM >= 150000 */ - ctl.replication_slot = replication_slot; - ctl.stop_socket = PGINVALID_SOCKET; - ctl.do_sync = false; /* We sync all files at the end of backup */ -// ctl.mark_done /* for future use in s3 */ -#if PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 110000 - /* StreamCtl.temp_slot used only for PG-10, in PG>10, temp_slots are created by calling CreateReplicationSlot() */ - ctl.temp_slot = temp_slot; -#endif /* PG_VERSION_NUM >= 100000 && PG_VERSION_NUM < 110000 */ -#else /* PG_VERSION_NUM < 100000 */ - ctl.basedir = (char *) stream_arg->basedir; -#endif /* PG_VERSION_NUM >= 100000 */ - - if (ReceiveXlogStream(stream_arg->conn, &ctl) == false) - { - interrupted = true; - elog(ERROR, "Problem in receivexlog"); - } - -#if PG_VERSION_NUM >= 100000 -#if PG_VERSION_NUM >= 160000 - if (!ctl.walmethod->ops->finish(ctl.walmethod)) -#else - if (!ctl.walmethod->finish()) -#endif - { - interrupted = true; - elog(ERROR, "Could not finish writing WAL files: %s", - strerror(errno)); - } -#endif /* PG_VERSION_NUM >= 100000 */ - } -#else /* PG_VERSION_NUM < 90600 */ - /* PG-9.5 */ - if (ReceiveXlogStream(stream_arg->conn, stream_arg->startpos, stream_arg->starttli, - NULL, (char *) stream_arg->basedir, stop_streaming, - standby_message_timeout, NULL, false, false) == false) - { - interrupted = true; - elog(ERROR, "Problem in receivexlog"); - } -#endif /* PG_VERSION_NUM >= 90600 */ - - /* be paranoid and sort xlog_files_list, - * so if stop_lsn segno is already in the list, - * then list must be sorted to detect duplicates. - */ - parray_qsort(xlog_files_list, pgFileCompareRelPathWithExternal); - - /* Add the last segment to the list */ - add_walsegment_to_filelist(xlog_files_list, stream_arg->starttli, - stop_stream_lsn, (char *) stream_arg->basedir, - instance_config.xlog_seg_size); - - /* append history file to walsegment filelist */ - add_history_file_to_filelist(xlog_files_list, stream_arg->starttli, (char *) stream_arg->basedir); - - /* - * TODO: remove redundant WAL segments - * walk pg_wal and remove files with segno greater that of stop_lsn`s segno +1 - */ - - elog(LOG, "finished streaming WAL at %X/%X (timeline %u)", - (uint32) (stop_stream_lsn >> 32), (uint32) stop_stream_lsn, stream_arg->starttli); - stream_arg->ret = 0; - - PQfinish(stream_arg->conn); - stream_arg->conn = NULL; - - return NULL; -} - -/* - * for ReceiveXlogStream - * - * The stream_stop callback will be called every time data - * is received, and whenever a segment is completed. If it returns - * true, the streaming will stop and the function - * return. As long as it returns false, streaming will continue - * indefinitely. - * - * Stop WAL streaming if current 'xlogpos' exceeds 'stop_backup_lsn', which is - * set by pg_stop_backup(). - * - */ -static bool -stop_streaming(XLogRecPtr xlogpos, uint32 timeline, bool segment_finished) -{ - static uint32 prevtimeline = 0; - static XLogRecPtr prevpos = InvalidXLogRecPtr; - - /* check for interrupt */ - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during WAL streaming"); - - /* we assume that we get called once at the end of each segment */ - if (segment_finished) - { - elog(VERBOSE, _("finished segment at %X/%X (timeline %u)"), - (uint32) (xlogpos >> 32), (uint32) xlogpos, timeline); - - add_walsegment_to_filelist(xlog_files_list, timeline, xlogpos, - (char*) stream_thread_arg.basedir, - instance_config.xlog_seg_size); - } - - /* - * Note that we report the previous, not current, position here. After a - * timeline switch, xlogpos points to the beginning of the segment because - * that's where we always begin streaming. Reporting the end of previous - * timeline isn't totally accurate, because the next timeline can begin - * slightly before the end of the WAL that we received on the previous - * timeline, but it's close enough for reporting purposes. - */ - if (prevtimeline != 0 && prevtimeline != timeline) - elog(LOG, _("switched to timeline %u at %X/%X\n"), - timeline, (uint32) (prevpos >> 32), (uint32) prevpos); - - if (!XLogRecPtrIsInvalid(stop_backup_lsn)) - { - if (xlogpos >= stop_backup_lsn) - { - stop_stream_lsn = xlogpos; - return true; - } - - /* pg_stop_backup() was executed, wait for the completion of stream */ - if (stream_stop_begin == 0) - { - elog(INFO, "Wait for LSN %X/%X to be streamed", - (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn); - - stream_stop_begin = time(NULL); - } - - if (time(NULL) - stream_stop_begin > stream_stop_timeout) - elog(ERROR, "Target LSN %X/%X could not be streamed in %d seconds", - (uint32) (stop_backup_lsn >> 32), (uint32) stop_backup_lsn, - stream_stop_timeout); - } - - prevtimeline = timeline; - prevpos = xlogpos; - - return false; -} - - -/* --- External API --- */ - -/* - * Maybe add a StreamOptions struct ? - * Backup conn only needed to calculate stream_stop_timeout. Think about refactoring it. - */ -parray* -get_history_streaming(ConnectionOptions *conn_opt, TimeLineID tli, parray *backup_list) -{ - PGresult *res; - PGconn *conn; - char *history; - char query[128]; - parray *result = NULL; - parray *tli_list = NULL; - timelineInfo *tlinfo = NULL; - int i,j; - - snprintf(query, sizeof(query), "TIMELINE_HISTORY %u", tli); - - /* - * Connect in replication mode to the server. - */ - conn = pgut_connect_replication(conn_opt->pghost, - conn_opt->pgport, - conn_opt->pgdatabase, - conn_opt->pguser, - false); - - if (!conn) - return NULL; - - res = PQexec(conn, query); - PQfinish(conn); - - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - elog(WARNING, "Could not send replication command \"%s\": %s", - query, PQresultErrorMessage(res)); - PQclear(res); - return NULL; - } - - /* - * The response to TIMELINE_HISTORY is a single row result set - * with two fields: filename and content - */ - - if (PQnfields(res) != 2 || PQntuples(res) != 1) - { - elog(WARNING, "Unexpected response to TIMELINE_HISTORY command: " - "got %d rows and %d fields, expected %d rows and %d fields", - PQntuples(res), PQnfields(res), 1, 2); - PQclear(res); - return NULL; - } - - history = pgut_strdup(PQgetvalue(res, 0, 1)); - result = parse_tli_history_buffer(history, tli); - - /* some cleanup */ - pg_free(history); - PQclear(res); - - if (result) - tlinfo = timelineInfoNew(tli); - else - return NULL; - - /* transform TimeLineHistoryEntry into timelineInfo */ - for (i = parray_num(result) -1; i >= 0; i--) - { - TimeLineHistoryEntry *tln = (TimeLineHistoryEntry *) parray_get(result, i); - - tlinfo->parent_tli = tln->tli; - tlinfo->switchpoint = tln->end; - - if (!tli_list) - tli_list = parray_new(); - - parray_append(tli_list, tlinfo); - - /* Next tli */ - tlinfo = timelineInfoNew(tln->tli); - - /* oldest tli */ - if (i == 0) - { - tlinfo->tli = tln->tli; - tlinfo->parent_tli = 0; - tlinfo->switchpoint = 0; - parray_append(tli_list, tlinfo); - } - } - - /* link parent to child */ - for (i = 0; i < parray_num(tli_list); i++) - { - tlinfo = (timelineInfo *) parray_get(tli_list, i); - - for (j = 0; j < parray_num(tli_list); j++) - { - timelineInfo *tlinfo_parent = (timelineInfo *) parray_get(tli_list, j); - - if (tlinfo->parent_tli == tlinfo_parent->tli) - { - tlinfo->parent_link = tlinfo_parent; - break; - } - } - } - - /* add backups to each timeline info */ - for (i = 0; i < parray_num(tli_list); i++) - { - tlinfo = parray_get(tli_list, i); - for (j = 0; j < parray_num(backup_list); j++) - { - pgBackup *backup = parray_get(backup_list, j); - if (tlinfo->tli == backup->tli) - { - if (tlinfo->backups == NULL) - tlinfo->backups = parray_new(); - parray_append(tlinfo->backups, backup); - } - } - } - - /* cleanup */ - parray_walk(result, pg_free); - pg_free(result); - - return tli_list; -} - -parray* -parse_tli_history_buffer(char *history, TimeLineID tli) -{ - char *curLine = history; - TimeLineHistoryEntry *entry; - TimeLineHistoryEntry *last_timeline = NULL; - parray *result = NULL; - - /* Parse timeline history buffer string by string */ - while (curLine) - { - char tempStr[1024]; - char *nextLine = strchr(curLine, '\n'); - int curLineLen = nextLine ? (nextLine-curLine) : strlen(curLine); - - memcpy(tempStr, curLine, curLineLen); - tempStr[curLineLen] = '\0'; // NUL-terminate! - curLine = nextLine ? (nextLine+1) : NULL; - - if (curLineLen > 0) - { - char *ptr; - TimeLineID tli; - uint32 switchpoint_hi; - uint32 switchpoint_lo; - int nfields; - - for (ptr = tempStr; *ptr; ptr++) - { - if (!isspace((unsigned char) *ptr)) - break; - } - if (*ptr == '\0' || *ptr == '#') - continue; - - nfields = sscanf(tempStr, "%u\t%X/%X", &tli, &switchpoint_hi, &switchpoint_lo); - - if (nfields < 1) - { - /* expect a numeric timeline ID as first field of line */ - elog(ERROR, "Syntax error in timeline history: \"%s\". Expected a numeric timeline ID.", tempStr); - } - if (nfields != 3) - elog(ERROR, "Syntax error in timeline history: \"%s\". Expected a transaction log switchpoint location.", tempStr); - - if (last_timeline && tli <= last_timeline->tli) - elog(ERROR, "Timeline IDs must be in increasing sequence: \"%s\"", tempStr); - - entry = pgut_new(TimeLineHistoryEntry); - entry->tli = tli; - entry->end = ((uint64) switchpoint_hi << 32) | switchpoint_lo; - - last_timeline = entry; - /* Build list with newest item first */ - if (!result) - result = parray_new(); - parray_append(result, entry); - elog(VERBOSE, "parse_tli_history_buffer() found entry: tli = %X, end = %X/%X", - tli, switchpoint_hi, switchpoint_lo); - - /* we ignore the remainder of each line */ - } - } - - return result; -} - -/* - * Maybe add a StreamOptions struct ? - * Backup conn only needed to calculate stream_stop_timeout. Think about refactoring it. - */ -void -start_WAL_streaming(PGconn *backup_conn, char *stream_dst_path, ConnectionOptions *conn_opt, - XLogRecPtr startpos, TimeLineID starttli, bool is_backup) -{ - /* calculate crc only when running backup, catchup has no need for it */ - do_crc = is_backup; - /* How long we should wait for streaming end after pg_stop_backup */ - stream_stop_timeout = checkpoint_timeout(backup_conn); - //TODO Add a comment about this calculation - stream_stop_timeout = stream_stop_timeout + stream_stop_timeout * 0.1; - - strlcpy(stream_thread_arg.basedir, stream_dst_path, sizeof(stream_thread_arg.basedir)); - - /* - * Connect in replication mode to the server. - */ - stream_thread_arg.conn = pgut_connect_replication(conn_opt->pghost, - conn_opt->pgport, - conn_opt->pgdatabase, - conn_opt->pguser, - true); - /* sanity check*/ - IdentifySystem(&stream_thread_arg); - - /* Set error exit code as default */ - stream_thread_arg.ret = 1; - /* we must use startpos as start_lsn from start_backup */ - stream_thread_arg.startpos = startpos; - stream_thread_arg.starttli = starttli; - - thread_interrupted = false; - pthread_create(&stream_thread, NULL, StreamLog, &stream_thread_arg); -} - -/* - * Wait for the completion of stream - * append list of streamed xlog files - * into backup_files_list (if it is not NULL) - */ -int -wait_WAL_streaming_end(parray *backup_files_list) -{ - pthread_join(stream_thread, NULL); - - if(backup_files_list != NULL) - parray_concat(backup_files_list, xlog_files_list); - parray_free(xlog_files_list); - return stream_thread_arg.ret; -} - -/* Append streamed WAL segment to filelist */ -void -add_walsegment_to_filelist(parray *filelist, uint32 timeline, XLogRecPtr xlogpos, char *basedir, uint32 xlog_seg_size) -{ - XLogSegNo xlog_segno; - char wal_segment_name[MAXFNAMELEN]; - char wal_segment_relpath[MAXPGPATH]; - char wal_segment_fullpath[MAXPGPATH]; - pgFile *file = NULL; - pgFile **existing_file = NULL; - - GetXLogSegNo(xlogpos, xlog_segno, xlog_seg_size); - - /* - * When xlogpos points to the zero offset (0/3000000), - * it means that previous segment was just successfully streamed. - * When xlogpos points to the positive offset, - * then current segment is successfully streamed. - */ - if (WalSegmentOffset(xlogpos, xlog_seg_size) == 0) - xlog_segno--; - - GetXLogFileName(wal_segment_name, timeline, xlog_segno, xlog_seg_size); - - join_path_components(wal_segment_fullpath, basedir, wal_segment_name); - join_path_components(wal_segment_relpath, PG_XLOG_DIR, wal_segment_name); - - file = pgFileNew(wal_segment_fullpath, wal_segment_relpath, false, 0, FIO_BACKUP_HOST); - - /* - * Check if file is already in the list - * stop_lsn segment can be added to this list twice, so - * try not to add duplicates - */ - - existing_file = (pgFile **) parray_bsearch(filelist, file, pgFileCompareRelPathWithExternal); - - if (existing_file) - { - if (do_crc) - (*existing_file)->crc = pgFileGetCRC(wal_segment_fullpath, true, false); - (*existing_file)->write_size = xlog_seg_size; - (*existing_file)->uncompressed_size = xlog_seg_size; - - return; - } - - if (do_crc) - file->crc = pgFileGetCRC(wal_segment_fullpath, true, false); - - /* Should we recheck it using stat? */ - file->write_size = xlog_seg_size; - file->uncompressed_size = xlog_seg_size; - - /* append file to filelist */ - parray_append(filelist, file); -} - -/* Append history file to filelist */ -void -add_history_file_to_filelist(parray *filelist, uint32 timeline, char *basedir) -{ - char filename[MAXFNAMELEN]; - char fullpath[MAXPGPATH]; - char relpath[MAXPGPATH]; - pgFile *file = NULL; - - /* Timeline 1 does not have a history file */ - if (timeline == 1) - return; - - snprintf(filename, lengthof(filename), "%08X.history", timeline); - join_path_components(fullpath, basedir, filename); - join_path_components(relpath, PG_XLOG_DIR, filename); - - file = pgFileNew(fullpath, relpath, false, 0, FIO_BACKUP_HOST); - - /* calculate crc */ - if (do_crc) - file->crc = pgFileGetCRC(fullpath, true, false); - file->write_size = file->size; - file->uncompressed_size = file->size; - - /* append file to filelist */ - parray_append(filelist, file); -} diff --git a/src/util.c b/src/util.c deleted file mode 100644 index 1407f03cc..000000000 --- a/src/util.c +++ /dev/null @@ -1,583 +0,0 @@ -/*------------------------------------------------------------------------- - * - * util.c: log messages to log file or stderr, and misc code. - * - * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2021, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -#include - -#include - -#include - -static const char *statusName[] = -{ - "UNKNOWN", - "OK", - "ERROR", - "RUNNING", - "MERGING", - "MERGED", - "DELETING", - "DELETED", - "DONE", - "ORPHAN", - "CORRUPT" -}; - -const char * -base36enc_to(long unsigned int value, char buf[ARG_SIZE_HINT base36bufsize]) -{ - const char base36[36] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - char buffer[base36bufsize]; - char *p; - - p = &buffer[sizeof(buffer)-1]; - *p = '\0'; - do { - *(--p) = base36[value % 36]; - } while (value /= 36); - - /* I know, it doesn't look safe */ - strncpy(buf, p, base36bufsize); - - return buf; -} - -long unsigned int -base36dec(const char *text) -{ - return strtoul(text, NULL, 36); -} - -static void -checkControlFile(ControlFileData *ControlFile) -{ - pg_crc32c crc; - - /* Calculate CRC */ - INIT_CRC32C(crc); - COMP_CRC32C(crc, (char *) ControlFile, offsetof(ControlFileData, crc)); - FIN_CRC32C(crc); - - /* Then compare it */ - if (!EQ_CRC32C(crc, ControlFile->crc)) - elog(ERROR, "Calculated CRC checksum does not match value stored in file.\n" - "Either the file is corrupt, or it has a different layout than this program\n" - "is expecting. The results below are untrustworthy."); - - if ((ControlFile->pg_control_version % 65536 == 0 || ControlFile->pg_control_version % 65536 > 10000) && - ControlFile->pg_control_version / 65536 != 0) - elog(ERROR, "Possible byte ordering mismatch\n" - "The byte ordering used to store the pg_control file might not match the one\n" - "used by this program. In that case the results below would be incorrect, and\n" - "the PostgreSQL installation would be incompatible with this data directory."); -} - -/* - * Verify control file contents in the buffer src, and copy it to *ControlFile. - */ -static void -digestControlFile(ControlFileData *ControlFile, char *src, size_t size) -{ -#if PG_VERSION_NUM >= 100000 - int ControlFileSize = PG_CONTROL_FILE_SIZE; -#else - int ControlFileSize = PG_CONTROL_SIZE; -#endif - - if (size != ControlFileSize) - elog(ERROR, "Unexpected control file size %d, expected %d", - (int) size, ControlFileSize); - - memcpy(ControlFile, src, sizeof(ControlFileData)); - - /* Additional checks on control file */ - checkControlFile(ControlFile); -} - -/* - * Write ControlFile to pg_control - */ -static void -writeControlFile(ControlFileData *ControlFile, const char *path, fio_location location) -{ - int fd; - char *buffer = NULL; - -#if PG_VERSION_NUM >= 100000 - int ControlFileSize = PG_CONTROL_FILE_SIZE; -#else - int ControlFileSize = PG_CONTROL_SIZE; -#endif - - /* copy controlFileSize */ - buffer = pg_malloc0(ControlFileSize); - memcpy(buffer, ControlFile, sizeof(ControlFileData)); - - /* Write pg_control */ - fd = fio_open(path, - O_RDWR | O_CREAT | O_TRUNC | PG_BINARY, location); - - if (fd < 0) - elog(ERROR, "Failed to open file: %s", path); - - if (fio_write(fd, buffer, ControlFileSize) != ControlFileSize) - elog(ERROR, "Failed to overwrite file: %s", path); - - if (fio_flush(fd) != 0) - elog(ERROR, "Failed to sync file: %s", path); - - fio_close(fd); - pg_free(buffer); -} - -/* - * Utility shared by backup and restore to fetch the current timeline - * used by a node. - */ -TimeLineID -get_current_timeline(PGconn *conn) -{ - - PGresult *res; - TimeLineID tli = 0; - char *val; - - res = pgut_execute_extended(conn, - "SELECT timeline_id FROM pg_catalog.pg_control_checkpoint()", 0, NULL, true, true); - - if (PQresultStatus(res) == PGRES_TUPLES_OK) - val = PQgetvalue(res, 0, 0); - else - return get_current_timeline_from_control(instance_config.pgdata, FIO_DB_HOST, false); - - if (!parse_uint32(val, &tli, 0)) - { - PQclear(res); - elog(WARNING, "Invalid value of timeline_id %s", val); - - /* TODO 3.0 remove it and just error out */ - return get_current_timeline_from_control(instance_config.pgdata, FIO_DB_HOST, false); - } - - return tli; -} - -/* Get timeline from pg_control file */ -TimeLineID -get_current_timeline_from_control(const char *pgdata_path, fio_location location, bool safe) -{ - ControlFileData ControlFile; - char *buffer; - size_t size; - - /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, - safe, location); - if (safe && buffer == NULL) - return 0; - - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); - - return ControlFile.checkPointCopy.ThisTimeLineID; -} - -/* - * Get last check point record ptr from pg_tonrol. - */ -XLogRecPtr -get_checkpoint_location(PGconn *conn) -{ -#if PG_VERSION_NUM >= 90600 - PGresult *res; - uint32 lsn_hi; - uint32 lsn_lo; - XLogRecPtr lsn; - -#if PG_VERSION_NUM >= 100000 - res = pgut_execute(conn, - "SELECT checkpoint_lsn FROM pg_catalog.pg_control_checkpoint()", - 0, NULL); -#else - res = pgut_execute(conn, - "SELECT checkpoint_location FROM pg_catalog.pg_control_checkpoint()", - 0, NULL); -#endif - XLogDataFromLSN(PQgetvalue(res, 0, 0), &lsn_hi, &lsn_lo); - PQclear(res); - /* Calculate LSN */ - lsn = ((uint64) lsn_hi) << 32 | lsn_lo; - - return lsn; -#else - char *buffer; - size_t size; - ControlFileData ControlFile; - - buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); - - return ControlFile.checkPoint; -#endif -} - -uint64 -get_system_identifier(const char *pgdata_path, fio_location location, bool safe) -{ - ControlFileData ControlFile; - char *buffer; - size_t size; - - /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, safe, location); - if (safe && buffer == NULL) - return 0; - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); - - return ControlFile.system_identifier; -} - -uint64 -get_remote_system_identifier(PGconn *conn) -{ -#if PG_VERSION_NUM >= 90600 - PGresult *res; - uint64 system_id_conn; - char *val; - - res = pgut_execute(conn, - "SELECT system_identifier FROM pg_catalog.pg_control_system()", - 0, NULL); - val = PQgetvalue(res, 0, 0); - if (!parse_uint64(val, &system_id_conn, 0)) - { - PQclear(res); - elog(ERROR, "%s is not system_identifier", val); - } - PQclear(res); - - return system_id_conn; -#else - char *buffer; - size_t size; - ControlFileData ControlFile; - - buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); - - return ControlFile.system_identifier; -#endif -} - -uint32 -get_xlog_seg_size(const char *pgdata_path) -{ -#if PG_VERSION_NUM >= 110000 - ControlFileData ControlFile; - char *buffer; - size_t size; - - /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); - - return ControlFile.xlog_seg_size; -#else - return (uint32) XLOG_SEG_SIZE; -#endif -} - -uint32 -get_data_checksum_version(bool safe) -{ - ControlFileData ControlFile; - char *buffer; - size_t size; - - /* First fetch file... */ - buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, - safe, FIO_DB_HOST); - if (buffer == NULL) - return 0; - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); - - return ControlFile.data_checksum_version; -} - -pg_crc32c -get_pgcontrol_checksum(const char *pgdata_path) -{ - ControlFileData ControlFile; - char *buffer; - size_t size; - - /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_BACKUP_HOST); - - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); - - return ControlFile.crc; -} - -void -get_redo(const char *pgdata_path, fio_location pgdata_location, RedoParams *redo) -{ - ControlFileData ControlFile; - char *buffer; - size_t size; - - /* First fetch file... */ - buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, pgdata_location); - - digestControlFile(&ControlFile, buffer, size); - pg_free(buffer); - - redo->lsn = ControlFile.checkPointCopy.redo; - redo->tli = ControlFile.checkPointCopy.ThisTimeLineID; - - if (ControlFile.minRecoveryPoint > 0 && - ControlFile.minRecoveryPoint < redo->lsn) - { - redo->lsn = ControlFile.minRecoveryPoint; - redo->tli = ControlFile.minRecoveryPointTLI; - } - - if (ControlFile.backupStartPoint > 0 && - ControlFile.backupStartPoint < redo->lsn) - { - redo->lsn = ControlFile.backupStartPoint; - redo->tli = ControlFile.checkPointCopy.ThisTimeLineID; - } - - redo->checksum_version = ControlFile.data_checksum_version; -} - -/* - * Rewrite minRecoveryPoint of pg_control in backup directory. minRecoveryPoint - * 'as-is' is not to be trusted. - */ -void -set_min_recovery_point(pgFile *file, const char *backup_path, - XLogRecPtr stop_backup_lsn) -{ - ControlFileData ControlFile; - char *buffer; - size_t size; - char fullpath[MAXPGPATH]; - - /* First fetch file content */ - buffer = slurpFile(instance_config.pgdata, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST); - digestControlFile(&ControlFile, buffer, size); - - elog(LOG, "Current minRecPoint %X/%X", - (uint32) (ControlFile.minRecoveryPoint >> 32), - (uint32) ControlFile.minRecoveryPoint); - - elog(LOG, "Setting minRecPoint to %X/%X", - (uint32) (stop_backup_lsn >> 32), - (uint32) stop_backup_lsn); - - ControlFile.minRecoveryPoint = stop_backup_lsn; - - /* Update checksum in pg_control header */ - INIT_CRC32C(ControlFile.crc); - COMP_CRC32C(ControlFile.crc, (char *) &ControlFile, - offsetof(ControlFileData, crc)); - FIN_CRC32C(ControlFile.crc); - - /* overwrite pg_control */ - join_path_components(fullpath, backup_path, XLOG_CONTROL_FILE); - writeControlFile(&ControlFile, fullpath, FIO_LOCAL_HOST); - - /* Update pg_control checksum in backup_list */ - file->crc = ControlFile.crc; - - pg_free(buffer); -} - -/* - * Copy pg_control file to backup. We do not apply compression to this file. - */ -void -copy_pgcontrol_file(const char *from_fullpath, fio_location from_location, - const char *to_fullpath, fio_location to_location, pgFile *file) -{ - ControlFileData ControlFile; - char *buffer; - size_t size; - - buffer = slurpFile(from_fullpath, "", &size, false, from_location); - - digestControlFile(&ControlFile, buffer, size); - - file->crc = ControlFile.crc; - file->read_size = size; - file->write_size = size; - file->uncompressed_size = size; - - writeControlFile(&ControlFile, to_fullpath, to_location); - - pg_free(buffer); -} - -/* - * Parse string representation of the server version. - */ -uint32 -parse_server_version(const char *server_version_str) -{ - int nfields; - uint32 result = 0; - int major_version = 0; - int minor_version = 0; - - nfields = sscanf(server_version_str, "%d.%d", &major_version, &minor_version); - if (nfields == 2) - { - /* Server version lower than 10 */ - if (major_version > 10) - elog(ERROR, "Server version format doesn't match major version %d", major_version); - result = major_version * 10000 + minor_version * 100; - } - else if (nfields == 1) - { - if (major_version < 10) - elog(ERROR, "Server version format doesn't match major version %d", major_version); - result = major_version * 10000; - } - else - elog(ERROR, "Unknown server version format %s", server_version_str); - - return result; -} - -/* - * Parse string representation of the program version. - */ -uint32 -parse_program_version(const char *program_version) -{ - int nfields; - int major = 0, - minor = 0, - micro = 0; - uint32 result = 0; - - if (program_version == NULL || program_version[0] == '\0') - return 0; - - nfields = sscanf(program_version, "%d.%d.%d", &major, &minor, µ); - if (nfields == 3) - result = major * 10000 + minor * 100 + micro; - else - elog(ERROR, "Unknown program version format %s", program_version); - - return result; -} - -const char * -status2str(BackupStatus status) -{ - if (status < BACKUP_STATUS_INVALID || BACKUP_STATUS_CORRUPT < status) - return "UNKNOWN"; - - return statusName[status]; -} - -const char * -status2str_color(BackupStatus status) -{ - char *status_str = pgut_malloc(20); - - /* UNKNOWN */ - if (status == BACKUP_STATUS_INVALID) - snprintf(status_str, 20, "%s%s%s", TC_YELLOW_BOLD, "UNKNOWN", TC_RESET); - /* CORRUPT, ERROR and ORPHAN */ - else if (status == BACKUP_STATUS_CORRUPT || status == BACKUP_STATUS_ERROR || - status == BACKUP_STATUS_ORPHAN) - snprintf(status_str, 20, "%s%s%s", TC_RED_BOLD, statusName[status], TC_RESET); - /* MERGING, MERGED, DELETING and DELETED */ - else if (status == BACKUP_STATUS_MERGING || status == BACKUP_STATUS_MERGED || - status == BACKUP_STATUS_DELETING || status == BACKUP_STATUS_DELETED) - snprintf(status_str, 20, "%s%s%s", TC_YELLOW_BOLD, statusName[status], TC_RESET); - /* OK and DONE */ - else - snprintf(status_str, 20, "%s%s%s", TC_GREEN_BOLD, statusName[status], TC_RESET); - - return status_str; -} - -BackupStatus -str2status(const char *status) -{ - BackupStatus i; - - for (i = BACKUP_STATUS_INVALID; i <= BACKUP_STATUS_CORRUPT; i++) - { - if (pg_strcasecmp(status, statusName[i]) == 0) return i; - } - - return BACKUP_STATUS_INVALID; -} - -bool -datapagemap_is_set(datapagemap_t *map, BlockNumber blkno) -{ - int offset; - int bitno; - - offset = blkno / 8; - bitno = blkno % 8; - - return (map->bitmapsize <= offset) ? false : (map->bitmap[offset] & (1 << bitno)) != 0; -} - -/* - * A debugging aid. Prints out the contents of the page map. - */ -void -datapagemap_print_debug(datapagemap_t *map) -{ - datapagemap_iterator_t *iter; - BlockNumber blocknum; - - iter = datapagemap_iterate(map); - while (datapagemap_next(iter, &blocknum)) - elog(VERBOSE, " block %u", blocknum); - - pg_free(iter); -} - -const char* -backup_id_of(pgBackup *backup) -{ - /* Change this Assert when backup_id will not be bound to start_time */ - Assert(backup->backup_id == backup->start_time || backup->start_time == 0); - - if (backup->backup_id_encoded[0] == '\x00') - { - base36enc_to(backup->backup_id, backup->backup_id_encoded); - } - return backup->backup_id_encoded; -} - -void -reset_backup_id(pgBackup *backup) -{ - backup->backup_id = INVALID_BACKUP_ID; - memset(backup->backup_id_encoded, 0, sizeof(backup->backup_id_encoded)); -} diff --git a/src/utils/configuration.c b/src/utils/configuration.c deleted file mode 100644 index 921555350..000000000 --- a/src/utils/configuration.c +++ /dev/null @@ -1,1602 +0,0 @@ -/*------------------------------------------------------------------------- - * - * configuration.c: - function implementations to work with pg_probackup - * configurations. - * - * Copyright (c) 2017-2019, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" -#include "configuration.h" -#include "logger.h" -#include "pgut.h" -#include "file.h" - -#include "datatype/timestamp.h" - -#include "getopt_long.h" - -#ifndef WIN32 -#include -#endif -#include -#include - -#define MAXPG_LSNCOMPONENT 8 - -/* - * Unit conversion tables. - * - * Copied from guc.c. - */ -#define MAX_UNIT_LEN 3 /* length of longest recognized unit string */ - -typedef struct -{ - char unit[MAX_UNIT_LEN + 1]; /* unit, as a string, like "kB" or - * "min" */ - int base_unit; /* OPTION_UNIT_XXX */ - int multiplier; /* If positive, multiply the value with this - * for unit -> base_unit conversion. If - * negative, divide (with the absolute value) */ -} unit_conversion; - -static const char *memory_units_hint = "Valid units for this parameter are \"kB\", \"MB\", \"GB\", and \"TB\"."; - -static const unit_conversion memory_unit_conversion_table[] = -{ - {"TB", OPTION_UNIT_KB, 1024 * 1024 * 1024}, - {"GB", OPTION_UNIT_KB, 1024 * 1024}, - {"MB", OPTION_UNIT_KB, 1024}, - {"KB", OPTION_UNIT_KB, 1}, - {"kB", OPTION_UNIT_KB, 1}, - - {"TB", OPTION_UNIT_BLOCKS, (1024 * 1024 * 1024) / (BLCKSZ / 1024)}, - {"GB", OPTION_UNIT_BLOCKS, (1024 * 1024) / (BLCKSZ / 1024)}, - {"MB", OPTION_UNIT_BLOCKS, 1024 / (BLCKSZ / 1024)}, - {"kB", OPTION_UNIT_BLOCKS, -(BLCKSZ / 1024)}, - - {"TB", OPTION_UNIT_XBLOCKS, (1024 * 1024 * 1024) / (XLOG_BLCKSZ / 1024)}, - {"GB", OPTION_UNIT_XBLOCKS, (1024 * 1024) / (XLOG_BLCKSZ / 1024)}, - {"MB", OPTION_UNIT_XBLOCKS, 1024 / (XLOG_BLCKSZ / 1024)}, - {"kB", OPTION_UNIT_XBLOCKS, -(XLOG_BLCKSZ / 1024)}, - - {""} /* end of table marker */ -}; - -static const char *time_units_hint = "Valid units for this parameter are \"ms\", \"s\", \"min\", \"h\", and \"d\"."; - -static const unit_conversion time_unit_conversion_table[] = -{ - {"d", OPTION_UNIT_MS, 1000 * 60 * 60 * 24}, - {"h", OPTION_UNIT_MS, 1000 * 60 * 60}, - {"min", OPTION_UNIT_MS, 1000 * 60}, - {"s", OPTION_UNIT_MS, 1000}, - {"ms", OPTION_UNIT_MS, 1}, - - {"d", OPTION_UNIT_S, 60 * 60 * 24}, - {"h", OPTION_UNIT_S, 60 * 60}, - {"min", OPTION_UNIT_S, 60}, - {"s", OPTION_UNIT_S, 1}, - {"ms", OPTION_UNIT_S, -1000}, - - {"d", OPTION_UNIT_MIN, 60 * 24}, - {"h", OPTION_UNIT_MIN, 60}, - {"min", OPTION_UNIT_MIN, 1}, - {"s", OPTION_UNIT_MIN, -60}, - {"ms", OPTION_UNIT_MIN, -1000 * 60}, - - {""} /* end of table marker */ -}; - -/* Order is important, keep it in sync with utils/configuration.h:enum ProbackupSubcmd declaration */ -static char const * const subcmd_names[] = -{ - "NO_CMD", - "init", - "add-instance", - "del-instance", - "archive-push", - "archive-get", - "backup", - "restore", - "validate", - "delete", - "merge", - "show", - "set-config", - "set-backup", - "show-config", - "checkdb", - "ssh", - "agent", - "help", - "version", - "catchup", -}; - -ProbackupSubcmd -parse_subcmd(char const * const subcmd_str) -{ - struct { - ProbackupSubcmd id; - char *name; - } - static const subcmd_additional_names[] = { - { HELP_CMD, "--help" }, - { HELP_CMD, "-?" }, - { VERSION_CMD, "--version" }, - { VERSION_CMD, "-V" }, - }; - - int i; - for(i = (int)NO_CMD + 1; i < sizeof(subcmd_names) / sizeof(subcmd_names[0]); ++i) - if(strcmp(subcmd_str, subcmd_names[i]) == 0) - return (ProbackupSubcmd)i; - for(i = 0; i < sizeof(subcmd_additional_names) / sizeof(subcmd_additional_names[0]); ++i) - if(strcmp(subcmd_str, subcmd_additional_names[i].name) == 0) - return subcmd_additional_names[i].id; - return NO_CMD; -} - -char const * -get_subcmd_name(ProbackupSubcmd const subcmd) -{ - Assert((int)subcmd < sizeof(subcmd_names) / sizeof(subcmd_names[0])); - return subcmd_names[(int)subcmd]; -} - -/* - * Reading functions. - */ - -static uint32 -option_length(const ConfigOption opts[]) -{ - uint32 len; - - for (len = 0; opts && opts[len].type; len++) { } - - return len; -} - -static int -option_has_arg(char type) -{ - switch (type) - { - case 'b': - case 'B': - return no_argument;//optional_argument; - default: - return required_argument; - } -} - -static void -option_copy(struct option dst[], const ConfigOption opts[], size_t len) -{ - size_t i; - - for (i = 0; i < len; i++) - { - dst[i].name = opts[i].lname; - dst[i].has_arg = option_has_arg(opts[i].type); - dst[i].flag = NULL; - dst[i].val = opts[i].sname; - } -} - -static ConfigOption * -option_find(int c, ConfigOption opts1[]) -{ - size_t i; - - for (i = 0; opts1 && opts1[i].type; i++) - if (opts1[i].sname == c) - return &opts1[i]; - - return NULL; /* not found */ -} - -static char * -longopts_to_optstring(const struct option opts[], const size_t len) -{ - size_t i; - char *result; - char *s; - - result = pgut_malloc(len * 2 + 1); - - s = result; - for (i = 0; i < len; i++) - { - if (!isprint(opts[i].val)) - continue; - *s++ = opts[i].val; - if (opts[i].has_arg != no_argument) - *s++ = ':'; - } - *s = '\0'; - - return result; -} - -/* - * Compare two strings ignore cases and ignore. - */ -static bool -key_equals(const char *lhs, const char *rhs) -{ - for (; *lhs && *rhs; lhs++, rhs++) - { - if (strchr("-_ ", *lhs)) - { - if (!strchr("-_ ", *rhs)) - return false; - } - else if (ToLower(*lhs) != ToLower(*rhs)) - return false; - } - - return *lhs == '\0' && *rhs == '\0'; -} - -static void -assign_option(ConfigOption *opt, const char *optarg, OptionSource src) -{ - const char *message; - - if (opt == NULL) - elog(ERROR, "Option is not found. Try \"%s --help\" for more information.\n", - PROGRAM_NAME); - - if (opt->source > src) - { - /* high prior value has been set already. */ - return; - } - /* Allow duplicate entries for function option */ - else if (src >= SOURCE_CMD && opt->source >= src && opt->type != 'f') - { - message = "specified only once"; - } - else - { - OptionSource orig_source = opt->source; - - /* can be overwritten if non-command line source */ - opt->source = src; - - switch (opt->type) - { - case 'b': - case 'B': - if (optarg == NULL) - { - *((bool *) opt->var) = (opt->type == 'b'); - return; - } - else if (parse_bool(optarg, (bool *) opt->var)) - { - return; - } - message = "a boolean"; - break; - case 'f': - ((option_assign_fn) opt->var)(opt, optarg); - return; - case 'i': - if (parse_int32(optarg, opt->var, opt->flags)) - return; - message = "a 32bit signed integer"; - break; - case 'u': - if (parse_uint32(optarg, opt->var, opt->flags)) - return; - message = "a 32bit unsigned integer"; - break; - case 'I': - if (parse_int64(optarg, opt->var, opt->flags)) - return; - message = "a 64bit signed integer"; - break; - case 'U': - if (parse_uint64(optarg, opt->var, opt->flags)) - return; - message = "a 64bit unsigned integer"; - break; - case 's': - if (orig_source != SOURCE_DEFAULT) - free(*(char **) opt->var); - - /* 'none' and 'off' are always disable the string parameter */ - //if (optarg && (pg_strcasecmp(optarg, "none") == 0)) - //{ - // *(char **) opt->var = "none"; - // return; - //} - - *(char **) opt->var = pgut_strdup(optarg); - if (strcmp(optarg,"") != 0) - return; - message = "a valid string"; - break; - case 't': - if (parse_time(optarg, opt->var, - opt->source == SOURCE_FILE)) - return; - message = "a time"; - break; - default: - elog(ERROR, "Invalid option type: %c", opt->type); - return; /* keep compiler quiet */ - } - } - - if (optarg) - { - if (isprint(opt->sname)) - elog(ERROR, "Option -%c, --%s should be %s: '%s'", - opt->sname, opt->lname, message, optarg); - else - elog(ERROR, "Option --%s should be %s: '%s'", - opt->lname, message, optarg); - } - else - { - if (isprint(opt->sname)) - elog(ERROR, "Option -%c, --%s should be %s", - opt->sname, opt->lname, message); - else - elog(ERROR, "Option --%s should be %s", - opt->lname, message); - } -} - -static const char * -skip_space(const char *str, const char *line) -{ - while (IsSpace(*str)) { str++; } - return str; -} - -static const char * -get_next_token(const char *src, char *dst, const char *line) -{ - const char *s; - int i; - int j; - - if ((s = skip_space(src, line)) == NULL) - return NULL; - - /* parse quoted string */ - if (*s == '\'') - { - s++; - for (i = 0, j = 0; s[i] != '\0'; i++) - { - if (s[i] == '\'') - { - i++; - /* doubled quote becomes just one quote */ - if (s[i] == '\'') - dst[j] = s[i]; - else - break; - } - else - dst[j] = s[i]; - j++; - } - } - else - { - i = j = strcspn(s, "#\n\r\t\v"); - memcpy(dst, s, j); - } - - dst[j] = '\0'; - return s + i; -} - -static bool -parse_pair(const char buffer[], char key[], char value[]) -{ - const char *start; - const char *end; - - key[0] = value[0] = '\0'; - - /* - * parse key - */ - start = buffer; - if ((start = skip_space(start, buffer)) == NULL) - return false; - - end = start + strcspn(start, "=# \n\r\t\v"); - - /* skip blank buffer */ - if (end - start <= 0) - { - if (*start == '=') - elog(ERROR, "Syntax error in \"%s\"", buffer); - return false; - } - - /* key found */ - strncpy(key, start, end - start); - key[end - start] = '\0'; - - /* find key and value split char */ - if ((start = skip_space(end, buffer)) == NULL) - return false; - - if (*start != '=') - { - elog(ERROR, "Syntax error in \"%s\"", buffer); - return false; - } - - start++; - - /* - * parse value - */ - if ((end = get_next_token(start, value, buffer)) == NULL) - return false; - - if ((start = skip_space(end, buffer)) == NULL) - return false; - - if (*start != '\0' && *start != '#') - { - elog(ERROR, "Syntax error in \"%s\"", buffer); - return false; - } - - return true; -} - -/* - * Returns the current user name. - */ -static const char * -get_username(void) -{ - const char *ret; - -#ifndef WIN32 - struct passwd *pw; - - pw = getpwuid(geteuid()); - ret = (pw ? pw->pw_name : NULL); -#else - static char username[128]; /* remains after function exit */ - DWORD len = sizeof(username) - 1; - - if (GetUserName(username, &len)) - ret = username; - else - { - _dosmaperr(GetLastError()); - ret = NULL; - } -#endif - - if (ret == NULL) - elog(ERROR, "Could not get current user name: %s", strerror(errno)); - return ret; -} - -/* - * Process options passed from command line. - * TODO: currectly argument parsing treat missing argument for options - * as invalid option - */ -int -config_get_opt(int argc, char **argv, ConfigOption cmd_options[], - ConfigOption options[]) -{ - int c; - int optindex = 0; - char *optstring; - struct option *longopts; - uint32 cmd_len, - len; - - cmd_len = option_length(cmd_options); - len = option_length(options); - - longopts = pgut_newarray(struct option, - cmd_len + len + 1 /* zero/end option */); - - /* Concatenate two options */ - option_copy(longopts, cmd_options, cmd_len); - option_copy(longopts + cmd_len, options, len + 1); - - optstring = longopts_to_optstring(longopts, cmd_len + len); - - opterr = 0; - /* Assign named options */ - while ((c = getopt_long(argc, argv, optstring, longopts, &optindex)) != -1) - { - ConfigOption *opt; - - if (c == '?') - { - elog(ERROR, "Option '%s' requires an argument. Try \"%s --help\" for more information.", - argv[optind-1], PROGRAM_NAME); - } - opt = option_find(c, cmd_options); - if (opt == NULL) - opt = option_find(c, options); - - if (opt - && opt->allowed < SOURCE_CMD && opt->allowed != SOURCE_CMD_STRICT) - elog(ERROR, "Option %s cannot be specified in command line", - opt->lname); - /* Check 'opt == NULL' is performed in assign_option() */ - assign_option(opt, optarg, SOURCE_CMD); - } - - pgut_free(optstring); - pgut_free(longopts); - - return optind; -} - -/* - * Get configuration from configuration file. - * Return number of parsed options. - */ -int -config_read_opt(const char *path, ConfigOption options[], int elevel, - bool strict, bool missing_ok) -{ - FILE *fp; - char buf[4096]; - char key[1024]; - char value[2048]; - int parsed_options = 0; - - if (!options) - return parsed_options; - - if ((fp = pgut_fopen(path, "rt", missing_ok)) == NULL) - return parsed_options; - - while (fgets(buf, lengthof(buf), fp)) - { - size_t i; - - for (i = strlen(buf); i > 0 && IsSpace(buf[i - 1]); i--) - buf[i - 1] = '\0'; - - if (parse_pair(buf, key, value)) - { - for (i = 0; options[i].type; i++) - { - ConfigOption *opt = &options[i]; - - if (key_equals(key, opt->lname)) - { - if (opt->allowed < SOURCE_FILE && - opt->allowed != SOURCE_FILE_STRICT) - elog(elevel, "Option %s cannot be specified in file", - opt->lname); - else if (opt->source <= SOURCE_FILE) - { - assign_option(opt, value, SOURCE_FILE); - parsed_options++; - } - break; - } - } - if (strict && !options[i].type) - elog(elevel, "Invalid option \"%s\" in file \"%s\"", key, path); - } - } - - if (ferror(fp)) - elog(ERROR, "Failed to read from file: \"%s\"", path); - - fio_close_stream(fp); - - return parsed_options; -} - -/* - * Process options passed as environment variables. - */ -void -config_get_opt_env(ConfigOption options[]) -{ - size_t i; - - for (i = 0; options && options[i].type; i++) - { - ConfigOption *opt = &options[i]; - const char *value = NULL; - - /* If option was already set do not check env */ - if (opt->source > SOURCE_ENV || opt->allowed < SOURCE_ENV) - continue; - - if (strcmp(opt->lname, "pgdata") == 0) - value = getenv("PGDATA"); - if (strcmp(opt->lname, "port") == 0) - value = getenv("PGPORT"); - if (strcmp(opt->lname, "host") == 0) - value = getenv("PGHOST"); - if (strcmp(opt->lname, "username") == 0) - value = getenv("PGUSER"); - if (strcmp(opt->lname, "pgdatabase") == 0) - { - value = getenv("PGDATABASE"); - if (value == NULL) - value = getenv("PGUSER"); - if (value == NULL) - value = get_username(); - } - - if (value) - assign_option(opt, value, SOURCE_ENV); - } -} - -/* - * Manually set source of the option. Find it by the pointer var. - */ -void -config_set_opt(ConfigOption options[], void *var, OptionSource source) -{ - int i; - - for (i = 0; options[i].type; i++) - { - ConfigOption *opt = &options[i]; - - if (opt->var == var) - { - if ((opt->allowed == SOURCE_FILE_STRICT && source != SOURCE_FILE) || - (opt->allowed == SOURCE_CMD_STRICT && source != SOURCE_CMD) || - (opt->allowed < source && opt->allowed >= SOURCE_ENV)) - elog(ERROR, "Invalid option source %d for %s", - source, opt->lname); - - opt->source = source; - break; - } - } -} - -/* - * Return value of the function in the string representation. Result is - * allocated string. - */ -char * -option_get_value(ConfigOption *opt) -{ - int64 value = 0; - uint64 value_u = 0; - const char *unit = NULL; - - /* - * If it is defined a unit for the option get readable value from base with - * unit name. - */ - if (opt->flags & OPTION_UNIT) - { - if (opt->type == 'i') - convert_from_base_unit(*((int32 *) opt->var), - opt->flags & OPTION_UNIT, &value, &unit); - else if (opt->type == 'I') - convert_from_base_unit(*((int64 *) opt->var), - opt->flags & OPTION_UNIT, &value, &unit); - else if (opt->type == 'u') - convert_from_base_unit_u(*((uint32 *) opt->var), - opt->flags & OPTION_UNIT, &value_u, &unit); - else if (opt->type == 'U') - convert_from_base_unit_u(*((uint64 *) opt->var), - opt->flags & OPTION_UNIT, &value_u, &unit); - } - - /* Get string representation itself */ - switch (opt->type) - { - case 'b': - case 'B': - return psprintf("%s", *((bool *) opt->var) ? "true" : "false"); - case 'i': - if (opt->flags & OPTION_UNIT) - return psprintf(INT64_FORMAT "%s", value, unit); - else - return psprintf("%d", *((int32 *) opt->var)); - case 'u': - if (opt->flags & OPTION_UNIT) - return psprintf(UINT64_FORMAT "%s", value_u, unit); - else - return psprintf("%u", *((uint32 *) opt->var)); - case 'I': - if (opt->flags & OPTION_UNIT) - return psprintf(INT64_FORMAT "%s", value, unit); - else - return psprintf(INT64_FORMAT, *((int64 *) opt->var)); - case 'U': - if (opt->flags & OPTION_UNIT) - return psprintf(UINT64_FORMAT "%s", value_u, unit); - else - return psprintf(UINT64_FORMAT, *((uint64 *) opt->var)); - case 's': - if (*((char **) opt->var) == NULL) - return NULL; - /* 'none' and 'off' are always disable the string parameter */ - //if ((pg_strcasecmp(*((char **) opt->var), "none") == 0) || - // (pg_strcasecmp(*((char **) opt->var), "off") == 0)) - // return NULL; - return pstrdup(*((char **) opt->var)); - case 't': - { - char *timestamp; - time_t t = *((time_t *) opt->var); - - if (t > 0) - { - timestamp = palloc(100); - time2iso(timestamp, 100, t, false); - } - else - timestamp = palloc0(1 /* just null termination */); - return timestamp; - } - default: - elog(ERROR, "Invalid option type: %c", opt->type); - return NULL; /* keep compiler quiet */ - } -} - -/* - * Parsing functions - */ - -/* - * Convert a value from one of the human-friendly units ("kB", "min" etc.) - * to the given base unit. 'value' and 'unit' are the input value and unit - * to convert from. The converted value is stored in *base_value. - * - * Returns true on success, false if the input unit is not recognized. - */ -static bool -convert_to_base_unit(int64 value, const char *unit, - int base_unit, int64 *base_value) -{ - const unit_conversion *table; - int i; - - if (base_unit & OPTION_UNIT_MEMORY) - table = memory_unit_conversion_table; - else - table = time_unit_conversion_table; - - for (i = 0; *table[i].unit; i++) - { - if (base_unit == table[i].base_unit && - strcmp(unit, table[i].unit) == 0) - { - if (table[i].multiplier < 0) - *base_value = value / (-table[i].multiplier); - else - { - /* Check for integer overflow first */ - if (value > PG_INT64_MAX / table[i].multiplier) - return false; - - *base_value = value * table[i].multiplier; - } - return true; - } - } - return false; -} - -/* - * Unsigned variant of convert_to_base_unit() - */ -static bool -convert_to_base_unit_u(uint64 value, const char *unit, - int base_unit, uint64 *base_value) -{ - const unit_conversion *table; - int i; - - if (base_unit & OPTION_UNIT_MEMORY) - table = memory_unit_conversion_table; - else - table = time_unit_conversion_table; - - for (i = 0; *table[i].unit; i++) - { - if (base_unit == table[i].base_unit && - strcmp(unit, table[i].unit) == 0) - { - if (table[i].multiplier < 0) - *base_value = value / (-table[i].multiplier); - else - { - /* Check for integer overflow first */ - if (value > PG_UINT64_MAX / table[i].multiplier) - return false; - - *base_value = value * table[i].multiplier; - } - return true; - } - } - return false; -} - -static bool -parse_unit(char *unit_str, int flags, int64 value, int64 *base_value) -{ - /* allow whitespace between integer and unit */ - while (isspace((unsigned char) *unit_str)) - unit_str++; - - /* Handle possible unit */ - if (*unit_str != '\0') - { - char unit[MAX_UNIT_LEN + 1]; - int unitlen; - bool converted = false; - - if ((flags & OPTION_UNIT) == 0) - return false; /* this setting does not accept a unit */ - - unitlen = 0; - while (*unit_str != '\0' && !isspace((unsigned char) *unit_str) && - unitlen < MAX_UNIT_LEN) - unit[unitlen++] = *(unit_str++); - unit[unitlen] = '\0'; - /* allow whitespace after unit */ - while (isspace((unsigned char) *unit_str)) - unit_str++; - - if (*unit_str == '\0') - converted = convert_to_base_unit(value, unit, (flags & OPTION_UNIT), - base_value); - if (!converted) - return false; - } - - return true; -} - -/* - * Unsigned variant of parse_unit() - */ -static bool -parse_unit_u(char *unit_str, int flags, uint64 value, uint64 *base_value) -{ - /* allow whitespace between integer and unit */ - while (isspace((unsigned char) *unit_str)) - unit_str++; - - /* Handle possible unit */ - if (*unit_str != '\0') - { - char unit[MAX_UNIT_LEN + 1]; - int unitlen; - bool converted = false; - - if ((flags & OPTION_UNIT) == 0) - return false; /* this setting does not accept a unit */ - - unitlen = 0; - while (*unit_str != '\0' && !isspace((unsigned char) *unit_str) && - unitlen < MAX_UNIT_LEN) - unit[unitlen++] = *(unit_str++); - unit[unitlen] = '\0'; - /* allow whitespace after unit */ - while (isspace((unsigned char) *unit_str)) - unit_str++; - - if (*unit_str == '\0') - converted = convert_to_base_unit_u(value, unit, - (flags & OPTION_UNIT), - base_value); - if (!converted) - return false; - } - - return true; -} - -/* - * Try to interpret value as boolean value. Valid values are: true, - * false, yes, no, on, off, 1, 0; as well as unique prefixes thereof. - * If the string parses okay, return true, else false. - * If okay and result is not NULL, return the value in *result. - */ -bool -parse_bool(const char *value, bool *result) -{ - return parse_bool_with_len(value, strlen(value), result); -} - -bool -parse_bool_with_len(const char *value, size_t len, bool *result) -{ - switch (*value) - { - case 't': - case 'T': - if (pg_strncasecmp(value, "true", len) == 0) - { - if (result) - *result = true; - return true; - } - break; - case 'f': - case 'F': - if (pg_strncasecmp(value, "false", len) == 0) - { - if (result) - *result = false; - return true; - } - break; - case 'y': - case 'Y': - if (pg_strncasecmp(value, "yes", len) == 0) - { - if (result) - *result = true; - return true; - } - break; - case 'n': - case 'N': - if (pg_strncasecmp(value, "no", len) == 0) - { - if (result) - *result = false; - return true; - } - break; - case 'o': - case 'O': - /* 'o' is not unique enough */ - if (pg_strncasecmp(value, "on", (len > 2 ? len : 2)) == 0) - { - if (result) - *result = true; - return true; - } - else if (pg_strncasecmp(value, "off", (len > 2 ? len : 2)) == 0) - { - if (result) - *result = false; - return true; - } - break; - case '1': - if (len == 1) - { - if (result) - *result = true; - return true; - } - break; - case '0': - if (len == 1) - { - if (result) - *result = false; - return true; - } - break; - default: - break; - } - - if (result) - *result = false; /* suppress compiler warning */ - return false; -} - -/* - * Parse string as 32bit signed int. - * valid range: -2147483648 ~ 2147483647 - */ -bool -parse_int32(const char *value, int32 *result, int flags) -{ - int64 val; - char *endptr; - - if (strcmp(value, INFINITE_STR) == 0) - { - *result = PG_INT32_MAX; - return true; - } - - errno = 0; - val = strtol(value, &endptr, 0); - if (endptr == value || (*endptr && flags == 0)) - return false; - - /* Check for integer overflow */ - if (errno == ERANGE || val != (int64) ((int32) val)) - return false; - - if (!parse_unit(endptr, flags, val, &val)) - return false; - - /* Check for integer overflow again */ - if (val != (int64) ((int32) val)) - return false; - - *result = val; - - return true; -} - -/* - * Parse string as 32bit unsigned int. - * valid range: 0 ~ 4294967295 (2^32-1) - */ -bool -parse_uint32(const char *value, uint32 *result, int flags) -{ - uint64 val; - char *endptr; - - if (strcmp(value, INFINITE_STR) == 0) - { - *result = PG_UINT32_MAX; - return true; - } - - errno = 0; - val = strtoul(value, &endptr, 0); - if (endptr == value || (*endptr && flags == 0)) - return false; - - /* Check for integer overflow */ - if (errno == ERANGE || val != (uint64) ((uint32) val)) - return false; - - if (!parse_unit_u(endptr, flags, val, &val)) - return false; - - /* Check for integer overflow again */ - if (val != (uint64) ((uint32) val)) - return false; - - *result = val; - - return true; -} - -/* - * Parse string as int64 - * valid range: -9223372036854775808 ~ 9223372036854775807 - */ -bool -parse_int64(const char *value, int64 *result, int flags) -{ - int64 val; - char *endptr; - - if (strcmp(value, INFINITE_STR) == 0) - { - *result = PG_INT64_MAX; - return true; - } - - errno = 0; -#if defined(HAVE_LONG_INT_64) - val = strtol(value, &endptr, 0); -#elif defined(HAVE_LONG_LONG_INT_64) - val = strtoll(value, &endptr, 0); -#else - val = strtol(value, &endptr, 0); -#endif - if (endptr == value || (*endptr && flags == 0)) - return false; - - if (errno == ERANGE) - return false; - - if (!parse_unit(endptr, flags, val, &val)) - return false; - - *result = val; - - return true; -} - -/* - * Parse string as uint64 - * valid range: 0 ~ (2^64-1) - */ -bool -parse_uint64(const char *value, uint64 *result, int flags) -{ - uint64 val; - char *endptr; - - if (strcmp(value, INFINITE_STR) == 0) - { - *result = PG_UINT64_MAX; - return true; - } - - errno = 0; -#if defined(HAVE_LONG_INT_64) - val = strtoul(value, &endptr, 0); -#elif defined(HAVE_LONG_LONG_INT_64) - val = strtoull(value, &endptr, 0); -#else - val = strtoul(value, &endptr, 0); -#endif - if (endptr == value || (*endptr && flags == 0)) - return false; - - if (errno == ERANGE) - return false; - - if (!parse_unit_u(endptr, flags, val, &val)) - return false; - - *result = val; - - return true; -} - -/* - * Convert ISO-8601 format string to time_t value. - * - * If utc_default is true, then if timezone offset isn't specified tz will be - * +00:00. - * - * TODO: '0' converted into '2000-01-01 00:00:00'. Example: set-backup --expire-time=0 - */ -bool -parse_time(const char *value, time_t *result, bool utc_default) -{ - size_t len; - int fields_num, - tz = 0, - i; - bool tz_set = false; - char *tmp; - struct tm tm; - char junk[2]; - - char *local_tz = getenv("TZ"); - - /* tmp = replace( value, !isalnum, ' ' ) */ - tmp = pgut_malloc(strlen(value) + 1); - if(!tmp) return false; - len = 0; - fields_num = 1; - - while (*value) - { - if (IsAlnum(*value)) - { - tmp[len++] = *value; - value++; - } - else if (fields_num < 6) - { - fields_num++; - tmp[len++] = ' '; - value++; - } - /* timezone field is 7th */ - else if ((*value == '-' || *value == '+') && fields_num == 6) - { - int hr, - min, - sec = 0; - char *cp; - - errno = 0; - hr = strtol(value + 1, &cp, 10); - if ((value + 1) == cp || errno == ERANGE) - { - pfree(tmp); - return false; - } - - /* explicit delimiter? */ - if (*cp == ':') - { - errno = 0; - min = strtol(cp + 1, &cp, 10); - if (errno == ERANGE) - { - pfree(tmp); - return false; - } - if (*cp == ':') - { - errno = 0; - sec = strtol(cp + 1, &cp, 10); - if (errno == ERANGE) - { - pfree(tmp); - return false; - } - } - } - /* otherwise, might have run things together... */ - else if (*cp == '\0' && strlen(value) > 3) - { - min = hr % 100; - hr = hr / 100; - /* we could, but don't, support a run-together hhmmss format */ - } - else - min = 0; - - /* Range-check the values; see notes in datatype/timestamp.h */ - if (hr < 0 || hr > MAX_TZDISP_HOUR) - { - pfree(tmp); - return false; - } - if (min < 0 || min >= MINS_PER_HOUR) - { - pfree(tmp); - return false; - } - if (sec < 0 || sec >= SECS_PER_MINUTE) - { - pfree(tmp); - return false; - } - - tz = (hr * MINS_PER_HOUR + min) * SECS_PER_MINUTE + sec; - if (*value == '-') - tz = -tz; - - tz_set = true; - - fields_num++; - value = cp; - } - /* wrong format */ - else if (!IsSpace(*value)) - { - pfree(tmp); - return false; - } - else - value++; - } - tmp[len] = '\0'; - - /* parse for "YYYY-MM-DD HH:MI:SS" */ - memset(&tm, 0, sizeof(tm)); - tm.tm_year = 0; /* tm_year is year - 1900 */ - tm.tm_mon = 0; /* tm_mon is 0 - 11 */ - tm.tm_mday = 1; /* tm_mday is 1 - 31 */ - tm.tm_hour = 0; - tm.tm_min = 0; - tm.tm_sec = 0; - i = sscanf(tmp, "%04d %02d %02d %02d %02d %02d%1s", - &tm.tm_year, &tm.tm_mon, &tm.tm_mday, - &tm.tm_hour, &tm.tm_min, &tm.tm_sec, junk); - pfree(tmp); - - if (i < 3 || i > 6) - return false; - - /* adjust year */ - if (tm.tm_year < 100) - tm.tm_year += 2000 - 1900; - else if (tm.tm_year >= 1900) - tm.tm_year -= 1900; - - /* adjust month */ - if (i > 1) - tm.tm_mon -= 1; - - /* determine whether Daylight Saving Time is in effect */ - tm.tm_isdst = -1; - - /* - * If tz is not set, - * treat it as UTC if requested, otherwise as local timezone - */ - if (tz_set || utc_default) - { - /* set timezone to UTC */ - pgut_setenv("TZ", "UTC"); - tzset(); - } - - /* convert time to utc unix time */ - *result = mktime(&tm); - - /* return old timezone back if any */ - if (local_tz) - pgut_setenv("TZ", local_tz); - else - pgut_unsetenv("TZ"); - - tzset(); - - /* adjust time zone */ - if (tz_set || utc_default) - { - /* UTC time */ - *result -= tz; - } - - return true; -} - -/* - * Try to parse value as an integer. The accepted formats are the - * usual decimal, octal, or hexadecimal formats, optionally followed by - * a unit name if "flags" indicates a unit is allowed. - * - * If the string parses okay, return true, else false. - * If okay and result is not NULL, return the value in *result. - * If not okay and hintmsg is not NULL, *hintmsg is set to a suitable - * HINT message, or NULL if no hint provided. - */ -bool -parse_int(const char *value, int *result, int flags, const char **hintmsg) -{ - int64 val; - char *endptr; - - /* To suppress compiler warnings, always set output params */ - if (result) - *result = 0; - if (hintmsg) - *hintmsg = NULL; - - /* We assume here that int64 is at least as wide as long */ - errno = 0; - val = strtol(value, &endptr, 0); - - if (endptr == value) - return false; /* no HINT for integer syntax error */ - - if (errno == ERANGE || val != (int64) ((int32) val)) - { - if (hintmsg) - *hintmsg = "Value exceeds integer range."; - return false; - } - - /* allow whitespace between integer and unit */ - while (isspace((unsigned char) *endptr)) - endptr++; - - /* Handle possible unit */ - if (*endptr != '\0') - { - char unit[MAX_UNIT_LEN + 1]; - int unitlen; - bool converted = false; - - if ((flags & OPTION_UNIT) == 0) - return false; /* this setting does not accept a unit */ - - unitlen = 0; - while (*endptr != '\0' && !isspace((unsigned char) *endptr) && - unitlen < MAX_UNIT_LEN) - unit[unitlen++] = *(endptr++); - unit[unitlen] = '\0'; - /* allow whitespace after unit */ - while (isspace((unsigned char) *endptr)) - endptr++; - - if (*endptr == '\0') - converted = convert_to_base_unit(val, unit, (flags & OPTION_UNIT), - &val); - if (!converted) - { - /* invalid unit, or garbage after the unit; set hint and fail. */ - if (hintmsg) - { - if (flags & OPTION_UNIT_MEMORY) - *hintmsg = memory_units_hint; - else - *hintmsg = time_units_hint; - } - return false; - } - - /* Check for overflow due to units conversion */ - if (val != (int64) ((int32) val)) - { - if (hintmsg) - *hintmsg = "Value exceeds integer range."; - return false; - } - } - - if (result) - *result = (int) val; - return true; -} - -bool -parse_lsn(const char *value, XLogRecPtr *result) -{ - uint32 xlogid; - uint32 xrecoff; - int len1; - int len2; - - len1 = strspn(value, "0123456789abcdefABCDEF"); - if (len1 < 1 || len1 > MAXPG_LSNCOMPONENT || value[len1] != '/') - elog(ERROR, "Invalid LSN \"%s\"", value); - len2 = strspn(value + len1 + 1, "0123456789abcdefABCDEF"); - if (len2 < 1 || len2 > MAXPG_LSNCOMPONENT || value[len1 + 1 + len2] != '\0') - elog(ERROR, "Invalid LSN \"%s\"", value); - - if (sscanf(value, "%X/%X", &xlogid, &xrecoff) == 2) - *result = (XLogRecPtr) ((uint64) xlogid << 32) | xrecoff; - else - { - elog(ERROR, "Invalid LSN \"%s\"", value); - return false; - } - - return true; -} - -/* - * Convert a value in some base unit to a human-friendly unit. The output - * unit is chosen so that it's the greatest unit that can represent the value - * without loss. For example, if the base unit is GUC_UNIT_KB, 1024 is - * converted to 1 MB, but 1025 is represented as 1025 kB. - */ -void -convert_from_base_unit(int64 base_value, int base_unit, - int64 *value, const char **unit) -{ - const unit_conversion *table; - int i; - - *unit = NULL; - - if (base_unit & OPTION_UNIT_MEMORY) - table = memory_unit_conversion_table; - else - table = time_unit_conversion_table; - - for (i = 0; *table[i].unit; i++) - { - if (base_unit == table[i].base_unit) - { - /* - * Accept the first conversion that divides the value evenly. We - * assume that the conversions for each base unit are ordered from - * greatest unit to the smallest! - */ - if (table[i].multiplier < 0) - { - /* Check for integer overflow first */ - if (base_value > PG_INT64_MAX / (-table[i].multiplier)) - continue; - - *value = base_value * (-table[i].multiplier); - *unit = table[i].unit; - break; - } - else if (base_value % table[i].multiplier == 0) - { - *value = base_value / table[i].multiplier; - *unit = table[i].unit; - break; - } - } - } - - Assert(*unit != NULL); -} - -/* - * Unsigned variant of convert_from_base_unit() - */ -void -convert_from_base_unit_u(uint64 base_value, int base_unit, - uint64 *value, const char **unit) -{ - const unit_conversion *table; - int i; - - *unit = NULL; - - if (base_unit & OPTION_UNIT_MEMORY) - table = memory_unit_conversion_table; - else - table = time_unit_conversion_table; - - for (i = 0; *table[i].unit; i++) - { - if (base_unit == table[i].base_unit) - { - /* - * Accept the first conversion that divides the value evenly. We - * assume that the conversions for each base unit are ordered from - * greatest unit to the smallest! - */ - if (table[i].multiplier < 0) - { - /* Check for integer overflow first */ - if (base_value > PG_UINT64_MAX / (-table[i].multiplier)) - continue; - - *value = base_value * (-table[i].multiplier); - *unit = table[i].unit; - break; - } - else if (base_value % table[i].multiplier == 0) - { - *value = base_value / table[i].multiplier; - *unit = table[i].unit; - break; - } - } - } - - Assert(*unit != NULL); -} - -/* - * Convert time_t value to ISO-8601 format string. Always set timezone offset. - */ -void -time2iso(char *buf, size_t len, time_t time, bool utc) -{ - struct tm *ptm = NULL; - time_t gmt; - time_t offset; - char *ptr = buf; - - /* set timezone to UTC if requested */ - if (utc) - { - ptm = gmtime(&time); - strftime(ptr, len, "%Y-%m-%d %H:%M:%S+00", ptm); - return; - } - - ptm = gmtime(&time); - gmt = mktime(ptm); - ptm = localtime(&time); - - /* adjust timezone offset */ - offset = time - gmt + (ptm->tm_isdst ? 3600 : 0); - - strftime(ptr, len, "%Y-%m-%d %H:%M:%S", ptm); - - ptr += strlen(ptr); - snprintf(ptr, len - (ptr - buf), "%c%02d", - (offset >= 0) ? '+' : '-', - abs((int) offset) / SECS_PER_HOUR); - - if (abs((int) offset) % SECS_PER_HOUR != 0) - { - ptr += strlen(ptr); - snprintf(ptr, len - (ptr - buf), ":%02d", - abs((int) offset % SECS_PER_HOUR) / SECS_PER_MINUTE); - } -} diff --git a/src/utils/configuration.h b/src/utils/configuration.h deleted file mode 100644 index 2c6ea3eec..000000000 --- a/src/utils/configuration.h +++ /dev/null @@ -1,134 +0,0 @@ -/*------------------------------------------------------------------------- - * - * configuration.h: - prototypes of functions and structures for - * configuration. - * - * Copyright (c) 2018-2019, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#ifndef CONFIGURATION_H -#define CONFIGURATION_H - -#include "postgres_fe.h" -#include "access/xlogdefs.h" - -#define INFINITE_STR "INFINITE" - -/* Order is important, keep it in sync with configuration.c:subcmd_names[] and help.c:help_command() */ -typedef enum ProbackupSubcmd -{ - NO_CMD = 0, - INIT_CMD, - ADD_INSTANCE_CMD, - DELETE_INSTANCE_CMD, - ARCHIVE_PUSH_CMD, - ARCHIVE_GET_CMD, - BACKUP_CMD, - RESTORE_CMD, - VALIDATE_CMD, - DELETE_CMD, - MERGE_CMD, - SHOW_CMD, - SET_CONFIG_CMD, - SET_BACKUP_CMD, - SHOW_CONFIG_CMD, - CHECKDB_CMD, - SSH_CMD, - AGENT_CMD, - HELP_CMD, - VERSION_CMD, - CATCHUP_CMD, -} ProbackupSubcmd; - -typedef enum OptionSource -{ - SOURCE_DEFAULT, - SOURCE_FILE_STRICT, - SOURCE_CMD_STRICT, - SOURCE_ENV, - SOURCE_FILE, - SOURCE_CMD, - SOURCE_CONST -} OptionSource; - -typedef struct ConfigOption ConfigOption; - -typedef void (*option_assign_fn) (ConfigOption *opt, const char *arg); -/* Returns allocated string value */ -typedef char *(*option_get_fn) (ConfigOption *opt); - -/* - * type: - * b: bool (true) - * B: bool (false) - * f: option_fn - * i: 32bit signed integer - * u: 32bit unsigned integer - * I: 64bit signed integer - * U: 64bit unsigned integer - * s: string - * t: time_t - */ -struct ConfigOption -{ - char type; - uint8 sname; /* short name */ - const char *lname; /* long name */ - void *var; /* pointer to variable */ - OptionSource allowed; /* allowed source */ - OptionSource source; /* actual source */ - const char *group; /* option group name */ - int flags; /* option unit */ - option_get_fn get_value; /* function to get the value as a string, - should return allocated string*/ -}; - -/* - * bit values in "flags" of an option - */ -#define OPTION_UNIT_KB 0x1000 /* value is in kilobytes */ -#define OPTION_UNIT_BLOCKS 0x2000 /* value is in blocks */ -#define OPTION_UNIT_XBLOCKS 0x3000 /* value is in xlog blocks */ -#define OPTION_UNIT_XSEGS 0x4000 /* value is in xlog segments */ -#define OPTION_UNIT_MEMORY 0xF000 /* mask for size-related units */ - -#define OPTION_UNIT_MS 0x10000 /* value is in milliseconds */ -#define OPTION_UNIT_S 0x20000 /* value is in seconds */ -#define OPTION_UNIT_MIN 0x30000 /* value is in minutes */ -#define OPTION_UNIT_TIME 0xF0000 /* mask for time-related units */ - -#define OPTION_UNIT (OPTION_UNIT_MEMORY | OPTION_UNIT_TIME) - -extern ProbackupSubcmd parse_subcmd(char const * const subcmd_str); -extern char const *get_subcmd_name(ProbackupSubcmd const subcmd); -extern int config_get_opt(int argc, char **argv, ConfigOption cmd_options[], - ConfigOption options[]); -extern int config_read_opt(const char *path, ConfigOption options[], int elevel, - bool strict, bool missing_ok); -extern void config_get_opt_env(ConfigOption options[]); -extern void config_set_opt(ConfigOption options[], void *var, - OptionSource source); - -extern char *option_get_value(ConfigOption *opt); - -extern bool parse_bool(const char *value, bool *result); -extern bool parse_bool_with_len(const char *value, size_t len, bool *result); -extern bool parse_int32(const char *value, int32 *result, int flags); -extern bool parse_uint32(const char *value, uint32 *result, int flags); -extern bool parse_int64(const char *value, int64 *result, int flags); -extern bool parse_uint64(const char *value, uint64 *result, int flags); -extern bool parse_time(const char *value, time_t *result, bool utc_default); -extern bool parse_int(const char *value, int *result, int flags, - const char **hintmsg); -extern bool parse_lsn(const char *value, XLogRecPtr *result); - -extern void time2iso(char *buf, size_t len, time_t time, bool utc); - -extern void convert_from_base_unit(int64 base_value, int base_unit, - int64 *value, const char **unit); -extern void convert_from_base_unit_u(uint64 base_value, int base_unit, - uint64 *value, const char **unit); - -#endif /* CONFIGURATION_H */ diff --git a/src/utils/file.c b/src/utils/file.c deleted file mode 100644 index e062a2133..000000000 --- a/src/utils/file.c +++ /dev/null @@ -1,3999 +0,0 @@ -#include -#include - -#include "pg_probackup.h" -/* sys/stat.h must be included after pg_probackup.h (see problems with compilation for windows described in PGPRO-5750) */ -#include - -#include "file.h" -#include "storage/checksum.h" - -#define PRINTF_BUF_SIZE 1024 -#define FILE_PERMISSIONS 0600 - -static __thread unsigned long fio_fdset = 0; -static __thread void* fio_stdin_buffer; -static __thread int fio_stdout = 0; -static __thread int fio_stdin = 0; -static __thread int fio_stderr = 0; -static char *async_errormsg = NULL; - -#define PAGE_ZEROSEARCH_COARSE_GRANULARITY 4096 -#define PAGE_ZEROSEARCH_FINE_GRANULARITY 64 -static const char zerobuf[PAGE_ZEROSEARCH_COARSE_GRANULARITY] = {0}; - -fio_location MyLocation; - -typedef struct -{ - BlockNumber nblocks; - BlockNumber segmentno; - XLogRecPtr horizonLsn; - uint32 checksumVersion; - int calg; - int clevel; - int bitmapsize; - int path_len; -} fio_send_request; - -typedef struct -{ - char path[MAXPGPATH]; - bool exclude; - bool follow_symlink; - bool add_root; - bool backup_logs; - bool exclusive_backup; - bool skip_hidden; - int external_dir_num; -} fio_list_dir_request; - -typedef struct -{ - mode_t mode; - size_t size; - time_t mtime; - bool is_datafile; - Oid tblspcOid; - Oid dbOid; - Oid relOid; - ForkName forkName; - int segno; - int external_dir_num; - int linked_len; -} fio_pgFile; - -typedef struct -{ - BlockNumber n_blocks; - BlockNumber segmentno; - XLogRecPtr stop_lsn; - uint32 checksumVersion; -} fio_checksum_map_request; - -typedef struct -{ - BlockNumber n_blocks; - BlockNumber segmentno; - XLogRecPtr shift_lsn; - uint32 checksumVersion; -} fio_lsn_map_request; - - -/* Convert FIO pseudo handle to index in file descriptor array */ -#define fio_fileno(f) (((size_t)f - 1) | FIO_PIPE_MARKER) - -#if defined(WIN32) -#undef open(a, b, c) -#undef fopen(a, b) -#endif - -void -setMyLocation(ProbackupSubcmd const subcmd) -{ - -#ifdef WIN32 - if (IsSshProtocol()) - elog(ERROR, "Currently remote operations on Windows are not supported"); -#endif - - MyLocation = IsSshProtocol() - ? (subcmd == ARCHIVE_PUSH_CMD || subcmd == ARCHIVE_GET_CMD) - ? FIO_DB_HOST - : (subcmd == BACKUP_CMD || subcmd == RESTORE_CMD || subcmd == ADD_INSTANCE_CMD || subcmd == CATCHUP_CMD) - ? FIO_BACKUP_HOST - : FIO_LOCAL_HOST - : FIO_LOCAL_HOST; -} - -/* Use specified file descriptors as stdin/stdout for FIO functions */ -void -fio_redirect(int in, int out, int err) -{ - fio_stdin = in; - fio_stdout = out; - fio_stderr = err; -} - -void -fio_error(int rc, int size, char const* file, int line) -{ - if (remote_agent) - { - fprintf(stderr, "%s:%d: processed %d bytes instead of %d: %s\n", file, line, rc, size, rc >= 0 ? "end of data" : strerror(errno)); - exit(EXIT_FAILURE); - } - else - { - char buf[PRINTF_BUF_SIZE+1]; -// Assert(false); - int err_size = read(fio_stderr, buf, PRINTF_BUF_SIZE); - if (err_size > 0) - { - buf[err_size] = '\0'; - elog(ERROR, "Agent error: %s", buf); - } - else - elog(ERROR, "Communication error: %s", rc >= 0 ? "end of data" : strerror(errno)); - } -} - -/* Check if file descriptor is local or remote (created by FIO) */ -static bool -fio_is_remote_fd(int fd) -{ - return (fd & FIO_PIPE_MARKER) != 0; -} - -#ifdef WIN32 - -#undef stat - -/* - * The stat() function in win32 is not guaranteed to update the st_size - * field when run. So we define our own version that uses the Win32 API - * to update this field. - */ -static int -fio_safestat(const char *path, struct stat *buf) -{ - int r; - WIN32_FILE_ATTRIBUTE_DATA attr; - - r = stat(path, buf); - if (r < 0) - return r; - - if (!GetFileAttributesEx(path, GetFileExInfoStandard, &attr)) - { - errno = ENOENT; - return -1; - } - - /* - * XXX no support for large files here, but we don't do that in general on - * Win32 yet. - */ - buf->st_size = attr.nFileSizeLow; - - return 0; -} - -#define stat(x, y) fio_safestat(x, y) - -/* TODO: use real pread on Linux */ -static ssize_t -pread(int fd, void* buf, size_t size, off_t off) -{ - off_t rc = lseek(fd, off, SEEK_SET); - if (rc != off) - return -1; - return read(fd, buf, size); -} - -static int -remove_file_or_dir(char const* path) -{ - int rc = remove(path); -#ifdef WIN32 - if (rc < 0 && errno == EACCESS) - rc = rmdir(path); -#endif - return rc; -} -#else -#define remove_file_or_dir(path) remove(path) -#endif - -/* Check if specified location is local for current node */ -bool -fio_is_remote(fio_location location) -{ - bool is_remote = MyLocation != FIO_LOCAL_HOST - && location != FIO_LOCAL_HOST - && location != MyLocation; - if (is_remote && !fio_stdin && !launch_agent()) - elog(ERROR, "Failed to establish SSH connection: %s", strerror(errno)); - return is_remote; -} - -/* Check if specified location is local for current node */ -bool -fio_is_remote_simple(fio_location location) -{ - bool is_remote = MyLocation != FIO_LOCAL_HOST - && location != FIO_LOCAL_HOST - && location != MyLocation; - return is_remote; -} - -/* Try to read specified amount of bytes unless error or EOF are encountered */ -static ssize_t -fio_read_all(int fd, void* buf, size_t size) -{ - size_t offs = 0; - while (offs < size) - { - ssize_t rc = read(fd, (char*)buf + offs, size - offs); - if (rc < 0) - { - if (errno == EINTR) - continue; - elog(ERROR, "fio_read_all error, fd %i: %s", fd, strerror(errno)); - return rc; - } - else if (rc == 0) - break; - - offs += rc; - } - return offs; -} - -/* Try to write specified amount of bytes unless error is encountered */ -static ssize_t -fio_write_all(int fd, void const* buf, size_t size) -{ - size_t offs = 0; - while (offs < size) - { - ssize_t rc = write(fd, (char*)buf + offs, size - offs); - if (rc <= 0) - { - if (errno == EINTR) - continue; - - elog(ERROR, "fio_write_all error, fd %i: %s", fd, strerror(errno)); - - return rc; - } - offs += rc; - } - return offs; -} - -/* Get version of remote agent */ -void -fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size) -{ - fio_header hdr; - hdr.cop = FIO_AGENT_VERSION; - hdr.size = 0; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - if (hdr.size > payload_buf_size) - { - elog(ERROR, "Corrupted remote compatibility protocol: insufficient payload_buf_size=%zu", payload_buf_size); - } - - *protocol = hdr.arg; - IO_CHECK(fio_read_all(fio_stdin, payload_buf, hdr.size), hdr.size); -} - -/* Open input stream. Remote file is fetched to the in-memory buffer and then accessed through Linux fmemopen */ -FILE* -fio_open_stream(char const* path, fio_location location) -{ - FILE* f; - if (fio_is_remote(location)) - { - fio_header hdr; - hdr.cop = FIO_LOAD; - hdr.size = strlen(path) + 1; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); - - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_SEND); - if (hdr.size > 0) - { - Assert(fio_stdin_buffer == NULL); - fio_stdin_buffer = pgut_malloc(hdr.size); - IO_CHECK(fio_read_all(fio_stdin, fio_stdin_buffer, hdr.size), hdr.size); -#ifdef WIN32 - f = tmpfile(); - IO_CHECK(fwrite(fio_stdin_buffer, 1, hdr.size, f), hdr.size); - SYS_CHECK(fseek(f, 0, SEEK_SET)); -#else - f = fmemopen(fio_stdin_buffer, hdr.size, "r"); -#endif - } - else - { - f = NULL; - } - } - else - { - f = fopen(path, "rt"); - } - return f; -} - -/* Close input stream */ -int -fio_close_stream(FILE* f) -{ - if (fio_stdin_buffer) - { - free(fio_stdin_buffer); - fio_stdin_buffer = NULL; - } - return fclose(f); -} - -/* Open directory */ -DIR* -fio_opendir(char const* path, fio_location location) -{ - DIR* dir; - if (fio_is_remote(location)) - { - int i; - fio_header hdr; - unsigned long mask; - - mask = fio_fdset; - for (i = 0; (mask & 1) != 0; i++, mask >>= 1); - if (i == FIO_FDMAX) { - elog(ERROR, "Descriptor pool for remote files is exhausted, " - "probably too many remote directories are opened"); - } - hdr.cop = FIO_OPENDIR; - hdr.handle = i; - hdr.size = strlen(path) + 1; - fio_fdset |= 1 << i; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); - - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - - if (hdr.arg != 0) - { - errno = hdr.arg; - fio_fdset &= ~(1 << hdr.handle); - return NULL; - } - dir = (DIR*)(size_t)(i + 1); - } - else - { - dir = opendir(path); - } - return dir; -} - -/* Get next directory entry */ -struct dirent* -fio_readdir(DIR *dir) -{ - if (fio_is_remote_file((FILE*)dir)) - { - fio_header hdr; - static __thread struct dirent entry; - - hdr.cop = FIO_READDIR; - hdr.handle = (size_t)dir - 1; - hdr.size = 0; - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_SEND); - if (hdr.size) { - Assert(hdr.size == sizeof(entry)); - IO_CHECK(fio_read_all(fio_stdin, &entry, sizeof(entry)), sizeof(entry)); - } - - return hdr.size ? &entry : NULL; - } - else - { - return readdir(dir); - } -} - -/* Close directory */ -int -fio_closedir(DIR *dir) -{ - if (fio_is_remote_file((FILE*)dir)) - { - fio_header hdr; - hdr.cop = FIO_CLOSEDIR; - hdr.handle = (size_t)dir - 1; - hdr.size = 0; - fio_fdset &= ~(1 << hdr.handle); - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - return 0; - } - else - { - return closedir(dir); - } -} - -/* Open file */ -int -fio_open(char const* path, int mode, fio_location location) -{ - int fd; - if (fio_is_remote(location)) - { - int i; - fio_header hdr; - unsigned long mask; - - mask = fio_fdset; - for (i = 0; (mask & 1) != 0; i++, mask >>= 1); - if (i == FIO_FDMAX) - elog(ERROR, "Descriptor pool for remote files is exhausted, " - "probably too many remote files are opened"); - - hdr.cop = FIO_OPEN; - hdr.handle = i; - hdr.size = strlen(path) + 1; - hdr.arg = mode; -// hdr.arg = mode & ~O_EXCL; -// elog(INFO, "PATH: %s MODE: %i, %i", path, mode, O_EXCL); -// elog(INFO, "MODE: %i", hdr.arg); - fio_fdset |= 1 << i; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, hdr.size), hdr.size); - - /* check results */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - - if (hdr.arg != 0) - { - errno = hdr.arg; - fio_fdset &= ~(1 << hdr.handle); - return -1; - } - fd = i | FIO_PIPE_MARKER; - } - else - { - fd = open(path, mode, FILE_PERMISSIONS); - } - return fd; -} - - -/* Close ssh session */ -void -fio_disconnect(void) -{ - if (fio_stdin) - { - fio_header hdr; - hdr.cop = FIO_DISCONNECT; - hdr.size = 0; - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_DISCONNECTED); - SYS_CHECK(close(fio_stdin)); - SYS_CHECK(close(fio_stdout)); - SYS_CHECK(close(fio_stderr)); - fio_stdin = 0; - fio_stdout = 0; - fio_stderr = 0; - wait_ssh(); - } -} - -/* Open stdio file */ -FILE* -fio_fopen(char const* path, char const* mode, fio_location location) -{ - FILE *f = NULL; - - if (fio_is_remote(location)) - { - int flags = 0; - int fd; - if (strcmp(mode, PG_BINARY_W) == 0) { - flags = O_TRUNC|PG_BINARY|O_RDWR|O_CREAT; - } else if (strcmp(mode, "w") == 0) { - flags = O_TRUNC|O_RDWR|O_CREAT; - } else if (strcmp(mode, PG_BINARY_R) == 0) { - flags = O_RDONLY|PG_BINARY; - } else if (strcmp(mode, "r") == 0) { - flags = O_RDONLY; - } else if (strcmp(mode, PG_BINARY_R "+") == 0) { - /* stdio fopen("rb+") actually doesn't create unexisted file, but probackup frequently - * needs to open existed file or create new one if not exists. - * In stdio it can be done using two fopen calls: fopen("r+") and if failed then fopen("w"). - * But to eliminate extra call which especially critical in case of remote connection - * we change r+ semantic to create file if not exists. - */ - flags = O_RDWR|O_CREAT|PG_BINARY; - } else if (strcmp(mode, "r+") == 0) { /* see comment above */ - flags |= O_RDWR|O_CREAT; - } else if (strcmp(mode, "a") == 0) { - flags |= O_CREAT|O_RDWR|O_APPEND; - } else { - Assert(false); - } - fd = fio_open(path, flags, location); - if (fd >= 0) - f = (FILE*)(size_t)((fd + 1) & ~FIO_PIPE_MARKER); - } - else - { - f = fopen(path, mode); - if (f == NULL && strcmp(mode, PG_BINARY_R "+") == 0) - f = fopen(path, PG_BINARY_W); - } - return f; -} - -/* Format output to file stream */ -int -fio_fprintf(FILE* f, char const* format, ...) -{ - int rc; - va_list args; - va_start (args, format); - if (fio_is_remote_file(f)) - { - char buf[PRINTF_BUF_SIZE]; -#ifdef HAS_VSNPRINTF - rc = vsnprintf(buf, sizeof(buf), format, args); -#else - rc = vsprintf(buf, format, args); -#endif - if (rc > 0) { - fio_fwrite(f, buf, rc); - } - } - else - { - rc = vfprintf(f, format, args); - } - va_end (args); - return rc; -} - -/* Flush stream data (does nothing for remote file) */ -int -fio_fflush(FILE* f) -{ - int rc = 0; - if (!fio_is_remote_file(f)) - rc = fflush(f); - return rc; -} - -/* Sync file to the disk (does nothing for remote file) */ -int -fio_flush(int fd) -{ - return fio_is_remote_fd(fd) ? 0 : fsync(fd); -} - -/* Close output stream */ -int -fio_fclose(FILE* f) -{ - return fio_is_remote_file(f) - ? fio_close(fio_fileno(f)) - : fclose(f); -} - -/* Close file */ -int -fio_close(int fd) -{ - if (fio_is_remote_fd(fd)) - { - fio_header hdr; - - hdr.cop = FIO_CLOSE; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = 0; - fio_fdset &= ~(1 << hdr.handle); - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - - /* Wait for response */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - - if (hdr.arg != 0) - { - errno = hdr.arg; - return -1; - } - - return 0; - } - else - { - return close(fd); - } -} - -/* Close remote file implementation */ -static void -fio_close_impl(int fd, int out) -{ - fio_header hdr; - - hdr.cop = FIO_CLOSE; - hdr.arg = 0; - - if (close(fd) != 0) - hdr.arg = errno; - - /* send header */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); -} - -/* Truncate stdio file */ -int -fio_ftruncate(FILE* f, off_t size) -{ - return fio_is_remote_file(f) - ? fio_truncate(fio_fileno(f), size) - : ftruncate(fileno(f), size); -} - -/* Truncate file - * TODO: make it synchronous - */ -int -fio_truncate(int fd, off_t size) -{ - if (fio_is_remote_fd(fd)) - { - fio_header hdr; - - hdr.cop = FIO_TRUNCATE; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = 0; - hdr.arg = size; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - - return 0; - } - else - { - return ftruncate(fd, size); - } -} - - -/* - * Read file from specified location. - */ -int -fio_pread(FILE* f, void* buf, off_t offs) -{ - if (fio_is_remote_file(f)) - { - int fd = fio_fileno(f); - fio_header hdr; - - hdr.cop = FIO_PREAD; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = 0; - hdr.arg = offs; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_SEND); - if (hdr.size != 0) - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - - /* TODO: error handling */ - - return hdr.arg; - } - else - { - /* For local file, opened by fopen, we should use stdio functions */ - int rc = fseek(f, offs, SEEK_SET); - - if (rc < 0) - return rc; - - return fread(buf, 1, BLCKSZ, f); - } -} - -/* Set position in stdio file */ -int -fio_fseek(FILE* f, off_t offs) -{ - return fio_is_remote_file(f) - ? fio_seek(fio_fileno(f), offs) - : fseek(f, offs, SEEK_SET); -} - -/* Set position in file */ -/* TODO: make it synchronous or check async error */ -int -fio_seek(int fd, off_t offs) -{ - if (fio_is_remote_fd(fd)) - { - fio_header hdr; - - hdr.cop = FIO_SEEK; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = 0; - hdr.arg = offs; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - - return 0; - } - else - { - return lseek(fd, offs, SEEK_SET); - } -} - -/* seek is asynchronous */ -static void -fio_seek_impl(int fd, off_t offs) -{ - int rc; - - /* Quick exit for tainted agent */ - if (async_errormsg) - return; - - rc = lseek(fd, offs, SEEK_SET); - - if (rc < 0) - { - async_errormsg = pgut_malloc(ERRMSG_MAX_LEN); - snprintf(async_errormsg, ERRMSG_MAX_LEN, "%s", strerror(errno)); - } -} - -/* Write data to stdio file */ -size_t -fio_fwrite(FILE* f, void const* buf, size_t size) -{ - if (fio_is_remote_file(f)) - return fio_write(fio_fileno(f), buf, size); - else - return fwrite(buf, 1, size, f); -} - -/* - * Write buffer to descriptor by calling write(), - * If size of written data is less than buffer size, - * then try to write what is left. - * We do this to get honest errno if there are some problems - * with filesystem, since writing less than buffer size - * is not considered an error. - */ -static ssize_t -durable_write(int fd, const char* buf, size_t size) -{ - off_t current_pos = 0; - size_t bytes_left = size; - - while (bytes_left > 0) - { - int rc = write(fd, buf + current_pos, bytes_left); - - if (rc <= 0) - return rc; - - bytes_left -= rc; - current_pos += rc; - } - - return size; -} - -/* Write data to the file synchronously */ -ssize_t -fio_write(int fd, void const* buf, size_t size) -{ - if (fio_is_remote_fd(fd)) - { - fio_header hdr; - - hdr.cop = FIO_WRITE; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = size; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, buf, size), size); - - /* check results */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - - /* set errno */ - if (hdr.arg > 0) - { - errno = hdr.arg; - return -1; - } - - return size; - } - else - { - return durable_write(fd, buf, size); - } -} - -static void -fio_write_impl(int fd, void const* buf, size_t size, int out) -{ - int rc; - fio_header hdr; - - rc = durable_write(fd, buf, size); - - hdr.arg = 0; - hdr.size = 0; - - if (rc < 0) - hdr.arg = errno; - - /* send header */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - - return; -} - -size_t -fio_fwrite_async(FILE* f, void const* buf, size_t size) -{ - return fio_is_remote_file(f) - ? fio_write_async(fio_fileno(f), buf, size) - : fwrite(buf, 1, size, f); -} - -/* Write data to the file */ -/* TODO: support async report error */ -ssize_t -fio_write_async(int fd, void const* buf, size_t size) -{ - if (size == 0) - return 0; - - if (fio_is_remote_fd(fd)) - { - fio_header hdr; - - hdr.cop = FIO_WRITE_ASYNC; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = size; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, buf, size), size); - } - else - return durable_write(fd, buf, size); - - return size; -} - -static void -fio_write_async_impl(int fd, void const* buf, size_t size, int out) -{ - /* Quick exit if agent is tainted */ - if (async_errormsg) - return; - - if (durable_write(fd, buf, size) <= 0) - { - async_errormsg = pgut_malloc(ERRMSG_MAX_LEN); - snprintf(async_errormsg, ERRMSG_MAX_LEN, "%s", strerror(errno)); - } -} - -int32 -fio_decompress(void* dst, void const* src, size_t size, int compress_alg, char **errormsg) -{ - const char *internal_errormsg = NULL; - int32 uncompressed_size = do_decompress(dst, BLCKSZ, - src, - size, - compress_alg, &internal_errormsg); - - if (uncompressed_size < 0 && internal_errormsg != NULL) - { - *errormsg = pgut_malloc(ERRMSG_MAX_LEN); - snprintf(*errormsg, ERRMSG_MAX_LEN, "An error occured during decompressing block: %s", internal_errormsg); - return -1; - } - - if (uncompressed_size != BLCKSZ) - { - *errormsg = pgut_malloc(ERRMSG_MAX_LEN); - snprintf(*errormsg, ERRMSG_MAX_LEN, "Page uncompressed to %d bytes != BLCKSZ", uncompressed_size); - return -1; - } - return uncompressed_size; -} - -/* Write data to the file */ -ssize_t -fio_fwrite_async_compressed(FILE* f, void const* buf, size_t size, int compress_alg) -{ - if (fio_is_remote_file(f)) - { - fio_header hdr; - - hdr.cop = FIO_WRITE_COMPRESSED_ASYNC; - hdr.handle = fio_fileno(f) & ~FIO_PIPE_MARKER; - hdr.size = size; - hdr.arg = compress_alg; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, buf, size), size); - - return size; - } - else - { - char *errormsg = NULL; - char decompressed_buf[BLCKSZ]; - int32 decompressed_size = fio_decompress(decompressed_buf, buf, size, compress_alg, &errormsg); - - if (decompressed_size < 0) - elog(ERROR, "%s", errormsg); - - return fwrite(decompressed_buf, 1, decompressed_size, f); - } -} - -static void -fio_write_compressed_impl(int fd, void const* buf, size_t size, int compress_alg) -{ - int32 decompressed_size; - char decompressed_buf[BLCKSZ]; - - /* If the previous command already have failed, - * then there is no point in bashing a head against the wall - */ - if (async_errormsg) - return; - - /* decompress chunk */ - decompressed_size = fio_decompress(decompressed_buf, buf, size, compress_alg, &async_errormsg); - - if (decompressed_size < 0) - return; - - if (durable_write(fd, decompressed_buf, decompressed_size) <= 0) - { - async_errormsg = pgut_malloc(ERRMSG_MAX_LEN); - snprintf(async_errormsg, ERRMSG_MAX_LEN, "%s", strerror(errno)); - } -} - -/* check if remote agent encountered any error during execution of async operations */ -int -fio_check_error_file(FILE* f, char **errmsg) -{ - if (fio_is_remote_file(f)) - { - fio_header hdr; - - hdr.cop = FIO_GET_ASYNC_ERROR; - hdr.size = 0; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - - /* check results */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - - if (hdr.size > 0) - { - *errmsg = pgut_malloc(ERRMSG_MAX_LEN); - IO_CHECK(fio_read_all(fio_stdin, *errmsg, hdr.size), hdr.size); - return 1; - } - } - - return 0; -} - -/* check if remote agent encountered any error during execution of async operations */ -int -fio_check_error_fd(int fd, char **errmsg) -{ - if (fio_is_remote_fd(fd)) - { - fio_header hdr; - - hdr.cop = FIO_GET_ASYNC_ERROR; - hdr.size = 0; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - - /* check results */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - - if (hdr.size > 0) - { - *errmsg = pgut_malloc(ERRMSG_MAX_LEN); - IO_CHECK(fio_read_all(fio_stdin, *errmsg, hdr.size), hdr.size); - return 1; - } - } - return 0; -} - -static void -fio_get_async_error_impl(int out) -{ - fio_header hdr; - hdr.cop = FIO_GET_ASYNC_ERROR; - - /* send error message */ - if (async_errormsg) - { - hdr.size = strlen(async_errormsg) + 1; - - /* send header */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - - /* send message itself */ - IO_CHECK(fio_write_all(out, async_errormsg, hdr.size), hdr.size); - - //TODO: should we reset the tainted state ? -// pg_free(async_errormsg); -// async_errormsg = NULL; - } - else - { - hdr.size = 0; - /* send header */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - } -} - -/* Read data from stdio file */ -ssize_t -fio_fread(FILE* f, void* buf, size_t size) -{ - size_t rc; - if (fio_is_remote_file(f)) - return fio_read(fio_fileno(f), buf, size); - rc = fread(buf, 1, size, f); - return rc == 0 && !feof(f) ? -1 : rc; -} - -/* Read data from file */ -ssize_t -fio_read(int fd, void* buf, size_t size) -{ - if (fio_is_remote_fd(fd)) - { - fio_header hdr; - - hdr.cop = FIO_READ; - hdr.handle = fd & ~FIO_PIPE_MARKER; - hdr.size = 0; - hdr.arg = size; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_SEND); - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - - return hdr.size; - } - else - { - return read(fd, buf, size); - } -} - -/* Get information about file */ -int -fio_stat(char const* path, struct stat* st, bool follow_symlink, fio_location location) -{ - if (fio_is_remote(location)) - { - fio_header hdr; - size_t path_len = strlen(path) + 1; - - hdr.cop = FIO_STAT; - hdr.handle = -1; - hdr.arg = follow_symlink; - hdr.size = path_len; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); - - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_STAT); - IO_CHECK(fio_read_all(fio_stdin, st, sizeof(*st)), sizeof(*st)); - - if (hdr.arg != 0) - { - errno = hdr.arg; - return -1; - } - return 0; - } - else - { - return follow_symlink ? stat(path, st) : lstat(path, st); - } -} - -/* - * Compare, that filename1 and filename2 is the same file - * in windows compare only filenames - */ -bool -fio_is_same_file(char const* filename1, char const* filename2, bool follow_symlink, fio_location location) -{ -#ifndef WIN32 - struct stat stat1, stat2; - - if (fio_stat(filename1, &stat1, follow_symlink, location) < 0) - elog(ERROR, "Can't stat file \"%s\": %s", filename1, strerror(errno)); - - if (fio_stat(filename2, &stat2, follow_symlink, location) < 0) - elog(ERROR, "Can't stat file \"%s\": %s", filename2, strerror(errno)); - - return stat1.st_ino == stat2.st_ino && stat1.st_dev == stat2.st_dev; -#else - char *abs_name1 = make_absolute_path(filename1); - char *abs_name2 = make_absolute_path(filename2); - bool result = strcmp(abs_name1, abs_name2) == 0; - free(abs_name2); - free(abs_name1); - return result; -#endif -} - -/* - * Read value of a symbolic link - * this is a wrapper about readlink() syscall - * side effects: string truncation occur (and it - * can be checked by caller by comparing - * returned value >= valsiz) - */ -ssize_t -fio_readlink(const char *path, char *value, size_t valsiz, fio_location location) -{ - if (!fio_is_remote(location)) - { - /* readlink don't place trailing \0 */ - ssize_t len = readlink(path, value, valsiz); - value[len < valsiz ? len : valsiz] = '\0'; - return len; - } - else - { - fio_header hdr; - size_t path_len = strlen(path) + 1; - - hdr.cop = FIO_READLINK; - hdr.handle = -1; - Assert(valsiz <= UINT_MAX); /* max value of fio_header.arg */ - hdr.arg = valsiz; - hdr.size = path_len; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); - - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_READLINK); - Assert(hdr.size <= valsiz); - IO_CHECK(fio_read_all(fio_stdin, value, hdr.size), hdr.size); - value[hdr.size < valsiz ? hdr.size : valsiz] = '\0'; - return hdr.size; - } -} - -/* Check presence of the file */ -int -fio_access(char const* path, int mode, fio_location location) -{ - if (fio_is_remote(location)) - { - fio_header hdr; - size_t path_len = strlen(path) + 1; - hdr.cop = FIO_ACCESS; - hdr.handle = -1; - hdr.size = path_len; - hdr.arg = mode; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); - - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_ACCESS); - - if (hdr.arg != 0) - { - errno = hdr.arg; - return -1; - } - return 0; - } - else - { - return access(path, mode); - } -} - -/* Create symbolic link */ -int -fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location) -{ - if (fio_is_remote(location)) - { - fio_header hdr; - size_t target_len = strlen(target) + 1; - size_t link_path_len = strlen(link_path) + 1; - hdr.cop = FIO_SYMLINK; - hdr.handle = -1; - hdr.size = target_len + link_path_len; - hdr.arg = overwrite ? 1 : 0; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, target, target_len), target_len); - IO_CHECK(fio_write_all(fio_stdout, link_path, link_path_len), link_path_len); - - return 0; - } - else - { - if (overwrite) - remove_file_or_dir(link_path); - - return symlink(target, link_path); - } -} - -static void -fio_symlink_impl(int out, char *buf, bool overwrite) -{ - char *linked_path = buf; - char *link_path = buf + strlen(buf) + 1; - - if (overwrite) - remove_file_or_dir(link_path); - - if (symlink(linked_path, link_path)) - elog(ERROR, "Could not create symbolic link \"%s\": %s", - link_path, strerror(errno)); -} - -/* Rename file */ -int -fio_rename(char const* old_path, char const* new_path, fio_location location) -{ - if (fio_is_remote(location)) - { - fio_header hdr; - size_t old_path_len = strlen(old_path) + 1; - size_t new_path_len = strlen(new_path) + 1; - hdr.cop = FIO_RENAME; - hdr.handle = -1; - hdr.size = old_path_len + new_path_len; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, old_path, old_path_len), old_path_len); - IO_CHECK(fio_write_all(fio_stdout, new_path, new_path_len), new_path_len); - - //TODO: wait for confirmation. - - return 0; - } - else - { - return rename(old_path, new_path); - } -} - -/* Sync file to disk */ -int -fio_sync(char const* path, fio_location location) -{ - if (fio_is_remote(location)) - { - fio_header hdr; - size_t path_len = strlen(path) + 1; - hdr.cop = FIO_SYNC; - hdr.handle = -1; - hdr.size = path_len; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - - if (hdr.arg != 0) - { - errno = hdr.arg; - return -1; - } - - return 0; - } - else - { - int fd; - - fd = open(path, O_WRONLY | PG_BINARY, FILE_PERMISSIONS); - if (fd < 0) - return -1; - - if (fsync(fd) < 0) - { - close(fd); - return -1; - } - close(fd); - - return 0; - } -} - -enum { - GET_CRC32_DECOMPRESS = 1, - GET_CRC32_MISSING_OK = 2, - GET_CRC32_TRUNCATED = 4 -}; - -/* Get crc32 of file */ -static pg_crc32 -fio_get_crc32_ex(const char *file_path, fio_location location, - bool decompress, bool missing_ok, bool truncated) -{ - if (decompress && truncated) - elog(ERROR, "Could not calculate CRC for compressed truncated file"); - - if (fio_is_remote(location)) - { - fio_header hdr; - size_t path_len = strlen(file_path) + 1; - pg_crc32 crc = 0; - hdr.cop = FIO_GET_CRC32; - hdr.handle = -1; - hdr.size = path_len; - hdr.arg = 0; - - if (decompress) - hdr.arg = GET_CRC32_DECOMPRESS; - if (missing_ok) - hdr.arg |= GET_CRC32_MISSING_OK; - if (truncated) - hdr.arg |= GET_CRC32_TRUNCATED; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, file_path, path_len), path_len); - IO_CHECK(fio_read_all(fio_stdin, &crc, sizeof(crc)), sizeof(crc)); - - return crc; - } - else - { - if (decompress) - return pgFileGetCRCgz(file_path, true, missing_ok); - else if (truncated) - return pgFileGetCRCTruncated(file_path, true, missing_ok); - else - return pgFileGetCRC(file_path, true, missing_ok); - } -} - -pg_crc32 -fio_get_crc32(const char *file_path, fio_location location, - bool decompress, bool missing_ok) -{ - return fio_get_crc32_ex(file_path, location, decompress, missing_ok, false); -} - -pg_crc32 -fio_get_crc32_truncated(const char *file_path, fio_location location, - bool missing_ok) -{ - return fio_get_crc32_ex(file_path, location, false, missing_ok, true); -} - -/* Remove file */ -int -fio_unlink(char const* path, fio_location location) -{ - if (fio_is_remote(location)) - { - fio_header hdr; - size_t path_len = strlen(path) + 1; - hdr.cop = FIO_UNLINK; - hdr.handle = -1; - hdr.size = path_len; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); - - // TODO: error is swallowed ? - return 0; - } - else - { - return remove_file_or_dir(path); - } -} - -/* Create directory - * TODO: add strict flag - */ -int -fio_mkdir(char const* path, int mode, fio_location location) -{ - if (fio_is_remote(location)) - { - fio_header hdr; - size_t path_len = strlen(path) + 1; - hdr.cop = FIO_MKDIR; - hdr.handle = -1; - hdr.size = path_len; - hdr.arg = mode; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); - - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - Assert(hdr.cop == FIO_MKDIR); - - return hdr.arg; - } - else - { - return dir_create_dir(path, mode, false); - } -} - -/* Change file mode */ -int -fio_chmod(char const* path, int mode, fio_location location) -{ - if (fio_is_remote(location)) - { - fio_header hdr; - size_t path_len = strlen(path) + 1; - hdr.cop = FIO_CHMOD; - hdr.handle = -1; - hdr.size = path_len; - hdr.arg = mode; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, path, path_len), path_len); - - return 0; - } - else - { - return chmod(path, mode); - } -} - -#ifdef HAVE_LIBZ - -#define ZLIB_BUFFER_SIZE (64*1024) -#define MAX_WBITS 15 /* 32K LZ77 window */ -#define DEF_MEM_LEVEL 8 -/* last bit used to differenciate remote gzFile from local gzFile - * TODO: this is insane, we should create our own scructure for this, - * not flip some bits in someone's else and hope that it will not break - * between zlib versions. - */ -#define FIO_GZ_REMOTE_MARKER 1 - -typedef struct fioGZFile -{ - z_stream strm; - int fd; - int errnum; - bool compress; - bool eof; - Bytef buf[ZLIB_BUFFER_SIZE]; -} fioGZFile; - -/* check if remote agent encountered any error during execution of async operations */ -int -fio_check_error_fd_gz(gzFile f, char **errmsg) -{ - if (f && ((size_t)f & FIO_GZ_REMOTE_MARKER)) - { - fio_header hdr; - - hdr.cop = FIO_GET_ASYNC_ERROR; - hdr.size = 0; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - - /* check results */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - - if (hdr.size > 0) - { - *errmsg = pgut_malloc(ERRMSG_MAX_LEN); - IO_CHECK(fio_read_all(fio_stdin, *errmsg, hdr.size), hdr.size); - return 1; - } - } - return 0; -} - -/* On error returns NULL and errno should be checked */ -gzFile -fio_gzopen(char const* path, char const* mode, int level, fio_location location) -{ - int rc; - if (fio_is_remote(location)) - { - fioGZFile* gz = (fioGZFile*) pgut_malloc(sizeof(fioGZFile)); - memset(&gz->strm, 0, sizeof(gz->strm)); - gz->eof = 0; - gz->errnum = Z_OK; - /* check if file opened for writing */ - if (strcmp(mode, PG_BINARY_W) == 0) /* compress */ - { - gz->strm.next_out = gz->buf; - gz->strm.avail_out = ZLIB_BUFFER_SIZE; - rc = deflateInit2(&gz->strm, - level, - Z_DEFLATED, - MAX_WBITS + 16, DEF_MEM_LEVEL, - Z_DEFAULT_STRATEGY); - if (rc == Z_OK) - { - gz->compress = 1; - gz->fd = fio_open(path, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY, location); - if (gz->fd < 0) - { - free(gz); - return NULL; - } - } - } - else - { - gz->strm.next_in = gz->buf; - gz->strm.avail_in = ZLIB_BUFFER_SIZE; - rc = inflateInit2(&gz->strm, 15 + 16); - gz->strm.avail_in = 0; - if (rc == Z_OK) - { - gz->compress = 0; - gz->fd = fio_open(path, O_RDONLY | PG_BINARY, location); - if (gz->fd < 0) - { - free(gz); - return NULL; - } - } - } - if (rc != Z_OK) - { - elog(ERROR, "zlib internal error when opening file %s: %s", - path, gz->strm.msg); - } - return (gzFile)((size_t)gz + FIO_GZ_REMOTE_MARKER); - } - else - { - gzFile file; - /* check if file opened for writing */ - if (strcmp(mode, PG_BINARY_W) == 0) - { - int fd = open(path, O_WRONLY | O_CREAT | O_EXCL | PG_BINARY, FILE_PERMISSIONS); - if (fd < 0) - return NULL; - file = gzdopen(fd, mode); - } - else - file = gzopen(path, mode); - if (file != NULL && level != Z_DEFAULT_COMPRESSION) - { - if (gzsetparams(file, level, Z_DEFAULT_STRATEGY) != Z_OK) - elog(ERROR, "Cannot set compression level %d: %s", - level, strerror(errno)); - } - return file; - } -} - -int -fio_gzread(gzFile f, void *buf, unsigned size) -{ - if ((size_t)f & FIO_GZ_REMOTE_MARKER) - { - int rc; - fioGZFile* gz = (fioGZFile*)((size_t)f - FIO_GZ_REMOTE_MARKER); - - if (gz->eof) - { - return 0; - } - - gz->strm.next_out = (Bytef *)buf; - gz->strm.avail_out = size; - - while (1) - { - if (gz->strm.avail_in != 0) /* If there is some data in receiver buffer, then decompress it */ - { - rc = inflate(&gz->strm, Z_NO_FLUSH); - if (rc == Z_STREAM_END) - { - gz->eof = 1; - } - else if (rc != Z_OK) - { - gz->errnum = rc; - return -1; - } - if (gz->strm.avail_out != size) - { - return size - gz->strm.avail_out; - } - if (gz->strm.avail_in == 0) - { - gz->strm.next_in = gz->buf; - } - } - else - { - gz->strm.next_in = gz->buf; - } - rc = fio_read(gz->fd, gz->strm.next_in + gz->strm.avail_in, - gz->buf + ZLIB_BUFFER_SIZE - gz->strm.next_in - gz->strm.avail_in); - if (rc > 0) - { - gz->strm.avail_in += rc; - } - else - { - if (rc == 0) - { - gz->eof = 1; - } - return rc; - } - } - } - else - { - return gzread(f, buf, size); - } -} - -int -fio_gzwrite(gzFile f, void const* buf, unsigned size) -{ - if ((size_t)f & FIO_GZ_REMOTE_MARKER) - { - int rc; - fioGZFile* gz = (fioGZFile*)((size_t)f - FIO_GZ_REMOTE_MARKER); - - gz->strm.next_in = (Bytef *)buf; - gz->strm.avail_in = size; - - do - { - if (gz->strm.avail_out == ZLIB_BUFFER_SIZE) /* Compress buffer is empty */ - { - gz->strm.next_out = gz->buf; /* Reset pointer to the beginning of buffer */ - - if (gz->strm.avail_in != 0) /* Has something in input buffer */ - { - rc = deflate(&gz->strm, Z_NO_FLUSH); - Assert(rc == Z_OK); - gz->strm.next_out = gz->buf; /* Reset pointer to the beginning of buffer */ - } - else - { - break; - } - } - rc = fio_write_async(gz->fd, gz->strm.next_out, ZLIB_BUFFER_SIZE - gz->strm.avail_out); - if (rc >= 0) - { - gz->strm.next_out += rc; - gz->strm.avail_out += rc; - } - else - { - return rc; - } - } while (gz->strm.avail_out != ZLIB_BUFFER_SIZE || gz->strm.avail_in != 0); - - return size; - } - else - { - return gzwrite(f, buf, size); - } -} - -int -fio_gzclose(gzFile f) -{ - if ((size_t)f & FIO_GZ_REMOTE_MARKER) - { - fioGZFile* gz = (fioGZFile*)((size_t)f - FIO_GZ_REMOTE_MARKER); - int rc; - if (gz->compress) - { - gz->strm.next_out = gz->buf; - rc = deflate(&gz->strm, Z_FINISH); - Assert(rc == Z_STREAM_END && gz->strm.avail_out != ZLIB_BUFFER_SIZE); - deflateEnd(&gz->strm); - rc = fio_write(gz->fd, gz->buf, ZLIB_BUFFER_SIZE - gz->strm.avail_out); - if (rc != ZLIB_BUFFER_SIZE - gz->strm.avail_out) - { - return -1; - } - } - else - { - inflateEnd(&gz->strm); - } - rc = fio_close(gz->fd); - free(gz); - return rc; - } - else - { - return gzclose(f); - } -} - -int -fio_gzeof(gzFile f) -{ - if ((size_t)f & FIO_GZ_REMOTE_MARKER) - { - fioGZFile* gz = (fioGZFile*)((size_t)f - FIO_GZ_REMOTE_MARKER); - return gz->eof; - } - else - { - return gzeof(f); - } -} - -const char* -fio_gzerror(gzFile f, int *errnum) -{ - if ((size_t)f & FIO_GZ_REMOTE_MARKER) - { - fioGZFile* gz = (fioGZFile*)((size_t)f - FIO_GZ_REMOTE_MARKER); - if (errnum) - *errnum = gz->errnum; - return gz->strm.msg; - } - else - { - return gzerror(f, errnum); - } -} - -z_off_t -fio_gzseek(gzFile f, z_off_t offset, int whence) -{ - Assert(!((size_t)f & FIO_GZ_REMOTE_MARKER)); - return gzseek(f, offset, whence); -} - - -#endif - -/* Send file content - * Note: it should not be used for large files. - */ -static void -fio_load_file(int out, char const* path) -{ - int fd = open(path, O_RDONLY); - fio_header hdr; - void* buf = NULL; - - hdr.cop = FIO_SEND; - hdr.size = 0; - - if (fd >= 0) - { - off_t size = lseek(fd, 0, SEEK_END); - buf = pgut_malloc(size); - lseek(fd, 0, SEEK_SET); - IO_CHECK(fio_read_all(fd, buf, size), size); - hdr.size = size; - SYS_CHECK(close(fd)); - } - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - if (buf) - { - IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); - free(buf); - } -} - -/* - * Return number of actually(!) readed blocks, attempts or - * half-readed block are not counted. - * Return values in case of error: - * FILE_MISSING - * OPEN_FAILED - * READ_ERROR - * PAGE_CORRUPTION - * WRITE_FAILED - * - * If none of the above, this function return number of blocks - * readed by remote agent. - * - * In case of DELTA mode horizonLsn must be a valid lsn, - * otherwise it should be set to InvalidXLogRecPtr. - */ -int -fio_send_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, - XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, - bool use_pagemap, BlockNumber* err_blknum, char **errormsg, - BackupPageHeader2 **headers) -{ - FILE *out = NULL; - char *out_buf = NULL; - struct { - fio_header hdr; - fio_send_request arg; - } req; - BlockNumber n_blocks_read = 0; - BlockNumber blknum = 0; - - /* send message with header - - 16bytes 24bytes var var - -------------------------------------------------------------- - | fio_header | fio_send_request | FILE PATH | BITMAP(if any) | - -------------------------------------------------------------- - */ - - req.hdr.cop = FIO_SEND_PAGES; - - if (use_pagemap) - { - req.hdr.size = sizeof(fio_send_request) + (*file).pagemap.bitmapsize + strlen(from_fullpath) + 1; - req.arg.bitmapsize = (*file).pagemap.bitmapsize; - - /* TODO: add optimization for the case of pagemap - * containing small number of blocks with big serial numbers: - * https://github.com/postgrespro/pg_probackup/blob/remote_page_backup/src/utils/file.c#L1211 - */ - } - else - { - req.hdr.size = sizeof(fio_send_request) + strlen(from_fullpath) + 1; - req.arg.bitmapsize = 0; - } - - req.arg.nblocks = file->size/BLCKSZ; - req.arg.segmentno = file->segno * RELSEG_SIZE; - req.arg.horizonLsn = horizonLsn; - req.arg.checksumVersion = checksum_version; - req.arg.calg = calg; - req.arg.clevel = clevel; - req.arg.path_len = strlen(from_fullpath) + 1; - - file->compress_alg = calg; /* TODO: wtf? why here? */ - -//<----- -// datapagemap_iterator_t *iter; -// BlockNumber blkno; -// iter = datapagemap_iterate(pagemap); -// while (datapagemap_next(iter, &blkno)) -// elog(INFO, "block %u", blkno); -// pg_free(iter); -//<----- - - /* send header */ - IO_CHECK(fio_write_all(fio_stdout, &req, sizeof(req)), sizeof(req)); - - /* send file path */ - IO_CHECK(fio_write_all(fio_stdout, from_fullpath, req.arg.path_len), req.arg.path_len); - - /* send pagemap if any */ - if (use_pagemap) - IO_CHECK(fio_write_all(fio_stdout, (*file).pagemap.bitmap, (*file).pagemap.bitmapsize), (*file).pagemap.bitmapsize); - - while (true) - { - fio_header hdr; - char buf[BLCKSZ + sizeof(BackupPageHeader)]; - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - - if (interrupted) - elog(ERROR, "Interrupted during page reading"); - - if (hdr.cop == FIO_ERROR) - { - /* FILE_MISSING, OPEN_FAILED and READ_FAILED */ - if (hdr.size > 0) - { - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - *errormsg = pgut_malloc(hdr.size); - snprintf(*errormsg, hdr.size, "%s", buf); - } - - return hdr.arg; - } - else if (hdr.cop == FIO_SEND_FILE_CORRUPTION) - { - *err_blknum = hdr.arg; - - if (hdr.size > 0) - { - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - *errormsg = pgut_malloc(hdr.size); - snprintf(*errormsg, hdr.size, "%s", buf); - } - return PAGE_CORRUPTION; - } - else if (hdr.cop == FIO_SEND_FILE_EOF) - { - /* n_blocks_read reported by EOF */ - n_blocks_read = hdr.arg; - - /* receive headers if any */ - if (hdr.size > 0) - { - *headers = pgut_malloc(hdr.size); - IO_CHECK(fio_read_all(fio_stdin, *headers, hdr.size), hdr.size); - file->n_headers = (hdr.size / sizeof(BackupPageHeader2)) -1; - } - - break; - } - else if (hdr.cop == FIO_PAGE) - { - blknum = hdr.arg; - - Assert(hdr.size <= sizeof(buf)); - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - - COMP_FILE_CRC32(true, file->crc, buf, hdr.size); - - /* lazily open backup file */ - if (!out) - out = open_local_file_rw(to_fullpath, &out_buf, STDIO_BUFSIZE); - - if (fio_fwrite(out, buf, hdr.size) != hdr.size) - { - fio_fclose(out); - *err_blknum = blknum; - return WRITE_FAILED; - } - file->write_size += hdr.size; - file->uncompressed_size += BLCKSZ; - } - else - elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop); - } - - if (out) - fclose(out); - pg_free(out_buf); - - return n_blocks_read; -} - -/* - * Return number of actually(!) readed blocks, attempts or - * half-readed block are not counted. - * Return values in case of error: - * FILE_MISSING - * OPEN_FAILED - * READ_ERROR - * PAGE_CORRUPTION - * WRITE_FAILED - * - * If none of the above, this function return number of blocks - * readed by remote agent. - * - * In case of DELTA mode horizonLsn must be a valid lsn, - * otherwise it should be set to InvalidXLogRecPtr. - * Взято из fio_send_pages - */ -int -fio_copy_pages(const char *to_fullpath, const char *from_fullpath, pgFile *file, - XLogRecPtr horizonLsn, int calg, int clevel, uint32 checksum_version, - bool use_pagemap, BlockNumber* err_blknum, char **errormsg) -{ - FILE *out = NULL; - char *out_buf = NULL; - struct { - fio_header hdr; - fio_send_request arg; - } req; - BlockNumber n_blocks_read = 0; - BlockNumber blknum = 0; - - /* send message with header - - 16bytes 24bytes var var - -------------------------------------------------------------- - | fio_header | fio_send_request | FILE PATH | BITMAP(if any) | - -------------------------------------------------------------- - */ - - req.hdr.cop = FIO_SEND_PAGES; - - if (use_pagemap) - { - req.hdr.size = sizeof(fio_send_request) + (*file).pagemap.bitmapsize + strlen(from_fullpath) + 1; - req.arg.bitmapsize = (*file).pagemap.bitmapsize; - - /* TODO: add optimization for the case of pagemap - * containing small number of blocks with big serial numbers: - * https://github.com/postgrespro/pg_probackup/blob/remote_page_backup/src/utils/file.c#L1211 - */ - } - else - { - req.hdr.size = sizeof(fio_send_request) + strlen(from_fullpath) + 1; - req.arg.bitmapsize = 0; - } - - req.arg.nblocks = file->size/BLCKSZ; - req.arg.segmentno = file->segno * RELSEG_SIZE; - req.arg.horizonLsn = horizonLsn; - req.arg.checksumVersion = checksum_version; - req.arg.calg = calg; - req.arg.clevel = clevel; - req.arg.path_len = strlen(from_fullpath) + 1; - - file->compress_alg = calg; /* TODO: wtf? why here? */ - -//<----- -// datapagemap_iterator_t *iter; -// BlockNumber blkno; -// iter = datapagemap_iterate(pagemap); -// while (datapagemap_next(iter, &blkno)) -// elog(INFO, "block %u", blkno); -// pg_free(iter); -//<----- - - /* send header */ - IO_CHECK(fio_write_all(fio_stdout, &req, sizeof(req)), sizeof(req)); - - /* send file path */ - IO_CHECK(fio_write_all(fio_stdout, from_fullpath, req.arg.path_len), req.arg.path_len); - - /* send pagemap if any */ - if (use_pagemap) - IO_CHECK(fio_write_all(fio_stdout, (*file).pagemap.bitmap, (*file).pagemap.bitmapsize), (*file).pagemap.bitmapsize); - - out = fio_fopen(to_fullpath, PG_BINARY_R "+", FIO_BACKUP_HOST); - if (out == NULL) - elog(ERROR, "Cannot open restore target file \"%s\": %s", to_fullpath, strerror(errno)); - - /* update file permission */ - if (fio_chmod(to_fullpath, file->mode, FIO_BACKUP_HOST) == -1) - elog(ERROR, "Cannot change mode of \"%s\": %s", to_fullpath, - strerror(errno)); - - elog(VERBOSE, "ftruncate file \"%s\" to size %lu", - to_fullpath, file->size); - if (fio_ftruncate(out, file->size) == -1) - elog(ERROR, "Cannot ftruncate file \"%s\" to size %lu: %s", - to_fullpath, file->size, strerror(errno)); - - if (!fio_is_remote_file(out)) - { - out_buf = pgut_malloc(STDIO_BUFSIZE); - setvbuf(out, out_buf, _IOFBF, STDIO_BUFSIZE); - } - - while (true) - { - fio_header hdr; - char buf[BLCKSZ + sizeof(BackupPageHeader)]; - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - - if (interrupted) - elog(ERROR, "Interrupted during page reading"); - - if (hdr.cop == FIO_ERROR) - { - /* FILE_MISSING, OPEN_FAILED and READ_FAILED */ - if (hdr.size > 0) - { - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - *errormsg = pgut_malloc(hdr.size); - snprintf(*errormsg, hdr.size, "%s", buf); - } - - return hdr.arg; - } - else if (hdr.cop == FIO_SEND_FILE_CORRUPTION) - { - *err_blknum = hdr.arg; - - if (hdr.size > 0) - { - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - *errormsg = pgut_malloc(hdr.size); - snprintf(*errormsg, hdr.size, "%s", buf); - } - return PAGE_CORRUPTION; - } - else if (hdr.cop == FIO_SEND_FILE_EOF) - { - /* n_blocks_read reported by EOF */ - n_blocks_read = hdr.arg; - - /* receive headers if any */ - if (hdr.size > 0) - { - char *tmp = pgut_malloc(hdr.size); - IO_CHECK(fio_read_all(fio_stdin, tmp, hdr.size), hdr.size); - pg_free(tmp); - } - - break; - } - else if (hdr.cop == FIO_PAGE) - { - blknum = hdr.arg; - - Assert(hdr.size <= sizeof(buf)); - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - - COMP_FILE_CRC32(true, file->crc, buf, hdr.size); - - if (fio_fseek(out, blknum * BLCKSZ) < 0) - { - elog(ERROR, "Cannot seek block %u of \"%s\": %s", - blknum, to_fullpath, strerror(errno)); - } - // должен прилетать некомпрессированный блок с заголовком - // Вставить assert? - if (fio_fwrite(out, buf + sizeof(BackupPageHeader), hdr.size - sizeof(BackupPageHeader)) != BLCKSZ) - { - fio_fclose(out); - *err_blknum = blknum; - return WRITE_FAILED; - } - file->write_size += BLCKSZ; - file->uncompressed_size += BLCKSZ; - } - else - elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop); - } - - if (out) - fclose(out); - pg_free(out_buf); - - return n_blocks_read; -} - -/* TODO: read file using large buffer - * Return codes: - * FIO_ERROR: - * FILE_MISSING (-1) - * OPEN_FAILED (-2) - * READ_FAILED (-3) - - * FIO_SEND_FILE_CORRUPTION - * FIO_SEND_FILE_EOF - */ -static void -fio_send_pages_impl(int out, char* buf) -{ - FILE *in = NULL; - BlockNumber blknum = 0; - int current_pos = 0; - BlockNumber n_blocks_read = 0; - PageState page_st; - char read_buffer[BLCKSZ+1]; - char in_buf[STDIO_BUFSIZE]; - fio_header hdr; - fio_send_request *req = (fio_send_request*) buf; - char *from_fullpath = (char*) buf + sizeof(fio_send_request); - bool with_pagemap = req->bitmapsize > 0 ? true : false; - /* error reporting */ - char *errormsg = NULL; - /* parse buffer */ - datapagemap_t *map = NULL; - datapagemap_iterator_t *iter = NULL; - /* page headers */ - int32 hdr_num = -1; - int32 cur_pos_out = 0; - BackupPageHeader2 *headers = NULL; - - /* open source file */ - in = fopen(from_fullpath, PG_BINARY_R); - if (!in) - { - hdr.cop = FIO_ERROR; - - /* do not send exact wording of ENOENT error message - * because it is a very common error in our case, so - * error code is enough. - */ - if (errno == ENOENT) - { - hdr.arg = FILE_MISSING; - hdr.size = 0; - } - else - { - hdr.arg = OPEN_FAILED; - errormsg = pgut_malloc(ERRMSG_MAX_LEN); - /* Construct the error message */ - snprintf(errormsg, ERRMSG_MAX_LEN, "Cannot open file \"%s\": %s", - from_fullpath, strerror(errno)); - hdr.size = strlen(errormsg) + 1; - } - - /* send header and message */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - if (errormsg) - IO_CHECK(fio_write_all(out, errormsg, hdr.size), hdr.size); - - goto cleanup; - } - - if (with_pagemap) - { - map = pgut_malloc(sizeof(datapagemap_t)); - map->bitmapsize = req->bitmapsize; - map->bitmap = (char*) buf + sizeof(fio_send_request) + req->path_len; - - /* get first block */ - iter = datapagemap_iterate(map); - datapagemap_next(iter, &blknum); - - setvbuf(in, NULL, _IONBF, BUFSIZ); - } - else - setvbuf(in, in_buf, _IOFBF, STDIO_BUFSIZE); - - /* TODO: what is this barrier for? */ - read_buffer[BLCKSZ] = 1; /* barrier */ - - while (blknum < req->nblocks) - { - int rc = 0; - size_t read_len = 0; - int retry_attempts = PAGE_READ_ATTEMPTS; - - /* TODO: handle signals on the agent */ - if (interrupted) - elog(ERROR, "Interrupted during remote page reading"); - - /* read page, check header and validate checksumms */ - for (;;) - { - /* - * Optimize stdio buffer usage, fseek only when current position - * does not match the position of requested block. - */ - if (current_pos != blknum*BLCKSZ) - { - current_pos = blknum*BLCKSZ; - if (fseek(in, current_pos, SEEK_SET) != 0) - elog(ERROR, "fseek to position %u is failed on remote file '%s': %s", - current_pos, from_fullpath, strerror(errno)); - } - - read_len = fread(read_buffer, 1, BLCKSZ, in); - - current_pos += read_len; - - /* report error */ - if (ferror(in)) - { - hdr.cop = FIO_ERROR; - hdr.arg = READ_FAILED; - - errormsg = pgut_malloc(ERRMSG_MAX_LEN); - /* Construct the error message */ - snprintf(errormsg, ERRMSG_MAX_LEN, "Cannot read block %u of '%s': %s", - blknum, from_fullpath, strerror(errno)); - hdr.size = strlen(errormsg) + 1; - - /* send header and message */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, errormsg, hdr.size), hdr.size); - goto cleanup; - } - - if (read_len == BLCKSZ) - { - rc = validate_one_page(read_buffer, req->segmentno + blknum, - InvalidXLogRecPtr, &page_st, - req->checksumVersion); - - /* TODO: optimize copy of zeroed page */ - if (rc == PAGE_IS_ZEROED) - break; - else if (rc == PAGE_IS_VALID) - break; - } - - if (feof(in)) - goto eof; -// else /* readed less than BLKSZ bytes, retry */ - - /* File is either has insane header or invalid checksum, - * retry. If retry attempts are exhausted, report corruption. - */ - if (--retry_attempts == 0) - { - hdr.cop = FIO_SEND_FILE_CORRUPTION; - hdr.arg = blknum; - - /* Construct the error message */ - if (rc == PAGE_HEADER_IS_INVALID) - get_header_errormsg(read_buffer, &errormsg); - else if (rc == PAGE_CHECKSUM_MISMATCH) - get_checksum_errormsg(read_buffer, &errormsg, - req->segmentno + blknum); - - /* if error message is not empty, set payload size to its length */ - hdr.size = errormsg ? strlen(errormsg) + 1 : 0; - - /* send header */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - - /* send error message if any */ - if (errormsg) - IO_CHECK(fio_write_all(out, errormsg, hdr.size), hdr.size); - - goto cleanup; - } - } - - n_blocks_read++; - - /* - * horizonLsn is not 0 only in case of delta and ptrack backup. - * As far as unsigned number are always greater or equal than zero, - * there is no sense to add more checks. - */ - if ((req->horizonLsn == InvalidXLogRecPtr) || /* full, page */ - (page_st.lsn == InvalidXLogRecPtr) || /* zeroed page */ - (req->horizonLsn > 0 && page_st.lsn > req->horizonLsn)) /* delta, ptrack */ - { - int compressed_size = 0; - char write_buffer[BLCKSZ*2]; - BackupPageHeader* bph = (BackupPageHeader*)write_buffer; - - /* compress page */ - hdr.cop = FIO_PAGE; - hdr.arg = blknum; - - compressed_size = do_compress(write_buffer + sizeof(BackupPageHeader), - sizeof(write_buffer) - sizeof(BackupPageHeader), - read_buffer, BLCKSZ, req->calg, req->clevel, - NULL); - - if (compressed_size <= 0 || compressed_size >= BLCKSZ) - { - /* Do not compress page */ - memcpy(write_buffer + sizeof(BackupPageHeader), read_buffer, BLCKSZ); - compressed_size = BLCKSZ; - } - bph->block = blknum; - bph->compressed_size = compressed_size; - - hdr.size = compressed_size + sizeof(BackupPageHeader); - - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, write_buffer, hdr.size), hdr.size); - - /* set page header for this file */ - hdr_num++; - if (!headers) - headers = (BackupPageHeader2 *) pgut_malloc(sizeof(BackupPageHeader2)); - else - headers = (BackupPageHeader2 *) pgut_realloc(headers, (hdr_num+1) * sizeof(BackupPageHeader2)); - - headers[hdr_num].block = blknum; - headers[hdr_num].lsn = page_st.lsn; - headers[hdr_num].checksum = page_st.checksum; - headers[hdr_num].pos = cur_pos_out; - - cur_pos_out += hdr.size; - } - - /* next block */ - if (with_pagemap) - { - /* exit if pagemap is exhausted */ - if (!datapagemap_next(iter, &blknum)) - break; - } - else - blknum++; - } - -eof: - /* We are done, send eof */ - hdr.cop = FIO_SEND_FILE_EOF; - hdr.arg = n_blocks_read; - hdr.size = 0; - - if (headers) - { - hdr.size = (hdr_num+2) * sizeof(BackupPageHeader2); - - /* add dummy header */ - headers = (BackupPageHeader2 *) pgut_realloc(headers, (hdr_num+2) * sizeof(BackupPageHeader2)); - headers[hdr_num+1].pos = cur_pos_out; - } - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - - if (headers) - IO_CHECK(fio_write_all(out, headers, hdr.size), hdr.size); - -cleanup: - pg_free(map); - pg_free(iter); - pg_free(errormsg); - pg_free(headers); - if (in) - fclose(in); - return; -} - -/* Receive chunks of compressed data, decompress them and write to - * destination file. - * Return codes: - * FILE_MISSING (-1) - * OPEN_FAILED (-2) - * READ_FAILED (-3) - * WRITE_FAILED (-4) - * ZLIB_ERROR (-5) - * REMOTE_ERROR (-6) - */ -int -fio_send_file_gz(const char *from_fullpath, FILE* out, char **errormsg) -{ - fio_header hdr; - int exit_code = SEND_OK; - char *in_buf = pgut_malloc(CHUNK_SIZE); /* buffer for compressed data */ - char *out_buf = pgut_malloc(OUT_BUF_SIZE); /* 1MB buffer for decompressed data */ - size_t path_len = strlen(from_fullpath) + 1; - /* decompressor */ - z_stream *strm = NULL; - - hdr.cop = FIO_SEND_FILE; - hdr.size = path_len; - -// elog(VERBOSE, "Thread [%d]: Attempting to open remote compressed WAL file '%s'", -// thread_num, from_fullpath); - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, from_fullpath, path_len), path_len); - - for (;;) - { - fio_header hdr; - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - - if (hdr.cop == FIO_SEND_FILE_EOF) - { - break; - } - else if (hdr.cop == FIO_ERROR) - { - /* handle error, reported by the agent */ - if (hdr.size > 0) - { - IO_CHECK(fio_read_all(fio_stdin, in_buf, hdr.size), hdr.size); - *errormsg = pgut_malloc(hdr.size); - snprintf(*errormsg, hdr.size, "%s", in_buf); - } - exit_code = hdr.arg; - goto cleanup; - } - else if (hdr.cop == FIO_PAGE) - { - int rc; - Assert(hdr.size <= CHUNK_SIZE); - IO_CHECK(fio_read_all(fio_stdin, in_buf, hdr.size), hdr.size); - - /* We have received a chunk of compressed data, lets decompress it */ - if (strm == NULL) - { - /* Initialize decompressor */ - strm = pgut_malloc(sizeof(z_stream)); - memset(strm, 0, sizeof(z_stream)); - - /* The fields next_in, avail_in initialized before init */ - strm->next_in = (Bytef *)in_buf; - strm->avail_in = hdr.size; - - rc = inflateInit2(strm, 15 + 16); - - if (rc != Z_OK) - { - *errormsg = pgut_malloc(ERRMSG_MAX_LEN); - snprintf(*errormsg, ERRMSG_MAX_LEN, - "Failed to initialize decompression stream for file '%s': %i: %s", - from_fullpath, rc, strm->msg); - exit_code = ZLIB_ERROR; - goto cleanup; - } - } - else - { - strm->next_in = (Bytef *)in_buf; - strm->avail_in = hdr.size; - } - - strm->next_out = (Bytef *)out_buf; /* output buffer */ - strm->avail_out = OUT_BUF_SIZE; /* free space in output buffer */ - - /* - * From zlib documentation: - * The application must update next_in and avail_in when avail_in - * has dropped to zero. It must update next_out and avail_out when - * avail_out has dropped to zero. - */ - while (strm->avail_in != 0) /* while there is data in input buffer, decompress it */ - { - /* decompress until there is no data to decompress, - * or buffer with uncompressed data is full - */ - rc = inflate(strm, Z_NO_FLUSH); - if (rc == Z_STREAM_END) - /* end of stream */ - break; - else if (rc != Z_OK) - { - /* got an error */ - *errormsg = pgut_malloc(ERRMSG_MAX_LEN); - snprintf(*errormsg, ERRMSG_MAX_LEN, - "Decompression failed for file '%s': %i: %s", - from_fullpath, rc, strm->msg); - exit_code = ZLIB_ERROR; - goto cleanup; - } - - if (strm->avail_out == 0) - { - /* Output buffer is full, write it out */ - if (fwrite(out_buf, 1, OUT_BUF_SIZE, out) != OUT_BUF_SIZE) - { - exit_code = WRITE_FAILED; - goto cleanup; - } - - strm->next_out = (Bytef *)out_buf; /* output buffer */ - strm->avail_out = OUT_BUF_SIZE; - } - } - - /* write out leftovers if any */ - if (strm->avail_out != OUT_BUF_SIZE) - { - int len = OUT_BUF_SIZE - strm->avail_out; - - if (fwrite(out_buf, 1, len, out) != len) - { - exit_code = WRITE_FAILED; - goto cleanup; - } - } - } - else - elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop); - } - -cleanup: - if (exit_code < OPEN_FAILED) - fio_disconnect(); /* discard possible pending data in pipe */ - - if (strm) - { - inflateEnd(strm); - pg_free(strm); - } - - pg_free(in_buf); - pg_free(out_buf); - return exit_code; -} - -typedef struct send_file_state { - bool calc_crc; - uint32_t crc; - int64_t read_size; - int64_t write_size; -} send_file_state; - -/* find page border of all-zero tail */ -static size_t -find_zero_tail(char *buf, size_t len) -{ - size_t i, l; - size_t granul = sizeof(zerobuf); - - if (len == 0) - return 0; - - /* fast check for last bytes */ - l = Min(len, PAGE_ZEROSEARCH_FINE_GRANULARITY); - i = len - l; - if (memcmp(buf + i, zerobuf, l) != 0) - return len; - - /* coarse search for zero tail */ - i = (len-1) & ~(granul-1); - l = len - i; - for (;;) - { - if (memcmp(buf+i, zerobuf, l) != 0) - { - i += l; - break; - } - if (i == 0) - break; - i -= granul; - l = granul; - } - - len = i; - /* search zero tail with finer granularity */ - for (granul = sizeof(zerobuf)/2; - len > 0 && granul >= PAGE_ZEROSEARCH_FINE_GRANULARITY; - granul /= 2) - { - if (granul > l) - continue; - i = (len-1) & ~(granul-1); - l = len - i; - if (memcmp(buf+i, zerobuf, l) == 0) - len = i; - } - - return len; -} - -static void -fio_send_file_crc(send_file_state* st, char *buf, size_t len) -{ - int64_t write_size; - - if (!st->calc_crc) - return; - - write_size = st->write_size; - while (st->read_size > write_size) - { - size_t crc_len = Min(st->read_size - write_size, sizeof(zerobuf)); - COMP_FILE_CRC32(true, st->crc, zerobuf, crc_len); - write_size += crc_len; - } - - if (len > 0) - COMP_FILE_CRC32(true, st->crc, buf, len); -} - -static bool -fio_send_file_write(FILE* out, send_file_state* st, char *buf, size_t len) -{ - if (len == 0) - return true; - -#ifdef WIN32 - if (st->read_size > st->write_size && - _chsize_s(fileno(out), st->read_size) != 0) - { - elog(WARNING, "Could not change file size to %lld: %m", st->read_size); - return false; - } -#endif - if (st->read_size > st->write_size && - fseeko(out, st->read_size, SEEK_SET) != 0) - { - return false; - } - - if (fwrite(buf, 1, len, out) != len) - { - return false; - } - - st->read_size += len; - st->write_size = st->read_size; - - return true; -} - -/* Receive chunks of data and write them to destination file. - * Return codes: - * SEND_OK (0) - * FILE_MISSING (-1) - * OPEN_FAILED (-2) - * READ_FAILED (-3) - * WRITE_FAILED (-4) - * - * OPEN_FAILED and READ_FAIL should also set errormsg. - * If pgFile is not NULL then we must calculate crc and read_size for it. - */ -int -fio_send_file(const char *from_fullpath, FILE* out, bool cut_zero_tail, - pgFile *file, char **errormsg) -{ - fio_header hdr; - int exit_code = SEND_OK; - size_t path_len = strlen(from_fullpath) + 1; - char *buf = pgut_malloc(CHUNK_SIZE); /* buffer */ - send_file_state st = {false, 0, 0, 0}; - - memset(&hdr, 0, sizeof(hdr)); - - if (file) - { - st.calc_crc = true; - st.crc = file->crc; - } - - hdr.cop = FIO_SEND_FILE; - hdr.size = path_len; - -// elog(VERBOSE, "Thread [%d]: Attempting to open remote WAL file '%s'", -// thread_num, from_fullpath); - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, from_fullpath, path_len), path_len); - - for (;;) - { - /* receive data */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - - if (hdr.cop == FIO_SEND_FILE_EOF) - { - if (st.write_size < st.read_size) - { - if (!cut_zero_tail) - { - /* - * We still need to calc crc for zero tail. - */ - fio_send_file_crc(&st, NULL, 0); - - /* - * Let's write single zero byte to the end of file to restore - * logical size. - * Well, it would be better to use ftruncate here actually, - * but then we need to change interface. - */ - st.read_size -= 1; - buf[0] = 0; - if (!fio_send_file_write(out, &st, buf, 1)) - { - exit_code = WRITE_FAILED; - break; - } - } - } - - if (file) - { - file->crc = st.crc; - file->read_size = st.read_size; - file->write_size = st.write_size; - } - break; - } - else if (hdr.cop == FIO_ERROR) - { - /* handle error, reported by the agent */ - if (hdr.size > 0) - { - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - *errormsg = pgut_malloc(hdr.size); - snprintf(*errormsg, hdr.size, "%s", buf); - } - exit_code = hdr.arg; - break; - } - else if (hdr.cop == FIO_PAGE) - { - Assert(hdr.size <= CHUNK_SIZE); - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - - /* We have received a chunk of data data, lets write it out */ - fio_send_file_crc(&st, buf, hdr.size); - if (!fio_send_file_write(out, &st, buf, hdr.size)) - { - exit_code = WRITE_FAILED; - break; - } - } - else if (hdr.cop == FIO_PAGE_ZERO) - { - Assert(hdr.size == 0); - Assert(hdr.arg <= CHUNK_SIZE); - - /* - * We have received a chunk of zero data, lets just think we - * wrote it. - */ - st.read_size += hdr.arg; - } - else - { - /* TODO: fio_disconnect may get assert fail when running after this */ - elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop); - } - } - - if (exit_code < OPEN_FAILED) - fio_disconnect(); /* discard possible pending data in pipe */ - - pg_free(buf); - return exit_code; -} - -int -fio_send_file_local(const char *from_fullpath, FILE* out, bool cut_zero_tail, - pgFile *file, char **errormsg) -{ - FILE* in; - char* buf; - size_t read_len, non_zero_len; - int exit_code = SEND_OK; - send_file_state st = {false, 0, 0, 0}; - - if (file) - { - st.calc_crc = true; - st.crc = file->crc; - } - - /* open source file for read */ - in = fopen(from_fullpath, PG_BINARY_R); - if (in == NULL) - { - /* maybe deleted, it's not error in case of backup */ - if (errno == ENOENT) - return FILE_MISSING; - - - *errormsg = psprintf("Cannot open file \"%s\": %s", from_fullpath, - strerror(errno)); - return OPEN_FAILED; - } - - /* disable stdio buffering for local input/output files to avoid triple buffering */ - setvbuf(in, NULL, _IONBF, BUFSIZ); - setvbuf(out, NULL, _IONBF, BUFSIZ); - - /* allocate 64kB buffer */ - buf = pgut_malloc(CHUNK_SIZE); - - /* copy content and calc CRC */ - for (;;) - { - read_len = fread(buf, 1, CHUNK_SIZE, in); - - if (ferror(in)) - { - *errormsg = psprintf("Cannot read from file \"%s\": %s", - from_fullpath, strerror(errno)); - exit_code = READ_FAILED; - goto cleanup; - } - - if (read_len > 0) - { - non_zero_len = find_zero_tail(buf, read_len); - /* - * It is dirty trick to silence warnings in CFS GC process: - * backup at least cfs header size bytes. - */ - if (st.read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY && - st.read_size + read_len > 0) - { - non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY, - st.read_size + read_len); - non_zero_len -= st.read_size; - } - if (non_zero_len > 0) - { - fio_send_file_crc(&st, buf, non_zero_len); - if (!fio_send_file_write(out, &st, buf, non_zero_len)) - { - exit_code = WRITE_FAILED; - goto cleanup; - } - } - if (non_zero_len < read_len) - { - /* Just pretend we wrote it. */ - st.read_size += read_len - non_zero_len; - } - } - - if (feof(in)) - break; - } - - if (st.write_size < st.read_size) - { - if (!cut_zero_tail) - { - /* - * We still need to calc crc for zero tail. - */ - fio_send_file_crc(&st, NULL, 0); - - /* - * Let's write single zero byte to the end of file to restore - * logical size. - * Well, it would be better to use ftruncate here actually, - * but then we need to change interface. - */ - st.read_size -= 1; - buf[0] = 0; - if (!fio_send_file_write(out, &st, buf, 1)) - { - exit_code = WRITE_FAILED; - goto cleanup; - } - } - } - - if (file) - { - file->crc = st.crc; - file->read_size = st.read_size; - file->write_size = st.write_size; - } - -cleanup: - free(buf); - fclose(in); - return exit_code; -} - -/* Send file content - * On error we return FIO_ERROR message with following codes - * FIO_ERROR: - * FILE_MISSING (-1) - * OPEN_FAILED (-2) - * READ_FAILED (-3) - * - * FIO_PAGE - * FIO_SEND_FILE_EOF - * - */ -static void -fio_send_file_impl(int out, char const* path) -{ - FILE *fp; - fio_header hdr; - char *buf = pgut_malloc(CHUNK_SIZE); - size_t read_len = 0; - int64_t read_size = 0; - char *errormsg = NULL; - - /* open source file for read */ - /* TODO: check that file is regular file */ - fp = fopen(path, PG_BINARY_R); - if (!fp) - { - hdr.cop = FIO_ERROR; - - /* do not send exact wording of ENOENT error message - * because it is a very common error in our case, so - * error code is enough. - */ - if (errno == ENOENT) - { - hdr.arg = FILE_MISSING; - hdr.size = 0; - } - else - { - hdr.arg = OPEN_FAILED; - errormsg = pgut_malloc(ERRMSG_MAX_LEN); - /* Construct the error message */ - snprintf(errormsg, ERRMSG_MAX_LEN, "Cannot open file '%s': %s", path, strerror(errno)); - hdr.size = strlen(errormsg) + 1; - } - - /* send header and message */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - if (errormsg) - IO_CHECK(fio_write_all(out, errormsg, hdr.size), hdr.size); - - goto cleanup; - } - - /* disable stdio buffering */ - setvbuf(fp, NULL, _IONBF, BUFSIZ); - - /* copy content */ - for (;;) - { - read_len = fread(buf, 1, CHUNK_SIZE, fp); - memset(&hdr, 0, sizeof(hdr)); - - /* report error */ - if (ferror(fp)) - { - hdr.cop = FIO_ERROR; - errormsg = pgut_malloc(ERRMSG_MAX_LEN); - hdr.arg = READ_FAILED; - /* Construct the error message */ - snprintf(errormsg, ERRMSG_MAX_LEN, "Cannot read from file '%s': %s", path, strerror(errno)); - hdr.size = strlen(errormsg) + 1; - /* send header and message */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, errormsg, hdr.size), hdr.size); - - goto cleanup; - } - - if (read_len > 0) - { - /* send chunk */ - int64_t non_zero_len = find_zero_tail(buf, read_len); - /* - * It is dirty trick to silence warnings in CFS GC process: - * backup at least cfs header size bytes. - */ - if (read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY && - read_size + read_len > 0) - { - non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY, - read_size + read_len); - non_zero_len -= read_size; - } - - if (non_zero_len > 0) - { - hdr.cop = FIO_PAGE; - hdr.size = non_zero_len; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, buf, non_zero_len), non_zero_len); - } - - if (non_zero_len < read_len) - { - hdr.cop = FIO_PAGE_ZERO; - hdr.size = 0; - hdr.arg = read_len - non_zero_len; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - } - - read_size += read_len; - } - - if (feof(fp)) - break; - } - - /* we are done, send eof */ - hdr.cop = FIO_SEND_FILE_EOF; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - -cleanup: - if (fp) - fclose(fp); - pg_free(buf); - pg_free(errormsg); - return; -} - -/* - * Read the local file to compute its CRC. - * We cannot make decision about file decompression because - * user may ask to backup already compressed files and we should be - * obvious about it. - */ -pg_crc32 -pgFileGetCRC(const char *file_path, bool use_crc32c, bool missing_ok) -{ - FILE *fp; - pg_crc32 crc = 0; - char *buf; - size_t len = 0; - - INIT_FILE_CRC32(use_crc32c, crc); - - /* open file in binary read mode */ - fp = fopen(file_path, PG_BINARY_R); - if (fp == NULL) - { - if (errno == ENOENT) - { - if (missing_ok) - { - FIN_FILE_CRC32(use_crc32c, crc); - return crc; - } - } - - elog(ERROR, "Cannot open file \"%s\": %s", - file_path, strerror(errno)); - } - - /* disable stdio buffering */ - setvbuf(fp, NULL, _IONBF, BUFSIZ); - buf = pgut_malloc(STDIO_BUFSIZE); - - /* calc CRC of file */ - for (;;) - { - if (interrupted) - elog(ERROR, "interrupted during CRC calculation"); - - len = fread(buf, 1, STDIO_BUFSIZE, fp); - - if (ferror(fp)) - elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno)); - - /* update CRC */ - COMP_FILE_CRC32(use_crc32c, crc, buf, len); - - if (feof(fp)) - break; - } - - FIN_FILE_CRC32(use_crc32c, crc); - fclose(fp); - pg_free(buf); - - return crc; -} - -/* - * Read the local file to compute CRC for it extened to real_size. - */ -pg_crc32 -pgFileGetCRCTruncated(const char *file_path, bool use_crc32c, bool missing_ok) -{ - FILE *fp; - char *buf; - size_t len = 0; - size_t non_zero_len; - send_file_state st = {true, 0, 0, 0}; - - INIT_FILE_CRC32(use_crc32c, st.crc); - - /* open file in binary read mode */ - fp = fopen(file_path, PG_BINARY_R); - if (fp == NULL) - { - if (errno == ENOENT) - { - if (missing_ok) - { - FIN_FILE_CRC32(use_crc32c, st.crc); - return st.crc; - } - } - - elog(ERROR, "Cannot open file \"%s\": %s", - file_path, strerror(errno)); - } - - /* disable stdio buffering */ - setvbuf(fp, NULL, _IONBF, BUFSIZ); - buf = pgut_malloc(CHUNK_SIZE); - - /* calc CRC of file */ - for (;;) - { - if (interrupted) - elog(ERROR, "interrupted during CRC calculation"); - - len = fread(buf, 1, STDIO_BUFSIZE, fp); - - if (ferror(fp)) - elog(ERROR, "Cannot read \"%s\": %s", file_path, strerror(errno)); - - non_zero_len = find_zero_tail(buf, len); - /* same trick as in fio_send_file */ - if (st.read_size + non_zero_len < PAGE_ZEROSEARCH_FINE_GRANULARITY && - st.read_size + len > 0) - { - non_zero_len = Min(PAGE_ZEROSEARCH_FINE_GRANULARITY, - st.read_size + len); - non_zero_len -= st.read_size; - } - if (non_zero_len) - { - fio_send_file_crc(&st, buf, non_zero_len); - st.write_size += st.read_size + non_zero_len; - } - st.read_size += len; - - if (feof(fp)) - break; - } - - FIN_FILE_CRC32(use_crc32c, st.crc); - fclose(fp); - pg_free(buf); - - return st.crc; -} - -/* - * Read the local file to compute its CRC. - * We cannot make decision about file decompression because - * user may ask to backup already compressed files and we should be - * obvious about it. - */ -pg_crc32 -pgFileGetCRCgz(const char *file_path, bool use_crc32c, bool missing_ok) -{ - gzFile fp; - pg_crc32 crc = 0; - int len = 0; - int err; - char *buf; - - INIT_FILE_CRC32(use_crc32c, crc); - - /* open file in binary read mode */ - fp = gzopen(file_path, PG_BINARY_R); - if (fp == NULL) - { - if (errno == ENOENT) - { - if (missing_ok) - { - FIN_FILE_CRC32(use_crc32c, crc); - return crc; - } - } - - elog(ERROR, "Cannot open file \"%s\": %s", - file_path, strerror(errno)); - } - - buf = pgut_malloc(STDIO_BUFSIZE); - - /* calc CRC of file */ - for (;;) - { - if (interrupted) - elog(ERROR, "interrupted during CRC calculation"); - - len = gzread(fp, buf, STDIO_BUFSIZE); - - if (len <= 0) - { - /* we either run into eof or error */ - if (gzeof(fp)) - break; - else - { - const char *err_str = NULL; - - err_str = gzerror(fp, &err); - elog(ERROR, "Cannot read from compressed file %s", err_str); - } - } - - /* update CRC */ - COMP_FILE_CRC32(use_crc32c, crc, buf, len); - } - - FIN_FILE_CRC32(use_crc32c, crc); - gzclose(fp); - pg_free(buf); - - return crc; -} - -/* Compile the array of files located on remote machine in directory root */ -static void -fio_list_dir_internal(parray *files, const char *root, bool exclude, - bool follow_symlink, bool add_root, bool backup_logs, - bool skip_hidden, int external_dir_num) -{ - fio_header hdr; - fio_list_dir_request req; - char *buf = pgut_malloc(CHUNK_SIZE); - - /* Send to the agent message with parameters for directory listing */ - snprintf(req.path, MAXPGPATH, "%s", root); - req.exclude = exclude; - req.follow_symlink = follow_symlink; - req.add_root = add_root; - req.backup_logs = backup_logs; - req.exclusive_backup = exclusive_backup; - req.skip_hidden = skip_hidden; - req.external_dir_num = external_dir_num; - - hdr.cop = FIO_LIST_DIR; - hdr.size = sizeof(req); - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, &req, hdr.size), hdr.size); - - for (;;) - { - /* receive data */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - - if (hdr.cop == FIO_SEND_FILE_EOF) - { - /* the work is done */ - break; - } - else if (hdr.cop == FIO_SEND_FILE) - { - pgFile *file = NULL; - fio_pgFile fio_file; - - /* receive rel_path */ - IO_CHECK(fio_read_all(fio_stdin, buf, hdr.size), hdr.size); - file = pgFileInit(buf); - - /* receive metainformation */ - IO_CHECK(fio_read_all(fio_stdin, &fio_file, sizeof(fio_file)), sizeof(fio_file)); - - file->mode = fio_file.mode; - file->size = fio_file.size; - file->mtime = fio_file.mtime; - file->is_datafile = fio_file.is_datafile; - file->tblspcOid = fio_file.tblspcOid; - file->dbOid = fio_file.dbOid; - file->relOid = fio_file.relOid; - file->forkName = fio_file.forkName; - file->segno = fio_file.segno; - file->external_dir_num = fio_file.external_dir_num; - - if (fio_file.linked_len > 0) - { - IO_CHECK(fio_read_all(fio_stdin, buf, fio_file.linked_len), fio_file.linked_len); - - file->linked = pgut_malloc(fio_file.linked_len); - snprintf(file->linked, fio_file.linked_len, "%s", buf); - } - -// elog(INFO, "Received file: %s, mode: %u, size: %lu, mtime: %lu", -// file->rel_path, file->mode, file->size, file->mtime); - - parray_append(files, file); - } - else - { - /* TODO: fio_disconnect may get assert fail when running after this */ - elog(ERROR, "Remote agent returned message of unexpected type: %i", hdr.cop); - } - } - - pg_free(buf); -} - - -/* - * To get the arrays of files we use the same function dir_list_file(), - * that is used for local backup. - * After that we iterate over arrays and for every file send at least - * two messages to main process: - * 1. rel_path - * 2. metainformation (size, mtime, etc) - * 3. link path (optional) - * - * TODO: replace FIO_SEND_FILE and FIO_SEND_FILE_EOF with dedicated messages - */ -static void -fio_list_dir_impl(int out, char* buf) -{ - int i; - fio_header hdr; - fio_list_dir_request *req = (fio_list_dir_request*) buf; - parray *file_files = parray_new(); - - /* - * Disable logging into console any messages with exception of ERROR messages, - * because currently we have no mechanism to notify the main process - * about then message been sent. - * TODO: correctly send elog messages from agent to main process. - */ - instance_config.logger.log_level_console = ERROR; - exclusive_backup = req->exclusive_backup; - - dir_list_file(file_files, req->path, req->exclude, req->follow_symlink, - req->add_root, req->backup_logs, req->skip_hidden, - req->external_dir_num, FIO_LOCAL_HOST); - - /* send information about files to the main process */ - for (i = 0; i < parray_num(file_files); i++) - { - fio_pgFile fio_file; - pgFile *file = (pgFile *) parray_get(file_files, i); - - fio_file.mode = file->mode; - fio_file.size = file->size; - fio_file.mtime = file->mtime; - fio_file.is_datafile = file->is_datafile; - fio_file.tblspcOid = file->tblspcOid; - fio_file.dbOid = file->dbOid; - fio_file.relOid = file->relOid; - fio_file.forkName = file->forkName; - fio_file.segno = file->segno; - fio_file.external_dir_num = file->external_dir_num; - - if (file->linked) - fio_file.linked_len = strlen(file->linked) + 1; - else - fio_file.linked_len = 0; - - hdr.cop = FIO_SEND_FILE; - hdr.size = strlen(file->rel_path) + 1; - - /* send rel_path first */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, file->rel_path, hdr.size), hdr.size); - - /* now send file metainformation */ - IO_CHECK(fio_write_all(out, &fio_file, sizeof(fio_file)), sizeof(fio_file)); - - /* If file is a symlink, then send link path */ - if (file->linked) - IO_CHECK(fio_write_all(out, file->linked, fio_file.linked_len), fio_file.linked_len); - - pgFileFree(file); - } - - parray_free(file_files); - hdr.cop = FIO_SEND_FILE_EOF; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); -} - -/* Wrapper for directory listing */ -void -fio_list_dir(parray *files, const char *root, bool exclude, - bool follow_symlink, bool add_root, bool backup_logs, - bool skip_hidden, int external_dir_num) -{ - if (fio_is_remote(FIO_DB_HOST)) - fio_list_dir_internal(files, root, exclude, follow_symlink, add_root, - backup_logs, skip_hidden, external_dir_num); - else - dir_list_file(files, root, exclude, follow_symlink, add_root, - backup_logs, skip_hidden, external_dir_num, FIO_LOCAL_HOST); -} - -PageState * -fio_get_checksum_map(const char *fullpath, uint32 checksum_version, int n_blocks, - XLogRecPtr dest_stop_lsn, BlockNumber segmentno, fio_location location) -{ - if (fio_is_remote(location)) - { - fio_header hdr; - fio_checksum_map_request req_hdr; - PageState *checksum_map = NULL; - size_t path_len = strlen(fullpath) + 1; - - req_hdr.n_blocks = n_blocks; - req_hdr.segmentno = segmentno; - req_hdr.stop_lsn = dest_stop_lsn; - req_hdr.checksumVersion = checksum_version; - - hdr.cop = FIO_GET_CHECKSUM_MAP; - hdr.size = sizeof(req_hdr) + path_len; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, &req_hdr, sizeof(req_hdr)), sizeof(req_hdr)); - IO_CHECK(fio_write_all(fio_stdout, fullpath, path_len), path_len); - - /* receive data */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - - if (hdr.size > 0) - { - checksum_map = pgut_malloc(n_blocks * sizeof(PageState)); - memset(checksum_map, 0, n_blocks * sizeof(PageState)); - IO_CHECK(fio_read_all(fio_stdin, checksum_map, hdr.size * sizeof(PageState)), hdr.size * sizeof(PageState)); - } - - return checksum_map; - } - else - { - - return get_checksum_map(fullpath, checksum_version, - n_blocks, dest_stop_lsn, segmentno); - } -} - -static void -fio_get_checksum_map_impl(int out, char *buf) -{ - fio_header hdr; - PageState *checksum_map = NULL; - char *fullpath = (char*) buf + sizeof(fio_checksum_map_request); - fio_checksum_map_request *req = (fio_checksum_map_request*) buf; - - checksum_map = get_checksum_map(fullpath, req->checksumVersion, - req->n_blocks, req->stop_lsn, req->segmentno); - hdr.size = req->n_blocks; - - /* send array of PageState`s to main process */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - if (hdr.size > 0) - IO_CHECK(fio_write_all(out, checksum_map, hdr.size * sizeof(PageState)), hdr.size * sizeof(PageState)); - - pg_free(checksum_map); -} - -datapagemap_t * -fio_get_lsn_map(const char *fullpath, uint32 checksum_version, - int n_blocks, XLogRecPtr shift_lsn, BlockNumber segmentno, - fio_location location) -{ - datapagemap_t* lsn_map = NULL; - - if (fio_is_remote(location)) - { - fio_header hdr; - fio_lsn_map_request req_hdr; - size_t path_len = strlen(fullpath) + 1; - - req_hdr.n_blocks = n_blocks; - req_hdr.segmentno = segmentno; - req_hdr.shift_lsn = shift_lsn; - req_hdr.checksumVersion = checksum_version; - - hdr.cop = FIO_GET_LSN_MAP; - hdr.size = sizeof(req_hdr) + path_len; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, &req_hdr, sizeof(req_hdr)), sizeof(req_hdr)); - IO_CHECK(fio_write_all(fio_stdout, fullpath, path_len), path_len); - - /* receive data */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - - if (hdr.size > 0) - { - lsn_map = pgut_malloc(sizeof(datapagemap_t)); - memset(lsn_map, 0, sizeof(datapagemap_t)); - - lsn_map->bitmap = pgut_malloc(hdr.size); - lsn_map->bitmapsize = hdr.size; - - IO_CHECK(fio_read_all(fio_stdin, lsn_map->bitmap, hdr.size), hdr.size); - } - } - else - { - lsn_map = get_lsn_map(fullpath, checksum_version, n_blocks, - shift_lsn, segmentno); - } - - return lsn_map; -} - -static void -fio_get_lsn_map_impl(int out, char *buf) -{ - fio_header hdr; - datapagemap_t *lsn_map = NULL; - char *fullpath = (char*) buf + sizeof(fio_lsn_map_request); - fio_lsn_map_request *req = (fio_lsn_map_request*) buf; - - lsn_map = get_lsn_map(fullpath, req->checksumVersion, req->n_blocks, - req->shift_lsn, req->segmentno); - if (lsn_map) - hdr.size = lsn_map->bitmapsize; - else - hdr.size = 0; - - /* send bitmap to main process */ - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - if (hdr.size > 0) - IO_CHECK(fio_write_all(out, lsn_map->bitmap, hdr.size), hdr.size); - - if (lsn_map) - { - pg_free(lsn_map->bitmap); - pg_free(lsn_map); - } -} - -/* - * Return pid of postmaster process running in given pgdata on local machine. - * Return 0 if there is none. - * Return 1 if postmaster.pid is mangled. - */ -static pid_t -local_check_postmaster(const char *pgdata) -{ - FILE *fp; - pid_t pid; - char pid_file[MAXPGPATH]; - - join_path_components(pid_file, pgdata, "postmaster.pid"); - - fp = fopen(pid_file, "r"); - if (fp == NULL) - { - /* No pid file, acceptable*/ - if (errno == ENOENT) - return 0; - else - elog(ERROR, "Cannot open file \"%s\": %s", - pid_file, strerror(errno)); - } - - if (fscanf(fp, "%i", &pid) != 1) - { - /* something is wrong with the file content */ - pid = 1; - } - - if (pid > 1) - { - if (kill(pid, 0) != 0) - { - /* process no longer exists */ - if (errno == ESRCH) - pid = 0; - else - elog(ERROR, "Failed to send signal 0 to a process %d: %s", - pid, strerror(errno)); - } - } - - fclose(fp); - return pid; -} - -/* - * Go to the remote host and get postmaster pid from file postmaster.pid - * and check that process is running, if process is running, return its pid number. - */ -pid_t -fio_check_postmaster(const char *pgdata, fio_location location) -{ - if (fio_is_remote(location)) - { - fio_header hdr; - - hdr.cop = FIO_CHECK_POSTMASTER; - hdr.size = strlen(pgdata) + 1; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, pgdata, hdr.size), hdr.size); - - /* receive result */ - IO_CHECK(fio_read_all(fio_stdin, &hdr, sizeof(hdr)), sizeof(hdr)); - return hdr.arg; - } - else - return local_check_postmaster(pgdata); -} - -static void -fio_check_postmaster_impl(int out, char *buf) -{ - fio_header hdr; - pid_t postmaster_pid; - char *pgdata = (char*) buf; - - postmaster_pid = local_check_postmaster(pgdata); - - /* send arrays of checksums to main process */ - hdr.arg = postmaster_pid; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); -} - -/* - * Delete file pointed by the pgFile. - * If the pgFile points directory, the directory must be empty. - */ -void -fio_delete(mode_t mode, const char *fullpath, fio_location location) -{ - if (fio_is_remote(location)) - { - fio_header hdr; - - hdr.cop = FIO_DELETE; - hdr.size = strlen(fullpath) + 1; - hdr.arg = mode; - - IO_CHECK(fio_write_all(fio_stdout, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(fio_stdout, fullpath, hdr.size), hdr.size); - - } - else - pgFileDelete(mode, fullpath); -} - -static void -fio_delete_impl(mode_t mode, char *buf) -{ - char *fullpath = (char*) buf; - - pgFileDelete(mode, fullpath); -} - -/* Execute commands at remote host */ -void -fio_communicate(int in, int out) -{ - /* - * Map of file and directory descriptors. - * The same mapping is used in agent and master process, so we - * can use the same index at both sides. - */ - int fd[FIO_FDMAX]; - DIR* dir[FIO_FDMAX]; - struct dirent* entry; - size_t buf_size = 128*1024; - char* buf = (char*)pgut_malloc(buf_size); - fio_header hdr; - struct stat st; - int rc; - int tmp_fd; - pg_crc32 crc; - -#ifdef WIN32 - SYS_CHECK(setmode(in, _O_BINARY)); - SYS_CHECK(setmode(out, _O_BINARY)); -#endif - - /* Main loop until end of processing all master commands */ - while ((rc = fio_read_all(in, &hdr, sizeof hdr)) == sizeof(hdr)) { - if (hdr.size != 0) { - if (hdr.size > buf_size) { - /* Extend buffer on demand */ - buf_size = hdr.size; - buf = (char*)realloc(buf, buf_size); - } - IO_CHECK(fio_read_all(in, buf, hdr.size), hdr.size); - } - errno = 0; /* reset errno */ - switch (hdr.cop) { - case FIO_LOAD: /* Send file content */ - fio_load_file(out, buf); - break; - case FIO_OPENDIR: /* Open directory for traversal */ - dir[hdr.handle] = opendir(buf); - hdr.arg = dir[hdr.handle] == NULL ? errno : 0; - hdr.size = 0; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - break; - case FIO_READDIR: /* Get next directory entry */ - hdr.cop = FIO_SEND; - entry = readdir(dir[hdr.handle]); - if (entry != NULL) - { - hdr.size = sizeof(*entry); - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, entry, hdr.size), hdr.size); - } - else - { - hdr.size = 0; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - } - break; - case FIO_CLOSEDIR: /* Finish directory traversal */ - SYS_CHECK(closedir(dir[hdr.handle])); - break; - case FIO_OPEN: /* Open file */ - fd[hdr.handle] = open(buf, hdr.arg, FILE_PERMISSIONS); - hdr.arg = fd[hdr.handle] < 0 ? errno : 0; - hdr.size = 0; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - break; - case FIO_CLOSE: /* Close file */ - fio_close_impl(fd[hdr.handle], out); - break; - case FIO_WRITE: /* Write to the current position in file */ -// IO_CHECK(fio_write_all(fd[hdr.handle], buf, hdr.size), hdr.size); - fio_write_impl(fd[hdr.handle], buf, hdr.size, out); - break; - case FIO_WRITE_ASYNC: /* Write to the current position in file */ - fio_write_async_impl(fd[hdr.handle], buf, hdr.size, out); - break; - case FIO_WRITE_COMPRESSED_ASYNC: /* Write to the current position in file */ - fio_write_compressed_impl(fd[hdr.handle], buf, hdr.size, hdr.arg); - break; - case FIO_READ: /* Read from the current position in file */ - if ((size_t)hdr.arg > buf_size) { - buf_size = hdr.arg; - buf = (char*)realloc(buf, buf_size); - } - rc = read(fd[hdr.handle], buf, hdr.arg); - hdr.cop = FIO_SEND; - hdr.size = rc > 0 ? rc : 0; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - if (hdr.size != 0) - IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); - break; - case FIO_PREAD: /* Read from specified position in file, ignoring pages beyond horizon of delta backup */ - rc = pread(fd[hdr.handle], buf, BLCKSZ, hdr.arg); - hdr.cop = FIO_SEND; - hdr.arg = rc; - hdr.size = rc >= 0 ? rc : 0; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - if (hdr.size != 0) - IO_CHECK(fio_write_all(out, buf, hdr.size), hdr.size); - break; - case FIO_AGENT_VERSION: - { - size_t payload_size = prepare_compatibility_str(buf, buf_size); - - hdr.arg = AGENT_PROTOCOL_VERSION; - hdr.size = payload_size; - - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, buf, payload_size), payload_size); - break; - } - case FIO_STAT: /* Get information about file with specified path */ - hdr.size = sizeof(st); - rc = hdr.arg ? stat(buf, &st) : lstat(buf, &st); - hdr.arg = rc < 0 ? errno : 0; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - IO_CHECK(fio_write_all(out, &st, sizeof(st)), sizeof(st)); - break; - case FIO_ACCESS: /* Check presence of file with specified name */ - hdr.size = 0; - hdr.arg = access(buf, hdr.arg) < 0 ? errno : 0; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - break; - case FIO_RENAME: /* Rename file */ - SYS_CHECK(rename(buf, buf + strlen(buf) + 1)); - break; - case FIO_SYMLINK: /* Create symbolic link */ - fio_symlink_impl(out, buf, hdr.arg > 0 ? true : false); - break; - case FIO_UNLINK: /* Remove file or directory (TODO: Win32) */ - SYS_CHECK(remove_file_or_dir(buf)); - break; - case FIO_MKDIR: /* Create directory */ - hdr.size = 0; - hdr.arg = dir_create_dir(buf, hdr.arg, false); - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - break; - case FIO_CHMOD: /* Change file mode */ - SYS_CHECK(chmod(buf, hdr.arg)); - break; - case FIO_SEEK: /* Set current position in file */ - fio_seek_impl(fd[hdr.handle], hdr.arg); - break; - case FIO_TRUNCATE: /* Truncate file */ - SYS_CHECK(ftruncate(fd[hdr.handle], hdr.arg)); - break; - case FIO_LIST_DIR: - fio_list_dir_impl(out, buf); - break; - case FIO_SEND_PAGES: - // buf contain fio_send_request header and bitmap. - fio_send_pages_impl(out, buf); - break; - case FIO_SEND_FILE: - fio_send_file_impl(out, buf); - break; - case FIO_SYNC: - /* open file and fsync it */ - tmp_fd = open(buf, O_WRONLY | PG_BINARY, FILE_PERMISSIONS); - if (tmp_fd < 0) - hdr.arg = errno; - else - { - if (fsync(tmp_fd) == 0) - hdr.arg = 0; - else - hdr.arg = errno; - } - close(tmp_fd); - - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - break; - case FIO_GET_CRC32: - Assert((hdr.arg & GET_CRC32_TRUNCATED) == 0 || - (hdr.arg & (GET_CRC32_TRUNCATED|GET_CRC32_DECOMPRESS)) == GET_CRC32_TRUNCATED); - /* calculate crc32 for a file */ - if ((hdr.arg & GET_CRC32_DECOMPRESS)) - crc = pgFileGetCRCgz(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0); - else if ((hdr.arg & GET_CRC32_TRUNCATED)) - crc = pgFileGetCRCTruncated(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0); - else - crc = pgFileGetCRC(buf, true, (hdr.arg & GET_CRC32_MISSING_OK) != 0); - IO_CHECK(fio_write_all(out, &crc, sizeof(crc)), sizeof(crc)); - break; - case FIO_GET_CHECKSUM_MAP: - /* calculate crc32 for a file */ - fio_get_checksum_map_impl(out, buf); - break; - case FIO_GET_LSN_MAP: - /* calculate crc32 for a file */ - fio_get_lsn_map_impl(out, buf); - break; - case FIO_CHECK_POSTMASTER: - /* calculate crc32 for a file */ - fio_check_postmaster_impl(out, buf); - break; - case FIO_DELETE: - /* delete file */ - fio_delete_impl(hdr.arg, buf); - break; - case FIO_DISCONNECT: - hdr.cop = FIO_DISCONNECTED; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - free(buf); - return; - case FIO_GET_ASYNC_ERROR: - fio_get_async_error_impl(out); - break; - case FIO_READLINK: /* Read content of a symbolic link */ - { - /* - * We need a buf for a arguments and for a result at the same time - * hdr.size = strlen(symlink_name) + 1 - * hdr.arg = bufsize for a answer (symlink content) - */ - size_t filename_size = (size_t)hdr.size; - if (filename_size + hdr.arg > buf_size) { - buf_size = hdr.arg; - buf = (char*)realloc(buf, buf_size); - } - rc = readlink(buf, buf + filename_size, hdr.arg); - hdr.cop = FIO_READLINK; - hdr.size = rc > 0 ? rc : 0; - IO_CHECK(fio_write_all(out, &hdr, sizeof(hdr)), sizeof(hdr)); - if (hdr.size != 0) - IO_CHECK(fio_write_all(out, buf + filename_size, hdr.size), hdr.size); - } - break; - default: - Assert(false); - } - } - free(buf); - if (rc != 0) { /* Not end of stream: normal pipe close */ - perror("read"); - exit(EXIT_FAILURE); - } -} diff --git a/src/utils/file.h b/src/utils/file.h deleted file mode 100644 index 01e5a24f4..000000000 --- a/src/utils/file.h +++ /dev/null @@ -1,154 +0,0 @@ -#ifndef __FILE__H__ -#define __FILE__H__ - -#include "storage/bufpage.h" -#include -#include -#include - -#ifdef HAVE_LIBZ -#include -#endif - -typedef enum -{ - /* message for compatibility check */ - FIO_AGENT_VERSION, /* never move this */ - FIO_OPEN, - FIO_CLOSE, - FIO_WRITE, - FIO_SYNC, - FIO_RENAME, - FIO_SYMLINK, - FIO_UNLINK, - FIO_MKDIR, - FIO_CHMOD, - FIO_SEEK, - FIO_TRUNCATE, - FIO_DELETE, - FIO_PREAD, - FIO_READ, - FIO_LOAD, - FIO_STAT, - FIO_SEND, - FIO_ACCESS, - FIO_OPENDIR, - FIO_READDIR, - FIO_CLOSEDIR, - FIO_PAGE, - FIO_WRITE_COMPRESSED_ASYNC, - FIO_GET_CRC32, - /* used for incremental restore */ - FIO_GET_CHECKSUM_MAP, - FIO_GET_LSN_MAP, - /* used in fio_send_pages */ - FIO_SEND_PAGES, - FIO_ERROR, - FIO_SEND_FILE, -// FIO_CHUNK, - FIO_SEND_FILE_EOF, - FIO_SEND_FILE_CORRUPTION, - FIO_SEND_FILE_HEADERS, - /* messages for closing connection */ - FIO_DISCONNECT, - FIO_DISCONNECTED, - FIO_LIST_DIR, - FIO_CHECK_POSTMASTER, - FIO_GET_ASYNC_ERROR, - FIO_WRITE_ASYNC, - FIO_READLINK, - FIO_PAGE_ZERO -} fio_operations; - -typedef enum -{ - FIO_LOCAL_HOST, /* data is locate at local host */ - FIO_DB_HOST, /* data is located at Postgres server host */ - FIO_BACKUP_HOST, /* data is located at backup host */ - FIO_REMOTE_HOST /* date is located at remote host */ -} fio_location; - -#define FIO_FDMAX 64 -#define FIO_PIPE_MARKER 0x40000000 - -#define SYS_CHECK(cmd) do if ((cmd) < 0) { fprintf(stderr, "%s:%d: (%s) %s\n", __FILE__, __LINE__, #cmd, strerror(errno)); exit(EXIT_FAILURE); } while (0) -#define IO_CHECK(cmd, size) do { int _rc = (cmd); if (_rc != (size)) fio_error(_rc, size, __FILE__, __LINE__); } while (0) - -typedef struct -{ -// fio_operations cop; -// 16 - unsigned cop : 32; - unsigned handle : 32; - unsigned size : 32; - unsigned arg; -} fio_header; - -extern fio_location MyLocation; - -/* Check if FILE handle is local or remote (created by FIO) */ -#define fio_is_remote_file(file) ((size_t)(file) <= FIO_FDMAX) - -extern void fio_redirect(int in, int out, int err); -extern void fio_communicate(int in, int out); - -extern void fio_get_agent_version(int* protocol, char* payload_buf, size_t payload_buf_size); -extern FILE* fio_fopen(char const* name, char const* mode, fio_location location); -extern size_t fio_fwrite(FILE* f, void const* buf, size_t size); -extern ssize_t fio_fwrite_async_compressed(FILE* f, void const* buf, size_t size, int compress_alg); -extern size_t fio_fwrite_async(FILE* f, void const* buf, size_t size); -extern int fio_check_error_file(FILE* f, char **errmsg); -extern int fio_check_error_fd(int fd, char **errmsg); -extern int fio_check_error_fd_gz(gzFile f, char **errmsg); -extern ssize_t fio_fread(FILE* f, void* buf, size_t size); -extern int fio_pread(FILE* f, void* buf, off_t offs); -extern int fio_fprintf(FILE* f, char const* arg, ...) pg_attribute_printf(2, 3); -extern int fio_fflush(FILE* f); -extern int fio_fseek(FILE* f, off_t offs); -extern int fio_ftruncate(FILE* f, off_t size); -extern int fio_fclose(FILE* f); -extern int fio_ffstat(FILE* f, struct stat* st); -extern void fio_error(int rc, int size, char const* file, int line); - -extern int fio_open(char const* name, int mode, fio_location location); -extern ssize_t fio_write(int fd, void const* buf, size_t size); -extern ssize_t fio_write_async(int fd, void const* buf, size_t size); -extern ssize_t fio_read(int fd, void* buf, size_t size); -extern int fio_flush(int fd); -extern int fio_seek(int fd, off_t offs); -extern int fio_fstat(int fd, struct stat* st); -extern int fio_truncate(int fd, off_t size); -extern int fio_close(int fd); -extern void fio_disconnect(void); -extern int fio_sync(char const* path, fio_location location); -extern pg_crc32 fio_get_crc32(const char *file_path, fio_location location, - bool decompress, bool missing_ok); -extern pg_crc32 fio_get_crc32_truncated(const char *file_path, fio_location location, - bool missing_ok); - -extern int fio_rename(char const* old_path, char const* new_path, fio_location location); -extern int fio_symlink(char const* target, char const* link_path, bool overwrite, fio_location location); -extern int fio_unlink(char const* path, fio_location location); -extern int fio_mkdir(char const* path, int mode, fio_location location); -extern int fio_chmod(char const* path, int mode, fio_location location); -extern int fio_access(char const* path, int mode, fio_location location); -extern int fio_stat(char const* path, struct stat* st, bool follow_symlinks, fio_location location); -extern bool fio_is_same_file(char const* filename1, char const* filename2, bool follow_symlink, fio_location location); -extern ssize_t fio_readlink(const char *path, char *value, size_t valsiz, fio_location location); -extern DIR* fio_opendir(char const* path, fio_location location); -extern struct dirent * fio_readdir(DIR *dirp); -extern int fio_closedir(DIR *dirp); -extern FILE* fio_open_stream(char const* name, fio_location location); -extern int fio_close_stream(FILE* f); - -#ifdef HAVE_LIBZ -extern gzFile fio_gzopen(char const* path, char const* mode, int level, fio_location location); -extern int fio_gzclose(gzFile file); -extern int fio_gzread(gzFile f, void *buf, unsigned size); -extern int fio_gzwrite(gzFile f, void const* buf, unsigned size); -extern int fio_gzeof(gzFile f); -extern z_off_t fio_gzseek(gzFile f, z_off_t offset, int whence); -extern const char* fio_gzerror(gzFile file, int *errnum); -#endif - -#endif diff --git a/src/utils/json.c b/src/utils/json.c deleted file mode 100644 index 2c8e0fe9b..000000000 --- a/src/utils/json.c +++ /dev/null @@ -1,164 +0,0 @@ -/*------------------------------------------------------------------------- - * - * json.c: - make json document. - * - * Copyright (c) 2018-2019, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "json.h" - -static void json_add_indent(PQExpBuffer buf, int32 level); -static void json_add_escaped(PQExpBuffer buf, const char *str); - -static bool add_comma = false; - -/* - * Start or end json token. Currently it is a json object or array. - * - * Function modifies level value and adds indent if it appropriate. - */ -void -json_add(PQExpBuffer buf, JsonToken type, int32 *level) -{ - switch (type) - { - case JT_BEGIN_ARRAY: - appendPQExpBufferChar(buf, '['); - *level += 1; - add_comma = false; - break; - case JT_END_ARRAY: - *level -= 1; - if (*level == 0) - appendPQExpBufferChar(buf, '\n'); - else - json_add_indent(buf, *level); - appendPQExpBufferChar(buf, ']'); - add_comma = true; - break; - case JT_BEGIN_OBJECT: - json_add_indent(buf, *level); - appendPQExpBufferChar(buf, '{'); - *level += 1; - add_comma = false; - break; - case JT_END_OBJECT: - *level -= 1; - if (*level == 0) - appendPQExpBufferChar(buf, '\n'); - else - json_add_indent(buf, *level); - appendPQExpBufferChar(buf, '}'); - add_comma = true; - break; - default: - break; - } -} - -/* - * Add json object's key. If it isn't first key we need to add a comma. - */ -void -json_add_key(PQExpBuffer buf, const char *name, int32 level) -{ - if (add_comma) - appendPQExpBufferChar(buf, ','); - json_add_indent(buf, level); - - json_add_escaped(buf, name); - appendPQExpBufferStr(buf, ": "); - - add_comma = true; -} - -/* - * Add json object's key and value. If it isn't first key we need to add a - * comma. - */ -void -json_add_value(PQExpBuffer buf, const char *name, const char *value, - int32 level, bool escaped) -{ - json_add_key(buf, name, level); - - if (escaped) - json_add_escaped(buf, value); - else - appendPQExpBufferStr(buf, value); -} - -static void -json_add_indent(PQExpBuffer buf, int32 level) -{ - uint16 i; - - if (level == 0) - return; - - appendPQExpBufferChar(buf, '\n'); - for (i = 0; i < level; i++) - appendPQExpBufferStr(buf, " "); -} - -static void -json_add_escaped(PQExpBuffer buf, const char *str) -{ - const char *p; - - appendPQExpBufferChar(buf, '"'); - for (p = str; *p; p++) - { - switch (*p) - { - case '\b': - appendPQExpBufferStr(buf, "\\b"); - break; - case '\f': - appendPQExpBufferStr(buf, "\\f"); - break; - case '\n': - appendPQExpBufferStr(buf, "\\n"); - break; - case '\r': - appendPQExpBufferStr(buf, "\\r"); - break; - case '\t': - appendPQExpBufferStr(buf, "\\t"); - break; - case '"': - appendPQExpBufferStr(buf, "\\\""); - break; - case '\\': - appendPQExpBufferStr(buf, "\\\\"); - break; - default: - if ((unsigned char) *p < ' ') - appendPQExpBuffer(buf, "\\u%04x", (int) *p); - else - appendPQExpBufferChar(buf, *p); - break; - } - } - appendPQExpBufferChar(buf, '"'); -} - -void -json_add_min(PQExpBuffer buf, JsonToken type) -{ - switch (type) - { - case JT_BEGIN_OBJECT: - appendPQExpBufferChar(buf, '{'); - add_comma = false; - break; - case JT_END_OBJECT: - appendPQExpBufferStr(buf, "}\n"); - add_comma = true; - break; - default: - break; - } -} diff --git a/src/utils/json.h b/src/utils/json.h deleted file mode 100644 index f80832e69..000000000 --- a/src/utils/json.h +++ /dev/null @@ -1,33 +0,0 @@ -/*------------------------------------------------------------------------- - * - * json.h: - prototypes of json output functions. - * - * Copyright (c) 2018-2019, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#ifndef PROBACKUP_JSON_H -#define PROBACKUP_JSON_H - -#include "postgres_fe.h" -#include "pqexpbuffer.h" - -/* - * Json document tokens. - */ -typedef enum -{ - JT_BEGIN_ARRAY, - JT_END_ARRAY, - JT_BEGIN_OBJECT, - JT_END_OBJECT -} JsonToken; - -extern void json_add(PQExpBuffer buf, JsonToken type, int32 *level); -extern void json_add_min(PQExpBuffer buf, JsonToken type); -extern void json_add_key(PQExpBuffer buf, const char *name, int32 level); -extern void json_add_value(PQExpBuffer buf, const char *name, const char *value, - int32 level, bool escaped); - -#endif /* PROBACKUP_JSON_H */ diff --git a/src/utils/logger.c b/src/utils/logger.c deleted file mode 100644 index 7ea41f74e..000000000 --- a/src/utils/logger.c +++ /dev/null @@ -1,980 +0,0 @@ -/*------------------------------------------------------------------------- - * - * logger.c: - log events into log file or stderr. - * - * Copyright (c) 2017-2019, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "postgres_fe.h" - -#include - -#include "pg_probackup.h" -#include "logger.h" -#include "pgut.h" -#include "thread.h" -#include - -#include "utils/configuration.h" - -#include "json.h" - -/* Logger parameters */ -LoggerConfig logger_config = { - LOG_LEVEL_CONSOLE_DEFAULT, - LOG_LEVEL_FILE_DEFAULT, - LOG_FILENAME_DEFAULT, - NULL, - NULL, - LOG_ROTATION_SIZE_DEFAULT, - LOG_ROTATION_AGE_DEFAULT, - LOG_FORMAT_CONSOLE_DEFAULT, - LOG_FORMAT_FILE_DEFAULT -}; - -/* Implementation for logging.h */ - -typedef enum -{ - PG_DEBUG, - PG_PROGRESS, - PG_WARNING, - PG_FATAL -} eLogType; - -#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING -#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004 -#endif - -void pg_log(eLogType type, const char *fmt,...) pg_attribute_printf(2, 3); - -static void elog_internal(int elevel, bool file_only, const char *message); -static void elog_stderr(int elevel, const char *fmt, ...) - pg_attribute_printf(2, 3); -static char *get_log_message(const char *fmt, va_list args) pg_attribute_printf(1, 0); - -/* Functions to work with log files */ -static void open_logfile(FILE **file, const char *filename_format); -static void release_logfile(bool fatal, void *userdata); -static char *logfile_getname(const char *format, time_t timestamp); -static FILE *logfile_open(const char *filename, const char *mode); - -/* Static variables */ - -static FILE *log_file = NULL; -static FILE *error_log_file = NULL; - -static bool exit_hook_registered = false; -/* Logging of the current thread is in progress */ -static bool loggin_in_progress = false; - -static pthread_mutex_t log_file_mutex = PTHREAD_MUTEX_INITIALIZER; - -/* - * Initialize logger. - * - * If log_directory wasn't set by a user we use full path: - * backup_directory/log - */ -void -init_logger(const char *root_path, LoggerConfig *config) -{ - /* - * If logging to file is enabled and log_directory wasn't set - * by user, init the path with default value: backup_directory/log/ - * */ - if (config->log_level_file != LOG_OFF - && config->log_directory == NULL) - { - config->log_directory = pgut_malloc(MAXPGPATH); - join_path_components(config->log_directory, - root_path, LOG_DIRECTORY_DEFAULT); - } - - if (config->log_directory != NULL) - canonicalize_path(config->log_directory); - - logger_config = *config; - -#if PG_VERSION_NUM >= 120000 - /* Setup logging for functions from other modules called by pg_probackup */ - pg_logging_init(PROGRAM_NAME); - errno = 0; /* sometimes pg_logging_init sets errno */ - - switch (logger_config.log_level_console) - { - case VERBOSE: - pg_logging_set_level(PG_LOG_DEBUG); - break; - case INFO: - case NOTICE: - case LOG: - pg_logging_set_level(PG_LOG_INFO); - break; - case WARNING: - pg_logging_set_level(PG_LOG_WARNING); - break; - case ERROR: - pg_logging_set_level(PG_LOG_ERROR); - break; - default: - break; - }; -#endif -} - -/* - * Check that we are connected to terminal and - * enable ANSI escape codes for Windows if possible - */ -void -init_console(void) -{ - - /* no point in tex coloring if we do not connected to terminal */ - if (!isatty(fileno(stderr)) || - !isatty(fileno(stdout))) - { - show_color = false; - return; - } - -#ifdef WIN32 - HANDLE hOut = INVALID_HANDLE_VALUE; - HANDLE hErr = INVALID_HANDLE_VALUE; - DWORD dwMode_out = 0; - DWORD dwMode_err = 0; - - hOut = GetStdHandle(STD_OUTPUT_HANDLE); - if (hOut == INVALID_HANDLE_VALUE || !hOut) - { - show_color = false; - _dosmaperr(GetLastError()); - elog(WARNING, "Failed to get terminal stdout handle: %s", strerror(errno)); - return; - } - - hErr = GetStdHandle(STD_ERROR_HANDLE); - if (hErr == INVALID_HANDLE_VALUE || !hErr) - { - show_color = false; - _dosmaperr(GetLastError()); - elog(WARNING, "Failed to get terminal stderror handle: %s", strerror(errno)); - return; - } - - if (!GetConsoleMode(hOut, &dwMode_out)) - { - show_color = false; - _dosmaperr(GetLastError()); - elog(WARNING, "Failed to get console mode for stdout: %s", strerror(errno)); - return; - } - - if (!GetConsoleMode(hErr, &dwMode_err)) - { - show_color = false; - _dosmaperr(GetLastError()); - elog(WARNING, "Failed to get console mode for stderr: %s", strerror(errno)); - return; - } - - /* Add ANSI codes support */ - dwMode_out |= ENABLE_VIRTUAL_TERMINAL_PROCESSING; - dwMode_err |= ENABLE_VIRTUAL_TERMINAL_PROCESSING; - - if (!SetConsoleMode(hOut, dwMode_out)) - { - show_color = false; - _dosmaperr(GetLastError()); - elog(WARNING, "Cannot set console mode for stdout: %s", strerror(errno)); - return; - } - - if (!SetConsoleMode(hErr, dwMode_err)) - { - show_color = false; - _dosmaperr(GetLastError()); - elog(WARNING, "Cannot set console mode for stderr: %s", strerror(errno)); - return; - } -#endif -} - -static void -write_elevel(FILE *stream, int elevel) -{ - switch (elevel) - { - case VERBOSE: - fputs("VERBOSE: ", stream); - break; - case LOG: - fputs("LOG: ", stream); - break; - case INFO: - fputs("INFO: ", stream); - break; - case NOTICE: - fputs("NOTICE: ", stream); - break; - case WARNING: - fputs("WARNING: ", stream); - break; - case ERROR: - fputs("ERROR: ", stream); - break; - default: - elog_stderr(ERROR, "invalid logging level: %d", elevel); - break; - } -} - -static void -write_elevel_for_json(PQExpBuffer buf, int elevel) -{ - switch (elevel) - { - case VERBOSE: - appendPQExpBufferStr(buf, "\"VERBOSE\""); - break; - case LOG: - appendPQExpBufferStr(buf, "\"LOG\""); - break; - case INFO: - appendPQExpBufferStr(buf, "\"INFO\""); - break; - case NOTICE: - appendPQExpBufferStr(buf, "\"NOTICE\""); - break; - case WARNING: - appendPQExpBufferStr(buf, "\"WARNING\""); - break; - case ERROR: - appendPQExpBufferStr(buf, "\"ERROR\""); - break; - default: - elog_stderr(ERROR, "invalid logging level: %d", elevel); - break; - } -} - -/* - * Exit with code if it is an error. - * Check for in_cleanup flag to avoid deadlock in case of ERROR in cleanup - * routines. - */ -static void -exit_if_necessary(int elevel) -{ - if (elevel > WARNING && !in_cleanup) - { - if (loggin_in_progress) - { - loggin_in_progress = false; - pthread_mutex_unlock(&log_file_mutex); - } - - if (remote_agent) - sleep(1); /* Let parent receive sent messages */ - - /* If this is not the main thread then don't call exit() */ - if (main_tid != pthread_self()) - { - /* Interrupt other possible routines */ - thread_interrupted = true; -#ifdef WIN32 - ExitThread(elevel); -#else - pthread_exit(NULL); -#endif - } - else - exit(elevel); - } -} - -/* - * Logs to stderr or to log file and exit if ERROR. - * - * Actual implementation for elog() and pg_log(). - */ -static void -elog_internal(int elevel, bool file_only, const char *message) -{ - bool write_to_file, - write_to_error_log, - write_to_stderr; - time_t log_time = (time_t) time(NULL); - char strfbuf[128]; - char str_pid[128]; - char str_pid_json[128]; - char str_thread_json[64]; - PQExpBufferData show_buf; - PQExpBuffer buf_json = &show_buf; - int8 format_console, - format_file; - - write_to_file = elevel >= logger_config.log_level_file - && logger_config.log_directory - && logger_config.log_directory[0] != '\0'; - write_to_error_log = elevel >= ERROR && logger_config.error_log_filename && - logger_config.log_directory && logger_config.log_directory[0] != '\0'; - write_to_stderr = elevel >= logger_config.log_level_console && !file_only; - format_console = logger_config.log_format_console; - format_file = logger_config.log_format_file; - - if (remote_agent) - { - write_to_stderr |= write_to_error_log | write_to_file; - write_to_error_log = write_to_file = false; - } - pthread_lock(&log_file_mutex); - loggin_in_progress = true; - - if (write_to_file || write_to_error_log || is_archive_cmd || - format_console == JSON) - strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", - localtime(&log_time)); - - if (format_file == JSON || format_console == JSON) - { - snprintf(str_pid_json, sizeof(str_pid_json), "%d", my_pid); - snprintf(str_thread_json, sizeof(str_thread_json), "[%d-1]", my_thread_num); - - initPQExpBuffer(&show_buf); - json_add_min(buf_json, JT_BEGIN_OBJECT); - json_add_value(buf_json, "ts", strfbuf, 0, true); - json_add_value(buf_json, "pid", str_pid_json, 0, true); - json_add_key(buf_json, "level", 0); - write_elevel_for_json(buf_json, elevel); - json_add_value(buf_json, "msg", message, 0, true); - json_add_value(buf_json, "my_thread_num", str_thread_json, 0, true); - json_add_min(buf_json, JT_END_OBJECT); - } - - snprintf(str_pid, sizeof(str_pid), "[%d]:", my_pid); - - /* - * Write message to log file. - * Do not write to file if this error was raised during write previous - * message. - */ - if (write_to_file) - { - if (log_file == NULL) - open_logfile(&log_file, logger_config.log_filename ? logger_config.log_filename : LOG_FILENAME_DEFAULT); - if (format_file == JSON) - { - fputs(buf_json->data, log_file); - } - else - { - fprintf(log_file, "%s ", strfbuf); - fprintf(log_file, "%s ", str_pid); - write_elevel(log_file, elevel); - - fprintf(log_file, "%s\n", message); - } - fflush(log_file); - } - - /* - * Write error message to error log file. - * Do not write to file if this error was raised during write previous - * message. - */ - if (write_to_error_log) - { - if (error_log_file == NULL) - open_logfile(&error_log_file, logger_config.error_log_filename); - - if (format_file == JSON) - { - fputs(buf_json->data, error_log_file); - } - else - { - fprintf(error_log_file, "%s ", strfbuf); - fprintf(error_log_file, "%s ", str_pid); - write_elevel(error_log_file, elevel); - - fprintf(error_log_file, "%s\n", message); - } - fflush(error_log_file); - } - - /* - * Write to stderr if the message was not written to log file. - * Write to stderr if the message level is greater than WARNING anyway. - */ - if (write_to_stderr) - { - if (format_console == JSON) - { - fprintf(stderr, "%s", buf_json->data); - } - else - { - if (is_archive_cmd) - { - char str_thread[64]; - /* [Issue #213] fix pgbadger parsing */ - snprintf(str_thread, sizeof(str_thread), "[%d-1]:", my_thread_num); - - fprintf(stderr, "%s ", strfbuf); - fprintf(stderr, "%s ", str_pid); - fprintf(stderr, "%s ", str_thread); - } - else if (show_color) - { - /* color WARNING and ERROR messages */ - if (elevel == WARNING) - fprintf(stderr, "%s", TC_YELLOW_BOLD); - else if (elevel == ERROR) - fprintf(stderr, "%s", TC_RED_BOLD); - } - - write_elevel(stderr, elevel); - - /* main payload */ - fprintf(stderr, "%s", message); - - /* reset color to default */ - if (show_color && (elevel == WARNING || elevel == ERROR)) - fprintf(stderr, "%s", TC_RESET); - - fprintf(stderr, "\n"); - } - - if (format_file == JSON || format_console == JSON) - { - termPQExpBuffer(buf_json); - } - - fflush(stderr); - } - - exit_if_necessary(elevel); - - loggin_in_progress = false; - pthread_mutex_unlock(&log_file_mutex); -} - -/* - * Log only to stderr. It is called only within elog_internal() when another - * logging already was started. - */ -static void -elog_stderr(int elevel, const char *fmt, ...) -{ - va_list args; - PQExpBufferData show_buf; - PQExpBuffer buf_json = &show_buf; - time_t log_time = (time_t) time(NULL); - char strfbuf[128]; - char str_pid[128]; - char str_thread_json[64]; - char *message; - int8 format_console; - - /* - * Do not log message if severity level is less than log_level. - * It is the little optimisation to put it here not in elog_internal(). - */ - if (elevel < logger_config.log_level_console && elevel < ERROR) - return; - - va_start(args, fmt); - - format_console = logger_config.log_format_console; - - if (format_console == JSON) - { - strftime(strfbuf, sizeof(strfbuf), "%Y-%m-%d %H:%M:%S %Z", - localtime(&log_time)); - snprintf(str_pid, sizeof(str_pid), "%d", my_pid); - snprintf(str_thread_json, sizeof(str_thread_json), "[%d-1]", my_thread_num); - - initPQExpBuffer(&show_buf); - json_add_min(buf_json, JT_BEGIN_OBJECT); - json_add_value(buf_json, "ts", strfbuf, 0, true); - json_add_value(buf_json, "pid", str_pid, 0, true); - json_add_key(buf_json, "level", 0); - write_elevel_for_json(buf_json, elevel); - message = get_log_message(fmt, args); - json_add_value(buf_json, "msg", message, 0, true); - json_add_value(buf_json, "my_thread_num", str_thread_json, 0, true); - json_add_min(buf_json, JT_END_OBJECT); - fputs(buf_json->data, stderr); - pfree(message); - termPQExpBuffer(buf_json); - } - else - { - write_elevel(stderr, elevel); - vfprintf(stderr, fmt, args); - fputc('\n', stderr); - } - - fflush(stderr); - va_end(args); - - exit_if_necessary(elevel); -} - -/* - * Formats text data under the control of fmt and returns it in an allocated - * buffer. - */ -static char * -get_log_message(const char *fmt, va_list args) -{ - size_t len = 256; /* initial assumption about buffer size */ - - for (;;) - { - char *result; - size_t newlen; - va_list copy_args; - - result = (char *) pgut_malloc(len); - - /* Try to format the data */ - va_copy(copy_args, args); - newlen = pvsnprintf(result, len, fmt, copy_args); - va_end(copy_args); - - if (newlen < len) - return result; /* success */ - - /* Release buffer and loop around to try again with larger len. */ - pfree(result); - len = newlen; - } -} - -/* - * Logs to stderr or to log file and exit if ERROR. - */ -void -elog(int elevel, const char *fmt, ...) -{ - char *message; - va_list args; - - /* - * Do not log message if severity level is less than log_level. - * It is the little optimisation to put it here not in elog_internal(). - */ - if (elevel < logger_config.log_level_console && - elevel < logger_config.log_level_file && elevel < ERROR) - return; - - va_start(args, fmt); - message = get_log_message(fmt, args); - va_end(args); - - elog_internal(elevel, false, message); - pfree(message); -} - -/* - * Logs only to log file and exit if ERROR. - */ -void -elog_file(int elevel, const char *fmt, ...) -{ - char *message; - va_list args; - - /* - * Do not log message if severity level is less than log_level. - * It is the little optimisation to put it here not in elog_internal(). - */ - if (elevel < logger_config.log_level_file && elevel < ERROR) - return; - - va_start(args, fmt); - message = get_log_message(fmt, args); - va_end(args); - - elog_internal(elevel, true, message); - pfree(message); -} - -/* - * Implementation of pg_log() from logging.h. - */ -void -pg_log(eLogType type, const char *fmt, ...) -{ - char *message; - va_list args; - int elevel = INFO; - - /* Transform logging level from eLogType to utils/logger.h levels */ - switch (type) - { - case PG_DEBUG: - elevel = LOG; - break; - case PG_PROGRESS: - elevel = INFO; - break; - case PG_WARNING: - elevel = WARNING; - break; - case PG_FATAL: - elevel = ERROR; - break; - default: - elog(ERROR, "invalid logging level: %d", type); - break; - } - - /* - * Do not log message if severity level is less than log_level. - * It is the little optimisation to put it here not in elog_internal(). - */ - if (elevel < logger_config.log_level_console && - elevel < logger_config.log_level_file && elevel < ERROR) - return; - - va_start(args, fmt); - message = get_log_message(fmt, args); - va_end(args); - - elog_internal(elevel, false, message); - pfree(message); -} - -/* - * Parses string representation of log level. - */ -int -parse_log_level(const char *level) -{ - const char *v = level; - size_t len; - - /* Skip all spaces detected */ - while (isspace((unsigned char)*v)) - v++; - len = strlen(v); - - if (len == 0) - elog(ERROR, "log-level is empty"); - - if (pg_strncasecmp("off", v, len) == 0) - return LOG_OFF; - else if (pg_strncasecmp("verbose", v, len) == 0) - return VERBOSE; - else if (pg_strncasecmp("log", v, len) == 0) - return LOG; - else if (pg_strncasecmp("info", v, len) == 0) - return INFO; - else if (pg_strncasecmp("notice", v, len) == 0) - return NOTICE; - else if (pg_strncasecmp("warning", v, len) == 0) - return WARNING; - else if (pg_strncasecmp("error", v, len) == 0) - return ERROR; - - /* Log level is invalid */ - elog(ERROR, "invalid log-level \"%s\"", level); - return 0; -} - -int -parse_log_format(const char *format) -{ - const char *v = format; - size_t len; - - if (v == NULL) - { - elog(ERROR, "log-format got invalid value"); - return 0; - } - - /* Skip all spaces detected */ - while (isspace((unsigned char)*v)) - v++; - len = strlen(v); - - if (len == 0) - elog(ERROR, "log-format is empty"); - - if (pg_strncasecmp("plain", v, len) == 0) - return PLAIN; - else if (pg_strncasecmp("json", v, len) == 0) - return JSON; - - /* Log format is invalid */ - elog(ERROR, "invalid log-format \"%s\"", format); - return 0; -} - -/* - * Converts integer representation of log level to string. - */ -const char * -deparse_log_level(int level) -{ - switch (level) - { - case LOG_OFF: - return "OFF"; - case VERBOSE: - return "VERBOSE"; - case LOG: - return "LOG"; - case INFO: - return "INFO"; - case NOTICE: - return "NOTICE"; - case WARNING: - return "WARNING"; - case ERROR: - return "ERROR"; - default: - elog(ERROR, "invalid log-level %d", level); - } - - return NULL; -} - -const char * -deparse_log_format(int format) -{ - switch (format) - { - case PLAIN: - return "PLAIN"; - case JSON: - return "JSON"; - default: - elog(ERROR, "invalid log-format %d", format); - } - - return NULL; -} - -/* - * Construct logfile name using timestamp information. - * - * Result is palloc'd. - */ -static char * -logfile_getname(const char *format, time_t timestamp) -{ - char *filename; - size_t len; - struct tm *tm = localtime(×tamp); - - if (logger_config.log_directory == NULL || - logger_config.log_directory[0] == '\0') - elog_stderr(ERROR, "logging path is not set"); - - filename = (char *) pgut_malloc(MAXPGPATH); - - snprintf(filename, MAXPGPATH, "%s/", logger_config.log_directory); - - len = strlen(filename); - - /* Treat log_filename as a strftime pattern */ -#ifdef WIN32 - if (pg_strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) -#else - if (strftime(filename + len, MAXPGPATH - len, format, tm) <= 0) -#endif - elog_stderr(ERROR, "strftime(%s) failed: %s", format, strerror(errno)); - - return filename; -} - -/* - * Open a new log file. - */ -static FILE * -logfile_open(const char *filename, const char *mode) -{ - FILE *fh; - - /* - * Create log directory if not present; ignore errors - */ - mkdir(logger_config.log_directory, S_IRWXU); - - fh = fopen(filename, mode); - - if (fh) - setvbuf(fh, NULL, PG_IOLBF, 0); - else - { - int save_errno = errno; - - elog_stderr(ERROR, "could not open log file \"%s\": %s", - filename, strerror(errno)); - errno = save_errno; - } - - return fh; -} - -/* - * Open the log file. - */ -static void -open_logfile(FILE **file, const char *filename_format) -{ - char *filename; - char control[MAXPGPATH]; - struct stat st; - FILE *control_file; - time_t cur_time = time(NULL); - bool rotation_requested = false, - logfile_exists = false, - rotation_file_exists = false; - - filename = logfile_getname(filename_format, cur_time); - - /* "log_directory" was checked in logfile_getname() */ - snprintf(control, MAXPGPATH, "%s.rotation", filename); - - if (stat(filename, &st) == -1) - { - if (errno == ENOENT) - { - /* There is no file "filename" and rotation does not need */ - goto logfile_open; - } - else - elog_stderr(ERROR, "cannot stat log file \"%s\": %s", - filename, strerror(errno)); - } - /* Found log file "filename" */ - logfile_exists = true; - - /* First check for rotation */ - if (logger_config.log_rotation_size > 0 || - logger_config.log_rotation_age > 0) - { - /* Check for rotation by age */ - if (logger_config.log_rotation_age > 0) - { - struct stat control_st; - - if (stat(control, &control_st) < 0) - { - if (errno == ENOENT) - /* '.rotation' file is not found, force its recreation */ - elog_stderr(WARNING, "missing rotation file: \"%s\"", - control); - else - elog_stderr(ERROR, "cannot stat rotation file \"%s\": %s", - control, strerror(errno)); - } - else - { - /* rotation file exists */ - char buf[1024]; - - control_file = fopen(control, "r"); - if (control_file == NULL) - elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", - control, strerror(errno)); - - rotation_file_exists = true; - - if (fgets(buf, lengthof(buf), control_file)) - { - time_t creation_time; - - if (!parse_int64(buf, (int64 *) &creation_time, 0)) - { - /* Inability to parse value from .rotation file is - * concerning but not a critical error - */ - elog_stderr(WARNING, "rotation file \"%s\" has wrong " - "creation timestamp \"%s\"", - control, buf); - rotation_file_exists = false; - } - else - /* Parsed creation time */ - rotation_requested = (cur_time - creation_time) > - /* convert to seconds from milliseconds */ - logger_config.log_rotation_age / 1000; - } - else - { - /* truncated .rotation file is not a critical error */ - elog_stderr(WARNING, "cannot read creation timestamp from " - "rotation file \"%s\"", control); - rotation_file_exists = false; - } - - fclose(control_file); - } - } - - /* Check for rotation by size */ - if (!rotation_requested && logger_config.log_rotation_size > 0) - rotation_requested = st.st_size >= - /* convert to bytes */ - logger_config.log_rotation_size * 1024L; - } - -logfile_open: - if (rotation_requested) - *file = logfile_open(filename, "w"); - else - *file = logfile_open(filename, "a"); - pfree(filename); - - /* Rewrite rotation control file */ - if (rotation_requested || !logfile_exists || !rotation_file_exists) - { - time_t timestamp = time(NULL); - - control_file = fopen(control, "w"); - if (control_file == NULL) - elog_stderr(ERROR, "cannot open rotation file \"%s\": %s", - control, strerror(errno)); - - fprintf(control_file, "%ld", timestamp); - - fclose(control_file); - } - - /* - * Arrange to close opened file at proc_exit. - */ - if (!exit_hook_registered) - { - pgut_atexit_push(release_logfile, NULL); - exit_hook_registered = true; - } -} - -/* - * Closes opened file. - */ -static void -release_logfile(bool fatal, void *userdata) -{ - if (log_file) - { - fclose(log_file); - log_file = NULL; - } - if (error_log_file) - { - fclose(error_log_file); - error_log_file = NULL; - } -} diff --git a/src/utils/logger.h b/src/utils/logger.h deleted file mode 100644 index adc5061e0..000000000 --- a/src/utils/logger.h +++ /dev/null @@ -1,69 +0,0 @@ -/*------------------------------------------------------------------------- - * - * logger.h: - prototypes of logger functions. - * - * Copyright (c) 2017-2019, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#ifndef LOGGER_H -#define LOGGER_H - -#define LOG_NONE (-10) - -/* Log level */ -#define VERBOSE (-5) -#define LOG (-4) -#define INFO (-3) -#define NOTICE (-2) -#define WARNING (-1) -#define ERROR 1 -#define LOG_OFF 10 - -#define PLAIN 0 -#define JSON 1 - -typedef struct LoggerConfig -{ - int log_level_console; - int log_level_file; - char *log_filename; - char *error_log_filename; - char *log_directory; - /* Maximum size of an individual log file in kilobytes */ - uint64 log_rotation_size; - /* Maximum lifetime of an individual log file in minutes */ - uint64 log_rotation_age; - int8 log_format_console; - int8 log_format_file; -} LoggerConfig; - -/* Logger parameters */ -extern LoggerConfig logger_config; - -#define LOG_ROTATION_SIZE_DEFAULT 0 -#define LOG_ROTATION_AGE_DEFAULT 0 - -#define LOG_LEVEL_CONSOLE_DEFAULT INFO -#define LOG_LEVEL_FILE_DEFAULT LOG_OFF - -#define LOG_FORMAT_CONSOLE_DEFAULT PLAIN -#define LOG_FORMAT_FILE_DEFAULT PLAIN - -#define LOG_FILENAME_DEFAULT "pg_probackup.log" -#define LOG_DIRECTORY_DEFAULT "log" - -#undef elog -extern void elog(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); -extern void elog_file(int elevel, const char *fmt, ...) pg_attribute_printf(2, 3); - -extern void init_logger(const char *root_path, LoggerConfig *config); -extern void init_console(void); - -extern int parse_log_level(const char *level); -extern const char *deparse_log_level(int level); - -extern int parse_log_format(const char *format); -extern const char *deparse_log_format(int format); -#endif /* LOGGER_H */ diff --git a/src/utils/parray.c b/src/utils/parray.c deleted file mode 100644 index 65377c001..000000000 --- a/src/utils/parray.c +++ /dev/null @@ -1,246 +0,0 @@ -/*------------------------------------------------------------------------- - * - * parray.c: pointer array collection. - * - * Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * - *------------------------------------------------------------------------- - */ - -#include "postgres_fe.h" - -#include "parray.h" -#include "pgut.h" - -/* members of struct parray are hidden from client. */ -struct parray -{ - void **data; /* pointer array, expanded if necessary */ - size_t alloced; /* number of elements allocated */ - size_t used; /* number of elements in use */ -}; - -/* - * Create new parray object. - * Never returns NULL. - */ -parray * -parray_new(void) -{ - parray *a = pgut_new(parray); - - a->data = NULL; - a->used = 0; - a->alloced = 0; - - parray_expand(a, 1024); - - return a; -} - -/* - * Expand array pointed by data to newsize. - * Elements in expanded area are initialized to NULL. - * Note: never returns NULL. - */ -void -parray_expand(parray *array, size_t newsize) -{ - void **p; - - /* already allocated */ - if (newsize <= array->alloced) - return; - - p = pgut_realloc(array->data, sizeof(void *) * newsize); - - /* initialize expanded area to NULL */ - memset(p + array->alloced, 0, (newsize - array->alloced) * sizeof(void *)); - - array->alloced = newsize; - array->data = p; -} - -void -parray_free(parray *array) -{ - if (array == NULL) - return; - free(array->data); - free(array); -} - -void -parray_append(parray *array, void *elem) -{ - if (array->used + 1 > array->alloced) - parray_expand(array, array->alloced * 2); - - array->data[array->used++] = elem; -} - -void -parray_insert(parray *array, size_t index, void *elem) -{ - if (array->used + 1 > array->alloced) - parray_expand(array, array->alloced * 2); - - memmove(array->data + index + 1, array->data + index, - (array->alloced - index - 1) * sizeof(void *)); - array->data[index] = elem; - - /* adjust used count */ - if (array->used < index + 1) - array->used = index + 1; - else - array->used++; -} - -/* - * Concatenate two parray. - * parray_concat() appends the copy of the content of src to the end of dest. - */ -parray * -parray_concat(parray *dest, const parray *src) -{ - /* expand head array */ - parray_expand(dest, dest->used + src->used); - - /* copy content of src after content of dest */ - memcpy(dest->data + dest->used, src->data, src->used * sizeof(void *)); - dest->used += parray_num(src); - - return dest; -} - -void -parray_set(parray *array, size_t index, void *elem) -{ - if (index > array->alloced - 1) - parray_expand(array, index + 1); - - array->data[index] = elem; - - /* adjust used count */ - if (array->used < index + 1) - array->used = index + 1; -} - -void * -parray_get(const parray *array, size_t index) -{ - if (index > array->alloced - 1) - return NULL; - return array->data[index]; -} - -void * -parray_remove(parray *array, size_t index) -{ - void *val; - - /* removing unused element */ - if (index > array->used) - return NULL; - - val = array->data[index]; - - /* Do not move if the last element was removed. */ - if (index < array->alloced - 1) - memmove(array->data + index, array->data + index + 1, - (array->alloced - index - 1) * sizeof(void *)); - - /* adjust used count */ - array->used--; - - return val; -} - -bool -parray_rm(parray *array, const void *key, int(*compare)(const void *, const void *)) -{ - int i; - - for (i = 0; i < array->used; i++) - { - if (compare(&key, &array->data[i]) == 0) - { - parray_remove(array, i); - return true; - } - } - return false; -} - -size_t -parray_num(const parray *array) -{ - return array!= NULL ? array->used : (size_t) 0; -} - -void -parray_qsort(parray *array, int(*compare)(const void *, const void *)) -{ - qsort(array->data, array->used, sizeof(void *), compare); -} - -void -parray_walk(parray *array, void (*action)(void *)) -{ - int i; - for (i = 0; i < array->used; i++) - action(array->data[i]); -} - -void * -parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const void *)) -{ - return bsearch(&key, array->data, array->used, sizeof(void *), compare); -} - -int -parray_bsearch_index(parray *array, const void *key, int(*compare)(const void *, const void *)) -{ - void **elem = parray_bsearch(array, key, compare); - return elem != NULL ? elem - array->data : -1; -} - -/* checks that parray contains element */ -bool parray_contains(parray *array, void *elem) -{ - int i; - - for (i = 0; i < parray_num(array); i++) - { - if (parray_get(array, i) == elem) - return true; - } - return false; -} - -/* effectively remove elements that satisfy certain criterion */ -void -parray_remove_if(parray *array, criterion_fn criterion, void *args, cleanup_fn clean) { - int i = 0; - int j = 0; - - /* removing certain elements */ - while(j < parray_num(array)) { - void *value = array->data[j]; - // if the value satisfies the criterion, clean it up - if(criterion(value, args)) { - clean(value); - j++; - continue; - } - - if(i != j) - array->data[i] = array->data[j]; - - i++; - j++; - } - - /* adjust the number of used elements */ - array->used -= j - i; -} diff --git a/src/utils/parray.h b/src/utils/parray.h deleted file mode 100644 index 08846f252..000000000 --- a/src/utils/parray.h +++ /dev/null @@ -1,41 +0,0 @@ -/*------------------------------------------------------------------------- - * - * parray.h: pointer array collection. - * - * Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * - *------------------------------------------------------------------------- - */ - -#ifndef PARRAY_H -#define PARRAY_H - -/* - * "parray" hold pointers to objects in a linear memory area. - * Client use "parray *" to access parray object. - */ -typedef struct parray parray; - -typedef bool (*criterion_fn)(void *value, void *args); -typedef void (*cleanup_fn)(void *ref); - -extern parray *parray_new(void); -extern void parray_expand(parray *array, size_t newnum); -extern void parray_free(parray *array); -extern void parray_append(parray *array, void *val); -extern void parray_insert(parray *array, size_t index, void *val); -extern parray *parray_concat(parray *head, const parray *tail); -extern void parray_set(parray *array, size_t index, void *val); -extern void *parray_get(const parray *array, size_t index); -extern void *parray_remove(parray *array, size_t index); -extern bool parray_rm(parray *array, const void *key, int(*compare)(const void *, const void *)); -extern size_t parray_num(const parray *array); -extern void parray_qsort(parray *array, int(*compare)(const void *, const void *)); -extern void *parray_bsearch(parray *array, const void *key, int(*compare)(const void *, const void *)); -extern int parray_bsearch_index(parray *array, const void *key, int(*compare)(const void *, const void *)); -extern void parray_walk(parray *array, void (*action)(void *)); -extern bool parray_contains(parray *array, void *elem); -extern void parray_remove_if(parray *array, criterion_fn criterion, void *args, cleanup_fn clean); - -#endif /* PARRAY_H */ - diff --git a/src/utils/pgut.c b/src/utils/pgut.c deleted file mode 100644 index 9559fa644..000000000 --- a/src/utils/pgut.c +++ /dev/null @@ -1,1375 +0,0 @@ -/*------------------------------------------------------------------------- - * - * pgut.c - * - * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2017-2021, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" -#include "postgres_fe.h" - -#include "getopt_long.h" -#include "libpq-fe.h" -#include "libpq/pqsignal.h" -#include "pqexpbuffer.h" - -#if PG_VERSION_NUM >= 140000 -#include "common/string.h" -#endif - -#if PG_VERSION_NUM >= 100000 -#include "common/connect.h" -#else -#include "fe_utils/connect.h" -#endif - -#include - -#include "pgut.h" -#include "logger.h" -#include "file.h" - - -static char *password = NULL; -bool prompt_password = true; -bool force_password = false; - -/* Database connections */ -static PGcancel *volatile cancel_conn = NULL; - -/* Interrupted by SIGINT (Ctrl+C) ? */ -bool interrupted = false; -bool in_cleanup = false; -bool in_password = false; - -/* critical section when adding disconnect callbackups */ -static pthread_mutex_t atexit_callback_disconnect_mutex = PTHREAD_MUTEX_INITIALIZER; - -/* Connection routines */ -static void init_cancel_handler(void); -static void on_before_exec(PGconn *conn, PGcancel *thread_cancel_conn); -static void on_after_exec(PGcancel *thread_cancel_conn); -static void on_interrupt(void); -static void on_cleanup(void); -static pqsigfunc oldhandler = NULL; - -static char ** pgut_pgfnames(const char *path, bool strict); -static void pgut_pgfnames_cleanup(char **filenames); - -void discard_response(PGconn *conn); - -/* Note that atexit handlers always called on the main thread */ -void -pgut_init(void) -{ - init_cancel_handler(); - atexit(on_cleanup); -} - -/* - * Ask the user for a password; 'username' is the username the - * password is for, if one has been explicitly specified. - * Set malloc'd string to the global variable 'password'. - */ -static void -prompt_for_password(const char *username) -{ - in_password = true; - - if (password) - { - free(password); - password = NULL; - } - -#if PG_VERSION_NUM >= 140000 - if (username == NULL) - password = simple_prompt("Password: ", false); - else - { - char message[256]; - snprintf(message, lengthof(message), "Password for user %s: ", username); - password = simple_prompt(message , false); - } -#elif PG_VERSION_NUM >= 100000 - password = (char *) pgut_malloc(sizeof(char) * 100 + 1); - if (username == NULL) - simple_prompt("Password: ", password, 100, false); - else - { - char message[256]; - snprintf(message, lengthof(message), "Password for user %s: ", username); - simple_prompt(message, password, 100, false); - } -#else - if (username == NULL) - password = simple_prompt("Password: ", 100, false); - else - { - char message[256]; - snprintf(message, lengthof(message), "Password for user %s: ", username); - password = simple_prompt(message, 100, false); - } -#endif - - in_password = false; -} - -/* - * Copied from pg_basebackup.c - * Escape a parameter value so that it can be used as part of a libpq - * connection string, e.g. in: - * - * application_name= - * - * The returned string is malloc'd. Return NULL on out-of-memory. - */ -static char * -escapeConnectionParameter(const char *src) -{ - bool need_quotes = false; - bool need_escaping = false; - const char *p; - char *dstbuf; - char *dst; - - /* - * First check if quoting is needed. Any quote (') or backslash (\) - * characters need to be escaped. Parameters are separated by whitespace, - * so any string containing whitespace characters need to be quoted. An - * empty string is represented by ''. - */ - if (strchr(src, '\'') != NULL || strchr(src, '\\') != NULL) - need_escaping = true; - - for (p = src; *p; p++) - { - if (isspace((unsigned char) *p)) - { - need_quotes = true; - break; - } - } - - if (*src == '\0') - return pg_strdup("''"); - - if (!need_quotes && !need_escaping) - return pg_strdup(src); /* no quoting or escaping needed */ - - /* - * Allocate a buffer large enough for the worst case that all the source - * characters need to be escaped, plus quotes. - */ - dstbuf = pg_malloc(strlen(src) * 2 + 2 + 1); - - dst = dstbuf; - if (need_quotes) - *(dst++) = '\''; - for (; *src; src++) - { - if (*src == '\'' || *src == '\\') - *(dst++) = '\\'; - *(dst++) = *src; - } - if (need_quotes) - *(dst++) = '\''; - *dst = '\0'; - - return dstbuf; -} - -/* Construct a connection string for possible future use in recovery.conf */ -char * -pgut_get_conninfo_string(PGconn *conn) -{ - PQconninfoOption *connOptions; - PQconninfoOption *option; - PQExpBuffer buf = createPQExpBuffer(); - char *connstr; - bool firstkeyword = true; - char *escaped; - - connOptions = PQconninfo(conn); - if (connOptions == NULL) - elog(ERROR, "out of memory"); - - /* Construct a new connection string in key='value' format. */ - for (option = connOptions; option && option->keyword; option++) - { - /* - * Do not emit this setting if: - the setting is "replication", - * "dbname" or "fallback_application_name", since these would be - * overridden by the libpqwalreceiver module anyway. - not set or - * empty. - */ - if (strcmp(option->keyword, "replication") == 0 || - strcmp(option->keyword, "dbname") == 0 || - strcmp(option->keyword, "fallback_application_name") == 0 || - (option->val == NULL) || - (option->val != NULL && option->val[0] == '\0')) - continue; - - /* do not print password, passfile and options into the file */ - if (strcmp(option->keyword, "password") == 0 || - strcmp(option->keyword, "passfile") == 0 || - strcmp(option->keyword, "options") == 0) - continue; - - if (!firstkeyword) - appendPQExpBufferChar(buf, ' '); - - firstkeyword = false; - - escaped = escapeConnectionParameter(option->val); - appendPQExpBuffer(buf, "%s=%s", option->keyword, escaped); - free(escaped); - } - - connstr = pg_strdup(buf->data); - destroyPQExpBuffer(buf); - return connstr; -} - -/* TODO: it is better to use PQconnectdbParams like in psql - * It will allow to set application_name for pg_probackup - */ -PGconn * -pgut_connect(const char *host, const char *port, - const char *dbname, const char *username) -{ - PGconn *conn; - - if (interrupted && !in_cleanup) - elog(ERROR, "interrupted"); - - if (force_password && !prompt_password) - elog(ERROR, "You cannot specify --password and --no-password options together"); - - if (!password && force_password) - prompt_for_password(username); - - /* Start the connection. Loop until we have a password if requested by backend. */ - for (;;) - { - conn = PQsetdbLogin(host, port, NULL, NULL, - dbname, username, password); - - if (PQstatus(conn) == CONNECTION_OK) - { - pthread_lock(&atexit_callback_disconnect_mutex); - pgut_atexit_push(pgut_disconnect_callback, conn); - pthread_mutex_unlock(&atexit_callback_disconnect_mutex); - break; - } - - if (conn && PQconnectionNeedsPassword(conn) && prompt_password) - { - PQfinish(conn); - prompt_for_password(username); - - if (interrupted) - elog(ERROR, "interrupted"); - - if (password == NULL || password[0] == '\0') - elog(ERROR, "no password supplied"); - - continue; - } - elog(ERROR, "could not connect to database %s: %s", - dbname, PQerrorMessage(conn)); - - PQfinish(conn); - return NULL; - } - - /* - * Fix for CVE-2018-1058. This code was taken with small modification from - * src/bin/pg_basebackup/streamutil.c:GetConnection() - */ - if (dbname != NULL) - { - PGresult *res; - - res = PQexec(conn, ALWAYS_SECURE_SEARCH_PATH_SQL); - if (PQresultStatus(res) != PGRES_TUPLES_OK) - { - elog(ERROR, "could not clear search_path: %s", - PQerrorMessage(conn)); - PQclear(res); - PQfinish(conn); - return NULL; - } - PQclear(res); - } - - return conn; -} - -PGconn * -pgut_connect_replication(const char *host, const char *port, - const char *dbname, const char *username, - bool strict) -{ - PGconn *tmpconn; - int argcount = 7; /* dbname, replication, fallback_app_name, - * host, user, port, password */ - int i; - const char **keywords; - const char **values; - - if (interrupted && !in_cleanup) - elog(ERROR, "interrupted"); - - if (force_password && !prompt_password) - elog(ERROR, "You cannot specify --password and --no-password options together"); - - if (!password && force_password) - prompt_for_password(username); - - i = 0; - - keywords = pg_malloc0((argcount + 1) * sizeof(*keywords)); - values = pg_malloc0((argcount + 1) * sizeof(*values)); - - - keywords[i] = "dbname"; - values[i] = "replication"; - i++; - keywords[i] = "replication"; - values[i] = "true"; - i++; - keywords[i] = "fallback_application_name"; - values[i] = PROGRAM_NAME; - i++; - - if (host) - { - keywords[i] = "host"; - values[i] = host; - i++; - } - if (username) - { - keywords[i] = "user"; - values[i] = username; - i++; - } - if (port) - { - keywords[i] = "port"; - values[i] = port; - i++; - } - - /* Use (or reuse, on a subsequent connection) password if we have it */ - if (password) - { - keywords[i] = "password"; - values[i] = password; - } - else - { - keywords[i] = NULL; - values[i] = NULL; - } - - for (;;) - { - tmpconn = PQconnectdbParams(keywords, values, true); - - - if (PQstatus(tmpconn) == CONNECTION_OK) - { - free(values); - free(keywords); - return tmpconn; - } - - if (tmpconn && PQconnectionNeedsPassword(tmpconn) && prompt_password) - { - PQfinish(tmpconn); - prompt_for_password(username); - keywords[i] = "password"; - values[i] = password; - continue; - } - - elog(strict ? ERROR : WARNING, "could not connect to database %s: %s", - dbname, PQerrorMessage(tmpconn)); - PQfinish(tmpconn); - free(values); - free(keywords); - return NULL; - } -} - - -void -pgut_disconnect(PGconn *conn) -{ - if (conn) - PQfinish(conn); - - pthread_lock(&atexit_callback_disconnect_mutex); - pgut_atexit_pop(pgut_disconnect_callback, conn); - pthread_mutex_unlock(&atexit_callback_disconnect_mutex); -} - - -PGresult * -pgut_execute_parallel(PGconn* conn, - PGcancel* thread_cancel_conn, const char *query, - int nParams, const char **params, - bool text_result, bool ok_error, bool async) -{ - PGresult *res; - - if (interrupted && !in_cleanup) - elog(ERROR, "interrupted"); - - /* write query to elog if verbose */ - if (logger_config.log_level_console <= VERBOSE || - logger_config.log_level_file <= VERBOSE) - { - int i; - - if (strchr(query, '\n')) - elog(VERBOSE, "(query)\n%s", query); - else - elog(VERBOSE, "(query) %s", query); - for (i = 0; i < nParams; i++) - elog(VERBOSE, "\t(param:%d) = %s", i, params[i] ? params[i] : "(null)"); - } - - if (conn == NULL) - { - elog(ERROR, "not connected"); - return NULL; - } - - //on_before_exec(conn, thread_cancel_conn); - if (async) - { - /* clean any old data */ - discard_response(conn); - - if (nParams == 0) - PQsendQuery(conn, query); - else - PQsendQueryParams(conn, query, nParams, NULL, params, NULL, NULL, - /* - * Specify zero to obtain results in text format, - * or one to obtain results in binary format. - */ - (text_result) ? 0 : 1); - - /* wait for processing, TODO: timeout */ - for (;;) - { - if (interrupted) - { - pgut_cancel(conn); - pgut_disconnect(conn); - elog(ERROR, "interrupted"); - } - - if (!PQconsumeInput(conn)) - elog(ERROR, "query failed: %s query was: %s", - PQerrorMessage(conn), query); - - /* query is no done */ - if (!PQisBusy(conn)) - break; - - usleep(10000); - } - - res = PQgetResult(conn); - } - else - { - if (nParams == 0) - res = PQexec(conn, query); - else - res = PQexecParams(conn, query, nParams, NULL, params, NULL, NULL, - /* - * Specify zero to obtain results in text format, - * or one to obtain results in binary format. - */ - (text_result) ? 0 : 1); - } - //on_after_exec(thread_cancel_conn); - - switch (PQresultStatus(res)) - { - case PGRES_TUPLES_OK: - case PGRES_COMMAND_OK: - case PGRES_COPY_IN: - break; - default: - if (ok_error && PQresultStatus(res) == PGRES_FATAL_ERROR) - break; - - elog(ERROR, "query failed: %squery was: %s", - PQerrorMessage(conn), query); - break; - } - - return res; -} - -PGresult * -pgut_execute(PGconn* conn, const char *query, int nParams, const char **params) -{ - return pgut_execute_extended(conn, query, nParams, params, true, false); -} - -PGresult * -pgut_execute_extended(PGconn* conn, const char *query, int nParams, - const char **params, bool text_result, bool ok_error) -{ - PGresult *res; - ExecStatusType res_status; - - if (interrupted && !in_cleanup) - elog(ERROR, "interrupted"); - - /* write query to elog if verbose */ - if (logger_config.log_level_console <= VERBOSE || - logger_config.log_level_file <= VERBOSE) - { - int i; - - if (strchr(query, '\n')) - elog(VERBOSE, "(query)\n%s", query); - else - elog(VERBOSE, "(query) %s", query); - for (i = 0; i < nParams; i++) - elog(VERBOSE, "\t(param:%d) = %s", i, params[i] ? params[i] : "(null)"); - } - - if (conn == NULL) - { - elog(ERROR, "not connected"); - return NULL; - } - - on_before_exec(conn, NULL); - if (nParams == 0) - res = PQexec(conn, query); - else - res = PQexecParams(conn, query, nParams, NULL, params, NULL, NULL, - /* - * Specify zero to obtain results in text format, - * or one to obtain results in binary format. - */ - (text_result) ? 0 : 1); - on_after_exec(NULL); - - res_status = PQresultStatus(res); - switch (res_status) - { - case PGRES_TUPLES_OK: - case PGRES_COMMAND_OK: - case PGRES_COPY_IN: - break; - default: - if (ok_error && res_status == PGRES_FATAL_ERROR) - break; - - elog(ERROR, "query failed: %squery was: %s", - PQerrorMessage(conn), query); - break; - } - - return res; -} - -bool -pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elevel) -{ - int res; - - if (interrupted && !in_cleanup) - elog(ERROR, "interrupted"); - - /* write query to elog if verbose */ - if (logger_config.log_level_console <= VERBOSE || - logger_config.log_level_file <= VERBOSE) - { - int i; - - if (strchr(query, '\n')) - elog(VERBOSE, "(query)\n%s", query); - else - elog(VERBOSE, "(query) %s", query); - for (i = 0; i < nParams; i++) - elog(VERBOSE, "\t(param:%d) = %s", i, params[i] ? params[i] : "(null)"); - } - - if (conn == NULL) - { - elog(elevel, "not connected"); - return false; - } - - if (nParams == 0) - res = PQsendQuery(conn, query); - else - res = PQsendQueryParams(conn, query, nParams, NULL, params, NULL, NULL, 0); - - if (res != 1) - { - elog(elevel, "query failed: %squery was: %s", - PQerrorMessage(conn), query); - return false; - } - - return true; -} - -void -pgut_cancel(PGconn* conn) -{ - PGcancel *cancel_conn = PQgetCancel(conn); - char errbuf[256]; - - if (cancel_conn != NULL) - { - if (PQcancel(cancel_conn, errbuf, sizeof(errbuf))) - elog(WARNING, "Cancel request sent"); - else - elog(WARNING, "Cancel request failed"); - } - - if (cancel_conn) - PQfreeCancel(cancel_conn); -} - -int -pgut_wait(int num, PGconn *connections[], struct timeval *timeout) -{ - /* all connections are busy. wait for finish */ - while (!interrupted) - { - int i; - fd_set mask; - int maxsock; - - FD_ZERO(&mask); - - maxsock = -1; - for (i = 0; i < num; i++) - { - int sock; - - if (connections[i] == NULL) - continue; - sock = PQsocket(connections[i]); - if (sock >= 0) - { - FD_SET(sock, &mask); - if (maxsock < sock) - maxsock = sock; - } - } - - if (maxsock == -1) - { - errno = ENOENT; - return -1; - } - - i = wait_for_sockets(maxsock + 1, &mask, timeout); - if (i == 0) - break; /* timeout */ - - for (i = 0; i < num; i++) - { - if (connections[i] && FD_ISSET(PQsocket(connections[i]), &mask)) - { - PQconsumeInput(connections[i]); - if (PQisBusy(connections[i])) - continue; - return i; - } - } - } - - errno = EINTR; - return -1; -} - -#ifdef WIN32 -static CRITICAL_SECTION cancelConnLock; -#endif - -/* - * on_before_exec - * - * Set cancel_conn to point to the current database connection. - */ -static void -on_before_exec(PGconn *conn, PGcancel *thread_cancel_conn) -{ - PGcancel *old; - - if (in_cleanup) - return; /* forbid cancel during cleanup */ - -#ifdef WIN32 - EnterCriticalSection(&cancelConnLock); -#endif - - if (thread_cancel_conn) - { - //elog(WARNING, "Handle tread_cancel_conn. on_before_exec"); - old = thread_cancel_conn; - - /* be sure handle_interrupt doesn't use pointer while freeing */ - thread_cancel_conn = NULL; - - if (old != NULL) - PQfreeCancel(old); - - thread_cancel_conn = PQgetCancel(conn); - } - else - { - /* Free the old one if we have one */ - old = cancel_conn; - - /* be sure handle_interrupt doesn't use pointer while freeing */ - cancel_conn = NULL; - - if (old != NULL) - PQfreeCancel(old); - - cancel_conn = PQgetCancel(conn); - } - -#ifdef WIN32 - LeaveCriticalSection(&cancelConnLock); -#endif -} - -/* - * on_after_exec - * - * Free the current cancel connection, if any, and set to NULL. - */ -static void -on_after_exec(PGcancel *thread_cancel_conn) -{ - PGcancel *old; - - if (in_cleanup) - return; /* forbid cancel during cleanup */ - -#ifdef WIN32 - EnterCriticalSection(&cancelConnLock); -#endif - - if (thread_cancel_conn) - { - //elog(WARNING, "Handle tread_cancel_conn. on_after_exec"); - old = thread_cancel_conn; - - /* be sure handle_interrupt doesn't use pointer while freeing */ - thread_cancel_conn = NULL; - - if (old != NULL) - PQfreeCancel(old); - } - else - { - old = cancel_conn; - - /* be sure handle_interrupt doesn't use pointer while freeing */ - cancel_conn = NULL; - - if (old != NULL) - PQfreeCancel(old); - } -#ifdef WIN32 - LeaveCriticalSection(&cancelConnLock); -#endif -} - -/* - * Handle interrupt signals by cancelling the current command. - */ -static void -on_interrupt(void) -{ - int save_errno = errno; - char errbuf[256]; - - /* Set interrupted flag */ - interrupted = true; - - /* - * User prompts password, call on_cleanup() byhand. Unless we do that we will - * get stuck forever until a user enters a password. - */ - if (in_password) - { - on_cleanup(); - - pqsignal(SIGINT, oldhandler); - kill(0, SIGINT); - } - - /* Send QueryCancel if we are processing a database query */ - if (!in_cleanup && cancel_conn != NULL && - PQcancel(cancel_conn, errbuf, sizeof(errbuf))) - { - elog(WARNING, "Cancel request sent"); - } - - errno = save_errno; /* just in case the write changed it */ -} - -typedef struct pgut_atexit_item pgut_atexit_item; -struct pgut_atexit_item -{ - pgut_atexit_callback callback; - void *userdata; - pgut_atexit_item *next; -}; - -static pgut_atexit_item *pgut_atexit_stack = NULL; - -void -pgut_disconnect_callback(bool fatal, void *userdata) -{ - PGconn *conn = (PGconn *) userdata; - if (conn) - pgut_disconnect(conn); -} - -void -pgut_atexit_push(pgut_atexit_callback callback, void *userdata) -{ - pgut_atexit_item *item; - - AssertArg(callback != NULL); - - item = pgut_new(pgut_atexit_item); - item->callback = callback; - item->userdata = userdata; - item->next = pgut_atexit_stack; - - pgut_atexit_stack = item; -} - -void -pgut_atexit_pop(pgut_atexit_callback callback, void *userdata) -{ - pgut_atexit_item *item; - pgut_atexit_item **prev; - - for (item = pgut_atexit_stack, prev = &pgut_atexit_stack; - item; - prev = &item->next, item = item->next) - { - if (item->callback == callback && item->userdata == userdata) - { - *prev = item->next; - free(item); - break; - } - } -} - -static void -call_atexit_callbacks(bool fatal) -{ - pgut_atexit_item *item; - pgut_atexit_item *next; - - for (item = pgut_atexit_stack; item; item = next) - { - next = item->next; - item->callback(fatal, item->userdata); - } -} - -static void -on_cleanup(void) -{ - in_cleanup = true; - interrupted = false; - call_atexit_callbacks(false); -} - -void * -pgut_malloc(size_t size) -{ - char *ret; - - if ((ret = malloc(size)) == NULL) - elog(ERROR, "could not allocate memory (%lu bytes): %s", - (unsigned long) size, strerror(errno)); - return ret; -} - -void * -pgut_malloc0(size_t size) -{ - char *ret; - - ret = pgut_malloc(size); - memset(ret, 0, size); - - return ret; -} - -void * -pgut_realloc(void *p, size_t size) -{ - char *ret; - - if ((ret = realloc(p, size)) == NULL) - elog(ERROR, "could not re-allocate memory (%lu bytes): %s", - (unsigned long) size, strerror(errno)); - return ret; -} - -char * -pgut_strdup(const char *str) -{ - char *ret; - - if (str == NULL) - return NULL; - - if ((ret = strdup(str)) == NULL) - elog(ERROR, "could not duplicate string \"%s\": %s", - str, strerror(errno)); - return ret; -} - -char * -pgut_strndup(const char *str, size_t n) -{ - char *ret; - - if (str == NULL) - return NULL; - -#if _POSIX_C_SOURCE >= 200809L - if ((ret = strndup(str, n)) == NULL) - elog(ERROR, "could not duplicate string \"%s\": %s", - str, strerror(errno)); -#else /* WINDOWS doesn't have strndup() */ - if ((ret = malloc(n + 1)) == NULL) - elog(ERROR, "could not duplicate string \"%s\": %s", - str, strerror(errno)); - - memcpy(ret, str, n); - ret[n] = '\0'; -#endif - return ret; -} - -/* - * Allocates new string, that contains part of filepath string minus trailing filename string - * If trailing filename string not found, returns copy of filepath. - * Result must be free by caller. - */ -char * -pgut_str_strip_trailing_filename(const char *filepath, const char *filename) -{ - size_t fp_len = strlen(filepath); - size_t fn_len = strlen(filename); - if (strncmp(filepath + fp_len - fn_len, filename, fn_len) == 0) - return pgut_strndup(filepath, fp_len - fn_len); - else - return pgut_strndup(filepath, fp_len); -} - -void -pgut_free(void *p) -{ - free(p); -} - -FILE * -pgut_fopen(const char *path, const char *mode, bool missing_ok) -{ - FILE *fp; - - if ((fp = fio_open_stream(path, FIO_BACKUP_HOST)) == NULL) - { - if (missing_ok && errno == ENOENT) - return NULL; - - elog(ERROR, "could not open file \"%s\": %s", - path, strerror(errno)); - } - - return fp; -} - -#ifdef WIN32 -static int select_win32(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, const struct timeval * timeout); -#define select select_win32 -#endif - -int -wait_for_socket(int sock, struct timeval *timeout) -{ - fd_set fds; - - FD_ZERO(&fds); - FD_SET(sock, &fds); - return wait_for_sockets(sock + 1, &fds, timeout); -} - -int -wait_for_sockets(int nfds, fd_set *fds, struct timeval *timeout) -{ - int i; - - for (;;) - { - i = select(nfds, fds, NULL, NULL, timeout); - if (i < 0) - { - if (interrupted) - elog(ERROR, "interrupted"); - else if (errno != EINTR) - elog(ERROR, "select failed: %s", strerror(errno)); - } - else - return i; - } -} - -#ifndef WIN32 -static void -handle_interrupt(SIGNAL_ARGS) -{ - on_interrupt(); -} - -/* Handle various inrerruptions in the same way */ -static void -init_cancel_handler(void) -{ - oldhandler = pqsignal(SIGINT, handle_interrupt); - pqsignal(SIGQUIT, handle_interrupt); - pqsignal(SIGTERM, handle_interrupt); - pqsignal(SIGPIPE, handle_interrupt); -} -#else /* WIN32 */ - -/* - * Console control handler for Win32. Note that the control handler will - * execute on a *different thread* than the main one, so we need to do - * proper locking around those structures. - */ -static BOOL WINAPI -consoleHandler(DWORD dwCtrlType) -{ - if (dwCtrlType == CTRL_C_EVENT || - dwCtrlType == CTRL_BREAK_EVENT) - { - EnterCriticalSection(&cancelConnLock); - on_interrupt(); - LeaveCriticalSection(&cancelConnLock); - return TRUE; - } - else - /* Return FALSE for any signals not being handled */ - return FALSE; -} - -static void -init_cancel_handler(void) -{ - InitializeCriticalSection(&cancelConnLock); - - SetConsoleCtrlHandler(consoleHandler, TRUE); -} - -int -sleep(unsigned int seconds) -{ - Sleep(seconds * 1000); - return 0; -} - -int -usleep(unsigned int usec) -{ - Sleep((usec + 999) / 1000); /* rounded up */ - return 0; -} - -#undef select -static int -select_win32(int nfds, fd_set *readfds, fd_set *writefds, fd_set *exceptfds, const struct timeval * timeout) -{ - struct timeval remain; - - if (timeout != NULL) - remain = *timeout; - else - { - remain.tv_usec = 0; - remain.tv_sec = LONG_MAX; /* infinite */ - } - - /* sleep only one second because Ctrl+C doesn't interrupt select. */ - while (remain.tv_sec > 0 || remain.tv_usec > 0) - { - int ret; - struct timeval onesec; - - if (remain.tv_sec > 0) - { - onesec.tv_sec = 1; - onesec.tv_usec = 0; - remain.tv_sec -= 1; - } - else - { - onesec.tv_sec = 0; - onesec.tv_usec = remain.tv_usec; - remain.tv_usec = 0; - } - - ret = select(nfds, readfds, writefds, exceptfds, &onesec); - if (ret != 0) - { - /* succeeded or error */ - return ret; - } - else if (interrupted) - { - errno = EINTR; - return 0; - } - } - - return 0; /* timeout */ -} - -#endif /* WIN32 */ - -void -discard_response(PGconn *conn) -{ - PGresult *res; - - do - { - res = PQgetResult(conn); - if (res) - PQclear(res); - } while (res); -} - -/* - * pgfnames - * - * return a list of the names of objects in the argument directory. Caller - * must call pgfnames_cleanup later to free the memory allocated by this - * function. - */ -char ** -pgut_pgfnames(const char *path, bool strict) -{ - DIR *dir; - struct dirent *file; - char **filenames; - int numnames = 0; - int fnsize = 200; /* enough for many small dbs */ - - dir = opendir(path); - if (dir == NULL) - { - elog(strict ? ERROR : WARNING, "could not open directory \"%s\": %m", path); - return NULL; - } - - filenames = (char **) palloc(fnsize * sizeof(char *)); - - while (errno = 0, (file = readdir(dir)) != NULL) - { - if (strcmp(file->d_name, ".") != 0 && strcmp(file->d_name, "..") != 0) - { - if (numnames + 1 >= fnsize) - { - fnsize *= 2; - filenames = (char **) repalloc(filenames, - fnsize * sizeof(char *)); - } - filenames[numnames++] = pstrdup(file->d_name); - } - } - - filenames[numnames] = NULL; - - if (errno) - { - elog(strict ? ERROR : WARNING, "could not read directory \"%s\": %m", path); - pgut_pgfnames_cleanup(filenames); - closedir(dir); - return NULL; - } - - - if (closedir(dir)) - { - elog(strict ? ERROR : WARNING, "could not close directory \"%s\": %m", path); - return NULL; - } - - return filenames; -} - -/* - * pgfnames_cleanup - * - * deallocate memory used for filenames - */ -void -pgut_pgfnames_cleanup(char **filenames) -{ - char **fn; - - for (fn = filenames; *fn; fn++) - pfree(*fn); - - pfree(filenames); -} - -/* Shamelessly stolen from commom/rmtree.c */ -bool -pgut_rmtree(const char *path, bool rmtopdir, bool strict) -{ - bool result = true; - char pathbuf[MAXPGPATH]; - char **filenames; - char **filename; - struct stat statbuf; - - /* - * we copy all the names out of the directory before we start modifying - * it. - */ - filenames = pgut_pgfnames(path, strict); - - if (filenames == NULL) - return false; - - /* now we have the names we can start removing things */ - for (filename = filenames; *filename; filename++) - { - join_path_components(pathbuf, path, *filename); - - if (lstat(pathbuf, &statbuf) != 0) - { - elog(strict ? ERROR : WARNING, "could not stat file or directory \"%s\": %m", pathbuf); - result = false; - break; - } - - if (S_ISDIR(statbuf.st_mode)) - { - /* call ourselves recursively for a directory */ - if (!pgut_rmtree(pathbuf, true, strict)) - { - result = false; - break; - } - } - else - { - if (unlink(pathbuf) != 0) - { - elog(strict ? ERROR : WARNING, "could not remove file or directory \"%s\": %m", pathbuf); - result = false; - break; - } - } - } - - if (rmtopdir) - { - if (rmdir(path) != 0) - { - elog(strict ? ERROR : WARNING, "could not remove file or directory \"%s\": %m", path); - result = false; - } - } - - pgut_pgfnames_cleanup(filenames); - - return result; -} - -/* cross-platform setenv */ -void -pgut_setenv(const char *key, const char *val) -{ -#ifdef WIN32 - char *envstr = NULL; - envstr = psprintf("%s=%s", key, val); - putenv(envstr); -#else - setenv(key, val, 1); -#endif -} - -/* stolen from unsetenv.c */ -void -pgut_unsetenv(const char *key) -{ -#ifdef WIN32 - char *envstr = NULL; - - if (getenv(key) == NULL) - return; /* no work */ - - /* - * The technique embodied here works if libc follows the Single Unix Spec - * and actually uses the storage passed to putenv() to hold the environ - * entry. When we clobber the entry in the second step we are ensuring - * that we zap the actual environ member. However, there are some libc - * implementations (notably recent BSDs) that do not obey SUS but copy the - * presented string. This method fails on such platforms. Hopefully all - * such platforms have unsetenv() and thus won't be using this hack. See: - * http://www.greenend.org.uk/rjk/2008/putenv.html - * - * Note that repeatedly setting and unsetting a var using this code will - * leak memory. - */ - - envstr = (char *) pgut_malloc(strlen(key) + 2); - if (!envstr) /* not much we can do if no memory */ - return; - - /* Override the existing setting by forcibly defining the var */ - sprintf(envstr, "%s=", key); - putenv(envstr); - - /* Now we can clobber the variable definition this way: */ - strcpy(envstr, "="); - - /* - * This last putenv cleans up if we have multiple zero-length names as a - * result of unsetting multiple things. - */ - putenv(envstr); -#else - unsetenv(key); -#endif -} diff --git a/src/utils/pgut.h b/src/utils/pgut.h deleted file mode 100644 index 1b7b7864c..000000000 --- a/src/utils/pgut.h +++ /dev/null @@ -1,128 +0,0 @@ -/*------------------------------------------------------------------------- - * - * pgut.h - * - * Portions Copyright (c) 2009-2013, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2017-2021, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#ifndef PGUT_H -#define PGUT_H - -#include "postgres_fe.h" -#include "libpq-fe.h" - -typedef void (*pgut_atexit_callback)(bool fatal, void *userdata); - -extern void pgut_help(bool details); - -/* - * pgut framework variables and functions - */ -extern bool prompt_password; -extern bool force_password; - -extern bool interrupted; -extern bool in_cleanup; -extern bool in_password; /* User prompts password */ - -extern void pgut_atexit_push(pgut_atexit_callback callback, void *userdata); -extern void pgut_atexit_pop(pgut_atexit_callback callback, void *userdata); - -extern void pgut_init(void); - -/* - * Database connections - */ -extern char *pgut_get_conninfo_string(PGconn *conn); -extern PGconn *pgut_connect(const char *host, const char *port, - const char *dbname, const char *username); -extern PGconn *pgut_connect_replication(const char *host, const char *port, - const char *dbname, const char *username, - bool strict); -extern void pgut_disconnect(PGconn *conn); -extern void pgut_disconnect_callback(bool fatal, void *userdata); -extern PGresult *pgut_execute(PGconn* conn, const char *query, int nParams, - const char **params); -extern PGresult *pgut_execute_extended(PGconn* conn, const char *query, int nParams, - const char **params, bool text_result, bool ok_error); -extern PGresult *pgut_execute_parallel(PGconn* conn, PGcancel* thread_cancel_conn, - const char *query, int nParams, - const char **params, bool text_result, bool ok_error, bool async); -extern bool pgut_send(PGconn* conn, const char *query, int nParams, const char **params, int elevel); -extern void pgut_cancel(PGconn* conn); -extern int pgut_wait(int num, PGconn *connections[], struct timeval *timeout); - -/* - * memory allocators - */ -extern void *pgut_malloc(size_t size); -extern void *pgut_malloc0(size_t size); -extern void *pgut_realloc(void *p, size_t size); -extern char *pgut_strdup(const char *str); -extern char *pgut_strndup(const char *str, size_t n); -extern char *pgut_str_strip_trailing_filename(const char *filepath, const char *filename); -extern void pgut_free(void *p); - -#define pgut_new(type) ((type *) pgut_malloc(sizeof(type))) -#define pgut_new0(type) ((type *) pgut_malloc0(sizeof(type))) -#define pgut_newarray(type, n) ((type *) pgut_malloc(sizeof(type) * (n))) - -/* - * file operations - */ -extern FILE *pgut_fopen(const char *path, const char *mode, bool missing_ok); - -/* - * Assert - */ -#undef Assert -#undef AssertArg -#undef AssertMacro - -#ifdef USE_ASSERT_CHECKING -#define Assert(x) assert(x) -#define AssertArg(x) assert(x) -#define AssertMacro(x) assert(x) -#else -#define Assert(x) ((void) 0) -#define AssertArg(x) ((void) 0) -#define AssertMacro(x) ((void) 0) -#endif - -#define IsSpace(c) (isspace((unsigned char)(c))) -#define IsAlpha(c) (isalpha((unsigned char)(c))) -#define IsAlnum(c) (isalnum((unsigned char)(c))) -#define ToLower(c) (tolower((unsigned char)(c))) -#define ToUpper(c) (toupper((unsigned char)(c))) - -/* - * socket operations - */ -extern int wait_for_socket(int sock, struct timeval *timeout); -extern int wait_for_sockets(int nfds, fd_set *fds, struct timeval *timeout); - -#ifdef WIN32 -extern int sleep(unsigned int seconds); -extern int usleep(unsigned int usec); -#endif - -#ifdef _MSC_VER -#define ARG_SIZE_HINT -#else -#define ARG_SIZE_HINT static -#endif - -static inline uint32 hash_mix32_2(uint32 a, uint32 b) -{ - b ^= (a<<7)|(a>>25); - a *= 0xdeadbeef; - b *= 0xcafeabed; - a ^= a >> 16; - b ^= b >> 15; - return a^b; -} - -#endif /* PGUT_H */ diff --git a/src/utils/remote.c b/src/utils/remote.c deleted file mode 100644 index 7ef8d3239..000000000 --- a/src/utils/remote.c +++ /dev/null @@ -1,379 +0,0 @@ -#include -#include -#include -#include -#include -#include - -#ifdef WIN32 -#define __thread __declspec(thread) -#else -#include -#endif - -#include "pg_probackup.h" -#include "file.h" - -#define MAX_CMDLINE_LENGTH 4096 -#define MAX_CMDLINE_OPTIONS 256 -#define ERR_BUF_SIZE 4096 -#define PIPE_SIZE (64*1024) - -static int split_options(int argc, char* argv[], int max_options, char* options) -{ - char* opt = options; - char in_quote = '\0'; - while (true) { - switch (*opt) { - case '\'': - case '\"': - if (!in_quote) { - in_quote = *opt++; - continue; - } - if (*opt == in_quote && *++opt != in_quote) { - in_quote = '\0'; - continue; - } - break; - case '\0': - if (opt != options) { - argv[argc++] = options; - if (argc >= max_options) - elog(ERROR, "Too much options"); - } - return argc; - case ' ': - argv[argc++] = options; - if (argc >= max_options) - elog(ERROR, "Too much options"); - *opt++ = '\0'; - options = opt; - continue; - default: - break; - } - opt += 1; - } - return argc; -} - -static __thread int child_pid; - -#if 0 -static void kill_child(void) -{ - kill(child_pid, SIGTERM); -} -#endif - - -void wait_ssh(void) -{ -/* - * We need to wait termination of SSH process to eliminate zombies. - * There is no waitpid() function at Windows but there are no zombie processes caused by lack of wait/waitpid. - * So just disable waitpid for Windows. - */ -#ifndef WIN32 - int status; - waitpid(child_pid, &status, 0); - elog(LOG, "SSH process %d is terminated with status %d", child_pid, status); -#endif -} - -/* - * On windows we launch a new pbk process via 'pg_probackup ssh ...' - * so this process would new that it should exec ssh, because - * there is no fork on Windows. - */ -#ifdef WIN32 -void launch_ssh(char* argv[]) -{ - int infd = atoi(argv[2]); - int outfd = atoi(argv[3]); - - SYS_CHECK(close(STDIN_FILENO)); - SYS_CHECK(close(STDOUT_FILENO)); - - SYS_CHECK(dup2(infd, STDIN_FILENO)); - SYS_CHECK(dup2(outfd, STDOUT_FILENO)); - - SYS_CHECK(execvp(argv[4], argv+4)); -} -#endif - -static bool needs_quotes(char const* path) -{ - return strchr(path, ' ') != NULL; -} - -bool launch_agent(void) -{ - char cmd[MAX_CMDLINE_LENGTH]; - char* ssh_argv[MAX_CMDLINE_OPTIONS]; - int ssh_argc; - int outfd[2]; - int infd[2]; - int errfd[2]; - int agent_version; - - ssh_argc = 0; -#ifdef WIN32 - ssh_argv[ssh_argc++] = PROGRAM_NAME_FULL; - ssh_argv[ssh_argc++] = "ssh"; - ssh_argc += 2; /* reserve space for pipe descriptors */ -#endif - ssh_argv[ssh_argc++] = instance_config.remote.proto; - if (instance_config.remote.port != NULL) { - ssh_argv[ssh_argc++] = "-p"; - ssh_argv[ssh_argc++] = instance_config.remote.port; - } - if (instance_config.remote.user != NULL) { - ssh_argv[ssh_argc++] = "-l"; - ssh_argv[ssh_argc++] = instance_config.remote.user; - } - if (instance_config.remote.ssh_config != NULL) { - ssh_argv[ssh_argc++] = "-F"; - ssh_argv[ssh_argc++] = instance_config.remote.ssh_config; - } - if (instance_config.remote.ssh_options != NULL) { - ssh_argc = split_options(ssh_argc, ssh_argv, MAX_CMDLINE_OPTIONS, pg_strdup(instance_config.remote.ssh_options)); - } - - ssh_argv[ssh_argc++] = "-o"; - ssh_argv[ssh_argc++] = "PasswordAuthentication=no"; - - ssh_argv[ssh_argc++] = "-o"; - ssh_argv[ssh_argc++] = "Compression=no"; - - ssh_argv[ssh_argc++] = "-o"; - ssh_argv[ssh_argc++] = "ControlMaster=no"; - - ssh_argv[ssh_argc++] = "-o"; - ssh_argv[ssh_argc++] = "LogLevel=error"; - - ssh_argv[ssh_argc++] = instance_config.remote.host; - ssh_argv[ssh_argc++] = cmd; - ssh_argv[ssh_argc] = NULL; - - if (instance_config.remote.path) - { - char const* probackup = PROGRAM_NAME_FULL; - char* sep = strrchr(probackup, '/'); - if (sep != NULL) { - probackup = sep + 1; - } -#ifdef WIN32 - else { - sep = strrchr(probackup, '\\'); - if (sep != NULL) { - probackup = sep + 1; - } - } - if (needs_quotes(instance_config.remote.path) || needs_quotes(PROGRAM_NAME_FULL)) - snprintf(cmd, sizeof(cmd), "\"%s\\%s\" agent", - instance_config.remote.path, probackup); - else - snprintf(cmd, sizeof(cmd), "%s\\%s agent", - instance_config.remote.path, probackup); -#else - if (needs_quotes(instance_config.remote.path) || needs_quotes(PROGRAM_NAME_FULL)) - snprintf(cmd, sizeof(cmd), "\"%s/%s\" agent", - instance_config.remote.path, probackup); - else - snprintf(cmd, sizeof(cmd), "%s/%s agent", - instance_config.remote.path, probackup); -#endif - } else { - if (needs_quotes(PROGRAM_NAME_FULL)) - snprintf(cmd, sizeof(cmd), "\"%s\" agent", PROGRAM_NAME_FULL); - else - snprintf(cmd, sizeof(cmd), "%s agent", PROGRAM_NAME_FULL); - } - -#ifdef WIN32 - SYS_CHECK(_pipe(infd, PIPE_SIZE, _O_BINARY)) ; - SYS_CHECK(_pipe(outfd, PIPE_SIZE, _O_BINARY)); - ssh_argv[2] = psprintf("%d", outfd[0]); - ssh_argv[3] = psprintf("%d", infd[1]); - { - intptr_t pid = _spawnvp(_P_NOWAIT, ssh_argv[0], ssh_argv); - if (pid < 0) - return false; - child_pid = GetProcessId((HANDLE)pid); -#else - SYS_CHECK(pipe(infd)); - SYS_CHECK(pipe(outfd)); - SYS_CHECK(pipe(errfd)); - - SYS_CHECK(child_pid = fork()); - - if (child_pid == 0) { /* child */ - SYS_CHECK(close(STDIN_FILENO)); - SYS_CHECK(close(STDOUT_FILENO)); - SYS_CHECK(close(STDERR_FILENO)); - - SYS_CHECK(dup2(outfd[0], STDIN_FILENO)); - SYS_CHECK(dup2(infd[1], STDOUT_FILENO)); - SYS_CHECK(dup2(errfd[1], STDERR_FILENO)); - - SYS_CHECK(close(infd[0])); - SYS_CHECK(close(infd[1])); - SYS_CHECK(close(outfd[0])); - SYS_CHECK(close(outfd[1])); - SYS_CHECK(close(errfd[0])); - SYS_CHECK(close(errfd[1])); - - if (execvp(ssh_argv[0], ssh_argv) < 0) - return false; - } else { -#endif - elog(LOG, "Start SSH client process, pid %d, cmd \"%s\"", child_pid, cmd); - SYS_CHECK(close(infd[1])); /* These are being used by the child */ - SYS_CHECK(close(outfd[0])); - SYS_CHECK(close(errfd[1])); - /*atexit(kill_child);*/ - - fio_redirect(infd[0], outfd[1], errfd[0]); /* write to stdout */ - } - - - /* Make sure that remote agent has the same version, fork and other features to be binary compatible */ - { - char payload_buf[1024]; - fio_get_agent_version(&agent_version, payload_buf, sizeof payload_buf); - check_remote_agent_compatibility(agent_version, payload_buf, sizeof payload_buf); - } - - return true; -} - -#ifdef PGPRO_EDITION -/* PGPRO 10-13 checks to be "(certified)", with exceptional case PGPRO_11 conforming to "(standard certified)" */ -static bool check_certified() -{ - return strstr(PGPRO_VERSION_STR, "(certified)") || - strstr(PGPRO_VERSION_STR, "(standard certified)"); -} -#endif - -static char* extract_pg_edition_str() -{ - static char *vanilla = "vanilla"; -#ifdef PGPRO_EDITION - static char *_1C = "1C"; - static char *std = "standard"; - static char *ent = "enterprise"; - static char *std_cert = "standard-certified"; - static char *ent_cert = "enterprise-certified"; - - if (strcmp(PGPRO_EDITION, _1C) == 0) - return vanilla; - - if (PG_VERSION_NUM < 100000) - return PGPRO_EDITION; - - /* these "certified" checks are applicable to PGPRO from 10 up to 12 versions. - * 13+ certified versions are compatible to non-certified ones */ - if (PG_VERSION_NUM < 130000 && check_certified()) - { - if (strcmp(PGPRO_EDITION, std) == 0) - return std_cert; - else if (strcmp(PGPRO_EDITION, ent) == 0) - return ent_cert; - else - Assert("Bad #define PGPRO_EDITION value" == 0); - } - - return PGPRO_EDITION; -#else - return vanilla; -#endif -} - -#define COMPATIBILITY_VAL_STR(macro) { #macro, macro, 0 } -#define COMPATIBILITY_VAL_INT(macro) { #macro, NULL, macro } - -#define COMPATIBILITY_VAL_SEPARATOR "=" -#define COMPATIBILITY_LINE_SEPARATOR "\n" - -/* - * Compose compatibility string to be sent by pg_probackup agent - * through ssh and to be verified by pg_probackup peer. - * Compatibility string contains postgres essential vars as strings - * in format "var_name" + COMPATIBILITY_VAL_SEPARATOR + "var_value" + COMPATIBILITY_LINE_SEPARATOR - */ -size_t prepare_compatibility_str(char* compatibility_buf, size_t compatibility_buf_size) -{ - typedef struct compatibility_param_tag { - const char* name; - const char* strval; - int intval; - } compatibility_param; - - compatibility_param compatibility_params[] = { - COMPATIBILITY_VAL_STR(PG_MAJORVERSION), - { "edition", extract_pg_edition_str(), 0 }, - COMPATIBILITY_VAL_INT(SIZEOF_VOID_P), - }; - - size_t result_size = 0; - int i; - *compatibility_buf = '\0'; - - for (i = 0; i < (sizeof compatibility_params / sizeof(compatibility_param)); i++) - { - if (compatibility_params[i].strval != NULL) - result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, - "%s" COMPATIBILITY_VAL_SEPARATOR "%s" COMPATIBILITY_LINE_SEPARATOR, - compatibility_params[i].name, - compatibility_params[i].strval); - else - result_size += snprintf(compatibility_buf + result_size, compatibility_buf_size - result_size, - "%s" COMPATIBILITY_VAL_SEPARATOR "%d" COMPATIBILITY_LINE_SEPARATOR, - compatibility_params[i].name, - compatibility_params[i].intval); - Assert(result_size < compatibility_buf_size); - } - return result_size + 1; -} - -/* - * Check incoming remote agent's compatibility params for equality to local ones. - */ -void check_remote_agent_compatibility(int agent_version, char *compatibility_str, size_t compatibility_str_max_size) -{ - elog(LOG, "Agent version=%d\n", agent_version); - - if (agent_version != AGENT_PROTOCOL_VERSION) - { - char agent_version_str[1024]; - sprintf(agent_version_str, "%d.%d.%d", - agent_version / 10000, - (agent_version / 100) % 100, - agent_version % 100); - - elog(ERROR, "Remote agent protocol version %s does not match local program protocol version %s, " - "consider to upgrade pg_probackup binary", - agent_version_str, AGENT_PROTOCOL_VERSION_STR); - } - - /* checking compatibility params */ - if (strnlen(compatibility_str, compatibility_str_max_size) == compatibility_str_max_size) - { - elog(ERROR, "Corrupted remote compatibility protocol: compatibility string has no terminating \\0"); - } - - elog(LOG, "Agent compatibility params:\n%s", compatibility_str); - - { - char buf[1024]; - - prepare_compatibility_str(buf, sizeof buf); - if(strcmp(compatibility_str, buf)) - { - elog(ERROR, "Incompatible remote agent params, expected:\n%s, actual:\n:%s", buf, compatibility_str); - } - } -} diff --git a/src/utils/remote.h b/src/utils/remote.h deleted file mode 100644 index dc98644ab..000000000 --- a/src/utils/remote.h +++ /dev/null @@ -1,24 +0,0 @@ -/*------------------------------------------------------------------------- - * - * remote.h: - prototypes of remote functions. - * - * Copyright (c) 2017-2019, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#ifndef REMOTE_H -#define REMOTE_H - -typedef struct RemoteConfig -{ - char* proto; - char* host; - char* port; - char* path; - char* user; - char *ssh_config; - char *ssh_options; -} RemoteConfig; - -#endif diff --git a/src/utils/thread.c b/src/utils/thread.c deleted file mode 100644 index 1c469bd29..000000000 --- a/src/utils/thread.c +++ /dev/null @@ -1,113 +0,0 @@ -/*------------------------------------------------------------------------- - * - * thread.c: - multi-platform pthread implementations. - * - * Copyright (c) 2018-2019, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "postgres_fe.h" - -#include "thread.h" - -/* - * Global var used to detect error condition (not signal interrupt!) in threads, - * so if one thread errored out, then others may abort - */ -bool thread_interrupted = false; - -#ifdef WIN32 -DWORD main_tid = 0; -#else -pthread_t main_tid = 0; -#endif -#ifdef WIN32 -#include - -typedef struct win32_pthread -{ - HANDLE handle; - void *(*routine) (void *); - void *arg; - void *result; -} win32_pthread; - -static long mutex_initlock = 0; - -static unsigned __stdcall -win32_pthread_run(void *arg) -{ - win32_pthread *th = (win32_pthread *)arg; - - th->result = th->routine(th->arg); - - return 0; -} - -int -pthread_create(pthread_t *thread, - pthread_attr_t *attr, - void *(*start_routine) (void *), - void *arg) -{ - int save_errno; - win32_pthread *th; - - th = (win32_pthread *)pg_malloc(sizeof(win32_pthread)); - th->routine = start_routine; - th->arg = arg; - th->result = NULL; - - th->handle = (HANDLE)_beginthreadex(NULL, 0, win32_pthread_run, th, 0, NULL); - if (th->handle == NULL) - { - save_errno = errno; - free(th); - return save_errno; - } - - *thread = th; - return 0; -} - -int -pthread_join(pthread_t th, void **thread_return) -{ - if (th == NULL || th->handle == NULL) - return errno = EINVAL; - - if (WaitForSingleObject(th->handle, INFINITE) != WAIT_OBJECT_0) - { - _dosmaperr(GetLastError()); - return errno; - } - - if (thread_return) - *thread_return = th->result; - - CloseHandle(th->handle); - free(th); - return 0; -} - -#endif /* WIN32 */ - -int -pthread_lock(pthread_mutex_t *mp) -{ -#ifdef WIN32 - if (*mp == NULL) - { - while (InterlockedExchange(&mutex_initlock, 1) == 1) - /* loop, another thread own the lock */ ; - if (*mp == NULL) - { - if (pthread_mutex_init(mp, NULL)) - return -1; - } - InterlockedExchange(&mutex_initlock, 0); - } -#endif - return pthread_mutex_lock(mp); -} diff --git a/src/utils/thread.h b/src/utils/thread.h deleted file mode 100644 index 2eaa5fb45..000000000 --- a/src/utils/thread.h +++ /dev/null @@ -1,41 +0,0 @@ -/*------------------------------------------------------------------------- - * - * thread.h: - multi-platform pthread implementations. - * - * Copyright (c) 2018-2019, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#ifndef PROBACKUP_THREAD_H -#define PROBACKUP_THREAD_H - -#ifdef WIN32 -#include "postgres_fe.h" -#include "port/pthread-win32.h" - -/* Use native win32 threads on Windows */ -typedef struct win32_pthread *pthread_t; -typedef int pthread_attr_t; - -#define PTHREAD_MUTEX_INITIALIZER NULL //{ NULL, 0 } -#define PTHREAD_ONCE_INIT false - -extern int pthread_create(pthread_t *thread, pthread_attr_t *attr, void *(*start_routine) (void *), void *arg); -extern int pthread_join(pthread_t th, void **thread_return); -#else -/* Use platform-dependent pthread capability */ -#include -#endif - -#ifdef WIN32 -extern DWORD main_tid; -#else -extern pthread_t main_tid; -#endif - -extern bool thread_interrupted; - -extern int pthread_lock(pthread_mutex_t *mp); - -#endif /* PROBACKUP_THREAD_H */ diff --git a/src/validate.c b/src/validate.c deleted file mode 100644 index 0887b2e7a..000000000 --- a/src/validate.c +++ /dev/null @@ -1,752 +0,0 @@ -/*------------------------------------------------------------------------- - * - * validate.c: validate backup files. - * - * Portions Copyright (c) 2009-2011, NIPPON TELEGRAPH AND TELEPHONE CORPORATION - * Portions Copyright (c) 2015-2019, Postgres Professional - * - *------------------------------------------------------------------------- - */ - -#include "pg_probackup.h" - -#include -#include - -#include "utils/thread.h" - -static void *pgBackupValidateFiles(void *arg); -static void do_validate_instance(InstanceState *instanceState); - -static bool corrupted_backup_found = false; -static bool skipped_due_to_lock = false; - -typedef struct -{ - const char *base_path; - parray *files; - bool corrupted; - XLogRecPtr stop_lsn; - uint32 checksum_version; - uint32 backup_version; - BackupMode backup_mode; - parray *dbOid_exclude_list; - const char *external_prefix; - HeaderMap *hdr_map; - - /* - * Return value from the thread. - * 0 means there is no error, 1 - there is an error. - */ - int ret; -} validate_files_arg; - -/* - * Validate backup files. - * TODO: partial validation. - */ -void -pgBackupValidate(pgBackup *backup, pgRestoreParams *params) -{ - char external_prefix[MAXPGPATH]; - parray *files = NULL; - bool corrupted = false; - bool validation_isok = true; - /* arrays with meta info for multi threaded validate */ - pthread_t *threads; - validate_files_arg *threads_args; - int i; -// parray *dbOid_exclude_list = NULL; - - /* Check backup program version */ - if (parse_program_version(backup->program_version) > parse_program_version(PROGRAM_VERSION)) - elog(ERROR, "pg_probackup binary version is %s, but backup %s version is %s. " - "pg_probackup do not guarantee to be forward compatible. " - "Please upgrade pg_probackup binary.", - PROGRAM_VERSION, backup_id_of(backup), backup->program_version); - - /* Check backup server version */ - if (strcmp(backup->server_version, PG_MAJORVERSION) != 0) - elog(ERROR, "Backup %s has server version %s, but current pg_probackup binary " - "compiled with server version %s", - backup_id_of(backup), backup->server_version, PG_MAJORVERSION); - - if (backup->status == BACKUP_STATUS_RUNNING) - { - elog(WARNING, "Backup %s has status %s, change it to ERROR and skip validation", - backup_id_of(backup), status2str(backup->status)); - write_backup_status(backup, BACKUP_STATUS_ERROR, true); - corrupted_backup_found = true; - return; - } - - /* Revalidation is attempted for DONE, ORPHAN and CORRUPT backups */ - if (backup->status != BACKUP_STATUS_OK && - backup->status != BACKUP_STATUS_DONE && - backup->status != BACKUP_STATUS_ORPHAN && - backup->status != BACKUP_STATUS_MERGING && - backup->status != BACKUP_STATUS_CORRUPT) - { - elog(WARNING, "Backup %s has status %s. Skip validation.", - backup_id_of(backup), status2str(backup->status)); - corrupted_backup_found = true; - return; - } - - /* additional sanity */ - if (backup->backup_mode == BACKUP_MODE_FULL && - backup->status == BACKUP_STATUS_MERGING) - { - elog(WARNING, "Full backup %s has status %s, skip validation", - backup_id_of(backup), status2str(backup->status)); - return; - } - - if (backup->status == BACKUP_STATUS_OK || backup->status == BACKUP_STATUS_DONE || - backup->status == BACKUP_STATUS_MERGING) - elog(INFO, "Validating backup %s", backup_id_of(backup)); - else - elog(INFO, "Revalidating backup %s", backup_id_of(backup)); - - if (backup->backup_mode != BACKUP_MODE_FULL && - backup->backup_mode != BACKUP_MODE_DIFF_PAGE && - backup->backup_mode != BACKUP_MODE_DIFF_PTRACK && - backup->backup_mode != BACKUP_MODE_DIFF_DELTA) - elog(WARNING, "Invalid backup_mode of backup %s", backup_id_of(backup)); - - join_path_components(external_prefix, backup->root_dir, EXTERNAL_DIR); - files = get_backup_filelist(backup, false); - - if (!files) - { - elog(WARNING, "Backup %s file list is corrupted", backup_id_of(backup)); - backup->status = BACKUP_STATUS_CORRUPT; - write_backup_status(backup, BACKUP_STATUS_CORRUPT, true); - return; - } - -// if (params && params->partial_db_list) -// dbOid_exclude_list = get_dbOid_exclude_list(backup, files, params->partial_db_list, -// params->partial_restore_type); - - /* setup threads */ - pfilearray_clear_locks(files); - - /* init thread args with own file lists */ - threads = (pthread_t *) palloc(sizeof(pthread_t) * num_threads); - threads_args = (validate_files_arg *) - palloc(sizeof(validate_files_arg) * num_threads); - - /* Validate files */ - thread_interrupted = false; - for (i = 0; i < num_threads; i++) - { - validate_files_arg *arg = &(threads_args[i]); - - arg->base_path = backup->database_dir; - arg->files = files; - arg->corrupted = false; - arg->backup_mode = backup->backup_mode; - arg->stop_lsn = backup->stop_lsn; - arg->checksum_version = backup->checksum_version; - arg->backup_version = parse_program_version(backup->program_version); - arg->external_prefix = external_prefix; - arg->hdr_map = &(backup->hdr_map); -// arg->dbOid_exclude_list = dbOid_exclude_list; - /* By default there are some error */ - threads_args[i].ret = 1; - - pthread_create(&threads[i], NULL, pgBackupValidateFiles, arg); - } - - /* Wait theads */ - for (i = 0; i < num_threads; i++) - { - validate_files_arg *arg = &(threads_args[i]); - - pthread_join(threads[i], NULL); - if (arg->corrupted) - corrupted = true; - if (arg->ret == 1) - validation_isok = false; - } - if (!validation_isok) - elog(ERROR, "Data files validation failed"); - - pfree(threads); - pfree(threads_args); - - /* cleanup */ - parray_walk(files, pgFileFree); - parray_free(files); - cleanup_header_map(&(backup->hdr_map)); - - /* Update backup status */ - if (corrupted) - backup->status = BACKUP_STATUS_CORRUPT; - - write_backup_status(backup, corrupted ? BACKUP_STATUS_CORRUPT : - BACKUP_STATUS_OK, true); - - if (corrupted) - elog(WARNING, "Backup %s data files are corrupted", backup_id_of(backup)); - else - elog(INFO, "Backup %s data files are valid", backup_id_of(backup)); - - /* Issue #132 kludge */ - if (!corrupted && - ((parse_program_version(backup->program_version) == 20104)|| - (parse_program_version(backup->program_version) == 20105)|| - (parse_program_version(backup->program_version) == 20201))) - { - char path[MAXPGPATH]; - - join_path_components(path, backup->root_dir, DATABASE_FILE_LIST); - - if (pgFileSize(path) >= (BLCKSZ*500)) - { - elog(WARNING, "Backup %s is a victim of metadata corruption. " - "Additional information can be found here: " - "https://github.com/postgrespro/pg_probackup/issues/132", - backup_id_of(backup)); - backup->status = BACKUP_STATUS_CORRUPT; - write_backup_status(backup, BACKUP_STATUS_CORRUPT, true); - } - } -} - -/* - * Validate files in the backup. - * NOTE: If file is not valid, do not use ERROR log message, - * rather throw a WARNING and set arguments->corrupted = true. - * This is necessary to update backup status. - */ -static void * -pgBackupValidateFiles(void *arg) -{ - int i; - validate_files_arg *arguments = (validate_files_arg *)arg; - int num_files = parray_num(arguments->files); - pg_crc32 crc; - - for (i = 0; i < num_files; i++) - { - struct stat st; - pgFile *file = (pgFile *) parray_get(arguments->files, i); - char file_fullpath[MAXPGPATH]; - - if (interrupted || thread_interrupted) - elog(ERROR, "Interrupted during validate"); - - /* Validate only regular files */ - if (!S_ISREG(file->mode)) - continue; - - /* - * If in partial validate, check if the file belongs to the database - * we exclude. Only files from pgdata can be skipped. - */ - //if (arguments->dbOid_exclude_list && file->external_dir_num == 0 - // && parray_bsearch(arguments->dbOid_exclude_list, - // &file->dbOid, pgCompareOid)) - //{ - // elog(VERBOSE, "Skip file validation due to partial restore: \"%s\"", - // file->rel_path); - // continue; - //} - - if (!pg_atomic_test_set_flag(&file->lock)) - continue; - - if (progress) - elog(INFO, "Progress: (%d/%d). Validate file \"%s\"", - i + 1, num_files, file->rel_path); - - /* - * Skip files which has no data, because they - * haven't changed between backups. - */ - if (file->write_size == BYTES_INVALID) - { - /* TODO: lookup corresponding merge bug */ - if (arguments->backup_mode == BACKUP_MODE_FULL) - { - /* It is illegal for file in FULL backup to have BYTES_INVALID */ - elog(WARNING, "Backup file \"%s\" has invalid size. Possible metadata corruption.", - file->rel_path); - arguments->corrupted = true; - break; - } - else - continue; - } - - /* no point in trying to open empty file */ - if (file->write_size == 0) - continue; - - if (file->external_dir_num) - { - char temp[MAXPGPATH]; - - makeExternalDirPathByNum(temp, arguments->external_prefix, file->external_dir_num); - join_path_components(file_fullpath, temp, file->rel_path); - } - else - join_path_components(file_fullpath, arguments->base_path, file->rel_path); - - /* TODO: it is redundant to check file existence using stat */ - if (stat(file_fullpath, &st) == -1) - { - if (errno == ENOENT) - elog(WARNING, "Backup file \"%s\" is not found", file_fullpath); - else - elog(WARNING, "Cannot stat backup file \"%s\": %s", - file_fullpath, strerror(errno)); - arguments->corrupted = true; - break; - } - - if (file->write_size != st.st_size) - { - elog(WARNING, "Invalid size of backup file \"%s\" : " INT64_FORMAT ". Expected %lu", - file_fullpath, (unsigned long) st.st_size, file->write_size); - arguments->corrupted = true; - break; - } - - /* - * If option skip-block-validation is set, compute only file-level CRC for - * datafiles, otherwise check them block by block. - * Currently we don't compute checksums for - * cfs_compressed data files, so skip block validation for them. - */ - if (!file->is_datafile || skip_block_validation || file->is_cfs) - { - /* - * Pre 2.0.22 we use CRC-32C, but in newer version of pg_probackup we - * use CRC-32. - * - * pg_control stores its content and checksum of the content, calculated - * using CRC-32C. If we calculate checksum of the whole pg_control using - * CRC-32C we get same checksum constantly. It might be because of the - * CRC-32C algorithm. - * To avoid this problem we need to use different algorithm, CRC-32 in - * this case. - * - * Starting from 2.0.25 we calculate crc of pg_control differently. - */ - if (arguments->backup_version >= 20025 && - strcmp(file->name, "pg_control") == 0 && - !file->external_dir_num) - crc = get_pgcontrol_checksum(arguments->base_path); - else - crc = pgFileGetCRC(file_fullpath, - arguments->backup_version <= 20021 || - arguments->backup_version >= 20025, - false); - if (crc != file->crc) - { - elog(WARNING, "Invalid CRC of backup file \"%s\" : %X. Expected %X", - file_fullpath, crc, file->crc); - arguments->corrupted = true; - } - } - else - { - /* - * validate relation block by block - * check page headers, checksums (if enabled) - * and compute checksum of the file - */ - if (!validate_file_pages(file, file_fullpath, arguments->stop_lsn, - arguments->checksum_version, - arguments->backup_version, - arguments->hdr_map)) - arguments->corrupted = true; - } - } - - /* Data files validation is successful */ - arguments->ret = 0; - - return NULL; -} - -/* - * Validate all backups in the backup catalog. - * If --instance option was provided, validate only backups of this instance. - * - * TODO: split into two functions: do_validate_catalog and do_validate_instance. - */ -int -do_validate_all(CatalogState *catalogState, InstanceState *instanceState) -{ - corrupted_backup_found = false; - skipped_due_to_lock = false; - - if (instanceState == NULL) - { - /* Show list of instances */ - DIR *dir; - struct dirent *dent; - - /* open directory and list contents */ - dir = opendir(catalogState->backup_subdir_path); - if (dir == NULL) - elog(ERROR, "Cannot open directory \"%s\": %s", catalogState->backup_subdir_path, strerror(errno)); - - errno = 0; - while ((dent = readdir(dir))) - { - char child[MAXPGPATH]; - struct stat st; - - /* skip entries point current dir or parent dir */ - if (strcmp(dent->d_name, ".") == 0 || - strcmp(dent->d_name, "..") == 0) - continue; - - join_path_components(child, catalogState->backup_subdir_path, dent->d_name); - - if (lstat(child, &st) == -1) - elog(ERROR, "Cannot stat file \"%s\": %s", child, strerror(errno)); - - if (!S_ISDIR(st.st_mode)) - continue; - - /* - * Initialize instance configuration. - */ - instanceState = pgut_new(InstanceState); /* memory leak */ - strncpy(instanceState->instance_name, dent->d_name, MAXPGPATH); - - join_path_components(instanceState->instance_backup_subdir_path, - catalogState->backup_subdir_path, instanceState->instance_name); - join_path_components(instanceState->instance_wal_subdir_path, - catalogState->wal_subdir_path, instanceState->instance_name); - join_path_components(instanceState->instance_config_path, - instanceState->instance_backup_subdir_path, BACKUP_CATALOG_CONF_FILE); - - if (config_read_opt(instanceState->instance_config_path, instance_options, ERROR, false, - true) == 0) - { - elog(WARNING, "Configuration file \"%s\" is empty", instanceState->instance_config_path); - corrupted_backup_found = true; - continue; - } - - do_validate_instance(instanceState); - } - } - else - { - do_validate_instance(instanceState); - } - - /* TODO: Probably we should have different exit code for every condition - * and they combination: - * 0 - all backups are valid - * 1 - some backups are corrupt - * 2 - some backups where skipped due to concurrent locks - * 3 - some backups are corrupt and some are skipped due to concurrent locks - */ - - if (skipped_due_to_lock) - elog(WARNING, "Some backups weren't locked and they were skipped"); - - if (corrupted_backup_found) - { - elog(WARNING, "Some backups are not valid"); - return 1; - } - - if (!skipped_due_to_lock && !corrupted_backup_found) - elog(INFO, "All backups are valid"); - - return 0; -} - -/* - * Validate all backups in the given instance of the backup catalog. - */ -static void -do_validate_instance(InstanceState *instanceState) -{ - int i; - int j; - parray *backups; - pgBackup *current_backup = NULL; - - elog(INFO, "Validate backups of the instance '%s'", instanceState->instance_name); - - /* Get list of all backups sorted in order of descending start time */ - backups = catalog_get_backup_list(instanceState, INVALID_BACKUP_ID); - - /* Examine backups one by one and validate them */ - for (i = 0; i < parray_num(backups); i++) - { - pgBackup *base_full_backup; - - current_backup = (pgBackup *) parray_get(backups, i); - - /* Find ancestor for incremental backup */ - if (current_backup->backup_mode != BACKUP_MODE_FULL) - { - pgBackup *tmp_backup = NULL; - int result; - - result = scan_parent_chain(current_backup, &tmp_backup); - - /* chain is broken */ - if (result == ChainIsBroken) - { - const char *parent_backup_id; - const char *current_backup_id; - /* determine missing backup ID */ - - parent_backup_id = base36enc(tmp_backup->parent_backup); - current_backup_id = backup_id_of(current_backup); - corrupted_backup_found = true; - - /* orphanize current_backup */ - if (current_backup->status == BACKUP_STATUS_OK || - current_backup->status == BACKUP_STATUS_DONE) - { - write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, true); - elog(WARNING, "Backup %s is orphaned because his parent %s is missing", - current_backup_id, parent_backup_id); - } - else - { - elog(WARNING, "Backup %s has missing parent %s", - current_backup_id, parent_backup_id); - } - continue; - } - /* chain is whole, but at least one parent is invalid */ - else if (result == ChainIsInvalid) - { - /* Oldest corrupt backup has a chance for revalidation */ - if (current_backup->start_time != tmp_backup->start_time) - { - /* orphanize current_backup */ - if (current_backup->status == BACKUP_STATUS_OK || - current_backup->status == BACKUP_STATUS_DONE) - { - write_backup_status(current_backup, BACKUP_STATUS_ORPHAN, true); - elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", - backup_id_of(current_backup), - backup_id_of(tmp_backup), - status2str(tmp_backup->status)); - } - else - { - elog(WARNING, "Backup %s has parent %s with status: %s", - backup_id_of(current_backup), - backup_id_of(tmp_backup), - status2str(tmp_backup->status)); - } - continue; - } - base_full_backup = find_parent_full_backup(current_backup); - - /* sanity */ - if (!base_full_backup) - elog(ERROR, "Parent full backup for the given backup %s was not found", - backup_id_of(current_backup)); - } - /* chain is whole, all parents are valid at first glance, - * current backup validation can proceed - */ - else - base_full_backup = tmp_backup; - } - else - base_full_backup = current_backup; - - /* Do not interrupt, validate the next backup */ - if (!lock_backup(current_backup, true, false)) - { - elog(WARNING, "Cannot lock backup %s directory, skip validation", - backup_id_of(current_backup)); - skipped_due_to_lock = true; - continue; - } - /* Valiate backup files*/ - pgBackupValidate(current_backup, NULL); - - /* Validate corresponding WAL files */ - if (current_backup->status == BACKUP_STATUS_OK) - validate_wal(current_backup, instanceState->instance_wal_subdir_path, 0, - 0, 0, current_backup->tli, - instance_config.xlog_seg_size); - - /* - * Mark every descendant of corrupted backup as orphan - */ - if (current_backup->status != BACKUP_STATUS_OK) - { - /* This is ridiculous but legal. - * PAGE_b2 <- OK - * PAGE_a2 <- OK - * PAGE_b1 <- ORPHAN - * PAGE_a1 <- CORRUPT - * FULL <- OK - */ - - corrupted_backup_found = true; - - for (j = i - 1; j >= 0; j--) - { - pgBackup *backup = (pgBackup *) parray_get(backups, j); - - if (is_parent(current_backup->start_time, backup, false)) - { - if (backup->status == BACKUP_STATUS_OK || - backup->status == BACKUP_STATUS_DONE) - { - write_backup_status(backup, BACKUP_STATUS_ORPHAN, true); - - elog(WARNING, "Backup %s is orphaned because his parent %s has status: %s", - backup_id_of(backup), - backup_id_of(current_backup), - status2str(current_backup->status)); - } - } - } - } - - /* For every OK backup we try to revalidate all his ORPHAN descendants. */ - if (current_backup->status == BACKUP_STATUS_OK) - { - /* revalidate all ORPHAN descendants - * be very careful not to miss a missing backup - * for every backup we must check that he is descendant of current_backup - */ - for (j = i - 1; j >= 0; j--) - { - pgBackup *backup = (pgBackup *) parray_get(backups, j); - pgBackup *tmp_backup = NULL; - int result; - - //PAGE_b2 ORPHAN - //PAGE_b1 ORPHAN ----- - //PAGE_a5 ORPHAN | - //PAGE_a4 CORRUPT | - //PAGE_a3 missing | - //PAGE_a2 missing | - //PAGE_a1 ORPHAN | - //PAGE OK <- we are here<-| - //FULL OK - - if (is_parent(current_backup->start_time, backup, false)) - { - /* Revalidation make sense only if parent chain is whole. - * is_parent() do not guarantee that. - */ - result = scan_parent_chain(backup, &tmp_backup); - - if (result == ChainIsInvalid) - { - /* revalidation make sense only if oldest invalid backup is current_backup - */ - - if (tmp_backup->start_time != backup->start_time) - continue; - - if (backup->status == BACKUP_STATUS_ORPHAN) - { - /* Do not interrupt, validate the next backup */ - if (!lock_backup(backup, true, false)) - { - elog(WARNING, "Cannot lock backup %s directory, skip validation", - backup_id_of(backup)); - skipped_due_to_lock = true; - continue; - } - /* Revalidate backup files*/ - pgBackupValidate(backup, NULL); - - if (backup->status == BACKUP_STATUS_OK) - { - - /* Revalidation successful, validate corresponding WAL files */ - validate_wal(backup, instanceState->instance_wal_subdir_path, 0, - 0, 0, backup->tli, - instance_config.xlog_seg_size); - } - } - - if (backup->status != BACKUP_STATUS_OK) - { - corrupted_backup_found = true; - continue; - } - } - } - } - } - } - - /* cleanup */ - parray_walk(backups, pgBackupFree); - parray_free(backups); -} - -/* - * Validate tablespace_map checksum. - * Error out in case of checksum mismatch. - * Return 'false' if there are no tablespaces in backup. - * - * TODO: it is a bad, that we read the whole filelist just for - * the sake of tablespace_map. Probably pgBackup should come with - * already filled pgBackup.files - */ -bool -validate_tablespace_map(pgBackup *backup, bool no_validate) -{ - char map_path[MAXPGPATH]; - pgFile *dummy = NULL; - pgFile **tablespace_map = NULL; - pg_crc32 crc; - parray *files = get_backup_filelist(backup, true); - bool use_crc32c = parse_program_version(backup->program_version) <= 20021 || - parse_program_version(backup->program_version) >= 20025; - - parray_qsort(files, pgFileCompareRelPathWithExternal); - join_path_components(map_path, backup->database_dir, PG_TABLESPACE_MAP_FILE); - - dummy = pgFileInit(PG_TABLESPACE_MAP_FILE); - tablespace_map = (pgFile **) parray_bsearch(files, dummy, pgFileCompareRelPathWithExternal); - - if (!tablespace_map) - { - elog(LOG, "there is no file tablespace_map"); - parray_walk(files, pgFileFree); - parray_free(files); - return false; - } - - /* Exit if database/tablespace_map doesn't exist */ - if (!fileExists(map_path, FIO_BACKUP_HOST)) - elog(ERROR, "Tablespace map is missing: \"%s\", " - "probably backup %s is corrupt, validate it", - map_path, backup_id_of(backup)); - - /* check tablespace map checksumms */ - if (!no_validate) - { - crc = pgFileGetCRC(map_path, use_crc32c, false); - - if ((*tablespace_map)->crc != crc) - elog(ERROR, "Invalid CRC of tablespace map file \"%s\" : %X. Expected %X, " - "probably backup %s is corrupt, validate it", - map_path, crc, (*tablespace_map)->crc, backup_id_of(backup)); - } - - pgFileFree(dummy); - parray_walk(files, pgFileFree); - parray_free(files); - return true; -} diff --git a/tests/archive_test.py b/tests/archive_test.py index 034223aa4..5cc744ea1 100644 --- a/tests/archive_test.py +++ b/tests/archive_test.py @@ -7,6 +7,7 @@ import subprocess from sys import exit from time import sleep +from pathlib import PurePath from testgres import ProcessType diff --git a/tests/auth_test.py b/tests/auth_test.py index d1a7c707c..0a8ee5909 100644 --- a/tests/auth_test.py +++ b/tests/auth_test.py @@ -1,20 +1,348 @@ +""" +The Test suite check behavior of pg_probackup utility, if password is required for connection to PostgreSQL instance. + - https://confluence.postgrespro.ru/pages/viewpage.action?pageId=16777522 +""" + +import os +import unittest +import signal +import time + from .helpers.ptrack_helpers import ProbackupTest +from testgres import StartNodeException + +skip_test = False + +try: + from pexpect import * +except ImportError: + skip_test = True + + +class SimpleAuthTest(ProbackupTest): + + # @unittest.skip("skip") + def test_backup_via_unprivileged_user(self): + """ + Make node, create unprivileged user, try to + run a backups without EXECUTE rights on + certain functions + """ + node = self.pg_node.make_simple('node', + set_replication=True, + ptrack_enable=self.ptrack) + + self.pb.init() + self.pb.add_instance('node', node) + self.pb.set_archiving('node', node) + node.slow_start() + + if self.ptrack: + node.safe_psql( + "postgres", + "CREATE EXTENSION ptrack") + + node.safe_psql("postgres", "CREATE ROLE backup with LOGIN") + + self.pb.backup_node('node', node, options=['-U', 'backup'], + expect_error='due to missing grant on EXECUTE') + if self.pg_config_version < 150000: + self.assertMessage(contains= + "ERROR: Query failed: ERROR: permission denied " + "for function pg_start_backup") + else: + self.assertMessage(contains= + "ERROR: Query failed: ERROR: permission denied " + "for function pg_backup_start") + + if self.pg_config_version < 150000: + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION" + " pg_start_backup(text, boolean, boolean) TO backup;") + else: + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION" + " pg_backup_start(text, boolean) TO backup;") + + node.safe_psql( + 'postgres', + "GRANT EXECUTE ON FUNCTION pg_catalog.pg_switch_wal() TO backup") + + self.pb.backup_node('node', node, + options=['-U', 'backup'], + expect_error='due to missing grant on EXECUTE') + self.assertMessage(contains= + "ERROR: Query failed: ERROR: permission denied for function " + "pg_create_restore_point\nquery was: " + "SELECT pg_catalog.pg_create_restore_point($1)") + + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION" + " pg_create_restore_point(text) TO backup;") + + self.pb.backup_node('node', node, + options=['-U', 'backup'], + expect_error='due to missing grant on EXECUTE') + if self.pg_config_version < 150000: + self.assertMessage(contains= + "ERROR: Query failed: ERROR: permission denied " + "for function pg_stop_backup") + else: + self.assertMessage(contains= + "ERROR: Query failed: ERROR: permission denied " + "for function pg_backup_stop") + + if self.pg_config_version < self.version_to_num('15.0'): + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean, boolean) TO backup;") + else: + node.safe_psql( + "postgres", + "GRANT EXECUTE ON FUNCTION pg_backup_stop(boolean) TO backup;") + + self.pb.backup_node('node', node, options=['-U', 'backup']) + + node.safe_psql("postgres", "CREATE DATABASE test1") + + self.pb.backup_node('node', node, options=['-U', 'backup']) + + node.safe_psql( + "test1", "create table t1 as select generate_series(0,100)") + + node.stop() + node.slow_start() + + node.safe_psql( + "postgres", + "ALTER ROLE backup REPLICATION") + + # FULL + self.pb.backup_node('node', node, options=['-U', 'backup']) + + # PTRACK + if self.ptrack: + self.pb.backup_node('node', node, + backup_type='ptrack', options=['-U', 'backup']) + + +class AuthTest(ProbackupTest): + pb = None + node = None + + # TODO move to object scope, replace module_name + @unittest.skipIf(skip_test, "Module pexpect isn't installed. You need to install it.") + def setUp(self): + + super().setUp() + + self.node = self.pg_node.make_simple("node", + set_replication=True, + initdb_params=['--auth-host=md5'], + pg_options={'archive_timeout': '5s'}, + ) + + self.modify_pg_hba(self.node) + + self.pb.init() + self.pb.add_instance(self.node.name, self.node) + self.pb.set_archiving(self.node.name, self.node) + try: + self.node.slow_start() + except StartNodeException: + raise unittest.skip("Node hasn't started") + + + version = self.pg_config_version + if version < 150000: + self.node.safe_psql( + "postgres", + "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_start_backup(text, boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_stop_backup() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_stop_backup(boolean, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") + else: + self.node.safe_psql( + "postgres", + "CREATE ROLE backup WITH LOGIN PASSWORD 'password'; " + "GRANT USAGE ON SCHEMA pg_catalog TO backup; " + "GRANT EXECUTE ON FUNCTION current_setting(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_is_in_recovery() TO backup; " + "GRANT EXECUTE ON FUNCTION pg_backup_start(text, boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_backup_stop(boolean) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_create_restore_point(text) TO backup; " + "GRANT EXECUTE ON FUNCTION pg_switch_wal() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_current_snapshot() TO backup; " + "GRANT EXECUTE ON FUNCTION txid_snapshot_xmax(txid_snapshot) TO backup;") + + if version >= 150000: + home_dir = os.path.join(self.test_path, "home") + os.makedirs(home_dir, exist_ok=True) + self.test_env['HOME'] = home_dir + self.pgpass_file = os.path.join(home_dir, '.pgpass') + self.pgpass_file_lock = None + else: + # before PGv15 only true home dir were inspected. + # Since we can't have separate file per test, we have to serialize + # tests. + self.pgpass_file = os.path.join(os.path.expanduser('~'), '.pgpass') + self.pgpass_file_lock = self.pgpass_file + '~probackup_test_lock' + # have to lock pgpass by creating file in exclusive mode + for i in range(120): + try: + open(self.pgpass_file_lock, "x").close() + except FileExistsError: + time.sleep(1) + else: + break + else: + raise TimeoutError("can't create ~/.pgpass~probackup_test_lock for 120 seconds") + + self.pb_cmd = ['backup', + '--instance', self.node.name, + '-h', '127.0.0.1', + '-p', str(self.node.port), + '-U', 'backup', + '-d', 'postgres', + '-b', 'FULL', + '--no-sync' + ] + + def tearDown(self): + super().tearDown() + if not self.pgpass_file_lock: + return + if hasattr(self, "pgpass_line") and os.path.exists(self.pgpass_file): + with open(self.pgpass_file, 'r') as fl: + lines = fl.readlines() + if self.pgpass_line in lines: + lines.remove(self.pgpass_line) + if len(lines) == 0: + os.remove(self.pgpass_file) + else: + with open(self.pgpass_file, 'w') as fl: + fl.writelines(lines) + os.remove(self.pgpass_file_lock) + + def test_empty_password(self): + """ Test case: PGPB_AUTH03 - zero password length """ + try: + self.assertIn("ERROR: no password supplied", + self.run_pb_with_auth('\0\r\n')) + except (TIMEOUT, ExceptionPexpect) as e: + self.fail(e.value) + + def test_wrong_password(self): + """ Test case: PGPB_AUTH04 - incorrect password """ + self.assertIn("password authentication failed", + self.run_pb_with_auth('wrong_password\r\n')) + + def test_right_password(self): + """ Test case: PGPB_AUTH01 - correct password """ + self.assertIn("completed", + self.run_pb_with_auth('password\r\n')) + + def test_right_password_and_wrong_pgpass(self): + """ Test case: PGPB_AUTH05 - correct password and incorrect .pgpass (-W)""" + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'wrong_password']) + self.create_pgpass(self.pgpass_file, line) + self.assertIn("completed", + self.run_pb_with_auth('password\r\n', add_args=["-W"])) + + def test_ctrl_c_event(self): + """ Test case: PGPB_AUTH02 - send interrupt signal """ + try: + self.run_pb_with_auth(kill=True) + except TIMEOUT: + self.fail("Error: CTRL+C event ignored") + + def test_pgpassfile_env(self): + """ Test case: PGPB_AUTH06 - set environment var PGPASSFILE """ + path = os.path.join(self.test_path, 'pgpass.conf') + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'password']) + self.create_pgpass(path, line) + self.test_env["PGPASSFILE"] = path + self.assertEqual( + "OK", + self.pb.show(self.node.name, self.pb.run(self.pb_cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + + def test_pgpass(self): + """ Test case: PGPB_AUTH07 - Create file .pgpass in home dir. """ + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'password']) + self.create_pgpass(self.pgpass_file, line) + self.assertEqual( + "OK", + self.pb.show(self.node.name, self.pb.run(self.pb_cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + + def test_pgpassword(self): + """ Test case: PGPB_AUTH08 - set environment var PGPASSWORD """ + self.test_env["PGPASSWORD"] = "password" + self.assertEqual( + "OK", + self.pb.show(self.node.name, self.pb.run(self.pb_cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + + def test_pgpassword_and_wrong_pgpass(self): + """ Test case: PGPB_AUTH09 - Check priority between PGPASSWORD and .pgpass file""" + line = ":".join(['127.0.0.1', str(self.node.port), 'postgres', 'backup', 'wrong_password']) + self.create_pgpass(self.pgpass_file, line) + self.test_env["PGPASSWORD"] = "password" + self.assertEqual( + "OK", + self.pb.show(self.node.name, self.pb.run(self.pb_cmd + ['-w']))["status"], + "ERROR: Full backup status is not valid." + ) + def run_pb_with_auth(self, password=None, add_args = [], kill=False): + cmd = [*self.pb_cmd, *add_args, *self.backup_dir.pb_args] + with spawn(self.probackup_path, cmd, + encoding='utf-8', timeout=60, env=self.test_env) as probackup: + result = probackup.expect(u"Password for user .*:", 10) + if kill: + probackup.kill(signal.SIGINT) + elif result == 0: + probackup.sendline(password) + probackup.expect(EOF) + return str(probackup.before) + else: + raise ExceptionPexpect("Other pexpect errors.") -class AuthorizationTest(ProbackupTest): - """ - Check connect to S3 via pre_start_checks() function - calling pg_probackup init --s3 - test that s3 keys allow to connect to all types of storages - """ + def modify_pg_hba(self, node): + """ + Description: + Add trust authentication for user postgres. Need for add new role and set grant. + :param node: + :return None: + """ + hba_conf = os.path.join(node.data_dir, "pg_hba.conf") + with open(hba_conf, 'r+') as fio: + data = fio.read() + fio.seek(0) + fio.write('host\tall\t%s\t127.0.0.1/0\ttrust\n%s' % (self.username, data)) - def test_s3_auth_test(self): - console_output = self.pb.init(options=["--log-level-console=VERBOSE"]) - self.assertNotIn(': 403', console_output) # Because we can have just '403' substring in timestamp - self.assertMessage(console_output, contains='S3_pre_start_check successful') - self.assertMessage(console_output, contains='HTTP response: 200') - self.assertIn( - f"INFO: Backup catalog '{self.backup_dir}' successfully initialized", - console_output) + def create_pgpass(self, path, line): + self.pgpass_line = line+"\n" + with open(path, 'a') as passfile: + # host:port:db:username:password + passfile.write(self.pgpass_line) + os.chmod(path, 0o600) diff --git a/tests/backup_test.py b/tests/backup_test.py index 2e0695b6c..bc90636a2 100644 --- a/tests/backup_test.py +++ b/tests/backup_test.py @@ -933,19 +933,19 @@ def test_persistent_slot_for_stream_backup(self): "postgres", "SELECT pg_create_physical_replication_slot('slot_1')") - # FULL backup. By default, --temp-slot=true. + # FULL backup self.pb.backup_node('node', node, - options=['--stream', '--slot=slot_1'], - expect_error="because replication slot already exist") + options=['--stream', '--slot=slot_1', '--temp-slot'], + expect_error="because a replication slot with this name already exists") self.assertMessage(contains='ERROR: replication slot "slot_1" already exists') # FULL backup self.pb.backup_node('node', node, - options=['--stream', '--slot=slot_1', '--temp-slot=false']) + options=['--stream', '--slot=slot_1']) # FULL backup self.pb.backup_node('node', node, - options=['--stream', '--slot=slot_1', '--temp-slot=false']) + options=['--stream', '--slot=slot_1']) # @unittest.skip("skip") def test_basic_temp_slot_for_stream_backup(self): @@ -964,9 +964,9 @@ def test_basic_temp_slot_for_stream_backup(self): self.pb.backup_node('node', node, options=['--stream', '--temp-slot']) - # FULL backup. By default, --temp-slot=true. + # FULL backup self.pb.backup_node('node', node, - options=['--stream', '--slot=slot_1']) + options=['--stream', '--slot=slot_1', '--temp-slot=on']) # FULL backup self.pb.backup_node('node', node, diff --git a/tests/catchup_test.py b/tests/catchup_test.py index 117ac0407..148a670f3 100644 --- a/tests/catchup_test.py +++ b/tests/catchup_test.py @@ -1201,7 +1201,7 @@ def test_catchup_with_replication_slot(self): destination_node = dst_pg, options = [ '-d', 'postgres', '-p', str(src_pg.port), '--stream', - '--slot=nonexistentslot_1a', '--temp-slot=false' + '--slot=nonexistentslot_1a' ], expect_error="because replication slot does not exist" ) @@ -1216,7 +1216,7 @@ def test_catchup_with_replication_slot(self): destination_node = dst_pg, options = [ '-d', 'postgres', '-p', str(src_pg.port), '--stream', - '--slot=existentslot_1b', '--temp-slot=false' + '--slot=existentslot_1b' ] ) @@ -2060,3 +2060,42 @@ def test_waldir_dry_run_catchup_full(self): # Cleanup src_pg.stop() + def test_custom_replication_slot(self): + # preparation + my_slot = "my_slot" + + src_pg = self.pg_node.make_simple('src', + set_replication=True + ) + src_pg.slow_start() + src_pg.safe_psql( + "postgres", + "CREATE TABLE ultimate_question AS SELECT 42 AS answer") + src_pg.safe_psql("postgres", f"SELECT * FROM pg_create_physical_replication_slot('{my_slot}');") + + src_query_result = src_pg.table_checksum("ultimate_question") + + # do full catchup + dst_pg = self.pg_node.make_empty('dst') + self.pb.catchup_node( + backup_mode='FULL', + source_pgdata=src_pg.data_dir, + destination_node=dst_pg, + options=['-d', 'postgres', '-p', str(src_pg.port), '--stream', '-S', my_slot] + ) + + # 1st check: compare data directories + self.compare_pgdata( + self.pgdata_content(src_pg.data_dir, exclude_dirs=['pg_replslot']), + self.pgdata_content(dst_pg.data_dir, exclude_dirs=['pg_replslot']), + ) + + # run&recover catchup'ed instance + src_pg.stop() + dst_options = {'port': str(dst_pg.port)} + dst_pg.set_auto_conf(dst_options) + dst_pg.slow_start() + + # 2nd check: run verification query + dst_query_result = dst_pg.table_checksum("ultimate_question") + self.assertEqual(src_query_result, dst_query_result, 'Different answer from copy') diff --git a/tests/delete_test.py b/tests/delete_test.py index 761aa36f3..111a48d60 100644 --- a/tests/delete_test.py +++ b/tests/delete_test.py @@ -778,15 +778,10 @@ def test_basic_dry_run_del_instance(self): self.pb.init() self.pb.add_instance('node', node) - self.pb.set_archiving('node', node) node.slow_start() # full backup - self.pb.backup_node('node', node) - # restore - node.cleanup() - self.pb.restore_node('node', node=node) - node.slow_start() + self.pb.backup_node('node', node, options=['--stream']) content_before = self.pgdata_content(self.backup_dir) # Delete instance diff --git a/tests/helpers/ptrack_helpers.py b/tests/helpers/ptrack_helpers.py index f06629012..f9695d3f8 100644 --- a/tests/helpers/ptrack_helpers.py +++ b/tests/helpers/ptrack_helpers.py @@ -875,8 +875,11 @@ def __init__(self, data): self.data = data @contextlib.contextmanager - def modify_backup_control(self, backup_dir, instance, backup_id): - path = os.path.join('backups', instance, backup_id, 'backup.control') + def modify_backup_control(self, backup_dir, instance, backup_id, content=False): + file = 'backup.control' + if content: + file = 'backup_content.control' + path = os.path.join('backups', instance, backup_id, file) control_file = backup_dir.read_file(path) cf = ProbackupTest.ControlFileContainer(control_file) yield cf diff --git a/tests/logging_test.py b/tests/logging_test.py index 85e646c1e..e2767d923 100644 --- a/tests/logging_test.py +++ b/tests/logging_test.py @@ -92,7 +92,7 @@ def test_truncate_rotation_file(self): output = self.pb.backup_node('node', node, options=[ '--stream', - '--log-level-file=LOG'], + '--log-level-file=INFO'], return_id=False) # check that log file wasn`t rotated @@ -152,7 +152,7 @@ def test_unlink_rotation_file(self): output = self.pb.backup_node('node', node, options=[ '--stream', - '--log-level-file=LOG'], + '--log-level-file=INFO'], return_id=False) # check that log file wasn`t rotated @@ -211,7 +211,7 @@ def test_garbage_in_rotation_file(self): output = self.pb.backup_node('node', node, options=[ '--stream', - '--log-level-file=LOG'], + '--log-level-file=INFO'], return_id=False) # check that log file wasn`t rotated diff --git a/tests/option_test.py b/tests/option_test.py index 89c5c52e0..a66331822 100644 --- a/tests/option_test.py +++ b/tests/option_test.py @@ -21,7 +21,7 @@ def test_without_backup_path_3(self): self.pb.run(["backup", "-b", "full"], expect_error="because '-B' parameter is not specified", use_backup_dir=None) self.assertMessage(contains="No backup catalog path specified.\n" - "Please specify it either using environment variable BACKUP_DIR or\n" + "Please specify it either using environment variable BACKUP_PATH or\n" "command line option --backup-path (-B)") def test_options_4(self): diff --git a/tests/page_test.py b/tests/page_test.py index 10959bd8b..780466684 100644 --- a/tests/page_test.py +++ b/tests/page_test.py @@ -731,22 +731,32 @@ def test_page_backup_with_alien_wal_segment(self): "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,10000) i;") - alien_node.safe_psql( + alien_node.execute( "postgres", "create database alien") - alien_node.safe_psql( + wal_file = alien_node.execute( + "alien", + "SELECT pg_walfile_name(pg_current_wal_lsn());" + ) + filename = wal_file[0][0] + self.compress_suffix + + alien_node.execute( "alien", "create sequence t_seq; " "create table t_heap_alien as select i as id, " "md5(i::text) as text, " "md5(repeat(i::text,10))::tsvector as tsvector " "from generate_series(0,10000) i;") + alien_node.execute( + "alien", + "select pg_switch_wal()") + node.execute( + "postgres", + "select pg_switch_wal()") - # copy latest wal segment - wals = self.get_instance_wal_list(backup_dir, 'alien_node') - filename = max(wals) - # wait `node` did archived same file + # wait nodes archived same file + self.wait_instance_wal_exists(backup_dir, 'alien_node', filename) self.wait_instance_wal_exists(backup_dir, 'node', filename) file_content = self.read_instance_wal(backup_dir, 'alien_node', filename) self.write_instance_wal(backup_dir, 'node', filename, file_content) diff --git a/tests/pbckp1242_test.py b/tests/pbckp1242_test.py new file mode 100644 index 000000000..7e55f460f --- /dev/null +++ b/tests/pbckp1242_test.py @@ -0,0 +1,662 @@ +import unittest +import os +import re +from time import sleep, time +from datetime import datetime + +from pg_probackup2.gdb import needs_gdb + +from .helpers.ptrack_helpers import base36enc, ProbackupTest +from .helpers.ptrack_helpers import fs_backup_class +import subprocess + +tblspace_name = 'some_tblspace' + +class Pbckp1242Test(ProbackupTest): + + def setup_node(self): + node = self.pg_node.make_simple( + "node", + set_replication=True, + initdb_params=['--data-checksums'] + ) + self.pb.init() + self.pb.add_instance( 'node', node) + node.slow_start() + return node + + def jump_the_oid(self, node): + pg_connect = node.connect("postgres", autocommit=True) + gdb = self.gdb_attach(pg_connect.pid) + gdb._execute('set ShmemVariableCache->nextOid=1<<31') + gdb._execute('set ShmemVariableCache->oidCount=0') + gdb.detach() + + @needs_gdb + def test_table_with_giga_oid(self): + node = self.setup_node() + self.jump_the_oid(node) + + node.execute(f'CREATE TABLE t1 (i int)') + node.execute('INSERT INTO t1 (i) SELECT generate_series(1, 1000)') + node.execute('CHECKPOINT') + + table1_checksum = node.table_checksum('t1') + + backup_id = self.pb.backup_node('node', node, backup_type='full', + options=['--stream']) + + node.stop() + node.cleanup() + + self.pb.restore_node('node', node, + backup_id=backup_id) + + node.slow_start() + + new_table1_checksum = node.table_checksum('t1') + + self.assertEqual(new_table1_checksum, table1_checksum, "table checksums doesn't match") + + @needs_gdb + def test_database_with_giga_oid(self): + node = self.setup_node() + self.jump_the_oid(node) + + node.execute(f'CREATE DATABASE db2') + + node.execute('db2', f'CREATE TABLE t1 (i int)') + node.execute('db2', 'INSERT INTO t1 (i) SELECT generate_series(1, 1000)') + node.execute('CHECKPOINT') + + table1_checksum = node.table_checksum('t1', 'db2') + + backup_id = self.pb.backup_node('node', node, backup_type='full', + options=['--stream']) + + node.stop() + node.cleanup() + + self.pb.restore_node('node', node, + backup_id=backup_id) + + node.slow_start() + + new_table1_checksum = node.table_checksum('t1', 'db2') + + self.assertEqual(new_table1_checksum, table1_checksum, "table checksums doesn't match") + + @needs_gdb + def test_table_with_giga_oid_in_tablespace(self): + node = self.setup_node() + self.create_tblspace_in_node(node, tblspace_name) + + self.jump_the_oid(node) + + node.execute(f'CREATE TABLE t1 (i int) TABLESPACE {tblspace_name}') + node.execute('INSERT INTO t1 (i) SELECT generate_series(1, 1000)') + node.execute('CHECKPOINT') + + table1_checksum = node.table_checksum('t1') + + backup_id = self.pb.backup_node('node', node, backup_type='full', + options=['--stream']) + + node.stop() + node.cleanup() + + self.pb.restore_node('node', node, + backup_id=backup_id) + + node.slow_start() + + new_table1_checksum = node.table_checksum('t1') + + self.assertEqual(new_table1_checksum, table1_checksum, "table checksums doesn't match") + + + @needs_gdb + def test_database_with_giga_oid_in_tablespace(self): + node = self.setup_node() + self.create_tblspace_in_node(node, tblspace_name) + + self.jump_the_oid(node) + + node.execute(f'CREATE DATABASE db2 TABLESPACE {tblspace_name}') + + node.execute('db2', f'CREATE TABLE t1 (i int)') + node.execute('db2', 'INSERT INTO t1 (i) SELECT generate_series(1, 1000)') + node.execute('CHECKPOINT') + + table1_checksum = node.table_checksum('t1', 'db2') + + backup_id = self.pb.backup_node('node', node, backup_type='full', + options=['--stream']) + + node.stop() + node.cleanup() + + self.pb.restore_node('node', node, + backup_id=backup_id) + + node.slow_start() + + new_table1_checksum = node.table_checksum('t1', 'db2') + + self.assertEqual(new_table1_checksum, table1_checksum, "table checksums doesn't match") + + @needs_gdb + def test_database_with_giga_oid_in_tablespace_2(self): + node = self.setup_node() + self.create_tblspace_in_node(node, tblspace_name) + + self.jump_the_oid(node) + + node.execute(f'CREATE DATABASE db2') + + node.execute('db2', f'CREATE TABLE t1 (i int) TABLESPACE {tblspace_name}') + node.execute('db2', 'INSERT INTO t1 (i) SELECT generate_series(1, 1000)') + node.execute('CHECKPOINT') + + table1_checksum = node.table_checksum('t1', 'db2') + + backup_id = self.pb.backup_node('node', node, backup_type='full', + options=['--stream']) + + node.stop() + node.cleanup() + + self.pb.restore_node('node', node, + backup_id=backup_id) + + node.slow_start() + + new_table1_checksum = node.table_checksum('t1', 'db2') + + self.assertEqual(new_table1_checksum, table1_checksum, "table checksums doesn't match") + + @needs_gdb + def test_detect_database_with_giga_oid_in_tablespace(self): + node = self.setup_node() + self.create_tblspace_in_node(node, tblspace_name) + + self.jump_the_oid(node) + + node.execute(f'CREATE DATABASE db2 TABLESPACE {tblspace_name}') + + node.execute('db2', f'CREATE TABLE t1 (i int)') + node.execute('db2', 'INSERT INTO t1 (i) SELECT generate_series(1, 1000)') + node.execute('CHECKPOINT') + + backup_id = self.pb.backup_node('node', node, backup_type='full', + options=['--stream']) + + node.stop() + node.cleanup() + + self.prepare_backup_for_detect_missed_database(backup_id) + + self.pb.restore_node('node', node, + backup_id=backup_id, + expect_error="database with giga oid") + self.assertMessage(contains="probably has missing files in") + self.assertMessage(contains="were created by misbehaving") + + def test_nodetect_database_without_giga_oid_in_tablespace(self): + node = self.setup_node() + self.create_tblspace_in_node(node, tblspace_name) + + node.execute(f'CREATE DATABASE db2 TABLESPACE {tblspace_name}') + + node.execute('db2', f'CREATE TABLE t1 (i int)') + node.execute('db2', 'INSERT INTO t1 (i) SELECT generate_series(1, 1000)') + node.execute('CHECKPOINT') + + table1_checksum = node.table_checksum('t1', 'db2') + + backup_id = self.pb.backup_node('node', node, backup_type='full', + options=['--stream']) + + node.stop() + node.cleanup() + + self.prepare_backup_for_detect_missed_database(backup_id) + + self.pb.restore_node('node', node, + backup_id=backup_id) + + node.slow_start() + + new_table1_checksum = node.table_checksum('t1', 'db2') + + self.assertEqual(new_table1_checksum, table1_checksum, "table checksums doesn't match") + + @needs_gdb + def test_tablespace_with_giga_oid(self): + node = self.setup_node() + node.execute(f'CREATE DATABASE db2') + + node.execute('db2', f'CREATE TABLE t1 (i int)') + node.execute('db2', 'INSERT INTO t1 (i) SELECT generate_series(1, 1000)') + + table1_checksum = node.table_checksum('t1', 'db2') + + self.jump_the_oid(node) + + self.create_tblspace_in_node(node, tblspace_name) + + node.execute(f'ALTER DATABASE db2 SET TABLESPACE {tblspace_name}') + node.execute('CHECKPOINT') + + backup_id = self.pb.backup_node('node', node, backup_type='full', + options=['--stream']) + + node.stop() + node.cleanup() + + self.pb.restore_node('node', node, + backup_id=backup_id) + + node.slow_start() + + new_table1_checksum = node.table_checksum('t1', 'db2') + + self.assertEqual(new_table1_checksum, table1_checksum, "table checksums doesn't match") + + @needs_gdb + def test_detect_tablespace_with_giga_oid(self): + node = self.setup_node() + node.execute(f'CREATE DATABASE db2') + + node.execute('db2', f'CREATE TABLE t1 (i int)') + node.execute('db2', 'INSERT INTO t1 (i) SELECT generate_series(1, 1000)') + + table1_checksum = node.table_checksum('t1', 'db2') + + self.jump_the_oid(node) + + self.create_tblspace_in_node(node, tblspace_name) + + node.execute(f'ALTER DATABASE db2 SET TABLESPACE {tblspace_name}') + node.execute('CHECKPOINT') + + backup_id = self.pb.backup_node('node', node, backup_type='full', + options=['--stream']) + + node.stop() + node.cleanup() + + self.prepare_backup_for_detect_missed_tablespace(backup_id) + + self.pb.restore_node('node', node, + backup_id=backup_id, + expect_error='tablespace with gigaoid') + + self.assertMessage(contains="has missing tablespace") + self.assertMessage(contains="were created by misbehaving") + + def test_nodetect_tablespace_without_giga_oid(self): + node = self.setup_node() + node.execute(f'CREATE DATABASE db2') + + node.execute('db2', f'CREATE TABLE t1 (i int)') + node.execute('db2', 'INSERT INTO t1 (i) SELECT generate_series(1, 1000)') + + table1_checksum = node.table_checksum('t1', 'db2') + + self.create_tblspace_in_node(node, tblspace_name) + + node.execute(f'ALTER DATABASE db2 SET TABLESPACE {tblspace_name}') + node.execute('CHECKPOINT') + + backup_id = self.pb.backup_node('node', node, backup_type='full', + options=['--stream']) + + node.stop() + node.cleanup() + + self.prepare_backup_for_detect_missed_tablespace(backup_id) + + self.pb.restore_node('node', node, + backup_id=backup_id) + + node.slow_start() + + new_table1_checksum = node.table_checksum('t1', 'db2') + + self.assertEqual(new_table1_checksum, table1_checksum, "table checksums doesn't match") + + @needs_gdb + def test_detect_giga_oid_table(self): + """Detect we couldn't increment based on backup with misdetected file type""" + node = self.setup_node() + self.jump_the_oid(node) + + node.execute(f'CREATE TABLE t1 (i int)') + node.execute('INSERT INTO t1 (i) SELECT generate_series(1, 1000)') + node.execute('CHECKPOINT') + + backup_id = self.pb.backup_node('node', node, backup_type='full', + options=['--stream']) + + self.prepare_backup_for_detect_nondatafile_relation(backup_id) + + self.pb.backup_node('node', node, backup_type='delta', + options=['--stream'], + expect_error="relation is mistakenly marked as non-datafile") + self.assertMessage(contains="were created by misbehaving") + self.assertMessage(contains="Could not use it as a parent for increment") + + def test_nodetect_giga_oid_table(self): + """Detect we could increment based on backup without misdetected file type""" + node = self.setup_node() + + node.execute(f'CREATE TABLE t1 (i int)') + node.execute('INSERT INTO t1 (i) SELECT generate_series(1, 1000)') + node.execute('CHECKPOINT') + + backup_id = self.pb.backup_node('node', node, backup_type='full', + options=['--stream']) + + self.prepare_backup_for_detect_nondatafile_relation(backup_id) + + node.execute('INSERT INTO t1 (i) SELECT generate_series(2000, 3000)') + + table1_checksum = node.table_checksum('t1') + + backup_id2 = self.pb.backup_node('node', node, backup_type='delta', + options=['--stream']) + + node.stop() + node.cleanup() + + self.pb.restore_node('node', node, + backup_id=backup_id2) + + node.slow_start() + + new_table1_checksum = node.table_checksum('t1') + + self.assertEqual(new_table1_checksum, table1_checksum, "table checksums doesn't match") + + @needs_gdb + def test_detect_giga_oid_table_in_merge_restore(self): + """Detect we cann't merge/restore mixed increment chain with misdetected file type""" + node = self.setup_node() + self.jump_the_oid(node) + + node.execute(f'CREATE TABLE t1 (i int)') + node.execute('INSERT INTO t1 (i) SELECT generate_series(1, 1000)') + node.execute('CHECKPOINT') + + backup_id1 = self.pb.backup_node('node', node, backup_type='full', + options=['--stream']) + + backup_id2 = self.pb.backup_node('node', node, backup_type='delta', + options=['--stream']) + self.backup_control_version_to_2_7_3(backup_id1) + self.prepare_backup_for_detect_nondatafile_relation(backup_id2) + + self.pb.merge_backup('node', backup_id2, + options=['--no-validate'], + expect_error="due to chain of mixed bug/nobug backups") + + self.assertMessage(contains="kind reg detected is_datafile=0 stored=1") + + node.stop() + node.cleanup() + + self.pb.restore_node('node', node, + backup_id=backup_id2, + options=['--no-validate'], + expect_error="due to chain of mixed bug/nobug backups") + + self.assertMessage(contains="kind reg detected is_datafile=0 stored=1") + + self.pb.merge_backup('node', backup_id2, + options=['--no-validate'], + expect_error="due to chain of mixed bug/nobug backups") + + self.assertMessage(contains="kind reg detected is_datafile=0 stored=1") + + + @needs_gdb + def test_allow_giga_oid_table_in_restore(self): + """Detect we can restore uniform increment chain with misdetected file type""" + node = self.setup_node() + self.jump_the_oid(node) + + node.execute(f'CREATE TABLE t1 (i int)') + node.execute('INSERT INTO t1 (i) SELECT generate_series(1, 1000)') + node.execute('CHECKPOINT') + + backup_id1 = self.pb.backup_node('node', node, backup_type='full', + options=['--stream']) + + backup_id2 = self.pb.backup_node('node', node, backup_type='delta', + options=['--stream']) + self.prepare_backup_for_detect_nondatafile_relation(backup_id1) + self.prepare_backup_for_detect_nondatafile_relation(backup_id2) + + node.stop() + node.cleanup() + + self.pb.restore_node('node', node, backup_id=backup_id2) + # although we did restore, we could not check table checksum, + # because we backup relations as datafiles + # (because we backuped with fixed pbckp1242), + # and restore relation as non-datafile (ie with probackup's headers) + + self.pb.merge_backup('node', backup_id2, + expect_error="because of backups with bug") + self.assertMessage(contains='backups with 2.8.0/2.8.1') + self.assertMessage(contains="Could not merge them.") + + @needs_gdb + def test_nodetect_giga_oid_table_in_merge_restore(self): + """Detect we can merge/restore mixed increment chain without misdetected file type""" + node = self.setup_node() + + node.execute(f'CREATE TABLE t1 (i int)') + node.execute('INSERT INTO t1 (i) SELECT generate_series(1, 1000)') + node.execute('CHECKPOINT') + + backup_id1 = self.pb.backup_node('node', node, backup_type='full', + options=['--stream']) + + node.execute('INSERT INTO t1 (i) SELECT generate_series(2000, 3000)') + node.execute('CHECKPOINT') + + table1_checksum = node.table_checksum('t1') + + backup_id2 = self.pb.backup_node('node', node, backup_type='delta', + options=['--stream']) + self.backup_control_version_to_2_7_3(backup_id1) + self.prepare_backup_for_detect_nondatafile_relation(backup_id2) + + node.stop() + node.cleanup() + + self.pb.restore_node('node', node, backup_id=backup_id2) + + node.slow_start() + + new_table1_checksum = node.table_checksum('t1') + + self.assertEqual(new_table1_checksum, table1_checksum, "table checksums doesn't match") + + self.pb.merge_backup('node', backup_id2) + + @needs_gdb + def test_detect_giga_oid_database_in_merge_restore(self): + """Detect we cann't merge/restore mixed increment chain with misdetected file type""" + node = self.setup_node() + self.jump_the_oid(node) + + node.execute(f'CREATE DATABASE db2') + + node.execute('db2', f'CREATE TABLE t1 (i int)') + node.execute('db2', 'INSERT INTO t1 (i) SELECT generate_series(1, 1000)') + node.execute('CHECKPOINT') + + backup_id1 = self.pb.backup_node('node', node, backup_type='full', + options=['--stream']) + + node.execute('db2', f'CREATE TABLE t2 (i int)') + node.execute('db2', 'INSERT INTO t1 (i) SELECT generate_series(2000, 3000)') + node.execute('db2', 'INSERT INTO t2 (i) SELECT generate_series(2000, 3000)') + node.execute('CHECKPOINT') + + backup_id2 = self.pb.backup_node('node', node, backup_type='delta', + options=['--stream']) + self.backup_control_version_to_2_7_3(backup_id1) + self.prepare_backup_for_detect_gigaoid_database(backup_id2) + + self.pb.merge_backup('node', backup_id2, + options=['--no-validate'], + expect_error="due to chain of mixed bug/nobug backups") + + self.assertMessage(contains="kind reg detected is_datafile=0 stored=1") + + node.stop() + node.cleanup() + + self.pb.restore_node('node', node, + backup_id=backup_id2, + options=['--no-validate'], + expect_error="due to chain of mixed bug/nobug backups") + + self.assertMessage(contains="kind reg detected is_datafile=0 stored=1") + + self.pb.merge_backup('node', backup_id2, + options=['--no-validate'], + expect_error="due to chain of mixed bug/nobug backups") + + self.assertMessage(contains="kind reg detected is_datafile=0 stored=1") + + @needs_gdb + def test_allow_giga_oid_database_in_restore(self): + """Detect we can restore uniform increment chain with misdetected file type""" + node = self.setup_node() + self.jump_the_oid(node) + + node.execute(f'CREATE DATABASE db2') + + node.execute('db2', f'CREATE TABLE t1 (i int)') + node.execute('db2', 'INSERT INTO t1 (i) SELECT generate_series(1, 1000)') + node.execute('CHECKPOINT') + + backup_id1 = self.pb.backup_node('node', node, backup_type='full', + options=['--stream']) + + node.execute('db2', f'CREATE TABLE t2 (i int)') + node.execute('db2', 'INSERT INTO t1 (i) SELECT generate_series(2000, 3000)') + node.execute('db2', 'INSERT INTO t2 (i) SELECT generate_series(2000, 3000)') + node.execute('CHECKPOINT') + + backup_id2 = self.pb.backup_node('node', node, backup_type='delta', + options=['--stream']) + self.prepare_backup_for_detect_gigaoid_database(backup_id1) + self.prepare_backup_for_detect_gigaoid_database(backup_id2) + + node.stop() + node.cleanup() + + self.pb.restore_node('node', node, + backup_id=backup_id2) + # although we did restore, we could not check table checksum, + # because we backup relations as datafiles + # (because we backuped with fixed pbckp1242), + # and restore relation as non-datafile (ie with probackup's headers) + + self.pb.merge_backup('node', backup_id2, + expect_error="because of backups with bug") + self.assertMessage(contains='backups with 2.8.0/2.8.1') + self.assertMessage(contains="Could not merge them.") + + def backup_control_version_to(self, version, backup_id): + with self.modify_backup_control(self.backup_dir, 'node', backup_id) as control: + new = [] + for line in control.data.splitlines(True): + if line.startswith('program-version'): + line = f'program-version = {version}\n' + elif line.startswith('content-crc'): + line = 'content-crc = 0\n' + new.append(line) + control.data = "".join(new) + + def backup_control_version_to_2_8_1(self, backup_id): + self.backup_control_version_to('2.8.1', backup_id) + + def backup_control_version_to_2_7_3(self, backup_id): + self.backup_control_version_to('2.7.3', backup_id) + + def prepare_backup_for_detect_missed_database(self, backup_id): + self.backup_control_version_to_2_8_1(backup_id) + + with self.modify_backup_control(self.backup_dir, 'node', backup_id, content=True) as content: + new = [] + for line in content.data.splitlines(True): + if 'pg_tblspc' in line: + st = line.index('pg_tblspc') + en = line.index('"', st) + path = line[st:en] + elems = path.split('/') + if len(elems) > 4 and len(elems[3]) >= 10: + # delete all files in database folder with giga-oid + continue + new.append(line) + content.data = "".join(new) + + def prepare_backup_for_detect_missed_tablespace(self, backup_id): + self.backup_control_version_to_2_8_1(backup_id) + + with self.modify_backup_control(self.backup_dir, 'node', backup_id, content=True) as content: + new = [] + for line in content.data.splitlines(True): + if 'pg_tblspc' in line: + st = line.index('pg_tblspc') + en = line.index('"', st) + path = line[st:en] + elems = path.split('/') + if len(elems) >= 2 and len(elems[1]) >= 10: + # delete giga-oid tablespace completely + continue + new.append(line) + content.data = "".join(new) + + def prepare_backup_for_detect_nondatafile_relation(self, backup_id): + self.backup_control_version_to_2_8_1(backup_id) + + with self.modify_backup_control(self.backup_dir, 'node', backup_id, content=True) as content: + new = [] + for line in content.data.splitlines(True): + if 'base/' in line: + st = line.index('base/') + en = line.index('"', st) + path = line[st:en] + elems = path.split('/') + if len(elems) == 3 and len(elems[2]) >= 10 and elems[2].isdecimal(): + # pretend it is not datafile + line = line.replace('"is_datafile":"1"', '"is_datafile":"0"') + new.append(line) + content.data = "".join(new) + + def prepare_backup_for_detect_gigaoid_database(self, backup_id): + self.backup_control_version_to_2_8_1(backup_id) + + with self.modify_backup_control(self.backup_dir, 'node', backup_id, content=True) as content: + new = [] + for line in content.data.splitlines(True): + if 'base/' in line: + st = line.index('base/') + en = line.index('"', st) + path = line[st:en] + elems = path.split('/') + if len(elems) == 3 and len(elems[1]) >= 10 and elems[2].isdecimal(): + # 1. change dbOid = dbOid / 10 + # 2. pretend it is not datafile + line = line.replace('"is_datafile":"1"', '"is_datafile":"0"') + line = line.replace(f'"dbOid":"{elems[1]}"', f'"dbOid":"{int(elems[1])//10}"') + new.append(line) + content.data = "".join(new) diff --git a/tests/replica_test.py b/tests/replica_test.py index 7ff539b34..afadab17d 100644 --- a/tests/replica_test.py +++ b/tests/replica_test.py @@ -989,7 +989,9 @@ def test_replica_promote_2(self): master.safe_psql( 'postgres', 'CREATE TABLE t1 AS ' - 'SELECT i, repeat(md5(i::text),5006056) AS fat_attr ' + 'SELECT i,' + ' (select string_agg(md5((i^j)::text), \',\')' + ' from generate_series(1,5006056) j) AS fat_attr ' 'FROM generate_series(0,1) i') self.wait_until_replica_catch_with_master(master, replica) diff --git a/tests/requirements.txt b/tests/requirements.txt index 32910a3b6..eea7b8c42 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -6,10 +6,10 @@ # 3. From a local directory # /path/to/local/directory/testgres testgres==1.10.0 -git+https://github.com/postgrespro/testgres.git@fix-json-parse-in-show#egg=testgres_pg_probackup2&subdirectory=testgres/plugins/pg_probackup2 +testgres-pg-probackup2==0.0.2 allure-pytest deprecation -minio==7.2.5 +minio pexpect pytest==7.4.3 pytest-xdist diff --git a/tests/s3_auth_test.py b/tests/s3_auth_test.py new file mode 100644 index 000000000..d1a7c707c --- /dev/null +++ b/tests/s3_auth_test.py @@ -0,0 +1,20 @@ +from .helpers.ptrack_helpers import ProbackupTest + + +class AuthorizationTest(ProbackupTest): + """ + Check connect to S3 via pre_start_checks() function + calling pg_probackup init --s3 + + test that s3 keys allow to connect to all types of storages + """ + + def test_s3_auth_test(self): + console_output = self.pb.init(options=["--log-level-console=VERBOSE"]) + + self.assertNotIn(': 403', console_output) # Because we can have just '403' substring in timestamp + self.assertMessage(console_output, contains='S3_pre_start_check successful') + self.assertMessage(console_output, contains='HTTP response: 200') + self.assertIn( + f"INFO: Backup catalog '{self.backup_dir}' successfully initialized", + console_output) diff --git a/tests/time_consuming_test.py b/tests/time_consuming_test.py index 3da2208db..4fe77e565 100644 --- a/tests/time_consuming_test.py +++ b/tests/time_consuming_test.py @@ -68,6 +68,8 @@ def test_pbckp150(self): pgbenchval.kill() pgbench.wait() pgbenchval.wait() + pgbench.stdout.close() + pgbenchval.stdout.close() backups = self.pb.show('node') for b in backups: diff --git a/tests/validate_test.py b/tests/validate_test.py index 3b97171d4..bf608ad25 100644 --- a/tests/validate_test.py +++ b/tests/validate_test.py @@ -2305,6 +2305,9 @@ def test_corrupt_pg_control_via_resetxlog(self): os.mkdir( os.path.join( self.backup_dir, 'backups', 'node', backup_id, 'database', wal_dir, 'archive_status')) + os.mkdir( + os.path.join( + self.backup_dir, 'backups', 'node', backup_id, 'database', wal_dir, 'summaries')) pg_control_path = os.path.join( self.backup_dir, 'backups', 'node', From 29a2ebca1a353f13169caeb4275cbdf35296e6ef Mon Sep 17 00:00:00 2001 From: vshepard Date: Thu, 5 Sep 2024 16:41:28 +0200 Subject: [PATCH 4/4] tests 2.8.2 --- tests/helpers/validators/show_validator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/helpers/validators/show_validator.py b/tests/helpers/validators/show_validator.py index d7df177a8..ac5a9d8e0 100644 --- a/tests/helpers/validators/show_validator.py +++ b/tests/helpers/validators/show_validator.py @@ -21,8 +21,8 @@ class ShowJsonResultValidator(TestCase): and do not worry about the readability of the error result. """ - def __init__(self): - super().__init__() + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.backup_id = None self.parent_backup_id = None self.backup_mode = None