Skip to content

Commit

Permalink
Fix Bigquery templates
Browse files Browse the repository at this point in the history
  • Loading branch information
judahrand committed Apr 19, 2022
1 parent 4de979f commit 18eb716
Show file tree
Hide file tree
Showing 3 changed files with 72 additions and 0 deletions.
5 changes: 5 additions & 0 deletions tests/end_to_end/test-project/tap_postgres_to_bq.yml.template
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,10 @@ schemas:
- column: "email"
type: "MASK-STRING-SKIP-ENDS-6"

### Empty table
- table_name: "empty_table"
replication_method: "INCREMENTAL"
replication_key: "id"

### SOURCE SCHEMA 2: public2
- source_schema: "public2"
Expand Down Expand Up @@ -116,3 +120,4 @@ schemas:
# target_schema: "ppw_e2e_tap_postgres_logical2"
# tables:
# - table_name: "logical2_table1"

Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
---

# ------------------------------------------------------------------------------
# General Properties
# ------------------------------------------------------------------------------
id: "postgres_to_bq_archive_load_files"
name: "PostgreSQL source test database"
type: "tap-postgres"
owner: "test-runner"


# ------------------------------------------------------------------------------
# Source (Tap) - PostgreSQL connection details
# ------------------------------------------------------------------------------
db_conn:
host: "${TAP_POSTGRES_HOST}" # PostgreSQL host
logical_poll_total_seconds: 3 # Time out if no LOG_BASED changes received for 3 seconds
port: ${TAP_POSTGRES_PORT} # PostgreSQL port
user: "${TAP_POSTGRES_USER}" # PostgreSQL user
password: "${TAP_POSTGRES_PASSWORD}" # Plain string or vault encrypted
dbname: "${TAP_POSTGRES_DB}" # PostgreSQL database name


# ------------------------------------------------------------------------------
# Destination (Target) - Target properties
# Connection details should be in the relevant target YAML file
# ------------------------------------------------------------------------------
target: "bigquery" # ID of the target connector where the data will be loaded
batch_size_rows: 1000 # Batch size for the stream to optimise load performance
stream_buffer_size: 0 # In-memory buffer size (MB) between taps and targets for asynchronous data pipes
archive_load_files: True # Archive the load files in dedicated S3 folder
archive_load_files_gcs_prefix: archive_folder # Archive folder


# ------------------------------------------------------------------------------
# Source to target Schema mapping
# ------------------------------------------------------------------------------
schemas:

### SOURCE SCHEMA 1: public
- source_schema: "public"
target_schema: "ppw_e2e_tap_postgres${TARGET_SNOWFLAKE_SCHEMA_POSTFIX}"

tables:

### Table with INCREMENTAL replication
- table_name: "city"
replication_method: "INCREMENTAL"
replication_key: "id"

### Table with FULL_TABLE replication
- table_name: "country"
replication_method: "FULL_TABLE"

### SOURCE SCHEMA 2: public2
- source_schema: "public2"
target_schema: "ppw_e2e_tap_postgres_public2${TARGET_SNOWFLAKE_SCHEMA_POSTFIX}"

tables:
### Table with FULL_TABLE replication
- table_name: "wearehere"
replication_method: "FULL_TABLE"
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,11 @@ schemas:
replication_method: "INCREMENTAL"
replication_key: "id"

### Empty table
- table_name: "empty_table"
replication_method: "INCREMENTAL"
replication_key: "id"

### Table with space and mixed upper and lowercase characters
- table_name: "table_with_space and UPPERCase"
replication_method: "LOG_BASED"
Expand Down

0 comments on commit 18eb716

Please sign in to comment.