From 833e6b84c6fd68e7cd5fcc0c87469fe76ec3e158 Mon Sep 17 00:00:00 2001 From: CL Abeel <89160012+fivetran-clgritton@users.noreply.github.com> Date: Wed, 8 Jan 2025 11:16:41 -0500 Subject: [PATCH 1/8] Create album_tracks.py --- .../source_examples/spotify/album_tracks.py | 144 ++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 examples/source_examples/spotify/album_tracks.py diff --git a/examples/source_examples/spotify/album_tracks.py b/examples/source_examples/spotify/album_tracks.py new file mode 100644 index 0000000..c65fa10 --- /dev/null +++ b/examples/source_examples/spotify/album_tracks.py @@ -0,0 +1,144 @@ +# See the Technical Reference documentation (https://fivetran.com/docs/connectors/connector-sdk/technical-reference#update) +# and the Best Practices documentation (https://fivetran.com/docs/connectors/connector-sdk/best-practices) for details + +# Import requests to make HTTP calls to API +import requests as rq +import traceback +import json +import spotipy +from spotipy.oauth2 import SpotifyClientCredentials + +# Import required classes from fivetran_connector_sdk +from fivetran_connector_sdk import Connector +from fivetran_connector_sdk import Logging as log +from fivetran_connector_sdk import Operations as op + +# Define the schema function which lets you configure the schema your connector delivers. +# See the technical reference documentation for more details on the schema function: +# https://fivetran.com/docs/connectors/connector-sdk/technical-reference#schema +# The schema function takes one parameter: +# - configuration: a dictionary that holds the configuration settings for the connector. +def schema(configuration: dict): + + return [ + { + "table": "album", + "primary_key": ["id"] + }, + { + "table": "track", + "primary_key": ["id"] + } + ] + +# Define the update function, which is a required function, and is called by Fivetran during each sync. +# See the technical reference documentation for more details on the update function +# https://fivetran.com/docs/connectors/connector-sdk/technical-reference#update +# The function takes two parameters: +# - configuration: dictionary contains any secrets or payloads you configure when deploying the connector +# - state: a dictionary contains whatever state you have chosen to checkpoint during the prior sync +# The state dictionary is empty for the first sync or for any full re-sync +def update(configuration: dict, state: dict): + # business_cursor = state["business_cursor"] if "business_cursor" in state else '0001-01-01T00:00:00Z' + # department_cursor = state["department_cursor"] if "department_cursor" in state else {} + + try: + conf = configuration + client_id = conf['client_id'] + client_secret = conf['client_secret'] + artist_url = conf['artist_url'] + album_params = {"artist_id": artist_url} + auth_manager = SpotifyClientCredentials(client_id=client_id, client_secret=client_secret) + sp = spotipy.Spotify(auth_manager=auth_manager) + + yield from sync_items(sp, album_params) + + except Exception as e: + # Return error response + exception_message = str(e) + stack_trace = traceback.format_exc() + detailed_message = f"Error Message: {exception_message}\nStack Trace:\n{stack_trace}" + raise RuntimeError(detailed_message) + + +# The function takes three parameters: +# - obj: The spotify connection object. +# - payload: A dictionary of query parameters send with the method, if needed +def sync_items(obj, payload): + + try: + # For this artist, get one page of albums in a response from the API call. + albums_page = get_api_response(obj, "artist_albums", payload) + albums = albums_page["items"] + for a in albums: + album_data = remove_lists(a) + album_name = album_data["name"] + log.fine(f"adding album {album_name}") + yield op.upsert(table="album", data=album_data) + + # For the current album, get one page of tracks in a response from the API call. + track_params = {"album_id": a["id"]} + tracks_page = get_api_response(obj, "album_tracks", track_params) + tracks = tracks_page["items"] + for t in tracks: + track_data = remove_lists(t) + yield op.upsert(table="track", data=track_data) + + # Save the progress by checkpointing the state. This is important for ensuring that the sync process can resume + # from the correct position in case of interruptions. + yield op.checkpoint({}) + + except Exception as e: + # Return error response + exception_message = str(e) + stack_trace = traceback.format_exc() + detailed_message = f"Error Message: {exception_message}\nStack Trace:\n{stack_trace}" + raise RuntimeError(detailed_message) + +# The get_api_response function uses the Spotipy python library to get API response from Spotify. +# +# The function takes three parameters: +# - obj: The spotify connection object. +# - method_name: The spotipy method to use +# - payload: a dictionary of parameters to send with the method, if needed +# +# Returns: +# - response_page: A dictionary containing the parsed JSON response from the API. +def get_api_response(obj, method_name, payload=None): + if payload is None: + payload = {} + method = getattr(obj, method_name) + response_page = method(**payload) + return response_page + +# The remove_lists function removes keys from a dictionary if the value is a list +# +# The function takes one parameter: +# - d: a dictionary +# +# Returns: +# - new_dict: A dictionary without any values that are lists +def remove_lists(d): + new_dict = {} + for key, value in d.items(): + if isinstance(value, list): + pass + else: + new_dict[key] = value + + return new_dict + +# This creates the connector object that will use the update function defined in this connector.py file. +connector = Connector(update=update, schema=schema) + +# Check if the script is being run as the main module. This is Python's standard entry method allowing your script to +# be run directly from the command line or IDE 'run' button. This is useful for debugging while you write your code. +# Note this method is not called by Fivetran when executing your connector in production. Please test using the +# Fivetran debug command prior to finalizing and deploying your connector. +if __name__ == "main": + # Open the configuration.json file and load its contents into a dictionary. + with open("configuration.json", 'r') as f: + configuration = json.load(f) + # Adding this code to your `connector.py` allows you to test your connector by running your file directly from your IDE. + connector.debug(configuration=configuration) + From b5244c06ef4ed26ff91a822caa3b4313c540db8f Mon Sep 17 00:00:00 2001 From: CL Abeel <89160012+fivetran-clgritton@users.noreply.github.com> Date: Wed, 8 Jan 2025 11:17:31 -0500 Subject: [PATCH 2/8] Rename album_tracks.py to connector.py --- .../source_examples/spotify/{album_tracks.py => connector.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename examples/source_examples/spotify/{album_tracks.py => connector.py} (100%) diff --git a/examples/source_examples/spotify/album_tracks.py b/examples/source_examples/spotify/connector.py similarity index 100% rename from examples/source_examples/spotify/album_tracks.py rename to examples/source_examples/spotify/connector.py From f4b862f1db86b668b8b01ca13ccfd5d761827ad5 Mon Sep 17 00:00:00 2001 From: CL Abeel <89160012+fivetran-clgritton@users.noreply.github.com> Date: Wed, 8 Jan 2025 11:18:30 -0500 Subject: [PATCH 3/8] Create configuration.json --- examples/source_examples/spotify/configuration.json | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 examples/source_examples/spotify/configuration.json diff --git a/examples/source_examples/spotify/configuration.json b/examples/source_examples/spotify/configuration.json new file mode 100644 index 0000000..891f8c4 --- /dev/null +++ b/examples/source_examples/spotify/configuration.json @@ -0,0 +1,5 @@ +{ + "client_secret": "", + "client_id": "", + "artist_url": "" +} From 21462add0af4f586b11ce9221f2923c54443f203 Mon Sep 17 00:00:00 2001 From: CL Abeel <89160012+fivetran-clgritton@users.noreply.github.com> Date: Thu, 9 Jan 2025 08:47:53 -0500 Subject: [PATCH 4/8] Update examples/source_examples/spotify/connector.py Co-authored-by: Rishabh Ghosh <105623976+fivetran-rishabhghosh@users.noreply.github.com> --- examples/source_examples/spotify/connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/source_examples/spotify/connector.py b/examples/source_examples/spotify/connector.py index c65fa10..4a248d6 100644 --- a/examples/source_examples/spotify/connector.py +++ b/examples/source_examples/spotify/connector.py @@ -135,7 +135,7 @@ def remove_lists(d): # be run directly from the command line or IDE 'run' button. This is useful for debugging while you write your code. # Note this method is not called by Fivetran when executing your connector in production. Please test using the # Fivetran debug command prior to finalizing and deploying your connector. -if __name__ == "main": +if __name__ == "__main__": # Open the configuration.json file and load its contents into a dictionary. with open("configuration.json", 'r') as f: configuration = json.load(f) From 78b1daaff044d76c90a78b975eda3349569f53f1 Mon Sep 17 00:00:00 2001 From: CL Abeel <89160012+fivetran-clgritton@users.noreply.github.com> Date: Thu, 9 Jan 2025 08:48:25 -0500 Subject: [PATCH 5/8] Update examples/source_examples/spotify/configuration.json Co-authored-by: Rishabh Ghosh <105623976+fivetran-rishabhghosh@users.noreply.github.com> --- examples/source_examples/spotify/configuration.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/source_examples/spotify/configuration.json b/examples/source_examples/spotify/configuration.json index 891f8c4..8a06cb4 100644 --- a/examples/source_examples/spotify/configuration.json +++ b/examples/source_examples/spotify/configuration.json @@ -1,5 +1,7 @@ { - "client_secret": "", + "client_secret": "", + "client_id": "", + "artist_url": "" "client_id": "", "artist_url": "" } From 12612ec047a5fd8359dd1087e0004be41bfcecb6 Mon Sep 17 00:00:00 2001 From: CL Abeel <89160012+fivetran-clgritton@users.noreply.github.com> Date: Thu, 9 Jan 2025 08:50:44 -0500 Subject: [PATCH 6/8] Update configuration.json --- examples/source_examples/spotify/configuration.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/examples/source_examples/spotify/configuration.json b/examples/source_examples/spotify/configuration.json index 8a06cb4..c1f21d9 100644 --- a/examples/source_examples/spotify/configuration.json +++ b/examples/source_examples/spotify/configuration.json @@ -2,6 +2,4 @@ "client_secret": "", "client_id": "", "artist_url": "" - "client_id": "", - "artist_url": "" } From 8c2b9c6e8d14c261d79454bf6a455e1027fdd3f4 Mon Sep 17 00:00:00 2001 From: CL Abeel <89160012+fivetran-clgritton@users.noreply.github.com> Date: Thu, 9 Jan 2025 08:54:27 -0500 Subject: [PATCH 7/8] added comments about what this connector does --- examples/source_examples/spotify/connector.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/examples/source_examples/spotify/connector.py b/examples/source_examples/spotify/connector.py index 4a248d6..97e1513 100644 --- a/examples/source_examples/spotify/connector.py +++ b/examples/source_examples/spotify/connector.py @@ -1,3 +1,6 @@ +# This is a simple example that retrieves a single page of albums from Spotify, +# and then a single page of tracks for each album. +# The required inputs are Spotify client ID and client secret, and an artist URL. # See the Technical Reference documentation (https://fivetran.com/docs/connectors/connector-sdk/technical-reference#update) # and the Best Practices documentation (https://fivetran.com/docs/connectors/connector-sdk/best-practices) for details @@ -43,10 +46,9 @@ def update(configuration: dict, state: dict): # department_cursor = state["department_cursor"] if "department_cursor" in state else {} try: - conf = configuration - client_id = conf['client_id'] - client_secret = conf['client_secret'] - artist_url = conf['artist_url'] + client_id = configuration['client_id'] + client_secret = configuration['client_secret'] + artist_url = configuration['artist_url'] album_params = {"artist_id": artist_url} auth_manager = SpotifyClientCredentials(client_id=client_id, client_secret=client_secret) sp = spotipy.Spotify(auth_manager=auth_manager) From 6811222d7ea4e04d3047659e9649d3fd0c94fef0 Mon Sep 17 00:00:00 2001 From: CL Abeel <89160012+fivetran-clgritton@users.noreply.github.com> Date: Mon, 13 Jan 2025 16:45:01 -0500 Subject: [PATCH 8/8] Create requirements.txt --- examples/source_examples/spotify/requirements.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 examples/source_examples/spotify/requirements.txt diff --git a/examples/source_examples/spotify/requirements.txt b/examples/source_examples/spotify/requirements.txt new file mode 100644 index 0000000..32d023e --- /dev/null +++ b/examples/source_examples/spotify/requirements.txt @@ -0,0 +1 @@ +spotipy==2.24.0