Bump facebook_business SDK to v21.0.5

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 1.21.0
+  * Bump facebook_business SDK to v21.0.5 [#242](https://github.com/singer-io/tap-facebook/pull/242)
+
 ## 1.20.2
   * Bump facebook_business SDK to v19.0.2 [#238](https://github.com/singer-io/tap-facebook/pull/239)
 

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup
 
 setup(name='tap-facebook',
-      version='1.20.2',
+      version='1.21.0',
       description='Singer.io tap for extracting data from the Facebook Ads API',
       author='Stitch',
       url='https://singer.io',
@@ -12,7 +12,7 @@
       install_requires=[
           'attrs==17.3.0',
           'backoff==2.2.1',
-          'facebook_business==19.0.2',
+          'facebook_business==21.0.5',
           'pendulum==1.2.0',
           'requests==2.20.0',
           'singer-python==6.0.0',

tests/test_facebook_all_fields.py

@@ -2,6 +2,7 @@
 Test that with no fields selected for a stream all fields are still replicated
 """
 
+from tap_tester import LOGGER
 from tap_tester.base_suite_tests.all_fields_test import AllFieldsTest
 from base_new_frmwrk import FacebookBaseTest
 import base
@@ -208,18 +209,33 @@ class FacebookAllFieldsTest(AllFieldsTest, FacebookBaseTest):
         }
     }
 
+    # TODO: https://jira.talendforge.org/browse/TDL-26640
+    EXCLUDE_STREAMS = {
+        'ads_insights_hourly_advertiser',   # TDL-24312, TDL-26640
+        'ads_insights_platform_and_device', # TDL-26640
+        'ads_insights',                     # TDL-26640
+        'ads_insights_age_and_gender',      # TDL-26640
+        'ads_insights_country',             # TDL-26640
+        'ads_insights_dma',                 # TDL-26640
+        'ads_insights_region'               # TDL-26640
+    }
 
     @staticmethod
     def name():
         return "tt_facebook_all_fields_test"
 
     def streams_to_test(self):
-        #return set(self.expected_metadata().keys())
-        # Fail the test when the JIRA card is done to allow stream to be re-added and tested
-        if self.is_done is None:
-            self.is_done = base.JIRA_CLIENT.get_status_category("TDL-24312") == 'done'
-            self.assert_message = ("JIRA ticket has moved to done, re-add the "
-                                   "ads_insights_hourly_advertiser stream to the test.")
-        assert self.is_done != True, self.assert_message
+        exprected_streams = self.expected_metadata().keys()
+        self.assert_message = f"JIRA ticket has moved to done, \
+                                re-add the applicable stream to the test: {0}"
+        assert base.JIRA_CLIENT.get_status_category("TDL-24312") != 'done',\
+            self.assert_message.format('ads_insights_hourly_advertiser')
+        exprected_streams = self.expected_metadata().keys() - {'ads_insights_hourly_advertiser'}
+        LOGGER.warn(f"Skipped streams: {'ads_insights_hourly_advertiser'}")
+
+        assert base.JIRA_CLIENT.get_status_category("TDL-26640") != 'done',\
+            self.assert_message.format(self.EXCLUDE_STREAMS)
+        exprected_streams = self.expected_metadata().keys() - self.EXCLUDE_STREAMS
+        LOGGER.warn(f"Skipped streams: {self.EXCLUDE_STREAMS}")
 
-        return self.expected_metadata().keys() - {'ads_insights_hourly_advertiser'}
+        return exprected_streams

tests/test_facebook_attribution_window.py

@@ -2,8 +2,8 @@
 import os
 
 from tap_tester import runner, connections
-
-from base import FacebookBaseTest
+from tap_tester.base_case import BaseCase as base_case
+from base import FacebookBaseTest, LOGGER
 
 
 class FacebookAttributionWindow(FacebookBaseTest):
@@ -43,6 +43,8 @@ def get_properties(self, original: bool = True):
         return_value["start_date"] = self.start_date
         return return_value
 
+    # TODO: https://jira.talendforge.org/browse/TDL-26640
+    @base_case.skipUnless(base.JIRA_CLIENT.get_status_category("TDL-26640") == "done", "TDL-26640")
     def test_run(self):
         """
         For the test ad set up in facebook ads manager we see data

tests/test_facebook_automatic_fields.py

@@ -3,20 +3,46 @@
 """
 import os
 
-from tap_tester import runner, connections
-
+from tap_tester import runner, connections, LOGGER
+import base
 from base import FacebookBaseTest
 
 
 class FacebookAutomaticFields(FacebookBaseTest):
     """Test that with no fields selected for a stream automatic fields are still replicated"""
 
+    is_done = None
+
+    # TODO: https://jira.talendforge.org/browse/TDL-26640
+    EXCLUDE_STREAMS = {
+        'ads_insights_hourly_advertiser',   # TDL-24312, TDL-26640
+        'ads_insights_platform_and_device', # TDL-26640
+        'ads_insights',                     # TDL-26640
+        'ads_insights_age_and_gender',      # TDL-26640
+        'ads_insights_country',             # TDL-26640
+        'ads_insights_dma',                 # TDL-26640
+        'ads_insights_region'               # TDL-26640
+    }
+
     @staticmethod
     def name():
         return "tap_tester_facebook_automatic_fields"
 
     def streams_to_test(self):
-        return self.expected_streams()
+        exprected_streams = self.expected_metadata().keys()
+        self.assert_message = f"JIRA ticket has moved to done, \
+                                re-add the applicable stream to the test: {0}"
+        assert base.JIRA_CLIENT.get_status_category("TDL-24312") != 'done',\
+            self.assert_message.format('ads_insights_hourly_advertiser')
+        exprected_streams = self.expected_metadata().keys() - {'ads_insights_hourly_advertiser'}
+        LOGGER.warn(f"Skipped streams: {'ads_insights_hourly_advertiser'}")
+
+        assert base.JIRA_CLIENT.get_status_category("TDL-26640") != 'done',\
+            self.assert_message.format(self.EXCLUDE_STREAMS)
+        exprected_streams = self.expected_metadata().keys() - self.EXCLUDE_STREAMS
+        LOGGER.warn(f"Skipped streams: {self.EXCLUDE_STREAMS}")
+
+        return exprected_streams
 
     def get_properties(self, original: bool = True):
         """Configuration properties required for the tap."""

tests/test_facebook_bookmarks.py

@@ -4,7 +4,7 @@
 import os
 import pytz
 
-from tap_tester import runner, menagerie, connections
+from tap_tester import runner, menagerie, connections, LOGGER
 
 from base import FacebookBaseTest
 
@@ -13,19 +13,36 @@ class FacebookBookmarks(FacebookBaseTest):
 
     is_done = None
 
+    # TODO: https://jira.talendforge.org/browse/TDL-26640
+    EXCLUDE_STREAMS = {
+        'ads_insights_hourly_advertiser',   # TDL-24312, TDL-26640
+        'ads_insights_platform_and_device', # TDL-26640
+        'ads_insights',                     # TDL-26640
+        'ads_insights_age_and_gender',      # TDL-26640
+        'ads_insights_country',             # TDL-26640
+        'ads_insights_dma',                 # TDL-26640
+        'ads_insights_region'               # TDL-26640
+    }
+
     @staticmethod
     def name():
         return "tap_tester_facebook_bookmarks"
 
     def streams_to_test(self):
-        # Fail the test when the JIRA card is done to allow stream to be re-added and tested
-        if self.is_done is None:
-            self.is_done = base.JIRA_CLIENT.get_status_category("TDL-24312") == 'done'
-            self.assert_message = ("JIRA ticket has moved to done, re-add the "
-                                   "ads_insights_hourly_advertiser stream to the test.")
-        assert self.is_done != True, self.assert_message
+        exprected_streams = self.expected_metadata().keys()
+        self.assert_message = f"JIRA ticket has moved to done, \
+                                re-add the applicable stream to the test: {0}"
+        assert base.JIRA_CLIENT.get_status_category("TDL-24312") != 'done',\
+            self.assert_message.format('ads_insights_hourly_advertiser')
+        exprected_streams = self.expected_metadata().keys() - {'ads_insights_hourly_advertiser'}
+        LOGGER.warn(f"Skipped streams: {'ads_insights_hourly_advertiser'}")
+
+        assert base.JIRA_CLIENT.get_status_category("TDL-26640") != 'done',\
+            self.assert_message.format(self.EXCLUDE_STREAMS)
+        exprected_streams = self.expected_metadata().keys() - self.EXCLUDE_STREAMS
+        LOGGER.warn(f"Skipped streams: {self.EXCLUDE_STREAMS}")
 
-        return self.expected_streams() - {'ads_insights_hourly_advertiser'}
+        return exprected_streams
 
     @staticmethod
     def convert_state_to_utc(date_str):
@@ -103,7 +120,8 @@ def test_run(self):
         # Testing against ads insights objects
         self.start_date = self.get_properties()['start_date']
         self.end_date = self.get_properties()['end_date']
-        self.bookmarks_test(insight_streams)
+        # TODO: https://jira.talendforge.org/browse/TDL-26640
+        # self.bookmarks_test(insight_streams)
 
         # Testing against core objects
         self.end_date = '2021-02-09T00:00:00Z'

tests/test_facebook_start_date.py

@@ -12,19 +12,36 @@ class FacebookStartDateTest(FacebookBaseTest):
     start_date_1 = ""
     start_date_2 = ""
 
+    # TODO: https://jira.talendforge.org/browse/TDL-26640
+    EXCLUDE_STREAMS = {
+        'ads_insights_hourly_advertiser',   # TDL-24312, TDL-26640
+        'ads_insights_platform_and_device', # TDL-26640
+        'ads_insights',                     # TDL-26640
+        'ads_insights_age_and_gender',      # TDL-26640
+        'ads_insights_country',             # TDL-26640
+        'ads_insights_dma',                 # TDL-26640
+        'ads_insights_region'               # TDL-26640
+    }
+
     @staticmethod
     def name():
         return "tap_tester_facebook_start_date_test"
 
     def streams_to_test(self):
-        # Fail the test when the JIRA card is done to allow stream to be re-added and tested
-        if self.is_done is None:
-            self.is_done = base.JIRA_CLIENT.get_status_category("TDL-24312") == 'done'
-            self.assert_message = ("JIRA ticket has moved to done, re-add the "
-                                   "ads_insights_hourly_advertiser stream to the test.")
-        assert self.is_done != True, self.assert_message
-
-        return self.expected_streams() - {'ads_insights_hourly_advertiser'}
+        exprected_streams = self.expected_metadata().keys()
+        self.assert_message = f"JIRA ticket has moved to done, \
+                                re-add the applicable stream to the test: {0}"
+        assert base.JIRA_CLIENT.get_status_category("TDL-24312") != 'done',\
+            self.assert_message.format('ads_insights_hourly_advertiser')
+        exprected_streams = self.expected_metadata().keys() - {'ads_insights_hourly_advertiser'}
+        LOGGER.warn(f"Skipped streams: {'ads_insights_hourly_advertiser'}")
+
+        assert base.JIRA_CLIENT.get_status_category("TDL-26640") != 'done',\
+            self.assert_message.format(self.EXCLUDE_STREAMS)
+        exprected_streams = self.expected_metadata().keys() - self.EXCLUDE_STREAMS
+        LOGGER.warn(f"Skipped streams: {self.EXCLUDE_STREAMS}")
+
+        return exprected_streams
 
     def test_run(self):
         """Instantiate start date according to the desired data set and run the test"""

tests/test_facebook_table_reset.py

@@ -1,26 +1,54 @@
 import os
 import dateutil.parser
 import datetime
+import base
 from base_new_frmwrk import FacebookBaseTest
 from tap_tester.base_suite_tests.table_reset_test import TableResetTest
+from tap_tester import LOGGER
 
 
 class FacebookTableResetTest(TableResetTest, FacebookBaseTest):
     """tap-salesforce Table reset test implementation
     Currently tests only the stream with Incremental replication method"""
 
+    is_done = None
+
     @staticmethod
     def name():
         return "tt_facebook_table_reset"
 
+    # TODO: https://jira.talendforge.org/browse/TDL-26640
+    EXCLUDE_STREAMS = {
+        'ads_insights_hourly_advertiser',   # TDL-24312, TDL-26640
+        'ads_insights_platform_and_device', # TDL-26640
+        'ads_insights',                     # TDL-26640
+        'ads_insights_age_and_gender',      # TDL-26640
+        'ads_insights_country',             # TDL-26640
+        'ads_insights_dma',                 # TDL-26640
+        'ads_insights_region'               # TDL-26640
+    }
+
     def streams_to_test(self):
-        return self.expected_stream_names()
+        exprected_streams = self.expected_metadata().keys()
+        self.assert_message = f"JIRA ticket has moved to done, \
+                                re-add the applicable stream to the test: {0}"
+        assert base.JIRA_CLIENT.get_status_category("TDL-24312") != 'done',\
+            self.assert_message.format('ads_insights_hourly_advertiser')
+        exprected_streams = self.expected_metadata().keys() - {'ads_insights_hourly_advertiser'}
+        LOGGER.warn(f"Skipped streams: {'ads_insights_hourly_advertiser'}")
+
+        assert base.JIRA_CLIENT.get_status_category("TDL-26640") != 'done',\
+            self.assert_message.format(self.EXCLUDE_STREAMS)
+        exprected_streams = self.expected_metadata().keys() - self.EXCLUDE_STREAMS
+        LOGGER.warn(f"Skipped streams: {self.EXCLUDE_STREAMS}")
+
+        return exprected_streams
 
     @property
     def reset_stream(self):
-        return ('ads_insights_dma')
+        return ('ads')
+
 
-
     def calculated_states_by_stream(self, current_state):
 
         """        The following streams barely make the cut: