From 7c1c0dc7652a617bf63509fad2152e26b83b0015 Mon Sep 17 00:00:00 2001 From: Yeganathan S Date: Thu, 16 Jan 2025 15:33:28 +0530 Subject: [PATCH 1/3] Remove hubspot, automations, and integration-sync-worker (#2755) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Uroš Marolt --- backend/.env.dist.local | 2 +- .../config/custom-environment-variables.json | 5 - backend/config/default.json | 8 +- backend/id-openapi.yaml | 4 +- backend/src/api/auth/authMe.ts | 14 - .../src/api/automation/automationCreate.ts | 31 - .../src/api/automation/automationDestroy.ts | 27 - .../api/automation/automationExecutionFind.ts | 40 -- backend/src/api/automation/automationFind.ts | 24 - backend/src/api/automation/automationList.ts | 51 -- .../api/automation/automationSlackCallback.ts | 19 - .../api/automation/automationSlackConnect.ts | 21 - .../src/api/automation/automationUpdate.ts | 32 - backend/src/api/automation/index.ts | 51 -- backend/src/api/index.ts | 1 - .../api/integration/helpers/hubspotConnect.ts | 9 - .../integration/helpers/hubspotGetLists.ts | 9 - .../helpers/hubspotGetMappableFields.ts | 9 - .../api/integration/helpers/hubspotOnboard.ts | 9 - .../helpers/hubspotStopSyncMember.ts | 9 - .../helpers/hubspotStopSyncOrganization.ts | 9 - .../integration/helpers/hubspotSyncMember.ts | 9 - .../helpers/hubspotSyncOrganization.ts | 9 - .../helpers/hubspotUpdateProperties.ts | 9 - backend/src/api/integration/index.ts | 45 -- backend/src/api/tenant/tenantFind.ts | 8 - backend/src/conf/configTypes.ts | 11 - backend/src/conf/index.ts | 9 +- backend/src/database/models/automation.ts | 74 -- .../database/models/automationExecution.ts | 90 --- backend/src/database/models/index.ts | 2 - .../automationExecutionRepository.ts | 170 ----- .../repositories/automationRepository.ts | 295 -------- .../repositories/integrationRepository.ts | 28 - .../database/repositories/memberRepository.ts | 16 - .../memberSyncRemoteRepository.ts | 250 ------- .../organizationSyncRemoteRepository.ts | 252 ------- .../repositories/types/automationTypes.ts | 35 - backend/src/security/permissions.ts | 16 - backend/src/serverless/utils/queueService.ts | 15 - backend/src/services/activityService.ts | 47 +- .../auth/passportStrategies/slackStrategy.ts | 26 +- .../services/automationExecutionService.ts | 78 --- backend/src/services/automationService.ts | 197 ------ backend/src/services/integrationService.ts | 529 +-------------- backend/src/services/memberService.ts | 32 - backend/src/types/automationTypes.ts | 68 -- pnpm-lock.yaml | 104 --- scripts/builders/automations-worker.env | 4 - scripts/builders/integration-sync-worker.env | 5 - scripts/cli | 8 +- scripts/services/automations-worker.yaml | 65 -- .../docker/Dockerfile.automations_worker | 23 - ...Dockerfile.automations_worker.dockerignore | 19 - .../docker/Dockerfile.integration_sync_worker | 22 - ...rfile.integration_sync_worker.dockerignore | 19 - scripts/services/integration-sync-worker.yaml | 65 -- services/apps/automations_worker/package.json | 36 - .../apps/automations_worker/src/activities.ts | 15 - .../src/activities/newActivityAutomations.ts | 36 - .../src/activities/newMemberAutomations.ts | 36 - services/apps/automations_worker/src/main.ts | 40 -- .../src/services/automation.service.ts | 502 -------------- .../src/services/slack/newActivityBlocks.ts | 220 ------ .../src/services/slack/newMemberBlocks.ts | 146 ---- .../apps/automations_worker/src/workflows.ts | 15 - .../src/workflows/newActivityAutomations.ts | 52 -- .../src/workflows/newMemberAutomations.ts | 54 -- .../apps/automations_worker/tsconfig.json | 4 - .../apps/data_sink_worker/config/default.json | 4 +- .../apps/data_sink_worker/src/conf/index.ts | 10 +- .../src/service/activity.service.ts | 39 +- .../src/service/member.service.ts | 41 +- .../apps/integration_run_worker/src/main.ts | 9 - .../integration_run_worker/src/queue/index.ts | 3 - .../src/service/integrationRunService.ts | 64 +- .../config/custom-environment-variables.json | 32 - .../config/default.json | 10 - .../config/docker.json | 1 - .../config/production.json | 1 - .../config/staging.json | 1 - .../apps/integration_sync_worker/package.json | 32 - .../integration_sync_worker/src/conf/index.ts | 54 -- .../integration_sync_worker/src/errors.ts | 5 - .../apps/integration_sync_worker/src/main.ts | 36 - .../src/queue/index.ts | 125 ---- .../src/service/member.sync.service.ts | 638 ------------------ .../src/service/opensearch.data.ts | 5 - .../src/service/opensearch.service.ts | 53 -- .../src/service/organization.sync.service.ts | 367 ---------- .../apps/integration_sync_worker/src/types.ts | 0 .../integration_sync_worker/tsconfig.json | 4 - services/libs/common/src/env.ts | 12 - services/libs/common/src/i18n/en.ts | 8 - .../src/services/emitters/index.ts | 1 - .../emitters/integrationSyncWorker.emitter.ts | 167 ----- .../automations_worker/automation.repo.ts | 134 ---- .../integration_run_worker/automation.repo.ts | 43 -- .../automation.repo.ts | 13 - .../automationExecution.repo.ts | 29 - .../hubspot/api/addContactsToList.ts | 36 - .../hubspot/api/batchCreateMembers.ts | 178 ----- .../hubspot/api/batchCreateOrganizations.ts | 167 ----- .../hubspot/api/batchUpdateMembers.ts | 119 ---- .../hubspot/api/batchUpdateOrganizations.ts | 104 --- .../src/integrations/hubspot/api/common.ts | 1 - .../src/integrations/hubspot/api/companies.ts | 100 --- .../integrations/hubspot/api/companyById.ts | 52 -- .../hubspot/api/contactAssociations.ts | 40 -- .../integrations/hubspot/api/contactById.ts | 52 -- .../src/integrations/hubspot/api/contacts.ts | 147 ---- .../src/integrations/hubspot/api/lists.ts | 44 -- .../integrations/hubspot/api/properties.ts | 33 - .../src/integrations/hubspot/api/tokenInfo.ts | 30 - .../src/integrations/hubspot/api/types.ts | 25 - .../api/utils/getOrganizationDomain.ts | 10 - .../field-mapper/hubspotFieldMapper.ts | 115 ---- .../hubspot/field-mapper/mapperFactory.ts | 25 - .../hubspot/field-mapper/memberFieldMapper.ts | 237 ------- .../field-mapper/organizationFieldMapper.ts | 301 --------- .../field-mapper/utils/serialization.ts | 19 - .../integrations/hubspot/generateStreams.ts | 67 -- .../src/integrations/hubspot/index.ts | 21 - .../integrations/hubspot/memberAttributes.ts | 23 - .../src/integrations/hubspot/processData.ts | 60 -- .../src/integrations/hubspot/processStream.ts | 115 ---- .../integrations/hubspot/processSyncRemote.ts | 148 ---- .../integrations/hubspot/startSyncRemote.ts | 58 -- .../src/integrations/hubspot/types.ts | 175 ----- .../integrations/src/integrations/index.ts | 7 - services/libs/integrations/src/types.ts | 34 - services/libs/queue/src/config.ts | 5 - services/libs/queue/src/types.ts | 1 - .../libs/queue/src/vendors/kafka/config.ts | 11 - services/libs/queue/src/vendors/sqs/config.ts | 12 - services/libs/types/src/automations.ts | 104 --- .../libs/types/src/enums/organizations.ts | 1 - services/libs/types/src/enums/platforms.ts | 4 - services/libs/types/src/enums/temporal.ts | 3 - services/libs/types/src/index.ts | 3 - .../queue/integration_sync_worker/index.ts | 36 - .../libs/types/src/temporal/automations.ts | 19 - services/libs/types/src/temporal/index.ts | 1 - 143 files changed, 24 insertions(+), 8854 deletions(-) delete mode 100644 backend/src/api/automation/automationCreate.ts delete mode 100644 backend/src/api/automation/automationDestroy.ts delete mode 100644 backend/src/api/automation/automationExecutionFind.ts delete mode 100644 backend/src/api/automation/automationFind.ts delete mode 100644 backend/src/api/automation/automationList.ts delete mode 100644 backend/src/api/automation/automationSlackCallback.ts delete mode 100644 backend/src/api/automation/automationSlackConnect.ts delete mode 100644 backend/src/api/automation/automationUpdate.ts delete mode 100644 backend/src/api/automation/index.ts delete mode 100644 backend/src/api/integration/helpers/hubspotConnect.ts delete mode 100644 backend/src/api/integration/helpers/hubspotGetLists.ts delete mode 100644 backend/src/api/integration/helpers/hubspotGetMappableFields.ts delete mode 100644 backend/src/api/integration/helpers/hubspotOnboard.ts delete mode 100644 backend/src/api/integration/helpers/hubspotStopSyncMember.ts delete mode 100644 backend/src/api/integration/helpers/hubspotStopSyncOrganization.ts delete mode 100644 backend/src/api/integration/helpers/hubspotSyncMember.ts delete mode 100644 backend/src/api/integration/helpers/hubspotSyncOrganization.ts delete mode 100644 backend/src/api/integration/helpers/hubspotUpdateProperties.ts delete mode 100644 backend/src/database/models/automation.ts delete mode 100644 backend/src/database/models/automationExecution.ts delete mode 100644 backend/src/database/repositories/automationExecutionRepository.ts delete mode 100644 backend/src/database/repositories/automationRepository.ts delete mode 100644 backend/src/database/repositories/memberSyncRemoteRepository.ts delete mode 100644 backend/src/database/repositories/organizationSyncRemoteRepository.ts delete mode 100644 backend/src/database/repositories/types/automationTypes.ts delete mode 100644 backend/src/services/automationExecutionService.ts delete mode 100644 backend/src/services/automationService.ts delete mode 100644 backend/src/types/automationTypes.ts delete mode 100644 scripts/builders/automations-worker.env delete mode 100644 scripts/builders/integration-sync-worker.env delete mode 100644 scripts/services/automations-worker.yaml delete mode 100644 scripts/services/docker/Dockerfile.automations_worker delete mode 100644 scripts/services/docker/Dockerfile.automations_worker.dockerignore delete mode 100644 scripts/services/docker/Dockerfile.integration_sync_worker delete mode 100644 scripts/services/docker/Dockerfile.integration_sync_worker.dockerignore delete mode 100644 scripts/services/integration-sync-worker.yaml delete mode 100644 services/apps/automations_worker/package.json delete mode 100644 services/apps/automations_worker/src/activities.ts delete mode 100644 services/apps/automations_worker/src/activities/newActivityAutomations.ts delete mode 100644 services/apps/automations_worker/src/activities/newMemberAutomations.ts delete mode 100644 services/apps/automations_worker/src/main.ts delete mode 100644 services/apps/automations_worker/src/services/automation.service.ts delete mode 100644 services/apps/automations_worker/src/services/slack/newActivityBlocks.ts delete mode 100644 services/apps/automations_worker/src/services/slack/newMemberBlocks.ts delete mode 100644 services/apps/automations_worker/src/workflows.ts delete mode 100644 services/apps/automations_worker/src/workflows/newActivityAutomations.ts delete mode 100644 services/apps/automations_worker/src/workflows/newMemberAutomations.ts delete mode 100644 services/apps/automations_worker/tsconfig.json delete mode 100644 services/apps/integration_sync_worker/config/custom-environment-variables.json delete mode 100644 services/apps/integration_sync_worker/config/default.json delete mode 100644 services/apps/integration_sync_worker/config/docker.json delete mode 100644 services/apps/integration_sync_worker/config/production.json delete mode 100644 services/apps/integration_sync_worker/config/staging.json delete mode 100644 services/apps/integration_sync_worker/package.json delete mode 100644 services/apps/integration_sync_worker/src/conf/index.ts delete mode 100644 services/apps/integration_sync_worker/src/errors.ts delete mode 100644 services/apps/integration_sync_worker/src/main.ts delete mode 100644 services/apps/integration_sync_worker/src/queue/index.ts delete mode 100644 services/apps/integration_sync_worker/src/service/member.sync.service.ts delete mode 100644 services/apps/integration_sync_worker/src/service/opensearch.data.ts delete mode 100644 services/apps/integration_sync_worker/src/service/opensearch.service.ts delete mode 100644 services/apps/integration_sync_worker/src/service/organization.sync.service.ts delete mode 100644 services/apps/integration_sync_worker/src/types.ts delete mode 100644 services/apps/integration_sync_worker/tsconfig.json delete mode 100644 services/libs/common_services/src/services/emitters/integrationSyncWorker.emitter.ts delete mode 100644 services/libs/data-access-layer/src/old/apps/automations_worker/automation.repo.ts delete mode 100644 services/libs/data-access-layer/src/old/apps/integration_run_worker/automation.repo.ts delete mode 100644 services/libs/data-access-layer/src/old/apps/integration_sync_worker/automation.repo.ts delete mode 100644 services/libs/data-access-layer/src/old/apps/integration_sync_worker/automationExecution.repo.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/api/addContactsToList.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/api/batchCreateMembers.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/api/batchCreateOrganizations.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/api/batchUpdateMembers.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/api/batchUpdateOrganizations.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/api/common.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/api/companies.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/api/companyById.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/api/contactAssociations.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/api/contactById.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/api/contacts.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/api/lists.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/api/properties.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/api/tokenInfo.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/api/types.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/api/utils/getOrganizationDomain.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/field-mapper/hubspotFieldMapper.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/field-mapper/mapperFactory.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/field-mapper/memberFieldMapper.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/field-mapper/organizationFieldMapper.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/field-mapper/utils/serialization.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/generateStreams.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/index.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/memberAttributes.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/processData.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/processStream.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/processSyncRemote.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/startSyncRemote.ts delete mode 100644 services/libs/integrations/src/integrations/hubspot/types.ts delete mode 100644 services/libs/types/src/automations.ts delete mode 100644 services/libs/types/src/queue/integration_sync_worker/index.ts delete mode 100644 services/libs/types/src/temporal/automations.ts diff --git a/backend/.env.dist.local b/backend/.env.dist.local index c4bb933dbc..6081f91955 100755 --- a/backend/.env.dist.local +++ b/backend/.env.dist.local @@ -109,7 +109,7 @@ CROWD_STACKEXCHANGE_KEY= # Nango settings CROWD_NANGO_URL=http://localhost:3003 CROWD_NANGO_SECRET_KEY=424242 -CROWD_NANGO_INTEGRATIONS=reddit,linkedin,stackexchange,hubspot +CROWD_NANGO_INTEGRATIONS=reddit,linkedin,stackexchange # Cohere settings CROWD_COHERE_API_KEY= diff --git a/backend/config/custom-environment-variables.json b/backend/config/custom-environment-variables.json index 91e5fcf1a8..739611c36e 100644 --- a/backend/config/custom-environment-variables.json +++ b/backend/config/custom-environment-variables.json @@ -120,11 +120,6 @@ "stackexchange": { "key": "CROWD_STACKEXCHANGE_KEY" }, - "hubspot": { - "appId": "CROWD_HUBSPOT_APP_ID", - "clientId": "CROWD_HUBSPOT_CLIENT_ID", - "clientSecret": "CROWD_HUBSPOT_CLIENT_SECRET" - }, "reddit": { "clientId": "CROWD_REDDIT_CLIENT_ID", "clientSecret": "CROWD_REDDIT_CLIENT_SECRET" diff --git a/backend/config/default.json b/backend/config/default.json index 8f5753a3e7..aac59ef26d 100644 --- a/backend/config/default.json +++ b/backend/config/default.json @@ -46,11 +46,11 @@ "crowdAnalytics": { "isEnabled": "false" }, - "temporal": { - "automationsTaskQueue": "automations" - }, + "temporal": {}, "searchSyncApi": {}, "encryption": {}, "openStatusApi": {}, - "gitlab": {} + "gitlab": {}, + "jiraIssueReporter": {}, + "snowflake": {} } diff --git a/backend/id-openapi.yaml b/backend/id-openapi.yaml index ff7c816df8..6b8c7ae5be 100644 --- a/backend/id-openapi.yaml +++ b/backend/id-openapi.yaml @@ -103,7 +103,7 @@ paths: description: Organization ID source: type: string - enum: ['ui', 'email-domain', 'enrichment', 'hubspot', 'github'] + enum: ['ui', 'email-domain', 'enrichment', 'github'] example: 'ui' description: Data source. For manual updates, always use 'ui' title: @@ -188,7 +188,7 @@ paths: description: Organization ID source: type: string - enum: ['ui', 'email-domain', 'enrichment', 'hubspot', 'github'] + enum: ['ui', 'email-domain', 'enrichment', 'github'] example: 'ui' description: Data source. For manual updates, always use 'ui' title: diff --git a/backend/src/api/auth/authMe.ts b/backend/src/api/auth/authMe.ts index 2844b54f37..7046d6dcf6 100644 --- a/backend/src/api/auth/authMe.ts +++ b/backend/src/api/auth/authMe.ts @@ -1,7 +1,5 @@ import { Error403 } from '@crowd/common' -import AutomationRepository from '@/database/repositories/automationRepository' - export default async (req, res) => { if (!req.currentUser || !req.currentUser.id) { await req.responseHandler.error(req, res, new Error403(req.language)) @@ -10,17 +8,5 @@ export default async (req, res) => { const payload = req.currentUser - payload.tenants = await Promise.all( - payload.tenants.map(async (tenantUser) => { - tenantUser.tenant.dataValues = { - ...tenantUser.tenant.dataValues, - automationCount: - Number(await AutomationRepository.countAllActive(req.database, tenantUser.tenant.id)) || - 0, - } - return tenantUser - }), - ) - await req.responseHandler.success(req, res, payload) } diff --git a/backend/src/api/automation/automationCreate.ts b/backend/src/api/automation/automationCreate.ts deleted file mode 100644 index b980b1bd73..0000000000 --- a/backend/src/api/automation/automationCreate.ts +++ /dev/null @@ -1,31 +0,0 @@ -import Permissions from '../../security/permissions' -import identifyTenant from '../../segment/identifyTenant' -import track from '../../segment/track' -import AutomationService from '../../services/automationService' -import PermissionChecker from '../../services/user/permissionChecker' - -/** - * POST /tenant/{tenantId}/automation - * @summary Create an automation - * @tag Automations - * @security Bearer - * @description Create a new automation for the tenant. - * @pathParam {string} tenantId - Your workspace/tenant ID - * @bodyContent {AutomationCreateInput} application/json - * @response 200 - Ok - * @responseContent {Automation} 200.application/json - * @responseExample {Automation} 200.application/json.Automation - * @response 401 - Unauthorized - * @response 429 - Too many requests - */ -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.automationCreate) - - const payload = await new AutomationService(req).create(req.body.data) - - track('Automation Created', { ...req.body.data }, { ...req }) - - identifyTenant(req) - - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/automation/automationDestroy.ts b/backend/src/api/automation/automationDestroy.ts deleted file mode 100644 index 5387f01420..0000000000 --- a/backend/src/api/automation/automationDestroy.ts +++ /dev/null @@ -1,27 +0,0 @@ -import Permissions from '../../security/permissions' -import identifyTenant from '../../segment/identifyTenant' -import track from '../../segment/track' -import AutomationService from '../../services/automationService' -import PermissionChecker from '../../services/user/permissionChecker' - -/** - * DELETE /tenant/{tenantId}/automation/{automationId} - * @summary Destroy an automation - * @tag Automations - * @security Bearer - * @description Destroys an existing automation in the tenant. - * @pathParam {string} tenantId - Your workspace/tenant ID - * @pathParam {string} automationId - Automation ID that you want to update - * @response 204 - Ok - No content - * @response 401 - Unauthorized - * @response 429 - Too many requests - */ -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.automationDestroy) - await new AutomationService(req).destroy(req.params.automationId) - - track('Automation Destroyed', { id: req.params.automationId }, { ...req }) - identifyTenant(req) - - await req.responseHandler.success(req, res, true, 204) -} diff --git a/backend/src/api/automation/automationExecutionFind.ts b/backend/src/api/automation/automationExecutionFind.ts deleted file mode 100644 index 89f06cabbd..0000000000 --- a/backend/src/api/automation/automationExecutionFind.ts +++ /dev/null @@ -1,40 +0,0 @@ -import Permissions from '../../security/permissions' -import AutomationExecutionService from '../../services/automationExecutionService' -import PermissionChecker from '../../services/user/permissionChecker' - -/** - * GET /tenant/{tenantId}/automation/{automationId}/executions - * @summary Get automation history - * @tag Automations - * @security Bearer - * @description Get all automation execution history for tenant and automation - * @pathParam {string} tenantId - Your workspace/tenant ID - * @pathParam {string} automationId - Your workspace/tenant ID - * @queryParam {integer} [offset=0] - How many elements from the beginning would you like to skip - * @queryParam {integer} [limit=10] - How many elements would you like to fetch - * @response 200 - Ok - * @responseContent {AutomationExecutionPage} 200.application/json - * @responseExample {AutomationExecutionPage} 200.application/json.AutomationExecutionPage - * @response 401 - Unauthorized - * @response 429 - Too many requests - */ -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.automationRead) - - let offset = 0 - if (req.query.offset) { - offset = parseInt(req.query.offset, 10) - } - let limit = 10 - if (req.query.limit) { - limit = parseInt(req.query.limit, 10) - } - - const payload = await new AutomationExecutionService(req).findAndCountAll({ - automationId: req.params.automationId, - offset, - limit, - }) - - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/automation/automationFind.ts b/backend/src/api/automation/automationFind.ts deleted file mode 100644 index 8784f624ca..0000000000 --- a/backend/src/api/automation/automationFind.ts +++ /dev/null @@ -1,24 +0,0 @@ -import Permissions from '../../security/permissions' -import AutomationService from '../../services/automationService' -import PermissionChecker from '../../services/user/permissionChecker' - -/** - * GET /tenant/{tenantId}/automation/{automationId} - * @summary Find an automation - * @tag Automations - * @security Bearer - * @description Get an existing automation data in the tenant. - * @pathParam {string} tenantId - Your workspace/tenant ID - * @pathParam {string} automationId - Automation ID that you want to find - * @response 200 - Ok - * @responseContent {Automation} 200.application/json - * @responseExample {Automation} 200.application/json.Automation - * @response 401 - Unauthorized - * @response 429 - Too many requests - */ -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.automationRead) - const payload = await new AutomationService(req).findById(req.params.automationId) - - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/automation/automationList.ts b/backend/src/api/automation/automationList.ts deleted file mode 100644 index 3fc770ee10..0000000000 --- a/backend/src/api/automation/automationList.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { AutomationState, AutomationTrigger, AutomationType } from '@crowd/types' - -import Permissions from '../../security/permissions' -import AutomationService from '../../services/automationService' -import PermissionChecker from '../../services/user/permissionChecker' -import { AutomationCriteria } from '../../types/automationTypes' - -/** - * GET /tenant/{tenantId}/automation - * @summary List automations - * @tag Automations - * @security Bearer - * @description Get all existing automation data for tenant. - * @pathParam {string} tenantId - Your workspace/tenant ID - * @queryParam {string} [filter[type]] - Filter by type of automation - * @queryParam {string} [filter[trigger]] - Filter by trigger type of automation - * @queryParam {string} [filter[state]] - Filter by state of automation - * @queryParam {number} [offset] - Skip the first n results. Default 0. - * @queryParam {number} [limit] - Limit the number of results. Default 50. - * @response 200 - Ok - * @responseContent {AutomationPage} 200.application/json - * @responseExample {AutomationPage} 200.application/json.AutomationPage - * @response 401 - Unauthorized - * @response 429 - Too many requests - */ -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.automationRead) - - let offset = 0 - if (req.query.offset) { - offset = parseInt(req.query.offset, 10) - } - let limit = 50 - if (req.query.limit) { - limit = parseInt(req.query.limit, 10) - } - - const criteria: AutomationCriteria = { - type: req.query.filter?.type ? (req.query.filter.type as AutomationType) : undefined, - trigger: req.query.filter?.trigger - ? (req.query.filter?.trigger as AutomationTrigger) - : undefined, - state: req.query.filter?.state ? (req.query.filter.state as AutomationState) : undefined, - limit, - offset, - } - - const payload = await new AutomationService(req).findAndCountAll(criteria) - - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/automation/automationSlackCallback.ts b/backend/src/api/automation/automationSlackCallback.ts deleted file mode 100644 index 609035c9af..0000000000 --- a/backend/src/api/automation/automationSlackCallback.ts +++ /dev/null @@ -1,19 +0,0 @@ -import Axios from 'axios' - -import Permissions from '../../security/permissions' -import SettingsService from '../../services/settingsService' -import PermissionChecker from '../../services/user/permissionChecker' - -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.automationCreate) - const { redirectUrl } = JSON.parse(Buffer.from(req.query.state, 'base64').toString()) - - const { url } = req.account - - await SettingsService.save({ slackWebHook: url }, req) - await Axios.post(url, { - text: 'crowd.dev notifier has been successfully connected.', - }) - - res.redirect(redirectUrl) -} diff --git a/backend/src/api/automation/automationSlackConnect.ts b/backend/src/api/automation/automationSlackConnect.ts deleted file mode 100644 index 3be7282be4..0000000000 --- a/backend/src/api/automation/automationSlackConnect.ts +++ /dev/null @@ -1,21 +0,0 @@ -import passport from 'passport' - -import Permissions from '../../security/permissions' -import { getSlackNotifierStrategy } from '../../services/auth/passportStrategies/slackStrategy' -import PermissionChecker from '../../services/user/permissionChecker' - -export default async (req, res, next) => { - new PermissionChecker(req).validateHas(Permissions.values.automationCreate) - const state = { - tenantId: req.params.tenantId, - redirectUrl: req.query.redirectUrl, - crowdToken: req.query.crowdToken, - } - - const authenticator = passport.authenticate(getSlackNotifierStrategy(), { - scope: ['incoming-webhook'], - state: Buffer.from(JSON.stringify(state)).toString('base64'), - }) - - authenticator(req, res, next) -} diff --git a/backend/src/api/automation/automationUpdate.ts b/backend/src/api/automation/automationUpdate.ts deleted file mode 100644 index c0fbed6948..0000000000 --- a/backend/src/api/automation/automationUpdate.ts +++ /dev/null @@ -1,32 +0,0 @@ -import Permissions from '../../security/permissions' -import track from '../../segment/track' -import AutomationService from '../../services/automationService' -import PermissionChecker from '../../services/user/permissionChecker' - -/** - * PUT /tenant/{tenantId}/automation/{automationId} - * @summary Update an automation - * @tag Automations - * @security Bearer - * @description Updates an existing automation in the tenant. - * @pathParam {string} tenantId - Your workspace/tenant ID - * @pathParam {string} automationId - Automation ID that you want to update - * @bodyContent {AutomationUpdateInput} application/json - * @response 200 - Ok - * @responseContent {Automation} 200.application/json - * @responseExample {Automation} 200.application/json.Automation - * @response 401 - Unauthorized - * @response 429 - Too many requests - */ -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.automationUpdate) - const payload = await new AutomationService(req).update(req.params.automationId, req.body.data) - - track( - 'Automation Updated', - { automationId: req.params.automationId, data: req.body.data }, - { ...req }, - ) - - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/automation/index.ts b/backend/src/api/automation/index.ts deleted file mode 100644 index 63fa7eacd2..0000000000 --- a/backend/src/api/automation/index.ts +++ /dev/null @@ -1,51 +0,0 @@ -import passport from 'passport' - -import { API_CONFIG } from '../../conf' -import { authMiddleware } from '../../middlewares/authMiddleware' -import { safeWrap } from '../../middlewares/errorMiddleware' -import { getSlackNotifierStrategy } from '../../services/auth/passportStrategies/slackStrategy' -import TenantService from '../../services/tenantService' - -export default (app) => { - app.get( - '/tenant/:tenantId/automation/slack', - safeWrap(require('./automationSlackConnect').default), - ) - app.get( - '/tenant/automation/slack/callback', - passport.authorize(getSlackNotifierStrategy(), { - session: false, - failureRedirect: `${API_CONFIG.frontendUrl}/settings?activeTab=automations&error=true`, - }), - (req, _res, next) => { - const { crowdToken } = JSON.parse(Buffer.from(req.query.state, 'base64').toString()) - req.headers.authorization = `Bearer ${crowdToken}` - next() - }, - authMiddleware, - async (req, _res, next) => { - const { tenantId } = JSON.parse(Buffer.from(req.query.state, 'base64').toString()) - req.currentTenant = await new TenantService(req).findById(tenantId) - next() - }, - safeWrap(require('./automationSlackCallback').default), - ) - app.post('/tenant/:tenantId/automation', safeWrap(require('./automationCreate').default)) - app.put( - '/tenant/:tenantId/automation/:automationId', - safeWrap(require('./automationUpdate').default), - ) - app.delete( - '/tenant/:tenantId/automation/:automationId', - safeWrap(require('./automationDestroy').default), - ) - app.get( - '/tenant/:tenantId/automation/:automationId/executions', - safeWrap(require('./automationExecutionFind').default), - ) - app.get( - '/tenant/:tenantId/automation/:automationId', - safeWrap(require('./automationFind').default), - ) - app.get('/tenant/:tenantId/automation', safeWrap(require('./automationList').default)) -} diff --git a/backend/src/api/index.ts b/backend/src/api/index.ts index 562f17b446..82453d85f7 100644 --- a/backend/src/api/index.ts +++ b/backend/src/api/index.ts @@ -193,7 +193,6 @@ setImmediate(async () => { require('./microservice').default(routes) require('./conversation').default(routes) require('./eagleEyeContent').default(routes) - require('./automation').default(routes) require('./organization').default(routes) require('./slack').default(routes) require('./segment').default(routes) diff --git a/backend/src/api/integration/helpers/hubspotConnect.ts b/backend/src/api/integration/helpers/hubspotConnect.ts deleted file mode 100644 index 4b6332d3ae..0000000000 --- a/backend/src/api/integration/helpers/hubspotConnect.ts +++ /dev/null @@ -1,9 +0,0 @@ -import Permissions from '../../../security/permissions' -import IntegrationService from '../../../services/integrationService' -import PermissionChecker from '../../../services/user/permissionChecker' - -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.integrationEdit) - const payload = await new IntegrationService(req).hubspotConnect() - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/integration/helpers/hubspotGetLists.ts b/backend/src/api/integration/helpers/hubspotGetLists.ts deleted file mode 100644 index 9723f44ef4..0000000000 --- a/backend/src/api/integration/helpers/hubspotGetLists.ts +++ /dev/null @@ -1,9 +0,0 @@ -import Permissions from '../../../security/permissions' -import IntegrationService from '../../../services/integrationService' -import PermissionChecker from '../../../services/user/permissionChecker' - -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.integrationEdit) - const payload = await new IntegrationService(req).hubspotGetLists() - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/integration/helpers/hubspotGetMappableFields.ts b/backend/src/api/integration/helpers/hubspotGetMappableFields.ts deleted file mode 100644 index 524f1846b1..0000000000 --- a/backend/src/api/integration/helpers/hubspotGetMappableFields.ts +++ /dev/null @@ -1,9 +0,0 @@ -import Permissions from '../../../security/permissions' -import IntegrationService from '../../../services/integrationService' -import PermissionChecker from '../../../services/user/permissionChecker' - -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) - const payload = await new IntegrationService(req).hubspotGetMappableFields() - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/integration/helpers/hubspotOnboard.ts b/backend/src/api/integration/helpers/hubspotOnboard.ts deleted file mode 100644 index 68b447238a..0000000000 --- a/backend/src/api/integration/helpers/hubspotOnboard.ts +++ /dev/null @@ -1,9 +0,0 @@ -import Permissions from '../../../security/permissions' -import IntegrationService from '../../../services/integrationService' -import PermissionChecker from '../../../services/user/permissionChecker' - -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) - const payload = await new IntegrationService(req).hubspotOnboard(req.body) - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/integration/helpers/hubspotStopSyncMember.ts b/backend/src/api/integration/helpers/hubspotStopSyncMember.ts deleted file mode 100644 index 8843e17d55..0000000000 --- a/backend/src/api/integration/helpers/hubspotStopSyncMember.ts +++ /dev/null @@ -1,9 +0,0 @@ -import Permissions from '../../../security/permissions' -import IntegrationService from '../../../services/integrationService' -import PermissionChecker from '../../../services/user/permissionChecker' - -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.integrationEdit) - const payload = await new IntegrationService(req).hubspotStopSyncMember(req.body) - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/integration/helpers/hubspotStopSyncOrganization.ts b/backend/src/api/integration/helpers/hubspotStopSyncOrganization.ts deleted file mode 100644 index ef84437137..0000000000 --- a/backend/src/api/integration/helpers/hubspotStopSyncOrganization.ts +++ /dev/null @@ -1,9 +0,0 @@ -import Permissions from '../../../security/permissions' -import IntegrationService from '../../../services/integrationService' -import PermissionChecker from '../../../services/user/permissionChecker' - -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.integrationEdit) - const payload = await new IntegrationService(req).hubspotStopSyncOrganization(req.body) - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/integration/helpers/hubspotSyncMember.ts b/backend/src/api/integration/helpers/hubspotSyncMember.ts deleted file mode 100644 index ce9d23fe4b..0000000000 --- a/backend/src/api/integration/helpers/hubspotSyncMember.ts +++ /dev/null @@ -1,9 +0,0 @@ -import Permissions from '../../../security/permissions' -import IntegrationService from '../../../services/integrationService' -import PermissionChecker from '../../../services/user/permissionChecker' - -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.integrationEdit) - const payload = await new IntegrationService(req).hubspotSyncMember(req.body) - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/integration/helpers/hubspotSyncOrganization.ts b/backend/src/api/integration/helpers/hubspotSyncOrganization.ts deleted file mode 100644 index f540146516..0000000000 --- a/backend/src/api/integration/helpers/hubspotSyncOrganization.ts +++ /dev/null @@ -1,9 +0,0 @@ -import Permissions from '../../../security/permissions' -import IntegrationService from '../../../services/integrationService' -import PermissionChecker from '../../../services/user/permissionChecker' - -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.integrationEdit) - const payload = await new IntegrationService(req).hubspotSyncOrganization(req.body) - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/integration/helpers/hubspotUpdateProperties.ts b/backend/src/api/integration/helpers/hubspotUpdateProperties.ts deleted file mode 100644 index cbd4e92d6a..0000000000 --- a/backend/src/api/integration/helpers/hubspotUpdateProperties.ts +++ /dev/null @@ -1,9 +0,0 @@ -import Permissions from '../../../security/permissions' -import IntegrationService from '../../../services/integrationService' -import PermissionChecker from '../../../services/user/permissionChecker' - -export default async (req, res) => { - new PermissionChecker(req).validateHas(Permissions.values.tenantEdit) - const payload = await new IntegrationService(req).hubspotUpdateProperties() - await req.responseHandler.success(req, res, payload) -} diff --git a/backend/src/api/integration/index.ts b/backend/src/api/integration/index.ts index 85c7b12d22..1a00029afc 100644 --- a/backend/src/api/integration/index.ts +++ b/backend/src/api/integration/index.ts @@ -128,51 +128,6 @@ export default (app) => { safeWrap(require('./helpers/discourseTestWebhook').default), ) - app.post( - '/tenant/:tenantId/hubspot-connect', - safeWrap(require('./helpers/hubspotConnect').default), - ) - - app.post( - '/tenant/:tenantId/hubspot-onboard', - safeWrap(require('./helpers/hubspotOnboard').default), - ) - - app.post( - '/tenant/:tenantId/hubspot-update-properties', - safeWrap(require('./helpers/hubspotUpdateProperties').default), - ) - - app.get( - '/tenant/:tenantId/hubspot-mappable-fields', - safeWrap(require('./helpers/hubspotGetMappableFields').default), - ) - - app.get( - '/tenant/:tenantId/hubspot-get-lists', - safeWrap(require('./helpers/hubspotGetLists').default), - ) - - app.post( - '/tenant/:tenantId/hubspot-sync-member', - safeWrap(require('./helpers/hubspotSyncMember').default), - ) - - app.post( - '/tenant/:tenantId/hubspot-stop-sync-member', - safeWrap(require('./helpers/hubspotStopSyncMember').default), - ) - - app.post( - '/tenant/:tenantId/hubspot-sync-organization', - safeWrap(require('./helpers/hubspotSyncOrganization').default), - ) - - app.post( - '/tenant/:tenantId/hubspot-stop-sync-organization', - safeWrap(require('./helpers/hubspotStopSyncOrganization').default), - ) - app.post( '/tenant/:tenantId/groupsio-connect', safeWrap(require('./helpers/groupsioConnectOrUpdate').default), diff --git a/backend/src/api/tenant/tenantFind.ts b/backend/src/api/tenant/tenantFind.ts index b9ffaf07b4..2b12000341 100644 --- a/backend/src/api/tenant/tenantFind.ts +++ b/backend/src/api/tenant/tenantFind.ts @@ -1,7 +1,5 @@ import { Error404 } from '@crowd/common' -import AutomationRepository from '@/database/repositories/automationRepository' - import Permissions from '../../security/permissions' import identifyTenant from '../../segment/identifyTenant' import TenantService from '../../services/tenantService' @@ -17,12 +15,6 @@ export default async (req, res) => { payload = await new TenantService(req).findByUrl(req.query.url) } - payload.dataValues = { - ...payload.dataValues, - automationCount: - Number(await AutomationRepository.countAllActive(req.database, payload.id)) || 0, - } - payload.dataValues.settings[0].dataValues = { ...payload.dataValues.settings[0].dataValues, slackWebHook: !!payload.settings[0].dataValues.slackWebHook, diff --git a/backend/src/conf/configTypes.ts b/backend/src/conf/configTypes.ts index 47bf548c4b..fa5a108683 100644 --- a/backend/src/conf/configTypes.ts +++ b/backend/src/conf/configTypes.ts @@ -1,5 +1,3 @@ -import { ITemporalConfig } from '@crowd/temporal' - export enum ServiceType { API = 'api', JOB_GENERATOR = 'job-generator', @@ -101,11 +99,6 @@ export interface SlackConfiguration { appToken?: string } -export interface SlackNotifierConfiguration { - clientId: string - clientSecret: string -} - export interface GoogleConfiguration { clientId: string clientSecret: string @@ -179,10 +172,6 @@ export interface CrowdAnalyticsConfiguration { apiToken: string } -export interface IBackendTemporalConfig extends ITemporalConfig { - automationsTaskQueue: string -} - export interface EncryptionConfiguration { secretKey: string initVector: string diff --git a/backend/src/conf/index.ts b/backend/src/conf/index.ts index 727b4b37e9..d2fe5b5185 100644 --- a/backend/src/conf/index.ts +++ b/backend/src/conf/index.ts @@ -4,6 +4,7 @@ import { IDatabaseConfig } from '@crowd/data-access-layer/src/database' import { ISearchSyncApiConfig } from '@crowd/opensearch' import { IQueueClientConfig } from '@crowd/queue' import { IRedisConfiguration } from '@crowd/redis' +import { ITemporalConfig } from '@crowd/temporal' import { IGithubIssueReporterConfiguration, IJiraIssueReporterConfiguration } from '@crowd/types' import { @@ -21,7 +22,6 @@ import { GithubTokenConfiguration, GitlabConfiguration, GoogleConfiguration, - IBackendTemporalConfig, IOpenSearchConfig, IOpenStatusApiConfig, IRedditConfig, @@ -34,7 +34,6 @@ import { ServiceType, SlackAlertingConfiguration, SlackConfiguration, - SlackNotifierConfiguration, SnowflakeConfiguration, StackExchangeConfiguration, TenantMode, @@ -100,9 +99,6 @@ export const TWITTER_CONFIG: TwitterConfiguration = config.get('slack') -export const SLACK_NOTIFIER_CONFIG: SlackNotifierConfiguration = - config.get('slackNotifier') - export const GOOGLE_CONFIG: GoogleConfiguration = config.get('google') export const DISCORD_CONFIG: DiscordConfiguration = config.get('discord') @@ -144,8 +140,7 @@ export const INTEGRATION_PROCESSING_CONFIG: IntegrationProcessingConfiguration = export const CROWD_ANALYTICS_CONFIG: CrowdAnalyticsConfiguration = config.get('crowdAnalytics') -export const TEMPORAL_CONFIG: IBackendTemporalConfig = - config.get('temporal') +export const TEMPORAL_CONFIG: ITemporalConfig = config.get('temporal') export const SEARCH_SYNC_API_CONFIG: ISearchSyncApiConfig = config.get('searchSyncApi') diff --git a/backend/src/database/models/automation.ts b/backend/src/database/models/automation.ts deleted file mode 100644 index a2726ba29e..0000000000 --- a/backend/src/database/models/automation.ts +++ /dev/null @@ -1,74 +0,0 @@ -import { DataTypes } from 'sequelize' - -export default (sequelize) => { - const automation = sequelize.define( - 'automation', - { - id: { - type: DataTypes.UUID, - defaultValue: DataTypes.UUIDV4, - primaryKey: true, - allowNull: false, - }, - type: { - type: DataTypes.STRING(80), - allowNull: false, - validate: { - notEmpty: true, - }, - }, - name: { - type: DataTypes.STRING(255), - }, - tenantId: { - type: DataTypes.UUID, - allowNull: false, - }, - trigger: { - type: DataTypes.STRING(80), - allowNull: false, - validate: { - notEmpty: true, - }, - }, - settings: { - type: DataTypes.JSONB, - allowNull: false, - }, - state: { - type: DataTypes.STRING(80), - allowNull: false, - validate: { - notEmpty: true, - }, - }, - }, - { - indexes: [ - { - fields: ['type', 'tenantId', 'trigger', 'state'], - }, - ], - timestamps: true, - }, - ) - - automation.associate = (models) => { - models.automation.belongsTo(models.tenant, { - as: 'tenant', - foreignKey: { - allowNull: false, - }, - }) - - models.automation.belongsTo(models.user, { - as: 'createdBy', - }) - - models.automation.belongsTo(models.user, { - as: 'updatedBy', - }) - } - - return automation -} diff --git a/backend/src/database/models/automationExecution.ts b/backend/src/database/models/automationExecution.ts deleted file mode 100644 index 72b59291ce..0000000000 --- a/backend/src/database/models/automationExecution.ts +++ /dev/null @@ -1,90 +0,0 @@ -import { DataTypes } from 'sequelize' - -export default (sequelize) => { - const automationExecution = sequelize.define( - 'automationExecution', - { - id: { - type: DataTypes.UUID, - defaultValue: DataTypes.UUIDV4, - primaryKey: true, - allowNull: false, - }, - automationId: { - type: DataTypes.UUID, - allowNull: false, - }, - type: { - type: DataTypes.STRING(80), - allowNull: false, - validate: { - notEmpty: true, - }, - }, - tenantId: { - type: DataTypes.UUID, - allowNull: false, - }, - trigger: { - type: DataTypes.STRING(80), - allowNull: false, - validate: { - notEmpty: true, - }, - }, - state: { - type: DataTypes.STRING(80), - allowNull: false, - validate: { - notEmpty: true, - }, - }, - error: { - type: DataTypes.JSON, - allowNull: true, - }, - executedAt: { - type: DataTypes.DATE, - allowNull: false, - }, - eventId: { - type: DataTypes.STRING(255), - allowNull: false, - validate: { - notEmpty: true, - }, - }, - payload: { - type: DataTypes.JSON, - allowNull: false, - }, - }, - { - indexes: [ - { - fields: ['automationId'], - }, - ], - timestamps: false, - paranoid: false, - }, - ) - - automationExecution.associate = (models) => { - models.automationExecution.belongsTo(models.tenant, { - as: 'tenant', - foreignKey: { - allowNull: false, - }, - }) - - models.automationExecution.belongsTo(models.automation, { - as: 'automation', - foreignKey: { - allowNull: false, - }, - }) - } - - return automationExecution -} diff --git a/backend/src/database/models/index.ts b/backend/src/database/models/index.ts index 28816ce2aa..6806261462 100644 --- a/backend/src/database/models/index.ts +++ b/backend/src/database/models/index.ts @@ -131,8 +131,6 @@ async function models(queryTimeoutMilliseconds: number, databaseHostnameOverride require('./conversationSettings').default, require('./eagleEyeContent').default, require('./eagleEyeAction').default, - require('./automation').default, - require('./automationExecution').default, require('./organization').default, require('./memberAttributeSettings').default, require('./segment').default, diff --git a/backend/src/database/repositories/automationExecutionRepository.ts b/backend/src/database/repositories/automationExecutionRepository.ts deleted file mode 100644 index 9d38fe03ff..0000000000 --- a/backend/src/database/repositories/automationExecutionRepository.ts +++ /dev/null @@ -1,170 +0,0 @@ -/* eslint-disable class-methods-use-this */ - -/* eslint-disable @typescript-eslint/no-unused-vars */ -import { QueryTypes } from 'sequelize' - -import { AutomationExecutionState, PageData } from '@crowd/types' - -import { AutomationExecution, AutomationExecutionCriteria } from '../../types/automationTypes' - -import { IRepositoryOptions } from './IRepositoryOptions' -import { RepositoryBase } from './repositoryBase' -import { DbAutomationExecutionInsertData } from './types/automationTypes' - -export default class AutomationExecutionRepository extends RepositoryBase< - AutomationExecution, - string, - DbAutomationExecutionInsertData, - unknown, - AutomationExecutionCriteria -> { - public constructor(options: IRepositoryOptions) { - super(options, false) - } - - override async create(data: DbAutomationExecutionInsertData): Promise { - const transaction = this.transaction - - return this.database.automationExecution.create( - { - automationId: data.automationId, - type: data.type, - tenantId: data.tenantId, - trigger: data.trigger, - state: data.state, - error: data.error, - executedAt: data.executedAt, - eventId: data.eventId, - payload: data.payload, - }, - { transaction }, - ) - } - - override async findAndCountAll( - criteria: AutomationExecutionCriteria, - ): Promise> { - // get current tenant that was used to make a request - const currentTenant = this.currentTenant - - // get plain sequelize object to use with a raw query - const seq = this.seq - - // construct a query with pagination - const query = ` - select id, - "automationId", - state, - error, - "executedAt", - "eventId", - payload, - count(*) over () as "paginatedItemsCount" - from "automationExecutions" - where "tenantId" = :tenantId - and "automationId" = :automationId - order by "executedAt" desc - limit ${criteria.limit} offset ${criteria.offset} - - ` - - const results = await seq.query(query, { - replacements: { - tenantId: currentTenant.id, - automationId: criteria.automationId, - }, - type: QueryTypes.SELECT, - }) - - if (results.length === 0) { - return { - rows: [], - count: 0, - offset: criteria.offset, - limit: criteria.limit, - } - } - - const count = parseInt((results[0] as any).paginatedItemsCount, 10) - const rows: AutomationExecution[] = results.map((r) => { - const row = r as any - return { - id: row.id, - automationId: row.automationId, - executedAt: row.executedAt, - eventId: row.eventId, - payload: row.payload, - error: row.error, - state: row.state, - } - }) - - return { - rows, - count, - offset: criteria.offset, - limit: criteria.limit, - } - } - - public async hasAlreadyBeenTriggered(automationId: string, eventId: string): Promise { - const seq = this.seq - - const query = ` - select id - from "automationExecutions" - where "automationId" = :automationId - and "eventId" = :eventId - and state = '${AutomationExecutionState.SUCCESS}'; - ` - - const results = await seq.query(query, { - replacements: { - automationId, - eventId, - }, - type: QueryTypes.SELECT, - }) - - return results.length > 0 - } - - override async update(id: string, data: unknown): Promise { - throw new Error('Method not implemented.') - } - - override async destroy(id: string): Promise { - throw new Error('Method not implemented.') - } - - async destroyAllAutomation(automationIds: string[]): Promise { - const transaction = this.transaction - - const seq = this.seq - - const currentTenant = this.currentTenant - - const query = ` - delete - from "automationExecutions" - where "automationId" in (:automationIds) - and "tenantId" = :tenantId;` - - await seq.query(query, { - replacements: { - automationIds, - tenantId: currentTenant.id, - }, - type: QueryTypes.DELETE, - transaction, - }) - } - - override async destroyAll(ids: string[]): Promise { - throw new Error('Method not implemented.') - } - - override async findById(id: string): Promise { - throw new Error('Method not implemented.') - } -} diff --git a/backend/src/database/repositories/automationRepository.ts b/backend/src/database/repositories/automationRepository.ts deleted file mode 100644 index a09c79480d..0000000000 --- a/backend/src/database/repositories/automationRepository.ts +++ /dev/null @@ -1,295 +0,0 @@ -import Sequelize, { QueryTypes } from 'sequelize' - -import { Error404 } from '@crowd/common' -import { AutomationState, AutomationSyncTrigger, IAutomationData, PageData } from '@crowd/types' - -import { AutomationCriteria } from '../../types/automationTypes' - -import { IRepositoryOptions } from './IRepositoryOptions' -import AuditLogRepository from './auditLogRepository' -import { RepositoryBase } from './repositoryBase' -import { DbAutomationInsertData, DbAutomationUpdateData } from './types/automationTypes' - -const { Op } = Sequelize - -export default class AutomationRepository extends RepositoryBase< - IAutomationData, - string, - DbAutomationInsertData, - DbAutomationUpdateData, - AutomationCriteria -> { - public constructor(options: IRepositoryOptions) { - super(options, true) - } - - override async create(data: DbAutomationInsertData): Promise { - const currentUser = this.currentUser - - const tenant = this.currentTenant - - const transaction = this.transaction - - const record = await this.database.automation.create( - { - name: data.name, - type: data.type, - trigger: data.trigger, - settings: data.settings, - state: data.state, - tenantId: tenant.id, - createdById: currentUser.id, - updatedById: currentUser.id, - }, - { - transaction, - }, - ) - - await this.createAuditLog('automation', AuditLogRepository.CREATE, record, data) - - return this.findById(record.id) - } - - override async update(id, data: DbAutomationUpdateData): Promise { - const currentUser = this.currentUser - - const currentTenant = this.currentTenant - - const transaction = this.transaction - - let record = await this.database.automation.findOne({ - where: { - id, - tenantId: currentTenant.id, - }, - transaction, - }) - - if (!record) { - throw new Error404() - } - - record = await record.update( - { - name: data.name, - trigger: data.trigger, - settings: data.settings, - state: data.state, - updatedById: currentUser.id, - }, - { - transaction, - }, - ) - - await this.createAuditLog('automation', AuditLogRepository.UPDATE, record, data) - - return this.findById(record.id) - } - - override async destroyAll(ids: string[]): Promise { - const transaction = this.transaction - - const currentTenant = this.currentTenant - - const records = await this.database.automation.findAll({ - where: { - id: { - [Op.in]: ids, - }, - tenantId: currentTenant.id, - }, - transaction, - }) - - if (ids.some((id) => records.find((r) => r.id === id) === undefined)) { - throw new Error404() - } - - await Promise.all( - records.flatMap((r) => [ - r.destroy({ transaction }), - this.createAuditLog('automation', AuditLogRepository.DELETE, r, r), - ]), - ) - } - - override async findById(id: string): Promise { - const results = await this.findAndCountAll({ - id, - offset: 0, - limit: 1, - }) - - if (results.count === 1) { - return results.rows[0] - } - - if (results.count === 0) { - throw new Error404() - } - - throw new Error('More than one row returned when fetching by automation unique ID!') - } - - override async findAndCountAll(criteria: AutomationCriteria): Promise> { - // get current tenant that was used to make a request - const currentTenant = this.currentTenant - - // we need transaction if there is one set because some records were perhaps created/updated in the same transaction - const transaction = this.transaction - - // get plain sequelize object to use with a raw query - const seq = this.seq - - // build a where condition based on tenant and other criteria passed as parameter - const conditions = ['a."tenantId" = :tenantId'] - const parameters: any = { - tenantId: currentTenant.id, - } - - if (criteria.id) { - conditions.push('a.id = :id') - parameters.id = criteria.id - } - - if (criteria.state) { - conditions.push('a.state = :state') - parameters.state = criteria.state - } - - if (criteria.type) { - conditions.push('a.type = :type') - parameters.type = criteria.type - } - - if (criteria.trigger) { - conditions.push('a.trigger = :trigger') - parameters.trigger = criteria.trigger - } - - const conditionsString = conditions.join(' and ') - - const query = ` - -- common table expression (CTE) to prepare the last execution information for each automationId - with latest_executions as (select distinct on ("automationId") "automationId", "executedAt", state, error - from "automationExecutions" - order by "automationId", "executedAt" desc) - select a.id, - a.name, - a.type, - a."tenantId", - a.trigger, - a.settings, - a.state, - a."createdAt", - a."updatedAt", - le."executedAt" as "lastExecutionAt", - le.state as "lastExecutionState", - le.error as "lastExecutionError", - count(*) over () as "paginatedItemsCount" - from automations a - left join latest_executions le on a.id = le."automationId" - where ${conditionsString} - order by "updatedAt" desc - ${this.getPaginationString(criteria)} - ` - // fetch all automations for a tenant - // and include the latest execution data if available - const results = await seq.query(query, { - replacements: parameters, - type: QueryTypes.SELECT, - transaction, - }) - - if (results.length === 0) { - return { - rows: [], - count: 0, - offset: criteria.offset, - limit: criteria.limit, - } - } - - const count = parseInt((results[0] as any).paginatedItemsCount, 10) - const rows: IAutomationData[] = results.map((r) => { - const row = r as any - return { - id: row.id, - name: row.name, - type: row.type, - tenantId: row.tenantId, - trigger: row.trigger, - settings: row.settings, - state: row.state, - createdAt: row.createdAt, - updatedAt: row.updatedAt, - lastExecutionAt: row.lastExecutionAt, - lastExecutionState: row.lastExecutionState, - lastExecutionError: row.lastExecutionError, - } - }) - - return { - rows, - count, - offset: criteria.offset, - limit: criteria.limit, - } - } - - static async countAllActive(database: any, tenantId: string): Promise { - const automationCount = await database.automation.count({ - where: { - tenantId, - state: AutomationState.ACTIVE, - }, - useMaster: true, - }) - - return automationCount - } - - public async findSyncAutomations( - tenantId: string, - platform: string, - ): Promise { - const seq = this.seq - - const transaction = this.transaction - - const pageSize = 10 - const syncAutomations: IAutomationData[] = [] - - let results - let offset - - do { - offset = results ? pageSize + offset : 0 - - results = await seq.query( - `select * from automations - where type = :platform and "tenantId" = :tenantId and trigger in (:syncAutomationTriggers) - limit :limit offset :offset`, - { - replacements: { - tenantId, - platform, - syncAutomationTriggers: [ - AutomationSyncTrigger.MEMBER_ATTRIBUTES_MATCH, - AutomationSyncTrigger.ORGANIZATION_ATTRIBUTES_MATCH, - ], - limit: pageSize, - offset, - }, - type: QueryTypes.SELECT, - transaction, - }, - ) - syncAutomations.push(...results) - } while (results.length > 0) - - return syncAutomations - } -} diff --git a/backend/src/database/repositories/integrationRepository.ts b/backend/src/database/repositories/integrationRepository.ts index 4617d53e16..ecd49e3ff2 100644 --- a/backend/src/database/repositories/integrationRepository.ts +++ b/backend/src/database/repositories/integrationRepository.ts @@ -16,12 +16,8 @@ import SequelizeFilterUtils from '../utils/sequelizeFilterUtils' import { IRepositoryOptions } from './IRepositoryOptions' import AuditLogRepository from './auditLogRepository' -import AutomationExecutionRepository from './automationExecutionRepository' -import AutomationRepository from './automationRepository' import QueryParser from './filters/queryParser' import { QueryOutput } from './filters/queryTypes' -import MemberSyncRemoteRepository from './memberSyncRemoteRepository' -import OrganizationSyncRemoteRepository from './organizationSyncRemoteRepository' import SequelizeRepository from './sequelizeRepository' const { Op } = Sequelize @@ -172,30 +168,6 @@ class IntegrationRepository { }, ) - // delete syncRemote rows coming from integration - await new MemberSyncRemoteRepository({ ...options, transaction }).destroyAllIntegration([ - record.id, - ]) - await new OrganizationSyncRemoteRepository({ ...options, transaction }).destroyAllIntegration([ - record.id, - ]) - - // destroy existing automations for outgoing integrations - const syncAutomationIds = ( - await new AutomationRepository({ ...options, transaction }).findSyncAutomations( - currentTenant.id, - record.platform, - ) - ).map((a) => a.id) - - if (syncAutomationIds.length > 0) { - await new AutomationExecutionRepository({ ...options, transaction }).destroyAllAutomation( - syncAutomationIds, - ) - } - - await new AutomationRepository({ ...options, transaction }).destroyAll(syncAutomationIds) - await this._createAuditLog(AuditLogRepository.DELETE, record, record, options) } diff --git a/backend/src/database/repositories/memberRepository.ts b/backend/src/database/repositories/memberRepository.ts index 632fbe033d..d6b6381932 100644 --- a/backend/src/database/repositories/memberRepository.ts +++ b/backend/src/database/repositories/memberRepository.ts @@ -67,7 +67,6 @@ import { SegmentProjectGroupNestedData, SegmentProjectNestedData, SegmentType, - SyncStatus, } from '@crowd/types' import { KUBE_MODE, SERVICE } from '@/conf' @@ -85,7 +84,6 @@ import { IRepositoryOptions } from './IRepositoryOptions' import AuditLogRepository from './auditLogRepository' import MemberAttributeSettingsRepository from './memberAttributeSettingsRepository' import MemberSegmentAffiliationRepository from './memberSegmentAffiliationRepository' -import MemberSyncRemoteRepository from './memberSyncRemoteRepository' import OrganizationRepository from './organizationRepository' import SegmentRepository from './segmentRepository' import SequelizeRepository from './sequelizeRepository' @@ -2678,20 +2676,6 @@ class MemberRepository { output.affiliations = await this.getAffiliations(record.id, options) - const manualSyncRemote = await new MemberSyncRemoteRepository(options).findMemberManualSync( - record.id, - ) - - for (const syncRemote of manualSyncRemote) { - if (output.attributes?.syncRemote) { - output.attributes.syncRemote[syncRemote.platform] = syncRemote.status === SyncStatus.ACTIVE - } else { - output.attributes.syncRemote = { - [syncRemote.platform]: syncRemote.status === SyncStatus.ACTIVE, - } - } - } - return output } diff --git a/backend/src/database/repositories/memberSyncRemoteRepository.ts b/backend/src/database/repositories/memberSyncRemoteRepository.ts deleted file mode 100644 index 861068fc07..0000000000 --- a/backend/src/database/repositories/memberSyncRemoteRepository.ts +++ /dev/null @@ -1,250 +0,0 @@ -import { QueryTypes } from 'sequelize' - -import { generateUUIDv1 as uuid } from '@crowd/common' -import { IMemberSyncRemoteData, SyncStatus } from '@crowd/types' - -import { IRepositoryOptions } from './IRepositoryOptions' -import { RepositoryBase } from './repositoryBase' -import SequelizeRepository from './sequelizeRepository' - -class MemberSyncRemoteRepository extends RepositoryBase< - IMemberSyncRemoteData, - string, - IMemberSyncRemoteData, - unknown, - unknown -> { - public constructor(options: IRepositoryOptions) { - super(options, true) - } - - async stopSyncingAutomation(automationId: string) { - await this.options.database.sequelize.query( - `update "membersSyncRemote" set status = :status where "syncFrom" = :automationId - `, - { - replacements: { - status: SyncStatus.STOPPED, - automationId, - }, - type: QueryTypes.UPDATE, - }, - ) - } - - async findRemoteSync(integrationId: string, memberId: string, syncFrom: string) { - const transaction = SequelizeRepository.getTransaction(this.options) - - const records = await this.options.database.sequelize.query( - `SELECT * - FROM "membersSyncRemote" - WHERE "integrationId" = :integrationId and "memberId" = :memberId and "syncFrom" = :syncFrom; - `, - { - replacements: { - integrationId, - memberId, - syncFrom, - }, - type: QueryTypes.SELECT, - transaction, - }, - ) - - if (records.length === 0) { - return null - } - - return records[0] - } - - async startManualSync(id: string, sourceId: string) { - const transaction = SequelizeRepository.getTransaction(this.options) - - await this.options.database.sequelize.query( - `update "membersSyncRemote" set status = :status, "sourceId" = :sourceId where "id" = :id - `, - { - replacements: { - status: SyncStatus.ACTIVE, - id, - sourceId: sourceId || null, - }, - type: QueryTypes.UPDATE, - transaction, - }, - ) - } - - async stopMemberManualSync(memberId: string) { - await this.options.database.sequelize.query( - `update "membersSyncRemote" set status = :status where "memberId" = :memberId and "syncFrom" = :manualSync - `, - { - replacements: { - status: SyncStatus.STOPPED, - memberId, - manualSync: 'manual', - }, - type: QueryTypes.UPDATE, - }, - ) - } - - async destroyAllAutomation(automationIds: string[]): Promise { - const transaction = this.transaction - - const seq = this.seq - - const query = ` - delete - from "membersSyncRemote" - where "syncFrom" in (:automationIds);` - - await seq.query(query, { - replacements: { - automationIds, - }, - type: QueryTypes.DELETE, - transaction, - }) - } - - async destroyAllIntegration(integrationIds: string[]): Promise { - const transaction = this.transaction - - const seq = this.seq - - const query = ` - delete - from "membersSyncRemote" - where "integrationId" in (:integrationIds);` - - await seq.query(query, { - replacements: { - integrationIds, - }, - type: QueryTypes.DELETE, - transaction, - }) - } - - async markMemberForSyncing(data: IMemberSyncRemoteData): Promise { - const transaction = SequelizeRepository.getTransaction(this.options) - - const existingSyncRemote = await this.findByMemberId(data.memberId) - - if (existingSyncRemote) { - data.sourceId = existingSyncRemote.sourceId - } - - const existingManualSyncRemote = await this.findRemoteSync( - data.integrationId, - data.memberId, - data.syncFrom, - ) - - if (existingManualSyncRemote) { - await this.startManualSync(existingManualSyncRemote.id, data.sourceId) - return existingManualSyncRemote - } - - const memberSyncRemoteInserted = await this.options.database.sequelize.query( - `insert into "membersSyncRemote" ("id", "memberId", "sourceId", "integrationId", "syncFrom", "metaData", "lastSyncedAt", "status") - values - (:id, :memberId, :sourceId, :integrationId, :syncFrom, :metaData, :lastSyncedAt, :status) - returning "id" - `, - { - replacements: { - id: uuid(), - memberId: data.memberId, - integrationId: data.integrationId, - syncFrom: data.syncFrom, - metaData: data.metaData, - lastSyncedAt: data.lastSyncedAt || null, - sourceId: data.sourceId || null, - status: SyncStatus.ACTIVE, - }, - type: QueryTypes.INSERT, - transaction, - }, - ) - - const memberSyncRemote = await this.findById(memberSyncRemoteInserted[0][0].id) - return memberSyncRemote - } - - async findMemberManualSync(memberId: string) { - const transaction = SequelizeRepository.getTransaction(this.options) - - const records = await this.options.database.sequelize.query( - `select i.platform, msr.status from "membersSyncRemote" msr - inner join integrations i on msr."integrationId" = i.id - where msr."syncFrom" = :syncFrom and msr."memberId" = :memberId; - `, - { - replacements: { - memberId, - syncFrom: 'manual', - }, - type: QueryTypes.SELECT, - transaction, - }, - ) - - return records - } - - async findByMemberId(memberId: string): Promise { - const transaction = SequelizeRepository.getTransaction(this.options) - - const records = await this.options.database.sequelize.query( - `SELECT * - FROM "membersSyncRemote" - WHERE "memberId" = :memberId - and "sourceId" is not null - limit 1; - `, - { - replacements: { - memberId, - }, - type: QueryTypes.SELECT, - transaction, - }, - ) - - if (records.length === 0) { - return null - } - - return records[0] - } - - async findById(id: string): Promise { - const transaction = SequelizeRepository.getTransaction(this.options) - - const records = await this.options.database.sequelize.query( - `SELECT * - FROM "membersSyncRemote" - WHERE id = :id; - `, - { - replacements: { - id, - }, - type: QueryTypes.SELECT, - transaction, - }, - ) - - if (records.length === 0) { - return null - } - - return records[0] - } -} - -export default MemberSyncRemoteRepository diff --git a/backend/src/database/repositories/organizationSyncRemoteRepository.ts b/backend/src/database/repositories/organizationSyncRemoteRepository.ts deleted file mode 100644 index ef2eb3b306..0000000000 --- a/backend/src/database/repositories/organizationSyncRemoteRepository.ts +++ /dev/null @@ -1,252 +0,0 @@ -import { QueryTypes } from 'sequelize' - -import { generateUUIDv1 as uuid } from '@crowd/common' -import { IOrganizationSyncRemoteData, SyncStatus } from '@crowd/types' - -import { IRepositoryOptions } from './IRepositoryOptions' -import { RepositoryBase } from './repositoryBase' -import SequelizeRepository from './sequelizeRepository' - -class OrganizationSyncRemoteRepository extends RepositoryBase< - IOrganizationSyncRemoteData, - string, - IOrganizationSyncRemoteData, - unknown, - unknown -> { - public constructor(options: IRepositoryOptions) { - super(options, true) - } - - async stopSyncingAutomation(automationId: string) { - await this.options.database.sequelize.query( - `update "organizationsSyncRemote" set status = :status where "syncFrom" = :automationId - `, - { - replacements: { - status: SyncStatus.STOPPED, - automationId, - }, - type: QueryTypes.UPDATE, - }, - ) - } - - async stopOrganizationManualSync(organizationId: string) { - await this.options.database.sequelize.query( - `update "organizationsSyncRemote" set status = :status where "organizationId" = :organizationId and "syncFrom" = :manualSync - `, - { - replacements: { - status: SyncStatus.STOPPED, - organizationId, - manualSync: 'manual', - }, - type: QueryTypes.UPDATE, - }, - ) - } - - async startManualSync(id: string, sourceId: string) { - const transaction = SequelizeRepository.getTransaction(this.options) - - await this.options.database.sequelize.query( - `update "organizationsSyncRemote" set status = :status, "sourceId" = :sourceId where "id" = :id - `, - { - replacements: { - status: SyncStatus.ACTIVE, - id, - sourceId: sourceId || null, - }, - type: QueryTypes.UPDATE, - transaction, - }, - ) - } - - async findRemoteSync(integrationId: string, organizationId: string, syncFrom: string) { - const transaction = SequelizeRepository.getTransaction(this.options) - - const records = await this.options.database.sequelize.query( - `SELECT * - FROM "organizationsSyncRemote" - WHERE "integrationId" = :integrationId and "organizationId" = :organizationId and "syncFrom" = :syncFrom; - `, - { - replacements: { - integrationId, - organizationId, - syncFrom, - }, - type: QueryTypes.SELECT, - transaction, - }, - ) - - if (records.length === 0) { - return null - } - - return records[0] - } - - async markOrganizationForSyncing( - data: IOrganizationSyncRemoteData, - ): Promise { - const transaction = SequelizeRepository.getTransaction(this.options) - - const existingSyncRemote = await this.findByOrganizationId(data.organizationId) - - if (existingSyncRemote) { - data.sourceId = existingSyncRemote.sourceId - } - - const existingManualSyncRemote = await this.findRemoteSync( - data.integrationId, - data.organizationId, - data.syncFrom, - ) - - if (existingManualSyncRemote) { - await this.startManualSync(existingManualSyncRemote.id, data.sourceId) - return existingManualSyncRemote - } - - const organizationSyncRemoteInserted = await this.options.database.sequelize.query( - `insert into "organizationsSyncRemote" ("id", "organizationId", "sourceId", "integrationId", "syncFrom", "metaData", "lastSyncedAt", "status") - VALUES - (:id, :organizationId, :sourceId, :integrationId, :syncFrom, :metaData, :lastSyncedAt, :status) - returning "id" - `, - { - replacements: { - id: uuid(), - organizationId: data.organizationId, - integrationId: data.integrationId, - syncFrom: data.syncFrom, - metaData: data.metaData, - lastSyncedAt: data.lastSyncedAt || null, - sourceId: data.sourceId || null, - status: SyncStatus.ACTIVE, - }, - type: QueryTypes.INSERT, - transaction, - }, - ) - - const organizationSyncRemote = await this.findById(organizationSyncRemoteInserted[0][0].id) - return organizationSyncRemote - } - - async destroyAllAutomation(automationIds: string[]): Promise { - const transaction = this.transaction - - const seq = this.seq - - const query = ` - delete - from "organizationsSyncRemote" - where "syncFrom" in (:automationIds);` - - await seq.query(query, { - replacements: { - automationIds, - }, - type: QueryTypes.DELETE, - transaction, - }) - } - - async destroyAllIntegration(integrationIds: string[]): Promise { - const transaction = this.transaction - - const seq = this.seq - - const query = ` - delete - from "organizationsSyncRemote" - where "integrationId" in (:integrationIds);` - - await seq.query(query, { - replacements: { - integrationIds, - }, - type: QueryTypes.DELETE, - transaction, - }) - } - - async findOrganizationManualSync(organizationId: string) { - const transaction = SequelizeRepository.getTransaction(this.options) - - const records = await this.options.database.sequelize.query( - `select i.platform, osr.status from "organizationsSyncRemote" osr - inner join integrations i on osr."integrationId" = i.id - where osr."syncFrom" = :syncFrom and osr."organizationId" = :organizationId; - `, - { - replacements: { - organizationId, - syncFrom: 'manual', - }, - type: QueryTypes.SELECT, - transaction, - }, - ) - - return records - } - - async findByOrganizationId(organizationId: string): Promise { - const transaction = SequelizeRepository.getTransaction(this.options) - - const records = await this.options.database.sequelize.query( - `SELECT * - FROM "organizationsSyncRemote" - WHERE "organizationId" = :organizationId - and "sourceId" is not null - limit 1; - `, - { - replacements: { - organizationId, - }, - type: QueryTypes.SELECT, - transaction, - }, - ) - - if (records.length === 0) { - return null - } - - return records[0] - } - - async findById(id: string): Promise { - const transaction = SequelizeRepository.getTransaction(this.options) - - const records = await this.options.database.sequelize.query( - `SELECT * - FROM "organizationsSyncRemote" - WHERE id = :id; - `, - { - replacements: { - id, - }, - type: QueryTypes.SELECT, - transaction, - }, - ) - - if (records.length === 0) { - return null - } - - return records[0] - } -} - -export default OrganizationSyncRemoteRepository diff --git a/backend/src/database/repositories/types/automationTypes.ts b/backend/src/database/repositories/types/automationTypes.ts deleted file mode 100644 index 3a9da8ced7..0000000000 --- a/backend/src/database/repositories/types/automationTypes.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { - AutomationExecutionState, - AutomationSettings, - AutomationState, - AutomationSyncTrigger, - AutomationTrigger, - AutomationType, -} from '@crowd/types' - -export interface DbAutomationInsertData { - name: string - type: AutomationType - trigger: AutomationTrigger | AutomationSyncTrigger - settings: AutomationSettings - state: AutomationState -} - -export interface DbAutomationUpdateData { - name: string - trigger: AutomationTrigger - settings: AutomationSettings - state: AutomationState -} - -export interface DbAutomationExecutionInsertData { - automationId: string - type: AutomationType - tenantId: string - trigger: AutomationTrigger | AutomationSyncTrigger - state: AutomationExecutionState - error: any | null - executedAt: Date - eventId: string - payload: any -} diff --git a/backend/src/security/permissions.ts b/backend/src/security/permissions.ts index 6594004e60..3927e4432a 100644 --- a/backend/src/security/permissions.ts +++ b/backend/src/security/permissions.ts @@ -111,22 +111,6 @@ class Permissions { id: 'activityAutocomplete', allowedRoles: [roles.admin, roles.projectAdmin, roles.readonly], }, - automationCreate: { - id: 'automationCreate', - allowedRoles: [roles.admin, roles.projectAdmin], - }, - automationUpdate: { - id: 'automationUpdate', - allowedRoles: [roles.admin, roles.projectAdmin], - }, - automationDestroy: { - id: 'automationDestroy', - allowedRoles: [roles.admin, roles.projectAdmin], - }, - automationRead: { - id: 'automationRead', - allowedRoles: [roles.admin, roles.projectAdmin, roles.readonly], - }, tagImport: { id: 'tagImport', allowedRoles: [roles.admin, roles.projectAdmin], diff --git a/backend/src/serverless/utils/queueService.ts b/backend/src/serverless/utils/queueService.ts index edd732002a..30b42239c8 100644 --- a/backend/src/serverless/utils/queueService.ts +++ b/backend/src/serverless/utils/queueService.ts @@ -2,7 +2,6 @@ import { DataSinkWorkerEmitter, IntegrationRunWorkerEmitter, IntegrationStreamWorkerEmitter, - IntegrationSyncWorkerEmitter, QueuePriorityContextLoader, SearchSyncWorkerEmitter, } from '@crowd/common_services' @@ -93,20 +92,6 @@ export const getSearchSyncWorkerEmitter = async (): Promise => { - if (integrationSyncWorkerEmitter) return integrationSyncWorkerEmitter - - integrationSyncWorkerEmitter = new IntegrationSyncWorkerEmitter( - QUEUE_CLIENT(), - await REDIS_CLIENT(), - await QUEUE_PRIORITY_LOADER(), - log, - ) - await integrationSyncWorkerEmitter.init() - return integrationSyncWorkerEmitter -} - let dataSinkWorkerEmitter: DataSinkWorkerEmitter export const getDataSinkWorkerEmitter = async (): Promise => { if (dataSinkWorkerEmitter) return dataSinkWorkerEmitter diff --git a/backend/src/services/activityService.ts b/backend/src/services/activityService.ts index e86c656564..638e98be11 100644 --- a/backend/src/services/activityService.ts +++ b/backend/src/services/activityService.ts @@ -19,20 +19,13 @@ import { import { optionsQx } from '@crowd/data-access-layer/src/queryExecutor' import { ActivityDisplayService } from '@crowd/integrations' import { LoggerBase, logExecutionTime } from '@crowd/logging' -import { WorkflowIdReusePolicy } from '@crowd/temporal' -import { - IMemberIdentity, - IntegrationResultType, - PlatformType, - SegmentData, - TemporalWorkflowId, -} from '@crowd/types' +import { IMemberIdentity, IntegrationResultType, PlatformType, SegmentData } from '@crowd/types' import { IRepositoryOptions } from '@/database/repositories/IRepositoryOptions' import OrganizationRepository from '@/database/repositories/organizationRepository' import { getDataSinkWorkerEmitter } from '@/serverless/utils/queueService' -import { GITHUB_CONFIG, IS_DEV_ENV, IS_TEST_ENV, TEMPORAL_CONFIG } from '../conf' +import { GITHUB_CONFIG, IS_DEV_ENV, IS_TEST_ENV } from '../conf' import ActivityRepository from '../database/repositories/activityRepository' import MemberRepository from '../database/repositories/memberRepository' import SegmentRepository from '../database/repositories/segmentRepository' @@ -251,42 +244,6 @@ export default class ActivityService extends LoggerBase { }) } - if (!existing && fireCrowdWebhooks) { - try { - const handle = await this.options.temporal.workflow.start( - 'processNewActivityAutomation', - { - workflowId: `${TemporalWorkflowId.NEW_ACTIVITY_AUTOMATION}/${record.id}`, - taskQueue: TEMPORAL_CONFIG.automationsTaskQueue, - workflowIdReusePolicy: - WorkflowIdReusePolicy.WORKFLOW_ID_REUSE_POLICY_REJECT_DUPLICATE, - retry: { - maximumAttempts: 100, - }, - args: [ - { - tenantId: this.options.currentTenant.id, - activityId: record.id, - }, - ], - searchAttributes: { - TenantId: [this.options.currentTenant.id], - }, - }, - ) - this.log.info( - { workflowId: handle.workflowId }, - 'Started temporal workflow to process new activity automation!', - ) - } catch (err) { - this.log.error( - err, - { activityId: record.id }, - 'Error triggering new activity automation!', - ) - } - } - if (!fireCrowdWebhooks) { this.log.info('Ignoring outgoing webhooks because of fireCrowdWebhooks!') } diff --git a/backend/src/services/auth/passportStrategies/slackStrategy.ts b/backend/src/services/auth/passportStrategies/slackStrategy.ts index 7e12b1c5d8..7f727116ce 100644 --- a/backend/src/services/auth/passportStrategies/slackStrategy.ts +++ b/backend/src/services/auth/passportStrategies/slackStrategy.ts @@ -3,7 +3,7 @@ import SlackStrategy from 'passport-slack' import { PlatformType } from '@crowd/types' -import { API_CONFIG, SLACK_CONFIG, SLACK_NOTIFIER_CONFIG } from '../../../conf' +import { API_CONFIG, SLACK_CONFIG } from '../../../conf' export function getSlackStrategy(): SlackStrategy { return new SlackStrategy.Strategy( @@ -41,27 +41,3 @@ export function getSlackStrategy(): SlackStrategy { }, ) } -export function getSlackNotifierStrategy(): SlackStrategy { - return new SlackStrategy.Strategy( - { - clientID: SLACK_NOTIFIER_CONFIG.clientId, - clientSecret: SLACK_NOTIFIER_CONFIG.clientSecret, - callbackURL: `${API_CONFIG.url}/tenant/automation/slack/callback`, - skipUserProfile: true, - }, - (req, accessToken, webhookData, profile, done) => { - if (!done) { - throw new TypeError( - 'Missing req in verifyCallback; did you enable passReqToCallback in your strategy?', - ) - } - return done(null, { - accessToken: webhookData.access_token, - url: webhookData.incoming_webhook.url, - configurationUrl: webhookData.incoming_webhook.url, - channelId: webhookData.incoming_webhook.url, - channelName: webhookData.incoming_webhook.channel, - }) - }, - ) -} diff --git a/backend/src/services/automationExecutionService.ts b/backend/src/services/automationExecutionService.ts deleted file mode 100644 index cc8ad274b1..0000000000 --- a/backend/src/services/automationExecutionService.ts +++ /dev/null @@ -1,78 +0,0 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ - -/* eslint-disable class-methods-use-this */ -import { PageData } from '@crowd/types' - -import AutomationExecutionRepository from '../database/repositories/automationExecutionRepository' -import SequelizeRepository from '../database/repositories/sequelizeRepository' -import { - AutomationExecution, - AutomationExecutionCriteria, - CreateAutomationExecutionRequest, -} from '../types/automationTypes' - -import { IServiceOptions } from './IServiceOptions' -import { ServiceBase } from './serviceBase' - -export default class AutomationExecutionService extends ServiceBase< - AutomationExecution, - string, - CreateAutomationExecutionRequest, - unknown, - AutomationExecutionCriteria -> { - public constructor(options: IServiceOptions) { - super(options) - } - - /** - * Method used by service that is processing automations as they are triggered - * @param data {CreateAutomationExecutionRequest} all the necessary data to log a new automation execution - */ - override async create(data: CreateAutomationExecutionRequest): Promise { - const transaction = await SequelizeRepository.createTransaction(this.options) - - try { - const record = await new AutomationExecutionRepository(this.options).create({ - automationId: data.automation.id, - type: data.automation.type, - tenantId: data.automation.tenantId, - trigger: data.automation.trigger, - error: data.error !== undefined ? data.error : null, - executedAt: new Date(), - state: data.state, - eventId: data.eventId, - payload: data.payload, - }) - await SequelizeRepository.commitTransaction(transaction) - - return record - } catch (error) { - await SequelizeRepository.rollbackTransaction(transaction) - throw error - } - } - - /** - * Method used to fetch all automation executions. - * @param criteria {AutomationExecutionCriteria} filters to be used when returning automation executions - * @returns {PageData>} - */ - override async findAndCountAll( - criteria: AutomationExecutionCriteria, - ): Promise> { - return new AutomationExecutionRepository(this.options).findAndCountAll(criteria) - } - - override async update(id: string, data: unknown): Promise { - throw new Error('Method not implemented.') - } - - override async destroyAll(ids: string[]): Promise { - throw new Error('Method not implemented.') - } - - override async findById(id: string): Promise { - throw new Error('Method not implemented.') - } -} diff --git a/backend/src/services/automationService.ts b/backend/src/services/automationService.ts deleted file mode 100644 index b2635af2c5..0000000000 --- a/backend/src/services/automationService.ts +++ /dev/null @@ -1,197 +0,0 @@ -import { Error404 } from '@crowd/common' -import { - AutomationState, - AutomationSyncTrigger, - AutomationType, - IAutomationData, - PageData, - PlatformType, -} from '@crowd/types' - -import AutomationExecutionRepository from '@/database/repositories/automationExecutionRepository' -import IntegrationRepository from '@/database/repositories/integrationRepository' -import MemberSyncRemoteRepository from '@/database/repositories/memberSyncRemoteRepository' -import OrganizationSyncRemoteRepository from '@/database/repositories/organizationSyncRemoteRepository' -import { getIntegrationSyncWorkerEmitter } from '@/serverless/utils/queueService' - -import AutomationRepository from '../database/repositories/automationRepository' -import SequelizeRepository from '../database/repositories/sequelizeRepository' -import { - AutomationCriteria, - CreateAutomationRequest, - UpdateAutomationRequest, -} from '../types/automationTypes' - -import { IServiceOptions } from './IServiceOptions' -import { ServiceBase } from './serviceBase' - -export default class AutomationService extends ServiceBase< - IAutomationData, - string, - CreateAutomationRequest, - UpdateAutomationRequest, - AutomationCriteria -> { - public constructor(options: IServiceOptions) { - super(options) - } - - /** - * Creates a new active automation - * @param req {CreateAutomationRequest} data used to create a new automation - * @returns {IAutomationData} object for frontend to use - */ - override async create(req: CreateAutomationRequest): Promise { - const txOptions = await this.getTxRepositoryOptions() - - try { - // create an automation - const result = await new AutomationRepository(txOptions).create({ - ...req, - state: AutomationState.ACTIVE, - }) - - // check automation type, if hubspot trigger an automation onboard - if (req.type === AutomationType.HUBSPOT) { - let integration - - try { - integration = await IntegrationRepository.findByPlatform(PlatformType.HUBSPOT, { - ...this.options, - }) - } catch (err) { - this.options.log.error(err, 'Error while fetching HubSpot integration from DB!') - throw new Error404() - } - - // enable sync remote for integration - integration = await IntegrationRepository.update( - integration.id, - { - settings: { - ...integration.settings, - syncRemoteEnabled: true, - }, - }, - txOptions, - ) - - const integrationSyncWorkerEmitter = await getIntegrationSyncWorkerEmitter() - await integrationSyncWorkerEmitter.triggerOnboardAutomation( - this.options.currentTenant.id, - integration.id, - result.id, - req.trigger as AutomationSyncTrigger, - ) - } - - await SequelizeRepository.commitTransaction(txOptions.transaction) - - return result - } catch (error) { - await SequelizeRepository.rollbackTransaction(txOptions.transaction) - throw error - } - } - - /** - * Updates an existing automation. - * Also used to change automation state - to enable or disable an automation. - * It updates all the columns at once so all the properties in the request parameter - * have to be filled. - * @param id of the existing automation that is being updated - * @param req {UpdateAutomationRequest} data used to update an existing automation - * @returns {IAutomationData} object for frontend to use - */ - override async update(id: string, req: UpdateAutomationRequest): Promise { - const txOptions = await this.getTxRepositoryOptions() - - try { - // update an existing automation including its state - const result = await new AutomationRepository(txOptions).update(id, req) - await SequelizeRepository.commitTransaction(txOptions.transaction) - // check automation type, if hubspot trigger an automation onboard - if (result.type === AutomationType.HUBSPOT) { - let integration - - try { - integration = await IntegrationRepository.findByPlatform(PlatformType.HUBSPOT, { - ...this.options, - }) - } catch (err) { - this.options.log.error(err, 'Error while fetching HubSpot integration from DB!') - throw new Error404() - } - - if ( - result.trigger === AutomationSyncTrigger.MEMBER_ATTRIBUTES_MATCH || - result.trigger === AutomationSyncTrigger.ORGANIZATION_ATTRIBUTES_MATCH - ) { - if (result.state === AutomationState.ACTIVE) { - const integrationSyncWorkerEmitter = await getIntegrationSyncWorkerEmitter() - await integrationSyncWorkerEmitter.triggerOnboardAutomation( - this.options.currentTenant.id, - integration.id, - result.id, - result.trigger as AutomationSyncTrigger, - ) - } else if (result.trigger === AutomationSyncTrigger.MEMBER_ATTRIBUTES_MATCH) { - // disable memberSyncRemote for given automationId - const syncRepo = new MemberSyncRemoteRepository(this.options) - await syncRepo.stopSyncingAutomation(result.id) - } else if (result.trigger === AutomationSyncTrigger.ORGANIZATION_ATTRIBUTES_MATCH) { - // disable organizationSyncRemote for given automationId - const syncRepo = new OrganizationSyncRemoteRepository(this.options) - await syncRepo.stopSyncingAutomation(result.id) - } - } - } - return result - } catch (error) { - await SequelizeRepository.rollbackTransaction(txOptions.transaction) - throw error - } - } - - /** - * Method used to fetch all tenants automation with filters available in the criteria parameter - * @param criteria {AutomationCriteria} filters to be used when returning automations - * @returns {PageData>} - */ - override async findAndCountAll(criteria: AutomationCriteria): Promise> { - return new AutomationRepository(this.options).findAndCountAll(criteria) - } - - /** - * Method used to fetch a single automation by its id - * @param id automation id - * @returns {IAutomationData} - */ - override async findById(id: string): Promise { - return new AutomationRepository(this.options).findById(id) - } - - /** - * Deletes existing automations by id - * @param ids automation unique IDs to be deleted - */ - override async destroyAll(ids: string[]): Promise { - const txOptions = await this.getTxRepositoryOptions() - - try { - // delete automation executions - await new AutomationExecutionRepository(txOptions).destroyAllAutomation(ids) - - // delete syncRemote rows coming from automations - await new MemberSyncRemoteRepository(txOptions).destroyAllAutomation(ids) - await new OrganizationSyncRemoteRepository(txOptions).destroyAllAutomation(ids) - - const result = await new AutomationRepository(txOptions).destroyAll(ids) - await SequelizeRepository.commitTransaction(txOptions.transaction) - return result - } catch (error) { - await SequelizeRepository.rollbackTransaction(txOptions.transaction) - throw error - } - } -} diff --git a/backend/src/services/integrationService.ts b/backend/src/services/integrationService.ts index 33e9e638d6..47b19dc31d 100644 --- a/backend/src/services/integrationService.ts +++ b/backend/src/services/integrationService.ts @@ -5,28 +5,12 @@ import lodash from 'lodash' import moment from 'moment' import { EDITION, Error400, Error404 } from '@crowd/common' -import { MemberField, findMemberById } from '@crowd/data-access-layer/src/members' -import { - HubspotEndpoint, - HubspotEntity, - HubspotFieldMapperFactory, - IHubspotManualSyncPayload, - IHubspotOnboardingSettings, - IHubspotProperty, - IHubspotTokenInfo, - IProcessStreamContext, - getHubspotLists, - getHubspotProperties, - getHubspotTokenInfo, -} from '@crowd/integrations' import { RedisCache } from '@crowd/redis' import { Edition, PlatformType } from '@crowd/types' import { IRepositoryOptions } from '@/database/repositories/IRepositoryOptions' import GitlabReposRepository from '@/database/repositories/gitlabReposRepository' import IntegrationProgressRepository from '@/database/repositories/integrationProgressRepository' -import MemberSyncRemoteRepository from '@/database/repositories/memberSyncRemoteRepository' -import OrganizationSyncRemoteRepository from '@/database/repositories/organizationSyncRemoteRepository' import { IntegrationProgress } from '@/serverless/integrations/types/regularTypes' import { fetchAllGitlabGroups, @@ -49,13 +33,10 @@ import { GITLAB_CONFIG, IS_TEST_ENV, KUBE_MODE, - NANGO_CONFIG, } from '../conf/index' import GithubReposRepository from '../database/repositories/githubReposRepository' import IntegrationRepository from '../database/repositories/integrationRepository' -import MemberAttributeSettingsRepository from '../database/repositories/memberAttributeSettingsRepository' import SequelizeRepository from '../database/repositories/sequelizeRepository' -import TenantRepository from '../database/repositories/tenantRepository' import telemetryTrack from '../segment/telemetryTrack' import track from '../segment/track' import { ILinkedInOrganization } from '../serverless/integrations/types/linkedinTypes' @@ -65,15 +46,10 @@ import { } from '../serverless/integrations/usecases/github/rest/getRemoteStats' import { getOrganizations } from '../serverless/integrations/usecases/linkedin/getOrganizations' import getToken from '../serverless/integrations/usecases/nango/getToken' -import { - getIntegrationRunWorkerEmitter, - getIntegrationSyncWorkerEmitter, -} from '../serverless/utils/queueService' +import { getIntegrationRunWorkerEmitter } from '../serverless/utils/queueService' import { encryptData } from '../utils/crypto' import { IServiceOptions } from './IServiceOptions' -import OrganizationService from './organizationService' -import SearchSyncService from './searchSyncService' const discordToken = DISCORD_CONFIG.token || DISCORD_CONFIG.token2 @@ -755,509 +731,6 @@ export default class IntegrationService { throw new Error404(this.options.language, 'errors.linkedin.cantOnboardWrongStatus') } - async hubspotStopSyncMember(payload: IHubspotManualSyncPayload) { - if (!payload.memberId) { - throw new Error('memberId is required in the payload while syncing member to hubspot!') - } - - const transaction = await SequelizeRepository.createTransaction(this.options) - - try { - const qx = SequelizeRepository.getQueryExecutor(this.options, transaction) - const member = await findMemberById(qx, payload.memberId, [MemberField.ID]) - - const memberSyncRemoteRepository = new MemberSyncRemoteRepository({ - ...this.options, - transaction, - }) - await memberSyncRemoteRepository.stopMemberManualSync(member.id) - - await SequelizeRepository.commitTransaction(transaction) - } catch (err) { - this.options.log.error(err, 'Error while stopping hubspot member sync!') - await SequelizeRepository.rollbackTransaction(transaction) - throw err - } - } - - async hubspotSyncMember(payload: IHubspotManualSyncPayload) { - if (!payload.memberId) { - throw new Error('memberId is required in the payload while syncing member to hubspot!') - } - - const transaction = await SequelizeRepository.createTransaction(this.options) - - let integration - let member: { id: string } - let memberSyncRemote - - try { - integration = await IntegrationRepository.findByPlatform(PlatformType.HUBSPOT, { - ...this.options, - transaction, - }) - - const qx = SequelizeRepository.getQueryExecutor(this.options, transaction) - member = await findMemberById(qx, payload.memberId, [MemberField.ID]) - - const memberSyncRemoteRepo = new MemberSyncRemoteRepository({ ...this.options, transaction }) - - memberSyncRemote = await memberSyncRemoteRepo.markMemberForSyncing({ - integrationId: integration.id, - memberId: member.id, - metaData: null, - syncFrom: 'manual', - lastSyncedAt: null, - }) - - integration = await this.createOrUpdate( - { - platform: PlatformType.HUBSPOT, - settings: { - ...integration.settings, - syncRemoteEnabled: true, - }, - }, - transaction, - ) - - await SequelizeRepository.commitTransaction(transaction) - } catch (err) { - this.options.log.error(err, 'Error while starting Hubspot member sync!') - await SequelizeRepository.rollbackTransaction(transaction) - throw err - } - - const integrationSyncWorkerEmitter = await getIntegrationSyncWorkerEmitter() - await integrationSyncWorkerEmitter.triggerSyncMember( - this.options.currentTenant.id, - integration.id, - payload.memberId, - memberSyncRemote.id, - ) - - const searchSyncService = new SearchSyncService(this.options) - - // send it to opensearch because in member.update we bypass while passing transactions - await searchSyncService.triggerMemberSync(this.options.currentTenant.id, member.id, { - withAggs: true, - }) - } - - async hubspotStopSyncOrganization(payload: IHubspotManualSyncPayload) { - if (!payload.organizationId) { - throw new Error( - 'organizationId is required in the payload while stopping organization sync to hubspot!', - ) - } - - const transaction = await SequelizeRepository.createTransaction(this.options) - - try { - const organizationService = new OrganizationService(this.options) - - const organization = await organizationService.findById(payload.organizationId) - - const organizationSyncRemoteRepository = new OrganizationSyncRemoteRepository({ - ...this.options, - transaction, - }) - await organizationSyncRemoteRepository.stopOrganizationManualSync(organization.id) - } catch (err) { - this.options.log.error(err, 'Error while stopping Hubspot organization sync!') - await SequelizeRepository.rollbackTransaction(transaction) - throw err - } - } - - async hubspotSyncOrganization(payload: IHubspotManualSyncPayload) { - if (!payload.organizationId) { - throw new Error( - 'organizationId is required in the payload while syncing organization to hubspot!', - ) - } - - const transaction = await SequelizeRepository.createTransaction(this.options) - - let integration - let organization - let organizationSyncRemote - - try { - integration = await IntegrationRepository.findByPlatform(PlatformType.HUBSPOT, { - ...this.options, - transaction, - }) - - const organizationService = new OrganizationService(this.options) - - organization = await organizationService.findById(payload.organizationId) - - const organizationSyncRemoteRepo = new OrganizationSyncRemoteRepository({ - ...this.options, - transaction, - }) - - organizationSyncRemote = await organizationSyncRemoteRepo.markOrganizationForSyncing({ - integrationId: integration.id, - organizationId: organization.id, - metaData: null, - syncFrom: 'manual', - lastSyncedAt: null, - }) - - integration = await this.createOrUpdate( - { - platform: PlatformType.HUBSPOT, - settings: { - ...integration.settings, - syncRemoteEnabled: true, - }, - }, - transaction, - ) - - await SequelizeRepository.commitTransaction(transaction) - - const integrationSyncWorkerEmitter = await getIntegrationSyncWorkerEmitter() - await integrationSyncWorkerEmitter.triggerSyncOrganization( - this.options.currentTenant.id, - integration.id, - payload.organizationId, - organizationSyncRemote.id, - ) - } catch (err) { - this.options.log.error(err, 'Error while starting Hubspot organization sync!') - await SequelizeRepository.rollbackTransaction(transaction) - throw err - } - } - - async hubspotOnboard(onboardSettings: IHubspotOnboardingSettings) { - if (onboardSettings.enabledFor.length === 0) { - throw new Error400(this.options.language, 'errors.hubspot.missingEnabledEntities') - } - - if ( - !onboardSettings.attributesMapping.members && - !onboardSettings.attributesMapping.organizations - ) { - throw new Error400(this.options.language, 'errors.hubspot.missingAttributesMapping') - } - - if ( - onboardSettings.enabledFor.includes(HubspotEntity.MEMBERS) && - !onboardSettings.attributesMapping.members - ) { - throw new Error400(this.options.language, 'errors.hubspot.missingAttributesMapping') - } - - if ( - onboardSettings.enabledFor.includes(HubspotEntity.ORGANIZATIONS) && - !onboardSettings.attributesMapping.organizations - ) { - throw new Error400(this.options.language, 'errors.hubspot.missingAttributesMapping') - } - - const tenantId = this.options.currentTenant.id - - let integration - - try { - integration = await IntegrationRepository.findByPlatform(PlatformType.HUBSPOT, { - ...this.options, - }) - } catch (err) { - this.options.log.error(err, 'Error while fetching HubSpot integration from DB!') - throw new Error404() - } - - const memberAttributeSettings = ( - await MemberAttributeSettingsRepository.findAndCountAll({}, this.options) - ).rows - - const platforms = (await TenantRepository.getAvailablePlatforms(tenantId, this.options)).map( - (p) => p.platform, - ) - - const hubspotId = integration.settings.hubspotId - - const memberMapper = HubspotFieldMapperFactory.getFieldMapper( - HubspotEntity.MEMBERS, - hubspotId, - memberAttributeSettings, - platforms, - ) - const organizationMapper = HubspotFieldMapperFactory.getFieldMapper( - HubspotEntity.ORGANIZATIONS, - hubspotId, - ) - - // validate members - if (onboardSettings.attributesMapping.members) { - for (const field of Object.keys(onboardSettings.attributesMapping.members)) { - const hubspotProperty: IHubspotProperty = - integration.settings.hubspotProperties.members.find( - (p) => p.name === onboardSettings.attributesMapping.members[field], - ) - if (!memberMapper.isFieldMappableToHubspotType(field, hubspotProperty.type)) { - throw new Error( - `Member field ${field} has incompatible type with hubspot property ${hubspotProperty.name}`, - ) - } - } - } - - // validate organizations - if (onboardSettings.attributesMapping.organizations) { - for (const field of Object.keys(onboardSettings.attributesMapping.organizations)) { - const hubspotProperty: IHubspotProperty = - integration.settings.hubspotProperties.organizations.find( - (p) => p.name === onboardSettings.attributesMapping.organizations[field], - ) - if (!organizationMapper.isFieldMappableToHubspotType(field, hubspotProperty.type)) { - throw new Error( - `Organization field ${field} has incompatible type with hubspot property ${hubspotProperty.name}`, - ) - } - } - } - - const transaction = await SequelizeRepository.createTransaction(this.options) - - // save attribute mapping and enabledFor - try { - integration = await this.createOrUpdate( - { - platform: PlatformType.HUBSPOT, - settings: { - ...integration.settings, - attributesMapping: onboardSettings.attributesMapping, - enabledFor: onboardSettings.enabledFor, - crowdAttributes: memberAttributeSettings, - platforms, - }, - status: 'in-progress', - }, - transaction, - ) - await SequelizeRepository.commitTransaction(transaction) - } catch (err) { - await SequelizeRepository.rollbackTransaction(transaction) - throw err - } - - // Send queue message that starts the hubspot integration - const emitter = await getIntegrationRunWorkerEmitter() - await emitter.triggerIntegrationRun( - integration.tenantId, - integration.platform, - integration.id, - true, - ) - } - - async hubspotGetLists() { - const tenantId = this.options.currentTenant.id - const nangoId = `${tenantId}-${PlatformType.HUBSPOT}` - - let token: string - try { - token = await getToken(nangoId, PlatformType.HUBSPOT, this.options.log) - } catch (err) { - this.options.log.error(err, 'Error while verifying HubSpot tenant token in Nango!') - throw new Error400(this.options.language, 'errors.noNangoToken.message') - } - - if (!token) { - throw new Error400(this.options.language, 'errors.noNangoToken.message') - } - - const context = { - log: this.options.log, - serviceSettings: { - nangoId, - nangoUrl: NANGO_CONFIG.url, - nangoSecretKey: NANGO_CONFIG.secretKey, - }, - } as IProcessStreamContext - - const memberLists = await getHubspotLists(nangoId, context) - - return { - members: memberLists, - organizations: [], // hubspot doesn't support company lists yet - } - } - - async hubspotGetMappableFields() { - const memberAttributeSettings = ( - await MemberAttributeSettingsRepository.findAndCountAll({}, this.options) - ).rows - - const identities = await TenantRepository.getAvailablePlatforms( - this.options.currentTenant.id, - this.options, - ) - - // hubspotId is not used while getting the typemap, we can send it null - const memberMapper = HubspotFieldMapperFactory.getFieldMapper( - HubspotEntity.MEMBERS, - null, - memberAttributeSettings, - identities.map((i) => i.platform), - ) - const organizationMapper = HubspotFieldMapperFactory.getFieldMapper( - HubspotEntity.ORGANIZATIONS, - null, - ) - - return { - members: memberMapper.getTypeMap(), - organizations: organizationMapper.getTypeMap(), - } - } - - async hubspotUpdateProperties(): Promise { - const tenantId = this.options.currentTenant.id - const nangoId = `${tenantId}-${PlatformType.HUBSPOT}` - - let integration - - try { - integration = await IntegrationRepository.findByPlatform(PlatformType.HUBSPOT, { - ...this.options, - }) - } catch (err) { - this.options.log.error(err, 'Error while fetching HubSpot integration from DB!') - throw new Error404() - } - - let token: string - try { - token = await getToken(nangoId, PlatformType.HUBSPOT, this.options.log) - } catch (err) { - this.options.log.error(err, 'Error while verifying HubSpot tenant token in Nango!') - throw new Error400(this.options.language, 'errors.noNangoToken.message') - } - - if (!token) { - throw new Error400(this.options.language, 'errors.noNangoToken.message') - } - - const transaction = await SequelizeRepository.createTransaction(this.options) - - const context = { - log: this.options.log, - serviceSettings: { - nangoId, - nangoUrl: NANGO_CONFIG.url, - nangoSecretKey: NANGO_CONFIG.secretKey, - }, - } as IProcessStreamContext - - const hubspotMemberProperties = await getHubspotProperties( - nangoId, - HubspotEndpoint.CONTACTS, - context, - ) - const hubspotOrganizationProperties = await getHubspotProperties( - nangoId, - HubspotEndpoint.COMPANIES, - context, - ) - - try { - integration = await this.createOrUpdate( - { - platform: PlatformType.HUBSPOT, - settings: { - ...integration.settings, - updateMemberAttributes: true, - hubspotProperties: { - [HubspotEntity.MEMBERS]: hubspotMemberProperties, - [HubspotEntity.ORGANIZATIONS]: hubspotOrganizationProperties, - }, - }, - }, - transaction, - ) - await SequelizeRepository.commitTransaction(transaction) - } catch (err) { - await SequelizeRepository.rollbackTransaction(transaction) - throw err - } - - return integration.settings.hubspotProperties - } - - async hubspotConnect() { - const tenantId = this.options.currentTenant.id - const nangoId = `${tenantId}-${PlatformType.HUBSPOT}` - - let token: string - try { - token = await getToken(nangoId, PlatformType.HUBSPOT, this.options.log) - } catch (err) { - this.options.log.error(err, 'Error while verifying HubSpot tenant token in Nango!') - throw new Error400(this.options.language, 'errors.noNangoToken.message') - } - - if (!token) { - throw new Error400(this.options.language, 'errors.noNangoToken.message') - } - - const transaction = await SequelizeRepository.createTransaction(this.options) - let integration - - const context = { - log: this.options.log, - serviceSettings: { - nangoId, - nangoUrl: NANGO_CONFIG.url, - nangoSecretKey: NANGO_CONFIG.secretKey, - }, - } as IProcessStreamContext - - const hubspotMemberProperties: IHubspotProperty[] = await getHubspotProperties( - nangoId, - HubspotEndpoint.CONTACTS, - context, - ) - - const hubspotOrganizationProperties: IHubspotProperty[] = await getHubspotProperties( - nangoId, - HubspotEndpoint.COMPANIES, - context, - ) - - const hubspotInfo: IHubspotTokenInfo = await getHubspotTokenInfo(nangoId, context) - - try { - integration = await this.createOrUpdate( - { - platform: PlatformType.HUBSPOT, - settings: { - updateMemberAttributes: true, - hubspotProperties: { - [HubspotEntity.MEMBERS]: hubspotMemberProperties, - [HubspotEntity.ORGANIZATIONS]: hubspotOrganizationProperties, - }, - hubspotId: hubspotInfo.hub_id, - }, - status: 'pending-action', - }, - transaction, - ) - await SequelizeRepository.commitTransaction(transaction) - } catch (err) { - await SequelizeRepository.rollbackTransaction(transaction) - throw err - } - - return integration - } - async linkedinConnect() { const tenantId = this.options.currentTenant.id const nangoId = `${tenantId}-${PlatformType.LINKEDIN}` diff --git a/backend/src/services/memberService.ts b/backend/src/services/memberService.ts index ea0ca23e88..d17f572bdd 100644 --- a/backend/src/services/memberService.ts +++ b/backend/src/services/memberService.ts @@ -27,7 +27,6 @@ import { QueryExecutor, optionsQx } from '@crowd/data-access-layer/src/queryExec // import { getActivityCountOfMemberIdentities } from '@crowd/data-access-layer' import { fetchManySegments } from '@crowd/data-access-layer/src/segments' import { LoggerBase } from '@crowd/logging' -import { WorkflowIdReusePolicy } from '@crowd/temporal' import { IMemberIdentity, IMemberRoleWithOrganization, @@ -43,10 +42,8 @@ import { MergeActionType, OrganizationIdentityType, SyncMode, - TemporalWorkflowId, } from '@crowd/types' -import { TEMPORAL_CONFIG } from '@/conf' import MemberOrganizationRepository from '@/database/repositories/memberOrganizationRepository' import { MergeActionsRepository } from '@/database/repositories/mergeActionsRepository' import OrganizationRepository from '@/database/repositories/organizationRepository' @@ -539,35 +536,6 @@ export default class MemberService extends LoggerBase { await searchSyncService.triggerMemberSync(this.options.currentTenant.id, record.id) } - if (!existing && fireCrowdWebhooks) { - try { - const handle = await this.options.temporal.workflow.start('processNewMemberAutomation', { - workflowId: `${TemporalWorkflowId.NEW_MEMBER_AUTOMATION}/${record.id}`, - taskQueue: TEMPORAL_CONFIG.automationsTaskQueue, - workflowIdReusePolicy: WorkflowIdReusePolicy.WORKFLOW_ID_REUSE_POLICY_REJECT_DUPLICATE, - retry: { - maximumAttempts: 100, - }, - - args: [ - { - tenantId: this.options.currentTenant.id, - memberId: record.id, - }, - ], - searchAttributes: { - TenantId: [this.options.currentTenant.id], - }, - }) - this.log.info( - { workflowId: handle.workflowId }, - 'Started temporal workflow to process new member automation!', - ) - } catch (err) { - logger.error(err, `Error triggering new member automation - ${record.id}!`) - } - } - if (!fireCrowdWebhooks) { this.log.info('Ignoring outgoing webhooks because of fireCrowdWebhooks!') } diff --git a/backend/src/types/automationTypes.ts b/backend/src/types/automationTypes.ts deleted file mode 100644 index 9278820077..0000000000 --- a/backend/src/types/automationTypes.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { - AutomationExecutionState, - AutomationSettings, - AutomationState, - AutomationSyncTrigger, - AutomationTrigger, - AutomationType, - IAutomationData, - SearchCriteria, -} from '@crowd/types' - -/** - * This data is used to create a new automation - */ -export interface CreateAutomationRequest { - name: string - type: AutomationType - trigger: AutomationTrigger | AutomationSyncTrigger - settings: AutomationSettings -} - -/** - * This data is used to update an existing automation - */ -export interface UpdateAutomationRequest { - name: string - trigger: AutomationTrigger - settings: AutomationSettings - state: AutomationState -} - -/** - * What filters we have available to list all automations - */ -export interface AutomationCriteria extends SearchCriteria { - id?: string - type?: AutomationType - trigger?: AutomationTrigger - state?: AutomationState -} - -export interface CreateAutomationExecutionRequest { - automation: IAutomationData - eventId: string - payload: any - state: AutomationExecutionState - error?: any -} - -/** - * Data about specific automation execution that was processed when a trigger was detected - */ -export interface AutomationExecution { - id: string - automationId: string - state: AutomationExecutionState - error: any | null - executedAt: string - eventId: string - payload: any -} - -/** - * What filters we have available to list all automations - */ -export interface AutomationExecutionCriteria extends SearchCriteria { - automationId: string -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8fc6038a4a..86b7861bfc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -489,64 +489,6 @@ importers: specifier: ^3.0.1 version: 3.1.9 - services/apps/automations_worker: - dependencies: - '@crowd/archetype-standard': - specifier: workspace:* - version: link:../../archetypes/standard - '@crowd/archetype-worker': - specifier: workspace:* - version: link:../../archetypes/worker - '@crowd/common': - specifier: workspace:* - version: link:../../libs/common - '@crowd/data-access-layer': - specifier: workspace:* - version: link:../../libs/data-access-layer - '@crowd/integrations': - specifier: workspace:* - version: link:../../libs/integrations - '@crowd/logging': - specifier: workspace:* - version: link:../../libs/logging - '@crowd/redis': - specifier: workspace:* - version: link:../../libs/redis - '@crowd/types': - specifier: workspace:* - version: link:../../libs/types - '@temporalio/workflow': - specifier: ~1.11.1 - version: 1.11.5 - html-to-mrkdwn-ts: - specifier: ^1.1.0 - version: 1.1.0 - lodash.clonedeep: - specifier: ^4.5.0 - version: 4.5.0 - lodash.merge: - specifier: ^4.6.2 - version: 4.6.2 - superagent: - specifier: ^8.1.2 - version: 8.1.2 - tsx: - specifier: ^4.7.1 - version: 4.19.2 - typescript: - specifier: ^5.6.3 - version: 5.7.2 - devDependencies: - '@types/node': - specifier: ^20.8.2 - version: 20.17.12 - '@types/superagent': - specifier: ^4.1.20 - version: 4.1.24 - nodemon: - specifier: ^3.0.1 - version: 3.1.9 - services/apps/cache_worker: dependencies: '@crowd/archetype-standard': @@ -874,52 +816,6 @@ importers: specifier: ^2.0.22 version: 2.0.22 - services/apps/integration_sync_worker: - dependencies: - '@crowd/common': - specifier: workspace:* - version: link:../../libs/common - '@crowd/data-access-layer': - specifier: workspace:* - version: link:../../libs/data-access-layer - '@crowd/integrations': - specifier: workspace:* - version: link:../../libs/integrations - '@crowd/logging': - specifier: workspace:* - version: link:../../libs/logging - '@crowd/opensearch': - specifier: workspace:* - version: link:../../libs/opensearch - '@crowd/queue': - specifier: workspace:* - version: link:../../libs/queue - '@crowd/types': - specifier: workspace:* - version: link:../../libs/types - '@opensearch-project/opensearch': - specifier: ^2.11.0 - version: 2.13.0 - config: - specifier: ^3.3.9 - version: 3.3.12 - tsx: - specifier: ^4.7.1 - version: 4.19.2 - typescript: - specifier: ^5.6.3 - version: 5.7.2 - devDependencies: - '@types/config': - specifier: ^3.3.0 - version: 3.3.5 - '@types/node': - specifier: ^18.16.3 - version: 18.19.70 - nodemon: - specifier: ^2.0.22 - version: 2.0.22 - services/apps/members_enrichment_worker: dependencies: '@crowd/archetype-standard': diff --git a/scripts/builders/automations-worker.env b/scripts/builders/automations-worker.env deleted file mode 100644 index c58bab3e3f..0000000000 --- a/scripts/builders/automations-worker.env +++ /dev/null @@ -1,4 +0,0 @@ -DOCKERFILE="./services/docker/Dockerfile.automations_worker" -CONTEXT="../" -REPO="crowddotdev/automations-worker" -SERVICES="automations-worker" \ No newline at end of file diff --git a/scripts/builders/integration-sync-worker.env b/scripts/builders/integration-sync-worker.env deleted file mode 100644 index be41c3d293..0000000000 --- a/scripts/builders/integration-sync-worker.env +++ /dev/null @@ -1,5 +0,0 @@ -DOCKERFILE="./services/docker/Dockerfile.integration_sync_worker" -CONTEXT="../" -REPO="crowddotdev/integration-sync-worker" -SERVICES="integration-sync-worker" -PRIORITIZED="integration-sync-worker" \ No newline at end of file diff --git a/scripts/cli b/scripts/cli index 95f18f67f3..99bf145f15 100755 --- a/scripts/cli +++ b/scripts/cli @@ -751,7 +751,7 @@ while test $# -gt 0; do exit ;; start-dev) - INGORED_SERVICES=("python-worker" "job-generator" "discord-ws" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker" "automations-worker") + INGORED_SERVICES=("python-worker" "job-generator" "discord-ws" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker") DEV=1 start exit @@ -762,7 +762,7 @@ while test $# -gt 0; do exit ;; clean-start-dev) - # INGORED_SERVICES=("python-worker" "job-generator" "discord-ws" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker" "automations-worker") + # INGORED_SERVICES=("python-worker" "job-generator" "discord-ws" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker") CLEAN_START=1 DEV=1 start @@ -847,14 +847,14 @@ while test $# -gt 0; do exit ;; service-restart-fe-dev) - IGNORED_SERVICES=("frontend" "python-worker" "job-generator" "discord-ws" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker" "automations-worker") + IGNORED_SERVICES=("frontend" "python-worker" "job-generator" "discord-ws" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker") DEV=1 kill_all_containers service_start exit ;; clean-start-fe-dev) - INGORED_SERVICES=("frontend" "python-worker" "job-generator" "discord-ws" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker" "automations-worker") + INGORED_SERVICES=("frontend" "python-worker" "job-generator" "discord-ws" "webhook-api" "profiles-worker" "organizations-enrichment-worker" "merge-suggestions-worker" "members-enrichment-worker" "exports-worker" "entity-merging-worker") CLEAN_START=1 DEV=1 start diff --git a/scripts/services/automations-worker.yaml b/scripts/services/automations-worker.yaml deleted file mode 100644 index 4e9f648977..0000000000 --- a/scripts/services/automations-worker.yaml +++ /dev/null @@ -1,65 +0,0 @@ -version: '3.1' - -x-env-args: &env-args - DOCKER_BUILDKIT: 1 - NODE_ENV: docker - SERVICE: automations-worker - CROWD_TEMPORAL_TASKQUEUE: automations - SHELL: /bin/sh - -services: - automations-worker: - build: - context: ../../ - dockerfile: ./scripts/services/docker/Dockerfile.automations_worker - command: 'pnpm run start' - working_dir: /usr/crowd/app/services/apps/automations_worker - env_file: - - ../../backend/.env.dist.local - - ../../backend/.env.dist.composed - - ../../backend/.env.override.local - - ../../backend/.env.override.composed - environment: - <<: *env-args - restart: always - networks: - - crowd-bridge - - automations-worker-dev: - build: - context: ../../ - dockerfile: ./scripts/services/docker/Dockerfile.automations_worker - command: 'pnpm run dev' - working_dir: /usr/crowd/app/services/apps/automations_worker - # user: '${USER_ID}:${GROUP_ID}' - env_file: - - ../../backend/.env.dist.local - - ../../backend/.env.dist.composed - - ../../backend/.env.override.local - - ../../backend/.env.override.composed - environment: - <<: *env-args - hostname: automations-worker - networks: - - crowd-bridge - volumes: - - ../../services/libs/audit-logs/src:/usr/crowd/app/services/libs/audit-logs/src - - ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src - - ../../services/libs/common_services/src:/usr/crowd/app/services/libs/common_services/src - - ../../services/libs/data-access-layer/src:/usr/crowd/app/services/libs/data-access-layer/src - - ../../services/libs/database/src:/usr/crowd/app/services/libs/database/src - - ../../services/libs/feature-flags/src:/usr/crowd/app/services/libs/feature-flags/src - - ../../services/libs/integrations/src:/usr/crowd/app/services/libs/integrations/src - - ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src - - ../../services/libs/opensearch/src:/usr/crowd/app/services/libs/opensearch/src - - ../../services/libs/questdb/src:/usr/crowd/app/services/libs/questdb/src - - ../../services/libs/queue/src:/usr/crowd/app/services/libs/queue/src - - ../../services/libs/redis/src:/usr/crowd/app/services/libs/redis/src - - ../../services/libs/telemetry/src:/usr/crowd/app/services/libs/telemetry/src - - ../../services/libs/temporal/src:/usr/crowd/app/services/libs/temporal/src - - ../../services/libs/types/src:/usr/crowd/app/services/libs/types/src - - ../../services/apps/automations_worker/src:/usr/crowd/app/services/apps/automations_worker/src - -networks: - crowd-bridge: - external: true diff --git a/scripts/services/docker/Dockerfile.automations_worker b/scripts/services/docker/Dockerfile.automations_worker deleted file mode 100644 index 1c39510dd7..0000000000 --- a/scripts/services/docker/Dockerfile.automations_worker +++ /dev/null @@ -1,23 +0,0 @@ -FROM node:20-alpine as builder - -RUN apk add --no-cache python3 make g++ - -WORKDIR /usr/crowd/app -RUN corepack enable - -COPY ./pnpm-workspace.yaml ./pnpm-lock.yaml ./ -RUN pnpm fetch - -COPY ./services ./services -RUN pnpm i --frozen-lockfile - -FROM node:20-bookworm-slim as runner - -WORKDIR /usr/crowd/app -RUN corepack enable && apt update && apt install -y ca-certificates --no-install-recommends && rm -rf /var/lib/apt/lists/* - -COPY --from=builder /usr/crowd/app/node_modules ./node_modules -COPY --from=builder /usr/crowd/app/services/base.tsconfig.json ./services/base.tsconfig.json -COPY --from=builder /usr/crowd/app/services/libs ./services/libs -COPY --from=builder /usr/crowd/app/services/archetypes/ ./services/archetypes -COPY --from=builder /usr/crowd/app/services/apps/automations_worker/ ./services/apps/automations_worker diff --git a/scripts/services/docker/Dockerfile.automations_worker.dockerignore b/scripts/services/docker/Dockerfile.automations_worker.dockerignore deleted file mode 100644 index 4eaeed9149..0000000000 --- a/scripts/services/docker/Dockerfile.automations_worker.dockerignore +++ /dev/null @@ -1,19 +0,0 @@ -**/.git -**/node_modules -**/venv* -**/.webpack -**/.serverless -**/.env -**/.env.* -**/.idea -**/.vscode -**/dist -backend/src/serverless/microservices/python -.vscode/ -.github/ -frontend/ -scripts/ -.flake8 -*.md -Makefile -backend/ \ No newline at end of file diff --git a/scripts/services/docker/Dockerfile.integration_sync_worker b/scripts/services/docker/Dockerfile.integration_sync_worker deleted file mode 100644 index 762313a320..0000000000 --- a/scripts/services/docker/Dockerfile.integration_sync_worker +++ /dev/null @@ -1,22 +0,0 @@ -FROM node:20-alpine as builder - -RUN apk add --no-cache python3 make g++ - -WORKDIR /usr/crowd/app -RUN corepack enable - -COPY ./pnpm-workspace.yaml ./pnpm-lock.yaml ./ -RUN pnpm fetch - -COPY ./services ./services -RUN pnpm i --frozen-lockfile - -FROM node:20-alpine as runner - -WORKDIR /usr/crowd/app -RUN corepack enable - -COPY --from=builder /usr/crowd/app/node_modules ./node_modules -COPY --from=builder /usr/crowd/app/services/base.tsconfig.json ./services/base.tsconfig.json -COPY --from=builder /usr/crowd/app/services/libs ./services/libs -COPY --from=builder /usr/crowd/app/services/apps/integration_sync_worker/ ./services/apps/integration_sync_worker diff --git a/scripts/services/docker/Dockerfile.integration_sync_worker.dockerignore b/scripts/services/docker/Dockerfile.integration_sync_worker.dockerignore deleted file mode 100644 index 4eaeed9149..0000000000 --- a/scripts/services/docker/Dockerfile.integration_sync_worker.dockerignore +++ /dev/null @@ -1,19 +0,0 @@ -**/.git -**/node_modules -**/venv* -**/.webpack -**/.serverless -**/.env -**/.env.* -**/.idea -**/.vscode -**/dist -backend/src/serverless/microservices/python -.vscode/ -.github/ -frontend/ -scripts/ -.flake8 -*.md -Makefile -backend/ \ No newline at end of file diff --git a/scripts/services/integration-sync-worker.yaml b/scripts/services/integration-sync-worker.yaml deleted file mode 100644 index 22c0e41016..0000000000 --- a/scripts/services/integration-sync-worker.yaml +++ /dev/null @@ -1,65 +0,0 @@ -version: '3.1' - -x-env-args: &env-args - DOCKER_BUILDKIT: 1 - NODE_ENV: docker - SERVICE: integration-sync-worker - SHELL: /bin/sh - QUEUE_PRIORITY_LEVEL: normal - -services: - integration-sync-worker: - build: - context: ../../ - dockerfile: ./scripts/services/docker/Dockerfile.integration_sync_worker - command: 'pnpm run start' - working_dir: /usr/crowd/app/services/apps/integration_sync_worker - env_file: - - ../../backend/.env.dist.local - - ../../backend/.env.dist.composed - - ../../backend/.env.override.local - - ../../backend/.env.override.composed - environment: - <<: *env-args - restart: always - networks: - - crowd-bridge - - integration-sync-worker-dev: - build: - context: ../../ - dockerfile: ./scripts/services/docker/Dockerfile.integration_sync_worker - command: 'pnpm run dev' - working_dir: /usr/crowd/app/services/apps/integration_sync_worker - # user: '${USER_ID}:${GROUP_ID}' - env_file: - - ../../backend/.env.dist.local - - ../../backend/.env.dist.composed - - ../../backend/.env.override.local - - ../../backend/.env.override.composed - environment: - <<: *env-args - hostname: integration-sync-worker - networks: - - crowd-bridge - volumes: - - ../../services/libs/audit-logs/src:/usr/crowd/app/services/libs/audit-logs/src - - ../../services/libs/common/src:/usr/crowd/app/services/libs/common/src - - ../../services/libs/common_services/src:/usr/crowd/app/services/libs/common_services/src - - ../../services/libs/data-access-layer/src:/usr/crowd/app/services/libs/data-access-layer/src - - ../../services/libs/database/src:/usr/crowd/app/services/libs/database/src - - ../../services/libs/feature-flags/src:/usr/crowd/app/services/libs/feature-flags/src - - ../../services/libs/integrations/src:/usr/crowd/app/services/libs/integrations/src - - ../../services/libs/logging/src:/usr/crowd/app/services/libs/logging/src - - ../../services/libs/opensearch/src:/usr/crowd/app/services/libs/opensearch/src - - ../../services/libs/questdb/src:/usr/crowd/app/services/libs/questdb/src - - ../../services/libs/queue/src:/usr/crowd/app/services/libs/queue/src - - ../../services/libs/redis/src:/usr/crowd/app/services/libs/redis/src - - ../../services/libs/telemetry/src:/usr/crowd/app/services/libs/telemetry/src - - ../../services/libs/temporal/src:/usr/crowd/app/services/libs/temporal/src - - ../../services/libs/types/src:/usr/crowd/app/services/libs/types/src - - ../../services/apps/integration_sync_worker/src:/usr/crowd/app/services/apps/integration_sync_worker/src - -networks: - crowd-bridge: - external: true diff --git a/services/apps/automations_worker/package.json b/services/apps/automations_worker/package.json deleted file mode 100644 index fd1e28795b..0000000000 --- a/services/apps/automations_worker/package.json +++ /dev/null @@ -1,36 +0,0 @@ -{ - "name": "@crowd/automations-worker", - "scripts": { - "start": "CROWD_TEMPORAL_TASKQUEUE=automations SERVICE=automations-worker tsx src/main.ts", - "start:debug:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && CROWD_TEMPORAL_TASKQUEUE=automations SERVICE=automations-worker LOG_LEVEL=trace tsx --inspect=0.0.0.0:9232 src/main.ts", - "start:debug": "CROWD_TEMPORAL_TASKQUEUE=automations SERVICE=automations-worker LOG_LEVEL=trace tsx --inspect=0.0.0.0:9232 src/main.ts", - "dev:local": "nodemon --watch src --watch ../../libs --ext ts --exec pnpm run start:debug:local", - "dev": "nodemon --watch src --watch ../../libs --ext ts --exec pnpm run start:debug", - "lint": "npx eslint --ext .ts src --max-warnings=0", - "format": "npx prettier --write \"src/**/*.ts\"", - "format-check": "npx prettier --check .", - "tsc-check": "tsc --noEmit" - }, - "dependencies": { - "@crowd/archetype-standard": "workspace:*", - "@crowd/archetype-worker": "workspace:*", - "@crowd/common": "workspace:*", - "@crowd/data-access-layer": "workspace:*", - "@crowd/integrations": "workspace:*", - "@crowd/logging": "workspace:*", - "@crowd/redis": "workspace:*", - "@crowd/types": "workspace:*", - "@temporalio/workflow": "~1.11.1", - "html-to-mrkdwn-ts": "^1.1.0", - "lodash.clonedeep": "^4.5.0", - "lodash.merge": "^4.6.2", - "superagent": "^8.1.2", - "tsx": "^4.7.1", - "typescript": "^5.6.3" - }, - "devDependencies": { - "@types/node": "^20.8.2", - "@types/superagent": "^4.1.20", - "nodemon": "^3.0.1" - } -} diff --git a/services/apps/automations_worker/src/activities.ts b/services/apps/automations_worker/src/activities.ts deleted file mode 100644 index 0d0970b3d2..0000000000 --- a/services/apps/automations_worker/src/activities.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { - detectNewActivityAutomations, - triggerActivityAutomationExecution, -} from './activities/newActivityAutomations' -import { - detectNewMemberAutomations, - triggerMemberAutomationExecution, -} from './activities/newMemberAutomations' - -export { - detectNewActivityAutomations, - triggerActivityAutomationExecution, - detectNewMemberAutomations, - triggerMemberAutomationExecution, -} diff --git a/services/apps/automations_worker/src/activities/newActivityAutomations.ts b/services/apps/automations_worker/src/activities/newActivityAutomations.ts deleted file mode 100644 index 04bc6802d3..0000000000 --- a/services/apps/automations_worker/src/activities/newActivityAutomations.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { getChildLogger } from '@crowd/logging' - -import { svc } from '../main' -import { AutomationService } from '../services/automation.service' - -export async function triggerActivityAutomationExecution( - automationId: string, - activityId: string, -): Promise { - const log = getChildLogger('triggerActivityAutomationExecution', svc.log, { - automationId, - activityId, - }) - - const service = new AutomationService(svc.postgres.writer, svc.redis, log) - const payload = await service.getActivity(activityId) - if (!payload) { - log.warn('Activity not found, skipping execution') - return - } - await service.triggerAutomationExecution(automationId, activityId, payload) -} - -// returns automation ids to trigger -export async function detectNewActivityAutomations( - tenantId: string, - activityId: string, -): Promise { - const log = getChildLogger('detectNewActivityAutomations', svc.log, { - tenantId, - activityId, - }) - - const service = new AutomationService(svc.postgres.reader, svc.redis, log) - return service.detectNewActivityAutomations(tenantId, activityId) -} diff --git a/services/apps/automations_worker/src/activities/newMemberAutomations.ts b/services/apps/automations_worker/src/activities/newMemberAutomations.ts deleted file mode 100644 index bedc5c29a8..0000000000 --- a/services/apps/automations_worker/src/activities/newMemberAutomations.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { getChildLogger } from '@crowd/logging' - -import { svc } from '../main' -import { AutomationService } from '../services/automation.service' - -export async function triggerMemberAutomationExecution( - automationId: string, - memberId: string, -): Promise { - const log = getChildLogger('triggerMemberAutomationExecution', svc.log, { - automationId, - memberId, - }) - - const service = new AutomationService(svc.postgres.writer, svc.redis, log) - const payload = await service.getMember(memberId) - if (!payload) { - log.warn('Member not found, skipping execution') - return - } - await service.triggerAutomationExecution(automationId, memberId, payload) -} - -// returns automation ids to trigger -export async function detectNewMemberAutomations( - tenantId: string, - memberId: string, -): Promise { - const log = getChildLogger('detectNewMemberAutomations', svc.log, { - tenantId, - memberId, - }) - - const service = new AutomationService(svc.postgres.reader, svc.redis, log) - return service.detectNewMemberAutomations(tenantId, memberId) -} diff --git a/services/apps/automations_worker/src/main.ts b/services/apps/automations_worker/src/main.ts deleted file mode 100644 index b6cb8a6f36..0000000000 --- a/services/apps/automations_worker/src/main.ts +++ /dev/null @@ -1,40 +0,0 @@ -import { Config } from '@crowd/archetype-standard' -import { Options, ServiceWorker } from '@crowd/archetype-worker' - -const config: Config = { - envvars: [], - producer: { - enabled: false, - }, - temporal: { - enabled: false, - }, - questdb: { - enabled: false, - }, - redis: { - enabled: true, - }, -} - -const options: Options = { - postgres: { - enabled: true, - }, - opensearch: { - enabled: false, - }, -} - -export const svc = new ServiceWorker(config, options) - -export const FRONTEND_URL = process.env.CROWD_API_FRONTEND_URL - -if (!FRONTEND_URL) { - throw new Error('CROWD_API_FRONTEND_URL is not set') -} - -setImmediate(async () => { - await svc.init() - await svc.start() -}) diff --git a/services/apps/automations_worker/src/services/automation.service.ts b/services/apps/automations_worker/src/services/automation.service.ts deleted file mode 100644 index e434f819ba..0000000000 --- a/services/apps/automations_worker/src/services/automation.service.ts +++ /dev/null @@ -1,502 +0,0 @@ -import request from 'superagent' - -import { DbStore } from '@crowd/data-access-layer/src/database' -import { - AutomationRepository, - IRelevantAutomationData, -} from '@crowd/data-access-layer/src/old/apps/automations_worker/automation.repo' -import { DataRepository } from '@crowd/data-access-layer/src/old/apps/automations_worker/data.repo' -import { - IActivityData, - IMemberData, -} from '@crowd/data-access-layer/src/old/apps/automations_worker/types' -import { Logger, getChildLogger } from '@crowd/logging' -import { RedisCache, RedisClient } from '@crowd/redis' -import { - AutomationExecutionState, - AutomationState, - AutomationTrigger, - AutomationType, - NewActivitySettings, - NewMemberSettings, - WebhookSettings, -} from '@crowd/types' - -import { newActivityBlocks } from './slack/newActivityBlocks' -import { newMemberBlocks } from './slack/newMemberBlocks' - -export class AutomationService { - private readonly log: Logger - private readonly automationRepo: AutomationRepository - private readonly dataRepo: DataRepository - - private readonly automationCache: RedisCache - private readonly memberCache: RedisCache - private readonly activityCache: RedisCache - - constructor(dbStore: DbStore, redis: RedisClient, parentLog: Logger) { - this.log = getChildLogger(this.constructor.name, parentLog) - - this.automationRepo = new AutomationRepository(dbStore, this.log) - this.dataRepo = new DataRepository(dbStore, this.log) - - this.automationCache = new RedisCache('automations:definitions', redis, this.log) - this.memberCache = new RedisCache('automations:members', redis, this.log) - this.activityCache = new RedisCache('automations:activities', redis, this.log) - } - - async shouldProcessMember( - member: IMemberData, - automation: IRelevantAutomationData, - ): Promise { - const settings = automation.settings as NewMemberSettings - - let shouldProcess = true - - // check if member joined after automation was created - if (new Date(automation.createdAt) > new Date(member.joinedAt)) { - this.log.warn( - `Ignoring automation ${automation.id} - Member ${member.id} joined before automation!`, - ) - shouldProcess = false - } - - // check whether member platforms matches - if (shouldProcess && settings.platforms && settings.platforms.length > 0) { - const platforms = Object.keys(member.username) - if (!platforms.some((platform) => settings.platforms.includes(platform))) { - this.log.warn( - `Ignoring automation ${automation.id} - Member ${ - member.id - } platforms do not include any of automation setting platforms: [${settings.platforms.join( - ', ', - )}]`, - ) - shouldProcess = false - } - } - - if (shouldProcess) { - const hasAlreadyBeenTriggered = await this.automationRepo.hasAlreadyBeenTriggered( - automation.id, - member.id, - ) - if (hasAlreadyBeenTriggered) { - this.log.warn( - `Ignoring automation ${automation.id} - Member ${member.id} was already processed!`, - ) - shouldProcess = false - } - } - - return shouldProcess - } - - async shouldProcessActivity( - activity: IActivityData, - automation: IRelevantAutomationData, - ): Promise { - const settings = automation.settings as NewActivitySettings - - let shouldProcess = true - - // check if activity created after automation was created - if (new Date(automation.createdAt) > new Date(activity.timestamp)) { - this.log.warn( - `Ignoring automation ${automation.id} - Activity ${activity.id} was created before automation!`, - ) - shouldProcess = false - } - - // check whether activity type matches - if (shouldProcess && settings.types && settings.types.length > 0) { - if (!settings.types.includes(activity.type)) { - this.log.warn( - `Ignoring automation ${automation.id} - Activity ${activity.id} type '${ - activity.type - }' does not match automation setting types: [${settings.types.join(', ')}]`, - ) - shouldProcess = false - } - } - - // check whether activity platform matches - if (shouldProcess && settings.platforms && settings.platforms.length > 0) { - if (!settings.platforms.includes(activity.platform)) { - this.log.warn( - `Ignoring automation ${automation.id} - Activity ${activity.id} platform '${ - activity.platform - }' does not match automation setting platforms: [${settings.platforms.join(', ')}]`, - ) - shouldProcess = false - } - } - - // check whether activity content contains any of the keywords - if (shouldProcess && settings.keywords && settings.keywords.length > 0 && activity.body) { - const body = (activity.body as string).toLowerCase() - if (!settings.keywords.some((keyword) => body.includes(keyword.trim().toLowerCase()))) { - this.log.warn( - `Ignoring automation ${automation.id} - Activity ${ - activity.id - } content does not match automation setting keywords: [${settings.keywords.join(', ')}]`, - ) - shouldProcess = false - } - } - - if ( - shouldProcess && - !settings.teamMemberActivities && - activity.member.attributes.isTeamMember && - activity.member.attributes.isTeamMember.default - ) { - this.log.warn( - `Ignoring automation ${automation.id} - Activity ${activity.id} belongs to a team member!`, - ) - shouldProcess = false - } - - if (activity?.member?.attributes?.isBot && activity?.member?.attributes?.isBot.default) { - this.log.warn( - `Ignoring automation ${automation.id} - Activity ${activity.id} belongs to a bot, cannot be processed automaticaly!`, - ) - shouldProcess = false - } - - if (shouldProcess) { - const hasAlreadyBeenTriggered = await this.automationRepo.hasAlreadyBeenTriggered( - automation.id, - activity.id, - ) - if (hasAlreadyBeenTriggered) { - this.log.warn( - `Ignoring automation ${automation.id} - Activity ${activity.id} was already processed!`, - ) - shouldProcess = false - } - } - - return shouldProcess - } - - /** - * @returns automation ids to trigger - */ - async detectNewMemberAutomations(tenantId: string, memberId: string): Promise { - this.log.debug('Detecting new member automations!') - - // load automations for this tenant and this type from database - const relevantAutomations = await this.automationRepo.findRelevant( - tenantId, - AutomationTrigger.NEW_MEMBER, - AutomationState.ACTIVE, - ) - - if (relevantAutomations.length > 0) { - this.log.info(`Found ${relevantAutomations.length} automations to process!`) - - const automationsToTrigger = [] - - const member = await this.getMember(memberId) - - if (!member) { - throw new Error(`Member ${memberId} not found!`) - } - - const promises = relevantAutomations.map((a) => this.shouldProcessMember(member, a)) - const results = await Promise.all(promises) - for (let i = 0; i < results.length; i++) { - if (results[i]) { - const automation = relevantAutomations[i] - this.log.info({ automationId: automation.id }, 'Automation should be processed!') - - await this.cacheAutomation(automation) - automationsToTrigger.push(automation.id) - } - } - - if (automationsToTrigger.length > 0) { - await this.cacheMember(member) - } - - return automationsToTrigger - } - - return [] - } - - /** - * @returns automation ids to trigger - */ - async detectNewActivityAutomations(tenantId: string, activityId: string): Promise { - this.log.debug('Detecting new member automations!') - - // load automations for this tenant and this type from database - const relevantAutomations = await this.automationRepo.findRelevant( - tenantId, - AutomationTrigger.NEW_ACTIVITY, - AutomationState.ACTIVE, - ) - - if (relevantAutomations.length > 0) { - this.log.info(`Found ${relevantAutomations.length} automations to process!`) - - const automationsToTrigger = [] - - const activity = await this.getActivity(activityId) - - if (!activity) { - throw new Error(`Activity ${activityId} not found!`) - } - - const promises = relevantAutomations.map((a) => this.shouldProcessActivity(activity, a)) - const results = await Promise.all(promises) - for (let i = 0; i < results.length; i++) { - if (results[i]) { - const automation = relevantAutomations[i] - this.log.info({ automationId: automation.id }, 'Automation should be processed!') - - await this.cacheAutomation(automation) - automationsToTrigger.push(automation.id) - } - } - - if (automationsToTrigger.length > 0) { - await this.cacheActivity(activity) - } - - return automationsToTrigger - } - - return [] - } - - async triggerAutomationExecution( - automationId: string, - eventId: string, - payload: unknown, - ): Promise { - this.log.info('Triggering automation execution!') - - const automation = await this.getAutomation(automationId) - - if (!automation) { - this.log.warn('Automation not found!') - return - } - - let executeMethod: ( - automation: IRelevantAutomationData, - eventId: string, - payload: unknown, - ) => Promise - - switch (automation.type) { - case AutomationType.WEBHOOK: { - executeMethod = this.executeWebhook - break - } - - case AutomationType.SLACK: { - executeMethod = this.executeSlack - break - } - default: - this.log.error(`Automation type '${automation.type}' is not supported!`) - return - } - - try { - await executeMethod.bind(this)(automation, eventId, payload) - await this.automationRepo.createExecution({ - automationId: automation.id, - type: automation.type, - trigger: automation.trigger, - tenantId: automation.tenantId, - executedAt: new Date(), - state: AutomationExecutionState.SUCCESS, - eventId, - payload, - error: undefined, - }) - } catch (err) { - this.log.error(err, 'Error while executing automation!') - await this.automationRepo.createExecution({ - automationId: automation.id, - type: automation.type, - trigger: automation.trigger, - tenantId: automation.tenantId, - executedAt: new Date(), - state: AutomationExecutionState.ERROR, - eventId, - payload, - error: err, - }) - } - } - - private async executeWebhook( - automation: IRelevantAutomationData, - eventId: string, - payload: unknown, - ): Promise { - const settings = automation.settings as WebhookSettings - const now = new Date() - - this.log.info('Firing webhook automation!') - const eventPayload = { - eventId, - eventType: automation.trigger, - eventExecutedAt: now.toISOString(), - eventPayload: payload, - } - - try { - const result = await request - .post(settings.url) - .send(eventPayload) - .set('User-Agent', 'Crowd.dev Automations Executor') - .set('X-CrowdDotDev-Event-Type', automation.trigger) - .set('X-CrowdDotDev-Event-ID', eventId) - this.log.debug(`Webhook response code ${result.statusCode}!`) - } catch (err) { - this.log.error( - err, - `Error while firing webhook automation ${automation.id} for event ${eventId} to url '${settings.url}'!`, - ) - - let error: Record - - if (err.syscall && err.code) { - error = { - type: 'network', - message: `Could not access ${settings.url}!`, - } - } else if (err.status) { - error = { - type: 'http_status', - message: `POST @ ${settings.url} returned ${err.statusCode} - ${err.statusMessage}!`, - body: err.res !== undefined ? err.res.body : undefined, - } - } else { - error = { - type: 'unknown', - message: err.message, - errorObject: err, - } - } - - throw error - } - } - - private async executeSlack( - automation: IRelevantAutomationData, - eventId: string, - payload: Record, - ): Promise { - this.log.info(`Firing slack automation ${automation.id} for event ${eventId}!`) - - if (!automation.slackWebHook) { - throw new Error( - `Automation ${automation.id} does not have a slack webhook set in tenant settings!`, - ) - } - - try { - let slackMessage: Record - if (automation.trigger === AutomationTrigger.NEW_MEMBER) { - slackMessage = { - text: `${payload.displayName} has joined your community!`, - ...newMemberBlocks(payload), - } - } else if (automation.trigger === AutomationTrigger.NEW_ACTIVITY) { - slackMessage = { - text: ':satellite_antenna: New activity', - ...newActivityBlocks(payload), - } - } else { - this.log.warn(`Error no slack handler for automation trigger ${automation.trigger}!`) - return - } - - const result = await request.post(automation.slackWebHook).send(slackMessage) - this.log.debug(`Slack response code ${result.statusCode}!`) - } catch (err) { - this.log.error( - err, - `Error while firing slack automation ${automation.id} for event ${eventId}!`, - ) - - let error: Record - - if (err.status === 404) { - error = { - type: 'connect', - message: `Could not access slack workspace!`, - } - } else { - error = { - type: 'connect', - } - } - - throw error - } - } - - private async cacheAutomation(automation: IRelevantAutomationData): Promise { - await this.automationCache.set(automation.id, JSON.stringify(automation), 60) - } - - private async getAutomation(automationId: string): Promise { - const cached = await this.automationCache.get(automationId) - if (!cached) { - const automation = await this.automationRepo.get(automationId) - if (!automation) { - return null - } - - return automation - } - - return JSON.parse(cached) - } - - private async cacheMember(member: IMemberData): Promise { - await this.memberCache.set(member.id, JSON.stringify(member), 60) - } - - async getMember(memberId: string): Promise { - const cached = await this.memberCache.get(memberId) - if (!cached) { - const members = await this.dataRepo.getMembers([memberId]) - if (members.length === 0) { - return null - } - - return members[0] - } - - return JSON.parse(cached) - } - - private async cacheActivity(activity: IActivityData): Promise { - await this.activityCache.set(activity.id, JSON.stringify(activity), 60) - } - - async getActivity(activityId: string): Promise { - const cached = await this.activityCache.get(activityId) - if (!cached) { - const activities = await this.dataRepo.getActivities([activityId]) - if (activities.length === 0) { - return null - } - - return activities[0] - } - - return JSON.parse(cached) - } -} diff --git a/services/apps/automations_worker/src/services/slack/newActivityBlocks.ts b/services/apps/automations_worker/src/services/slack/newActivityBlocks.ts deleted file mode 100644 index 7c6583ef09..0000000000 --- a/services/apps/automations_worker/src/services/slack/newActivityBlocks.ts +++ /dev/null @@ -1,220 +0,0 @@ -import htmlToMrkdwn from 'html-to-mrkdwn-ts' - -import { integrationLabel, integrationProfileUrl } from '@crowd/types' - -import { FRONTEND_URL } from '../../main' - -const defaultAvatarUrl = - 'https://uploads-ssl.webflow.com/635150609746eee5c60c4aac/6502afc9d75946873c1efa93_image%20(292).png' - -const computeEngagementLevel = (score) => { - if (score <= 1) { - return 'Silent' - } - if (score <= 3) { - return 'Quiet' - } - if (score <= 6) { - return 'Engaged' - } - if (score <= 8) { - return 'Fan' - } - if (score <= 10) { - return 'Ultra' - } - return '' -} - -const replacements: Record = { - '/images/integrations/linkedin-reactions/like.svg': ':thumbsup:', - '/images/integrations/linkedin-reactions/maybe.svg': ':thinking_face:', - '/images/integrations/linkedin-reactions/praise.svg': ':clap:', - '/images/integrations/linkedin-reactions/appreciation.svg': ':heart_hands:', - '/images/integrations/linkedin-reactions/empathy.svg': ':heart:', - '/images/integrations/linkedin-reactions/entertainment.svg': ':laughing:', - '/images/integrations/linkedin-reactions/interest.svg': ':bulb:', - 'href="/': `href="${FRONTEND_URL}/`, -} - -const replaceHeadline = (text) => { - Object.keys(replacements).forEach((key) => { - text = text.replaceAll(key, replacements[key]) - }) - return text -} - -const truncateText = (text: string, characters = 60): string => { - if (text.length > characters) { - return `${text.substring(0, characters)}...` - } - return text -} - -export const newActivityBlocks = (activity) => { - // Which platform identities are displayed as buttons and which ones go to menu - let buttonPlatforms = ['github', 'twitter', 'linkedin'] - - const display = htmlToMrkdwn(replaceHeadline(activity.display.default)) - const reach = activity.member.reach?.[activity.platform] || activity.member.reach?.total - - const { member } = activity - const memberProperties = [] - if (member.attributes.jobTitle?.default) { - memberProperties.push(`*💼 Job title:* ${member.attributes.jobTitle?.default}`) - } - if (member.organizations.length > 0) { - const orgs = member.organizations.map( - (org) => `<${`${FRONTEND_URL}/organizations/${org.id}`}|${org.name || org.displayName}>`, - ) - memberProperties.push(`*🏢 Organization:* ${orgs.join(' | ')}`) - } - if (reach > 0) { - memberProperties.push(`*👥 Reach:* ${reach} followers`) - } - if (member.attributes?.location?.default) { - memberProperties.push(`*📍 Location:* ${member.attributes?.location?.default}`) - } - if (member.emails.length > 0) { - const [email] = member.emails - memberProperties.push(`*✉️ Email:* `) - } - const engagementLevel = computeEngagementLevel(activity.member.score || activity.engagement) - if (engagementLevel.length > 0) { - memberProperties.push(`*📊 Engagement level:* ${engagementLevel}`) - } - if (activity.member.activeOn) { - const platforms = activity.member.activeOn - .map((platform) => integrationLabel[platform] || platform) - .join(' | ') - memberProperties.push(`*💬 Active on:* ${platforms}`) - } - - const profiles = Object.keys(member.username) - .map((p) => { - const username = (member.username?.[p] || []).length > 0 ? member.username[p][0] : null - const url = - member.attributes?.url?.[p] || (username && integrationProfileUrl[p](username)) || null - return { - platform: p, - url, - } - }) - .filter((p) => !!p.url) - - if (!buttonPlatforms.includes(activity.platform)) { - buttonPlatforms = [activity.platform, ...buttonPlatforms] - } - - const buttonProfiles = buttonPlatforms - .map((platform) => profiles.find((profile) => profile.platform === platform)) - .filter((profiles) => !!profiles) - - const menuProfiles = profiles.filter((profile) => !buttonPlatforms.includes(profile.platform)) - - return { - blocks: [ - { - type: 'section', - text: { - type: 'mrkdwn', - text: `*<${FRONTEND_URL}/contacts/${activity.member.id}|${activity.member.displayName}>* *${display.text}*`, - }, - ...(activity.url - ? { - accessory: { - type: 'button', - text: { - type: 'plain_text', - text: `:arrow_upper_right: ${ - activity.platform !== 'other' - ? `Open on ${integrationLabel[activity.platform]}` - : 'Open link' - }`, - emoji: true, - }, - url: activity.url, - }, - } - : {}), - }, - ...(activity.title || activity.body - ? [ - { - type: 'section', - text: { - type: 'mrkdwn', - text: `>${ - activity.title && activity.title !== activity.display.default - ? `*${truncateText(htmlToMrkdwn(activity.title).text, 120).replaceAll( - '\n', - '\n>', - )}* \n>` - : '' - }${truncateText(htmlToMrkdwn(activity.body).text, 260).replaceAll('\n', '\n>')}`, - }, - }, - ] - : []), - ...(memberProperties.length > 0 - ? [ - { - type: 'divider', - }, - { - type: 'section', - text: { - type: 'mrkdwn', - text: memberProperties.join('\n'), - }, - accessory: { - type: 'image', - image_url: member.attributes?.avatarUrl?.default ?? defaultAvatarUrl, - alt_text: 'computer thumbnail', - }, - }, - ] - : []), - { - type: 'actions', - elements: [ - { - type: 'button', - text: { - type: 'plain_text', - text: 'View in crowd.dev', - emoji: true, - }, - url: `${FRONTEND_URL}/contacts/${member.id}`, - }, - ...(buttonProfiles || []) - .map(({ platform, url }) => ({ - type: 'button', - text: { - type: 'plain_text', - text: `${integrationLabel[platform] ?? platform} profile`, - emoji: true, - }, - url, - })) - .filter((action) => !!action.url), - ...(menuProfiles.length > 0 - ? [ - { - type: 'overflow', - options: menuProfiles.map(({ platform, url }) => ({ - text: { - type: 'plain_text', - text: `${integrationLabel[platform] ?? platform} profile`, - emoji: true, - }, - url, - })), - }, - ] - : []), - ], - }, - ], - } -} diff --git a/services/apps/automations_worker/src/services/slack/newMemberBlocks.ts b/services/apps/automations_worker/src/services/slack/newMemberBlocks.ts deleted file mode 100644 index d74bee96d6..0000000000 --- a/services/apps/automations_worker/src/services/slack/newMemberBlocks.ts +++ /dev/null @@ -1,146 +0,0 @@ -import { integrationLabel, integrationProfileUrl } from '@crowd/types' - -import { FRONTEND_URL } from '../../main' - -const defaultAvatarUrl = - 'https://uploads-ssl.webflow.com/635150609746eee5c60c4aac/6502afc9d75946873c1efa93_image%20(292).png' - -export const newMemberBlocks = (member) => { - // Which platform identities are displayed as buttons and which ones go to menu - let buttonPlatforms = ['github', 'twitter', 'linkedin'] - - const platforms = member.activeOn - const reach = - platforms && platforms.length > 0 ? member.reach?.[platforms[0]] : member.reach?.total - const details = [] - if (member.attributes.jobTitle?.default) { - details.push(`*💼 Job title:* ${member.attributes.jobTitle?.default}`) - } - if (member.organizations.length > 0) { - const orgs = member.organizations.map( - (org) => `<${`${FRONTEND_URL}/organizations/${org.id}`}|${org.name || org.displayName}>`, - ) - details.push(`*🏢 Organization:* ${orgs.join(' | ')}`) - } - if (reach > 0) { - details.push(`*👥 Reach:* ${reach} followers`) - } - if (member.attributes?.location?.default) { - details.push(`*📍 Location:* ${member.attributes?.location?.default}`) - } - if (member.emails.length > 0) { - const [email] = member.emails - details.push(`*✉️ Email:* `) - } - const profiles = Object.keys(member.username) - .map((p) => { - const username = (member.username?.[p] || []).length > 0 ? member.username[p][0] : null - const url = - member.attributes?.url?.[p] || (username && integrationProfileUrl[p](username)) || null - return { - platform: p, - url, - } - }) - .filter((p) => !!p.url) - - if (platforms.length > 0 && !buttonPlatforms.includes(platforms[0])) { - buttonPlatforms = [platforms[0], ...buttonPlatforms] - } - - const buttonProfiles = buttonPlatforms - .map((platform) => profiles.find((profile) => profile.platform === platform)) - .filter((profiles) => !!profiles) - - const menuProfiles = profiles.filter((profile) => !buttonPlatforms.includes(profile.platform)) - - return { - blocks: [ - { - type: 'header', - text: { - type: 'plain_text', - text: member.displayName, - emoji: true, - }, - }, - ...(platforms && platforms.length > 0 - ? [ - { - type: 'context', - elements: [ - { - type: 'mrkdwn', - text: `Joined your community on *${ - integrationLabel[platforms[0]] || platforms[0] - }*`, - }, - ], - }, - ] - : []), - ...(details.length > 0 - ? [ - { - type: 'divider', - }, - { - type: 'section', - text: { - type: 'mrkdwn', - text: details.length > 0 ? details.join('\n') : '\n', - }, - accessory: { - type: 'image', - image_url: member.attributes?.avatarUrl?.default ?? defaultAvatarUrl, - alt_text: 'computer thumbnail', - }, - }, - { - type: 'divider', - }, - ] - : []), - { - type: 'actions', - elements: [ - { - type: 'button', - text: { - type: 'plain_text', - text: 'View in crowd.dev', - emoji: true, - }, - url: `${FRONTEND_URL}/contacts/${member.id}`, - }, - ...(buttonProfiles || []) - .map(({ platform, url }) => ({ - type: 'button', - text: { - type: 'plain_text', - text: `${integrationLabel[platform] ?? platform} profile`, - emoji: true, - }, - url, - })) - .filter((action) => !!action.url), - ...(menuProfiles.length > 0 - ? [ - { - type: 'overflow', - options: menuProfiles.map(({ platform, url }) => ({ - text: { - type: 'plain_text', - text: `${integrationLabel[platform] ?? platform} profile`, - emoji: true, - }, - url, - })), - }, - ] - : []), - ], - }, - ], - } -} diff --git a/services/apps/automations_worker/src/workflows.ts b/services/apps/automations_worker/src/workflows.ts deleted file mode 100644 index 72454b7b0b..0000000000 --- a/services/apps/automations_worker/src/workflows.ts +++ /dev/null @@ -1,15 +0,0 @@ -import { - processNewActivityAutomation, - triggerActivityAutomationExecution, -} from './workflows/newActivityAutomations' -import { - processNewMemberAutomation, - triggerMemberAutomationExecution, -} from './workflows/newMemberAutomations' - -export { - processNewActivityAutomation, - processNewMemberAutomation, - triggerActivityAutomationExecution, - triggerMemberAutomationExecution, -} diff --git a/services/apps/automations_worker/src/workflows/newActivityAutomations.ts b/services/apps/automations_worker/src/workflows/newActivityAutomations.ts deleted file mode 100644 index ff474fbe5f..0000000000 --- a/services/apps/automations_worker/src/workflows/newActivityAutomations.ts +++ /dev/null @@ -1,52 +0,0 @@ -import { - WorkflowIdReusePolicy, - executeChild, - proxyActivities, - workflowInfo, -} from '@temporalio/workflow' - -import { IProcessNewActivityAutomationArgs, ITriggerActivityAutomationArgs } from '@crowd/types' - -import * as activities from '../activities/newActivityAutomations' - -const activity = proxyActivities({ startToCloseTimeout: '1 minute' }) - -export async function processNewActivityAutomation( - args: IProcessNewActivityAutomationArgs, -): Promise { - const automationsToTrigger = await activity.detectNewActivityAutomations( - args.tenantId, - args.activityId, - ) - - const info = workflowInfo() - - if (automationsToTrigger.length > 0) { - await Promise.all( - automationsToTrigger.map((a) => - executeChild(triggerActivityAutomationExecution, { - workflowId: `${info.workflowId}/${a}`, - workflowIdReusePolicy: WorkflowIdReusePolicy.WORKFLOW_ID_REUSE_POLICY_REJECT_DUPLICATE, - retry: { - maximumAttempts: info.retryPolicy?.maximumAttempts ?? 100, - }, - args: [ - { - automationId: a, - activityId: args.activityId, - }, - ], - searchAttributes: { - TenantId: [args.tenantId], - }, - }), - ), - ) - } -} - -export async function triggerActivityAutomationExecution( - args: ITriggerActivityAutomationArgs, -): Promise { - await activity.triggerActivityAutomationExecution(args.automationId, args.activityId) -} diff --git a/services/apps/automations_worker/src/workflows/newMemberAutomations.ts b/services/apps/automations_worker/src/workflows/newMemberAutomations.ts deleted file mode 100644 index 291aba11f9..0000000000 --- a/services/apps/automations_worker/src/workflows/newMemberAutomations.ts +++ /dev/null @@ -1,54 +0,0 @@ -import { - WorkflowIdReusePolicy, - executeChild, - proxyActivities, - workflowInfo, -} from '@temporalio/workflow' - -import { IProcessNewMemberAutomationArgs, ITriggerMemberAutomationArgs } from '@crowd/types' - -import * as activities from '../activities/newMemberAutomations' - -const activity = proxyActivities({ startToCloseTimeout: '1 minute' }) - -export async function processNewMemberAutomation( - args: IProcessNewMemberAutomationArgs, -): Promise { - // first detect if there are any automations to trigger - const automationsToTrigger = await activity.detectNewMemberAutomations( - args.tenantId, - args.memberId, - ) - - const info = workflowInfo() - - // if there are any automations to trigger, trigger them - if (automationsToTrigger.length > 0) { - await Promise.all( - automationsToTrigger.map((a) => - executeChild(triggerMemberAutomationExecution, { - workflowId: `${info.workflowId}/${a}`, - workflowIdReusePolicy: WorkflowIdReusePolicy.WORKFLOW_ID_REUSE_POLICY_REJECT_DUPLICATE, - retry: { - maximumAttempts: info.retryPolicy?.maximumAttempts ?? 100, - }, - args: [ - { - automationId: a, - memberId: args.memberId, - }, - ], - searchAttributes: { - TenantId: [args.tenantId], - }, - }), - ), - ) - } -} - -export async function triggerMemberAutomationExecution( - args: ITriggerMemberAutomationArgs, -): Promise { - await activity.triggerMemberAutomationExecution(args.automationId, args.memberId) -} diff --git a/services/apps/automations_worker/tsconfig.json b/services/apps/automations_worker/tsconfig.json deleted file mode 100644 index bf7f183850..0000000000 --- a/services/apps/automations_worker/tsconfig.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": "../../base.tsconfig.json", - "include": ["src/**/*"] -} diff --git a/services/apps/data_sink_worker/config/default.json b/services/apps/data_sink_worker/config/default.json index 2410d7096c..3e51559501 100644 --- a/services/apps/data_sink_worker/config/default.json +++ b/services/apps/data_sink_worker/config/default.json @@ -2,9 +2,7 @@ "db": {}, "queue": {}, "redis": {}, - "temporal": { - "automationsTaskQueue": "automations" - }, + "temporal": {}, "worker": { "maxStreamRetries": 5, "queuePriorityLevel": "normal" diff --git a/services/apps/data_sink_worker/src/conf/index.ts b/services/apps/data_sink_worker/src/conf/index.ts index 42962612c3..55759a9f05 100644 --- a/services/apps/data_sink_worker/src/conf/index.ts +++ b/services/apps/data_sink_worker/src/conf/index.ts @@ -56,15 +56,11 @@ export const SLACK_ALERTING_CONFIG = (): ISlackAlertingConfig => { return slackAlertingConfig } -export interface IDataSinkWorkerTemporalConfig extends ITemporalConfig { - automationsTaskQueue: string -} - -let temporalConfig: IDataSinkWorkerTemporalConfig | undefined -export const TEMPORAL_CONFIG = (): IDataSinkWorkerTemporalConfig | undefined => { +let temporalConfig: ITemporalConfig | undefined +export const TEMPORAL_CONFIG = (): ITemporalConfig | undefined => { if (temporalConfig) return temporalConfig - temporalConfig = config.get('temporal') + temporalConfig = config.get('temporal') return temporalConfig } diff --git a/services/apps/data_sink_worker/src/service/activity.service.ts b/services/apps/data_sink_worker/src/service/activity.service.ts index 68cf867b9e..fd73f69832 100644 --- a/services/apps/data_sink_worker/src/service/activity.service.ts +++ b/services/apps/data_sink_worker/src/service/activity.service.ts @@ -4,7 +4,6 @@ import mergeWith from 'lodash.mergewith' import moment from 'moment-timezone' import { - EDITION, escapeNullByte, generateUUIDv4, isObjectEmpty, @@ -36,19 +35,15 @@ import { DEFAULT_ACTIVITY_TYPE_SETTINGS, GithubActivityType } from '@crowd/integ import { GitActivityType } from '@crowd/integrations/src/integrations/git/types' import { Logger, LoggerBase, getChildLogger } from '@crowd/logging' import { RedisClient } from '@crowd/redis' -import { Client as TemporalClient, WorkflowIdReusePolicy } from '@crowd/temporal' +import { Client as TemporalClient } from '@crowd/temporal' import { - Edition, IActivityData, ISentimentAnalysisResult, MemberAttributeName, MemberIdentityType, PlatformType, - TemporalWorkflowId, } from '@crowd/types' -import { TEMPORAL_CONFIG } from '../conf' - import { IActivityCreateData, IActivityUpdateData, ISentimentActivityInput } from './activity.data' import { UnrepeatableError } from './common' import MemberService from './member.service' @@ -139,38 +134,6 @@ export default class ActivityService extends LoggerBase { return activity.id }) - if (EDITION !== Edition.LFX) { - try { - const handle = await this.temporal.workflow.start('processNewActivityAutomation', { - workflowId: `${TemporalWorkflowId.NEW_ACTIVITY_AUTOMATION}/${id}`, - taskQueue: TEMPORAL_CONFIG().automationsTaskQueue, - workflowIdReusePolicy: WorkflowIdReusePolicy.WORKFLOW_ID_REUSE_POLICY_REJECT_DUPLICATE, - retry: { - maximumAttempts: 100, - }, - args: [ - { - tenantId, - activityId: id, - }, - ], - searchAttributes: { - TenantId: [tenantId], - }, - }) - this.log.info( - { workflowId: handle.workflowId }, - 'Started temporal workflow to process new activity automation!', - ) - } catch (err) { - this.log.error( - err, - 'Error while starting temporal workflow to process new activity automation!', - ) - throw err - } - } - return id } catch (err) { this.log.error(err, 'Error while creating an activity!') diff --git a/services/apps/data_sink_worker/src/service/member.service.ts b/services/apps/data_sink_worker/src/service/member.service.ts index caa156cfcb..31f6822022 100644 --- a/services/apps/data_sink_worker/src/service/member.service.ts +++ b/services/apps/data_sink_worker/src/service/member.service.ts @@ -3,7 +3,6 @@ import mergeWith from 'lodash.mergewith' import uniqby from 'lodash.uniqby' import { - EDITION, getEarliestValidDate, getProperDisplayName, isDomainExcluded, @@ -21,9 +20,8 @@ import { import MemberRepository from '@crowd/data-access-layer/src/old/apps/data_sink_worker/repo/member.repo' import { Logger, LoggerBase, getChildLogger } from '@crowd/logging' import { RedisClient } from '@crowd/redis' -import { Client as TemporalClient, WorkflowIdReusePolicy } from '@crowd/temporal' +import { Client as TemporalClient } from '@crowd/temporal' import { - Edition, IMemberData, IMemberIdentity, IOrganizationIdSource, @@ -32,11 +30,8 @@ import { OrganizationIdentityType, OrganizationSource, PlatformType, - TemporalWorkflowId, } from '@crowd/types' -import { TEMPORAL_CONFIG } from '../conf' - import { IMemberCreateData, IMemberUpdateData } from './member.data' import MemberAttributeService from './memberAttribute.service' import { OrganizationService } from './organization.service' @@ -162,40 +157,6 @@ export default class MemberService extends LoggerBase { } }) - if (EDITION !== Edition.LFX) { - try { - const handle = await this.temporal.workflow.start('processNewMemberAutomation', { - workflowId: `${TemporalWorkflowId.NEW_MEMBER_AUTOMATION}/${id}`, - taskQueue: TEMPORAL_CONFIG().automationsTaskQueue, - workflowIdReusePolicy: WorkflowIdReusePolicy.WORKFLOW_ID_REUSE_POLICY_REJECT_DUPLICATE, - retry: { - maximumAttempts: 100, - }, - - args: [ - { - tenantId, - memberId: id, - }, - ], - searchAttributes: { - TenantId: [tenantId], - }, - }) - - this.log.info( - { workflowId: handle.workflowId }, - 'Started temporal workflow to process new member automation!', - ) - } catch (err) { - this.log.error( - err, - 'Error while starting temporal workflow to process new member automation!', - ) - throw err - } - } - if (fireSync) { await this.searchSyncWorkerEmitter.triggerMemberSync(tenantId, id, onboarding, segmentId) } diff --git a/services/apps/integration_run_worker/src/main.ts b/services/apps/integration_run_worker/src/main.ts index 4d88dfa7dc..e27a061974 100644 --- a/services/apps/integration_run_worker/src/main.ts +++ b/services/apps/integration_run_worker/src/main.ts @@ -1,7 +1,6 @@ import { IntegrationRunWorkerEmitter, IntegrationStreamWorkerEmitter, - IntegrationSyncWorkerEmitter, PriorityLevelContextRepository, QueuePriorityContextLoader, SearchSyncWorkerEmitter, @@ -32,12 +31,6 @@ setImmediate(async () => { const runWorkerEmitter = new IntegrationRunWorkerEmitter(queueClient, redis, loader, log) const streamWorkerEmitter = new IntegrationStreamWorkerEmitter(queueClient, redis, loader, log) const searchSyncWorkerEmitter = new SearchSyncWorkerEmitter(queueClient, redis, loader, log) - const integrationSyncWorkerEmitter = new IntegrationSyncWorkerEmitter( - queueClient, - redis, - loader, - log, - ) const apiPubSubEmitter = new ApiPubSubEmitter(redis, log) @@ -49,7 +42,6 @@ setImmediate(async () => { streamWorkerEmitter, runWorkerEmitter, searchSyncWorkerEmitter, - integrationSyncWorkerEmitter, apiPubSubEmitter, log, MAX_CONCURRENT_PROCESSING, @@ -59,7 +51,6 @@ setImmediate(async () => { await streamWorkerEmitter.init() await runWorkerEmitter.init() await searchSyncWorkerEmitter.init() - await integrationSyncWorkerEmitter.init() await queue.start() } catch (err) { log.error({ err }, 'Failed to start queues!') diff --git a/services/apps/integration_run_worker/src/queue/index.ts b/services/apps/integration_run_worker/src/queue/index.ts index 0241187896..d7c3434443 100644 --- a/services/apps/integration_run_worker/src/queue/index.ts +++ b/services/apps/integration_run_worker/src/queue/index.ts @@ -1,7 +1,6 @@ import { IntegrationRunWorkerEmitter, IntegrationStreamWorkerEmitter, - IntegrationSyncWorkerEmitter, SearchSyncWorkerEmitter, } from '@crowd/common_services' import { DbConnection, DbStore } from '@crowd/data-access-layer/src/database' @@ -30,7 +29,6 @@ export class WorkerQueueReceiver extends PrioritizedQueueReciever { private readonly streamWorkerEmitter: IntegrationStreamWorkerEmitter, private readonly runWorkerEmitter: IntegrationRunWorkerEmitter, private readonly searchSyncWorkerEmitter: SearchSyncWorkerEmitter, - private readonly integrationSyncWorkerEmitter: IntegrationSyncWorkerEmitter, private readonly apiPubSubEmitter: ApiPubSubEmitter, parentLog: Logger, maxConcurrentProcessing: number, @@ -53,7 +51,6 @@ export class WorkerQueueReceiver extends PrioritizedQueueReciever { this.streamWorkerEmitter, this.runWorkerEmitter, this.searchSyncWorkerEmitter, - this.integrationSyncWorkerEmitter, this.apiPubSubEmitter, new DbStore(this.log, this.dbConn), this.log, diff --git a/services/apps/integration_run_worker/src/service/integrationRunService.ts b/services/apps/integration_run_worker/src/service/integrationRunService.ts index f9f5c8dc15..179c20483d 100644 --- a/services/apps/integration_run_worker/src/service/integrationRunService.ts +++ b/services/apps/integration_run_worker/src/service/integrationRunService.ts @@ -2,18 +2,12 @@ import { singleOrDefault } from '@crowd/common' import { IntegrationRunWorkerEmitter, IntegrationStreamWorkerEmitter, - IntegrationSyncWorkerEmitter, SearchSyncWorkerEmitter, } from '@crowd/common_services' import { DbStore } from '@crowd/data-access-layer/src/database' -import { AutomationRepository } from '@crowd/data-access-layer/src/old/apps/integration_run_worker/automation.repo' import IntegrationRunRepository from '@crowd/data-access-layer/src/old/apps/integration_run_worker/integrationRun.repo' import MemberAttributeSettingsRepository from '@crowd/data-access-layer/src/old/apps/integration_run_worker/memberAttributeSettings.repo' -import { - IGenerateStreamsContext, - IIntegrationStartRemoteSyncContext, - INTEGRATION_SERVICES, -} from '@crowd/integrations' +import { IGenerateStreamsContext, INTEGRATION_SERVICES } from '@crowd/integrations' import { Logger, LoggerBase, getChildLogger } from '@crowd/logging' import { ApiPubSubEmitter, RedisCache, RedisClient } from '@crowd/redis' import { IntegrationRunState, IntegrationStreamState } from '@crowd/types' @@ -22,14 +16,12 @@ import { NANGO_CONFIG, PLATFORM_CONFIG, WORKER_CONFIG } from '../conf' export default class IntegrationRunService extends LoggerBase { private readonly repo: IntegrationRunRepository - private readonly automationRepo: AutomationRepository constructor( private readonly redisClient: RedisClient, private readonly streamWorkerEmitter: IntegrationStreamWorkerEmitter, private readonly runWorkerEmitter: IntegrationRunWorkerEmitter, private readonly searchSyncWorkerEmitter: SearchSyncWorkerEmitter, - private readonly integrationSyncWorkerEmitter: IntegrationSyncWorkerEmitter, private readonly apiPubSubEmitter: ApiPubSubEmitter, private readonly store: DbStore, parentLog: Logger, @@ -37,7 +29,6 @@ export default class IntegrationRunService extends LoggerBase { super(parentLog) this.repo = new IntegrationRunRepository(store, this.log) - this.automationRepo = new AutomationRepository(store, this.log) } public async handleStreamProcessed(runId: string): Promise { @@ -132,59 +123,6 @@ export default class IntegrationRunService extends LoggerBase { this.log.error({ err }, 'Error while post processing integration settings!') } - try { - this.log.info('Trying to process startSyncRemote!') - - const service = singleOrDefault( - INTEGRATION_SERVICES, - (s) => s.type === runInfo.integrationType, - ) - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const settings = runInfo.integrationSettings as any - - if ( - service.startSyncRemote && - settings.syncRemoteEnabled && - settings.blockSyncRemote !== true - ) { - const syncAutomations = await this.automationRepo.findSyncAutomations( - runInfo.tenantId, - runInfo.integrationType, - ) - - const syncRemoteContext: IIntegrationStartRemoteSyncContext = { - integrationSyncWorkerEmitter: this.integrationSyncWorkerEmitter, - integration: { - id: runInfo.integrationId, - identifier: runInfo.integrationIdentifier, - platform: runInfo.integrationType, - status: runInfo.integrationState, - settings: runInfo.integrationSettings, - token: runInfo.integrationToken, - refreshToken: runInfo.integrationRefreshToken, - }, - automations: syncAutomations, - tenantId: runInfo.tenantId, - log: this.log, - } - - await service.startSyncRemote(syncRemoteContext) - - this.log.info('Finished processing startSyncRemote!') - } else { - this.log.info( - { - syncRemoteEnabled: settings.syncRemoteEnabled, - blockSyncRemote: settings.blockSyncRemote, - }, - `Integration does not have a startSyncRemote function or settings disable sync remote!`, - ) - } - } catch (err) { - this.log.error({ err }, 'Error while starting integration sync remote!') - } - this.log.info('Marking run and integration as successfully processed!') await this.repo.markRunProcessed(runId) await this.repo.markIntegration(runId, 'done') diff --git a/services/apps/integration_sync_worker/config/custom-environment-variables.json b/services/apps/integration_sync_worker/config/custom-environment-variables.json deleted file mode 100644 index 63ca0e12b7..0000000000 --- a/services/apps/integration_sync_worker/config/custom-environment-variables.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "service": { - "edition": "CROWD_EDITION", - "queuePriorityLevel": "QUEUE_PRIORITY_LEVEL" - }, - "db": { - "host": "CROWD_DB_WRITE_HOST", - "port": "CROWD_DB_PORT", - "database": "CROWD_DB_DATABASE", - "user": "CROWD_DB_USERNAME", - "password": "CROWD_DB_PASSWORD" - }, - "queue": { - "brokers": "CROWD_KAFKA_BROKERS", - "extra": "CROWD_KAFKA_EXTRA" - }, - "redis": { - "username": "CROWD_REDIS_USERNAME", - "password": "CROWD_REDIS_PASSWORD", - "host": "CROWD_REDIS_HOST", - "port": "CROWD_REDIS_PORT" - }, - "opensearch": { - "node": "CROWD_OPENSEARCH_NODE", - "username": "CROWD_OPENSEARCH_USERNAME", - "password": "CROWD_OPENSEARCH_PASSWORD" - }, - "nango": { - "url": "CROWD_NANGO_URL", - "secretKey": "CROWD_NANGO_SECRET_KEY" - } -} diff --git a/services/apps/integration_sync_worker/config/default.json b/services/apps/integration_sync_worker/config/default.json deleted file mode 100644 index 808bb07ab4..0000000000 --- a/services/apps/integration_sync_worker/config/default.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "service": { - "queuePriorityLevel": "normal" - }, - "db": {}, - "queue": {}, - "redis": {}, - "opensearch": {}, - "nango": {} -} diff --git a/services/apps/integration_sync_worker/config/docker.json b/services/apps/integration_sync_worker/config/docker.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/services/apps/integration_sync_worker/config/docker.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/services/apps/integration_sync_worker/config/production.json b/services/apps/integration_sync_worker/config/production.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/services/apps/integration_sync_worker/config/production.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/services/apps/integration_sync_worker/config/staging.json b/services/apps/integration_sync_worker/config/staging.json deleted file mode 100644 index 0967ef424b..0000000000 --- a/services/apps/integration_sync_worker/config/staging.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/services/apps/integration_sync_worker/package.json b/services/apps/integration_sync_worker/package.json deleted file mode 100644 index 44cfa55f30..0000000000 --- a/services/apps/integration_sync_worker/package.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "@crowd/integration-sync-worker", - "scripts": { - "start": "SERVICE=integration-sync-worker tsx src/main.ts", - "start:debug:local": "set -a && . ../../../backend/.env.dist.local && . ../../../backend/.env.override.local && set +a && SERVICE=integration-sync-worker LOG_LEVEL=trace tsx --inspect=0.0.0.0:9235 src/main.ts", - "start:debug": "SERVICE=integration-sync-worker LOG_LEVEL=trace tsx --inspect=0.0.0.0:9235 src/main.ts", - "dev:local": "nodemon --watch src --watch ../../libs --ext ts --exec pnpm run start:debug:local", - "dev": "nodemon --watch src --watch ../../libs --ext ts --exec pnpm run start:debug", - "lint": "npx eslint --ext .ts src --max-warnings=0", - "format": "npx prettier --write \"src/**/*.ts\"", - "format-check": "npx prettier --check .", - "tsc-check": "tsc --noEmit" - }, - "dependencies": { - "@crowd/common": "workspace:*", - "@crowd/data-access-layer": "workspace:*", - "@crowd/integrations": "workspace:*", - "@crowd/logging": "workspace:*", - "@crowd/opensearch": "workspace:*", - "@crowd/queue": "workspace:*", - "@crowd/types": "workspace:*", - "@opensearch-project/opensearch": "^2.11.0", - "config": "^3.3.9", - "tsx": "^4.7.1", - "typescript": "^5.6.3" - }, - "devDependencies": { - "@types/config": "^3.3.0", - "@types/node": "^18.16.3", - "nodemon": "^2.0.22" - } -} diff --git a/services/apps/integration_sync_worker/src/conf/index.ts b/services/apps/integration_sync_worker/src/conf/index.ts deleted file mode 100644 index 5df2c660c1..0000000000 --- a/services/apps/integration_sync_worker/src/conf/index.ts +++ /dev/null @@ -1,54 +0,0 @@ -import config from 'config' - -import { IDatabaseConfig } from '@crowd/data-access-layer/src/database' -import { IQueueClientConfig } from '@crowd/queue' -import { IOpenSearchConfig, QueuePriorityLevel } from '@crowd/types' - -export interface IServiceConfig { - edition: string - queuePriorityLevel: QueuePriorityLevel -} -export interface INangoConfig { - url: string - secretKey: string -} - -let serviceConfig: IServiceConfig -export const SERVICE_CONFIG = (): IServiceConfig => { - if (serviceConfig) return serviceConfig - - serviceConfig = config.get('service') - return serviceConfig -} - -let queueConfig: IQueueClientConfig -export const QUEUE_CONFIG = (): IQueueClientConfig => { - if (queueConfig) return queueConfig - - queueConfig = config.get('queue') - return queueConfig -} - -let dbConfig: IDatabaseConfig -export const DB_CONFIG = (): IDatabaseConfig => { - if (dbConfig) return dbConfig - - dbConfig = config.get('db') - return dbConfig -} - -let openSearchConfig: IOpenSearchConfig -export const OPENSEARCH_CONFIG = (): IOpenSearchConfig => { - if (openSearchConfig) return openSearchConfig - - openSearchConfig = config.get('opensearch') - return openSearchConfig -} - -let nangoConfig: INangoConfig -export const NANGO_CONFIG = (): INangoConfig => { - if (nangoConfig) return nangoConfig - - nangoConfig = config.get('nango') - return nangoConfig -} diff --git a/services/apps/integration_sync_worker/src/errors.ts b/services/apps/integration_sync_worker/src/errors.ts deleted file mode 100644 index 3096292c3c..0000000000 --- a/services/apps/integration_sync_worker/src/errors.ts +++ /dev/null @@ -1,5 +0,0 @@ -export const integrationNotFound = (integrationId: string): string => - `Integration ${integrationId} not found!` - -export const automationNotFound = (automationId: string): string => - `Automation ${automationId} not found!` diff --git a/services/apps/integration_sync_worker/src/main.ts b/services/apps/integration_sync_worker/src/main.ts deleted file mode 100644 index 0bec5a813f..0000000000 --- a/services/apps/integration_sync_worker/src/main.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { getDbConnection } from '@crowd/data-access-layer/src/database' -import { getServiceLogger } from '@crowd/logging' -import { getOpensearchClient } from '@crowd/opensearch' -import { QueueFactory } from '@crowd/queue' - -import { DB_CONFIG, OPENSEARCH_CONFIG, QUEUE_CONFIG, SERVICE_CONFIG } from './conf' -import { WorkerQueueReceiver } from './queue' - -const log = getServiceLogger() - -const MAX_CONCURRENT_PROCESSING = 2 - -setImmediate(async () => { - log.info('Starting integration sync worker...') - const queueClient = QueueFactory.createQueueService(QUEUE_CONFIG()) - - const dbConnection = await getDbConnection(DB_CONFIG(), MAX_CONCURRENT_PROCESSING) - - const opensearchClient = await getOpensearchClient(OPENSEARCH_CONFIG()) - - const worker = new WorkerQueueReceiver( - SERVICE_CONFIG().queuePriorityLevel, - queueClient, - dbConnection, - opensearchClient, - log, - MAX_CONCURRENT_PROCESSING, - ) - - try { - await worker.start() - } catch (err) { - log.error(err, 'Failed to start!') - process.exit(1) - } -}) diff --git a/services/apps/integration_sync_worker/src/queue/index.ts b/services/apps/integration_sync_worker/src/queue/index.ts deleted file mode 100644 index a3e630d17f..0000000000 --- a/services/apps/integration_sync_worker/src/queue/index.ts +++ /dev/null @@ -1,125 +0,0 @@ -import { Client } from '@opensearch-project/opensearch' - -import { DbConnection, DbStore } from '@crowd/data-access-layer/src/database' -import { Logger } from '@crowd/logging' -import { CrowdQueue, IQueue, PrioritizedQueueReciever } from '@crowd/queue' -import { - AutomationSyncTrigger, - IQueueMessage, - IntegrationSyncWorkerQueueMessageType, - QueuePriorityLevel, -} from '@crowd/types' - -import { MemberSyncService } from '../service/member.sync.service' -import { OpenSearchService } from '../service/opensearch.service' -import { OrganizationSyncService } from '../service/organization.sync.service' - -export class WorkerQueueReceiver extends PrioritizedQueueReciever { - constructor( - level: QueuePriorityLevel, - client: IQueue, - private readonly dbConn: DbConnection, - private readonly openSearchClient: Client, - parentLog: Logger, - maxConcurrentProcessing: number, - ) { - super( - level, - client, - client.getQueueChannelConfig(CrowdQueue.INTEGRATION_SYNC_WORKER), - maxConcurrentProcessing, - parentLog, - ) - } - - private initMemberService(): MemberSyncService { - return new MemberSyncService( - new DbStore(this.log, this.dbConn), - new OpenSearchService(this.log, this.openSearchClient), - this.log, - ) - } - - private initOrganizationService(): OrganizationSyncService { - return new OrganizationSyncService(new DbStore(this.log, this.dbConn), this.log) - } - - public override async processMessage(message: T): Promise { - try { - this.log.trace({ messageType: message.type }, 'Processing message!') - - const type = message.type as IntegrationSyncWorkerQueueMessageType - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const data = message as any - - switch (type) { - // members - case IntegrationSyncWorkerQueueMessageType.SYNC_ALL_MARKED_MEMBERS: - await this.initMemberService().syncAllMarkedMembers(data.tenantId, data.integrationId) - - break - case IntegrationSyncWorkerQueueMessageType.SYNC_MEMBER: - await this.initMemberService().syncMember( - data.tenantId, - data.integrationId, - data.memberId, - data.syncRemoteId, - ) - break - case IntegrationSyncWorkerQueueMessageType.SYNC_ORGANIZATION: - await this.initOrganizationService().syncOrganization( - data.tenantId, - data.integrationId, - data.organizationId, - data.syncRemoteId, - ) - break - case IntegrationSyncWorkerQueueMessageType.SYNC_ALL_MARKED_ORGANIZATIONS: - await this.initOrganizationService().syncAllMarkedOrganizations( - data.tenantId, - data.integrationId, - ) - break - case IntegrationSyncWorkerQueueMessageType.ONBOARD_AUTOMATION: - if (data.automationTrigger === AutomationSyncTrigger.MEMBER_ATTRIBUTES_MATCH) { - await this.initMemberService().syncAllFilteredMembers( - data.tenantId, - data.integrationId, - data.automationId, - ) - } else if ( - data.automationTrigger === AutomationSyncTrigger.ORGANIZATION_ATTRIBUTES_MATCH - ) { - const organizationIds = - await this.initOrganizationService().syncAllFilteredOrganizations( - data.tenantId, - data.integrationId, - data.automationId, - ) - - // also sync organization members if syncAllFilteredOrganizations return the ids - while (organizationIds.length > 0) { - const organizationId = organizationIds.shift() - await this.initMemberService().syncOrganizationMembers( - data.tenantId, - data.integrationId, - data.automationId, - organizationId, - ) - } - } else { - const errorMessage = `Unsupported trigger for onboard automation message!` - this.log.error({ message }, errorMessage) - throw new Error(errorMessage) - } - break - - default: - throw new Error(`Unknown message type: ${message.type}`) - } - } catch (err) { - this.log.error(err, 'Error while processing message!') - throw err - } - } -} diff --git a/services/apps/integration_sync_worker/src/service/member.sync.service.ts b/services/apps/integration_sync_worker/src/service/member.sync.service.ts deleted file mode 100644 index 33e79a05be..0000000000 --- a/services/apps/integration_sync_worker/src/service/member.sync.service.ts +++ /dev/null @@ -1,638 +0,0 @@ -import { singleOrDefault } from '@crowd/common' -import { DbStore } from '@crowd/data-access-layer/src/database' -import { AutomationRepository } from '@crowd/data-access-layer/src/old/apps/integration_sync_worker/automation.repo' -import { AutomationExecutionRepository } from '@crowd/data-access-layer/src/old/apps/integration_sync_worker/automationExecution.repo' -import { IDbIntegration } from '@crowd/data-access-layer/src/old/apps/integration_sync_worker/integration.data' -import { IntegrationRepository } from '@crowd/data-access-layer/src/old/apps/integration_sync_worker/integration.repo' -import { MemberRepository } from '@crowd/data-access-layer/src/old/apps/integration_sync_worker/member.repo' -import { - IBatchCreateMembersResult, - IBatchUpdateMembersResult, - IIntegrationProcessRemoteSyncContext, - INTEGRATION_SERVICES, -} from '@crowd/integrations' -import { Logger, LoggerBase } from '@crowd/logging' -import { FieldTranslatorFactory, OpensearchQueryParser } from '@crowd/opensearch' -import { Entity, HubspotSettings, IMember, OpenSearchIndex } from '@crowd/types' -import { Edition } from '@crowd/types' - -import { NANGO_CONFIG, SERVICE_CONFIG } from '../conf' -import { automationNotFound, integrationNotFound } from '../errors' - -import { ISearchHit } from './opensearch.data' -import { OpenSearchService } from './opensearch.service' - -export class MemberSyncService extends LoggerBase { - private readonly memberRepo: MemberRepository - private readonly integrationRepo: IntegrationRepository - private readonly automationRepo: AutomationRepository - private readonly automationExecutionRepo: AutomationExecutionRepository - - constructor( - store: DbStore, - private readonly openSearchService: OpenSearchService, - parentLog: Logger, - ) { - super(parentLog) - - this.memberRepo = new MemberRepository(store, this.log) - this.integrationRepo = new IntegrationRepository(store, this.log) - this.automationRepo = new AutomationRepository(store, this.log) - this.automationExecutionRepo = new AutomationExecutionRepository(store, this.log) - } - - public async syncMember( - tenantId: string, - integrationId: string, - memberId: string, - syncRemoteId: string, - ): Promise { - const integration = await this.integrationRepo.findById(integrationId) - - if (!integration) { - const message = integrationNotFound(integrationId) - this.log.warn(message) - return - } - - const member = await this.memberRepo.findMember(memberId) - - if (!member) { - this.log.warn(`Member ${memberId} is not found for syncing remote!`) - return - } - - const syncRemote = await this.memberRepo.findSyncRemoteById(syncRemoteId) - - const membersToCreate = [] - const membersToUpdate = [] - - if (syncRemote.sourceId) { - member.attributes = { - ...member.attributes, - sourceId: { - ...(member.attributes.sourceId || {}), - [integration.platform]: syncRemote.sourceId, - }, - } - - membersToUpdate.push(member) - } else { - membersToCreate.push(member) - } - - const service = singleOrDefault(INTEGRATION_SERVICES, (s) => s.type === integration.platform) - - this.log.info(`Syncing member ${memberId} to ${integration.platform} remote!`) - - if (service.processSyncRemote) { - const context: IIntegrationProcessRemoteSyncContext = { - integration, - log: this.log, - serviceSettings: { - nangoId: `${tenantId}-${integration.platform}`, - nangoUrl: NANGO_CONFIG().url, - nangoSecretKey: NANGO_CONFIG().secretKey, - }, - tenantId, - } - - const { created, updated } = await service.processSyncRemote( - membersToCreate, - membersToUpdate, - Entity.MEMBERS, - context, - ) - - if (created.length > 0) { - const memberCreated = created[0] as IBatchCreateMembersResult - await this.memberRepo.setSyncRemoteSourceId(syncRemoteId, memberCreated.sourceId) - await this.memberRepo.setLastSyncedAtBySyncRemoteId( - syncRemoteId, - memberCreated.lastSyncedPayload, - ) - } - - if (updated.length > 0) { - const memberUpdated = updated[0] as IBatchUpdateMembersResult - await this.memberRepo.setLastSyncedAtBySyncRemoteId( - syncRemoteId, - memberUpdated.lastSyncedPayload, - ) - } - } else { - this.log.warn(`Integration ${integration.platform} has no processSyncRemote function!`) - } - } - - public async syncAllMarkedMembers( - tenantId: string, - integrationId: string, - batchSize = 100, - ): Promise { - const integration: IDbIntegration = await this.integrationRepo.findById(integrationId) - - if (!integration) { - const message = integrationNotFound(integrationId) - this.log.warn(message) - return - } - - const platforms = await this.memberRepo.getExistingPlatforms(tenantId) - - const attributes = await this.memberRepo.getTenantMemberAttributes(tenantId) - - const isMultiSegment = SERVICE_CONFIG().edition === Edition.LFX - - const translator = FieldTranslatorFactory.getTranslator(OpenSearchIndex.MEMBERS, attributes, [ - 'default', - 'custom', - 'crowd', - 'enrichment', - integration.platform, // it could be that no members have the outgoing identity yet, but member could be marked through attributes - ...platforms, - ]) - - let markedMembers - let offset - - do { - const membersToCreate: IMember[] = [] - const membersToUpdate: IMember[] = [] - - offset = markedMembers ? offset + batchSize : 0 - markedMembers = await this.memberRepo.findMarkedMembers(integration.id, batchSize, offset) - - while (markedMembers.length > 0) { - const memberToSync = markedMembers.shift() - - this.log.info(`Syncing member ${memberToSync.memberId} to ${integration.platform} remote!`) - - // find member in opensearch - const query = { - bool: { - must: [ - { - term: { - [`uuid_memberId`]: memberToSync.memberId, - }, - }, - ], - }, - // eslint-disable-next-line @typescript-eslint/no-explicit-any - } as any - - if (isMultiSegment) { - query.must.push({ - term: { - uuid_segmentId: integration.segmentId, - }, - }) - } - - const memberFromOpensearch = (await this.openSearchService.search( - OpenSearchIndex.MEMBERS, - query, - undefined, - 1, - undefined, - undefined, - )) as ISearchHit[] - - const translatedMembers: IMember[] = memberFromOpensearch.reduce((acc, member) => { - const membersWithCrowdFields = translator.translateObjectToCrowd(member._source) - acc.push(membersWithCrowdFields) - return acc - }, []) - - if (translatedMembers.length !== 1) { - this.log.warn('Found more than one member matching the id from opensearch - exiting!') - return - } - - if (memberToSync.sourceId) { - // append sourceId to object - it'll be used for updating the remote counterpart - translatedMembers[0].attributes = { - ...translatedMembers[0].attributes, - sourceId: { - ...(translatedMembers[0].attributes.sourceId || {}), - [integration.platform]: memberToSync.sourceId, - }, - } - membersToUpdate.push(translatedMembers[0]) - } else { - membersToCreate.push(translatedMembers[0]) - } - } - - const service = singleOrDefault(INTEGRATION_SERVICES, (s) => s.type === integration.platform) - - if (service.processSyncRemote) { - const context: IIntegrationProcessRemoteSyncContext = { - integration, - log: this.log, - serviceSettings: { - nangoId: `${tenantId}-${integration.platform}`, - nangoUrl: NANGO_CONFIG().url, - nangoSecretKey: NANGO_CONFIG().secretKey, - }, - tenantId, - } - - const { created, updated } = await service.processSyncRemote( - membersToCreate, - membersToUpdate, - Entity.MEMBERS, - context, - ) - - for (const newMember of created as IBatchCreateMembersResult[]) { - await this.memberRepo.setIntegrationSourceId( - newMember.memberId, - integration.id, - newMember.sourceId, - ) - await this.memberRepo.setLastSyncedAt( - newMember.memberId, - integration.id, - newMember.lastSyncedPayload, - ) - } - - for (const updatedMember of updated as IBatchUpdateMembersResult[]) { - await this.memberRepo.setLastSyncedAt( - updatedMember.memberId, - integration.id, - updatedMember.lastSyncedPayload, - ) - } - } else { - this.log.warn(`Integration ${integration.platform} has no processSyncRemote function!`) - } - } while (markedMembers.length > 0) - } - - public async syncAllFilteredMembers( - tenantId: string, - integrationId: string, - automationId: string, - batchSize = 50, - ) { - const integration: IDbIntegration = await this.integrationRepo.findById(integrationId) - - if (!integration) { - const message = integrationNotFound(integrationId) - this.log.warn(message) - return - } - const automation = await this.automationRepo.findById(automationId) - - if (!automation) { - const message = automationNotFound(automationId) - this.log.warn(message) - return - } - - const platforms = await this.memberRepo.getExistingPlatforms(tenantId) - - const attributes = await this.memberRepo.getTenantMemberAttributes(tenantId) - - const isMultiSegment = SERVICE_CONFIG().edition === Edition.LFX - - const translator = FieldTranslatorFactory.getTranslator(OpenSearchIndex.MEMBERS, attributes, [ - 'default', - 'custom', - 'crowd', - 'enrichment', - automation.type, // it could be that no members have the outgoing identity yet, but member could be marked through attributes - ...platforms, - ]) - - const parseOptions = { - filter: (automation.settings as HubspotSettings).filter, - limit: batchSize, - offset: 0, - orderBy: 'createdAt_DESC', - } - - const parsed = OpensearchQueryParser.parse(parseOptions, OpenSearchIndex.MEMBERS, translator) - - // add tenant filter - parsed.query.bool.must.push({ - term: { - [`uuid_tenantId`]: tenantId, - }, - }) - - // discard team members, bots and organizations - parsed.query.bool.must.push({ - bool: { - must: [ - { - bool: { - must_not: { - term: { - [`obj_attributes.obj_isTeamMember.bool_default`]: true, - }, - }, - }, - }, - { - bool: { - must_not: { - term: { - [`obj_attributes.obj_isBot.bool_default`]: true, - }, - }, - }, - }, - { - bool: { - must_not: { - term: { - [`obj_attributes.obj_isOrganization.bool_default`]: true, - }, - }, - }, - }, - ], - }, - }) - - if (isMultiSegment) { - parsed.query.bool.must.push({ - term: { - uuid_segmentId: integration.segmentId, - }, - }) - } - - let filteredMembers - let offset - - this.log.trace( - { filter: (automation.settings as HubspotSettings).filter }, - 'Trying to sync members that conform to sent filter!', - ) - - try { - do { - const membersToCreate: IMember[] = [] - const membersToUpdate: IMember[] = [] - - offset = filteredMembers ? offset + batchSize : 0 - parsed.from = offset - - filteredMembers = await this.openSearchService.client.search({ - index: OpenSearchIndex.MEMBERS, - body: parsed, - }) - - this.log.trace('Found members: ') - this.log.trace(filteredMembers) - - const translatedMembers: IMember[] = filteredMembers.body.hits.hits.reduce( - (acc, member) => { - const membersWithCrowdFields = translator.translateObjectToCrowd(member._source) - acc.push(membersWithCrowdFields) - return acc - }, - [], - ) - - while (translatedMembers.length > 0) { - const memberToSync = translatedMembers.shift() - - this.log.info(`Syncing member ${memberToSync.id} to ${integration.platform} remote!`) - - let syncRemote = await this.memberRepo.findSyncRemote( - memberToSync.id, - integration.id, - automation.id, - ) - - // member isn't marked yet, mark it - if (!syncRemote) { - syncRemote = await this.memberRepo.markMemberForSyncing({ - integrationId: integration.id, - memberId: memberToSync.id, - metaData: null, - syncFrom: automation.id, - }) - } - - if (syncRemote.sourceId) { - memberToSync.attributes = { - ...memberToSync.attributes, - sourceId: { - ...(memberToSync.attributes.sourceId || {}), - [integration.platform]: syncRemote.sourceId, - }, - } - membersToUpdate.push(memberToSync) - } else { - membersToCreate.push(memberToSync) - } - } - - const service = singleOrDefault( - INTEGRATION_SERVICES, - (s) => s.type === integration.platform, - ) - - if (service.processSyncRemote) { - const context: IIntegrationProcessRemoteSyncContext = { - integration, - log: this.log, - serviceSettings: { - nangoId: `${tenantId}-${integration.platform}`, - nangoUrl: NANGO_CONFIG().url, - nangoSecretKey: NANGO_CONFIG().secretKey, - }, - tenantId, - automation, - } - - const { created, updated } = await service.processSyncRemote( - membersToCreate, - membersToUpdate, - Entity.MEMBERS, - context, - ) - - for (const newMember of created as IBatchCreateMembersResult[]) { - await this.memberRepo.setIntegrationSourceId( - newMember.memberId, - integration.id, - newMember.sourceId, - ) - - await this.memberRepo.setLastSyncedAt( - newMember.memberId, - integration.id, - newMember.lastSyncedPayload, - ) - } - - for (const updatedMember of updated as IBatchUpdateMembersResult[]) { - await this.memberRepo.setLastSyncedAt( - updatedMember.memberId, - integration.id, - updatedMember.lastSyncedPayload, - ) - } - } else { - this.log.warn(`Integration ${integration.platform} has no processSyncRemote function!`) - } - } while (filteredMembers.length > 0) - - await this.automationExecutionRepo.addExecution({ - automationId: automation.id, - type: automation.type, - trigger: automation.trigger, - tenantId: automation.tenantId, - state: 'success', - payload: { - type: automation.type, - trigger: automation.trigger, - }, - error: null, - }) - } catch (e) { - await this.automationExecutionRepo.addExecution({ - automationId: automation.id, - type: automation.type, - trigger: automation.trigger, - tenantId: automation.tenantId, - state: 'error', - payload: { - type: automation.type, - trigger: automation.trigger, - }, - error: JSON.stringify(e), - }) - - throw e - } - } - - public async syncOrganizationMembers( - tenantId: string, - integrationId: string, - automationId: string, - organizationId: string, - batchSize = 100, - ) { - const integration: IDbIntegration = await this.integrationRepo.findById(integrationId) - - if (!integration) { - const message = integrationNotFound(integrationId) - this.log.warn(message) - return - } - - const automation = await this.automationRepo.findById(automationId) - - if (!automation) { - const message = automationNotFound(automationId) - this.log.warn(message) - return - } - - let organizationMembers - let offset - - do { - const membersToCreate: IMember[] = [] - const membersToUpdate: IMember[] = [] - - offset = organizationMembers ? offset + batchSize : 0 - organizationMembers = await this.memberRepo.findOrganizationMembers( - organizationId, - batchSize, - offset, - ) - - while (organizationMembers.length > 0) { - const memberToSync = organizationMembers.shift() - - this.log.info(`Syncing member ${memberToSync.memberId} to ${integration.platform} remote!`) - const member = await this.memberRepo.findMember(memberToSync.memberId) - - let syncRemote = await this.memberRepo.findSyncRemote( - member.id, - integration.id, - automation.id, - ) - - // member isn't marked yet, mark it - if (!syncRemote) { - syncRemote = await this.memberRepo.markMemberForSyncing({ - integrationId: integration.id, - memberId: member.id, - metaData: null, - syncFrom: automation.id, - }) - } - - if (syncRemote.sourceId) { - member.attributes = { - ...member.attributes, - sourceId: { - ...(member.attributes.sourceId || {}), - [integration.platform]: syncRemote.sourceId, - }, - } - membersToUpdate.push(member) - } else { - membersToCreate.push(member) - } - } - - const service = singleOrDefault(INTEGRATION_SERVICES, (s) => s.type === integration.platform) - - if (service.processSyncRemote) { - const context: IIntegrationProcessRemoteSyncContext = { - integration, - log: this.log, - serviceSettings: { - nangoId: `${tenantId}-${integration.platform}`, - nangoUrl: NANGO_CONFIG().url, - nangoSecretKey: NANGO_CONFIG().secretKey, - }, - tenantId, - } - - const { created, updated } = await service.processSyncRemote( - membersToCreate, - membersToUpdate, - Entity.MEMBERS, - context, - ) - - for (const newMember of created as IBatchCreateMembersResult[]) { - await this.memberRepo.setIntegrationSourceId( - newMember.memberId, - integration.id, - newMember.sourceId, - ) - await this.memberRepo.setLastSyncedAt( - newMember.memberId, - integration.id, - newMember.lastSyncedPayload, - ) - } - - for (const updatedMember of updated as IBatchUpdateMembersResult[]) { - await this.memberRepo.setLastSyncedAt( - updatedMember.memberId, - integration.id, - updatedMember.lastSyncedPayload, - ) - } - } else { - this.log.warn(`Integration ${integration.platform} has no processSyncRemote function!`) - } - } while (organizationMembers.length > 0) - } -} diff --git a/services/apps/integration_sync_worker/src/service/opensearch.data.ts b/services/apps/integration_sync_worker/src/service/opensearch.data.ts deleted file mode 100644 index 13deac5cb8..0000000000 --- a/services/apps/integration_sync_worker/src/service/opensearch.data.ts +++ /dev/null @@ -1,5 +0,0 @@ -export interface ISearchHit { - _id: string - - _source?: T -} diff --git a/services/apps/integration_sync_worker/src/service/opensearch.service.ts b/services/apps/integration_sync_worker/src/service/opensearch.service.ts deleted file mode 100644 index f99f727de2..0000000000 --- a/services/apps/integration_sync_worker/src/service/opensearch.service.ts +++ /dev/null @@ -1,53 +0,0 @@ -import { Client } from '@opensearch-project/opensearch' - -import { Logger, LoggerBase } from '@crowd/logging' -import { OpenSearchIndex } from '@crowd/types' - -import { ISearchHit } from './opensearch.data' - -export class OpenSearchService extends LoggerBase { - public readonly client: Client - - constructor(parentLog: Logger, client: Client) { - super(parentLog) - this.client = client - } - - public async search( - index: OpenSearchIndex, - query?: unknown, - aggs?: unknown, - size?: number, - sort?: unknown[], - searchAfter?: unknown, - sourceIncludeFields?: string[], - sourceExcludeFields?: string[], - ): Promise[] | unknown> { - try { - const payload = { - index, - _source_excludes: sourceExcludeFields, - _source_includes: sourceIncludeFields, - body: { - size: aggs ? 0 : undefined, - query, - aggs, - search_after: searchAfter ? [searchAfter] : undefined, - sort, - }, - size, - } - - const data = await this.client.search(payload) - - if (query) { - return data.body.hits.hits - } else { - return data.body.aggregations - } - } catch (err) { - this.log.error(err, { index, query }, 'Failed to search documents!') - throw new Error('Failed to search documents!') - } - } -} diff --git a/services/apps/integration_sync_worker/src/service/organization.sync.service.ts b/services/apps/integration_sync_worker/src/service/organization.sync.service.ts deleted file mode 100644 index 74c6957753..0000000000 --- a/services/apps/integration_sync_worker/src/service/organization.sync.service.ts +++ /dev/null @@ -1,367 +0,0 @@ -import { singleOrDefault } from '@crowd/common' -import { DbStore } from '@crowd/data-access-layer/src/database' -import { AutomationRepository } from '@crowd/data-access-layer/src/old/apps/integration_sync_worker/automation.repo' -import { AutomationExecutionRepository } from '@crowd/data-access-layer/src/old/apps/integration_sync_worker/automationExecution.repo' -import { IDbIntegration } from '@crowd/data-access-layer/src/old/apps/integration_sync_worker/integration.data' -import { IntegrationRepository } from '@crowd/data-access-layer/src/old/apps/integration_sync_worker/integration.repo' -import { OrganizationRepository } from '@crowd/data-access-layer/src/old/apps/integration_sync_worker/organization.repo' -import { - IBatchCreateOrganizationsResult, - IBatchUpdateOrganizationsResult, - IIntegrationProcessRemoteSyncContext, - INTEGRATION_SERVICES, -} from '@crowd/integrations' -import { Logger, LoggerBase } from '@crowd/logging' -import { Entity, HubspotSettings, IOrganization, IOrganizationSyncRemoteData } from '@crowd/types' - -import { NANGO_CONFIG } from '../conf' - -export class OrganizationSyncService extends LoggerBase { - private readonly organizationRepo: OrganizationRepository - private readonly integrationRepo: IntegrationRepository - private readonly automationRepo: AutomationRepository - private readonly automationExecutionRepo: AutomationExecutionRepository - - constructor(store: DbStore, parentLog: Logger) { - super(parentLog) - - this.integrationRepo = new IntegrationRepository(store, this.log) - this.organizationRepo = new OrganizationRepository(store, this.log) - this.automationRepo = new AutomationRepository(store, this.log) - this.automationExecutionRepo = new AutomationExecutionRepository(store, this.log) - } - - public async syncOrganization( - tenantId: string, - integrationId: string, - organizationId: string, - syncRemoteId: string, - ): Promise { - const integration = await this.integrationRepo.findById(integrationId) - - const organization = await this.organizationRepo.findOrganization( - organizationId, - tenantId, - integration.segmentId, - ) - - const syncRemote = await this.organizationRepo.findSyncRemoteById(syncRemoteId) - - const oranizationsToCreate = [] - const organizationsToUpdate = [] - - if (syncRemote.sourceId) { - // organization.attributes = { - // ...organization.attributes, - // sourceId: { - // ...(organization.attributes.sourceId || {}), - // [integration.platform]: syncRemote.sourceId, - // }, - // } - organizationsToUpdate.push(organization) - } else { - oranizationsToCreate.push(organization) - } - - const service = singleOrDefault(INTEGRATION_SERVICES, (s) => s.type === integration.platform) - - this.log.info(`Syncing organization ${organizationId} to ${integration.platform} remote!`) - - if (service.processSyncRemote) { - const context: IIntegrationProcessRemoteSyncContext = { - integration, - log: this.log, - serviceSettings: { - nangoId: `${tenantId}-${integration.platform}`, - nangoUrl: NANGO_CONFIG().url, - nangoSecretKey: NANGO_CONFIG().secretKey, - }, - tenantId, - } - - const { created, updated } = await service.processSyncRemote( - oranizationsToCreate, - organizationsToUpdate, - Entity.ORGANIZATIONS, - context, - ) - - if (created.length > 0) { - const orgCreated = created[0] as IBatchCreateOrganizationsResult - await this.organizationRepo.setSyncRemoteSourceId(syncRemoteId, orgCreated.sourceId) - await this.organizationRepo.setLastSyncedAtBySyncRemoteId( - syncRemoteId, - orgCreated.lastSyncedPayload, - ) - } - - if (updated.length > 0) { - const orgUpdated = updated[0] as IBatchUpdateOrganizationsResult - await this.organizationRepo.setLastSyncedAtBySyncRemoteId( - syncRemoteId, - orgUpdated.lastSyncedPayload, - ) - } - } else { - this.log.warn(`Integration ${integration.platform} has no processSyncRemote function!`) - } - } - - public async syncAllMarkedOrganizations( - tenantId: string, - integrationId: string, - batchSize = 100, - ): Promise { - const integration: IDbIntegration = await this.integrationRepo.findById(integrationId) - - let markedOrganizations: IOrganizationSyncRemoteData[] - let offset - - do { - const organizationsToCreate: IOrganization[] = [] - const organizationsToUpdate: IOrganization[] = [] - - offset = markedOrganizations ? offset + batchSize : 0 - - markedOrganizations = await this.organizationRepo.findMarkedOrganizations( - integration.id, - batchSize, - offset, - ) - - for (const organizationToSync of markedOrganizations) { - this.log.info( - `Syncing organization ${organizationToSync.organizationId} to ${integration.platform} remote!`, - ) - - const organization = await this.organizationRepo.findOrganization( - organizationToSync.organizationId, - tenantId, - integration.segmentId, - ) - - if (organizationToSync.sourceId) { - // append sourceId to object - it'll be used for updating the remote counterpart - // organization.attributes = organization.attributes || {} - - // organization.attributes = { - // ...organization.attributes, - // sourceId: { - // ...(organization.attributes.sourceId || {}), - // [integration.platform]: organizationToSync.sourceId, - // }, - // } - organizationsToUpdate.push(organization) - } else { - organizationsToCreate.push(organization) - } - } - - const service = singleOrDefault(INTEGRATION_SERVICES, (s) => s.type === integration.platform) - - if (service.processSyncRemote) { - const context: IIntegrationProcessRemoteSyncContext = { - integration, - log: this.log, - serviceSettings: { - nangoId: `${tenantId}-${integration.platform}`, - nangoUrl: NANGO_CONFIG().url, - nangoSecretKey: NANGO_CONFIG().secretKey, - }, - tenantId, - } - - const { created, updated } = await service.processSyncRemote( - organizationsToCreate, - organizationsToUpdate, - Entity.ORGANIZATIONS, - context, - ) - - for (const newOrganization of created as IBatchCreateOrganizationsResult[]) { - await this.organizationRepo.setIntegrationSourceId( - newOrganization.organizationId, - integration.id, - newOrganization.sourceId, - ) - - await this.organizationRepo.setLastSyncedAt( - newOrganization.organizationId, - integration.id, - newOrganization.lastSyncedPayload, - ) - } - - for (const updatedOrganization of updated as IBatchUpdateOrganizationsResult[]) { - await this.organizationRepo.setLastSyncedAt( - updatedOrganization.organizationId, - integration.id, - updatedOrganization.lastSyncedPayload, - ) - } - } else { - this.log.warn(`Integration ${integration.platform} has no processSyncRemote function!`) - } - } while (markedOrganizations.length > 0) - } - - public async syncAllFilteredOrganizations( - tenantId: string, - integrationId: string, - automationId: string, - batchSize = 50, - ) { - const integration: IDbIntegration = await this.integrationRepo.findById(integrationId) - const automation = await this.automationRepo.findById(automationId) - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const syncOrganizationMembers = (automation.settings as any)?.syncCompanyContacts === true - - const organizationsIdsSyncedToReturn = [] - - let filteredOrganizations: { id: string }[] - let offset - - try { - do { - const organizationsToCreate: IOrganization[] = [] - const organizationsToUpdate: IOrganization[] = [] - - offset = filteredOrganizations ? offset + batchSize : 0 - - filteredOrganizations = await this.organizationRepo.findFilteredOrganizations( - tenantId, - [integration.segmentId], - integration.platform, - (automation.settings as HubspotSettings).filter, - batchSize, - offset, - ) - - for (const organizationToSync of filteredOrganizations) { - this.log.info( - `Syncing organization ${organizationToSync.id} to ${integration.platform} remote!`, - ) - - let syncRemote = await this.organizationRepo.findSyncRemote( - organizationToSync.id, - integration.id, - automation.id, - ) - - // member isn't marked yet, mark it - if (!syncRemote) { - syncRemote = await this.organizationRepo.markOrganizationForSyncing({ - integrationId: integration.id, - organizationId: organizationToSync.id, - metaData: null, - syncFrom: automation.id, - }) - } - - const organization = await this.organizationRepo.findOrganization( - organizationToSync.id, - tenantId, - integration.segmentId, - ) - - if (syncRemote.sourceId) { - // organization.attributes = { - // ...organization.attributes, - // sourceId: { - // ...(organization.attributes.sourceId || {}), - // [integration.platform]: syncRemote.sourceId, - // }, - // } - organizationsToUpdate.push(organization) - } else { - organizationsToCreate.push(organization) - } - } - - const service = singleOrDefault( - INTEGRATION_SERVICES, - (s) => s.type === integration.platform, - ) - - if (service.processSyncRemote) { - const context: IIntegrationProcessRemoteSyncContext = { - integration, - log: this.log, - serviceSettings: { - nangoId: `${tenantId}-${integration.platform}`, - nangoUrl: NANGO_CONFIG().url, - nangoSecretKey: NANGO_CONFIG().secretKey, - }, - tenantId, - } - - const { created, updated } = await service.processSyncRemote( - organizationsToCreate, - organizationsToUpdate, - Entity.ORGANIZATIONS, - context, - ) - - for (const newOrganization of created as IBatchCreateOrganizationsResult[]) { - await this.organizationRepo.setIntegrationSourceId( - newOrganization.organizationId, - integration.id, - newOrganization.sourceId, - ) - - await this.organizationRepo.setLastSyncedAt( - newOrganization.organizationId, - integration.id, - newOrganization.lastSyncedPayload, - ) - } - - for (const updatedOrganization of updated as IBatchUpdateOrganizationsResult[]) { - await this.organizationRepo.setLastSyncedAt( - updatedOrganization.organizationId, - integration.id, - updatedOrganization.lastSyncedPayload, - ) - } - - if (syncOrganizationMembers) { - organizationsIdsSyncedToReturn.push(...organizationsToCreate.map((o) => o.id)) - organizationsIdsSyncedToReturn.push(...organizationsToUpdate.map((o) => o.id)) - } - } else { - this.log.warn(`Integration ${integration.platform} has no processSyncRemote function!`) - } - } while (filteredOrganizations.length > 0) - - await this.automationExecutionRepo.addExecution({ - automationId: automation.id, - type: automation.type, - trigger: automation.trigger, - tenantId: automation.tenantId, - state: 'success', - payload: { - type: automation.type, - trigger: automation.trigger, - }, - error: null, - }) - } catch (e) { - await this.automationExecutionRepo.addExecution({ - automationId: automation.id, - type: automation.type, - trigger: automation.trigger, - tenantId: automation.tenantId, - state: 'error', - payload: { - type: automation.type, - trigger: automation.trigger, - }, - error: JSON.stringify(e), - }) - throw e - } - - return syncOrganizationMembers ? organizationsIdsSyncedToReturn : [] - } -} diff --git a/services/apps/integration_sync_worker/src/types.ts b/services/apps/integration_sync_worker/src/types.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/services/apps/integration_sync_worker/tsconfig.json b/services/apps/integration_sync_worker/tsconfig.json deleted file mode 100644 index bf7f183850..0000000000 --- a/services/apps/integration_sync_worker/tsconfig.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "extends": "../../base.tsconfig.json", - "include": ["src/**/*"] -} diff --git a/services/libs/common/src/env.ts b/services/libs/common/src/env.ts index b30d402409..75e0c94936 100644 --- a/services/libs/common/src/env.ts +++ b/services/libs/common/src/env.ts @@ -43,18 +43,6 @@ export const SEARCH_SYNC_WORKER_PARTITIONS: Record = : undefined, } -export const INTEGRATION_SYNC_WORKER_PARTITIONS: Record = { - [QueuePriorityLevel.HIGH]: process.env.INTEGRATION_SYNC_WORKER_HIGH_PARTITIONS - ? Number(process.env.INTEGRATION_SYNC_WORKER_HIGH_PARTITIONS) - : undefined, - [QueuePriorityLevel.NORMAL]: process.env.INTEGRATION_SYNC_WORKER_NORMAL_PARTITIONS - ? Number(process.env.INTEGRATION_SYNC_WORKER_NORMAL_PARTITIONS) - : undefined, - [QueuePriorityLevel.SYSTEM]: process.env.INTEGRATION_SYNC_WORKER_SYSTEM_PARTITIONS - ? Number(process.env.INTEGRATION_SYNC_WORKER_SYSTEM_PARTITIONS) - : undefined, -} - export const INTEGRATION_RUN_WORKER_PARTITIONS: Record = { [QueuePriorityLevel.HIGH]: process.env.INTEGRATION_RUN_WORKER_HIGH_PARTITIONS ? Number(process.env.INTEGRATION_RUN_WORKER_HIGH_PARTITIONS) diff --git a/services/libs/common/src/i18n/en.ts b/services/libs/common/src/i18n/en.ts index 01144fd9af..0197586f2d 100644 --- a/services/libs/common/src/i18n/en.ts +++ b/services/libs/common/src/i18n/en.ts @@ -143,9 +143,6 @@ const en = { 'Can not trigger nboarding because integration is not in state pending-action!', noOrganizationFound: 'No organization found for given id!', }, - hubspot: { - notInPlan: 'Hubspot integration requires Scale plan.', - }, members: { activeList: { activityTimestampFrom: 'activityTimestampFrom is required query parameter!', @@ -202,11 +199,6 @@ const en = { discourse: 'Discourse', }, }, - automation: { - errors: { - planLimitExceeded: 'You have exceeded # of automations you can have in your plan.', - }, - }, eagleEye: { errors: { planLimitExceeded: diff --git a/services/libs/common_services/src/services/emitters/index.ts b/services/libs/common_services/src/services/emitters/index.ts index cdb07551dc..51a995b3f0 100644 --- a/services/libs/common_services/src/services/emitters/index.ts +++ b/services/libs/common_services/src/services/emitters/index.ts @@ -1,5 +1,4 @@ export * from './dataSinkWorker.emitter' export * from './integrationRunWorker.emitter' export * from './integrationStreamWorker.emitter' -export * from './integrationSyncWorker.emitter' export * from './searchSyncWorker.emitter' diff --git a/services/libs/common_services/src/services/emitters/integrationSyncWorker.emitter.ts b/services/libs/common_services/src/services/emitters/integrationSyncWorker.emitter.ts deleted file mode 100644 index 9017fbaad3..0000000000 --- a/services/libs/common_services/src/services/emitters/integrationSyncWorker.emitter.ts +++ /dev/null @@ -1,167 +0,0 @@ -import { Logger } from '@crowd/logging' -import { CrowdQueue, IQueue } from '@crowd/queue' -import { RedisClient } from '@crowd/redis' -import { - AutomationSyncTrigger, - IIntegrationSyncWorkerEmitter, - IntegrationSyncWorkerQueueMessageType, -} from '@crowd/types' - -import { QueuePriorityContextLoader, QueuePriorityService } from '../priority.service' - -export class IntegrationSyncWorkerEmitter - extends QueuePriorityService - implements IIntegrationSyncWorkerEmitter -{ - public constructor( - client: IQueue, - redis: RedisClient, - priorityLevelCalculationContextLoader: QueuePriorityContextLoader, - parentLog: Logger, - ) { - super( - CrowdQueue.INTEGRATION_SYNC_WORKER, - client.getQueueChannelConfig(CrowdQueue.INTEGRATION_SYNC_WORKER), - client, - redis, - priorityLevelCalculationContextLoader, - parentLog, - ) - } - - public async triggerSyncMarkedMembers(tenantId: string, integrationId: string): Promise { - if (!tenantId) { - throw new Error('tenantId is required!') - } - if (!integrationId) { - throw new Error('integrationId is required!') - } - await this.sendMessage( - tenantId, - integrationId, - { - type: IntegrationSyncWorkerQueueMessageType.SYNC_ALL_MARKED_MEMBERS, - tenantId, - integrationId, - }, - integrationId, - ) - } - - public async triggerSyncMember( - tenantId: string, - integrationId: string, - memberId: string, - syncRemoteId: string, - ): Promise { - if (!tenantId) { - throw new Error('tenantId is required!') - } - if (!integrationId) { - throw new Error('integrationId is required!') - } - - if (!memberId) { - throw new Error('memberId is required!') - } - - if (!syncRemoteId) { - throw new Error('syncRemoteId is required!') - } - - await this.sendMessage( - tenantId, - memberId, - { - type: IntegrationSyncWorkerQueueMessageType.SYNC_MEMBER, - tenantId, - integrationId, - memberId, - syncRemoteId, - }, - memberId, - ) - } - - public async triggerOnboardAutomation( - tenantId: string, - integrationId: string, - automationId: string, - automationTrigger: AutomationSyncTrigger, - ): Promise { - if (!tenantId) { - throw new Error('tenantId is required!') - } - if (!automationId) { - throw new Error('automationId is required!') - } - if (!integrationId) { - throw new Error('integrationId is required!') - } - await this.sendMessage( - tenantId, - automationId, - { - type: IntegrationSyncWorkerQueueMessageType.ONBOARD_AUTOMATION, - tenantId, - integrationId, - automationId, - automationTrigger, - }, - automationId, - ) - } - - public async triggerSyncMarkedOrganizations( - tenantId: string, - integrationId: string, - ): Promise { - if (!tenantId) { - throw new Error('tenantId is required!') - } - if (!integrationId) { - throw new Error('integrationId is required!') - } - await this.sendMessage( - tenantId, - integrationId, - { - type: IntegrationSyncWorkerQueueMessageType.SYNC_ALL_MARKED_ORGANIZATIONS, - tenantId, - integrationId, - }, - integrationId, - ) - } - - public async triggerSyncOrganization( - tenantId: string, - integrationId: string, - organizationId: string, - syncRemoteId: string, - ): Promise { - if (!tenantId) { - throw new Error('tenantId is required!') - } - if (!integrationId) { - throw new Error('integrationId is required!') - } - - if (!syncRemoteId) { - throw new Error('syncRemoteId is required!') - } - - await this.sendMessage( - tenantId, - organizationId, - { - type: IntegrationSyncWorkerQueueMessageType.SYNC_ORGANIZATION, - tenantId, - integrationId, - organizationId, - syncRemoteId, - }, - organizationId, - ) - } -} diff --git a/services/libs/data-access-layer/src/old/apps/automations_worker/automation.repo.ts b/services/libs/data-access-layer/src/old/apps/automations_worker/automation.repo.ts deleted file mode 100644 index ef9da6f6ed..0000000000 --- a/services/libs/data-access-layer/src/old/apps/automations_worker/automation.repo.ts +++ /dev/null @@ -1,134 +0,0 @@ -import { generateUUIDv1 } from '@crowd/common' -import { DbColumnSet, DbStore, RepositoryBase } from '@crowd/database' -import { Logger } from '@crowd/logging' -import { - AutomationExecutionState, - AutomationSettings, - AutomationState, - AutomationSyncTrigger, - AutomationTrigger, - AutomationType, -} from '@crowd/types' - -export interface IRelevantAutomationData { - id: string - tenantId: string - type: AutomationType - settings: AutomationSettings - trigger: AutomationTrigger - slackWebHook: string | null - createdAt: string -} - -export interface IDbAutomationExecutionInsertData { - automationId: string - type: AutomationType - tenantId: string - trigger: AutomationTrigger | AutomationSyncTrigger - state: AutomationExecutionState - error: unknown | null - executedAt: Date - eventId: string - payload: unknown -} - -export class AutomationRepository extends RepositoryBase { - private insertExecutionColumnSet: DbColumnSet - - constructor(dbStore: DbStore, parentLog: Logger) { - super(dbStore, parentLog) - - this.insertExecutionColumnSet = new this.dbInstance.helpers.ColumnSet( - [ - 'id', - 'automationId', - 'type', - 'tenantId', - 'trigger', - 'state', - 'error', - 'executedAt', - 'eventId', - 'payload', - ], - { - table: { - table: 'automationExecutions', - }, - }, - ) - } - - async get(automationId: string): Promise { - const result = await this.db().oneOrNone( - ` - select a.id, - a."tenantId", - a.type, - a.trigger, - a.settings, - s."slackWebHook" - from automations a - inner join settings s on s."tenantId" = a."tenantId" - where a.id = $(automationId) - `, - { - automationId, - }, - ) - - return result - } - - async findRelevant( - tenantId: string, - trigger: AutomationTrigger, - state: AutomationState, - ): Promise { - const results = await this.db().any( - ` - select a.id, - a."tenantId", - a.type, - a.trigger, - a.settings, - s."slackWebHook", - a."createdAt" - from automations a - inner join settings s on s."tenantId" = a."tenantId" - where a."tenantId" = $(tenantId) and a.trigger = $(trigger) and a.state = $(state) - `, - { - tenantId, - trigger, - state, - }, - ) - - return results - } - - async hasAlreadyBeenTriggered(automationId: string, eventId: string): Promise { - const query = ` - select id - from "automationExecutions" - where "automationId" = $(automationId) - and "eventId" = $(eventId) - and state = '${AutomationExecutionState.SUCCESS}'; - ` - - const results = await this.db().any(query, { - automationId, - eventId, - }) - - return results.length > 0 - } - - async createExecution(data: IDbAutomationExecutionInsertData): Promise { - const id = generateUUIDv1() - const prepared = RepositoryBase.prepare({ id, ...data }, this.insertExecutionColumnSet) - const query = this.dbInstance.helpers.insert(prepared, this.insertExecutionColumnSet) - await this.db().none(query) - } -} diff --git a/services/libs/data-access-layer/src/old/apps/integration_run_worker/automation.repo.ts b/services/libs/data-access-layer/src/old/apps/integration_run_worker/automation.repo.ts deleted file mode 100644 index def38980a1..0000000000 --- a/services/libs/data-access-layer/src/old/apps/integration_run_worker/automation.repo.ts +++ /dev/null @@ -1,43 +0,0 @@ -import { DbStore, RepositoryBase } from '@crowd/database' -import { Logger } from '@crowd/logging' -import { AutomationSyncTrigger, IAutomationData } from '@crowd/types' - -export class AutomationRepository extends RepositoryBase { - constructor(dbStore: DbStore, parentLog: Logger) { - super(dbStore, parentLog) - } - - public async findSyncAutomations( - tenantId: string, - platform: string, - ): Promise { - const pageSize = 10 - const syncAutomations: IAutomationData[] = [] - - let results - let offset - - do { - offset = results ? pageSize + offset : 0 - results = await this.db().any( - `select * from automations - where type = $(platform) and "tenantId" = $(tenantId) and trigger in ($(syncAutomationTriggers:csv)) - limit $(limit) offset $(offset)`, - { - tenantId, - platform, - syncAutomationTriggers: [ - AutomationSyncTrigger.MEMBER_ATTRIBUTES_MATCH, - AutomationSyncTrigger.ORGANIZATION_ATTRIBUTES_MATCH, - ], - limit: pageSize, - offset, - }, - ) - - syncAutomations.push(...results) - } while (results.length > 0) - - return syncAutomations - } -} diff --git a/services/libs/data-access-layer/src/old/apps/integration_sync_worker/automation.repo.ts b/services/libs/data-access-layer/src/old/apps/integration_sync_worker/automation.repo.ts deleted file mode 100644 index 5fa898c00a..0000000000 --- a/services/libs/data-access-layer/src/old/apps/integration_sync_worker/automation.repo.ts +++ /dev/null @@ -1,13 +0,0 @@ -import { DbStore, RepositoryBase } from '@crowd/database' -import { Logger } from '@crowd/logging' -import { IAutomationData } from '@crowd/types' - -export class AutomationRepository extends RepositoryBase { - constructor(dbStore: DbStore, parentLog: Logger) { - super(dbStore, parentLog) - } - - public async findById(id: string): Promise { - return await this.db().oneOrNone(`select * from automations where id = $(id)`, { id }) - } -} diff --git a/services/libs/data-access-layer/src/old/apps/integration_sync_worker/automationExecution.repo.ts b/services/libs/data-access-layer/src/old/apps/integration_sync_worker/automationExecution.repo.ts deleted file mode 100644 index 1403275feb..0000000000 --- a/services/libs/data-access-layer/src/old/apps/integration_sync_worker/automationExecution.repo.ts +++ /dev/null @@ -1,29 +0,0 @@ -import { generateUUIDv1 as uuid } from '@crowd/common' -import { DbStore, RepositoryBase } from '@crowd/database' -import { Logger } from '@crowd/logging' -import { IAutomationExecution } from '@crowd/types' - -export class AutomationExecutionRepository extends RepositoryBase { - constructor(dbStore: DbStore, parentLog: Logger) { - super(dbStore, parentLog) - } - - public async addExecution(data: IAutomationExecution): Promise { - await this.db().none( - `insert into "automationExecutions" ("id", "automationId", "type", "tenantId", "trigger", "state", "error", "executedAt", "eventId", "payload") - values - ($(id), $(automationId), $(type), $(tenantId), $(trigger), $(state), $(error), now(), $(eventId), $(payload))`, - { - id: uuid(), - automationId: data.automationId, - type: data.type, - tenantId: data.tenantId, - trigger: data.trigger, - state: data.state, - error: data.error || null, - eventId: data.eventId || uuid(), - payload: JSON.stringify(data.payload) || JSON.stringify({}), - }, - ) - } -} diff --git a/services/libs/integrations/src/integrations/hubspot/api/addContactsToList.ts b/services/libs/integrations/src/integrations/hubspot/api/addContactsToList.ts deleted file mode 100644 index c0ef1650e9..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/api/addContactsToList.ts +++ /dev/null @@ -1,36 +0,0 @@ -import axios, { AxiosRequestConfig } from 'axios' - -import { RequestThrottler } from '@crowd/common' -import { PlatformType } from '@crowd/types' - -import { IGenerateStreamsContext, IProcessStreamContext } from '../../../types' -import { getNangoToken } from '../../nango' - -export const addContactsToList = async ( - nangoId: string, - listId: string, - contactIds: string[], - ctx: IProcessStreamContext | IGenerateStreamsContext, - throttler: RequestThrottler, -): Promise => { - const config: AxiosRequestConfig = { - method: 'post', - url: `https://api.hubapi.com/contacts/v1/lists/${listId}/add`, - data: {}, - } - try { - ctx.log.debug({ nangoId }, `Adding contacts to list ${listId} in HubSpot!`) - - const accessToken = await getNangoToken(nangoId, PlatformType.HUBSPOT, ctx, throttler) - config.headers = { Authorization: `Bearer ${accessToken}` } - - config.data = { - vids: contactIds, - } - - await throttler.throttle(() => axios(config)) - } catch (err) { - ctx.log.error({ err }, `Error while adding contacts to hubspot list!`) - throw err - } -} diff --git a/services/libs/integrations/src/integrations/hubspot/api/batchCreateMembers.ts b/services/libs/integrations/src/integrations/hubspot/api/batchCreateMembers.ts deleted file mode 100644 index ca105c6c31..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/api/batchCreateMembers.ts +++ /dev/null @@ -1,178 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import axios, { AxiosRequestConfig } from 'axios' - -import { RequestThrottler } from '@crowd/common' -import { IMember, MemberIdentityType, PlatformType } from '@crowd/types' - -import { IGenerateStreamsContext, IProcessStreamContext } from '../../../types' -import { getNangoToken } from '../../nango' -import { HubspotMemberFieldMapper } from '../field-mapper/memberFieldMapper' - -import { batchUpdateMembers } from './batchUpdateMembers' -import { getContactById } from './contactById' -import { IBatchCreateMembersResult } from './types' - -export const batchCreateMembers = async ( - nangoId: string, - members: IMember[], - memberMapper: HubspotMemberFieldMapper, - ctx: IProcessStreamContext | IGenerateStreamsContext, - throttler: RequestThrottler, -): Promise => { - const config: AxiosRequestConfig = { - method: 'post', - url: `https://api.hubapi.com/crm/v3/objects/contacts/batch/create`, - data: {}, - headers: { - Authorization: '', - }, - } - - try { - const hubspotMembers = [] - - for (const member of members) { - const primaryEmail = - member.identities.filter((i) => i.type === MemberIdentityType.EMAIL && i.verified).length > - 0 - ? member.identities.find((i) => i.type === MemberIdentityType.EMAIL && i.verified)?.value - : null - - if (!primaryEmail) { - ctx.log.warn( - `Member ${member.id} can't be created in hubspot! Member doesn't have any associated email.`, - ) - } else { - const hsMember = { - properties: { - email: primaryEmail, - }, - } as any - - const fields = memberMapper.getAllCrowdFields() - - for (const crowdField of fields) { - const hubspotField = memberMapper.getHubspotFieldName(crowdField) - // if hubspot e-mail field is mapped to a crowd field, we should ignore it because - // we handle this manually above - if (hubspotField && hubspotField !== 'email') { - if (crowdField.startsWith('attributes')) { - const attributeName = crowdField.split('.')[1] || null - - if ( - attributeName && - hubspotField && - member.attributes[attributeName]?.default !== undefined - ) { - hsMember.properties[hubspotField] = member.attributes[attributeName].default - } - } else if (crowdField.startsWith('identities')) { - const identityPlatform = crowdField.split('.')[1] || null - - const identityFound = member.identities.find( - (i) => i.platform === identityPlatform && i.type === MemberIdentityType.USERNAME, - ) - - if (identityPlatform && hubspotField && identityFound) { - hsMember.properties[hubspotField] = identityFound.value - } - } else if (crowdField === 'organizationName') { - // send latest org of member as value - } else if (hubspotField && member[crowdField] !== undefined) { - hsMember.properties[hubspotField] = memberMapper.getHubspotValue(member, crowdField) - } - } - } - - if (Object.keys(hsMember.properties).length > 0) { - hubspotMembers.push(hsMember) - } - } - } - - config.data = { - inputs: hubspotMembers, - } - - // Get an access token from Nango - const accessToken = await getNangoToken(nangoId, PlatformType.HUBSPOT, ctx, throttler) - - ctx.log.debug({ nangoId, accessToken, data: config.data }, 'Creating bulk contacts in HubSpot') - - config.headers.Authorization = `Bearer ${accessToken}` - - const result = await throttler.throttle(() => axios(config)) - - // return hubspot ids back to sync worker for saving - return result.data.results.reduce((acc, m) => { - const member = members.find( - (crowdMember) => - crowdMember.identities.filter((i) => i.type === MemberIdentityType.EMAIL && i.verified) - .length > 0 && - crowdMember.identities - .filter((i) => i.type === MemberIdentityType.EMAIL && i.verified) - .map((i) => i.value) - .includes(m.properties.email), - ) - - const hubspotPayload = hubspotMembers.find( - (hubspotMember) => hubspotMember.properties.email === m.properties.email, - ) - - acc.push({ - memberId: member.id, - sourceId: m.id, - lastSyncedPayload: hubspotPayload, - }) - return acc - }, []) - } catch (err) { - // this means that member actually exists in hubspot but we tried re-creating it - // handle it gracefully - if (err.response?.data?.category === 'CONFLICT') { - ctx.log.info( - { err }, - 'Conflict while batch create contacts in HubSpot. Trying to resolve the conflicts.', - ) - const match = err.response?.data?.message.match(/ID: (\d+)/) - const id = match ? match[1] : null - if (id) { - const member = await getContactById(nangoId, id, memberMapper, ctx, throttler) - - if (member) { - // exclude found member from batch payload - const createMembers = members.filter( - (m) => - !m.identities - .filter((i) => i.type === MemberIdentityType.EMAIL && i.verified) - .map((i) => i.value.toLowerCase()) - .includes((member.properties as any).email.toLowerCase()), - ) - - const updateMembers = members - .filter((m) => - m.identities - .filter((i) => i.type === MemberIdentityType.EMAIL && i.verified) - .map((i) => i.value.toLowerCase()) - .includes((member.properties as any).email.toLowerCase()), - ) - .map((m) => { - m.attributes = { - ...m.attributes, - sourceId: { - hubspot: id, - }, - } - return m - }) - - await batchUpdateMembers(nangoId, updateMembers, memberMapper, ctx, throttler) - return await batchCreateMembers(nangoId, createMembers, memberMapper, ctx, throttler) - } - } - } else { - ctx.log.error({ err }, 'Error while batch create contacts to HubSpot') - throw err - } - } -} diff --git a/services/libs/integrations/src/integrations/hubspot/api/batchCreateOrganizations.ts b/services/libs/integrations/src/integrations/hubspot/api/batchCreateOrganizations.ts deleted file mode 100644 index 59e91d62a2..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/api/batchCreateOrganizations.ts +++ /dev/null @@ -1,167 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import axios, { AxiosRequestConfig } from 'axios' - -import { RequestThrottler } from '@crowd/common' -import { IOrganization, OrganizationIdentityType, PlatformType } from '@crowd/types' - -import { IGenerateStreamsContext, IProcessStreamContext } from '../../../types' -import { getNangoToken } from '../../nango' -import { HubspotOrganizationFieldMapper } from '../field-mapper/organizationFieldMapper' - -import { IBatchCreateOrganizationsResult } from './types' -import { getOrganizationDomain } from './utils/getOrganizationDomain' - -export const batchCreateOrganizations = async ( - nangoId: string, - organizations: IOrganization[], - organizationMapper: HubspotOrganizationFieldMapper, - ctx: IProcessStreamContext | IGenerateStreamsContext, - throttler: RequestThrottler, -): Promise => { - const config: AxiosRequestConfig = { - method: 'post', - url: `https://api.hubapi.com/crm/v3/objects/companies/batch/create`, - data: {}, - headers: { - Authorization: '', - }, - } - - try { - const hubspotCompanies = [] - - for (const organization of organizations) { - const verifiedPrimaryDomains = organization.identities.filter( - (i) => i.type === OrganizationIdentityType.PRIMARY_DOMAIN && i.verified, - ) - const domains = verifiedPrimaryDomains - .map((i) => getOrganizationDomain(i.value)) - .filter((i) => i !== null) - - if (domains.length === 0) { - ctx.log.info( - `Organization ${organization.id} can't be created in hubspot! Organization doesn't have any associated website or domain can't be derived from existing website.`, - ) - } else { - const organizationDomain = domains[0] - - const hubspotCompany = { - properties: { - domain: organizationDomain, - }, - } as any - - const fields = organizationMapper.getAllCrowdFields() - - for (const crowdField of fields) { - const hubspotField = organizationMapper.getHubspotFieldName(crowdField) - // if hubspot domain field is mapped to a crowd field, we should ignore it - // because we handle this manually above - if (hubspotField && hubspotField !== 'domain') { - if (organization[crowdField] !== undefined) { - hubspotCompany.properties[hubspotField] = organizationMapper.getHubspotValue( - organization, - crowdField, - ) - } - } - } - - if (Object.keys(hubspotCompany.properties).length > 0) { - hubspotCompanies.push(hubspotCompany) - } - } - } - - config.data = { - inputs: hubspotCompanies, - } - - // Get an access token from Nango - const accessToken = await getNangoToken(nangoId, PlatformType.HUBSPOT, ctx, throttler) - - ctx.log.debug( - { nangoId, accessToken, data: config.data }, - 'Creating bulk companies in HubSpot!', - ) - - config.headers.Authorization = `Bearer ${accessToken}` - - const result = await throttler.throttle(() => axios(config)) - - // return hubspot ids back to sync worker for saving - return result.data.results.reduce((acc, o) => { - const organization = organizations.find((crowdOrganization) => { - const domains = crowdOrganization.identities - .filter((i) => i.type === OrganizationIdentityType.PRIMARY_DOMAIN && i.verified) - .map((i) => getOrganizationDomain(i.value)) - .filter((i) => i !== null) - return domains.includes(o.properties.domain) - }) - - const hubspotPayload = hubspotCompanies.find( - (hubspotCompany) => hubspotCompany.properties.domain === o.properties.domain, - ) - - acc.push({ - organizationId: organization.id, - sourceId: o.id, - lastSyncedPayload: hubspotPayload, - }) - - return acc - }, []) - } catch (err) { - // this means that organization actually exists in hubspot but we tried re-creating it - // handle it gracefully - if (err.response?.data?.category === 'CONFLICT') { - ctx.log.warn( - { err }, - 'Conflict while batch create companies in HubSpot. Trying to resolve the conflicts.', - ) - // TODO uros fix this - // const match = err.response?.data?.message.match(/ID: (\d+)/) - // const id = match ? match[1] : null - // if (id) { - // const organization = await getCompanyById(nangoId, id, organizationMapper, ctx, throttler) - - // if (organization) { - // // exclude found organization from batch payload - // const createOrganizations = organizations.filter( - // (o) => !o.website.includes((organization.properties as any).domain), - // ) - - // const updateOrganizations = organizations - // .filter((o) => o.website.includes((organization.properties as any).domain)) - // .map((o) => { - // o.attributes = { - // ...o.attributes, - // sourceId: { - // hubspot: id, - // }, - // } - // return o - // }) - - // await batchUpdateOrganizations( - // nangoId, - // updateOrganizations, - // organizationMapper, - // ctx, - // throttler, - // ) - // return await batchCreateOrganizations( - // nangoId, - // createOrganizations, - // organizationMapper, - // ctx, - // throttler, - // ) - // } - // } - } else { - ctx.log.error({ err }, 'Error while batch create companies to HubSpot') - throw err - } - } -} diff --git a/services/libs/integrations/src/integrations/hubspot/api/batchUpdateMembers.ts b/services/libs/integrations/src/integrations/hubspot/api/batchUpdateMembers.ts deleted file mode 100644 index a44b602301..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/api/batchUpdateMembers.ts +++ /dev/null @@ -1,119 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import axios, { AxiosRequestConfig } from 'axios' - -import { RequestThrottler } from '@crowd/common' -import { IMember, MemberIdentityType, PlatformType } from '@crowd/types' - -import { IGenerateStreamsContext, IProcessStreamContext } from '../../../types' -import { getNangoToken } from '../../nango' -import { HubspotMemberFieldMapper } from '../field-mapper/memberFieldMapper' - -import { IBatchUpdateMembersResult } from './types' - -export const batchUpdateMembers = async ( - nangoId: string, - members: IMember[], - memberMapper: HubspotMemberFieldMapper, - ctx: IProcessStreamContext | IGenerateStreamsContext, - throttler: RequestThrottler, -): Promise => { - const config: AxiosRequestConfig = { - method: 'post', - url: `https://api.hubapi.com/crm/v3/objects/contacts/batch/update`, - data: {}, - headers: { - Authorization: '', - }, - } - - try { - const hubspotMembers = [] - - for (const member of members) { - if (member) { - const hubspotSourceId = member.attributes?.sourceId?.hubspot - - if (!hubspotSourceId) { - ctx.log.warn( - `Member ${member.id} can't be updated in hubspot! Member doesn't have a hubspot sourceId in attributes.`, - ) - } else { - const hsMember = { - id: hubspotSourceId, - properties: {}, - } as any - - const fields = memberMapper.getAllCrowdFields() - - for (const crowdField of fields) { - const hubspotField = memberMapper.getHubspotFieldName(crowdField) - // if hubspot e-mail field is mapped to a crowd field, we should ignore it because - // we handle this manually above - if (hubspotField && hubspotField !== 'email') { - if (crowdField.startsWith('attributes')) { - const attributeName = crowdField.split('.')[1] || null - - if ( - attributeName && - hubspotField && - member.attributes[attributeName]?.default !== undefined - ) { - hsMember.properties[hubspotField] = member.attributes[attributeName].default - } - } else if (crowdField.startsWith('identities')) { - const identityPlatform = crowdField.split('.')[1] || null - - const identityFound = member.identities.find( - (i) => i.platform === identityPlatform && i.type === MemberIdentityType.USERNAME, - ) - - if (identityPlatform && hubspotField && identityFound) { - hsMember.properties[hubspotField] = identityFound.value - } - } else if (crowdField === 'organizationName') { - // send latest org of member as value - } else if (hubspotField && member[crowdField] !== undefined) { - hsMember.properties[hubspotField] = memberMapper.getHubspotValue(member, crowdField) - } - } - } - - if (Object.keys(hsMember.properties).length > 0) { - hubspotMembers.push(hsMember) - } - } - } - } - - config.data = { - inputs: hubspotMembers, - } - - // Get an access token from Nango - const accessToken = await getNangoToken(nangoId, PlatformType.HUBSPOT, ctx, throttler) - - ctx.log.debug({ nangoId, accessToken, data: config.data }, 'Updating bulk contacts in HubSpot') - - config.headers.Authorization = `Bearer ${accessToken}` - - const result = await throttler.throttle(() => axios(config)) - - return result.data.results.reduce((acc, m) => { - const member = members.find( - (crowdMember) => crowdMember.attributes?.sourceId?.hubspot === m.id, - ) - - const hubspotPayload = hubspotMembers.find((hubspotMember) => hubspotMember.id === m.id) - - acc.push({ - memberId: member.id, - sourceId: m.id, - lastSyncedPayload: hubspotPayload, - }) - return acc - }, []) - } catch (err) { - ctx.log.error({ err }, 'Error while batch update contacts to HubSpot') - throw err - } -} diff --git a/services/libs/integrations/src/integrations/hubspot/api/batchUpdateOrganizations.ts b/services/libs/integrations/src/integrations/hubspot/api/batchUpdateOrganizations.ts deleted file mode 100644 index 1063862689..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/api/batchUpdateOrganizations.ts +++ /dev/null @@ -1,104 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import axios, { AxiosRequestConfig } from 'axios' - -import { RequestThrottler } from '@crowd/common' -import { IOrganization, PlatformType } from '@crowd/types' - -import { IGenerateStreamsContext, IProcessStreamContext } from '../../../types' -import { getNangoToken } from '../../nango' -import { HubspotOrganizationFieldMapper } from '../field-mapper/organizationFieldMapper' - -import { IBatchUpdateOrganizationsResult } from './types' - -export const batchUpdateOrganizations = async ( - nangoId: string, - organizations: IOrganization[], - organizationMapper: HubspotOrganizationFieldMapper, - ctx: IProcessStreamContext | IGenerateStreamsContext, - throttler: RequestThrottler, -): Promise => { - const config: AxiosRequestConfig = { - method: 'post', - url: `https://api.hubapi.com/crm/v3/objects/companies/batch/update`, - data: {}, - headers: { - Authorization: '', - }, - } - - try { - const hubspotCompanies = [] - - for (const organization of organizations) { - if (organization) { - // const hubspotSourceId = organization.attributes?.sourceId?.hubspot - const hubspotSourceId = undefined - - if (!hubspotSourceId) { - ctx.log.warn( - `Organization ${organization.id} can't be updated in hubspot! Organization doesn't have a hubspot sourceId.`, - ) - } else { - const hubspotCompany = { - id: hubspotSourceId, - properties: {}, - } as any - - const fields = organizationMapper.getAllCrowdFields() - - for (const crowdField of fields) { - const hubspotField = organizationMapper.getHubspotFieldName(crowdField) - // if hubspot domain field is mapped to a crowd field, we should ignore it - // because we handle this manually above - if (hubspotField && hubspotField !== 'domain') { - if (organization[crowdField] !== undefined) { - hubspotCompany.properties[hubspotField] = organizationMapper.getHubspotValue( - organization, - crowdField, - ) - } - } - } - - if (Object.keys(hubspotCompany.properties).length > 0) { - hubspotCompanies.push(hubspotCompany) - } - } - } - } - - config.data = { - inputs: hubspotCompanies, - } - - // Get an access token from Nango - const accessToken = await getNangoToken(nangoId, PlatformType.HUBSPOT, ctx, throttler) - - ctx.log.debug({ nangoId, accessToken, data: config.data }, 'Updating bulk companies in HubSpot') - - config.headers.Authorization = `Bearer ${accessToken}` - - // eslint-disable-next-line @typescript-eslint/no-unused-vars - const result = await throttler.throttle(() => axios(config)) - - // return result.data.results.reduce((acc, o) => { - // const organization = organizations.find( - // (crowdOrganization) => crowdOrganization.attributes?.sourceId?.hubspot === o.id, - // ) - - // const hubspotPayload = hubspotCompanies.find((hubspotCompany) => hubspotCompany.id === o.id) - - // acc.push({ - // organizationId: organization.id, - // sourceId: o.id, - // lastSyncedPayload: hubspotPayload, - // }) - - // return acc - // }, []) - return [] - } catch (err) { - ctx.log.error({ err }, 'Error while batch update companies in HubSpot') - throw err - } -} diff --git a/services/libs/integrations/src/integrations/hubspot/api/common.ts b/services/libs/integrations/src/integrations/hubspot/api/common.ts deleted file mode 100644 index 4d2bce6999..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/api/common.ts +++ /dev/null @@ -1 +0,0 @@ -export const HUBSPOT_API_PAGE_SIZE = 50 diff --git a/services/libs/integrations/src/integrations/hubspot/api/companies.ts b/services/libs/integrations/src/integrations/hubspot/api/companies.ts deleted file mode 100644 index 598cc5940a..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/api/companies.ts +++ /dev/null @@ -1,100 +0,0 @@ -import axios, { AxiosRequestConfig } from 'axios' - -import { RequestThrottler } from '@crowd/common' -import { PlatformType } from '@crowd/types' - -import { IGenerateStreamsContext, IProcessStreamContext } from '../../../types' -import { getNangoToken } from '../../nango' -import { HubspotOrganizationFieldMapper } from '../field-mapper/organizationFieldMapper' -import { IHubspotContact, IHubspotObject } from '../types' - -import { HUBSPOT_API_PAGE_SIZE } from './common' -import { IPaginatedResponse } from './types' - -export const getCompanies = async ( - nangoId: string, - organizationMapper: HubspotOrganizationFieldMapper, - ctx: IProcessStreamContext | IGenerateStreamsContext, - throttler: RequestThrottler, - after?: string, -): Promise> => { - const config: AxiosRequestConfig = { - method: 'get', - url: `https://api.hubapi.com/crm/v3/objects/companies`, - params: { - limit: HUBSPOT_API_PAGE_SIZE, - properties: `name,domain,${organizationMapper.getAllHubspotFields().join(',')}`, - }, - headers: { - Authorization: '', - }, - } - - // If we're not onboarding, only get data that was updated in last 8 hours - if (!ctx.onboarding) { - const date = new Date() - date.setHours(date.getHours() - 8) - - config.params.filterGroups = JSON.stringify([ - { - filters: [ - { - value: date.getTime(), - propertyName: 'hs_lastmodifieddate', - operator: 'GT', - }, - ], - }, - ]) - } - - try { - // Get an access token from Nango - const accessToken = await getNangoToken(nangoId, PlatformType.HUBSPOT, ctx, throttler) - - ctx.log.debug({ nangoId, accessToken }, 'Fetching contacts from HubSpot') - - config.headers.Authorization = `Bearer ${accessToken}` - - // set pagination - config.params.after = after - - const result = await throttler.throttle(() => axios(config)) - - const elements = result.data.results as IHubspotObject[] - - if (result.data.paging?.next?.after) { - return { - elements, - after: result.data.paging.next.after, - } - } - - return { - elements, - } - } catch (err) { - ctx.log.error({ err }, 'Error while fetching companies from HubSpot') - throw err - } -} - -export async function* getAllCompanies( - nangoId: string, - organizationMapper: HubspotOrganizationFieldMapper, - ctx: IProcessStreamContext | IGenerateStreamsContext, - throttler: RequestThrottler, -): AsyncGenerator { - let hasNextPage = true - let after = undefined - - while (hasNextPage) { - const response = await getCompanies(nangoId, organizationMapper, ctx, throttler, after) - - hasNextPage = response.after !== undefined - - after = response.after - - yield response.elements - } -} diff --git a/services/libs/integrations/src/integrations/hubspot/api/companyById.ts b/services/libs/integrations/src/integrations/hubspot/api/companyById.ts deleted file mode 100644 index ef5e94efe1..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/api/companyById.ts +++ /dev/null @@ -1,52 +0,0 @@ -import axios, { AxiosRequestConfig } from 'axios' - -import { RequestThrottler } from '@crowd/common' -import { PlatformType } from '@crowd/types' - -import { IGenerateStreamsContext, IProcessStreamContext } from '../../../types' -import { getNangoToken } from '../../nango' -import { HubspotOrganizationFieldMapper } from '../field-mapper/organizationFieldMapper' -import { IHubspotObject } from '../types' - -export const getCompanyById = async ( - nangoId: string, - companyId: string, - organizationMapper: HubspotOrganizationFieldMapper, - ctx: IProcessStreamContext | IGenerateStreamsContext, - throttler: RequestThrottler, -): Promise => { - const config: AxiosRequestConfig = { - method: 'get', - url: `https://api.hubapi.com/crm/v3/objects/companies/${companyId}`, - params: { - limit: 1, - properties: `name,domain,${organizationMapper.getAllHubspotFields().join(',')}`, - }, - headers: { - Authorization: '', - }, - } - - try { - ctx.log.debug({ nangoId, companyId }, 'Fetching company by id from HubSpot') - - // Get an access token from Nango - const accessToken = await getNangoToken(nangoId, PlatformType.HUBSPOT, ctx, throttler) - - ctx.log.debug({ accessToken }, `nango token`) - config.headers.Authorization = `Bearer ${accessToken}` - - const result = await throttler.throttle(() => axios(config)) - - const element = result.data - - if (!element) { - return null - } - - return element - } catch (err) { - ctx.log.error({ err }, 'Error while fetching contacts from HubSpot') - throw err - } -} diff --git a/services/libs/integrations/src/integrations/hubspot/api/contactAssociations.ts b/services/libs/integrations/src/integrations/hubspot/api/contactAssociations.ts deleted file mode 100644 index 96feb5519f..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/api/contactAssociations.ts +++ /dev/null @@ -1,40 +0,0 @@ -import axios, { AxiosRequestConfig } from 'axios' - -import { RequestThrottler } from '@crowd/common' -import { PlatformType } from '@crowd/types' - -import { IGenerateStreamsContext, IProcessStreamContext } from '../../../types' -import { getNangoToken } from '../../nango' -import { HubspotAssociationType, HubspotEndpoint, IHubspotAssociation } from '../types' - -export const getContactAssociations = async ( - nangoId: string, - association: HubspotEndpoint, - type: HubspotAssociationType, - contactId: string, - ctx: IProcessStreamContext | IGenerateStreamsContext, - throttler: RequestThrottler, -): Promise => { - const config: AxiosRequestConfig = { - method: 'get', - url: `https://api.hubapi.com/crm/v3/objects/contacts/${contactId}/associations/${association}`, - } - try { - ctx.log.debug( - { nangoId }, - `Fetching contact associations [${association}] of contact ${contactId} from HubSpot`, - ) - - const accessToken = await getNangoToken(nangoId, PlatformType.HUBSPOT, ctx, throttler) - config.headers = { Authorization: `Bearer ${accessToken}` } - - const result = await throttler.throttle(() => axios(config)) - - const contactAssociations: IHubspotAssociation[] = result?.data?.results || [] - - return contactAssociations.filter((a) => a.type === type) - } catch (err) { - ctx.log.error({ err }, `Error while getting hubspot contact associations [${association}]`) - throw err - } -} diff --git a/services/libs/integrations/src/integrations/hubspot/api/contactById.ts b/services/libs/integrations/src/integrations/hubspot/api/contactById.ts deleted file mode 100644 index 415e2f96e1..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/api/contactById.ts +++ /dev/null @@ -1,52 +0,0 @@ -import axios, { AxiosRequestConfig } from 'axios' - -import { RequestThrottler } from '@crowd/common' -import { PlatformType } from '@crowd/types' - -import { IGenerateStreamsContext, IProcessStreamContext } from '../../../types' -import { getNangoToken } from '../../nango' -import { HubspotMemberFieldMapper } from '../field-mapper/memberFieldMapper' -import { IHubspotObject } from '../types' - -export const getContactById = async ( - nangoId: string, - contactId: string, - memberMapper: HubspotMemberFieldMapper, - ctx: IProcessStreamContext | IGenerateStreamsContext, - throttler: RequestThrottler, -): Promise => { - const config: AxiosRequestConfig = { - method: 'get', - url: `https://api.hubapi.com/crm/v3/objects/contacts/${contactId}`, - params: { - limit: 1, - properties: `email,${memberMapper.getAllHubspotFields().join(',')}`, - }, - headers: { - Authorization: '', - }, - } - - try { - ctx.log.debug({ nangoId, contactId }, 'Fetching contact by id from HubSpot') - - // Get an access token from Nango - const accessToken = await getNangoToken(nangoId, PlatformType.HUBSPOT, ctx, throttler) - - ctx.log.debug({ accessToken }, `nango token`) - config.headers.Authorization = `Bearer ${accessToken}` - - const result = await throttler.throttle(() => axios(config)) - - const element = result.data - - if (!element) { - return null - } - - return element - } catch (err) { - ctx.log.error({ err }, 'Error while fetching contacts from HubSpot') - throw err - } -} diff --git a/services/libs/integrations/src/integrations/hubspot/api/contacts.ts b/services/libs/integrations/src/integrations/hubspot/api/contacts.ts deleted file mode 100644 index cc8d34361c..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/api/contacts.ts +++ /dev/null @@ -1,147 +0,0 @@ -import axios, { AxiosRequestConfig } from 'axios' - -import { RequestThrottler } from '@crowd/common' -import { PlatformType } from '@crowd/types' - -import { IGenerateStreamsContext, IProcessStreamContext } from '../../../types' -import { getNangoToken } from '../../nango' -import { HubspotMemberFieldMapper } from '../field-mapper/memberFieldMapper' -import { HubspotOrganizationFieldMapper } from '../field-mapper/organizationFieldMapper' -import { HubspotAssociationType, HubspotEndpoint, IHubspotContact, IHubspotObject } from '../types' - -import { HUBSPOT_API_PAGE_SIZE } from './common' -import { getCompanyById } from './companyById' -import { getContactAssociations } from './contactAssociations' -import { IPaginatedResponse } from './types' - -export const getContacts = async ( - nangoId: string, - memberMapper: HubspotMemberFieldMapper, - organizationMapper: HubspotOrganizationFieldMapper, - ctx: IProcessStreamContext | IGenerateStreamsContext, - throttler: RequestThrottler, - includeOrganizations = false, - after?: string, -): Promise> => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const config: AxiosRequestConfig = { - method: 'get', - url: `https://api.hubapi.com/crm/v3/objects/contacts`, - params: { - limit: HUBSPOT_API_PAGE_SIZE, - properties: `email,${memberMapper.getAllHubspotFields().join(',')}`, - }, - headers: { - Authorization: '', - }, - } - - // If we're not onboarding, only get data that was updated in last 8 hours - if (!ctx.onboarding) { - const date = new Date() - date.setHours(date.getHours() - 8) - - config.params.filterGroups = JSON.stringify([ - { - filters: [ - { - value: date.getTime(), - propertyName: 'hs_lastmodifieddate', - operator: 'GT', - }, - ], - }, - ]) - } - - try { - ctx.log.debug({ nangoId }, 'Fetching contacts from HubSpot') - - // Get an access token from Nango - const accessToken = await getNangoToken(nangoId, PlatformType.HUBSPOT, ctx, throttler) - - ctx.log.debug({ accessToken }, `nango token`) - config.headers.Authorization = `Bearer ${accessToken}` - - // set pagination - config.params.after = after - - const result = await throttler.throttle(() => axios(config)) - - const elements = result.data.results as IHubspotContact[] - - if (includeOrganizations) { - for (const element of elements) { - // check association - const companyAssociations = await getContactAssociations( - nangoId, - HubspotEndpoint.COMPANIES, - HubspotAssociationType.CONTACT_TO_COMPANY, - element.id, - ctx, - throttler, - ) - - if (companyAssociations.length > 0) { - // get company - const company = await getCompanyById( - nangoId, - companyAssociations[0].id, - organizationMapper, - ctx, - throttler, - ) - - // eslint-disable-next-line @typescript-eslint/no-explicit-any - if ((company?.properties as any)?.name) { - element.organization = company - } - } - } - } - - if (result.data.paging?.next?.after) { - return { - elements, - after: result.data.paging.next.after, - } - } - - return { - elements, - } - } catch (err) { - ctx.log.error({ err }, 'Error while fetching contacts from HubSpot') - throw err - } -} - -export async function* getAllContacts( - nangoId: string, - memberMapper: HubspotMemberFieldMapper, - organizationMapper: HubspotOrganizationFieldMapper, - ctx: IProcessStreamContext | IGenerateStreamsContext, - throttler: RequestThrottler, - includeOrganizations = false, -): AsyncGenerator { - let hasNextPage = true - let after = undefined - - while (hasNextPage) { - const response = await getContacts( - nangoId, - memberMapper, - organizationMapper, - ctx, - throttler, - includeOrganizations, - after, - ) - - hasNextPage = response.after !== undefined - - after = response.after - - yield response.elements - } -} diff --git a/services/libs/integrations/src/integrations/hubspot/api/lists.ts b/services/libs/integrations/src/integrations/hubspot/api/lists.ts deleted file mode 100644 index 0d9fff643c..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/api/lists.ts +++ /dev/null @@ -1,44 +0,0 @@ -import axios, { AxiosRequestConfig } from 'axios' - -import { PlatformType } from '@crowd/types' - -import { IGenerateStreamsContext, IProcessStreamContext } from '../../../types' -import { getNangoToken } from '../../nango' -import { IHubspotList } from '../types' - -export const getLists = async ( - nangoId: string, - ctx: IProcessStreamContext | IGenerateStreamsContext, -): Promise => { - const PAGE_SIZE = 100 - const config: AxiosRequestConfig = { - method: 'get', - url: `https://api.hubapi.com/contacts/v1/lists`, - } - try { - ctx.log.debug({ nangoId }, `Fetching lists from HubSpot`) - - const accessToken = await getNangoToken(nangoId, PlatformType.HUBSPOT, ctx) - config.headers = { Authorization: `Bearer ${accessToken}` } - - let response - let offset - const hubspotLists: IHubspotList[] = [] - - do { - offset = response === undefined ? 0 : response.offset - config.url = `https://api.hubapi.com/contacts/v1/lists?count=${PAGE_SIZE}&offset=${offset}` - - ctx.log.debug({ PAGE_SIZE, offset, url: config.url }, `Getting lists!`) - - response = (await axios(config)).data - - hubspotLists.push(...response.lists.filter((l) => l.dynamic !== true)) - } while (response[`has-more`] !== false) - - return hubspotLists - } catch (err) { - ctx.log.error({ err }, `Error while getting hubspot lists!`) - throw err - } -} diff --git a/services/libs/integrations/src/integrations/hubspot/api/properties.ts b/services/libs/integrations/src/integrations/hubspot/api/properties.ts deleted file mode 100644 index 02eaf6d05b..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/api/properties.ts +++ /dev/null @@ -1,33 +0,0 @@ -import axios, { AxiosRequestConfig } from 'axios' - -import { PlatformType } from '@crowd/types' - -import { IGenerateStreamsContext, IProcessStreamContext } from '../../../types' -import { getNangoToken } from '../../nango' -import { HubspotEndpoint, IHubspotProperty } from '../types' - -export const getProperties = async ( - nangoId: string, - endpoint: HubspotEndpoint, - ctx: IProcessStreamContext | IGenerateStreamsContext, -): Promise => { - const config: AxiosRequestConfig = { - method: 'get', - url: `https://api.hubapi.com/crm/v3/properties/${endpoint}`, - } - try { - ctx.log.debug({ nangoId }, `Fetching ${endpoint} properties from HubSpot`) - - const accessToken = await getNangoToken(nangoId, PlatformType.HUBSPOT, ctx) - config.headers = { Authorization: `Bearer ${accessToken}` } - - const response = (await axios(config)).data - - const hubspotProperties: IHubspotProperty[] = response.results - - return hubspotProperties.filter((p) => !p.calculated && !p.modificationMetadata.readOnlyValue) - } catch (err) { - ctx.log.error({ err }, `Error while getting hubspot ${endpoint} properties!`) - throw err - } -} diff --git a/services/libs/integrations/src/integrations/hubspot/api/tokenInfo.ts b/services/libs/integrations/src/integrations/hubspot/api/tokenInfo.ts deleted file mode 100644 index 6caf573e61..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/api/tokenInfo.ts +++ /dev/null @@ -1,30 +0,0 @@ -import axios, { AxiosRequestConfig } from 'axios' - -import { PlatformType } from '@crowd/types' - -import { IGenerateStreamsContext, IProcessStreamContext } from '../../../types' -import { getNangoToken } from '../../nango' -import { IHubspotTokenInfo } from '../types' - -export const getTokenInfo = async ( - nangoId: string, - ctx: IProcessStreamContext | IGenerateStreamsContext, -): Promise => { - try { - ctx.log.debug({ nangoId }, 'Fetching custom properties from HubSpot') - - const accessToken = await getNangoToken(nangoId, PlatformType.HUBSPOT, ctx) - - const config: AxiosRequestConfig = { - method: 'get', - url: `https://api.hubapi.com/oauth/v1/access-tokens/${accessToken}`, - } - - const response: IHubspotTokenInfo = (await axios(config)).data - - return response - } catch (err) { - ctx.log.error({ err }, 'Error while getting hubspot token information!') - throw err - } -} diff --git a/services/libs/integrations/src/integrations/hubspot/api/types.ts b/services/libs/integrations/src/integrations/hubspot/api/types.ts deleted file mode 100644 index 26f69af3dd..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/api/types.ts +++ /dev/null @@ -1,25 +0,0 @@ -export interface IPaginatedResponse { - elements: T[] - after?: string -} - -export interface IBatchCreateOrganizationsResult { - organizationId: string - sourceId: string - lastSyncedPayload: unknown -} - -export type IBatchUpdateOrganizationsResult = IBatchCreateOrganizationsResult - -export interface IBatchCreateMembersResult { - memberId: string - sourceId: string - lastSyncedPayload: unknown -} - -export type IBatchUpdateMembersResult = IBatchCreateMembersResult - -export interface IBatchOperationResult { - created: IBatchCreateMembersResult[] | IBatchCreateOrganizationsResult[] - updated: IBatchCreateMembersResult[] | IBatchCreateOrganizationsResult[] -} diff --git a/services/libs/integrations/src/integrations/hubspot/api/utils/getOrganizationDomain.ts b/services/libs/integrations/src/integrations/hubspot/api/utils/getOrganizationDomain.ts deleted file mode 100644 index 883ca1f40a..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/api/utils/getOrganizationDomain.ts +++ /dev/null @@ -1,10 +0,0 @@ -export const getOrganizationDomain = (website: string): string => { - try { - if (website.startsWith('http')) { - return new URL(website).host - } - return website - } catch (e) { - return null - } -} diff --git a/services/libs/integrations/src/integrations/hubspot/field-mapper/hubspotFieldMapper.ts b/services/libs/integrations/src/integrations/hubspot/field-mapper/hubspotFieldMapper.ts deleted file mode 100644 index 2538be0de7..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/field-mapper/hubspotFieldMapper.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { IMemberAttribute } from '@crowd/types' - -import { - HubspotEntity, - HubspotPropertyType, - IFieldProperty, - IHubspotObject, - ITypeInfo, -} from '../types' - -export abstract class HubspotFieldMapper { - protected fieldProperties: Record - - public fieldMap: Record - - public entity: HubspotEntity - - public hubspotId: number - - constructor(hubspotId: number) { - this.hubspotId = hubspotId - } - - abstract getFieldProperties( - attributes?: IMemberAttribute[], - platforms?: string[], - ): Record - - abstract getEntity(input: IHubspotObject, args?: unknown): unknown - - public isFieldMappableToHubspotType(field: string, type: HubspotPropertyType) { - if (!this.fieldProperties) { - throw new Error( - `${this.entity} field properties aren't initialized. Instance should be created with customAttributes and identities.`, - ) - } - - if (this.fieldProperties[field] === undefined) { - throw new Error(`${this.entity} property ${field} not found!`) - } - - return this.fieldProperties[field].hubspotType === type - } - - public getHubspotValue(entity: unknown, crowdKey: string) { - let value = entity[crowdKey] - - if (this.fieldProperties[crowdKey].serialize) { - value = this.fieldProperties[crowdKey].serialize(entity[crowdKey]) - } - - return value - } - - public getTypeMap(): Record { - if (!this.fieldProperties) { - throw new Error(`Can't find field properties of ${this.entity}!`) - } - - const typeMap: Record = {} - - Object.keys(this.fieldProperties).forEach((propertyName) => { - typeMap[propertyName] = { - hubspotType: this.fieldProperties[propertyName].hubspotType, - readonly: this.fieldProperties[propertyName].readonly || false, - } - }) - - return typeMap - } - - public getCrowdFieldName(hubspotAttributeName: string): string { - if (!this.fieldMap) { - throw new Error(`${this.entity} field map is not set!`) - } - - const crowdField = Object.keys(this.fieldMap).find( - (crowdFieldName) => this.fieldMap[crowdFieldName] === hubspotAttributeName, - ) - - return crowdField - } - - public getHubspotFieldName(crowdAttributeName: string): string { - this.ensureFieldMapExists() - - return this.fieldMap[crowdAttributeName] - } - - public getAllHubspotFields(): string[] { - this.ensureFieldMapExists() - - return Object.values(this.fieldMap) - } - - public getAllCrowdFields(): string[] { - this.ensureFieldMapExists() - - return Object.keys(this.fieldMap) - } - - public setFieldMap(fieldMap: Record): void { - this.fieldMap = fieldMap - } - - public setHubspotId(hubspotId: number): void { - this.hubspotId = hubspotId - } - - protected ensureFieldMapExists(): void { - if (!this.fieldMap) { - throw new Error(`${this.entity} field map is not set!`) - } - } -} diff --git a/services/libs/integrations/src/integrations/hubspot/field-mapper/mapperFactory.ts b/services/libs/integrations/src/integrations/hubspot/field-mapper/mapperFactory.ts deleted file mode 100644 index 2923bde81a..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/field-mapper/mapperFactory.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { IMemberAttribute } from '@crowd/types' - -import { HubspotEntity } from '../types' - -import { HubspotFieldMapper } from './hubspotFieldMapper' -import { HubspotMemberFieldMapper } from './memberFieldMapper' -import { HubspotOrganizationFieldMapper } from './organizationFieldMapper' - -export class HubspotFieldMapperFactory { - static getFieldMapper( - entity: HubspotEntity, - hubspotId: number, - attributes?: IMemberAttribute[], - platforms?: string[], - ): HubspotFieldMapper { - switch (entity) { - case HubspotEntity.MEMBERS: - return new HubspotMemberFieldMapper(hubspotId, attributes, platforms) - case HubspotEntity.ORGANIZATIONS: - return new HubspotOrganizationFieldMapper(hubspotId) - default: - throw new Error(`Field mapper for ${entity} not found!`) - } - } -} diff --git a/services/libs/integrations/src/integrations/hubspot/field-mapper/memberFieldMapper.ts b/services/libs/integrations/src/integrations/hubspot/field-mapper/memberFieldMapper.ts deleted file mode 100644 index 1f4db995ab..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/field-mapper/memberFieldMapper.ts +++ /dev/null @@ -1,237 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import { - IMemberAttribute, - IMemberData, - ITagOpensearch, - MemberAttributeName, - MemberAttributeType, - MemberIdentityType, - OrganizationIdentityType, - OrganizationSource, - PlatformType, -} from '@crowd/types' - -import { HubspotPropertyType, IFieldProperty, IHubspotContact } from '../types' - -import { HubspotFieldMapper } from './hubspotFieldMapper' -import { HubspotOrganizationFieldMapper } from './organizationFieldMapper' -import { serializeDate } from './utils/serialization' - -export class HubspotMemberFieldMapper extends HubspotFieldMapper { - protected fieldProperties: Record = { - displayName: { - hubspotType: HubspotPropertyType.STRING, - }, - score: { - hubspotType: HubspotPropertyType.NUMBER, - readonly: true, - }, - joinedAt: { - hubspotType: HubspotPropertyType.DATE, - readonly: true, - serialize: serializeDate, - }, - createdAt: { - hubspotType: HubspotPropertyType.DATE, - readonly: true, - serialize: serializeDate, - }, - reach: { - hubspotType: HubspotPropertyType.NUMBER, - readonly: true, - }, - numberOfOpensourceContributions: { - hubspotType: HubspotPropertyType.NUMBER, - readonly: true, - }, - activityCount: { - hubspotType: HubspotPropertyType.NUMBER, - readonly: true, - }, - activeDaysCount: { - hubspotType: HubspotPropertyType.NUMBER, - readonly: true, - }, - lastActive: { - hubspotType: HubspotPropertyType.DATE, - readonly: true, - serialize: serializeDate, - }, - averageSentiment: { - hubspotType: HubspotPropertyType.NUMBER, - readonly: true, - }, - tags: { - hubspotType: HubspotPropertyType.STRING, - readonly: true, - serialize: (tags: ITagOpensearch[]) => { - return tags.map((t) => t.name).toString() - }, - }, - organizationName: { - hubspotType: HubspotPropertyType.STRING, - }, - } - - public customAttributes: IMemberAttribute[] - - public platforms: string[] - - constructor(hubspotId: number, customAttributes: IMemberAttribute[], platforms: string[]) { - super(hubspotId) - - this.customAttributes = customAttributes - this.platforms = platforms - this.fieldProperties = this.getFieldProperties(customAttributes, platforms) - } - - getFieldProperties( - customAttributes: IMemberAttribute[], - platforms: string[], - ): Record { - const dynamicProperties = this.getDynamicFieldProperties(customAttributes, platforms) - return { ...this.fieldProperties, ...dynamicProperties } - } - - getDynamicFieldProperties( - customAttributes: IMemberAttribute[], - platforms: string[], - ): Record { - const dynamicFieldProperties: Record = {} - - for (const customAttribute of customAttributes.filter((a) => a.show)) { - if (customAttribute.type === MemberAttributeType.BOOLEAN) { - dynamicFieldProperties[`attributes.${customAttribute.name}`] = { - hubspotType: HubspotPropertyType.BOOL, - } - } else if (customAttribute.type === MemberAttributeType.DATE) { - dynamicFieldProperties[`attributes.${customAttribute.name}`] = { - hubspotType: HubspotPropertyType.DATE, - } - } else if (customAttribute.type === MemberAttributeType.NUMBER) { - dynamicFieldProperties[`attributes.${customAttribute.name}`] = { - hubspotType: HubspotPropertyType.NUMBER, - } - } else if ( - [MemberAttributeType.EMAIL, MemberAttributeType.STRING, MemberAttributeType.URL].includes( - customAttribute.type, - ) - ) { - dynamicFieldProperties[`attributes.${customAttribute.name}`] = { - hubspotType: HubspotPropertyType.STRING, - } - } - } - - for (const platform of platforms) { - dynamicFieldProperties[`identities.${platform}`] = { - hubspotType: HubspotPropertyType.STRING, - } - } - - return dynamicFieldProperties - } - - override getEntity( - hubspotContact: IHubspotContact, - organizationMapper: HubspotOrganizationFieldMapper, - ): IMemberData { - this.ensureFieldMapExists() - - if (!this.hubspotId) { - throw new Error('Hubspot Id should be set before parsing the member!') - } - - const contactProperties = hubspotContact.properties as any - - if (!contactProperties.email) { - throw new Error( - 'Member returned from HubSpot API is missing the unique identifier e-mail field!', - ) - } - - // staticly defined member fields - const member: IMemberData = { - identities: [ - { - platform: PlatformType.HUBSPOT, - value: contactProperties.email, - type: MemberIdentityType.EMAIL, - sourceId: hubspotContact.id, - verified: true, - }, - ], - attributes: { - [MemberAttributeName.SOURCE_ID]: { - [PlatformType.HUBSPOT]: hubspotContact.id, - }, - [MemberAttributeName.URL]: { - [PlatformType.HUBSPOT]: `https://app.hubspot.com/contacts/${this.hubspotId}/contact/${hubspotContact.id}`, - }, - }, - } - - // loop through member properties - for (const hubspotPropertyName of Object.keys(contactProperties)) { - const crowdKey = this.getCrowdFieldName(hubspotPropertyName) - - // discard readonly fields, readonly fields will be only used when pushing data back to hubspot - if (crowdKey && !this.fieldProperties[crowdKey].readonly) { - // For incoming integrations, we already get the member email from hubspot defined field `email` - // if user mapped crowd field `emails` to some other field - // this will be saved to the mapped field when sending the member back to hubspot - if (contactProperties[hubspotPropertyName] !== null) { - if (crowdKey.startsWith('attributes')) { - const crowdAttributeName = crowdKey.split('.')[1] || null - - if (crowdAttributeName) { - member.attributes[crowdAttributeName] = { - [PlatformType.HUBSPOT]: contactProperties[hubspotPropertyName], - } - } - } else if (crowdKey.startsWith('identities')) { - const identityPlatform = crowdKey.split('.')[1] || null - - if (identityPlatform) { - member.identities.push({ - value: contactProperties[hubspotPropertyName], - type: MemberIdentityType.USERNAME, - platform: identityPlatform, - verified: false, - }) - } - } else if (crowdKey === 'organizationName') { - // TODO uros check if this is verified or not with anil - member.organizations = [ - { - // names: [contactProperties[hubspotPropertyName]], // TODO migrate to attributes - identities: [ - { - value: contactProperties[hubspotPropertyName], - type: OrganizationIdentityType.USERNAME, - platform: PlatformType.HUBSPOT, - verified: true, - }, - ], - source: OrganizationSource.HUBSPOT, - }, - ] - } else { - member[crowdKey] = contactProperties[hubspotPropertyName] - } - } - } - } - - if (hubspotContact.organization) { - const organization = organizationMapper.getEntity(hubspotContact.organization) - if (member.organizations && member.organizations.length > 0) { - member.organizations.push(organization) - } else { - member.organizations = [organization] - } - } - - return member - } -} diff --git a/services/libs/integrations/src/integrations/hubspot/field-mapper/organizationFieldMapper.ts b/services/libs/integrations/src/integrations/hubspot/field-mapper/organizationFieldMapper.ts deleted file mode 100644 index cd0f1cc28b..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/field-mapper/organizationFieldMapper.ts +++ /dev/null @@ -1,301 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import { - IOrganization, - OrganizationIdentityType, - OrganizationSource, - PlatformType, -} from '@crowd/types' - -import { HubspotPropertyType, IFieldProperty, IHubspotObject } from '../types' - -import { HubspotFieldMapper } from './hubspotFieldMapper' -import { serializeArray } from './utils/serialization' - -export class HubspotOrganizationFieldMapper extends HubspotFieldMapper { - protected fieldProperties: Record = { - name: { - hubspotType: HubspotPropertyType.STRING, - readonly: true, - }, - url: { - hubspotType: HubspotPropertyType.STRING, - }, - description: { - hubspotType: HubspotPropertyType.STRING, - }, - emails: { - hubspotType: HubspotPropertyType.STRING, - readonly: true, - serialize: serializeArray, - }, - names: { - hubspotType: HubspotPropertyType.STRING, - readonly: true, - serialize: serializeArray, - }, - logo: { - hubspotType: HubspotPropertyType.STRING, - }, - tags: { - hubspotType: HubspotPropertyType.STRING, - readonly: true, - serialize: serializeArray, - }, - // revenueRange: { - // hubspotType: HubspotPropertyType.STRING, - // readonly: true, - // serialize: (revenueRange: any) => { - // return JSON.stringify(revenueRange) - // }, - // }, - revenueRangeMin: { - hubspotType: HubspotPropertyType.NUMBER, - readonly: true, - }, - revenueRangeMax: { - hubspotType: HubspotPropertyType.NUMBER, - readonly: true, - }, - employeeCountByCountry: { - hubspotType: HubspotPropertyType.STRING, - readonly: true, - serialize: (employeeCountByCountry: any) => { - return JSON.stringify(employeeCountByCountry) - }, - }, - employees: { - hubspotType: HubspotPropertyType.NUMBER, - }, - location: { - hubspotType: HubspotPropertyType.STRING, - }, - type: { - hubspotType: HubspotPropertyType.ENUMERATION, - }, - size: { - hubspotType: HubspotPropertyType.STRING, - }, - headline: { - hubspotType: HubspotPropertyType.STRING, - }, - industry: { - hubspotType: HubspotPropertyType.ENUMERATION, - }, - founded: { - hubspotType: HubspotPropertyType.STRING, - }, - activityCount: { - hubspotType: HubspotPropertyType.NUMBER, - readonly: true, - }, - memberCount: { - hubspotType: HubspotPropertyType.NUMBER, - readonly: true, - }, - allSubsidiaries: { - hubspotType: HubspotPropertyType.STRING, - readonly: true, - serialize: serializeArray, - }, - alternativeNames: { - hubspotType: HubspotPropertyType.STRING, - readonly: true, - serialize: serializeArray, - }, - averageEmployeeTenure: { - hubspotType: HubspotPropertyType.NUMBER, - readonly: true, - }, - averageTenureByLevel: { - hubspotType: HubspotPropertyType.STRING, - readonly: true, - serialize: (averageTenureByLevel: any) => { - return JSON.stringify(averageTenureByLevel) - }, - }, - averageTenureByRole: { - hubspotType: HubspotPropertyType.STRING, - readonly: true, - serialize: (averageTenureByRole: any) => { - return JSON.stringify(averageTenureByRole) - }, - }, - directSubsidiaries: { - hubspotType: HubspotPropertyType.STRING, - readonly: true, - serialize: serializeArray, - }, - // employeeChurnRate: { - // hubspotType: HubspotPropertyType.STRING, - // readonly: true, - // serialize: (employeeChurnRate: any) => { - // return JSON.stringify(employeeChurnRate) - // }, - // }, - employeeCountByMonth: { - hubspotType: HubspotPropertyType.STRING, - readonly: true, - serialize: (employeeCountByMonth: any) => { - return JSON.stringify(employeeCountByMonth) - }, - }, - employeeChurnRate12Month: { - hubspotType: HubspotPropertyType.NUMBER, - readonly: true, - }, - // employeeGrowthRate: { - // hubspotType: HubspotPropertyType.STRING, - // readonly: true, - // serialize: (employeeGrowthRate: any) => { - // return JSON.stringify(employeeGrowthRate) - // }, - // }, - employeeGrowthRate12Month: { - hubspotType: HubspotPropertyType.NUMBER, - readonly: true, - }, - employeeCountByMonthByLevel: { - hubspotType: HubspotPropertyType.STRING, - readonly: true, - serialize: (employeeCountByMonthByLevel: any) => { - return JSON.stringify(employeeCountByMonthByLevel) - }, - }, - employeeCountByMonthByRole: { - hubspotType: HubspotPropertyType.STRING, - readonly: true, - serialize: (employeeCountByMonthByRole: any) => { - return JSON.stringify(employeeCountByMonthByRole) - }, - }, - gicsSector: { - hubspotType: HubspotPropertyType.STRING, - readonly: true, - }, - grossAdditionsByMonth: { - hubspotType: HubspotPropertyType.STRING, - readonly: true, - serialize: (grossAdditionsByMonth: any) => { - return JSON.stringify(grossAdditionsByMonth) - }, - }, - grossDeparturesByMonth: { - hubspotType: HubspotPropertyType.STRING, - readonly: true, - serialize: (grossDeparturesByMonth: any) => { - return JSON.stringify(grossDeparturesByMonth) - }, - }, - immediateParent: { - hubspotType: HubspotPropertyType.STRING, - readonly: true, - }, - ultimateParent: { - hubspotType: HubspotPropertyType.STRING, - readonly: true, - }, - } - - override getFieldProperties(): Record { - return this.fieldProperties - } - - public getSocialUrl(platform: string, handle: string): string { - if (!platform || !handle) { - return null - } - - switch (platform) { - case PlatformType.TWITTER: - return `https://twitter.com/${handle}` - case PlatformType.LINKEDIN: - return `https://linkedin.com/company/${handle}` - case PlatformType.CRUNCHBASE: - return `https://crunchbase.com/organization/${handle}` - case PlatformType.GITHUB: - return `https://github.com/${handle}` - default: - return null - } - } - - override getEntity(hubspotOrganization: IHubspotObject): IOrganization { - this.ensureFieldMapExists() - - if (!this.hubspotId) { - throw new Error('Hubspot Id should be set before parsing the organization!') - } - - const organizationProperties = hubspotOrganization.properties as any - - if (!organizationProperties.name) { - throw new Error( - 'Organization returned from HubSpot API is missing the unique identifier name field!', - ) - } - - // TODO uros check if this is verified or not with anil - const organization: IOrganization = { - // names: [organizationProperties.name], // TODO migrate to attributes - identities: [ - { - value: `${this.hubspotId}:${hubspotOrganization.id}`, - platform: PlatformType.HUBSPOT, - sourceId: hubspotOrganization.id, - type: OrganizationIdentityType.USERNAME, - verified: true, - }, - ], - // attributes: { - // [OrganizationAttributeName.SOURCE_ID]: { - // [PlatformType.HUBSPOT]: hubspotOrganization.id, - // }, - // [OrganizationAttributeName.URL]: { - // [PlatformType.HUBSPOT]: `https://app.hubspot.com/contacts/${this.hubspotId}/company/${hubspotOrganization.id}`, - // }, - // [OrganizationAttributeName.DOMAIN]: { - // [PlatformType.HUBSPOT]: organizationProperties.domain, - // }, - // }, - source: OrganizationSource.HUBSPOT, - } - - // loop through organization properties - for (const hubspotPropertyName of Object.keys(organizationProperties)) { - const crowdKey = this.getCrowdFieldName(hubspotPropertyName) - - // discard readonly fields, readonly fields will be only used when pushing data back to hubspot - if (crowdKey && !this.fieldProperties[crowdKey].readonly) { - if (organizationProperties[hubspotPropertyName] !== null) { - organization[crowdKey] = organizationProperties[hubspotPropertyName] - - // add additional identities to org using social fields come from hubspot - if ( - [ - PlatformType.LINKEDIN, - PlatformType.TWITTER, - PlatformType.GITHUB, - PlatformType.CRUNCHBASE, - ].includes(crowdKey as PlatformType) - ) { - // fix for linkedin social, it comes as a full url - if (crowdKey === PlatformType.LINKEDIN) { - const linkedinHandle = organizationProperties[hubspotPropertyName].split('/').pop() - organization[crowdKey] = linkedinHandle - } - - // TODO uros - check if this is verified or not with anil - organization.identities.push({ - value: organization[crowdKey], - type: OrganizationIdentityType.USERNAME, - platform: crowdKey, - sourceId: null, - verified: false, - }) - } - } - } - } - return organization - } -} diff --git a/services/libs/integrations/src/integrations/hubspot/field-mapper/utils/serialization.ts b/services/libs/integrations/src/integrations/hubspot/field-mapper/utils/serialization.ts deleted file mode 100644 index a6f327342c..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/field-mapper/utils/serialization.ts +++ /dev/null @@ -1,19 +0,0 @@ -export const serializeArray = (array: string[]): string => { - if (!array || array.length === 0) { - return undefined - } - return array.toString() -} - -export const deserializeArray = (string: string): string[] => { - return string.split(',') -} - -export const serializeDate = (date: string): number => { - const dateObj = new Date(date) - if (isNaN(dateObj.getTime()) || dateObj.getTime() === 0) { - return undefined - } - dateObj.setUTCHours(0, 0, 0, 0) - return dateObj.getTime() -} diff --git a/services/libs/integrations/src/integrations/hubspot/generateStreams.ts b/services/libs/integrations/src/integrations/hubspot/generateStreams.ts deleted file mode 100644 index aa613def2f..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/generateStreams.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { GenerateStreamsHandler } from '../../types' - -import { - HubspotEntity, - HubspotStream, - IHubspotAttributeMap, - IHubspotIntegrationSettings, -} from './types' - -const handler: GenerateStreamsHandler = async (ctx) => { - // eslint-disable-next-line @typescript-eslint/no-explicit-any - - const settings = ctx.integration.settings as IHubspotIntegrationSettings - - if (!settings.enabledFor || settings.enabledFor.length === 0) { - await ctx.abortRunWithError('Integration is not enabled for members or organizations!') - return - } - - const streams: HubspotStream[] = [] - - if (settings.enabledFor.includes(HubspotEntity.MEMBERS)) { - const fieldMap: IHubspotAttributeMap = settings.attributesMapping?.members - - if (!fieldMap) { - await ctx.abortRunWithError( - `Integration is enabled for members, but field mapping for members doesn't exist!`, - ) - return - } - - if (!settings.crowdAttributes) { - await ctx.abortRunWithError( - `Member attributes that are mapped to a hubspot field are required for member field mapper.`, - ) - return - } - - if (!settings.platforms) { - await ctx.abortRunWithError( - `Identity platforms that are mapped to a hubspot field are required for member field mapper.`, - ) - return - } - - streams.push(HubspotStream.MEMBERS) - } - - if (settings.enabledFor.includes(HubspotEntity.ORGANIZATIONS)) { - const fieldMap: IHubspotAttributeMap = settings.attributesMapping?.organizations - - if (!fieldMap) { - await ctx.abortRunWithError( - `Integration is enabled for organizations, but field mapping for organizations doesn't exist!`, - ) - return - } - - streams.push(HubspotStream.ORGANIZATIONS) - } - - if (streams.length > 0) { - await ctx.publishStream(HubspotStream.ROOT, streams) - } -} - -export default handler diff --git a/services/libs/integrations/src/integrations/hubspot/index.ts b/services/libs/integrations/src/integrations/hubspot/index.ts deleted file mode 100644 index 91a49d0c58..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/index.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { IIntegrationDescriptor } from '../../types' - -import generateStreams from './generateStreams' -import { HUBSPOT_MEMBER_ATTRIBUTES } from './memberAttributes' -import processData from './processData' -import processStream from './processStream' -import processSyncRemote from './processSyncRemote' -import startSyncRemote from './startSyncRemote' - -const descriptor: IIntegrationDescriptor = { - type: 'hubspot', - memberAttributes: HUBSPOT_MEMBER_ATTRIBUTES, - checkEvery: 8 * 60, // 8 hours - generateStreams, - processStream, - processData, - startSyncRemote, - processSyncRemote, -} - -export default descriptor diff --git a/services/libs/integrations/src/integrations/hubspot/memberAttributes.ts b/services/libs/integrations/src/integrations/hubspot/memberAttributes.ts deleted file mode 100644 index da7d4ef728..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/memberAttributes.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { - IMemberAttribute, - MemberAttributeName, - MemberAttributeType, - MemberAttributes, -} from '@crowd/types' - -export const HUBSPOT_MEMBER_ATTRIBUTES: IMemberAttribute[] = [ - { - name: MemberAttributes[MemberAttributeName.URL].name, - label: MemberAttributes[MemberAttributeName.URL].label, - type: MemberAttributeType.URL, - canDelete: false, - show: true, - }, - { - name: MemberAttributes[MemberAttributeName.SOURCE_ID].name, - label: MemberAttributes[MemberAttributeName.SOURCE_ID].label, - type: MemberAttributeType.STRING, - canDelete: false, - show: false, - }, -] diff --git a/services/libs/integrations/src/integrations/hubspot/processData.ts b/services/libs/integrations/src/integrations/hubspot/processData.ts deleted file mode 100644 index 648cec816d..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/processData.ts +++ /dev/null @@ -1,60 +0,0 @@ -/* eslint-disable @typescript-eslint/no-explicit-any */ -import { IntegrationResultType } from '@crowd/types' - -import { ProcessDataHandler } from '../../types' - -import { HubspotFieldMapperFactory } from './field-mapper/mapperFactory' -import { HubspotEntity, HubspotStream, IHubspotData, IHubspotIntegrationSettings } from './types' - -const processContact: ProcessDataHandler = async (ctx) => { - const data = ctx.data as IHubspotData - - const settings = ctx.integration.settings as IHubspotIntegrationSettings - - const memberMapper = HubspotFieldMapperFactory.getFieldMapper( - HubspotEntity.MEMBERS, - settings.hubspotId, - settings.crowdAttributes, - settings.platforms, - ) - const orgMapper = HubspotFieldMapperFactory.getFieldMapper( - HubspotEntity.ORGANIZATIONS, - settings.hubspotId, - ) - - memberMapper.setFieldMap(settings.attributesMapping[HubspotEntity.MEMBERS]) - orgMapper.setFieldMap(settings.attributesMapping[HubspotEntity.ORGANIZATIONS]) - - const member = memberMapper.getEntity(data.element, orgMapper) - - await ctx.publishCustom(member, IntegrationResultType.MEMBER_ENRICH) -} - -const processCompany: ProcessDataHandler = async (ctx) => { - const data = ctx.data as IHubspotData - const settings = ctx.integration.settings as IHubspotIntegrationSettings - - const orgMapper = HubspotFieldMapperFactory.getFieldMapper( - HubspotEntity.ORGANIZATIONS, - settings.hubspotId, - ) - - orgMapper.setFieldMap(settings.attributesMapping[HubspotEntity.ORGANIZATIONS]) - - const organization = orgMapper.getEntity(data.element, orgMapper) - - await ctx.publishCustom(organization, IntegrationResultType.ORGANIZATION_ENRICH) -} - -const handler: ProcessDataHandler = async (ctx) => { - const data = ctx.data as IHubspotData - - // only process contacts with e-mails, this will be the unique identifier when enriching - if (data.type === HubspotStream.MEMBERS && (data.element.properties as any).email) { - await processContact(ctx) - } else if (data.type === HubspotStream.ORGANIZATIONS && (data.element.properties as any).name) { - await processCompany(ctx) - } -} - -export default handler diff --git a/services/libs/integrations/src/integrations/hubspot/processStream.ts b/services/libs/integrations/src/integrations/hubspot/processStream.ts deleted file mode 100644 index 6c8d8a61a8..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/processStream.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { RequestThrottler } from '@crowd/common' - -import { ProcessStreamHandler } from '../../types' - -import { getAllCompanies } from './api/companies' -import { getAllContacts } from './api/contacts' -import { HubspotFieldMapperFactory } from './field-mapper/mapperFactory' -import { HubspotMemberFieldMapper } from './field-mapper/memberFieldMapper' -import { HubspotOrganizationFieldMapper } from './field-mapper/organizationFieldMapper' -import { - HubspotEntity, - HubspotStream, - IHubspotContact, - IHubspotData, - IHubspotIntegrationSettings, - IHubspotObject, -} from './types' - -const processRootStream: ProcessStreamHandler = async (ctx) => { - const throttler = new RequestThrottler(99, 11000, ctx.log) - - const settings = ctx.integration.settings as IHubspotIntegrationSettings - - // hubspot might have long running root stream, change stream queue message visibility to 7 hours - await ctx.setMessageVisibilityTimeout(60 * 60 * 7) - - const streams = ctx.stream.data as HubspotStream[] - - if (streams.includes(HubspotStream.MEMBERS)) { - const memberMapper = HubspotFieldMapperFactory.getFieldMapper( - HubspotEntity.MEMBERS, - settings.hubspotId, - settings.crowdAttributes, - settings.platforms, - ) as HubspotMemberFieldMapper - memberMapper.setFieldMap(settings.attributesMapping.members) - memberMapper.setHubspotId(settings.hubspotId) - - const organizationsEnabled = settings.enabledFor.includes(HubspotEntity.ORGANIZATIONS) - - let organizationMapper - - if (organizationsEnabled) { - organizationMapper = HubspotFieldMapperFactory.getFieldMapper( - HubspotEntity.ORGANIZATIONS, - settings.hubspotId, - ) as HubspotOrganizationFieldMapper - - organizationMapper.setFieldMap(settings.attributesMapping.organizations) - } - - const contactsGenerator = getAllContacts( - ctx.serviceSettings.nangoId, - memberMapper, - organizationMapper, - ctx, - throttler, - organizationsEnabled, - ) - - let contactsPage = await contactsGenerator.next() - - while (!contactsPage.done) { - const contacts = contactsPage.value as IHubspotContact[] - - while (contacts.length > 0) { - const contact = contacts.shift() - await ctx.processData({ - type: HubspotStream.MEMBERS, - element: contact as IHubspotContact, - }) - } - - contactsPage = await contactsGenerator.next() - } - } - - if (streams.includes(HubspotStream.ORGANIZATIONS)) { - const organizationMapper = HubspotFieldMapperFactory.getFieldMapper( - HubspotEntity.ORGANIZATIONS, - settings.hubspotId, - ) as HubspotOrganizationFieldMapper - - organizationMapper.setFieldMap(settings.attributesMapping.organizations) - - const companyGenerator = getAllCompanies( - ctx.serviceSettings.nangoId, - organizationMapper, - ctx, - throttler, - ) - - let companyPage = await companyGenerator.next() - - while (!companyPage.done) { - const companies = companyPage.value as IHubspotObject[] - - while (companies.length > 0) { - const company = companies.shift() - await ctx.processData({ - type: HubspotStream.ORGANIZATIONS, - element: company as IHubspotObject, - }) - } - - companyPage = await companyGenerator.next() - } - } -} - -const handler: ProcessStreamHandler = async (ctx) => { - await processRootStream(ctx) -} - -export default handler diff --git a/services/libs/integrations/src/integrations/hubspot/processSyncRemote.ts b/services/libs/integrations/src/integrations/hubspot/processSyncRemote.ts deleted file mode 100644 index bf650a5a6e..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/processSyncRemote.ts +++ /dev/null @@ -1,148 +0,0 @@ -import { RequestThrottler } from '@crowd/common' -import { - AutomationSyncTrigger, - Entity, - HubspotSettings, - IMember, - IOrganization, -} from '@crowd/types' - -import { - IGenerateStreamsContext, - IIntegrationProcessRemoteSyncContext, - ProcessIntegrationSyncHandler, -} from '../../types' - -import { addContactsToList } from './api/addContactsToList' -import { batchCreateMembers } from './api/batchCreateMembers' -import { batchCreateOrganizations } from './api/batchCreateOrganizations' -import { batchUpdateMembers } from './api/batchUpdateMembers' -import { batchUpdateOrganizations } from './api/batchUpdateOrganizations' -import { IBatchOperationResult } from './api/types' -import { HubspotFieldMapperFactory } from './field-mapper/mapperFactory' -import { HubspotMemberFieldMapper } from './field-mapper/memberFieldMapper' -import { HubspotOrganizationFieldMapper } from './field-mapper/organizationFieldMapper' -import { HubspotEntity, IHubspotIntegrationSettings } from './types' - -const handler: ProcessIntegrationSyncHandler = async ( - toCreate: T[], - toUpdate: T[], - entity: Entity, - ctx: IIntegrationProcessRemoteSyncContext, -): Promise => { - const nangoId = `${ctx.tenantId}-${ctx.integration.platform}` - - const integrationContext = { - log: ctx.log, - serviceSettings: { - nangoId, - nangoUrl: ctx.serviceSettings.nangoUrl, - nangoSecretKey: ctx.serviceSettings.nangoSecretKey, - }, - } as IGenerateStreamsContext - - const settings = ctx.integration.settings as IHubspotIntegrationSettings - - const throttler = new RequestThrottler(100, 10000, ctx.log) - - switch (entity) { - case Entity.MEMBERS: { - let membersCreatedInHubspot = [] - let membersUpdatedInHubspot = [] - - const memberMapper = HubspotFieldMapperFactory.getFieldMapper( - HubspotEntity.MEMBERS, - settings.hubspotId, - settings.crowdAttributes, - settings.platforms, - ) as HubspotMemberFieldMapper - - memberMapper.setFieldMap( - (ctx.integration.settings as IHubspotIntegrationSettings).attributesMapping.members, - ) - - if (toCreate.length > 0) { - membersCreatedInHubspot = await batchCreateMembers( - nangoId, - toCreate as IMember[], - memberMapper, - integrationContext, - throttler, - ) - } - - if (toUpdate.length > 0) { - membersUpdatedInHubspot = await batchUpdateMembers( - nangoId, - toUpdate as IMember[], - memberMapper, - integrationContext, - throttler, - ) - } - - // we should also add members to hubspot lists, if it's coming from an automation - if ( - ctx.automation && - ctx.automation.trigger === AutomationSyncTrigger.MEMBER_ATTRIBUTES_MATCH - ) { - const vids: string[] = [ - ...membersCreatedInHubspot.map((m) => m.sourceId), - ...(toUpdate as IMember[]).map((m) => m.attributes.sourceId.hubspot), - ] - - await addContactsToList( - nangoId, - (ctx.automation.settings as HubspotSettings).contactList, - vids, - integrationContext, - throttler, - ) - } - - return { created: membersCreatedInHubspot, updated: membersUpdatedInHubspot } - } - case Entity.ORGANIZATIONS: { - let companiesCreatedInHubspot = [] - let companiesUpdatedInHubspot = [] - - const organizationMapper = HubspotFieldMapperFactory.getFieldMapper( - HubspotEntity.ORGANIZATIONS, - settings.hubspotId, - ) as HubspotOrganizationFieldMapper - - organizationMapper.setFieldMap( - (ctx.integration.settings as IHubspotIntegrationSettings).attributesMapping.organizations, - ) - - if (toCreate.length > 0) { - companiesCreatedInHubspot = await batchCreateOrganizations( - nangoId, - toCreate as IOrganization[], - organizationMapper, - integrationContext, - throttler, - ) - } - - if (toUpdate.length > 0) { - companiesUpdatedInHubspot = await batchUpdateOrganizations( - nangoId, - toUpdate as IOrganization[], - organizationMapper, - integrationContext, - throttler, - ) - } - - return { created: companiesCreatedInHubspot, updated: companiesUpdatedInHubspot } - } - default: { - const message = `Unsupported entity ${entity} while processing HubSpot sync remote!` - ctx.log.error(message) - throw new Error(message) - } - } -} - -export default handler diff --git a/services/libs/integrations/src/integrations/hubspot/startSyncRemote.ts b/services/libs/integrations/src/integrations/hubspot/startSyncRemote.ts deleted file mode 100644 index e7a88c7443..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/startSyncRemote.ts +++ /dev/null @@ -1,58 +0,0 @@ -import { AutomationState, AutomationSyncTrigger } from '@crowd/types' - -import { StartIntegrationSyncHandler } from '../../types' - -import { HubspotEntity, IHubspotIntegrationSettings } from './types' - -const handler: StartIntegrationSyncHandler = async (ctx) => { - const settings = ctx.integration.settings as IHubspotIntegrationSettings - if (settings.enabledFor.includes(HubspotEntity.MEMBERS)) { - // sync members - await ctx.integrationSyncWorkerEmitter.triggerSyncMarkedMembers( - ctx.tenantId, - ctx.integration.id, - ) - - const memberSyncAutomations = ctx.automations.filter( - (a) => - a.trigger === AutomationSyncTrigger.MEMBER_ATTRIBUTES_MATCH && - a.state === AutomationState.ACTIVE, - ) - - // sync filter automations - for (const automation of memberSyncAutomations) { - await ctx.integrationSyncWorkerEmitter.triggerOnboardAutomation( - ctx.tenantId, - ctx.integration.id, - automation.id, - automation.trigger as AutomationSyncTrigger, - ) - } - } - - if (settings.enabledFor.includes(HubspotEntity.ORGANIZATIONS)) { - // sync orgs - await ctx.integrationSyncWorkerEmitter.triggerSyncMarkedOrganizations( - ctx.tenantId, - ctx.integration.id, - ) - - const organizationSyncAutomations = ctx.automations.filter( - (a) => - a.trigger === AutomationSyncTrigger.ORGANIZATION_ATTRIBUTES_MATCH && - a.state === AutomationState.ACTIVE, - ) - - // sync filter automations - for (const automation of organizationSyncAutomations) { - await ctx.integrationSyncWorkerEmitter.triggerOnboardAutomation( - ctx.tenantId, - ctx.integration.id, - automation.id, - automation.trigger as AutomationSyncTrigger, - ) - } - } -} - -export default handler diff --git a/services/libs/integrations/src/integrations/hubspot/types.ts b/services/libs/integrations/src/integrations/hubspot/types.ts deleted file mode 100644 index 7c0dcf5185..0000000000 --- a/services/libs/integrations/src/integrations/hubspot/types.ts +++ /dev/null @@ -1,175 +0,0 @@ -import { IMemberAttribute } from '@crowd/types' - -export enum HubspotPropertyType { - BOOL = 'bool', - ENUMERATION = 'enumeration', - DATE = 'date', - DATETIME = 'dateTime', - STRING = 'string', - NUMBER = 'number', -} - -export interface IFieldProperty { - hubspotType: HubspotPropertyType - readonly?: boolean - // eslint-disable-next-line @typescript-eslint/no-explicit-any - serialize?: (object: any) => string | number - deserialize?: (string: string) => object -} - -export enum HubspotEndpoint { - CONTACTS = 'contacts', - COMPANIES = 'companies', -} - -export enum HubspotEntity { - MEMBERS = 'members', - ORGANIZATIONS = 'organizations', -} - -export enum HubspotStream { - MEMBERS = 'members', - ORGANIZATIONS = 'organizations', - ROOT = 'root', -} - -export enum HubspotAssociationType { - CONTACT_TO_COMPANY = 'contact_to_company', -} - -export interface IHubspotTokenInfo { - token: string - user: string - hub_domain: string - scopes: string[] - scope_to_scope_group_pks: number[] - hub_id: number - app_id: number - expires_in: number - user_id: number - token_type: string -} - -export interface IHubspotPropertyEnumerationOption { - label: string - value: string - hidden: boolean - description: string - displayOrder: number -} - -export interface IHubspotPropertyModificationMetadata { - archivable: boolean - readOnlyValue: boolean - readOnlyOptions: boolean - readOnlyDefinition: boolean -} - -export interface IHubspotProperty { - name: string - type: HubspotPropertyType - label: string - hidden: boolean - options: IHubspotPropertyEnumerationOption[] - createdAt: string - fieldType: string - formField: boolean - groupName: string - updatedAt: string - calculated: boolean - description: string - displayOrder: number - hasUniqueValue: boolean - hubspotDefined: boolean - externalOptions: boolean - modificationMetadata: IHubspotPropertyModificationMetadata -} - -export interface IHubspotListMetadata { - size: number - lastSizeChangeAt: number - processing: string - lastProcessingStateChangeAt: number - error: string - listReferencesCount: number - parentFolderId: number -} - -export interface IHubspotList { - portalId: number - listId: number - createdAt: number - updatedAt: number - name: string - listType: string - authorId: number - parentId: number - filters: string[] - metaData: IHubspotListMetadata - archived: boolean - teamIds: number[] - ilsFilterBranch: string - readOnly: boolean - internal: boolean - limitExempt: boolean - dynamic: boolean -} - -export interface ITypeInfo { - hubspotType: HubspotPropertyType - readonly: boolean -} - -export interface IHubspotObject { - id: string - properties: unknown - createdAt: string - updatedAt: string - archived: boolean -} - -export interface IHubspotContact extends IHubspotObject { - organization?: IHubspotObject -} - -export interface IHubspotAssociation { - id: string - type: HubspotAssociationType -} - -export interface IHubspotData { - type: HubspotStream - element: IHubspotObject | IHubspotContact -} - -export interface IHubspotAttributeMap { - [key: string]: string -} - -export interface IHubspotOnboardingSettings { - enabledFor: HubspotEntity[] - attributesMapping: { - [HubspotEntity.MEMBERS]?: IHubspotAttributeMap - [HubspotEntity.ORGANIZATIONS]?: IHubspotAttributeMap - } -} - -export interface IHubspotIntegrationSettings extends IHubspotOnboardingSettings { - hubspotId: number - hubspotProperties: IHubspotProperty[] - crowdAttributes: IMemberAttribute[] - platforms: string[] - updateMemberAttributes: boolean - syncRemoteEnabled?: boolean - blockSyncRemote?: boolean -} - -export interface IHubspotManualSyncPayload { - memberId?: string - organizationId?: string - segmentId: string -} - -export interface IHubspotBaseStream { - start?: number -} diff --git a/services/libs/integrations/src/integrations/index.ts b/services/libs/integrations/src/integrations/index.ts index 99508139b3..19ac68de40 100644 --- a/services/libs/integrations/src/integrations/index.ts +++ b/services/libs/integrations/src/integrations/index.ts @@ -28,13 +28,6 @@ export * from './linkedin/grid' export * from './linkedin/types' export * from './linkedin/memberAttributes' -export * from './hubspot/types' -export * from './hubspot/api/types' -export * from './hubspot/field-mapper/mapperFactory' -export { getProperties as getHubspotProperties } from './hubspot/api/properties' -export { getTokenInfo as getHubspotTokenInfo } from './hubspot/api/tokenInfo' -export { getLists as getHubspotLists } from './hubspot/api/lists' - export * from './reddit/grid' export * from './reddit/types' export * from './reddit/memberAttributes' diff --git a/services/libs/integrations/src/types.ts b/services/libs/integrations/src/types.ts index f81c2def26..52ed010b96 100644 --- a/services/libs/integrations/src/types.ts +++ b/services/libs/integrations/src/types.ts @@ -1,21 +1,16 @@ import { DbConnection, DbTransaction } from '@crowd/data-access-layer/src/database' import { Logger } from '@crowd/logging' import { - Entity, IActivityData, - IAutomationData, ICache, IConcurrentRequestLimiter, IIntegration, IIntegrationStream, - IIntegrationSyncWorkerEmitter, IMemberAttribute, IRateLimiter, IntegrationResultType, } from '@crowd/types' -import { IBatchOperationResult } from './integrations/hubspot/api/types' - export interface IIntegrationContext { onboarding?: boolean integration: IIntegration @@ -33,20 +28,11 @@ export interface IIntegrationContext { abortRunWithError: (message: string, metadata?: unknown, error?: Error) => Promise } -export interface IIntegrationStartRemoteSyncContext { - integrationSyncWorkerEmitter: IIntegrationSyncWorkerEmitter - integration: IIntegration - automations: IAutomationData[] - tenantId: string - log: Logger -} - export interface IIntegrationProcessRemoteSyncContext { tenantId: string integration: IIntegration log: Logger serviceSettings: IIntegrationServiceSettings - automation?: IAutomationData } export interface IGenerateStreamsContext extends IIntegrationContext { @@ -134,13 +120,6 @@ export type GenerateStreamsHandler = (ctx: IGenerateStreamsContext) => Promise Promise export type ProcessWebhookStreamHandler = (ctx: IProcessWebhookStreamContext) => Promise export type ProcessDataHandler = (ctx: IProcessDataContext) => Promise -export type StartIntegrationSyncHandler = (ctx: IIntegrationStartRemoteSyncContext) => Promise -export type ProcessIntegrationSyncHandler = ( - toCreate: T[], - toUpdate: T[], - entity: Entity, - ctx: IIntegrationProcessRemoteSyncContext, -) => Promise export interface IIntegrationDescriptor { /** @@ -205,19 +184,6 @@ export interface IIntegrationDescriptor { // if undefined it will never check // if 0 it will check the same as if it was 1 - every minute checkEvery?: number - - /** - * Function that will be called if defined, after an integration goes into done state. - * Mainly responsible for sending queue messages to integration-sync-worker - */ - startSyncRemote?: StartIntegrationSyncHandler - - /** - * Function that will be called from integration sync worker for outgoing integrations. - * Gets two arrays, entities to create and entities to update. - * Logic for calling the required api endpoints per integration lives here. - */ - processSyncRemote?: ProcessIntegrationSyncHandler } export interface IIntegrationServiceSettings { diff --git a/services/libs/queue/src/config.ts b/services/libs/queue/src/config.ts index 23bd128f6b..f114e68620 100644 --- a/services/libs/queue/src/config.ts +++ b/services/libs/queue/src/config.ts @@ -21,8 +21,3 @@ export const SEARCH_SYNC_WORKER_QUEUE_SETTINGS = { [QueueVendor.KAFKA]: KafkaQueueConfig.SEARCH_SYNC_WORKER_QUEUE_SETTINGS, [QueueVendor.SQS]: SqsQueueConfig.SEARCH_SYNC_WORKER_QUEUE_SETTINGS, } - -export const INTEGRATION_SYNC_WORKER_QUEUE_SETTINGS = { - [QueueVendor.KAFKA]: KafkaQueueConfig.INTEGRATION_SYNC_WORKER_QUEUE_SETTINGS, - [QueueVendor.SQS]: SqsQueueConfig.INTEGRATION_SYNC_WORKER_QUEUE_SETTINGS, -} diff --git a/services/libs/queue/src/types.ts b/services/libs/queue/src/types.ts index 9519ea4472..bd2842417e 100644 --- a/services/libs/queue/src/types.ts +++ b/services/libs/queue/src/types.ts @@ -76,7 +76,6 @@ export enum CrowdQueue { INTEGRATION_STREAM_WORKER = 'integration-stream-worker', DATA_SINK_WORKER = 'data-sink-worker', SEARCH_SYNC_WORKER = 'search-sync-worker', - INTEGRATION_SYNC_WORKER = 'integration-sync-worker', } export enum QueueVendor { diff --git a/services/libs/queue/src/vendors/kafka/config.ts b/services/libs/queue/src/vendors/kafka/config.ts index a31407a30d..5b0e2b2f14 100644 --- a/services/libs/queue/src/vendors/kafka/config.ts +++ b/services/libs/queue/src/vendors/kafka/config.ts @@ -2,7 +2,6 @@ import { DATA_SINK_WORKER_PARTITIONS, INTEGRATION_RUN_WORKER_PARTITIONS, INTEGRATION_STREAM_WORKER_PARTITIONS, - INTEGRATION_SYNC_WORKER_PARTITIONS, SEARCH_SYNC_WORKER_PARTITIONS, } from '@crowd/common' @@ -37,15 +36,6 @@ export const SEARCH_SYNC_WORKER_QUEUE_SETTINGS: IKafkaChannelConfig = { }, } -export const INTEGRATION_SYNC_WORKER_QUEUE_SETTINGS: IKafkaChannelConfig = { - name: CrowdQueue.INTEGRATION_SYNC_WORKER, - replicationFactor: 1, - partitions: { - default: 1, - ...INTEGRATION_SYNC_WORKER_PARTITIONS, - }, -} - export const INTEGRATION_STREAM_WORKER_QUEUE_SETTINGS: IKafkaChannelConfig = { name: CrowdQueue.INTEGRATION_STREAM_WORKER, replicationFactor: 1, @@ -60,5 +50,4 @@ export const configMap = { [CrowdQueue.INTEGRATION_STREAM_WORKER]: INTEGRATION_STREAM_WORKER_QUEUE_SETTINGS, [CrowdQueue.DATA_SINK_WORKER]: DATA_SINK_WORKER_QUEUE_SETTINGS, [CrowdQueue.SEARCH_SYNC_WORKER]: SEARCH_SYNC_WORKER_QUEUE_SETTINGS, - [CrowdQueue.INTEGRATION_SYNC_WORKER]: INTEGRATION_SYNC_WORKER_QUEUE_SETTINGS, } diff --git a/services/libs/queue/src/vendors/sqs/config.ts b/services/libs/queue/src/vendors/sqs/config.ts index 593135d034..70e030e335 100644 --- a/services/libs/queue/src/vendors/sqs/config.ts +++ b/services/libs/queue/src/vendors/sqs/config.ts @@ -51,21 +51,9 @@ export const SEARCH_SYNC_WORKER_QUEUE_SETTINGS: ISqsConfig = { fifoThroughputLimit: SqsFifoThroughputLimitType.PER_MESSAGE_GROUP_ID, } -export const INTEGRATION_SYNC_WORKER_QUEUE_SETTINGS: ISqsConfig = { - name: CrowdQueue.INTEGRATION_SYNC_WORKER, - type: SqsQueueType.FIFO, - waitTimeSeconds: 20, // seconds - visibilityTimeout: 30, // seconds - messageRetentionPeriod: 345600, // 4 days - deliveryDelay: 0, - deduplicationScope: SqsQueueDeduplicationType.MESSAGE_GROUP, - fifoThroughputLimit: SqsFifoThroughputLimitType.PER_MESSAGE_GROUP_ID, -} - export const configMap = { [CrowdQueue.INTEGRATION_RUN_WORKER]: INTEGRATION_RUN_WORKER_QUEUE_SETTINGS, [CrowdQueue.INTEGRATION_STREAM_WORKER]: INTEGRATION_STREAM_WORKER_QUEUE_SETTINGS, [CrowdQueue.DATA_SINK_WORKER]: DATA_SINK_WORKER_QUEUE_SETTINGS, [CrowdQueue.SEARCH_SYNC_WORKER]: SEARCH_SYNC_WORKER_QUEUE_SETTINGS, - [CrowdQueue.INTEGRATION_SYNC_WORKER]: INTEGRATION_SYNC_WORKER_QUEUE_SETTINGS, } diff --git a/services/libs/types/src/automations.ts b/services/libs/types/src/automations.ts deleted file mode 100644 index b91d951028..0000000000 --- a/services/libs/types/src/automations.ts +++ /dev/null @@ -1,104 +0,0 @@ -export enum AutomationSyncTrigger { - MEMBER_ATTRIBUTES_MATCH = 'member_attributes_match', - ORGANIZATION_ATTRIBUTES_MATCH = 'organization_attributes_match', -} - -/** - * all automation types that we are currently supporting - */ -export enum AutomationType { - WEBHOOK = 'webhook', - SLACK = 'slack', - HUBSPOT = 'hubspot', -} - -/** - * automation can either be active or disabled - */ -export enum AutomationState { - ACTIVE = 'active', - DISABLED = 'disabled', -} - -/** - * To determine the result of the execution if state == error -> error column will also be available - */ -export enum AutomationExecutionState { - SUCCESS = 'success', - ERROR = 'error', -} - -/** - * What can trigger this automation - */ -export enum AutomationTrigger { - NEW_ACTIVITY = 'new_activity', - NEW_MEMBER = 'new_member', -} - -/** - * For webhook automation we only need URL to which we will post information - */ -export interface WebhookSettings { - url: string -} - -/** - * Settings for new activity trigger based automations - */ -export interface NewActivitySettings { - types: string[] - platforms: string[] - keywords: string[] - teamMemberActivities: boolean -} - -/** - * Settings for new member trigger based automations - */ -export interface NewMemberSettings { - platforms: string[] -} - -export interface HubspotSettings { - contactList: string - // eslint-disable-next-line @typescript-eslint/no-explicit-any - filter: any -} - -/** - * Union type to contain all different types of settings - */ -export type AutomationSettings = - | WebhookSettings - | NewActivitySettings - | NewMemberSettings - | HubspotSettings - -export interface IAutomationData { - id: string - name: string - type: AutomationType - tenantId: string - trigger: AutomationTrigger | AutomationSyncTrigger - settings: AutomationSettings - state: AutomationState - createdAt: string - lastExecutionAt: string | null - lastExecutionState: AutomationExecutionState | null - lastExecutionError: unknown | null -} - -export interface IAutomationExecution { - id?: string - automationId: string - type: string - tenantId: string - trigger: string - state: string - error: string - executedAt?: string - eventId?: string - // eslint-disable-next-line @typescript-eslint/no-explicit-any - payload: any -} diff --git a/services/libs/types/src/enums/organizations.ts b/services/libs/types/src/enums/organizations.ts index b7fbed720d..ef73b84f31 100644 --- a/services/libs/types/src/enums/organizations.ts +++ b/services/libs/types/src/enums/organizations.ts @@ -10,7 +10,6 @@ export enum OrganizationSource { ENRICHMENT_PROGAI = 'enrichment-progai', ENRICHMENT_CLEARBIT = 'enrichment-clearbit', ENRICHMENT_CRUSTDATA = 'enrichment-crustdata', - HUBSPOT = 'hubspot', GITHUB = 'github', UI = 'ui', } diff --git a/services/libs/types/src/enums/platforms.ts b/services/libs/types/src/enums/platforms.ts index 8f1b44487d..b2eac7f5c0 100644 --- a/services/libs/types/src/enums/platforms.ts +++ b/services/libs/types/src/enums/platforms.ts @@ -19,7 +19,6 @@ export enum PlatformType { DISCOURSE = 'discourse', GIT = 'git', CRUNCHBASE = 'crunchbase', - HUBSPOT = 'hubspot', GROUPSIO = 'groupsio', CONFLUENCE = 'confluence', GERRIT = 'gerrit', @@ -48,7 +47,6 @@ export enum IntegrationType { STACKOVERFLOW = 'stackoverflow', DISCOURSE = 'discourse', GIT = 'git', - HUBSPOT = 'hubspot', } export const integrationLabel: Record = { @@ -65,7 +63,6 @@ export const integrationLabel: Record = { [IntegrationType.STACKOVERFLOW]: 'Stack Overflow', [IntegrationType.DISCOURSE]: 'Discourse', [IntegrationType.GIT]: 'Git', - [IntegrationType.HUBSPOT]: 'HubSpot', } // Backup url from username if profile url not present in member.attributes.url @@ -84,5 +81,4 @@ export const integrationProfileUrl: Record `https://stackoverflow.com/users/${username}`, [IntegrationType.DISCOURSE]: () => null, [IntegrationType.GIT]: () => null, - [IntegrationType.HUBSPOT]: () => null, } diff --git a/services/libs/types/src/enums/temporal.ts b/services/libs/types/src/enums/temporal.ts index 464a3dafa2..836ae29477 100644 --- a/services/libs/types/src/enums/temporal.ts +++ b/services/libs/types/src/enums/temporal.ts @@ -1,7 +1,4 @@ export enum TemporalWorkflowId { - NEW_ACTIVITY_AUTOMATION = 'new-activity-automation', - NEW_MEMBER_AUTOMATION = 'new-member-automation', - EMAIL_WEEKLY_ANALYTICS = 'email-weekly-analytics', EMAIL_EAGLEEYE_DIGEST = 'email-eagleeye-digest', diff --git a/services/libs/types/src/index.ts b/services/libs/types/src/index.ts index 3a15bcd8fc..2570828f43 100644 --- a/services/libs/types/src/index.ts +++ b/services/libs/types/src/index.ts @@ -5,7 +5,6 @@ export * from './queue/integration_run_worker' export * from './queue/integration_stream_worker' export * from './queue/data_sink_worker' export * from './queue/search_sync_worker' -export * from './queue/integration_sync_worker' export * from './dashboard' @@ -33,8 +32,6 @@ export * from './tags' export * from './attributes' -export * from './automations' - export * from './query' export * from './sync' diff --git a/services/libs/types/src/queue/integration_sync_worker/index.ts b/services/libs/types/src/queue/integration_sync_worker/index.ts deleted file mode 100644 index 42d7b4791e..0000000000 --- a/services/libs/types/src/queue/integration_sync_worker/index.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { AutomationSyncTrigger } from '../../automations' - -export enum IntegrationSyncWorkerQueueMessageType { - SYNC_ALL_MARKED_MEMBERS = 'sync_all_marked_members', - SYNC_MEMBER = 'sync_member', - SYNC_ALL_MARKED_ORGANIZATIONS = 'sync_all_marked_organizations', - SYNC_ORGANIZATION = 'sync_organization', - ONBOARD_AUTOMATION = 'onboard_automation', -} - -export interface IIntegrationSyncWorkerEmitter { - triggerSyncMarkedMembers(tenantId: string, integrationId: string): Promise - - triggerSyncMember( - tenantId: string, - integrationId: string, - memberId: string, - syncRemoteId: string, - ): Promise - - triggerOnboardAutomation( - tenantId: string, - integrationId: string, - automationId: string, - automationTrigger: AutomationSyncTrigger, - ): Promise - - triggerSyncMarkedOrganizations(tenantId: string, integrationId: string): Promise - - triggerSyncOrganization( - tenantId: string, - integrationId: string, - organizationId: string, - syncRemoteId: string, - ): Promise -} diff --git a/services/libs/types/src/temporal/automations.ts b/services/libs/types/src/temporal/automations.ts deleted file mode 100644 index 1264ca4f30..0000000000 --- a/services/libs/types/src/temporal/automations.ts +++ /dev/null @@ -1,19 +0,0 @@ -export interface IProcessNewActivityAutomationArgs { - tenantId: string - activityId: string -} - -export interface ITriggerActivityAutomationArgs { - automationId: string - activityId: string -} - -export interface IProcessNewMemberAutomationArgs { - tenantId: string - memberId: string -} - -export interface ITriggerMemberAutomationArgs { - automationId: string - memberId: string -} diff --git a/services/libs/types/src/temporal/index.ts b/services/libs/types/src/temporal/index.ts index ea27f66aab..18ac0264ae 100644 --- a/services/libs/types/src/temporal/index.ts +++ b/services/libs/types/src/temporal/index.ts @@ -1,3 +1,2 @@ -export * from './automations' export * from './cache' export * from './exports' From 546aaf56409e4591245f7e75a44c0c02dbb27ca4 Mon Sep 17 00:00:00 2001 From: emlimlf <52259294+emlimlf@users.noreply.github.com> Date: Thu, 16 Jan 2025 18:10:50 +0800 Subject: [PATCH 2/3] Bugfix/cm 1959 integration loading (#2767) Signed-off-by: Efren Lim --- .../integration/components/integration-list-item.vue | 2 ++ .../modules/integration/pages/integration-list.page.vue | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/frontend/src/modules/admin/modules/integration/components/integration-list-item.vue b/frontend/src/modules/admin/modules/integration/components/integration-list-item.vue index 47b2596757..ed694956e7 100644 --- a/frontend/src/modules/admin/modules/integration/components/integration-list-item.vue +++ b/frontend/src/modules/admin/modules/integration/components/integration-list-item.vue @@ -47,6 +47,7 @@
{{ props.config.name }} integration failed to connect due to an API error.
+ + -
- +
+