Pull request update/241003

diworker/diworker/importers/aws.py

@@ -85,6 +85,10 @@ def __init__(self, *args, **kwargs):
     def use_edp_discount(self):
         return self.cloud_acc['config'].get('use_edp_discount', False)
 
+    @staticmethod
+    def short_resource_id(resource_id):
+        return resource_id[resource_id.find('/') + 1:]
+
     @staticmethod
     def unzip_report(report_path, dest_dir):
         LOG.info('Extracting %s as zip archive to %s', report_path, dest_dir)
@@ -412,8 +416,8 @@ def load_csv_report(self, report_path, account_id_ca_id_map,
                 row['_rec_n'] = record_number
                 row['cloud_account_id'] = cloud_account_id
                 if 'lineItem/ResourceId' in row:
-                    r_id = row['lineItem/ResourceId']
-                    row['resource_id'] = r_id[r_id.find('/') + 1:]
+                    row['resource_id'] = self.short_resource_id(
+                        row['lineItem/ResourceId'])
                 start_date = self._datetime_from_expense(
                     row, 'lineItem/UsageStartDate').replace(
                     hour=0, minute=0, second=0)
@@ -475,7 +479,8 @@ def load_parquet_report(self, report_path, account_id_ca_id_map,
                         chunk[expense_num]['cloud_account_id'] = cloud_account_id
                         self.detected_cloud_accounts.add(cloud_account_id)
                     elif field_name == 'lineItem/ResourceId' and value:
-                        chunk[expense_num]['resource_id'] = value[value.find('/') + 1:]
+                        chunk[expense_num][
+                            'resource_id'] = self.short_resource_id(value)
                     elif field_name == 'lineItem/UsageStartDate':
                         start_date = self._datetime_from_value(value).replace(
                             hour=0, minute=0, second=0)
@@ -775,7 +780,9 @@ def get_resource_info_map(self, chunk):
             # are related to SP resource
             for exp in expenses:
                 if exp['lineItem/LineItemType'] == 'SavingsPlanCoveredUsage':
-                    sp_covered_chunk[exp['lineItem/ResourceId']].append(exp)
+                    resource_id = self.short_resource_id(
+                        exp['lineItem/ResourceId'])
+                    sp_covered_chunk[resource_id].append(exp)
 
         # use predefined resource type to avoid detecting SP fields
         info_map = {
@@ -887,9 +894,9 @@ def compose_resource_id(self, expense):
         if item_type in IGNORE_EXPENSE_TYPES:
             return
         elif 'SavingsPlan' in item_type and sp_id:
-            return sp_id[sp_id.find('/') + 1:]
+            return self.short_resource_id(sp_id)
         elif ri_id:
-            return ri_id[ri_id.find('/') + 1:]
+            return self.short_resource_id(ri_id)
         parts = self.ITEM_TYPE_ID_FIELDS.get(item_type)
         if parts:
             resource_id = ' '.join([expense.get(k)

diworker/diworker/importers/environment.py

@@ -126,6 +126,20 @@ def get_resource_data(self, r_id, info,
             **self._get_fake_cad_extras(info)
         }
 
+    def get_resource_ids(self, cloud_account_id, period_start):
+        all_resource_ids = super().get_resource_ids(cloud_account_id,
+                                                    period_start)
+        not_deleted_resource_ids = []
+        for i in range(0, len(all_resource_ids), CHUNK_SIZE):
+            chunk = all_resource_ids[i:i+CHUNK_SIZE]
+            chunk_res_ids = [
+                x['cloud_resource_id'] for x in self.mongo_resources.find({
+                    'cloud_account_id': self.cloud_acc_id,
+                    'cloud_resource_id': {'$in': chunk},
+                    'deleted_at': 0}, {'cloud_resource_id': 1})]
+            not_deleted_resource_ids.extend(chunk_res_ids)
+        return not_deleted_resource_ids
+
     def generate_clean_records(self, regeneration=False):
         if regeneration:
             self.period_start = None

diworker/diworker/migrations/2024092712400000_remove_extra_sp_covered_resources.py

@@ -0,0 +1,122 @@
+import logging
+from pymongo import DeleteOne, UpdateOne
+from optscale_client.rest_api_client.client_v2 import Client as RestClient
+from diworker.diworker.migrations.base import BaseMigration
+"""
+Remove duplicated resources created by SP covered resources and change long
+cloud_resource_id like 'arn:aws:service:account_id:type/short_id' to short_id
+"""
+CHUNK_SIZE = 1000
+LOG = logging.getLogger(__name__)
+
+
+class Migration(BaseMigration):
+    @property
+    def mongo_raw(self):
+        return self.db.raw_expenses
+
+    @property
+    def mongo_resources(self):
+        return self.db.resources
+
+    @property
+    def rest_cl(self):
+        if self._rest_cl is None:
+            self._rest_cl = RestClient(
+                url=self.config_cl.restapi_url(),
+                secret=self.config_cl.cluster_secret())
+        return self._rest_cl
+
+    def get_cloud_accs(self):
+        cloud_accounts_ids = set()
+        _, organizations = self.rest_cl.organization_list({
+            'with_connected_accounts': True, 'is_demo': False})
+        for org in organizations['organizations']:
+            _, accounts = self.rest_cl.cloud_account_list(
+                org['id'], type='aws_cnr')
+            for cloud_account in accounts['cloud_accounts']:
+                if cloud_account['auto_import']:
+                    cloud_accounts_ids.add(cloud_account['id'])
+        return cloud_accounts_ids
+
+    @staticmethod
+    def short_resource_id(resource_id):
+        return resource_id[resource_id.find('/') + 1:]
+
+    def upgrade(self):
+        cloud_accs = self.get_cloud_accs()
+        for i, cloud_acc_id in enumerate(list(cloud_accs)):
+            LOG.info('Starting processing for cloud account %s (%s/%s)' % (
+                cloud_acc_id, i + 1, len(cloud_accs)))
+            sps = self.mongo_resources.find({
+                'cloud_account_id': cloud_acc_id,
+                'resource_type': 'Savings Plan',
+                'active': {'$exists': False}}, {'cloud_resource_id': 1})
+            sps_ids = [x['cloud_resource_id'] for x in sps]
+            if not sps_ids:
+                continue
+            LOG.info('Found %s Savings Plan resources', len(sps_ids))
+            long_res_ids = list(
+                set(x['lineItem/ResourceId'] for x in self.mongo_raw.find({
+                    'cloud_account_id': cloud_acc_id,
+                    'resource_id': {'$in': sps_ids},
+                    'box_usage': True,
+                    'lineItem/LineItemType': 'SavingsPlanCoveredUsage',
+                    'lineItem/ResourceId': {'$regex': 'arn:aws(.*)'}
+                }, {'lineItem/ResourceId': 1})))
+            long_short_map = {}
+            if long_res_ids:
+                # collect ids map and filter out resource ids that have
+                # correct ids
+                for long_res_id in long_res_ids:
+                    short_res_id = self.short_resource_id(long_res_id)
+                    if short_res_id != long_res_id:
+                        long_short_map[long_res_id] = short_res_id
+                long_res_ids = list(long_short_map)
+                if not long_res_ids:
+                    continue
+
+                actions = []
+                updated = 0
+                deleted = 0
+                for j in range(0, len(long_res_ids), CHUNK_SIZE):
+                    if len(actions) >= CHUNK_SIZE:
+                        self.mongo_resources.bulk_write(actions)
+                        actions = []
+                    long_chunk = long_res_ids[j:j + CHUNK_SIZE]
+                    short_chunk = [long_short_map[x] for x in long_chunk]
+                    db_short_resources = self.mongo_resources.find(
+                        {'cloud_account_id': cloud_acc_id,
+                         'cloud_resource_id': {'$in': short_chunk}},
+                        {'cloud_resource_id': 1}
+                    )
+                    db_short_res_ids = [x['cloud_resource_id']
+                                        for x in db_short_resources]
+                    # resources created by SP without expenses
+                    db_long_resources = self.mongo_resources.find({
+                        'cloud_account_id': cloud_acc_id,
+                        'cloud_resource_id': {'$in': long_chunk},
+                        'resource_type': 'Instance',
+                        'total_cost': {'$exists': False}
+                    })
+                    for resource in db_long_resources:
+                        long_id = resource['cloud_resource_id']
+                        short_id = long_short_map[long_id]
+                        if short_id in db_short_res_ids:
+                            # delete resource duplicate
+                            actions.append(DeleteOne({'_id': resource['_id']}))
+                            deleted = deleted + 1
+                        else:
+                            # update resource id to short id
+                            actions.append(UpdateOne(
+                                filter={'_id': resource['_id']},
+                                update={
+                                    '$set': {'cloud_resource_id': short_id}}))
+                            updated = updated + 1
+                if actions:
+                    self.mongo_resources.bulk_write(actions)
+                LOG.info('Deleted %s resources, updated %s resources', deleted,
+                         updated)
+
+    def downgrade(self):
+        pass

ngui/ui/package.json

@@ -18,12 +18,12 @@
     "@mui/icons-material": "^5.15.0",
     "@mui/material": "^5.15.0",
     "@mui/system": "^5.15.0",
-    "@nivo/bar": "^0.84.0",
-    "@nivo/colors": "^0.84.0",
-    "@nivo/core": "^0.84.0",
-    "@nivo/line": "^0.84.0",
-    "@nivo/pie": "^0.84.0",
-    "@nivo/scales": "^0.84.0",
+    "@nivo/bar": "^0.87.0",
+    "@nivo/colors": "^0.87.0",
+    "@nivo/core": "^0.87.0",
+    "@nivo/line": "^0.87.0",
+    "@nivo/pie": "^0.87.0",
+    "@nivo/scales": "^0.87.0",
     "@tanstack/react-table": "^8.7.6",
     "@types/react": "^18.2.45",
     "@types/react-dom": "^18.2.18",