Pull request update/240927

be3855a OS-7863. Update rollup
68428d2 OS-7806. Added use_edp_discount flag for aws cloud account
fa1f43c OS-7814. cryptography==43.0.1 to fix dependabot issue

auth/requirements.txt

@@ -8,7 +8,7 @@ apispec==6.3.1
 google-auth==1.23.0
 requests==2.32.3
 PyJWT>=2.4.0
-cryptography==42.0.4
+cryptography==43.0.1
 pyyaml==6.0.1
 zcrmsdk==3.1.0
 # OptScale packages

diworker/diworker/importers/aws.py

@@ -12,6 +12,7 @@
 import json
 from collections import defaultdict, OrderedDict
 from datetime import datetime, timedelta, timezone
+from functools import cached_property
 
 from diworker.diworker.importers.base import CSVBaseReportImporter
 
@@ -80,6 +81,10 @@ def __init__(self, *args, **kwargs):
         self.import_start_ts = int(datetime.utcnow().timestamp())
         self.current_billing_period = None
 
+    @cached_property
+    def use_edp_discount(self):
+        return self.cloud_acc['config'].get('use_edp_discount', False)
+
     @staticmethod
     def unzip_report(report_path, dest_dir):
         LOG.info('Extracting %s as zip archive to %s', report_path, dest_dir)
@@ -168,6 +173,7 @@ def get_unique_field_list(include_date=True):
 
     def get_update_fields(self):
         return [
+            'discount/EdpDiscount',
             'lineItem/BlendedRate',
             'lineItem/BlendedCost',
             'lineItem/UnblendedRate',
@@ -419,6 +425,8 @@ def load_csv_report(self, report_path, account_id_ca_id_map,
                     row, 'lineItem/UsageEndDate')
                 row['cost'] = float(row['lineItem/BlendedCost']) if row[
                     'lineItem/BlendedCost'] else 0
+                if self.use_edp_discount:
+                    row['cost'] += float(row.get('discount/EdpDiscount') or 0)
                 if self._is_flavor_usage(row):
                     row['box_usage'] = True
                 for k, v in row.copy().items():
@@ -442,7 +450,7 @@ def load_parquet_report(self, report_path, account_id_ca_id_map,
         for i in range(0, dataframe.shape[0], CHUNK_SIZE):
             expense_chunk = self._extract_nested_objects(
                 dataframe.iloc[i:i + CHUNK_SIZE, :].to_dict(), parquet=True)
-            chunk = [{} for _ in range(0, CHUNK_SIZE)]
+            chunk = [{'cost': 0} for _ in range(0, CHUNK_SIZE)]
             skipped_rows = set()
             for field_name, values_dict in expense_chunk.items():
                 for n, value in values_dict.items():
@@ -476,6 +484,9 @@ def load_parquet_report(self, report_path, account_id_ca_id_map,
                             value)
                     elif field_name == 'lineItem/BlendedCost':
                         chunk[expense_num]['cost'] = float(value) if value else 0
+                    elif (self.use_edp_discount and
+                          field_name == 'discount/EdpDiscount' and value):
+                        chunk[expense_num]['cost'] += float(value)
                     elif field_name == 'lineItem/UsageType':
                         if 'BoxUsage' in value:
                             chunk[expense_num]['box_usage'] = True

optscale-deploy/ansible/roles/common/files/requirements.txt

@@ -1,4 +1,4 @@
 docker==3.5.1
 kubernetes==10.0.1
 pyyaml==6.0.1
-cryptography==42.0.4
+cryptography==43.0.1

rest_api/requirements.txt

@@ -2,7 +2,7 @@ SQLAlchemy==1.3.24
 mysql-connector-python==8.1.0
 tornado==6.4.1
 alembic==1.13.1
-cryptography==42.0.4
+cryptography==43.0.1
 apispec==6.3.1
 pymongo==4.6.3
 boto3==1.34.7

risp/risp_worker/requirements.txt

@@ -1,4 +1,4 @@
-cryptography==42.0.4
+cryptography==43.0.1
 pymongo==4.6.3
 boto3==1.34.7
 kombu==5.3.4

tools/cloud_adapter/clouds/aws.py

@@ -104,7 +104,8 @@ class Aws(S3CloudMixin):
         CloudParameter(name='linked', type=bool, required=False),
 
         # Service parameters
-        CloudParameter(name='cur_version', type=int, required=False)
+        CloudParameter(name='cur_version', type=int, required=False),
+        CloudParameter(name='use_edp_discount', type=bool, required=False)
     ]
     DEFAULT_S3_REGION_NAME = 'eu-central-1'
     SUPPORTS_REPORT_UPLOAD = True