OS-7535. Implement CUR2.0 data export support

## Description Implement CUR2.0 data export support ## Related issue number OS-7535 ## Checklist * [ ] The pull request title is a good summary of the changes * [ ] Unit tests for the changes exist * [ ] New and existing unit tests pass locally
hystax · Jun 3, 2024 · 7ec1a76 · 7ec1a76
1 parent 61f9a2f
commit 7ec1a76
Show file tree

Hide file tree

Showing 5 changed files with 194 additions and 262 deletions.
diff --git a/bumiworker/bumiworker/modules/recommendations/short_living_instances.py b/bumiworker/bumiworker/modules/recommendations/short_living_instances.py
@@ -1,3 +1,4 @@
+import re
 from collections import OrderedDict
 from datetime import datetime, timedelta
 
@@ -59,6 +60,13 @@ def _get_work_hrs(exp, cpu_count=None):
             return float(exp['pricing_quantity']) / cpu_count
         return 0
 
+    @staticmethod
+    def _datetime_from_value(value):
+        dt_format = '%Y-%m-%dT%H:%M:%SZ'
+        if re.match(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d+Z", value):
+            dt_format = '%Y-%m-%dT%H:%M:%S.%fZ'
+        return datetime.strptime(value, dt_format)
+
     def _get(self):
         (days_threshold, excluded_pools,
          skip_cloud_accounts) = self.get_options_values()
@@ -111,8 +119,8 @@ def _get(self):
                 if r_id in inst_to_remove:
                     continue
                 if exp.get('lineItem/UsageStartDate'):
-                    exp_start_date = datetime.strptime(
-                        exp['lineItem/UsageStartDate'], '%Y-%m-%dT%H:%M:%SZ')
+                    exp_start_date = self._datetime_from_value(
+                        exp['lineItem/UsageStartDate'])
                 else:
                     exp_start_date = exp['start_date']
                 if exp_start_date < start_date:

diff --git a/diworker/diworker/constants.py b/diworker/diworker/constants.py
@@ -1,192 +0,0 @@
-# according to https://docs.aws.amazon.com/cur/latest/userguide/data-dictionary.html
-AWS_PARQUET_CSV_MAP = {
-    'identity_line_item_id': 'identity/LineItemId',
-    'identity_time_interval': 'identity/TimeInterval',
-    'bill_invoice_id': 'bill/InvoiceId',
-    'bill_billing_entity': 'bill/BillingEntity',
-    'bill_bill_type': 'bill/BillType',
-    'bill_payer_account_id': 'bill/PayerAccountId',
-    'bill_billing_period_start_date': 'bill/BillingPeriodStartDate',
-    'bill_billing_period_end_date': 'bill/BillingPeriodEndDate',
-    'bill_invoicing_entity': 'bill/InvoicingEntity',
-    'line_item_usage_account_id': 'lineItem/UsageAccountId',
-    'line_item_line_item_type': 'lineItem/LineItemType',
-    'line_item_usage_start_date': 'lineItem/UsageStartDate',
-    'line_item_usage_end_date': 'lineItem/UsageEndDate',
-    'line_item_product_code': 'lineItem/ProductCode',
-    'line_item_usage_type': 'lineItem/UsageType',
-    'line_item_operation': 'lineItem/Operation',
-    'line_item_availability_zone': 'lineItem/AvailabilityZone',
-    'line_item_resource_id': 'lineItem/ResourceId',
-    'line_item_usage_amount': 'lineItem/UsageAmount',
-    'line_item_net_unblended_cost': 'lineItem/NetUnblendedCost',
-    'line_item_normalization_factor': 'lineItem/NormalizationFactor',
-    'line_item_normalized_usage_amount': 'lineItem/NormalizedUsageAmount',
-    'line_item_currency_code': 'lineItem/CurrencyCode',
-    'line_item_unblended_rate': 'lineItem/UnblendedRate',
-    'line_item_unblended_cost': 'lineItem/UnblendedCost',
-    'line_item_blended_rate': 'lineItem/BlendedRate',
-    'line_item_blended_cost': 'lineItem/BlendedCost',
-    'line_item_line_item_description': 'lineItem/LineItemDescription',
-    'line_item_tax_type': 'lineItem/TaxType',
-    'line_item_legal_entity': 'lineItem/LegalEntity',
-    'product_product_name': 'product/ProductName',
-    'product_purchase_option': 'product/PurchaseOption',
-    'product_account_assistance': 'product/accountAssistance',
-    'product_architectural_review': 'product/architecturalReview',
-    'product_architecture_support': 'product/architectureSupport',
-    'product_availability': 'product/availability',
-    'product_availability_zone': 'product/availabilityZone',
-    'product_best_practices': 'product/bestPractices',
-    'product_capacitystatus': 'product/capacitystatus',
-    'product_case_severityresponse_times': 'product/caseSeverityresponseTimes',
-    'product_classicnetworkingsupport': 'product/classicnetworkingsupport',
-    'product_clock_speed': 'product/clockSpeed',
-    'product_current_generation': 'product/currentGeneration',
-    'product_customer_service_and_communities': 'product/customerServiceAndCommunities',
-    'product_database_engine': 'product/databaseEngine',
-    'product_dedicated_ebs_throughput': 'product/dedicatedEbsThroughput',
-    'product_durability': 'product/durability',
-    'product_ecu': 'product/ecu',
-    'product_engine_code': 'product/engineCode',
-    'product_enhanced_networking_supported': 'product/enhancedNetworkingSupported',
-    'product_fee_code': 'product/feeCode',
-    'product_fee_description': 'product/feeDescription',
-    'product_from_location': 'product/fromLocation',
-    'product_from_location_type': 'product/fromLocationType',
-    'product_group': 'product/group',
-    'product_group_description': 'product/groupDescription',
-    'product_included_services': 'product/includedServices',
-    'product_insightstype': 'product/insightstype',
-    'product_instance_family': 'product/instanceFamily',
-    'product_instance_type': 'product/instanceType',
-    'product_instance_type_family': 'product/instanceTypeFamily',
-    'product_intel_avx2_available': 'product/intelAvx2Available',
-    'product_intel_avx_available': 'product/intelAvxAvailable',
-    'product_intel_turbo_available': 'product/intelTurboAvailable',
-    'product_launch_support': 'product/launchSupport',
-    'product_license_model': 'product/licenseModel',
-    'product_location': 'product/location',
-    'product_location_type': 'product/locationType',
-    'product_marketoption': 'product/marketoption',
-    'product_max_iops_burst_performance': 'product/maxIopsBurstPerformance',
-    'product_max_iopsvolume': 'product/maxIopsvolume',
-    'product_max_throughputvolume': 'product/maxThroughputvolume',
-    'product_max_volume_size': 'product/maxVolumeSize',
-    'product_memory': 'product/memory',
-    'product_network_performance': 'product/networkPerformance',
-    'product_normalization_size_factor': 'product/normalizationSizeFactor',
-    'product_operating_system': 'product/operatingSystem',
-    'product_operation': 'product/operation',
-    'product_operations_support': 'product/operationsSupport',
-    'product_physical_processor': 'product/physicalProcessor',
-    'product_platousagetype': 'product/platousagetype',
-    'product_platovolumetype': 'product/platovolumetype',
-    'product_pre_installed_sw': 'product/preInstalledSw',
-    'product_proactive_guidance': 'product/proactiveGuidance',
-    'product_processor_architecture': 'product/processorArchitecture',
-    'product_processor_features': 'product/processorFeatures',
-    'product_product_family': 'product/productFamily',
-    'product_programmatic_case_management': 'product/programmaticCaseManagement',
-    'product_provisioned': 'product/provisioned',
-    'product_purchaseterm': 'product/purchaseterm',
-    'product_region': 'product/region',
-    'product_servicecode': 'product/servicecode',
-    'product_servicename': 'product/servicename',
-    'product_sku': 'product/sku',
-    'product_storage': 'product/storage',
-    'product_storage_class': 'product/storageClass',
-    'product_storage_media': 'product/storageMedia',
-    'product_technical_support': 'product/technicalSupport',
-    'product_tenancy': 'product/tenancy',
-    'product_thirdparty_software_support': 'product/thirdpartySoftwareSupport',
-    'product_tiertype': 'product/tiertype',
-    'product_to_location': 'product/toLocation',
-    'product_to_location_type': 'product/toLocationType',
-    'product_training': 'product/training',
-    'product_transfer_type': 'product/transferType',
-    'product_usagetype': 'product/usagetype',
-    'product_vcpu': 'product/vcpu',
-    'product_version': 'product/version',
-    'product_volume_api_name': 'product/volumeApiName',
-    'product_volume_type': 'product/volumeType',
-    'product_vpcnetworkingsupport': 'product/vpcnetworkingsupport',
-    'product_who_can_open_cases': 'product/whoCanOpenCases',
-    'pricing_lease_contract_length': 'pricing/LeaseContractLength',
-    'pricing_offering_class': 'pricing/OfferingClass',
-    'pricing_purchase_option': 'pricing/PurchaseOption',
-    'pricing_rate_code': 'pricing/RateCode',
-    'pricing_rate_id': 'pricing/RateId',
-    'pricing_currency': 'pricing/currency',
-    'pricing_public_on_demand_cost': 'pricing/publicOnDemandCost',
-    'pricing_public_on_demand_rate': 'pricing/publicOnDemandRate',
-    'pricing_term': 'pricing/term',
-    'pricing_unit': 'pricing/unit',
-    'reservation_amortized_upfront_cost_for_usage':
-        'reservation/AmortizedUpfrontCostForUsage',
-    'reservation_amortized_upfront_fee_for_billing_period':
-        'reservation/AmortizedUpfrontFeeForBillingPeriod',
-    'reservation_effective_cost': 'reservation/EffectiveCost',
-    'reservation_end_time': 'reservation/EndTime',
-    'reservation_modification_status': 'reservation/ModificationStatus',
-    'reservation_net_amortized_upfront_cost_for_usage':
-        'reservation/NetAmortizedUpfrontCostForUsage',
-    'reservation_net_amortized_upfront_fee_for_billing_period':
-        'reservation/NetAmortizedUpfrontFeeForBillingPeriod',
-    'reservation_net_recurring_fee_for_usage':
-        'reservation/NetRecurringFeeForUsage',
-    'reservation_net_unused_recurring_fee': 'reservation/NetUnusedRecurringFee',
-    'reservation_net_upfront_value': 'reservation/NetUpfrontValue',
-    'reservation_net_unused_amortized_upfront_fee_for_billing_period':
-        'reservation/NetUnusedAmortizedUpfrontFeeForBillingPeriod',
-    'reservation_net_effective_cost': 'reservation/NetEffectiveCost',
-    'reservation_normalized_units_per_reservation':
-        'reservation/NormalizedUnitsPerReservation',
-    'reservation_number_of_reservations': 'reservation/NumberOfReservations',
-    'reservation_recurring_fee_for_usage': 'reservation/RecurringFeeForUsage',
-    'reservation_reservation_a_r_n': 'reservation/ReservationARN',
-    'reservation_start_time': 'reservation/StartTime',
-    'reservation_subscription_id': 'reservation/SubscriptionId',
-    'reservation_total_reserved_normalized_units':
-        'reservation/TotalReservedNormalizedUnits',
-    'reservation_total_reserved_units': 'reservation/TotalReservedUnits',
-    'reservation_units_per_reservation': 'reservation/UnitsPerReservation',
-    'reservation_unused_amortized_upfront_fee_for_billing_period':
-        'reservation/UnusedAmortizedUpfrontFeeForBillingPeriod',
-    'reservation_unused_normalized_unit_quantity':
-        'reservation/UnusedNormalizedUnitQuantity',
-    'reservation_unused_quantity': 'reservation/UnusedQuantity',
-    'reservation_unused_recurring_fee': 'reservation/UnusedRecurringFee',
-    'reservation_upfront_value': 'reservation/UpfrontValue',
-    'savings_plan_total_commitment_to_date':
-        'savingsPlan/TotalCommitmentToDate',
-    'savings_plan_savings_plan_a_r_n': 'savingsPlan/SavingsPlanARN',
-    'savings_plan_savings_plan_rate': 'savingsPlan/SavingsPlanRate',
-    'savings_plan_used_commitment': 'savingsPlan/UsedCommitment',
-    'savings_plan_savings_plan_effective_cost':
-        'savingsPlan/SavingsPlanEffectiveCost',
-    'savings_plan_amortized_upfront_commitment_for_billing_period':
-        'savingsPlan/AmortizedUpfrontCommitmentForBillingPeriod',
-    'savings_plan_recurring_commitment_for_billing_period':
-        'savingsPlan/RecurringCommitmentForBillingPeriod',
-    'savings_plan_start_time': 'savingsPlan/StartTime',
-    'savings_plan_end_time': 'savingsPlan/EndTime',
-    'savings_plan_offering_type': 'savingsPlan/OfferingType',
-    'savings_plan_payment_option': 'savingsPlan/PaymentOption',
-    'savings_plan_purchase_term': 'savingsPlan/PurchaseTerm',
-    'savings_plan_region': 'savingsPlan/Region',
-    'savings_plan_net_savings_plan_effective_cost':
-        'savingsPlan/NetSavingsPlanEffectiveCost',
-    'savings_plan_net_amortized_upfront_commitment_for_billing_period':
-        'savingsPlan/NetAmortizedUpfrontCommitmentForBillingPeriod',
-    'savings_plan_net_recurring_commitment_for_billing_period':
-        'savingsPlan/NetRecurringCommitmentForBillingPeriod',
-    'resource_tags_user_name': 'resourceTags/user:Name',
-    'resource_tags_aws_cloudformation_stack_id':
-        'resourceTags/aws:cloudformation:stack-id',
-    'resource_tags_aws_cloudformation_logical_id':
-        'resourceTags/aws:cloudformation:logical-id',
-    'resource_tags_aws_cloudformation_stack_name':
-        'resourceTags/aws:cloudformation:stack-name',
-    'resource_tags_aws_created_by': 'resourceTags/aws:createdBy',
-}