From f721f6384c0d916240088440ac62fea828bb887f Mon Sep 17 00:00:00 2001 From: tinogis Date: Thu, 13 Jul 2023 16:59:53 +0200 Subject: [PATCH 01/12] PowerProfileQh improvements & get_hourly_profile --- powerprofile/powerprofile.py | 75 +++++++++++++++++++++--------------- 1 file changed, 45 insertions(+), 30 deletions(-) diff --git a/powerprofile/powerprofile.py b/powerprofile/powerprofile.py index 7d3f35c..3efc1c3 100644 --- a/powerprofile/powerprofile.py +++ b/powerprofile/powerprofile.py @@ -24,6 +24,8 @@ class PowerProfile(): + SAMPLING_INTERVAL = 3600 + def __init__(self, datetime_field='timestamp', data_fields=DEFAULT_DATA_FIELDS): self.start = None @@ -87,11 +89,22 @@ def dump(self): return data + @property + def samples(self): + if self.SAMPLING_INTERVAL == 900: + return self.quart_hours + else: + return self.hours + @property def hours(self): - return self.curve.count()[self.datetime_field] + return int(self.curve.count()[self.datetime_field] / (self.SAMPLING_INTERVAL / 3600)) - def is_complete(self): + @property + def quart_hours(self): + return int(self.curve.count()[self.datetime_field] / (self.SAMPLING_INTERVAL / 900)) + + def is_complete_counter(self, counter): ''' Checks completeness of curve ''' start = self.start if self.start.tzinfo is None or self.start.tzinfo.utcoffset(self.start) is None: @@ -100,11 +113,11 @@ def is_complete(self): if self.end.tzinfo is None or self.end.tzinfo.utcoffset(self.end) is None: end = TIMEZONE.localize(self.end) - hours = ((end - start).total_seconds() + 3600) / 3600 - if self.hours != hours: + samples = ((end - start).total_seconds() + self.SAMPLING_INTERVAL) / self.SAMPLING_INTERVAL + if counter != samples: ids = set(self.curve[self.datetime_field]) dt = start - df_hours = set([TIMEZONE.normalize(dt + timedelta(hours=x)) for x in range(0, int(hours))]) + df_hours = set([TIMEZONE.normalize(dt + timedelta(seconds=x * self.SAMPLING_INTERVAL)) for x in range(0, int(samples))]) not_found = sorted(list(df_hours - ids)) if len(not_found): first_not_found = not_found[0] @@ -113,6 +126,9 @@ def is_complete(self): return False, first_not_found return True, None + def is_complete(self): + return self.is_complete_counter(self.hours) + def is_fixed(self, fields=['cch_fact', 'valid']): """ Given a list of fields, check all values are True in every register @@ -128,15 +144,18 @@ def is_fixed(self, fields=['cch_fact', 'valid']): raise PowerProfileMissingField(field) return True - def has_duplicates(self): + def has_duplicates_counter(self, counter): ''' Checks for duplicated hours''' uniques = len(self.curve[self.datetime_field].unique()) - if uniques != self.hours: + if uniques != counter: ids = self.curve[self.datetime_field] first_occurrence = self.curve[ids.isin(ids[ids.duplicated()])][self.datetime_field].min() return True, first_occurrence return False, None + def has_duplicates(self): + return self.has_duplicates_counter(self.hours) + def is_positive(self, fields=DEFAULT_DATA_FIELDS): """ Checks if the curve does not have any negative value @@ -541,31 +560,27 @@ def convert_numpydate_to_datetime(date, to_string=False): class PowerProfileQh(PowerProfile): - @property - def hours(self): - return self.curve.count()[self.datetime_field] / 4.0 - - @property - def quart_hours(self): - return self.curve.count()[self.datetime_field] + SAMPLING_INTERVAL = 900 def has_duplicates(self): - ''' Checks for duplicated hours''' - uniques = len(self.curve[self.datetime_field].unique()) - if uniques != self.quart_hours: - return True - return False + return self.has_duplicates_counter(self.quart_hours) def is_complete(self): - ''' Checks completeness of curve ''' - start = self.start - if self.start.tzinfo is None or self.start.tzinfo.utcoffset(self.start) is None: - start = TIMEZONE.localize(self.start) - end = self.end - if self.end.tzinfo is None or self.end.tzinfo.utcoffset(self.end) is None: - end = TIMEZONE.localize(self.end) + return self.is_complete_counter(self.quart_hours) - quart_hours = (((end - start)).total_seconds() + 900) / 900 - if self.quart_hours != quart_hours: - return False - return True + def get_hourly_profile(self): + ''' + Returns a Powerprofile aggregating quarter-hour curve by hour + :return: + New Powerprofile + ''' + + new_curve = PowerProfile() + + new_curve.curve = self.curve.resample('1H', closed='right', label='right', on=self.datetime_field).sum() + new_curve.curve.sort_values(by=new_curve.datetime_field, inplace=True) + new_curve.curve = new_curve.curve.reset_index() + new_curve.start = new_curve.curve[new_curve.datetime_field].min() + new_curve.end = new_curve.curve[new_curve.datetime_field].max() + + return new_curve From e6370195ba91f3e64eee945f050421c169b7d3c8 Mon Sep 17 00:00:00 2001 From: tinogis Date: Thu, 13 Jul 2023 17:02:08 +0200 Subject: [PATCH 02/12] Class QH compat usin __class__ on new instance --- powerprofile/powerprofile.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/powerprofile/powerprofile.py b/powerprofile/powerprofile.py index 3efc1c3..95c6de6 100644 --- a/powerprofile/powerprofile.py +++ b/powerprofile/powerprofile.py @@ -194,7 +194,7 @@ def __getitem__(self, item): # test bounds self.curve.iloc[item.start] self.curve.iloc[item.stop] - powpro = PowerProfile() + powpro = self.__class__() powpro.curve = res powpro.start = res.iloc[0][self.datetime_field] powpro.end = res.iloc[-1][self.datetime_field] @@ -413,7 +413,7 @@ def copy(self): Returns an identical copy of the same profile :return: PowerProfile Object """ - new = PowerProfile(self.datetime_field) + new = self.__class__(self.datetime_field) new.start = self.start new.end = self.end new.curve = copy.copy(self.curve) @@ -505,10 +505,10 @@ def get_complete_daily_subcurve(self): if last_hour >= self.start: data = self.curve[self.curve[self.datetime_field] <= last_hour] data = data.to_dict('records') - res = PowerProfile() + res = self.__class__() res.load(data, datetime_field=self.datetime_field) else: - res = PowerProfile() + res = self.__class__() return res From bd38123be9205452ddfa05f6049a1e40f43dcd85 Mon Sep 17 00:00:00 2001 From: tinogis Date: Thu, 13 Jul 2023 17:03:37 +0200 Subject: [PATCH 03/12] Fix dragger. Uses magn for units if available --- powerprofile/powerprofile.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/powerprofile/powerprofile.py b/powerprofile/powerprofile.py index 95c6de6..dd45a4e 100644 --- a/powerprofile/powerprofile.py +++ b/powerprofile/powerprofile.py @@ -286,11 +286,11 @@ def drag(self, magns, drag_key=None): draggers = Dragger() # Dragg field is specified and exists in curve if drag_key is not None and drag_key in self.curve: - self.curve[magn] = self.curve.apply(lambda row: draggers.drag(round(row[magn] / 1000, 6), + self.curve[magn] = self.curve.apply(lambda row: draggers.drag(round(row[magn] / (1000 / row.get('magn', 1)), 6), key=row[drag_key]), axis=1) else: - self.curve[magn] = self.curve.apply(lambda row: draggers.drag(round(row[magn] / 1000, 6)), axis=1) - self.curve[magn] = self.curve.apply(lambda row: row[magn] * 1000, axis=1) + self.curve[magn] = self.curve.apply(lambda row: draggers.drag(round(row[magn] / (1000 / row.get('magn', 1)), 6)), axis=1) + self.curve[magn] = self.curve.apply(lambda row: row[magn] * (1000 / row.get('magn', 1)), axis=1) def Min(self, magn1='ae', magn2='ai', sufix='ac'): """ From 35bcd785a044957e967c77fef64c1a63a98a47e9 Mon Sep 17 00:00:00 2001 From: tinogis Date: Thu, 13 Jul 2023 17:04:38 +0200 Subject: [PATCH 04/12] NEW fill. Creates a profile with default values --- powerprofile/powerprofile.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/powerprofile/powerprofile.py b/powerprofile/powerprofile.py index dd45a4e..25b2044 100644 --- a/powerprofile/powerprofile.py +++ b/powerprofile/powerprofile.py @@ -76,6 +76,37 @@ def load(self, data, start=None, end=None, datetime_field=None, data_fields=None auto_data_fields.append(field) self.data_fields = auto_data_fields + def fill(self, default_data, start, end): + ''' + Fills curve with default data + :param data: dict with field and default value, ie: {'ai': 0, 'ae': 0, 'cch_bruta': False} + ''' + if not isinstance(default_data, dict): + raise TypeError("ERROR: [default_data] must be a dict") + + if not isinstance(start, datetime) or not start.tzinfo: + raise TypeError("ERROR: [start] must be a localized datetime") + + if not isinstance(end, datetime) or not end.tzinfo: + raise TypeError("ERROR: [end] must be a localized datetime") + + self.start = start + self.end = end + + data = [] + sample_counter = 0 + ts = copy.copy(self.start) + while self.end > ts: + append_data = {} + ts = TIMEZONE.normalize(self.start + timedelta(seconds=sample_counter * self.SAMPLING_INTERVAL)) + append_data[self.datetime_field] = ts + append_data.update(default_data) + data.append(append_data) + + sample_counter += 1 + + self.load(data) + def ensure_localized_dt(self, row): dt = row[self.datetime_field] if dt.tzinfo is None: From 859ed30012188940daaf3f63f30294b0b8051942 Mon Sep 17 00:00:00 2001 From: tinogis Date: Thu, 13 Jul 2023 17:06:00 +0200 Subject: [PATCH 05/12] FIX extract when setting data_fields --- powerprofile/powerprofile.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/powerprofile/powerprofile.py b/powerprofile/powerprofile.py index 25b2044..d0aab73 100644 --- a/powerprofile/powerprofile.py +++ b/powerprofile/powerprofile.py @@ -489,8 +489,10 @@ def extract(self, cols): new.curve.rename(columns=cols, inplace=True) - new_data_fields = [x for x in final_trans_cols if x != self.datetime_field] - new.data_fields = new_data_fields + final_cols = final_trans_cols[:] + + new_data_fields = [x for x in final_cols if x != self.datetime_field] + new.data_fields = new_data_fields return new From bbc1bb46cf0a3ff1c9932a33fcb9aff50c116683 Mon Sep 17 00:00:00 2001 From: tinogis Date: Thu, 13 Jul 2023 17:08:16 +0200 Subject: [PATCH 06/12] IMP check_data_fields function --- powerprofile/powerprofile.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/powerprofile/powerprofile.py b/powerprofile/powerprofile.py index d0aab73..817876d 100644 --- a/powerprofile/powerprofile.py +++ b/powerprofile/powerprofile.py @@ -339,6 +339,18 @@ def Min(self, magn1='ae', magn2='ai', sufix='ac'): self.curve[magn1 + sufix] = self.curve.apply(lambda row: min(row[magn1], row[magn2]), axis=1) # Operators + def check_data_fields(self, right): + if len(self.data_fields) != len(right.data_fields): + raise PowerProfileIncompatible('ERROR: right data fields "{}" are not the same: {}'.format( + self.data_fields, right.data_fields) + ) + for field in self.data_fields: + if field not in right.data_fields: + raise PowerProfileIncompatible('ERROR: right profile does not contains field "{}": {}'.format( + field, right.data_fields) + ) + return True + # Binary def similar(self, right, data_fields=False): """Ensures two PowerProfiles are "compatible", that is: @@ -357,15 +369,8 @@ def similar(self, right, data_fields=False): field, getattr(right, field), getattr(self, field))) if data_fields: - if len(self.data_fields) != len(right.data_fields): - raise PowerProfileIncompatible('ERROR: right data fields "{}" are not the same: {}'.format( - self.data_fields, right.data_fields) - ) - for field in self.data_fields: - if field not in right.data_fields: - raise PowerProfileIncompatible('ERROR: right profile does not contains field "{}": {}'.format( - field, right.data_fields) - ) + self.check_data_fields(right) + return True def __operate(self, right, op='mul'): From 7748a3625eca075523c28abbf941e87dc435a66a Mon Sep 17 00:00:00 2001 From: tinogis Date: Thu, 13 Jul 2023 17:09:11 +0200 Subject: [PATCH 07/12] IMP append function to add registers on a profile --- powerprofile/powerprofile.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/powerprofile/powerprofile.py b/powerprofile/powerprofile.py index 817876d..f17519a 100644 --- a/powerprofile/powerprofile.py +++ b/powerprofile/powerprofile.py @@ -443,6 +443,36 @@ def extend(self, right): return new + def append(self, new_profile): + '''Appends data to to current curve. Usefull to fill gaps or strech the profile''' + if not isinstance(new_profile, PowerProfile): + raise TypeError('ERROR append: Appended Profile must be a PowerProfile') + + #if type(self) is not type(new_profile): + if self.SAMPLING_INTERVAL != new_profile.SAMPLING_INTERVAL: + raise PowerProfileIncompatible( + "ERROR: Can't append profiles of different profile type: {} != {}".format(self.__class__, new_profile.__class__) + ) + + if self.datetime_field != new_profile.datetime_field: + raise PowerProfileIncompatible( + "ERROR: Can't append profiles of different datetime field: {} != {}".format( + self.datetime_field , new_profile.datetime_field + ) + ) + + self.check_data_fields(new_profile) + + new_curve = self.copy() + + new_curve.curve = pd.concat([new_curve.curve, new_profile.curve]) + new_curve.curve.sort_values(by=new_curve.datetime_field, inplace=True) + new_curve.curve.reset_index(inplace=True, drop=True) + new_curve.start = new_curve.curve[new_curve.datetime_field].min() + new_curve.end = new_curve.curve[new_curve.datetime_field].max() + + return new_curve + # Unary def copy(self): """ From 895a56a961f4dad5ed86cf7d5342dd1a3b019640 Mon Sep 17 00:00:00 2001 From: tinogis Date: Thu, 13 Jul 2023 17:10:19 +0200 Subject: [PATCH 08/12] TESTS --- spec/powerprofile_spec.py | 104 ++++++++++++++++++++++- spec/powerprofileqh_spec.py | 165 ++++++++++++++++++++++++++++++++++++ 2 files changed, 268 insertions(+), 1 deletion(-) create mode 100644 spec/powerprofileqh_spec.py diff --git a/spec/powerprofile_spec.py b/spec/powerprofile_spec.py index e9aee01..5f870dd 100644 --- a/spec/powerprofile_spec.py +++ b/spec/powerprofile_spec.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from expects.testing import failure from expects import * -from powerprofile.powerprofile import PowerProfile, DEFAULT_DATA_FIELDS +from powerprofile.powerprofile import PowerProfile, PowerProfileQh, DEFAULT_DATA_FIELDS from datetime import datetime, timedelta from dateutil.relativedelta import relativedelta from dateutil.parser import parse as parse_datetime @@ -173,6 +173,48 @@ def read_csv(txt): for idx, hour in powerprofile.curve.iterrows(): assert hour[powerprofile.datetime_field].tzinfo is not None + with context('fill function'): + with context('with bad data'): + with it('raises TypeError exception'): + powerprofile = PowerProfile() + + expect(lambda: powerprofile.fill(['a', 'b'], datetime(2022, 8, 1, 1, 0, 0), datetime(2022, 9, 1, 0, 0, 0))).to( + raise_error(TypeError, "ERROR: [default_data] must be a dict") + ) + + expect(lambda: powerprofile.fill( + {'ae': 1.0, "ai": 13.0}, '2020-01-31 10:00:00', datetime(2022, 9, 1, 0, 0, 0))).to( + raise_error(TypeError, "ERROR: [start] must be a localized datetime") + ) + expect(lambda: powerprofile.fill( + {'ae': 1.0, "ai": 13.0}, datetime(2022, 9, 1, 0, 0, 0), datetime(2022, 9, 1, 0, 0, 0))).to( + raise_error(TypeError, "ERROR: [start] must be a localized datetime") + ) + + expect(lambda: powerprofile.fill( + {'ae': 1.0, "ai": 13.0}, LOCAL_TZ.localize(datetime(2022, 8, 1, 1, 0, 0)), '2020-01-31 10:00:00')).to( + raise_error(TypeError, "ERROR: [end] must be a localized datetime") + ) + expect(lambda: powerprofile.fill( + {'ae': 1.0, "ai": 13.0}, LOCAL_TZ.localize(datetime(2022, 9, 1, 0, 0, 0)), datetime(2022, 9, 1, 0, 0, 0))).to( + raise_error(TypeError, "ERROR: [end] must be a localized datetime") + ) + + with context('correctly'): + with it('with correct params'): + powerprofile = PowerProfile() + + default_data = {'name': '12345678', 'cch_bruta': True, 'bc': 0x06} + start = LOCAL_TZ.localize(datetime(2022, 8, 1, 1, 0, 0)) + end = LOCAL_TZ.localize(datetime(2022, 9, 1, 0, 0, 0)) + powerprofile.fill(default_data, start, end) + + powerprofile.check() + expect(powerprofile.hours).to(equal(744)) + for field, value in default_data.items(): + expect(powerprofile[0][field]).to(equal(value)) + expect(powerprofile[-1][field]).to(equal(value)) + with context('dump function'): with before.all: self.curve = [] @@ -872,6 +914,66 @@ def read_csv(txt): for field in expected_cols: expect(first_register.keys()).to(contain(field)) + ## End Extend tests + + with context('Append'): + with it('Raises Type error if not powerprofile param'): + curve_a = PowerProfile('utc_datetime') + curve_a.load(self.erp_curve['curve']) + + expect( + lambda: curve_a.append(self.erp_curve['curve']) + ).to(raise_error(TypeError, 'ERROR append: Appended Profile must be a PowerProfile')) + + with context('Tests profiles and'): + with before.all: + self.curve_a = PowerProfile('utc_datetime') + self.curve_a.load(self.erp_curve['curve']) + + with it('raises a PowerProfileIncompatible with different profile type'): + curve_b = PowerProfileQh('utc_datetime') + curve_b.load(self.erp_curve['curve']) + + try: + self.curve_a.append(curve_b) + except Exception as e: + expect(str(e)).to(contain('different profile type')) + + with it('raises a PowerProfileIncompatible with different datetime field'): + curve_b = PowerProfile('local_datetime') + curve_b.load(self.erp_curve['curve']) + + try: + self.curve_a.append(curve_b) + except Exception as e: + expect(str(e)).to(contain('datetime field')) + + with it('raises a PowerProfileIncompatible with different fields'): + curve_b = PowerProfile('utc_datetime') + curve_b.load(self.erp_curve['curve']) + curve_c = curve_b.extract(['ai', 'ae']) + + try: + self.curve_a.append(curve_c) + except PowerProfileIncompatible as e: + expect(str(e)).to(contain('ai_fact')) + + with it('returns a new power profile with both data'): + curve_a = PowerProfile('utc_datetime') + curve_a.load(self.erp_curve['curve']) + + curve_b = PowerProfile('utc_datetime') + curve_b.load(self.erp_curve['curve']) + + curve_c = curve_a.append(curve_b) + + expect(curve_a.hours * 2).to(equal(curve_c.hours)) + + expect(curve_c.start).to(equal(curve_a.start)) + expect(curve_c.end).to(equal(curve_b.end)) + + ## End Append tests + with context('Arithmetic'): with before.all: self.data_path = './spec/data/' diff --git a/spec/powerprofileqh_spec.py b/spec/powerprofileqh_spec.py new file mode 100644 index 0000000..44e0eaf --- /dev/null +++ b/spec/powerprofileqh_spec.py @@ -0,0 +1,165 @@ +# -*- coding: utf-8 -*- +from expects.testing import failure +from expects import * +from powerprofile.powerprofile import PowerProfileQh, DEFAULT_DATA_FIELDS +from datetime import datetime, timedelta +from dateutil.relativedelta import relativedelta +from dateutil.parser import parse as parse_datetime +from pytz import timezone +from copy import copy, deepcopy +from powerprofile.exceptions import * +import json +import random +try: + # Python 2 + from cStringIO import StringIO +except ImportError: + # Python 3 + from io import StringIO +import csv +import pandas as pd + +LOCAL_TZ = timezone('Europe/Madrid') +UTC_TZ = timezone('UTC') + +with description('PowerProfileQh class'): + + with context('Instance'): + + with it('Returns PowerProfile Object'): + powerprofile = PowerProfileQh() + + expect(powerprofile).to(be_a(PowerProfileQh)) + with context('load function'): + with context('with bad data'): + with it('raises TypeError exception'): + powerprofile = PowerProfileQh() + + expect(lambda: powerprofile.load({'timestamp': '2020-01-31 10:00:00', "ai": 13.0})).to( + raise_error(TypeError, "ERROR: [data] must be a list of dicts ordered by timestamp") + ) + expect(lambda: powerprofile.load( + [{'timestamp': '2020-01-31 10:00:00', "ai": 13.0}], start='2020-03-11')).to( + raise_error(TypeError, "ERROR: [start] must be a localized datetime") + ) + expect(lambda: powerprofile.load( + [{'timestamp': '2020-01-31 10:00:00', "ai": 13.0}], end='2020-03-11')).to( + raise_error(TypeError, "ERROR: [end] must be a localized datetime") + ) + + with context('correctly'): + with before.all: + self.curve = [] + self.start = LOCAL_TZ.localize(datetime(2020, 3, 11, 0, 15, 0)) + self.end = LOCAL_TZ.localize(datetime(2020, 3, 12, 0, 0, 0)) + for hours in range(0, 24): + for minutes in [0, 15, 30, 45]: + self.curve.append({'timestamp': self.start + timedelta(minutes=hours*60 + minutes), 'value': 100 + hours * 10 + minutes}) + self.powpro = PowerProfileQh() + + with it('without dates'): + self.powpro.load(self.curve) + expect(lambda: self.powpro.check()).not_to(raise_error) + expect(self.powpro.data_fields).to(equal(['value'])) + + with it('with start date'): + self.powpro.load(self.curve, start=self.start) + expect(lambda: self.powpro.check()).not_to(raise_error) + expect(self.powpro.data_fields).to(equal(['value'])) + + with it('with end date'): + self.powpro.load(self.curve, end=self.end) + expect(lambda: self.powpro.check()).not_to(raise_error) + expect(self.powpro.data_fields).to(equal(['value'])) + + with it('with start and end date'): + self.powpro.load(self.curve, start=self.start, end=self.end) + expect(lambda: self.powpro.check()).not_to(raise_error) + expect(self.powpro.data_fields).to(equal(['value'])) + + with it('with datetime_field field in load'): + + curve_name = [] + for row in self.curve: + new_row = copy(row) + new_row['datetime'] = new_row['timestamp'] + new_row.pop('timestamp') + curve_name.append(new_row) + + powpro = PowerProfileQh() + + expect(lambda: powpro.load(curve_name)).to(raise_error(TypeError)) + + expect(lambda: powpro.load(curve_name, datetime_field='datetime')).to_not(raise_error(TypeError)) + expect(powpro[0]).to(have_key('datetime')) + + with it('with datetime_field field in constructor'): + + curve_name = [] + for row in self.curve: + new_row = copy(row) + new_row['datetime'] = new_row['timestamp'] + new_row.pop('timestamp') + curve_name.append(new_row) + + powpro = PowerProfileQh(datetime_field='datetime') + + expect(lambda: powpro.load(curve_name)).to_not(raise_error(TypeError)) + expect(powpro[0]).to(have_key('datetime')) + + with it('with data_fields field in load'): + + powpro = PowerProfileQh() + + powpro.load(self.curve, data_fields=['value']) + expect(powpro.data_fields).to(equal(['value'])) + + expect(lambda: powpro.load(curve_name, data_fields=['value'])).to_not(raise_error(TypeError)) + expect(powpro[0]).to(have_key('value')) + + with context('with unlocalized datetimes'): + with before.all: + self.curve = [] + self.start = datetime(2022, 8, 1, 1, 0, 0) + self.end = datetime(2022, 9, 1, 0, 0, 0) + for hours in range(0, 24): + self.curve.append({'timestamp': self.start + timedelta(hours=hours), 'value': 100 + hours}) + with it('should localize datetimes on load'): + powerprofile = PowerProfileQh() + powerprofile.load(self.curve) + for idx, hour in powerprofile.curve.iterrows(): + assert hour[powerprofile.datetime_field].tzinfo is not None + + with context('dump function'): + with before.all: + self.curve = [] + self.erp_curve = [] + self.start = LOCAL_TZ.localize(datetime(2020, 3, 11, 1, 0, 0)) + self.end = LOCAL_TZ.localize(datetime(2020, 3, 12, 0, 0, 0)) + for hours in range(0, 24): + for minutes in [15, 30, 45, 60]: + self.curve.append({'timestamp': self.start + timedelta(hours=hours, minutes=minutes), 'value': 100 + hours*10 + minutes}) + self.erp_curve.append( + { + 'local_datetime': self.start + timedelta(hours=hours, minutes=minutes), + 'utc_datetime': (self.start + timedelta(hours=hours, minutes=minutes)).astimezone(UTC_TZ), + 'value': 100 + hours, + 'valid': bool(hours % 2), + 'period': 'P' + str(hours % 3), + } + ) + + with context('performs complet curve -> load -> dump -> curve circuit '): + + with it('works with simple format'): + powpro = PowerProfileQh() + powpro.load(self.curve) + curve = powpro.dump() + expect(curve).to(equal(self.curve)) + + with it('works with ERP curve API'): + powpro = PowerProfileQh(datetime_field='utc_datetime') + powpro.load(self.erp_curve) + erp_curve = powpro.dump() + + expect(erp_curve).to(equal(self.erp_curve)) \ No newline at end of file From 25bd5f80f579b75e2a2d327b663977f8fcac1242 Mon Sep 17 00:00:00 2001 From: tinogis Date: Thu, 13 Jul 2023 17:32:39 +0200 Subject: [PATCH 09/12] TESTS get_hourly_profile --- spec/powerprofileqh_spec.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/spec/powerprofileqh_spec.py b/spec/powerprofileqh_spec.py index 44e0eaf..cb2872c 100644 --- a/spec/powerprofileqh_spec.py +++ b/spec/powerprofileqh_spec.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from expects.testing import failure from expects import * -from powerprofile.powerprofile import PowerProfileQh, DEFAULT_DATA_FIELDS +from powerprofile.powerprofile import PowerProfile, PowerProfileQh, DEFAULT_DATA_FIELDS from datetime import datetime, timedelta from dateutil.relativedelta import relativedelta from dateutil.parser import parse as parse_datetime @@ -162,4 +162,32 @@ powpro.load(self.erp_curve) erp_curve = powpro.dump() - expect(erp_curve).to(equal(self.erp_curve)) \ No newline at end of file + expect(erp_curve).to(equal(self.erp_curve)) + + with description('PowerProfileQH Operators'): + with before.all: + self.curve = [] + self.start = LOCAL_TZ.localize(datetime(2020, 3, 11, 0, 15, 0)) + self.end = LOCAL_TZ.localize(datetime(2020, 3, 12, 0, 0, 0)) + for hours in range(0, 24): + for minutes in [0, 15, 30, 45]: + self.curve.append({'timestamp': self.start + timedelta(minutes=hours * 60 + minutes), + 'value': 100 + hours * 10 + minutes}) + self.powpro = PowerProfileQh() + self.powpro.load(self.curve) + + with description('Unary Operator'): + with context('get_hourly_profile'): + with it('returns a PowerProfile instance'): + pph = self.powpro.get_hourly_profile() + + expect(pph).to(be_an(PowerProfile)) + expect(pph).not_to(be_an(PowerProfileQh)) + + pph.check() + expect(pph.start).to(equal(LOCAL_TZ.localize(datetime(2020, 3, 11, 1, 0, 0)))) + expect(pph.end).to(equal(self.powpro.end)) + expect(pph.sum(['value'])).to(equal(self.powpro.sum(['value']))) + expect(pph.hours).to(equal(24)) + expected_value = sum([x['value'] for x in self.powpro[:4]]) + expect(pph[0]['value']).to(equal(expected_value)) \ No newline at end of file From 74790538a353e1c66e6e22c21ebc3a77a6c1e791 Mon Sep 17 00:00:00 2001 From: davidmunoznovoa Date: Thu, 13 Jul 2023 17:49:44 +0200 Subject: [PATCH 10/12] Fix Python 2.7 tests --- .github/workflows/python2.7-app.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python2.7-app.yml b/.github/workflows/python2.7-app.yml index f428488..9d41a3b 100644 --- a/.github/workflows/python2.7-app.yml +++ b/.github/workflows/python2.7-app.yml @@ -16,10 +16,15 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Set up Python 2.7 - uses: actions/setup-python@v2 - with: - python-version: "2.7" + - name: Install Python 2.7 + run: | + sudo apt update + sudo apt install python2 python-pip + sudo update-alternatives --install /usr/bin/python python /usr/bin/python2 1 + sudo update-alternatives --install /usr/bin/python python /usr/bin/python3 2 + printf '1\n' | sudo update-alternatives --config python + cd /usr/bin + sudo ln -s /usr/bin/pip2 ./pip - name: Install dependencies run: | python -m pip install --upgrade pip From df66ebf51fd6cca8f5f734287ec5f8d9cf2aab1a Mon Sep 17 00:00:00 2001 From: davidmunoznovoa Date: Thu, 13 Jul 2023 18:02:14 +0200 Subject: [PATCH 11/12] Fix Python 3.x tests --- spec/powerprofileqh_spec.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/powerprofileqh_spec.py b/spec/powerprofileqh_spec.py index cb2872c..3d5b3ed 100644 --- a/spec/powerprofileqh_spec.py +++ b/spec/powerprofileqh_spec.py @@ -189,5 +189,7 @@ expect(pph.end).to(equal(self.powpro.end)) expect(pph.sum(['value'])).to(equal(self.powpro.sum(['value']))) expect(pph.hours).to(equal(24)) - expected_value = sum([x['value'] for x in self.powpro[:4]]) + expected_value = 0 + for x in range(4): + expected_value += self.powpro[x]['value'] expect(pph[0]['value']).to(equal(expected_value)) \ No newline at end of file From f4fbc5b4e17932e28bd23bb825852699e08edd31 Mon Sep 17 00:00:00 2001 From: davidmunoznovoa Date: Thu, 13 Jul 2023 18:19:15 +0200 Subject: [PATCH 12/12] Improve solution (fix __getitem__ with slice in Python 3) --- powerprofile/powerprofile.py | 4 ++-- spec/powerprofileqh_spec.py | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/powerprofile/powerprofile.py b/powerprofile/powerprofile.py index f17519a..9f37993 100644 --- a/powerprofile/powerprofile.py +++ b/powerprofile/powerprofile.py @@ -223,8 +223,8 @@ def __getitem__(self, item): res = self.curve.iloc[item] #interger slice [a:b] # test bounds - self.curve.iloc[item.start] - self.curve.iloc[item.stop] + self.curve.iloc[item.start or 0] # Python 3 returns None istead of 0 when empty + self.curve.iloc[item.stop or -1] # Python 3 returns None instead of -1 when empty powpro = self.__class__() powpro.curve = res powpro.start = res.iloc[0][self.datetime_field] diff --git a/spec/powerprofileqh_spec.py b/spec/powerprofileqh_spec.py index 3d5b3ed..cb2872c 100644 --- a/spec/powerprofileqh_spec.py +++ b/spec/powerprofileqh_spec.py @@ -189,7 +189,5 @@ expect(pph.end).to(equal(self.powpro.end)) expect(pph.sum(['value'])).to(equal(self.powpro.sum(['value']))) expect(pph.hours).to(equal(24)) - expected_value = 0 - for x in range(4): - expected_value += self.powpro[x]['value'] + expected_value = sum([x['value'] for x in self.powpro[:4]]) expect(pph[0]['value']).to(equal(expected_value)) \ No newline at end of file