From 3dbc3f157e0d912d5d0d0e6db3cbd3409e6e68b2 Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 23 Oct 2024 14:25:11 -0600 Subject: [PATCH 01/42] Running black to fix formatting --- gtep/driver.py | 6 +- gtep/driver_config_test.py | 5 +- gtep/driver_jsc.py | 6 +- gtep/gtep_model.py | 3 +- gtep/tests/unit/test_validation.py | 19 ++++-- gtep/validation.py | 98 +++++++++++++++++++++--------- 6 files changed, 91 insertions(+), 46 deletions(-) diff --git a/gtep/driver.py b/gtep/driver.py index 8137df4..900bdba 100644 --- a/gtep/driver.py +++ b/gtep/driver.py @@ -19,13 +19,13 @@ num_dispatch=4, ) mod_object.create_model() -TransformationFactory("gdp.bound_pretransformation").apply_to(mod_object.model) -TransformationFactory("gdp.bigm").apply_to(mod_object.model) +TransformationFactory("gdp.bound_pretransformation").apply_to(mod_object.model) +TransformationFactory("gdp.bigm").apply_to(mod_object.model) # opt = SolverFactory("gurobi") # opt = Gurobi() opt = Highs() # # mod_object.results = opt.solve(mod_object.model, tee=True) -mod_object.results = opt.solve(mod_object.model) +mod_object.results = opt.solve(mod_object.model) sol_object = ExpansionPlanningSolution() sol_object.load_from_model(mod_object) diff --git a/gtep/driver_config_test.py b/gtep/driver_config_test.py index 899f9ae..ba93e8a 100644 --- a/gtep/driver_config_test.py +++ b/gtep/driver_config_test.py @@ -12,7 +12,6 @@ data_object.load_prescient(data_path) - mod_object = ExpansionPlanningModel( stages=1, data=data_object.md, @@ -22,8 +21,8 @@ num_dispatch=4, ) -for k,v in mod_object.config.items(): - ic(k,v) +for k, v in mod_object.config.items(): + ic(k, v) quit() diff --git a/gtep/driver_jsc.py b/gtep/driver_jsc.py index 4ab5bac..e99f78c 100644 --- a/gtep/driver_jsc.py +++ b/gtep/driver_jsc.py @@ -33,7 +33,7 @@ opt = Highs() # opt = SolverFactory("gurobi", solver_io="python") # opt = Gurobi() -#mod_object.results = opt.solve(mod_object.model, tee=True) +# mod_object.results = opt.solve(mod_object.model, tee=True) mod_object.results = opt.solve(mod_object.model) @@ -48,9 +48,9 @@ if load_numerical_results: sol_object.read_json("./gtep_solution_jscTest.json") # sol_object.read_json("./gtep_solution.json") - #sol_object.read_json("./bigger_longer_wigglier_gtep_solution.json") + # sol_object.read_json("./bigger_longer_wigglier_gtep_solution.json") plot_results = True if plot_results: sol_object.plot_levels(save_dir="./plots/") -pass \ No newline at end of file +pass diff --git a/gtep/gtep_model.py b/gtep/gtep_model.py index ce87db0..41b554e 100644 --- a/gtep/gtep_model.py +++ b/gtep/gtep_model.py @@ -4,7 +4,7 @@ # date: 01/04/2024 # Model available at http://www.optimization-online.org/DB_FILE/2017/08/6162.pdf -from pyomo.environ import * +from pyomo.environ import * from pyomo.environ import units as u # from pyomo.gdp import * @@ -756,7 +756,6 @@ def capacity_factor(b, renewableGen): == m.renewableCapacity[renewableGen] ) - ## TODO: (@jkskolf) add renewableExtended to this and anywhere else @b.Constraint(m.renewableGenerators) def operational_renewables_only(b, renewableGen): diff --git a/gtep/tests/unit/test_validation.py b/gtep/tests/unit/test_validation.py index 19e01ef..99bb450 100644 --- a/gtep/tests/unit/test_validation.py +++ b/gtep/tests/unit/test_validation.py @@ -6,11 +6,17 @@ from pyomo.contrib.appsi.solvers.highs import Highs import logging -from gtep.validation import clone_timeseries, filter_pointers, populate_generators, populate_transmission +from gtep.validation import ( + clone_timeseries, + filter_pointers, + populate_generators, + populate_transmission, +) input_data_source = "./gtep/data/5bus" output_data_source = "./gtep/tests/data/5bus_out" + def test_solution(): data_object = ExpansionPlanningData() data_object.load_prescient(input_data_source) @@ -24,26 +30,27 @@ def test_solution(): num_dispatch=4, ) mod_object.create_model() - TransformationFactory("gdp.bound_pretransformation").apply_to(mod_object.model) - TransformationFactory("gdp.bigm").apply_to(mod_object.model) + TransformationFactory("gdp.bound_pretransformation").apply_to(mod_object.model) + TransformationFactory("gdp.bigm").apply_to(mod_object.model) # opt = SolverFactory("gurobi") # opt = Gurobi() opt = Highs() # # mod_object.results = opt.solve(mod_object.model, tee=True) - mod_object.results = opt.solve(mod_object.model) + mod_object.results = opt.solve(mod_object.model) sol_object = ExpansionPlanningSolution() sol_object.load_from_model(mod_object) sol_object.dump_json("./gtep/tests/test_solution.json") return sol_object + solution = test_solution() + class TestValidation(unittest.TestCase): def test_populate_generators(self): populate_generators(input_data_source, solution, output_data_source) - def test_populate_transmission(self): populate_transmission(input_data_source, solution, output_data_source) @@ -51,4 +58,4 @@ def test_filter_pointers(self): filter_pointers(input_data_source, output_data_source) def test_clone_timeseries(self): - clone_timeseries(input_data_source, output_data_source) \ No newline at end of file + clone_timeseries(input_data_source, output_data_source) diff --git a/gtep/validation.py b/gtep/validation.py index 96782a7..f63ecae 100644 --- a/gtep/validation.py +++ b/gtep/validation.py @@ -10,7 +10,10 @@ logger = logging.getLogger(__name__) -def populate_generators(data_input_path: str, sol_object: ExpansionPlanningSolution, data_output_path: str): + +def populate_generators( + data_input_path: str, sol_object: ExpansionPlanningSolution, data_output_path: str +): # load existing and candidate generators from initial prescient data # note that -c in name indicates candidate input_df = pd.read_csv(data_input_path + "/gen.csv") @@ -19,34 +22,55 @@ def populate_generators(data_input_path: str, sol_object: ExpansionPlanningSolut # for thermal: # generator should exist in future grid if the status in the final stage # is installed, operational, or extended - + def gen_name_filter(gen_name): - return 'gen' in gen_name and ('Ext' in gen_name or 'Ope' in gen_name or 'Ins' in gen_name) - solution_dict = sol_object._to_dict()['results']['primals_tree'] + return "gen" in gen_name and ( + "Ext" in gen_name or "Ope" in gen_name or "Ins" in gen_name + ) + + solution_dict = sol_object._to_dict()["results"]["primals_tree"] end_investment_stage = list(solution_dict.keys())[0] - end_investment_solution_dict = {k: v['value'] for k,v in solution_dict[end_investment_stage].items() if gen_name_filter(k) and v['value'] > 0.5} - end_investment_gens = [re.search(r'\[.*\]', k).group(0)[1:-1] for k in end_investment_solution_dict.keys()] - + end_investment_solution_dict = { + k: v["value"] + for k, v in solution_dict[end_investment_stage].items() + if gen_name_filter(k) and v["value"] > 0.5 + } + end_investment_gens = [ + re.search(r"\[.*\]", k).group(0)[1:-1] + for k in end_investment_solution_dict.keys() + ] + # for renewable: # total capacity should be installed + operational + extended values def renewable_name_filter(gen_name): - return 'renew' in gen_name and ('Ext' in gen_name or 'Ope' in gen_name or 'Ins' in gen_name) - end_investment_renewable_dict = {k: v['value'] for k,v in solution_dict[end_investment_stage].items() if renewable_name_filter(k)} - end_investment_renewable_gens = {re.search(r'\[.*\]', k).group(0)[1:-1]: 0 for k in end_investment_renewable_dict.keys()} - for k,v in end_investment_renewable_dict.items(): - end_investment_renewable_gens[re.search(r'\[.*\]', k).group(0)[1:-1]] += v + return "renew" in gen_name and ( + "Ext" in gen_name or "Ope" in gen_name or "Ins" in gen_name + ) + + end_investment_renewable_dict = { + k: v["value"] + for k, v in solution_dict[end_investment_stage].items() + if renewable_name_filter(k) + } + end_investment_renewable_gens = { + re.search(r"\[.*\]", k).group(0)[1:-1]: 0 + for k in end_investment_renewable_dict.keys() + } + for k, v in end_investment_renewable_dict.items(): + end_investment_renewable_gens[re.search(r"\[.*\]", k).group(0)[1:-1]] += v for k, v in end_investment_renewable_gens.items(): ## NOTE: (@jkskolf) this will break in pandas 3.0 - input_df["PMax MW"].mask(input_df['GEN UID'] == k,v, inplace=True) + input_df["PMax MW"].mask(input_df["GEN UID"] == k, v, inplace=True) end_investment_gens += [k for k in end_investment_renewable_gens.keys()] # populate output dataframe - output_df = input_df[input_df['GEN UID'].isin(end_investment_gens)] + output_df = input_df[input_df["GEN UID"].isin(end_investment_gens)] # TODO: (@jkskolf) should we update prices here? I think no, but ... if not os.path.exists(data_output_path): os.makedirs(data_output_path) - output_df.to_csv(data_output_path + '/gen.csv',index=False) + output_df.to_csv(data_output_path + "/gen.csv", index=False) + def populate_transmission(data_input_path, sol_object, data_output_path): # load existing and candidate generators from initial prescient data @@ -55,16 +79,27 @@ def populate_transmission(data_input_path, sol_object, data_output_path): # pull final stage solution variables for transmission def branch_name_filter(gen_name): - return 'bran' in gen_name and ('Ext' in gen_name or 'Ope' in gen_name or 'Ins' in gen_name) - solution_dict = sol_object._to_dict()['results']['primals_tree'] + return "bran" in gen_name and ( + "Ext" in gen_name or "Ope" in gen_name or "Ins" in gen_name + ) + + solution_dict = sol_object._to_dict()["results"]["primals_tree"] end_investment_stage = list(solution_dict.keys())[0] - end_investment_solution_dict = {k: v['value'] for k,v in solution_dict[end_investment_stage].items() if branch_name_filter(k) and v['value'] > 0.5} - end_investment_branches = [re.search(r'\[.*\]', k).group(0)[1:-1] for k in end_investment_solution_dict.keys()] - output_df = input_df[input_df['UID'].isin(end_investment_branches)] - + end_investment_solution_dict = { + k: v["value"] + for k, v in solution_dict[end_investment_stage].items() + if branch_name_filter(k) and v["value"] > 0.5 + } + end_investment_branches = [ + re.search(r"\[.*\]", k).group(0)[1:-1] + for k in end_investment_solution_dict.keys() + ] + output_df = input_df[input_df["UID"].isin(end_investment_branches)] + if not os.path.exists(data_output_path): os.makedirs(data_output_path) - output_df.to_csv(data_output_path + '/branch.csv' ,index=False) + output_df.to_csv(data_output_path + "/branch.csv", index=False) + def filter_pointers(data_input_path, data_output_path): # load initial timeseries pointers @@ -77,19 +112,24 @@ def filter_pointers(data_input_path, data_output_path): # keep generators that exist at the final investment stage and remove the rest # keep all non-generator timeseries pointers - matching_gen_list = [gen for gen in output_generators_df['GEN UID']] - output_df = input_pointers_df[input_pointers_df['Object'].isin(matching_gen_list) | input_pointers_df['Category'] != 'Generator'] + matching_gen_list = [gen for gen in output_generators_df["GEN UID"]] + output_df = input_pointers_df[ + input_pointers_df["Object"].isin(matching_gen_list) + | input_pointers_df["Category"] + != "Generator" + ] if not os.path.exists(data_output_path): os.makedirs(data_output_path) - output_df.to_csv(data_output_path + '/timeseries_pointers.csv') + output_df.to_csv(data_output_path + "/timeseries_pointers.csv") + def clone_timeseries(data_input_path, data_output_path): file_list = os.listdir(data_input_path) - file_list.remove('timeseries_pointers.csv') - file_list.remove('gen.csv') - file_list.remove('branch.csv') + file_list.remove("timeseries_pointers.csv") + file_list.remove("gen.csv") + file_list.remove("branch.csv") # @jkskolf, I don't think I like this ... for fname in file_list: - shutil.copy(data_input_path + "/" + fname, data_output_path + "/" + fname) \ No newline at end of file + shutil.copy(data_input_path + "/" + fname, data_output_path + "/" + fname) From b8565514a7b932e8e8dc87d47997b3e9d58fe6c9 Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 23 Oct 2024 14:42:03 -0600 Subject: [PATCH 02/42] Enable GHA branch testing and fix GHA PR testing --- .github/workflows/test_branches.yml | 79 ++++++++++++++++++++++++++ .github/workflows/test_pr_and_main.yml | 5 ++ 2 files changed, 84 insertions(+) create mode 100644 .github/workflows/test_branches.yml diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml new file mode 100644 index 0000000..3a08036 --- /dev/null +++ b/.github/workflows/test_branches.yml @@ -0,0 +1,79 @@ +name: GitHub Branch CI + +on: + push: + branches-ignore: + - main + workflow_dispatch: + inputs: + git-ref: + description: Git Hash (Optional) + required: false + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +defaults: + run: + # -l: login shell, needed when using Conda run: + shell: bash -l {0} + +jobs: + code-formatting: + name: Check code formatting (Black) + # OS and/or Python version don't make a difference, so we choose ubuntu and 3.10 for performance + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Install Black + # unlike the other jobs, we don't need to install IDAES and/or all the dev dependencies, + # but we still want to specify the Black version to use in requirements-dev.txt for local development + # so we extract the relevant line and pass it to a simple `pip install` + run: | + pip install black + - name: Run Black to verify that the committed code is formatted + run: | + black --check . + + spell-check: + name: Check Spelling + runs-on: ubuntu-latest + steps: + - name: Checkout source + uses: actions/checkout@v4 + - name: Run Spell Checker + uses: crate-ci/typos@master + with: + config: ./.github/workflows/typos.toml + + build: + name: ${{ matrix.os }}/${{ matrix.python }} + needs: [code-formatting, spell-check] + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, windows-latest] + python: [3.9, '3.10', '3.11'] + steps: + - name: Checkout source + uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + - name: Install Pyomo + run: | + pip install git+https://github.com/Pyomo/pyomo.git + - name: Install idaes-gtep + run: | + python setup.py develop + - name: Run Tests + run: | + pip install pytest + pip install highspy + pytest -v gtep + diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 6239ed4..c6ce4ec 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -7,6 +7,11 @@ on: pull_request: branches: - main + types: + - opened + - reopened + - synchronize + - ready_for_review workflow_dispatch: inputs: git-ref: From add9e871331d4459d8d98b468f2aa95515020bae Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 23 Oct 2024 15:24:29 -0600 Subject: [PATCH 03/42] Fixing typos --- .github/workflows/typos.toml | 5 +++++ gtep/gtep_solution.py | 14 +++++++------- gtep/map_bus_to_vars.py | 6 +++--- gtep/tests/unit/pglib_opf_case5_pjm.m | 2 +- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/.github/workflows/typos.toml b/.github/workflows/typos.toml index 049543a..3b7b5c5 100644 --- a/.github/workflows/typos.toml +++ b/.github/workflows/typos.toml @@ -5,3 +5,8 @@ extend-exclude = ["*.eps"] # Ignore IDAES IDAES = "IDAES" idaes = "idaes" +# Ignore ND +ND = "ND" +nd = "nd" +# Ignore opf +opf = "opf" \ No newline at end of file diff --git a/gtep/gtep_solution.py b/gtep/gtep_solution.py index d2cf53a..7089863 100644 --- a/gtep/gtep_solution.py +++ b/gtep/gtep_solution.py @@ -379,7 +379,7 @@ def _plot_workhorse_relational( ) ax_koi_list.append(ax_koi) - for iy, this_voi in enumerate(vars): + for _, this_voi in enumerate(vars): ax_koi.plot( df[level_key], df[f"{this_koi}_{this_voi}_value"], @@ -404,7 +404,7 @@ def _plot_workhorse_relational( # plot variables of interest ax_voi_list = [] - # plot generations and curtailmentsagainst each outher + # plot generations and curtailments against each other for ix_voi, this_voi in enumerate(vars): ax_voi = fig.add_subplot( gs[(ix_voi * var_gridspec_div) : ((ix_voi + 1) * var_gridspec_div), 1] @@ -469,7 +469,7 @@ def _plot_workhose_binaries( axline_ix + 0.5, color="grey", linewidth=3, - ) # draw a seperator line between each level + ) # draw a separator line between each level for axline_ix in range(len(df[level_key])): ax_bins.axvline( axline_ix + 0.5, @@ -477,7 +477,7 @@ def _plot_workhose_binaries( linewidth=3, linestyle="dotted", alpha=0.5, - ) # draw a seperator line between each level + ) # draw a separator line between each level for ix_key, this_koi in enumerate(keys): # make a dummy line to steal the color cycler and make a single item for the legend @@ -550,7 +550,7 @@ def _level_relationship_df_to_plot( # check if ALL the possible things to look at are binaries all_binaries = True for ix, this_voi in enumerate(vars): - for iy, this_koi in enumerate(keys): + for _, this_koi in enumerate(keys): if not (df[f"{this_koi}_{this_voi}_value"].dtype == "bool"): all_binaries = False break @@ -1048,13 +1048,13 @@ def generate_flow_glyphs( PathPatch(mpath.Path(verts, codes), ec="none"), ) - rotation_transofrm = Affine2D().rotate_around( + rotation_transform = Affine2D().rotate_around( glyph_anchor_coord[0] + patch_width * 0.5, glyph_anchor_coord[1] + patch_height * 0.5, glyph_rotation, ) # rescale y to make it fit in a 1x1 box - flow_glyphs[-1].set_transform(rotation_transofrm) + flow_glyphs[-1].set_transform(rotation_transform) return flow_glyphs diff --git a/gtep/map_bus_to_vars.py b/gtep/map_bus_to_vars.py index 7d1bd03..7c59aa1 100644 --- a/gtep/map_bus_to_vars.py +++ b/gtep/map_bus_to_vars.py @@ -49,7 +49,7 @@ # Everything outside of Solar PV requires "Low Urban Density", so we're just going to completely ignore that entirely. # Wind needs to map to Wind Zones # Solar Thermal need Solar Resource and Water Vailability -# NG CT needs NG Supply, and Non-attainment is requird if replacing higher-emission generation +# NG CT needs NG Supply, and Non-attainment is required if replacing higher-emission generation # NG CC needs NG Supply, and Water Availability # Coal needs Rail, Can't be in Non-Attainment Area, and Water Availability # Biomass needs Rail and Biomass @@ -178,7 +178,7 @@ # Hydrothermal is probably Hydrothermal in both cases. That only really makes sense for Binary, because the map says that the highest temperature is # ~160F (71C), which is well below the 200C needed for Flash. Geopressure can get high enough temperature, but is deep. # Geopressure is very deep (13000 ft ~= 3.9 km), so this likely applies to the Deep EGS (3-6 km). Let's ignore it for now. -# Hot Dry Rock is very explicity "little or no water", so let's seperate Hydro from NFEGS +# Hot Dry Rock is very explicitly "little or no water", so let's separate Hydro from NFEGS # C.2 only gives us High, Medium and NA. Everything is pain. # Looking at the workbook, Flash and Binary appear to be identical in the data and we can ignore Hydro and NFEGS, until you get to costs. # If it was the other way around, we could instead map "Medium" and "High" to production. @@ -191,7 +191,7 @@ # *************************************************************************************************************************************************** # --- Solar - Utility PV --- -# For some reason, we can't assign any actual units here, even though we did so on the other solar classificaitons. Cool. Thanks. Appreciate that. +# For some reason, we can't assign any actual units here, even though we did so on the other solar classifications. Cool. Thanks. Appreciate that. # Instead we get "Very High", "High", and "Medium" from C.1 and "Good" or "Avg." from C.2. # Mapping C.1 to C.2 we get that Very High = Good, High = Good and Medium = Avg. So let's use C.1 because it's more detailed. # The workbook gives us Class 1-10. Lower is better. Everything is pain. diff --git a/gtep/tests/unit/pglib_opf_case5_pjm.m b/gtep/tests/unit/pglib_opf_case5_pjm.m index 2cc5414..3cc4c9d 100644 --- a/gtep/tests/unit/pglib_opf_case5_pjm.m +++ b/gtep/tests/unit/pglib_opf_case5_pjm.m @@ -20,7 +20,7 @@ % Licensed under the Creative Commons Attribution 4.0 % International license, http://creativecommons.org/licenses/by/4.0/ % -% Contact M.E. Brennan (me.brennan@ieee.org) for inquries on further reuse of +% Contact M.E. Brennan (me.brennan@ieee.org) for inquiries on further reuse of % this dataset. % function mpc = pglib_opf_case5_pjm From c6fd77e5c2be6fb408f72b56f10ada3bb8e103b8 Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 23 Oct 2024 15:29:44 -0600 Subject: [PATCH 04/42] Setting "fail-fast" to False in GHA workflows --- .github/workflows/test_branches.yml | 1 + .github/workflows/test_pr_and_main.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 3a08036..7192458 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -55,6 +55,7 @@ jobs: needs: [code-formatting, spell-check] runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest] python: [3.9, '3.10', '3.11'] diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index c6ce4ec..c28a694 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -63,6 +63,7 @@ jobs: needs: [code-formatting, spell-check] runs-on: ${{ matrix.os }} strategy: + fail-fast: false matrix: os: [ubuntu-latest, windows-latest] python: [3.9, '3.10', '3.11'] From e8d276981ada075991d695dde2a3139a6307ecaa Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 23 Oct 2024 15:35:09 -0600 Subject: [PATCH 05/42] Update Action versions to address Node.js deprecation --- .github/workflows/test_branches.yml | 2 +- .github/workflows/test_pr_and_main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 7192458..c33f11b 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -26,7 +26,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' - name: Install Black diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index c28a694..1b1ed2e 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -71,7 +71,7 @@ jobs: - name: Checkout source uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - name: Install Pyomo From 431532762d3dad3e5a1360cce0bb6fef20b45ff3 Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 23 Oct 2024 15:39:19 -0600 Subject: [PATCH 06/42] Update Action versions to address Node.js deprecation --- .github/workflows/test_branches.yml | 2 +- .github/workflows/test_pr_and_main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index c33f11b..030c174 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -63,7 +63,7 @@ jobs: - name: Checkout source uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - name: Install Pyomo diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 1b1ed2e..7b01af6 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -34,7 +34,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: '3.10' - name: Install Black From 8151e523e32f87c004f3c9f4e94ec06d306f0fe5 Mon Sep 17 00:00:00 2001 From: blnicho Date: Thu, 24 Oct 2024 09:07:30 -0600 Subject: [PATCH 07/42] Debugging tests --- gtep/gtep_model.py | 2 +- setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/gtep/gtep_model.py b/gtep/gtep_model.py index 41b554e..68e790e 100644 --- a/gtep/gtep_model.py +++ b/gtep/gtep_model.py @@ -20,7 +20,7 @@ from math import ceil -from config_options import _get_model_config +from gtep.config_options import _get_model_config # Define what a USD is for pyomo units purposes # This will be set to a base year and we will do NPV calculations diff --git a/setup.py b/setup.py index b2e0966..c78b747 100644 --- a/setup.py +++ b/setup.py @@ -17,6 +17,7 @@ "gridx-prescient", "anyio==3.1", "pint", + "icecream" ] setup( name="gtep", From a058f9028a3e565cafaa3e02ec70a5665691f1f1 Mon Sep 17 00:00:00 2001 From: blnicho Date: Thu, 24 Oct 2024 09:08:02 -0600 Subject: [PATCH 08/42] Try relaxing anyio version requirement --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c78b747..90a0f42 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,7 @@ "scipy", "gridx-egret", "gridx-prescient", - "anyio==3.1", + "anyio", "pint", "icecream" ] From 3e7d822bf0077218ef381b81509ccc7d0732b96b Mon Sep 17 00:00:00 2001 From: blnicho Date: Thu, 24 Oct 2024 09:09:57 -0600 Subject: [PATCH 09/42] Forgot to run black --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 90a0f42..2061406 100644 --- a/setup.py +++ b/setup.py @@ -17,7 +17,7 @@ "gridx-prescient", "anyio", "pint", - "icecream" + "icecream", ] setup( name="gtep", From 2bd95fe48ccf467b483e71340b2422f4609d3807 Mon Sep 17 00:00:00 2001 From: blnicho Date: Thu, 24 Oct 2024 10:43:58 -0600 Subject: [PATCH 10/42] Try changing version of pyomo used for testing --- .github/workflows/test_branches.yml | 2 +- .github/workflows/test_pr_and_main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 030c174..7d839df 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -68,7 +68,7 @@ jobs: python-version: ${{ matrix.python }} - name: Install Pyomo run: | - pip install git+https://github.com/Pyomo/pyomo.git + pip install pyomo - name: Install idaes-gtep run: | python setup.py develop diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 7b01af6..81066f5 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -76,7 +76,7 @@ jobs: python-version: ${{ matrix.python }} - name: Install Pyomo run: | - pip install git+https://github.com/Pyomo/pyomo.git + pip install pyomo - name: Install idaes-gtep run: | python setup.py develop From 6a3cb0ae40eb53aff2e1ba88d9bf7a5fdd41f1a2 Mon Sep 17 00:00:00 2001 From: blnicho Date: Thu, 24 Oct 2024 10:59:01 -0600 Subject: [PATCH 11/42] Debugging GHA environment setup --- .github/workflows/test_branches.yml | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 7d839df..7ea167b 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -66,12 +66,23 @@ jobs: uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} + - name: Install Python Packages (pip) + shell: bash # DO NOT REMOVE: see note above + run: | + python -c 'import sys;print(sys.executable)' + python -m pip install --upgrade pip + + python -c 'import sys; print("PYTHON_EXE=%s" \ + % (sys.executable,))' >> $GITHUB_ENV + echo "" + echo "Final pip environment:" + python -m pip list | sed 's/^/ /' - name: Install Pyomo run: | pip install pyomo - name: Install idaes-gtep run: | - python setup.py develop + $PYTHON_EXE setup.py develop - name: Run Tests run: | pip install pytest From e0f92971c7dfbd96137499a7b9595958ea6448bc Mon Sep 17 00:00:00 2001 From: blnicho Date: Thu, 24 Oct 2024 11:02:36 -0600 Subject: [PATCH 12/42] Manually install scipy in GHA test environment --- .github/workflows/test_branches.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 7ea167b..c97070f 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -69,8 +69,7 @@ jobs: - name: Install Python Packages (pip) shell: bash # DO NOT REMOVE: see note above run: | - python -c 'import sys;print(sys.executable)' - python -m pip install --upgrade pip + pip install scipy python -c 'import sys; print("PYTHON_EXE=%s" \ % (sys.executable,))' >> $GITHUB_ENV From e7958e570f3577e10bedb8019430c7108f1498ab Mon Sep 17 00:00:00 2001 From: blnicho Date: Thu, 24 Oct 2024 11:31:01 -0600 Subject: [PATCH 13/42] Debugging GHA Python environment --- .github/workflows/test_branches.yml | 62 ++++++++++++++++++++++------- 1 file changed, 48 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index c97070f..c8713ea 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -62,29 +62,63 @@ jobs: steps: - name: Checkout source uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v5 + # - name: Set up Python ${{ matrix.python }} + # uses: actions/setup-python@v5 + # with: + # python-version: ${{ matrix.python }} + # - name: Install Python Packages (pip) + # shell: bash # DO NOT REMOVE: see note above + # run: | + # pip install scipy + + # python -c 'import sys; print("PYTHON_EXE=%s" \ + # % (sys.executable,))' >> $GITHUB_ENV + # echo "" + # echo "Final pip environment:" + # python -m pip list | sed 's/^/ /' + + - name: Set up Conda environment + uses: conda-incubator/setup-miniconda@v3 with: python-version: ${{ matrix.python }} - - name: Install Python Packages (pip) - shell: bash # DO NOT REMOVE: see note above + - name: Intall Python packages (conda) run: | - pip install scipy + # Set up environment + conda config --set always_yes yes + conda config --set auto_update_conda false + conda config --remove channels defaults + conda config --append channels nodefaults + conda config --append channels conda-forge + + # Print environment info + echo "*** CONDA environment: ***" + conda info + conda config --show-sources + conda config --show channels + conda list --show-channel-urls + which python + python --version + + echo "Install Python packages" + conda install --yes --quiet pip setuptools wheel + conda install --yes --quiet scipy pyomo pytest highspy + # remember this python interpreter python -c 'import sys; print("PYTHON_EXE=%s" \ % (sys.executable,))' >> $GITHUB_ENV - echo "" - echo "Final pip environment:" - python -m pip list | sed 's/^/ /' - - name: Install Pyomo - run: | - pip install pyomo + + echo "Final conda environment:" + conda list | sed 's/^/ /' + + #- name: Install Pyomo + # run: | + # pip install pyomo - name: Install idaes-gtep run: | - $PYTHON_EXE setup.py develop + python setup.py develop - name: Run Tests run: | - pip install pytest - pip install highspy + #pip install pytest + #pip install highspy pytest -v gtep From 5cf466823e3cbf79fea3205119dd481b5c0a8ff3 Mon Sep 17 00:00:00 2001 From: blnicho Date: Thu, 24 Oct 2024 11:43:49 -0600 Subject: [PATCH 14/42] More debugging --- .github/workflows/test_branches.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index c8713ea..d1031db 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -115,7 +115,7 @@ jobs: # pip install pyomo - name: Install idaes-gtep run: | - python setup.py develop + pip install -e - name: Run Tests run: | #pip install pytest From 01fbafa9701b2dd790d00008cade811cbbeb3900 Mon Sep 17 00:00:00 2001 From: blnicho Date: Thu, 24 Oct 2024 11:46:43 -0600 Subject: [PATCH 15/42] More debugging --- .github/workflows/test_branches.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index d1031db..2f0ed8d 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -89,15 +89,6 @@ jobs: conda config --remove channels defaults conda config --append channels nodefaults conda config --append channels conda-forge - - # Print environment info - echo "*** CONDA environment: ***" - conda info - conda config --show-sources - conda config --show channels - conda list --show-channel-urls - which python - python --version echo "Install Python packages" conda install --yes --quiet pip setuptools wheel @@ -115,7 +106,7 @@ jobs: # pip install pyomo - name: Install idaes-gtep run: | - pip install -e + pip install -e . - name: Run Tests run: | #pip install pytest From a8e7a28f3f332d7742f819e6ff88f33ce1376f09 Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 30 Oct 2024 15:19:17 -0600 Subject: [PATCH 16/42] More edits to GHA branch testing workflow --- .github/workflows/test_branches.yml | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 2f0ed8d..8210b2b 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -62,25 +62,12 @@ jobs: steps: - name: Checkout source uses: actions/checkout@v4 - # - name: Set up Python ${{ matrix.python }} - # uses: actions/setup-python@v5 - # with: - # python-version: ${{ matrix.python }} - # - name: Install Python Packages (pip) - # shell: bash # DO NOT REMOVE: see note above - # run: | - # pip install scipy - - # python -c 'import sys; print("PYTHON_EXE=%s" \ - # % (sys.executable,))' >> $GITHUB_ENV - # echo "" - # echo "Final pip environment:" - # python -m pip list | sed 's/^/ /' - - name: Set up Conda environment + - name: Set up Miniconda Python uses: conda-incubator/setup-miniconda@v3 with: python-version: ${{ matrix.python }} + - name: Intall Python packages (conda) run: | # Set up environment @@ -94,22 +81,13 @@ jobs: conda install --yes --quiet pip setuptools wheel conda install --yes --quiet scipy pyomo pytest highspy - # remember this python interpreter - python -c 'import sys; print("PYTHON_EXE=%s" \ - % (sys.executable,))' >> $GITHUB_ENV - echo "Final conda environment:" conda list | sed 's/^/ /' - #- name: Install Pyomo - # run: | - # pip install pyomo - name: Install idaes-gtep run: | pip install -e . - name: Run Tests run: | - #pip install pytest - #pip install highspy pytest -v gtep From 86d47d2fb6575c2854908ffc4e485c8add64523a Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 30 Oct 2024 15:29:53 -0600 Subject: [PATCH 17/42] Renaming driver_config_test.py to fix error in pytest --- gtep/{driver_config_test.py => driver_config_check.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename gtep/{driver_config_test.py => driver_config_check.py} (100%) diff --git a/gtep/driver_config_test.py b/gtep/driver_config_check.py similarity index 100% rename from gtep/driver_config_test.py rename to gtep/driver_config_check.py From 1c421db2140ec73f2f40f643025408f3d64cf95f Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 6 Nov 2024 14:25:11 -0700 Subject: [PATCH 18/42] Updating paths in validation tests --- gtep/tests/unit/test_validation.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/gtep/tests/unit/test_validation.py b/gtep/tests/unit/test_validation.py index 99bb450..b574a35 100644 --- a/gtep/tests/unit/test_validation.py +++ b/gtep/tests/unit/test_validation.py @@ -1,3 +1,4 @@ +from os.path import abspath, join, dirname import pyomo.common.unittest as unittest from gtep.gtep_model import ExpansionPlanningModel from gtep.gtep_data import ExpansionPlanningData @@ -13,8 +14,9 @@ populate_transmission, ) -input_data_source = "./gtep/data/5bus" -output_data_source = "./gtep/tests/data/5bus_out" +curr_dir = dirname(abspath(__file__)) +input_data_source = abspath(join(curr_dir, "..", "..", "data", "5bus")) +output_data_source = abspath(join(curr_dir, "..", "..", "data", "5bus_out")) def test_solution(): From b8495c2f83bcae9f52231bc858ac5ec3d75b63ea Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 6 Nov 2024 14:35:27 -0700 Subject: [PATCH 19/42] Minor change the GHA workflow for testing branches --- .github/workflows/test_branches.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 8210b2b..052da01 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -52,7 +52,6 @@ jobs: build: name: ${{ matrix.os }}/${{ matrix.python }} - needs: [code-formatting, spell-check] runs-on: ${{ matrix.os }} strategy: fail-fast: false From 9562f466a795f9af86e0826e44cce4b90567a919 Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 6 Nov 2024 14:53:18 -0700 Subject: [PATCH 20/42] Fixing bug in validation.py --- gtep/validation.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/gtep/validation.py b/gtep/validation.py index f63ecae..529beca 100644 --- a/gtep/validation.py +++ b/gtep/validation.py @@ -125,6 +125,10 @@ def filter_pointers(data_input_path, data_output_path): def clone_timeseries(data_input_path, data_output_path): + + if not os.path.exists(data_output_path): + os.makedirs(data_output_path) + file_list = os.listdir(data_input_path) file_list.remove("timeseries_pointers.csv") file_list.remove("gen.csv") @@ -132,4 +136,6 @@ def clone_timeseries(data_input_path, data_output_path): # @jkskolf, I don't think I like this ... for fname in file_list: - shutil.copy(data_input_path + "/" + fname, data_output_path + "/" + fname) + from_file = os.path.join(data_input_path, fname) + to_file = os.path.join(data_output_path, fname) + shutil.copy(from_file, to_file) From 1ed36cae575b1a3f7be08b09a185d891af0b3124 Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 6 Nov 2024 14:54:31 -0700 Subject: [PATCH 21/42] Running black --- gtep/validation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gtep/validation.py b/gtep/validation.py index 529beca..94e8e06 100644 --- a/gtep/validation.py +++ b/gtep/validation.py @@ -125,10 +125,10 @@ def filter_pointers(data_input_path, data_output_path): def clone_timeseries(data_input_path, data_output_path): - + if not os.path.exists(data_output_path): os.makedirs(data_output_path) - + file_list = os.listdir(data_input_path) file_list.remove("timeseries_pointers.csv") file_list.remove("gen.csv") From d654f5cc5298f140e13e0fe48f1b4494f40c74a0 Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 20 Nov 2024 15:04:03 -0700 Subject: [PATCH 22/42] Cleaning up file paths in validation.py --- gtep/tests/unit/test_gtep_model.py | 2 +- gtep/validation.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index d616225..a1b44c5 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -103,7 +103,7 @@ def test_solve_bigm(self): TransformationFactory("gdp.bound_pretransformation").apply_to(modObject.model) TransformationFactory("gdp.bigm").apply_to(modObject.model) modObject.results = opt.solve(modObject.model) - modObject.model.pprint() + # modObject.model.pprint() self.assertAlmostEqual( value(modObject.model.total_cost_objective_rule), 6078.86, places=1 ) diff --git a/gtep/validation.py b/gtep/validation.py index 94e8e06..ec521e6 100644 --- a/gtep/validation.py +++ b/gtep/validation.py @@ -16,7 +16,7 @@ def populate_generators( ): # load existing and candidate generators from initial prescient data # note that -c in name indicates candidate - input_df = pd.read_csv(data_input_path + "/gen.csv") + input_df = pd.read_csv(os.path.join(data_input_path, "gen.csv")) # pull final stage solution variables for thermal and renewable investments # for thermal: @@ -69,13 +69,13 @@ def renewable_name_filter(gen_name): # TODO: (@jkskolf) should we update prices here? I think no, but ... if not os.path.exists(data_output_path): os.makedirs(data_output_path) - output_df.to_csv(data_output_path + "/gen.csv", index=False) + output_df.to_csv(os.path.join(data_output_path, "gen.csv"), index=False) def populate_transmission(data_input_path, sol_object, data_output_path): # load existing and candidate generators from initial prescient data # note that -c in name indicates candidate - input_df = pd.read_csv(data_input_path + "/branch.csv") + input_df = pd.read_csv(os.path.join(data_input_path, "branch.csv")) # pull final stage solution variables for transmission def branch_name_filter(gen_name): @@ -98,17 +98,17 @@ def branch_name_filter(gen_name): if not os.path.exists(data_output_path): os.makedirs(data_output_path) - output_df.to_csv(data_output_path + "/branch.csv", index=False) + output_df.to_csv(os.path.join(data_output_path, "branch.csv"), index=False) def filter_pointers(data_input_path, data_output_path): # load initial timeseries pointers - input_pointers_df = pd.read_csv(data_input_path + "/timeseries_pointers.csv") + input_pointers_df = pd.read_csv(os.path.join(data_input_path, "timeseries_pointers.csv")) # load final generators # NOTE: must be run _after_ populate_generators and with the same data_output_path # to pull resulting generator objects - output_generators_df = pd.read_csv(data_output_path + "/gen.csv") + output_generators_df = pd.read_csv(os.path.join(data_output_path, "gen.csv")) # keep generators that exist at the final investment stage and remove the rest # keep all non-generator timeseries pointers @@ -121,7 +121,7 @@ def filter_pointers(data_input_path, data_output_path): if not os.path.exists(data_output_path): os.makedirs(data_output_path) - output_df.to_csv(data_output_path + "/timeseries_pointers.csv") + output_df.to_csv(os.path.join(data_output_path, "timeseries_pointers.csv")) def clone_timeseries(data_input_path, data_output_path): From 9f991f0dfa082a5209ede1d32a67a9f6bd0b487e Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 20 Nov 2024 15:08:47 -0700 Subject: [PATCH 23/42] Fixing black formatting --- .github/workflows/test_branches.yml | 2 +- gtep/validation.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 052da01..de5db14 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -37,7 +37,7 @@ jobs: pip install black - name: Run Black to verify that the committed code is formatted run: | - black --check . + black --check --diff . spell-check: name: Check Spelling diff --git a/gtep/validation.py b/gtep/validation.py index ec521e6..7e0dfd8 100644 --- a/gtep/validation.py +++ b/gtep/validation.py @@ -103,7 +103,9 @@ def branch_name_filter(gen_name): def filter_pointers(data_input_path, data_output_path): # load initial timeseries pointers - input_pointers_df = pd.read_csv(os.path.join(data_input_path, "timeseries_pointers.csv")) + input_pointers_df = pd.read_csv( + os.path.join(data_input_path, "timeseries_pointers.csv") + ) # load final generators # NOTE: must be run _after_ populate_generators and with the same data_output_path From 505710e6501b6791df9b9d0c72027046da7f6778 Mon Sep 17 00:00:00 2001 From: Kyle Skolfield Date: Mon, 25 Nov 2024 14:00:47 -0700 Subject: [PATCH 24/42] fixing units in model and objective value in test_gtep_model --- gtep/config_options.py | 13 +++++-- gtep/gtep_model.py | 61 +++++++++++++++--------------- gtep/tests/unit/test_gtep_model.py | 6 ++- 3 files changed, 45 insertions(+), 35 deletions(-) diff --git a/gtep/config_options.py b/gtep/config_options.py index 5bdea95..379ad43 100644 --- a/gtep/config_options.py +++ b/gtep/config_options.py @@ -19,6 +19,13 @@ def _get_model_config(): CONFIG = ConfigBlock("GTEPModelConfig") + + CONFIG.declare("include_planning", ConfigValue(default = True, domain=Bool, description="Enable inclusion of any investment options.")) + + CONFIG.declare("include_commitment", ConfigValue(default = True, domain=Bool, description="Include unit commitment formulation.")) + + CONFIG.declare("include_dispatch", ConfigValue(default = True, domain=Bool, description="Include economic dispatch formulation (i.e., OPF).")) + CONFIG.declare( "flow_model", ConfigValue( @@ -34,11 +41,11 @@ def _get_model_config(): ), ) CONFIG.declare( - "dispatch_randomizations", + "dispatch_randomization", ConfigValue( default=True, domain=Bool, - description="Introduces random dispatch information rather than having fixed values per-commitment period.", + description="Introduces random dispatch information rather than having fixed values per commitment period.", ), ) return CONFIG @@ -79,7 +86,7 @@ def _add_investment_configs(CONFIG): description="Include transmission investment options", ), ) - pass + def _add_solver_configs(CONFIG): diff --git a/gtep/gtep_model.py b/gtep/gtep_model.py index ce87db0..a8eab8a 100644 --- a/gtep/gtep_model.py +++ b/gtep/gtep_model.py @@ -125,7 +125,7 @@ def create_model(self): ) m.commitmentPeriodLength = Param(within=PositiveReals, default=1, units=u.hr) # TODO: index by dispatch period? Certainly index by commitment period - m.dispatchPeriodLength = Param(within=PositiveReals, default=15, units=u.min) + m.dispatchPeriodLength = Param(within=PositiveReals, default=0.25, units=u.hr) model_data_references(m) model_create_investment_stages(m, self.stages) @@ -253,21 +253,20 @@ def branchInvestStatus(disj, branch): # Renewable generator MW values (operational, installed, retired, extended) b.renewableOperational = Var( - m.renewableGenerators, within=NonNegativeReals, initialize=0 + m.renewableGenerators, within=NonNegativeReals, initialize=0, units = u.MW ) b.renewableInstalled = Var( - m.renewableGenerators, within=NonNegativeReals, initialize=0 + m.renewableGenerators, within=NonNegativeReals, initialize=0, units = u.MW ) b.renewableRetired = Var( - m.renewableGenerators, within=NonNegativeReals, initialize=0 + m.renewableGenerators, within=NonNegativeReals, initialize=0, units=u.MW ) b.renewableExtended = Var( - m.renewableGenerators, within=NonNegativeReals, initialize=0 + m.renewableGenerators, within=NonNegativeReals, initialize=0, units=u.MW ) # Track and accumulate costs and penalties - b.quotaDeficit = Var(within=NonNegativeReals, initialize=0, units=u.MW) - b.operatingCostInvestment = Var(within=Reals, initialize=0, units=u.USD) + b.quotaDeficit = Var(within=NonNegativeReals, initialize=0, units=u.MW * u.hr) b.expansionCost = Var(within=Reals, initialize=0, units=u.USD) b.renewableCurtailmentInvestment = Var( within=NonNegativeReals, initialize=0, units=u.USD @@ -379,7 +378,6 @@ def renewable_generation_requirement(b): for com_per in b.representativePeriod[rep_per].commitmentPeriods: renewableSurplusRepresentative += ( m.weights[rep_per] - * m.commitmentPeriodLength * b.representativePeriod[rep_per] .commitmentPeriod[com_per] .renewableSurplusCommitment @@ -397,13 +395,12 @@ def operatingCostInvestment(b): for com_per in b.representativePeriod[rep_per].commitmentPeriods: operatingCostRepresentative += ( m.weights[rep_per] - * m.commitmentPeriodLength * b.representativePeriod[rep_per] .commitmentPeriod[com_per] .operatingCostCommitment ) return m.investmentFactor[investment_stage] * operatingCostRepresentative - + # Investment costs for investment period ## FIXME: investment cost definition needs to be revisited AND possibly depends on ## data format. It is _rare_ for these values to be defined at all, let alone consistently. @@ -692,7 +689,7 @@ def spinning_reserve_limits(b, thermalGen): domain=NonNegativeReals, bounds=spinning_reserve_limits, initialize=0, - units=u.MW, + units=u.MW * u.hr, ) # Define bounds on thermal generator quickstart reserve supply @@ -707,7 +704,7 @@ def quickstart_reserve_limits(b, thermalGen): domain=NonNegativeReals, bounds=quickstart_reserve_limits, initialize=0, - units=u.MW, + units=u.MW * u.hr, ) @@ -1003,14 +1000,12 @@ def renewableSurplusCommitment(b): def operatingCostCommitment(b): return ( sum( - ## FIXME: update test objective value when this changes; ready to uncomment - # (m.dispatchPeriodLength / 60) * b.dispatchPeriod[disp_per].operatingCostDispatch for disp_per in b.dispatchPeriods ) + sum( m.fixedOperatingCost[gen] - # * m.thermalCapacity[gen] + * b.commitmentPeriodLength * ( b.genOn[gen].indicator_var.get_associated_binary() + b.genShutdown[gen].indicator_var.get_associated_binary() @@ -1020,7 +1015,7 @@ def operatingCostCommitment(b): ) ## FIXME: how do we do assign fixed operating costs to renewables; flat per location or per MW + sum( - m.fixedOperatingCost[gen] + m.fixedOperatingCost[gen] * b.commitmentPeriodLength # * m.renewableCapacity[gen] for gen in m.renewableGenerators ) @@ -1031,6 +1026,7 @@ def operatingCostCommitment(b): ) ) + # Define total curtailment for commitment block @b.Expression() def renewableCurtailmentCommitment(b): @@ -1051,6 +1047,7 @@ def commitment_period_rule(b, commitment_period): i_p = r_p.parent_block() b.commitmentPeriod = commitment_period + b.commitmentPeriodLength = Param(within=PositiveReals, default=1, units=u.hr) b.dispatchPeriods = RangeSet(m.numDispatchPeriods[r_p.currentPeriod]) b.carbonTax = Param(default=0) b.dispatchPeriod = Block(b.dispatchPeriods) @@ -1674,8 +1671,8 @@ def model_data_references(m): ## TODO: don't lazily approx NPV, add it into unit handling and calculate from actual time frames for stage in m.stages: m.investmentFactor[stage] *= 1 / ((1.04) ** (5 * stage)) - m.fixedOperatingCost = Param(m.generators, default=1, units=u.USD) - m.deficitPenalty = Param(m.stages, default=1, units=u.USD / u.MW) + m.fixedOperatingCost = Param(m.generators, default=1, units=u.USD / u.hr) + m.deficitPenalty = Param(m.stages, default=1, units=u.USD / (u.MW *u.hr)) # Amount of fuel required to be consumed for startup process for each generator m.startFuel = { @@ -1683,23 +1680,23 @@ def model_data_references(m): for gen in m.generators } + fuelCost = {} # Cost per unit of fuel at each generator if "RTS-GMLC" in m.md.data["system"]["name"]: - m.fuelCost = { - gen: m.md.data["elements"]["generator"][gen]["fuel_cost"] - for gen in m.thermalGenerators - } + for gen in m.thermalGenerators: + fuelCost[gen] = m.md.data["elements"]["generator"][gen]["fuel_cost"] + else: - m.fuelCost = { - gen: m.md.data["elements"]["generator"][gen]["p_cost"]["values"][1] - for gen in m.thermalGenerators - } + for gen in m.thermalGenerators: + fuelCost[gen] = m.md.data["elements"]["generator"][gen]["p_cost"]["values"][1] + + m.fuelCost = Param(m.thermalGenerators, initialize=fuelCost, units = u.USD / (u.MW * u.hr)) # Cost per MW of curtailed renewable energy # NOTE: what should this be valued at? This being both curtailment and load shed. # TODO: update valuations - m.curtailmentCost = 2 * max(m.fuelCost.values()) - m.loadShedCost = 1000 * m.curtailmentCost + m.curtailmentCost = Param(initialize = 2 * max(value(item) for item in m.fuelCost.values()), units = u.USD / (u.MW * u.hr)) + m.loadShedCost = Param(initialize = 1000 * m.curtailmentCost, units= u.USD / (u.MW * u.hr)) # Full lifecycle CO_2 emission factor for each generator m.emissionsFactor = { @@ -1709,15 +1706,19 @@ def model_data_references(m): # Flat startup cost for each generator if "RTS-GMLC" in m.md.data["system"]["name"]: - m.startupCost = { + startupCost = { gen: m.md.data["elements"]["generator"][gen]["non_fuel_startup_cost"] for gen in m.thermalGenerators } + m.startupCost = Param(m.thermalGenerators, initialize = startupCost, units = u.USD) else: - m.startupCost = { + startupCost = { gen: m.md.data["elements"]["generator"][gen]["startup_cost"] for gen in m.generators } + m.startupCost = Param(m.generators, initialize = startupCost, units = u.USD) + + # (Arbitrary) multiplier for new generator investments corresponds to depreciation schedules # for individual technologies; higher values are indicative of slow depreciation diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index d616225..c8285d2 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -103,9 +103,11 @@ def test_solve_bigm(self): TransformationFactory("gdp.bound_pretransformation").apply_to(modObject.model) TransformationFactory("gdp.bigm").apply_to(modObject.model) modObject.results = opt.solve(modObject.model) - modObject.model.pprint() + # modObject.model.pprint() + # modObject.model.total_cost_objective_rule.pprint() + # previous successful objective values: 9207.95, 6078.86 self.assertAlmostEqual( - value(modObject.model.total_cost_objective_rule), 6078.86, places=1 + value(modObject.model.total_cost_objective_rule), 9207.95, places=1 ) # Is it finally time to fix units self.assertEqual( From 53b6525357c957a7c32ddee8295e24b2bd62b9ee Mon Sep 17 00:00:00 2001 From: Kyle Skolfield Date: Mon, 25 Nov 2024 14:05:50 -0700 Subject: [PATCH 25/42] unbreaking unit test autosearch... --- gtep/{driver_config_test.py => driver_config_work.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename gtep/{driver_config_test.py => driver_config_work.py} (100%) diff --git a/gtep/driver_config_test.py b/gtep/driver_config_work.py similarity index 100% rename from gtep/driver_config_test.py rename to gtep/driver_config_work.py From 4cfa7658c3cb2aa26085859bd40f795a119c7bfd Mon Sep 17 00:00:00 2001 From: blnicho Date: Mon, 25 Nov 2024 15:33:41 -0700 Subject: [PATCH 26/42] Ran black --- gtep/config_options.py | 28 ++++++++++++++++++++++++---- gtep/gtep_model.py | 32 +++++++++++++++++++------------- 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/gtep/config_options.py b/gtep/config_options.py index 379ad43..4fa78a6 100644 --- a/gtep/config_options.py +++ b/gtep/config_options.py @@ -20,11 +20,32 @@ def _get_model_config(): CONFIG = ConfigBlock("GTEPModelConfig") - CONFIG.declare("include_planning", ConfigValue(default = True, domain=Bool, description="Enable inclusion of any investment options.")) + CONFIG.declare( + "include_planning", + ConfigValue( + default=True, + domain=Bool, + description="Enable inclusion of any investment options.", + ), + ) - CONFIG.declare("include_commitment", ConfigValue(default = True, domain=Bool, description="Include unit commitment formulation.")) + CONFIG.declare( + "include_commitment", + ConfigValue( + default=True, + domain=Bool, + description="Include unit commitment formulation.", + ), + ) - CONFIG.declare("include_dispatch", ConfigValue(default = True, domain=Bool, description="Include economic dispatch formulation (i.e., OPF).")) + CONFIG.declare( + "include_dispatch", + ConfigValue( + default=True, + domain=Bool, + description="Include economic dispatch formulation (i.e., OPF).", + ), + ) CONFIG.declare( "flow_model", @@ -88,6 +109,5 @@ def _add_investment_configs(CONFIG): ) - def _add_solver_configs(CONFIG): pass diff --git a/gtep/gtep_model.py b/gtep/gtep_model.py index 2ae067a..fc870d3 100644 --- a/gtep/gtep_model.py +++ b/gtep/gtep_model.py @@ -253,10 +253,10 @@ def branchInvestStatus(disj, branch): # Renewable generator MW values (operational, installed, retired, extended) b.renewableOperational = Var( - m.renewableGenerators, within=NonNegativeReals, initialize=0, units = u.MW + m.renewableGenerators, within=NonNegativeReals, initialize=0, units=u.MW ) b.renewableInstalled = Var( - m.renewableGenerators, within=NonNegativeReals, initialize=0, units = u.MW + m.renewableGenerators, within=NonNegativeReals, initialize=0, units=u.MW ) b.renewableRetired = Var( m.renewableGenerators, within=NonNegativeReals, initialize=0, units=u.MW @@ -400,7 +400,7 @@ def operatingCostInvestment(b): .operatingCostCommitment ) return m.investmentFactor[investment_stage] * operatingCostRepresentative - + # Investment costs for investment period ## FIXME: investment cost definition needs to be revisited AND possibly depends on ## data format. It is _rare_ for these values to be defined at all, let alone consistently. @@ -1025,7 +1025,6 @@ def operatingCostCommitment(b): ) ) - # Define total curtailment for commitment block @b.Expression() def renewableCurtailmentCommitment(b): @@ -1671,7 +1670,7 @@ def model_data_references(m): for stage in m.stages: m.investmentFactor[stage] *= 1 / ((1.04) ** (5 * stage)) m.fixedOperatingCost = Param(m.generators, default=1, units=u.USD / u.hr) - m.deficitPenalty = Param(m.stages, default=1, units=u.USD / (u.MW *u.hr)) + m.deficitPenalty = Param(m.stages, default=1, units=u.USD / (u.MW * u.hr)) # Amount of fuel required to be consumed for startup process for each generator m.startFuel = { @@ -1687,15 +1686,24 @@ def model_data_references(m): else: for gen in m.thermalGenerators: - fuelCost[gen] = m.md.data["elements"]["generator"][gen]["p_cost"]["values"][1] + fuelCost[gen] = m.md.data["elements"]["generator"][gen]["p_cost"]["values"][ + 1 + ] - m.fuelCost = Param(m.thermalGenerators, initialize=fuelCost, units = u.USD / (u.MW * u.hr)) + m.fuelCost = Param( + m.thermalGenerators, initialize=fuelCost, units=u.USD / (u.MW * u.hr) + ) # Cost per MW of curtailed renewable energy # NOTE: what should this be valued at? This being both curtailment and load shed. # TODO: update valuations - m.curtailmentCost = Param(initialize = 2 * max(value(item) for item in m.fuelCost.values()), units = u.USD / (u.MW * u.hr)) - m.loadShedCost = Param(initialize = 1000 * m.curtailmentCost, units= u.USD / (u.MW * u.hr)) + m.curtailmentCost = Param( + initialize=2 * max(value(item) for item in m.fuelCost.values()), + units=u.USD / (u.MW * u.hr), + ) + m.loadShedCost = Param( + initialize=1000 * m.curtailmentCost, units=u.USD / (u.MW * u.hr) + ) # Full lifecycle CO_2 emission factor for each generator m.emissionsFactor = { @@ -1709,15 +1717,13 @@ def model_data_references(m): gen: m.md.data["elements"]["generator"][gen]["non_fuel_startup_cost"] for gen in m.thermalGenerators } - m.startupCost = Param(m.thermalGenerators, initialize = startupCost, units = u.USD) + m.startupCost = Param(m.thermalGenerators, initialize=startupCost, units=u.USD) else: startupCost = { gen: m.md.data["elements"]["generator"][gen]["startup_cost"] for gen in m.generators } - m.startupCost = Param(m.generators, initialize = startupCost, units = u.USD) - - + m.startupCost = Param(m.generators, initialize=startupCost, units=u.USD) # (Arbitrary) multiplier for new generator investments corresponds to depreciation schedules # for individual technologies; higher values are indicative of slow depreciation From 71e39f9c6eddb4b26ecfca59ec950cfa1262d132 Mon Sep 17 00:00:00 2001 From: blnicho Date: Mon, 25 Nov 2024 15:59:26 -0700 Subject: [PATCH 27/42] Removing unnecessary pass statement --- gtep/driver_jsc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/gtep/driver_jsc.py b/gtep/driver_jsc.py index e99f78c..ed03289 100644 --- a/gtep/driver_jsc.py +++ b/gtep/driver_jsc.py @@ -52,5 +52,3 @@ plot_results = True if plot_results: sol_object.plot_levels(save_dir="./plots/") - -pass From b459b03595094599cc8216bdaae7801d98aceb34 Mon Sep 17 00:00:00 2001 From: Kyle Skolfield Date: Mon, 25 Nov 2024 16:44:16 -0700 Subject: [PATCH 28/42] fixing rng setting for deterministic results with multiple model creation events --- gtep/gtep_model.py | 7 ++++--- gtep/tests/unit/test_gtep_model.py | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/gtep/gtep_model.py b/gtep/gtep_model.py index a8eab8a..5b179a9 100644 --- a/gtep/gtep_model.py +++ b/gtep/gtep_model.py @@ -27,7 +27,7 @@ # based on automatic pyomo unit transformations u.load_definitions_from_strings(["USD = [currency]"]) -rng = np.random.default_rng(seed=123186) + #################################### ########## New Work Here ########### @@ -73,7 +73,7 @@ def __init__( :param num_dispatch: integer number of dispatch periods per commitment period :return: Pyomo model for full GTEP """ - + self.stages = stages self.formulation = formulation self.data = data @@ -89,6 +89,7 @@ def create_model(self): self.timer.tic("Creating GTEP Model") m = ConcreteModel() + m.rng = np.random.default_rng(seed=123186) ## TODO: checks for active/built/inactive/unbuilt/etc. gen ## NOTE: scale_ModelData_to_pu doesn't account for expansion data -- does it need to? @@ -716,7 +717,7 @@ def add_dispatch_constraints(b, disp_per): i_p = r_p.parent_block() for key in m.loads.keys(): - m.loads[key] *= max(0, rng.normal(0.5, 0.2)) + m.loads[key] *= max(0, m.rng.normal(0.5, 0.2)) # Energy balance constraint @b.Constraint(m.buses) diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index c8285d2..a454bdb 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -107,7 +107,7 @@ def test_solve_bigm(self): # modObject.model.total_cost_objective_rule.pprint() # previous successful objective values: 9207.95, 6078.86 self.assertAlmostEqual( - value(modObject.model.total_cost_objective_rule), 9207.95, places=1 + value(modObject.model.total_cost_objective_rule), 6078.86, places=1 ) # Is it finally time to fix units self.assertEqual( From 85626a1f1304de771d7cd61b4801172e7ec98386 Mon Sep 17 00:00:00 2001 From: blnicho Date: Mon, 25 Nov 2024 16:55:03 -0700 Subject: [PATCH 29/42] Running black --- gtep/gtep_model.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gtep/gtep_model.py b/gtep/gtep_model.py index 8e57243..7c96248 100644 --- a/gtep/gtep_model.py +++ b/gtep/gtep_model.py @@ -28,7 +28,6 @@ u.load_definitions_from_strings(["USD = [currency]"]) - #################################### ########## New Work Here ########### #################################### @@ -73,7 +72,7 @@ def __init__( :param num_dispatch: integer number of dispatch periods per commitment period :return: Pyomo model for full GTEP """ - + self.stages = stages self.formulation = formulation self.data = data From 8de4ae8d83d8bf7619915a9ee49473e0a1b8a63f Mon Sep 17 00:00:00 2001 From: Kyle Skolfield Date: Mon, 2 Dec 2024 20:19:45 -0700 Subject: [PATCH 30/42] minor config additions; running Black --- gtep/config_options.py | 47 +++++++++-- gtep/driver.py | 7 +- gtep/driver_config_work.py | 15 ++-- gtep/driver_jsc.py | 6 +- gtep/gtep_model.py | 123 +++++++++++++---------------- gtep/gtep_solution.py | 35 +++----- gtep/tests/unit/test_validation.py | 19 +++-- gtep/validation.py | 78 +++++++++++++----- setup.py | 9 +-- 9 files changed, 187 insertions(+), 152 deletions(-) diff --git a/gtep/config_options.py b/gtep/config_options.py index 379ad43..82272ff 100644 --- a/gtep/config_options.py +++ b/gtep/config_options.py @@ -20,11 +20,32 @@ def _get_model_config(): CONFIG = ConfigBlock("GTEPModelConfig") - CONFIG.declare("include_planning", ConfigValue(default = True, domain=Bool, description="Enable inclusion of any investment options.")) + CONFIG.declare( + "include_planning", + ConfigValue( + default=True, + domain=Bool, + description="Enable inclusion of any investment options.", + ), + ) - CONFIG.declare("include_commitment", ConfigValue(default = True, domain=Bool, description="Include unit commitment formulation.")) + CONFIG.declare( + "include_commitment", + ConfigValue( + default=True, + domain=Bool, + description="Include unit commitment formulation.", + ), + ) - CONFIG.declare("include_dispatch", ConfigValue(default = True, domain=Bool, description="Include economic dispatch formulation (i.e., OPF).")) + CONFIG.declare( + "include_dispatch", + ConfigValue( + default=True, + domain=Bool, + description="Include economic dispatch formulation (i.e., OPF).", + ), + ) CONFIG.declare( "flow_model", @@ -34,12 +55,21 @@ def _get_model_config(): description="Power flow approximation to use.", ), ) + + CONFIG.declare( + "time_period_subsets", + ConfigList( + description="Time period counts for fixed-length and fixed-subset periods." + ), + ) + CONFIG.declare( "time_period_dict", ConfigDict( description="Time period dict, specified as \{(investment period #, length): \{(commitment period #, length): \{dispatch period #: length\}\}\}" ), ) + CONFIG.declare( "dispatch_randomization", ConfigValue( @@ -52,7 +82,15 @@ def _get_model_config(): def _add_common_configs(CONFIG): - pass + + CONFIG.declare( + "scale_loads", + ConfigValue( + default=False, + domain=Bool, + description="Allow scaling of load values into future years; i.e., load scaling is represented in the model but not the data.", + ), + ) def _add_investment_configs(CONFIG): @@ -88,6 +126,5 @@ def _add_investment_configs(CONFIG): ) - def _add_solver_configs(CONFIG): pass diff --git a/gtep/driver.py b/gtep/driver.py index 578bd36..1c73889 100644 --- a/gtep/driver.py +++ b/gtep/driver.py @@ -12,12 +12,7 @@ data_object.load_prescient(data_path) mod_object = ExpansionPlanningModel( - stages=2, - data=data_object.md, - num_reps=2, - len_reps=1, - num_commit=6, - num_dispatch=4, + stages=2, data=data_object.md, num_reps=2, len_reps=1, num_commit=6, num_dispatch=4 ) mod_object.create_model() TransformationFactory("gdp.bound_pretransformation").apply_to(mod_object.model) diff --git a/gtep/driver_config_work.py b/gtep/driver_config_work.py index 899f9ae..76d703d 100644 --- a/gtep/driver_config_work.py +++ b/gtep/driver_config_work.py @@ -12,20 +12,15 @@ data_object.load_prescient(data_path) - mod_object = ExpansionPlanningModel( - stages=1, - data=data_object.md, - num_reps=1, - len_reps=1, - num_commit=24, - num_dispatch=4, + stages=1, data=data_object.md, num_reps=1, len_reps=1, num_commit=24, num_dispatch=4 ) -for k,v in mod_object.config.items(): - ic(k,v) +for k, v in mod_object.config.items(): + print(f"k: {k}", f"v: {v}") -quit() + +exit() mod_object.create_model() diff --git a/gtep/driver_jsc.py b/gtep/driver_jsc.py index 4ab5bac..e99f78c 100644 --- a/gtep/driver_jsc.py +++ b/gtep/driver_jsc.py @@ -33,7 +33,7 @@ opt = Highs() # opt = SolverFactory("gurobi", solver_io="python") # opt = Gurobi() -#mod_object.results = opt.solve(mod_object.model, tee=True) +# mod_object.results = opt.solve(mod_object.model, tee=True) mod_object.results = opt.solve(mod_object.model) @@ -48,9 +48,9 @@ if load_numerical_results: sol_object.read_json("./gtep_solution_jscTest.json") # sol_object.read_json("./gtep_solution.json") - #sol_object.read_json("./bigger_longer_wigglier_gtep_solution.json") + # sol_object.read_json("./bigger_longer_wigglier_gtep_solution.json") plot_results = True if plot_results: sol_object.plot_levels(save_dir="./plots/") -pass \ No newline at end of file +pass diff --git a/gtep/gtep_model.py b/gtep/gtep_model.py index 5b179a9..4f4c8e9 100644 --- a/gtep/gtep_model.py +++ b/gtep/gtep_model.py @@ -4,7 +4,7 @@ # date: 01/04/2024 # Model available at http://www.optimization-online.org/DB_FILE/2017/08/6162.pdf -from pyomo.environ import * +from pyomo.environ import * from pyomo.environ import units as u # from pyomo.gdp import * @@ -28,7 +28,6 @@ u.load_definitions_from_strings(["USD = [currency]"]) - #################################### ########## New Work Here ########### #################################### @@ -73,7 +72,7 @@ def __init__( :param num_dispatch: integer number of dispatch periods per commitment period :return: Pyomo model for full GTEP """ - + self.stages = stages self.formulation = formulation self.data = data @@ -174,10 +173,7 @@ def report_large_coefficients(self, outfile, magnitude_cutoff=1e5): #################################### -def add_investment_variables( - b, - investment_stage, -): +def add_investment_variables(b, investment_stage): """Add variables to investment stage block. :param b: Investment block @@ -254,10 +250,10 @@ def branchInvestStatus(disj, branch): # Renewable generator MW values (operational, installed, retired, extended) b.renewableOperational = Var( - m.renewableGenerators, within=NonNegativeReals, initialize=0, units = u.MW + m.renewableGenerators, within=NonNegativeReals, initialize=0, units=u.MW ) b.renewableInstalled = Var( - m.renewableGenerators, within=NonNegativeReals, initialize=0, units = u.MW + m.renewableGenerators, within=NonNegativeReals, initialize=0, units=u.MW ) b.renewableRetired = Var( m.renewableGenerators, within=NonNegativeReals, initialize=0, units=u.MW @@ -274,10 +270,7 @@ def branchInvestStatus(disj, branch): ) -def add_investment_constraints( - b, - investment_stage, -): +def add_investment_constraints(b, investment_stage): """Add standard inequalities (i.e., those not involving disjunctions) to investment stage block.""" m = b.model() @@ -401,7 +394,7 @@ def operatingCostInvestment(b): .operatingCostCommitment ) return m.investmentFactor[investment_stage] * operatingCostRepresentative - + # Investment costs for investment period ## FIXME: investment cost definition needs to be revisited AND possibly depends on ## data format. It is _rare_ for these values to be defined at all, let alone consistently. @@ -468,10 +461,7 @@ def renewable_curtailment_cost(b): ) -def add_dispatch_variables( - b, - dispatch_period, -): +def add_dispatch_variables(b, dispatch_period): """Add dispatch-associated variables to representative period block.""" m = b.model() @@ -560,10 +550,7 @@ def loadShedCost(b, bus): # Define bounds on transmission line capacity - restrictions on flow over # uninvested lines are enforced in a disjuction below def power_flow_limits(b, branch): - return ( - -m.transmissionCapacity[branch], - m.transmissionCapacity[branch], - ) + return (-m.transmissionCapacity[branch], m.transmissionCapacity[branch]) # NOTE: this is an abuse of units and needs to be fixed for variable temporal resolution b.powerFlow = Var( @@ -617,23 +604,28 @@ def delta_bus_angle_rule(disj): # def max_delta_bus_angle(disj): # return abs(disj.deltaBusAngle) <= math.pi/6 - @disj.Constraint() - def dc_power_flow(disj): - fb = m.transmission[branch]["from_bus"] - tb = m.transmission[branch]["to_bus"] - reactance = m.md.data["elements"]["branch"][branch]["reactance"] - if m.md.data["elements"]["branch"][branch]["branch_type"] == "transformer": - reactance *= m.md.data["elements"]["branch"][branch][ - "transformer_tap_ratio" - ] - shift = m.md.data["elements"]["branch"][branch][ - "transformer_phase_shift" - ] - else: - shift = 0 - return b.powerFlow[branch] == (-1 / reactance) * ( - disj.busAngle[tb] - disj.busAngle[fb] + shift - ) + if m.config["flow_model"] == "DC": + + @disj.Constraint() + def dc_power_flow(disj): + fb = m.transmission[branch]["from_bus"] + tb = m.transmission[branch]["to_bus"] + reactance = m.md.data["elements"]["branch"][branch]["reactance"] + if ( + m.md.data["elements"]["branch"][branch]["branch_type"] + == "transformer" + ): + reactance *= m.md.data["elements"]["branch"][branch][ + "transformer_tap_ratio" + ] + shift = m.md.data["elements"]["branch"][branch][ + "transformer_phase_shift" + ] + else: + shift = 0 + return b.powerFlow[branch] == (-1 / reactance) * ( + disj.busAngle[tb] - disj.busAngle[fb] + shift + ) @b.Disjunct(m.transmission) def branchNotInUse(disj, branch): @@ -650,10 +642,7 @@ def dc_power_flow(disj): # provide the basis for transmission switching in the future @b.Disjunction(m.transmission) def branchInUseStatus(disj, branch): - return [ - disj.branchInUse[branch], - disj.branchNotInUse[branch], - ] + return [disj.branchInUse[branch], disj.branchNotInUse[branch]] # JSC update - If a branch is in use, it must be active # Update this when switching is implemented @@ -754,7 +743,6 @@ def capacity_factor(b, renewableGen): == m.renewableCapacity[renewableGen] ) - ## TODO: (@jkskolf) add renewableExtended to this and anywhere else @b.Constraint(m.renewableGenerators) def operational_renewables_only(b, renewableGen): @@ -976,10 +964,7 @@ def commit_active_gens_only(b, generator): ) -def add_commitment_constraints( - b, - comm_per, -): +def add_commitment_constraints(b, comm_per): """Add commitment-associated disjunctions and constraints to representative period block.""" m = b.model() r_p = b.parent_block() @@ -1027,7 +1012,6 @@ def operatingCostCommitment(b): ) ) - # Define total curtailment for commitment block @b.Expression() def renewableCurtailmentCommitment(b): @@ -1383,10 +1367,7 @@ def consistent_commitment_start_after_downtime(b, commitmentPeriod, thermalGen): ) -def representative_period_rule( - b, - representative_period, -): +def representative_period_rule(b, representative_period): """Create representative period block. :b: Representative period block @@ -1404,10 +1385,7 @@ def representative_period_rule( add_representative_period_constraints(b, representative_period) -def investment_stage_rule( - b, - investment_stage, -): +def investment_stage_rule(b, investment_stage): """Creates investment stage block. :b: Investment block @@ -1501,8 +1479,7 @@ def model_set_declaration(m, stages, rep_per=["a", "b"], com_per=2, dis_per=2): } m.generators = Set( - initialize=m.md.data["elements"]["generator"].keys(), - doc="All generators", + initialize=m.md.data["elements"]["generator"].keys(), doc="All generators" ) m.thermalGenerators = Set( @@ -1538,8 +1515,7 @@ def model_set_declaration(m, stages, rep_per=["a", "b"], com_per=2, dis_per=2): m.stages = RangeSet(stages, doc="Set of planning periods") m.representativePeriods = Set( - initialize=rep_per, - doc="Set of representative periods for each planning period", + initialize=rep_per, doc="Set of representative periods for each planning period" ) @@ -1673,7 +1649,7 @@ def model_data_references(m): for stage in m.stages: m.investmentFactor[stage] *= 1 / ((1.04) ** (5 * stage)) m.fixedOperatingCost = Param(m.generators, default=1, units=u.USD / u.hr) - m.deficitPenalty = Param(m.stages, default=1, units=u.USD / (u.MW *u.hr)) + m.deficitPenalty = Param(m.stages, default=1, units=u.USD / (u.MW * u.hr)) # Amount of fuel required to be consumed for startup process for each generator m.startFuel = { @@ -1689,15 +1665,24 @@ def model_data_references(m): else: for gen in m.thermalGenerators: - fuelCost[gen] = m.md.data["elements"]["generator"][gen]["p_cost"]["values"][1] + fuelCost[gen] = m.md.data["elements"]["generator"][gen]["p_cost"]["values"][ + 1 + ] - m.fuelCost = Param(m.thermalGenerators, initialize=fuelCost, units = u.USD / (u.MW * u.hr)) + m.fuelCost = Param( + m.thermalGenerators, initialize=fuelCost, units=u.USD / (u.MW * u.hr) + ) # Cost per MW of curtailed renewable energy # NOTE: what should this be valued at? This being both curtailment and load shed. # TODO: update valuations - m.curtailmentCost = Param(initialize = 2 * max(value(item) for item in m.fuelCost.values()), units = u.USD / (u.MW * u.hr)) - m.loadShedCost = Param(initialize = 1000 * m.curtailmentCost, units= u.USD / (u.MW * u.hr)) + m.curtailmentCost = Param( + initialize=2 * max(value(item) for item in m.fuelCost.values()), + units=u.USD / (u.MW * u.hr), + ) + m.loadShedCost = Param( + initialize=1000 * m.curtailmentCost, units=u.USD / (u.MW * u.hr) + ) # Full lifecycle CO_2 emission factor for each generator m.emissionsFactor = { @@ -1711,15 +1696,13 @@ def model_data_references(m): gen: m.md.data["elements"]["generator"][gen]["non_fuel_startup_cost"] for gen in m.thermalGenerators } - m.startupCost = Param(m.thermalGenerators, initialize = startupCost, units = u.USD) + m.startupCost = Param(m.thermalGenerators, initialize=startupCost, units=u.USD) else: startupCost = { gen: m.md.data["elements"]["generator"][gen]["startup_cost"] for gen in m.generators } - m.startupCost = Param(m.generators, initialize = startupCost, units = u.USD) - - + m.startupCost = Param(m.generators, initialize=startupCost, units=u.USD) # (Arbitrary) multiplier for new generator investments corresponds to depreciation schedules # for individual technologies; higher values are indicative of slow depreciation diff --git a/gtep/gtep_solution.py b/gtep/gtep_solution.py index d2cf53a..ff51f1f 100644 --- a/gtep/gtep_solution.py +++ b/gtep/gtep_solution.py @@ -177,9 +177,7 @@ def nested_set(this_dict, key, val): split_name = key.split(".") # start at the bottom and nest accordingly - tmp_dict = { - "value": val, - } + tmp_dict = {"value": val} # allocate the nested dictionary def nested_set(this_dict, key, val): @@ -466,9 +464,7 @@ def _plot_workhose_binaries( ax_bins.xaxis.set_major_locator(MaxNLocator(integer=True)) for axline_ix in range(total_height): ax_bins.axhline( - axline_ix + 0.5, - color="grey", - linewidth=3, + axline_ix + 0.5, color="grey", linewidth=3 ) # draw a seperator line between each level for axline_ix in range(len(df[level_key])): ax_bins.axvline( @@ -481,12 +477,7 @@ def _plot_workhose_binaries( for ix_key, this_koi in enumerate(keys): # make a dummy line to steal the color cycler and make a single item for the legend - (line,) = ax_bins.plot( - [None], - [None], - label=f"{this_koi}", - linewidth=5, - ) + (line,) = ax_bins.plot([None], [None], label=f"{this_koi}", linewidth=5) for ix_var, this_voi in enumerate(vars): for tx, is_it_on in zip( df[level_key], df[f"{this_koi}_{this_voi}_value"] @@ -586,13 +577,7 @@ def _level_relationship_df_to_plot( vars = config["order_branch_invest_state"] self._plot_workhose_binaries( - level_key, - df, - keys, - vars, - parent_key_string, - pretty_title, - save_dir, + level_key, df, keys, vars, parent_key_string, pretty_title, save_dir ) else: @@ -807,10 +792,11 @@ def _level_plot_workhorse( tmp_koi = sorted(keys_of_interest) # make a df for debug and also easy tabularness for plots - this_df_of_interest, this_df_units = ( - self._level_relationship_dict_to_df_workhorse( - level_key, level_timeseries, tmp_koi, tmp_voi - ) + ( + this_df_of_interest, + this_df_units, + ) = self._level_relationship_dict_to_df_workhorse( + level_key, level_timeseries, tmp_koi, tmp_voi ) # check if we got anything in the df @@ -914,7 +900,6 @@ def draw_single_edge_flow( norm=Normalize(vmin=None, vmax=None), glyph_type="custom", ): - def generate_flow_glyphs( num_glyphs, spacing=0.05, @@ -1045,7 +1030,7 @@ def generate_flow_glyphs( ) # go to home flow_glyphs.append( - PathPatch(mpath.Path(verts, codes), ec="none"), + PathPatch(mpath.Path(verts, codes), ec="none") ) rotation_transofrm = Affine2D().rotate_around( diff --git a/gtep/tests/unit/test_validation.py b/gtep/tests/unit/test_validation.py index 19e01ef..99bb450 100644 --- a/gtep/tests/unit/test_validation.py +++ b/gtep/tests/unit/test_validation.py @@ -6,11 +6,17 @@ from pyomo.contrib.appsi.solvers.highs import Highs import logging -from gtep.validation import clone_timeseries, filter_pointers, populate_generators, populate_transmission +from gtep.validation import ( + clone_timeseries, + filter_pointers, + populate_generators, + populate_transmission, +) input_data_source = "./gtep/data/5bus" output_data_source = "./gtep/tests/data/5bus_out" + def test_solution(): data_object = ExpansionPlanningData() data_object.load_prescient(input_data_source) @@ -24,26 +30,27 @@ def test_solution(): num_dispatch=4, ) mod_object.create_model() - TransformationFactory("gdp.bound_pretransformation").apply_to(mod_object.model) - TransformationFactory("gdp.bigm").apply_to(mod_object.model) + TransformationFactory("gdp.bound_pretransformation").apply_to(mod_object.model) + TransformationFactory("gdp.bigm").apply_to(mod_object.model) # opt = SolverFactory("gurobi") # opt = Gurobi() opt = Highs() # # mod_object.results = opt.solve(mod_object.model, tee=True) - mod_object.results = opt.solve(mod_object.model) + mod_object.results = opt.solve(mod_object.model) sol_object = ExpansionPlanningSolution() sol_object.load_from_model(mod_object) sol_object.dump_json("./gtep/tests/test_solution.json") return sol_object + solution = test_solution() + class TestValidation(unittest.TestCase): def test_populate_generators(self): populate_generators(input_data_source, solution, output_data_source) - def test_populate_transmission(self): populate_transmission(input_data_source, solution, output_data_source) @@ -51,4 +58,4 @@ def test_filter_pointers(self): filter_pointers(input_data_source, output_data_source) def test_clone_timeseries(self): - clone_timeseries(input_data_source, output_data_source) \ No newline at end of file + clone_timeseries(input_data_source, output_data_source) diff --git a/gtep/validation.py b/gtep/validation.py index 96782a7..8a15800 100644 --- a/gtep/validation.py +++ b/gtep/validation.py @@ -10,7 +10,10 @@ logger = logging.getLogger(__name__) -def populate_generators(data_input_path: str, sol_object: ExpansionPlanningSolution, data_output_path: str): + +def populate_generators( + data_input_path: str, sol_object: ExpansionPlanningSolution, data_output_path: str +): # load existing and candidate generators from initial prescient data # note that -c in name indicates candidate input_df = pd.read_csv(data_input_path + "/gen.csv") @@ -19,25 +22,45 @@ def populate_generators(data_input_path: str, sol_object: ExpansionPlanningSolut # for thermal: # generator should exist in future grid if the status in the final stage # is installed, operational, or extended - + def gen_name_filter(gen_name): - return 'gen' in gen_name and ('Ext' in gen_name or 'Ope' in gen_name or 'Ins' in gen_name) + return 'gen' in gen_name and ( + 'Ext' in gen_name or 'Ope' in gen_name or 'Ins' in gen_name + ) + solution_dict = sol_object._to_dict()['results']['primals_tree'] end_investment_stage = list(solution_dict.keys())[0] - end_investment_solution_dict = {k: v['value'] for k,v in solution_dict[end_investment_stage].items() if gen_name_filter(k) and v['value'] > 0.5} - end_investment_gens = [re.search(r'\[.*\]', k).group(0)[1:-1] for k in end_investment_solution_dict.keys()] - + end_investment_solution_dict = { + k: v['value'] + for k, v in solution_dict[end_investment_stage].items() + if gen_name_filter(k) and v['value'] > 0.5 + } + end_investment_gens = [ + re.search(r'\[.*\]', k).group(0)[1:-1] + for k in end_investment_solution_dict.keys() + ] + # for renewable: # total capacity should be installed + operational + extended values def renewable_name_filter(gen_name): - return 'renew' in gen_name and ('Ext' in gen_name or 'Ope' in gen_name or 'Ins' in gen_name) - end_investment_renewable_dict = {k: v['value'] for k,v in solution_dict[end_investment_stage].items() if renewable_name_filter(k)} - end_investment_renewable_gens = {re.search(r'\[.*\]', k).group(0)[1:-1]: 0 for k in end_investment_renewable_dict.keys()} - for k,v in end_investment_renewable_dict.items(): + return 'renew' in gen_name and ( + 'Ext' in gen_name or 'Ope' in gen_name or 'Ins' in gen_name + ) + + end_investment_renewable_dict = { + k: v['value'] + for k, v in solution_dict[end_investment_stage].items() + if renewable_name_filter(k) + } + end_investment_renewable_gens = { + re.search(r'\[.*\]', k).group(0)[1:-1]: 0 + for k in end_investment_renewable_dict.keys() + } + for k, v in end_investment_renewable_dict.items(): end_investment_renewable_gens[re.search(r'\[.*\]', k).group(0)[1:-1]] += v for k, v in end_investment_renewable_gens.items(): ## NOTE: (@jkskolf) this will break in pandas 3.0 - input_df["PMax MW"].mask(input_df['GEN UID'] == k,v, inplace=True) + input_df["PMax MW"].mask(input_df['GEN UID'] == k, v, inplace=True) end_investment_gens += [k for k in end_investment_renewable_gens.keys()] # populate output dataframe @@ -46,7 +69,8 @@ def renewable_name_filter(gen_name): # TODO: (@jkskolf) should we update prices here? I think no, but ... if not os.path.exists(data_output_path): os.makedirs(data_output_path) - output_df.to_csv(data_output_path + '/gen.csv',index=False) + output_df.to_csv(data_output_path + '/gen.csv', index=False) + def populate_transmission(data_input_path, sol_object, data_output_path): # load existing and candidate generators from initial prescient data @@ -55,16 +79,27 @@ def populate_transmission(data_input_path, sol_object, data_output_path): # pull final stage solution variables for transmission def branch_name_filter(gen_name): - return 'bran' in gen_name and ('Ext' in gen_name or 'Ope' in gen_name or 'Ins' in gen_name) + return 'bran' in gen_name and ( + 'Ext' in gen_name or 'Ope' in gen_name or 'Ins' in gen_name + ) + solution_dict = sol_object._to_dict()['results']['primals_tree'] end_investment_stage = list(solution_dict.keys())[0] - end_investment_solution_dict = {k: v['value'] for k,v in solution_dict[end_investment_stage].items() if branch_name_filter(k) and v['value'] > 0.5} - end_investment_branches = [re.search(r'\[.*\]', k).group(0)[1:-1] for k in end_investment_solution_dict.keys()] + end_investment_solution_dict = { + k: v['value'] + for k, v in solution_dict[end_investment_stage].items() + if branch_name_filter(k) and v['value'] > 0.5 + } + end_investment_branches = [ + re.search(r'\[.*\]', k).group(0)[1:-1] + for k in end_investment_solution_dict.keys() + ] output_df = input_df[input_df['UID'].isin(end_investment_branches)] - + if not os.path.exists(data_output_path): os.makedirs(data_output_path) - output_df.to_csv(data_output_path + '/branch.csv' ,index=False) + output_df.to_csv(data_output_path + '/branch.csv', index=False) + def filter_pointers(data_input_path, data_output_path): # load initial timeseries pointers @@ -78,12 +113,17 @@ def filter_pointers(data_input_path, data_output_path): # keep generators that exist at the final investment stage and remove the rest # keep all non-generator timeseries pointers matching_gen_list = [gen for gen in output_generators_df['GEN UID']] - output_df = input_pointers_df[input_pointers_df['Object'].isin(matching_gen_list) | input_pointers_df['Category'] != 'Generator'] + output_df = input_pointers_df[ + input_pointers_df['Object'].isin(matching_gen_list) + | input_pointers_df['Category'] + != 'Generator' + ] if not os.path.exists(data_output_path): os.makedirs(data_output_path) output_df.to_csv(data_output_path + '/timeseries_pointers.csv') + def clone_timeseries(data_input_path, data_output_path): file_list = os.listdir(data_input_path) file_list.remove('timeseries_pointers.csv') @@ -92,4 +132,4 @@ def clone_timeseries(data_input_path, data_output_path): # @jkskolf, I don't think I like this ... for fname in file_list: - shutil.copy(data_input_path + "/" + fname, data_output_path + "/" + fname) \ No newline at end of file + shutil.copy(data_input_path + "/" + fname, data_output_path + "/" + fname) diff --git a/setup.py b/setup.py index b2e0966..c641d54 100644 --- a/setup.py +++ b/setup.py @@ -10,14 +10,7 @@ # "pint", # ] -requires = [ - "pyomo", - "scipy", - "gridx-egret", - "gridx-prescient", - "anyio==3.1", - "pint", -] +requires = ["pyomo", "scipy", "gridx-egret", "gridx-prescient", "anyio==3.1", "pint"] setup( name="gtep", version="0.1.dev0", From 181b64c7258ed7c120c44b2d18d71a399564c754 Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 11 Dec 2024 14:08:11 -0700 Subject: [PATCH 31/42] Ran black --- gtep/tests/unit/test_validation.py | 1 - gtep/validation.py | 33 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/gtep/tests/unit/test_validation.py b/gtep/tests/unit/test_validation.py index f35747f..b574a35 100644 --- a/gtep/tests/unit/test_validation.py +++ b/gtep/tests/unit/test_validation.py @@ -19,7 +19,6 @@ output_data_source = abspath(join(curr_dir, "..", "..", "data", "5bus_out")) - def test_solution(): data_object = ExpansionPlanningData() data_object.load_prescient(input_data_source) diff --git a/gtep/validation.py b/gtep/validation.py index 0a2ca72..7e0dfd8 100644 --- a/gtep/validation.py +++ b/gtep/validation.py @@ -24,43 +24,43 @@ def populate_generators( # is installed, operational, or extended def gen_name_filter(gen_name): - return 'gen' in gen_name and ( - 'Ext' in gen_name or 'Ope' in gen_name or 'Ins' in gen_name + return "gen" in gen_name and ( + "Ext" in gen_name or "Ope" in gen_name or "Ins" in gen_name ) - solution_dict = sol_object._to_dict()['results']['primals_tree'] + solution_dict = sol_object._to_dict()["results"]["primals_tree"] end_investment_stage = list(solution_dict.keys())[0] end_investment_solution_dict = { - k: v['value'] + k: v["value"] for k, v in solution_dict[end_investment_stage].items() - if gen_name_filter(k) and v['value'] > 0.5 + if gen_name_filter(k) and v["value"] > 0.5 } end_investment_gens = [ - re.search(r'\[.*\]', k).group(0)[1:-1] + re.search(r"\[.*\]", k).group(0)[1:-1] for k in end_investment_solution_dict.keys() ] # for renewable: # total capacity should be installed + operational + extended values def renewable_name_filter(gen_name): - return 'renew' in gen_name and ( - 'Ext' in gen_name or 'Ope' in gen_name or 'Ins' in gen_name + return "renew" in gen_name and ( + "Ext" in gen_name or "Ope" in gen_name or "Ins" in gen_name ) end_investment_renewable_dict = { - k: v['value'] + k: v["value"] for k, v in solution_dict[end_investment_stage].items() if renewable_name_filter(k) } end_investment_renewable_gens = { - re.search(r'\[.*\]', k).group(0)[1:-1]: 0 + re.search(r"\[.*\]", k).group(0)[1:-1]: 0 for k in end_investment_renewable_dict.keys() } for k, v in end_investment_renewable_dict.items(): - end_investment_renewable_gens[re.search(r'\[.*\]', k).group(0)[1:-1]] += v + end_investment_renewable_gens[re.search(r"\[.*\]", k).group(0)[1:-1]] += v for k, v in end_investment_renewable_gens.items(): ## NOTE: (@jkskolf) this will break in pandas 3.0 - input_df["PMax MW"].mask(input_df['GEN UID'] == k, v, inplace=True) + input_df["PMax MW"].mask(input_df["GEN UID"] == k, v, inplace=True) end_investment_gens += [k for k in end_investment_renewable_gens.keys()] # populate output dataframe @@ -114,11 +114,11 @@ def filter_pointers(data_input_path, data_output_path): # keep generators that exist at the final investment stage and remove the rest # keep all non-generator timeseries pointers - matching_gen_list = [gen for gen in output_generators_df['GEN UID']] + matching_gen_list = [gen for gen in output_generators_df["GEN UID"]] output_df = input_pointers_df[ - input_pointers_df['Object'].isin(matching_gen_list) - | input_pointers_df['Category'] - != 'Generator' + input_pointers_df["Object"].isin(matching_gen_list) + | input_pointers_df["Category"] + != "Generator" ] if not os.path.exists(data_output_path): @@ -126,7 +126,6 @@ def filter_pointers(data_input_path, data_output_path): output_df.to_csv(os.path.join(data_output_path, "timeseries_pointers.csv")) - def clone_timeseries(data_input_path, data_output_path): if not os.path.exists(data_output_path): From 35265fdeed1fbc7c653e038569019050ab0caf2e Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 11 Dec 2024 15:12:37 -0700 Subject: [PATCH 32/42] Debugging validation test --- gtep/tests/unit/test_validation.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gtep/tests/unit/test_validation.py b/gtep/tests/unit/test_validation.py index b574a35..4ec2f7f 100644 --- a/gtep/tests/unit/test_validation.py +++ b/gtep/tests/unit/test_validation.py @@ -50,14 +50,16 @@ def test_solution(): class TestValidation(unittest.TestCase): - def test_populate_generators(self): + def test_populate_generators_filter_pointers(self): populate_generators(input_data_source, solution, output_data_source) + # filter_pointers needs to access the gen.csv file created in populate_generators + filter_pointers(input_data_source, output_data_source) def test_populate_transmission(self): populate_transmission(input_data_source, solution, output_data_source) - def test_filter_pointers(self): - filter_pointers(input_data_source, output_data_source) + # def test_filter_pointers(self): + # filter_pointers(input_data_source, output_data_source) def test_clone_timeseries(self): clone_timeseries(input_data_source, output_data_source) From 80806c8170fc312be64d42faf874d30b6a727d1a Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 11 Dec 2024 15:14:46 -0700 Subject: [PATCH 33/42] Fixing typo --- gtep/gtep_solution.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gtep/gtep_solution.py b/gtep/gtep_solution.py index 8c91142..14ad8eb 100644 --- a/gtep/gtep_solution.py +++ b/gtep/gtep_solution.py @@ -473,7 +473,7 @@ def _plot_workhose_binaries( linewidth=3, linestyle="dotted", alpha=0.5, - ) # draw a separator line between each level + ) # draw a seperator line between each level for ix_key, this_koi in enumerate(keys): # make a dummy line to steal the color cycler and make a single item for the legend From 3440691e0e6cb5a1c7a220a905312aaaa9dbd27d Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 11 Dec 2024 15:16:41 -0700 Subject: [PATCH 34/42] Fixing typo --- gtep/gtep_solution.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gtep/gtep_solution.py b/gtep/gtep_solution.py index 14ad8eb..0cbe939 100644 --- a/gtep/gtep_solution.py +++ b/gtep/gtep_solution.py @@ -465,7 +465,7 @@ def _plot_workhose_binaries( for axline_ix in range(total_height): ax_bins.axhline( axline_ix + 0.5, color="grey", linewidth=3 - ) # draw a seperator line between each level + ) # draw a separator line between each level for axline_ix in range(len(df[level_key])): ax_bins.axvline( axline_ix + 0.5, @@ -473,7 +473,7 @@ def _plot_workhose_binaries( linewidth=3, linestyle="dotted", alpha=0.5, - ) # draw a seperator line between each level + ) # draw a separator line between each level for ix_key, this_koi in enumerate(keys): # make a dummy line to steal the color cycler and make a single item for the legend From 5fc4223492a3f34f20ca7e081392d797341a4a0a Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 11 Dec 2024 17:10:03 -0700 Subject: [PATCH 35/42] Cleaning up test_validation.py --- gtep/tests/unit/test_validation.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/gtep/tests/unit/test_validation.py b/gtep/tests/unit/test_validation.py index 4ec2f7f..6a78723 100644 --- a/gtep/tests/unit/test_validation.py +++ b/gtep/tests/unit/test_validation.py @@ -19,7 +19,7 @@ output_data_source = abspath(join(curr_dir, "..", "..", "data", "5bus_out")) -def test_solution(): +def get_solution_object(): data_object = ExpansionPlanningData() data_object.load_prescient(input_data_source) @@ -42,24 +42,24 @@ def test_solution(): sol_object = ExpansionPlanningSolution() sol_object.load_from_model(mod_object) - sol_object.dump_json("./gtep/tests/test_solution.json") + # sol_object.dump_json("./gtep/tests/test_solution.json") return sol_object -solution = test_solution() +class TestValidation(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.solution = get_solution_object() -class TestValidation(unittest.TestCase): def test_populate_generators_filter_pointers(self): - populate_generators(input_data_source, solution, output_data_source) + populate_generators(input_data_source, self.solution, output_data_source) # filter_pointers needs to access the gen.csv file created in populate_generators + # so these functions need to be tested together filter_pointers(input_data_source, output_data_source) def test_populate_transmission(self): - populate_transmission(input_data_source, solution, output_data_source) - - # def test_filter_pointers(self): - # filter_pointers(input_data_source, output_data_source) + populate_transmission(input_data_source, self.solution, output_data_source) def test_clone_timeseries(self): clone_timeseries(input_data_source, output_data_source) From d4a04ec5deef2af4dfce0a5ab1c6720df694f1b1 Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 11 Dec 2024 17:13:49 -0700 Subject: [PATCH 36/42] Updating GHA workflows to be consistent and adding Python 3.12 testing --- .github/workflows/test_branches.yml | 4 ++-- .github/workflows/test_pr_and_main.yml | 33 ++++++++++++++++++-------- 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index de5db14..32cda90 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -57,7 +57,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest] - python: [3.9, '3.10', '3.11'] + python: [3.9, '3.10', '3.11', '3.12'] steps: - name: Checkout source uses: actions/checkout@v4 @@ -86,7 +86,7 @@ jobs: - name: Install idaes-gtep run: | pip install -e . + - name: Run Tests run: | pytest -v gtep - diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 81066f5..7f30c88 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -45,7 +45,7 @@ jobs: pip install black - name: Run Black to verify that the committed code is formatted run: | - black --check . + black --check --diff . spell-check: name: Check Spelling @@ -66,23 +66,36 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, windows-latest] - python: [3.9, '3.10', '3.11'] + python: [3.9, '3.10', '3.11', '3.12'] steps: - name: Checkout source uses: actions/checkout@v4 - - name: Set up Python ${{ matrix.python }} - uses: actions/setup-python@v5 + + - name: Set up Miniconda Python + uses: conda-incubator/setup-miniconda@v3 with: python-version: ${{ matrix.python }} - - name: Install Pyomo + + - name: Intall Python packages (conda) run: | - pip install pyomo + # Set up environment + conda config --set always_yes yes + conda config --set auto_update_conda false + conda config --remove channels defaults + conda config --append channels nodefaults + conda config --append channels conda-forge + + echo "Install Python packages" + conda install --yes --quiet pip setuptools wheel + conda install --yes --quiet scipy pyomo pytest highspy + + echo "Final conda environment:" + conda list | sed 's/^/ /' + - name: Install idaes-gtep run: | - python setup.py develop + pip install -e . + - name: Run Tests run: | - pip install pytest - pip install highspy pytest -v gtep - From 927a404afe9b8b7346472af05579255a6c3e882c Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 11 Dec 2024 17:18:36 -0700 Subject: [PATCH 37/42] switch to raw string to silence deprecation warning --- gtep/config_options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gtep/config_options.py b/gtep/config_options.py index 82272ff..27754bb 100644 --- a/gtep/config_options.py +++ b/gtep/config_options.py @@ -66,7 +66,7 @@ def _get_model_config(): CONFIG.declare( "time_period_dict", ConfigDict( - description="Time period dict, specified as \{(investment period #, length): \{(commitment period #, length): \{dispatch period #: length\}\}\}" + description=r"Time period dict, specified as \{(investment period #, length): \{(commitment period #, length): \{dispatch period #: length\}\}\}" ), ) From cb2ac37aae50f54bb8f24b43ed6d5645e2913008 Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 11 Dec 2024 17:25:24 -0700 Subject: [PATCH 38/42] Use raw strings to silence deprecation warnings --- gtep/gtep_solution.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gtep/gtep_solution.py b/gtep/gtep_solution.py index 0cbe939..7fcf67c 100644 --- a/gtep/gtep_solution.py +++ b/gtep/gtep_solution.py @@ -622,7 +622,7 @@ def _expressions_plot_workhorse( for this_key in upper_level_dict: level_period_number = int( - re.split("\[|\]", this_key.split(level_key)[1])[1] + re.split(r"\[|\]", this_key.split(level_key)[1])[1] ) vals_dict.setdefault(level_period_number, {}) for this_val_key in keys_of_vals_of_interest: @@ -726,7 +726,7 @@ def _level_plot_workhorse( level_period_dict = {} # cut out which dispatch period this is level_period_number = int( - re.split("\[|\]", this_key.split(level_key)[1])[1] + re.split(r"\[|\]", this_key.split(level_key)[1])[1] ) # print(level_period_number) From c3240d4a963c85f059e1982f4a642e632f086e2d Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 15 Jan 2025 13:51:08 -0700 Subject: [PATCH 39/42] Removing commented debugging code from tests --- gtep/tests/unit/test_gtep_model.py | 3 +-- gtep/tests/unit/test_validation.py | 4 ---- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index a454bdb..620214f 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -103,8 +103,7 @@ def test_solve_bigm(self): TransformationFactory("gdp.bound_pretransformation").apply_to(modObject.model) TransformationFactory("gdp.bigm").apply_to(modObject.model) modObject.results = opt.solve(modObject.model) - # modObject.model.pprint() - # modObject.model.total_cost_objective_rule.pprint() + # previous successful objective values: 9207.95, 6078.86 self.assertAlmostEqual( value(modObject.model.total_cost_objective_rule), 6078.86, places=1 diff --git a/gtep/tests/unit/test_validation.py b/gtep/tests/unit/test_validation.py index 6a78723..157333d 100644 --- a/gtep/tests/unit/test_validation.py +++ b/gtep/tests/unit/test_validation.py @@ -34,15 +34,11 @@ def get_solution_object(): mod_object.create_model() TransformationFactory("gdp.bound_pretransformation").apply_to(mod_object.model) TransformationFactory("gdp.bigm").apply_to(mod_object.model) - # opt = SolverFactory("gurobi") - # opt = Gurobi() opt = Highs() - # # mod_object.results = opt.solve(mod_object.model, tee=True) mod_object.results = opt.solve(mod_object.model) sol_object = ExpansionPlanningSolution() sol_object.load_from_model(mod_object) - # sol_object.dump_json("./gtep/tests/test_solution.json") return sol_object From 150fb25c6f272ab6b09206253f6bd54486817394 Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 15 Jan 2025 13:53:21 -0700 Subject: [PATCH 40/42] Run black --- gtep/tests/unit/test_gtep_model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gtep/tests/unit/test_gtep_model.py b/gtep/tests/unit/test_gtep_model.py index 620214f..128144c 100644 --- a/gtep/tests/unit/test_gtep_model.py +++ b/gtep/tests/unit/test_gtep_model.py @@ -103,7 +103,7 @@ def test_solve_bigm(self): TransformationFactory("gdp.bound_pretransformation").apply_to(modObject.model) TransformationFactory("gdp.bigm").apply_to(modObject.model) modObject.results = opt.solve(modObject.model) - + # previous successful objective values: 9207.95, 6078.86 self.assertAlmostEqual( value(modObject.model.total_cost_objective_rule), 6078.86, places=1 From 5f1c71148514329fbb6a6881c0cac6a283a0a0f8 Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 15 Jan 2025 14:02:59 -0700 Subject: [PATCH 41/42] Removing incorrect comments in the GHA workflows --- .github/workflows/test_branches.yml | 3 --- .github/workflows/test_pr_and_main.yml | 3 --- 2 files changed, 6 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 32cda90..6bc1bb6 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -30,9 +30,6 @@ jobs: with: python-version: '3.10' - name: Install Black - # unlike the other jobs, we don't need to install IDAES and/or all the dev dependencies, - # but we still want to specify the Black version to use in requirements-dev.txt for local development - # so we extract the relevant line and pass it to a simple `pip install` run: | pip install black - name: Run Black to verify that the committed code is formatted diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 7f30c88..0554437 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -38,9 +38,6 @@ jobs: with: python-version: '3.10' - name: Install Black - # unlike the other jobs, we don't need to install IDAES and/or all the dev dependencies, - # but we still want to specify the Black version to use in requirements-dev.txt for local development - # so we extract the relevant line and pass it to a simple `pip install` run: | pip install black - name: Run Black to verify that the committed code is formatted From d7a014fa93175ee1fb453703dbc48cc191892a02 Mon Sep 17 00:00:00 2001 From: blnicho Date: Wed, 15 Jan 2025 14:11:37 -0700 Subject: [PATCH 42/42] NFC: Fixing typo in the GHA workflows --- .github/workflows/test_branches.yml | 2 +- .github/workflows/test_pr_and_main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_branches.yml b/.github/workflows/test_branches.yml index 6bc1bb6..f5260a9 100644 --- a/.github/workflows/test_branches.yml +++ b/.github/workflows/test_branches.yml @@ -64,7 +64,7 @@ jobs: with: python-version: ${{ matrix.python }} - - name: Intall Python packages (conda) + - name: Install Python packages (conda) run: | # Set up environment conda config --set always_yes yes diff --git a/.github/workflows/test_pr_and_main.yml b/.github/workflows/test_pr_and_main.yml index 0554437..4dc5c11 100644 --- a/.github/workflows/test_pr_and_main.yml +++ b/.github/workflows/test_pr_and_main.yml @@ -73,7 +73,7 @@ jobs: with: python-version: ${{ matrix.python }} - - name: Intall Python packages (conda) + - name: Install Python packages (conda) run: | # Set up environment conda config --set always_yes yes