diff --git a/.github/workflows/pymake-requests.yml b/.github/workflows/pymake-requests.yml index 93b841a0..2b57b501 100644 --- a/.github/workflows/pymake-requests.yml +++ b/.github/workflows/pymake-requests.yml @@ -32,7 +32,7 @@ jobs: - name: Run pytest run: | - pytest -v -m requests --durations=0 --cov=pymake --cov-report=xml autotest/ + pytest -v -n=auto -m requests --durations=0 --cov=pymake --cov-report=xml autotest/ - name: Run scheduled tests if: ${{ github.event_name == 'schedule' }} diff --git a/.gitignore b/.gitignore index 36baeb66..b73444ff 100644 --- a/.gitignore +++ b/.gitignore @@ -62,18 +62,14 @@ target/ # files in the autotest directory autotest/temp*/ +autotest/*.json +autotest/code.md mod_temp/ obj_temp/ src_temp/ *.exe Dependencies -# files in the examples directory -examples/win32/ -examples/win64/ -examples/mac/ -examples/linux/ -examples/temp/ - +# files in the doc directory docs/source/ diff --git a/autotest/test_requests.py b/autotest/test_requests.py index 64959221..ea952336 100644 --- a/autotest/test_requests.py +++ b/autotest/test_requests.py @@ -42,12 +42,12 @@ def initialize_working_dir(): os.makedirs(dstpth, exist_ok=True) -def export_code_json(): +def export_code_json(file_name="code.json"): # make sure the test directory exists initialize_working_dir() # make the json file - fpth = os.path.join(dstpth, "code.test.json") + fpth = os.path.join(dstpth, file_name) pymake.usgs_program_data.export_json( fpth=fpth, current=True, @@ -302,7 +302,7 @@ def test_target_keys(): @pytest.mark.requests def test_usgsprograms_export_json(): # export code.json and return json file path - fpth = export_code_json() + fpth = export_code_json(file_name="code.export.json") # test the json export with open(fpth, "r") as f: @@ -353,7 +353,7 @@ def test_usgsprograms_load_json(): print("test_usgsprograms_load_json()") # export code.json and return json file path - fpth = export_code_json() + fpth = export_code_json(file_name="code.load.json") json_dict = pymake.usgs_program_data.load_json(fpth) @@ -383,7 +383,7 @@ def test_usgsprograms_list_json(): print("test_usgsprograms_list_json()") # export code.json and return json file path - fpth = export_code_json() + fpth = export_code_json(file_name="code.list.json") # list the contents of the json file pymake.usgs_program_data.list_json(fpth=fpth) diff --git a/pymake/cmds/createjson.py b/pymake/cmds/createjson.py index 60de4ccc..e82c9483 100755 --- a/pymake/cmds/createjson.py +++ b/pymake/cmds/createjson.py @@ -91,7 +91,7 @@ def main() -> None: "tag": ("--write_markdown",), "help": "If True, write markdown file that includes the " + "target name, version, and the last-modified date of " - + "the download asset (url). Default is False.", + + "the download asset (url). Default is True.", "default": True, "choices": None, "action": "store_true", diff --git a/pymake/utils/download.py b/pymake/utils/download.py index 9a6dc8fb..d1cdac32 100644 --- a/pymake/utils/download.py +++ b/pymake/utils/download.py @@ -15,8 +15,8 @@ import shutil import sys import tarfile -import time import timeit +from http.client import responses from zipfile import ZIP_DEFLATED, ZipFile, ZipInfo import requests @@ -202,32 +202,38 @@ def _request_get(url, verify=True, timeout=1, max_requests=10, verbose=False): request object for url """ + if verbose: + print(f"request url '{url}'") + for idx in range(max_requests): if verbose: - msg = f"open request attempt {idx + 1} of {max_requests}" - print(msg) + print(f" request attempt {idx + 1} of {max_requests}") req = None try: req = requests.get( - url, stream=True, verify=verify, timeout=timeout + url, + stream=True, + verify=verify, + timeout=timeout, ) + if verbose: + print(f" status: {responses[req.status_code]}") except: - if idx < max_requests - 1: - time.sleep(13) - continue - else: - msg = "Cannot open request from:\n" + f" {url}\n\n" - print(msg) - if req is not None: - req.raise_for_status() + continue - # successful request - break + if req.status_code == 200: + break + + # final test for success + if req is None: + raise ConnectionError(f"Could not get data from: {url}") + else: + req.raise_for_status() return req -def _request_header(url, max_requests=10, verbose=False): +def _request_header(url, max_requests=10, timeout=1, verbose=False): """Get the headers from a url Parameters @@ -236,6 +242,8 @@ def _request_header(url, max_requests=10, verbose=False): url address for the zip file max_requests : int number of url download request attempts (default is 10) + timeout : int + url request time out length (default is 1 seconds) verbose : bool boolean indicating if output will be printed to the terminal (default is False) @@ -246,23 +254,32 @@ def _request_header(url, max_requests=10, verbose=False): request header object for url """ + if verbose: + print(f"request url: '{url}'") + for idx in range(max_requests): if verbose: - msg = f"open request attempt {idx + 1} of {max_requests}" - print(msg) + print(f" request attempt {idx + 1} of {max_requests}") + header = None + try: + header = requests.head( + url, + allow_redirects=True, + timeout=timeout, + ) + if verbose: + print(f" status: {responses[header.status_code]}") + except: + continue - header = requests.head(url, allow_redirects=True) - if header.status_code != 200: - if idx < max_requests - 1: - time.sleep(13) - continue - else: - msg = "Cannot open request from:\n" + f" {url}\n\n" - print(msg) - header.raise_for_status() + if header.status_code == 200: + break - # successful header request - break + # final test for success + if header is None: + raise ConnectionError(f"Could not get header from: {url}") + else: + header.raise_for_status() return header diff --git a/pymake/utils/usgsprograms.py b/pymake/utils/usgsprograms.py index 8c3a6b6b..16e5475e 100644 --- a/pymake/utils/usgsprograms.py +++ b/pymake/utils/usgsprograms.py @@ -21,6 +21,7 @@ import datetime import json import os +import pathlib as pl import sys from .download import _request_header @@ -341,12 +342,25 @@ def export_json( print( f'writing a json file ("{fpth}") ' + f"of {sel} USGS programs\n" - + f'in the "{program_data_file}" database.' + + f'in the "{program_data_file}" database.\n' ) if prog_data is not None: for idx, key in enumerate(prog_data.keys()): print(f" {idx + 1:>2d}: {key}") - print("\n") + + # process the passed file path into appdir and file_name + appdir = pl.Path(".") + file_name = pl.Path(fpth) + if file_name.parent != str(appdir): + appdir = file_name.parent + file_name = file_name.name + else: + for idx, argv in enumerate(sys.argv): + if argv in ("--appdir", "-ad"): + appdir = pl.Path(sys.argv[idx + 1]) + + if str(appdir) != ".": + appdir.mkdir(parents=True, exist_ok=True) # get usgs program data udata = usgs_program_data.get_program_dict() @@ -392,48 +406,38 @@ def export_json( # export file try: - with open(fpth, "w") as f: - json.dump(prog_data, f, indent=4) + with open(file_name, "w") as file_obj: + json.dump(prog_data, file_obj, indent=4, sort_keys=True) except: - msg = f'could not export json file "{fpth}"' + msg = f'could not export json file "{file_name}"' raise IOError(msg) - # export code.json to --appdir directory, if the - # command line argument was specified. Only done if not CI - # command line argument was specified. Only done if not CI - appdir = "." - for idx, argv in enumerate(sys.argv): - if argv in ("--appdir", "-ad"): - appdir = sys.argv[idx + 1] - - # make appdir if it does not already exist - if not os.path.isdir(appdir): - os.makedirs(appdir) - # write code.json - if appdir != ".": - dst = os.path.join(appdir, fpth) - with open(dst, "w") as f: - json.dump(prog_data, f, indent=4) + if str(appdir) != ".": + dst = appdir / file_name + with open(dst, "w") as file_obj: + json.dump(prog_data, file_obj, indent=4, sort_keys=True) # write code.md if prog_data is not None and write_markdown: - file_obj = open("code.md", "w") - line = "| Program | Version | UTC Date |" - file_obj.write(line + "\n") - line = "| ------- | ------- | ---- |" - file_obj.write(line + "\n") - for target, target_dict in prog_data.items(): - keys = list(target_dict.keys()) - line = f"| {target} | {target_dict['version']} |" - date_key = "url_download_asset_date" - if date_key in keys: - line += f" {target_dict[date_key]} |" - else: - line += " |" - line += "\n" - file_obj.write(line) - file_obj.close() + sorted_prog_data = { + key: prog_data[key] for key in sorted(list(prog_data.keys())) + } + with open("code.md", "w") as file_obj: + line = "| Program | Version | UTC Date |" + file_obj.write(line + "\n") + line = "| ------- | ------- | ---- |" + file_obj.write(line + "\n") + for target, target_dict in sorted_prog_data.items(): + keys = list(target_dict.keys()) + line = f"| {target} | {target_dict['version']} |" + date_key = "url_download_asset_date" + if date_key in keys: + line += f" {target_dict[date_key]} |" + else: + line += " |" + line += "\n" + file_obj.write(line) return diff --git a/pymake/utils/usgsprograms.txt b/pymake/utils/usgsprograms.txt index 244142ff..0fcc9ad0 100644 --- a/pymake/utils/usgsprograms.txt +++ b/pymake/utils/usgsprograms.txt @@ -1,7 +1,7 @@ target , version, current, url , dirname , srcdir , standard_switch, double_switch, shared_object -mf6 , 6.4.0 , True , https://github.com/MODFLOW-USGS/modflow6/releases/download/6.4.0/mf6.4.0_linux.zip , mf6.4.0 , src , True , False , False -zbud6 , 6.4.0 , True , https://github.com/MODFLOW-USGS/modflow6/releases/download/6.4.0/mf6.4.0_linux.zip , mf6.4.0 , utils/zonebudget/src, True , False , False -libmf6 , 6.4.0 , True , https://github.com/MODFLOW-USGS/modflow6/releases/download/6.4.0/mf6.4.0_linux.zip , mf6.4.0 , srcbmi , True , False , True +mf6 , 6.4.1 , True , https://github.com/MODFLOW-USGS/modflow6/releases/download/6.4.1/mf6.4.1_linux.zip , mf6.4.1_linux , src , True , False , False +zbud6 , 6.4.1 , True , https://github.com/MODFLOW-USGS/modflow6/releases/download/6.4.1/mf6.4.1_linux.zip , mf6.4.1_linux , utils/zonebudget/src, True , False , False +libmf6 , 6.4.1 , True , https://github.com/MODFLOW-USGS/modflow6/releases/download/6.4.1/mf6.4.1_linux.zip , mf6.4.1_linux , srcbmi , True , False , True mp7 , 7.2.001, True , https://water.usgs.gov/water-resources/software/MODPATH/modpath_7_2_001.zip , modpath_7_2_001 , source , True , False , False mt3dms , 5.3.0 , True , https://hydro.geo.ua.edu/mt3d/mt3dms_530.exe , mt3dms5.3.0 , src/true-binary , True , False , False mt3dusgs , 1.1.0 , True , https://water.usgs.gov/water-resources/software/MT3D-USGS/mt3dusgs1.1.0.zip , mt3dusgs1.1.0 , src , True , False , False diff --git a/pytest.ini b/pytest.ini index 24a239aa..77dd21cf 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,5 +1,6 @@ [pytest] markers = base: base tests - regression: base and regression tests + regression: regression tests requests: usgsprograms requests tests + schedule: tests to run if a scheduled job