From a743d07ba7d212d302fd2c0b6f4bc71e6127796f Mon Sep 17 00:00:00 2001 From: Dusan Figala Date: Thu, 11 May 2023 15:52:07 +0200 Subject: [PATCH 01/13] Add CI build config --- .github/ci-config.yml | 8 ++++++++ .github/ci-hpc-config.yml | 8 ++++++++ 2 files changed, 16 insertions(+) create mode 100644 .github/ci-config.yml create mode 100644 .github/ci-hpc-config.yml diff --git a/.github/ci-config.yml b/.github/ci-config.yml new file mode 100644 index 00000000..1c76674c --- /dev/null +++ b/.github/ci-config.yml @@ -0,0 +1,8 @@ +self_coverage: true +dependencies: | + ecmwf/ecbuild + MathisRosenhauer/libaec@master + ecmwf/eccodes + ecmwf/eckit +dependency_branch: develop +parallelism_factor: 8 diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml new file mode 100644 index 00000000..15adf120 --- /dev/null +++ b/.github/ci-hpc-config.yml @@ -0,0 +1,8 @@ +build: + modules: + - ninja + dependencies: + - ecmwf/ecbuild@develop + - ecmwf/eccodes@develop + - ecmwf/eckit@develop + parallel: 64 From b19eb2efa8cd3a5e0cdc5560e0f3944d897bafdd Mon Sep 17 00:00:00 2001 From: Dusan Figala Date: Thu, 11 May 2023 17:59:25 +0200 Subject: [PATCH 02/13] Remove self coverage input --- .github/ci-config.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/ci-config.yml b/.github/ci-config.yml index 1c76674c..0920dfcd 100644 --- a/.github/ci-config.yml +++ b/.github/ci-config.yml @@ -1,4 +1,3 @@ -self_coverage: true dependencies: | ecmwf/ecbuild MathisRosenhauer/libaec@master From ccdb0b803f1e397121fca0ac24a80e5f2347fae5 Mon Sep 17 00:00:00 2001 From: madd Date: Tue, 16 May 2023 11:38:41 +0000 Subject: [PATCH 03/13] added dotw for eefo --- share/metkit/language.yaml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/share/metkit/language.yaml b/share/metkit/language.yaml index 5cd54d00..6f5054af 100644 --- a/share/metkit/language.yaml +++ b/share/metkit/language.yaml @@ -890,8 +890,13 @@ disseminate: type: enum values: - bc - - thursday - monday + - tuesday + - wednesday + - thursday + - friday + - saturday + - sunday option: default: normal From 00848a9e4419b2ceaf4abd2554cbf062da935030 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Tue, 25 Apr 2023 16:24:48 +0100 Subject: [PATCH 04/13] version bump --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 3cc3669f..b0dd7532 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.10.8 +1.10.9 From 4998b5c12e2a9ba4484afbd02f974431b4891c07 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Tue, 16 May 2023 14:51:47 +0100 Subject: [PATCH 05/13] added new ext ens streams --- share/metkit/params.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/share/metkit/params.yaml b/share/metkit/params.yaml index 43868ede..ae3c48e5 100644 --- a/share/metkit/params.yaml +++ b/share/metkit/params.yaml @@ -8015,6 +8015,7 @@ - 130 - 131 - 132 + - 156 - 157 - - levtype: sfc stream: oper From 908e591c7c3bd31159c9c17c1ee42410c4e352b0 Mon Sep 17 00:00:00 2001 From: Dusan Figala Date: Wed, 26 Jul 2023 19:38:55 +0200 Subject: [PATCH 06/13] Remove obsolete workflows --- .github/workflows/ci.yml | 10 +------ .github/workflows/reusable-ci-hpc.yml | 33 ----------------------- .github/workflows/reusable-ci.yml | 38 --------------------------- 3 files changed, 1 insertion(+), 80 deletions(-) delete mode 100644 .github/workflows/reusable-ci-hpc.yml delete mode 100644 .github/workflows/reusable-ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 474acf70..a189d823 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,6 +27,7 @@ jobs: uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main with: metkit: ecmwf/metkit@${{ github.event.pull_request.head.sha || github.sha }} + codecov_upload: true secrets: inherit # Run CI of private downstream packages on self-hosted runners @@ -73,12 +74,3 @@ jobs: repository: private-downstream-ci event_type: downstream-ci-hpc payload: '{"metkit": "ecmwf/metkit@${{ github.event.pull_request.head.sha || github.sha }}"}' - - codecov: - name: code-coverage - if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ./.github/workflows/reusable-ci.yml - with: - metkit: ecmwf/metkit@${{ github.event.pull_request.head.sha || github.sha }} - codecov: true - secrets: inherit diff --git a/.github/workflows/reusable-ci-hpc.yml b/.github/workflows/reusable-ci-hpc.yml deleted file mode 100644 index 189e80fe..00000000 --- a/.github/workflows/reusable-ci-hpc.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: reusable-ci-hpc - -on: - workflow_call: - inputs: - metkit: - required: false - type: string - eccodes: - required: false - type: string - eckit: - required: false - type: string - ecbuild: - required: false - type: string -jobs: - ci-hpc: - name: ci-hpc - uses: ecmwf-actions/reusable-workflows/.github/workflows/ci-hpc.yml@v2 - with: - name-prefix: metkit- - build-inputs: | - --package: ${{ inputs.metkit || 'ecmwf/metkit@develop' }} - --modules: | - ninja - --dependencies: | - ${{ inputs.ecbuild || 'ecmwf/ecbuild@develop' }} - ${{ inputs.eccodes || 'ecmwf/eccodes@develop' }} - ${{ inputs.eckit || 'ecmwf/eckit@develop' }} - --parallel: 64 - secrets: inherit diff --git a/.github/workflows/reusable-ci.yml b/.github/workflows/reusable-ci.yml deleted file mode 100644 index e926dbe4..00000000 --- a/.github/workflows/reusable-ci.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: reusable-ci - -on: - workflow_call: - inputs: - metkit: - required: false - type: string - eccodes: - required: false - type: string - eckit: - required: false - type: string - codecov: - required: false - type: boolean - default: false - -jobs: - ci: - name: metkit-ci - uses: ecmwf-actions/reusable-workflows/.github/workflows/ci.yml@v2 - with: - repository: ${{ inputs.metkit || 'ecmwf/metkit@develop' }} - name_prefix: metkit- - codecov_upload: ${{ inputs.codecov }} - build_package_inputs: | - repository: ${{ inputs.metkit || 'ecmwf/metkit@develop' }} - self_coverage: true - dependencies: | - ecmwf/ecbuild - MathisRosenhauer/libaec@master - ${{ inputs.eccodes || 'ecmwf/eccodes' }} - ${{ inputs.eckit || 'ecmwf/eckit' }} - dependency_branch: develop - parallelism_factor: 8 - secrets: inherit From 2e7b16b23a2dbcd8e8d9f93a26da8026b65a855b Mon Sep 17 00:00:00 2001 From: Dusan Figala Date: Wed, 26 Jul 2023 19:39:21 +0200 Subject: [PATCH 07/13] Add CD --- .github/workflows/cd.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .github/workflows/cd.yml diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml new file mode 100644 index 00000000..ef99a217 --- /dev/null +++ b/.github/workflows/cd.yml @@ -0,0 +1,11 @@ +name: cd + +on: + push: + tags: + - '**' + +jobs: + deploy: + uses: ecmwf-actions/reusable-workflows/.github/workflows/create-package.yml@v2 + secrets: inherit From 64b6bd012f131bf61f263a297a8df8fdbe2c9109 Mon Sep 17 00:00:00 2001 From: Simon Smart Date: Tue, 19 Sep 2023 12:36:17 +0100 Subject: [PATCH 08/13] METK-113: First re-implementation of marsgen --- src/metkit/mars/MarsParser.cc | 2 +- src/metkit/mars/MarsRequest.cc | 7 + src/metkit/mars/MarsRequest.h | 2 + src/tools/CMakeLists.txt | 9 + src/tools/mars-archive-script.cc | 303 +++++++++++++++++++++++++++++++ 5 files changed, 322 insertions(+), 1 deletion(-) create mode 100644 src/tools/mars-archive-script.cc diff --git a/src/metkit/mars/MarsParser.cc b/src/metkit/mars/MarsParser.cc index d211d48b..46cd1abd 100644 --- a/src/metkit/mars/MarsParser.cc +++ b/src/metkit/mars/MarsParser.cc @@ -99,7 +99,7 @@ std::string MarsParser::parseString(char quote) } static bool inindent(char c) { - return isalnum(c) || c == '_' || c == ':' || c == '-' || c == '.'; + return isalnum(c) || c == '_' || c == ':' || c == '-' || c == '.' || c == '@'; } diff --git a/src/metkit/mars/MarsRequest.cc b/src/metkit/mars/MarsRequest.cc index 7ae2a82f..124cfeb5 100644 --- a/src/metkit/mars/MarsRequest.cc +++ b/src/metkit/mars/MarsRequest.cc @@ -401,6 +401,13 @@ std::list::iterator MarsRequest::find(const std::string& name) { return params_.end(); } +void MarsRequest::erase(const std::string& name) { + auto it = find(name); + if (it != params_.end()) { + params_.erase(it); + } +} + //---------------------------------------------------------------------------------------------------------------------- std::vector MarsRequest::parse(std::istream& in, bool strict) { diff --git a/src/metkit/mars/MarsRequest.h b/src/metkit/mars/MarsRequest.h index dd863c3a..96edecf8 100644 --- a/src/metkit/mars/MarsRequest.h +++ b/src/metkit/mars/MarsRequest.h @@ -106,6 +106,8 @@ class MarsRequest { MarsRequest extract(const std::string& category) const; + void erase(const std::string& param); + public: // static methods static MarsRequest parse(const std::string& s, bool strict = false); static std::vector parse(std::istream&, bool strict = false); diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt index 44919c09..fb3bb9fb 100644 --- a/src/tools/CMakeLists.txt +++ b/src/tools/CMakeLists.txt @@ -58,6 +58,15 @@ ecbuild_add_executable( TARGET bufr-sanity-check NO_AS_NEEDED LIBS metkit eckit_option ) +ecbuild_add_executable( + TARGET mars-archive-script + SOURCES mars-archive-script.cc + CONDITION HAVE_BUILD_TOOLS + INCLUDES ${ECKIT_INCLUDE_DIRS} + NO_AS_NEEDED + LIBS metkit +) + # ecbuild_add_executable( TARGET ncmerge # SOURCES ncmerge.cc # CONDITION HAVE_NETCDF diff --git a/src/tools/mars-archive-script.cc b/src/tools/mars-archive-script.cc new file mode 100644 index 00000000..135ce3aa --- /dev/null +++ b/src/tools/mars-archive-script.cc @@ -0,0 +1,303 @@ + +#include "eckit/io/DataHandle.h" +#include "eckit/io/FileDescHandle.h" +#include "eckit/io/FileHandle.h" +#include "eckit/option/CmdArgs.h" +#include "eckit/option/SimpleOption.h" + +#include "metkit/mars/MarsParser.h" +#include "metkit/tool/MetkitTool.h" + +#include + +#include + + +using namespace metkit; +using namespace metkit::mars; +using namespace eckit; +using namespace eckit::option; + +//---------------------------------------------------------------------------------------------------------------------- + +class MarsArchiveScript : public MetkitTool { + + using OverridesDict = std::map>>; + +public: // methods + + MarsArchiveScript(int argc, char** argv); + ~MarsArchiveScript() override = default; + +private: // methods + + void init(const CmdArgs& args) override; + void execute(const CmdArgs& args) override; + void usage(const std::string& tool) const override; + + OverridesDict extractOverrides(MarsRequest& request); + + template + void setOverrides(MarsRequest& rq, const OverridesDict& overrides, const char* overrideName, Args... more); + + void setOverrides(MarsRequest& rq, const OverridesDict& overrides); + + std::string srcfile(int cnt) const; + std::string arcfile(int cnt) const; + std::string cmpfile(int cnt) const; + +private: // members + + std::string infile_; + std::string outfile_; + std::string tempPrefix_; + + bool obs_; + bool compare_; + + // Output environment variables + + std::string preObs_; + std::string retrieveMars_; + std::string archiveMars_; + std::string compareMars_; + + std::string unblockCommand_; + + std::string retrieveComplete_; + std::string archiveComplete_; + std::string compareComplete_; + std::string compareCommand_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +MarsArchiveScript::MarsArchiveScript(int argc, char** argv) : + MetkitTool(argc, argv), + tempPrefix_("mars"), + obs_(false), + compare_(false), + preObs_("PREOBS"), + retrieveMars_("RETRIEVE_MARS"), + archiveMars_("ARCHIVE_MARS"), + compareMars_("COMPARE_MARS"), + unblockCommand_("UNBLOCK"), + retrieveComplete_("RETRIEVE_COMPLETE"), + archiveComplete_("ARCHIVE_COMPLETE"), + compareComplete_("COMPARE_COMPLETE"), + compareCommand_("COMPARE") { + + options_.push_back( + new SimpleOption("out", "Output filename (defaults output to stdout)")); + + options_.push_back( + new SimpleOption("in", "Input filename (defaults input to stdin)")); + + options_.push_back( + new SimpleOption("obs", "Handle observations rather than output GRIBs")); + + options_.push_back( + new SimpleOption("compare", "Re-retrieve and compare results")); + + options_.push_back( + new SimpleOption("legacy", "Use legacy environment variable names for old suite compatability")); + + options_.push_back( + new SimpleOption("prefix", "Prefix for the temporary files. Typically \"mars\"")); +} + +void MarsArchiveScript::usage(const std::string& tool) const { + + Log::info() << "Usage: " << tool << " [options]" << eckit::newl + << eckit::newl + << "Note: The output of this tool assumes that we are running in an ecflow suite with" << eckit::newl + << " failure trapping enabled, as well as the following environment variables:" << eckit::newl + << " RETRIEVE_MARS, ARCHIVE_MARS, COMPARE_MARS, UNBLOCK, RETRIEVE_COMPLETE" << eckit::newl + << " ARCHIVE_COMPLETE, COMPARE_COMPLETE" << eckit::newl + << eckit::newl + + << "Examples:" << eckit::newl + << "=========" << eckit::newl + << std::endl; +} + +void MarsArchiveScript::init(const CmdArgs& args) { + + infile_ = args.getString("in", infile_); + outfile_ = args.getString("out", outfile_); + + compare_ = args.getBool("compare", compare_); + tempPrefix_ = args.getString("prefix", tempPrefix_); + obs_ = args.getBool("obs", obs_); + + if (args.getBool("legacy", false)) { + retrieveMars_ = "MARS_FROM_FDB"; + archiveMars_ = "MARS_TO_IBM"; + compareMars_ = "MARS_FROM_IBM"; + unblockCommand_ = "UNBLOCK"; + retrieveComplete_ = "FDB_COMPLETE"; + archiveComplete_ = "ARC_COMPLETE"; + compareComplete_ = "CMP_COMPLETE"; + } + + if (obs_) compareCommand_ = "COMPOBS"; +} + +std::string MarsArchiveScript::srcfile(int cnt) const { + return tempPrefix_ + ".source." + translate(cnt); +} + +std::string MarsArchiveScript::arcfile(int cnt) const { + return tempPrefix_ + ".archive." + translate(cnt); +} + +std::string MarsArchiveScript::cmpfile(int cnt) const { + return tempPrefix_ + ".compare." + translate(cnt); +} + +MarsArchiveScript::OverridesDict MarsArchiveScript::extractOverrides(MarsRequest& request) { + + OverridesDict ret; + + std::vector keys; + request.getParams(keys); + + for (const auto& k : keys) { + int pos = k.find('@'); + if (pos != std::string::npos) { + std::vector vals; + request.getValues(k, vals); + ret[k.substr(0, pos)][k.substr(pos+1)] = vals; + request.erase(k); + } + } + + return ret; +} + +template +void MarsArchiveScript::setOverrides(MarsRequest& rq, const OverridesDict& overrides, const char* overrideName, Args... more) { + + auto namedset = overrides.find(overrideName); + if (namedset != overrides.end()) { + for (const auto& kv : namedset->second) { + rq.setValue(kv.first, kv.second); + } + } + + setOverrides(rq, overrides, more...); +} + +void MarsArchiveScript::setOverrides(MarsRequest& rq, const OverridesDict& overrides) {} + +void MarsArchiveScript::execute(const CmdArgs& args) { + + // Input from file, or from stdin? + + std::unique_ptr in_file; + if (!infile_.empty()) { + in_file = std::make_unique(infile_.c_str(), std::ios::in | std::ios::binary); + in_file->exceptions(std::ios::badbit); + } + std::istream& in(in_file ? *in_file : std::cin); + + // Output to file, or to stdout? + + std::unique_ptr out_file; + if (!outfile_.empty()) { + out_file = std::make_unique(outfile_.c_str(), std::ios::out | std::ios::binary); + out_file->exceptions(std::ios::badbit); + } + std::ostream& out(out_file ? *out_file : std::cout); + + // Parse the input request + + MarsParser parser(in); + auto requests = parser.parse(); + + std::vector overrides; + for (auto& rq : requests) { + overrides.emplace_back(extractOverrides(rq)); + } + + // 1. Retrieve the source data (typically from the FDB) + // n.b. loop over non-const, non-reference requests --> mutable copy in the loops + + int cnt = 0; + + if (obs_) { + + for (cnt = 0; cnt < requests.size(); ++cnt) { + out << "$" << preObs_ << " $header." << translate(cnt+1) + << " $data." << translate(cnt+1) + << " " << arcfile(cnt) << "\n\n"; + } + + } else { + + out << "$" << retrieveMars_ << " << @\n\n"; + + cnt = 0; + for (auto rq: requests) { + rq.verb("retrieve"); + rq.setValue("target", srcfile(cnt)); + setOverrides(rq, overrides[cnt++], "default", "retrieve", "fdb_retrieve"); + rq.dump(out); + } + + out << "@\n\n$" << retrieveComplete_ << "\n\n"; + + // 1.a) Any intermediate step required. Typically ln -s (historically handle "blocked" data output by older Fortran + // based systems) + + for (cnt = 0; cnt < requests.size(); ++cnt) { + out << "$" << unblockCommand_ << " " << srcfile(cnt) << " " << arcfile(cnt) << "\n\n"; + } + } + + // 2. Archive the data + + out << "$" << archiveMars_ << " << @\n\n"; + + cnt = 0; + for (auto rq : requests) { + rq.verb("archive"); + rq.setValue("source", arcfile(cnt)); + setOverrides(rq, overrides[cnt++], "default", "archive"); + rq.dump(out); + } + + out << "@\n\n$" << archiveComplete_ << "\n\n"; + + // 3. Comparison of re-retrieved data + + if (compare_) { + + out << "$" << compareMars_ << " << @\n\n"; + + cnt = 0; + for (MarsParsedRequest rq : requests) { + rq.verb("retrieve"); + rq.setValue("target", cmpfile(cnt)); + setOverrides(rq, overrides[cnt++], "default", "compare", "ibm_retrieve"); + rq.dump(out); + } + + out << "@\n\n"; + + for (cnt = 0; cnt < requests.size(); ++cnt) { + out << "$" << compareCommand_ << " " << arcfile(cnt) << " " << cmpfile(cnt) << "\n\n"; + } + + out << "$" << compareComplete_ << "\n"; + } + + out << std::endl; +} + +//---------------------------------------------------------------------------------------------------------------------- + +int main(int argc, char** argv) { + MarsArchiveScript tool(argc, argv); + return tool.start(); +} \ No newline at end of file From 27c1734b1f742b867db78335abf34af0a8d7cf21 Mon Sep 17 00:00:00 2001 From: Simon Smart Date: Tue, 19 Sep 2023 15:03:01 +0100 Subject: [PATCH 09/13] METK-113: Add some tests --- src/tools/mars-archive-script.cc | 9 ++-- tests/CMakeLists.txt | 1 + tests/marsgen/CMakeLists.txt | 17 ++++++ tests/marsgen/genfile | 19 +++++++ tests/marsgen/genfile.default | 16 ++++++ tests/marsgen/genfile.legacy | 19 +++++++ tests/marsgen/metkit_marsgen_compare.sh.in | 60 ++++++++++++++++++++++ tests/marsgen/metkit_marsgen_legacy.sh.in | 60 ++++++++++++++++++++++ tests/marsgen/metkit_marsgen_obs.sh.in | 39 ++++++++++++++ tests/marsgen/metkit_marsgen_simple.sh.in | 60 ++++++++++++++++++++++ 10 files changed, 294 insertions(+), 6 deletions(-) create mode 100644 tests/marsgen/CMakeLists.txt create mode 100644 tests/marsgen/genfile create mode 100644 tests/marsgen/genfile.default create mode 100644 tests/marsgen/genfile.legacy create mode 100755 tests/marsgen/metkit_marsgen_compare.sh.in create mode 100755 tests/marsgen/metkit_marsgen_legacy.sh.in create mode 100755 tests/marsgen/metkit_marsgen_obs.sh.in create mode 100755 tests/marsgen/metkit_marsgen_simple.sh.in diff --git a/src/tools/mars-archive-script.cc b/src/tools/mars-archive-script.cc index 135ce3aa..04d98f4f 100644 --- a/src/tools/mars-archive-script.cc +++ b/src/tools/mars-archive-script.cc @@ -1,6 +1,5 @@ #include "eckit/io/DataHandle.h" -#include "eckit/io/FileDescHandle.h" #include "eckit/io/FileHandle.h" #include "eckit/option/CmdArgs.h" #include "eckit/option/SimpleOption.h" @@ -10,8 +9,6 @@ #include -#include - using namespace metkit; using namespace metkit::mars; @@ -267,13 +264,13 @@ void MarsArchiveScript::execute(const CmdArgs& args) { rq.dump(out); } - out << "@\n\n$" << archiveComplete_ << "\n\n"; + out << "@\n\n$" << archiveComplete_; // 3. Comparison of re-retrieved data if (compare_) { - out << "$" << compareMars_ << " << @\n\n"; + out << "\n\n$" << compareMars_ << " << @\n\n"; cnt = 0; for (MarsParsedRequest rq : requests) { @@ -289,7 +286,7 @@ void MarsArchiveScript::execute(const CmdArgs& args) { out << "$" << compareCommand_ << " " << arcfile(cnt) << " " << cmpfile(cnt) << "\n\n"; } - out << "$" << compareComplete_ << "\n"; + out << "$" << compareComplete_; } out << std::endl; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index fdcac05d..05a781a3 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -77,3 +77,4 @@ endforeach() # endif() add_subdirectory(regressions) +add_subdirectory(marsgen) diff --git a/tests/marsgen/CMakeLists.txt b/tests/marsgen/CMakeLists.txt new file mode 100644 index 00000000..6392ec99 --- /dev/null +++ b/tests/marsgen/CMakeLists.txt @@ -0,0 +1,17 @@ + +list( APPEND marsgen_tests + simple + compare + obs + legacy) + + +foreach( _test ${marsgen_tests} ) + + configure_file( metkit_marsgen_${_test}.sh.in metkit_marsgen_${_test}.sh @ONLY ) + + ecbuild_add_test( + TYPE SCRIPT + COMMAND metkit_marsgen_${_test}.sh ) + +endforeach() \ No newline at end of file diff --git a/tests/marsgen/genfile b/tests/marsgen/genfile new file mode 100644 index 00000000..4371c4b0 --- /dev/null +++ b/tests/marsgen/genfile @@ -0,0 +1,19 @@ +verb, + retrieve@expver = 9999, + archive@expver = 9999, + retrieve@class = EA, + archive@class = EA, + retrieve@stream = LWDA, + archive@stream = DA, + + class=EA, + stream=LWDA, + expver=1001, + type=analysis,anoffset=3, + param=129/152/130/138/155/75/76/133/203.128/246/247/248, + levtype=modellevels, + levels=1/to/137, + pass="runda", + date=20200601, + time=06, + step=00 diff --git a/tests/marsgen/genfile.default b/tests/marsgen/genfile.default new file mode 100644 index 00000000..71ec9f4f --- /dev/null +++ b/tests/marsgen/genfile.default @@ -0,0 +1,16 @@ +verb, + default@expver = 9999, + default@class = EA, + default@stream = LWDA, + + class=EA, + stream=LWDA, + expver=1001, + type=analysis,anoffset=3, + param=129/152/130/138/155/75/76/133/203.128/246/247/248, + levtype=modellevels, + levels=1/to/137, + pass="runda", + date=20200601, + time=06, + step=00 diff --git a/tests/marsgen/genfile.legacy b/tests/marsgen/genfile.legacy new file mode 100644 index 00000000..142ca8a7 --- /dev/null +++ b/tests/marsgen/genfile.legacy @@ -0,0 +1,19 @@ +verb, + fdb_retrieve@expver = 9999, + archive@expver = 9999, + fdb_retrieve@class = EA, + archive@class = EA, + fdb_retrieve@stream = LWDA, + archive@stream = DA, + + class=EA, + stream=LWDA, + expver=1001, + type=analysis,anoffset=3, + param=129/152/130/138/155/75/76/133/203.128/246/247/248, + levtype=modellevels, + levels=1/to/137, + pass="runda", + date=20200601, + time=06, + step=00 diff --git a/tests/marsgen/metkit_marsgen_compare.sh.in b/tests/marsgen/metkit_marsgen_compare.sh.in new file mode 100755 index 00000000..6f799afa --- /dev/null +++ b/tests/marsgen/metkit_marsgen_compare.sh.in @@ -0,0 +1,60 @@ +#!/bin/bash + +set -uex + +[ -e outfile.compare ] && rm outfile.compare + +MARSGEN="@CMAKE_BINARY_DIR@/bin/mars-archive-script" +GENFILE="@CMAKE_CURRENT_SOURCE_DIR@/genfile" +OUTFILE="outfile.compare" +CHECKFILE="checkfile.compare" + +$MARSGEN < $GENFILE > $OUTFILE + +cat << EOF > $CHECKFILE +\$RETRIEVE_MARS << @ + +retrieve, + class=EA, + stream=LWDA, + expver=9999, + type=analysis, + anoffset=3, + param=129/152/130/138/155/75/76/133/203.128/246/247/248, + levtype=modellevels, + levels=1/to/137, + pass=runda, + date=20200601, + time=06, + step=00, + target=mars.source.0 + +@ + +\$RETRIEVE_COMPLETE + +\$UNBLOCK mars.source.0 mars.archive.0 + +\$ARCHIVE_MARS << @ + +archive, + class=EA, + stream=DA, + expver=9999, + type=analysis, + anoffset=3, + param=129/152/130/138/155/75/76/133/203.128/246/247/248, + levtype=modellevels, + levels=1/to/137, + pass=runda, + date=20200601, + time=06, + step=00, + source=mars.archive.0 + +@ + +\$ARCHIVE_COMPLETE +EOF + +cmp $CHECKFILE $OUTFILE \ No newline at end of file diff --git a/tests/marsgen/metkit_marsgen_legacy.sh.in b/tests/marsgen/metkit_marsgen_legacy.sh.in new file mode 100755 index 00000000..ee297f3d --- /dev/null +++ b/tests/marsgen/metkit_marsgen_legacy.sh.in @@ -0,0 +1,60 @@ +#!/bin/bash + +set -uex + +[ -e outfile.legacy ] && rm outfile.legacy + +MARSGEN="@CMAKE_BINARY_DIR@/bin/mars-archive-script" +GENFILE="@CMAKE_CURRENT_SOURCE_DIR@/genfile.legacy" +OUTFILE="outfile.legacy" +CHECKFILE="checkfile.legacy" + +$MARSGEN --in="$GENFILE" --out="$OUTFILE" + +cat << EOF > $CHECKFILE +\$RETRIEVE_MARS << @ + +retrieve, + class=EA, + stream=LWDA, + expver=9999, + type=analysis, + anoffset=3, + param=129/152/130/138/155/75/76/133/203.128/246/247/248, + levtype=modellevels, + levels=1/to/137, + pass=runda, + date=20200601, + time=06, + step=00, + target=mars.source.0 + +@ + +\$RETRIEVE_COMPLETE + +\$UNBLOCK mars.source.0 mars.archive.0 + +\$ARCHIVE_MARS << @ + +archive, + class=EA, + stream=DA, + expver=9999, + type=analysis, + anoffset=3, + param=129/152/130/138/155/75/76/133/203.128/246/247/248, + levtype=modellevels, + levels=1/to/137, + pass=runda, + date=20200601, + time=06, + step=00, + source=mars.archive.0 + +@ + +\$ARCHIVE_COMPLETE +EOF + +cmp $CHECKFILE $OUTFILE diff --git a/tests/marsgen/metkit_marsgen_obs.sh.in b/tests/marsgen/metkit_marsgen_obs.sh.in new file mode 100755 index 00000000..b6cbf2a8 --- /dev/null +++ b/tests/marsgen/metkit_marsgen_obs.sh.in @@ -0,0 +1,39 @@ +#!/bin/bash + +set -uex + +[ -e outfile.simple ] && rm outfile.simple + +MARSGEN="@CMAKE_BINARY_DIR@/bin/mars-archive-script" +GENFILE="@CMAKE_CURRENT_SOURCE_DIR@/genfile" +OUTFILE="outfile.obs" +CHECKFILE="checkfile.obs" + +$MARSGEN --obs < $GENFILE > $OUTFILE + +cat << EOF > $CHECKFILE +\$PREOBS \$header.1 \$data.1 mars.archive.0 + +\$ARCHIVE_MARS << @ + +archive, + class=EA, + stream=DA, + expver=9999, + type=analysis, + anoffset=3, + param=129/152/130/138/155/75/76/133/203.128/246/247/248, + levtype=modellevels, + levels=1/to/137, + pass=runda, + date=20200601, + time=06, + step=00, + source=mars.archive.0 + +@ + +\$ARCHIVE_COMPLETE +EOF + +cmp $CHECKFILE $OUTFILE diff --git a/tests/marsgen/metkit_marsgen_simple.sh.in b/tests/marsgen/metkit_marsgen_simple.sh.in new file mode 100755 index 00000000..e4df82d6 --- /dev/null +++ b/tests/marsgen/metkit_marsgen_simple.sh.in @@ -0,0 +1,60 @@ +#!/bin/bash + +set -uex + +[ -e outfile.simple ] && rm outfile.simple + +MARSGEN="@CMAKE_BINARY_DIR@/bin/mars-archive-script" +GENFILE="@CMAKE_CURRENT_SOURCE_DIR@/genfile" +OUTFILE="outfile.simple" +CHECKFILE="checkfile.simple" + +$MARSGEN --in="$GENFILE" --out="$OUTFILE" + +cat << EOF > $CHECKFILE +\$RETRIEVE_MARS << @ + +retrieve, + class=EA, + stream=LWDA, + expver=9999, + type=analysis, + anoffset=3, + param=129/152/130/138/155/75/76/133/203.128/246/247/248, + levtype=modellevels, + levels=1/to/137, + pass=runda, + date=20200601, + time=06, + step=00, + target=mars.source.0 + +@ + +\$RETRIEVE_COMPLETE + +\$UNBLOCK mars.source.0 mars.archive.0 + +\$ARCHIVE_MARS << @ + +archive, + class=EA, + stream=DA, + expver=9999, + type=analysis, + anoffset=3, + param=129/152/130/138/155/75/76/133/203.128/246/247/248, + levtype=modellevels, + levels=1/to/137, + pass=runda, + date=20200601, + time=06, + step=00, + source=mars.archive.0 + +@ + +\$ARCHIVE_COMPLETE +EOF + +cmp $CHECKFILE $OUTFILE From d7a4a2d2a62089a1c3ac8128d04b1a283927c55f Mon Sep 17 00:00:00 2001 From: Philipp Geier Date: Thu, 21 Sep 2023 15:29:19 +0200 Subject: [PATCH 10/13] METK-114: Add useProxyHostAsCallback option and override host in PassiveProxyCallback on DHSProtocol --- src/metkit/mars/DHSProtocol.cc | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/metkit/mars/DHSProtocol.cc b/src/metkit/mars/DHSProtocol.cc index 286cb9c8..5706f749 100644 --- a/src/metkit/mars/DHSProtocol.cc +++ b/src/metkit/mars/DHSProtocol.cc @@ -194,7 +194,7 @@ class PassiveProxyCallback : public BaseCallbackConnection { public: - explicit PassiveProxyCallback(const Endpoint& proxyhost) : + explicit PassiveProxyCallback(const Endpoint& proxyhost, bool useProxyHostAsCallback = true) : control_(net::TCPClient().connect(proxyhost)), proxyHost_(proxyhost) { @@ -213,6 +213,10 @@ class PassiveProxyCallback : public BaseCallbackConnection { LOG_DEBUG_LIB(LibMetkit) << "Remote address. host=" << remoteAddr_<< std::endl; passiveAddr_ = Endpoint(control_); + + if (useProxyHostAsCallback) { + passiveAddr_ = Endpoint(proxyhost.host(), passiveAddr_.port()); + } LOG_DEBUG_LIB(LibMetkit) << "Passive address. host=" << passiveAddr_<< std::endl; @@ -222,7 +226,7 @@ class PassiveProxyCallback : public BaseCallbackConnection { } explicit PassiveProxyCallback(const Configuration& config) : - PassiveProxyCallback(selectProxyHost(config)) {} + PassiveProxyCallback(selectProxyHost(config), config.getBool("useProxyHostAsCallback", true)) {} explicit PassiveProxyCallback(Stream& s) : PassiveProxyCallback(Endpoint(s)) {} From dc464335cee626b35f2d38bbbce555165bd2573f Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Tue, 26 Sep 2023 12:57:52 +0100 Subject: [PATCH 11/13] fix marsgen tests (disabled if tools are not built) --- tests/marsgen/CMakeLists.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/marsgen/CMakeLists.txt b/tests/marsgen/CMakeLists.txt index 6392ec99..5bf1b607 100644 --- a/tests/marsgen/CMakeLists.txt +++ b/tests/marsgen/CMakeLists.txt @@ -11,7 +11,8 @@ foreach( _test ${marsgen_tests} ) configure_file( metkit_marsgen_${_test}.sh.in metkit_marsgen_${_test}.sh @ONLY ) ecbuild_add_test( - TYPE SCRIPT - COMMAND metkit_marsgen_${_test}.sh ) + TYPE SCRIPT + CONDITION HAVE_BUILD_TOOLS + COMMAND metkit_marsgen_${_test}.sh ) endforeach() \ No newline at end of file From 8a7948d3528fb75e39eed0cfa004d0442ff599f2 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Thu, 12 Oct 2023 13:01:36 +0100 Subject: [PATCH 12/13] Exposed AxisOrder class --- src/metkit/hypercube/HyperCube.cc | 42 +++++++++---------------------- src/metkit/hypercube/HyperCube.h | 15 +++++++++++ src/metkit/mars/MarsRequest.cc | 14 ++++++----- src/metkit/mars/MarsRequest.h | 2 +- 4 files changed, 36 insertions(+), 37 deletions(-) diff --git a/src/metkit/hypercube/HyperCube.cc b/src/metkit/hypercube/HyperCube.cc index 7c7ee5ef..2b1820e1 100644 --- a/src/metkit/hypercube/HyperCube.cc +++ b/src/metkit/hypercube/HyperCube.cc @@ -19,45 +19,28 @@ #include "metkit/mars/MarsRequest.h" #include "metkit/mars/Type.h" -namespace metkit { -namespace hypercube { +namespace metkit::hypercube { static metkit::mars::Type& type(const std::string& name) { static metkit::mars::MarsLanguage language("retrieve"); return *language.type(name); } -namespace { -class AxisOrder { -public: // methods - static AxisOrder& instance() { - static AxisOrder instance; - return instance; - } - - const std::vector& axes() { - return axes_; - } - -private: // methods - AxisOrder() { - eckit::Value axis = eckit::YAMLParser::decodeFile(axisYamlFile()); - const eckit::Value axesNames = axis["axes"]; - - for (size_t i = 0; i < axesNames.size(); ++i) { - axes_.push_back(axesNames[i]); - } - } +AxisOrder::AxisOrder() { + eckit::Value axis = eckit::YAMLParser::decodeFile(axisYamlFile()); + const eckit::Value axesNames = axis["axes"]; - eckit::PathName axisYamlFile() { - return "~metkit/share/metkit/axis.yaml"; + for (size_t i = 0; i < axesNames.size(); ++i) { + axes_.push_back(axesNames[i]); } +} -private: // members - std::vector axes_; -}; +AxisOrder& AxisOrder::instance() { + static AxisOrder instance; + return instance; } + class Axis { public: Axis(const std::string& name, const std::vector& values) : @@ -308,5 +291,4 @@ size_t HyperCube::fieldOrdinal(const metkit::mars::MarsRequest& r, bool noholes) return idx; } -} // namespace hypercube -} // namespace metkit +} // namespace metkit::hypercube diff --git a/src/metkit/hypercube/HyperCube.h b/src/metkit/hypercube/HyperCube.h index 32134987..8c720896 100644 --- a/src/metkit/hypercube/HyperCube.h +++ b/src/metkit/hypercube/HyperCube.h @@ -31,6 +31,21 @@ namespace hypercube { class Axis; +class AxisOrder { +public: // methods + static AxisOrder& instance(); + + const std::vector& axes() { return axes_; } + +private: // methods + AxisOrder(); + + eckit::PathName axisYamlFile() { return "~metkit/share/metkit/axis.yaml"; } + +private: // members + std::vector axes_; +}; + class HyperCube { public: HyperCube(const metkit::mars::MarsRequest&); diff --git a/src/metkit/mars/MarsRequest.cc b/src/metkit/mars/MarsRequest.cc index 124cfeb5..bb9f725e 100644 --- a/src/metkit/mars/MarsRequest.cc +++ b/src/metkit/mars/MarsRequest.cc @@ -121,18 +121,20 @@ bool MarsRequest::empty() const { void MarsRequest::print(std::ostream& s) const { - dump(s, "", ""); + dump(s, "", "", true); } -void MarsRequest::dump(std::ostream& s, const char* cr, const char* tab) const { +void MarsRequest::dump(std::ostream& s, const char* cr, const char* tab, bool verb) const { std::list::const_iterator begin = params_.begin(); std::list::const_iterator end = params_.end(); - - s << verb_; - + if (verb) { + s << verb_ << ','; + } + std::string separator = ""; if (begin != end) { - s << ',' << cr << tab; + s << separator << cr << tab; + separator = ","; int a = 0; for (std::list::const_iterator i = begin; i != end; ++i) { diff --git a/src/metkit/mars/MarsRequest.h b/src/metkit/mars/MarsRequest.h index 96edecf8..42985197 100644 --- a/src/metkit/mars/MarsRequest.h +++ b/src/metkit/mars/MarsRequest.h @@ -94,7 +94,7 @@ class MarsRequest { void md5(eckit::MD5&) const; - void dump(std::ostream&, const char* cr = "\n", const char* tab = "\t") const; + void dump(std::ostream&, const char* cr = "\n", const char* tab = "\t", bool verb = true) const; void setValuesTyped(Type*, const std::vector&); From 24fb431d2c226037e43ed47f9c17924df8d26b68 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Fri, 13 Oct 2023 10:18:18 +0100 Subject: [PATCH 13/13] version bump --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index ccb12466..0e73a4b8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.10.16 +1.10.17