From 75f43bea7feb38d5f1c3b3dc8451674843592213 Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Wed, 27 Nov 2024 15:30:02 +0000 Subject: [PATCH 1/2] improved unit tests --- VERSION | 2 +- tests/test_expand.cc | 330 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 295 insertions(+), 37 deletions(-) diff --git a/VERSION b/VERSION index 45333f3..33fb4ac 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.11.22 +1.11.23 diff --git a/tests/test_expand.cc b/tests/test_expand.cc index 5baaa66..1ca0faa 100644 --- a/tests/test_expand.cc +++ b/tests/test_expand.cc @@ -30,65 +30,183 @@ namespace test { //----------------------------------------------------------------------------- -CASE( "test_metkit_expand_1" ) { - const char* text = "ret,date=-5/to/-1"; - MarsRequest r = MarsRequest::parse(text); - r.dump(std::cout); +void expand(const MarsRequest& r, const std::string& verb, std::map> expected, std::vector dates) { + // MarsExpension exp(false); + // MarsRequest r = exp.expand(req); + std::cout << "comparing " << r << " with " << expected << " dates " << dates << std::endl; + ASSERT(r.verb() == verb); + for(const auto& [key, vals] : expected) { + ASSERT(r.has(key)); + auto vv = r.values(key); + ASSERT(vv.size() == vals.size()); + for (int i=0; i 0) { + ASSERT(r.has("date")); + auto dd = r.values("date"); + ASSERT(dd.size() == dates.size()); + for (int i=0; i& dates = r.values("date"); - EXPECT(dates.size() == 1); - - eckit::Date yesterday(-1); - std::ostringstream oss; - - oss << yesterday.yyyymmdd(); - EXPECT(dates[0] == oss.str()); - +void expand(const std::string& text, const std::string& verb, std::map> expected, std::vector dates) { + MarsRequest r = MarsRequest::parse(text, true); + expand(r, verb, expected, dates); +} +CASE( "test_metkit_expand_1" ) { + const char* text = "ret,date=-5/to/-1."; + std::map> expected{ + {"class", {"od"}}, + {"domain", {"g"}}, + {"expver", {"0001"}}, + {"levelist", {"1000","850","700","500","400","300"}}, + {"levtype", {"pl"}}, + {"param", {"129"}}, + {"step", {"0"}}, + {"stream", {"oper"}}, + {"time", {"1200"}}, + {"type", {"an"}} + }; + expand(text, "retrieve", expected, {-5,-4,-3,-2,-1}); } +// CASE( "test_metkit_expand_2" ) { +// const char* text = "ret"; +// std::map> expected{ +// {"class", {"od"}}, +// {"domain", {"g"}}, +// {"expver", {"0001"}}, +// {"levelist", {"1000","850","700","500","400","300"}}, +// {"levtype", {"pl"}}, +// {"param", {"129"}}, +// {"step", {"0"}}, +// {"stream", {"oper"}}, +// {"time", {"1200"}}, +// {"type", {"an"}} +// }; +// expand(text, "retrieve", expected, {-1}); +// } + CASE( "test_metkit_expand_3" ) { const char* text = "ret,date=-5/to/-1,grid=n640"; - MarsRequest r = MarsRequest::parse(text); - r.dump(std::cout); + std::map> expected{ + {"class", {"od"}}, + {"domain", {"g"}}, + {"expver", {"0001"}}, + {"levelist", {"1000","850","700","500","400","300"}}, + {"levtype", {"pl"}}, + {"param", {"129"}}, + {"step", {"0"}}, + {"stream", {"oper"}}, + {"time", {"1200"}}, + {"type", {"an"}}, + {"grid", {"N640"}} + }; + expand(text, "retrieve", expected, {-5,-4,-3,-2,-1}); } CASE( "test_metkit_expand_4" ) { const char* text = "ret,date=-5/to/-1,grid=o640"; - MarsRequest r = MarsRequest::parse(text); - r.dump(std::cout); + std::map> expected{ + {"class", {"od"}}, + {"domain", {"g"}}, + {"expver", {"0001"}}, + {"levelist", {"1000","850","700","500","400","300"}}, + {"levtype", {"pl"}}, + {"param", {"129"}}, + {"step", {"0"}}, + {"stream", {"oper"}}, + {"time", {"1200"}}, + {"type", {"an"}}, + {"grid", {"O640"}} + }; + expand(text, "retrieve", expected, {-5,-4,-3,-2,-1}); } CASE( "test_metkit_expand_5" ) { const char* text = "retrieve,class=od,date=20050601,diagnostic=1,expver=1,iteration=0,levelist=1,levtype=ml,param=155.129,stream=sens,time=1200,type=sg"; - MarsRequest r = MarsRequest::parse(text); - r.dump(std::cout); + std::map> expected{ + {"class", {"od"}}, + {"diagnostic", {"1"}}, + {"domain", {"g"}}, + {"expver", {"0001"}}, + {"iteration", {"0"}}, + {"levelist", {"1"}}, + {"levtype", {"ml"}}, + {"param", {"129155"}}, + {"step", {"0"}}, + {"stream", {"sens"}}, + {"time", {"1200"}}, + {"type", {"sg"}} + }; + expand(text, "retrieve", expected, {20050601}); } CASE( "test_metkit_expand_6" ) { const char* text = "retrieve,class=rd,expver=hl1m,stream=oper,date=20000801,time=0000,domain=g,type=fc,levtype=pl,step=24,param=129,levelist=1/to/31"; - MarsRequest r = MarsRequest::parse(text); - r.dump(std::cout); + std::map> expected{ + {"class", {"rd"}}, + {"expver", {"hl1m"}}, + {"stream", {"oper"}}, + {"time", {"0000"}}, + {"domain", {"g"}}, + {"type", {"fc"}}, + {"levtype", {"pl"}}, + {"step", {"24"}}, + {"param", {"129"}} + }; + std::vector levelist; + for (int i=1; i<=31; i++) { + levelist.push_back(std::to_string(i)); + } + expected["levelist"] = levelist; + expand(text, "retrieve", expected, {20000801}); } CASE( "test_metkit_expand_7" ) { const char* text = "retrieve,class=rd,expver=hl1m,stream=oper,date=20000801,time=0000,domain=g,type=fc,levtype=pl,step=24,param=129,levelist=0.01/0.7"; - MarsRequest r = MarsRequest::parse(text); - r.dump(std::cout); + std::map> expected{ + {"class", {"rd"}}, + {"expver", {"hl1m"}}, + {"stream", {"oper"}}, + {"time", {"0000"}}, + {"domain", {"g"}}, + {"type", {"fc"}}, + {"levtype", {"pl"}}, + {"step", {"24"}}, + {"param", {"129"}}, + {"levelist", {".01", ".7"}} + }; + expand(text, "retrieve", expected, {20000801}); } CASE( "test_metkit_expand_8" ) { const char* text = "retrieve,class=rd,expver=hl1m,stream=oper,date=20000801,time=0000,domain=g,type=fc,levtype=pl,step=24,param=129,levelist=0.1/to/0.7/by/0.2"; - MarsRequest r = MarsRequest::parse(text); - r.dump(std::cout); + std::map> expected{ + {"class", {"rd"}}, + {"expver", {"hl1m"}}, + {"stream", {"oper"}}, + {"time", {"0000"}}, + {"domain", {"g"}}, + {"type", {"fc"}}, + {"levtype", {"pl"}}, + {"step", {"24"}}, + {"param", {"129"}}, + {"levelist", {".1", ".3", ".5", ".7"}} + }; + expand(text, "retrieve", expected, {20000801}); } CASE( "test_metkit_expand_9_strict" ) { @@ -124,14 +242,58 @@ CASE( "test_metkit_expand_10_strict" ) { ASSERT(v.size() == 1); v[0].dump(std::cout); } - { - std::istringstream in(text); - MarsParser parser(in); - MarsExpension expand(false, true); - EXPECT_THROWS(expand.expand(parser.parse())); - } + // { + // std::istringstream in(text); + // MarsParser parser(in); + // MarsExpension expand(false, true); + // EXPECT_THROWS(expand.expand(parser.parse())); + // } +} + +CASE( "test_metkit_expand_multirequest-1" ) { + const std::string text = "ret,date=-5/to/-2.\nret,date=-1"; + std::istringstream in(text); + std::vector reqs = MarsRequest::parse(in, true); + EXPECT_EQUAL(reqs.size(), 2); + std::map> expected{ + {"class", {"od"}}, + {"domain", {"g"}}, + {"expver", {"0001"}}, + {"levelist", {"1000","850","700","500","400","300"}}, + {"levtype", {"pl"}}, + {"param", {"129"}}, + {"step", {"0"}}, + {"stream", {"oper"}}, + {"time", {"1200"}}, + {"type", {"an"}} + }; + expand(reqs.at(0), "retrieve", expected, {-5,-4,-3,-2}); + expand(reqs.at(1), "retrieve", expected, {-1}); } +// CASE( "test_metkit_expand_multirequest-2" ) { +// const std::string text = "retrieve,\n type=ai,date=20230616,time=12,class=OD,stream=LWDA,\n obstype=airs,expver=0001,target=\"airs\"."; +// std::istringstream in(text); +// std::vector reqs = MarsRequest::parse(in, true); +// EXPECT_EQUAL(reqs.size(), 1); +// std::map> expected{ +// {"class", {"od"}}, +// {"domain", {"g"}}, +// {"expver", {"0001"}}, +// {"stream", {"lwda"}}, +// {"time", {"1200"}}, +// {"type", {"ai"}}, +// {"target", {"\"airs\""}}, +// // {"repres", {"bu"}}, +// {"obstype", {"57"}}, +// {"duplicates", {"keep"}} +// }; + +// expand(reqs.at(0), "retrieve", expected, {20230616}); +// } + + + void expandKeyThrows(const std::string& key, std::vector values) { DummyContext ctx; static metkit::mars::MarsLanguage language("retrieve"); @@ -437,6 +599,102 @@ CASE( "test_metkit_expand_param" ) { } } +CASE( "test_metkit_expand_d1" ) { + { + const char* text = "retrieve,class=d1,dataset=extremes-dt,date=-1"; + std::map> expected{ + {"class", {"d1"}}, + {"dataset", {"extremes-dt"}}, + {"domain", {"g"}}, + {"expver", {"0001"}}, + {"levelist", {"1000","850","700","500","400","300"}}, + {"levtype", {"pl"}}, + {"param", {"129"}}, + {"step", {"0"}}, + {"stream", {"oper"}}, + {"time", {"1200"}}, + {"type", {"an"}} + }; + expand(text, "retrieve", expected, {-1}); + } { + const char* text = "retrieve,class=d1,dataset=extreme-dt,date=-1"; + std::map> expected{ + {"class", {"d1"}}, + {"dataset", {"extremes-dt"}}, + {"expver", {"0001"}}, + {"levelist", {"1000","850","700","500","400","300"}}, + {"levtype", {"pl"}}, + {"param", {"129"}}, + {"step", {"0"}}, + {"stream", {"oper"}}, + {"time", {"1200"}}, + {"type", {"an"}} + }; + expand(text, "retrieve", expected, {-1}); + } + // { + // const char* text = "retrieve,class=d1,dataset=extreme-dt,date=-1"; + // std::istringstream in(text); + // MarsParser parser(in); + // MarsExpension expand(false, true); + // EXPECT_THROWS(expand.expand(parser.parse())); + // } + { + const char* text = "retrieve,class=d1,dataset=climate-dt,levtype=pl,date=20000101,activity=CMIP6,experiment=hist,model=IFS-NEMO,generation=1,realization=1,resolution=high,stream=clte,type=fc,param=134/137"; + std::map> expected{ + {"class", {"d1"}}, + {"dataset", {"climate-dt"}}, + {"activity", {"cmip6"}}, + {"experiment", {"hist"}}, + {"model", {"ifs-nemo"}}, + {"generation", {"1"}}, + {"realization", {"1"}}, + {"resolution", {"high"}}, + {"expver", {"0001"}}, + {"date", {"20000101"}}, + {"time", {"1200"}}, + {"stream", {"clte"}}, + {"type", {"fc"}}, + {"levtype", {"pl"}}, + {"levelist", {"1000","850","700","500","400","300"}}, + {"param", {"134","137"}} + }; + expand(text, "retrieve", expected, {20000101}); + } +} + + +CASE( "test_metkit_expand_ng" ) { + { + const char* text = "retrieve,class=ng,date=20000101,activity=CMIP6,experiment=hist,model=IFS-NEMO,generation=1,realization=1,resolution=high,stream=clte,type=fc,levtype=pl,param=134/137"; + std::map> expected { + {"class", {"ng"}}, + {"levtype", {"pl"}}, + {"levelist", {"1000","850","700","500","400","300"}}, + {"activity", {"cmip6"}}, + {"experiment", {"hist"}}, + {"model", {"ifs-nemo"}}, + {"generation", {"1"}}, + {"realization", {"1"}}, + {"resolution", {"high"}}, + {"expver", {"0001"}}, + {"date", {"20000101"}}, + {"time", {"1200"}}, + {"stream", {"clte"}}, + {"type", {"fc"}}, + {"param", {"134","137"}} + }; + expand(text, "retrieve", expected, {20000101}); + } + { + const char* text = "retrieve,class=ng,dataset=climate-dt,date=20000101,activity=CMIP6,experiment=hist,model=IFS-NEMO,generation=1,realization=1,resolution=high,stream=clte,type=fc,levtype=sfc,param=134/137"; + std::istringstream in(text); + MarsParser parser(in); + MarsExpension expand(false, true); + EXPECT_THROWS(expand.expand(parser.parse())); + } +} + //----------------------------------------------------------------------------- } // namespace test From 963c2950b8a9d4fb17436c5a8de2c40c17d3795d Mon Sep 17 00:00:00 2001 From: Emanuele Danovaro Date: Mon, 9 Dec 2024 12:45:02 +0000 Subject: [PATCH 2/2] cleanup unused code --- src/metkit/mars/StepRange.cc | 63 +++++++++++++++++----------------- src/tools/bufr-sanity-check.cc | 2 +- 2 files changed, 32 insertions(+), 33 deletions(-) diff --git a/src/metkit/mars/StepRange.cc b/src/metkit/mars/StepRange.cc index eb98104..c8b8c3d 100644 --- a/src/metkit/mars/StepRange.cc +++ b/src/metkit/mars/StepRange.cc @@ -23,25 +23,25 @@ using namespace eckit; namespace { -enum TimeUnit { - Second = 0, - Minute = 1, - Hour = 2, - Day = 3 -}; - -TimeUnit maxUnit(const eckit::Time& t) { - if (t.seconds() == 0) { - if (t.minutes() == 0) { - if (t.hours() == 0) { - return TimeUnit::Day; - } - return TimeUnit::Hour; - } - return TimeUnit::Minute; - } - return TimeUnit::Second; -} +// enum TimeUnit { +// Second = 0, +// Minute = 1, +// Hour = 2, +// Day = 3 +// }; + +// TimeUnit maxUnit(const eckit::Time& t) { +// if (t.seconds() == 0) { +// if (t.minutes() == 0) { +// if (t.hours() == 0) { +// return TimeUnit::Day; +// } +// return TimeUnit::Hour; +// } +// return TimeUnit::Minute; +// } +// return TimeUnit::Second; +// } std::string canonical(const eckit::Time& time) { @@ -65,24 +65,23 @@ std::string canonical(const eckit::Time& time) { return out; } -std::string canonical(const eckit::Time& time, TimeUnit unit) { - switch (unit) { - case TimeUnit::Second: - return std::to_string(time.seconds()+60*time.minutes()+3600*time.hours()) + "s"; - case TimeUnit::Minute: - return std::to_string(time.minutes()+60*time.hours()) + "m"; - case TimeUnit::Day: - case TimeUnit::Hour: - default: - return std::to_string(time.hours()); - } -} +// std::string canonical(const eckit::Time& time, TimeUnit unit) { +// switch (unit) { +// case TimeUnit::Second: +// return std::to_string(time.seconds()+60*time.minutes()+3600*time.hours()) + "s"; +// case TimeUnit::Minute: +// return std::to_string(time.minutes()+60*time.hours()) + "m"; +// case TimeUnit::Day: +// case TimeUnit::Hour: +// default: +// return std::to_string(time.hours()); +// } +// } } namespace metkit::mars { - //---------------------------------------------------------------------------------------------------------------------- StepRange::operator std::string() const diff --git a/src/tools/bufr-sanity-check.cc b/src/tools/bufr-sanity-check.cc index 9fa56e3..c2e4b4f 100644 --- a/src/tools/bufr-sanity-check.cc +++ b/src/tools/bufr-sanity-check.cc @@ -380,7 +380,7 @@ void BufrCheck::process(const PathName& input, const PathName& output) { numMessage++; } } - catch(eckit::Exception e) { + catch(eckit::Exception& e) { Log::warning() << " Error parsing message " << numMessage << " - offset " << pos << std::endl; if (verbose_) { e.dumpStackTrace(Log::warning());