diff --git a/integration/tests_ok/html.ps1 b/integration/tests_ok/html.ps1 index 7ace2d54f31..57782709e7e 100644 --- a/integration/tests_ok/html.ps1 +++ b/integration/tests_ok/html.ps1 @@ -1,6 +1,8 @@ Set-StrictMode -Version latest $ErrorActionPreference = 'Stop' +# Some tests are failing but we want to continue until the end +$ErrorActionPreference = "Continue" hurl --test ` --report-html build\tmp\ ` --glob "tests_ok\test.*.hurl" diff --git a/integration/tests_ok/html.sh b/integration/tests_ok/html.sh index 4efd460ba4b..3930097f5ac 100755 --- a/integration/tests_ok/html.sh +++ b/integration/tests_ok/html.sh @@ -1,6 +1,8 @@ #!/bin/bash set -Eeuo pipefail +# Some tests are failing but we want to continue until the end +set +euo pipefail hurl --test \ --report-html build/ \ --glob "tests_ok/test.*.hurl" diff --git a/integration/tests_ok/junit.err.pattern b/integration/tests_ok/junit.err.pattern index ed5ccf1b005..b9983810f0c 100755 --- a/integration/tests_ok/junit.err.pattern +++ b/integration/tests_ok/junit.err.pattern @@ -1,15 +1,22 @@ -tests_ok~test.1.hurl: Running [1/2] -tests_ok~test.1.hurl: Success (1 request(s) in ~~~ ms) -tests_ok~test.2.hurl: Running [2/2] -tests_ok~test.2.hurl: Success (1 request(s) in ~~~ ms) +tests_ok/test.1.hurl: Running [1/2] +tests_ok/test.1.hurl: Success (1 request(s) in ~~~ ms) +tests_ok/test.2.hurl: Running [2/2] +error: Assert body value + --> tests_ok/test.2.hurl:8:1 + | + 8 | `Goodbye World!` + | ^^^^^^^^^^^^^^^^ actual value is + | + +tests_ok/test.2.hurl: Failure (2 request(s) in ~~~ ms) -------------------------------------------------------------------------------- Executed files: 2 -Succeeded files: 2 (100.0%) -Failed files: 0 (0.0%) -Duration: ~ ms +Succeeded files: 1 (50.0%) +Failed files: 1 (50.0%) +Duration: ~~~ ms -tests_ok~test.3.hurl: Running [1/1] -tests_ok~test.3.hurl: Success (1 request(s) in ~~~ ms) +tests_ok/test.3.hurl: Running [1/1] +tests_ok/test.3.hurl: Success (1 request(s) in ~~~ ms) -------------------------------------------------------------------------------- Executed files: 1 Succeeded files: 1 (100.0%) diff --git a/integration/tests_ok/junit.out.pattern b/integration/tests_ok/junit.out.pattern index 762e77be906..3be6fc6c1df 100644 --- a/integration/tests_ok/junit.out.pattern +++ b/integration/tests_ok/junit.out.pattern @@ -1 +1,6 @@ - \ No newline at end of file +Assert body value + --> tests_ok/test.2.hurl:8:1 + | + 8 | `Goodbye World!` + | ^^^^^^^^^^^^^^^^ actual value is <Hello World!> + | \ No newline at end of file diff --git a/integration/tests_ok/junit.sh b/integration/tests_ok/junit.sh index 2d96b811bd7..620c1961482 100755 --- a/integration/tests_ok/junit.sh +++ b/integration/tests_ok/junit.sh @@ -1,6 +1,11 @@ #!/bin/bash set -Eeuo pipefail rm -f build/result.xml + +# test.2.hurl is KO but we want the script to continue until the end +set +eo pipefail hurl --test --report-junit build/result.xml tests_ok/test.1.hurl tests_ok/test.2.hurl hurl --test --report-junit build/result.xml tests_ok/test.3.hurl +set -Eeuo pipefail + cat build/result.xml diff --git a/integration/tests_ok/tap.out b/integration/tests_ok/tap.out index 535bc81cfed..2791be5b7c1 100644 --- a/integration/tests_ok/tap.out +++ b/integration/tests_ok/tap.out @@ -1,5 +1,5 @@ TAP version 13 1..3 ok 1 - tests_ok/test.1.hurl -ok 2 - tests_ok/test.2.hurl +not ok 2 - tests_ok/test.2.hurl ok 3 - tests_ok/test.3.hurl diff --git a/integration/tests_ok/tap.ps1 b/integration/tests_ok/tap.ps1 index a36df3dddd0..c0760847386 100644 --- a/integration/tests_ok/tap.ps1 +++ b/integration/tests_ok/tap.ps1 @@ -3,6 +3,11 @@ $ErrorActionPreference = 'Stop' if (Test-Path build/result.tap) { Remove-Item build/result.tap } + +# test.2.hurl is KO but we want the script to continue until the end +$ErrorActionPreference = 'Continue' hurl --test --report-tap build/result.tap tests_ok/test.1.hurl tests_ok/test.2.hurl hurl --test --report-tap build/result.tap tests_ok/test.3.hurl +$ErrorActionPreference = 'Stop' + Write-Host (Get-Content build/result.tap -Raw) -NoNewLine diff --git a/integration/tests_ok/tap.sh b/integration/tests_ok/tap.sh index 105d33e0548..06005d174df 100755 --- a/integration/tests_ok/tap.sh +++ b/integration/tests_ok/tap.sh @@ -1,6 +1,11 @@ #!/bin/bash set -Eeuo pipefail rm -f build/result.tap + +# test.2.hurl is KO but we want the script to continue until the end +set +eo pipefail hurl --test --report-tap build/result.tap tests_ok/test.1.hurl tests_ok/test.2.hurl hurl --test --report-tap build/result.tap tests_ok/test.3.hurl +set -Eeuo pipefail + cat build/result.tap diff --git a/integration/tests_ok/test.2.html b/integration/tests_ok/test.2.html index e460aee8c1d..4121b91d9b6 100644 --- a/integration/tests_ok/test.2.html +++ b/integration/tests_ok/test.2.html @@ -1,5 +1,9 @@ 
GET http://localhost:8000/hello
-
-HTTP 200
+HTTP 200
 `Hello World!`
+
+
+GET http://localhost:8000/hello
+HTTP 200
+`Goodbye World!`
diff --git a/integration/tests_ok/test.2.hurl b/integration/tests_ok/test.2.hurl index 19f3b0475c6..81264ec2730 100644 --- a/integration/tests_ok/test.2.hurl +++ b/integration/tests_ok/test.2.hurl @@ -1,4 +1,8 @@ GET http://localhost:8000/hello - HTTP 200 `Hello World!` + + +GET http://localhost:8000/hello +HTTP 200 +`Goodbye World!` diff --git a/integration/tests_ok/test.2.json b/integration/tests_ok/test.2.json index 01ffc165bbf..e271bf9d44a 100644 --- a/integration/tests_ok/test.2.json +++ b/integration/tests_ok/test.2.json @@ -1 +1 @@ -{"entries":[{"request":{"method":"GET","url":"http://localhost:8000/hello"},"response":{"status":200,"body":{"type":"text","value":"Hello World!"}}}]} +{"entries":[{"request":{"method":"GET","url":"http://localhost:8000/hello"},"response":{"status":200,"body":{"type":"text","value":"Hello World!"}}},{"request":{"method":"GET","url":"http://localhost:8000/hello"},"response":{"status":200,"body":{"type":"text","value":"Goodbye World!"}}}]} diff --git a/integration/tests_ok/test.err.pattern b/integration/tests_ok/test.err.pattern index 0797e87223a..1ee1657e071 100644 --- a/integration/tests_ok/test.err.pattern +++ b/integration/tests_ok/test.err.pattern @@ -1,12 +1,19 @@ tests_ok~test.1.hurl: Running [1/3] tests_ok~test.1.hurl: Success (1 request(s) in ~~~ ms) tests_ok~test.2.hurl: Running [2/3] -tests_ok~test.2.hurl: Success (1 request(s) in ~~~ ms) +error: Assert body value + --> tests_ok~test.2.hurl:8:1 + | + 8 | `Goodbye World!` + | ^^^^^^^^^^^^^^^^ actual value is ~ + | + +tests_ok~test.2.hurl: Failure (2 request(s) in ~~~ ms) tests_ok~test.3.hurl: Running [3/3] tests_ok~test.3.hurl: Success (1 request(s) in ~~~ ms) -------------------------------------------------------------------------------- Executed files: 3 -Succeeded files: 3 (100.0%) -Failed files: 0 (0.0%) +Succeeded files: 2 (66.7%) +Failed files: 1 (33.3%) Duration: ~~~ ms diff --git a/integration/tests_ok/test.exit b/integration/tests_ok/test.exit new file mode 100644 index 00000000000..bf0d87ab1b2 --- /dev/null +++ b/integration/tests_ok/test.exit @@ -0,0 +1 @@ +4 \ No newline at end of file diff --git a/packages/hurl/src/report/tap/report.rs b/packages/hurl/src/report/tap/report.rs index fb0164221ff..5daaf62b999 100644 --- a/packages/hurl/src/report/tap/report.rs +++ b/packages/hurl/src/report/tap/report.rs @@ -56,10 +56,10 @@ fn write_tap_file(filename: &str, testcases: &[&Testcase]) -> Result<(), Error> s.push_str(format!("{start}..{end}\n").as_str()); for (i, testcase) in testcases.iter().enumerate() { - let success = if testcase.success { "" } else { "not " }; + let state = if testcase.success { "ok" } else { "not ok" }; let number = i + 1; let description = &testcase.description; - s.push_str(format!("{success}ok {number} - {description}\n").as_str()); + s.push_str(format!("{state} {number} - {description}\n").as_str()); } match file.write_all(s.as_bytes()) { Ok(_) => Ok(()), @@ -149,7 +149,7 @@ mod tests { let s = r#"1..3 ok 1 - tests_ok/test.1.hurl ok 2 -tests_ok/test.2.hurl -nok 3 - tests_ok/test.3.hurl +not ok 3 - tests_ok/test.3.hurl "#; assert_eq!( @@ -177,7 +177,7 @@ nok 3 - tests_ok/test.3.hurl 1..3 ok 1 - tests_ok/test.1.hurl ok 2 -tests_ok/test.2.hurl -nok 3 - tests_ok/test.3.hurl +not ok 3 - tests_ok/test.3.hurl "#; assert_eq!( diff --git a/packages/hurl/src/report/tap/testcase.rs b/packages/hurl/src/report/tap/testcase.rs index 054391961eb..57386a2cb90 100644 --- a/packages/hurl/src/report/tap/testcase.rs +++ b/packages/hurl/src/report/tap/testcase.rs @@ -44,8 +44,8 @@ impl Testcase { let success = if line.starts_with("ok") { line = &line[2..]; true - } else if line.starts_with("nok") { - line = &line[3..]; + } else if line.starts_with("not ok") { + line = &line[6..]; false } else { return Err(Error {