From 45ee4a06886aa29d96d1ed2ace82f304447f9564 Mon Sep 17 00:00:00 2001 From: Sara Thon Date: Tue, 25 Aug 2026 14:02:20 +0200 Subject: [PATCH 1/5] CFE-3041: change regular acceptance test to .error.cf Ticket: CFE-3041 --- tests/acceptance/00_basics/03_bodies/{112.cf => 112.error.cf} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/acceptance/00_basics/03_bodies/{112.cf => 112.error.cf} (100%) diff --git a/tests/acceptance/00_basics/03_bodies/112.cf b/tests/acceptance/00_basics/03_bodies/112.error.cf similarity index 100% rename from tests/acceptance/00_basics/03_bodies/112.cf rename to tests/acceptance/00_basics/03_bodies/112.error.cf From 0a625700e2dc90c864f17dd27f4b503e6d78396c Mon Sep 17 00:00:00 2001 From: Ole Herman Schumacher Elgesem Date: Wed, 7 Oct 2026 14:47:31 +0200 Subject: [PATCH 2/5] Added new type of test to testall - errorpass Signed-off-by: Ole Herman Schumacher Elgesem --- tests/acceptance/testall | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/tests/acceptance/testall b/tests/acceptance/testall index 4166191fe56..02de29d2b38 100755 --- a/tests/acceptance/testall +++ b/tests/acceptance/testall @@ -323,9 +323,11 @@ runtest() { case "$TEST" in *.x.cf) TEST_TYPE=errorexit && - EXTRATEXT='(should exit with non-zero, but not crash)' ;; + EXTRATEXT='(should exit with non-zero, but not crash. Printing pass or fail is irrelevant)' ;; *.error.cf) TEST_TYPE=errorlog && - EXTRATEXT='(should log error(s), but exit with 0)' ;; + EXTRATEXT='(should log error(s), but exit with 0. Printing pass or fail is irrelevant)' ;; + *.error.pass.cf) TEST_TYPE=errorpass && + EXTRATEXT='(should log error(s), print pass message and, exit with 0)' ;; */staging/*) TEST_TYPE=staging ;; */unsafe/*) TEST_TYPE=unsafe ;; */network/*) TEST_TYPE=network ;; @@ -565,6 +567,24 @@ export CFENGINE_TEST_OVERRIDE_WORKDIR TEMP CFENGINE_TEST_OVERRIDE_EXTENSION_LIBR RESULT=FAIL RESULT_MSG="${COLOR_FAILURE}FAIL (No errors printed)${COLOR_NORMAL}" fi + elif [ $TEST_TYPE = errorpass ] + then + if [ $RETVAL != 0 ] + then + RESULT=FAIL + RESULT_MSG="${COLOR_FAILURE}FAIL (NON-ZERO EXIT CODE!)${COLOR_NORMAL}" + elif ! grep -F 'error: ' "$OUTFILE" > /dev/null + then + RESULT=FAIL + RESULT_MSG="${COLOR_FAILURE}FAIL (No error message!)${COLOR_NORMAL}" + elif ! grep -E "R: .*$ESCAPED_TEST Pass" $OUTFILE > /dev/null + then + RESULT=FAIL + RESULT_MSG="${COLOR_FAILURE}FAIL (Pass not printed!)${COLOR_NORMAL}" + else + RESULT=Pass + RESULT_MSG="${COLOR_SUCCESS}Pass${COLOR_NORMAL}" + fi else # TEST_TYPE is not errorlog or errorexit # We need to be careful when matching test outcomes. Because of convergence # passes, a test may output FAIL before it outputs Pass; in this case the From 0f7d959e0ae5ffbaeb589cad567f3c1b642dabe8 Mon Sep 17 00:00:00 2001 From: Ole Herman Schumacher Elgesem Date: Wed, 7 Oct 2026 14:48:37 +0200 Subject: [PATCH 3/5] Moved acceptance test to error.pass.cf test Signed-off-by: Ole Herman Schumacher Elgesem --- .../00_basics/03_bodies/{112.error.cf => 112.error.pass.cf} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/acceptance/00_basics/03_bodies/{112.error.cf => 112.error.pass.cf} (100%) diff --git a/tests/acceptance/00_basics/03_bodies/112.error.cf b/tests/acceptance/00_basics/03_bodies/112.error.pass.cf similarity index 100% rename from tests/acceptance/00_basics/03_bodies/112.error.cf rename to tests/acceptance/00_basics/03_bodies/112.error.pass.cf From 453630a331db9f1815431b0cfe1459f27bb6b958 Mon Sep 17 00:00:00 2001 From: Ole Herman Schumacher Elgesem Date: Wed, 7 Oct 2026 16:55:47 +0200 Subject: [PATCH 4/5] Fix errorpass test(?) Signed-off-by: Ole Herman Schumacher Elgesem --- tests/acceptance/testall | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/acceptance/testall b/tests/acceptance/testall index 02de29d2b38..2773f6aab4b 100755 --- a/tests/acceptance/testall +++ b/tests/acceptance/testall @@ -256,7 +256,7 @@ usage() { echo " defaults to $DEFLIBTOOL." echo " --include Include the file or directory given in the workdir before the test" echo " starts. This option may be given several times." - echo " --tests=common,network,serial,timed,slow,errorexit,errorlog,libxml2,libcurl" + echo " --tests=common,network,serial,timed,slow,errorexit,errorlog,errorpass,libxml2,libcurl" echo " This is the default value, you can also add from the following:" echo " unsafe,staging" echo " NOTE: 'staging' tests are not expected to pass" @@ -345,6 +345,10 @@ runtest() { then SKIP=1 SKIPREASON="${COLOR_WARNING}'errorexit' tests are disabled${COLOR_NORMAL}" + elif [ $ERRORPASS_TESTS = 0 -a $TEST_TYPE = errorpass ] + then + SKIP=1 + SKIPREASON="${COLOR_WARNING}'errorpass' tests are disabled${COLOR_NORMAL}" elif [ $ERRORLOG_TESTS = 0 -a $TEST_TYPE = errorlog ] then SKIP=1 @@ -569,6 +573,7 @@ export CFENGINE_TEST_OVERRIDE_WORKDIR TEMP CFENGINE_TEST_OVERRIDE_EXTENSION_LIBR fi elif [ $TEST_TYPE = errorpass ] then + ESCAPED_TEST="$(echo "($TEST|dcs.sub.cf)" | sed -e 's/\./\\./g')" if [ $RETVAL != 0 ] then RESULT=FAIL @@ -585,7 +590,7 @@ export CFENGINE_TEST_OVERRIDE_WORKDIR TEMP CFENGINE_TEST_OVERRIDE_EXTENSION_LIBR RESULT=Pass RESULT_MSG="${COLOR_SUCCESS}Pass${COLOR_NORMAL}" fi - else # TEST_TYPE is not errorlog or errorexit + else # TEST_TYPE is not errorlog or errorexit or errorpass # We need to be careful when matching test outcomes. Because of convergence # passes, a test may output FAIL before it outputs Pass; in this case the # latter trumps the former. Also be careful when matching an [XS]FAIL; the @@ -834,6 +839,7 @@ do SLOW_TESTS=0 ERROREXIT_TESTS=0 ERRORLOG_TESTS=0 + ERRORPASS_TESTS=0 SERIAL_TESTS=0 NETWORK_TESTS=0 LIBXML2_TESTS=0 @@ -850,6 +856,7 @@ do slow) SLOW_TESTS=1;; errorexit) ERROREXIT_TESTS=1;; errorlog) ERRORLOG_TESTS=1;; + errorpass) ERRORPASS_TESTS=1;; serial) SERIAL_TESTS=1;; network) NETWORK_TESTS=1;; libxml2) LIBXML2_TESTS=1;; From 1bafc0c22ed63a1a479437bb829a9b7d53b0adec Mon Sep 17 00:00:00 2001 From: Ole Herman Schumacher Elgesem Date: Wed, 7 Oct 2026 17:07:49 +0200 Subject: [PATCH 5/5] Enabled errorpass tests in GH Actions Signed-off-by: Ole Herman Schumacher Elgesem --- .github/workflows/acceptance_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/acceptance_tests.yml b/.github/workflows/acceptance_tests.yml index 6e1d1a7ba89..31ef03c8b31 100644 --- a/.github/workflows/acceptance_tests.yml +++ b/.github/workflows/acceptance_tests.yml @@ -20,4 +20,4 @@ jobs: run: | cd tests/acceptance chmod -R go-w . - ./testall --printlog --tests=common,errorlog,errorexit + ./testall --printlog --tests=common,errorlog,errorexit,errorpass