This is an automated email from the ASF dual-hosted git repository.
pcongiusti pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/camel-k.git
The following commit(s) were added to refs/heads/main by this push:
new df2c5d4b3 Add support for TEST_SKIP_AFTER_FAILURE_COUNT
df2c5d4b3 is described below
commit df2c5d4b3962b40a861cea9e35bce2a38ba53964
Author: Thomas Diesler <[email protected]>
AuthorDate: Tue May 7 14:19:12 2024 +0200
Add support for TEST_SKIP_AFTER_FAILURE_COUNT
---
e2e/README.md | 54 +++++++++++++++++++++++++++---------------------------
script/Makefile | 52 ++++++++++++++++++++++++++++++++--------------------
2 files changed, 59 insertions(+), 47 deletions(-)
diff --git a/e2e/README.md b/e2e/README.md
index 0f44a1cc4..a65207527 100644
--- a/e2e/README.md
+++ b/e2e/README.md
@@ -2,6 +2,33 @@
This directory contains the suite of test that are run on a CI to ensure the
stability of the product and no regression are introduced at each PR. The full
documentation can be found at
https://camel.apache.org/camel-k/next/contributing/e2e.html
+## Environment variables
+
+You can set some environment variables to change the behavior of the E2E test
suite.
+
+| Env | Default
| Description
|
+|-----------------------------------------|-----------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------|
+| CAMEL_K_TEST_SKIP_PROBLEMATIC | false
| Skips tests that are marked to be problematic (flaky) on certain
environments (e.g. on OpenShift). |
+| CAMEL_K_TEST_SAVE_FAILED_TEST_NAMESPACE | false
| Used to not remove the temporary test namespaces after the test run.
Enables better analysis of resources after the test |
+| CAMEL_K_TEST_LOG_LEVEL | info
| Logging level used to run the tests and used in Maven commands run by
the operator (if level is `debug` the Maven commands use `-X` option). |
+| CAMEL_K_TEST_MAVEN_CLI_OPTIONS | {}
| Maven CLI options used to run Camel K integrations during the tests.
|
+| CAMEL_K_TEST_OPERATOR_IMAGE |
docker.io/apache/camel-k:2.4.0-SNAPSHOT | Camel K operator image used in
operator installation.
|
+| CAMEL_K_TEST_OPERATOR_IMAGE_PULL_POLICY | -
| Operator image pull policy.
|
+| CAMEL_K_TEST_IMAGE_NAME | docker.io/apache/camel-k
| Camel K operator image name used in operator installation.
|
+| CAMEL_K_TEST_IMAGE_VERSION | 2.4.0-SNAPSHOT
| Camel K operator image version used in operator installation. Value is
retrieved from `pkg/util/defaults/defaults.go` |
+| CAMEL_K_TEST_NO_OLM_OPERATOR_IMAGE |
docker.io/apache/camel-k:2.4.0-SNAPSHOT | Camel K operator image used in non
OLM based operator installation.
|
+| CAMEL_K_TEST_RUNTIME_VERSION | 3.8.1
| Camel K runtime version used for the integrations. Value is retrieved
from `pkg/util/defaults/defaults.go` |
+| CAMEL_K_TEST_BASE_IMAGE | eclipse-temurin:17
| Camel K runtime base image used for the integrations. Value is retrieved
from `pkg/util/defaults/defaults.go` |
+| CAMEL_K_TEST_TIMEOUT_SHORT | 1
| Customize the timeouts (in minutes) used in test assertions.
|
+| CAMEL_K_TEST_TIMEOUT_MEDIUM | 5
| Customize the timeouts (in minutes) used in test assertions.
|
+| CAMEL_K_TEST_TIMEOUT_LONG | 15
| Customize the timeouts (in minutes) used in test assertions.
|
+| CAMEL_K_TEST_MAVEN_CA_PEM_PATH | -
| Optional Maven certificate path.
|
+| CAMEL_K_TEST_COPY_CATALOG | true
| Enable/disable the optimization to copy the Camel Catalog from default
operator namespace for each test namespace. |
+| CAMEL_K_TEST_COPY_INTEGRATION_KITS | true
| Enable/disable the optimization to copy integration kits from default
operator namespace for each test namespace. |
+| CAMEL_K_TEST_NS | -
| Custom test namespace name used to create temporary namespaces.
|
+| CAMEL_K_TEST_MAKE_DIR | -
| Used in Helm and Kustomize install tests as Makefile root dir.
|
+| CAMEL_K_TEST_MAKE_ARGS | -
| Used in Helm and Kustomize install tests as Makefile arguments.
|
+
## Structure of the directory
NOTE: dear contributor, please, keep this organization as clean as you can,
updating any documentation if any change is done.
@@ -46,30 +73,3 @@ Test suite that cover the features associated with Telemetry
feature. The test e
### Yaks
Test suite that cover certain Knative features together with YAKS operator.
-
-## Environment variables
-
-You can set some environment variables to change the behavior of the E2E test
suite.
-
-| Env | Default
| Description
|
-|-----------------------------------------|-----------------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------|
-| CAMEL_K_TEST_SKIP_PROBLEMATIC | false
| Skips tests that are marked to be problematic (flaky) on certain
environments (e.g. on OpenShift). |
-| CAMEL_K_TEST_SAVE_FAILED_TEST_NAMESPACE | false
| Used to not remove the temporary test namespaces after the test run.
Enables better analysis of resources after the test |
-| CAMEL_K_TEST_LOG_LEVEL | info
| Logging level used to run the tests and used in Maven commands run by
the operator (if level is `debug` the Maven commands use `-X` option). |
-| CAMEL_K_TEST_MAVEN_CLI_OPTIONS | {}
| Maven CLI options used to run Camel K integrations during the tests.
|
-| CAMEL_K_TEST_OPERATOR_IMAGE |
docker.io/apache/camel-k:2.4.0-SNAPSHOT | Camel K operator image used in
operator installation.
|
-| CAMEL_K_TEST_OPERATOR_IMAGE_PULL_POLICY | -
| Operator image pull policy.
|
-| CAMEL_K_TEST_IMAGE_NAME | docker.io/apache/camel-k
| Camel K operator image name used in operator installation.
|
-| CAMEL_K_TEST_IMAGE_VERSION | 2.4.0-SNAPSHOT
| Camel K operator image version used in operator installation. Value is
retrieved from `pkg/util/defaults/defaults.go` |
-| CAMEL_K_TEST_NO_OLM_OPERATOR_IMAGE |
docker.io/apache/camel-k:2.4.0-SNAPSHOT | Camel K operator image used in non
OLM based operator installation.
|
-| CAMEL_K_TEST_RUNTIME_VERSION | 3.8.1
| Camel K runtime version used for the integrations. Value is retrieved
from `pkg/util/defaults/defaults.go` |
-| CAMEL_K_TEST_BASE_IMAGE | eclipse-temurin:17
| Camel K runtime base image used for the integrations. Value is retrieved
from `pkg/util/defaults/defaults.go` |
-| CAMEL_K_TEST_TIMEOUT_SHORT | 1
| Customize the timeouts (in minutes) used in test assertions.
|
-| CAMEL_K_TEST_TIMEOUT_MEDIUM | 5
| Customize the timeouts (in minutes) used in test assertions.
|
-| CAMEL_K_TEST_TIMEOUT_LONG | 15
| Customize the timeouts (in minutes) used in test assertions.
|
-| CAMEL_K_TEST_MAVEN_CA_PEM_PATH | -
| Optional Maven certificate path.
|
-| CAMEL_K_TEST_COPY_CATALOG | true
| Enable/disable the optimization to copy the Camel Catalog from default
operator namespace for each test namespace. |
-| CAMEL_K_TEST_COPY_INTEGRATION_KITS | true
| Enable/disable the optimization to copy integration kits from default
operator namespace for each test namespace. |
-| CAMEL_K_TEST_NS | -
| Custom test namespace name used to create temporary namespaces.
|
-| CAMEL_K_TEST_MAKE_DIR | -
| Used in Helm and Kustomize install tests as Makefile root dir.
|
-| CAMEL_K_TEST_MAKE_ARGS | -
| Used in Helm and Kustomize install tests as Makefile arguments.
|
diff --git a/script/Makefile b/script/Makefile
index 9898b30e8..49d919ca7 100644
--- a/script/Makefile
+++ b/script/Makefile
@@ -272,35 +272,47 @@ endif
#
test-common: do-build
FAILED=0; STAGING_RUNTIME_REPO="$(STAGING_RUNTIME_REPO)"; \
- go test -timeout 90m -v ./e2e/common/... -tags=integration
-parallel=$(TEST_COMMON_PARALLEL_COUNT) $(TEST_INTEGRATION_COMMON_LANG_RUN)
$(GOTESTFMT) || FAILED=1; \
+ go test -timeout 90m -v ./e2e/common/... -tags=integration
-parallel=$(TEST_COMMON_PARALLEL_COUNT) $(TEST_INTEGRATION_COMMON_LANG_RUN)
$(GOTESTFMT) || ((FAILED++)); \
exit $${FAILED}
#
# Smoke tests are a subset of common tests used to quickly verify the operator
#
test-smoke: do-build
+ TEST_SKIP_AFTER_FAILURE_COUNT="$(TEST_SKIP_AFTER_FAILURE_COUNT)"; \
+ if [[ $$TEST_SKIP_AFTER_FAILURE_COUNT = "" ]]; then \
+ TEST_SKIP_AFTER_FAILURE_COUNT=0; \
+ fi; \
FAILED=0; STAGING_RUNTIME_REPO="$(STAGING_RUNTIME_REPO)"; \
- go test -timeout 10m -v ./e2e/common/main_test.go -tags=integration
$(TEST_INTEGRATION_COMMON_LANG_RUN) $(GOTESTFMT) || FAILED=1; \
- go test -timeout 30m -v ./e2e/common/languages -tags=integration
-parallel=$(TEST_COMMON_PARALLEL_COUNT) $(TEST_INTEGRATION_COMMON_LANG_RUN)
$(GOTESTFMT) || FAILED=1; \
- go test -timeout 30m -v \
- ./e2e/common/misc/cron_test.go \
- ./e2e/common/misc/kamelet_test.go \
- ./e2e/common/misc/pipe_test.go \
- -tags=integration -parallel=$(TEST_COMMON_PARALLEL_COUNT)
$(TEST_INTEGRATION_COMMON_LANG_RUN) $(GOTESTFMT) || FAILED=1; \
- go test -timeout 30m -v \
- ./e2e/common/traits/camel_test.go \
- ./e2e/common/traits/container_test.go \
- ./e2e/common/traits/openapi_test.go \
- ./e2e/common/traits/service_test.go \
- -tags=integration -parallel=$(TEST_COMMON_PARALLEL_COUNT)
$(TEST_INTEGRATION_COMMON_LANG_RUN) $(GOTESTFMT) || FAILED=1; \
- exit $${FAILED}
+ echo "TEST_COMMON_PARALLEL_COUNT=$$TEST_COMMON_PARALLEL_COUNT"; \
+ echo "TEST_SKIP_AFTER_FAILURE_COUNT=$$TEST_SKIP_AFTER_FAILURE_COUNT"; \
+ go test -timeout 10m -count=1 -v ./e2e/common/main_test.go
-tags=integration $(TEST_INTEGRATION_COMMON_LANG_RUN) $(GOTESTFMT) ||
((FAILED++)); \
+ if [ $$FAILED -le $$TEST_SKIP_AFTER_FAILURE_COUNT ]; then \
+ go test -timeout 30m -count=1 -v ./e2e/common/languages
-tags=integration -parallel=$(TEST_COMMON_PARALLEL_COUNT)
$(TEST_INTEGRATION_COMMON_LANG_RUN) $(GOTESTFMT) || ((FAILED++)); \
+ fi; \
+ if [ $$FAILED -le $$TEST_SKIP_AFTER_FAILURE_COUNT ]; then \
+ go test -timeout 30m -count=1 -v \
+ ./e2e/common/misc/cron_test.go \
+ ./e2e/common/misc/kamelet_test.go \
+ ./e2e/common/misc/pipe_test.go \
+ -tags=integration
-parallel=$(TEST_COMMON_PARALLEL_COUNT) $(TEST_INTEGRATION_COMMON_LANG_RUN)
$(GOTESTFMT) || ((FAILED++)); \
+ fi; \
+ if [ $$FAILED -le $$TEST_SKIP_AFTER_FAILURE_COUNT ]; then \
+ go test -timeout 30m -count=1 -v \
+ ./e2e/common/traits/camel_test.go \
+ ./e2e/common/traits/container_test.go \
+ ./e2e/common/traits/openapi_test.go \
+ ./e2e/common/traits/service_test.go \
+ -tags=integration
-parallel=$(TEST_COMMON_PARALLEL_COUNT) $(TEST_INTEGRATION_COMMON_LANG_RUN)
$(GOTESTFMT) || ((FAILED++)); \
+ fi; \
+ exit $$FAILED
#
# Common tests that require some particular operator setting or need to be
installed in multiple namespaces
#
test-advanced: do-build
FAILED=0; STAGING_RUNTIME_REPO="$(STAGING_RUNTIME_REPO)"; \
- go test -timeout 90m -v ./e2e/advanced -tags=integration
-parallel=$(TEST_ADVANCED_PARALLEL_COUNT) $(TEST_INSTALL_RUN) $(GOTESTFMT) ||
FAILED=1; \
+ go test -timeout 90m -v ./e2e/advanced -tags=integration
-parallel=$(TEST_ADVANCED_PARALLEL_COUNT) $(TEST_INSTALL_RUN) $(GOTESTFMT) ||
((FAILED++)); \
exit $${FAILED}
#
@@ -308,9 +320,9 @@ test-advanced: do-build
#
test-install: do-build
FAILED=0; STAGING_RUNTIME_REPO="$(STAGING_RUNTIME_REPO)"; \
- go test -timeout 40m -v ./e2e/install/cli -tags=integration
$(TEST_INSTALL_RUN) $(GOTESTFMT) || FAILED=1; \
- go test -timeout 40m -v ./e2e/install/kustomize -tags=integration
$(TEST_INSTALL_RUN) $(GOTESTFMT) || FAILED=1; \
- go test -timeout 40m -v ./e2e/install/helm -tags=integration
$(TEST_INSTALL_RUN) $(GOTESTFMT) || FAILED=1; \
+ go test -timeout 40m -v ./e2e/install/cli -tags=integration
$(TEST_INSTALL_RUN) $(GOTESTFMT) || ((FAILED++)); \
+ go test -timeout 40m -v ./e2e/install/kustomize -tags=integration
$(TEST_INSTALL_RUN) $(GOTESTFMT) || ((FAILED++)); \
+ go test -timeout 40m -v ./e2e/install/helm -tags=integration
$(TEST_INSTALL_RUN) $(GOTESTFMT) || ((FAILED++)); \
exit $${FAILED}
#
@@ -332,7 +344,7 @@ test-install-upgrade: do-build
#
test-knative: do-build
STAGING_RUNTIME_REPO="$(STAGING_RUNTIME_REPO)"; \
- go test -timeout 60m -v ./e2e/knative -tags=integration
$(TEST_KNATIVE_RUN) $(GOTESTFMT) || FAILED=1; \
+ go test -timeout 60m -v ./e2e/knative -tags=integration
$(TEST_KNATIVE_RUN) $(GOTESTFMT) || ((FAILED++)); \
exit $${FAILED}
#
