Normally the first failed assertion causes the test to stop. Provide a
flag to allow the test to continue, as a means to discover other
problems later in the test.
Note that the utility of this depends on the test, since some tests will
be broken if any step fails. But this can be useful when there are lots
of test adjustments to make.
Signed-off-by: Simon Glass <s...@chromium.org>
---
(no changes since v1)
arch/sandbox/cpu/start.c | 10 ++++++
arch/sandbox/include/asm/state.h | 1 +
include/test/test.h | 13 ++++++++
include/test/ut.h | 57 +++++++++++++++++++++-----------
test/test-main.c | 1 +
5 files changed, 63 insertions(+), 19 deletions(-)
diff --git a/arch/sandbox/cpu/start.c b/arch/sandbox/cpu/start.c
index d3bb8ff429e..ae004a69c98 100644
--- a/arch/sandbox/cpu/start.c
+++ b/arch/sandbox/cpu/start.c
@@ -468,6 +468,16 @@ static int sandbox_cmdline_cb_noflat(struct sandbox_state
*state,
}
SANDBOX_CMDLINE_OPT_SHORT(noflat, 'F', 0, "Don't run second set of DM tests");
+static int sandbox_cmdline_cb_soft_fail(struct sandbox_state *state,
+ const char *arg)
+{
+ state->soft_fail = true;
+
+ return 0;
+}
+SANDBOX_CMDLINE_OPT_SHORT(soft_fail, 'f', 0,
+ "continue test execution even after it fails");
+
void state_show(struct sandbox_state *state)
{
char **p;
diff --git a/arch/sandbox/include/asm/state.h b/arch/sandbox/include/asm/state.h
index b0dfc87f1f9..926761db180 100644
--- a/arch/sandbox/include/asm/state.h
+++ b/arch/sandbox/include/asm/state.h
@@ -103,6 +103,7 @@ struct sandbox_state {
bool upl; /* Enable Universal Payload (UPL) */
bool native; /* Adjust to reflect host arch */
bool no_flattree_tests; /* Don't run second set of DM tests */
+ bool soft_fail; /* Continue on failure */
/* Pointer to information for each SPI bus/cs */
struct sandbox_spi_info spi[CONFIG_SANDBOX_SPI_MAX_BUS]
diff --git a/include/test/test.h b/include/test/test.h
index cadf39a0c9d..51609e799b7 100644
--- a/include/test/test.h
+++ b/include/test/test.h
@@ -51,6 +51,7 @@ struct ut_stats {
* @runs_per_test: Number of times to run each test (typically 1)
* @force_run: true to run tests marked with the UTF_MANUAL flag
* @old_bloblist: stores the old gd->bloblist pointer
+ * @soft_fail: continue execution of the test even after it fails
* @expect_str: Temporary string used to hold expected string value
* @actual_str: Temporary string used to hold actual string value
*/
@@ -76,6 +77,7 @@ struct unit_test_state {
int runs_per_test;
bool force_run;
void *old_bloblist;
+ bool soft_fail;
char expect_str[512];
char actual_str[512];
};
@@ -299,4 +301,15 @@ static inline bool test_flattree_test_enabled(void)
#endif
}
+static inline bool test_soft_fail(void)
+{
+#ifdef CONFIG_SANDBOX
+ struct sandbox_state *state = state_get_current();
+
+ return state->soft_fail;
+#else
+ return false;
+#endif
+}
+
#endif /* __TEST_TEST_H */
diff --git a/include/test/ut.h b/include/test/ut.h
index be5502e03a1..6731b43dba9 100644
--- a/include/test/ut.h
+++ b/include/test/ut.h
@@ -147,7 +147,8 @@ int ut_check_console_dump(struct unit_test_state *uts, int
total_bytes);
\
if (!(cond)) { \
ut_fail(uts, __FILE__, __LINE__, __func__, #cond); \
- return CMD_RET_FAILURE; \
+ if (!uts->soft_fail) \
+ return CMD_RET_FAILURE; \
} \
__ret; \
})
@@ -159,7 +160,8 @@ int ut_check_console_dump(struct unit_test_state *uts, int
total_bytes);
if (!(cond)) { \
ut_failf(uts, __FILE__, __LINE__, __func__, #cond, \
fmt, ##args); \
- return CMD_RET_FAILURE; \
+ if (!uts->soft_fail) \
+ return CMD_RET_FAILURE; \
} \
__ret; \
})
@@ -174,7 +176,8 @@ int ut_check_console_dump(struct unit_test_state *uts, int
total_bytes);
#expr1 " == " #expr2, \
"Expected %#x (%d), got %#x (%d)", \
_val1, _val1, _val2, _val2); \
- return CMD_RET_FAILURE; \
+ if (!uts->soft_fail) \
+ return CMD_RET_FAILURE; \
} \
__ret; \
})
@@ -192,7 +195,8 @@ int ut_check_console_dump(struct unit_test_state *uts, int
total_bytes);
(unsigned long long)_val1, \
(unsigned long long)_val2, \
(unsigned long long)_val2); \
- return CMD_RET_FAILURE; \
+ if (!uts->soft_fail) \
+ return CMD_RET_FAILURE; \
} \
__ret; \
})
@@ -206,7 +210,8 @@ int ut_check_console_dump(struct unit_test_state *uts, int
total_bytes);
ut_failf(uts, __FILE__, __LINE__, __func__, \
#expr1 " = " #expr2, \
"Expected \"%s\", got \"%s\"", _val1, _val2); \
- return CMD_RET_FAILURE; \
+ if (!uts->soft_fail) \
+ return CMD_RET_FAILURE; \
} \
__ret; \
})
@@ -225,7 +230,8 @@ int ut_check_console_dump(struct unit_test_state *uts, int
total_bytes);
#expr1 " = " #expr2, \
"Expected \"%.*s\", got \"%.*s\"", \
_len, _val1, _len, _val2); \
- return CMD_RET_FAILURE; \
+ if (!uts->soft_fail) \
+ return CMD_RET_FAILURE; \
} \
__ret; \
})
@@ -245,7 +251,8 @@ int ut_check_console_dump(struct unit_test_state *uts, int
total_bytes);
#expr1 " = " #expr2, \
"Expected \"%s\", got \"%s\"", \
__buf1, __buf2); \
- return CMD_RET_FAILURE; \
+ if (!uts->soft_fail) \
+ return CMD_RET_FAILURE; \
} \
__ret; \
})
@@ -259,7 +266,8 @@ int ut_check_console_dump(struct unit_test_state *uts, int
total_bytes);
ut_failf(uts, __FILE__, __LINE__, __func__, \
#expr1 " = " #expr2, \
"Expected %p, got %p", _val1, _val2); \
- return CMD_RET_FAILURE; \
+ if (!uts->soft_fail) \
+ return CMD_RET_FAILURE; \
} \
__ret; \
})
@@ -274,7 +282,8 @@ int ut_check_console_dump(struct unit_test_state *uts, int
total_bytes);
ut_failf(uts, __FILE__, __LINE__, __func__, \
#expr1 " = " #expr2, \
"Expected %lx, got %lx", _val1, _val2); \
- return CMD_RET_FAILURE; \
+ if (!uts->soft_fail) \
+ return CMD_RET_FAILURE; \
} \
__ret; \
})
@@ -288,7 +297,8 @@ int ut_check_console_dump(struct unit_test_state *uts, int
total_bytes);
ut_failf(uts, __FILE__, __LINE__, __func__, \
#expr " != NULL", \
"Expected NULL, got %p", _val); \
- return CMD_RET_FAILURE; \
+ if (!uts->soft_fail) \
+ return CMD_RET_FAILURE; \
} \
__ret; \
})
@@ -302,7 +312,8 @@ int ut_check_console_dump(struct unit_test_state *uts, int
total_bytes);
ut_failf(uts, __FILE__, __LINE__, __func__, \
#expr " = NULL", \
"Expected non-null, got NULL"); \
- return CMD_RET_FAILURE; \
+ if (!uts->soft_fail) \
+ return CMD_RET_FAILURE; \
} \
__ret; \
})
@@ -317,7 +328,8 @@ int ut_check_console_dump(struct unit_test_state *uts, int
total_bytes);
#expr " = NULL", \
"Expected pointer, got error %ld", \
PTR_ERR(_val)); \
- return CMD_RET_FAILURE; \
+ if (!uts->soft_fail) \
+ return CMD_RET_FAILURE; \
} \
__ret; \
})
@@ -333,7 +345,8 @@ int ut_check_console_dump(struct unit_test_state *uts, int
total_bytes);
ut_failf(uts, __FILE__, __LINE__, __func__, \
"console", "\nExpected '%s',\n got '%s'", \
uts->expect_str, uts->actual_str); \
- return CMD_RET_FAILURE; \
+ if (!uts->soft_fail) \
+ return CMD_RET_FAILURE; \
} \
__ret; \
})
@@ -346,7 +359,8 @@ int ut_check_console_dump(struct unit_test_state *uts, int
total_bytes);
ut_failf(uts, __FILE__, __LINE__, __func__, \
"console", "\nExpected '%s',\n got '%s'", \
uts->expect_str, uts->actual_str); \
- return CMD_RET_FAILURE; \
+ if (!uts->soft_fail) \
+ return CMD_RET_FAILURE; \
} \
__ret; \
})
@@ -358,7 +372,8 @@ int ut_check_console_dump(struct unit_test_state *uts, int
total_bytes);
if (ut_check_skipline(uts)) { \
ut_failf(uts, __FILE__, __LINE__, __func__, \
"console", "\nExpected a line, got end"); \
- return CMD_RET_FAILURE; \
+ if (!uts->soft_fail) \
+ return CMD_RET_FAILURE; \
} \
__ret; \
})
@@ -371,7 +386,8 @@ int ut_check_console_dump(struct unit_test_state *uts, int
total_bytes);
ut_failf(uts, __FILE__, __LINE__, __func__, \
"console", "\nExpected '%s',\n got to '%s'", \
uts->expect_str, uts->actual_str); \
- return CMD_RET_FAILURE; \
+ if (!uts->soft_fail) \
+ return CMD_RET_FAILURE; \
} \
__ret; \
})
@@ -384,7 +400,8 @@ int ut_check_console_dump(struct unit_test_state *uts, int
total_bytes);
ut_failf(uts, __FILE__, __LINE__, __func__, \
"console", "\nExpected '%s',\n got to '%s'", \
uts->expect_str, uts->actual_str); \
- return CMD_RET_FAILURE; \
+ if (!uts->soft_fail) \
+ return CMD_RET_FAILURE; \
} \
__ret; \
})
@@ -397,7 +414,8 @@ int ut_check_console_dump(struct unit_test_state *uts, int
total_bytes);
ut_failf(uts, __FILE__, __LINE__, __func__, \
"console", "Expected no more output, got '%s'",\
uts->actual_str); \
- return CMD_RET_FAILURE; \
+ if (!uts->soft_fail) \
+ return CMD_RET_FAILURE; \
} \
__ret; \
})
@@ -411,7 +429,8 @@ int ut_check_console_dump(struct unit_test_state *uts, int
total_bytes);
"console", \
"Expected dump of length %x bytes, got '%s'", \
total_bytes, uts->actual_str); \
- return CMD_RET_FAILURE; \
+ if (!uts->soft_fail) \
+ return CMD_RET_FAILURE; \
} \
__ret; \
})
diff --git a/test/test-main.c b/test/test-main.c
index aa3eb5a3d8e..8515a77fe42 100644
--- a/test/test-main.c
+++ b/test/test-main.c
@@ -77,6 +77,7 @@ void ut_set_state(struct unit_test_state *uts)
void ut_init_state(struct unit_test_state *uts)
{
memset(uts, '\0', sizeof(*uts));
+ uts->soft_fail = test_soft_fail();
}
void ut_uninit_state(struct unit_test_state *uts)
--
2.43.0
