summaryrefslogtreecommitdiffstats
path: root/tools/testing
diff options
context:
space:
mode:
authorLucas Bates <lucasb@mojatatu.com>2019-02-28 17:38:40 -0500
committerDavid S. Miller <davem@davemloft.net>2019-03-01 23:05:06 -0800
commit255c1c7279abf991113e7c9c29f91e9f1a5776a9 (patch)
tree4f6240e2b26aebed6b08666983290e33b345761c /tools/testing
parent745732c8ecf89ad60f7e393094d4993284bd1869 (diff)
downloadlinux-255c1c7279abf991113e7c9c29f91e9f1a5776a9.tar.bz2
tc-testing: Allow test cases to be skipped
By adding a check for an optional key/value pair to the test case data, individual test cases may be skipped to prevent tdc from aborting a test run due to setup or teardown failure. If a test case is skipped, it will still appear in the results output to allow for a consistent number of executed tests in each run. However, the test will be marked as skipped. This support for skipping extends to any plugins that may generate additional results for each executed test. Signed-off-by: Lucas Bates <lucasb@mojatatu.com> Signed-off-by: David S. Miller <davem@davemloft.net>
Diffstat (limited to 'tools/testing')
-rw-r--r--tools/testing/selftests/tc-testing/TdcPlugin.py4
-rw-r--r--tools/testing/selftests/tc-testing/creating-testcases/AddingTestCases.txt5
-rw-r--r--tools/testing/selftests/tc-testing/plugin-lib/valgrindPlugin.py16
-rwxr-xr-xtools/testing/selftests/tc-testing/tdc.py15
4 files changed, 27 insertions, 13 deletions
diff --git a/tools/testing/selftests/tc-testing/TdcPlugin.py b/tools/testing/selftests/tc-testing/TdcPlugin.py
index 1d9e279331eb..b980a565fa89 100644
--- a/tools/testing/selftests/tc-testing/TdcPlugin.py
+++ b/tools/testing/selftests/tc-testing/TdcPlugin.py
@@ -18,13 +18,13 @@ class TdcPlugin:
if self.args.verbose > 1:
print(' -- {}.post_suite'.format(self.sub_class))
- def pre_case(self, test_ordinal, testid, test_name):
+ def pre_case(self, testid, test_name, test_skip):
'''run commands before test_runner does one test'''
if self.args.verbose > 1:
print(' -- {}.pre_case'.format(self.sub_class))
self.args.testid = testid
self.args.test_name = test_name
- self.args.test_ordinal = test_ordinal
+ self.args.test_skip = test_skip
def post_case(self):
'''run commands after test_runner does one test'''
diff --git a/tools/testing/selftests/tc-testing/creating-testcases/AddingTestCases.txt b/tools/testing/selftests/tc-testing/creating-testcases/AddingTestCases.txt
index 17b267dedbd9..a28571aff0e1 100644
--- a/tools/testing/selftests/tc-testing/creating-testcases/AddingTestCases.txt
+++ b/tools/testing/selftests/tc-testing/creating-testcases/AddingTestCases.txt
@@ -33,6 +33,11 @@ Each test case has required data:
id: A unique alphanumeric value to identify a particular test case
name: Descriptive name that explains the command under test
+skip: A completely optional key, if the corresponding value is "yes"
+ then tdc will not execute the test case in question. However,
+ this test case will still appear in the results output but
+ marked as skipped. This key can be placed anywhere inside the
+ test case at the top level.
category: A list of single-word descriptions covering what the command
under test is testing. Example: filter, actions, u32, gact, etc.
setup: The list of commands required to ensure the command under test
diff --git a/tools/testing/selftests/tc-testing/plugin-lib/valgrindPlugin.py b/tools/testing/selftests/tc-testing/plugin-lib/valgrindPlugin.py
index e00c798de0bb..4bb866575ea1 100644
--- a/tools/testing/selftests/tc-testing/plugin-lib/valgrindPlugin.py
+++ b/tools/testing/selftests/tc-testing/plugin-lib/valgrindPlugin.py
@@ -102,6 +102,14 @@ class SubPlugin(TdcPlugin):
if not self.args.valgrind:
return
+ res = TestResult('{}-mem'.format(self.args.testid),
+ '{} memory leak check'.format(self.args.test_name))
+ if self.args.test_skip:
+ res.set_result(ResultState.skip)
+ res.set_errormsg('Test case designated as skipped.')
+ self._add_results(res)
+ return
+
self.definitely_lost_re = re.compile(
r'definitely lost:\s+([,0-9]+)\s+bytes in\s+([,0-9]+)\sblocks', re.MULTILINE | re.DOTALL)
self.indirectly_lost_re = re.compile(
@@ -134,8 +142,6 @@ class SubPlugin(TdcPlugin):
nle_num = int(nle_mo.group(1))
mem_results = ''
- res = TestResult('{}-mem'.format(self.args.testid),
- '{} memory leak check'.format(self.args.test_name))
if (def_num > 0) or (ind_num > 0) or (pos_num > 0) or (nle_num > 0):
mem_results += 'not '
res.set_result(ResultState.fail)
@@ -146,12 +152,6 @@ class SubPlugin(TdcPlugin):
self._add_results(res)
- mem_results += 'ok {} - {}-mem # {}\n'.format(
- self.args.test_ordinal, self.args.testid, 'memory leak check')
- self._add_to_tap(mem_results)
- if mem_results.startswith('not '):
- print('{}'.format(content))
- self._add_to_tap(content)
def _add_results(self, res):
self._tsr.add_resultdata(res)
diff --git a/tools/testing/selftests/tc-testing/tdc.py b/tools/testing/selftests/tc-testing/tdc.py
index e6e4ce80a726..5cee15659e5f 100755
--- a/tools/testing/selftests/tc-testing/tdc.py
+++ b/tools/testing/selftests/tc-testing/tdc.py
@@ -61,10 +61,10 @@ class PluginMgr:
for pgn_inst in reversed(self.plugin_instances):
pgn_inst.post_suite(index)
- def call_pre_case(self, test_ordinal, testid, test_name):
+ def call_pre_case(self, testid, test_name, *, test_skip=False):
for pgn_inst in self.plugin_instances:
try:
- pgn_inst.pre_case(test_ordinal, testid, test_name)
+ pgn_inst.pre_case(testid, test_name, test_skip)
except Exception as ee:
print('exception {} in call to pre_case for {} plugin'.
format(ee, pgn_inst.__class__))
@@ -192,10 +192,19 @@ def run_one_test(pm, args, index, tidx):
print("\t====================\n=====> ", end="")
print("Test " + tidx["id"] + ": " + tidx["name"])
+ if 'skip' in tidx:
+ if tidx['skip'] == 'yes':
+ res = TestResult(tidx['id'], tidx['name'])
+ res.set_result(ResultState.skip)
+ res.set_errormsg('Test case designated as skipped.')
+ pm.call_pre_case(tidx['id'], tidx['name'], test_skip=True)
+ pm.call_post_execute()
+ return res
+
# populate NAMES with TESTID for this test
NAMES['TESTID'] = tidx['id']
- pm.call_pre_case(index, tidx['id'], tidx['name'])
+ pm.call_pre_case(tidx['id'], tidx['name'])
prepare_env(args, pm, 'setup', "-----> prepare stage", tidx["setup"])
if (args.verbose > 0):