summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorScott Murray <scott.murray@konsulko.com>2020-09-09 18:13:35 -0400
committerScott Murray <scott.murray@konsulko.com>2020-09-10 10:24:45 -0400
commit6cffabd9cf2f0e5b1528c5e3e55558e3d8aab201 (patch)
tree17e16d328cd9c2d3d7227270dbf720ee61955bf3
parentf583233db298a709a813d8c0af97769ba30ae002 (diff)
Improve LAVA output
Changes: - Disable progress and verbose output when in LAVA output mode to clean up output. - Disable color output in LAVA output mode. - Removed STARTTC and ENDTC lines for passed/skipped tests to reduce output volume, they are still printed around the output of failed tests. - Tweaked error output in LAVA output mode so it will be usefully parsed out by LAVA and show up in results. - Fixed skipped test output in LAVA output mode, the logic needed tweaking to get a TESTCASE line output for them. Also, tweaked the LAVA result to 'skip' instead of 'pass' for skipped tests, as LAVA understands 'skip'. - By request, add 0.25 second delay after each test in LAVA output mode to slow down serial output. - Updated documentation to add some invocation examples. Bug-AGL: SPEC-3572 Signed-off-by: Scott Murray <scott.murray@konsulko.com> Change-Id: I2e2b213dba2a9751210512d698afb67f7543cbe1
-rw-r--r--README.md33
-rw-r--r--pyagl/conftest.py91
2 files changed, 110 insertions, 14 deletions
diff --git a/README.md b/README.md
index 6f690e5..be8c283 100644
--- a/README.md
+++ b/README.md
@@ -46,11 +46,9 @@ Until the package is uploaded onto PyPI, either:
* `pip install .`
or
-
* `pip install` the generated zip from the repository(when public)
or
-
* git clone \<repo\>
* mkdir packages && cd packages
* pip wheel ../\<repo\>
@@ -84,6 +82,21 @@ Note that the tests have been labelled with `pytest` markers to allow selecting
* regular - all regular verb tests with expected values
* hwrequired - verb tests requiring available physical hardware
+#### Examples
+Running just the tests for a single binding (audiomixer):
+```
+pytest -k "audiomixer" /usr/lib/python3.8/site-packages/pyagl/tests
+```
+Note that the per-binding markers cannot use dashes ('-') in their names, so generating the marker for a specific binding can be done with something like:
+```
+echo agl-service-can-low-level | cut -d- -f3- | tr - _
+```
+Running tests with LAVA compatible output:
+```
+pytest --lava "audiomixer" /usr/lib/python3.8/site-packages/pyagl/tests
+```
+Note that `--lava` and `-L` are equivalent, either will activate LAVA compatible output. As well, note that when using LAVA output, the `pytest` `-v` verbose option is ignored.
+
### Test Configuration
Running the tests remotely involves the export of the following environment variables:
* AGL_TGT_IP - required - point at an IP Address with AGL instance
@@ -101,3 +114,19 @@ Some specific tests are dependent on additional configuration via the following
* AGL_BT_TEST_ADDR - optional, for the Bluetooth tests pair/connect/disconnect with an actual Bluetooth device(phone)'s address . The address should have the DBus address style as "dev_00_01_02_03_04_05" instead of a regular colon-separated MAC address.
Should some of the optional variables be omitted, the fixtures and their dependent tests should fail gracefully with a "XFAIL" (expected fail) or "SKIP" result. There are tests that are dependent on other tests, therefore if the dependent tests fail, the whole test chain will be skipped.
+
+### Running Inside QEMU
+To run `QEMU` from inside the AGL build environment to be able to run tests, the `runqemu` command can be used. For example:
+```
+$ runqemu kvm publicvnc serial slirp audio
+```
+A note on some of the `runqemu` options:
+ - serial - enables a serial console
+ - slirp - enables user networking (no root login / `sudo` required)
+ - audio - enables audio
+
+The tests can then be invoked after logging in:
+```
+ AGL_AVAILABLE_INTERFACES=ethernet pytest -k "not hwrequired" /usr/lib/python3.8/site-packages/pyagl/tests
+```
+If running `QEMU` outside of the AGL build environment, note that you will likely want to pass the `-soundhw hda` option to enable audio support if running the audiomixer tests, and an appropriate `-netdev` option to enable some form of networking for the network binding tests.
diff --git a/pyagl/conftest.py b/pyagl/conftest.py
index ae28e79..f37de31 100644
--- a/pyagl/conftest.py
+++ b/pyagl/conftest.py
@@ -15,18 +15,53 @@
import pytest
+import argparse
+import time
+
+
+class LavaAction(argparse.Action):
+ def __init__(self, option_strings, dest, nargs=0, **kwargs):
+ if nargs != 0:
+ raise ValueError("nargs not allowed")
+ super(LavaAction, self).__init__(option_strings, dest, nargs=nargs, **kwargs)
+ def __call__(self, parser, namespace, values, option_string=None):
+ setattr(namespace, self.dest, True)
+ setattr(namespace, 'color', 'no')
def pytest_addoption(parser):
- parser.addoption('-L', '--lava', action='store_true', help='enable LAVA signals')
+ parser.addoption('-L', '--lava', action=LavaAction, help='enable LAVA signals')
+
+
+def pytest_configure(config):
+ # Force normal progress and verbose output off when doing LAVA output
+ terminal = config.pluginmanager.getplugin('terminal')
+ class QuietReporter(terminal.TerminalReporter):
+ def _determine_show_progress_info(self):
+ return False
+
+ @property
+ def verbosity(self):
+ return 0
+
+ @property
+ def showlongtestinfo(self):
+ return False
+
+ @property
+ def showfspath(self):
+ return False
+
+ if config.getoption('lava'):
+ terminal.TerminalReporter = QuietReporter
def lava_result_convert(pytest_outcome):
- """ Convert the pytestoutcome to the string expected by LAVA."""
+ """ Convert the pytest outcome to the string expected by LAVA."""
if pytest_outcome == 'passed':
return 'pass'
elif pytest_outcome == 'skipped':
- return 'pass'
+ return 'skip'
elif pytest_outcome == 'xfailed':
return 'pass'
else:
@@ -35,14 +70,46 @@ def lava_result_convert(pytest_outcome):
def pytest_report_teststatus(config, report):
""" Insert strings that LAVA expects to capture test results."""
-# Get pytest test name and remove the 'test_' prefix
- if config.getoption('--lava'):
- test_name = report.location[2][5:]
+ done = False
+ if config.getoption('lava'):
+ # Convert pytest test file and name into a LAVA test name
+ test_file = report.location[0].split('/')[-1]
+ test_file = test_file.replace('test_', '', 1)
+ if test_file.endswith('.py'):
+ test_file = test_file[:-3]
+ test_name = test_file + '_' + report.location[2][5:]
+ test_result = lava_result_convert(report.outcome)
+
+ # Generate expected LAVA testcase output
if report.when == 'setup':
- print('\n')
- print(f'<LAVA_SIGNAL_STARTTC {test_name}>')
+ if report.outcome == 'skipped':
+ done = True
elif report.when == 'call':
- test_result = lava_result_convert(report.outcome)
- print('\n')
- print(f'<LAVA_SIGNAL_ENDTC {test_name}>')
- print(f'<LAVA_SIGNAL_TESTCASE TEST_CASE_ID={test_name} RESULT={test_result}>')
+ done = True
+ if report.outcome == 'failed':
+ print(f'<LAVA_SIGNAL_STARTTC {test_name}>')
+ print('ERROR:\n')
+ print(report.longrepr)
+ print(f'<LAVA_SIGNAL_ENDTC {test_name}>')
+ if done:
+ print(f'<LAVA_SIGNAL_TESTCASE TEST_CASE_ID={test_name} RESULT={test_result}>\n')
+ # Delay to slow down serial output for LAVA
+ time.sleep(0.25)
+
+ # Quiet short result output
+ category, short, verbose = '', '', ''
+ if hasattr(report, 'wasxfail'):
+ if report.skipped:
+ category = 'xfailed'
+ elif report.passed:
+ category = 'xpassed'
+ return (category, short, verbose)
+ elif report.when in ('setup', 'teardown'):
+ if report.failed:
+ category = 'error'
+ elif report.skipped:
+ category = 'skipped'
+ return (category, short, verbose)
+ category = report.outcome
+ return (category, short, verbose)
+