diff options
| -rw-r--r-- | README.md | 33 | ||||
| -rw-r--r-- | pyagl/conftest.py | 91 |
2 files changed, 110 insertions, 14 deletions
@@ -46,11 +46,9 @@ Until the package is uploaded onto PyPI, either: * `pip install .` or - * `pip install` the generated zip from the repository(when public) or - * git clone \<repo\> * mkdir packages && cd packages * pip wheel ../\<repo\> @@ -84,6 +82,21 @@ Note that the tests have been labelled with `pytest` markers to allow selecting * regular - all regular verb tests with expected values * hwrequired - verb tests requiring available physical hardware +#### Examples +Running just the tests for a single binding (audiomixer): +``` +pytest -k "audiomixer" /usr/lib/python3.8/site-packages/pyagl/tests +``` +Note that the per-binding markers cannot use dashes ('-') in their names, so generating the marker for a specific binding can be done with something like: +``` +echo agl-service-can-low-level | cut -d- -f3- | tr - _ +``` +Running tests with LAVA compatible output: +``` +pytest --lava "audiomixer" /usr/lib/python3.8/site-packages/pyagl/tests +``` +Note that `--lava` and `-L` are equivalent, either will activate LAVA compatible output. As well, note that when using LAVA output, the `pytest` `-v` verbose option is ignored. + ### Test Configuration Running the tests remotely involves the export of the following environment variables: * AGL_TGT_IP - required - point at an IP Address with AGL instance @@ -101,3 +114,19 @@ Some specific tests are dependent on additional configuration via the following * AGL_BT_TEST_ADDR - optional, for the Bluetooth tests pair/connect/disconnect with an actual Bluetooth device(phone)'s address . The address should have the DBus address style as "dev_00_01_02_03_04_05" instead of a regular colon-separated MAC address. Should some of the optional variables be omitted, the fixtures and their dependent tests should fail gracefully with a "XFAIL" (expected fail) or "SKIP" result. There are tests that are dependent on other tests, therefore if the dependent tests fail, the whole test chain will be skipped. + +### Running Inside QEMU +To run `QEMU` from inside the AGL build environment to be able to run tests, the `runqemu` command can be used. For example: +``` +$ runqemu kvm publicvnc serial slirp audio +``` +A note on some of the `runqemu` options: + - serial - enables a serial console + - slirp - enables user networking (no root login / `sudo` required) + - audio - enables audio + +The tests can then be invoked after logging in: +``` + AGL_AVAILABLE_INTERFACES=ethernet pytest -k "not hwrequired" /usr/lib/python3.8/site-packages/pyagl/tests +``` +If running `QEMU` outside of the AGL build environment, note that you will likely want to pass the `-soundhw hda` option to enable audio support if running the audiomixer tests, and an appropriate `-netdev` option to enable some form of networking for the network binding tests. diff --git a/pyagl/conftest.py b/pyagl/conftest.py index ae28e79..f37de31 100644 --- a/pyagl/conftest.py +++ b/pyagl/conftest.py @@ -15,18 +15,53 @@ import pytest +import argparse +import time + + +class LavaAction(argparse.Action): + def __init__(self, option_strings, dest, nargs=0, **kwargs): + if nargs != 0: + raise ValueError("nargs not allowed") + super(LavaAction, self).__init__(option_strings, dest, nargs=nargs, **kwargs) + def __call__(self, parser, namespace, values, option_string=None): + setattr(namespace, self.dest, True) + setattr(namespace, 'color', 'no') def pytest_addoption(parser): - parser.addoption('-L', '--lava', action='store_true', help='enable LAVA signals') + parser.addoption('-L', '--lava', action=LavaAction, help='enable LAVA signals') + + +def pytest_configure(config): + # Force normal progress and verbose output off when doing LAVA output + terminal = config.pluginmanager.getplugin('terminal') + class QuietReporter(terminal.TerminalReporter): + def _determine_show_progress_info(self): + return False + + @property + def verbosity(self): + return 0 + + @property + def showlongtestinfo(self): + return False + + @property + def showfspath(self): + return False + + if config.getoption('lava'): + terminal.TerminalReporter = QuietReporter def lava_result_convert(pytest_outcome): - """ Convert the pytestoutcome to the string expected by LAVA.""" + """ Convert the pytest outcome to the string expected by LAVA.""" if pytest_outcome == 'passed': return 'pass' elif pytest_outcome == 'skipped': - return 'pass' + return 'skip' elif pytest_outcome == 'xfailed': return 'pass' else: @@ -35,14 +70,46 @@ def lava_result_convert(pytest_outcome): def pytest_report_teststatus(config, report): """ Insert strings that LAVA expects to capture test results.""" -# Get pytest test name and remove the 'test_' prefix - if config.getoption('--lava'): - test_name = report.location[2][5:] + done = False + if config.getoption('lava'): + # Convert pytest test file and name into a LAVA test name + test_file = report.location[0].split('/')[-1] + test_file = test_file.replace('test_', '', 1) + if test_file.endswith('.py'): + test_file = test_file[:-3] + test_name = test_file + '_' + report.location[2][5:] + test_result = lava_result_convert(report.outcome) + + # Generate expected LAVA testcase output if report.when == 'setup': - print('\n') - print(f'<LAVA_SIGNAL_STARTTC {test_name}>') + if report.outcome == 'skipped': + done = True elif report.when == 'call': - test_result = lava_result_convert(report.outcome) - print('\n') - print(f'<LAVA_SIGNAL_ENDTC {test_name}>') - print(f'<LAVA_SIGNAL_TESTCASE TEST_CASE_ID={test_name} RESULT={test_result}>') + done = True + if report.outcome == 'failed': + print(f'<LAVA_SIGNAL_STARTTC {test_name}>') + print('ERROR:\n') + print(report.longrepr) + print(f'<LAVA_SIGNAL_ENDTC {test_name}>') + if done: + print(f'<LAVA_SIGNAL_TESTCASE TEST_CASE_ID={test_name} RESULT={test_result}>\n') + # Delay to slow down serial output for LAVA + time.sleep(0.25) + + # Quiet short result output + category, short, verbose = '', '', '' + if hasattr(report, 'wasxfail'): + if report.skipped: + category = 'xfailed' + elif report.passed: + category = 'xpassed' + return (category, short, verbose) + elif report.when in ('setup', 'teardown'): + if report.failed: + category = 'error' + elif report.skipped: + category = 'skipped' + return (category, short, verbose) + category = report.outcome + return (category, short, verbose) + |