Improve LAVA output

Changes:
- Disable progress and verbose output when in LAVA output mode to
  clean up output.
- Disable color output in LAVA output mode.
- Removed STARTTC and ENDTC lines for passed/skipped tests to
  reduce output volume, they are still printed around the output
  of failed tests.
- Tweaked error output in LAVA output mode so it will be usefully
  parsed out by LAVA and show up in results.
- Fixed skipped test output in LAVA output mode, the logic needed
  tweaking to get a TESTCASE line output for them.  Also, tweaked
  the LAVA result to 'skip' instead of 'pass' for skipped tests,
  as LAVA understands 'skip'.
- By request, add 0.25 second delay after each test in LAVA output
  mode to slow down serial output.
- Updated documentation to add some invocation examples.

Bug-AGL: SPEC-3572

Signed-off-by: Scott Murray <scott.murray@konsulko.com>
Change-Id: I2e2b213dba2a9751210512d698afb67f7543cbe1

1 files changed, 79 insertions, 12 deletions

diff --git a/pyagl/conftest.py b/pyagl/conftest.py
index ae28e79..f37de31 100644
--- a/pyagl/conftest.py
+++ b/pyagl/conftest.py
@@ -15,18 +15,53 @@
 
 
 import pytest
+import argparse
+import time
+
+
+class LavaAction(argparse.Action):
+    def __init__(self, option_strings, dest, nargs=0, **kwargs):
+        if nargs !=  0:
+            raise ValueError("nargs not allowed")
+        super(LavaAction, self).__init__(option_strings, dest, nargs=nargs, **kwargs)
+    def __call__(self, parser, namespace, values, option_string=None):
+        setattr(namespace, self.dest, True)
+        setattr(namespace, 'color', 'no')
 
 
 def pytest_addoption(parser):
-    parser.addoption('-L', '--lava', action='store_true', help='enable LAVA signals')
+    parser.addoption('-L', '--lava', action=LavaAction, help='enable LAVA signals')
+
+
+def pytest_configure(config):
+    # Force normal progress and verbose output off when doing LAVA output
+    terminal = config.pluginmanager.getplugin('terminal')
+    class QuietReporter(terminal.TerminalReporter):
+        def _determine_show_progress_info(self):
+            return False
+
+        @property
+        def verbosity(self):
+            return 0
+
+        @property
+        def showlongtestinfo(self):
+            return False
+
+        @property
+        def showfspath(self):
+            return False
+
+    if config.getoption('lava'):
+        terminal.TerminalReporter = QuietReporter
 
 
 def lava_result_convert(pytest_outcome):
-    """ Convert the pytestoutcome to the string expected by LAVA."""
+    """ Convert the pytest outcome to the string expected by LAVA."""
     if pytest_outcome == 'passed':
         return 'pass'
     elif pytest_outcome == 'skipped':
-        return 'pass'
+        return 'skip'
     elif pytest_outcome == 'xfailed':
         return 'pass'
     else:
@@ -35,14 +70,46 @@ def lava_result_convert(pytest_outcome):
 
 def pytest_report_teststatus(config, report):
     """ Insert strings that LAVA expects to capture test results."""
-# Get pytest test name and remove the 'test_' prefix
-    if config.getoption('--lava'):
-        test_name = report.location[2][5:]
+    done = False
+    if config.getoption('lava'):
+        # Convert pytest test file and name into a LAVA test name
+        test_file = report.location[0].split('/')[-1]
+        test_file = test_file.replace('test_', '', 1)
+        if test_file.endswith('.py'):
+            test_file = test_file[:-3]
+        test_name = test_file + '_' + report.location[2][5:]
+        test_result = lava_result_convert(report.outcome)
+
+        # Generate expected LAVA testcase output
         if report.when == 'setup':
-            print('\n')
-            print(f'<LAVA_SIGNAL_STARTTC {test_name}>')
+            if report.outcome == 'skipped':
+                done = True
         elif report.when == 'call':
-            test_result = lava_result_convert(report.outcome)
-            print('\n')
-            print(f'<LAVA_SIGNAL_ENDTC {test_name}>')
-            print(f'<LAVA_SIGNAL_TESTCASE TEST_CASE_ID={test_name} RESULT={test_result}>')
+            done = True
+            if report.outcome == 'failed':
+                print(f'<LAVA_SIGNAL_STARTTC {test_name}>')
+                print('ERROR:\n')
+                print(report.longrepr)
+                print(f'<LAVA_SIGNAL_ENDTC {test_name}>')
+        if done:
+            print(f'<LAVA_SIGNAL_TESTCASE TEST_CASE_ID={test_name} RESULT={test_result}>\n')
+            # Delay to slow down serial output for LAVA
+            time.sleep(0.25)
+
+        # Quiet short result output
+        category, short, verbose = '', '', ''
+        if hasattr(report, 'wasxfail'):
+            if report.skipped:
+                category = 'xfailed'
+            elif report.passed:
+                category = 'xpassed'
+            return (category, short, verbose)
+        elif report.when in ('setup', 'teardown'):
+            if report.failed:
+                category = 'error'
+            elif report.skipped:
+                category = 'skipped'
+            return (category, short, verbose)
+        category = report.outcome
+        return (category, short, verbose)
+