Add submodule dependency filesHEADmaster

Change-Id: Iaf8d18082d3991dec7c0ebbea540f092188eb4ec

2 files changed, 330 insertions, 0 deletions

diff --git a/roms/skiboot/external/fwts/generate-fwts-olog b/roms/skiboot/external/fwts/generate-fwts-olog
new file mode 100755
index 000000000..c89fbf6df
--- /dev/null
+++ b/roms/skiboot/external/fwts/generate-fwts-olog
@@ -0,0 +1,227 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+#
+# Copyright 2016 Jeremy Kerr <jk@ozlabs.org>
+
+import os.path
+import re
+import sys
+import string
+import json
+import argparse
+import subprocess
+from pyparsing import Regex, Literal, Word, Combine, OneOrMore, QuotedString, \
+         lineno
+
+json_params = {
+    'indent': 1,
+    'sort_keys': True,
+    'separators': (',', ': '),
+}
+
+def create_parser():
+    # Match a C-style comment starting with two *s
+    comment = Regex(r'/\*\*(?P<content>.*?)\*/', re.DOTALL)
+
+    # Match an @fwts-<tag> annotation (within the comment), plus the proceeding
+    # text
+    annotation = Regex(r'@fwts-(?P<tag>\w+)\W+(?P<text>.*?)(?=@fwts-|\Z)',
+                re.DOTALL)
+
+    # Match the following prlog() call
+    log_call = (((Literal("prerror") + Literal('(').suppress()) |
+                 (Literal("prlog")   + Literal('(').suppress() +
+                  Word(string.ascii_letters + string.digits + '_') +
+                  Literal(',').suppress())) +
+                Combine(OneOrMore(QuotedString('"')), adjacent=False) +
+                (Literal(')') | Literal(',')).suppress()
+               )
+
+    pattern = comment + log_call
+    pattern.setWhitespaceChars(string.whitespace + '\n')
+
+    def comment_action(tok):
+        patterns = {}
+        for result in annotation.scanString(tok['content']):
+            patterns.update(result[0][0])
+        return patterns
+
+    def annotation_action(tok):
+        return {
+            tok['tag']: cleanup_content(tok['text'])
+        }
+
+    comment.setParseAction(comment_action)
+    annotation.setParseAction(annotation_action)
+    pattern.parseWithTabs()
+
+    return pattern
+
+def find_sources(dirname):
+    sources = []
+
+    def is_source(fname):
+        return fname.endswith('.c')
+
+    for directory, dirnames, filenames in os.walk(dirname):
+        sources.extend([ os.path.join(directory, fname)
+                         for fname in filenames if is_source(fname) ])
+    return sources
+
+def cleanup_content(content):
+    comment_prefix_re = re.compile(r'^\s*\*\s*', re.MULTILINE)
+    whitespace_re = re.compile(r'\s+')
+
+    content = comment_prefix_re.sub(' ', content)
+    content = whitespace_re.sub(' ', content)
+    return content.strip()
+
+def warn(loc, message):
+    print >>sys.stderr, 'WARNING:%s:%d: %s' % (loc[0], loc[1], message)
+
+def log_level_to_fwts(level):
+    level_map = {
+        'PR_EMERG':     'LOG_LEVEL_CRITICAL',
+        'PR_ALERT':     'LOG_LEVEL_CRITICAL',
+        'PR_CRIT':      'LOG_LEVEL_CRITICAL',
+        'PR_ERR':       'LOG_LEVEL_CRITICAL',
+        'PR_WARNING':   'LOG_LEVEL_HIGH',
+        'PR_NOTICE':    'LOG_LEVEL_MEDIUM',
+        'PR_PRINTF':    'LOG_LEVEL_MEDIUM',
+    }
+    return level_map.get(level, 'LOG_LEVEL_LOW')
+
+def message_to_pattern(loc, msg):
+    """ Convert a C printf()-style template to a pattern suitable for fwts """
+
+    # Somewhat-simplified match for a %-template
+    template_re = re.compile(
+            '%(?P<flag>[-#0 +]*)'
+            '(?P<width>(?:[0-9]*|\*))?'
+            '(?P<precision>\.*(?:[1-9][0-9]*|\*))?'
+            '(?:hh|h|ll|l|L|j|z|t)?'
+            '(?P<conversion>[a-zA-Z%])')
+    global is_regex
+    is_regex = False
+
+    def expand_template(match):
+        global is_regex
+        c = match.group('conversion').lower()
+        if c == '%':
+            return '%'
+        is_regex = True
+        if c in ['d', 'i', 'u']:
+            return '[0-9]+'
+        elif c == 'o':
+            return '[0-7]+'
+        elif c == 'x':
+            return '[0-9a-f]+'
+        elif c == 'p':
+            return '(0x[0-9a-f]+|nil)'
+        elif c == 's':
+            return '.*'
+        else:
+            warn(loc, "Unknown template conversion '%s'" % match.group(0))
+            return '.*'
+
+    escape_re = re.compile(r'\\(?P<char>.)', re.DOTALL)
+    def expand_escape(match):
+        global is_regex
+        c = match.group('char')
+        if c == 'n':
+            return '\n'
+        elif c in ['\\', '"']:
+            return c
+        else:
+            warn(loc, "Unhandled escape sequence '%s'" % match.group(0))
+            is_regex = True
+            return '.'
+
+    pattern = template_re.sub(expand_template, msg)
+    pattern = escape_re.sub(expand_escape, pattern)
+    pattern = pattern.strip()
+
+    compare_mode = "string"
+    if is_regex:
+        compare_mode = "regex"
+
+    return (compare_mode, pattern)
+
+def parse_patterns(parser, fname, tag):
+    patterns = []
+    data = open(fname).read()
+    i = 1
+    for result in parser.scanString(data):
+        (token, loc, _) = result
+        if token[1] == 'prlog':
+            (annotations, logfn, level, msg) = token
+        else:
+            (annotations, logfn, msg) = token
+            level = 'PR_ERR'
+
+        loc = (fname, lineno(loc, data))
+
+        if logfn != 'prlog' and logfn != 'prerror':
+            warn(loc, "unknown log output function '%s'" % logfn)
+
+        compare_mode, pattern_str = message_to_pattern(loc, msg)
+
+        pattern = {
+            'log_level': log_level_to_fwts(level),
+            'compare_mode': compare_mode,
+            'pattern': pattern_str,
+            'last_tag': tag,
+        }
+
+        pattern.update(annotations)
+
+        if not 'label' in pattern:
+            warn(loc, "missing label")
+            pattern['label'] = '%s:%d' % (fname, i)
+            i += 1
+
+        if not 'advice' in pattern:
+            warn(loc, "missing advice")
+
+        allowed_data = ['compare_mode', 'log_level',
+                        'pattern', 'advice', 'label', 'last_tag']
+        extras = set(pattern.keys()) - set(allowed_data)
+        if extras:
+            warn(loc, "unknown pattern annotation: %s" %
+                    ','.join([ "'%s'" % e for e in extras]))
+            for e in extras:
+                del pattern[e]
+
+        patterns.append(pattern)
+
+    return patterns
+
+if __name__ == '__main__':
+    argparser = argparse.ArgumentParser(
+            description='Generate FWTS olog definitions from the skiboot '
+                        'source tree')
+    argparser.add_argument('directories', metavar='DIR', nargs='*',
+            help='path to source files (default .)', default=['.'])
+    argparser.add_argument('--output', '-o', metavar='FILE',
+            type=argparse.FileType('w'), default=sys.stdout,
+            help='output to FILE (default to stdout)', nargs='?')
+    args = argparser.parse_args()
+
+    sources = []
+    for directory in args.directories:
+        try:
+            git_tag = subprocess.check_output(["git","-C", directory, "describe", "--abbrev=0" ], text=True)
+        except:
+            git_tag = "???"
+        git_tag = git_tag.replace("\n", "")
+        sources.extend([ (x, git_tag) for x in find_sources(directory)])
+
+    parser = create_parser()
+    patterns = []
+    for source, tag in sources:
+        patterns.extend(parse_patterns(parser, source, tag))
+
+    data = {'olog_error_warning_patterns': patterns}
+
+    args.output.write(json.dumps(data, **json_params) + '\n')
+
diff --git a/roms/skiboot/external/fwts/merge-fwts-olog b/roms/skiboot/external/fwts/merge-fwts-olog
new file mode 100755
index 000000000..de70d6cfe
--- /dev/null
+++ b/roms/skiboot/external/fwts/merge-fwts-olog
@@ -0,0 +1,103 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0 OR GPL-2.0-or-later
+#
+# Copyright 2016 IBM Corp.
+
+import json
+import sys
+
+json_params = {
+    'indent': 1,
+    'sort_keys': True,
+    'separators': (',', ': '),
+}
+
+def get_input():
+    while True:
+        resp = input("Update pattern to match both? (y/n): ")
+        if resp in [ "y", "Y" ]:
+            break
+        elif resp in [ "n", "N" ]:
+            print("New entry will be added.")
+            return False
+        else:
+            print("???")
+            continue
+
+    return input("New pattern: ")
+
+def main():
+    if len(sys.argv) != 4:
+        print("USAGE: merge.py old_olog new_olog output")
+        sys.exit(-1)
+
+    old_olog = sys.argv[1]
+    cur_olog = sys.argv[2]
+    out_olog = sys.argv[3]
+
+    try:
+        old_file = open(old_olog, "r")
+        old_json = json.load(old_file)
+        old_file.close()
+    except Exception as e:
+        print("Failed to parse old olog: %s" % e)
+        sys.exit(-1)
+
+    try:
+        cur_file = open(cur_olog, "r")
+        cur_json = json.load(cur_file)
+        cur_file.close()
+    except Exception as e:
+        print("Failed to parse new olog: %s" % e)
+        sys.exit(-1)
+
+    try:
+        out_file = open(out_olog, "w")
+    except Exception as e:
+        print("Failed to open output file: %s" % e)
+        sys.exit(-1)
+
+    cur_patterns = cur_json["olog_error_warning_patterns"]
+    old_patterns = old_json["olog_error_warning_patterns"]
+
+    # Match current patterns to old definitions, detect when pattern is
+    # different.
+
+    for cp in cur_patterns:
+        for op in old_patterns:
+            if cp["label"] != op["label"]:
+                continue
+
+            if cp["pattern"] != op["pattern"]:
+                print("Pattern update detected.")
+                print("Label: %s" % cp["label"])
+                print("")
+                print("Cur Pattern: %s" % cp["pattern"])
+                print("New Pattern: %s" % op["pattern"])
+                print("")
+
+                user_pattern = get_input()
+
+                if user_pattern == False:
+                    continue
+
+                cp["pattern"] = user_pattern
+
+            op["found"] = True
+            break
+
+    # Take any old patterns that are no longer current and move them over
+
+    for op in old_patterns:
+        if "found" in op:
+            continue
+
+        cur_patterns.append(op)
+
+    json.dump(cur_json, out_file, **json_params)
+    out_file.close()
+
+    print("OK")
+
+if __name__ == "__main__":
+    main()