Callback procedure update

New callback procedure. Now the arguments are:
- '--callback-from' LAVA lab
- '--callback-to' kCI instance

This new procedure helps differentiating the lab and the kCI target.

Callback config files updated accordingly.

Documentation on callback updated.

Change-Id: I41b9dc86e33783f1d6a8164fc783557c81678c7f
Signed-off-by: Loys Ollivier <lollivier@baylibre.com>

9 files changed, 82 insertions, 42 deletions

diff --git a/templates/base/agl-base.jinja2 b/templates/base/agl-base.jinja2
index 069eeb2..ad3b955 100644
--- a/templates/base/agl-base.jinja2
+++ b/templates/base/agl-base.jinja2
@@ -3,7 +3,7 @@
 {%- block metadata %}
 {% include 'base/agl-metadata.jinja2' %}
 {% endblock %}
-{% if callback_name %}
+{% if do_callback %}
 {% include 'base/agl-callback.jinja2' %}
 {% endif %}
 {%- block main %}
diff --git a/templates/base/agl-callback.jinja2 b/templates/base/agl-callback.jinja2
index 6798b17..b0e083d 100644
--- a/templates/base/agl-callback.jinja2
+++ b/templates/base/agl-callback.jinja2
@@ -3,7 +3,7 @@ notify:
   criteria:
     status: finished
   callback:
-    url: {{ backend_fqdn }}/callback/{{ callback_name }}?lab_name={{ lab_name }}&status={STATUS}&status_string={STATUS_STRING}
+    url: {{ backend_fqdn }}/callback/lava/test?lab_name={{ lab_name }}&status={STATUS}&status_string={STATUS_STRING}
     method: POST
     dataset: all
     token: {{ lab_token }}
diff --git a/templates/callback/callback_readme.md b/templates/callback/callback_readme.md
new file mode 100644
index 0000000..73c4c82
--- /dev/null
+++ b/templates/callback/callback_readme.md
@@ -0,0 +1,37 @@
+## Using --callback-* arguments ##
+- - - -
+
+For each LAVA lab target, a configuration file must be created in this repo.
+The file name must correspond to the LAVA lab name as registered in KernelCI.\
+Using the argument "--callback-from LAB_NAME", the script will search for a file called "LAB_NAME.cfg" in this folder.\
+Using the argument "--callback-to KCI_NAME", the script will search in the previous cfg file for a section called KCI_NAME. This section must contain three fields described below.
+
+### Requirements: ###
+
+* Filetype: .cfg
+* Filename: <LAB_NAME>.cfg
+    * [default] section
+        * section = "The section containing the callback data to use by default"
+    * [CUSTOM] section
+        * backend_fqdn = "The FQDN of the kernelCI backend to callback"
+        * lab_name = "The lab name as registered for the kernelCI backend"
+        * lab_token = "The kernelCI backend lab token. Usually this is the 'LAVA description' string as shorthand of the token. Alternatively it can be the token itself."
+
+Example file: _lab-mylab.cfg_
+```
+[default]
+section = centralized-kci
+
+[centralized-kci]
+backend_fqdn = http(s)://api.centralized-kci.org
+lab_name = lab-my-lab
+lab_token = lab-my-lab-callback-centralized-kci
+
+[my-dev-kci]
+backend_fqdn = http(s)://api.my-dev-kci.org
+lab_name = lab-my-lab
+lab_token = f1d130b4-8198-4d34-9841-bad7d4ea64d9
+```
+
+_Warning:_\
+If the lab_token is a LAVA description string, make sure that the actual token is registered in the LAVA interface and matches this string.
diff --git a/templates/callback/callback_readme.txt b/templates/callback/callback_readme.txt
deleted file mode 100644
index 7bfed0d..0000000
--- a/templates/callback/callback_readme.txt
+++ /dev/null
@@ -1,19 +0,0 @@
-## The callbacks info must be in this repo ##
-- - - -
-
-### Requirements: ###
-
-* Filetype: .cfg
-* Filename: <LAB_NAME>.cfg
-    * [default] section
-    * backend_fqdn = "The FQDN of the kernelCI backend to callback"
-    * lab_name = "The lab name as registered for the kernelCI backend"
-    * lab_token = "The kernelCI backend lab token"
-
-Example file: lab-mylab.cfg
-`
-[default]
-backend_fqdn = http(s)://api.mylab.com
-lab_name = lab-mylab
-lab_token = 123g5789-45f4-4f21-a485-7412589df235
-`
diff --git a/templates/callback/lab-agl-core.cfg b/templates/callback/lab-agl-core.cfg
index 6be0098..08df284 100644
--- a/templates/callback/lab-agl-core.cfg
+++ b/templates/callback/lab-agl-core.cfg
@@ -1,4 +1,12 @@
 [default]
+section = AGL
+
+[AGL]
+backend_fqdn = http://kernelci.automotivelinux.org:8888
+lab_name = AGL-core-lab-1
+lab_token = lab-agl-core-callback-LF
+
+[Baylibre]
 backend_fqdn = http://api.dev.baylibre.com
 lab_name = lab-agl-core
-lab_token = lab-agl-core-callback
+lab_token = lab-agl-core-callback-baylibre
\ No newline at end of file
diff --git a/templates/callback/lab-baylibre-lavabox.cfg b/templates/callback/lab-baylibre-lavabox.cfg
deleted file mode 100644
index e8b849e..0000000
--- a/templates/callback/lab-baylibre-lavabox.cfg
+++ /dev/null
@@ -1,4 +0,0 @@
-[default]
-backend_fqdn = http://api.lavabox
-lab_name = lab-baylibre-lavabox
-lab_token = callback-demo
diff --git a/templates/callback/lab-baylibre-legacy.cfg b/templates/callback/lab-baylibre-legacy.cfg
index c9e38b5..e5ac5a0 100644
--- a/templates/callback/lab-baylibre-legacy.cfg
+++ b/templates/callback/lab-baylibre-legacy.cfg
@@ -1,4 +1,12 @@
 [default]
+section = AGL
+
+[AGL]
+backend_fqdn = http://kernelci.automotivelinux.org:8888
+lab_name = lab-baylibre
+lab_token = lab-baylibre-legacy-callback-LF
+
+[Baylibre]
 backend_fqdn = http://api.dev.baylibre.com
 lab_name = lab-baylibre-legacy
-lab_token = lab-baylibre-legacy-callback
+lab_token = lab-baylibre-legacy-callback-baylibre
\ No newline at end of file
diff --git a/utils/agljobtemplate.py b/utils/agljobtemplate.py
index be46e92..381f7f4 100644
--- a/utils/agljobtemplate.py
+++ b/utils/agljobtemplate.py
@@ -20,15 +20,21 @@ def parse_url_file(template_path, url_file, url_type):
     except IOError:
         raise IOError, "Unable to read from file {}".format(url_file_path)
 
-def parse_callback_file(template_path, callback_file, job):
-    callback_file_path = template_path + '/callback/' + callback_file + '.cfg'
+def parse_callback_file(template_path, lava_callback, kci_callback):
+    callback_file_path = template_path + '/callback/' + lava_callback + '.cfg'
     try:
         with open(callback_file_path):
             cfg = ConfigParser.ConfigParser()
             cfg.read(callback_file_path)
-            job['backend_fqdn'] = cfg.get('default', 'backend_fqdn')
-            job['lab_name'] = cfg.get('default', 'lab_name')
-            job['lab_token'] = cfg.get('default', 'lab_token')
+            if kci_callback is None:
+                kci_callback = cfg.get('default', 'section')
+            cb_data = dict(cfg.items(kci_callback))
+            return cb_data
+    except ConfigParser.NoSectionError:
+        str_err = "Please make sure: --callback-to " + kci_callback
+        str_err += " is correct and corresponds to a section in: "
+        str_err += lava_callback + ".cfg"
+        raise ConfigParser.NoSectionError, str_err
     except IOError:
         raise IOError, "Unable to read from file {}".format(callback_file_path)
 
@@ -71,7 +77,8 @@ class Agljobtemplate(object):
 
     def render_job(self, url, machine, url_branch=None, url_version=None,
                    job_name="AGL-short-smoke", priority="medium", tests=[], rfs_type=None,
-                   kci_callback=None, rfs_image=None, kernel_image=None, dtb_image=None, modules_image=None,
+                   lava_callback=None, kci_callback=None,
+                   rfs_image=None, kernel_image=None, dtb_image=None, modules_image=None,
                    build_version=None):
         test_templates = []
 
@@ -134,12 +141,10 @@ class Agljobtemplate(object):
         if modules_image is not None:
             job['modules'] = modules_image
 
-        if kci_callback:
-            if test_templates:
-                job['callback_name'] = 'lava/test'
-            else:
-                job['callback_name'] = 'lava/boot'
-            parse_callback_file(self._template_path, kci_callback, job)
+        if lava_callback:
+            job['do_callback'] = True
+            callback_data = parse_callback_file(self._template_path, lava_callback, kci_callback)
+            job.update(callback_data)
 
         env = jinja2.Environment(loader=jinja2.FileSystemLoader(self._template_path))
         env.filters['get_extension'] = get_extension
diff --git a/utils/create-jobs.py b/utils/create-jobs.py
index 67afe4c..5a146c4 100755
--- a/utils/create-jobs.py
+++ b/utils/create-jobs.py
@@ -24,8 +24,12 @@ def parse_cmdline(machines, tests, rfs_types):
                         help='The version (or patchset) to generate the job for.')
     parser.add_argument('--boot', action='store', dest='rfs_type',
                         choices=rfs_types, help='select boot type')
-    parser.add_argument('--callback', action='store', dest='callback',
-                        help='url to notify when job is done. Please read: ./templates/callback/callback_readme.txt')
+    parser.add_argument('--callback-from', action='store', dest='callback_from',
+                        help='The LAVA lab (name) that will be responsible of doing the callback. '
+                        'Please read: ./templates/callback/callback_readme.txt')
+    parser.add_argument('--callback-to', action='store', dest='callback_to',
+                        help='The KernelCI instance (name) that will receive the callback from LAVA. '
+                        'Please read: ./templates/callback/callback_readme.txt')
     parser.add_argument('--test', dest='tests',  action='store', choices=tests + ['all'],
                         help="add these test to the job", nargs='*', default=[])
     parser.add_argument('-o', '--output', dest='job_file',  action='store',
@@ -72,7 +76,8 @@ def main():
 
     job = ajt.render_job(url=args.url, url_branch=args.url_branch, url_version=args.url_version,
                          machine=args.machine, tests=args.tests, priority=args.priority,
-                         rfs_type=args.rfs_type, job_name=args.job_name, kci_callback=args.callback,
+                         rfs_type=args.rfs_type, job_name=args.job_name,
+                         lava_callback=args.callback_from, kci_callback=args.callback_to,
                          rfs_image=args.rootfs_img,
                          kernel_image=args.kernel_img,
                          dtb_image=args.dtb_img,