aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJan-Simon Möller <dl9pf@gmx.de>2016-08-30 23:15:27 +0200
committerJan-Simon Moeller <jsmoeller@linuxfoundation.org>2016-09-01 21:40:17 +0000
commit379bec2b01b3a27a66f20858caf2f2cc25d258b7 (patch)
treebcfd0b302c310590f46e9832bad9adf0e59c5e70
parent5839362875ee425e4e434c19e178e3e0e1fc216c (diff)
AGL-JTA integration for using boards through LAVA
This is the initial version quite tailored to the AGL setup. Todo: generalize scripts Change-Id: Ifc1eeaf4130b7ea67595dc0c24170055fdf2c323 Signed-off-by: Jan-Simon Möller <dl9pf@gmx.de>
-rw-r--r--Dockerfile25
-rw-r--r--userdata/conf/boards/porter.board20
-rw-r--r--userdata/conf/lava/.lava.yaml4
-rwxr-xr-xuserdata/conf/lava/boardready.py65
-rw-r--r--userdata/conf/lava/deploy.sh10
-rw-r--r--userdata/conf/lava/lava-boot/README203
-rw-r--r--userdata/conf/lava/lava-boot/hack.yaml22
-rwxr-xr-xuserdata/conf/lava/lava-boot/lava-boot373
-rwxr-xr-xuserdata/conf/lava/lava-boot/proxy.py82
-rw-r--r--userdata/conf/lava/lava-boot/template.yaml19
-rw-r--r--userdata/conf/lava/lava-boot/test_definition.yaml20
-rw-r--r--userdata/conf/lava/porter_nbd.yaml20
-rw-r--r--userdata/conf/lava/porter_nbd_snapshot.yaml20
-rw-r--r--userdata/conf/lava/porter_nbd_snapshot_withvar.yaml20
-rw-r--r--userdata/conf/lava/upload4lava.sh14
15 files changed, 903 insertions, 14 deletions
diff --git a/Dockerfile b/Dockerfile
index be67bb4..38c24a4 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -4,10 +4,12 @@
FROM debian:jessie
MAINTAINER qiaonuohan@cn.fujitsu.com
+MAINTAINER jsmoeller@linuxfoundation.org
ENV INST_JTA_ENGINE_PATH /home/jenkins
ENV INST_JTA_FRONTEND_PATH /var/lib/jenkins
-ENV INST_JTA_CORE_GIT_REVISION 0b96e0564d44bcc0a2bc0a8c2a24b0877c6e9ac3
+ENV INST_JTA_CORE_GIT_REVISION 35d13c8c4eb1f21189d1f6836df977199c980c39
+#ENV INST_JTA_CORE_GIT_REVISION origin/master
# ==============================================================================
# Prepare basic image
@@ -17,9 +19,9 @@ WORKDIR /jta-install
RUN dpkg --add-architecture i386
#RUN echo deb http://mirrors.163.com/debian/ jessie main non-free > /etc/apt/sources.list
#RUN echo deb http://mirrors.163.com/debian/ jessie-updates main non-free >> /etc/apt/sources.list
-RUN echo deb http://ftp.us.debian.org/debian jessie main non-free >> /etc/apt/sources.list
-RUN echo deb http://ftp.us.debian.org/debian jessie-updates main non-free >> /etc/apt/sources.list
-RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get -yV install apt-utils daemon gcc make python-paramiko python-lxml python-simplejson python-matplotlib libtool xmlstarlet autoconf automake rsync openjdk-7-jre openjdk-7-jdk iperf netperf netpipe-tcp texlive-latex-base sshpass wget git sudo net-tools vim openssh-server curl bzip2
+RUN echo deb http://httpredir.debian.org/debian jessie main non-free > /etc/apt/sources.list
+RUN echo deb http://httpredir.debian.org/debian jessie-updates main non-free >> /etc/apt/sources.list
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get -yV install apt-utils daemon gcc make python-paramiko python-lxml python-simplejson python-matplotlib libtool xmlstarlet autoconf automake rsync openjdk-7-jre openjdk-7-jdk iperf netperf netpipe-tcp texlive-latex-base sshpass wget git sudo net-tools vim openssh-server curl bzip2 mc python-yaml python-requests
RUN /bin/bash -c 'echo "dash dash/sh boolean false" | debconf-set-selections ; DEBIAN_FRONTEND=noninteractive dpkg-reconfigure dash'
#COPY frontend-install/jenkins_1.509.2_all.deb /jta-install/
#RUN dpkg -i /jta-install/jenkins_1.509.2_all.deb
@@ -59,6 +61,21 @@ RUN ln -s /userdata/conf/users $INST_JTA_FRONTEND_PATH/users
RUN ln -s /userdata/conf/tools.sh $INST_JTA_ENGINE_PATH/scripts/tools.sh
#RUN mkdir $INST_JTA_ENGINE_PATH/logs/logruns
+# persist jenkins user account data
+RUN rm -rf $INST_JTA_FRONTEND_PATH/users
+RUN ln -s /userdata/users $INST_JTA_FRONTEND_PATH/users
+
+# ==============================================================================
+# get LAVA-boot via git
+# ==============================================================================
+
+RUN mkdir -p $INST_JTA_ENGINE_PATH
+#RUN git clone https://git.linaro.org/people/riku.voipio/lava-boot.git $INST_JTA_ENGINE_PATH/lava-boot
+run ln -s /userdata/conf/lava/lava-boot $INST_JTA_ENGINE_PATH/lava-boot
+#link .lava.yaml template
+RUN ln -s /userdata/conf/lava/.lava.yaml $INST_JTA_FRONTEND_PATH/.lava.yaml
+
+
# ==============================================================================
# Initialize Jenkins plugin configs
# ==============================================================================
diff --git a/userdata/conf/boards/porter.board b/userdata/conf/boards/porter.board
index a01577d..f4316a5 100644
--- a/userdata/conf/boards/porter.board
+++ b/userdata/conf/boards/porter.board
@@ -1,14 +1,14 @@
inherit "base-board"
include "base-params"
-IPV4_NET="192.168.30"
+IPV4_NET="192.168.111"
-IPADDR="192.168.30.64"
+IPADDR="10.0.0.247"
L_IPV4_HOST="64"
L_HOST_IF="eth0"
-SSH_PORT="22"
+SSH_PORT="2222"
LOGIN="root"
-JTA_HOME="/home/a"
+JTA_HOME="/home/"
#PASSWORD="root"
PLATFORM="porter"
TRANSPORT="ssh"
@@ -40,18 +40,18 @@ LTP_EXPECT_FAIL_CASES="abort01 access06 acct01 adjtimex02 cacheflush01 chmod06 c
# for network test of ltp
# name of the remote host machine
-REMOTE_HOST="192.168.30.63"
-R_IPV4_HOST="63"
+REMOTE_HOST="192.168.111.1"
+R_IPV4_HOST="1"
R_HOST_IF="eth0"
# root passwd of the remote host machine
REMOTE_PASSWD="root"
# configuration for CIAT
# configuration for building images
-BUILD_SERVERIP="192.168.30.70"
-BUILD_USER="vince"
-BUILD_PASSWORD="root"
-BUILD_PATH="/compile/agl-demo/CIAT"
+#BUILD_SERVERIP="192.168.30.70"
+#BUILD_USER="vince"
+#BUILD_PASSWORD="root"
+#BUILD_PATH="/compile/agl-demo/CIAT"
# the directory for image will be $BUILD_PATH/repoclone/$IMAGE_PATH/
IMAGE_PATH="build/tmp/deploy/images"
UIMAGE="uImage"
diff --git a/userdata/conf/lava/.lava.yaml b/userdata/conf/lava/.lava.yaml
new file mode 100644
index 0000000..c88b7ac
--- /dev/null
+++ b/userdata/conf/lava/.lava.yaml
@@ -0,0 +1,4 @@
+server: porter.automotivelinux.org
+user: replace_with_user
+token: replace_with_token
+https: true \ No newline at end of file
diff --git a/userdata/conf/lava/boardready.py b/userdata/conf/lava/boardready.py
new file mode 100755
index 0000000..45639f6
--- /dev/null
+++ b/userdata/conf/lava/boardready.py
@@ -0,0 +1,65 @@
+#!/usr/bin/python
+
+import xmlrpclib
+import ssl
+import sys
+import time
+
+# bug, ssl chain cannot be verified (letsencrypt)
+# to be fixed
+ssl._create_default_https_context = ssl._create_unverified_context
+#server = xmlrpclib.ServerProxy("https://porter.automotivelinux.org/RPC2")
+
+print "Starting ..."
+sys.stdout.flush()
+
+arg = None
+
+if len(sys.argv) > 1:
+ arg = sys.argv[1]
+
+if arg == None:
+ print("No argument, need lava jobnumber as argument.")
+ sys.exit(1)
+
+username = "replace-with-username"
+token = "replace-with-token"
+hostname = "porter.automotivelinux.org"
+
+server = xmlrpclib.ServerProxy("https://%s:%s@%s/RPC2" % (username, token, hostname))
+
+#print(arg)
+#sys.stdout.flush()
+
+# Poll loop
+while True:
+ try:
+ x = server.scheduler.job_status(arg)['job_status']
+ except:
+ print("Error, quitting.")
+ sys.stdout.flush()
+ sys.exit(1)
+ break
+ if 'Cancelled' in x:
+ sys.exit(1)
+ break
+ if 'Submitted' in x:
+ print("Job submitted - pending")
+ sys.stdout.flush()
+ time.sleep(20)
+ continue
+ if 'Running' in x:
+ print("Job Running now.")
+ print("Remote boot takes around 6-10 minutes to complete ...")
+ sys.stdout.flush()
+ time.sleep(600)
+ # todo: check for ssh port and wait a little longer if not
+ y = server.scheduler.job_status(arg)['job_status']
+ if 'Running' in y:
+ break
+ else:
+ continue
+ break
+ break
+
+# end
diff --git a/userdata/conf/lava/deploy.sh b/userdata/conf/lava/deploy.sh
new file mode 100644
index 0000000..a358144
--- /dev/null
+++ b/userdata/conf/lava/deploy.sh
@@ -0,0 +1,10 @@
+#!/bin/bash
+
+#set -x
+set -e
+
+PORTERYAML="porter_nbd_snapshot.yaml"
+
+
+/userdata/conf/lava/boardready.py $(/home/jenkins/lava-boot/lava-boot -j /userdata/conf/lava/${PORTERYAML} -a -q | sed -e "s#.*job/##g")
+
diff --git a/userdata/conf/lava/lava-boot/README b/userdata/conf/lava/lava-boot/README
new file mode 100644
index 0000000..7484a9e
--- /dev/null
+++ b/userdata/conf/lava/lava-boot/README
@@ -0,0 +1,203 @@
+Lava-boot
+=========
+
+Lava-boot is a synchronous alternative to the official lava-tool command line
+tool. Lava-boot will submit a job and exit when the job finishes.
+
+The key use case is quick turnaround testing of locally built kernel in LAVA.
+
+Lava-boot features include variable substitution in job/test definition files,
+running scripts on target devices (with exit code return) and even
+integrated hacking session support.
+
+Quick start
+===========
+
+Go to your lava web interface and get your API key from api->tokens
+
+Create the minimal ~/.lava.yaml configuration file:
+
+-~/lava.yaml-
+server: lava-server
+user: username
+token: lava token string
+https: true
+-~/lava.yaml-
+
+The simplest invocation with just an existing json file:
+
+$ lava-boot -j test.json
+[12:32:46] LAVA Job 10, URL: http://lava-server/scheduler/job/10
+
+[12:32:46] LAVA Job 10, status: Submitted .
+[12:33:47] LAVA Job 10, status: Running ...................
+[12:43:56] LAVA Job 10 finished, status: Complete
+...output of lava job...
+
+Booting a local built kernel
+============================
+
+To boot a kernel in lava, the kernel needs to be placed somewhere where lava
+can fetch it from. lava-boot launches a HTTP server and uploads kernel and
+other artifacts there. The server is configured with the "proxy-ssh" and
+"proxy-hostname" variables below:
+
+-~/lava.yaml-
+server: lava-server
+user: username
+token: output of lava token
+https: true
+
+proxy-ssh: username@jumphost
+proxy-hostname: jumphost
+-~/lava.yaml-
+
+proxy-ssh is the username@host combo for the ssh to host which can be reached
+from the lava dispatcher and lava target machines. proxy-hostname is the
+hostname as seen from dispatcher.
+
+In case your PC and lava dispatcher are in the same network, you don't need
+proxy-ssh setting - just set proxy-hostname t the hostname or IP of the PC
+where you are running lava-boot from.
+
+You will now need a lava job definition with a few variable expansions. You
+can take an existing .json file and set the dtb to ${PROXY}/${DTB} and kernel
+to ${PROXY}/${KERNEL}, or you can use the minimal yaml file below as beginning
+point. lava-boot will convert the yaml file to json file on the fly.
+
+-test.yaml-
+actions:
+ - command: deploy_linaro_kernel
+ parameters:
+ dtb: '${PROXY}/${DTB}'
+ kernel: '${PROXY}/${KERNEL}'
+ nfsrootfs: ${ROOTFS}
+ - command: boot_linaro_image
+device_type: '${DEVICE_TYPE}'
+job_name: '${JOB_NAME}'
+timeout: 1800
+-test-yaml-
+
+Now say, you have a kernel for apm mustang compiled under build/ directory,
+you can now boot with:
+
+lava-boot -k build/ -d arndale -j test.yaml
+[16:00:26] Proxy started at: user@jumphost, URL: http://jumphost:29871
+[16:00:28] Uploading kernel build/arch/arm/boot/zImage
+[16:00:34] Uploading dtb build/arch/arm/boot/dts/exynos5250-arndale.dtb for arndale
+[16:00:35] LAVA Job 11, URL: http://lava-server/scheduler/job/11
+
+[16:00:35] LAVA Job 11, status: Submitted
+[16:01:06] LAVA Job 11, status: Running .
+
+
+Running a script on device and fetch it's exit code
+===================================================
+
+The original motivation for this script was to run arbitrary scripts against
+the locally built kernel and get the results back. For example, to bisect a
+regression on where the rtc device vanished, you might want a script like
+
+--has-rtc.sh--
+#!/bin/sh -e
+test -c /dev/rtc
+--has-rtc.sh--
+
+$ lava-boot --quiet -k build/ -d arndale -r has-rtc.sh
+[16:00:26] Proxy started at: user@jumphost, URL: http://jumphost:29871
+[16:00:28] Uploading kernel build/arch/arm/boot/zImage
+[16:00:34] Uploading dtb build/arch/arm/boot/dts/exynos5250-arndale.dtb for arndale
+[16:00:34] Uploading test-rtc
+[16:00:35] Uploading test_definition.yaml
+[16:00:35] LAVA Job 11, URL: http://lava-server/scheduler/job/11
+
+[16:00:35] LAVA Job 11, status: Submitted
+[16:01:06] LAVA Job 11, status: Running ......
+[16:04:38] LAVA Job 11, script finished: fail
+$
+
+The target of this feature is to help bisecting.
+
+Automatic hacking sessions
+==========================
+
+One advanced hack in this script has is automatic hack sessions: Add your ssh
+key settings to config. sshfile is optional if you only have one private ssh
+key. sshkey variable is the contents of your public ssh key, usually in file
+~/.ssh/id_rsa.pub.
+
+-~/lava.yaml-
+sshfile: "~/.ssh/private_file"
+sshkey: "ssh-rsa AAAAB3i...."
+-~/lava.yaml-
+
+Add the --hack option:
+
+$ lava-boot -k build -d arndale --hack
+[14:27:19] Proxy started at: user@jumphost, URL: http://jumphost:36766
+[14:27:21] Uploading kernel build/arch/arm/boot/zImage
+[14:27:27] Uploading dtb build/arch/arm/boot/dts/exynos5250-arndale.dtb for arndale
+[14:27:27] LAVA Job 243722, URL: http://lava-server/scheduler/job/243722
+
+[14:27:27] LAVA Job 243722, status: Submitted
+[14:27:58] LAVA Job 243722, status: Running ........
+[14:32:31] LAVA Job 243722, hacking session: 10.1.1.1
+[14:32:32] session started with: ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o 'ProxyCommand=ssh user@jumphost netcat %h %p' root@10.1.1.1
+Warning: Permanently added '10.1.1.1' (ECDSA) to the list of known hosts.
+Welcome to Ubuntu 14.10 (GNU/Linux 3.19.0-rc5 armv7l)
+
+ * Documentation: http://www.linaro.org
+Last login: Sat Jan 1 00:00:13 2000
+root@linaro-nano:~#
+lava-boot -k build/ -d mustang --hack
+
+
+Variable substitution
+=====================
+
+Automatic Variables defined by lava-boot:
+
+GIT_COMMIT = git commit id of kernel
+TESTCASE = lava_test_shell test case passed from command line
+KERNEL = kernel filename
+DTB = device tree file used
+DEVICE_TYPE = lava device type
+
+PROXY = proxy HTTP URL
+RESULT_URL = URL to poll back for results
+JOB_NAME = generated name for job
+
+ROOTFS = rootfs URL
+ lava-boot fill this with rootfs_arm for 32bit target and
+ rootfs_arm64 for 64bit targets. these variables can be set
+ in ~/.lava.yaml
+
+All variables from ~/.lava.yaml are expanded too. You can add arbitrary
+key/value pairs to the configuration file, and lava-boot will used them
+in substitution phase.
+
+Multi-server configuration
+==========================
+
+The configuration file supports defining several servers, using yaml list
+format:
+
+- server: lavaserver.net
+ user: ss
+ token: ...
+- server: staging.lavaserver.net
+ user: xx
+ token: ...
+
+In this format by default lava-boot will pick up first server defined. other
+servers can be selected using "-s staging.lavaserver.net" in command line
+
+TODO
+====
+
+* test suite!
+* printing output from LAVA line-by-line
+* instead of passing success/fail via proxy, support reading lava-test-shell
+ results from result bundle
+* make is possible to upload the job artifacts directly to LAVA so proxy can
+ be eliminated.
diff --git a/userdata/conf/lava/lava-boot/hack.yaml b/userdata/conf/lava/lava-boot/hack.yaml
new file mode 100644
index 0000000..dd5e65b
--- /dev/null
+++ b/userdata/conf/lava/lava-boot/hack.yaml
@@ -0,0 +1,22 @@
+actions:
+ - command: deploy_linaro_kernel
+ parameters:
+ dtb: '${PROXY}/${DTB}'
+ kernel: '${PROXY}/${KERNEL}'
+ nfsrootfs: '${ROOTFS}'
+ target_type: ubuntu
+ metadata:
+ git commit: '${GIT_COMMIT}'
+ - command: boot_linaro_image
+ - command: lava_test_shell
+ parameters:
+ testdef_repos:
+ - git-repo: http://git.linaro.org/lava-team/hacking-session.git
+ testdef: hacking-session-debian.yaml
+ parameters:
+ NOTIFY_URL: '${PROXY}/result'
+ PUB_KEY: '${SSHKEY}'
+ timeout: 60000
+device_type: '${DEVICE_TYPE}'
+job_name: '${JOB_NAME}'
+timeout: 60000
diff --git a/userdata/conf/lava/lava-boot/lava-boot b/userdata/conf/lava/lava-boot/lava-boot
new file mode 100755
index 0000000..b6d3d7f
--- /dev/null
+++ b/userdata/conf/lava/lava-boot/lava-boot
@@ -0,0 +1,373 @@
+#!/usr/bin/python
+
+import argparse
+import atexit
+import json
+import os
+import re
+import requests
+import string
+import subprocess
+import sys
+import time
+import xmlrpclib
+import yaml
+from random import randint
+
+import ssl
+
+ssl._create_default_https_context = ssl._create_unverified_context
+
+
+# mapping device tree blob - device type
+device = {
+ 'beaglebone-black' : { 'dtb' : 'am335x-boneblack.dtb', 'kernel' : 'zImage' },
+ 'arndale' : { 'dtb' : 'exynos5250-arndale.dtb', 'kernel' : 'zImage' },
+ 'arndale-octa' : { 'dtb' : 'exynos5250-arndale-octa.dtb', 'kernel' : 'zImage'},
+ 'mustang' : { 'dtb' : 'apm-mustang.dtb', 'kernel' : 'Image', 'arch' : 'arm64'},
+ 'juno' : { 'dtb' : 'juno.dtb', 'kernel' : 'Image', 'arch' : 'arm64'},
+}
+
+replacetable = {
+'ROOTFS_ARM' : "https://releases.linaro.org/14.12/ubuntu/utopic-images/nano/linaro-utopic-nano-20141212-693.tar.gz",
+'ROOTFS_ARM64' : "http://people.linaro.org/~riku.voipio/linaro-utopic-nano64-20150114-87.tar.gz"
+}
+
+
+def obfuscate_credentials(s):
+ return re.sub(r'([^ ]:).+?(@)', r'\1xxx\2', s)
+
+def print_time():
+ return "["+time.strftime("%H:%M:%S", time.gmtime())+"]"
+
+def get_path(filename):
+ if os.path.exists(filename):
+ return filename
+ corepath = os.path.dirname(sys.argv[0])
+ longpath=os.path.join(corepath,filename)
+ if os.path.exists(longpath):
+ return longpath
+ else:
+ print '%s Error unable to open %s' % (print_time(), filename)
+ sys.exit(1)
+
+def read_settings(arguments):
+ settings_file=os.path.expanduser("~/.lava.yaml")
+ with open(settings_file) as f:
+ settings=yaml.safe_load(f.read())
+
+ # user asked for specific server
+ if arguments.server is not None:
+ for setting in settings:
+ if ( 'server' in setting and
+ setting['server'] == arguments.server ):
+ return setting
+ sys.stderr.write("server %s not found in ~/.lava.yaml\n" % arguments.server)
+ sys.exit(3)
+
+ # users with only one lava server in their config
+ if 'server' in settings:
+ return settings
+
+ # if a list of servers, use first
+ for setting in settings:
+ if 'server' in setting:
+ return setting
+
+ # give up!
+ sys.stderr.write("no useful server ~/.lava.yaml!\n")
+ sys.exit(3)
+
+def setup_replacetable(settings, variables, mydtb):
+ for key, value in settings.iteritems():
+ replacetable[key.upper()]=value
+
+ device_type=None
+ if not mydtb.endswith(".dtb") and mydtb in device:
+ device_type=mydtb
+ else:
+ for machine, features in device.iteritems():
+ if features['dtb'] == mydtb:
+ device_type = machine
+ break
+
+ if device_type == None:
+ sys.stderr.write("Invalid value for dtb/device: %s\n" % mydtb )
+ sys.exit(2)
+
+ replacetable['DEVICE_TYPE']=device_type
+
+ if 'arch' in device[device_type] and device[device_type]['arch'] == 'arm64':
+ replacetable['ROOTFS'] = replacetable['ROOTFS_ARM64']
+ else:
+ replacetable['ROOTFS'] = replacetable['ROOTFS_ARM']
+
+ if os.path.isdir(".git"):
+ try:
+ mystr=subprocess.check_output(["git","log","-n1","--pretty=oneline"]).splitlines()[0]
+ replacetable['GIT_COMMIT']=re.sub(r'([^\s\w.-]|_)+','',mystr)
+ except subprocess.CalledProcessError, e:
+ print "%s Git commit unavailable" %s (print_time())
+
+ if variables is None:
+ return
+
+ for variable in variables:
+ varlist=variable.split("=")
+ if len(varlist) == 2:
+ replacetable[varlist[0].upper()]=varlist[1]
+ else:
+ sys.stderr.write("Invalid --variable parameter: %s\n" % variable )
+ sys.exit(2)
+
+
+def substitute_file(filename):
+ with open(get_path(filename)) as f:
+ template = string.Template(f.read())
+ try:
+ replaced=template.substitute(replacetable)
+ except KeyError, e:
+ sys.stderr.write("Missing key in replacement table: %s\n" % str(e) )
+ sys.exit(2)
+ return replaced
+
+def upload_file(upload_url, filename, replacename):
+ upload_string=substitute_file(filename)
+ basename=os.path.basename(filename)
+ replacetable[replacename]=basename
+ print '%s Uploading %s' % (print_time(), basename)
+ r = requests.post(upload_url, files={'filename': (basename, upload_string) })
+
+def upload_kernel(upload_url, searchdir):
+
+ device_type=replacetable['DEVICE_TYPE']
+ lava_device=device[device_type]
+ kernel_name= lava_device['kernel']
+ dtb = lava_device['dtb']
+
+ for root, dirs, files in os.walk(searchdir + "/arch/"):
+ for f in files:
+ if f == kernel_name:
+ replacetable['KERNEL']=kernel_name
+ kernel=os.path.join(root,f)
+ print '%s Uploading kernel %s' % (print_time(), kernel)
+ r = requests.post(upload_url,
+ files={'filename': open(kernel, 'rb')})
+
+ if f == dtb:
+ replacetable['DTB']=dtb
+ dtb=os.path.join(root,f)
+ print '%s Uploading dtb %s for %s' % (print_time(), dtb, device_type)
+ r = requests.post(upload_url,
+ files={'filename': open(dtb, 'rb')})
+
+ if not 'DTB' in replacetable:
+ sys.stderr.write("Failed to upload dtb: %s \n" % dtb )
+ sys.exit(2)
+ if not 'KERNEL' in replacetable:
+ sys.stderr.write("Failed to upload kernel: %s \n" % kernel_name )
+ sys.exit(2)
+
+def poll_job(server, job_id, settings, upload_url, poll_type):
+ return_code=2
+ prev_status=""
+ while True:
+ status=server.scheduler.job_status(job_id)
+ s=status['job_status']
+ if s == 'Complete':
+ if poll_type=='script':
+ r = requests.get("%s/return_code" % upload_url)
+ if r.status_code == 200:
+ sys.stdout.write("\n%s LAVA Job %s, script finished: %s\n" % (print_time(), job_id, r.text))
+ sys.stdout.flush()
+ if r.text=='success':
+ return 0
+ else:
+ return 3
+
+ return_code=0
+ break
+ elif s == 'Incomplete' or s == 'Canceled' or s == 'Canceling':
+ return_code=1
+ if prev_status == 'Submitted':
+ return_code=4
+ break
+
+ if prev_status != s:
+ sys.stdout.write("\n%s LAVA Job %s, status: %s " % (print_time(), job_id, s))
+ sys.stdout.flush()
+ elif poll_type=='hack' and s == "Running":
+ r = requests.get("%s/ssh" % upload_url)
+ if r.status_code == 200:
+ sys.stdout.write("\n%s LAVA Job %s, hacking session: %s\n" % (print_time(), job_id, r.text))
+ sys.stdout.flush()
+ start_hacking(settings, r.text)
+
+ sys.stdout.write(".")
+ sys.stdout.flush()
+ else:
+ sys.stdout.write(".")
+ sys.stdout.flush()
+ prev_status=s
+ time.sleep(30)
+ sys.stdout.write("\n%s LAVA Job %s finished, status: %s\n" % (print_time(), job_id, s))
+ sys.stdout.flush()
+ return return_code
+
+def start_hacking(settings, ipaddr):
+
+ sshfile=""
+ jump=""
+ if 'proxy-ssh' in settings:
+ jump="-o 'ProxyCommand=ssh %s netcat %%h %%p'" % (settings['proxy-ssh'])
+ if 'sshfile' in settings:
+ sshfile="-o IdentityFile=%s" % (settings['sshfile'])
+
+ # FIXME check that ipaddr is really only and ipaddr
+ ssh_command = "ssh %s -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null %s root@%s" % (sshfile, jump, ipaddr)
+ print "%s session started with: %s" %(print_time(), ssh_command)
+ subprocess.call(ssh_command, shell=True)
+ # FIXME xmlrpc call cancel session to tear it down
+ sys.exit(0)
+
+def random_port():
+ return str(randint(8000,50000))
+
+def start_proxy(settings):
+ # FIXME: try another TCP port if first one is reserved
+ proxyport=random_port()
+ proxy="http://"+settings['proxy-hostname']+":"+proxyport
+ replacetable['PROXY']=proxy
+ if 'proxy-ssh' in settings:
+ ssh_host=settings['proxy-ssh']
+ port_forward=proxyport+":localhost:"+proxyport
+ subprocess.check_call(["scp", get_path("proxy.py"), ssh_host+":"],
+ stdout=subprocess.PIPE)
+ proxy_process=subprocess.Popen(["ssh","-t","-t","-L" ,port_forward, ssh_host, "~/proxy.py", proxyport],
+ stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+ print '%s Proxy started at: %s, URL: %s' % (print_time(), ssh_host, proxy)
+ else:
+ ssh_host=None
+ proxy_process=subprocess.Popen([get_path("proxy.py"), proxyport],
+ stdout=subprocess.PIPE, stdin=subprocess.PIPE, stderr=subprocess.PIPE)
+ print '%s Proxy started at: localhost, URL: %s' % (print_time(), proxy)
+ atexit.register(exit_cleanup, proxy_process)
+ time.sleep(2)
+ return "http://localhost:"+proxyport+"/"
+
+def exit_cleanup(proxy_process):
+ proxy_process.kill()
+
+
+def lava_submit(template, settings):
+ lava_server=settings['server']
+ lava_user=settings['user']
+ lava_token=settings['token']
+ if 'https' in settings and settings['https'] is not True:
+ server_url = 'http://{lava_user:>s}:{lava_token:>s}@{lava_server:>s}/RPC2'
+ result_url = 'http://' + lava_server + "/RPC2"
+ else:
+ server_url = 'https://{lava_user:>s}:{lava_token:>s}@{lava_server:>s}/RPC2'
+ result_url = 'https://' + lava_server + "/RPC2"
+ replacetable['RESULT_URL']= result_url
+ replacetable['JOB_NAME']="lava-boot job for "+lava_user
+
+ config=substitute_file(template)
+ if template.endswith(".yaml"):
+ config=json.dumps(yaml.safe_load(config), indent=2)
+
+ try:
+ server = \
+ xmlrpclib.ServerProxy(server_url.format(
+ lava_user=lava_user,
+ lava_token=lava_token,
+ lava_server=lava_server))
+ lava_job_id = server.scheduler.submit_job(config)
+ except xmlrpclib.ProtocolError, e:
+ print 'Error making a LAVA request:', obfuscate_credentials(str(e))
+
+ print '%s LAVA Job %s, URL: http://%s/scheduler/job/%s' % \
+ (print_time(), lava_job_id, lava_server, lava_job_id)
+ return server, lava_job_id
+
+def main():
+ parser = argparse.ArgumentParser(
+ description='Generate LAVA job definition')
+ parser.add_argument('-s', '--server',
+ help='server selection from ~/.lava.yaml, default first one')
+ parser.add_argument('-j', '--job',
+ help='Job template for LAVA, json or yaml')
+ parser.add_argument('-t', '--testcase',
+ help='lava-test-shell definition (optional)')
+ parser.add_argument('-r', '--run-script',
+ help='run script on target device')
+ parser.add_argument('-k', '--kerneldir',
+ help='kernel build dir')
+ parser.add_argument('-d', '--device', default='arndale',
+ help='lava device type or dtb file to use')
+ parser.add_argument('-x', '--hack', action='store_true',
+ help='start a hacking session')
+ parser.add_argument('-z','--skip-lava', action='store_true',
+ help="Dont submit to lava")
+ parser.add_argument('-q','--quiet', action='store_true',
+ help="Dont print job log")
+ parser.add_argument('-v','--variable', action='append',
+ help="with -v foo=bar ${FOO} will replaced with bar in job template")
+ parser.add_argument('-a','--async', action='store_true',
+ help="dont wait for lava job to finish")
+
+ arguments = parser.parse_args()
+
+ settings=read_settings(arguments)
+ setup_replacetable(settings, arguments.variable, arguments.device)
+ poll_type=None
+
+ if ( arguments.kerneldir is not None or
+ arguments.testcase is not None or
+ arguments.run_script is not None or
+ arguments.hack is True ):
+ upload_url=start_proxy(settings)
+ if arguments.testcase is not None:
+ testcase=arguments.testcase
+ else:
+ testcase=None
+ if arguments.kerneldir is not None:
+ upload_kernel(upload_url, arguments.kerneldir)
+ if arguments.run_script is not None:
+ poll_type='script'
+ upload_file(upload_url, arguments.run_script, 'SCRIPT')
+ if arguments.testcase is None:
+ testcase="test_definition.yaml"
+ if testcase is not None:
+ upload_file(upload_url, testcase, 'TESTCASE')
+ else:
+ upload_url=None
+
+ if arguments.skip_lava:
+ sys.exit(0)
+ if arguments.hack:
+ poll_type='hack'
+ if arguments.job is not None:
+ job=arguments.job
+ elif arguments.hack:
+ job= "hack.yaml"
+ elif arguments.run_script:
+ job="template.yaml"
+ else:
+ print "no job definition given"
+ sys.exit(1)
+
+ server, job_id = lava_submit(job, settings)
+ if arguments.async:
+ sys.exit(0)
+
+ return_code=poll_job(server, job_id, settings, upload_url, poll_type)
+
+ if arguments.quiet is False and return_code != 4:
+ job_log=server.scheduler.job_output(job_id)
+ print job_log
+
+ sys.exit(return_code)
+
+if __name__ == '__main__':
+ main()
diff --git a/userdata/conf/lava/lava-boot/proxy.py b/userdata/conf/lava/lava-boot/proxy.py
new file mode 100755
index 0000000..c72efb8
--- /dev/null
+++ b/userdata/conf/lava/lava-boot/proxy.py
@@ -0,0 +1,82 @@
+#!/usr/bin/python
+
+# simple http file upload/download server
+# you can upload either using POST method, or by key/value upload
+
+# GET result/kernelversion/`uname -r`
+# POST method:
+# import requests
+# r = requests.post('http://localhost:8000/', files={'filename': open('proxy.py', 'rb')})
+
+
+import string,cgi,time,os,sys,urllib
+from BaseHTTPServer import BaseHTTPRequestHandler, HTTPServer
+
+IN_MEMORY_FILES = {}
+
+class MyHandler(BaseHTTPRequestHandler):
+
+ def do_GET(self):
+ try:
+ filename=os.path.basename(self.path)
+ if filename in IN_MEMORY_FILES:
+ self.send_response(200)
+ self.send_header('Content-type','application/octet-stream')
+ self.end_headers()
+ self.wfile.write(IN_MEMORY_FILES[filename])
+ return
+ else:
+ # GET /result?key=value as way to store values with GET
+ # This makes it possible to ping back from lava
+ elements=self.path.split("?")
+ if (len(elements) == 2 and elements[0] == '/result'):
+ (key, data) = elements[1].split("=",1)
+ IN_MEMORY_FILES[key] = urllib.unquote(data)
+ self.send_response(200)
+ self.end_headers()
+ else:
+ self.send_response(404)
+ self.end_headers()
+ return
+ except IOError as e :
+ print e
+ self.send_error(404,'File Not Found: %s' % self.path)
+
+ def do_POST(self):
+ try:
+ ctype, pdict = cgi.parse_header(self.headers.getheader('content-type'))
+ if ctype == 'multipart/form-data' :
+ fs = cgi.FieldStorage( fp = self.rfile,
+ headers = self.headers, # headers_,
+ environ={ 'REQUEST_METHOD':'POST' }
+ )
+
+ else: raise Exception("Unexpected POST request")
+
+ fs_up = fs['filename']
+ filename = os.path.basename(fs_up.filename)
+ IN_MEMORY_FILES[filename] = fs_up.file.read()
+
+ self.send_response(200)
+ self.end_headers()
+ except Exception as e:
+ print e
+ self.send_error(404,'POST to "%s" failed: %s' % (self.path, str(e)) )
+
+def main():
+ if len(sys.argv) == 2:
+ port = int(sys.argv[1])
+ else:
+ port = 8080
+
+ try:
+ server = HTTPServer(('', port), MyHandler)
+ print 'started file sharing http server at port %d' % port
+ server.serve_forever()
+ except KeyboardInterrupt:
+ print '^C received, shutting down server'
+ server.socket.close()
+
+if __name__ == '__main__':
+ main()
+
diff --git a/userdata/conf/lava/lava-boot/template.yaml b/userdata/conf/lava/lava-boot/template.yaml
new file mode 100644
index 0000000..5338ed5
--- /dev/null
+++ b/userdata/conf/lava/lava-boot/template.yaml
@@ -0,0 +1,19 @@
+actions:
+ - command: deploy_linaro_kernel
+ parameters:
+ dtb: '${PROXY}/${DTB}'
+ kernel: '${PROXY}/${KERNEL}'
+ nfsrootfs: '${ROOTFS}'
+ target_type: ubuntu
+ metadata:
+ git commit: '${GIT_COMMIT}'
+ - command: boot_linaro_image
+ - command: "lava_test_shell"
+ parameters:
+ testdef_urls:
+ - '${PROXY}/${TESTCASE}'
+ timeout: 600
+device_type: '${DEVICE_TYPE}'
+logging_level: INFO
+job_name: '${JOB_NAME}'
+timeout: 1800
diff --git a/userdata/conf/lava/lava-boot/test_definition.yaml b/userdata/conf/lava/lava-boot/test_definition.yaml
new file mode 100644
index 0000000..78ffecb
--- /dev/null
+++ b/userdata/conf/lava/lava-boot/test_definition.yaml
@@ -0,0 +1,20 @@
+metadata:
+ name: "lava-boot-test"
+ format: "Lava-Test-Shell Test Definition 1.0"
+ description: "Run a script for lava-boot"
+ version: "1.0"
+ maintainer:
+ - riku.voipio@linaro.org
+ os:
+ - ubuntu
+ environment:
+ - lava-test-shell
+install:
+ deps:
+ - wget
+
+run:
+ steps:
+ - "wget ${PROXY}/${SCRIPT}"
+ - "sh ./${SCRIPT} && wget ${PROXY}/result?return_code=success||wget ${PROXY}/result?return_code=fail"
+
diff --git a/userdata/conf/lava/porter_nbd.yaml b/userdata/conf/lava/porter_nbd.yaml
new file mode 100644
index 0000000..2a6e153
--- /dev/null
+++ b/userdata/conf/lava/porter_nbd.yaml
@@ -0,0 +1,20 @@
+actions:
+ - command: deploy_linaro_kernel
+ parameters:
+ kernel: 'http://localhost/porter/upload/uImage+dtb'
+ nbdroot: 'http://localhost/porter/upload/agl-demo-platform-porter.ext4'
+ ramdisk: 'http://localhost/porter/upload/initramfs-netboot-image-porter.ext4.gz.u-boot'
+ login_prompt: 'porter login:'
+ username: 'root'
+ - command: boot_linaro_image
+ parameters:
+ test_image_prompt: 'root@porter:~#'
+ - command: lava_command_run
+ parameters:
+ commands:
+ - "while test ! -f /jta.done ; do echo \"Waiting for JTA to finish ... \" ; sleep 20 ; done"
+ timeout: 22100
+device_type: 'renesas-porter'
+logging_level: INFO
+job_name: '${JOB_NAME}'
+timeout: 22600
diff --git a/userdata/conf/lava/porter_nbd_snapshot.yaml b/userdata/conf/lava/porter_nbd_snapshot.yaml
new file mode 100644
index 0000000..91e5449
--- /dev/null
+++ b/userdata/conf/lava/porter_nbd_snapshot.yaml
@@ -0,0 +1,20 @@
+actions:
+ - command: deploy_linaro_kernel
+ parameters:
+ kernel: 'https://download.automotivelinux.org/AGL/snapshots/master/latest/porter-nogfx/deploy/images/porter/uImage+dtb'
+ nbdroot: 'https://download.automotivelinux.org/AGL/snapshots/master/latest/porter-nogfx/deploy/images/porter/core-image-minimal-porter.ext4'
+ ramdisk: 'https://download.automotivelinux.org/AGL/snapshots/master/latest/porter-nogfx/deploy/images/porter/initramfs-netboot-image-porter.ext4.gz.u-boot'
+ login_prompt: 'porter login:'
+ username: 'root'
+ - command: boot_linaro_image
+ parameters:
+ test_image_prompt: 'root@porter:~#'
+ - command: lava_command_run
+ parameters:
+ commands:
+ - "while test ! -f /jta.done ; do echo \"Waiting for JTA to finish ... \" ; sleep 20 ; done"
+ timeout: 22100
+device_type: 'renesas-porter'
+logging_level: INFO
+job_name: '${JOB_NAME}'
+timeout: 22600
diff --git a/userdata/conf/lava/porter_nbd_snapshot_withvar.yaml b/userdata/conf/lava/porter_nbd_snapshot_withvar.yaml
new file mode 100644
index 0000000..0658495
--- /dev/null
+++ b/userdata/conf/lava/porter_nbd_snapshot_withvar.yaml
@@ -0,0 +1,20 @@
+actions:
+ - command: deploy_linaro_kernel
+ parameters:
+ kernel: '${KERNELIMAGE}'
+ nbdroot: '${NBDIMAGE}'
+ ramdisk: '${INITRD}'
+ login_prompt: 'porter login:'
+ username: 'root'
+ - command: boot_linaro_image
+ parameters:
+ test_image_prompt: 'root@porter:~#'
+ - command: lava_command_run
+ parameters:
+ commands:
+ - "while test ! -f /jta.done ; do echo \"Waiting for JTA to finish ... \" ; sleep 20 ; done"
+ timeout: 22100
+device_type: 'renesas-porter'
+logging_level: INFO
+job_name: '${JOB_NAME}'
+timeout: 22600
diff --git a/userdata/conf/lava/upload4lava.sh b/userdata/conf/lava/upload4lava.sh
new file mode 100644
index 0000000..e55f00f
--- /dev/null
+++ b/userdata/conf/lava/upload4lava.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+#set -x
+
+if test x"" != x"$1"; then
+
+Y=$(echo "$1" | sed -e "s#\.\.##g" -e "s#/##g")
+
+curl -T "$Y" https://porter.automotivelinux.org/porter/upload/jta/$Y --insecure
+
+else
+
+echo "Help: $0 file"
+
+fi