Raspberry 4: XEN support

Added the support for DOM0 in raspberry pi4. The feature agl-virt-xen should be enabled to activate this code.

1512 MB of memory is used for DOM0 machine.

There are limitations:
	- brcmfmac module disabled (bluethooth and Wifi) due to the crash
	- total_memory is set to the 3Gb, as it impacts the USB ports on rasbperry pi4 on Linux kernel 4.19

v2 (jsmoeller): make changes in virtualization-layer conditional to raspberrypi4

Bug-AGL: SPEC-3259
Change-Id: I873fbf1f5498e31ae5f468407502f16e1873f7fc
Signed-off-by: Leonid Lazarev <leonid.lazarev@mera.com>
Signed-off-by: Jan-Simon Möller <jsmoeller@linuxfoundation.org>
Reviewed-on: https://gerrit.automotivelinux.org/gerrit/c/AGL/meta-agl/+/24522
Tested-by: Jenkins Job builder account <agl-jobbuilder@automotivelinux.org>
ci-image-build: Jenkins Job builder account <agl-jobbuilder@automotivelinux.org>
ci-image-boot-test: Jenkins Job builder account <agl-jobbuilder@automotivelinux.org>

13 files changed, 263 insertions, 3 deletions

diff --git a/meta-agl-bsp/conf/include/agl_raspberrypi4.inc b/meta-agl-bsp/conf/include/agl_raspberrypi4.inc
index b91c330f2..ae302aae1 100644
--- a/meta-agl-bsp/conf/include/agl_raspberrypi4.inc
+++ b/meta-agl-bsp/conf/include/agl_raspberrypi4.inc
@@ -17,3 +17,12 @@ IMAGE_FEATURES_remove = "splash"
 
 # Build updatable image. Only takes effect when sota.bbclass is inherited
 DISTRO_FEATURES_append = " sota"
+
+# Add xen build, if the xen feature activated
+DISTRO_FEATURES_append = " ${@bb.utils.contains('AGL_XEN_WANTED', '1', ' xen', '', d)}"
+
+# add xen-tools to build
+IMAGE_INSTALL_append = " ${@bb.utils.contains('AGL_XEN_WANTED', '1', 'xen-tools xen-mkimage-rpi4', '', d)}"
+
+# Add xen-raspberry binary to boot files
+IMAGE_BOOT_FILES_append = " ${@bb.utils.contains('AGL_XEN_WANTED','1', 'xen-${MACHINE}', '',d)}"
diff --git a/meta-agl-bsp/meta-raspberrypi/recipes-bsp/bootfiles/rpi-config_git.bbappend b/meta-agl-bsp/meta-raspberrypi/recipes-bsp/bootfiles/rpi-config_git.bbappend
index 5eecff6f1..9891fa200 100644
--- a/meta-agl-bsp/meta-raspberrypi/recipes-bsp/bootfiles/rpi-config_git.bbappend
+++ b/meta-agl-bsp/meta-raspberrypi/recipes-bsp/bootfiles/rpi-config_git.bbappend
@@ -1,4 +1,5 @@
 DISABLE_OVERSCAN = "1"
+TOTAL_BOARD_MEM = "3072"
 
 do_deploy_append_raspberrypi4() {
     # ENABLE CAN
@@ -17,6 +18,10 @@ do_deploy_append_raspberrypi4() {
                 ;;
         esac
     fi
+
+    if [ "${AGL_XEN_WANTED}" = "1" ]; then
+        echo "total_mem=${TOTAL_BOARD_MEM}" >> ${DEPLOYDIR}/bcm2835-bootfiles/config.txt
+    fi
 }
 
 do_deploy_append() {
diff --git a/meta-agl-bsp/meta-raspberrypi/recipes-bsp/rpi-u-boot-scr/files/boot_xen.cmd b/meta-agl-bsp/meta-raspberrypi/recipes-bsp/rpi-u-boot-scr/files/boot_xen.cmd
new file mode 100644
index 000000000..24cfb1658
--- /dev/null
+++ b/meta-agl-bsp/meta-raspberrypi/recipes-bsp/rpi-u-boot-scr/files/boot_xen.cmd
@@ -0,0 +1,45 @@
+#
+# SPDX-License-Identifier: MIT
+#
+# Copyright (c) 2020, MERA
+#
+# Author: Leonid Lazarev
+#
+# Xen Boot Script
+#
+# https://www.raspberrypi.org/documentation/configuration/device-tree.md
+# We do not set fdt_addr, because device tree initially is loaded by raspberry pi firmware loader and the particular
+# modification are performed. The prepared DTS is propagated to u-boot and this prepared device tree has to be reused.
+
+setenv kernel_addr_r  0x00480000  # 16M
+setenv xen_addr_r     0x00200000 # 2M
+
+# Load xen to ${xen_addr_r}.
+fatload mmc 0:1 ${xen_addr_r} /xen-@@MACHINE@@
+
+#configure dom0
+fdt addr ${fdt_addr}
+
+#read prepared bootargs, rapsberry pi prepared initial list of the parameters for loading
+fdt get value bootargs /chosen bootargs
+fdt resize 8192
+
+# add device type for raspberry
+fdt set pcie0 device_type "pci"
+
+fdt chosen
+fdt set /chosen \#address-cells <1>
+fdt set /chosen \#size-cells <1>
+
+# Load Linux Image to ${kernel_addr_r}
+fatload mmc 0:1 ${kernel_addr_r} /@@KERNEL_IMAGETYPE@@
+
+# we load dom0 with 1512 MB of memory
+fdt mknod /chosen dom0
+fdt set /chosen xen,xen-bootargs "console=dtuart dtuart=/soc/serial@7e215040 sync_console dom0_mem=1512M bootscrub=0"
+fdt set /chosen xen,dom0-bootargs "${bootargs}"
+
+fdt set /chosen/dom0 compatible "xen,linux-zimage", "xen,multiboot-module"
+fdt set /chosen/dom0 reg <${kernel_addr_r} 0x${filesize} >
+
+@@KERNEL_BOOTCMD@@ ${xen_addr_r} - ${fdt_addr}
diff --git a/meta-agl-bsp/meta-raspberrypi/recipes-bsp/rpi-u-boot-scr/rpi-u-boot-scr.bbappend b/meta-agl-bsp/meta-raspberrypi/recipes-bsp/rpi-u-boot-scr/rpi-u-boot-scr.bbappend
new file mode 100644
index 000000000..2437d9a84
--- /dev/null
+++ b/meta-agl-bsp/meta-raspberrypi/recipes-bsp/rpi-u-boot-scr/rpi-u-boot-scr.bbappend
@@ -0,0 +1,18 @@
+FILESEXTRAPATHS_prepend := "${THISDIR}/files:"
+
+SRC_URI_append =" \
+    ${@bb.utils.contains('AGL_XEN_WANTED','1',' file://boot_xen.cmd','',d)} \
+"
+
+do_compile_append() {
+
+ # if xen feature is activated we overwirte the boot script with xen specific one
+   if [ "${AGL_XEN_WANTED}" = "1" ]; then
+        sed -e 's/@@KERNEL_IMAGETYPE@@/${KERNEL_IMAGETYPE}/' \
+      	    -e 's/@@KERNEL_BOOTCMD@@/${KERNEL_BOOTCMD}/' \
+    	    -e 's/@@MACHINE@@/${MACHINE}/' \
+              "${WORKDIR}/boot_xen.cmd" > "${WORKDIR}/boot.cmd"
+
+     	mkimage -A arm -T script -C none -n "Boot script" -d "${WORKDIR}/boot.cmd" boot.scr
+   fi
+}
\ No newline at end of file
diff --git a/meta-agl-bsp/meta-raspberrypi/recipes-kernel/linux/files/0002-Disable-DMA-in-sdhci-driver.patch b/meta-agl-bsp/meta-raspberrypi/recipes-kernel/linux/files/0002-Disable-DMA-in-sdhci-driver.patch
new file mode 100644
index 000000000..04a8733f2
--- /dev/null
+++ b/meta-agl-bsp/meta-raspberrypi/recipes-kernel/linux/files/0002-Disable-DMA-in-sdhci-driver.patch
@@ -0,0 +1,25 @@
+From 758bc9b917469bc4e527dc3aae821c91cbc3d426 Mon Sep 17 00:00:00 2001
+From: Stewart Hildebrand <stewart.hildebrand@dornerworks.com>
+Date: Thu, 18 Jul 2019 00:01:27 -0400
+Subject: [PATCH 2/4] Disable DMA in sdhci driver
+
+---
+ drivers/mmc/host/sdhci-iproc.c | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/drivers/mmc/host/sdhci-iproc.c b/drivers/mmc/host/sdhci-iproc.c
+index 983cddce9..150f7bfc8 100644
+--- a/drivers/mmc/host/sdhci-iproc.c
++++ b/drivers/mmc/host/sdhci-iproc.c
+@@ -252,6 +252,8 @@ static const struct sdhci_iproc_data bcm2835_data = {
+ 
+ static const struct sdhci_pltfm_data sdhci_bcm2838_pltfm_data = {
+ 	.ops = &sdhci_iproc_32only_ops,
++	.quirks = SDHCI_QUIRK_BROKEN_DMA |
++		  SDHCI_QUIRK_BROKEN_ADMA,
+ };
+ 
+ static const struct sdhci_iproc_data bcm2838_data = {
+-- 
+2.17.1
+
diff --git a/meta-agl-bsp/meta-raspberrypi/recipes-kernel/linux/files/0003-Fix-PCIe-in-dom0-for-RPi4.patch b/meta-agl-bsp/meta-raspberrypi/recipes-kernel/linux/files/0003-Fix-PCIe-in-dom0-for-RPi4.patch
new file mode 100644
index 000000000..835f36da3
--- /dev/null
+++ b/meta-agl-bsp/meta-raspberrypi/recipes-kernel/linux/files/0003-Fix-PCIe-in-dom0-for-RPi4.patch
@@ -0,0 +1,31 @@
+From 77765c2a47ce43911b8fe7b09f037c9bc13008ca Mon Sep 17 00:00:00 2001
+From: Jeff Kubascik <jeff.kubascik@dornerworks.com>
+Date: Tue, 30 Jul 2019 08:49:56 -0400
+Subject: [PATCH 3/4] Fix PCIe in dom0 for RPi4
+
+There is an issue where the Broadcom PCIe driver and Xen swiotlb layer
+invoke each other's dma alloc function recursively until the stack blows
+up. The cause appears to be due to arch_setup_dma_ops being called
+more than once for the device, possibly through of_dma_configure, and
+screwing up the dma_ops pointers. This patch adds a check to make sure
+that the xen_dma_ops are applied only once for a device.
+---
+ arch/arm64/mm/dma-mapping.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/arch/arm64/mm/dma-mapping.c b/arch/arm64/mm/dma-mapping.c
+index d3a5bb16f..243a3b9be 100644
+--- a/arch/arm64/mm/dma-mapping.c
++++ b/arch/arm64/mm/dma-mapping.c
+@@ -895,7 +895,7 @@ void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size,
+ 	__iommu_setup_dma_ops(dev, dma_base, size, iommu);
+ 
+ #ifdef CONFIG_XEN
+-	if (xen_initial_domain()) {
++	if (!dev->archdata.dev_dma_ops && xen_initial_domain()) {
+ 		dev->archdata.dev_dma_ops = dev->dma_ops;
+ 		dev->dma_ops = xen_dma_ops;
+ 	}
+-- 
+2.17.1
+
diff --git a/meta-agl-bsp/meta-raspberrypi/recipes-kernel/linux/files/xen-be.cfg b/meta-agl-bsp/meta-raspberrypi/recipes-kernel/linux/files/xen-be.cfg
new file mode 100644
index 000000000..36f390187
--- /dev/null
+++ b/meta-agl-bsp/meta-raspberrypi/recipes-kernel/linux/files/xen-be.cfg
@@ -0,0 +1,50 @@
+# global stuff - these enable us to allow some
+# of the not so generic stuff below for xen
+CONFIG_PARAVIRT=y
+CONFIG_NET=y
+CONFIG_NET_CORE=y
+CONFIG_NETDEVICES=y
+CONFIG_BLOCK=y
+CONFIG_WATCHDOG=y
+CONFIG_TARGET_CORE=y
+CONFIG_SCSI=y
+CONFIG_FB=y
+CONFIG_INPUT_MISC=y
+CONFIG_MEMORY_HOTPLUG=y
+CONFIG_TTY=y
+# Technically not required but otherwise produces
+# pretty useless systems starting from allnoconfig
+# You want TCP/IP and ELF binaries right?
+CONFIG_INET=y
+CONFIG_BINFMT_ELF=y
+# generic config
+CONFIG_XEN=y
+CONFIG_XEN_DOM0=y
+# backend drivers
+CONFIG_XEN_BACKEND=y
+CONFIG_XEN_BLKDEV_BACKEND=m
+CONFIG_XEN_NETDEV_BACKEND=m
+CONFIG_HVC_XEN=y
+CONFIG_XEN_WDT=m
+CONFIG_XEN_SCSI_BACKEND=m
+# frontend drivers
+CONFIG_XEN_FBDEV_FRONTEND=m
+CONFIG_HVC_XEN_FRONTEND=y
+CONFIG_INPUT_XEN_KBDDEV_FRONTEND=m
+CONFIG_XEN_SCSI_FRONTEND=m
+# others
+CONFIG_XEN_BALLOON=y
+CONFIG_XEN_SCRUB_PAGES=y
+CONFIG_XEN_DEV_EVTCHN=m
+CONFIG_XEN_BLKDEV_FRONTEND=m
+CONFIG_XEN_NETDEV_FRONTEND=m
+CONFIG_XENFS=m
+CONFIG_XEN_COMPAT_XENFS=y
+CONFIG_XEN_SYS_HYPERVISOR=y
+CONFIG_XEN_XENBUS_FRONTEND=y
+CONFIG_XEN_GNTDEV=m
+CONFIG_XEN_GRANT_DEV_ALLOC=m
+CONFIG_SWIOTLB_XEN=y
+CONFIG_XEN_PRIVCMD=m
+CONFIG_DEBUG_INFO=y
+CONFIG_FRAME_POINTER=y
diff --git a/meta-agl-bsp/meta-raspberrypi/recipes-kernel/linux/linux-raspberrypi_%.bbappend b/meta-agl-bsp/meta-raspberrypi/recipes-kernel/linux/linux-raspberrypi_%.bbappend
index ff17b2532..f78e0b5eb 100644
--- a/meta-agl-bsp/meta-raspberrypi/recipes-kernel/linux/linux-raspberrypi_%.bbappend
+++ b/meta-agl-bsp/meta-raspberrypi/recipes-kernel/linux/linux-raspberrypi_%.bbappend
@@ -5,8 +5,18 @@ require recipes-kernel/linux/linux-agl.inc
 SRC_URI_append = "\
     ${@oe.utils.conditional('USE_FAYTECH_MONITOR', '1', 'file://0002-faytech-fix-rpi.patch', '', d)} \
 "
+#take in account that linux under xen should use the hvc0 console
+SERIAL_OPTION = "${@bb.utils.contains('AGL_XEN_WANTED','1','hvc0','115200;ttyS0',d)}"
+SERIAL = "${@oe.utils.conditional("ENABLE_UART", "1", "console=${SERIAL_OPTION}", "", d)}"
 
 CMDLINE_DEBUG = ""
+
+#XEN related option
+CMDLINE_append = ' ${@bb.utils.contains('AGL_XEN_WANTED','1','clk_ignore_unused','',d)}'
+
+#workaround for crash during brcmfmac loading. Disable it at this moment
+CMDLINE_append = ' ${@bb.utils.contains('AGL_XEN_WANTED','1','modprobe.blacklist=brcmfmac','',d)}'
+
 CMDLINE_append = " usbhid.mousepoll=0"
 
 # Add options to allow CMA to operate
diff --git a/meta-agl-bsp/meta-raspberrypi/recipes-kernel/linux/linux-raspberrypi_4.19.bbappend b/meta-agl-bsp/meta-raspberrypi/recipes-kernel/linux/linux-raspberrypi_4.19.bbappend
index 782d61597..7f17f0c3f 100644
--- a/meta-agl-bsp/meta-raspberrypi/recipes-kernel/linux/linux-raspberrypi_4.19.bbappend
+++ b/meta-agl-bsp/meta-raspberrypi/recipes-kernel/linux/linux-raspberrypi_4.19.bbappend
@@ -2,3 +2,19 @@ require recipes-kernel/linux/linux-agl-4.19.inc
 
 ENABLE_UART_raspberrypi4 = "1"
 
+FILESEXTRAPATHS_prepend := "${THISDIR}/files:"
+
+
+# For Xen
+
+SRC_URI_append =" \
+    ${@bb.utils.contains('AGL_XEN_WANTED','1','file://0002-Disable-DMA-in-sdhci-driver.patch','',d)} \
+"
+
+SRC_URI_append =" \
+    ${@bb.utils.contains('AGL_XEN_WANTED','1','file://0003-Fix-PCIe-in-dom0-for-RPi4.patch','',d)} \
+"
+
+SRC_URI_append = " \
+    ${@bb.utils.contains('AGL_XEN_WANTED','1','file://xen-be.cfg','',d)} \
+"
diff --git a/meta-agl-bsp/virtualization-layer/recipes-extended/xen/files/0001-XEN-on-RPi4-1GB-lmitation-workaround-XEN-tries-to-al.patch b/meta-agl-bsp/virtualization-layer/recipes-extended/xen/files/0001-XEN-on-RPi4-1GB-lmitation-workaround-XEN-tries-to-al.patch
new file mode 100644
index 000000000..d83c65257
--- /dev/null
+++ b/meta-agl-bsp/virtualization-layer/recipes-extended/xen/files/0001-XEN-on-RPi4-1GB-lmitation-workaround-XEN-tries-to-al.patch
@@ -0,0 +1,28 @@
+From 2701d65271686dff0cb39ab7da1e821f5dede61e Mon Sep 17 00:00:00 2001
+From: Leonid Lazarev <leonid.lazarev@mera.com>
+Date: Tue, 21 Apr 2020 17:28:16 +0300
+Subject: [PATCH] XEN on RPi4 1GB lmitation workaround:  XEN tries to allocated
+ the memopry for dom0  within first 4GB, but for some reason raspberry pi only
+ allows to load kernel  from 1st GB. Push XEN to allocate first bank for dom0
+ from 1st GB
+
+---
+ xen/arch/arm/domain_build.c | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/xen/arch/arm/domain_build.c b/xen/arch/arm/domain_build.c
+index dd9c3b73ba..8501a8c8aa 100644
+--- a/xen/arch/arm/domain_build.c
++++ b/xen/arch/arm/domain_build.c
+@@ -281,7 +281,7 @@ static void __init allocate_memory_11(struct domain *d,
+      */
+     while ( order >= min_low_order )
+     {
+-        for ( bits = order ; bits <= (lowmem ? 32 : PADDR_BITS); bits++ )
++        for ( bits = order ; bits <= (lowmem ? 30 : PADDR_BITS); bits++ )
+         {
+             pg = alloc_domheap_pages(d, order, MEMF_bits(bits));
+             if ( pg != NULL )
+-- 
+2.17.1
+
diff --git a/meta-agl-bsp/virtualization-layer/recipes-extended/xen/xen-mkimage.bb b/meta-agl-bsp/virtualization-layer/recipes-extended/xen/xen-mkimage-rpi4.bb
index ae8fe71b6..ae8fe71b6 100644
--- a/meta-agl-bsp/virtualization-layer/recipes-extended/xen/xen-mkimage.bb
+++ b/meta-agl-bsp/virtualization-layer/recipes-extended/xen/xen-mkimage-rpi4.bb
diff --git a/meta-agl-bsp/virtualization-layer/recipes-extended/xen/xen-tools_git.bbappend b/meta-agl-bsp/virtualization-layer/recipes-extended/xen/xen-tools_git.bbappend
new file mode 100644
index 000000000..057f4564c
--- /dev/null
+++ b/meta-agl-bsp/virtualization-layer/recipes-extended/xen/xen-tools_git.bbappend
@@ -0,0 +1,11 @@
+
+# make the package specific to the machine for now
+PACKAGE_ARCH = "${MACHINE_ARCH}"
+
+# rpi4
+LIC_FILES_CHKSUM_raspberrypi4 = "file://COPYING;md5=4295d895d4b5ce9d070263d52f030e49"
+XEN_REL_raspberrypi4 = "4.13"
+SRCREV_raspberrypi4 = "721f2c323ca55c77857c93e7275b4a93a0e15e1f"
+SRC_URI_raspberrypi4 = " \
+    git://xenbits.xen.org/xen.git;branch=${XEN_BRANCH} \
+    "
diff --git a/meta-agl-bsp/virtualization-layer/recipes-extended/xen/xen_git.bbappend b/meta-agl-bsp/virtualization-layer/recipes-extended/xen/xen_git.bbappend
index 70c7fd415..1b9aaf4ca 100644
--- a/meta-agl-bsp/virtualization-layer/recipes-extended/xen/xen_git.bbappend
+++ b/meta-agl-bsp/virtualization-layer/recipes-extended/xen/xen_git.bbappend
@@ -1,6 +1,18 @@
 FILESEXTRAPATHS_prepend := "${THISDIR}/files:"
 
-XEN_REL = "4.13"
-XEN_BRANCH = "stable"
-SRCREV = "721f2c323ca55c77857c93e7275b4a93a0e15e1f"
+# make the package machine-specific
+PACKAGE_ARCH = "${MACHINE_ARCH}"
 
+# rpi4 specifics
+LIC_FILES_CHKSUM_raspberrypi4 = "file://COPYING;md5=4295d895d4b5ce9d070263d52f030e49"
+XEN_REL_raspberrypi4 = "4.13"
+
+SRCREV_raspberrypi4 = "721f2c323ca55c77857c93e7275b4a93a0e15e1f"
+SRC_URI_append_raspberrypi4 = " file://0001-XEN-on-RPi4-1GB-lmitation-workaround-XEN-tries-to-al.patch"
+
+#due to incorrect xen binary preparation in external library, we add additional deploy
+do_deploy_append_raspberrypi4() {
+    if [ -f ${B}/xen/xen ]; then
+        install -m 0644 ${B}/xen/xen ${DEPLOYDIR}/xen-${MACHINE}
+    fi
+}