diff options
| -rw-r--r-- | Makefile | 7 | ||||
| -rw-r--r-- | README.md | 14 | ||||
| -rw-r--r-- | loopback_driver.c | 1148 | ||||
| -rw-r--r-- | loopback_driver.h | 516 | ||||
| -rw-r--r-- | virtio_loopback_device.c | 902 | ||||
| -rw-r--r-- | virtio_loopback_driver.c | 629 | ||||
| -rw-r--r-- | virtio_loopback_driver.h | 253 |
7 files changed, 1797 insertions, 1672 deletions
@@ -15,16 +15,17 @@ # Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # -NAME_C=loopback_driver - LINUX_DIR ?= /lib/modules/$(shell uname -r)/build -obj-m += $(NAME_C).o +virtio_loopback-objs := virtio_loopback_driver.o virtio_loopback_device.o +obj-m += virtio_loopback.o ifeq ($(DEBUG), 1) ccflags-y += -DDEBUG endif +CFLAGS := -Wall -Wextra -Werror + all: make -C $(LINUX_DIR) M=$(PWD) modules @@ -1,16 +1,20 @@ # virtio-loopback transport repository -This repository includes the beta version of the "virtio_loopback_transport" driver which is part of the Virtio Loopback Design presented in this [document](https://git.virtualopensystems.com/virtio-loopback/docs/-/blob/master/design_docs). This work carried on by Virtual Open Systems in the [Automotive Grade Linux](https://www.automotivegradelinux.org) community. +This repository includes the latest version of the "virtio_loopback_transport" driver which is part of the Virtio Loopback Design presented in this [document](https://git.virtualopensystems.com/virtio-loopback/docs/-/blob/master/design_docs). This work carried on by Virtual Open Systems in the [Automotive Grade Linux](https://www.automotivegradelinux.org) community. -As described in the design document, the transport is only a part of a more complex architecture. If you want to see the implementation and build the other components, refer to the [virtio-loopback docs repository](https://git.virtualopensystems.com/virtio-loopback/docs/-/tree/beta-release). +As described in the design document, the transport is only a part of a more complex architecture. If you want to see the implementation and build the other components, refer to the [virtio-loopback docs repository](https://git.virtualopensystems.com/virtio-loopback/docs/-/tree/epsilon-release). ## Build the virtio-loopback transport In order to build this project the next commands need to be used: - `make` for x86 - `make ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu-` for arm64 +- `make ARCH=riscv CROSS_COMPILE=riscv64-linux-gnu-` for riscv64 -**NOTE**: The `DEBUG=1` can be used in order to enable the driver's logs. - -This driver is tested with Linux v5.10 and it is NOT be compatible with newer versions ("fcheck_files" function is replaced by "files_lookup_fd_rcu". +**NOTE**: The `DEBUG=1` can be used in order to enable the driver's debug logs. +## Tested platforms +The driver has been tested with the following platforms (sorted by architecture): +- x86: QEMU (machine `pc`), Thinkpad e14 gen3, x86 servers etc. +- aarch64: QEMU (machine `virt`), Raspberry PI 4, AGL reference HW board (Rcar-H3) +- riscv64: LicheePi4A diff --git a/loopback_driver.c b/loopback_driver.c deleted file mode 100644 index 989b865..0000000 --- a/loopback_driver.c +++ /dev/null @@ -1,1148 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Based on virtio_mmio.c - * Copyright 2011-2014, ARM Ltd. - * - * Copyright 2022-2024 Virtual Open Systems SAS - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - */ - -#define pr_fmt(fmt) "virtio-loopback: " fmt - -#include <linux/cdev.h> -#include <linux/eventfd.h> -#include <linux/fdtable.h> -#include <linux/init.h> -#include <linux/module.h> -#include <linux/platform_device.h> -#include <linux/kernel.h> -#include <linux/device.h> -#include <linux/version.h> -#include <linux/slab.h> - -/* Virtio-loopback includes */ -#include <linux/dma-mapping.h> -#include <linux/list.h> -#include <linux/spinlock.h> -#include <linux/mutex.h> -#include <linux/virtio.h> -#include <linux/virtio_config.h> -#include <uapi/linux/virtio_mmio.h> -#include <linux/virtio_ring.h> -#include <linux/pid.h> -#include <linux/kthread.h> - -/* Loopback header file */ -#include "loopback_driver.h" - -/* Features */ -MODULE_LICENSE("GPL v2"); - -/* function declaration */ -static uint64_t read_adapter(uint64_t fn_id, uint64_t size); -static void write_adapter(uint64_t data, uint64_t fn_id, uint64_t size); - -struct mmap_info *info; -int mmap_index; -uint64_t sum_pgfaults; - -/* Waitq */ -wait_queue_head_t wq; -wait_queue_head_t wq_notify; - -/* Read write mutex */ -struct mutex read_write_lock; -struct mutex interrupt_lock; - -/* Notification spinlock */ -spinlock_t notify_q_spinlock; - -bool share_communication_struct; -uint32_t vq_index; -uint64_t vq_pfns[16], vq_pfn; - -struct virtio_mmio_device *vm_dev_irq; - -struct virtqueue *global_vq; -const struct vring *global_vring; - -/* counters */ -static int interrupt_cnt; -static int notify_sent, notify_received; - -/* Define a notification list */ -static struct list_head *notify_list; - -static struct platform_device *virtio_loopback_device; - -/* Virio-loopback device funcitonality */ -struct eventfd_ctx *efd_ctx; -static struct task_struct *start_loopback_thread; -static struct task_struct *start_notification_thread; - -/* global storage for device Major number */ -static int dev_major; -/* sysfs class structure */ -static struct class *loopback_class; -/* array of loopback_device_data for */ -static struct loopback_device_data *loopback_data; - -/* Allow only one process to open the driver */ -unsigned long loopback_flags; - -/* Current ram index */ -int cur_ram_idx; - -/* - * If this variable is true then read/write should wait - * the adapter to unlock this opertion by sending an - * eventfd. If it's equal to "false" then the oparetion - * does not wait for adapter's confirmation. - */ -bool valid_eventfd; - -/* Create a mapping array */ -struct share_mmap share_mmap_list[MMAP_LIMIT]; - -/* Configuration interface */ - -static u64 vm_get_features(struct virtio_device *vdev) -{ - u64 features; - - /* Take feature bits 0-31 */ - write_adapter(1, VIRTIO_MMIO_DEVICE_FEATURES_SEL, 4); - features = read_adapter(VIRTIO_MMIO_DEVICE_FEATURES, 4); - features <<= 32; - - /* Take feature bits 32-63 */ - write_adapter(0, VIRTIO_MMIO_DEVICE_FEATURES_SEL, 4); - features |= read_adapter(VIRTIO_MMIO_DEVICE_FEATURES, 4); - - return features; -} - -static int vm_finalize_features(struct virtio_device *vdev) -{ - struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); - - /* Give virtio_ring a chance to accept features. */ - vring_transport_features(vdev); - - /* Make sure there are no mixed devices */ - if (vm_dev->version == 2 && - !__virtio_test_bit(vdev, VIRTIO_F_VERSION_1)) { - dev_err(&vdev->dev, "New virtio-mmio devices (version 2) " - "must provide VIRTIO_F_VERSION_1 feature!\n"); - return -EINVAL; - } - - write_adapter(1, VIRTIO_MMIO_DRIVER_FEATURES_SEL, 4); - write_adapter((u32)(vdev->features >> 32), VIRTIO_MMIO_DRIVER_FEATURES, 4); - - write_adapter(0, VIRTIO_MMIO_DRIVER_FEATURES_SEL, 4); - write_adapter((u32)vdev->features, VIRTIO_MMIO_DRIVER_FEATURES, 4); - - return 0; -} - -static void vm_get(struct virtio_device *vdev, unsigned int offset, - void *buf, unsigned int len) -{ - struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); - u8 b; - __le16 w; - __le32 l; - - if (vm_dev->version == 1) { - u8 *ptr = buf; - int i; - - for (i = 0; i < len; i++) - ptr[i] = read_adapter(VIRTIO_MMIO_CONFIG + offset + i, 1); - return; - } - - switch (len) { - case 1: - b = read_adapter(VIRTIO_MMIO_CONFIG + offset, 1); - memcpy(buf, &b, sizeof(b)); - break; - case 2: - w = cpu_to_le16(read_adapter(VIRTIO_MMIO_CONFIG + offset, 2)); - memcpy(buf, &w, sizeof(w)); - break; - case 4: - l = cpu_to_le32(read_adapter(VIRTIO_MMIO_CONFIG + offset, 4)); - memcpy(buf, &l, sizeof(l)); - break; - case 8: - l = cpu_to_le32(read_adapter(VIRTIO_MMIO_CONFIG + offset, 4)); - memcpy(buf, &l, sizeof(l)); - l = cpu_to_le32(read_adapter(VIRTIO_MMIO_CONFIG + offset + sizeof(l), 4)); - memcpy(buf + sizeof(l), &l, sizeof(l)); - break; - default: - BUG(); - } -} - -static void vm_set(struct virtio_device *vdev, unsigned int offset, - const void *buf, unsigned int len) -{ - struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); - u8 b; - __le16 w; - __le32 l; - - if (vm_dev->version == 1) { - const u8 *ptr = buf; - int i; - - for (i = 0; i < len; i++) - write_adapter(ptr[i], VIRTIO_MMIO_CONFIG + offset + i, 1); - - return; - } - - switch (len) { - case 1: - memcpy(&b, buf, sizeof(b)); - write_adapter(b, VIRTIO_MMIO_CONFIG + offset, 1); - break; - case 2: - memcpy(&w, buf, sizeof(w)); - write_adapter(le16_to_cpu(w), VIRTIO_MMIO_CONFIG + offset, 2); - break; - case 4: - memcpy(&l, buf, sizeof(l)); - write_adapter(le32_to_cpu(l), VIRTIO_MMIO_CONFIG + offset, 4); - break; - case 8: - memcpy(&l, buf, sizeof(l)); - write_adapter(le32_to_cpu(l), VIRTIO_MMIO_CONFIG + offset, 4); - memcpy(&l, buf + sizeof(l), sizeof(l)); - write_adapter(le32_to_cpu(l), VIRTIO_MMIO_CONFIG + offset + sizeof(l), 4); - break; - default: - BUG(); - } -} - -static u32 vm_generation(struct virtio_device *vdev) -{ - struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); - - if (vm_dev->version == 1) - return 0; - else - return read_adapter(VIRTIO_MMIO_CONFIG_GENERATION, 4); -} - -static u8 vm_get_status(struct virtio_device *vdev) -{ - return read_adapter(VIRTIO_MMIO_STATUS, 4) & 0xff; -} - -static void vm_set_status(struct virtio_device *vdev, u8 status) -{ - write_adapter(status, VIRTIO_MMIO_STATUS, 4); -} - -static void vm_reset(struct virtio_device *vdev) -{ - /* 0 status means a reset. */ - write_adapter(0, VIRTIO_MMIO_STATUS, 4); -} - -int start_notification(void *data) -{ - struct notify_data *first_notification; - uint32_t index; - - DBG("Start notification\n"); - - (void)data; - - while(1) { - - spin_lock(¬ify_q_spinlock); - while (valid_eventfd && list_empty(notify_list) == 1) { - spin_unlock(¬ify_q_spinlock); - wait_event_timeout(wq_notify, list_empty(notify_list) != 1, 1 * HZ); - spin_lock(¬ify_q_spinlock); - } - - first_notification = list_first_entry(notify_list, struct notify_data, list); - index = first_notification->index; - list_del(&first_notification->list); - - DBG("notify_received: %d, VQ: %d\n", notify_received++, index); - spin_unlock(¬ify_q_spinlock); - write_adapter(index, VIRTIO_MMIO_QUEUE_NOTIFY, 4); - - if (!valid_eventfd) { - DBG("Exit notification thread\n"); - return 0; - } - } -} - -/* the notify function used when creating a virt queue */ -static bool vm_notify(struct virtqueue *vq) -{ - struct notify_data *data; - /* - * We write the queue's selector into - * the notification register to signal - * the other end - */ - spin_lock(¬ify_q_spinlock); - DBG("vm_notify\n"); - - data = kmalloc(sizeof(struct notify_data), GFP_KERNEL); - data->index = vq->index; - INIT_LIST_HEAD(&data->list); - list_add_tail(&data->list, notify_list); - spin_unlock(¬ify_q_spinlock); - - wake_up(&wq_notify); - - return true; -} - -/* Notify all virtqueues on an interrupt. */ -static void vm_interrupt(struct work_struct *work) -{ - struct virtio_mmio_device *vm_dev = vm_dev_irq; - struct virtio_mmio_vq_info *info; - int irq = 44; - unsigned long status; - - /* STATUS and ACK should ne done without any intermediate status change */ - mutex_lock(&interrupt_lock); - DBG("interrupt_cnt: %d\n", interrupt_cnt++); - - /* Read and acknowledge interrupts */ - status = read_adapter(VIRTIO_MMIO_INTERRUPT_STATUS, 4); - write_adapter(status, VIRTIO_MMIO_INTERRUPT_ACK, 4); - - if (unlikely(status & VIRTIO_MMIO_INT_CONFIG)) - virtio_config_changed(&vm_dev->vdev); - - if (likely(status & VIRTIO_MMIO_INT_VRING)) { - spin_lock(&vm_dev->lock); - list_for_each_entry(info, &vm_dev->virtqueues, node) { - (void)vring_interrupt(irq, info->vq); - } - spin_unlock(&vm_dev->lock); - } - mutex_unlock(&interrupt_lock); -} - -static void vm_del_vq(struct virtqueue *vq) -{ - struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vq->vdev); - struct virtio_mmio_vq_info *info = vq->priv; - unsigned long flags; - unsigned int index = vq->index; - - spin_lock_irqsave(&vm_dev->lock, flags); - list_del(&info->node); - spin_unlock_irqrestore(&vm_dev->lock, flags); - - /* Select and deactivate the queue */ - write_adapter(index, VIRTIO_MMIO_QUEUE_SEL, 4); - - if (vm_dev->version == 1) { - write_adapter(0, VIRTIO_MMIO_QUEUE_PFN, 4); - } else { - write_adapter(0, VIRTIO_MMIO_QUEUE_READY, 4); - WARN_ON(read_adapter(VIRTIO_MMIO_QUEUE_READY, 4)); - } - - vring_del_virtqueue(vq); - kfree(info); -} - -static void vm_del_vqs(struct virtio_device *vdev) -{ - struct virtqueue *vq, *n; - - list_for_each_entry_safe(vq, n, &vdev->vqs, list) - vm_del_vq(vq); -} - -static struct virtqueue *vm_setup_vq(struct virtio_device *vdev, unsigned int index, - void (*callback)(struct virtqueue *vq), - const char *name, bool ctx) -{ - struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); - struct virtio_mmio_vq_info *info; - struct virtqueue *vq; - unsigned long flags; - unsigned int num; - int err; - - if (!name) - return NULL; - - /* Select the queue we're interested in */ - write_adapter(index, VIRTIO_MMIO_QUEUE_SEL, 4); - - /* Queue shouldn't already be set up. */ - if (read_adapter((vm_dev->version == 1 ? - VIRTIO_MMIO_QUEUE_PFN : VIRTIO_MMIO_QUEUE_READY), 4)) { - err = -ENOENT; - goto error_available; - } - - /* Allocate and fill out our active queue description */ - info = kmalloc(sizeof(*info), GFP_KERNEL); - if (!info) { - err = -ENOMEM; - goto error_kmalloc; - } - - num = read_adapter(VIRTIO_MMIO_QUEUE_NUM_MAX, 4); - if (num == 0) { - err = -ENOENT; - goto error_new_virtqueue; - } - - /* Create the vring */ - vq = vring_create_virtqueue(index, num, VIRTIO_MMIO_VRING_ALIGN, vdev, - true, true, ctx, vm_notify, callback, name); - if (!vq) { - err = -ENOMEM; - goto error_new_virtqueue; - } - - -#if LINUX_VERSION_CODE > KERNEL_VERSION(6,0,0) - vq->num_max = num; -#endif - /* Activate the queue */ - write_adapter(virtqueue_get_vring_size(vq), VIRTIO_MMIO_QUEUE_NUM, 4); - if (vm_dev->version == 1) { - u64 q_pfn = virtqueue_get_desc_addr(vq); - - q_pfn = q_pfn >> PAGE_SHIFT; - - /* Copy the physical address and enable the mmap */ - vq_pfn = q_pfn; - vq_pfns[vq_index++] = q_pfn; - - /* Save the virtqueue in a global variable */ - global_vq = vq; - global_vring = virtqueue_get_vring(vq); - - /* - * virtio-mmio v1 uses a 32bit QUEUE PFN. If we have something - * that doesn't fit in 32bit, fail the setup rather than - * pretending to be successful. - */ - if (q_pfn >> 32) { - dev_err(&vdev->dev, - "platform bug: legacy virtio-mmio must not " - "be used with RAM above 0x%llxGB\n", - 0x1ULL << (32 + PAGE_SHIFT - 30)); - err = -E2BIG; - goto error_bad_pfn; - } - - write_adapter(PAGE_SIZE, VIRTIO_MMIO_QUEUE_ALIGN, 4); - write_adapter(q_pfn, VIRTIO_MMIO_QUEUE_PFN, 4); - } else { - u64 addr; - - addr = virtqueue_get_desc_addr(vq); - write_adapter((u32)addr, VIRTIO_MMIO_QUEUE_DESC_LOW, 4); - write_adapter((u32)(addr >> 32), VIRTIO_MMIO_QUEUE_DESC_HIGH, 4); - - addr = virtqueue_get_avail_addr(vq); - write_adapter((u32)addr, VIRTIO_MMIO_QUEUE_AVAIL_LOW, 4); - write_adapter((u32)(addr >> 32), VIRTIO_MMIO_QUEUE_AVAIL_HIGH, 4); - - addr = virtqueue_get_used_addr(vq); - write_adapter((u32)addr, VIRTIO_MMIO_QUEUE_USED_LOW, 4); - write_adapter((u32)(addr >> 32), VIRTIO_MMIO_QUEUE_USED_HIGH, 4); - - write_adapter(1, VIRTIO_MMIO_QUEUE_READY, 4); - } - - vq->priv = info; - info->vq = vq; - - spin_lock_irqsave(&vm_dev->lock, flags); - list_add(&info->node, &vm_dev->virtqueues); - spin_unlock_irqrestore(&vm_dev->lock, flags); - - return vq; - -error_bad_pfn: - vring_del_virtqueue(vq); -error_new_virtqueue: - if (vm_dev->version == 1) { - write_adapter(0, VIRTIO_MMIO_QUEUE_PFN, 4); - } else { - write_adapter(0, VIRTIO_MMIO_QUEUE_READY, 4); - WARN_ON(read_adapter(VIRTIO_MMIO_QUEUE_READY, 4)); - } - kfree(info); -error_kmalloc: -error_available: - return ERR_PTR(err); -} - -static int vm_find_vqs(struct virtio_device *vdev, unsigned int nvqs, - struct virtqueue *vqs[], - vq_callback_t *callbacks[], - const char * const names[], - const bool *ctx, - struct irq_affinity *desc) -{ - int i, queue_idx = 0; - - for (i = 0; i < nvqs; ++i) { - if (!names[i]) { - vqs[i] = NULL; - continue; - } - - vqs[i] = vm_setup_vq(vdev, queue_idx++, callbacks[i], names[i], - ctx ? ctx[i] : false); - if (IS_ERR(vqs[i])) { - vm_del_vqs(vdev); - return PTR_ERR(vqs[i]); - } - } - - return 0; -} - -static const char *vm_bus_name(struct virtio_device *vdev) -{ - struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); - - return vm_dev->pdev->name; -} - -static bool vm_get_shm_region(struct virtio_device *vdev, - struct virtio_shm_region *region, u8 id) -{ - u64 len, addr; - - /* Select the region we're interested in */ - write_adapter(id, VIRTIO_MMIO_SHM_SEL, 4); - - /* Read the region size */ - len = (u64) read_adapter(VIRTIO_MMIO_SHM_LEN_LOW, 4); - len |= (u64) read_adapter(VIRTIO_MMIO_SHM_LEN_HIGH, 4) << 32; - - region->len = len; - - /* Check if region length is -1. If that's the case, the shared memory - * region does not exist and there is no need to proceed further. - */ - if (len == ~(u64)0) - return false; - - /* Read the region base address */ - addr = (u64) read_adapter(VIRTIO_MMIO_SHM_BASE_LOW, 4); - addr |= (u64) read_adapter(VIRTIO_MMIO_SHM_BASE_HIGH, 4) << 32; - - region->addr = addr; - - return true; -} - -static const struct virtio_config_ops virtio_mmio_config_ops = { - .get = vm_get, - .set = vm_set, - .generation = vm_generation, - .get_status = vm_get_status, - .set_status = vm_set_status, - .reset = vm_reset, - .find_vqs = vm_find_vqs, - .del_vqs = vm_del_vqs, - .get_features = vm_get_features, - .finalize_features = vm_finalize_features, - .bus_name = vm_bus_name, - .get_shm_region = vm_get_shm_region, -}; - -static void virtio_mmio_release_dev(struct device *_d) -{ - struct virtio_device *vdev = container_of(_d, struct virtio_device, dev); - struct virtio_mmio_device *vm_dev = to_virtio_mmio_device(vdev); - struct platform_device *pdev = vm_dev->pdev; - - devm_kfree(&pdev->dev, vm_dev); -} - -static int virtio_mmio_probe(struct platform_device *pdev) -{ - struct virtio_mmio_device *vm_dev; - unsigned long magic; - int rc; - - vm_dev = devm_kzalloc(&pdev->dev, sizeof(*vm_dev), GFP_KERNEL); - if (!vm_dev) - return -ENOMEM; - - /* Save the device pointer globally */ - vm_dev_irq = vm_dev; - - vm_dev->vdev.dev.parent = &pdev->dev; - vm_dev->vdev.dev.release = virtio_mmio_release_dev; - vm_dev->vdev.config = &virtio_mmio_config_ops; - vm_dev->pdev = pdev; - INIT_LIST_HEAD(&vm_dev->virtqueues); - spin_lock_init(&vm_dev->lock); - - /* Check magic value */ - magic = read_adapter(VIRTIO_MMIO_MAGIC_VALUE, 4); - - if (magic != ('v' | 'i' << 8 | 'r' << 16 | 't' << 24)) { - dev_warn(&pdev->dev, "Wrong magic value 0x%08lx!\n", magic); - return -ENODEV; - } - - /* Check device version */ - vm_dev->version = read_adapter(VIRTIO_MMIO_VERSION, 4); - - if (vm_dev->version < 1 || vm_dev->version > 2) { - dev_err(&pdev->dev, "Version %ld not supported!\n", - vm_dev->version); - return -ENXIO; - } - - vm_dev->vdev.id.device = read_adapter(VIRTIO_MMIO_DEVICE_ID, 4); - - if (vm_dev->vdev.id.device == 0) { - /* - * virtio-mmio device with an ID 0 is a (dummy) placeholder - * with no function. End probing now with no error reported. - */ - return -ENODEV; - } - - vm_dev->vdev.id.vendor = read_adapter(VIRTIO_MMIO_VENDOR_ID, 4); - - if (vm_dev->version == 1) { - write_adapter(PAGE_SIZE, VIRTIO_MMIO_GUEST_PAGE_SIZE, 4); - - rc = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); - /* - * In the legacy case, ensure our coherently-allocated virtio - * ring will be at an address expressable as a 32-bit PFN. - */ - if (!rc) - dma_set_coherent_mask(&pdev->dev, - DMA_BIT_MASK(32 + PAGE_SHIFT)); - } else { - rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); - } - if (rc) - rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); - if (rc) - dev_warn(&pdev->dev, "Failed to enable 64-bit or 32-bit DMA." - "Trying to continue, but this might not work.\n"); - - platform_set_drvdata(pdev, vm_dev); - - rc = register_virtio_device(&vm_dev->vdev); - - if (rc) - put_device(&vm_dev->vdev.dev); - - return rc; -} - -static int virtio_mmio_remove(struct platform_device *pdev) -{ - struct virtio_mmio_device *vm_dev = platform_get_drvdata(pdev); - - unregister_virtio_device(&vm_dev->vdev); - DBG("unregister_virtio_device!\n"); - return 0; -} - -/* Not need of DTS and ACPI */ -static struct platform_driver virtio_mmio_driver = { - .probe = virtio_mmio_probe, - .remove = virtio_mmio_remove, - .driver = { - .name = "loopback-transport", - }, -}; - -void pf_mmap_close(struct vm_area_struct *vma) -{ - DBG("unmap\t-> vma->vm_start: 0x%lx\n", vma->vm_start); - DBG("unmap\t-> size: %lu\n", vma->vm_end - vma->vm_start); - share_mmap_rem(vma, share_mmap_list); -} - -vm_fault_t pf_mmap_fault(struct vm_fault *vmf) -{ - uint64_t corrected_pfn; - pfn_t corr_pfn_struct; - struct page *page; - int ret = 0; - - DBG("----- Page fault: %lld -----\n", sum_pgfaults++); - - /* Find the corrected pfn */ - corrected_pfn = share_mmap_exist_vma_return_correct_pfn(vmf->address, share_mmap_list); - corr_pfn_struct.val = corrected_pfn; - - /* Some debug prints */ - DBG("vma->vm_start: 0x%lx\n", vmf->vma->vm_start); - DBG("vma->vm_pgoff: 0x%lx\n", vmf->vma->vm_pgoff); - DBG("vmf->address: 0x%lx\n", vmf->address); - DBG("corrected_pfn: 0x%llx\n", corrected_pfn); - DBG("pfn_valid(corrected_pfn): 0x%x\n", pfn_valid(corrected_pfn)); - - BUG_ON(!pfn_valid(corrected_pfn)); - - /* After finding the page, correct the vmf->page */ - page = pfn_to_page(corrected_pfn); - BUG_ON(!virt_addr_valid(page_address(page))); - - /* Insert the correct page */ - ret = vmf_insert_pfn(vmf->vma, vmf->address, corrected_pfn); - DBG("vmf_insert_pfn -> ret: %d\n", ret); - - return ret; -} - -const struct vm_operations_struct pf_mmap_ops = { - .close = pf_mmap_close, - .fault = pf_mmap_fault, -}; - -int pf_mmap_vm_page(struct file *filp, struct vm_area_struct *vma) -{ - uint64_t size = (unsigned long)(vma->vm_end - vma->vm_start); - uint64_t pfn = ((cur_ram_idx++) * 0x40000); - -#if LINUX_VERSION_CODE < KERNEL_VERSION(6,3,0) - vma->vm_flags |= VM_PFNMAP; -#else - vm_flags_set(vma, VM_PFNMAP); -#endif - add_share_mmap(filp, pfn, vma->vm_start, size, share_mmap_list, &mmap_index); - return 0; -} - -int op_mmap(struct file *filp, struct vm_area_struct *vma) -{ - int ret = 0; - uint64_t size = (unsigned long)(vma->vm_end - vma->vm_start); - - DBG("op_mmap -> vma->vm_pgoff: 0x%lx", vma->vm_pgoff); - - if (share_communication_struct) { - DBG("MMAP communication struct\n"); - ret = mmap_communication_shared_space(filp, vma, share_mmap_list, &mmap_index); - share_communication_struct = false; - goto out; - } - - if (size > PAGE_SIZE * 100) { - ret = pf_mmap_vm_page(filp, vma); - goto out; - } - - ret = mmap_mix(filp, vma, share_mmap_list, &mmap_index, vq_pfn); - -out: - vma->vm_ops = &pf_mmap_ops; - return ret; -} - -static uint64_t read_adapter(uint64_t fn_id, uint64_t size) -{ - uint64_t result; - - mutex_lock(&read_write_lock); - - /* - * By enabling the following line all - * read messages will be printed: - * - * print_neg_flag(fn_id, 1); - */ - print_neg_flag(fn_id, 1); - - ((virtio_neg_t *)(info->data))->notification = fn_id; - ((virtio_neg_t *)(info->data))->data = 0; - ((virtio_neg_t *)(info->data))->size = size; - ((virtio_neg_t *)(info->data))->read = true; - - atomic_set(&((virtio_neg_t *)(info->data))->done, 0); - - eventfd_signal(efd_ctx, 1); - - /* - * There is a chance virtio-loopback adapter to call "wake_up" - * before the current thread sleep. This is the reason that - * "wait_event_timeout" is used instead of "wait_event". In this - * way, virtio-loopback driver will wake up even if has missed the - * "wake_up" kick, check the updated "done" value and return. - */ - - while (valid_eventfd && atomic_read(&((virtio_neg_t *)(info->data))->done) != 1) - wait_event_timeout(wq, atomic_read(&((virtio_neg_t *)(info->data))->done) == 1, 1 * HZ); - - result = ((virtio_neg_t *)(info->data))->data; - - mutex_unlock(&read_write_lock); - - return result; -} - -static void write_adapter(uint64_t data, uint64_t fn_id, uint64_t size) -{ - - mutex_lock(&read_write_lock); - - /* - * By enabling the following line all - * write messages will be printed: - * - * print_neg_flag(fn_id, 1); - */ - print_neg_flag(fn_id, 0); - - ((virtio_neg_t *)(info->data))->notification = fn_id; - ((virtio_neg_t *)(info->data))->data = data; - ((virtio_neg_t *)(info->data))->size = size; - ((virtio_neg_t *)(info->data))->read = false; - - atomic_set(&((virtio_neg_t *)(info->data))->done, 0); - - eventfd_signal(efd_ctx, 1); - - /* - * There is a chance virtio-loopback adapter to call "wake_up" - * before the current thread sleep. This is the reason that - * "wait_event_timeout" is used instead of "wait_event". In this - * way, virtio-loopback driver will wake up even if has missed the - * "wake_up" kick, check the updated "done" value and return. - */ - while (valid_eventfd && atomic_read(&((virtio_neg_t *)(info->data))->done) != 1) - wait_event_timeout(wq, atomic_read(&((virtio_neg_t *)(info->data))->done) == 1, 1 * HZ); - - mutex_unlock(&read_write_lock); -} - -/* Defined for future work */ -static ssize_t loopback_write(struct file *file, - const char __user *user_buffer, - size_t size, - loff_t *offset) -{ - ssize_t len = sizeof(int); - - DBG("loopback write function is called\n"); - if (len <= 0) - return 0; - - return len; -} - -/* Defined for future work */ -static ssize_t loopback_read(struct file *file, - char __user *user_buffer, - size_t size, loff_t *offset) -{ - DBG("loopback read function is called\n"); - return 0; -} - -loff_t loopback_seek(struct file *file, loff_t offset, int whence) -{ - loff_t new_pos; - - DBG("loopback seek function!\n"); - switch (whence) { - case SEEK_SET: - new_pos = offset; - break; - case SEEK_CUR: - new_pos = file->f_pos + offset; - break; - case SEEK_END: - new_pos = file->f_inode->i_size; - break; - default: - return -EINVAL; - } - - if (new_pos < 0 || new_pos > file->f_inode->i_size) - return -EINVAL; - - return new_pos; -} - -int start_loopback(void *data) -{ - (void)data; - /* Register mmio_trasmport */ - (void)platform_driver_register(&virtio_mmio_driver); - return 0; -} - -static long loopback_ioctl(struct file *file, - unsigned int cmd, unsigned long arg) -{ - efd_data_t efd_data; - struct task_struct *userspace_task; - struct file *efd_file; - int irq; - uint32_t queue_sel; - - switch (cmd) { - case EFD_INIT: - if (copy_from_user(&efd_data, (efd_data_t *) arg, - sizeof(efd_data_t))) - return -EFAULT; - - userspace_task = pid_task(find_vpid(efd_data.pid), PIDTYPE_PID); - - rcu_read_lock(); - -#if LINUX_VERSION_CODE < KERNEL_VERSION(5,11,220) - efd_file = fcheck_files(userspace_task->files, efd_data.efd[0]); -#else - efd_file = files_lookup_fd_rcu(userspace_task->files, efd_data.efd[0]); -#endif - - rcu_read_unlock(); - - efd_ctx = eventfd_ctx_fileget(efd_file); - if (!efd_ctx) - return -1; - - break; - case WAKEUP: - atomic_set(&((virtio_neg_t *)(info->data))->done, 1); - wake_up(&wq); - break; - case START_LOOPBACK: - start_notification_thread = kthread_run(start_notification, NULL, "start_notification"); - start_loopback_thread = kthread_run(start_loopback, NULL, "start_loopback"); - break; - case IRQ: - if (copy_from_user(&irq, (int *) arg, sizeof(int))) - return -EFAULT; - DBG("\nIRQ\n"); - /* - * Both of the interrupt ways work but a) is more stable - * and b) has better performance: - * a) vm_interrupt(NULL); - * b) queue_work(interrupt_workqueue, &async_interrupt); - */ - vm_interrupt(NULL); - break; - case SHARE_VQS: - if (copy_from_user(&queue_sel, (uint32_t *) arg, sizeof(uint32_t))) - return -EFAULT; - DBG("\n\nSHARE_VQS: %u\n\n", queue_sel); - vq_pfn = vq_pfns[queue_sel]; - DBG("Selected pfn is: 0x%llx", vq_pfn); - break; - case SHARE_COM_STRUCT: - share_communication_struct = true; - break; - default: - DBG("loopback ioctl: default, %u\n", cmd); - return -ENOTTY; - } - - return 0; -} - -/* - * The current implementation of the driver supports - * exclusive access to one user-space thread. Multi-device - * support will be added in future implementation. - */ -static int loopback_open(struct inode *inode, struct file *file) -{ - uint32_t val_1gb = 1024 * 1024 * 1024; // 1GB - virtio_neg_t device_neg = {.done = ATOMIC_INIT(0)}; - - /* Update the global variable, the driver is in use */ - if (test_and_set_bit(IN_USE_BIT, &loopback_flags)) { - DBG("Driver is busy\n"); - return -EBUSY; - } - - /* Set the i_size for the stat SYS_CALL*/ - file->f_inode->i_size = 10 * val_1gb; - - /* Init mmap funcitonality */ - info = kmalloc(sizeof(struct mmap_info), GFP_KERNEL); - info->data = (void *)get_zeroed_page(GFP_KERNEL); - memcpy(info->data, &device_neg, sizeof(virtio_neg_t)); - - /* assign this info struct to the file */ - file->private_data = info; - - /* Init notification list */ - notify_list = kmalloc(sizeof(struct list_head), GFP_KERNEL); - if (!notify_list) { - printk(KERN_ERR "Failed to allocate memory for list head\n"); - return 1; - } - INIT_LIST_HEAD(notify_list); - - /* Init global variables */ - mmap_index = 0; - sum_pgfaults = 0; - share_communication_struct = false; - valid_eventfd = true; - vq_index = 0; - cur_ram_idx = 0; - interrupt_cnt = 0; - notify_sent = 0; - notify_received = 0; - - return 0; -} - -static int loopback_release(struct inode *inode, struct file *file) -{ - int i; - - DBG("Release the device\n"); - - /* - * This make the read/write do not wait - * for the virtio-loopback-adapter if - * the last has closed the fd - */ - valid_eventfd = false; - - /* Unegister mmio_trasmport */ - platform_driver_unregister(&virtio_mmio_driver); - DBG("platform_driver_unregister!\n"); - - /* free communication structure */ - free_page((unsigned long)info->data); - kfree(info); - file->private_data = NULL; - DBG("Clean private_data\n"); - - /* Clear share_mmap_list */ - for(i = 0; i < MMAP_LIMIT; i++) { - share_mmap_list[i].uid = 0; - share_mmap_list[i].pfn = 0; - share_mmap_list[i].vm_start = 0; - share_mmap_list[i].size = 0; - } - - /* Update the global variable, the driver is not in use */ - smp_mb__before_atomic(); - clear_bit(IN_USE_BIT, &loopback_flags); - DBG("clear_bit!\n"); - - return 0; -} - -static const struct file_operations fops = { - .owner = THIS_MODULE, - .read = loopback_read, - .write = loopback_write, - .open = loopback_open, - .unlocked_ioctl = loopback_ioctl, - .mmap = op_mmap, - .llseek = loopback_seek, - .release = loopback_release -}; - -static int __init loopback_init(void) -{ - int err; - dev_t dev; - - err = alloc_chrdev_region(&dev, 0, MAX_DEV, "loopback"); - dev_major = MAJOR(dev); - -#if LINUX_VERSION_CODE < KERNEL_VERSION(6,4,0) - loopback_class = class_create(THIS_MODULE, "loopback"); -#else - loopback_class = class_create("loopback"); -#endif - if (IS_ERR(loopback_class)) { - printk(KERN_ERR "Failed to create class\n"); - return PTR_ERR(loopback_class); - } - - loopback_data = (struct loopback_device_data *)kmalloc( - sizeof(struct loopback_device_data), GFP_KERNEL); - - cdev_init(&loopback_data->cdev, &fops); - loopback_data->cdev.owner = THIS_MODULE; - cdev_add(&loopback_data->cdev, MKDEV(dev_major, 0), 1); - device_create(loopback_class, NULL, MKDEV(dev_major, 0), NULL, "loopback"); - - /* Init wq */ - init_waitqueue_head(&wq); - init_waitqueue_head(&wq_notify); - - /* Init mutex */ - mutex_init(&read_write_lock); - mutex_init(&interrupt_lock); - - /* Init spinlock */ - spin_lock_init(¬ify_q_spinlock); - - virtio_loopback_device = platform_device_register_simple("loopback-transport", -1, NULL, 0); - if (IS_ERR(virtio_loopback_device)) { - err = PTR_ERR(virtio_loopback_device); - pr_err("failed to register loopback-transport device: %d\n", err); - return err; - } - - return 0; -} - -void __exit loopback_exit(void) -{ - DBG("Exit driver!\n"); - - /* Unegister loopback device */ - platform_device_unregister(virtio_loopback_device); - DBG("platform_device_unregister!\n"); - - device_destroy(loopback_class, MKDEV(dev_major, 0)); - cdev_del(&loopback_data->cdev); - DBG("device_destroy!\n"); - - class_destroy(loopback_class); - DBG("class_destroy!\n"); - - kfree(loopback_data); - DBG("Free resources"); -} - -module_init(loopback_init); -module_exit(loopback_exit); diff --git a/loopback_driver.h b/loopback_driver.h deleted file mode 100644 index bb713aa..0000000 --- a/loopback_driver.h +++ /dev/null @@ -1,516 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-or-later -/* - * Based on virtio_mmio.c - * Copyright 2011-2014, ARM Ltd. - * - * Copyright 2022-2024 Virtual Open Systems SAS. - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. - */ -#ifndef __LOOPBACK_H__ -#define __LOOPBACK_H__ - -#define DRIVER "LOOPBACK" - -#include <linux/version.h> - -/* max Minor devices */ -#define MAX_DEV 1 - -/* Define mmap elements limit */ -#define MMAP_LIMIT 200 - -#ifdef DEBUG -#define DBG(...) pr_crit(__VA_ARGS__) -#else -#define DBG(...) -#endif /* DEBUG */ - -/* - * The alignment to use between consumer and producer parts of vring. - * Currently hardcoded to the page size. - */ -#define VIRTIO_MMIO_VRING_ALIGN PAGE_SIZE - -#define to_virtio_mmio_device(_plat_dev) \ - container_of(_plat_dev, struct virtio_mmio_device, vdev) - -typedef struct virtio_device_info_struct { - unsigned long magic; - unsigned long version; - unsigned long device_id; - unsigned long vendor; -} virtio_device_info_struct_t; - -typedef struct virtio_neg { - uint64_t notification; - uint64_t data; - uint64_t size; - bool read; - atomic_t done; -} virtio_neg_t; - -struct virtio_mmio_device { - struct virtio_device vdev; - struct platform_device *pdev; - - void __iomem *base; - unsigned long version; - - /* A list of queues so we can dispatch IRQs */ - spinlock_t lock; - struct list_head virtqueues; -}; - -struct virtio_mmio_vq_info { - /* the actual virtqueue */ - struct virtqueue *vq; - - /* the list node for the virtqueues list */ - struct list_head node; -}; - - -/* - * Print the pdev: - * - *static void print_virtio_pdev(struct platform_device *pdev) - *{ - * int i; - * - * pr_info("Print the pdev:\n"); - * pr_info("\t.name = %s\n", pdev->name); - * pr_info("\t.id = %d\n", pdev->id); - * pr_info("\t.num_resources = %d\n", pdev->num_resources); - * - * for (i=0; i < pdev->num_resources; i++) { - * pr_info("\t.num_resource = %d\n", i); - * pr_info("\t\t.start = 0x%llx\n", pdev->resource[i].start); - * pr_info("\t\t.end = 0x%llx\n", pdev->resource[i].end); - * pr_info("\t\t.flags = 0x%lx\n", pdev->resource[i].flags); - * } - *} - * - *Result: - * - * .name = a003e00.virtio_mmio - * .id = -1 - * .num_resources = 2 - * .num_resource = 0 - * .start = 0xa003e00 - * .end = 0xa003fff - * .flags = 0x200 - * .num_resource = 1 - * .start = 0x2c - * .end = 0x2c - * .flags = 0x401 - */ - -/* mmap finctionality */ -#ifndef VM_RESERVED -#define VM_RESERVED (VM_DONTEXPAND | VM_DONTDUMP) -#endif - -/* Define a bit for atomic test&set */ -#define IN_USE_BIT 0 - -struct mmap_info { - void *data; - int reference; -}; - -/* Define a structure for your notify_list */ -struct notify_data { - uint32_t index; - struct list_head list; -}; - -/* This stuct is used to share the eventfds between driver and userspace */ -typedef struct efd_data { - int efd[2]; - int pid; -} efd_data_t; - -/* mmap functionality related structures */ -struct share_mmap { - uint64_t pfn; - uint64_t vm_start; - uint32_t size; - uint32_t uid; - struct page *page; -}; - -/* Mmap help funcitons */ -/* - * This functions registers all mmap calls done by the user-space into an array - */ -void add_share_mmap(struct file *filp, uint64_t pfn, uint64_t vm_start, - uint64_t size, struct share_mmap *share_mmap_list, int *mmap_index) -{ - DBG("Add new mmaping! index: %d\n", *mmap_index); - DBG("pfn: 0x%llx", pfn); - DBG("vm_start: 0x%llx", vm_start); - DBG("size: 0x%llx", size); - - share_mmap_list[*mmap_index].pfn = pfn; - share_mmap_list[*mmap_index].vm_start = vm_start; - share_mmap_list[*mmap_index].size = size; - share_mmap_list[*mmap_index].uid = task_pid_nr(current); - (*mmap_index)++; -} - -/* - * This functions removes a record from mmap array - */ -void share_mmap_rem(struct vm_area_struct *vma, struct share_mmap *share_mmap_list) -{ - int i; - - for (i = 0; i < MMAP_LIMIT; i++) { - if (share_mmap_list[i].vm_start == vma->vm_start) { - DBG("share_mmap with pa: 0x%llx and size: %x is deleted from the list\n", - share_mmap_list[i].pfn, share_mmap_list[i].size); - share_mmap_list[i].uid = 0; - share_mmap_list[i].pfn = 0; - share_mmap_list[i].vm_start = 0; - share_mmap_list[i].size = 0; - } - } -} - -void print_mmap_idx(int i, struct share_mmap *share_mmap_list) -{ - DBG("share_mmap_list[%d].uid %x\n", i, share_mmap_list[i].uid); - DBG("share_mmap_list[%d].pfn %llx\n", i, share_mmap_list[i].pfn); - DBG("share_mmap_list[%d].vm_start %llx\n", i, share_mmap_list[i].vm_start); - DBG("share_mmap_list[%d].size %x\n", i, share_mmap_list[i].size); -} - - -void print_mmaps(struct share_mmap *share_mmap_list, int mmap_index) -{ - int i; - int limit = mmap_index == 0 ? MMAP_LIMIT : mmap_index; - - for (i = 0; i < limit; i++) - print_mmap_idx(i, share_mmap_list); -} - -/* - * This function return the corresponding pfn of a user-space address - * based on the mapping done during the initialization - */ -uint64_t share_mmap_exist_vma_return_correct_addr(uint64_t pfn, struct share_mmap *share_mmap_list) -{ - int i; - uint64_t corrected_addr; - - for (i = 0; i < MMAP_LIMIT; i++) { - if ((share_mmap_list[i].pfn <= pfn) && - (pfn < share_mmap_list[i].pfn + (share_mmap_list[i].size >> PAGE_SHIFT)) && - (share_mmap_list[i].uid == task_pid_nr(current))) { - DBG("pfn (0x%llx) exist in: 0x%llx - 0x%llx\n", pfn, share_mmap_list[i].pfn, - share_mmap_list[i].pfn + (share_mmap_list[i].size >> PAGE_SHIFT)); - corrected_addr = ((pfn - share_mmap_list[i].pfn) << PAGE_SHIFT) + share_mmap_list[i].vm_start; - DBG("The return addr is: 0x%llx\n", corrected_addr); - return corrected_addr; - } - } - return 0; -} -/* - * This function return the corresponding user-space address of a pfn - * based on the mapping done during the initialization - */ -uint64_t share_mmap_exist_vma_return_correct_pfn(uint64_t addr, struct share_mmap *share_mmap_list) -{ - int i; - uint64_t corrected_pfn; - - for (i = 0; i < MMAP_LIMIT; i++) { - if ((share_mmap_list[i].vm_start <= addr) && - (addr < share_mmap_list[i].vm_start + share_mmap_list[i].size)) { - DBG("addr (0x%llx) exist in: 0x%llx - 0x%llx\n", addr, share_mmap_list[i].vm_start, - share_mmap_list[i].vm_start + share_mmap_list[i].size); - DBG("((addr - share_mmap_list[i].vm_start) / PAGE_SIZE): 0x%llx\n", - ((addr - share_mmap_list[i].vm_start) / PAGE_SIZE)); - DBG("share_mmap_list[i].pfn: 0x%llx\n", share_mmap_list[i].pfn); - corrected_pfn = ((addr - share_mmap_list[i].vm_start) / PAGE_SIZE) + share_mmap_list[i].pfn; - return corrected_pfn; - } - } - return 0; -} - -/* - * This function returns the size of memory block area referrenced by the vrings - */ -uint64_t share_mmap_exist_vma_vring_size(uint64_t insert_pfn, struct vring *global_vring) -{ - int i = 0; - uint64_t next_pfn, mem_blk_size; - - while (((vring_desc_t)global_vring->desc[i]).addr != 0) { - - /* Get the current value of pfn and its size */ - next_pfn = ((vring_desc_t)global_vring->desc[i]).addr >> PAGE_SHIFT; - mem_blk_size = ((vring_desc_t)global_vring->desc[i]).len; - - /* Check if the insert_pfn is found */ - if (insert_pfn == next_pfn) { - - DBG("Found 0x%llx into the vring\n", insert_pfn); - /* Formalize the mem_blk_size to be multiple of PAGE_SIZE */ - mem_blk_size = mem_blk_size % PAGE_SIZE ? - (mem_blk_size & PAGE_MASK) + PAGE_SIZE : mem_blk_size; - DBG("The formalized size is %llu\n", mem_blk_size); - - return mem_blk_size; - } - - /* Go to next element into the vring array */ - i++; - } - - return PAGE_SIZE; -} - -/* - * This function tries to insert multiple PFNs into the user-space process. - * The pfn of the starting page is given as an argument and the number of - * pages to be inserted is calculated based on the memory block length found into - * the vrings. - */ -void vmf_insert_vring_pfns(struct vm_area_struct *vma, uint64_t vaddr, - uint64_t insert_pfn, struct vring *global_vring) -{ - int i, page_num, ret; - uint64_t mem_blk_size; - - /* Formalize the mem_blk_size to be multiple of PAGE_SIZE */ - mem_blk_size = share_mmap_exist_vma_vring_size(insert_pfn, global_vring); - - page_num = mem_blk_size / PAGE_SIZE; - DBG("page_num: %u, need to be inserted\n", page_num); - - for (i = 0; i < page_num; i++) { - DBG("\tTry to insert 0x%llx pfn into vaddr: 0x%llx with size of 0x%llx\n", insert_pfn, vaddr, mem_blk_size); - if (!pfn_valid(insert_pfn)) - break; - - ret = vmf_insert_pfn(vma, vaddr, insert_pfn); - DBG("vmf_insert_pfn returns: 0x%x\n", ret); - - /* Go to the next page of the memory block */ - vaddr += PAGE_SIZE; - insert_pfn++; - } -} - -int mmap_mix(struct file *filp, struct vm_area_struct *vma, - struct share_mmap *share_mmap_list, int *mmap_index, - uint64_t vq_pfn) -{ - int ret = 0; - unsigned long size = (unsigned long)(vma->vm_end - vma->vm_start); - - pr_crit("mmap mixx"); - - ret = remap_pfn_range(vma, vma->vm_start, vq_pfn, size, vma->vm_page_prot); - if (ret != 0) { - DBG("Mmap error\n"); - print_mmaps(share_mmap_list, *mmap_index); - goto out; - } - - add_share_mmap(filp, vq_pfn, vma->vm_start, size, share_mmap_list, mmap_index); - -out: - return ret; -} - -/* This funciton shares the communication struct with the userspace */ -int mmap_communication_shared_space(struct file *filp, struct vm_area_struct *vma, - struct share_mmap *share_mmap_list, int *mmap_index) -{ - unsigned long size = (unsigned long)(vma->vm_end - vma->vm_start); - struct mmap_info *com_mmap_virt = ((struct mmap_info *)(filp->private_data))->data; - uint64_t com_mmap_pfn = ((uint64_t)virt_to_phys(com_mmap_virt)) >> PAGE_SHIFT; - int ret; - -#if LINUX_VERSION_CODE < KERNEL_VERSION(5,16,0) - vma->vm_flags |= VM_RESERVED; -#else - vm_flags_set(vma, VM_RESERVED); -#endif - - ret = remap_pfn_range(vma, vma->vm_start, com_mmap_pfn, size, vma->vm_page_prot); - - if (ret != 0) { - DBG("Error to mmap communication shared space\n"); - goto out; - } - - add_share_mmap(filp, com_mmap_pfn, vma->vm_start, size, share_mmap_list, mmap_index); - -out: - return ret; -} - -/* A debug log function to help track the execution */ -void print_neg_flag(uint64_t neg_flag, bool read) -{ - if (read) - DBG("Read:\n"); - else - DBG("Write:\n"); - - switch (neg_flag) { - case VIRTIO_MMIO_MAGIC_VALUE: //0x000 - DBG("\tVIRTIO_MMIO_MAGIC_VALUE\n"); - break; - case VIRTIO_MMIO_VERSION: //0x004 - DBG("\tVIRTIO_MMIO_VERSION\n"); - break; - case VIRTIO_MMIO_DEVICE_ID: //0x008 - DBG("\tVIRTIO_MMIO_DEVICE_ID\n"); - break; - case VIRTIO_MMIO_VENDOR_ID: //0x00c - DBG("\tVIRTIO_MMIO_VENDOR_ID\n"); - break; - case VIRTIO_MMIO_DEVICE_FEATURES: //0x010 - DBG("\tVIRTIO_MMIO_DEVICE_FEATURES\n"); - break; - case VIRTIO_MMIO_DEVICE_FEATURES_SEL: //0x014 - DBG("\tVIRTIO_MMIO_DEVICE_FEATURES_SEL\n"); - break; - case VIRTIO_MMIO_DRIVER_FEATURES: //0x020 - DBG("\tVIRTIO_MMIO_DRIVER_FEATURES\n"); - break; - case VIRTIO_MMIO_DRIVER_FEATURES_SEL: //0x024 - DBG("\tVIRTIO_MMIO_DRIVER_FEATURES_SEL\n"); - break; - case VIRTIO_MMIO_GUEST_PAGE_SIZE: //0x028 - DBG("\tVIRTIO_MMIO_GUEST_PAGE_SIZE\n"); - break; - case VIRTIO_MMIO_QUEUE_SEL: //0x030 - DBG("\tVIRTIO_MMIO_QUEUE_SEL\n"); - break; - case VIRTIO_MMIO_QUEUE_NUM_MAX: //0x034 - DBG("\tVIRTIO_MMIO_QUEUE_NUM_MAX\n"); - break; - case VIRTIO_MMIO_QUEUE_NUM: //0x038 - DBG("\tVIRTIO_MMIO_QUEUE_NUM\n"); - break; - case VIRTIO_MMIO_QUEUE_ALIGN: //0x03c - DBG("\tVIRTIO_MMIO_QUEUE_ALIGN\n"); - break; - case VIRTIO_MMIO_QUEUE_PFN: //0x040 - DBG("\tVIRTIO_MMIO_QUEUE_PFN\n"); - break; - case VIRTIO_MMIO_QUEUE_READY: //0x044 - DBG("\tVIRTIO_MMIO_QUEUE_READY\n"); - break; - case VIRTIO_MMIO_QUEUE_NOTIFY: //0x050 - DBG("\tVIRTIO_MMIO_QUEUE_NOTIFY\n"); - break; - case VIRTIO_MMIO_INTERRUPT_STATUS: //0x060 - DBG("\tVIRTIO_MMIO_INTERRUPT_STATUS\n"); - break; - case VIRTIO_MMIO_INTERRUPT_ACK: //0x064 - DBG("\tVIRTIO_MMIO_INTERRUPT_ACK\n"); - break; - case VIRTIO_MMIO_STATUS: //0x070 - DBG("\tVIRTIO_MMIO_STATUS\n"); - break; - case VIRTIO_MMIO_QUEUE_DESC_LOW: //0x080 - DBG("\tVIRTIO_MMIO_QUEUE_DESC_LOW\n"); - break; - case VIRTIO_MMIO_QUEUE_DESC_HIGH: //0x084 - DBG("\tVIRTIO_MMIO_QUEUE_DESC_HIGH\n"); - break; - case VIRTIO_MMIO_QUEUE_AVAIL_LOW: //0x090 - DBG("\tVIRTIO_MMIO_QUEUE_AVAIL_LOW\n"); - break; - case VIRTIO_MMIO_QUEUE_AVAIL_HIGH: //0x094 - DBG("\tVIRTIO_MMIO_QUEUE_AVAIL_HIGH\n"); - break; - case VIRTIO_MMIO_QUEUE_USED_LOW: //0x0a0 - DBG("\tVIRTIO_MMIO_QUEUE_USED_LOW\n"); - break; - case VIRTIO_MMIO_QUEUE_USED_HIGH: //0x0a4 - DBG("\tVIRTIO_MMIO_QUEUE_USED_HIGH\n"); - break; - case VIRTIO_MMIO_SHM_SEL: //0x0ac - DBG("\tVIRTIO_MMIO_SHM_SEL\n"); - break; - case VIRTIO_MMIO_SHM_LEN_LOW: //0x0b0 - DBG("\tVIRTIO_MMIO_SHM_LEN_LOW\n"); - break; - case VIRTIO_MMIO_SHM_LEN_HIGH: //0x0b4 - DBG("\tVIRTIO_MMIO_SHM_LEN_HIGH\n"); - break; - case VIRTIO_MMIO_SHM_BASE_LOW: //0x0b8 - DBG("\tVIRTIO_MMIO_SHM_BASE_LOW\n"); - break; - case VIRTIO_MMIO_SHM_BASE_HIGH: //0x0bc - DBG("\tVIRTIO_MMIO_SHM_BASE_HIGH\n"); - break; - case VIRTIO_MMIO_CONFIG_GENERATION: //0x0fc - DBG("\tVIRTIO_MMIO_CONFIG_GENERATION\n"); - break; - default: - if (neg_flag >= VIRTIO_MMIO_CONFIG) - DBG("\tVIRTIO_MMIO_CONFIG\n"); - else - DBG("\tNegotiation flag Unknown: %lld\n", neg_flag); - return; - } -} - -void print_data(const void *buf, size_t size) -{ - int offset = 10; - int i, j; - - DBG("Print data from linux virtio-rng side:\n"); - - printk(KERN_CRIT ""); - - for (i = 0; i < size; i += offset) { - printk(KERN_CONT "\t\t"); - - for (j = i; (j < i + offset) && (j < size); j++) - printk(KERN_CONT "%d, ", *((uint8_t *)(buf + j))); - - printk(KERN_CRIT ""); - } -} - -/* IOCTL defines */ -#define EFD_INIT _IOC(_IOC_WRITE, 'k', 1, sizeof(efd_data)) -#define WAKEUP _IOC(_IOC_WRITE, 'k', 2, 0) -#define START_LOOPBACK _IOC(_IOC_WRITE, 'k', 3, sizeof(virtio_device_info_struct_t)) -#define IRQ _IOC(_IOC_WRITE, 'k', 4, sizeof(int)) -#define SHARE_VQS _IOC(_IOC_WRITE, 'k', 5, sizeof(uint32_t)) -#define SHARE_BUF _IOC(_IOC_WRITE, 'k', 6, sizeof(uint64_t)) -#define SHARE_COM_STRUCT _IOC(_IOC_WRITE, 'k', 7, 0) - -/* device data holder, this structure may be extended to hold additional data */ -struct loopback_device_data { - struct cdev cdev; -}; - -#endif /* __VIRTUALNET_H__ */ diff --git a/virtio_loopback_device.c b/virtio_loopback_device.c new file mode 100644 index 0000000..e0b19a6 --- /dev/null +++ b/virtio_loopback_device.c @@ -0,0 +1,902 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Based on virtio_mmio.c + * Copyright 2011-2014, ARM Ltd. + * + * Copyright 2022-2024 Virtual Open Systems SAS + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#define pr_fmt(fmt) "virtio-loopback-transport: " fmt + +/* Loopback header file */ +#include "virtio_loopback_driver.h" + +static void print_neg_flag(uint64_t neg_flag, bool read) +{ + if (read) + pr_debug("Read:\n"); + else + pr_debug("Write:\n"); + + switch (neg_flag) { + case VIRTIO_MMIO_MAGIC_VALUE: //0x000 + pr_debug("\tVIRTIO_MMIO_MAGIC_VALUE\n"); + break; + case VIRTIO_MMIO_VERSION: //0x004 + pr_debug("\tVIRTIO_MMIO_VERSION\n"); + break; + case VIRTIO_MMIO_DEVICE_ID: //0x008 + pr_debug("\tVIRTIO_MMIO_DEVICE_ID\n"); + break; + case VIRTIO_MMIO_VENDOR_ID: //0x00c + pr_debug("\tVIRTIO_MMIO_VENDOR_ID\n"); + break; + case VIRTIO_MMIO_DEVICE_FEATURES: //0x010 + pr_debug("\tVIRTIO_MMIO_DEVICE_FEATURES\n"); + break; + case VIRTIO_MMIO_DEVICE_FEATURES_SEL: //0x014 + pr_debug("\tVIRTIO_MMIO_DEVICE_FEATURES_SEL\n"); + break; + case VIRTIO_MMIO_DRIVER_FEATURES: //0x020 + pr_debug("\tVIRTIO_MMIO_DRIVER_FEATURES\n"); + break; + case VIRTIO_MMIO_DRIVER_FEATURES_SEL: //0x024 + pr_debug("\tVIRTIO_MMIO_DRIVER_FEATURES_SEL\n"); + break; + case VIRTIO_MMIO_GUEST_PAGE_SIZE: //0x028 + pr_debug("\tVIRTIO_MMIO_GUEST_PAGE_SIZE\n"); + break; + case VIRTIO_MMIO_QUEUE_SEL: //0x030 + pr_debug("\tVIRTIO_MMIO_QUEUE_SEL\n"); + break; + case VIRTIO_MMIO_QUEUE_NUM_MAX: //0x034 + pr_debug("\tVIRTIO_MMIO_QUEUE_NUM_MAX\n"); + break; + case VIRTIO_MMIO_QUEUE_NUM: //0x038 + pr_debug("\tVIRTIO_MMIO_QUEUE_NUM\n"); + break; + case VIRTIO_MMIO_QUEUE_ALIGN: //0x03c + pr_debug("\tVIRTIO_MMIO_QUEUE_ALIGN\n"); + break; + case VIRTIO_MMIO_QUEUE_PFN: //0x040 + pr_debug("\tVIRTIO_MMIO_QUEUE_PFN\n"); + break; + case VIRTIO_MMIO_QUEUE_READY: //0x044 + pr_debug("\tVIRTIO_MMIO_QUEUE_READY\n"); + break; + case VIRTIO_MMIO_QUEUE_NOTIFY: //0x050 + pr_debug("\tVIRTIO_MMIO_QUEUE_NOTIFY\n"); + break; + case VIRTIO_MMIO_INTERRUPT_STATUS: //0x060 + pr_debug("\tVIRTIO_MMIO_INTERRUPT_STATUS\n"); + break; + case VIRTIO_MMIO_INTERRUPT_ACK: //0x064 + pr_debug("\tVIRTIO_MMIO_INTERRUPT_ACK\n"); + break; + case VIRTIO_MMIO_STATUS: //0x070 + pr_debug("\tVIRTIO_MMIO_STATUS\n"); + break; + case VIRTIO_MMIO_QUEUE_DESC_LOW: //0x080 + pr_debug("\tVIRTIO_MMIO_QUEUE_DESC_LOW\n"); + break; + case VIRTIO_MMIO_QUEUE_DESC_HIGH: //0x084 + pr_debug("\tVIRTIO_MMIO_QUEUE_DESC_HIGH\n"); + break; + case VIRTIO_MMIO_QUEUE_AVAIL_LOW: //0x090 + pr_debug("\tVIRTIO_MMIO_QUEUE_AVAIL_LOW\n"); + break; + case VIRTIO_MMIO_QUEUE_AVAIL_HIGH: //0x094 + pr_debug("\tVIRTIO_MMIO_QUEUE_AVAIL_HIGH\n"); + break; + case VIRTIO_MMIO_QUEUE_USED_LOW: //0x0a0 + pr_debug("\tVIRTIO_MMIO_QUEUE_USED_LOW\n"); + break; + case VIRTIO_MMIO_QUEUE_USED_HIGH: //0x0a4 + pr_debug("\tVIRTIO_MMIO_QUEUE_USED_HIGH\n"); + break; + case VIRTIO_MMIO_SHM_SEL: //0x0ac + pr_debug("\tVIRTIO_MMIO_SHM_SEL\n"); + break; + case VIRTIO_MMIO_SHM_LEN_LOW: //0x0b0 + pr_debug("\tVIRTIO_MMIO_SHM_LEN_LOW\n"); + break; + case VIRTIO_MMIO_SHM_LEN_HIGH: //0x0b4 + pr_debug("\tVIRTIO_MMIO_SHM_LEN_HIGH\n"); + break; + case VIRTIO_MMIO_SHM_BASE_LOW: //0x0b8 + pr_debug("\tVIRTIO_MMIO_SHM_BASE_LOW\n"); + break; + case VIRTIO_MMIO_SHM_BASE_HIGH: //0x0bc + pr_debug("\tVIRTIO_MMIO_SHM_BASE_HIGH\n"); + break; + case VIRTIO_MMIO_CONFIG_GENERATION: //0x0fc + pr_debug("\tVIRTIO_MMIO_CONFIG_GENERATION\n"); + break; + default: + if (neg_flag >= VIRTIO_MMIO_CONFIG) + pr_debug("\tVIRTIO_MMIO_CONFIG\n"); + else + pr_debug("\tNegotiation flag Unknown: %lld\n", neg_flag); + return; + } +} + +/* + * Print the pdev: + * + *static void print_virtio_pdev(struct platform_device *pdev) + *{ + * int i; + * + * pr_info("Print the pdev:\n"); + * pr_info("\t.name = %s\n", pdev->name); + * pr_info("\t.id = %d\n", pdev->id); + * pr_info("\t.num_resources = %d\n", pdev->num_resources); + * + * for (i=0; i < pdev->num_resources; i++) { + * pr_info("\t.num_resource = %d\n", i); + * pr_info("\t\t.start = 0x%llx\n", pdev->resource[i].start); + * pr_info("\t\t.end = 0x%llx\n", pdev->resource[i].end); + * pr_info("\t\t.flags = 0x%lx\n", pdev->resource[i].flags); + * } + *} + * + *Result: + * + * .name = a003e00.virtio_loopback + * .id = -1 + * .num_resources = 2 + * .num_resource = 0 + * .start = 0xa003e00 + * .end = 0xa003fff + * .flags = 0x200 + * .num_resource = 1 + * .start = 0x2c + * .end = 0x2c + * .flags = 0x401 + */ + +/* function declaration */ +static uint64_t read_adapter(uint64_t fn_id, uint64_t size, struct device_data *dev_data); +static void write_adapter(uint64_t data, uint64_t fn_id, uint64_t size, struct device_data *dev_data); + +/* Configuration interface */ +static u64 vl_get_features(struct virtio_device *vdev) +{ + struct virtio_loopback_device *vl_dev = to_virtio_loopback_device(vdev, vdev); + struct device_data *data = vl_dev->data; + u64 features; + + /* Take feature bits 0-31 */ + write_adapter(1, VIRTIO_MMIO_DEVICE_FEATURES_SEL, 4, data); + features = read_adapter(VIRTIO_MMIO_DEVICE_FEATURES, 4, data); + features <<= 32; + + /* Take feature bits 32-63 */ + write_adapter(0, VIRTIO_MMIO_DEVICE_FEATURES_SEL, 4, data); + features |= read_adapter(VIRTIO_MMIO_DEVICE_FEATURES, 4, data); + + return features; +} + +static int vl_finalize_features(struct virtio_device *vdev) +{ + struct virtio_loopback_device *vl_dev = to_virtio_loopback_device(vdev, vdev); + struct device_data *data = vl_dev->data; + + /* Give virtio_ring a chance to accept features. */ + vring_transport_features(vdev); + + /* Make sure there are no mixed devices */ + if (vl_dev->version == 2 && + !__virtio_test_bit(vdev, VIRTIO_F_VERSION_1)) { + dev_err(&vdev->dev, "New virtio-loopback devices (version 2) " + "must provide VIRTIO_F_VERSION_1 feature!\n"); + return -EINVAL; + } + + write_adapter(1, VIRTIO_MMIO_DRIVER_FEATURES_SEL, 4, data); + write_adapter((u32)(vdev->features >> 32), VIRTIO_MMIO_DRIVER_FEATURES, 4, data); + + write_adapter(0, VIRTIO_MMIO_DRIVER_FEATURES_SEL, 4, data); + write_adapter((u32)vdev->features, VIRTIO_MMIO_DRIVER_FEATURES, 4, data); + + return 0; +} + +static void vl_get(struct virtio_device *vdev, unsigned int offset, + void *buf, unsigned int len) +{ + struct virtio_loopback_device *vl_dev = to_virtio_loopback_device(vdev, vdev); + struct device_data *data = vl_dev->data; + + u8 b; + __le16 w; + __le32 l; + + if (vl_dev->version == 1) { + u8 *ptr = buf; + int i; + + for (i = 0; i < len; i++) + ptr[i] = read_adapter(VIRTIO_MMIO_CONFIG + offset + i, 1, data); + return; + } + + switch (len) { + case 1: + b = read_adapter(VIRTIO_MMIO_CONFIG + offset, 1, data); + memcpy(buf, &b, sizeof(b)); + break; + case 2: + w = cpu_to_le16(read_adapter(VIRTIO_MMIO_CONFIG + offset, 2, data)); + memcpy(buf, &w, sizeof(w)); + break; + case 4: + l = cpu_to_le32(read_adapter(VIRTIO_MMIO_CONFIG + offset, 4, data)); + memcpy(buf, &l, sizeof(l)); + break; + case 8: + l = cpu_to_le32(read_adapter(VIRTIO_MMIO_CONFIG + offset, 4, data)); + memcpy(buf, &l, sizeof(l)); + l = cpu_to_le32(read_adapter(VIRTIO_MMIO_CONFIG + offset + sizeof(l), 4, data)); + memcpy(buf + sizeof(l), &l, sizeof(l)); + break; + default: + BUG(); + } +} + +static void vl_set(struct virtio_device *vdev, unsigned int offset, + const void *buf, unsigned int len) +{ + struct virtio_loopback_device *vl_dev = to_virtio_loopback_device(vdev, vdev); + struct device_data *data = vl_dev->data; + + u8 b; + __le16 w; + __le32 l; + + if (vl_dev->version == 1) { + const u8 *ptr = buf; + int i; + + for (i = 0; i < len; i++) + write_adapter(ptr[i], VIRTIO_MMIO_CONFIG + offset + i, 1, data); + + return; + } + + switch (len) { + case 1: + memcpy(&b, buf, sizeof(b)); + write_adapter(b, VIRTIO_MMIO_CONFIG + offset, 1, data); + break; + case 2: + memcpy(&w, buf, sizeof(w)); + write_adapter(le16_to_cpu(w), VIRTIO_MMIO_CONFIG + offset, 2, data); + break; + case 4: + memcpy(&l, buf, sizeof(l)); + write_adapter(le32_to_cpu(l), VIRTIO_MMIO_CONFIG + offset, 4, data); + break; + case 8: + memcpy(&l, buf, sizeof(l)); + write_adapter(le32_to_cpu(l), VIRTIO_MMIO_CONFIG + offset, 4, data); + memcpy(&l, buf + sizeof(l), sizeof(l)); + write_adapter(le32_to_cpu(l), VIRTIO_MMIO_CONFIG + offset + sizeof(l), 4, data); + break; + default: + BUG(); + } +} + +static u32 vl_generation(struct virtio_device *vdev) +{ + struct virtio_loopback_device *vl_dev = to_virtio_loopback_device(vdev, vdev); + struct device_data *data = vl_dev->data; + + if (vl_dev->version == 1) + return 0; + else + return read_adapter(VIRTIO_MMIO_CONFIG_GENERATION, 4, data); +} + +static u8 vl_get_status(struct virtio_device *vdev) +{ + struct virtio_loopback_device *vl_dev = to_virtio_loopback_device(vdev, vdev); + struct device_data *data = vl_dev->data; + + return read_adapter(VIRTIO_MMIO_STATUS, 4, data) & 0xff; +} + +static void vl_set_status(struct virtio_device *vdev, u8 status) +{ + struct virtio_loopback_device *vl_dev = to_virtio_loopback_device(vdev, vdev); + struct device_data *data = vl_dev->data; + + write_adapter(status, VIRTIO_MMIO_STATUS, 4, data); +} + +static void vl_reset(struct virtio_device *vdev) +{ + struct virtio_loopback_device *vl_dev = to_virtio_loopback_device(vdev, vdev); + struct device_data *data = vl_dev->data; + + /* 0 status means a reset. */ + write_adapter(0, VIRTIO_MMIO_STATUS, 4, data); +} + +/* Notify work handling function */ +static void notify_work_handler(struct work_struct *work) +{ + struct virtio_loopback_device *vl_dev = to_virtio_loopback_device(work, notify_work); + struct device_data *dev_data = vl_dev->data; + struct notify_data *entry, *tmp; + uint32_t index; + + spin_lock(&vl_dev->notify_q_lock); + list_for_each_entry_safe(entry, tmp, &vl_dev->notify_list, list) { + index = entry->index; + list_del(&entry->list); + kfree(entry); + /* Proceed in dispatching the notification to the adapter */ + spin_unlock(&vl_dev->notify_q_lock); + write_adapter(index, VIRTIO_MMIO_QUEUE_NOTIFY, 4, dev_data); + spin_lock(&vl_dev->notify_q_lock); + } + spin_unlock(&vl_dev->notify_q_lock); +} + +/* The notify function used when creating a virtqueue */ +static bool vl_notify(struct virtqueue *vq) +{ + struct virtio_loopback_device *vl_dev = to_virtio_loopback_device(vq->vdev, vdev); + struct notify_data *data; + int ret = 1; + + /* Create the new node */ + data = kmalloc(sizeof(struct notify_data), GFP_ATOMIC); + if (!data) + return false; + + data->index = vq->index; + INIT_LIST_HEAD(&data->list); + + /* Add in the notify_list, which should be protected! */ + spin_lock(&vl_dev->notify_q_lock); + list_add_tail(&data->list, &vl_dev->notify_list); + spin_unlock(&vl_dev->notify_q_lock); + + /* Schedule the element */ + while (ret) { + /* Force scheduling if queue_work fails and list is not empty */ + ret = !queue_work(loopback_data.notify_workqueue, &vl_dev->notify_work); + spin_lock(&vl_dev->notify_q_lock); + ret &= !list_empty(&vl_dev->notify_list); + spin_unlock(&vl_dev->notify_q_lock); + } + + return true; +} + +/* the interrupt function used when receiving an IRQ */ +bool vl_interrupt(struct virtio_loopback_device *vl_dev, int irq) +{ + struct device_data *data = vl_dev->data; + struct virtio_loopback_vq_info *info; + unsigned long status; + + pr_debug("Received interrupt!\n"); + /* STATUS and ACK should be done without any intermediate status change */ + /* Read and acknowledge interrupts */ + status = read_adapter(VIRTIO_MMIO_INTERRUPT_STATUS, 4, data); + write_adapter(status, VIRTIO_MMIO_INTERRUPT_ACK, 4, data); + + if (unlikely(status & VIRTIO_MMIO_INT_CONFIG)) + virtio_config_changed(&vl_dev->vdev); + + if (likely(status & VIRTIO_MMIO_INT_VRING)) { + spin_lock(&vl_dev->lock); + list_for_each_entry(info, &vl_dev->virtqueues, node) { + (void)vring_interrupt(irq, info->vq); + } + spin_unlock(&vl_dev->lock); + } + + return true; +} + +static void vl_del_vq(struct virtqueue *vq) +{ + struct virtio_loopback_device *vl_dev = to_virtio_loopback_device(vq->vdev, vdev); + struct device_data *data = vl_dev->data; + + struct virtio_loopback_vq_info *info = vq->priv; + unsigned long flags; + unsigned int index = vq->index; + + spin_lock_irqsave(&vl_dev->lock, flags); + list_del(&info->node); + spin_unlock_irqrestore(&vl_dev->lock, flags); + + /* Select and deactivate the queue */ + write_adapter(index, VIRTIO_MMIO_QUEUE_SEL, 4, data); + + if (vl_dev->version == 1) { + write_adapter(0, VIRTIO_MMIO_QUEUE_PFN, 4, data); + } else { + write_adapter(0, VIRTIO_MMIO_QUEUE_READY, 4, data); + WARN_ON(read_adapter(VIRTIO_MMIO_QUEUE_READY, 4, data)); + } + + vring_del_virtqueue(vq); + kfree(info); +} + +static void vl_del_vqs(struct virtio_device *vdev) +{ + struct virtqueue *vq, *n; + + list_for_each_entry_safe(vq, n, &vdev->vqs, list) + vl_del_vq(vq); +} + +static struct virtqueue *vl_setup_vq(struct virtio_device *vdev, unsigned int index, + void (*callback)(struct virtqueue *vq), + const char *name, bool ctx) +{ + struct virtio_loopback_device *vl_dev = to_virtio_loopback_device(vdev, vdev); + struct device_data *data = vl_dev->data; + struct virtio_loopback_vq_info *info; + struct virtqueue *vq; + unsigned long flags; + unsigned int num; + int err; + + if (!name) + return NULL; + + /* Select the queue we're interested in */ + write_adapter(index, VIRTIO_MMIO_QUEUE_SEL, 4, data); + + /* Queue shouldn't already be set up. */ + if (read_adapter((vl_dev->version == 1 ? + VIRTIO_MMIO_QUEUE_PFN : VIRTIO_MMIO_QUEUE_READY), 4, data)) { + err = -ENOENT; + goto error_available; + } + + /* Allocate and fill out our active queue description */ + info = kmalloc(sizeof(*info), GFP_KERNEL); + if (!info) { + err = -ENOMEM; + goto error_kmalloc; + } + + num = read_adapter(VIRTIO_MMIO_QUEUE_NUM_MAX, 4, data); + if (num == 0) { + err = -ENOENT; + goto error_new_virtqueue; + } + + /* Create the vring */ + vq = vring_create_virtqueue(index, num, VIRTIO_MMIO_VRING_ALIGN, vdev, + true, true, ctx, vl_notify, callback, name); + if (!vq) { + err = -ENOMEM; + goto error_new_virtqueue; + } + +#if LINUX_VERSION_CODE > KERNEL_VERSION(6, 0, 0) + vq->num_max = num; +#endif + + /* Activate the queue */ + write_adapter(virtqueue_get_vring_size(vq), VIRTIO_MMIO_QUEUE_NUM, 4, data); + if (vl_dev->version == 1) { + u64 q_pfn = virtqueue_get_desc_addr(vq); + + q_pfn = q_pfn >> PAGE_SHIFT; + + /* Copy the physical address and enable the mmap */ + data->vq_data.vq_pfn = q_pfn; + data->vq_data.vq_pfns[data->vq_data.vq_index++] = q_pfn; + + /* + * virtio-loopback v1 uses a 32bit QUEUE PFN. If we have something + * that doesn't fit in 32bit, fail the setup rather than + * pretending to be successful. + */ + if (q_pfn >> 32) { + dev_err(&vdev->dev, + "platform bug: legacy virtio-loopback must not " + "be used with RAM above 0x%llxGB\n", + 0x1ULL << (32 + PAGE_SHIFT - 30)); + err = -E2BIG; + goto error_bad_pfn; + } + + write_adapter(PAGE_SIZE, VIRTIO_MMIO_QUEUE_ALIGN, 4, data); + write_adapter(q_pfn, VIRTIO_MMIO_QUEUE_PFN, 4, data); + } else { + u64 addr; + + addr = virtqueue_get_desc_addr(vq); + write_adapter((u32)addr, VIRTIO_MMIO_QUEUE_DESC_LOW, 4, data); + write_adapter((u32)(addr >> 32), VIRTIO_MMIO_QUEUE_DESC_HIGH, 4, data); + + addr = virtqueue_get_avail_addr(vq); + write_adapter((u32)addr, VIRTIO_MMIO_QUEUE_AVAIL_LOW, 4, data); + write_adapter((u32)(addr >> 32), VIRTIO_MMIO_QUEUE_AVAIL_HIGH, 4, data); + + addr = virtqueue_get_used_addr(vq); + write_adapter((u32)addr, VIRTIO_MMIO_QUEUE_USED_LOW, 4, data); + write_adapter((u32)(addr >> 32), VIRTIO_MMIO_QUEUE_USED_HIGH, 4, data); + + write_adapter(1, VIRTIO_MMIO_QUEUE_READY, 4, data); + } + + vq->priv = info; + info->vq = vq; + + spin_lock_irqsave(&vl_dev->lock, flags); + list_add(&info->node, &vl_dev->virtqueues); + spin_unlock_irqrestore(&vl_dev->lock, flags); + + return vq; + +error_bad_pfn: + vring_del_virtqueue(vq); +error_new_virtqueue: + if (vl_dev->version == 1) { + write_adapter(0, VIRTIO_MMIO_QUEUE_PFN, 4, data); + } else { + write_adapter(0, VIRTIO_MMIO_QUEUE_READY, 4, data); + WARN_ON(read_adapter(VIRTIO_MMIO_QUEUE_READY, 4, data)); + } + kfree(info); +error_kmalloc: +error_available: + return ERR_PTR(err); +} + +#if LINUX_VERSION_CODE <= KERNEL_VERSION(6, 10, 8) +static int vl_find_vqs(struct virtio_device *vdev, unsigned int nvqs, + struct virtqueue *vqs[], + vq_callback_t *callbacks[], + const char * const names[], + const bool *ctx, + struct irq_affinity *desc) +{ + int i, queue_idx = 0; + + for (i = 0; i < nvqs; ++i) { + if (!names[i]) { + vqs[i] = NULL; + continue; + } + + vqs[i] = vl_setup_vq(vdev, queue_idx++, callbacks[i], names[i], + ctx ? ctx[i] : false); + if (IS_ERR(vqs[i])) { + vl_del_vqs(vdev); + return PTR_ERR(vqs[i]); + } + } + + return 0; +} +#else +static int vl_find_vqs(struct virtio_device *vdev, unsigned int nvqs, + struct virtqueue *vqs[], + struct virtqueue_info vqs_info[], + struct irq_affinity *desc) +{ + int i, queue_idx = 0; + + for (i = 0; i < nvqs; ++i) { + struct virtqueue_info *vqi = &vqs_info[i]; + + if (!vqi->name) { + vqs[i] = NULL; + continue; + } + + vqs[i] = vl_setup_vq(vdev, queue_idx++, vqi->callback, + vqi->name, vqi->ctx); + if (IS_ERR(vqs[i])) { + vl_del_vqs(vdev); + return PTR_ERR(vqs[i]); + } + } + + return 0; +} +#endif + +static const char *vl_bus_name(struct virtio_device *vdev) +{ + struct virtio_loopback_device *vl_dev = to_virtio_loopback_device(vdev, vdev); + + return vl_dev->pdev->name; +} + +static bool vl_get_shm_region(struct virtio_device *vdev, + struct virtio_shm_region *region, u8 id) +{ + struct virtio_loopback_device *vl_dev = to_virtio_loopback_device(vdev, vdev); + struct device_data *data = vl_dev->data; + u64 len, addr; + + /* Select the region we're interested in */ + write_adapter(id, VIRTIO_MMIO_SHM_SEL, 4, data); + + /* Read the region size */ + len = (u64) read_adapter(VIRTIO_MMIO_SHM_LEN_LOW, 4, data); + len |= (u64) read_adapter(VIRTIO_MMIO_SHM_LEN_HIGH, 4, data) << 32; + + region->len = len; + + /* Check if region length is -1. If that's the case, the shared memory + * region does not exist and there is no need to proceed further. + */ + if (len == ~(u64)0) + return false; + + /* Read the region base address */ + addr = (u64) read_adapter(VIRTIO_MMIO_SHM_BASE_LOW, 4, data); + addr |= (u64) read_adapter(VIRTIO_MMIO_SHM_BASE_HIGH, 4, data) << 32; + + region->addr = addr; + + return true; +} + +static const struct virtio_config_ops virtio_loopback_config_ops = { + .get = vl_get, + .set = vl_set, + .generation = vl_generation, + .get_status = vl_get_status, + .set_status = vl_set_status, + .reset = vl_reset, + .find_vqs = vl_find_vqs, + .del_vqs = vl_del_vqs, + .get_features = vl_get_features, + .finalize_features = vl_finalize_features, + .bus_name = vl_bus_name, + .get_shm_region = vl_get_shm_region, +}; + +static void virtio_loopback_release_dev(struct device *_d) +{ + struct virtio_device *vdev = container_of(_d, struct virtio_device, dev); + struct virtio_loopback_device *vl_dev = to_virtio_loopback_device(vdev, vdev); + struct platform_device *pdev = vl_dev->pdev; + + devm_kfree(&pdev->dev, vl_dev); +} + +/* Function to carry-out the registration of the virtio_loopback */ +int loopback_register_virtio_dev(struct virtio_loopback_device *vl_dev) +{ + struct platform_device *pdev = vl_dev->pdev; + struct device_data *data = vl_dev->data; + unsigned long magic; + int rc; + + /* Check magic value */ + magic = read_adapter(VIRTIO_MMIO_MAGIC_VALUE, 4, data); + + if (magic != ('v' | 'i' << 8 | 'r' << 16 | 't' << 24)) { + dev_warn(&pdev->dev, "Wrong magic value 0x%08lx!\n", magic); + return -ENODEV; + } + + /* Check device version */ + vl_dev->version = read_adapter(VIRTIO_MMIO_VERSION, 4, data); + + if (vl_dev->version < 1 || vl_dev->version > 2) { + dev_err(&pdev->dev, "Version %ld not supported!\n", + vl_dev->version); + return -ENXIO; + } + + vl_dev->vdev.id.device = read_adapter(VIRTIO_MMIO_DEVICE_ID, 4, data); + + if (vl_dev->vdev.id.device == 0) { + /* + * virtio-loopback device with an ID 0 is a (dummy) placeholder + * with no function. End probing now with no error reported. + */ + return -ENODEV; + } + + vl_dev->vdev.id.vendor = read_adapter(VIRTIO_MMIO_VENDOR_ID, 4, data); + + if (vl_dev->version == 1) { + write_adapter(PAGE_SIZE, VIRTIO_MMIO_GUEST_PAGE_SIZE, 4, data); + + rc = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); + /* + * In the legacy case, ensure our coherently-allocated virtio + * ring will be at an address expressable as a 32-bit PFN. + */ + if (!rc) + dma_set_coherent_mask(&pdev->dev, + DMA_BIT_MASK(32 + PAGE_SHIFT)); + } else { + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + } + if (rc) + rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (rc) + dev_warn(&pdev->dev, "Failed to enable 64-bit or 32-bit DMA." + "Trying to continue, but this might not work.\n"); + + /* Register the virtio device in the system */ + rc = register_virtio_device(&vl_dev->vdev); + if (rc) + put_device(&vl_dev->vdev.dev); + + return 0; +} + +static int virtio_loopback_probe(struct platform_device *pdev) +{ + int err; + struct virtio_loopback_device *vl_dev; + + pr_info("Entered probe with id: %d!\n", pdev->id); + vl_dev = devm_kzalloc(&pdev->dev, sizeof(*vl_dev), GFP_KERNEL); + if (!vl_dev) { + err = -ENOMEM; + goto out; + } + + vl_dev->vdev.dev.parent = &pdev->dev; + vl_dev->vdev.dev.release = virtio_loopback_release_dev; + vl_dev->vdev.config = &virtio_loopback_config_ops; + vl_dev->pdev = pdev; + INIT_LIST_HEAD(&vl_dev->virtqueues); + spin_lock_init(&vl_dev->lock); + /* Initialize the workqueue */ + INIT_WORK(&vl_dev->notify_work, notify_work_handler); + INIT_LIST_HEAD(&vl_dev->notify_list); + spin_lock_init(&vl_dev->notify_q_lock); + + platform_set_drvdata(pdev, vl_dev); + + /* Insert new entry data */ + err = insert_entry_data(vl_dev, pdev->id); + +out: + return err; +} + +#if LINUX_VERSION_CODE > KERNEL_VERSION(6, 10, 8) +void virtio_loopback_remove(struct platform_device *pdev) +#else +int virtio_loopback_remove(struct platform_device *pdev) +#endif +{ + struct virtio_loopback_device *vl_dev = platform_get_drvdata(pdev); + + if (vl_dev->data) { + unregister_virtio_device(&vl_dev->vdev); + pr_info("unregister_virtio_device!\n"); + /* Proceed to de-activating the data for this entry */ + vl_dev->data = NULL; + } + +#if LINUX_VERSION_CODE <= KERNEL_VERSION(6, 10, 8) + return 0; +#endif +} + +/* No need of DTS and ACPI */ +struct platform_driver virtio_loopback_driver = { + .probe = virtio_loopback_probe, + .remove = virtio_loopback_remove, + .driver = { + .name = "loopback-transport", + }, +}; + +static uint64_t read_adapter(uint64_t fn_id, uint64_t size, struct device_data *dev_data) +{ + uint64_t result; + + mutex_lock(&(dev_data)->read_write_lock); + + /* + * By enabling the following line all + * read messages will be printed: + * + * print_neg_flag(fn_id, 1); + */ + print_neg_flag(fn_id, 1); + + ((struct virtio_neg *)(dev_data->info->data))->notification = fn_id; + ((struct virtio_neg *)(dev_data->info->data))->data = 0; + ((struct virtio_neg *)(dev_data->info->data))->size = size; + ((struct virtio_neg *)(dev_data->info->data))->read = true; + + atomic_set(&((struct virtio_neg *)(dev_data->info->data))->done, 0); + +#if LINUX_VERSION_CODE > KERNEL_VERSION(6, 7, 12) + eventfd_signal(dev_data->efd_ctx); +#else + eventfd_signal(dev_data->efd_ctx, 1); +#endif + + /* + * There is a chance virtio-loopback adapter to call "wake_up" + * before the current thread sleep. This is the reason that + * "wait_event_timeout" is used instead of "wait_event". In this + * way, virtio-loopback driver will wake up even if has missed the + * "wake_up" kick, check the updated "done" value and return. + */ + + while (dev_data->valid_eventfd && atomic_read(&((struct virtio_neg *)(dev_data->info->data))->done) != 1) + wait_event_timeout(dev_data->wq, atomic_read(&((struct virtio_neg *)(dev_data->info->data))->done) == 1, 1 * HZ); + + result = ((struct virtio_neg *)(dev_data->info->data))->data; + + mutex_unlock(&(dev_data)->read_write_lock); + + return result; +} + +static void write_adapter(uint64_t data, uint64_t fn_id, uint64_t size, struct device_data *dev_data) +{ + + mutex_lock(&(dev_data)->read_write_lock); + + /* + * By enabling the following line all + * write messages will be printed: + * + * print_neg_flag(fn_id, 1); + */ + print_neg_flag(fn_id, 0); + + ((struct virtio_neg *)(dev_data->info->data))->notification = fn_id; + ((struct virtio_neg *)(dev_data->info->data))->data = data; + ((struct virtio_neg *)(dev_data->info->data))->size = size; + ((struct virtio_neg *)(dev_data->info->data))->read = false; + + atomic_set(&((struct virtio_neg *)(dev_data->info->data))->done, 0); + +#if LINUX_VERSION_CODE > KERNEL_VERSION(6, 7, 12) + eventfd_signal(dev_data->efd_ctx); +#else + eventfd_signal(dev_data->efd_ctx, 1); +#endif + + /* + * There is a chance virtio-loopback adapter to call "wake_up" + * before the current thread sleep. This is the reason that + * "wait_event_timeout" is used instead of "wait_event". In this + * way, virtio-loopback driver will wake up even if has missed the + * "wake_up" kick, check the updated "done" value and return. + */ + while (dev_data->valid_eventfd && atomic_read(&((struct virtio_neg *)(dev_data->info->data))->done) != 1) + wait_event_timeout(dev_data->wq, atomic_read(&((struct virtio_neg *)(dev_data->info->data))->done) == 1, 1 * HZ); + + mutex_unlock(&(dev_data)->read_write_lock); +} diff --git a/virtio_loopback_driver.c b/virtio_loopback_driver.c new file mode 100644 index 0000000..d822a3e --- /dev/null +++ b/virtio_loopback_driver.c @@ -0,0 +1,629 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2022-2024 Virtual Open Systems SAS + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ + +#define pr_fmt(fmt) "virtio-loopback: " fmt + +/* Loopback header file */ +#include "virtio_loopback_driver.h" + +/* Features */ +MODULE_LICENSE("GPL v2"); + +/* The global data for the loopback */ +struct loopback_device_data loopback_data; +struct loopback_devices_array loopback_devices; + +/* + * This functions registers all mmap calls done by the user-space into an array + */ +static void add_share_mmap(struct file *filp, uint64_t pfn, uint64_t vm_start, uint64_t size) +{ + struct file_priv_data *file_data = (struct file_priv_data *)(filp->private_data); + struct mmap_data *mm_data = (struct mmap_data *)file_data->mm_data; + + pr_debug("Add new mmaping! index: %d\n", mm_data->mmap_index); + pr_debug("pfn: 0x%llx", pfn); + pr_debug("vm_start: 0x%llx", vm_start); + pr_debug("size: 0x%llx", size); + + mm_data->share_mmap_list[mm_data->mmap_index].pfn = pfn; + mm_data->share_mmap_list[mm_data->mmap_index].vm_start = vm_start; + mm_data->share_mmap_list[mm_data->mmap_index].size = size; + mm_data->share_mmap_list[mm_data->mmap_index].uid = task_pid_nr(current); + mm_data->mmap_index++; +} + +/* + * This functions removes a record from mmap array + */ +static void share_mmap_rem(struct vm_area_struct *vma) +{ + struct file *file = vma->vm_file; + struct file_priv_data *file_data = (struct file_priv_data *)(file->private_data); + struct mmap_data *mm_data = (struct mmap_data *)file_data->mm_data; + int i; + + for (i = 0; i < MMAP_LIMIT; i++) { + if (mm_data->share_mmap_list[i].vm_start == vma->vm_start) { + pr_debug("share_mmap with pa: 0x%llx and size: %x is deleted from the list\n", + mm_data->share_mmap_list[i].pfn, mm_data->share_mmap_list[i].size); + mm_data->share_mmap_list[i].uid = 0; + mm_data->share_mmap_list[i].pfn = 0; + mm_data->share_mmap_list[i].vm_start = 0; + mm_data->share_mmap_list[i].size = 0; + } + } +} + +static void print_mmap_idx(struct mmap_data *mm_data, int i) +{ + pr_debug("share_mmap_list[%d].uid %x\n", i, mm_data->share_mmap_list[i].uid); + pr_debug("share_mmap_list[%d].pfn %llx\n", i, mm_data->share_mmap_list[i].pfn); + pr_debug("share_mmap_list[%d].vm_start %llx\n", i, mm_data->share_mmap_list[i].vm_start); + pr_debug("share_mmap_list[%d].size %x\n", i, mm_data->share_mmap_list[i].size); +} + +static void print_mmaps(struct mmap_data *mm_data) +{ + int i, limit = mm_data->mmap_index == 0 ? MMAP_LIMIT : mm_data->mmap_index; + + for (i = 0; i < limit; i++) + print_mmap_idx(mm_data, i); +} + +/* + * This function return the corresponding user-space address of a pfn + * based on the mapping done during the initialization + */ +static uint64_t share_mmap_exist_vma_return_correct_pfn(struct mmap_data *mm_data, uint64_t addr) +{ + int i; + uint64_t corrected_pfn; + + for (i = 0; i < MMAP_LIMIT; i++) { + if ((mm_data->share_mmap_list[i].vm_start <= addr) && + (addr < mm_data->share_mmap_list[i].vm_start + mm_data->share_mmap_list[i].size)) { + pr_debug("addr (0x%llx) exist in: 0x%llx - 0x%llx\n", addr, mm_data->share_mmap_list[i].vm_start, + mm_data->share_mmap_list[i].vm_start + mm_data->share_mmap_list[i].size); + pr_debug("((addr - share_mmap_list[i].vm_start) / PAGE_SIZE): 0x%llx\n", + ((addr - mm_data->share_mmap_list[i].vm_start) / PAGE_SIZE)); + pr_debug("share_mmap_list[i].pfn: 0x%llx\n", mm_data->share_mmap_list[i].pfn); + corrected_pfn = ((addr - mm_data->share_mmap_list[i].vm_start) / PAGE_SIZE) + mm_data->share_mmap_list[i].pfn; + return corrected_pfn; + } + } + return 0; +} + +static void pf_mmap_close(struct vm_area_struct *vma) +{ + pr_debug("unmap\t-> vma->vm_start: 0x%lx\n", vma->vm_start); + pr_debug("unmap\t-> size: %lu\n", vma->vm_end - vma->vm_start); + share_mmap_rem(vma); +} + +static vm_fault_t pf_mmap_fault(struct vm_fault *vmf) +{ + uint64_t corrected_pfn; + pfn_t corr_pfn_struct; + struct page *page; + int ret = 0; + + struct file *file = vmf->vma->vm_file; + struct file_priv_data *file_data = (struct file_priv_data *)(file->private_data); + struct mmap_data *mm_data = (struct mmap_data *)file_data->mm_data; + + pr_debug("----- Page fault: %lld -----\n", mm_data->sum_pgfaults); + mm_data->sum_pgfaults++; + + /* Find the corrected pfn */ + corrected_pfn = share_mmap_exist_vma_return_correct_pfn(mm_data, vmf->address); + corr_pfn_struct.val = corrected_pfn; + + /* Some debug prints */ + pr_debug("vma->vm_start: 0x%lx\n", vmf->vma->vm_start); + pr_debug("vma->vm_pgoff: 0x%lx\n", vmf->vma->vm_pgoff); + pr_debug("vmf->address: 0x%lx\n", vmf->address); + pr_debug("corrected_pfn: 0x%llx\n", corrected_pfn); + pr_debug("pfn_valid(corrected_pfn): 0x%x\n", pfn_valid(corrected_pfn)); + + BUG_ON(!pfn_valid(corrected_pfn)); + + /* After finding the page, correct the vmf->page */ + page = pfn_to_page(corrected_pfn); + BUG_ON(!virt_addr_valid(page_address(page))); + + /* Insert the correct page */ + ret = vmf_insert_pfn(vmf->vma, vmf->address, corrected_pfn); + pr_debug("vmf_insert_pfn -> ret: %d\n", ret); + + return ret; +} + +const struct vm_operations_struct pf_mmap_ops = { + .close = pf_mmap_close, + .fault = pf_mmap_fault, +}; + +static int pf_mmap_vm_page(struct file *filp, struct vm_area_struct *vma) +{ + uint64_t size = (unsigned long)(vma->vm_end - vma->vm_start); + struct file_priv_data *file_data = (struct file_priv_data *)(filp->private_data); + struct mmap_data *mm_data = (struct mmap_data *)file_data->mm_data; + uint64_t pfn = ((mm_data->cur_ram_idx++) * 0x40000); + +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 3, 0) + vma->vm_flags |= VM_PFNMAP; +#else + vm_flags_set(vma, VM_PFNMAP); +#endif + add_share_mmap(filp, pfn, vma->vm_start, size); + return 0; +} + +static int mmap_vqs_com_struct(struct file *filp, struct vm_area_struct *vma) +{ + int ret = 0; + unsigned long size = (unsigned long)(vma->vm_end - vma->vm_start); + struct file_priv_data *file_data = (struct file_priv_data *)(filp->private_data); + struct device_data *dev_data = (struct device_data *)file_data->dev_data; + struct mmap_data *mmap_data = (struct mmap_data *)file_data->mm_data; + struct mmap_info *com_mmap_virt = (struct mmap_info *)(file_data->dev_data->info)->data; + uint64_t com_mmap_pfn = ((uint64_t)virt_to_phys(com_mmap_virt)) >> PAGE_SHIFT; + uint64_t starting_pfn; + + if (mmap_data->share_communication_struct) { +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 16, 0) + vma->vm_flags |= VM_RESERVED; +#else + vm_flags_set(vma, VM_RESERVED); +#endif + mmap_data->share_communication_struct = false; + starting_pfn = com_mmap_pfn; + } else { + mmap_data->share_vqs = false; + starting_pfn = dev_data->vq_data.vq_pfn; + } + + ret = remap_pfn_range(vma, vma->vm_start, starting_pfn, size, vma->vm_page_prot); + if (ret != 0) { + pr_err("Mmap error\n"); + print_mmaps(mmap_data); + goto out; + } + + add_share_mmap(filp, starting_pfn, vma->vm_start, size); + +out: + return ret; +} + +static int op_mmap(struct file *filp, struct vm_area_struct *vma) +{ + struct file_priv_data *file_data = (struct file_priv_data *)(filp->private_data); + struct mmap_data *mmap_data = (struct mmap_data *)file_data->mm_data; + int ret = 0; + + pr_debug("MMAP SYS_CALL -> vma->vm_pgoff: 0x%lx", vma->vm_pgoff); + vma->vm_ops = &pf_mmap_ops; + + if (mmap_data->share_communication_struct || mmap_data->share_vqs) { + ret = mmap_vqs_com_struct(filp, vma); + goto out; + } + + ret = pf_mmap_vm_page(filp, vma); + +out: + return ret; +} + +/* Defined for future work */ +static ssize_t loopback_write(struct file *file, + const char __user *user_buffer, + size_t size, + loff_t *offset) +{ + ssize_t len = sizeof(int); + + pr_debug("loopback write function\n"); + if (len <= 0) + return 0; + + return len; +} + +/* Defined for future work */ +static ssize_t loopback_read(struct file *file, + char __user *user_buffer, + size_t size, loff_t *offset) +{ + pr_debug("loopback read function\n"); + return 0; +} + +static loff_t loopback_seek(struct file *file, loff_t offset, int whence) +{ + loff_t new_pos; + + pr_debug("loopback seek function!\n"); + switch (whence) { + case SEEK_SET: + new_pos = offset; + break; + case SEEK_CUR: + new_pos = file->f_pos + offset; + break; + case SEEK_END: + new_pos = file->f_inode->i_size; + break; + default: + return -EINVAL; + } + + if (new_pos < 0 || new_pos > file->f_inode->i_size) + return -EINVAL; + + return new_pos; +} + +static int register_virtio_loopback_dev(uint32_t device_id) +{ + struct platform_device *pdev; + int err = 0; + + pr_info("Received request to register a new virtio-loopback-dev\n"); + + /* Register a new loopback-transport device */ + pdev = platform_device_register_simple("loopback-transport", device_id, NULL, 0); + if (IS_ERR(pdev)) { + err = PTR_ERR(pdev); + pr_err("failed to register loopback-transport device: %d\n", err); + } + + return err; +} + +/* Insert new entry data for a discovered device */ +int insert_entry_data(struct virtio_loopback_device *vl_dev, int id) +{ + int err = 0; + /* Read and that value atomically */ + uint32_t max_used_dev_idx = atomic_read(&loopback_devices.device_num); + + /* Store the new vl_dev */ + if ((id <= MAX_PDEV) && (max_used_dev_idx < MAX_PDEV)) { + loopback_devices.devices[id] = vl_dev; + } else { + err = -ENOMEM; + } + + /* Mark the request as completed and free registration */ + complete(&loopback_devices.reg_vl_dev_completion[id]); + return err; +} + +/* Helper function to mark an entry as active */ +static struct virtio_loopback_device *activate_entry_data(struct device_data *data, uint32_t curr_dev_id) +{ + struct virtio_loopback_device *vl_dev = NULL; + + /* See if there is any available device */ + if (curr_dev_id < MAX_PDEV) { + /* Find and store the data */ + vl_dev = loopback_devices.devices[curr_dev_id]; + vl_dev->data = data; + } + + return vl_dev; +} + +static int start_loopback(struct file_priv_data *file_data, uint32_t curr_dev_id) +{ + struct virtio_loopback_device *vl_dev; + int rc; + + /* Activate the entry */ + vl_dev = activate_entry_data(file_data->dev_data, curr_dev_id); + if (vl_dev) { + file_data->vl_dev_irq = vl_dev; + /* Register the activated vl_dev in the system */ + rc = loopback_register_virtio_dev(vl_dev); + } else { + pr_debug("No available entry found!\n"); + file_data->vl_dev_irq = NULL; + rc = -EFAULT; + } + + return rc; +} + +static long loopback_ioctl(struct file *file, + unsigned int cmd, unsigned long arg) +{ + struct efd_data efd_data; + int irq, err; + uint32_t queue_sel; + struct file_priv_data *file_data = (struct file_priv_data *)(file->private_data); + struct mmap_data *mm_data = (struct mmap_data *)file_data->mm_data; + struct device_data *dev_data = (struct device_data *)file_data->dev_data; + uint32_t curr_avail_dev_id; + + switch (cmd) { + case EFD_INIT: { + struct task_struct *userspace_task; + struct file *efd_file; + + if (copy_from_user(&efd_data, (struct efd_data *) arg, + sizeof(struct efd_data))) + return -EFAULT; + + userspace_task = pid_task(find_vpid(efd_data.pid), PIDTYPE_PID); + + rcu_read_lock(); +#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 11, 220) + efd_file = fcheck_files(userspace_task->files, efd_data.efd[0]); +#else +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 7, 0) + efd_file = files_lookup_fd_rcu(userspace_task->files, efd_data.efd[0]); +#else + efd_file = files_lookup_fd_raw(userspace_task->files, efd_data.efd[0]); +#endif +#endif + rcu_read_unlock(); + + dev_data->efd_ctx = eventfd_ctx_fileget(efd_file); + if (!dev_data->efd_ctx) + return -1; + + break; + } + case WAKEUP: { + atomic_set(&((struct virtio_neg *)(dev_data->info->data))->done, 1); + wake_up(&(dev_data)->wq); + break; + } + case START_LOOPBACK: { + if (copy_from_user(&(file_data)->device_info, (struct virtio_device_info_struct *) arg, + sizeof(struct virtio_device_info_struct))) + return -EFAULT; + + /* Read and increase that value atomically */ + curr_avail_dev_id = atomic_add_return(1, &loopback_devices.device_num) - 1; + + /* Register a new loopback device */ + err = register_virtio_loopback_dev(curr_avail_dev_id); + if (err) + return -EFAULT; + + /* Wait for probe function to be called before return control to user-space app */ + wait_for_completion(&loopback_devices.reg_vl_dev_completion[curr_avail_dev_id]); + + /* Start the loopback */ + err = start_loopback(file_data, curr_avail_dev_id); + if (err) + return -EFAULT; + + break; + } + case IRQ: + if (copy_from_user(&irq, (int *) arg, sizeof(int))) + return -EFAULT; + pr_debug("\nIRQ\n"); + /* + * Both of the interrupt ways work but a) is more stable + * and b) has better performance: + * a) vl_interrupt(NULL); + * b) queue_work(interrupt_workqueue, &async_interrupt); + */ + /* Call the function */ + vl_interrupt(file_data->vl_dev_irq, irq); + break; + case SHARE_VQS: + if (copy_from_user(&queue_sel, (uint32_t *) arg, sizeof(uint32_t))) + return -EFAULT; + pr_debug("\n\nSHARE_VQS: %u\n\n", queue_sel); + dev_data->vq_data.vq_pfn = dev_data->vq_data.vq_pfns[queue_sel]; + pr_debug("Selected pfn is: 0x%llx", dev_data->vq_data.vq_pfn); + mm_data->share_vqs = true; + break; + case SHARE_COM_STRUCT: + mm_data->share_communication_struct = true; + break; + default: + pr_err("Unknown loopback ioctl: %u\n", cmd); + return -ENOTTY; + } + + return 0; +} + +static int loopback_open(struct inode *inode, struct file *file) +{ + uint32_t val_1gb = 1024 * 1024 * 1024; // 1GB + struct virtio_neg device_neg = {.done = ATOMIC_INIT(0)}; + /* Allocate file private data */ + struct file_priv_data *file_data = kmalloc(sizeof(struct file_priv_data), + GFP_KERNEL); + struct device_data *dev_data = kmalloc(sizeof(struct device_data), GFP_KERNEL); + struct mmap_data *mm_data = kmalloc(sizeof(struct mmap_data), GFP_KERNEL); + + if (!file_data || !dev_data || !mm_data) + goto error_kmalloc; + + /* Set the i_size for the stat SYS_CALL*/ + file->f_inode->i_size = 10 * val_1gb; + + /* Initialize the device data */ + dev_data->info = kmalloc(sizeof(struct mmap_info), GFP_KERNEL); + if (!dev_data->info) + goto error_kmalloc; + dev_data->info->data = (void *)get_zeroed_page(GFP_KERNEL); + memcpy(dev_data->info->data, &device_neg, sizeof(struct virtio_neg)); + + /* Init wq */ + init_waitqueue_head(&(dev_data)->wq); + + /* Init mutex */ + mutex_init(&(dev_data)->read_write_lock); + + /* Init vq_data */ + dev_data->vq_data.vq_index = 0; + dev_data->valid_eventfd = true; + file_data->dev_data = dev_data; + + /* Init file mmap_data */ + mm_data->mmap_index = 0; + mm_data->share_communication_struct = false; + mm_data->share_vqs = false; + mm_data->cur_ram_idx = 0; + mm_data->sum_pgfaults = 0; + file_data->mm_data = mm_data; + + /* Store in the private data as it should */ + file->private_data = (struct file_priv_data *)file_data; + + return 0; + +error_kmalloc: + kfree(file_data); + kfree(dev_data); + kfree(mm_data); + return -ENOMEM; +} + +static int loopback_release(struct inode *inode, struct file *file) +{ + struct file_priv_data *file_data = (struct file_priv_data *)(file->private_data); + struct device_data *dev_data = (struct device_data *)file_data->dev_data; + struct mmap_data *mm_data = (struct mmap_data *)file_data->mm_data; + + pr_info("Release the device\n"); + /* + * This makes the read/write do not wait + * for the virtio-loopback-adapter if + * the last has closed the fd + */ + dev_data->valid_eventfd = false; + /* Active entry found */ + if (file_data->vl_dev_irq) { + pr_debug("About to cancel the work\n"); + /* Cancel any pending work */ + cancel_work_sync(&file_data->vl_dev_irq->notify_work); + /* Continue with the vl_dev unregister */ + virtio_loopback_remove(file_data->vl_dev_irq->pdev); + file_data->vl_dev_irq = NULL; + } + /* Subsequently free the dev_data */ + free_page((unsigned long)dev_data->info->data); + kfree(dev_data->info); + eventfd_ctx_put(dev_data->efd_ctx); + dev_data->efd_ctx = NULL; + kfree(dev_data); + file_data->dev_data = NULL; + /* Continue with the mm_data */ + kfree(mm_data); + file_data->mm_data = NULL; + /* Last, free the private data */ + kfree(file_data); + file->private_data = NULL; + + return 0; +} + +static const struct file_operations fops = { + .owner = THIS_MODULE, + .read = loopback_read, + .write = loopback_write, + .open = loopback_open, + .unlocked_ioctl = loopback_ioctl, + .mmap = op_mmap, + .llseek = loopback_seek, + .release = loopback_release +}; + +static int __init loopback_init(void) +{ + int err, i; + dev_t dev; + + err = alloc_chrdev_region(&dev, 0, MAX_DEV, "loopback"); + + /* Set-up the loopback_data */ + loopback_data.dev_major = MAJOR(dev); +#if LINUX_VERSION_CODE < KERNEL_VERSION(6, 4, 0) + loopback_data.class = class_create(THIS_MODULE, "loopback"); +#else + loopback_data.class = class_create("loopback"); +#endif + if (IS_ERR(loopback_data.class)) { + pr_err("Failed to create class\n"); + return PTR_ERR(loopback_data.class); + } + cdev_init(&loopback_data.cdev, &fops); + loopback_data.cdev.owner = THIS_MODULE; + cdev_add(&loopback_data.cdev, MKDEV(loopback_data.dev_major, 0), 1); + device_create(loopback_data.class, NULL, MKDEV(loopback_data.dev_major, 0), + NULL, "loopback"); + /* Create the workqueues of the loopback driver */ + loopback_data.notify_workqueue = create_singlethread_workqueue("notify_workqueue"); + + /* Register virtio_loopback_transport */ + (void)platform_driver_register(&virtio_loopback_driver); + + /* Init loopback device array */ + atomic_set(&loopback_devices.device_num, 1); + + /* Init completion for all devices */ + for (i = 0; i < MAX_PDEV; i++) + init_completion(&loopback_devices.reg_vl_dev_completion[i]); + + return 0; +} + +static void __exit loopback_exit(void) +{ + int i; + uint32_t max_used_device_num = atomic_read(&loopback_devices.device_num); + + pr_info("Exit driver!\n"); + + /* Unregister loopback device */ + for (i = 0; i < max_used_device_num; i++) + if (loopback_devices.devices[i]) + platform_device_unregister(loopback_devices.devices[i]->pdev); + + /* Unregister virtio_loopback_transport */ + platform_driver_unregister(&virtio_loopback_driver); + pr_debug("platform_driver_unregister!\n"); + + /* Necessary actions for the loopback_data */ + device_destroy(loopback_data.class, MKDEV(loopback_data.dev_major, 0)); + cdev_del(&loopback_data.cdev); + pr_debug("device_destroy!\n"); + class_destroy(loopback_data.class); + pr_debug("class_destroy!\n"); + + /* Destroy the notify workqueue */ + flush_workqueue(loopback_data.notify_workqueue); + destroy_workqueue(loopback_data.notify_workqueue); +} + +module_init(loopback_init); +module_exit(loopback_exit); diff --git a/virtio_loopback_driver.h b/virtio_loopback_driver.h new file mode 100644 index 0000000..d0a744b --- /dev/null +++ b/virtio_loopback_driver.h @@ -0,0 +1,253 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright 2022-2024 Virtual Open Systems SAS. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software Foundation, + * Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. + */ +#ifndef __LOOPBACK_H__ +#define __LOOPBACK_H__ + +#define DRIVER "LOOPBACK" + +#include <linux/cdev.h> +#include <linux/eventfd.h> +#include <linux/fdtable.h> +#include <linux/init.h> +#include <linux/interrupt.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/of_address.h> +#include <linux/cpumask.h> +#include <linux/smp.h> +#include <linux/version.h> +#include <linux/completion.h> + +/* MMIO includes */ +#include <linux/acpi.h> +#include <linux/dma-mapping.h> +#include <linux/highmem.h> +#include <linux/io.h> +#include <linux/list.h> +#include <linux/spinlock.h> +#include <linux/virtio.h> +#include <linux/virtio_config.h> +#include <uapi/linux/virtio_mmio.h> +#include <linux/virtio_ring.h> + +#include <linux/kernel.h> +#include <linux/pid.h> +#include <linux/sched.h> +#include <linux/rcupdate.h> +#include <linux/kthread.h> + +/* mmap includes */ +#include <linux/kernel.h> +#include <linux/fs.h> +#include <linux/device.h> +#include <linux/mutex.h> + +#include <linux/pagemap.h> +#include <linux/delay.h> + +/* max Minor devices */ +#define MAX_DEV 1 +#define MAX_PDEV 100 +#define PDEV_TYPES 2 + +/* Define mmap elements limit */ +#define MMAP_LIMIT 200 + +/* + * The alignment to use between consumer and producer parts of vring. + * Currently hardcoded to the page size. + */ +#define VIRTIO_MMIO_VRING_ALIGN PAGE_SIZE + +#define to_virtio_loopback_device(ptr, field) \ + container_of(ptr, struct virtio_loopback_device, field) + +/* mmap functionality */ +#ifndef VM_RESERVED +#define VM_RESERVED (VM_DONTEXPAND | VM_DONTDUMP) +#endif + +/* IOCTL defines */ +#define EFD_INIT _IOC(_IOC_WRITE, 'k', 1, sizeof(efd_data)) +#define WAKEUP _IOC(_IOC_WRITE, 'k', 2, 0) +#define START_LOOPBACK _IOC(_IOC_WRITE, 'k', 3, sizeof(struct virtio_device_info_struct)) +#define IRQ _IOC(_IOC_WRITE, 'k', 4, sizeof(int)) +#define SHARE_VQS _IOC(_IOC_WRITE, 'k', 5, sizeof(uint32_t)) +#define SHARE_COM_STRUCT _IOC(_IOC_WRITE, 'k', 7, 0) + +/* Data structures */ +struct virtio_device_info_struct { + unsigned long magic; + unsigned long version; + unsigned long device_id; + unsigned long vendor; +}; + +struct virtio_neg { + uint64_t notification; + uint64_t data; + uint64_t size; + bool read; + atomic_t done; +}; + +struct share_mmap { + uint64_t pfn; + uint64_t vm_start; + uint32_t size; + uint32_t uid; + struct page *page; +}; + +struct mmap_data { + int mmap_index; + bool share_communication_struct; + bool share_vqs; + struct share_mmap share_mmap_list[MMAP_LIMIT]; + int cur_ram_idx; + uint64_t sum_pgfaults; +}; + +/* vq related data */ +struct vq_data { + uint32_t vq_index; + uint64_t vq_pfns[16]; + uint64_t vq_pfn; +}; + +/* Data describing each device private status */ +struct device_data { + /* Info needed for adapter ops */ + struct mmap_info *info; + /* Waitqueue for the adapter */ + wait_queue_head_t wq; + struct mutex read_write_lock; + struct eventfd_ctx *efd_ctx; + /* + * If this variable is true then read/write should wait + * the adapter to unlock this operation by sending an + * eventfd. If it's equal to "false" then the operation + * does not wait for adapter's confirmation. + */ + bool valid_eventfd; + /* vq data */ + struct vq_data vq_data; +}; + +/* Data describing each entry of the driver */ +struct loopback_devices_array { + /* Array of probed devices */ + struct virtio_loopback_device *devices[MAX_PDEV]; + /* Number of available devices */ + atomic_t device_num; + /* Registration completion */ + struct completion reg_vl_dev_completion[MAX_PDEV]; +}; + +/* Data concealed in the file private pointer */ +struct file_priv_data { + /* Device needed data */ + struct device_data *dev_data; + /* mmap needed data */ + struct mmap_data *mm_data; + /* Device info! */ + struct virtio_device_info_struct device_info; + /* The vl_dev pointer for the irq */ + struct virtio_loopback_device *vl_dev_irq; +}; + +struct virtio_loopback_device { + struct virtio_device vdev; + struct platform_device *pdev; + /* Corresponding data pointer */ + struct device_data *data; + + /* Status: -1 not initialized, 0 running, 1 paused */ + int status; + + void __iomem *base; + unsigned long version; + + /* A list of queues so we can dispatch IRQs */ + spinlock_t lock; + struct list_head virtqueues; + + /* Notify list and work struct */ + spinlock_t notify_q_lock; + struct list_head notify_list; + struct work_struct notify_work; +}; + +struct virtio_loopback_vq_info { + /* the actual virtqueue */ + struct virtqueue *vq; + /* the list node for the virtqueues list */ + struct list_head node; +}; + +/* Notify data*/ +struct notify_data { + uint32_t index; + struct list_head list; +}; + +/* Shared data structure between driver and user-space application */ +struct mmap_info { + void *data; + int reference; +}; + +/* + * This structure holds the eventfds shared between the driver + * and the user-space application. + */ +struct efd_data { + int efd[2]; + int pid; +}; + +/* device data holder, this structure may be extended to hold additional data */ +struct loopback_device_data { + /*device Major number */ + int dev_major; + /* sysfs class structure */ + struct class *class; + struct cdev cdev; + /* Define workqueue for notifications */ + struct workqueue_struct *notify_workqueue; +}; + +/* Global variables */ +extern struct loopback_device_data loopback_data; +extern struct loopback_devices_array loopback_devices; +extern struct platform_driver virtio_loopback_driver; + +/* Global functions */ +int insert_entry_data(struct virtio_loopback_device *vl_dev, int id); +int loopback_register_virtio_dev(struct virtio_loopback_device *vl_dev); +bool vl_interrupt(struct virtio_loopback_device *vl_dev, int irq); +#if LINUX_VERSION_CODE > KERNEL_VERSION(6, 10, 8) +void virtio_loopback_remove(struct platform_device *pdev); +#else +int virtio_loopback_remove(struct platform_device *pdev); +#endif + +#endif /* __LOOPBACK_H__ */ |