trusty: Add pKVM dma-buf heap module This patch introduces a standalone pKVM dma-buf heap module, serving as an example of how to use pKVM vendor hypercalls to protect/unprotect at stage-2. The module consists of: * EL1 driver: a modified system heap that issues pKVM vendor hypercalls to protect pages upon allocation and unprotect them upon release. It also exposes a device which is used to enable custom SMC forwarding. * EL2 module: handles stage-2 permission transitions and implements an SMC handler for guests to accept protected buffers. * Build integration: adds bazel build rules (ddk_module, ddk_library) integrating the module into main-kernel-build-2025, main-kernel-build-2026, and mainline. Note that the pKVM module is not built for mainline kernels. Based on aosp/3876668 by qperret@ Bug: 481446462 Test: tools/bazel run \ -- common-modules/trusty/build/main-kernel-build-2025:trusty_aarch64_dist \ --destdir out/trusty_aarch64_dist-2025 Test: tools/bazel run \ -- common-modules/trusty/build/main-kernel-build-2026:trusty_aarch64_dist \ --destdir out/trusty_aarch64_dist-2026 Test: tools/bazel run \ -- common-modules/trusty/build/mainline:trusty_aarch64_dist \ --destdir out/trusty_aarch64_dist-mainline Change-Id: I1e3076cc2d4875f8524b7ad9daebe8891d830391 Co-developed-by: Quentin Perret <[email protected]> Signed-off-by: Per Larsen <[email protected]>
diff --git a/build/main-kernel-build-2025/BUILD.bazel b/build/main-kernel-build-2025/BUILD.bazel index 3bd2e6c..2a54c55 100644 --- a/build/main-kernel-build-2025/BUILD.bazel +++ b/build/main-kernel-build-2025/BUILD.bazel
@@ -4,6 +4,7 @@ load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") load( "//build/kernel/kleaf:kernel.bzl", + "ddk_headers", "ddk_module", "kernel_abi", "kernel_build", @@ -32,6 +33,9 @@ "system_heap.fragment", ], strip_modules = True, + visibility = [ + "//common-modules/trusty:__subpackages__", + ], ) ddk_module( @@ -175,9 +179,41 @@ ], ) +ddk_headers( + name = "pkvm-dma-buf-heap-headers", + hdrs = glob([ + "drivers/pkvm-dma-buf-heap/include/*.h", + ]), + includes = [ + "drivers/pkvm-dma-buf-heap/include", + ], + visibility = [ + "//common-modules/trusty/build/main-kernel-build-2025/drivers/pkvm-dma-buf-heap/hyp:__pkg__", + ], +) + +# NOTE: it would be cleaner to put the ddk_library rule here but that leads +# to linking errors for the code running in EL2. The ddk_library rule is +# deliberately put in the build file for the hyp directory as a workaround. +ddk_module( + name = "pkvm-dma-buf-heap", + srcs = [ + "drivers/pkvm-dma-buf-heap/el1.c", + "drivers/pkvm-dma-buf-heap/system_heap_modified.c", + ], + out = "pkvm-dma-buf-heap.ko", + kernel_build = ":trusty_aarch64", + deps = [ + ":pkvm-dma-buf-heap-headers", + "//common-modules/trusty/build/main-kernel-build-2025/drivers/pkvm-dma-buf-heap/hyp:hyp-2025", + "//common-modules/trusty/build/main-kernel-build-2025/drivers/pkvm-dma-buf-heap/hyp:hyp-headers", + ], +) + kernel_module_group( name = "trusty_aarch64_external_modules", srcs = [ + ":pkvm-dma-buf-heap", ":trusty-core", ":trusty-ffa", ":trusty-ipc",
diff --git a/build/main-kernel-build-2026/BUILD.bazel b/build/main-kernel-build-2026/BUILD.bazel index 3bd2e6c..7570fda 100644 --- a/build/main-kernel-build-2026/BUILD.bazel +++ b/build/main-kernel-build-2026/BUILD.bazel
@@ -4,6 +4,7 @@ load("@rules_pkg//pkg:mappings.bzl", "pkg_files", "strip_prefix") load( "//build/kernel/kleaf:kernel.bzl", + "ddk_headers", "ddk_module", "kernel_abi", "kernel_build", @@ -32,6 +33,9 @@ "system_heap.fragment", ], strip_modules = True, + visibility = [ + "//common-modules/trusty:__subpackages__", + ], ) ddk_module( @@ -175,9 +179,41 @@ ], ) +ddk_headers( + name = "pkvm-dma-buf-heap-headers", + hdrs = glob([ + "drivers/pkvm-dma-buf-heap/include/*.h", + ]), + includes = [ + "drivers/pkvm-dma-buf-heap/include", + ], + visibility = [ + "//common-modules/trusty/build/main-kernel-build-2026/drivers/pkvm-dma-buf-heap/hyp:__pkg__", + ], +) + +# NOTE: it would be cleaner to put the ddk_library rule here but that leads +# to linking errors for the code running in EL2. The ddk_library rule is +# deliberately put in the build file for the hyp directory as a workaround. +ddk_module( + name = "pkvm-dma-buf-heap", + srcs = [ + "drivers/pkvm-dma-buf-heap/el1.c", + "drivers/pkvm-dma-buf-heap/system_heap_modified.c", + ], + out = "pkvm-dma-buf-heap.ko", + kernel_build = ":trusty_aarch64", + deps = [ + ":pkvm-dma-buf-heap-headers", + "//common-modules/trusty/build/main-kernel-build-2026/drivers/pkvm-dma-buf-heap/hyp:hyp-2026", + "//common-modules/trusty/build/main-kernel-build-2026/drivers/pkvm-dma-buf-heap/hyp:hyp-headers", + ], +) + kernel_module_group( name = "trusty_aarch64_external_modules", srcs = [ + ":pkvm-dma-buf-heap", ":trusty-core", ":trusty-ffa", ":trusty-ipc",
diff --git a/build/mainline/BUILD.bazel b/build/mainline/BUILD.bazel index 8e25a3c..e0559ff 100644 --- a/build/mainline/BUILD.bazel +++ b/build/mainline/BUILD.bazel
@@ -30,6 +30,9 @@ "system_heap.fragment", ], strip_modules = True, + visibility = [ + "//common-modules/trusty:__subpackages__", + ], ) ddk_module( @@ -173,6 +176,12 @@ ], ) +# NOTE: we are not building the pkvm-dma-buf-heap module for mainline kernels +# because they do not support building pKVM modules with DDK. The test is +# whether $(srctree)/arch/arm64/kvm/hyp/nvhe/Makefile.module exists or not. +# +# Source: comment in _define_common_kernels_additional_tests in +# build/kernel/kleaf/common_kernels.bzl. kernel_module_group( name = "trusty_aarch64_external_modules", srcs = [
diff --git a/drivers/pkvm-dma-buf-heap/el1.c b/drivers/pkvm-dma-buf-heap/el1.c new file mode 100644 index 0000000..54f85ee --- /dev/null +++ b/drivers/pkvm-dma-buf-heap/el1.c
@@ -0,0 +1,82 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include <asm/esr.h> +#include <asm/kvm_pkvm.h> +#include <asm/kvm_pkvm_module.h> +#include <asm/word-at-a-time.h> +#include <kunit/test.h> +#include <linux/dma-heap.h> +#include <linux/init.h> +#include <linux/miscdevice.h> +#include <linux/module.h> +#include <linux/kernel.h> + +#include "el2.h" + +#ifndef MODULE +BUILD_BUG("Example pKVM dma-buf heap must be compiled as a module"); +#endif + +unsigned long protect_page_hvc, unprotect_page_hvc; + +static long pkvm_dma_buf_heap_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) +{ + struct file *f; + int ret; + + if (ioctl != PKVM_DMA_BUF_HEAP_IOCTL_ENABLE_SMC) + return -ENOTSUPP; + f = fget(arg); + if (!f) + return -EINVAL; + + ret = pkvm_enable_smc_forwarding(f); + fput(f); + + return ret; +} + +static struct file_operations pkvm_dma_buf_heap_ops = { + .unlocked_ioctl = pkvm_dma_buf_heap_ioctl, +}; + +static struct miscdevice pkvm_dma_buf_heap_dev = { + MISC_DYNAMIC_MINOR, + "pkvm_dma_buf_heap", + &pkvm_dma_buf_heap_ops, +}; + +static int host_init(void) +{ + struct dma_heap_export_info exp_info; + struct dma_heap *ex_heap; + int ret; + + exp_info.name = "pkvm_dma_buf_heap"; + exp_info.ops = &system_heap_modified_ops; + exp_info.priv = NULL; + + ex_heap = dma_heap_add(&exp_info); + if (IS_ERR(ex_heap)) + return PTR_ERR(ex_heap); + + ret = pkvm_load_el2_module(__kvm_nvhe_hyp_init); + if (ret) + return ret; + + protect_page_hvc = pkvm_register_el2_mod_call(__kvm_nvhe_protect_page); + if (protect_page_hvc < 0) + return -EINVAL; + + unprotect_page_hvc = pkvm_register_el2_mod_call(__kvm_nvhe_unprotect_page); + if (unprotect_page_hvc < 0) + return -EINVAL; + + return misc_register(&pkvm_dma_buf_heap_dev); +} +module_init(host_init); + +MODULE_DESCRIPTION("pKVM dma-buf heap"); +MODULE_LICENSE("GPL v2"); +MODULE_IMPORT_NS("DMA_BUF"); +MODULE_IMPORT_NS("DMA_BUF_HEAP"); \ No newline at end of file
diff --git a/drivers/pkvm-dma-buf-heap/hyp/BUILD.bazel b/drivers/pkvm-dma-buf-heap/hyp/BUILD.bazel new file mode 100644 index 0000000..5e1819c --- /dev/null +++ b/drivers/pkvm-dma-buf-heap/hyp/BUILD.bazel
@@ -0,0 +1,63 @@ +# Copyright (C) 2025 The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +load("@kleaf//build/kernel/kleaf:kernel.bzl", "ddk_headers", "ddk_library") + +ddk_headers( + name = "hyp-headers", + hdrs = glob([ + "*.h", + ]), + includes = [ + ".", + ], + visibility = [ + "//common-modules/trusty:__subpackages__", + ], +) + +# NOTE: keep in sync with :hyp target modulo name and kernel_build +# TODO: get rid of duplicate ddk_library targets like this one +ddk_library( + name = "hyp-2025", + srcs = ["el2.c"], + kernel_build = "//common-modules/trusty/build/main-kernel-build-2025:trusty_aarch64", + # Indicate that this builds EL2 hypervisor code. + pkvm_el2 = True, + visibility = [ + "//common-modules/trusty:__subpackages__", + ], + deps = [ + ":hyp-headers", + "//common-modules/trusty/build/main-kernel-build-2025:pkvm-dma-buf-heap-headers", + ], +) + +# NOTE: keep in sync with :hyp target modulo name and kernel_build +# TODO: get rid of duplicate ddk_library targets like this one +ddk_library( + name = "hyp-2026", + srcs = ["el2.c"], + kernel_build = "//common-modules/trusty/build/main-kernel-build-2026:trusty_aarch64", + # Indicate that this builds EL2 hypervisor code. + pkvm_el2 = True, + visibility = [ + "//common-modules/trusty:__subpackages__", + ], + deps = [ + ":hyp-headers", + "//common-modules/trusty/build/main-kernel-build-2026:pkvm-dma-buf-heap-headers", + ], +) + +# NOTE: entry for mainline kernel omitted as it doesn't support pKVM modules
diff --git a/drivers/pkvm-dma-buf-heap/hyp/el2.c b/drivers/pkvm-dma-buf-heap/hyp/el2.c new file mode 100644 index 0000000..9b71822 --- /dev/null +++ b/drivers/pkvm-dma-buf-heap/hyp/el2.c
@@ -0,0 +1,77 @@ +//SPDX-License-Identifier: GPL-2.0 + +/* + * TODO: Avoid __NO_FORTIFY + * pkvm_module_ops.memcpy triggers the fortified memcpy macro. + */ +#define __NO_FORTIFY + +#include <asm/kvm_hyp.h> +#include <asm/kvm_pkvm_module.h> + +#include "el2.h" +#include "pkvm_dma_buf_heap.h" + +static const struct pkvm_module_ops *ops; + +void protect_page(struct user_pt_regs *regs) +{ + regs->regs[0] = SMCCC_RET_SUCCESS; + regs->regs[1] = ops->host_stage2_mod_prot(regs->regs[1], 0, regs->regs[2], true); +} + +void unprotect_page(struct user_pt_regs *regs) +{ + regs->regs[0] = SMCCC_RET_SUCCESS; + regs->regs[1] = ops->host_stage2_mod_prot(regs->regs[1], KVM_PGTABLE_PROT_RWX, + regs->regs[2], true); +} + +static enum pkvm_smc_handler_ret smc_handler(struct arm_smccc_1_2_regs *regs, + struct arm_smccc_1_2_regs *res, + pkvm_handle_t handle) +{ + u64 ipa, nr_pages; + int ret; + + ops->memcpy(res, regs, sizeof(*res)); + + if (regs->a0 != SMC_ACCEPT_SECURE_BUF) + return GUEST_SMC_NOT_HANDLED; + + /* TODO: validate guest identity using @handle !*/ + + ipa = regs->a1; + nr_pages = regs->a2; + ret = ops->guest_accept_module_prot_page(ipa, nr_pages); + if (ret == -ENOMEM) + return GUEST_SMC_NEED_TOPUP; + + res->a0 = (u64)ret; + + return GUEST_SMC_HANDLED; +} + +static int module_owned_fault_handler(u64 phys, u64 ipa, u64 size, pkvm_handle_t handle) +{ + /* TODO: check the pfn ! */ + + return 0; +} + +int hyp_init(const struct pkvm_module_ops *__ops) +{ + int ret; + + ops = __ops; + + ret = ops->register_guest_smc_handler(smc_handler); + if (ret) + return ret; + + ret = ops->register_guest_accept_module_owned_handler(module_owned_fault_handler); + if (ret) + return ret; + + return 0; +}
diff --git a/drivers/pkvm-dma-buf-heap/hyp/el2.h b/drivers/pkvm-dma-buf-heap/hyp/el2.h new file mode 100644 index 0000000..2268c61 --- /dev/null +++ b/drivers/pkvm-dma-buf-heap/hyp/el2.h
@@ -0,0 +1,46 @@ +#ifndef __PKVM_DMA_BUF_HEAP_EL2_H +#define __PKVM_DMA_BUF_HEAP_EL2_H + +#ifdef __KVM_NVHE_HYPERVISOR__ +int hyp_init(const struct pkvm_module_ops *__ops); +/** + * protect_page() - Protect memory pages from the host. + * @regs: Pointer to the CPU registers structure containing the hypercall arguments. + * + * Revokes host access to a memory range by stripping permissions in the host + * stage-2 page tables. Expected arguments are the base PFN in regs->regs[1] + * and the number of pages in regs->regs[2]. The result of the operation is + * stored back in regs->regs[1]. + */ +void protect_page(struct user_pt_regs *regs); + +/** + * unprotect_page() - Restore host access to memory pages. + * @regs: Pointer to the CPU registers structure containing the hypercall arguments. + * + * Restores Read, Write, and Execute (RWX) permissions for the host over a + * specified memory range in the host stage-2 page tables. Expected arguments + * are the base PFN in regs->regs[1] and the number of pages in regs->regs[2]. + * The result of the operation is stored back in regs->regs[1]. + */ +void unprotect_page(struct user_pt_regs *regs); +#else +int __kvm_nvhe_hyp_init(const struct pkvm_module_ops *__ops); +/** + * __kvm_nvhe_protect_page() - EL1 alias for protect_page + */ +void __kvm_nvhe_protect_page(struct user_pt_regs *regs); +/** + * __kvm_nvhe_unprotect_page() - EL1 alias for unprotect_page + */ +void __kvm_nvhe_unprotect_page(struct user_pt_regs *regs); + +extern unsigned long protect_page_hvc; +extern unsigned long unprotect_page_hvc; +extern const struct dma_heap_ops system_heap_modified_ops; + +#define PKVM_DMA_BUF_HEAP_IOC_MAGIC 'P' +#define PKVM_DMA_BUF_HEAP_IOCTL_ENABLE_SMC _IOWR(PKVM_DMA_BUF_HEAP_IOC_MAGIC, 0x0, __u32) +#endif /* __KVM_NVHE_HYPERVISOR__ */ + +#endif /* __PKVM_DMA_BUF_HEAP_EL2_H */ \ No newline at end of file
diff --git a/drivers/pkvm-dma-buf-heap/include/pkvm_dma_buf_heap.h b/drivers/pkvm-dma-buf-heap/include/pkvm_dma_buf_heap.h new file mode 100644 index 0000000..3bbead6 --- /dev/null +++ b/drivers/pkvm-dma-buf-heap/include/pkvm_dma_buf_heap.h
@@ -0,0 +1,6 @@ +#ifndef _PKVM_DMA_BUF_HEAP_H +#define _PKVM_DMA_BUF_HEAP_H + +#define SMC_ACCEPT_SECURE_BUF 0x123 + +#endif /* _PKVM_DMA_BUF_HEAP_H */ \ No newline at end of file
diff --git a/drivers/pkvm-dma-buf-heap/system_heap_modified.c b/drivers/pkvm-dma-buf-heap/system_heap_modified.c new file mode 100644 index 0000000..35c8d9f --- /dev/null +++ b/drivers/pkvm-dma-buf-heap/system_heap_modified.c
@@ -0,0 +1,488 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copied from drivers/dma-buf/heaps/system_heap.c with minimal + * changes (only a few extra pKVM protection hypercalls) for example + * purposes. + */ + +#include <linux/dma-buf.h> +#include <linux/dma-direct.h> +#include <linux/dma-mapping.h> +#include <linux/dma-heap.h> +#include <linux/err.h> +#include <linux/highmem.h> +#include <linux/iommu.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/printk.h> +#include <linux/scatterlist.h> +#include <linux/swiotlb.h> +#include <linux/vmalloc.h> + +#include <asm/kvm_pkvm_module.h> +#include "el2.h" + +struct system_heap_buffer { + struct dma_heap *heap; + struct list_head attachments; + struct mutex lock; + unsigned long len; + struct sg_table sg_table; + int vmap_cnt; + void *vaddr; + + bool uncached; +}; + +struct dma_heap_attachment { + struct device *dev; + struct sg_table table; + struct list_head list; + bool mapped; + + bool uncached; +}; + +#define LOW_ORDER_GFP (GFP_HIGHUSER | __GFP_ZERO) +#define HIGH_ORDER_GFP (((GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN \ + | __GFP_NORETRY) & ~__GFP_RECLAIM) \ + | __GFP_COMP) +static gfp_t order_flags[] = {HIGH_ORDER_GFP, HIGH_ORDER_GFP, LOW_ORDER_GFP}; +/* + * The selection of the orders used for allocation (1MB, 64K, 4K) is designed + * to match with the sizes often found in IOMMUs. Using order 4 pages instead + * of order 0 pages can significantly improve the performance of many IOMMUs + * by reducing TLB pressure and time spent updating page tables. + */ +static const unsigned int orders[] = {8, 4, 0}; +#define NUM_ORDERS ARRAY_SIZE(orders) + +static bool needs_swiotlb_bounce(struct device *dev, struct sg_table *table) +{ + struct iommu_domain *domain = iommu_get_domain_for_dev(dev); + struct scatterlist *sg; + int i; + + for_each_sgtable_dma_sg(table, sg, i) { + // SG_DMA_SWIOTLB is set only for dma-iommu, not for dma-direct + if (domain && IS_ENABLED(CONFIG_NEED_SG_DMA_FLAGS)) { + if (sg_dma_is_swiotlb(table->sgl)) + return true; + } else { + phys_addr_t paddr = domain ? + iommu_iova_to_phys(domain, sg_dma_address(sg)) : + dma_to_phys(dev, sg_dma_address(sg)); + if (swiotlb_find_pool(dev, paddr)) + return true; + } + } + return false; +} + +static int dup_sg_table(struct sg_table *from, struct sg_table *to) +{ + struct scatterlist *sg, *new_sg; + int ret, i; + + ret = sg_alloc_table(to, from->orig_nents, GFP_KERNEL); + if (ret) + return ret; + + new_sg = to->sgl; + for_each_sgtable_sg(from, sg, i) { + sg_set_page(new_sg, sg_page(sg), sg->length, sg->offset); + new_sg = sg_next(new_sg); + } + + return 0; +} + +static int system_heap_attach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attachment) +{ + struct system_heap_buffer *buffer = dmabuf->priv; + struct dma_heap_attachment *a; + int ret; + + a = kzalloc(sizeof(*a), GFP_KERNEL); + if (!a) + return -ENOMEM; + + ret = dup_sg_table(&buffer->sg_table, &a->table); + if (ret) { + kfree(a); + return ret; + } + + a->dev = attachment->dev; + INIT_LIST_HEAD(&a->list); + a->mapped = false; + a->uncached = buffer->uncached; + attachment->priv = a; + + mutex_lock(&buffer->lock); + list_add(&a->list, &buffer->attachments); + mutex_unlock(&buffer->lock); + + return 0; +} + +static void system_heap_detach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attachment) +{ + struct system_heap_buffer *buffer = dmabuf->priv; + struct dma_heap_attachment *a = attachment->priv; + + mutex_lock(&buffer->lock); + list_del(&a->list); + mutex_unlock(&buffer->lock); + + sg_free_table(&a->table); + kfree(a); +} + +static struct sg_table *system_heap_map_dma_buf(struct dma_buf_attachment *attachment, + enum dma_data_direction direction) +{ + struct dma_heap_attachment *a = attachment->priv; + struct sg_table *table = &a->table; + int attr = attachment->dma_map_attrs; + int ret; + + if (a->uncached) + attr |= DMA_ATTR_SKIP_CPU_SYNC; + + ret = dma_map_sgtable(attachment->dev, table, direction, attr); + if (ret) + return ERR_PTR(ret); + + if (a->uncached && needs_swiotlb_bounce(attachment->dev, table)) { + pr_err("Cannot map uncached system heap buffer for %s, as it requires SWIOTLB", + dev_name(attachment->dev)); + dma_unmap_sgtable(attachment->dev, table, direction, attr); + return ERR_PTR(-EINVAL); + } + + a->mapped = true; + return table; +} + +static void system_heap_unmap_dma_buf(struct dma_buf_attachment *attachment, + struct sg_table *table, + enum dma_data_direction direction) +{ + struct dma_heap_attachment *a = attachment->priv; + int attr = attachment->dma_map_attrs; + + if (a->uncached) + attr |= DMA_ATTR_SKIP_CPU_SYNC; + a->mapped = false; + dma_unmap_sgtable(attachment->dev, table, direction, attr); +} + +static int system_heap_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, + enum dma_data_direction direction) +{ + struct system_heap_buffer *buffer = dmabuf->priv; + struct dma_heap_attachment *a; + + mutex_lock(&buffer->lock); + + if (buffer->vmap_cnt) + invalidate_kernel_vmap_range(buffer->vaddr, buffer->len); + + if (!buffer->uncached) { + list_for_each_entry(a, &buffer->attachments, list) { + if (!a->mapped) + continue; + dma_sync_sgtable_for_cpu(a->dev, &a->table, direction); + } + } + mutex_unlock(&buffer->lock); + + return 0; +} + +static int system_heap_dma_buf_end_cpu_access(struct dma_buf *dmabuf, + enum dma_data_direction direction) +{ + struct system_heap_buffer *buffer = dmabuf->priv; + struct dma_heap_attachment *a; + + mutex_lock(&buffer->lock); + + if (buffer->vmap_cnt) + flush_kernel_vmap_range(buffer->vaddr, buffer->len); + + if (!buffer->uncached) { + list_for_each_entry(a, &buffer->attachments, list) { + if (!a->mapped) + continue; + dma_sync_sgtable_for_device(a->dev, &a->table, direction); + } + } + mutex_unlock(&buffer->lock); + + return 0; +} + +static int system_heap_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) +{ + struct system_heap_buffer *buffer = dmabuf->priv; + struct sg_table *table = &buffer->sg_table; + unsigned long addr = vma->vm_start; + struct sg_page_iter piter; + int ret; + + if (buffer->uncached) + vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot); + + for_each_sgtable_page(table, &piter, vma->vm_pgoff) { + struct page *page = sg_page_iter_page(&piter); + + ret = remap_pfn_range(vma, addr, page_to_pfn(page), PAGE_SIZE, + vma->vm_page_prot); + if (ret) + return ret; + addr += PAGE_SIZE; + if (addr >= vma->vm_end) + return 0; + } + return 0; +} + +static void *system_heap_do_vmap(struct system_heap_buffer *buffer) +{ + struct sg_table *table = &buffer->sg_table; + int npages = PAGE_ALIGN(buffer->len) / PAGE_SIZE; + struct page **pages = vmalloc(sizeof(struct page *) * npages); + struct page **tmp = pages; + struct sg_page_iter piter; + pgprot_t pgprot = PAGE_KERNEL; + void *vaddr; + + if (!pages) + return ERR_PTR(-ENOMEM); + + if (buffer->uncached) + pgprot = pgprot_writecombine(PAGE_KERNEL); + + for_each_sgtable_page(table, &piter, 0) { + WARN_ON(tmp - pages >= npages); + *tmp++ = sg_page_iter_page(&piter); + } + + vaddr = vmap(pages, npages, VM_MAP, pgprot); + vfree(pages); + + if (!vaddr) + return ERR_PTR(-ENOMEM); + + return vaddr; +} + +static int system_heap_vmap(struct dma_buf *dmabuf, struct iosys_map *map) +{ + struct system_heap_buffer *buffer = dmabuf->priv; + void *vaddr; + int ret = 0; + + mutex_lock(&buffer->lock); + if (buffer->vmap_cnt) { + buffer->vmap_cnt++; + iosys_map_set_vaddr(map, buffer->vaddr); + goto out; + } + + vaddr = system_heap_do_vmap(buffer); + if (IS_ERR(vaddr)) { + ret = PTR_ERR(vaddr); + goto out; + } + + buffer->vaddr = vaddr; + buffer->vmap_cnt++; + iosys_map_set_vaddr(map, buffer->vaddr); +out: + mutex_unlock(&buffer->lock); + + return ret; +} + +static void system_heap_vunmap(struct dma_buf *dmabuf, struct iosys_map *map) +{ + struct system_heap_buffer *buffer = dmabuf->priv; + + mutex_lock(&buffer->lock); + if (!--buffer->vmap_cnt) { + vunmap(buffer->vaddr); + buffer->vaddr = NULL; + } + mutex_unlock(&buffer->lock); + iosys_map_clear(map); +} + +static void system_heap_dma_buf_release(struct dma_buf *dmabuf) +{ + struct system_heap_buffer *buffer = dmabuf->priv; + struct sg_table *table; + struct scatterlist *sg; + int i; + + table = &buffer->sg_table; + for_each_sgtable_sg(table, sg, i) { + struct page *page = sg_page(sg); + + pkvm_el2_mod_call(unprotect_page_hvc, page_to_pfn(page), 1 << compound_order(page)); + __free_pages(page, compound_order(page)); + } + sg_free_table(table); + kfree(buffer); +} + +static const struct dma_buf_ops system_heap_buf_ops = { + .attach = system_heap_attach, + .detach = system_heap_detach, + .map_dma_buf = system_heap_map_dma_buf, + .unmap_dma_buf = system_heap_unmap_dma_buf, + .begin_cpu_access = system_heap_dma_buf_begin_cpu_access, + .end_cpu_access = system_heap_dma_buf_end_cpu_access, + .mmap = system_heap_mmap, + .vmap = system_heap_vmap, + .vunmap = system_heap_vunmap, + .release = system_heap_dma_buf_release, +}; + +static struct page *alloc_largest_available(unsigned long size, + unsigned int max_order) +{ + struct page *page; + int i; + + for (i = 0; i < NUM_ORDERS; i++) { + if (size < (PAGE_SIZE << orders[i])) + continue; + if (max_order < orders[i]) + continue; + + page = alloc_pages(order_flags[i], orders[i]); + if (!page) + continue; + return page; + } + return NULL; +} + +static struct dma_buf *system_heap_do_allocate(struct dma_heap *heap, + unsigned long len, + u32 fd_flags, + u64 heap_flags, + bool uncached) +{ + struct system_heap_buffer *buffer; + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + unsigned long size_remaining = len; + unsigned int max_order = orders[0]; + struct dma_buf *dmabuf; + struct sg_table *table; + struct scatterlist *sg; + struct list_head pages; + struct page *page, *tmp_page; + int i, ret = -ENOMEM; + + buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); + if (!buffer) + return ERR_PTR(-ENOMEM); + + INIT_LIST_HEAD(&buffer->attachments); + mutex_init(&buffer->lock); + buffer->heap = heap; + buffer->len = len; + buffer->uncached = uncached; + + INIT_LIST_HEAD(&pages); + i = 0; + while (size_remaining > 0) { + /* + * Avoid trying to allocate memory if the process + * has been killed by SIGKILL + */ + if (fatal_signal_pending(current)) { + ret = -EINTR; + goto free_buffer; + } + + page = alloc_largest_available(size_remaining, max_order); + if (!page) + goto free_buffer; + + list_add_tail(&page->lru, &pages); + size_remaining -= page_size(page); + max_order = compound_order(page); + i++; + } + + table = &buffer->sg_table; + if (sg_alloc_table(table, i, GFP_KERNEL)) + goto free_buffer; + + sg = table->sgl; + list_for_each_entry_safe(page, tmp_page, &pages, lru) { + sg_set_page(sg, page, page_size(page), 0); + sg = sg_next(sg); + list_del(&page->lru); + + pkvm_el2_mod_call(protect_page_hvc, page_to_pfn(page), 1 << compound_order(page)); + } + + /* create the dmabuf */ + exp_info.exp_name = dma_heap_get_name(heap); + exp_info.ops = &system_heap_buf_ops; + exp_info.size = buffer->len; + exp_info.flags = fd_flags; + exp_info.priv = buffer; + dmabuf = dma_buf_export(&exp_info); + if (IS_ERR(dmabuf)) { + ret = PTR_ERR(dmabuf); + goto free_pages; + } + + /* + * For uncached buffers, we need to initially flush cpu cache, since + * the __GFP_ZERO on the allocation means the zeroing was done by the + * cpu and thus it is likely cached. Map (and implicitly flush) and + * unmap it now so we don't get corruption later on. + */ + if (buffer->uncached) { + dma_map_sgtable(dma_heap_get_dev(heap), table, DMA_BIDIRECTIONAL, 0); + dma_unmap_sgtable(dma_heap_get_dev(heap), table, DMA_BIDIRECTIONAL, 0); + } + + return dmabuf; + +free_pages: + for_each_sgtable_sg(table, sg, i) { + struct page *p = sg_page(sg); + + __free_pages(p, compound_order(p)); + } + sg_free_table(table); +free_buffer: + list_for_each_entry_safe(page, tmp_page, &pages, lru) + __free_pages(page, compound_order(page)); + kfree(buffer); + + return ERR_PTR(ret); +} + +static struct dma_buf *system_heap_allocate(struct dma_heap *heap, + unsigned long len, + u32 fd_flags, + u64 heap_flags) +{ + return system_heap_do_allocate(heap, len, fd_flags, heap_flags, false); +} + +const struct dma_heap_ops system_heap_modified_ops = { + .allocate = system_heap_allocate, +}; \ No newline at end of file