gxp: [Copybara Auto Merge] Merge branch 'zumapro-v' into 'android14-gs-pixel-6.1'
gcip: Makefile supports standalone IIF
Bug: 312161537
gcip: unittests: Makefile supports standalone IIF
Bug: 333862288
gcip: alloc helper: update comments
gcip: pass IP type when submit fence waiters
Bug: 336524176
GCIP_MAIN_REV_ID: 44cb27076738eb87d7d3d235aee7df7a672b8fa6
gxp: remove LPM mapping from core
Bug: 299037074
gxp: direct mode: Request forced debug dump in case of mailbox timeout
Bug: 328551772
gxp: direct mode: Request forced debug dump in case of core crash
Bug: 328551772 (repeat)
gxp: Append fw start and stop logic to support direct mode debug dump.
Bug: 328551772 (repeat)
gxp: Add extra members to vd struct for direct mode
Bug: 328551772 (repeat)
gxp: Add get/set methods for debug dump flags
Bug: 328551772 (repeat)
gxp: Update gxp-host-device-structs.h
Bug: 328551772 (repeat)
gxp: increase gxp version number to 1.27
Bug: 328197763
gxp: Makefile supports standalone IIF
Bug: 332216049
gxp: makefile fix syntax
ganymede: enable gsa support
Bug: 329815240
gxp: Add gxp_thermal_control() implementation
Bug: 328688605
gxp: Add handler for GCIP_KCI_CODE_THERMAL_CONTROL
Bug: 328688605 (repeat)
gxp: pass IP type to fence waited function
Bug: 330929140 (repeat)
gxp: gsx01 add BCI and DSU support
Bug: 336947583
gxp: unittests: Makefile supports standalone IIF
Bug: 333862288 (repeat)
gxp: pass waiter IP type
Bug: 336524176 (repeat)
gxp: remove redundant header inclusion
Bug: 312161537 (repeat)
gcip: add sanitizer_config field in image configs
gcip: image config: support 36-bit mappings
Bug: 325138254
GCIP_MAIN_REV_ID: 673083751510a2e0a60702b1a87d74182ff33355
gxp: Add debugfs nodes for min/max freq limits
Bug: 328688605 (repeat)
gxp: adopt standalone IIF
Bug: 332216049 (repeat)
gxp: reset shadow memory before booting MCU
Bug: 302441258
gxp: support using dynamic memory for MCU fw
Bug: 312642303
gxp: support ns mapping in image configs
gxp: Add KCI handler for setting frequency limits
Bug: 328688605 (repeat)
gxp: add trace points for uci command and respond
Bug: 328197763 (repeat)
gxp: use IOMMU granule instead of PAGE_SIZE
Bug: 333336858
gcip: image config: always pass secure shared mappings to IP driver
Bug: 325139482
gcip: expose gcip_iommu_domain_granule
Bug: 333336858 (repeat)
gcip: fix page size on decoding image cfg
Bug: 333336858 (repeat)
gcip: iif: rename header file to iif-signaler-submission-waiter.h
GCIP_MAIN_REV_ID: 2d5859f668763d7e5a5624e701815b73b593923f
Bug: 330284596
gcip: remove get_cmd_queue_head callback
Bug: 330646127
gcip: make wait_for_cmd_queue_not_full mandatory
Bug: 330646127 (repeat)
gcip: fix context comments in gcip_mailbox_ops
gcip: Remove iif unittests
Bug: 326539224
gcip: guard dma_resv_[un]lock with LINUX_VERSION_CODE
Bug: 329016610
gcip: introduce gcip_fence_array_merge_ikf
Bug: 328046412
gcip: image config parse iova map flags from firmware
Bug: 324254320
Bug: 325139482 (repeat)
gcip: iommu add flag for MMIO device memory mappings
Bug: 324254320 (repeat)
gcip: image config add mapping flags in IOVA
Bug: 324254320 (repeat)
Bug: 325139482 (repeat)
gcip: unittests: remove Kconfig and update Makefile
Bug: 322448134
gcip: guard dma_buf map attachment with dma_resv
Bug: 328199067
gcip: add gcip_dma_fence_merge_fences
Bug: 320401031
iif: add lock_class_key to submitted_signalers_lock
Bug: 327075979
Merge "Merge branch 'headers' into HEAD" into main
GCIP_MAIN_REV_ID: f2f385cce0d35680ebe7042cc5a75dffaa27b801
Signed-off-by: Aurora zuma pro automerger <[email protected]>
GitOrigin-RevId: b27296515aa3cfdc9a29c412b7e4c65ba59a4f89
Change-Id: Id9b66ffe91c548bef14e1eb31d3814299b5483be
diff --git a/Kconfig b/Kconfig
index 174b3e4..f8c7358 100644
--- a/Kconfig
+++ b/Kconfig
@@ -25,5 +25,4 @@
To compile this driver as a module, choose M here. The module will be
called "gxp".
-
endmenu
diff --git a/Makefile b/Makefile
index e8be6aa..717f11a 100644
--- a/Makefile
+++ b/Makefile
@@ -6,6 +6,7 @@
GXP_CHIP := CALLISTO
CONFIG_$(GXP_CHIP) ?= m
GCIP_DIR := gcip-kernel-driver/drivers/gcip
+CURRENT_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
obj-$(CONFIG_$(GXP_CHIP)) += gxp.o
@@ -13,6 +14,7 @@
gxp-bpm.o \
gxp-client.o \
gxp-core-telemetry.o \
+ gxp-dci.o \
gxp-debug-dump.o \
gxp-dma-fence.o \
gxp-dma-iommu.o \
@@ -34,7 +36,6 @@
gxp-vd.o
gxp-mcu-objs := \
- gxp-dci.o \
gxp-kci.o \
gxp-mcu-firmware.o \
gxp-mcu-fs.o \
@@ -93,11 +94,12 @@
GXP_PLATFORM ?= SILICON
gxp-flags := -DCONFIG_GXP_$(GXP_PLATFORM) -DCONFIG_$(GXP_CHIP)=1 \
- -I$(M)/include -I$(M)/gcip-kernel-driver/include \
- -I$(srctree)/$(M)/include \
- -I$(srctree)/$(M)/gcip-kernel-driver/include \
- -I$(srctree)/drivers/gxp/include \
+ -I$(CURRENT_DIR)/include -I$(CURRENT_DIR)/gcip-kernel-driver/include \
-I$(KERNEL_SRC)/../private/google-modules/power/mitigation
+# TODO(b/336717718): Remove path of embedded IIF
+gxp-flags += -I$(CURRENT_DIR)/gcip-kernel-driver/drivers/gcip/iif/include \
+ -I$(KERNEL_SRC)/../private/google-modules/iif/include
+
ccflags-y += $(EXTRA_CFLAGS) $(gxp-flags)
# Flags needed for external modules.
ccflags-y += -DCONFIG_GOOGLE_BCL
@@ -107,11 +109,18 @@
ifneq ($(OUT_DIR),)
# Access TPU driver's exported symbols.
EXTRA_SYMBOLS += $(GMODULE_PATH)/edgetpu/$(EDGETPU_CHIP)/drivers/edgetpu/Module.symvers
+ifneq ($(wildcard $(GMODULE_PATH)/soc/gs/drivers/soc/google/gsa/Module.symvers),)
+EXTRA_SYMBOLS += $(GMODULE_PATH)/soc/gs/drivers/soc/google/gsa/Module.symvers
+endif
ifneq ($(GXP_POWER_MITIGATION), false)
EXTRA_SYMBOLS += $(GMODULE_PATH)/power/mitigation/Module.symvers
endif
+
+ifneq ($(wildcard $(GMODULE_PATH)/iif/Module.symvers),)
+EXTRA_SYMBOLS += $(GMODULE_PATH)/iif/Module.symvers
endif
+endif # OUT_DIR
modules modules_install:
$(MAKE) -C $(KERNEL_SRC) M=$(M)/$(GCIP_DIR) gcip.o
diff --git a/callisto/iova.h b/callisto/iova.h
index c09637b..9dd83c3 100644
--- a/callisto/iova.h
+++ b/callisto/iova.h
@@ -27,5 +27,6 @@
#define GXP_IREMAP_DATA_SIZE 0x1FF000 /* (2MB - 4KB) */
/* Last 4K page of IREMAP_DATA */
#define GXP_MCU_BOOT_MODE_OFFSET (GXP_IREMAP_DATA_OFFSET + GXP_IREMAP_DATA_SIZE)
+#define GXP_IREMAP_DYNAMIC_CODE_BASE 0x11000000
#endif /* __CALLISTO_IOVA_H__ */
diff --git a/gcip-kernel-driver/drivers/gcip/Makefile b/gcip-kernel-driver/drivers/gcip/Makefile
index 6971e51..a57f5ec 100644
--- a/gcip-kernel-driver/drivers/gcip/Makefile
+++ b/gcip-kernel-driver/drivers/gcip/Makefile
@@ -32,11 +32,15 @@
iif/iif-signaler-submission-waiter.o \
iif/iif-sync-file.o
-CURRENT_DIR=$(dir $(abspath $(lastword $(MAKEFILE_LIST))))
+CURRENT_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
-ccflags-y += -I$(CURRENT_DIR)/../../include
+# TODO(b/336717718): Remove path of embedded IIF
+ccflags-y += -I$(CURRENT_DIR)/../../include \
+ -I$(CURRENT_DIR)/iif/include \
+ -I$(KERNEL_SRC)/../private/google-modules/iif/include
modules modules_install clean:
- $(MAKE) -C $(KERNEL_SRC) M=$(M) W=1 $(KBUILD_OPTIONS) $(@)
+ $(MAKE) -C $(KERNEL_SRC) M=$(M) W=1 $(KBUILD_OPTIONS) \
+ KBUILD_EXTRA_SYMBOLS="$(EXTRA_SYMBOLS)" $(@)
diff --git a/gcip-kernel-driver/drivers/gcip/gcip-dma-fence.c b/gcip-kernel-driver/drivers/gcip/gcip-dma-fence.c
index 8053f77..b27fc40 100644
--- a/gcip-kernel-driver/drivers/gcip/gcip-dma-fence.c
+++ b/gcip-kernel-driver/drivers/gcip/gcip-dma-fence.c
@@ -207,12 +207,32 @@
spin_unlock_irq(&gfence->lock);
}
+struct dma_fence *gcip_dma_fence_merge_fences(int num_fences, struct dma_fence **fences)
+{
+ struct dma_fence *tmp;
+ struct dma_fence *array;
+ int i;
+
+ array = dma_fence_unwrap_merge(fences[0]);
+ if (!array)
+ return ERR_PTR(-ENOMEM);
+
+ for (i = 1; i < num_fences; i++) {
+ tmp = array;
+ array = dma_fence_unwrap_merge(tmp, fences[i]);
+ dma_fence_put(tmp);
+ if (!array)
+ return ERR_PTR(-ENOMEM);
+ }
+
+ return array;
+}
+
struct dma_fence *gcip_dma_fence_merge_fds(int num_fences, int *fence_fds)
{
struct dma_fence **fences;
- struct dma_fence *tmp;
struct dma_fence *result;
- int i = 0;
+ int i;
if (!num_fences)
return ERR_PTR(-EINVAL);
@@ -229,20 +249,7 @@
}
}
- result = dma_fence_unwrap_merge(fences[0]);
- if (!result) {
- result = ERR_PTR(-ENOMEM);
- goto out;
- }
- for (i = 1; i < num_fences; i++) {
- tmp = result;
- result = dma_fence_unwrap_merge(tmp, fences[i]);
- dma_fence_put(tmp);
- if (!result) {
- result = ERR_PTR(-ENOMEM);
- goto out;
- }
- }
+ result = gcip_dma_fence_merge_fences(num_fences, fences);
out:
for (i = 0; i < num_fences; i++)
diff --git a/gcip-kernel-driver/drivers/gcip/gcip-fence-array.c b/gcip-kernel-driver/drivers/gcip/gcip-fence-array.c
index 5ef7546..567e3ac 100644
--- a/gcip-kernel-driver/drivers/gcip/gcip-fence-array.c
+++ b/gcip-kernel-driver/drivers/gcip/gcip-fence-array.c
@@ -5,9 +5,11 @@
* Copyright (C) 2023 Google LLC
*/
+#include <linux/dma-fence.h>
#include <linux/kref.h>
#include <linux/slab.h>
+#include <gcip/gcip-dma-fence.h>
#include <gcip/gcip-fence-array.h>
#include <gcip/gcip-fence.h>
@@ -135,7 +137,7 @@
gcip_fence_signal(fence_array->fences[i], errno);
}
-void gcip_fence_array_waited(struct gcip_fence_array *fence_array)
+void gcip_fence_array_waited(struct gcip_fence_array *fence_array, enum iif_ip_type ip)
{
int i;
@@ -143,7 +145,7 @@
return;
for (i = 0; i < fence_array->size; i++)
- gcip_fence_waited(fence_array->fences[i]);
+ gcip_fence_waited(fence_array->fences[i], ip);
}
void gcip_fence_array_submit_signaler(struct gcip_fence_array *fence_array)
@@ -157,7 +159,7 @@
gcip_fence_submit_signaler(fence_array->fences[i]);
}
-void gcip_fence_array_submit_waiter(struct gcip_fence_array *fence_array)
+void gcip_fence_array_submit_waiter(struct gcip_fence_array *fence_array, enum iif_ip_type ip)
{
int i;
@@ -165,11 +167,12 @@
return;
for (i = 0; i < fence_array->size; i++)
- gcip_fence_submit_waiter(fence_array->fences[i]);
+ gcip_fence_submit_waiter(fence_array->fences[i], ip);
}
int gcip_fence_array_submit_waiter_and_signaler(struct gcip_fence_array *in_fences,
- struct gcip_fence_array *out_fences)
+ struct gcip_fence_array *out_fences,
+ enum iif_ip_type ip)
{
int i;
@@ -207,7 +210,7 @@
/* Submits a waiter to @in_fences. */
for (i = 0; in_fences && i < in_fences->size; i++)
- gcip_fence_submit_waiter(in_fences->fences[i]);
+ gcip_fence_submit_waiter(in_fences->fences[i], ip);
return 0;
}
@@ -256,3 +259,30 @@
return gcip_fence_wait_signaler_submission(fence_array->fences, fence_array->size, eventfd,
remaining_signalers);
}
+
+struct dma_fence *gcip_fence_array_merge_ikf(struct gcip_fence_array *fence_array)
+{
+ struct dma_fence **fences;
+ struct dma_fence *merged;
+ int i;
+
+ if (!fence_array || !fence_array->size || !fence_array->same_type ||
+ fence_array->type != GCIP_IN_KERNEL_FENCE)
+ return NULL;
+
+ if (fence_array->size == 1)
+ return dma_fence_get(fence_array->fences[0]->fence.ikf);
+
+ fences = kcalloc(fence_array->size, sizeof(*fences), GFP_KERNEL);
+ if (!fences)
+ return ERR_PTR(-ENOMEM);
+
+ for (i = 0; i < fence_array->size; i++)
+ fences[i] = fence_array->fences[i]->fence.ikf;
+
+ merged = gcip_dma_fence_merge_fences(fence_array->size, fences);
+
+ kfree(fences);
+
+ return merged;
+}
diff --git a/gcip-kernel-driver/drivers/gcip/gcip-fence.c b/gcip-kernel-driver/drivers/gcip/gcip-fence.c
index 8548818..1538b80 100644
--- a/gcip-kernel-driver/drivers/gcip/gcip-fence.c
+++ b/gcip-kernel-driver/drivers/gcip/gcip-fence.c
@@ -11,9 +11,9 @@
#include <gcip/gcip-dma-fence.h>
#include <gcip/gcip-fence.h>
-#include <gcip/iif/iif-fence.h>
-#include <gcip/iif/iif-signaler-submission-watier.h>
-#include <gcip/iif/iif.h>
+#include <iif/iif-fence.h>
+#include <iif/iif-signaler-submission-waiter.h>
+#include <iif/iif.h>
static struct gcip_fence *gcip_fence_alloc(enum gcip_fence_type type)
{
@@ -175,10 +175,10 @@
return -EOPNOTSUPP;
}
-int gcip_fence_submit_waiter(struct gcip_fence *fence)
+int gcip_fence_submit_waiter(struct gcip_fence *fence, enum iif_ip_type ip)
{
if (fence->type == GCIP_INTER_IP_FENCE)
- return iif_fence_submit_waiter(fence->fence.iif, IIF_IP_DSP);
+ return iif_fence_submit_waiter(fence->fence.iif, ip);
return -EOPNOTSUPP;
}
@@ -196,10 +196,10 @@
}
}
-void gcip_fence_waited(struct gcip_fence *fence)
+void gcip_fence_waited(struct gcip_fence *fence, enum iif_ip_type ip)
{
if (fence->type == GCIP_INTER_IP_FENCE)
- iif_fence_waited(fence->fence.iif);
+ iif_fence_waited(fence->fence.iif, ip);
}
/*
diff --git a/gcip-kernel-driver/drivers/gcip/gcip-image-config.c b/gcip-kernel-driver/drivers/gcip/gcip-image-config.c
index c345505..62cb6a8 100644
--- a/gcip-kernel-driver/drivers/gcip/gcip-image-config.c
+++ b/gcip-kernel-driver/drivers/gcip/gcip-image-config.c
@@ -12,6 +12,26 @@
#include <gcip/gcip-image-config.h>
+/*
+ * Return true for a secure config mapping that is not shared. The host IP driver doesn't need to
+ * process such mappings.
+ */
+static bool skip_secure_mapping(struct gcip_image_config *config, u32 map_flags)
+{
+ return gcip_image_config_is_secure(config) && !GCIP_IMAGE_CONFIG_MAP_SHARED(map_flags);
+}
+
+static dma_addr_t virt_address_to_dma(u32 virt_address)
+{
+ dma_addr_t daddr = virt_address & ~(GCIP_IMG_CFG_MAP_FLAGS_MASK);
+ u32 flags = virt_address & GCIP_IMG_CFG_MAP_FLAGS_MASK;
+
+ if (GCIP_IMAGE_CONFIG_MAP_36BIT(flags))
+ daddr <<= 4;
+
+ return daddr;
+}
+
static int setup_iommu_mappings(struct gcip_image_config_parser *parser,
struct gcip_image_config *config)
{
@@ -19,9 +39,13 @@
dma_addr_t daddr;
size_t size;
phys_addr_t paddr;
+ u32 map_flags;
for (i = 0; i < config->num_iommu_mappings; i++) {
- daddr = config->iommu_mappings[i].virt_address;
+ daddr = virt_address_to_dma(config->iommu_mappings[i].virt_address);
+ map_flags = config->iommu_mappings[i].virt_address & GCIP_IMG_CFG_MAP_FLAGS_MASK;
+ if (skip_secure_mapping(config, map_flags))
+ continue;
if (unlikely(!daddr)) {
dev_warn(parser->dev, "Invalid config, device address is zero");
ret = -EIO;
@@ -37,7 +61,7 @@
ret = -EOVERFLOW;
goto err;
}
- ret = parser->ops->map(parser->data, daddr, paddr, size,
+ ret = parser->ops->map(parser->data, daddr, paddr, size, map_flags,
GCIP_IMAGE_CONFIG_FLAGS_SECURE);
if (ret) {
dev_err(parser->dev,
@@ -51,9 +75,13 @@
err:
while (i--) {
- daddr = config->iommu_mappings[i].virt_address;
+ daddr = config->iommu_mappings[i].virt_address & ~(GCIP_IMG_CFG_MAP_FLAGS_MASK);
+ map_flags = config->iommu_mappings[i].virt_address & GCIP_IMG_CFG_MAP_FLAGS_MASK;
+ if (skip_secure_mapping(config, map_flags))
+ continue;
size = gcip_config_to_size(config->iommu_mappings[i].image_config_value);
- parser->ops->unmap(parser->data, daddr, size, GCIP_IMAGE_CONFIG_FLAGS_SECURE);
+ parser->ops->unmap(parser->data, daddr, size, map_flags,
+ GCIP_IMAGE_CONFIG_FLAGS_SECURE);
}
return ret;
}
@@ -63,14 +91,19 @@
{
dma_addr_t daddr;
size_t size;
+ u32 map_flags;
int i;
for (i = config->num_iommu_mappings - 1; i >= 0; i--) {
- daddr = config->iommu_mappings[i].virt_address;
+ daddr = virt_address_to_dma(config->iommu_mappings[i].virt_address);
+ map_flags = config->iommu_mappings[i].virt_address & GCIP_IMG_CFG_MAP_FLAGS_MASK;
+ if (skip_secure_mapping(config, map_flags))
+ continue;
size = gcip_config_to_size(config->iommu_mappings[i].image_config_value);
dev_dbg(parser->dev, "Image config removing IOMMU mapping: %pad size=%#lx", &daddr,
size);
- parser->ops->unmap(parser->data, daddr, size, GCIP_IMAGE_CONFIG_FLAGS_SECURE);
+ parser->ops->unmap(parser->data, daddr, size, map_flags,
+ GCIP_IMAGE_CONFIG_FLAGS_SECURE);
}
}
@@ -96,7 +129,7 @@
ret = -EOVERFLOW;
goto err;
}
- ret = parser->ops->map(parser->data, daddr, paddr, size, 0);
+ ret = parser->ops->map(parser->data, daddr, paddr, size, 0, 0);
if (ret)
goto err;
paddr += size;
@@ -108,7 +141,7 @@
while (i--) {
size = gcip_ns_config_to_size(config->ns_iommu_mappings[i]);
daddr = config->ns_iommu_mappings[i] & GCIP_IMG_CFG_ADDR_MASK;
- parser->ops->unmap(parser->data, daddr, size, 0);
+ parser->ops->unmap(parser->data, daddr, size, 0, 0);
}
return ret;
}
@@ -125,7 +158,7 @@
daddr = config->ns_iommu_mappings[i] & GCIP_IMG_CFG_ADDR_MASK;
dev_dbg(parser->dev, "Image config removing NS IOMMU mapping: %pad size=%#lx",
&daddr, size);
- parser->ops->unmap(parser->data, daddr, size, 0);
+ parser->ops->unmap(parser->data, daddr, size, 0, 0);
}
}
@@ -136,19 +169,16 @@
if (ret)
return ret;
- if (gcip_image_config_is_ns(config)) {
- ret = setup_iommu_mappings(parser, config);
- if (ret)
- clear_ns_iommu_mappings(parser, config);
- }
+ ret = setup_iommu_mappings(parser, config);
+ if (ret)
+ clear_ns_iommu_mappings(parser, config);
return ret;
}
static void unmap_image_config(struct gcip_image_config_parser *parser,
struct gcip_image_config *config)
{
- if (gcip_image_config_is_ns(config))
- clear_iommu_mappings(parser, config);
+ clear_iommu_mappings(parser, config);
clear_ns_iommu_mappings(parser, config);
}
diff --git a/gcip-kernel-driver/drivers/gcip/gcip-iommu.c b/gcip-kernel-driver/drivers/gcip/gcip-iommu.c
index 3b893d6..b285489 100644
--- a/gcip-kernel-driver/drivers/gcip/gcip-iommu.c
+++ b/gcip-kernel-driver/drivers/gcip/gcip-iommu.c
@@ -10,6 +10,7 @@
#include <linux/dma-buf.h>
#include <linux/dma-direction.h>
#include <linux/dma-mapping.h>
+#include <linux/dma-resv.h>
#include <linux/genalloc.h>
#include <linux/iova.h>
#include <linux/limits.h>
@@ -38,6 +39,7 @@
#define GCIP_MAP_MASK_DMA_COHERENT GCIP_MAP_MASK(DMA_COHERENT)
#define GCIP_MAP_MASK_DMA_ATTR GCIP_MAP_MASK(DMA_ATTR)
#define GCIP_MAP_MASK_RESTRICT_IOVA GCIP_MAP_MASK(RESTRICT_IOVA)
+#define GCIP_MAP_MASK_MMIO GCIP_MAP_MASK(MMIO)
#define GCIP_MAP_FLAGS_GET_VALUE(ATTR, flags) \
(((flags) & GCIP_MAP_MASK(ATTR)) >> (GCIP_MAP_FLAGS_##ATTR##_OFFSET))
@@ -45,6 +47,7 @@
#define GCIP_MAP_FLAGS_GET_DMA_COHERENT(flags) GCIP_MAP_FLAGS_GET_VALUE(DMA_COHERENT, flags)
#define GCIP_MAP_FLAGS_GET_DMA_ATTR(flags) GCIP_MAP_FLAGS_GET_VALUE(DMA_ATTR, flags)
#define GCIP_MAP_FLAGS_GET_RESTRICT_IOVA(flags) GCIP_MAP_FLAGS_GET_VALUE(RESTRICT_IOVA, flags)
+#define GCIP_MAP_FLAGS_GET_MMIO(flags) GCIP_MAP_FLAGS_GET_VALUE(MMIO, flags)
/* Restricted IOVA ceiling is for components with 32-bit DMA windows */
#define GCIP_RESTRICT_IOVA_CEILING UINT_MAX
@@ -94,13 +97,6 @@
}
}
-static inline size_t gcip_iommu_domain_granule(struct gcip_iommu_domain *domain)
-{
- if (unlikely(domain->default_domain))
- return PAGE_SIZE;
- return domain->domain_pool->granule;
-}
-
/*
* Allocates an IOVA for the scatterlist and maps it to @domain.
*
@@ -497,8 +493,16 @@
mapping->gcip_map_flags, false);
}
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 2, 0)
+ dma_resv_lock(dmabuf_mapping->dma_buf->resv, NULL);
dma_buf_unmap_attachment(dmabuf_mapping->dma_buf_attachment, dmabuf_mapping->sgt_default,
mapping->dir);
+ dma_resv_unlock(dmabuf_mapping->dma_buf->resv);
+#else
+ dma_buf_unmap_attachment(dmabuf_mapping->dma_buf_attachment, dmabuf_mapping->sgt_default,
+ mapping->dir);
+#endif
+
dma_buf_detach(dmabuf_mapping->dma_buf, dmabuf_mapping->dma_buf_attachment);
dma_buf_put(dmabuf_mapping->dma_buf);
kfree(dmabuf_mapping);
@@ -1293,7 +1297,13 @@
#endif
/* Map the attachment into the default domain. */
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 2, 0)
+ dma_resv_lock(dmabuf->resv, NULL);
sgt = dma_buf_map_attachment(attachment, dir);
+ dma_resv_unlock(dmabuf->resv);
+#else
+ sgt = dma_buf_map_attachment(attachment, dir);
+#endif
if (IS_ERR(sgt)) {
ret = PTR_ERR(sgt);
dev_err(dev, "Failed to get sgt from attachment (ret=%d, name=%s, size=%lu)\n", ret,
@@ -1311,7 +1321,13 @@
return mapping;
err_map_dma_buf_sgt:
+#if LINUX_VERSION_CODE >= KERNEL_VERSION(6, 2, 0)
+ dma_resv_lock(dmabuf->resv, NULL);
dma_buf_unmap_attachment(attachment, sgt, dir);
+ dma_resv_unlock(dmabuf->resv);
+#else
+ dma_buf_unmap_attachment(attachment, sgt, dir);
+#endif
err_map_attachment:
dma_buf_detach(dmabuf, attachment);
return ERR_PTR(ret);
@@ -1363,9 +1379,13 @@
{
enum dma_data_direction dir = GCIP_MAP_FLAGS_GET_DMA_DIRECTION(gcip_map_flags);
bool coherent = GCIP_MAP_FLAGS_GET_DMA_COHERENT(gcip_map_flags);
+ bool mmio = GCIP_MAP_FLAGS_GET_MMIO(gcip_map_flags);
unsigned long attrs = GCIP_MAP_FLAGS_GET_DMA_ATTR(gcip_map_flags);
int prot = dma_info_to_prot(dir, coherent, attrs);
+ if (mmio)
+ prot |= IOMMU_MMIO;
+
#if GCIP_IOMMU_MAP_HAS_GFP
return iommu_map(domain->domain, iova, paddr, size, prot, GFP_KERNEL);
#else
diff --git a/gcip-kernel-driver/drivers/gcip/gcip-kci.c b/gcip-kernel-driver/drivers/gcip/gcip-kci.c
index f4e904b..40da93c 100644
--- a/gcip-kernel-driver/drivers/gcip/gcip-kci.c
+++ b/gcip-kernel-driver/drivers/gcip/gcip-kci.c
@@ -14,13 +14,6 @@
#include <gcip/gcip-kci.h>
#include <gcip/gcip-mailbox.h>
-static u32 gcip_kci_get_cmd_queue_head(struct gcip_mailbox *mailbox)
-{
- struct gcip_kci *kci = gcip_mailbox_get_data(mailbox);
-
- return kci->ops->get_cmd_queue_head(kci);
-}
-
static u32 gcip_kci_get_cmd_queue_tail(struct gcip_mailbox *mailbox)
{
struct gcip_kci *kci = gcip_mailbox_get_data(mailbox);
@@ -255,7 +248,6 @@
}
static const struct gcip_mailbox_ops gcip_mailbox_ops = {
- .get_cmd_queue_head = gcip_kci_get_cmd_queue_head,
.get_cmd_queue_tail = gcip_kci_get_cmd_queue_tail,
.inc_cmd_queue_tail = gcip_kci_inc_cmd_queue_tail,
.acquire_cmd_queue_lock = gcip_kci_acquire_cmd_queue_lock,
diff --git a/gcip-kernel-driver/drivers/gcip/gcip-mailbox.c b/gcip-kernel-driver/drivers/gcip/gcip-mailbox.c
index 24849c9..a4b6b7d 100644
--- a/gcip-kernel-driver/drivers/gcip/gcip-mailbox.c
+++ b/gcip-kernel-driver/drivers/gcip/gcip-mailbox.c
@@ -21,7 +21,6 @@
#define TEST_TRIGGER_TIMEOUT_RACE(...)
#endif
-#define GET_CMD_QUEUE_HEAD() mailbox->ops->get_cmd_queue_head(mailbox)
#define GET_CMD_QUEUE_TAIL() mailbox->ops->get_cmd_queue_tail(mailbox)
#define INC_CMD_QUEUE_TAIL(inc) mailbox->ops->inc_cmd_queue_tail(mailbox, inc)
#define ACQUIRE_CMD_QUEUE_LOCK(try, atomic) \
@@ -162,33 +161,16 @@
gcip_mailbox_cmd_flags_t flags)
{
int ret = 0;
- u32 tail;
bool atomic = false;
ACQUIRE_CMD_QUEUE_LOCK(false, &atomic);
if (!(flags & GCIP_MAILBOX_CMD_FLAGS_SKIP_ASSIGN_SEQ))
SET_CMD_ELEM_SEQ(cmd, mailbox->cur_seq);
- /*
- * The lock ensures mailbox cmd_queue_tail cannot be changed by other processes (this
- * method should be the only one to modify the value of tail), therefore we can remember
- * its value here and use it in the condition of wait_event() call.
- */
- tail = GET_CMD_QUEUE_TAIL();
-
- if (mailbox->ops->wait_for_cmd_queue_not_full) {
- /* Wait until the cmd queue has a space for putting cmd. */
- ret = mailbox->ops->wait_for_cmd_queue_not_full(mailbox);
- if (ret)
- goto out;
- } else if (GET_CMD_QUEUE_HEAD() == (tail ^ mailbox->queue_wrap_bit)) {
- /*
- * Default logic of checking the fullness of cmd_queue. If the cmd_queue is full,
- * it's up to the caller to retry.
- */
- ret = -EAGAIN;
+ /* Wait until the cmd queue has a space for putting cmd. */
+ ret = mailbox->ops->wait_for_cmd_queue_not_full(mailbox);
+ if (ret)
goto out;
- }
if (async_resp->resp) {
/* Adds @resp to the wait_list only if the cmd can be pushed successfully. */
@@ -198,9 +180,11 @@
if (ret)
goto out;
}
+
/* Size of cmd_queue is a multiple of mailbox->cmd_elem_size. */
- memcpy(mailbox->cmd_queue + mailbox->cmd_elem_size *
- CIRC_QUEUE_REAL_INDEX(tail, mailbox->queue_wrap_bit),
+ memcpy(mailbox->cmd_queue +
+ mailbox->cmd_elem_size *
+ CIRC_QUEUE_REAL_INDEX(GET_CMD_QUEUE_TAIL(), mailbox->queue_wrap_bit),
cmd, mailbox->cmd_elem_size);
INC_CMD_QUEUE_TAIL(1);
if (mailbox->ops->after_enqueue_cmd) {
@@ -492,9 +476,9 @@
return 0;
}
- if (!ops->get_cmd_queue_head || !ops->get_cmd_queue_tail || !ops->inc_cmd_queue_tail ||
- !ops->acquire_cmd_queue_lock || !ops->release_cmd_queue_lock ||
- !ops->get_cmd_elem_seq || !ops->set_cmd_elem_seq || !ops->get_cmd_elem_code) {
+ if (!ops->get_cmd_queue_tail || !ops->inc_cmd_queue_tail || !ops->acquire_cmd_queue_lock ||
+ !ops->release_cmd_queue_lock || !ops->get_cmd_elem_seq || !ops->set_cmd_elem_seq ||
+ !ops->get_cmd_elem_code || !ops->wait_for_cmd_queue_not_full) {
dev_err(mailbox->dev, "Incomplete mailbox CMD queue ops.\n");
return -EINVAL;
}
diff --git a/gcip-kernel-driver/drivers/gcip/gcip-usage-stats.c b/gcip-kernel-driver/drivers/gcip/gcip-usage-stats.c
index 5c50705..90c0468 100644
--- a/gcip-kernel-driver/drivers/gcip/gcip-usage-stats.c
+++ b/gcip-kernel-driver/drivers/gcip/gcip-usage-stats.c
@@ -721,7 +721,7 @@
mutex_unlock(&ustats->dvfs_freqs_lock);
dvfs_freqs_num = ustats->ops->get_default_dvfs_freqs_num(ustats->data);
for (i = 0; i < dvfs_freqs_num; i++)
- written += scnprintf(buf + written, PAGE_SIZE - written, "%.*s%d", i, " ",
+ written += scnprintf(buf + written, PAGE_SIZE - written, "%.*s%u", i, " ",
ustats->ops->get_default_dvfs_freq(i, ustats->data));
} else {
dvfs_freqs_num = ustats->dvfs_freqs_num;
diff --git a/gcip-kernel-driver/drivers/gcip/iif/iif-fence-table.c b/gcip-kernel-driver/drivers/gcip/iif/iif-fence-table.c
index dcea5df..8af7ccb 100644
--- a/gcip-kernel-driver/drivers/gcip/iif/iif-fence-table.c
+++ b/gcip-kernel-driver/drivers/gcip/iif/iif-fence-table.c
@@ -1,8 +1,15 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
- * GCIP-integrated IIF driver fence table.
+ * Interface to utilize IIF fence tables, the wait table and the signal table. Both tables will have
+ * one entry per fence ID.
*
- * Copyright (C) 2023 Google LLC
+ * - Wait table: Describes which IPs are waiting on each fence. This table will be written by the
+ * kernel driver only.
+ *
+ * - Signal table: Describes how many signals are remaining to unblock each fence. This table will
+ * be initialized by the kernel driver and each signaler IP will update it.
+ *
+ * Copyright (C) 2023-2024 Google LLC
*/
#define pr_fmt(fmt) "iif: " fmt
@@ -12,7 +19,7 @@
#include <linux/of.h>
#include <linux/slab.h>
-#include <gcip/iif/iif-fence-table.h>
+#include <iif/iif-fence-table.h>
#define IIF_FENCE_WAIT_TABLE_PROP_NAME "iif-fence-wait-table-region"
#define IIF_FENCE_SIGNAL_TABLE_PROP_NAME "iif-fence-signal-table-region"
diff --git a/gcip-kernel-driver/drivers/gcip/iif/iif-fence.c b/gcip-kernel-driver/drivers/gcip/iif/iif-fence.c
index 5834af5..9a2563c 100644
--- a/gcip-kernel-driver/drivers/gcip/iif/iif-fence.c
+++ b/gcip-kernel-driver/drivers/gcip/iif/iif-fence.c
@@ -1,27 +1,34 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
- * GCIP-integrated IIF driver fence.
+ * The inter-IP fence.
*
- * Copyright (C) 2023 Google LLC
+ * The actual functionality (waiting and signaling) won't be done by the kernel driver. The main
+ * role of it is creating fences with assigning fence IDs, initializing the fence table and managing
+ * the life cycle of them.
+ *
+ * Copyright (C) 2023-2024 Google LLC
*/
#define pr_fmt(fmt) "iif: " fmt
#include <linux/atomic.h>
#include <linux/container_of.h>
+#include <linux/export.h>
#include <linux/file.h>
#include <linux/fs.h>
#include <linux/idr.h>
#include <linux/kref.h>
+#include <linux/lockdep.h>
+#include <linux/lockdep_types.h>
#include <linux/spinlock.h>
#include <linux/types.h>
#include <linux/version.h>
-#include <gcip/iif/iif-fence-table.h>
-#include <gcip/iif/iif-fence.h>
-#include <gcip/iif/iif-manager.h>
-#include <gcip/iif/iif-sync-file.h>
-#include <gcip/iif/iif.h>
+#include <iif/iif-fence-table.h>
+#include <iif/iif-fence.h>
+#include <iif/iif-manager.h>
+#include <iif/iif-sync-file.h>
+#include <iif/iif.h>
/*
* Returns the number of remaining signalers to be submitted. Returns 0 if all signalers are
@@ -86,21 +93,24 @@
/*
* Signals @fence.
*
+ * The function returns the number of remaining signals to unblock the fence on success. Otherwise,
+ * returns a negative errno.
+ *
+ * If the function returns 0, it means that the fence has been unblocked and the caller is expected
+ * to call the `iif_fence_notify_poll_cb` function to notify all others waiting on the fence.
+ *
* If @complete is true, it will make @fence have been signaled by all signalers. This must be used
- * only when @fence is going to be released before all signalers signal the fence and let the drive
- * side notice that there was some problem by triggering registered callbacks.
+ * only when @fence is going to be released before all signalers signal the fence.
*
* Caller must hold @fence->signaled_signalers_lock.
*/
-static void iif_fence_signal_locked(struct iif_fence *fence, bool complete)
+static int iif_fence_signal_locked(struct iif_fence *fence, bool complete)
{
- struct iif_fence_poll_cb *cur, *tmp;
-
lockdep_assert_held(&fence->signaled_signalers_lock);
if (iif_fence_is_signaled_locked(fence)) {
- pr_warn("The fence is already signaled, id=%u", fence->id);
- return;
+ pr_err("The fence is already signaled, id=%u", fence->id);
+ return -EBUSY;
}
if (!complete)
@@ -108,12 +118,22 @@
else
fence->signaled_signalers = fence->total_signalers;
- /* All signalers have signaled the fence. */
- if (iif_fence_is_signaled_locked(fence)) {
- list_for_each_entry_safe(cur, tmp, &fence->poll_cb_list, node) {
- list_del_init(&cur->node);
- cur->func(fence, cur);
- }
+ return fence->total_signalers - fence->signaled_signalers;
+}
+
+/*
+ * Notifies the poll callbacks registered to @fence.
+ *
+ * This function must be called only when @fence is signaled so that @fence->signaled_signalers_lock
+ * doesn't have to be held.
+ */
+static void iif_fence_notify_poll_cb(struct iif_fence *fence)
+{
+ struct iif_fence_poll_cb *cur, *tmp;
+
+ list_for_each_entry_safe(cur, tmp, &fence->poll_cb_list, node) {
+ list_del_init(&cur->node);
+ cur->func(fence, cur);
}
}
@@ -176,11 +196,27 @@
if (!list_empty(&fence->poll_cb_list) && !iif_fence_is_signaled_locked(fence)) {
iif_fence_set_signal_error_locked(fence, -EDEADLK);
+ /*
+ * We don't have to check the return value here since theoretically it will never
+ * fail.
+ */
iif_fence_signal_locked(fence, true);
}
spin_unlock_irqrestore(&fence->signaled_signalers_lock, flags);
+ /*
+ * It is always safe to call this function.
+ * - If the if-clause above was executed, it means that the fence has been signaled and it
+ * is good to call this function.
+ * - If @fence->poll_cb_list was empty, this function call will be NO-OP.
+ * - If `iif_fence_is_signaled_locked(fence)` was true, it means that the fence was already
+ * signaled and it is good to call it. (In this case, all callbacks should be called when
+ * the fence was signaled and @fence->poll_cb_list should be already empty. It means that
+ * the function call will be NO-OP theoretically.)
+ */
+ iif_fence_notify_poll_cb(fence);
+
/* Checks whether there is remaining all_signaler_submitted callback. */
iif_fence_submitted_signalers_lock(fence);
@@ -200,16 +236,30 @@
*/
iif_fence_retire(fence);
+#if IS_ENABLED(CONFIG_DEBUG_SPINLOCK)
+ lockdep_unregister_key(&fence->submitted_signalers_key);
+#endif /* IS_ENABLED(CONFIG_DEBUG_SPINLOCK) */
+
if (fence->ops && fence->ops->on_release)
fence->ops->on_release(fence);
}
+/*
+ * This callback will be registered to @fence when the fence is initialized as its signaler as AP.
+ * Also, it will be called when @fence has been unblocked.
+ */
+static void iif_fence_ap_poll_callback(struct iif_fence *fence, struct iif_fence_poll_cb *cb)
+{
+ iif_manager_broadcast_fence_unblocked_by_ap(fence->mgr, fence);
+}
+
int iif_fence_init(struct iif_manager *mgr, struct iif_fence *fence,
const struct iif_fence_ops *ops, enum iif_ip_type signaler_ip,
uint16_t total_signalers)
{
unsigned int id_min = signaler_ip * IIF_NUM_FENCES_PER_IP;
unsigned int id_max = id_min + IIF_NUM_FENCES_PER_IP - 1;
+ int ret = 0;
fence->id = ida_alloc_range(&mgr->idp, id_min, id_max, GFP_KERNEL);
if (fence->id < 0)
@@ -224,7 +274,14 @@
fence->ops = ops;
fence->state = IIF_FENCE_STATE_INITIALIZED;
kref_init(&fence->kref);
+#if IS_ENABLED(CONFIG_DEBUG_SPINLOCK)
+ lockdep_register_key(&fence->submitted_signalers_key);
+ __raw_spin_lock_init(spinlock_check(&fence->submitted_signalers_lock),
+ "&fence->submitted_signalers_lock", &fence->submitted_signalers_key,
+ LD_WAIT_CONFIG);
+#else
spin_lock_init(&fence->submitted_signalers_lock);
+#endif /* IS_ENABLED(CONFIG_DEBUG_SPINLOCK) */
spin_lock_init(&fence->signaled_signalers_lock);
spin_lock_init(&fence->outstanding_waiters_lock);
iif_fence_table_init_fence_entry(&mgr->fence_table, fence->id, total_signalers);
@@ -232,7 +289,11 @@
INIT_LIST_HEAD(&fence->all_signaler_submitted_cb_list);
atomic_set(&fence->num_sync_file, 0);
- return 0;
+ if (signaler_ip == IIF_IP_AP)
+ ret = iif_fence_add_poll_callback(fence, &fence->ap_poll_cb,
+ iif_fence_ap_poll_callback);
+
+ return ret;
}
int iif_fence_install_fd(struct iif_fence *fence)
@@ -333,13 +394,25 @@
return 0;
}
-void iif_fence_signal(struct iif_fence *fence)
+int iif_fence_signal(struct iif_fence *fence)
{
unsigned long flags;
+ int ret;
spin_lock_irqsave(&fence->signaled_signalers_lock, flags);
- iif_fence_signal_locked(fence, false);
+ ret = iif_fence_signal_locked(fence, false);
spin_unlock_irqrestore(&fence->signaled_signalers_lock, flags);
+
+ if (!ret) {
+ /*
+ * If @fence has been signaled, it is safe to execute all registered poll callbacks
+ * without holding @fence->signaled_signalers_lock since the drivers can't register
+ * callbacks anymore.
+ */
+ iif_fence_notify_poll_cb(fence);
+ }
+
+ return ret;
}
void iif_fence_set_signal_error(struct iif_fence *fence, int error)
@@ -378,7 +451,7 @@
return signaled;
}
-void iif_fence_waited(struct iif_fence *fence)
+void iif_fence_waited(struct iif_fence *fence, enum iif_ip_type ip)
{
unsigned long flags;
diff --git a/gcip-kernel-driver/drivers/gcip/iif/iif-manager.c b/gcip-kernel-driver/drivers/gcip/iif/iif-manager.c
index 253c36f..cc84870 100644
--- a/gcip-kernel-driver/drivers/gcip/iif/iif-manager.c
+++ b/gcip-kernel-driver/drivers/gcip/iif/iif-manager.c
@@ -1,19 +1,24 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
- * GCIP-integrated IIF driver manager.
+ * The manager of inter-IP fences.
*
- * Copyright (C) 2023 Google LLC
+ * It manages the pool of fence IDs. The IIF driver device will initialize a manager and each IP
+ * driver will fetch the manager from the IIF device.
+ *
+ * Copyright (C) 2023-2024 Google LLC
*/
#include <linux/container_of.h>
+#include <linux/export.h>
#include <linux/idr.h>
#include <linux/kref.h>
#include <linux/of.h>
#include <linux/slab.h>
#include <linux/version.h>
-#include <gcip/iif/iif-fence-table.h>
-#include <gcip/iif/iif-manager.h>
+#include <iif/iif-fence-table.h>
+#include <iif/iif-fence.h>
+#include <iif/iif-manager.h>
static void iif_manager_destroy(struct kref *kref)
{
@@ -54,3 +59,37 @@
{
kref_put(&mgr->kref, iif_manager_destroy);
}
+
+int iif_manager_register_ops(struct iif_manager *mgr, enum iif_ip_type ip,
+ const struct iif_manager_ops *ops, void *data)
+{
+ if (!ops || !ops->fence_unblocked_by_ap)
+ return -EINVAL;
+
+ mgr->ops[ip] = ops;
+ mgr->data[ip] = data;
+
+ return 0;
+}
+
+int iif_manager_acquire_block_wakelock(struct iif_manager *mgr, enum iif_ip_type ip)
+{
+ if (mgr->ops[ip] && mgr->ops[ip]->acquire_block_wakelock)
+ return mgr->ops[ip]->acquire_block_wakelock(mgr->data[ip]);
+ return 0;
+}
+
+void iif_manager_release_block_wakelock(struct iif_manager *mgr, enum iif_ip_type ip)
+{
+ if (mgr->ops[ip] && mgr->ops[ip]->release_block_wakelock)
+ mgr->ops[ip]->release_block_wakelock(mgr->data[ip]);
+}
+
+void iif_manager_broadcast_fence_unblocked_by_ap(struct iif_manager *mgr, struct iif_fence *fence)
+{
+ enum iif_ip_type ip;
+ unsigned int tmp;
+
+ for_each_waiting_ip(&mgr->fence_table, fence->id, ip, tmp)
+ mgr->ops[ip]->fence_unblocked_by_ap(fence, mgr->data[ip]);
+}
diff --git a/gcip-kernel-driver/drivers/gcip/iif/iif-signaler-submission-waiter.c b/gcip-kernel-driver/drivers/gcip/iif/iif-signaler-submission-waiter.c
index 98edc18..110cd9a 100644
--- a/gcip-kernel-driver/drivers/gcip/iif/iif-signaler-submission-waiter.c
+++ b/gcip-kernel-driver/drivers/gcip/iif/iif-signaler-submission-waiter.c
@@ -6,13 +6,14 @@
*/
#include <linux/eventfd.h>
+#include <linux/export.h>
#include <linux/kref.h>
#include <linux/list.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
-#include <gcip/iif/iif-fence.h>
-#include <gcip/iif/iif-signaler-submission-watier.h>
+#include <iif/iif-fence.h>
+#include <iif/iif-signaler-submission-waiter.h>
static struct iif_signaler_submission_waiter *
iif_signaler_submission_waiter_alloc(unsigned int eventfd, int pending_fences)
@@ -73,7 +74,7 @@
/*
* The `iif_all_signaler_submission_waiter_cancel` function will delete @cb from
- * @watier->cb_list, decrement the refcount of @waiter and release @cb instead.
+ * @waiter->cb_list, decrement the refcount of @waiter and release @cb instead.
*/
if (waiter->cancel) {
spin_unlock_irqrestore(&waiter->lock, flags);
diff --git a/gcip-kernel-driver/drivers/gcip/iif/iif-sync-file.c b/gcip-kernel-driver/drivers/gcip/iif/iif-sync-file.c
index f4982d3..3a1f88f 100644
--- a/gcip-kernel-driver/drivers/gcip/iif/iif-sync-file.c
+++ b/gcip-kernel-driver/drivers/gcip/iif/iif-sync-file.c
@@ -1,8 +1,12 @@
// SPDX-License-Identifier: GPL-2.0-only
/*
- * GCIP-integrated IIF driver sync file.
+ * IIF driver sync file.
*
- * Copyright (C) 2023 Google LLC
+ * To export fences to the userspace, the driver will allocate a sync file to a fence and will
+ * return its file descriptor to the user. The user can distinguish fences with it. The driver will
+ * convert the file descriptor to the corresponding fence ID and will pass it to the IP.
+ *
+ * Copyright (C) 2023-2024 Google LLC
*/
#include <linux/anon_inodes.h>
@@ -15,8 +19,8 @@
#include <linux/types.h>
#include <linux/uaccess.h>
-#include <gcip/iif/iif-fence.h>
-#include <gcip/iif/iif-sync-file.h>
+#include <iif/iif-fence.h>
+#include <iif/iif-sync-file.h>
static void iif_sync_file_fence_signaled(struct iif_fence *fence, struct iif_fence_poll_cb *poll_cb)
{
diff --git a/gcip-kernel-driver/include/gcip/iif/iif-fence-table.h b/gcip-kernel-driver/drivers/gcip/iif/include/iif/iif-fence-table.h
similarity index 61%
rename from gcip-kernel-driver/include/gcip/iif/iif-fence-table.h
rename to gcip-kernel-driver/drivers/gcip/iif/include/iif/iif-fence-table.h
index 0f10577..dfa52a3 100644
--- a/gcip-kernel-driver/include/gcip/iif/iif-fence-table.h
+++ b/gcip-kernel-driver/drivers/gcip/iif/include/iif/iif-fence-table.h
@@ -1,17 +1,39 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
- * GCIP-integrated IIF driver fence table.
+ * Interface to utilize IIF fence tables, the wait table and the signal table. Both tables will have
+ * one entry per fence ID.
*
- * Copyright (C) 2023 Google LLC
+ * - Wait table: Describes which IPs are waiting on each fence. This table will be written by the
+ * kernel driver only.
+ *
+ * - Signal table: Describes how many signals are remaining to unblock each fence. This table will
+ * be initialized by the kernel driver and each signaler IP will update it.
+ *
+ * Copyright (C) 2023-2024 Google LLC
*/
#ifndef __IIF_IIF_FENCE_TABLE_H__
#define __IIF_IIF_FENCE_TABLE_H__
+#include <linux/bitops.h>
#include <linux/of.h>
#include <linux/types.h>
-#include <gcip/iif/iif.h>
+#include <iif/iif.h>
+
+#define IIF_CLEAR_LSB(b) ((b) & ((b) - 1))
+
+/*
+ * Iterates the type of IPs waiting on the fence of @fence_id.
+ *
+ * fence_table (input): Pointer to the fence table.
+ * fence_id (input): ID of the fence to get IPs waiting on it.
+ * ip (output): Type of IP waiting on the fence (enum iif_ip_type).
+ * tmp (output): Temporary variable to iterate the wait table entry (int).
+ */
+#define for_each_waiting_ip(fence_table, fence_id, waiting_ip, tmp) \
+ for (tmp = (fence_table)->wait_table[fence_id].waiting_ips, waiting_ip = __ffs(tmp); tmp; \
+ tmp = IIF_CLEAR_LSB(tmp), waiting_ip = __ffs(tmp))
/* Entry of the wait table. */
struct iif_wait_table_entry {
diff --git a/gcip-kernel-driver/include/gcip/iif/iif-fence.h b/gcip-kernel-driver/drivers/gcip/iif/include/iif/iif-fence.h
similarity index 92%
rename from gcip-kernel-driver/include/gcip/iif/iif-fence.h
rename to gcip-kernel-driver/drivers/gcip/iif/include/iif/iif-fence.h
index befd745..67ee061 100644
--- a/gcip-kernel-driver/include/gcip/iif/iif-fence.h
+++ b/gcip-kernel-driver/drivers/gcip/iif/include/iif/iif-fence.h
@@ -1,19 +1,24 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
- * GCIP-integrated IIF driver fence.
+ * The inter-IP fence.
*
- * Copyright (C) 2023 Google LLC
+ * The actual functionality (waiting and signaling) won't be done by the kernel driver. The main
+ * role of it is creating fences with assigning fence IDs, initializing the fence table and managing
+ * the life cycle of them.
+ *
+ * Copyright (C) 2023-2024 Google LLC
*/
#ifndef __IIF_IIF_FENCE_H__
#define __IIF_IIF_FENCE_H__
#include <linux/kref.h>
+#include <linux/lockdep_types.h>
#include <linux/spinlock.h>
#include <linux/types.h>
-#include <gcip/iif/iif-manager.h>
-#include <gcip/iif/iif.h>
+#include <iif/iif-manager.h>
+#include <iif/iif.h>
struct iif_fence;
struct iif_fence_ops;
@@ -50,6 +55,33 @@
IIF_FENCE_STATE_RETIRED,
};
+/*
+ * Contains the callback function which will be called when all signalers have signaled the fence.
+ *
+ * The callback can be registered to the fence by the `iif_fence_add_poll_callback` function.
+ */
+struct iif_fence_poll_cb {
+ /* Node to be added to the list. */
+ struct list_head node;
+ /* Actual callback function to be called. */
+ iif_fence_poll_cb_t func;
+};
+
+/*
+ * Contains the callback function which will be called when all signalers have been submitted.
+ *
+ * The callback will be registered to the fence when the `iif_fence_submit_waiter` function fails
+ * in the submission.
+ */
+struct iif_fence_all_signaler_submitted_cb {
+ /* Node to be added to the list. */
+ struct list_head node;
+ /* Actual callback function to be called. */
+ iif_fence_all_signaler_submitted_cb_t func;
+ /* The number of remaining signalers to be submitted. */
+ int remaining_signalers;
+};
+
/* The fence object. */
struct iif_fence {
/* IIF manager. */
@@ -67,6 +99,9 @@
* @all_signaler_submitted_error.
*/
spinlock_t submitted_signalers_lock;
+#if IS_ENABLED(CONFIG_DEBUG_SPINLOCK)
+ struct lock_class_key submitted_signalers_key;
+#endif /* IS_ENABLED(CONFIG_DEBUG_SPINLOCK) */
/* The interrupt state before holding @submitted_signalers_lock. */
unsigned long submitted_signalers_lock_flags;
/* The number of signaled signalers. */
@@ -93,6 +128,8 @@
int all_signaler_submitted_error;
/* The number of sync_file(s) bound to the fence. */
atomic_t num_sync_file;
+ /* The callback called if the fence's signaler is AP and the fence is signaled. */
+ struct iif_fence_poll_cb ap_poll_cb;
};
/* Operators of `struct iif_fence`. */
@@ -108,33 +145,6 @@
};
/*
- * Contains the callback function which will be called when all signalers have signaled the fence.
- *
- * The callback can be registered to the fence by the `iif_fence_add_poll_callback` function.
- */
-struct iif_fence_poll_cb {
- /* Node to be added to the list. */
- struct list_head node;
- /* Actual callback function to be called. */
- iif_fence_poll_cb_t func;
-};
-
-/*
- * Contains the callback function which will be called when all signalers have been submitted.
- *
- * The callback will be registered to the fence when the `iif_fence_submit_waiter` function fails
- * in the submission.
- */
-struct iif_fence_all_signaler_submitted_cb {
- /* Node to be added to the list. */
- struct list_head node;
- /* Actual callback function to be called. */
- iif_fence_all_signaler_submitted_cb_t func;
- /* The number of remaining signalers to be submitted. */
- int remaining_signalers;
-};
-
-/*
* Initializes @fence which will be signaled by @signaler_ip IP. @total_signalers is the number of
* signalers which must be submitted to the fence. Its initial reference count is 1.
*
@@ -199,7 +209,7 @@
int iif_fence_submit_waiter(struct iif_fence *fence, enum iif_ip_type ip);
/* Signals @fence. If all signalers have signaled, it will notify polling FDs. */
-void iif_fence_signal(struct iif_fence *fence);
+int iif_fence_signal(struct iif_fence *fence);
/*
* Sets @fence->signal_error to let the user know that @fence has been signaled with an error.
@@ -226,8 +236,8 @@
*/
bool iif_fence_is_signaled(struct iif_fence *fence);
-/* Notifies the driver that a waiter finished waiting on @fence. */
-void iif_fence_waited(struct iif_fence *fence);
+/* Notifies the driver that a waiter of @ip finished waiting on @fence. */
+void iif_fence_waited(struct iif_fence *fence, enum iif_ip_type ip);
/*
* Registers a callback which will be called when all signalers of @fence signaled. Once the
diff --git a/gcip-kernel-driver/drivers/gcip/iif/include/iif/iif-manager.h b/gcip-kernel-driver/drivers/gcip/iif/include/iif/iif-manager.h
new file mode 100644
index 0000000..95c994c
--- /dev/null
+++ b/gcip-kernel-driver/drivers/gcip/iif/include/iif/iif-manager.h
@@ -0,0 +1,157 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * The manager of inter-IP fences.
+ *
+ * It manages the pool of fence IDs. The IIF driver device will initialize a manager and each IP
+ * driver will fetch the manager from the IIF device.
+ *
+ * Copyright (C) 2023-2024 Google LLC
+ */
+
+#ifndef __IIF_IIF_MANAGER_H__
+#define __IIF_IIF_MANAGER_H__
+
+#include <linux/cdev.h>
+#include <linux/device.h>
+#include <linux/idr.h>
+#include <linux/kref.h>
+#include <linux/of.h>
+#include <linux/types.h>
+
+#include <iif/iif-fence-table.h>
+
+struct iif_fence;
+
+/* Operators. */
+struct iif_manager_ops {
+ /* Following callbacks are required. */
+ /*
+ * Called when @fence is unblocked by AP.
+ *
+ * This callback is meaningful only when @fence's signaler type is AP. When the runtime
+ * signals @fence as many as the number of total signalers which was decided when the fence
+ * was created and the fence has been unblocked eventually, this callback will be called if
+ * the IP registered this operator is waiting on the fence to be unblocked.
+ *
+ * If this callback is called, the IP drivers should notify their IP that the fence has been
+ * unblocked (i.e., send @fence->id to IP) so that they will check whether they can start
+ * processing commands which were pended by the fence. Also, the IP drivers should check
+ * whether the fence was signaled with an error or not by checking @fence->signal_error and
+ * decide what to do if there was an error.
+ *
+ * This callback returns void since the IIF driver has nothing can do when the IP driver
+ * fails to notify their IP. It is the responsibility of the IP driver side if that happens.
+ *
+ * Note that if IP drivers want to poll the fence unblock in general cases, they should
+ * utilize the `iif_fence_{add,remove}_poll_callback` functions instead.
+ *
+ * Context: Normal.
+ */
+ void (*fence_unblocked_by_ap)(struct iif_fence *fence, void *data);
+
+ /* Following callbacks are optional. */
+ /*
+ * Acquires the block wakelock of IP.
+ *
+ * The callback will be executed when IP submits a waiter (i.e., `iif_fence_submit_waiter`
+ * is called).
+ *
+ * This callback is required if the waiter IPx doesn't allow to be notified by the signaler
+ * IPy when the IPx block is not powered on. If somehow the driver of waiter IPx submits a
+ * waiter command to the firmware late and the signaler IPy processes its command early by
+ * the race, IPy may try to notify IPx which is not powered on yet and it may cause a bug if
+ * the IPx spec doesn't allow that. Therefore, if the IPx implements this callback, the IIF
+ * driver will try to acquire the IPx block wakelock before submitting a waiter of IPx.
+ *
+ * Context: Depends on in which context the IPx calls the `iif_fence_submit_waiter`
+ * function.
+ */
+ int (*acquire_block_wakelock)(void *data);
+ /*
+ * Releases the block wakelock of IP.
+ *
+ * If both of the waiter IPx and the signaler IPy finishes waiting on / signaling the fence,
+ * (i.e., both `iif_fence_waited` and `iif_fence_signal` are called) the block wakelock of
+ * IPx will be released.
+ *
+ * If somehow IPx driver calls `iif_fence_waited` before the fence is signaled (For example,
+ * IPy is not responding and IPx has processed its command as timeout.), releasing the block
+ * wakelock of IPx will be pended until IPy processes its command normally or as timeout and
+ * IPy driver eventually calls `iif_fence_signal`.
+ *
+ * Context: Depends on in which context the IPx calls the `iif_fence_waited` function or
+ * the IPy calls the `iif_fence_signal` function. Since this callback will be
+ * implemented by the IPx and IPx may not be aware of in which context IPy calls
+ * the `iif_fence_signal` function, it's recommended to assume the context is any
+ * including IRQ context.
+ */
+ void (*release_block_wakelock)(void *data);
+};
+
+/*
+ * The structure overall data required by IIF driver such as fence table.
+ *
+ * Until we have stand-alone IIF driver, one of the IP drivers will initializes a manager by
+ * the `iif_init` function and every IP driver will share it.
+ */
+struct iif_manager {
+ /* Reference count of this instance. */
+ struct kref kref;
+ /* Fence ID pool. */
+ struct ida idp;
+ /* Fence table shared with the firmware. */
+ struct iif_fence_table fence_table;
+ /* Operators per IP. */
+ const struct iif_manager_ops *ops[IIF_IP_RESERVED];
+ /* User-data per IP. */
+ void *data[IIF_IP_RESERVED];
+ /* Platform bus device. */
+ struct device *dev;
+ /* Char device structure. */
+ struct cdev char_dev;
+ /* Char device number. */
+ dev_t char_dev_no;
+};
+
+/*
+ * Initializes IIF driver and returns its manager. Its initial reference count is 1. It will map
+ * the fence table by parsing the device tree via @np.
+ *
+ * The returned manager will be destroyed when its reference count becomes 0 by `iif_manager_put`
+ * function.
+ */
+struct iif_manager *iif_manager_init(const struct device_node *np);
+
+/* Increases the reference count of @mgr. */
+struct iif_manager *iif_manager_get(struct iif_manager *mgr);
+
+/* Decreases the reference count of @mgr and if it becomes 0, releases @mgr. */
+void iif_manager_put(struct iif_manager *mgr);
+
+/*
+ * Registers operators of @ip.
+ *
+ * Note that @ops must not be released until @ip won't be utilized as signaler or waiter anymore.
+ */
+int iif_manager_register_ops(struct iif_manager *mgr, enum iif_ip_type ip,
+ const struct iif_manager_ops *ops, void *data);
+
+/*
+ * Acquires the block wakelock of @ip.
+ *
+ * Returns 0 on success or @ip hasn't defined the `acquire_block_wakelock` operator. Otherwise,
+ * returns a negative errno.
+ */
+int iif_manager_acquire_block_wakelock(struct iif_manager *mgr, enum iif_ip_type ip);
+
+/* Releases the block wakelock of @ip. */
+void iif_manager_release_block_wakelock(struct iif_manager *mgr, enum iif_ip_type ip);
+
+/*
+ * Notifies @fence has been unblocked by AP to IPs waiting on the fence.
+ *
+ * This function will be called if @fence's signaler is AP and the fence has been unblocked.
+ */
+void iif_manager_broadcast_fence_unblocked_by_ap(struct iif_manager *mgr, struct iif_fence *fence);
+
+#endif /* __IIF_IIF_MANAGER_H__ */
diff --git a/gcip-kernel-driver/include/gcip/iif/iif-signaler-submission-watier.h b/gcip-kernel-driver/drivers/gcip/iif/include/iif/iif-signaler-submission-waiter.h
similarity index 98%
rename from gcip-kernel-driver/include/gcip/iif/iif-signaler-submission-watier.h
rename to gcip-kernel-driver/drivers/gcip/iif/include/iif/iif-signaler-submission-waiter.h
index 6692b80..3d53972 100644
--- a/gcip-kernel-driver/include/gcip/iif/iif-signaler-submission-watier.h
+++ b/gcip-kernel-driver/drivers/gcip/iif/include/iif/iif-signaler-submission-waiter.h
@@ -8,7 +8,7 @@
#ifndef __IIF_IIF_SIGNALER_SUBMISSION_WAITER_H__
#define __IIF_IIF_SIGNALER_SUBMISSION_WAITER_H__
-#include <gcip/iif/iif-fence.h>
+#include <iif/iif-fence.h>
#define IIF_NO_REGISTER_EVENTFD (~0u)
diff --git a/gcip-kernel-driver/include/gcip/iif/iif-sync-file.h b/gcip-kernel-driver/drivers/gcip/iif/include/iif/iif-sync-file.h
similarity index 75%
rename from gcip-kernel-driver/include/gcip/iif/iif-sync-file.h
rename to gcip-kernel-driver/drivers/gcip/iif/include/iif/iif-sync-file.h
index a79f25b..7b4711c 100644
--- a/gcip-kernel-driver/include/gcip/iif/iif-sync-file.h
+++ b/gcip-kernel-driver/drivers/gcip/iif/include/iif/iif-sync-file.h
@@ -1,8 +1,12 @@
/* SPDX-License-Identifier: GPL-2.0-only */
/*
- * GCIP-integrated IIF driver sync file.
+ * IIF driver sync file.
*
- * Copyright (C) 2023 Google LLC
+ * To export fences to the userspace, the driver will allocate a sync file to a fence and will
+ * return its file descriptor to the user. The user can distinguish fences with it. The driver will
+ * convert the file descriptor to the corresponding fence ID and will pass it to the IP.
+ *
+ * Copyright (C) 2023-2024 Google LLC
*/
#ifndef __IIF_IIF_SYNC_FILE_H__
@@ -11,7 +15,7 @@
#include <linux/file.h>
#include <linux/wait.h>
-#include <gcip/iif/iif-fence.h>
+#include <iif/iif-fence.h>
#define IIF_SYNC_FILE_FLAGS_POLL_ENABLED 0
diff --git a/gcip-kernel-driver/drivers/gcip/iif/include/iif/iif.h b/gcip-kernel-driver/drivers/gcip/iif/include/iif/iif.h
new file mode 100644
index 0000000..b71b69c
--- /dev/null
+++ b/gcip-kernel-driver/drivers/gcip/iif/include/iif/iif.h
@@ -0,0 +1,148 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/*
+ * Defines the interface of the IIF driver.
+ *
+ * Copyright (C) 2023 Google LLC
+ */
+
+#ifndef __IIF_IIF_H__
+#define __IIF_IIF_H__
+
+#include <linux/ioctl.h>
+#include <linux/types.h>
+
+/* Interface Version. */
+#define IIF_INTERFACE_VERSION_MAJOR 1
+#define IIF_INTERFACE_VERSION_MINOR 0
+
+#define IIF_IOCTL_BASE 'i'
+
+/* The ioctl number for the fence FDs will start from here. */
+#define IIF_FENCE_IOCTL_NUM_BASE 0x80
+
+/*
+ * The max number of fences can be created per IP.
+ * Increasing this value needs to increase the size of fence table.
+ */
+#define IIF_NUM_FENCES_PER_IP 1024
+
+/* The maximum number of fences can be passed to one ioctl request. */
+#define IIF_MAX_NUM_FENCES 64
+
+/*
+ * Type of IPs.
+ *
+ * The order of IP must be matched with the firmware side because the fence ID will be assigned
+ * according to the IP type.
+ */
+enum iif_ip_type {
+ IIF_IP_DSP,
+ IIF_IP_TPU,
+ IIF_IP_GPU,
+ IIF_IP_AP,
+ IIF_IP_NUM,
+
+ /* Reserve the number of IP type to expand the fence table easily in the future. */
+ IIF_IP_RESERVED = 16,
+};
+
+/*
+ * ioctls for /dev/iif.
+ */
+
+struct iif_create_fence_ioctl {
+ /*
+ * Input:
+ * The type of the fence signaler IP. (See enum iif_ip_type)
+ */
+ __u8 signaler_ip;
+ /*
+ * Input:
+ * The number of the signalers.
+ */
+ __u16 total_signalers;
+ /*
+ * Output:
+ * The file descriptor of the created fence.
+ */
+ __s32 fence;
+};
+
+/* Create an IIF fence. */
+#define IIF_CREATE_FENCE _IOWR(IIF_IOCTL_BASE, 0, struct iif_create_fence_ioctl)
+
+/*
+ * The ioctl won't register @eventfd and will simply return the number of
+ * remaining signalers of each fence.
+ */
+#define IIF_FENCE_REMAINING_SIGNALERS_NO_REGISTER_EVENTFD (~0u)
+
+struct iif_fence_remaining_signalers_ioctl {
+ /*
+ * Input:
+ * User-space pointer to an int array of inter-IP fence file descriptors
+ * to check whether there are remaining signalers to be submitted or
+ * not.
+ */
+ __u64 fences;
+ /*
+ * Input:
+ * The number of fences in `fence_array`.
+ * If > IIF_MAX_NUM_FENCES, the ioctl will fail with errno == EINVAL.
+ */
+ __u32 fences_count;
+ /*
+ * Input:
+ * The eventfd which will be triggered if there were fence(s) which
+ * haven't finished the signaler submission yet when the ioctl is called
+ * and when they eventually have finished the submission. Note that if
+ * all fences already finished the submission (i.e., all values in the
+ * returned @remaining_signalers are 0), this eventfd will be ignored.
+ *
+ * Note that if `IIF_FENCE_REMAINING_SIGNALERS_NO_REGISTER_EVENTFD` is
+ * passed, this ioctl will simply return the number of remaining
+ * signalers of each fence to @remaining_signalers.
+ */
+ __u32 eventfd;
+ /*
+ * Output:
+ * User-space pointer to an int array where the driver will write the
+ * number of remaining signalers to be submitted per fence. The order
+ * will be the same with @fences.
+ */
+ __u64 remaining_signalers;
+};
+
+/*
+ * Check whether there are remaining signalers to be submitted to fences.
+ * If all signalers have been submitted, the runtime is expected to send waiter
+ * commands right away. Otherwise, it will listen the eventfd to wait signaler
+ * submission to be finished.
+ */
+#define IIF_FENCE_REMAINING_SIGNALERS \
+ _IOWR(IIF_IOCTL_BASE, 1, struct iif_fence_remaining_signalers_ioctl)
+
+/*
+ * ioctls for inter-IP fence FDs.
+ */
+
+struct iif_fence_get_information_ioctl {
+ /* The type of the signaler IP. (enum iif_ip_type) */
+ __u8 signaler_ip;
+ /* The number of total signalers. */
+ __u16 total_signalers;
+ /* The number of submitted signalers. */
+ __u16 submitted_signalers;
+ /* The number of signaled signalers. */
+ __u16 signaled_signalers;
+ /* The number of outstanding waiters. */
+ __u16 outstanding_waiters;
+ /* Reserved. */
+ __u8 reserved[7];
+};
+
+/* Returns the fence information. */
+#define IIF_FENCE_GET_INFORMATION \
+ _IOR(IIF_IOCTL_BASE, IIF_FENCE_IOCTL_NUM_BASE, struct iif_fence_get_information_ioctl)
+
+#endif /* __IIF_IIF_H__ */
diff --git a/gcip-kernel-driver/include/gcip/gcip-alloc-helper.h b/gcip-kernel-driver/include/gcip/gcip-alloc-helper.h
index 17208bf..4f61135 100644
--- a/gcip-kernel-driver/include/gcip/gcip-alloc-helper.h
+++ b/gcip-kernel-driver/include/gcip/gcip-alloc-helper.h
@@ -13,12 +13,11 @@
#include <linux/types.h>
/*
- * The actual return value from the alloc_noncontiguous function.
- * The user should only care about @sgt. @pages is used internally for freeing memory.
+ * Used internally by the alloc_noncontiguous functions.
*/
struct gcip_sgt_handle {
- struct sg_table sgt;
- void *mem;
+ struct sg_table sgt; /* SG table for vmalloc'ed physical pages */
+ void *mem; /* kernel VA of vmalloc area */
};
/*
@@ -28,10 +27,11 @@
* @size: Total size in bytes. Will be page aligned.
* @gfp: The GFP flag for malloc internal structures.
*
- * Returns the SG table represents the non-contiguous region.
+ * Returns an SG table for the non-contiguous pages.
* Returns NULL on any error.
*/
struct sg_table *gcip_alloc_noncontiguous(struct device *dev, size_t size, gfp_t gfp);
+
/* Frees the memory allocated by gcip_alloc_noncontiguous. */
void gcip_free_noncontiguous(struct sg_table *sgt);
diff --git a/gcip-kernel-driver/include/gcip/gcip-dma-fence.h b/gcip-kernel-driver/include/gcip/gcip-dma-fence.h
index 87c7467..774bc98 100644
--- a/gcip-kernel-driver/include/gcip/gcip-dma-fence.h
+++ b/gcip-kernel-driver/include/gcip/gcip-dma-fence.h
@@ -161,7 +161,7 @@
void gcip_dma_fence_show(struct gcip_dma_fence *gfence, struct seq_file *s);
/**
- * Gets and merges an array of DMA fences from their FDs.
+ * gcip_dma_fence_merge_fences() - Merges an array of DMA fences with dma_fence_unwrap_merge().
*
* Creates a dma_fence_array from all of the provided fences and returns a dma_fence representing
* that array. If any of the provided fences are also arrays, the resulting array will include
@@ -171,12 +171,19 @@
*
* The returned fence must be released with `dma_fence_put()`.
*
+ * Returns a pointer to the fence on success. Otherwise a negative errno as an ERR_PTR.
+ */
+struct dma_fence *gcip_dma_fence_merge_fences(int num_fences, struct dma_fence **fences);
+
+/**
+ * gcip_dma_fence_merge_fds() - Gets and merges an array of DMA fences from their FDs.
+ *
+ * Creates a dma_fence_array from all of the provided fences.
* It is OK if @fence_fds do not refer to gcip_dma_fences.
*
* Returns a pointer to the fence on success. Otherwise a negative errno as an ERR_PTR.
* - If unable to allocate sufficient memory, returns ERR_PTR(-ENOMEM)
* - If any of @fence_fds are invalid, returns ERR_PTR(-ENOENT)
- * - If unable to merge the fences, returns NULL
*/
struct dma_fence *gcip_dma_fence_merge_fds(int num_fences, int *fence_fds);
diff --git a/gcip-kernel-driver/include/gcip/gcip-fence-array.h b/gcip-kernel-driver/include/gcip/gcip-fence-array.h
index afcc7ea..6b6dbfb 100644
--- a/gcip-kernel-driver/include/gcip/gcip-fence-array.h
+++ b/gcip-kernel-driver/include/gcip/gcip-fence-array.h
@@ -8,6 +8,7 @@
#ifndef __GCIP_FENCE_ARRAY_H__
#define __GCIP_FENCE_ARRAY_H__
+#include <linux/dma-fence.h>
#include <linux/kref.h>
#include <gcip/gcip-fence.h>
@@ -54,21 +55,22 @@
void gcip_fence_array_signal(struct gcip_fence_array *fence_array, int errno);
/*
- * Notifies the fences in @fence_array that a command which waited on them has finished their work.
+ * Notifies the fences in @fence_array that a command of @ip which waited on them has finished their
+ * work.
*
* This function is only meaningful when a fence is GCIP_INTER_IP_FENCE.
*/
-void gcip_fence_array_waited(struct gcip_fence_array *fence_array);
+void gcip_fence_array_waited(struct gcip_fence_array *fence_array, enum iif_ip_type ip);
/* Submits a signaler to the fences in @fence_array. */
void gcip_fence_array_submit_signaler(struct gcip_fence_array *fence_array);
-/* Submits a waiter to the fences in @fence_array. */
-void gcip_fence_array_submit_waiter(struct gcip_fence_array *fence_array);
+/* Submits a waiter of @ip to the fences in @fence_array. */
+void gcip_fence_array_submit_waiter(struct gcip_fence_array *fence_array, enum iif_ip_type ip);
/*
- * Submits a waiter to each fence in @in_fences and a signaler to each fence in @out_fences. Either
- * @in_fences or @out_fences is allowed to be NULL.
+ * Submits a waiter of @ip to each fence in @in_fences and a signaler to each fence in @out_fences.
+ * Either @in_fences or @out_fences is allowed to be NULL.
*
* For the waiter submission, if at least one fence of @in_fences haven't finished the signaler
* submission, this function will fail and return -EAGAIN.
@@ -82,7 +84,8 @@
* Otherwise, returns 0 on success.
*/
int gcip_fence_array_submit_waiter_and_signaler(struct gcip_fence_array *in_fences,
- struct gcip_fence_array *out_fences);
+ struct gcip_fence_array *out_fences,
+ enum iif_ip_type ip);
/*
* Allocates and returns the array of inter-IP fence IDs. The number of IIFs in @fence_array will
@@ -108,4 +111,22 @@
int gcip_fence_array_wait_signaler_submission(struct gcip_fence_array *fence_array,
unsigned int eventfd, int *remaining_signalers);
+/*
+ * Creates a merged dma_fence_array object of the underlying DMA fences of @fence_array.
+ *
+ * Note that this function is meaningful only when the fence type of all fences of @fence_array are
+ * the same (i.e., @fence_array->same_type is true) and the type is DMA fence (i.e., fence_array->
+ * type is GCIP_IN_KERNEL_FENCE). If that's not the case or there is no fence in @fence_array, the
+ * function will return NULL.
+ *
+ * If there is only one DMA fence in @fence_array, the function will return the DMA fence itself,
+ * not a base fence of merged dma_fence_array.
+ *
+ * Returns the representing fence of the merged DMA fences on success or NULL if @fence_array is not
+ * meaningful to be merged. Otherwise, returns an errno pointer.
+ *
+ * The returned fence must be released with `dma_fence_put()`.
+ */
+struct dma_fence *gcip_fence_array_merge_ikf(struct gcip_fence_array *fence_array);
+
#endif /* __GCIP_FENCE_ARRAY_H__ */
diff --git a/gcip-kernel-driver/include/gcip/gcip-fence.h b/gcip-kernel-driver/include/gcip/gcip-fence.h
index 5c0417a..004fa61 100644
--- a/gcip-kernel-driver/include/gcip/gcip-fence.h
+++ b/gcip-kernel-driver/include/gcip/gcip-fence.h
@@ -11,9 +11,9 @@
#include <linux/dma-fence.h>
#include <linux/kref.h>
-#include <gcip/iif/iif-fence.h>
-#include <gcip/iif/iif-manager.h>
-#include <gcip/iif/iif.h>
+#include <iif/iif-fence.h>
+#include <iif/iif-manager.h>
+#include <iif/iif.h>
#define GCIP_FENCE_REMAINING_SIGNALERS_NO_REGISTER_EVENTFD (~0u)
@@ -90,7 +90,7 @@
int gcip_fence_submit_signaler_locked(struct gcip_fence *fence);
/*
- * Submits a waiter.
+ * Submits a waiter of @ip.
* Note that the waiter submission will not be done when not all signalers have been submitted.
*
* This function is only meaningful when the fence type is GCIP_INTER_IP_FENCE and can be called in
@@ -100,7 +100,7 @@
* has been succeeded when the function returns 0.) Otherwise, returns a negative errno if it fails
* with other reasons.
*/
-int gcip_fence_submit_waiter(struct gcip_fence *fence);
+int gcip_fence_submit_waiter(struct gcip_fence *fence, enum iif_ip_type ip);
/*
* Signals @fence. If all signalers have signaled the fence, it will notify polling FDs.
@@ -110,11 +110,11 @@
void gcip_fence_signal(struct gcip_fence *fence, int errno);
/*
- * Notifies @fence that a command which waited the fence has finished their work.
+ * Notifies @fence that a command of @ip which waited the fence has finished their work.
*
* This function is only meaningful when the fence type is GCIP_INTER_IP_FENCE.
*/
-void gcip_fence_waited(struct gcip_fence *fence);
+void gcip_fence_waited(struct gcip_fence *fence, enum iif_ip_type ip);
/*
* Registers a callback which will be called when all signalers are submitted for @fence and
diff --git a/gcip-kernel-driver/include/gcip/gcip-image-config.h b/gcip-kernel-driver/include/gcip/gcip-image-config.h
index d6e66ad..bebae45 100644
--- a/gcip-kernel-driver/include/gcip/gcip-image-config.h
+++ b/gcip-kernel-driver/include/gcip/gcip-image-config.h
@@ -8,12 +8,12 @@
#ifndef __GCIP_IMAGE_CONFIG_H__
#define __GCIP_IMAGE_CONFIG_H__
-#include <asm/page.h>
#include <linux/bits.h>
+#include <linux/sizes.h>
#include <linux/types.h>
#define GCIP_FW_NUM_VERSIONS 4
-#define GCIP_IMG_CFG_MAX_IOMMU_MAPPINGS 20
+#define GCIP_IMG_CFG_MAX_IOMMU_MAPPINGS 19
#define GCIP_IMG_CFG_MAX_NS_IOMMU_MAPPINGS 5
#define GCIP_IMG_CFG_MAX_PROTECTED_MEMORY_MAPPINGS 3
@@ -21,6 +21,30 @@
#define GCIP_FW_PRIV_LEVEL_TZ 1
#define GCIP_FW_PRIV_LEVEL_NS 2
+#define GCIP_FW_ASAN_ENABLED (1 << 0)
+#define GCIP_FW_UBSAN_ENABLED (1 << 1)
+
+#define GCIP_IMAGE_CONFIG_SANITIZER_SHIFT 2
+#define GCIP_IMAGE_CONFIG_SANITIZER_MASK (BIT(GCIP_IMAGE_CONFIG_SANITIZER_SHIFT) - 1)
+#define GCIP_IMAGE_CONFIG_SANITIZER_STATUS(cfg) (cfg & GCIP_IMAGE_CONFIG_SANITIZER_MASK)
+#define GCIP_IMAGE_CONFIG_SANITIZER_INDEX(cfg) (cfg >> GCIP_IMAGE_CONFIG_SANITIZER_SHIFT)
+
+/*
+ * Mapping flags in lower 12 bits (regardless of page size in use by AP kernel) of requested
+ * virt_address in iommu_mappings. Must match definitions used by firmware.
+ */
+#define GCIP_IMG_CFG_MAP_FLAGS_MASK (SZ_4K - 1)
+
+/* The entry is for a CSR/device region and must be mapped with IOMMU_MMIO flag. */
+#define GCIP_IMAGE_CONFIG_MAP_MMIO_BIT (BIT(0))
+#define GCIP_IMAGE_CONFIG_MAP_MMIO(flags) (flags & GCIP_IMAGE_CONFIG_MAP_MMIO_BIT)
+/* The mapping must be replicated for each PASID/context. */
+#define GCIP_IMAGE_CONFIG_MAP_SHARED_BIT (BIT(1))
+#define GCIP_IMAGE_CONFIG_MAP_SHARED(flags) (flags & GCIP_IMAGE_CONFIG_MAP_SHARED_BIT)
+/* The mapping uses a 36-bit IOVA. The incoming value needs to be shifted left 4 bits */
+#define GCIP_IMAGE_CONFIG_MAP_36BIT_BIT (BIT(3))
+#define GCIP_IMAGE_CONFIG_MAP_36BIT(flags) (flags & GCIP_IMAGE_CONFIG_MAP_36BIT_BIT)
+
/*
* The image configuration attached to the signed firmware.
*/
@@ -35,7 +59,7 @@
__u32 remapped_region_size;
__u32 num_iommu_mappings;
struct {
- /* Device virtual address */
+ /* Device virtual address requested, with map flags in lower 12 bits */
__u32 virt_address;
/*
* Encodes a 12-bit aligned address and the corresponding size
@@ -44,6 +68,8 @@
*/
__u32 image_config_value;
} iommu_mappings[GCIP_IMG_CFG_MAX_IOMMU_MAPPINGS];
+ __u32 reserved;
+ __u32 sanitizer_config;
__u32 protected_memory_regions[GCIP_IMG_CFG_MAX_PROTECTED_MEMORY_MAPPINGS];
__u32 secure_telemetry_region_start;
__u32 remapped_data_start;
@@ -60,21 +86,24 @@
*
* It is ensured that there is no overflow on @paddr + @size before calling this function.
*
- * @flags is a bit-field with the following attributes:
- * [0:0] - Security. 1 for secure and 0 for non-secure.
+ * @cfg_map_flags holds GCIP_IMAGE_CONFIG_MAP_* flags masked from an iommu_mappings_entry,
+ * or zero for ns_iommu_mappings (GCIP_IMAGE_CONFIG_FLAGS_SECURE == 0)
+ * @cfg_op_flags is a bit-field with the following attributes:
+ * [0:0] - Secure: 1 = handle secure iommu_mappings[] entry, 0 = ns_iommu_mappings[]
* [31:1] - Reserved.
*
* Returns 0 on success. Otherwise a negative errno.
* Mandatory.
*/
int (*map)(void *data, dma_addr_t daddr, phys_addr_t paddr, size_t size,
- unsigned int flags);
+ unsigned int cfg_map_flags, unsigned int cfg_op_flags);
/*
* Removes the IOMMU mapping previously added by @map.
*
* Mandatory.
*/
- void (*unmap)(void *data, dma_addr_t daddr, size_t size, unsigned int flags);
+ void (*unmap)(void *data, dma_addr_t daddr, size_t size, unsigned int cfg_map_flags,
+ unsigned int cfg_op_flags);
};
struct gcip_image_config_parser {
@@ -183,4 +212,12 @@
return config->privilege_level == GCIP_FW_PRIV_LEVEL_NS;
}
+/*
+ * Returns whether the privilege level specified by @config is secure.
+ */
+static inline bool gcip_image_config_is_secure(struct gcip_image_config *config)
+{
+ return config->privilege_level != GCIP_FW_PRIV_LEVEL_NS;
+}
+
#endif /* __GCIP_IMAGE_CONFIG_H__ */
diff --git a/gcip-kernel-driver/include/gcip/gcip-iommu.h b/gcip-kernel-driver/include/gcip/gcip-iommu.h
index 425691d..96dd824 100644
--- a/gcip-kernel-driver/include/gcip/gcip-iommu.h
+++ b/gcip-kernel-driver/include/gcip/gcip-iommu.h
@@ -62,6 +62,12 @@
#define GCIP_MAP_FLAGS_RESTRICT_IOVA_TO_FLAGS(restrict) \
((u64)(restrict) << GCIP_MAP_FLAGS_RESTRICT_IOVA_OFFSET)
+#define GCIP_MAP_FLAGS_MMIO_OFFSET \
+ (GCIP_MAP_FLAGS_RESTRICT_IOVA_OFFSET + GCIP_MAP_FLAGS_RESTRICT_IOVA_BIT_SIZE)
+#define GCIP_MAP_FLAGS_MMIO_BIT_SIZE 1
+#define GCIP_MAP_FLAGS_MMIO_TO_FLAGS(mmio) \
+ ((u64)(mmio) << GCIP_MAP_FLAGS_RESTRICT_IOVA_OFFSET)
+
/* Helper macros to easily create the mapping direction flags. */
#define GCIP_MAP_FLAGS_DMA_RW GCIP_MAP_FLAGS_DMA_DIRECTION_TO_FLAGS(DMA_BIDIRECTIONAL)
#define GCIP_MAP_FLAGS_DMA_RO GCIP_MAP_FLAGS_DMA_DIRECTION_TO_FLAGS(DMA_TO_DEVICE)
@@ -80,7 +86,9 @@
* (See [REDACTED]
* [13:13] - RESTRICT_IOVA:
* Restrict the IOVA assignment to 32 bit address window.
- * [63:14] - RESERVED
+ * [14:14] - MMIO:
+ * Mapping is for device memory, use IOMMU_MMIO flag.
+ * [63:15] - RESERVED
* Set RESERVED bits to 0 to ensure backwards compatibility.
*
* One can use gcip_iommu_encode_gcip_map_flags or `GCIP_MAP_FLAGS_DMA_*_TO_FLAGS` macros to
@@ -539,6 +547,13 @@
mapping->data = data;
}
+static inline size_t gcip_iommu_domain_granule(struct gcip_iommu_domain *domain)
+{
+ if (unlikely(domain->default_domain))
+ return PAGE_SIZE;
+ return domain->domain_pool->granule;
+}
+
/**
* gcip_iommu_map() - Maps the desired mappings to the domain.
* @domain: The GCIP domain to be mapped to.
diff --git a/gcip-kernel-driver/include/gcip/gcip-mailbox.h b/gcip-kernel-driver/include/gcip/gcip-mailbox.h
index 1088303..13a0ea3 100644
--- a/gcip-kernel-driver/include/gcip/gcip-mailbox.h
+++ b/gcip-kernel-driver/include/gcip/gcip-mailbox.h
@@ -137,18 +137,15 @@
struct gcip_mailbox_ops {
/* Mandatory. */
/*
- * Gets the head of mailbox command queue.
- * Context: normal.
- */
- u32 (*get_cmd_queue_head)(struct gcip_mailbox *mailbox);
- /*
* Gets the tail of mailbox command queue.
- * Context: normal.
+ *
+ * Context: cmd_queue_lock.
*/
u32 (*get_cmd_queue_tail)(struct gcip_mailbox *mailbox);
/*
* Increases the tail of mailbox command queue by @inc.
- * Context: normal.
+ *
+ * Context: cmd_queue_lock.
*/
void (*inc_cmd_queue_tail)(struct gcip_mailbox *mailbox, u32 inc);
/*
@@ -167,43 +164,61 @@
int (*acquire_cmd_queue_lock)(struct gcip_mailbox *mailbox, bool try, bool *atomic);
/*
* Releases the lock of cmd_queue which is acquired by calling `acquire_cmd_queue_lock`.
- * Context: normal.
+ *
+ * Context: cmd_queue_lock.
*/
void (*release_cmd_queue_lock)(struct gcip_mailbox *mailbox);
/*
* Gets the sequence number of @cmd queue element.
- * Context: normal.
+ *
+ * Context: cmd_queue_lock.
*/
u64 (*get_cmd_elem_seq)(struct gcip_mailbox *mailbox, void *cmd);
/*
* Sets the sequence number of @cmd queue element.
- * Context: normal.
+ *
+ * Context: cmd_queue_lock.
*/
void (*set_cmd_elem_seq)(struct gcip_mailbox *mailbox, void *cmd, u64 seq);
/*
* Gets the code of @cmd queue element.
+ *
* Context: normal.
*/
u32 (*get_cmd_elem_code)(struct gcip_mailbox *mailbox, void *cmd);
+ /*
+ * Waits for the cmd queue of @mailbox has a available space for putting the command. If
+ * the queue has a space, returns 0. Otherwise, returns error as non-zero value. It depends
+ * on the implementation details, but it is okay to return right away with error when the
+ * queue is full. If this callback returns an error, `gcip_mailbox_send_cmd` function or
+ * `gcip_mailbox_put_cmd` function will return that error too.
+ *
+ * Context: cmd_queue_lock.
+ */
+ int (*wait_for_cmd_queue_not_full)(struct gcip_mailbox *mailbox);
/*
* Gets the size of mailbox response queue.
+ *
* Context: normal.
*/
u32 (*get_resp_queue_size)(struct gcip_mailbox *mailbox);
/*
* Gets the head of mailbox response queue.
- * Context: normal and in_interrupt().
+ *
+ * Context: resp_queue_lock.
*/
u32 (*get_resp_queue_head)(struct gcip_mailbox *mailbox);
/*
* Gets the tail of mailbox response queue.
- * Context: normal and in_interrupt().
+ *
+ * Context: resp_queue_lock.
*/
u32 (*get_resp_queue_tail)(struct gcip_mailbox *mailbox);
/*
* Increases the head of mailbox response queue by @inc.
- * Context: normal and in_interrupt().
+ *
+ * Context: resp_queue_lock.
*/
void (*inc_resp_queue_head)(struct gcip_mailbox *mailbox, u32 inc);
/*
@@ -224,17 +239,20 @@
int (*acquire_resp_queue_lock)(struct gcip_mailbox *mailbox, bool try, bool *atomic);
/*
* Releases the lock of resp_queue which is acquired by calling `acquire_resp_queue_lock`.
- * Context: normal and in_interrupt().
+ *
+ * Context: resp_queue_lock.
*/
void (*release_resp_queue_lock)(struct gcip_mailbox *mailbox);
/*
* Gets the sequence number of @resp queue element.
- * Context: normal and in_interrupt().
+ *
+ * Context: wait_list_lock.
*/
u64 (*get_resp_elem_seq)(struct gcip_mailbox *mailbox, void *resp);
/*
* Sets the sequence number of @resp queue element.
- * Context: normal and in_interrupt().
+ *
+ * Context: cmd_queue_lock.
*/
void (*set_resp_elem_seq)(struct gcip_mailbox *mailbox, void *resp, u64 seq);
@@ -249,6 +267,7 @@
* The lock can be a mutex lock or a spin lock. However, if @irqsave is considered and
* "_irqsave" is used, it must be spin lock only.
* The lock will be released by calling `release_wait_list_lock` callback.
+ *
* Context: normal and in_interrupt().
*/
void (*acquire_wait_list_lock)(struct gcip_mailbox *mailbox, bool irqsave,
@@ -257,28 +276,14 @@
* Releases the lock of wait_list which is acquired by calling `acquire_wait_list_lock`.
* If @irqsave is true, restores @flags from `acquire_wait_list_lock` to the irq state.
* Or it can be ignored, if @irqsave was not considered in the `acquire_wait_list_lock`.
- * Context: normal and in_interrupt().
+ *
+ * Context: wait_list_lock.
*/
void (*release_wait_list_lock)(struct gcip_mailbox *mailbox, bool irqrestore,
unsigned long flags);
/* Optional. */
/*
- * Waits for the cmd queue of @mailbox has a available space for putting the command. If
- * the queue has a space, returns 0. Otherwise, returns error as non-zero value. It depends
- * on the implementation details, but it is okay to return right away with error when the
- * queue is full. If this callback returns an error, `gcip_mailbox_send_cmd` function or
- * `gcip_mailbox_put_cmd` function will return that error too. This callback is called with
- * the `cmd_queue_lock` being held.
- *
- * Note: if this callback is NULL, it will simply check the fullness of cmd_queue and
- * return -EAGAIN error right away if it is full. Please refer the implementation of the
- * `gcip_mailbox_enqueue_cmd` function.
- *
- * Context: normal.
- */
- int (*wait_for_cmd_queue_not_full)(struct gcip_mailbox *mailbox);
- /*
* This callback will be called before putting the @resp into @mailbox->wait_list and
* putting @cmd of @resp into the command queue. After this callback returns, the consumer
* is able to start processing it and the mailbox is going to wait for it. Therefore, this
@@ -290,15 +295,15 @@
*
* If @resp is synchronous, @awaiter will be NULL.
*
- * Context: normal.
+ * Context: cmd_queue_lock.
*/
int (*before_enqueue_wait_list)(struct gcip_mailbox *mailbox, void *resp,
struct gcip_mailbox_resp_awaiter *awaiter);
/*
* This callback will be called after putting the @cmd to the command queue. It can be used
* for triggering the doorbell. Returns 0 on success, or returns error code otherwise.
- * This is called with the `cmd_queue_lock` being held.
- * Context: normal.
+ *
+ * Context: cmd_queue_lock.
*/
int (*after_enqueue_cmd)(struct gcip_mailbox *mailbox, void *cmd);
/*
@@ -306,6 +311,7 @@
* a signal to break up waiting consuming the response queue. This is called without
* holding any locks.
* - @num_resps: the number of fetched responses.
+ *
* Context: normal and in_interrupt().
*/
void (*after_fetch_resps)(struct gcip_mailbox *mailbox, u32 num_resps);
@@ -313,6 +319,7 @@
* Before handling each fetched responses, this callback will be called. If this callback
* is not defined or returns true, the mailbox will handle the @resp normally. If the @resp
* should not be handled, returns false. This is called without holding any locks.
+ *
* Context: normal and in_interrupt().
*/
bool (*before_handle_resp)(struct gcip_mailbox *mailbox, const void *resp);
@@ -321,6 +328,7 @@
* chip implementation. However, @awaiter should be released by calling the
* `gcip_mailbox_release_awaiter` function when the kernel driver doesn't need
* @awaiter anymore.
+ *
* Context: normal and in_interrupt().
*/
void (*handle_awaiter_arrived)(struct gcip_mailbox *mailbox,
@@ -330,6 +338,7 @@
* implementation. However, @awaiter should be released by calling the
* `gcip_mailbox_release_awaiter` function when the kernel driver doesn't need
* @awaiter anymore. This is called without holding any locks.
+ *
* Context: normal and in_interrupt().
*/
void (*handle_awaiter_timedout)(struct gcip_mailbox *mailbox,
@@ -339,20 +348,22 @@
* The @awaiter should be marked as unprocessable to make it not to be processed by
* the `handle_awaiter_arrived` or `handle_awaiter_timedout` callbacks in race
* conditions. Don't have to release @awaiter of this function by calling the
- * `gcip_mailbox_release_awaiter` function. It will be released internally. This is
- * called with the `wait_list_lock` being held.
- * Context: normal.
+ * `gcip_mailbox_release_awaiter` function. It will be released internally.
+ *
+ * Context: wait_list_lock.
*/
void (*flush_awaiter)(struct gcip_mailbox *mailbox,
struct gcip_mailbox_resp_awaiter *awaiter);
/*
* Releases the @data which was passed to the `gcip_mailbox_put_cmd` function. This is
* called without holding any locks.
+ *
* Context: normal and in_interrupt().
*/
void (*release_awaiter_data)(void *data);
/*
* Checks if the block is off.
+ *
* Context: in_interrupt()
*/
bool (*is_block_off)(struct gcip_mailbox *mailbox);
@@ -362,11 +373,13 @@
* be fetched from @cmd, @resp or @data passed to the `gcip_mailbox_put_cmd` function.
* Therefore, this callback passes all of them not only @cmd. This can be called without
* holding any locks.
+ *
* Context: normal.
*/
u32 (*get_cmd_timeout)(struct gcip_mailbox *mailbox, void *cmd, void *resp, void *data);
/*
* Called when a command fails to be sent.
+ *
* Context: normal.
*/
void (*on_error)(struct gcip_mailbox *mailbox, int err);
diff --git a/gcip-kernel-driver/include/gcip/gcip-usage-stats.h b/gcip-kernel-driver/include/gcip/gcip-usage-stats.h
index a4e0cb4..4c400eb 100644
--- a/gcip-kernel-driver/include/gcip/gcip-usage-stats.h
+++ b/gcip-kernel-driver/include/gcip/gcip-usage-stats.h
@@ -60,7 +60,7 @@
#define GCIP_USAGE_STATS_METRIC_SIZE_V1 20
/* Max number of frequencies to support. */
-#define GCIP_USAGE_STATS_MAX_DVFS_FREQ_NUM 10
+#define GCIP_USAGE_STATS_MAX_DVFS_FREQ_NUM 25
struct gcip_usage_stats_attr;
@@ -457,7 +457,7 @@
* @idx will not exceed the number of default DVFS frequencies which is returned by the
* `get_default_dvfs_freqs_num` operator.
*/
- int (*get_default_dvfs_freq)(int idx, void *data);
+ unsigned int (*get_default_dvfs_freq)(int idx, void *data);
};
/* Structure manages the information of usage stats and device attributes. */
diff --git a/gcip-kernel-driver/include/gcip/iif/iif-manager.h b/gcip-kernel-driver/include/gcip/iif/iif-manager.h
deleted file mode 100644
index 3432e09..0000000
--- a/gcip-kernel-driver/include/gcip/iif/iif-manager.h
+++ /dev/null
@@ -1,47 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * GCIP-integrated IIF driver manager.
- *
- * Copyright (C) 2023 Google LLC
- */
-
-#ifndef __IIF_IIF_MANAGER_H__
-#define __IIF_IIF_MANAGER_H__
-
-#include <linux/idr.h>
-#include <linux/kref.h>
-#include <linux/of.h>
-
-#include <gcip/iif/iif-fence-table.h>
-
-/*
- * The structure overall data required by IIF driver such as fence table.
- *
- * Until we have stand-alone IIF driver, one of the IP drivers will initializes a manager by
- * the `iif_init` function and every IP driver will share it.
- */
-struct iif_manager {
- /* Reference count of this instance. */
- struct kref kref;
- /* Fence ID pool. */
- struct ida idp;
- /* Fence table shared with the firmware. */
- struct iif_fence_table fence_table;
-};
-
-/*
- * Initializes IIF driver and returns its manager. Its initial reference count is 1. It will map
- * the fence table by parsing the device tree via @np.
- *
- * The returned manager will be destroyed when its reference count becomes 0 by `iif_manager_put`
- * function.
- */
-struct iif_manager *iif_manager_init(const struct device_node *np);
-
-/* Increases the reference count of @mgr. */
-struct iif_manager *iif_manager_get(struct iif_manager *mgr);
-
-/* Decreases the reference count of @mgr and if it becomes 0, releases @mgr. */
-void iif_manager_put(struct iif_manager *mgr);
-
-#endif /* __IIF_IIF_MANAGER_H__ */
diff --git a/gcip-kernel-driver/include/gcip/iif/iif.h b/gcip-kernel-driver/include/gcip/iif/iif.h
deleted file mode 100644
index 0ccdb72..0000000
--- a/gcip-kernel-driver/include/gcip/iif/iif.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/* SPDX-License-Identifier: GPL-2.0-only */
-/*
- * Defines the interface of the IIF driver.
- *
- * Copyright (C) 2023 Google LLC
- */
-
-#ifndef __IIF_IIF_H__
-#define __IIF_IIF_H__
-
-#include <linux/ioctl.h>
-#include <linux/types.h>
-
-/* Interface Version. */
-#define IIF_INTERFACE_VERSION_MAJOR 1
-#define IIF_INTERFACE_VERSION_MINOR 0
-
-#define IIF_IOCTL_BASE 'i'
-
-/* The ioctl number for the fence FDs will start from here. */
-#define IIF_FENCE_IOCTL_NUM_BASE 0x80
-
-/*
- * The max number of fences can be created per IP.
- * Increasing this value needs to increase the size of fence table.
- */
-#define IIF_NUM_FENCES_PER_IP 1024
-
-/*
- * Type of IPs.
- *
- * The order of IP must be matched with the firmware side because the fence ID will be assigned
- * according to the IP type.
- */
-enum iif_ip_type {
- IIF_IP_DSP,
- IIF_IP_TPU,
- IIF_IP_GPU,
- IIF_IP_NUM,
-
- /* Reserve the number of IP type to expand the fence table easily in the future. */
- IIF_IP_RESERVED = 16,
-};
-
-/*
- * ioctls for /dev/iif.
- * TODO(b/312161537): introduce ioctls once we have a standalone IIF driver.
- */
-
-struct iif_fence_get_information_ioctl {
- /* The type of the signaler IP. (enum iif_ip_type) */
- __u8 signaler_ip;
- /* The number of total signalers. */
- __u16 total_signalers;
- /* The number of submitted signalers. */
- __u16 submitted_signalers;
- /* The number of signaled signalers. */
- __u16 signaled_signalers;
- /* The number of outstanding waiters. */
- __u16 outstanding_waiters;
- /* Reserved. */
- __u8 reserved[7];
-};
-
-/*
- * ioctls for inter-IP fence FDs.
- */
-
-/* Returns the fence information. */
-#define IIF_FENCE_GET_INFORMATION \
- _IOR(IIF_IOCTL_BASE, IIF_FENCE_IOCTL_NUM_BASE, struct iif_fence_get_information_ioctl)
-
-#endif /* __IIF_IIF_H__ */
diff --git a/gxp-client.c b/gxp-client.c
index a2a3dd0..dea2f61 100644
--- a/gxp-client.c
+++ b/gxp-client.c
@@ -141,6 +141,7 @@
down_write(&client->semaphore);
if (client->vd && client->vd->state != GXP_VD_OFF) {
+ gxp_vd_check_and_wait_for_debug_dump(client->vd);
down_write(&gxp->vd_semaphore);
gxp_vd_stop(client->vd);
up_write(&gxp->vd_semaphore);
@@ -413,6 +414,8 @@
if (client->vd->state == GXP_VD_UNAVAILABLE)
return;
+ gxp_vd_check_and_wait_for_debug_dump(client->vd);
+
down_write(&gxp->vd_semaphore);
gxp_vd_suspend(client->vd);
up_write(&gxp->vd_semaphore);
diff --git a/gxp-common-platform.c b/gxp-common-platform.c
index a2a5443..0524fe1 100644
--- a/gxp-common-platform.c
+++ b/gxp-common-platform.c
@@ -31,18 +31,19 @@
#include "gxp-client.h"
#include "gxp-config.h"
#include "gxp-core-telemetry.h"
+#include "gxp-dci.h"
#include "gxp-debug-dump.h"
#include "gxp-dma-fence.h"
#include "gxp-dma.h"
#include "gxp-dmabuf.h"
#include "gxp-domain-pool.h"
-#include "gxp-firmware.h"
#include "gxp-firmware-data.h"
#include "gxp-firmware-loader.h"
+#include "gxp-firmware.h"
#include "gxp-internal.h"
#include "gxp-lpm.h"
-#include "gxp-mailbox.h"
#include "gxp-mailbox-driver.h"
+#include "gxp-mailbox.h"
#include "gxp-mapping.h"
#include "gxp-notification.h"
#include "gxp-pm.h"
@@ -55,11 +56,6 @@
#include <soc/google/tpu-ext.h>
#endif
-#if GXP_USE_LEGACY_MAILBOX
-#include "gxp-mailbox-impl.h"
-#else
-#include "gxp-dci.h"
-#endif
/* We will only have one gxp device */
#define GXP_DEV_COUNT 1
@@ -590,206 +586,6 @@
return 0;
}
-static int gxp_ioctl_etm_trace_start_command(struct gxp_client *client,
- struct gxp_etm_trace_start_ioctl __user *argp)
-{
- struct gxp_dev *gxp = client->gxp;
- struct gxp_etm_trace_start_ioctl ibuf;
- int phys_core;
- int ret = 0;
-
- if (copy_from_user(&ibuf, argp, sizeof(ibuf)))
- return -EFAULT;
-
- ibuf.trace_ram_enable &= ETM_TRACE_LSB_MASK;
- ibuf.atb_enable &= ETM_TRACE_LSB_MASK;
-
- if (!ibuf.trace_ram_enable && !ibuf.atb_enable)
- return -EINVAL;
-
- if (!(ibuf.sync_msg_period == 0 ||
- (ibuf.sync_msg_period <= ETM_TRACE_SYNC_MSG_PERIOD_MAX &&
- ibuf.sync_msg_period >= ETM_TRACE_SYNC_MSG_PERIOD_MIN &&
- is_power_of_2(ibuf.sync_msg_period))))
- return -EINVAL;
-
- if (ibuf.pc_match_mask_length > ETM_TRACE_PC_MATCH_MASK_LEN_MAX)
- return -EINVAL;
-
- /* Caller must hold VIRTUAL_DEVICE wakelock */
- down_read(&client->semaphore);
-
- if (!check_client_has_available_vd_wakelock(
- client, "GXP_ETM_TRACE_START_COMMAND")) {
- ret = -ENODEV;
- goto out_unlock_client_semaphore;
- }
-
- down_read(&gxp->vd_semaphore);
-
- phys_core =
- gxp_vd_virt_core_to_phys_core(client->vd, ibuf.virtual_core_id);
- if (phys_core < 0) {
- dev_err(gxp->dev, "Trace start failed: Invalid virtual core id (%u)\n",
- ibuf.virtual_core_id);
- ret = -EINVAL;
- goto out;
- }
-
-out:
- up_read(&gxp->vd_semaphore);
-out_unlock_client_semaphore:
- up_read(&client->semaphore);
-
- return ret;
-}
-
-static int gxp_ioctl_etm_trace_sw_stop_command(struct gxp_client *client, __u16 __user *argp)
-{
- struct gxp_dev *gxp = client->gxp;
- u16 virtual_core_id;
- int phys_core;
- int ret = 0;
-
- if (copy_from_user(&virtual_core_id, argp, sizeof(virtual_core_id)))
- return -EFAULT;
-
- /* Caller must hold VIRTUAL_DEVICE wakelock */
- down_read(&client->semaphore);
-
- if (!check_client_has_available_vd_wakelock(
- client, "GXP_ETM_TRACE_SW_STOP_COMMAND")) {
- ret = -ENODEV;
- goto out_unlock_client_semaphore;
- }
-
- down_read(&gxp->vd_semaphore);
-
- phys_core = gxp_vd_virt_core_to_phys_core(client->vd, virtual_core_id);
- if (phys_core < 0) {
- dev_err(gxp->dev, "Trace stop via software trigger failed: Invalid virtual core id (%u)\n",
- virtual_core_id);
- ret = -EINVAL;
- goto out;
- }
-out:
- up_read(&gxp->vd_semaphore);
-out_unlock_client_semaphore:
- up_read(&client->semaphore);
-
- return ret;
-}
-
-static int gxp_ioctl_etm_trace_cleanup_command(struct gxp_client *client, __u16 __user *argp)
-{
- struct gxp_dev *gxp = client->gxp;
- u16 virtual_core_id;
- int phys_core;
- int ret = 0;
-
- if (copy_from_user(&virtual_core_id, argp, sizeof(virtual_core_id)))
- return -EFAULT;
-
- /* Caller must hold VIRTUAL_DEVICE wakelock */
- down_read(&client->semaphore);
-
- if (!check_client_has_available_vd_wakelock(
- client, "GXP_ETM_TRACE_CLEANUP_COMMAND")) {
- ret = -ENODEV;
- goto out_unlock_client_semaphore;
- }
-
- down_read(&gxp->vd_semaphore);
-
- phys_core = gxp_vd_virt_core_to_phys_core(client->vd, virtual_core_id);
- if (phys_core < 0) {
- dev_err(gxp->dev, "Trace cleanup failed: Invalid virtual core id (%u)\n",
- virtual_core_id);
- ret = -EINVAL;
- goto out;
- }
-out:
- up_read(&gxp->vd_semaphore);
-out_unlock_client_semaphore:
- up_read(&client->semaphore);
-
- return ret;
-}
-
-static int gxp_ioctl_etm_get_trace_info_command(struct gxp_client *client,
- struct gxp_etm_get_trace_info_ioctl __user *argp)
-{
- struct gxp_dev *gxp = client->gxp;
- struct gxp_etm_get_trace_info_ioctl ibuf;
- int phys_core;
- u32 *trace_header;
- u32 *trace_data;
- int ret = 0;
-
- if (copy_from_user(&ibuf, argp, sizeof(ibuf)))
- return -EFAULT;
-
- if (ibuf.type > 1)
- return -EINVAL;
-
- /* Caller must hold VIRTUAL_DEVICE wakelock */
- down_read(&client->semaphore);
-
- if (!check_client_has_available_vd_wakelock(
- client, "GXP_ETM_GET_TRACE_INFO_COMMAND")) {
- ret = -ENODEV;
- goto out_unlock_client_semaphore;
- }
-
- down_read(&gxp->vd_semaphore);
-
- phys_core = gxp_vd_virt_core_to_phys_core(client->vd, ibuf.virtual_core_id);
- if (phys_core < 0) {
- dev_err(gxp->dev, "Get trace info failed: Invalid virtual core id (%u)\n",
- ibuf.virtual_core_id);
- ret = -EINVAL;
- goto out;
- }
-
- trace_header = kzalloc(GXP_TRACE_HEADER_SIZE, GFP_KERNEL);
- if (!trace_header) {
- ret = -ENOMEM;
- goto out;
- }
-
- trace_data = kzalloc(GXP_TRACE_RAM_SIZE, GFP_KERNEL);
- if (!trace_data) {
- ret = -ENOMEM;
- goto out_free_header;
- }
-
- if (copy_to_user((void __user *)ibuf.trace_header_addr, trace_header,
- GXP_TRACE_HEADER_SIZE)) {
- ret = -EFAULT;
- goto out_free_data;
- }
-
- if (ibuf.type == 1) {
- if (copy_to_user((void __user *)ibuf.trace_data_addr,
- trace_data, GXP_TRACE_RAM_SIZE)) {
- ret = -EFAULT;
- goto out_free_data;
- }
- }
-
-out_free_data:
- kfree(trace_data);
-out_free_header:
- kfree(trace_header);
-
-out:
- up_read(&gxp->vd_semaphore);
-out_unlock_client_semaphore:
- up_read(&client->semaphore);
-
- return ret;
-}
-
#if HAS_TPU_EXT
/*
@@ -1117,7 +913,7 @@
* We intentionally don't call `gcip_pm_*` functions while holding @client->semaphore.
*
* As the `gcip_pm_put` function cancels KCI works synchronously and the KCI works may hold
- * @client->semaphore in some logics such as MCU FW crash handler, it can cause deadlock
+ * @client->semaphore in some logic such as MCU FW crash handler, it can cause deadlock
* issues potentially if we call `gcip_pm_put` after holding @client->semaphore.
*
* Therefore, we decided to decouple calling the `gcip_pm_put` function from holding
@@ -1733,18 +1529,6 @@
case GXP_ALLOCATE_VIRTUAL_DEVICE:
ret = gxp_ioctl_allocate_vd(client, argp);
break;
- case GXP_ETM_TRACE_START_COMMAND:
- ret = gxp_ioctl_etm_trace_start_command(client, argp);
- break;
- case GXP_ETM_TRACE_SW_STOP_COMMAND:
- ret = gxp_ioctl_etm_trace_sw_stop_command(client, argp);
- break;
- case GXP_ETM_TRACE_CLEANUP_COMMAND:
- ret = gxp_ioctl_etm_trace_cleanup_command(client, argp);
- break;
- case GXP_ETM_GET_TRACE_INFO_COMMAND:
- ret = gxp_ioctl_etm_get_trace_info_command(client, argp);
- break;
case GXP_MAP_TPU_MBX_QUEUE:
ret = gxp_ioctl_map_tpu_mbx_queue(client, argp);
break;
@@ -2236,19 +2020,14 @@
dev_err(dev, "Failed to create mailbox manager: %d\n", ret);
goto err_dma_exit;
}
- if (gxp_is_direct_mode(gxp)) {
-#if GXP_USE_LEGACY_MAILBOX
- gxp_mailbox_init(gxp->mailbox_mgr);
-#else
+ if (gxp_is_direct_mode(gxp))
gxp_dci_init(gxp->mailbox_mgr);
-#endif
- }
#if IS_ENABLED(CONFIG_SUBSYSTEM_COREDUMP)
ret = gxp_debug_dump_init(gxp, &gxp_sscd_dev, &gxp_sscd_pdata);
#else
ret = gxp_debug_dump_init(gxp, NULL, NULL);
-#endif // !CONFIG_SUBSYSTEM_COREDUMP
+#endif /* !CONFIG_SUBSYSTEM_COREDUMP */
if (ret)
dev_warn(dev, "Failed to initialize debug dump\n");
@@ -2330,11 +2109,6 @@
if (ret)
goto err_dma_fence_destroy;
}
- /*
- * We only know where the system config region is after after_probe is
- * done so this can't be called earlier.
- */
- gxp_fw_data_populate_system_config(gxp);
ret = gxp_device_add(gxp);
if (ret)
diff --git a/gxp-config.h b/gxp-config.h
index 56a84bf..de6bdfc 100644
--- a/gxp-config.h
+++ b/gxp-config.h
@@ -32,10 +32,6 @@
#define SYNC_BARRIER_COUNT 16
-#ifndef GXP_USE_LEGACY_MAILBOX
-#define GXP_USE_LEGACY_MAILBOX 0
-#endif
-
#ifndef GXP_HAS_LAP
#define GXP_HAS_LAP 1
#endif
diff --git a/gxp-core-telemetry.c b/gxp-core-telemetry.c
index 096eb53..2e8339d 100644
--- a/gxp-core-telemetry.c
+++ b/gxp-core-telemetry.c
@@ -394,7 +394,7 @@
unsigned long orig_pgoff = vma->vm_pgoff;
if (!phys) {
- dev_err(gxp->dev, "Secure buffer is not mapped.\n");
+ dev_warn(gxp->dev, "Secure buffer is not mapped.\n");
return -ENODATA;
}
diff --git a/gxp-dci.c b/gxp-dci.c
index 5f658f8..29da8d2 100644
--- a/gxp-dci.c
+++ b/gxp-dci.c
@@ -2,7 +2,7 @@
/*
* Implementation of DCI (Direct Command Interface) using mailbox.
*
- * Copyright (C) 2022 Google LLC
+ * Copyright (C) 2022-2024 Google LLC
*/
#include <linux/kthread.h>
@@ -11,6 +11,7 @@
#include <uapi/linux/sched/types.h>
#include "gxp-dci.h"
+#include "gxp-debug-dump.h"
#include "gxp-dma.h"
#include "gxp-mailbox-driver.h"
#include "gxp-pm.h"
@@ -281,13 +282,14 @@
spin_unlock_irqrestore(async_resp->dest_queue_lock, flags);
}
-static void
-gxp_dci_handle_awaiter_timedout(struct gcip_mailbox *mailbox,
- struct gcip_mailbox_resp_awaiter *awaiter)
+static void gxp_dci_handle_awaiter_timedout(struct gcip_mailbox *mailbox,
+ struct gcip_mailbox_resp_awaiter *awaiter)
{
struct gxp_mailbox *mbx = mailbox->data;
struct gxp_dci_async_response *async_resp = awaiter->data;
struct gxp_dci_response *resp = &async_resp->resp;
+ struct gxp_dev *gxp = mbx->gxp;
+ struct gxp_virtual_device *vd;
unsigned long flags;
/*
@@ -304,8 +306,21 @@
list_add_tail(&async_resp->list_entry, async_resp->dest_queue);
spin_unlock_irqrestore(async_resp->dest_queue_lock, flags);
- gxp_pm_update_requested_power_states(
- mbx->gxp, async_resp->requested_states, off_states);
+ /* Get hold of the virtual device. */
+ down_read(&gxp->vd_semaphore);
+ if (gxp->core_to_vd[mbx->core_id]) {
+ vd = gxp->core_to_vd[mbx->core_id];
+ /*
+ * Response timeout most probably would happen because of core stall. Check
+ * if forced debug dump can be requested to the participating cores for the
+ * current vd.
+ */
+ gxp_debug_dump_send_forced_debug_dump_request(gxp, vd);
+ }
+ up_read(&gxp->vd_semaphore);
+
+ gxp_pm_update_requested_power_states(mbx->gxp, async_resp->requested_states,
+ off_states);
if (async_resp->eventfd) {
gxp_eventfd_signal(async_resp->eventfd);
@@ -337,7 +352,6 @@
}
static const struct gcip_mailbox_ops gxp_dci_gcip_mbx_ops = {
- .get_cmd_queue_head = gxp_mailbox_gcip_ops_get_cmd_queue_head,
.get_cmd_queue_tail = gxp_mailbox_gcip_ops_get_cmd_queue_tail,
.inc_cmd_queue_tail = gxp_mailbox_gcip_ops_inc_cmd_queue_tail,
.acquire_cmd_queue_lock = gxp_mailbox_gcip_ops_acquire_cmd_queue_lock,
@@ -460,25 +474,31 @@
.cmd_elem_size = sizeof(struct gxp_dci_command),
.resp_elem_size = sizeof(struct gxp_dci_response),
};
+ struct gxp_mailbox *mbx;
dci = kzalloc(sizeof(*dci), GFP_KERNEL);
if (!dci)
return ERR_PTR(-ENOMEM);
mbx_args.data = dci;
- dci->mbx = gxp_mailbox_alloc(mgr, vd, virt_core, core_id, &mbx_args);
- if (IS_ERR(dci->mbx))
+ mbx = gxp_mailbox_alloc(mgr, vd, virt_core, core_id, &mbx_args);
+ if (IS_ERR(mbx)) {
kfree(dci);
- else
- gxp_mailbox_generate_device_interrupt(dci->mbx, BIT(0));
+ return mbx;
+ }
+ dci->mbx = mbx;
+ gxp_mailbox_reinit(dci->mbx);
+ gxp_mailbox_generate_device_interrupt(dci->mbx, BIT(0));
- return dci->mbx;
+ return mbx;
}
void gxp_dci_release(struct gxp_mailbox_manager *mgr,
struct gxp_virtual_device *vd, uint virt_core,
struct gxp_mailbox *mbx)
{
+ /* Frees dci. */
+ kfree(mbx->data);
gxp_mailbox_release(mgr, vd, virt_core, mbx);
}
diff --git a/gxp-dci.h b/gxp-dci.h
index 09eaed4..7367939 100644
--- a/gxp-dci.h
+++ b/gxp-dci.h
@@ -2,7 +2,7 @@
/*
* Implementation of DCI (Direct Command Interface) using mailbox.
*
- * Copyright (C) 2022 Google LLC
+ * Copyright (C) 2022-2024 Google LLC
*/
#ifndef __GXP_DCI_H__
diff --git a/gxp-debug-dump.c b/gxp-debug-dump.c
index 381438e..e19bbc3 100644
--- a/gxp-debug-dump.c
+++ b/gxp-debug-dump.c
@@ -23,16 +23,14 @@
#include "gxp-debug-dump.h"
#include "gxp-dma.h"
#include "gxp-doorbell.h"
-#include "gxp-firmware.h"
#include "gxp-firmware-data.h"
#include "gxp-firmware-loader.h"
+#include "gxp-firmware.h"
#include "gxp-host-device-structs.h"
#include "gxp-internal.h"
#include "gxp-lpm.h"
#include "gxp-mailbox-driver.h"
#include "gxp-mapping.h"
-#include "gxp-mcu.h"
-#include "gxp-mcu-telemetry.h"
#include "gxp-notification.h"
#include "gxp-pm.h"
#include "gxp-vd.h"
@@ -41,6 +39,11 @@
#include <linux/platform_data/sscoredump.h>
#endif
+#if GXP_HAS_MCU
+#include "gxp-mcu-telemetry.h"
+#include "gxp-mcu.h"
+#endif /* GXP_HAS_MCU */
+
#define SSCD_MSG_LENGTH 64
#define GXP_SYNC_BARRIER_STRIDE (GXP_REG_SYNC_BARRIER_1 - GXP_REG_SYNC_BARRIER_0)
@@ -48,15 +51,6 @@
#define DEBUG_DUMP_MEMORY_SIZE 0x400000 /* size in bytes */
/*
- * The minimum wait time in millisecond to be enforced between two successive calls to the SSCD
- * module to prevent the overwrite of the previous generated core dump files. SSCD module generates
- * the files whose name are at second precision i.e.
- * crashinfo_<SUBSYSTEM_NAME>_<%Y-%m-%d_%H-%M-%S>.txt and
- * coredump_<SUBSYSTEM_NAME>_<%Y-%m-%d_%H-%M-%S>.bin.
- */
-#define SSCD_REPORT_WAIT_TIME (1000ULL)
-
-/*
* CORE_FIRMWARE_RW_STRIDE & CORE_FIRMWARE_RW_ADDR must match with their
* values defind in core firmware image config.
*/
@@ -613,6 +607,37 @@
core_dump_header->core_header.dump_available = 0;
}
+void gxp_debug_dump_send_forced_debug_dump_request(struct gxp_dev *gxp,
+ struct gxp_virtual_device *vd)
+{
+ uint phys_core;
+ uint generate_debug_dump;
+ uint debug_dump_generated;
+
+ lockdep_assert_held(&gxp->vd_semaphore);
+
+ for (phys_core = 0; phys_core < GXP_NUM_CORES; phys_core++) {
+ if (!(vd->core_list & BIT(phys_core)))
+ continue;
+
+ generate_debug_dump = gxp_firmware_get_generate_debug_dump(gxp, vd, phys_core);
+ debug_dump_generated = gxp_firmware_get_debug_dump_generated(gxp, vd, phys_core);
+ /*
+ * If neither the core has generated the debug dump nor has been requested to
+ * generate the forced debug dump.
+ */
+ if (!debug_dump_generated && !generate_debug_dump) {
+ if (!gxp_lpm_is_powered(gxp, CORE_TO_PSM(phys_core))) {
+ dev_dbg(gxp->dev, "Core%u not powered on.\n", phys_core);
+ continue;
+ }
+ /* Send the interrupt to the core for requesting the forced debug dump. */
+ gxp_firmware_set_generate_debug_dump(gxp, vd, phys_core, 1);
+ gxp_notification_send(gxp, phys_core, CORE_NOTIF_GENERATE_DEBUG_DUMP);
+ }
+ }
+}
+
/*
* Caller must make sure that gxp->debug_dump_mgr->common_dump and
* gxp->debug_dump_mgr->core_dump are not NULL.
@@ -790,10 +815,12 @@
uint core_id = debug_dump_work->core_id;
struct gxp_dev *gxp = debug_dump_work->gxp;
struct gxp_virtual_device *vd = NULL;
+ int old_core_dump_generated_list;
down_read(&gxp->vd_semaphore);
if (gxp->core_to_vd[core_id]) {
vd = gxp_vd_get(gxp->core_to_vd[core_id]);
+ gxp_debug_dump_send_forced_debug_dump_request(gxp, vd);
} else {
dev_warn(gxp->dev, "debug dump failed for null vd on core %d.", core_id);
up_read(&gxp->vd_semaphore);
@@ -810,6 +837,15 @@
gxp_generate_debug_dump(gxp, core_id, vd);
+ /* Update the debug dump processing status for the current core. */
+ old_core_dump_generated_list = atomic_fetch_or(BIT(core_id), &vd->core_dump_generated_list);
+ /*
+ * Event the wait queue in case debug dump processing has been finished for all the
+ * running cores for the vd.
+ */
+ if ((old_core_dump_generated_list | BIT(core_id)) == vd->core_list)
+ wake_up(&vd->finished_dump_processing_waitq);
+
mutex_unlock(&vd->debug_dump_lock);
gxp_vd_put(vd);
}
diff --git a/gxp-debug-dump.h b/gxp-debug-dump.h
index 5bcec03..e03155e 100644
--- a/gxp-debug-dump.h
+++ b/gxp-debug-dump.h
@@ -39,6 +39,15 @@
(GXP_NUM_COMMON_SEGMENTS + GXP_NUM_CORE_SEGMENTS + GXP_NUM_CORE_DATA_SEGMENTS + \
GXP_NUM_BUFFER_MAPPINGS)
+/*
+ * The minimum wait time in millisecond to be enforced between two successive calls to the SSCD
+ * module to prevent the overwrite of the previous generated core dump files. SSCD module generates
+ * the files whose name are at second precision i.e.
+ * crashinfo_<SUBSYSTEM_NAME>_<%Y-%m-%d_%H-%M-%S>.txt and
+ * coredump_<SUBSYSTEM_NAME>_<%Y-%m-%d_%H-%M-%S>.bin.
+ */
+#define SSCD_REPORT_WAIT_TIME (1000ULL)
+
#define GXP_Q7_ICACHE_SIZE 131072 /* I-cache size in bytes */
#define GXP_Q7_ICACHE_LINESIZE 64 /* I-cache line size in bytes */
#define GXP_Q7_ICACHE_WAYS 4
@@ -238,6 +247,22 @@
void gxp_debug_dump_invalidate_segments(struct gxp_dev *gxp, uint32_t core_id);
/**
+ * gxp_debug_dump_send_forced_debug_dump_request() - Sends the forced debug dump request to the
+ * running cores of the given vd.
+ * @gxp: The GXP device to obtain the handler for.
+ * @vd: vd whose cores to send the forced debug dump request.
+ *
+ * The caller must hold @gxp->vd_semaphore.
+ *
+ * In case of single core VD no forced debug dump request will be sent since there will be no other
+ * core. For multicore VD, forced debug dump request will be sent to other cores only during the
+ * first worker thread run. For successive worker thread runs since the forced request was already
+ * sent, it will not be sent again.
+ */
+void gxp_debug_dump_send_forced_debug_dump_request(struct gxp_dev *gxp,
+ struct gxp_virtual_device *vd);
+
+/**
* gxp_debug_dump_process_dump_mcu_mode() - Checks and process the debug dump
* for cores from core_list.
* @gxp: The GXP device to obtain the handler for
diff --git a/gxp-dma-iommu.c b/gxp-dma-iommu.c
index 32c86f2..77537b9 100644
--- a/gxp-dma-iommu.c
+++ b/gxp-dma-iommu.c
@@ -38,29 +38,26 @@
switch (fault->type) {
case IOMMU_FAULT_DMA_UNRECOV:
dev_err(gxp->dev, "Unrecoverable IOMMU fault!\n");
+ dev_err(gxp->dev, "reason = %08X\n", fault->event.reason);
+ dev_err(gxp->dev, "flags = %08X\n", fault->event.flags);
+ dev_err(gxp->dev, "pasid = %08X\n", fault->event.pasid);
+ dev_err(gxp->dev, "perm = %08X\n", fault->event.perm);
+ dev_err(gxp->dev, "addr = %llX\n", fault->event.addr);
+ dev_err(gxp->dev, "fetch_addr = %llX\n", fault->event.fetch_addr);
break;
case IOMMU_FAULT_PAGE_REQ:
dev_err(gxp->dev, "IOMMU page request fault!\n");
+ dev_err(gxp->dev, "flags = %08X\n", fault->prm.flags);
+ dev_err(gxp->dev, "pasid = %08X\n", fault->prm.pasid);
+ dev_err(gxp->dev, "grpid = %08X\n", fault->prm.grpid);
+ dev_err(gxp->dev, "perm = %08X\n", fault->prm.perm);
+ dev_err(gxp->dev, "addr = %llX\n", fault->prm.addr);
break;
default:
dev_err(gxp->dev, "Unexpected IOMMU fault type (%d)\n",
fault->type);
- return -EAGAIN;
}
- /*
- * Normally the iommu driver should fill out the `event` struct for
- * unrecoverable errors, and the `prm` struct for page request faults.
- * The SysMMU driver, instead, always fills out the `event` struct.
- *
- * Note that the `fetch_addr` and `perm` fields are never filled out,
- * so we skip printing them.
- */
- dev_err(gxp->dev, "reason = %08X\n", fault->event.reason);
- dev_err(gxp->dev, "flags = %08X\n", fault->event.flags);
- dev_err(gxp->dev, "pasid = %08X\n", fault->event.pasid);
- dev_err(gxp->dev, "addr = %llX\n", fault->event.addr);
-
// Tell the IOMMU driver to carry on
return -EAGAIN;
}
diff --git a/gxp-dmabuf.c b/gxp-dmabuf.c
index c51cd0b..a641ab7 100644
--- a/gxp-dmabuf.c
+++ b/gxp-dmabuf.c
@@ -25,12 +25,12 @@
dma_addr_t device_address = mapping->gcip_mapping->device_address;
size_t size = mapping->gcip_mapping->size;
- trace_gxp_mapping_destroy_start(device_address, size);
+ trace_gxp_dmabuf_mapping_destroy_start(device_address, size);
gcip_iommu_mapping_unmap(mapping->gcip_mapping);
kfree(mapping);
- trace_gxp_mapping_destroy_end(device_address, size);
+ trace_gxp_dmabuf_mapping_destroy_end(device_address, size);
}
struct gxp_mapping *gxp_dmabuf_map(struct gxp_dev *gxp, struct gcip_iommu_reserve_manager *mgr,
diff --git a/gxp-firmware-data.c b/gxp-firmware-data.c
index 4d1fd10..b5db087 100644
--- a/gxp-firmware-data.c
+++ b/gxp-firmware-data.c
@@ -38,16 +38,18 @@
* is provided to all VDs and all cores. This is the R/W section.
*/
struct gxp_system_descriptor_rw *sys_desc_rw;
+ /* The size of system config region. */
+ u32 sys_cfg_size;
};
/*
- * Here assumes sys_cfg contains gxp_system_descriptor_ro in the first page and
- * gxp_system_descriptor_rw in the second page.
+ * Here assumes sys_cfg contains gxp_system_descriptor_ro in the first half and
+ * gxp_system_descriptor_rw in the second half.
*/
static void set_system_cfg_region(struct gxp_dev *gxp, void *sys_cfg)
{
struct gxp_system_descriptor_ro *des_ro = sys_cfg;
- struct gxp_system_descriptor_rw *des_rw = sys_cfg + SZ_4K;
+ struct gxp_system_descriptor_rw *des_rw = sys_cfg + gxp->data_mgr->sys_cfg_size / 2;
struct gxp_core_telemetry_descriptor *descriptor =
&gxp->data_mgr->core_telemetry_desc;
struct telemetry_descriptor_ro *ro;
@@ -226,10 +228,11 @@
struct gxp_mapped_resource res = gxp_fw_data_resource(gxp);
/* Use the end of the shared region for system cfg. */
- return res.vaddr + res.size - GXP_FW_DATA_SYSCFG_SIZE;
+ return res.vaddr + res.size - gxp->data_mgr->sys_cfg_size;
}
-void gxp_fw_data_populate_system_config(struct gxp_dev *gxp)
+void gxp_fw_data_populate_system_config(struct gxp_dev *gxp, u32 sys_cfg_size)
{
+ gxp->data_mgr->sys_cfg_size = sys_cfg_size;
set_system_cfg_region(gxp, gxp_fw_data_system_cfg(gxp));
}
diff --git a/gxp-firmware-data.h b/gxp-firmware-data.h
index 028e610..e92f35b 100644
--- a/gxp-firmware-data.h
+++ b/gxp-firmware-data.h
@@ -15,8 +15,6 @@
#include "gxp-internal.h"
#include "gxp-vd.h"
-#define GXP_FW_DATA_SYSCFG_SIZE SZ_8K
-
enum gxp_fw_data_protocol {
/* Use the per-VD configuration region. */
FW_DATA_PROTOCOL_PER_VD_CONFIG = 2,
@@ -109,13 +107,14 @@
void *gxp_fw_data_system_cfg(struct gxp_dev *gxp);
/**
- * gxp_fw_data_populate_system_config() - Populate settings onto firmware system
- * config region.
- * @gxp: The GXP device
+ * gxp_fw_data_populate_system_config() - Populate settings onto firmware system config region.
+ * @gxp: The GXP device.
+ * @sys_cfg_size: The size of system config region.
*
- * This function is expected to be called after "after_probe" in the probe
- * procedure since it uses gxp_fw_data_system_cfg().
+ * This function is expected to be called after firmware loaded to know the correct size of the
+ * system config.
+ * The size will be recorded in gxp->data_mgr.
*/
-void gxp_fw_data_populate_system_config(struct gxp_dev *gxp);
+void gxp_fw_data_populate_system_config(struct gxp_dev *gxp, u32 sys_cfg_size);
#endif /* __GXP_FIRMWARE_DATA_H__ */
diff --git a/gxp-firmware-loader.c b/gxp-firmware-loader.c
index 1a145a3..6807484 100644
--- a/gxp-firmware-loader.c
+++ b/gxp-firmware-loader.c
@@ -12,6 +12,7 @@
#include <gcip/gcip-image-config.h>
#include "gxp-config.h"
+#include "gxp-firmware-data.h"
#include "gxp-firmware-loader.h"
#include "gxp-firmware.h"
#include "gxp-internal.h"
@@ -246,13 +247,27 @@
int gxp_firmware_loader_load_if_needed(struct gxp_dev *gxp)
{
struct gxp_firmware_loader_manager *mgr = gxp->fw_loader_mgr;
+ resource_size_t addr_size;
int ret = 0;
mutex_lock(&mgr->lock);
if (mgr->is_loaded)
- goto out;
+ goto out_unlock;
ret = gxp_firmware_loader_load_locked(gxp);
-out:
+ if (ret)
+ goto out_unlock;
+ /*
+ * This should be done only after the firmware is successfully loaded because the size of
+ * system config region is required.
+ */
+ addr_size = mgr->core_img_cfg.iommu_mappings[SYS_CFG_REGION_IDX].image_config_value;
+ mutex_unlock(&mgr->lock);
+
+ gxp_fw_data_populate_system_config(gxp, gcip_config_to_size(addr_size));
+
+ return 0;
+
+out_unlock:
mutex_unlock(&mgr->lock);
return ret;
}
diff --git a/gxp-firmware.c b/gxp-firmware.c
index f9049a0..9c0addc 100644
--- a/gxp-firmware.c
+++ b/gxp-firmware.c
@@ -334,6 +334,8 @@
core_cfg->top_access_ok = 0;
core_cfg->boot_status = GXP_BOOT_STATUS_NONE;
gxp_firmware_set_boot_mode(gxp, vd, core, GXP_BOOT_MODE_COLD_BOOT);
+ gxp_firmware_set_debug_dump_generated(gxp, vd, core, 0);
+ gxp_firmware_set_generate_debug_dump(gxp, vd, core, 0);
}
static int gxp_firmware_handshake(struct gxp_dev *gxp,
@@ -1008,7 +1010,7 @@
gxp_notification_unregister_handler(gxp, phys_core,
HOST_NOTIF_CORE_TELEMETRY_STATUS);
- if (gxp->mailbox_mgr->release_mailbox) {
+ if (gxp->mailbox_mgr->release_mailbox && gxp->mailbox_mgr->mailboxes[phys_core]) {
gxp->mailbox_mgr->release_mailbox(gxp->mailbox_mgr, vd, virt_core,
gxp->mailbox_mgr->mailboxes[phys_core]);
dev_notice(gxp->dev, "Mailbox %u released\n", phys_core);
@@ -1150,8 +1152,43 @@
core_cfg->boot_status = status;
}
-u32 gxp_firmware_get_boot_status(struct gxp_dev *gxp,
- struct gxp_virtual_device *vd, uint core)
+u32 gxp_firmware_get_generate_debug_dump(struct gxp_dev *gxp, struct gxp_virtual_device *vd,
+ uint core)
+{
+ struct gxp_host_control_region *core_cfg;
+
+ core_cfg = get_scratchpad_base(gxp, vd, core);
+ return core_cfg->generate_debug_dump;
+}
+
+void gxp_firmware_set_generate_debug_dump(struct gxp_dev *gxp, struct gxp_virtual_device *vd,
+ uint core, u32 generate_debug_dump)
+{
+ struct gxp_host_control_region *core_cfg;
+
+ core_cfg = get_scratchpad_base(gxp, vd, core);
+ core_cfg->generate_debug_dump = generate_debug_dump;
+}
+
+u32 gxp_firmware_get_debug_dump_generated(struct gxp_dev *gxp, struct gxp_virtual_device *vd,
+ uint core)
+{
+ struct gxp_host_control_region *core_cfg;
+
+ core_cfg = get_scratchpad_base(gxp, vd, core);
+ return core_cfg->debug_dump_generated;
+}
+
+void gxp_firmware_set_debug_dump_generated(struct gxp_dev *gxp, struct gxp_virtual_device *vd,
+ uint core, u32 debug_dump_generated)
+{
+ struct gxp_host_control_region *core_cfg;
+
+ core_cfg = get_scratchpad_base(gxp, vd, core);
+ core_cfg->debug_dump_generated = debug_dump_generated;
+}
+
+u32 gxp_firmware_get_boot_status(struct gxp_dev *gxp, struct gxp_virtual_device *vd, uint core)
{
struct gxp_host_control_region *core_cfg;
diff --git a/gxp-firmware.h b/gxp-firmware.h
index 40ee22b..85436e9 100644
--- a/gxp-firmware.h
+++ b/gxp-firmware.h
@@ -22,8 +22,6 @@
CORE_CFG_REGION_IDX,
VD_CFG_REGION_IDX,
SYS_CFG_REGION_IDX,
- /* TODO(b/299037074): Remove core's accesses to LPM. */
- REMOTE_LPM_IDX = 7,
};
struct gxp_firmware_manager {
@@ -126,6 +124,34 @@
struct gxp_virtual_device *vd, uint core);
/*
+ * Returns the `generate_debug_dump` flag from the given virtual device's shared host-core region
+ * for the specified core.
+ */
+u32 gxp_firmware_get_generate_debug_dump(struct gxp_dev *gxp, struct gxp_virtual_device *vd,
+ uint core);
+
+/*
+ * Sets the `generate_debug_dump` flag for the given virtual device's shared host-core region
+ * for the specified core.
+ */
+void gxp_firmware_set_generate_debug_dump(struct gxp_dev *gxp, struct gxp_virtual_device *vd,
+ uint core, u32 generate_debug_dump);
+
+/*
+ * Returns the `debug_dump_generated` flag from the given virtual device's shared host-core region
+ * for the specified core.
+ */
+u32 gxp_firmware_get_debug_dump_generated(struct gxp_dev *gxp, struct gxp_virtual_device *vd,
+ uint core);
+
+/*
+ * Sets the `generate_debug_dump` flag from the given virtual device's shared host-core region
+ * for the specified core.
+ */
+void gxp_firmware_set_debug_dump_generated(struct gxp_dev *gxp, struct gxp_virtual_device *vd,
+ uint core, u32 debug_dump_generated);
+
+/*
* Disable external interrupts to core.
*/
void gxp_firmware_disable_ext_interrupts(struct gxp_dev *gxp, uint core);
diff --git a/gxp-gsa.h b/gxp-gsa.h
index 6fcfb6c..3ff544b 100644
--- a/gxp-gsa.h
+++ b/gxp-gsa.h
@@ -2,7 +2,7 @@
/*
* Wrapper for GSA related APIs.
*
- * Copyright (C) 2023 Google LLC
+ * Copyright (C) 2023-2024 Google LLC
*/
#ifndef __GXP_GSA_H__
diff --git a/gxp-gsx01-mailbox.c b/gxp-gsx01-mailbox.c
index db4c1aa..22d06f6 100644
--- a/gxp-gsx01-mailbox.c
+++ b/gxp-gsx01-mailbox.c
@@ -18,19 +18,14 @@
#include "gxp-mailbox-driver.c"
-static u32 csr_read(struct gxp_mailbox *mailbox, uint reg_offset)
+static u32 gxp_mailbox_get_interrupt_status(struct gxp_mailbox *mailbox)
{
- return readl(mailbox->csr_reg_base + reg_offset);
-}
-
-static void csr_write(struct gxp_mailbox *mailbox, uint reg_offset, u32 value)
-{
- writel(value, mailbox->csr_reg_base + reg_offset);
+ return gxp_mailbox_csr_read(mailbox, MBOX_INTMSR1_OFFSET);
}
void gxp_mailbox_reset_hw(struct gxp_mailbox *mailbox)
{
- csr_write(mailbox, MBOX_MCUCTLR_OFFSET, 1);
+ gxp_mailbox_csr_write(mailbox, MBOX_MCUCTLR_OFFSET, 1);
}
/* Interrupt to signal a response from the device to host */
@@ -78,23 +73,18 @@
*/
wmb();
- csr_write(mailbox, MBOX_INTGR0_OFFSET, int_mask);
+ gxp_mailbox_csr_write(mailbox, MBOX_INTGR0_OFFSET, int_mask);
}
void gxp_mailbox_clear_interrupts(struct gxp_mailbox *mailbox, u32 intr_bits)
{
- csr_write(mailbox, MBOX_INTCR1_OFFSET, intr_bits);
+ gxp_mailbox_csr_write(mailbox, MBOX_INTCR1_OFFSET, intr_bits);
}
void gxp_mailbox_enable_interrupt(struct gxp_mailbox *mailbox)
{
}
-u32 gxp_mailbox_get_interrupt_status(struct gxp_mailbox *mailbox)
-{
- return csr_read(mailbox, MBOX_INTMSR1_OFFSET);
-}
-
int gxp_mailbox_wait_for_device_mailbox_init(struct gxp_mailbox *mailbox)
{
return 0;
diff --git a/gxp-gsx01-ssmt.c b/gxp-gsx01-ssmt.c
index 190316d..4efdf78 100644
--- a/gxp-gsx01-ssmt.c
+++ b/gxp-gsx01-ssmt.c
@@ -2,7 +2,7 @@
/*
* GXP SSMT driver.
*
- * Copyright (C) 2022 Google LLC
+ * Copyright (C) 2022-2024 Google LLC
*/
#include <linux/platform_device.h>
diff --git a/gxp-gsx01-ssmt.h b/gxp-gsx01-ssmt.h
index 16eb82a..1fed70a 100644
--- a/gxp-gsx01-ssmt.h
+++ b/gxp-gsx01-ssmt.h
@@ -2,7 +2,7 @@
/*
* GXP SSMT driver.
*
- * Copyright (C) 2022 Google LLC
+ * Copyright (C) 2022-2024 Google LLC
*/
#ifndef __GXP_SSMT_H__
diff --git a/gxp-host-device-structs.h b/gxp-host-device-structs.h
index 8fe8a78..e2c38fa 100644
--- a/gxp-host-device-structs.h
+++ b/gxp-host-device-structs.h
@@ -10,20 +10,41 @@
* Since the header is shared with the FW, it cannot rely on kernel-specific
* headers or data structures.
*
+ * Note: since the structures are shared across entities (cores, kernel, MCU),
+ * and they may change as the code is running (since they are used to build the
+ * system synchronization primitives), care should be taken to prevent the
+ * compiler from optimizing reads to such structures (for example, when polling
+ * on a value of a member in a struct; waiting for another core to change it).
+ * To achieve that, it's generally advised to access these structures as
+ * volatile to forbid the compiler from caching field values in CPU registers.
*/
#ifndef __GXP_HOST_DEVICE_STRUCTURES_H__
#define __GXP_HOST_DEVICE_STRUCTURES_H__
+/*
+ * The structure currently doesn't change its layout between the different
+ * HW generations; thus max core count is kept the same for all.
+ */
#define MAX_NUM_CORES 4
-/* The number of physical doorbells and sync barriers allocated to each VD */
+/*
+ * The number of physical doorbells, sync barriers and timers allocated to each
+ * VD. The HW supports a total of 16 sync barriers; divided across 4 active VDs.
+ * Some are reserved for system use such as the UART sync barrier. It also
+ * supports a total of 32 doorbells; divided across 4 active VDs. Some are
+ * reserved for system use; such as the 4 doorbells for waking up cores.
+ * 8 timers are also supported by the HW. Some are reserved for system use; such
+ * as timers 0-3 and timer 7.
+ */
#define GXP_NUM_DOORBELLS_PER_VD 7
#define GXP_NUM_SYNC_BARRIERS_PER_VD 4
+#define GXP_NUM_TIMERS_PER_VD 1
/* The first allowed doorbell and sync barrier to be used for VDs' usage */
#define GXP_DOORBELLS_START 4 /* The first 4 are used for boot */
#define GXP_SYNC_BARRIERS_START 1 /* The first 1 is used for UART */
+#define GXP_TIMERS_START 4 /* The first 4 are used for global and cores */
/* Definitions for host->device boot mode requests */
/*
@@ -89,12 +110,49 @@
#define GXP_BOOT_STATUS_INVALID_MODE 4
#define GXP_BOOT_STATUS_BOOTING 5
#define GXP_BOOT_STATUS_BOOTING_FAILED 6
-#define GXP_BOOT_STATUS_SUSPENDING 7
-#define GXP_BOOT_STATUS_SUSPENDING_FAILED 8
-#define GXP_BOOT_STATUS_SUSPENDING_FAILED_ACTIVE_WL 9
-#define GXP_BOOT_STATUS_WAITING_FOR_WORKLOAD 10
-#define GXP_BOOT_STATUS_WAITING_FOR_DMA 11
-#define GXP_BOOT_STATUS_SHUTTING_DOWN 12
+#define GXP_BOOT_STATUS_SUSPEND_WAITING_FOR_WL 20
+#define GXP_BOOT_STATUS_SUSPEND_WAITING_FOR_DMA 21
+#define GXP_BOOT_STATUS_SUSPEND_SAVING_TCM 22
+#define GXP_BOOT_STATUS_SUSPEND_SAVING_TCM_FAILED 23
+#define GXP_BOOT_STATUS_SUSPEND_SAVING_TOP 24
+#define GXP_BOOT_STATUS_SUSPEND_SAVING_CORE 25
+#define GXP_BOOT_STATUS_SUSPEND_FAILED 26
+#define GXP_BOOT_STATUS_RESUME_RESTORING_CORE 40
+#define GXP_BOOT_STATUS_RESUME_RESTORING_CORE_FAILED 41
+#define GXP_BOOT_STATUS_RESUME_RESTORING_MISC 42
+#define GXP_BOOT_STATUS_RESUME_RESTORING_TCM 43
+#define GXP_BOOT_STATUS_RESUME_RESTORING_TOP 44
+#define GXP_BOOT_STATUS_RESUME_FAILED 44
+#define GXP_BOOT_STATUS_SHUTTING_DOWN 60
+
+/* Definitions for host->device warm up cache action requests */
+
+/* No boot action is needed. */
+#define GXP_WARM_UP_CACHE_NONE 0
+
+/* Call SystemImpl::HandleUserDispatch with empty command after resuming. */
+#define GXP_WARM_UP_CACHE_CALL_HANDLE_USER_DISPATCH 1
+
+/* Timing info enums */
+/* Suspend */
+#define GXP_TIMING_MCU_INTENT_SUSPEND 0
+#define GXP_TIMING_DSP_ACK_SUSPEND 1
+#define GXP_TIMING_START_SUSPEND 2
+#define GXP_TIMING_CORE_QUIESCENCED 3
+#define GXP_TIMING_TCM_SAVED 4
+#define GXP_TIMING_TOP_SAVED 5
+#define GXP_TIMING_INTERNAL_SAVED 6
+#define GXP_TIMING_SUSPEND_COMPLETED 7
+
+/* Resume */
+#define GXP_TIMING_MCU_INTENT_RESUME 8
+#define GXP_TIMING_DSP_FIRST_INSTR 9
+#define GXP_TIMING_CACHE_RESET 10
+#define GXP_TIMING_INTERNAL_RESTORE_START 11
+#define GXP_TIMING_INTERNAL_RESTORED 12
+#define GXP_TIMING_TCM_RESTORED 13
+#define GXP_TIMING_TOP_RESTORED 14
+#define GXP_TIMING_RESUME_COMPLETED 15
/* Bit masks for the status fields in the core telemetry structures. */
/* The core telemetry buffers have been setup by the host. */
@@ -104,6 +162,9 @@
/* There was an attempt to use the buffers but their content was invalid. */
#define GXP_CORE_TELEMETRY_DEVICE_STATUS_SANITY_CHECK_FAILED (1 << 1)
+/* Mailbox command buffer descriptor invalid address for null command */
+#define GXP_DEVICE_ADDRESS_INVALID 0
+
/*
* A structure describing the core telemetry (logging and tracing) parameters
* and buffers.
@@ -199,21 +260,74 @@
*/
uint32_t boot_status;
+ /* GXP kernel driver major version. Host is responsible for updating it. */
+ uint16_t gxp_kernel_driver_major_version;
+
+ /* GXP kernel driver minor version. Host is responsible for updating it. */
+ uint16_t gxp_kernel_driver_minor_version;
+
/* Reserved fields for future expansion */
- uint32_t reserved_boot[12];
+ uint32_t reserved_boot[11];
/* To be used to communicate statistics for timing events during boot */
uint32_t timing_entries[16];
/* To be used to communicate crash events in case of failures */
- uint32_t valid_crash_info;
+ uint32_t crash_handler_stage;
uint32_t crash_exccause;
uint32_t crash_excvaddr;
uint32_t crash_epc1;
- uint32_t reserved_crash_info[12];
+ /* Written by host to request debug dump generation on core. */
+ uint32_t generate_debug_dump;
+ /* Written by core to notify the host about debug dump completion. */
+ uint32_t debug_dump_generated;
+ /* To be used by telemetry to check if IRQ needs to be sent to MCU/kernel. */
+ uint32_t telemetry_threshold_reached;
+
+ /* Debug keys to communicate runtime context */
+ /*
+ * Android Process ID in runtime that triggered the workload on this DSP core.
+ * It gets written by the runtime while preparing the workloads.
+ */
+ uint16_t android_pid;
+ /*
+ * Unique id of a GxpDevice within a process. It gets written by the runtime
+ * while preparing the workloads.
+ */
+ uint16_t device_id;
+ /*
+ * Unique id associated with each run command of a device. It gets written by
+ * the DSP firmware.
+ */
+ uint32_t job_id;
+
+ /* Currently loaded active library details on DSP firmware */
+ /*
+ * Base address of the currently loaded active library's code section on DSP
+ * firmware.
+ */
+ uint32_t code_section_base_addr;
+ /*
+ * Base address of the currently loaded active library's data0 section on DSP
+ * firmware.
+ */
+ uint32_t data0_section_base_addr;
+ /*
+ * Base address of the currently loaded active library's data1 section on DSP
+ * firmware.
+ */
+ uint32_t data1_section_base_addr;
+
+ uint32_t reserved_crash_info[4];
+
+ /*
+ * Written to by the host to specify the action after resuming the core. See
+ * the GXP_WARM_UP_* definitions for valid values.
+ */
+ uint8_t warm_up_cache;
/* Reserved for more categories */
- uint32_t reserved[16];
+ uint8_t reserved[63];
/*
* The per-core job descriptor. This struct will be inspected by the FW
@@ -223,6 +337,22 @@
};
/*
+ * A structure describing the external state of the VD. This structure is read
+ * once by the FW upon the first cold boot and is never checked again.
+ */
+struct gxp_vd_descriptor {
+ /* The ID for this GXP application. */
+ uint32_t application_id;
+
+ /*
+ * Whether or not this VD has been initialized by one of its cores.
+ * This variable is protected by sync barrier at offset 0. Should be
+ * initialized by the host to 0.
+ */
+ uint32_t vd_is_initialized;
+};
+
+/*
* A structure describing the telemetry (logging and tracing) parameters and
* buffers; this describes R/O aspects of the telemetry buffers.
*/
@@ -247,22 +377,6 @@
};
/*
- * A structure describing the external state of the VD. This structure is read
- * once by the FW upon the first cold boot and is never checked again.
- */
-struct gxp_vd_descriptor {
- /* The ID for this GXP application. */
- uint32_t application_id;
-
- /*
- * Whether or not this VD has been initialized by one of its cores.
- * This variable is protected by sync barrier at offset 0. Should be
- * initialized by the host to 0.
- */
- uint32_t vd_is_initialized;
-};
-
-/*
* A descriptor for data that is common to the entire system; usually accessed
* by physical core. This region is mapped as R/O for all VDs. Should be
* writable by the host (MCU/Kernel)
diff --git a/gxp-internal.h b/gxp-internal.h
index 35c0993..5985f27 100644
--- a/gxp-internal.h
+++ b/gxp-internal.h
@@ -23,9 +23,9 @@
#include <linux/rwsem.h>
#include <linux/spinlock.h>
-#include <gcip/iif/iif-manager.h>
#include <gcip/gcip-resource-accessor.h>
#include <gcip/gcip-thermal.h>
+#include <iif/iif-manager.h>
#include "gxp-config.h"
#include "gxp.h"
@@ -165,6 +165,7 @@
/* To manage IIF fences. */
struct iif_manager *iif_mgr;
+ struct device *iif_dev;
/* callbacks for chip-dependent implementations */
@@ -303,11 +304,8 @@
struct device_node *np;
np = of_parse_phandle(gxp->dev->of_node, phandle, 0);
- if (IS_ERR_OR_NULL(np)) {
- dev_err(gxp->dev, "Failed to find \"%s\" reserved memory\n",
- phandle);
+ if (IS_ERR_OR_NULL(np))
return -ENODEV;
- }
ret = of_address_to_resource(np, 0, r);
of_node_put(np);
diff --git a/gxp-kci.c b/gxp-kci.c
index 722722e..d86fb32 100644
--- a/gxp-kci.c
+++ b/gxp-kci.c
@@ -270,8 +270,7 @@
return -ENOMEM;
}
-static void gxp_kci_release_resources(struct gxp_mailbox *mailbox,
- struct gxp_virtual_device *vd,
+static void gxp_kci_release_resources(struct gxp_mailbox *mailbox, struct gxp_virtual_device *vd,
uint virt_core)
{
struct gxp_kci *gkci = mailbox->data;
@@ -376,11 +375,7 @@
{
struct gxp_mailbox *mailbox = gkci->mbx;
- gxp_mailbox_write_descriptor(mailbox, mailbox->descriptor_buf.dsp_addr);
- gxp_mailbox_reset(mailbox);
- gxp_mailbox_enable_interrupt(mailbox);
- gxp_mailbox_write_status(mailbox, 1);
-
+ gxp_mailbox_reinit(mailbox);
return 0;
}
@@ -678,3 +673,26 @@
return gxp_kci_send_cmd_with_data(injection->kci_data, GCIP_KCI_CODE_FAULT_INJECTION,
injection->opaque, sizeof(injection->opaque));
}
+
+int gxp_kci_set_freq_limits(struct gxp_kci *gkci, u32 min_freq, u32 max_freq)
+{
+ struct gcip_kci_command_element cmd = {
+ .code = GCIP_KCI_CODE_SET_FREQ_LIMITS,
+ .dma = {
+ .size = min_freq,
+ .flags = max_freq,
+ },
+ };
+ return gxp_kci_send_cmd(gkci->mbx, &cmd);
+}
+
+int gxp_kci_thermal_control(struct gxp_kci *gkci, bool enable)
+{
+ struct gcip_kci_command_element cmd = {
+ .code = GCIP_KCI_CODE_THERMAL_CONTROL,
+ .dma = {
+ .flags = (u32)enable,
+ },
+ };
+ return gxp_kci_send_cmd(gkci->mbx, &cmd);
+}
diff --git a/gxp-kci.h b/gxp-kci.h
index 4bf413f..a355227 100644
--- a/gxp-kci.h
+++ b/gxp-kci.h
@@ -251,4 +251,24 @@
*/
int gxp_kci_fault_injection(struct gcip_fault_inject *injection);
+/**
+ * gxp_kci_set_freq_limits() - Sends the frequency limits to be honoured to the firmware.
+ * @gkci: Reference to the GXP KCI mailbox struct.
+ * @min_freq: minimum frequency limit to be set.
+ * @max_freq: maximum frequency limit to be set.
+ *
+ * Return: 0 if the command is sent successfully.
+ */
+int gxp_kci_set_freq_limits(struct gxp_kci *gkci, u32 min_freq, u32 max_freq);
+
+/**
+ * gxp_kci_thermal_control() - Sends KCI command to enable/disable thermal throttling.
+ * @gkci: Reference to the GXP KCI mailbox struct.
+ * @enable: bool to indicate whether to enable or disable thermal throttling.
+ *
+ * Return: 0 if the command is sent successfully.
+ */
+
+int gxp_kci_thermal_control(struct gxp_kci *gkci, bool enable);
+
#endif /* __GXP_KCI_H__ */
diff --git a/gxp-mailbox-driver.c b/gxp-mailbox-driver.c
index 7257114..a528695 100644
--- a/gxp-mailbox-driver.c
+++ b/gxp-mailbox-driver.c
@@ -12,7 +12,8 @@
#include <linux/of_irq.h>
#include <linux/spinlock.h>
-#include "gxp-config.h" /* GXP_USE_LEGACY_MAILBOX */
+#include <trace/events/gxp.h>
+
#include "gxp-mailbox-driver.h"
#include "gxp-mailbox-regs.h"
#include "gxp-mailbox.h"
@@ -33,6 +34,8 @@
{
struct gxp_mailbox *mailbox = (struct gxp_mailbox *)arg;
+ trace_gxp_uci_rsp_start(irq);
+
gxp_mailbox_chip_irq_handler(mailbox);
return IRQ_HANDLED;
}
@@ -104,10 +107,9 @@
/* Nothing to cleanup */
}
-void gxp_mailbox_driver_enable_interrupts(struct gxp_mailbox *mailbox)
+void gxp_mailbox_driver_register_interrupts(struct gxp_mailbox *mailbox)
{
register_irq(mailbox);
- gxp_mailbox_enable_interrupt(mailbox);
}
void gxp_mailbox_driver_disable_interrupts(struct gxp_mailbox *mailbox)
@@ -339,14 +341,6 @@
return gxp_mailbox_inc_resp_queue_head_nolock(mailbox, inc, wrap_bit);
}
-#if !GXP_USE_LEGACY_MAILBOX
-u32 gxp_mailbox_gcip_ops_get_cmd_queue_head(struct gcip_mailbox *mailbox)
-{
- struct gxp_mailbox *gxp_mbx = mailbox->data;
-
- return gxp_mailbox_read_cmd_queue_head(gxp_mbx);
-}
-
u32 gxp_mailbox_gcip_ops_get_cmd_queue_tail(struct gcip_mailbox *mailbox)
{
struct gxp_mailbox *gxp_mbx = mailbox->data;
@@ -500,4 +494,3 @@
return gxp_pm_is_blk_down(gxp_mbx->gxp);
}
-#endif /* !GXP_USE_LEGACY_MAILBOX */
diff --git a/gxp-mailbox-driver.h b/gxp-mailbox-driver.h
index 4349827..b883f7e 100644
--- a/gxp-mailbox-driver.h
+++ b/gxp-mailbox-driver.h
@@ -2,18 +2,19 @@
/*
* GXP mailbox driver.
*
- * Copyright (C) 2020-2022 Google LLC
+ * Copyright (C) 2020-2024 Google LLC
*/
+
#ifndef __GXP_MAILBOX_DRIVER_H__
#define __GXP_MAILBOX_DRIVER_H__
+#include <linux/io.h>
+
+#include <gcip/gcip-mailbox.h>
+
#include "gxp-config.h"
#include "gxp-mailbox.h"
-#if !GXP_USE_LEGACY_MAILBOX
-#include <gcip/gcip-mailbox.h>
-#endif
-
/* Utilities of circular queue operations */
#define CIRCULAR_QUEUE_INDEX_MASK(wrap_bit) (wrap_bit - 1)
@@ -33,12 +34,22 @@
void gxp_mailbox_driver_init(struct gxp_mailbox *mailbox);
void gxp_mailbox_driver_exit(struct gxp_mailbox *mailbox);
-void gxp_mailbox_driver_enable_interrupts(struct gxp_mailbox *mailbox);
+void gxp_mailbox_driver_register_interrupts(struct gxp_mailbox *mailbox);
void gxp_mailbox_driver_disable_interrupts(struct gxp_mailbox *mailbox);
void __iomem *gxp_mailbox_get_csr_base(struct gxp_dev *gxp, uint index);
void __iomem *gxp_mailbox_get_data_base(struct gxp_dev *gxp, uint index);
+static inline u32 gxp_mailbox_csr_read(struct gxp_mailbox *mailbox, uint reg_offset)
+{
+ return readl(mailbox->csr_reg_base + reg_offset);
+}
+
+static inline void gxp_mailbox_csr_write(struct gxp_mailbox *mailbox, uint reg_offset, u32 value)
+{
+ writel(value, mailbox->csr_reg_base + reg_offset);
+}
+
void gxp_mailbox_reset_hw(struct gxp_mailbox *mailbox);
/**
* gxp_mailbox_generate_device_interrupt(): Trigger interrupt to device.
@@ -59,12 +70,6 @@
* @mailbox: Mailbox for which to enable interrupts.
*/
void gxp_mailbox_enable_interrupt(struct gxp_mailbox *mailbox);
-/**
- * gxp_mailbox_get_interrupt_status() - Retrieve the set interrupt bits coming
- * to AP/Host.
- * @mailbox: Mailbox for which to get interrupt status.
- */
-u32 gxp_mailbox_get_interrupt_status(struct gxp_mailbox *mailbox);
/* gxp_mailbox_wait_for_device_mailbox_init() - Wait for mailbox to get
* enabled/initialised by device.
* @mailbox: Mailbox to get it enabled from device end.
@@ -165,12 +170,10 @@
int gxp_mailbox_inc_resp_queue_head_locked(struct gxp_mailbox *mailbox, u32 inc,
u32 wrap_bit);
-#if !GXP_USE_LEGACY_MAILBOX
/*
* Following functions are used when setting the operators of `struct gcip_mailbox_ops`.
* To use these functions, @mailbox->data should be set as an instance of `struct gxp_mailbox`.
*/
-u32 gxp_mailbox_gcip_ops_get_cmd_queue_head(struct gcip_mailbox *mailbox);
u32 gxp_mailbox_gcip_ops_get_cmd_queue_tail(struct gcip_mailbox *mailbox);
void gxp_mailbox_gcip_ops_inc_cmd_queue_tail(struct gcip_mailbox *mailbox,
u32 inc);
@@ -201,6 +204,5 @@
void gxp_mailbox_gcip_ops_after_fetch_resps(struct gcip_mailbox *mailbox,
u32 num_resps);
bool gxp_mailbox_gcip_ops_is_block_off(struct gcip_mailbox *mailbox);
-#endif /* !GXP_USE_LEGACY_MAILBOX */
#endif /* __GXP_MAILBOX_DRIVER_H__ */
diff --git a/gxp-mailbox.c b/gxp-mailbox.c
index 5afd820..daf8fdb 100644
--- a/gxp-mailbox.c
+++ b/gxp-mailbox.c
@@ -15,7 +15,9 @@
#include <linux/spinlock.h>
#include <uapi/linux/sched/types.h>
-#include "gxp-config.h" /* GXP_USE_LEGACY_MAILBOX */
+#include <gcip/gcip-mailbox.h>
+
+#include "gxp-config.h"
#include "gxp-dma.h"
#include "gxp-internal.h"
#include "gxp-mailbox.h"
@@ -23,15 +25,12 @@
#include "gxp-pm.h"
#include "gxp.h"
-#if GXP_USE_LEGACY_MAILBOX
-#include "gxp-mailbox-impl.h"
-#else
-#include <gcip/gcip-mailbox.h>
+#if GXP_HAS_MCU
#include <gcip/gcip-kci.h>
#include "gxp-kci.h"
#include "gxp-mcu-telemetry.h"
-#endif
+#endif /* GXP_HAS_MCU */
/* Timeout of 1s by default */
int gxp_mbx_timeout = 2000;
@@ -50,14 +49,12 @@
struct gxp_mailbox *mailbox =
container_of(work, struct gxp_mailbox, response_work);
-#if GXP_USE_LEGACY_MAILBOX
- gxp_mailbox_consume_responses(mailbox);
-#else
if (gxp_is_direct_mode(mailbox->gxp))
gcip_mailbox_consume_responses_work(mailbox->mbx_impl.gcip_mbx);
+#if GXP_HAS_MCU
else if (mailbox->type == GXP_MBOX_TYPE_KCI)
gxp_mcu_telemetry_irq_handler(((struct gxp_kci *)mailbox->data)->mcu);
-#endif
+#endif /* GXP_HAS_MCU */
}
/*
@@ -198,7 +195,6 @@
kfree(mailbox);
}
-#if !GXP_USE_LEGACY_MAILBOX
static int init_gcip_mailbox(struct gxp_mailbox *mailbox)
{
const struct gcip_mailbox_args args = {
@@ -243,6 +239,8 @@
mailbox->mbx_impl.gcip_mbx = NULL;
}
+#if GXP_HAS_MCU
+
static int init_gcip_kci(struct gxp_mailbox *mailbox)
{
const struct gcip_kci_args args = {
@@ -285,44 +283,35 @@
kfree(gcip_kci);
mailbox->mbx_impl.gcip_kci = NULL;
}
-#endif /* !GXP_USE_LEGACY_MAILBOX */
+
+#endif /* GXP_HAS_MCU */
/*
* Initializes @mailbox->mbx_impl to start waiting and consuming responses.
* This will initializes GCIP mailbox modules according to the type of @mailbox.
* - GENERAL: will initialize @mailbox->mbx_impl.gcip_mbx
* - KCI: will initialize @mailbox->mbx_impl.kci_mbx
- *
- * Note: On `GXP_USE_LEGACY_MAILBOX`, it will initialize @mailbox itself as its
- * queuing logic is implemented in `gxp-mailbox-impl.c`.
*/
static int init_mailbox_impl(struct gxp_mailbox *mailbox)
{
int ret;
-#if GXP_USE_LEGACY_MAILBOX
- if (mailbox->type != GXP_MBOX_TYPE_GENERAL)
- return -EOPNOTSUPP;
-
- ret = gxp_mailbox_init_consume_responses(mailbox);
- if (ret)
- return ret;
-#else
switch (mailbox->type) {
case GXP_MBOX_TYPE_GENERAL:
ret = init_gcip_mailbox(mailbox);
if (ret)
return ret;
break;
+#if GXP_HAS_MCU
case GXP_MBOX_TYPE_KCI:
ret = init_gcip_kci(mailbox);
if (ret)
return ret;
break;
+#endif /* GXP_HAS_MCU */
default:
return -EOPNOTSUPP;
}
-#endif /* GXP_USE_LEGACY_MAILBOX */
return 0;
}
@@ -331,9 +320,6 @@
{
int ret;
- gxp_mailbox_write_descriptor(mailbox, mailbox->descriptor_buf.dsp_addr);
- gxp_mailbox_reset(mailbox);
-
ret = init_mailbox_impl(mailbox);
if (ret)
return ret;
@@ -343,14 +329,18 @@
kthread_init_work(&mailbox->response_work,
gxp_mailbox_consume_responses_work);
- /* Only enable interrupts once everything has been setup */
- gxp_mailbox_driver_enable_interrupts(mailbox);
- /* Enable the mailbox */
- gxp_mailbox_write_status(mailbox, 1);
-
+ gxp_mailbox_driver_register_interrupts(mailbox);
return 0;
}
+void gxp_mailbox_reinit(struct gxp_mailbox *mailbox)
+{
+ gxp_mailbox_write_descriptor(mailbox, mailbox->descriptor_buf.dsp_addr);
+ gxp_mailbox_reset(mailbox);
+ gxp_mailbox_enable_interrupt(mailbox);
+ gxp_mailbox_write_status(mailbox, 1);
+}
+
struct gxp_mailbox *gxp_mailbox_alloc(struct gxp_mailbox_manager *mgr,
struct gxp_virtual_device *vd,
uint virt_core, u8 core_id,
@@ -378,24 +368,21 @@
* This releases GCIP mailbox modules according to the type of @mailbox.
* - GENERAL: will release @mailbox->mbx_impl.gcip_mbx
* - KCI: will release @mailbox->mbx_impl.kci_mbx
- *
- * Note: On `GXP_USE_LEGACY_MAILBOX`, it will release @mailbox itself as its
- * queuing logic is implemented in `gxp-mailbox-impl.c`.
*/
static void release_mailbox_impl(struct gxp_mailbox *mailbox)
{
-#if GXP_USE_LEGACY_MAILBOX
- gxp_mailbox_release_consume_responses(mailbox);
-#else
switch (mailbox->type) {
case GXP_MBOX_TYPE_GENERAL:
release_gcip_mailbox(mailbox);
break;
+#if GXP_HAS_MCU
case GXP_MBOX_TYPE_KCI:
release_gcip_kci(mailbox);
break;
+#endif /* GXP_HAS_MCU */
+ default:
+ break;
}
-#endif
}
void gxp_mailbox_release(struct gxp_mailbox_manager *mgr,
@@ -480,16 +467,18 @@
return 0;
}
-#if !GXP_USE_LEGACY_MAILBOX
int gxp_mailbox_send_cmd(struct gxp_mailbox *mailbox, void *cmd, void *resp)
{
switch (mailbox->type) {
case GXP_MBOX_TYPE_GENERAL:
return gcip_mailbox_send_cmd(mailbox->mbx_impl.gcip_mbx, cmd, resp, 0);
+#if GXP_HAS_MCU
case GXP_MBOX_TYPE_KCI:
return gcip_kci_send_cmd(mailbox->mbx_impl.gcip_kci, cmd);
+#endif /* GXP_HAS_MCU */
+ default:
+ return -EOPNOTSUPP;
}
- return -EOPNOTSUPP;
}
struct gcip_mailbox_resp_awaiter *gxp_mailbox_put_cmd(struct gxp_mailbox *mailbox, void *cmd,
@@ -505,4 +494,3 @@
}
return ERR_PTR(-EOPNOTSUPP);
}
-#endif /* !GXP_USE_LEGACY_MAILBOX */
diff --git a/gxp-mailbox.h b/gxp-mailbox.h
index 7e3dd58..5fe0069 100644
--- a/gxp-mailbox.h
+++ b/gxp-mailbox.h
@@ -10,17 +10,14 @@
#include <linux/kthread.h>
#include <linux/spinlock.h>
+#include <gcip/gcip-mailbox.h>
+
+#include "gxp-config.h"
#include "gxp-client.h"
-#include "gxp-config.h" /* GXP_USE_LEGACY_MAILBOX */
#include "gxp-dma.h"
#include "gxp-internal.h"
#include "gxp-mailbox-manager.h"
-#if !GXP_USE_LEGACY_MAILBOX
-#include <gcip/gcip-kci.h>
-#include <gcip/gcip-mailbox.h>
-#endif
-
/* Pre-agreed values can be passed to gxp_mailbox_set_control(). */
#define GXP_MBOX_CONTROL_MAGIC_POWER_DOWN (0xcafebabeu)
@@ -66,8 +63,6 @@
enum gxp_mailbox_type {
/*
* Mailbox will utilize `gcip-mailbox.h` internally.
- * (Note: On `GXP_USE_LEGACY_MAILBOX`, it utilizes `gxp-mailbox-impl.h`
- * instead.)
* Mostly will be used for handling user commands.
*/
GXP_MBOX_TYPE_GENERAL = 0,
@@ -92,6 +87,8 @@
u32 resp_queue_size;
};
+struct gcip_kci;
+struct gcip_kci_ops;
struct gxp_mailbox;
/*
@@ -135,7 +132,6 @@
void (*release_resources)(struct gxp_mailbox *mailbox,
struct gxp_virtual_device *vd,
uint virt_core);
-#if !GXP_USE_LEGACY_MAILBOX
/*
* Operators which has dependency on the GCIP according to the type of mailbox.
* - GXP_MBOX_TYPE_GENERAL: @gcip_ops.mbx must be defined.
@@ -145,7 +141,6 @@
const struct gcip_mailbox_ops *mbx;
const struct gcip_kci_ops *kci;
} gcip_ops;
-#endif
};
struct gxp_mailbox_args {
@@ -203,13 +198,6 @@
struct gxp_mailbox_ops *ops;
void *data; /* private data */
-#if GXP_USE_LEGACY_MAILBOX
- u64 cur_seq;
- /* add to this list if a command needs to wait for a response */
- struct list_head wait_list;
- /* queue for waiting for the wait_list to be consumed */
- wait_queue_head_t wait_list_waitq;
-#else /* !GXP_USE_LEGACY_MAILBOX */
/*
* Implementation of the mailbox according to the type.
* - GXP_MBOX_TYPE_GENERAL: @gcip_mbx will be allocated.
@@ -219,7 +207,6 @@
struct gcip_mailbox *gcip_mbx;
struct gcip_kci *gcip_kci;
} mbx_impl;
-#endif /* GXP_USE_LEGACY_MAILBOX */
};
/* Mailbox APIs */
@@ -255,7 +242,12 @@
int gxp_mailbox_unregister_interrupt_handler(struct gxp_mailbox *mailbox,
u32 int_bit);
-#if !GXP_USE_LEGACY_MAILBOX
+/*
+ * Initialises the mailbox CSRs.
+ * Must be called only after mailbox is allocated via gxp_mailbox_alloc().
+ */
+void gxp_mailbox_reinit(struct gxp_mailbox *mailbox);
+
/*
* Executes command synchronously. If @resp is not NULL, the response will be returned to it.
* See the `gcip_mailbox_send_cmd` of `gcip-mailbox.h` or `gcip_kci_send_cmd` of `gcip-kci.h`
@@ -272,6 +264,5 @@
struct gcip_mailbox_resp_awaiter *gxp_mailbox_put_cmd(struct gxp_mailbox *mailbox, void *cmd,
void *resp, void *data,
gcip_mailbox_cmd_flags_t flags);
-#endif /* !GXP_USE_LEGACY_MAILBOX */
#endif /* __GXP_MAILBOX_H__ */
diff --git a/gxp-mcu-firmware.c b/gxp-mcu-firmware.c
index 6d19080..78a566f 100644
--- a/gxp-mcu-firmware.c
+++ b/gxp-mcu-firmware.c
@@ -16,6 +16,7 @@
#include <linux/string.h>
#include <linux/workqueue.h>
+#include <gcip/gcip-alloc-helper.h>
#include <gcip/gcip-common-image-header.h>
#include <gcip/gcip-fault-injection.h>
#include <gcip/gcip-image-config.h>
@@ -62,17 +63,33 @@
/*
* Programs instruction remap CSRs.
*/
-static void program_iremap_csr(struct gxp_dev *gxp,
- struct gxp_mapped_resource *buf)
+static int program_iremap_csr(struct gxp_dev *gxp, struct gxp_mapped_resource *buf)
{
+ struct gxp_mcu_firmware *mcu_fw = gxp_mcu_firmware_of(gxp);
+ size_t size;
+
dev_info(gxp->dev, "Program instruction remap CSRs");
gxp_soc_set_iremap_context(gxp);
- gxp_write_32(gxp, GXP_REG_CFGVECTABLE0, buf->daddr);
- gxp_write_32(gxp, GXP_REG_IREMAP_LOW, buf->daddr);
- gxp_write_32(gxp, GXP_REG_IREMAP_HIGH, buf->daddr + buf->size);
+ if (mcu_fw->dynamic_fw_buffer) {
+ if (buf->daddr + buf->size > GXP_IREMAP_DYNAMIC_CODE_BASE) {
+ dev_err(gxp->dev,
+ "Bad dynamic firmware base %x, carveout daddr: %pad size: %llx",
+ GXP_IREMAP_DYNAMIC_CODE_BASE, &buf->daddr, buf->size);
+ return -EINVAL;
+ }
+ gxp_write_32(gxp, GXP_REG_CFGVECTABLE0, GXP_IREMAP_DYNAMIC_CODE_BASE);
+ size = mcu_fw->dynamic_fw_buffer->size;
+ gxp_write_32(gxp, GXP_REG_IREMAP_LOW, buf->daddr);
+ gxp_write_32(gxp, GXP_REG_IREMAP_HIGH, GXP_IREMAP_DYNAMIC_CODE_BASE + size);
+ } else {
+ gxp_write_32(gxp, GXP_REG_CFGVECTABLE0, buf->daddr);
+ gxp_write_32(gxp, GXP_REG_IREMAP_LOW, buf->daddr);
+ gxp_write_32(gxp, GXP_REG_IREMAP_HIGH, buf->daddr + buf->size);
+ }
gxp_write_32(gxp, GXP_REG_IREMAP_TARGET, buf->daddr);
gxp_write_32(gxp, GXP_REG_IREMAP_ENABLE, 1);
+ return 0;
}
/*
@@ -260,6 +277,68 @@
return gxp_pg_by_recovery_boot(gxp, force);
}
+static struct gxp_mcu_firmware_ns_buffer *map_ns_buffer(struct gxp_dev *gxp, dma_addr_t daddr,
+ size_t size)
+{
+ struct gxp_mcu_firmware_ns_buffer *ns_buffer;
+ u64 gcip_map_flags = GCIP_MAP_FLAGS_DMA_RW;
+
+ ns_buffer = kzalloc(sizeof(*ns_buffer), GFP_KERNEL);
+ if (!ns_buffer)
+ return ERR_PTR(-ENOMEM);
+
+ ns_buffer->daddr = daddr;
+ ns_buffer->size = size;
+ ns_buffer->sgt = gcip_alloc_noncontiguous(gxp->dev, size, GFP_KERNEL);
+ if (!ns_buffer->sgt) {
+ kfree(ns_buffer);
+ return ERR_PTR(-ENOMEM);
+ }
+ if (!gcip_iommu_domain_map_sgt_to_iova(gxp_iommu_get_domain_for_dev(gxp), ns_buffer->sgt,
+ daddr, &gcip_map_flags)) {
+ dev_err(gxp->dev, "Failed to map NS buffer, daddr: %pad size: %#zx", &daddr, size);
+ gcip_free_noncontiguous(ns_buffer->sgt);
+ kfree(ns_buffer);
+ return ERR_PTR(-EBUSY);
+ }
+ return ns_buffer;
+}
+
+static void unmap_ns_buffer(struct gxp_dev *gxp, struct gxp_mcu_firmware_ns_buffer *ns_buffer)
+{
+ gcip_iommu_domain_unmap_sgt_from_iova(gxp_iommu_get_domain_for_dev(gxp), ns_buffer->sgt,
+ GCIP_MAP_FLAGS_DMA_RW);
+ gcip_free_noncontiguous(ns_buffer->sgt);
+ kfree(ns_buffer);
+}
+
+static void reset_shadow_memory(struct gxp_dev *gxp)
+{
+ struct gxp_mcu_firmware *mcu_fw = gxp_mcu_firmware_of(gxp);
+ struct gxp_mcu_firmware_ns_buffer *shadow_buffer = NULL, *cur;
+ u32 shadow_index =
+ GCIP_IMAGE_CONFIG_SANITIZER_INDEX(mcu_fw->cfg_parser.last_config.sanitizer_config);
+ u32 count = 0;
+
+ mutex_lock(&mcu_fw->ns_buffer_list_lock);
+ list_for_each_entry(cur, &mcu_fw->ns_buffer_list, list) {
+ if (count == shadow_index) {
+ shadow_buffer = cur;
+ break;
+ }
+ count++;
+ }
+ mutex_unlock(&mcu_fw->ns_buffer_list_lock);
+
+ if (shadow_buffer) {
+ memset(gcip_noncontiguous_sgt_to_mem(shadow_buffer->sgt), 0, shadow_buffer->size);
+ gxp_dma_sync_sg_for_device(gxp, shadow_buffer->sgt->sgl,
+ shadow_buffer->sgt->orig_nents, DMA_TO_DEVICE);
+ } else {
+ dev_warn(gxp->dev, "shadow buffer not found");
+ }
+}
+
int gxp_mcu_firmware_load(struct gxp_dev *gxp, char *fw_name,
const struct firmware **fw)
{
@@ -298,13 +377,6 @@
size = (*fw)->size - GCIP_FW_HEADER_SIZE;
- if (size > mcu_fw->image_buf.size) {
- dev_err(dev, "firmware %s size %#zx exceeds buffer size %#llx",
- fw_name, size, mcu_fw->image_buf.size);
- ret = -ENOSPC;
- goto err_release_firmware;
- }
-
imgcfg = get_image_config_from_hdr(hdr);
if (!imgcfg) {
dev_err(dev, "Unsupported image header generation");
@@ -332,8 +404,32 @@
}
mcu_fw->is_secure = !gcip_image_config_is_ns(imgcfg);
- memcpy(mcu_fw->image_buf.vaddr, (*fw)->data + GCIP_FW_HEADER_SIZE,
- size);
+ mcu_fw->sanitizer_status = GCIP_IMAGE_CONFIG_SANITIZER_STATUS(imgcfg->sanitizer_config);
+
+ if (size > mcu_fw->image_buf.size || (mcu_fw->sanitizer_status != 0)) {
+ if (mcu_fw->is_secure) {
+ dev_err(dev, "firmware %s size %#zx exceeds buffer size %#llx", fw_name,
+ size, mcu_fw->image_buf.size);
+ ret = -ENOSPC;
+ goto err_clear_config;
+ }
+
+ /*
+ * In non-secure mode, we support allocating buffers to put MCU firmware image
+ * instead of using carveouts.
+ */
+ mcu_fw->dynamic_fw_buffer = map_ns_buffer(gxp, GXP_IREMAP_DYNAMIC_CODE_BASE, size);
+ if (IS_ERR(mcu_fw->dynamic_fw_buffer))
+ goto err_clear_config;
+ memcpy(gcip_noncontiguous_sgt_to_mem(mcu_fw->dynamic_fw_buffer->sgt),
+ (*fw)->data + GCIP_FW_HEADER_SIZE, size);
+ gxp_dma_sync_sg_for_device(gxp, mcu_fw->dynamic_fw_buffer->sgt->sgl,
+ mcu_fw->dynamic_fw_buffer->sgt->orig_nents,
+ DMA_TO_DEVICE);
+ } else {
+ memcpy(mcu_fw->image_buf.vaddr, (*fw)->data + GCIP_FW_HEADER_SIZE, size);
+ }
+
out:
mutex_unlock(&mcu_fw->lock);
return 0;
@@ -358,6 +454,10 @@
mutex_unlock(&mcu_fw->lock);
return;
}
+ if (mcu_fw->dynamic_fw_buffer) {
+ unmap_ns_buffer(gxp, mcu_fw->dynamic_fw_buffer);
+ mcu_fw->dynamic_fw_buffer = NULL;
+ }
gcip_image_config_clear(&mcu_fw->cfg_parser);
mcu_fw->status = GCIP_FW_INVALID;
mutex_unlock(&mcu_fw->lock);
@@ -388,7 +488,11 @@
return -EIO;
}
} else {
- program_iremap_csr(gxp, &mcu_fw->image_buf);
+ ret = program_iremap_csr(gxp, &mcu_fw->image_buf);
+ if (ret) {
+ gxp_lpm_down(gxp, GXP_MCU_CORE_ID);
+ return ret;
+ }
/* Raise wakeup doorbell */
dev_dbg(gxp->dev, "Raising doorbell %d interrupt\n",
CORE_WAKEUP_DOORBELL(GXP_MCU_CORE_ID));
@@ -548,6 +652,9 @@
lockdep_assert_held(&mcu_fw->lock);
+ if (mcu_fw->sanitizer_status & GCIP_FW_ASAN_ENABLED)
+ reset_shadow_memory(gxp);
+
/*
* Resets UCI/KCI CSRs to ensure that no unconsumed commands are carried over from the last
* execution.
@@ -596,8 +703,10 @@
int ret;
ret = gxp_acquire_rmem_resource(gxp, &r, "gxp-mcu-fw-region");
- if (ret)
+ if (ret) {
+ dev_err(gxp->dev, "Failed to find reserved memory for MCU FW: %d", ret);
return ret;
+ }
buf->size = resource_size(&r);
buf->paddr = r.start;
buf->daddr = GXP_IREMAP_CODE_BASE;
@@ -734,24 +843,71 @@
.attrs = dev_attrs,
};
-static int image_config_map(void *data, dma_addr_t daddr, phys_addr_t paddr,
- size_t size, unsigned int flags)
+static int image_config_map_ns(struct gxp_dev *gxp, dma_addr_t daddr, size_t size)
{
- struct gxp_dev *gxp = data;
- const bool ns = !(flags & GCIP_IMAGE_CONFIG_FLAGS_SECURE);
+ struct gxp_mcu_firmware *mcu_fw = gxp_mcu_firmware_of(gxp);
+ struct gxp_mcu_firmware_ns_buffer *ns_buffer;
- if (ns) {
- dev_err(gxp->dev, "image config NS mappings are not supported");
- return -EINVAL;
- }
+ ns_buffer = map_ns_buffer(gxp, daddr, size);
+ if (IS_ERR(ns_buffer))
+ return PTR_ERR(ns_buffer);
- return gcip_iommu_map(gxp_iommu_get_domain_for_dev(gxp), daddr, paddr, size,
- GCIP_MAP_FLAGS_DMA_RW);
+ mutex_lock(&mcu_fw->ns_buffer_list_lock);
+ list_add_tail(&ns_buffer->list, &mcu_fw->ns_buffer_list);
+ mutex_unlock(&mcu_fw->ns_buffer_list_lock);
+ return 0;
}
-static void image_config_unmap(void *data, dma_addr_t daddr, size_t size, unsigned int flags)
+static void image_config_unmap_ns(struct gxp_dev *gxp, dma_addr_t daddr, size_t size)
+{
+ struct gxp_mcu_firmware *mcu_fw = gxp_mcu_firmware_of(gxp);
+ struct gxp_mcu_firmware_ns_buffer *ns_buffer = NULL, *cur;
+
+ mutex_lock(&mcu_fw->ns_buffer_list_lock);
+ list_for_each_entry(cur, &mcu_fw->ns_buffer_list, list) {
+ if (cur->daddr == daddr && cur->size == size) {
+ ns_buffer = cur;
+ list_del(&cur->list);
+ break;
+ }
+ }
+ mutex_unlock(&mcu_fw->ns_buffer_list_lock);
+
+ if (ns_buffer) {
+ unmap_ns_buffer(gxp, cur);
+ } else {
+ dev_warn(gxp->dev, "Failed to find NS buffer, daddr: %pad size: %#zx", &daddr,
+ size);
+ }
+}
+
+static int image_config_map(void *data, dma_addr_t daddr, phys_addr_t paddr, size_t size,
+ unsigned int cfg_map_flags, unsigned int cfg_op_flags)
{
struct gxp_dev *gxp = data;
+ const bool ns = !(cfg_op_flags & GCIP_IMAGE_CONFIG_FLAGS_SECURE);
+ u64 gcip_map_flags = GCIP_MAP_FLAGS_DMA_RW;
+
+ if (ns)
+ return image_config_map_ns(gxp, daddr, size);
+
+ if (GCIP_IMAGE_CONFIG_MAP_MMIO(cfg_map_flags))
+ gcip_map_flags |= GCIP_MAP_FLAGS_MMIO_TO_FLAGS(1);
+
+ return gcip_iommu_map(gxp_iommu_get_domain_for_dev(gxp), daddr, paddr, size,
+ gcip_map_flags);
+}
+
+static void image_config_unmap(void *data, dma_addr_t daddr, size_t size,
+ unsigned int cfg_map_flags, unsigned int cfg_op_flags)
+{
+ struct gxp_dev *gxp = data;
+ const bool ns = !(cfg_op_flags & GCIP_IMAGE_CONFIG_FLAGS_SECURE);
+
+ if (ns) {
+ image_config_unmap_ns(gxp, daddr, size);
+ return;
+ }
gcip_iommu_unmap(gxp_iommu_get_domain_for_dev(gxp), daddr, size);
}
@@ -814,6 +970,8 @@
mcu_fw->status = GCIP_FW_INVALID;
mcu_fw->crash_cnt = 0;
mutex_init(&mcu_fw->lock);
+ INIT_LIST_HEAD(&mcu_fw->ns_buffer_list);
+ mutex_init(&mcu_fw->ns_buffer_list_lock);
INIT_WORK(&mcu_fw->fw_crash_handler_work, gxp_mcu_firmware_crash_handler_work);
ret = device_add_group(gxp->dev, &firmware_attr_group);
diff --git a/gxp-mcu-firmware.h b/gxp-mcu-firmware.h
index 739c38b..e8df752 100644
--- a/gxp-mcu-firmware.h
+++ b/gxp-mcu-firmware.h
@@ -19,6 +19,17 @@
#define GXP_MCU_BOOT_MODE_NORMAL 0
#define GXP_MCU_BOOT_MODE_RECOVERY 1
+struct gxp_mcu_firmware_ns_buffer {
+ /* SG table for NS firmware buffer mappings. */
+ struct sg_table *sgt;
+ /* DMA address of the NS firmware buffer. */
+ dma_addr_t daddr;
+ /* Size of the NS firmware buffer. */
+ size_t size;
+ /* List of NS firmware buffer mappings for the device. */
+ struct list_head list;
+};
+
struct gxp_mcu_firmware {
struct gxp_dev *gxp;
/* resource for MCU firmware image */
@@ -35,6 +46,14 @@
struct work_struct fw_crash_handler_work;
/* The container of fault injection data. */
struct gcip_fault_inject *fault_inject;
+ /* List of all NS buffer mappings for the device. */
+ struct list_head ns_buffer_list;
+ /* Lock to protect @ns_buffer_list. */
+ struct mutex ns_buffer_list_lock;
+ /* The buffer of dynamic fw memory, which is only used in non-secure mode. */
+ struct gxp_mcu_firmware_ns_buffer *dynamic_fw_buffer;
+ /* The sanitizer enablement status for ASAN and UBSAN */
+ int sanitizer_status;
};
/*
diff --git a/gxp-mcu-fs.c b/gxp-mcu-fs.c
index 2c97e30..c2effcc 100644
--- a/gxp-mcu-fs.c
+++ b/gxp-mcu-fs.c
@@ -12,10 +12,13 @@
#include <linux/rwsem.h>
#include <linux/slab.h>
+#include <gcip/gcip-dma-fence.h>
#include <gcip/gcip-fence-array.h>
#include <gcip/gcip-fence.h>
#include <gcip/gcip-telemetry.h>
+#include <trace/events/gxp.h>
+
#include "gxp-client.h"
#include "gxp-internal.h"
#include "gxp-mcu-fs.h"
@@ -52,59 +55,6 @@
return 0;
}
-/**
- * polled_dma_fence_get() - Generate a dma-fence to represent all the fan-in fences.
- * @in_fences: The pointer to the gcip_fence_array object containing fan-in fances.
- *
- * Use dma_fence_array to handle all the fan-in fences if @in_fences->size > 1.
- * The caller should hold the reference count of the fences in @in_fences to make sure they will not
- * be released during the process.
- * The output fence will acquired 1 reference count in this function either with dma_fence_get() or
- * dma_fence_array_create().
- *
- * Return: The pointer to the generated dma-fence or the error pointer on error.
- * A NULL pointer is returned if no in-kernel fence is passed in.
- */
-static struct dma_fence *polled_dma_fence_get(struct gcip_fence_array *in_fences)
-{
- static int array_seq;
- const int size = in_fences->size;
- struct dma_fence_array *fence_array;
- struct dma_fence **in_dma_fences;
- int i;
-
- if (!in_fences->size || !in_fences->same_type || in_fences->type != GCIP_IN_KERNEL_FENCE)
- return NULL;
-
- /* TODO(b/320401031): Remove this constraint after dma-fence-unwrap adopted. */
- /* dma-fence-array as in-fences is currently not supported. */
- for (i = 0; i < size; i++) {
- if (dma_fence_is_array(in_fences->fences[i]->fence.ikf))
- return ERR_PTR(-EINVAL);
- }
-
- if (size == 1)
- return dma_fence_get(in_fences->fences[0]->fence.ikf);
-
- in_dma_fences = kcalloc(size, sizeof(*in_dma_fences), GFP_KERNEL);
- if (!in_dma_fences)
- return ERR_PTR(-ENOMEM);
-
- for (i = 0; i < size; i++)
- in_dma_fences[i] = dma_fence_get(in_fences->fences[i]->fence.ikf);
-
- /* fence_array will take care of the life cycle of in_dma_fences.*/
- fence_array = dma_fence_array_create(size, in_dma_fences, dma_fence_context_alloc(1),
- array_seq++, false);
- if (!fence_array) {
- kfree(in_dma_fences);
- /* dma_fence_array_create only returns NULL on allocation failure. */
- return ERR_PTR(-ENOMEM);
- }
-
- return &fence_array->base;
-}
-
/*
* Returns the number of fences. If the runtime passed an invalid fence, returns an errno
* accordingly.
@@ -145,6 +95,8 @@
cmd_seq = gcip_mailbox_inc_seq_num(mcu->uci.mbx->mbx_impl.gcip_mbx, 1);
+ trace_gxp_uci_cmd_start(cmd_seq);
+
num_in_fences = get_num_fences(ibuf.in_fences);
if (num_in_fences < 0)
return num_in_fences;
@@ -163,7 +115,8 @@
return PTR_ERR(out_fences);
}
- polled_dma_fence = polled_dma_fence_get(in_fences);
+ /* @polled_dma_fence can be NULL if @in_fences are not DMA fences or there are no fences. */
+ polled_dma_fence = gcip_fence_array_merge_ikf(in_fences);
if (IS_ERR(polled_dma_fence)) {
ret = PTR_ERR(polled_dma_fence);
goto out;
@@ -183,7 +136,7 @@
err_put_fence:
/*
- * Put the reference count of the fence acqurired in polled_dma_fence_get.
+ * Put the reference count of the fence acqurired in gcip_fence_array_merge_ikf.
* If the fence is a dma_fence_array and the callback is failed to be added,
* the whole object and the array it holds will be freed.
* If it is a NULL pointer, it's still safe to call this function.
@@ -192,6 +145,9 @@
out:
gcip_fence_array_put(out_fences);
gcip_fence_array_put(in_fences);
+
+ trace_gxp_uci_cmd_end(cmd_seq);
+
return ret;
}
diff --git a/gxp-mcu-platform.c b/gxp-mcu-platform.c
index 8e55dcd..edc66e4 100644
--- a/gxp-mcu-platform.c
+++ b/gxp-mcu-platform.c
@@ -5,11 +5,15 @@
* Copyright (C) 2022 Google LLC
*/
+#include <linux/device.h>
#include <linux/interrupt.h>
#include <linux/moduleparam.h>
#include <linux/of_irq.h>
+#include <linux/of_platform.h>
+#include <linux/of.h>
+#include <linux/platform_device.h>
-#include <gcip/iif/iif-manager.h>
+#include <iif/iif-manager.h>
#include "gxp-config.h"
#include "gxp-internal.h"
@@ -271,6 +275,81 @@
return GXP_CHIP_ANY;
}
+static void gxp_get_embedded_iif_mgr(struct gxp_dev *gxp)
+{
+ struct iif_manager *mgr;
+
+#if HAS_TPU_EXT
+ if (gxp->tpu_dev.dev) {
+ int ret = edgetpu_ext_driver_cmd(gxp->tpu_dev.dev, EDGETPU_EXTERNAL_CLIENT_TYPE_DSP,
+ GET_IIF_MANAGER, NULL, &mgr);
+
+ if (!ret) {
+ dev_info(gxp->dev, "Use the IIF manager of TPU driver");
+ /* Note that we shouldn't call `iif_manager_get` here. */
+ gxp->iif_mgr = mgr;
+ return;
+ }
+ }
+#endif /* HAS_TPU_EXT */
+
+ dev_info(gxp->dev, "Try to get an embedded IIF manager");
+
+ mgr = iif_manager_init(gxp->dev->of_node);
+ if (IS_ERR(mgr)) {
+ dev_warn(gxp->dev, "Failed to init an embedded IIF manager: %ld", PTR_ERR(mgr));
+ return;
+ }
+
+ gxp->iif_mgr = mgr;
+}
+
+static void gxp_get_iif_mgr(struct gxp_dev *gxp)
+{
+ struct platform_device *pdev;
+ struct device_node *node;
+ struct iif_manager *mgr;
+
+ node = of_parse_phandle(gxp->dev->of_node, "iif-device", 0);
+ if (IS_ERR_OR_NULL(node)) {
+ dev_warn(gxp->dev, "There is no iif-device node in the device tree");
+ goto get_embed;
+ }
+
+ pdev = of_find_device_by_node(node);
+ of_node_put(node);
+ if (!pdev) {
+ dev_warn(gxp->dev, "Failed to find the IIF device");
+ goto get_embed;
+ }
+
+ mgr = platform_get_drvdata(pdev);
+ if (!mgr) {
+ dev_warn(gxp->dev, "Failed to get a manager from IIF device");
+ goto put_device;
+ }
+
+ dev_info(gxp->dev, "Use the IIF manager of IIF device");
+
+ /* We don't need to call `get_device` since `of_find_device_by_node` takes a refcount. */
+ gxp->iif_dev = &pdev->dev;
+ gxp->iif_mgr = iif_manager_get(mgr);
+ return;
+
+put_device:
+ put_device(&pdev->dev);
+get_embed:
+ gxp_get_embedded_iif_mgr(gxp);
+}
+
+static void gxp_put_iif_mgr(struct gxp_dev *gxp)
+{
+ if (gxp->iif_mgr)
+ iif_manager_put(gxp->iif_mgr);
+ /* NO-OP if `gxp->iif_dev` is NULL. */
+ put_device(gxp->iif_dev);
+}
+
int gxp_mcu_platform_after_probe(struct gxp_dev *gxp)
{
int ret;
@@ -282,12 +361,7 @@
if (ret)
return ret;
- gxp->iif_mgr = iif_manager_init(gxp->dev->of_node);
- if (IS_ERR(gxp->iif_mgr)) {
- dev_err(gxp->dev, "Failed to init IIF manager: %ld", PTR_ERR(gxp->iif_mgr));
- gxp->iif_mgr = NULL;
- }
-
+ gxp_get_iif_mgr(gxp);
gxp_usage_stats_init(gxp);
return gxp_mcu_init(gxp, gxp_mcu_of(gxp));
}
@@ -299,9 +373,7 @@
gxp_mcu_exit(gxp_mcu_of(gxp));
gxp_usage_stats_exit(gxp);
-
- if (gxp->iif_mgr)
- iif_manager_put(gxp->iif_mgr);
+ gxp_put_iif_mgr(gxp);
}
void gxp_mcu_dev_init(struct gxp_mcu_dev *mcu_dev)
diff --git a/gxp-pm.c b/gxp-pm.c
index d294e02..c59ef17 100644
--- a/gxp-pm.c
+++ b/gxp-pm.c
@@ -27,11 +27,20 @@
#include "gxp-pm.h"
#include "mobile-soc.h"
+#if GXP_HAS_MCU
+#include <gcip/gcip-kci.h>
+
+#include "gxp-kci.h"
+#include "gxp-mcu.h"
+#endif /* GXP_HAS_MCU */
+
/* Don't attempt to touch the device when @busy_count equals this value. */
#define BUSY_COUNT_OFF (~0ull)
#define DEBUGFS_BLK_POWERSTATE "blk_powerstate"
#define DEBUGFS_WAKELOCK "wakelock"
+#define DEBUGFS_MIN_FREQ "min_freq"
+#define DEBUGFS_MAX_FREQ "max_freq"
#define SHUTDOWN_DELAY_US_MIN 200
#define SHUTDOWN_DELAY_US_MAX 400
@@ -403,8 +412,7 @@
return -EINVAL;
}
if (gxp->power_mgr->curr_state == AUR_OFF) {
- dev_err(gxp->dev,
- "Cannot request power state when BLK is off\n");
+ dev_warn(gxp->dev, "Cannot request power state when BLK is off\n");
return -EBUSY;
}
if (state == AUR_OFF)
@@ -731,36 +739,118 @@
return ret;
}
-static int gxp_pm_power_up(void *data)
+#if GXP_HAS_MCU
+static int gxp_pm_update_freq_limits_locked(struct gxp_dev *gxp)
{
- struct gxp_dev *gxp = data;
- int ret = gxp_pm_blk_on(gxp);
+ struct gxp_power_manager *mgr = gxp->power_mgr;
+ struct gxp_kci *kci = &(gxp_mcu_of(gxp)->kci);
+ int ret;
+ lockdep_assert_held(&gxp->power_mgr->freq_limits_lock);
+
+ ret = gxp_kci_set_freq_limits(kci, mgr->min_freq_limit, mgr->max_freq_limit);
if (ret) {
- dev_err(gxp->dev, "Failed to power on BLK_AUR (ret=%d)\n", ret);
- return ret;
- }
-
- if (gxp->pm_after_blk_on) {
- ret = gxp->pm_after_blk_on(gxp);
- if (ret) {
- gxp_pm_blk_off(gxp);
- return ret;
+ dev_warn(gxp->dev, "Set frequency limit request failed with error %d.", ret);
+ if (ret == GCIP_KCI_ERROR_INVALID_ARGUMENT) {
+ dev_warn(gxp->dev,
+ "Invalid values within debugfs frequency limits: [%u, %u]\n",
+ mgr->min_freq_limit, mgr->max_freq_limit);
+ ret = -EINVAL;
+ } else {
+ ret = -EIO;
}
+ mgr->min_freq_limit = 0;
+ mgr->max_freq_limit = 0;
+ } else {
+ dev_info(gxp->dev, "BLK frequency to remain in [%u, %u]kHz frequency limit.\n",
+ mgr->min_freq_limit, mgr->max_freq_limit);
}
+ return ret;
+}
+
+static int gxp_pm_set_freq_limit(struct gxp_dev *gxp, u32 val, u32 *limit_to_set)
+{
+ int ret = 0;
+
+ if (val == *limit_to_set)
+ return ret;
+ /*
+ * Need to hold pm lock to prevent races with power up/down when checking block state and
+ * sending the KCI command to update limits.
+ *
+ * Since power_up will also acquire freq_limits_lock to send initial limits, pm lock must be
+ * held first to avoid lock inversion.
+ */
+ gcip_pm_lock(gxp->power_mgr->pm);
+ mutex_lock(&gxp->power_mgr->freq_limits_lock);
+
+ *limit_to_set = val;
+ if (!gxp_pm_is_blk_down(gxp))
+ ret = gxp_pm_update_freq_limits_locked(gxp);
+
+ mutex_unlock(&gxp->power_mgr->freq_limits_lock);
+ gcip_pm_unlock(gxp->power_mgr->pm);
+ return ret;
+}
+
+static int debugfs_min_freq_limit_get(void *data, u64 *val)
+{
+ struct gxp_dev *gxp = (struct gxp_dev *)data;
+ struct gxp_power_manager *mgr = gxp->power_mgr;
+
+ mutex_lock(&mgr->freq_limits_lock);
+ *val = mgr->min_freq_limit;
+ mutex_unlock(&mgr->freq_limits_lock);
return 0;
}
-static int gxp_pm_power_down(void *data)
+static int debugfs_min_freq_limit_set(void *data, u64 val)
{
- struct gxp_dev *gxp = data;
+ struct gxp_dev *gxp = (struct gxp_dev *)data;
+ struct gxp_power_manager *mgr = gxp->power_mgr;
- if (gxp->pm_before_blk_off)
- gxp->pm_before_blk_off(gxp);
- return gxp_pm_blk_off(gxp);
+ if (val > UINT_MAX) {
+ dev_err(gxp->dev, "Requested debugfs min freq %llu must be <= %u (UINT_MAX)\n", val,
+ UINT_MAX);
+ return -EINVAL;
+ }
+
+ return gxp_pm_set_freq_limit(gxp, (u32)val, &mgr->min_freq_limit);
}
+DEFINE_DEBUGFS_ATTRIBUTE(debugfs_min_freq_limit_fops, debugfs_min_freq_limit_get,
+ debugfs_min_freq_limit_set, "%llu\n");
+
+static int debugfs_max_freq_limit_get(void *data, u64 *val)
+{
+ struct gxp_dev *gxp = (struct gxp_dev *)data;
+ struct gxp_power_manager *mgr = gxp->power_mgr;
+
+ mutex_lock(&mgr->freq_limits_lock);
+ *val = mgr->max_freq_limit;
+ mutex_unlock(&mgr->freq_limits_lock);
+ return 0;
+}
+
+static int debugfs_max_freq_limit_set(void *data, u64 val)
+{
+ struct gxp_dev *gxp = (struct gxp_dev *)data;
+ struct gxp_power_manager *mgr = gxp->power_mgr;
+
+ if (val > UINT_MAX) {
+ dev_err(gxp->dev, "Requested debugfs max freq %llu must be <= %u (UINT_MAX)\n", val,
+ UINT_MAX);
+ return -EINVAL;
+ }
+
+ return gxp_pm_set_freq_limit(gxp, (u32)val, &mgr->max_freq_limit);
+}
+
+DEFINE_DEBUGFS_ATTRIBUTE(debugfs_max_freq_limit_fops, debugfs_max_freq_limit_get,
+ debugfs_max_freq_limit_set, "%llu\n");
+#endif /* GXP_HAS_MCU */
+
static int debugfs_wakelock_set(void *data, u64 val)
{
struct gxp_dev *gxp = (struct gxp_dev *)data;
@@ -848,6 +938,42 @@
debugfs_blk_powerstate_get, debugfs_blk_powerstate_set,
"%llx\n");
+static int gxp_pm_power_up(void *data)
+{
+ struct gxp_dev *gxp = data;
+ int ret = gxp_pm_blk_on(gxp);
+
+ if (ret) {
+ dev_err(gxp->dev, "Failed to power on BLK_AUR (ret=%d)\n", ret);
+ return ret;
+ }
+
+ if (gxp->pm_after_blk_on) {
+ ret = gxp->pm_after_blk_on(gxp);
+ if (ret) {
+ gxp_pm_blk_off(gxp);
+ return ret;
+ }
+ }
+
+#if GXP_HAS_MCU
+ mutex_lock(&gxp->power_mgr->freq_limits_lock);
+ if (gxp->power_mgr->min_freq_limit || gxp->power_mgr->max_freq_limit)
+ gxp_pm_update_freq_limits_locked(gxp);
+ mutex_unlock(&gxp->power_mgr->freq_limits_lock);
+#endif /* GXP_HAS_MCU */
+ return 0;
+}
+
+static int gxp_pm_power_down(void *data)
+{
+ struct gxp_dev *gxp = data;
+
+ if (gxp->pm_before_blk_off)
+ gxp->pm_before_blk_off(gxp);
+ return gxp_pm_blk_off(gxp);
+}
+
static void gxp_pm_on_busy(struct gxp_dev *gxp)
{
set_cmu_pll_aur_mux_state(gxp, AUR_CMU_MUX_NORMAL);
@@ -862,6 +988,12 @@
}
}
+/**
+ * gxp_pm_parse_pmu_base() - Parse the pmu-aur-status from device tree.
+ * @gxp: The gxp_dev object.
+ *
+ * Parse the pmu status from property or fall back to try finding from reg instead.
+ */
static void gxp_pm_parse_pmu_base(struct gxp_dev *gxp)
{
u32 reg;
@@ -869,31 +1001,28 @@
struct platform_device *pdev =
container_of(gxp->dev, struct platform_device, dev);
struct device *dev = gxp->dev;
-
- /* TODO(b/309801480): Remove after DT updated */
- r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pmu_aur_status");
- if (r) {
- gxp->power_mgr->aur_status = devm_ioremap_resource(gxp->dev, r);
- if (IS_ERR(gxp->power_mgr->aur_status)) {
- dev_warn(gxp->dev, "Failed to map PMU register base, ret=%ld\n",
- PTR_ERR(gxp->power_mgr->aur_status));
- gxp->power_mgr->aur_status = NULL;
- }
- }
+ void __iomem *aur_status = ERR_PTR(-ENOENT);
/* "pmu-aur-status" DT property takes precedence over reg entry */
if (of_find_property(dev->of_node, "pmu-aur-status", NULL) &&
- !of_property_read_u32_index(dev->of_node, "pmu-aur-status", 0, ®)) {
- gxp->power_mgr->aur_status = devm_ioremap(dev, reg, 0x4);
- if (IS_ERR(gxp->power_mgr->aur_status)) {
- dev_warn(gxp->dev, "Failed to map PMU register base, ret=%ld\n",
- PTR_ERR(gxp->power_mgr->aur_status));
- gxp->power_mgr->aur_status = NULL;
- }
- } else {
- if (!r)
- dev_warn(gxp->dev, "Failed to find PMU register base\n");
+ !of_property_read_u32_index(dev->of_node, "pmu-aur-status", 0, ®)) {
+ aur_status = devm_ioremap(dev, reg, 0x4);
}
+
+ if (!IS_ERR(aur_status))
+ goto out;
+
+ /* Fall back to try pasring from resource if the property not found. */
+ r = platform_get_resource_byname(pdev, IORESOURCE_MEM, "pmu_aur_status");
+ if (r)
+ aur_status = devm_ioremap_resource(gxp->dev, r);
+
+out:
+ if (IS_ERR(aur_status))
+ dev_warn(gxp->dev, "Failed to get PMU register base, ret=%ld\n",
+ PTR_ERR(aur_status));
+
+ gxp->power_mgr->aur_status = IS_ERR(aur_status) ? NULL : aur_status;
}
int gxp_pm_init(struct gxp_dev *gxp)
@@ -906,6 +1035,7 @@
.power_down = gxp_pm_power_down,
};
uint i;
+ int ret;
mgr = devm_kzalloc(gxp->dev, sizeof(*mgr), GFP_KERNEL);
if (!mgr)
@@ -914,8 +1044,8 @@
mgr->pm = gcip_pm_create(&args);
if (IS_ERR(mgr->pm)) {
- devm_kfree(gxp->dev, mgr);
- return PTR_ERR(mgr->pm);
+ ret = PTR_ERR(mgr->pm);
+ goto err_free_mgr;
}
mutex_init(&mgr->pm_lock);
@@ -937,8 +1067,11 @@
}
mutex_init(&mgr->set_acpm_state_work_lock);
mutex_init(&mgr->req_pm_qos_work_lock);
- gxp->power_mgr->wq =
- create_singlethread_workqueue("gxp_power_work_queue");
+ gxp->power_mgr->wq = create_singlethread_workqueue("gxp_power_work_queue");
+ if (!gxp->power_mgr->wq) {
+ ret = -ENOMEM;
+ goto err_pm_destroy;
+ }
gxp->power_mgr->force_mux_normal_count = 0;
gxp->power_mgr->blk_switch_count = 0l;
spin_lock_init(&gxp->power_mgr->busy_lock);
@@ -951,12 +1084,24 @@
gxp_pm_chip_init(gxp);
gxp->debugfs_wakelock_held = false;
- debugfs_create_file(DEBUGFS_WAKELOCK, 0200, gxp->d_entry, gxp,
- &debugfs_wakelock_fops);
+#if GXP_HAS_MCU
+ mutex_init(&mgr->freq_limits_lock);
+ debugfs_create_file(DEBUGFS_MIN_FREQ, 0600, gxp->d_entry, gxp,
+ &debugfs_min_freq_limit_fops);
+ debugfs_create_file(DEBUGFS_MAX_FREQ, 0600, gxp->d_entry, gxp,
+ &debugfs_max_freq_limit_fops);
+#endif /* GXP_HAS_MCU */
+ debugfs_create_file(DEBUGFS_WAKELOCK, 0200, gxp->d_entry, gxp, &debugfs_wakelock_fops);
debugfs_create_file(DEBUGFS_BLK_POWERSTATE, 0600, gxp->d_entry, gxp,
&debugfs_blk_powerstate_fops);
return 0;
+
+err_pm_destroy:
+ gcip_pm_destroy(mgr->pm);
+err_free_mgr:
+ devm_kfree(gxp->dev, mgr);
+ return ret;
}
int gxp_pm_destroy(struct gxp_dev *gxp)
@@ -966,6 +1111,10 @@
if (IS_GXP_TEST && !mgr)
return 0;
+#if GXP_HAS_MCU
+ debugfs_remove(debugfs_lookup(DEBUGFS_MIN_FREQ, gxp->d_entry));
+ debugfs_remove(debugfs_lookup(DEBUGFS_MAX_FREQ, gxp->d_entry));
+#endif /* GXP_HAS_MCU */
debugfs_remove(debugfs_lookup(DEBUGFS_BLK_POWERSTATE, gxp->d_entry));
debugfs_remove(debugfs_lookup(DEBUGFS_WAKELOCK, gxp->d_entry));
diff --git a/gxp-pm.h b/gxp-pm.h
index 27a1398..b3a5fc7 100644
--- a/gxp-pm.h
+++ b/gxp-pm.h
@@ -124,6 +124,13 @@
bool curr_low_clkmux;
/* Last requested clock mux state */
bool last_scheduled_low_clkmux;
+ /*
+ * Min/Max frequency limits, in kHz, requested via debugfs.
+ * Protected by `freq_limits_lock`.
+ */
+ struct mutex freq_limits_lock;
+ u32 min_freq_limit;
+ u32 max_freq_limit;
int curr_state;
int curr_memory_state; /* Note: this state will not be maintained in the MCU mode. */
const struct gxp_pm_ops *ops;
diff --git a/gxp-thermal.c b/gxp-thermal.c
index 9807b09..8c851fb 100644
--- a/gxp-thermal.c
+++ b/gxp-thermal.c
@@ -33,7 +33,7 @@
struct gxp_dev *gxp = data;
int ret = 0;
- dev_warn(gxp->dev, "Received thermal throttling requests %lu", rate);
+ dev_warn(gxp->dev, "Received thermal throttling requests %lu.\n", rate);
if (!gxp_is_direct_mode(gxp)) {
#if GXP_HAS_MCU
struct gxp_mcu *mcu = gxp_mcu_of(gxp);
@@ -58,7 +58,19 @@
static int gxp_thermal_control(void *data, bool enable)
{
- return -EOPNOTSUPP;
+ int ret = -EOPNOTSUPP;
+#if GXP_HAS_MCU
+ struct gxp_dev *gxp = data;
+ struct gxp_mcu *mcu = gxp_mcu_of(gxp);
+
+ dev_warn(gxp->dev, "Received request to %s thermal throttling.\n",
+ enable ? "enable" : "disable");
+ ret = gxp_kci_thermal_control(&mcu->kci, enable);
+ if (ret)
+ dev_err(gxp->dev, "Error on %s thermal throttling: %d.\n",
+ enable ? "enabling" : "disabling", ret);
+#endif /* GXP_HAS_MCU */
+ return ret;
}
int gxp_thermal_init(struct gxp_dev *gxp)
diff --git a/gxp-trace.c b/gxp-trace.c
index 4d0202d..ebe15c2 100644
--- a/gxp-trace.c
+++ b/gxp-trace.c
@@ -2,7 +2,7 @@
/*
* GXP ftrace support.
*
- * Copyright (C) 2023 Google LLC
+ * Copyright (C) 2023-2024 Google LLC
*/
#define CREATE_TRACE_POINTS
diff --git a/gxp-uci.c b/gxp-uci.c
index 9ea2e43..6d655a8 100644
--- a/gxp-uci.c
+++ b/gxp-uci.c
@@ -11,9 +11,11 @@
#include <linux/slab.h>
#include <uapi/linux/sched/types.h>
-#include <gcip/iif/iif.h>
#include <gcip/gcip-fence-array.h>
#include <gcip/gcip-fence.h>
+#include <iif/iif.h>
+
+#include <trace/events/gxp.h>
#include "gxp-config.h"
#include "gxp-internal.h"
@@ -253,7 +255,7 @@
}
ret = gcip_fence_array_submit_waiter_and_signaler(async_resp->in_fences,
- async_resp->out_fences);
+ async_resp->out_fences, IIF_IP_DSP);
if (ret) {
dev_err(mailbox->dev, "Failed to submit waiter or signaler to fences, ret=%d", ret);
list_del_init(&async_resp->wait_list_entry);
@@ -294,10 +296,12 @@
spin_unlock_irqrestore(async_resp->queue_lock, flags);
gcip_fence_array_signal(async_resp->out_fences, (status != GXP_RESP_OK) ? -ETIMEDOUT : 0);
- gcip_fence_array_waited(async_resp->in_fences);
+ gcip_fence_array_waited(async_resp->in_fences, IIF_IP_DSP);
if (async_resp->eventfd)
gxp_eventfd_signal(async_resp->eventfd);
+ trace_gxp_uci_rsp_end(async_resp->resp.seq);
+
wake_up(async_resp->dest_queue_waitq);
}
@@ -356,7 +360,6 @@
}
static const struct gcip_mailbox_ops gxp_uci_gcip_mbx_ops = {
- .get_cmd_queue_head = gxp_mailbox_gcip_ops_get_cmd_queue_head,
.get_cmd_queue_tail = gxp_mailbox_gcip_ops_get_cmd_queue_tail,
.inc_cmd_queue_tail = gxp_mailbox_gcip_ops_inc_cmd_queue_tail,
.acquire_cmd_queue_lock = gxp_mailbox_gcip_ops_acquire_cmd_queue_lock,
@@ -574,10 +577,7 @@
{
struct gxp_mailbox *mailbox = uci->mbx;
- gxp_mailbox_write_descriptor(mailbox, mailbox->descriptor_buf.dsp_addr);
- gxp_mailbox_reset(uci->mbx);
- gxp_mailbox_enable_interrupt(mailbox);
- gxp_mailbox_write_status(mailbox, 1);
+ gxp_mailbox_reinit(mailbox);
return 0;
}
diff --git a/gxp-usage-stats.c b/gxp-usage-stats.c
index 44d6960..1268c25 100644
--- a/gxp-usage-stats.c
+++ b/gxp-usage-stats.c
@@ -73,7 +73,7 @@
return AUR_NUM_POWER_STATE;
}
-static int get_default_dvfs_freq(int idx, void *data)
+static unsigned int get_default_dvfs_freq(int idx, void *data)
{
if (idx >= AUR_NUM_POWER_STATE)
return 0;
diff --git a/gxp-vd.c b/gxp-vd.c
index 81237d6..9509c28 100644
--- a/gxp-vd.c
+++ b/gxp-vd.c
@@ -183,11 +183,11 @@
{
struct gxp_dev *gxp = vd->gxp;
int ret;
- const size_t ro_size = GXP_FW_DATA_SYSCFG_SIZE / 2;
+ const size_t ro_size = res->size / 2;
if (res->daddr == 0)
return 0;
- if (res->size != GXP_FW_DATA_SYSCFG_SIZE) {
+ if (!res->size || !IS_ALIGNED(res->size, gcip_iommu_domain_granule(vd->domain) * 2)) {
dev_err(gxp->dev, "invalid system cfg size: %#llx", res->size);
return -EINVAL;
}
@@ -239,8 +239,7 @@
* To keep compatibility, if not both mapping[0, 1] present then this function
* falls back to map the MCU-core shared region with hard-coded IOVA and size.
*/
-static int map_cfg_regions(struct gxp_virtual_device *vd,
- struct gcip_image_config *img_cfg)
+static int map_cfg_regions(struct gxp_virtual_device *vd, struct gcip_image_config *img_cfg)
{
struct gxp_dev *gxp = vd->gxp;
struct gxp_mapped_resource pool;
@@ -318,55 +317,27 @@
}
static int gxp_vd_imgcfg_map(void *data, dma_addr_t daddr, phys_addr_t paddr,
- size_t size, unsigned int flags)
+ size_t size, unsigned int cfg_map_flags, unsigned int cfg_op_flags)
{
struct gxp_virtual_device *vd = data;
- if (flags & GCIP_IMAGE_CONFIG_FLAGS_SECURE)
+ if (cfg_op_flags & GCIP_IMAGE_CONFIG_FLAGS_SECURE)
return 0;
return map_ns_region(vd, daddr, size);
}
static void gxp_vd_imgcfg_unmap(void *data, dma_addr_t daddr, size_t size,
- unsigned int flags)
+ unsigned int cfg_map_flags, unsigned int cfg_op_flags)
{
struct gxp_virtual_device *vd = data;
- if (flags & GCIP_IMAGE_CONFIG_FLAGS_SECURE)
+ if (cfg_op_flags & GCIP_IMAGE_CONFIG_FLAGS_SECURE)
return;
unmap_ns_region(vd, daddr);
}
-/* TODO(b/299037074): Remove core's accesses to LPM. */
-static int map_remote_lpm(struct gxp_virtual_device *vd,
- struct gcip_image_config *img_cfg)
-{
- struct gxp_mapped_resource res;
- int ret;
-
- if (img_cfg->num_iommu_mappings != REMOTE_LPM_IDX + 1)
- /* Core doesn't require remote lpm */
- return 0;
-
- res.daddr = img_cfg->iommu_mappings[REMOTE_LPM_IDX].virt_address;
- res.paddr = (img_cfg->iommu_mappings[REMOTE_LPM_IDX].image_config_value) &
- GCIP_IMG_CFG_ADDR_MASK;
- res.size = gcip_config_to_size(img_cfg->iommu_mappings[REMOTE_LPM_IDX].image_config_value);
- ret = map_resource(vd, &res);
- if (ret)
- return ret;
-
- vd->lpm = res;
- return 0;
-}
-
-static void unmap_remote_lpm(struct gxp_virtual_device *vd)
-{
- unmap_resource(vd, &vd->lpm);
-}
-
static int
map_fw_image_config(struct gxp_dev *gxp, struct gxp_virtual_device *vd,
struct gxp_firmware_loader_manager *fw_loader_mgr)
@@ -395,13 +366,6 @@
goto err;
}
- ret = map_remote_lpm(vd, cfg);
- if (ret) {
- dev_err(gxp->dev, "Remote LPM mapping failed");
- unmap_cfg_regions(vd);
- goto err;
- }
-
return 0;
err:
gcip_image_config_clear(&vd->cfg_parser);
@@ -411,7 +375,6 @@
static void unmap_fw_image_config(struct gxp_dev *gxp,
struct gxp_virtual_device *vd)
{
- unmap_remote_lpm(vd);
unmap_cfg_regions(vd);
gcip_image_config_clear(&vd->cfg_parser);
}
@@ -699,6 +662,8 @@
mutex_init(&vd->fence_list_lock);
INIT_LIST_HEAD(&vd->gxp_fence_list);
mutex_init(&vd->debug_dump_lock);
+ init_waitqueue_head(&vd->finished_dump_processing_waitq);
+ atomic_set(&vd->core_dump_generated_list, 0);
#ifdef GXP_USE_DEFAULT_DOMAIN
vd->domain = gxp_iommu_get_domain_for_dev(gxp);
@@ -1050,8 +1015,46 @@
debug_dump_unlock(vd);
}
-static inline uint select_core(struct gxp_virtual_device *vd, uint virt_core,
- uint phys_core)
+void gxp_vd_check_and_wait_for_debug_dump(struct gxp_virtual_device *vd)
+{
+ struct gxp_dev *gxp = vd->gxp;
+ bool vd_crashed = 0;
+ uint phys_core;
+ uint remaining_time;
+
+ if (!gxp_is_direct_mode(gxp))
+ return;
+
+ /*
+ * Check if any of the cores for the given virtual device has generated the debug dump or
+ * has been requested to generate the forced debug dump.
+ */
+ for (phys_core = 0; phys_core < GXP_NUM_CORES; phys_core++) {
+ if (!(vd->core_list & BIT(phys_core)))
+ continue;
+
+ vd_crashed |= gxp_firmware_get_generate_debug_dump(gxp, vd, phys_core) |
+ gxp_firmware_get_debug_dump_generated(gxp, vd, phys_core);
+ }
+
+ if (vd_crashed) {
+ /*
+ * Successive prccessing of debug dumps demand a delay a second. This
+ * delay is due to the current implementation of the SSCD module which
+ * generates the dump files whose names are at precision of a second i.e.
+ * coredump_<SUBSYSTEM_NAME>_<%Y-%m-%d_%H-%M-%S>.bin Thus max wait time is
+ * kept to be GXP_NUM_CORES seconds.
+ */
+ remaining_time = wait_event_timeout(
+ vd->finished_dump_processing_waitq,
+ atomic_read(&vd->core_dump_generated_list) == vd->core_list,
+ msecs_to_jiffies(GXP_NUM_CORES * SSCD_REPORT_WAIT_TIME));
+ if (!remaining_time)
+ dev_warn(gxp->dev, "Debug dump processing timedout.\n");
+ }
+}
+
+static inline uint select_core(struct gxp_virtual_device *vd, uint virt_core, uint phys_core)
{
return virt_core;
}
diff --git a/gxp-vd.h b/gxp-vd.h
index f49479d..5dd3fb2 100644
--- a/gxp-vd.h
+++ b/gxp-vd.h
@@ -155,6 +155,16 @@
bool mcu_crashed;
/* The manager of IOMMU reserve regions. */
struct gcip_iommu_reserve_manager *iommu_reserve_mgr;
+ /*
+ * Bitmap of cores for which KD has processed the core dump. This flag is used only
+ * in direct mode.
+ */
+ atomic_t core_dump_generated_list;
+ /*
+ * Wait queue to wait till the debug dump processing finishes for all the cores for
+ * the current VD before powering them down. This flag is used only in direct mode.
+ */
+ wait_queue_head_t finished_dump_processing_waitq;
};
/*
@@ -230,6 +240,23 @@
*/
void gxp_vd_stop(struct gxp_virtual_device *vd);
+/**
+ * gxp_vd_check_and_wait_for_debug_dump() - Checks if any of the cores for the given vd has
+ * generated the debug dump or has been requested to
+ * generate the forced debug dump. If yes then waits
+ * till all the debug dumps are processed.
+ * @vd: The virtual device to be checked for the crash.
+ *
+ * The caller should never lock the gxp->vd_semaphore before calling this function. The logic
+ * inside this function waits till the debug dumps for all the cores for the given VD are
+ * processed. Processing of debug dump logic requires acquiring of gxp->vd_semaphore which will
+ * end up in deadlock if this function is called after locking the gxp->vd_semaphore.
+ *
+ * This function is only meaningful in direct mode. On MCU mode it returns directly.
+ */
+
+void gxp_vd_check_and_wait_for_debug_dump(struct gxp_virtual_device *vd);
+
/*
* Returns the physical core ID for the specified virtual_core belonging to
* this virtual device or -EINVAL if this virtual core is not running on a
@@ -493,8 +520,14 @@
void gxp_vd_unlink_offload_vmbox(struct gxp_dev *gxp, struct gxp_virtual_device *vd,
u32 offload_client_id, u8 offload_chip_type);
#else /* !GXP_HAS_MCU */
-#define gxp_vd_release_vmbox(...)
-#define gxp_vd_unlink_offload_vmbox(...)
+static inline void gxp_vd_release_vmbox(struct gxp_dev *gxp, struct gxp_virtual_device *vd)
+{
+}
+
+static inline void gxp_vd_unlink_offload_vmbox(struct gxp_dev *gxp, struct gxp_virtual_device *vd,
+ u32 offload_client_id, u8 offload_chip_type)
+{
+}
#endif /* GXP_HAS_MCU */
/*
diff --git a/gxp.h b/gxp.h
index e4537b7..74010e5 100644
--- a/gxp.h
+++ b/gxp.h
@@ -13,7 +13,7 @@
/* Interface Version */
#define GXP_INTERFACE_VERSION_MAJOR 1
-#define GXP_INTERFACE_VERSION_MINOR 26
+#define GXP_INTERFACE_VERSION_MINOR 27
#define GXP_INTERFACE_VERSION_BUILD 0
/* mmap offsets for MCU logging and tracing buffers */
@@ -272,116 +272,6 @@
#define GXP_ALLOCATE_VIRTUAL_DEVICE \
_IOWR(GXP_IOCTL_BASE, 6, struct gxp_virtual_device_ioctl)
-#define ETM_TRACE_LSB_MASK 0x1
-#define ETM_TRACE_SYNC_MSG_PERIOD_MIN 8
-#define ETM_TRACE_SYNC_MSG_PERIOD_MAX 256
-#define ETM_TRACE_PC_MATCH_MASK_LEN_MAX 31
-
-/*
- * For all *_enable and pc_match_sense fields, only the least significant bit is
- * considered. All other bits are ignored.
- */
-struct gxp_etm_trace_start_ioctl {
- __u16 virtual_core_id;
- __u8 trace_ram_enable; /* Enables local trace memory. */
- /* When set, trace output is sent out on the ATB interface. */
- __u8 atb_enable;
- /* Enables embedding timestamp information in trace messages. */
- __u8 timestamp_enable;
- /*
- * Determines the rate at which synchronization messages are
- * automatically emitted in the output trace.
- * Valid values: 0, 8, 16, 32, 64, 128, 256
- * Eg. A value of 16 means 1 synchronization message will be emitted
- * every 16 messages.
- * A value of 0 means no synchronization messages will be emitted.
- */
- __u16 sync_msg_period;
- __u8 pc_match_enable; /* PC match causes Stop trigger. */
- /*
- * 32-bit address to compare to processor PC when pc_match_enable = 1.
- * A match for a given executed instruction triggers trace stop.
- * Note: trigger_pc is ignored when pc_match_enable = 0.
- */
- __u32 trigger_pc;
- /*
- * Indicates how many of the lower bits of trigger_pc to ignore.
- * Valid values: 0 to 31
- * Note: pc_match_mask_length is ignored when pc_match_enable = 0.
- */
- __u8 pc_match_mask_length;
- /* When 0, match when the processor's PC is in-range of trigger_pc and
- * mask. When 1, match when the processor's PC is out-of-range of
- * trigger_pc and mask.
- * Note: pc_match_sense is ignored when pc_match_enable = 0.
- */
- __u8 pc_match_sense;
-};
-
-/*
- * Configure ETM trace registers and start ETM tracing.
- *
- * The client must hold a VIRTUAL_DEVICE wakelock.
- */
-#define GXP_ETM_TRACE_START_COMMAND \
- _IOW(GXP_IOCTL_BASE, 7, struct gxp_etm_trace_start_ioctl)
-
-/*
- * Halts trace generation via a software trigger. The virtual core id is passed
- * in as an input.
- *
- * The client must hold a VIRTUAL_DEVICE wakelock.
- */
-#define GXP_ETM_TRACE_SW_STOP_COMMAND _IOW(GXP_IOCTL_BASE, 8, __u16)
-
-/*
- * Users should call this IOCTL after tracing has been stopped for the last
- * trace session of the core. Otherwise, there is a risk of having up to 3 bytes
- * of trace data missing towards the end of the trace session.
- * This is a workaround for b/180728272 and b/181623511.
- * The virtual core id is passed in as an input.
- *
- * The client must hold a VIRTUAL_DEVICE wakelock.
- */
-#define GXP_ETM_TRACE_CLEANUP_COMMAND _IOW(GXP_IOCTL_BASE, 9, __u16)
-
-#define GXP_TRACE_HEADER_SIZE 256
-#define GXP_TRACE_RAM_SIZE 4096
-struct gxp_etm_get_trace_info_ioctl {
- /*
- * Input:
- * The virtual core to fetch a response from.
- */
- __u16 virtual_core_id;
- /*
- * Input:
- * The type of data to retrieve.
- * 0: Trace Header only
- * 1: Trace Header + Trace Data in Trace RAM
- */
- __u8 type;
- /*
- * Input:
- * Trace header user space address to contain trace header information
- * that is used for decoding the trace.
- */
- __u64 trace_header_addr;
- /*
- * Input:
- * Trace data user space address to contain Trace RAM data.
- * Note: trace_data field will be empty if type == 0
- */
- __u64 trace_data_addr;
-};
-
-/*
- * Retrieves trace header and/or trace data for decoding purposes.
- *
- * The client must hold a VIRTUAL_DEVICE wakelock.
- */
-#define GXP_ETM_GET_TRACE_INFO_COMMAND \
- _IOWR(GXP_IOCTL_BASE, 10, struct gxp_etm_get_trace_info_ioctl)
-
#define GXP_TELEMETRY_TYPE_LOGGING (0)
#define GXP_TELEMETRY_TYPE_TRACING (1)
diff --git a/include/trace/events/gxp.h b/include/trace/events/gxp.h
index 095bd7c..93b699e 100644
--- a/include/trace/events/gxp.h
+++ b/include/trace/events/gxp.h
@@ -2,7 +2,7 @@
/*
* Trace events for gxp
*
- * Copyright (c) 2023 Google LLC
+ * Copyright (c) 2023-2024 Google LLC
*/
#undef TRACE_SYSTEM
@@ -215,6 +215,54 @@
TP_printk("vdid = %d", __entry->vdid));
+TRACE_EVENT(gxp_uci_cmd_start,
+
+ TP_PROTO(u64 seq),
+
+ TP_ARGS(seq),
+
+ TP_STRUCT__entry(__field(u64, seq)),
+
+ TP_fast_assign(__entry->seq = seq;),
+
+ TP_printk("seq = %llu", __entry->seq));
+
+TRACE_EVENT(gxp_uci_cmd_end,
+
+ TP_PROTO(u64 seq),
+
+ TP_ARGS(seq),
+
+ TP_STRUCT__entry(__field(u64, seq)),
+
+ TP_fast_assign(__entry->seq = seq;),
+
+ TP_printk("seq = %llu", __entry->seq));
+
+TRACE_EVENT(gxp_uci_rsp_start,
+
+ TP_PROTO(int irq),
+
+ TP_ARGS(irq),
+
+ TP_STRUCT__entry(__field(int, irq)),
+
+ TP_fast_assign(__entry->irq = irq;),
+
+ TP_printk("irq = %d", __entry->irq));
+
+TRACE_EVENT(gxp_uci_rsp_end,
+
+ TP_PROTO(u64 seq),
+
+ TP_ARGS(seq),
+
+ TP_STRUCT__entry(__field(u64, seq)),
+
+ TP_fast_assign(__entry->seq = seq;),
+
+ TP_printk("seq = %llu", __entry->seq));
+
#endif /* _TRACE_GXP_H */
/* This part must be outside protection */
diff --git a/mobile-soc-gsx01.c b/mobile-soc-gsx01.c
index 65e368a..51efd4e 100644
--- a/mobile-soc-gsx01.c
+++ b/mobile-soc-gsx01.c
@@ -2,21 +2,19 @@
/*
* SoC specific function definitions for GSx01.
*
- * Copyright (C) 2023 Google LLC
+ * Copyright (C) 2023-2024 Google LLC
*/
#include <linux/acpm_dvfs.h>
#include <linux/slab.h>
#include <soc/google/exynos_pm_qos.h>
-#include <gcip/gcip-kci.h>
#include <gcip/gcip-slc.h>
#include "gxp-config.h"
#include "gxp-core-telemetry.h"
#include "gxp-firmware.h"
#include "gxp-gsx01-ssmt.h"
-#include "gxp-kci.h"
#include "gxp-lpm.h"
#include "gxp-pm.h"
#include "mobile-soc-gsx01.h"
@@ -80,10 +78,14 @@
{
exynos_pm_qos_add_request(&gxp->soc_data->int_min, PM_QOS_DEVICE_THROUGHPUT, 0);
exynos_pm_qos_add_request(&gxp->soc_data->mif_min, PM_QOS_BUS_THROUGHPUT, 0);
+ exynos_pm_qos_add_request(&gxp->soc_data->bci_min, PM_QOS_BCI_THROUGHPUT, 0);
+ exynos_pm_qos_add_request(&gxp->soc_data->dsu_min, PM_QOS_DSU_THROUGHPUT, 0);
}
void gxp_soc_pm_exit(struct gxp_dev *gxp)
{
+ exynos_pm_qos_remove_request(&gxp->soc_data->dsu_min);
+ exynos_pm_qos_remove_request(&gxp->soc_data->bci_min);
exynos_pm_qos_remove_request(&gxp->soc_data->mif_min);
exynos_pm_qos_remove_request(&gxp->soc_data->int_min);
}
@@ -113,6 +115,8 @@
{
exynos_pm_qos_update_request(&gxp->soc_data->int_min, 0);
exynos_pm_qos_update_request(&gxp->soc_data->mif_min, 0);
+ exynos_pm_qos_update_request(&gxp->soc_data->bci_min, 0);
+ exynos_pm_qos_update_request(&gxp->soc_data->dsu_min, 0);
}
int gxp_soc_init(struct gxp_dev *gxp)
diff --git a/mobile-soc-gsx01.h b/mobile-soc-gsx01.h
index 2d12fe4..d817998 100644
--- a/mobile-soc-gsx01.h
+++ b/mobile-soc-gsx01.h
@@ -2,7 +2,7 @@
/*
* Header file for GSx01.
*
- * Copyright (C) 2023 Google LLC
+ * Copyright (C) 2023-2024 Google LLC
*/
#ifndef __MOBILE_SOC_GSX01_H__
@@ -18,6 +18,8 @@
/* INT/MIF requests for memory bandwidth. */
struct exynos_pm_qos_request int_min;
struct exynos_pm_qos_request mif_min;
+ struct exynos_pm_qos_request bci_min;
+ struct exynos_pm_qos_request dsu_min;
/* The Stream Security Mapping Table support. */
struct gxp_ssmt ssmt;
/* The SLC configurations. */
diff --git a/mobile-soc.h b/mobile-soc.h
index 9b378ce..42a5ec3 100644
--- a/mobile-soc.h
+++ b/mobile-soc.h
@@ -2,7 +2,7 @@
/*
* SoC-specific function headers.
*
- * Copyright (C) 2023 Google LLC
+ * Copyright (C) 2023-2024 Google LLC
*/
#ifndef __MOBILE_SOC_H__
