include/linux/io-pgtable-arm.h - kernel/common - Git at Google

 /* SPDX-License-Identifier: GPL-2.0-only */
 #ifndef IO_PGTABLE_H_
 #define IO_PGTABLE_H_

 #include <linux/io-pgtable.h>

 extern bool selftest_running;

 typedef u64 arm_lpae_iopte;

 struct arm_lpae_io_pgtable {
 	struct io_pgtable	iop;

 	int			pgd_bits;
 	int			start_level;
 	int			bits_per_level;

 	void			*pgd;

 	bool			idmapped; /* Used by hypervisor */
 };

 /* Struct accessors */
 #define io_pgtable_to_data(x)						\
 	container_of((x), struct arm_lpae_io_pgtable, iop)

 #define io_pgtable_ops_to_data(x)					\
 	io_pgtable_to_data(io_pgtable_ops_to_pgtable(x))

 /*
  * Calculate the right shift amount to get to the portion describing level l
  * in a virtual address mapped by the pagetable in d.
  */
 #define ARM_LPAE_LVL_SHIFT(l,d)						\
 	(((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level) +		\
 	ilog2(sizeof(arm_lpae_iopte)))

 #define ARM_LPAE_GRANULE(d)						\
 	(sizeof(arm_lpae_iopte) << (d)->bits_per_level)
 #define ARM_LPAE_PGD_SIZE(d)						\
 	(sizeof(arm_lpae_iopte) << (d)->pgd_bits)

 #define ARM_LPAE_PTES_PER_TABLE(d)					\
 	(ARM_LPAE_GRANULE(d) >> ilog2(sizeof(arm_lpae_iopte)))

 /*
  * Calculate the index at level l used to map virtual address a using the
  * pagetable in d.
  */
 #define ARM_LPAE_PGD_IDX(l,d)						\
 	((l) == (d)->start_level ? (d)->pgd_bits - (d)->bits_per_level : 0)

 #define ARM_LPAE_LVL_IDX(a,l,d)						\
 	(((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) &			\
 	 ((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1))

 /* Calculate the block/page mapping size at level l for pagetable in d. */
 #define ARM_LPAE_BLOCK_SIZE(l,d)	(1ULL << ARM_LPAE_LVL_SHIFT(l,d))

 /* Page table bits */
 #define ARM_LPAE_PTE_TYPE_SHIFT		0
 #define ARM_LPAE_PTE_TYPE_MASK		0x3

 #define ARM_LPAE_PTE_TYPE_BLOCK		1
 #define ARM_LPAE_PTE_TYPE_TABLE		3
 #define ARM_LPAE_PTE_TYPE_PAGE		3

 #define ARM_LPAE_PTE_ADDR_MASK		GENMASK_ULL(47,12)

 #define ARM_LPAE_PTE_NSTABLE		(((arm_lpae_iopte)1) << 63)
 #define ARM_LPAE_PTE_XN			(((arm_lpae_iopte)3) << 53)
 #define ARM_LPAE_PTE_AF			(((arm_lpae_iopte)1) << 10)
 #define ARM_LPAE_PTE_SH_NS		(((arm_lpae_iopte)0) << 8)
 #define ARM_LPAE_PTE_SH_OS		(((arm_lpae_iopte)2) << 8)
 #define ARM_LPAE_PTE_SH_IS		(((arm_lpae_iopte)3) << 8)
 #define ARM_LPAE_PTE_NS			(((arm_lpae_iopte)1) << 5)
 #define ARM_LPAE_PTE_VALID		(((arm_lpae_iopte)1) << 0)

 #define ARM_LPAE_PTE_ATTR_LO_MASK	(((arm_lpae_iopte)0x3ff) << 2)
 /* Ignore the contiguous bit for block splitting */
 #define ARM_LPAE_PTE_ATTR_HI_MASK	(((arm_lpae_iopte)6) << 52)
 #define ARM_LPAE_PTE_ATTR_MASK		(ARM_LPAE_PTE_ATTR_LO_MASK |	\
 					 ARM_LPAE_PTE_ATTR_HI_MASK)
 /* Software bit for solving coherency races */
 #define ARM_LPAE_PTE_SW_SYNC		(((arm_lpae_iopte)1) << 55)

 /* Stage-1 PTE */
 #define ARM_LPAE_PTE_AP_UNPRIV		(((arm_lpae_iopte)1) << 6)
 #define ARM_LPAE_PTE_AP_RDONLY		(((arm_lpae_iopte)2) << 6)
 #define ARM_LPAE_PTE_ATTRINDX_SHIFT	2
 #define ARM_LPAE_PTE_nG			(((arm_lpae_iopte)1) << 11)

 /* Stage-2 PTE */
 #define ARM_LPAE_PTE_HAP_FAULT		(((arm_lpae_iopte)0) << 6)
 #define ARM_LPAE_PTE_HAP_READ		(((arm_lpae_iopte)1) << 6)
 #define ARM_LPAE_PTE_HAP_WRITE		(((arm_lpae_iopte)2) << 6)
 #define ARM_LPAE_PTE_MEMATTR_OIWB	(((arm_lpae_iopte)0xf) << 2)
 #define ARM_LPAE_PTE_MEMATTR_NC		(((arm_lpae_iopte)0x5) << 2)
 #define ARM_LPAE_PTE_MEMATTR_DEV	(((arm_lpae_iopte)0x1) << 2)

 /* Register bits */
 #define ARM_LPAE_VTCR_SL0_MASK		0x3

 #define ARM_LPAE_TCR_T0SZ_SHIFT		0

 #define ARM_LPAE_TCR_TG0_4K		0
 #define ARM_LPAE_TCR_TG0_64K		1
 #define ARM_LPAE_TCR_TG0_16K		2

 #define ARM_LPAE_TCR_TG1_16K		1
 #define ARM_LPAE_TCR_TG1_4K		2
 #define ARM_LPAE_TCR_TG1_64K		3

 #define ARM_LPAE_TCR_SH_NS		0
 #define ARM_LPAE_TCR_SH_OS		2
 #define ARM_LPAE_TCR_SH_IS		3

 #define ARM_LPAE_TCR_RGN_NC		0
 #define ARM_LPAE_TCR_RGN_WBWA		1
 #define ARM_LPAE_TCR_RGN_WT		2
 #define ARM_LPAE_TCR_RGN_WB		3

 #define ARM_LPAE_TCR_PS_32_BIT		0x0ULL
 #define ARM_LPAE_TCR_PS_36_BIT		0x1ULL
 #define ARM_LPAE_TCR_PS_40_BIT		0x2ULL
 #define ARM_LPAE_TCR_PS_42_BIT		0x3ULL
 #define ARM_LPAE_TCR_PS_44_BIT		0x4ULL
 #define ARM_LPAE_TCR_PS_48_BIT		0x5ULL
 #define ARM_LPAE_TCR_PS_52_BIT		0x6ULL

 #define ARM_LPAE_VTCR_PS_SHIFT		16
 #define ARM_LPAE_VTCR_PS_MASK		0x7

 #define ARM_LPAE_MAIR_ATTR_SHIFT(n)	((n) << 3)
 #define ARM_LPAE_MAIR_ATTR_MASK		0xff
 #define ARM_LPAE_MAIR_ATTR_DEVICE	0x04
 #define ARM_LPAE_MAIR_ATTR_NC		0x44
 #define ARM_LPAE_MAIR_ATTR_INC_OWBRWA	0xf4
 #define ARM_LPAE_MAIR_ATTR_WBRWA	0xff
 #define ARM_LPAE_MAIR_ATTR_IDX_NC	0
 #define ARM_LPAE_MAIR_ATTR_IDX_CACHE	1
 #define ARM_LPAE_MAIR_ATTR_IDX_DEV	2
 #define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE	3

 #define ARM_MALI_LPAE_TTBR_ADRMODE_TABLE (3u << 0)
 #define ARM_MALI_LPAE_TTBR_READ_INNER	BIT(2)
 #define ARM_MALI_LPAE_TTBR_SHARE_OUTER	BIT(4)

 #define ARM_MALI_LPAE_MEMATTR_IMP_DEF	0x88ULL
 #define ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 0x8DULL

 #define ARM_LPAE_MAX_LEVELS		4

 #define iopte_type(pte)					\
 	(((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK)

 #define iopte_prot(pte)	((pte) & ARM_LPAE_PTE_ATTR_MASK)

 static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl,
 			      enum io_pgtable_fmt fmt)
 {
 	if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE)
 		return iopte_type(pte) == ARM_LPAE_PTE_TYPE_PAGE;

 	return iopte_type(pte) == ARM_LPAE_PTE_TYPE_BLOCK;
 }

 #ifdef __KVM_NVHE_HYPERVISOR__
 #include <nvhe/memory.h>
 #define __arm_lpae_virt_to_phys	hyp_virt_to_phys
 #define __arm_lpae_phys_to_virt	hyp_phys_to_virt
 #else
 #define __arm_lpae_virt_to_phys	__pa
 #define __arm_lpae_phys_to_virt	__va
 #endif

 /* Generic functions */
 void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
 			     arm_lpae_iopte *ptep);

 int arm_lpae_init_pgtable(struct io_pgtable_cfg *cfg,
 			  struct arm_lpae_io_pgtable *data);
 int arm_lpae_init_pgtable_s1(struct io_pgtable_cfg *cfg,
 			     struct arm_lpae_io_pgtable *data);
 int arm_lpae_init_pgtable_s2(struct io_pgtable_cfg *cfg,
 			     struct arm_lpae_io_pgtable *data);

 /* Host/hyp-specific functions */
 void *__arm_lpae_alloc_pages(size_t size, gfp_t gfp, struct io_pgtable_cfg *cfg);
 void __arm_lpae_free_pages(void *pages, size_t size, struct io_pgtable_cfg *cfg);
 void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries,
 			 struct io_pgtable_cfg *cfg);
 int arm_lpae_mapping_exists(struct arm_lpae_io_pgtable *data);
 void arm_lpae_mapping_missing(struct arm_lpae_io_pgtable *data);

 #endif /* IO_PGTABLE_H_ */
	/* SPDX-License-Identifier: GPL-2.0-only */
	#ifndef IO_PGTABLE_H_
	#define IO_PGTABLE_H_

	#include <linux/io-pgtable.h>

	extern bool selftest_running;

	typedef u64 arm_lpae_iopte;

	struct arm_lpae_io_pgtable {
	struct io_pgtable iop;

	int pgd_bits;
	int start_level;
	int bits_per_level;

	void *pgd;

	bool idmapped; /* Used by hypervisor */
	};

	/* Struct accessors */
	#define io_pgtable_to_data(x) \
	container_of((x), struct arm_lpae_io_pgtable, iop)

	#define io_pgtable_ops_to_data(x) \
	io_pgtable_to_data(io_pgtable_ops_to_pgtable(x))

	/*
	* Calculate the right shift amount to get to the portion describing level l
	* in a virtual address mapped by the pagetable in d.
	*/
	#define ARM_LPAE_LVL_SHIFT(l,d) \
	(((ARM_LPAE_MAX_LEVELS - (l)) * (d)->bits_per_level) + \
	ilog2(sizeof(arm_lpae_iopte)))

	#define ARM_LPAE_GRANULE(d) \
	(sizeof(arm_lpae_iopte) << (d)->bits_per_level)
	#define ARM_LPAE_PGD_SIZE(d) \
	(sizeof(arm_lpae_iopte) << (d)->pgd_bits)

	#define ARM_LPAE_PTES_PER_TABLE(d) \
	(ARM_LPAE_GRANULE(d) >> ilog2(sizeof(arm_lpae_iopte)))

	/*
	* Calculate the index at level l used to map virtual address a using the
	* pagetable in d.
	*/
	#define ARM_LPAE_PGD_IDX(l,d) \
	((l) == (d)->start_level ? (d)->pgd_bits - (d)->bits_per_level : 0)

	#define ARM_LPAE_LVL_IDX(a,l,d) \
	(((u64)(a) >> ARM_LPAE_LVL_SHIFT(l,d)) & \
	((1 << ((d)->bits_per_level + ARM_LPAE_PGD_IDX(l,d))) - 1))

	/* Calculate the block/page mapping size at level l for pagetable in d. */
	#define ARM_LPAE_BLOCK_SIZE(l,d) (1ULL << ARM_LPAE_LVL_SHIFT(l,d))

	/* Page table bits */
	#define ARM_LPAE_PTE_TYPE_SHIFT 0
	#define ARM_LPAE_PTE_TYPE_MASK 0x3

	#define ARM_LPAE_PTE_TYPE_BLOCK 1
	#define ARM_LPAE_PTE_TYPE_TABLE 3
	#define ARM_LPAE_PTE_TYPE_PAGE 3

	#define ARM_LPAE_PTE_ADDR_MASK GENMASK_ULL(47,12)

	#define ARM_LPAE_PTE_NSTABLE (((arm_lpae_iopte)1) << 63)
	#define ARM_LPAE_PTE_XN (((arm_lpae_iopte)3) << 53)
	#define ARM_LPAE_PTE_AF (((arm_lpae_iopte)1) << 10)
	#define ARM_LPAE_PTE_SH_NS (((arm_lpae_iopte)0) << 8)
	#define ARM_LPAE_PTE_SH_OS (((arm_lpae_iopte)2) << 8)
	#define ARM_LPAE_PTE_SH_IS (((arm_lpae_iopte)3) << 8)
	#define ARM_LPAE_PTE_NS (((arm_lpae_iopte)1) << 5)
	#define ARM_LPAE_PTE_VALID (((arm_lpae_iopte)1) << 0)

	#define ARM_LPAE_PTE_ATTR_LO_MASK (((arm_lpae_iopte)0x3ff) << 2)
	/* Ignore the contiguous bit for block splitting */
	#define ARM_LPAE_PTE_ATTR_HI_MASK (((arm_lpae_iopte)6) << 52)
	#define ARM_LPAE_PTE_ATTR_MASK (ARM_LPAE_PTE_ATTR_LO_MASK \| \
	ARM_LPAE_PTE_ATTR_HI_MASK)
	/* Software bit for solving coherency races */
	#define ARM_LPAE_PTE_SW_SYNC (((arm_lpae_iopte)1) << 55)

	/* Stage-1 PTE */
	#define ARM_LPAE_PTE_AP_UNPRIV (((arm_lpae_iopte)1) << 6)
	#define ARM_LPAE_PTE_AP_RDONLY (((arm_lpae_iopte)2) << 6)
	#define ARM_LPAE_PTE_ATTRINDX_SHIFT 2
	#define ARM_LPAE_PTE_nG (((arm_lpae_iopte)1) << 11)

	/* Stage-2 PTE */
	#define ARM_LPAE_PTE_HAP_FAULT (((arm_lpae_iopte)0) << 6)
	#define ARM_LPAE_PTE_HAP_READ (((arm_lpae_iopte)1) << 6)
	#define ARM_LPAE_PTE_HAP_WRITE (((arm_lpae_iopte)2) << 6)
	#define ARM_LPAE_PTE_MEMATTR_OIWB (((arm_lpae_iopte)0xf) << 2)
	#define ARM_LPAE_PTE_MEMATTR_NC (((arm_lpae_iopte)0x5) << 2)
	#define ARM_LPAE_PTE_MEMATTR_DEV (((arm_lpae_iopte)0x1) << 2)

	/* Register bits */
	#define ARM_LPAE_VTCR_SL0_MASK 0x3

	#define ARM_LPAE_TCR_T0SZ_SHIFT 0

	#define ARM_LPAE_TCR_TG0_4K 0
	#define ARM_LPAE_TCR_TG0_64K 1
	#define ARM_LPAE_TCR_TG0_16K 2

	#define ARM_LPAE_TCR_TG1_16K 1
	#define ARM_LPAE_TCR_TG1_4K 2
	#define ARM_LPAE_TCR_TG1_64K 3

	#define ARM_LPAE_TCR_SH_NS 0
	#define ARM_LPAE_TCR_SH_OS 2
	#define ARM_LPAE_TCR_SH_IS 3

	#define ARM_LPAE_TCR_RGN_NC 0
	#define ARM_LPAE_TCR_RGN_WBWA 1
	#define ARM_LPAE_TCR_RGN_WT 2
	#define ARM_LPAE_TCR_RGN_WB 3

	#define ARM_LPAE_TCR_PS_32_BIT 0x0ULL
	#define ARM_LPAE_TCR_PS_36_BIT 0x1ULL
	#define ARM_LPAE_TCR_PS_40_BIT 0x2ULL
	#define ARM_LPAE_TCR_PS_42_BIT 0x3ULL
	#define ARM_LPAE_TCR_PS_44_BIT 0x4ULL
	#define ARM_LPAE_TCR_PS_48_BIT 0x5ULL
	#define ARM_LPAE_TCR_PS_52_BIT 0x6ULL

	#define ARM_LPAE_VTCR_PS_SHIFT 16
	#define ARM_LPAE_VTCR_PS_MASK 0x7

	#define ARM_LPAE_MAIR_ATTR_SHIFT(n) ((n) << 3)
	#define ARM_LPAE_MAIR_ATTR_MASK 0xff
	#define ARM_LPAE_MAIR_ATTR_DEVICE 0x04
	#define ARM_LPAE_MAIR_ATTR_NC 0x44
	#define ARM_LPAE_MAIR_ATTR_INC_OWBRWA 0xf4
	#define ARM_LPAE_MAIR_ATTR_WBRWA 0xff
	#define ARM_LPAE_MAIR_ATTR_IDX_NC 0
	#define ARM_LPAE_MAIR_ATTR_IDX_CACHE 1
	#define ARM_LPAE_MAIR_ATTR_IDX_DEV 2
	#define ARM_LPAE_MAIR_ATTR_IDX_INC_OCACHE 3

	#define ARM_MALI_LPAE_TTBR_ADRMODE_TABLE (3u << 0)
	#define ARM_MALI_LPAE_TTBR_READ_INNER BIT(2)
	#define ARM_MALI_LPAE_TTBR_SHARE_OUTER BIT(4)

	#define ARM_MALI_LPAE_MEMATTR_IMP_DEF 0x88ULL
	#define ARM_MALI_LPAE_MEMATTR_WRITE_ALLOC 0x8DULL

	#define ARM_LPAE_MAX_LEVELS 4

	#define iopte_type(pte) \
	(((pte) >> ARM_LPAE_PTE_TYPE_SHIFT) & ARM_LPAE_PTE_TYPE_MASK)

	#define iopte_prot(pte) ((pte) & ARM_LPAE_PTE_ATTR_MASK)

	static inline bool iopte_leaf(arm_lpae_iopte pte, int lvl,
	enum io_pgtable_fmt fmt)
	{
	if (lvl == (ARM_LPAE_MAX_LEVELS - 1) && fmt != ARM_MALI_LPAE)
	return iopte_type(pte) == ARM_LPAE_PTE_TYPE_PAGE;

	return iopte_type(pte) == ARM_LPAE_PTE_TYPE_BLOCK;
	}

	#ifdef __KVM_NVHE_HYPERVISOR__
	#include <nvhe/memory.h>
	#define __arm_lpae_virt_to_phys hyp_virt_to_phys
	#define __arm_lpae_phys_to_virt hyp_phys_to_virt
	#else
	#define __arm_lpae_virt_to_phys __pa
	#define __arm_lpae_phys_to_virt __va
	#endif

	/* Generic functions */
	void __arm_lpae_free_pgtable(struct arm_lpae_io_pgtable *data, int lvl,
	arm_lpae_iopte *ptep);

	int arm_lpae_init_pgtable(struct io_pgtable_cfg *cfg,
	struct arm_lpae_io_pgtable *data);
	int arm_lpae_init_pgtable_s1(struct io_pgtable_cfg *cfg,
	struct arm_lpae_io_pgtable *data);
	int arm_lpae_init_pgtable_s2(struct io_pgtable_cfg *cfg,
	struct arm_lpae_io_pgtable *data);

	/* Host/hyp-specific functions */
	void __arm_lpae_alloc_pages(size_t size, gfp_t gfp, struct io_pgtable_cfg cfg);
	void __arm_lpae_free_pages(void pages, size_t size, struct io_pgtable_cfg cfg);
	void __arm_lpae_sync_pte(arm_lpae_iopte *ptep, int num_entries,
	struct io_pgtable_cfg *cfg);
	int arm_lpae_mapping_exists(struct arm_lpae_io_pgtable *data);
	void arm_lpae_mapping_missing(struct arm_lpae_io_pgtable *data);

	#endif /* IO_PGTABLE_H_ */