adreno_a5xx.c

// SPDX-License-Identifier: GPL-2.0-only
/*
 * Copyright (c) 2014-2021, The Linux Foundation. All rights reserved.
 * Copyright (c) 2022-2023 Qualcomm Innovation Center, Inc. All rights reserved.
 */

#include <linux/clk/qcom.h>
#include <linux/delay.h>
#include <linux/firmware.h>
#include <linux/of.h>
#include <linux/of_device.h>
#include <linux/qcom_scm.h>
#include <linux/slab.h>

#include "adreno.h"
#include "adreno_a5xx.h"
#include "adreno_a5xx_packets.h"
#include "adreno_pm4types.h"
#include "adreno_trace.h"
#include "kgsl_trace.h"

static int critical_packet_constructed;
static unsigned int crit_pkts_dwords;

static void a5xx_irq_storm_worker(struct work_struct *work);
static int _read_fw2_block_header(struct kgsl_device *device,
		uint32_t *header, uint32_t remain,
		uint32_t id, uint32_t major, uint32_t minor);
static void a5xx_gpmu_reset(struct work_struct *work);
static int a5xx_gpmu_init(struct adreno_device *adreno_dev);

/**
 * Number of times to check if the regulator enabled before
 * giving up and returning failure.
 */
#define PWR_RETRY 100

/**
 * Number of times to check if the GPMU firmware is initialized before
 * giving up and returning failure.
 */
#define GPMU_FW_INIT_RETRY 5000

#define A530_QFPROM_RAW_PTE_ROW0_MSB 0x134
#define A530_QFPROM_RAW_PTE_ROW2_MSB 0x144

#define A5XX_INT_MASK \
	((1 << A5XX_INT_RBBM_AHB_ERROR) |		\
	 (1 << A5XX_INT_RBBM_TRANSFER_TIMEOUT) |		\
	 (1 << A5XX_INT_RBBM_ME_MS_TIMEOUT) |		\
	 (1 << A5XX_INT_RBBM_PFP_MS_TIMEOUT) |		\
	 (1 << A5XX_INT_RBBM_ETS_MS_TIMEOUT) |		\
	 (1 << A5XX_INT_RBBM_ATB_ASYNC_OVERFLOW) |		\
	 (1 << A5XX_INT_RBBM_GPC_ERROR) |		\
	 (1 << A5XX_INT_CP_HW_ERROR) |	\
	 (1 << A5XX_INT_CP_CACHE_FLUSH_TS) |		\
	 (1 << A5XX_INT_RBBM_ATB_BUS_OVERFLOW) |	\
	 (1 << A5XX_INT_MISC_HANG_DETECT) |		\
	 (1 << A5XX_INT_UCHE_OOB_ACCESS) |		\
	 (1 << A5XX_INT_UCHE_TRAP_INTR) |		\
	 (1 << A5XX_INT_CP_SW) |			\
	 (1 << A5XX_INT_GPMU_FIRMWARE) |                \
	 (1 << A5XX_INT_GPMU_VOLTAGE_DROOP))

static int a5xx_probe(struct platform_device *pdev,
	u32 chipid, const struct adreno_gpu_core *gpucore)
{
	struct adreno_device *adreno_dev;
	struct kgsl_device *device;
	int ret;

	adreno_dev = (struct adreno_device *)
		of_device_get_match_data(&pdev->dev);

	memset(adreno_dev, 0, sizeof(*adreno_dev));

	adreno_dev->gpucore = gpucore;
	adreno_dev->chipid = chipid;

	adreno_reg_offset_init(gpucore->gpudev->reg_offsets);

	adreno_dev->sptp_pc_enabled =
		ADRENO_FEATURE(adreno_dev, ADRENO_SPTP_PC);

	if (adreno_is_a540(adreno_dev))
		adreno_dev->throttling_enabled = true;

	adreno_dev->hwcg_enabled = true;
	adreno_dev->lm_enabled =
		ADRENO_FEATURE(adreno_dev, ADRENO_LM);

	/* Setup defaults that might get changed by the fuse bits */
	adreno_dev->lm_leakage = 0x4e001a;

	device = KGSL_DEVICE(adreno_dev);

	timer_setup(&device->idle_timer, kgsl_timer, 0);

	INIT_WORK(&device->idle_check_ws, kgsl_idle_check);

	ret = adreno_device_probe(pdev, adreno_dev);
	if (ret)
		return ret;

	a5xx_coresight_init(adreno_dev);

	return adreno_dispatcher_init(adreno_dev);
}

static void _do_fixup(const struct adreno_critical_fixup *fixups, int count,
		uint64_t *gpuaddrs, unsigned int *buffer)
{
	int i;

	for (i = 0; i < count; i++) {
		buffer[fixups[i].lo_offset] =
			lower_32_bits(gpuaddrs[fixups[i].buffer]) |
			fixups[i].mem_offset;

		buffer[fixups[i].hi_offset] =
			upper_32_bits(gpuaddrs[fixups[i].buffer]);
	}
}

static int a5xx_critical_packet_construct(struct adreno_device *adreno_dev)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	unsigned int *cmds;
	uint64_t gpuaddrs[4];

	adreno_dev->critpkts = kgsl_allocate_global(device,
		PAGE_SIZE * 4, 0, 0, 0, "crit_pkts");
	if (IS_ERR(adreno_dev->critpkts))
		return PTR_ERR(adreno_dev->critpkts);

	adreno_dev->critpkts_secure = kgsl_allocate_global(device,
		PAGE_SIZE, 0, KGSL_MEMFLAGS_SECURE, 0, "crit_pkts_secure");
	if (IS_ERR(adreno_dev->critpkts_secure))
		return PTR_ERR(adreno_dev->critpkts_secure);

	cmds = adreno_dev->critpkts->hostptr;

	gpuaddrs[0] = adreno_dev->critpkts_secure->gpuaddr;
	gpuaddrs[1] = adreno_dev->critpkts->gpuaddr + PAGE_SIZE;
	gpuaddrs[2] = adreno_dev->critpkts->gpuaddr + (PAGE_SIZE * 2);
	gpuaddrs[3] = adreno_dev->critpkts->gpuaddr + (PAGE_SIZE * 3);

	crit_pkts_dwords = ARRAY_SIZE(_a5xx_critical_pkts);

	memcpy(cmds, _a5xx_critical_pkts, crit_pkts_dwords << 2);

	_do_fixup(critical_pkt_fixups, ARRAY_SIZE(critical_pkt_fixups),
		gpuaddrs, cmds);

	cmds = adreno_dev->critpkts->hostptr + PAGE_SIZE;
	memcpy(cmds, _a5xx_critical_pkts_mem01,
			ARRAY_SIZE(_a5xx_critical_pkts_mem01) << 2);

	cmds = adreno_dev->critpkts->hostptr + (PAGE_SIZE * 2);
	memcpy(cmds, _a5xx_critical_pkts_mem02,
			ARRAY_SIZE(_a5xx_critical_pkts_mem02) << 2);

	cmds = adreno_dev->critpkts->hostptr + (PAGE_SIZE * 3);
	memcpy(cmds, _a5xx_critical_pkts_mem03,
			ARRAY_SIZE(_a5xx_critical_pkts_mem03) << 2);

	_do_fixup(critical_pkt_mem03_fixups,
		ARRAY_SIZE(critical_pkt_mem03_fixups), gpuaddrs, cmds);

	critical_packet_constructed = 1;

	return 0;
}

static int a5xx_microcode_read(struct adreno_device *adreno_dev);

static int a5xx_init(struct adreno_device *adreno_dev)
{
	const struct adreno_a5xx_core *a5xx_core = to_a5xx_core(adreno_dev);
	int ret;

	ret = a5xx_ringbuffer_init(adreno_dev);
	if (ret)
		return ret;

	ret = a5xx_microcode_read(adreno_dev);
	if (ret)
		return ret;

	if (a5xx_has_gpmu(adreno_dev))
		INIT_WORK(&adreno_dev->gpmu_work, a5xx_gpmu_reset);

	adreno_dev->highest_bank_bit = a5xx_core->highest_bank_bit;

	INIT_WORK(&adreno_dev->irq_storm_work, a5xx_irq_storm_worker);

	if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_CRITICAL_PACKETS))
		a5xx_critical_packet_construct(adreno_dev);

	adreno_create_profile_buffer(adreno_dev);
	a5xx_crashdump_init(adreno_dev);

	return 0;
}

static const struct {
	u32 reg;
	u32 base;
	u32 count;
} a5xx_protected_blocks[] = {
	/* RBBM */
	{  A5XX_CP_PROTECT_REG_0,     0x004, 2 },
	{  A5XX_CP_PROTECT_REG_0 + 1, 0x008, 3 },
	{  A5XX_CP_PROTECT_REG_0 + 2, 0x010, 4 },
	{  A5XX_CP_PROTECT_REG_0 + 3, 0x020, 5 },
	{  A5XX_CP_PROTECT_REG_0 + 4, 0x040, 6 },
	{  A5XX_CP_PROTECT_REG_0 + 5, 0x080, 6 },
	/* Content protection */
	{  A5XX_CP_PROTECT_REG_0 + 6, A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO, 4 },
	{  A5XX_CP_PROTECT_REG_0 + 7, A5XX_RBBM_SECVID_TRUST_CNTL, 1 },
	/* CP */
	{  A5XX_CP_PROTECT_REG_0 + 8, 0x800, 6 },
	{  A5XX_CP_PROTECT_REG_0 + 9, 0x840, 3 },
	{  A5XX_CP_PROTECT_REG_0 + 10, 0x880, 5 },
	{  A5XX_CP_PROTECT_REG_0 + 11, 0xaa0, 0 },
	/* RB */
	{  A5XX_CP_PROTECT_REG_0 + 12, 0xcc0, 0 },
	{  A5XX_CP_PROTECT_REG_0 + 13, 0xcf0, 1 },
	/* VPC */
	{  A5XX_CP_PROTECT_REG_0 + 14, 0xe68, 3 },
	{  A5XX_CP_PROTECT_REG_0 + 15, 0xe70, 4 },
	/* UCHE */
	{  A5XX_CP_PROTECT_REG_0 + 16, 0xe80, 4 },
	/* A5XX_CP_PROTECT_REG_17 will be used for SMMU */
	/* A5XX_CP_PROTECT_REG_18 - A5XX_CP_PROTECT_REG_31 are available */
};

static void _setprotectreg(struct kgsl_device *device, u32 offset,
		u32 base, u32 count)
{
	kgsl_regwrite(device, offset, 0x60000000 | (count << 24) | (base << 2));
}

static void a5xx_protect_init(struct adreno_device *adreno_dev)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	u32 reg;
	int i;

	/* enable access protection to privileged registers */
	kgsl_regwrite(device, A5XX_CP_PROTECT_CNTL, 0x00000007);

	for (i = 0; i < ARRAY_SIZE(a5xx_protected_blocks); i++) {
		reg = a5xx_protected_blocks[i].reg;

		_setprotectreg(device, reg, a5xx_protected_blocks[i].base,
			a5xx_protected_blocks[i].count);
	}

	/*
	 * For a530 and a540 the SMMU region is 0x20000 bytes long and 0x10000
	 * bytes on all other targets. The base offset for both is 0x40000.
	 * Write it to the next available slot
	 */
	if (adreno_is_a530(adreno_dev) || adreno_is_a540(adreno_dev))
		_setprotectreg(device, reg + 1, 0x40000, ilog2(0x20000));
	else
		_setprotectreg(device, reg + 1, 0x40000, ilog2(0x10000));
}

/*
 * _poll_gdsc_status() - Poll the GDSC status register
 * @adreno_dev: The adreno device pointer
 * @status_reg: Offset of the status register
 * @status_value: The expected bit value
 *
 * Poll the status register till the power-on bit is equal to the
 * expected value or the max retries are exceeded.
 */
static int _poll_gdsc_status(struct adreno_device *adreno_dev,
				unsigned int status_reg,
				unsigned int status_value)
{
	unsigned int reg, retry = PWR_RETRY;

	/* Bit 20 is the power on bit of SPTP and RAC GDSC status register */
	do {
		udelay(1);
		kgsl_regread(KGSL_DEVICE(adreno_dev), status_reg, &reg);
	} while (((reg & BIT(20)) != (status_value << 20)) && retry--);
	if ((reg & BIT(20)) != (status_value << 20))
		return -ETIMEDOUT;
	return 0;
}

static void a5xx_restore_isense_regs(struct adreno_device *adreno_dev)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	unsigned int reg, i, ramp = GPMU_ISENSE_SAVE;
	static unsigned int isense_regs[6] = {0xFFFF}, isense_reg_addr[] = {
		A5XX_GPU_CS_DECIMAL_ALIGN,
		A5XX_GPU_CS_SENSOR_PARAM_CORE_1,
		A5XX_GPU_CS_SENSOR_PARAM_CORE_2,
		A5XX_GPU_CS_SW_OV_FUSE_EN,
		A5XX_GPU_CS_ENDPOINT_CALIBRATION_DONE,
		A5XX_GPMU_TEMP_SENSOR_CONFIG};

	if (!adreno_is_a540(adreno_dev))
		return;

	/* read signature */
	kgsl_regread(device, ramp++, &reg);

	if (reg == 0xBABEFACE) {
		/* store memory locations in buffer */
		for (i = 0; i < ARRAY_SIZE(isense_regs); i++)
			kgsl_regread(device, ramp + i, isense_regs + i);

		/* clear signature */
		kgsl_regwrite(device, GPMU_ISENSE_SAVE, 0x0);
	}

	/* if we never stored memory locations - do nothing */
	if (isense_regs[0] == 0xFFFF)
		return;

	/* restore registers from memory */
	for (i = 0; i < ARRAY_SIZE(isense_reg_addr); i++)
		kgsl_regwrite(device, isense_reg_addr[i], isense_regs[i]);

}

/*
 * a5xx_regulator_enable() - Enable any necessary HW regulators
 * @adreno_dev: The adreno device pointer
 *
 * Some HW blocks may need their regulators explicitly enabled
 * on a restart.  Clocks must be on during this call.
 */
static int a5xx_regulator_enable(struct adreno_device *adreno_dev)
{
	unsigned int ret;
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);

	if (test_bit(ADRENO_DEVICE_GPU_REGULATOR_ENABLED,
			&adreno_dev->priv))
		return 0;

	if (!(adreno_is_a530(adreno_dev) || adreno_is_a540(adreno_dev))) {
		/* Halt the sp_input_clk at HM level */
		kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL, 0x00000055);
		a5xx_hwcg_set(adreno_dev, true);
		/* Turn on sp_input_clk at HM level */
		kgsl_regrmw(device, A5XX_RBBM_CLOCK_CNTL, 0xFF, 0);

		set_bit(ADRENO_DEVICE_GPU_REGULATOR_ENABLED,
			&adreno_dev->priv);
		return 0;
	}

	/*
	 * Turn on smaller power domain first to reduce voltage droop.
	 * Set the default register values; set SW_COLLAPSE to 0.
	 */
	kgsl_regwrite(device, A5XX_GPMU_RBCCU_POWER_CNTL, 0x778000);
	/* Insert a delay between RAC and SPTP GDSC to reduce voltage droop */
	udelay(3);
	ret = _poll_gdsc_status(adreno_dev, A5XX_GPMU_RBCCU_PWR_CLK_STATUS, 1);
	if (ret) {
		dev_err(device->dev, "RBCCU GDSC enable failed\n");
		return ret;
	}

	kgsl_regwrite(device, A5XX_GPMU_SP_POWER_CNTL, 0x778000);
	ret = _poll_gdsc_status(adreno_dev, A5XX_GPMU_SP_PWR_CLK_STATUS, 1);
	if (ret) {
		dev_err(device->dev, "SPTP GDSC enable failed\n");
		return ret;
	}

	/* Disable SP clock */
	kgsl_regrmw(device, A5XX_GPMU_GPMU_SP_CLOCK_CONTROL,
		CNTL_IP_CLK_ENABLE, 0);
	/* Enable hardware clockgating */
	a5xx_hwcg_set(adreno_dev, true);
	/* Enable SP clock */
	kgsl_regrmw(device, A5XX_GPMU_GPMU_SP_CLOCK_CONTROL,
		CNTL_IP_CLK_ENABLE, 1);

	a5xx_restore_isense_regs(adreno_dev);

	set_bit(ADRENO_DEVICE_GPU_REGULATOR_ENABLED, &adreno_dev->priv);
	return 0;
}

/*
 * a5xx_regulator_disable() - Disable any necessary HW regulators
 * @adreno_dev: The adreno device pointer
 *
 * Some HW blocks may need their regulators explicitly disabled
 * on a power down to prevent current spikes.  Clocks must be on
 * during this call.
 */
static void a5xx_regulator_disable(struct adreno_device *adreno_dev)
{
	unsigned int reg;
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);

	if (adreno_is_a512(adreno_dev) || adreno_is_a508(adreno_dev))
		return;

	if (!test_and_clear_bit(ADRENO_DEVICE_GPU_REGULATOR_ENABLED,
		&adreno_dev->priv))
		return;

	/* If feature is not supported or not enabled */
	if (!adreno_dev->sptp_pc_enabled) {
		/* Set the default register values; set SW_COLLAPSE to 1 */
		kgsl_regwrite(device, A5XX_GPMU_SP_POWER_CNTL, 0x778001);
		/*
		 * Insert a delay between SPTP and RAC GDSC to reduce voltage
		 * droop.
		 */
		udelay(3);
		if (_poll_gdsc_status(adreno_dev,
					A5XX_GPMU_SP_PWR_CLK_STATUS, 0))
			dev_warn(device->dev, "SPTP GDSC disable failed\n");

		kgsl_regwrite(device, A5XX_GPMU_RBCCU_POWER_CNTL, 0x778001);
		if (_poll_gdsc_status(adreno_dev,
					A5XX_GPMU_RBCCU_PWR_CLK_STATUS, 0))
			dev_warn(device->dev, "RBCCU GDSC disable failed\n");
	} else if (test_bit(ADRENO_DEVICE_GPMU_INITIALIZED,
			&adreno_dev->priv)) {
		/* GPMU firmware is supposed to turn off SPTP & RAC GDSCs. */
		kgsl_regread(device, A5XX_GPMU_SP_PWR_CLK_STATUS, &reg);
		if (reg & BIT(20))
			dev_warn(device->dev, "SPTP GDSC is not disabled\n");
		kgsl_regread(device, A5XX_GPMU_RBCCU_PWR_CLK_STATUS, &reg);
		if (reg & BIT(20))
			dev_warn(device->dev, "RBCCU GDSC is not disabled\n");
		/*
		 * GPMU firmware is supposed to set GMEM to non-retention.
		 * Bit 14 is the memory core force on bit.
		 */
		kgsl_regread(device, A5XX_GPMU_RBCCU_CLOCK_CNTL, &reg);
		if (reg & BIT(14))
			dev_warn(device->dev, "GMEM is forced on\n");
	}

	if (adreno_is_a530(adreno_dev)) {
		/* Reset VBIF before PC to avoid popping bogus FIFO entries */
		kgsl_regwrite(device, A5XX_RBBM_BLOCK_SW_RESET_CMD,
			0x003C0000);
		kgsl_regwrite(device, A5XX_RBBM_BLOCK_SW_RESET_CMD, 0);
	}
}

/*
 * a5xx_enable_pc() - Enable the GPMU based power collapse of the SPTP and RAC
 * blocks
 * @adreno_dev: The adreno device pointer
 */
static void a5xx_enable_pc(struct adreno_device *adreno_dev)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);

	if (!adreno_dev->sptp_pc_enabled)
		return;

	kgsl_regwrite(device, A5XX_GPMU_PWR_COL_INTER_FRAME_CTRL, 0x0000007F);
	kgsl_regwrite(device, A5XX_GPMU_PWR_COL_BINNING_CTRL, 0);
	kgsl_regwrite(device, A5XX_GPMU_PWR_COL_INTER_FRAME_HYST, 0x000A0080);
	kgsl_regwrite(device, A5XX_GPMU_PWR_COL_STAGGER_DELAY, 0x00600040);

	trace_adreno_sp_tp((unsigned long) __builtin_return_address(0));
};

/*
 * The maximum payload of a type4 packet is the max size minus one for the
 * opcode
 */
#define TYPE4_MAX_PAYLOAD (PM4_TYPE4_PKT_SIZE_MAX - 1)

static int _gpmu_create_load_cmds(struct adreno_device *adreno_dev,
	uint32_t *ucode, uint32_t size)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	uint32_t *start, *cmds;
	uint32_t offset = 0;
	uint32_t cmds_size = size;

	/* Add a dword for each PM4 packet */
	cmds_size += (size / TYPE4_MAX_PAYLOAD) + 1;

	/* Add 4 dwords for the protected mode */
	cmds_size += 4;

	if (adreno_dev->gpmu_cmds != NULL)
		return 0;

	adreno_dev->gpmu_cmds = devm_kmalloc(&device->pdev->dev,
		cmds_size << 2, GFP_KERNEL);
	if (adreno_dev->gpmu_cmds == NULL)
		return -ENOMEM;

	cmds = adreno_dev->gpmu_cmds;
	start = cmds;

	/* Turn CP protection OFF */
	cmds += cp_protected_mode(adreno_dev, cmds, 0);

	/*
	 * Prebuild the cmd stream to send to the GPU to load
	 * the GPMU firmware
	 */
	while (size > 0) {
		int tmp_size = size;

		if (size >= TYPE4_MAX_PAYLOAD)
			tmp_size = TYPE4_MAX_PAYLOAD;

		*cmds++ = cp_type4_packet(
				A5XX_GPMU_INST_RAM_BASE + offset,
				tmp_size);

		memcpy(cmds, &ucode[offset], tmp_size << 2);

		cmds += tmp_size;
		offset += tmp_size;
		size -= tmp_size;
	}

	/* Turn CP protection ON */
	cmds += cp_protected_mode(adreno_dev, cmds, 1);

	adreno_dev->gpmu_cmds_size = (size_t) (cmds - start);

	return 0;
}


/*
 * _load_gpmu_firmware() - Load the ucode into the GPMU RAM
 * @adreno_dev: Pointer to adreno device
 */
static int _load_gpmu_firmware(struct adreno_device *adreno_dev)
{
	uint32_t *data;
	const struct firmware *fw = NULL;
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	const struct adreno_a5xx_core *a5xx_core = to_a5xx_core(adreno_dev);
	uint32_t *cmds, cmd_size;
	int ret =  -EINVAL;
	u32 gmu_major = 1;

	if (!a5xx_has_gpmu(adreno_dev))
		return 0;

	/* a530 used GMU major 1 and A540 used GMU major 3 */
	if (adreno_is_a540(adreno_dev))
		gmu_major = 3;

	/* gpmu fw already saved and verified so do nothing new */
	if (adreno_dev->gpmu_cmds_size != 0)
		return 0;

	if (a5xx_core->gpmufw_name == NULL)
		return 0;

	ret = request_firmware(&fw, a5xx_core->gpmufw_name, &device->pdev->dev);
	if (ret || fw == NULL) {
		dev_err(&device->pdev->dev,
			"request_firmware (%s) failed: %d\n",
			a5xx_core->gpmufw_name, ret);
		return ret;
	}

	data = (uint32_t *)fw->data;

	if (data[0] >= (fw->size / sizeof(uint32_t)) || data[0] < 2)
		goto err;

	if (data[1] != GPMU_FIRMWARE_ID)
		goto err;
	ret = _read_fw2_block_header(device, &data[2],
		data[0] - 2, GPMU_FIRMWARE_ID, gmu_major, 0);
	if (ret)
		goto err;

	/* Integer overflow check for cmd_size */
	if (data[2] > (data[0] - 2))
		goto err;

	cmds = data + data[2] + 3;
	cmd_size = data[0] - data[2] - 2;

	if (cmd_size > GPMU_INST_RAM_SIZE) {
		dev_err(device->dev,
			"GPMU firmware block size is larger than RAM size\n");
		goto err;
	}

	/* Everything is cool, so create some commands */
	ret = _gpmu_create_load_cmds(adreno_dev, cmds, cmd_size);
err:
	if (fw)
		release_firmware(fw);

	return ret;
}

static void a5xx_spin_idle_debug(struct adreno_device *adreno_dev,
				const char *str)
{
	struct kgsl_device *device = &adreno_dev->dev;
	unsigned int rptr, wptr;
	unsigned int status, status3, intstatus;
	unsigned int hwfault;

	dev_err(device->dev, str);

	kgsl_regread(device, A5XX_CP_RB_RPTR, &rptr);
	kgsl_regread(device, A5XX_CP_RB_WPTR, &wptr);

	kgsl_regread(device, A5XX_RBBM_STATUS, &status);
	kgsl_regread(device, A5XX_RBBM_STATUS3, &status3);
	kgsl_regread(device, A5XX_RBBM_INT_0_STATUS, &intstatus);
	kgsl_regread(device, A5XX_CP_HW_FAULT, &hwfault);


	dev_err(device->dev,
		"rb=%d pos=%X/%X rbbm_status=%8.8X/%8.8X int_0_status=%8.8X\n",
		adreno_dev->cur_rb->id, rptr, wptr, status, status3, intstatus);

	dev_err(device->dev, " hwfault=%8.8X\n", hwfault);

	kgsl_device_snapshot(device, NULL, NULL, false);
}

static int _gpmu_send_init_cmds(struct adreno_device *adreno_dev)
{
	struct adreno_ringbuffer *rb = adreno_dev->cur_rb;
	uint32_t *cmds;
	uint32_t size = adreno_dev->gpmu_cmds_size;
	int ret;

	if (size == 0 || adreno_dev->gpmu_cmds == NULL)
		return -EINVAL;

	cmds = adreno_ringbuffer_allocspace(rb, size);
	if (IS_ERR(cmds))
		return PTR_ERR(cmds);
	if (cmds == NULL)
		return -ENOSPC;

	/* Copy to the RB the predefined fw sequence cmds */
	memcpy(cmds, adreno_dev->gpmu_cmds, size << 2);

	ret = a5xx_ringbuffer_submit(rb, NULL, true);
	if (!ret) {
		ret = adreno_spin_idle(adreno_dev, 2000);
		if (ret)
			a5xx_spin_idle_debug(adreno_dev,
				"gpmu initialization failed to idle\n");
	}
	return ret;
}

/*
 * a5xx_gpmu_start() - Initialize and start the GPMU
 * @adreno_dev: Pointer to adreno device
 *
 * Load the GPMU microcode, set up any features such as hardware clock gating
 * or IFPC, and take the GPMU out of reset.
 */
static int a5xx_gpmu_start(struct adreno_device *adreno_dev)
{
	int ret;
	unsigned int reg, retry = GPMU_FW_INIT_RETRY;
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);

	if (!a5xx_has_gpmu(adreno_dev))
		return 0;

	ret = _gpmu_send_init_cmds(adreno_dev);
	if (ret)
		return ret;

	if (adreno_is_a530(adreno_dev)) {
		/* GPMU clock gating setup */
		kgsl_regwrite(device, A5XX_GPMU_WFI_CONFIG, 0x00004014);
	}
	/* Kick off GPMU firmware */
	kgsl_regwrite(device, A5XX_GPMU_CM3_SYSRESET, 0);
	/*
	 * The hardware team's estimation of GPMU firmware initialization
	 * latency is about 3000 cycles, that's about 5 to 24 usec.
	 */
	do {
		udelay(1);
		kgsl_regread(device, A5XX_GPMU_GENERAL_0, &reg);
	} while ((reg != 0xBABEFACE) && retry--);

	if (reg != 0xBABEFACE) {
		dev_err(device->dev,
			"GPMU firmware initialization timed out\n");
		return -ETIMEDOUT;
	}

	if (!adreno_is_a530(adreno_dev)) {
		kgsl_regread(device, A5XX_GPMU_GENERAL_1, &reg);

		if (reg) {
			dev_err(device->dev,
				"GPMU firmware initialization failed: %d\n",
				reg);
			return -EIO;
		}
	}
	set_bit(ADRENO_DEVICE_GPMU_INITIALIZED, &adreno_dev->priv);
	/*
	 *  We are in AWARE state and IRQ line from GPU to host is
	 *  disabled.
	 *  Read pending GPMU interrupts and clear GPMU_RBBM_INTR_INFO.
	 */
	kgsl_regread(device, A5XX_GPMU_RBBM_INTR_INFO, &reg);
	/*
	 * Clear RBBM interrupt mask if any of GPMU interrupts
	 * are pending.
	 */
	if (reg)
		kgsl_regwrite(device,
			A5XX_RBBM_INT_CLEAR_CMD,
			1 << A5XX_INT_GPMU_FIRMWARE);
	return ret;
}

void a5xx_hwcg_set(struct adreno_device *adreno_dev, bool on)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	const struct adreno_a5xx_core *a5xx_core = to_a5xx_core(adreno_dev);
	int i;

	if (!adreno_dev->hwcg_enabled)
		return;

	for (i = 0; i < a5xx_core->hwcg_count; i++)
		kgsl_regwrite(device, a5xx_core->hwcg[i].offset,
			on ? a5xx_core->hwcg[i].val : 0);

	/* enable top level HWCG */
	kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL, on ? 0xAAA8AA00 : 0);
	kgsl_regwrite(device, A5XX_RBBM_ISDB_CNT, on ? 0x00000182 : 0x00000180);
}

static int _read_fw2_block_header(struct kgsl_device *device,
		uint32_t *header, uint32_t remain,
		uint32_t id, uint32_t major, uint32_t minor)
{
	uint32_t header_size;
	int i = 1;

	if (header == NULL)
		return -ENOMEM;

	header_size = header[0];
	/* Headers have limited size and always occur as pairs of words */
	if (header_size >  MAX_HEADER_SIZE || header_size >= remain ||
				header_size % 2 || header_size == 0)
		return -EINVAL;
	/* Sequences must have an identifying id first thing in their header */
	if (id == GPMU_SEQUENCE_ID) {
		if (header[i] != HEADER_SEQUENCE ||
			(header[i + 1] >= MAX_SEQUENCE_ID))
			return -EINVAL;
		i += 2;
	}
	for (; i < header_size; i += 2) {
		switch (header[i]) {
		/* Major Version */
		case HEADER_MAJOR:
			if ((major > header[i + 1]) &&
				header[i + 1]) {
				dev_err(device->dev,
					"GPMU major version mis-match %d, %d\n",
					major, header[i + 1]);
				return -EINVAL;
			}
			break;
		case HEADER_MINOR:
			if (minor > header[i + 1])
				dev_err(device->dev,
					"GPMU minor version mis-match %d %d\n",
					minor, header[i + 1]);
			break;
		case HEADER_DATE:
		case HEADER_TIME:
			break;
		default:
			dev_err(device->dev, "GPMU unknown header ID %d\n",
					header[i]);
		}
	}
	return 0;
}

/*
 * Read in the register sequence file and save pointers to the
 * necessary sequences.
 *
 * GPU sequence file format (one dword per field unless noted):
 * Block 1 length (length dword field not inclusive)
 * Block 1 type = Sequence = 3
 * Block Header length (length dword field not inclusive)
 * BH field ID = Sequence field ID
 * BH field data = Sequence ID
 * BH field ID
 * BH field data
 * ...
 * Opcode 0 ID
 * Opcode 0 data M words
 * Opcode 1 ID
 * Opcode 1 data N words
 * ...
 * Opcode X ID
 * Opcode X data O words
 * Block 2 length...
 */
static void _load_regfile(struct adreno_device *adreno_dev)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	const struct adreno_a5xx_core *a5xx_core = to_a5xx_core(adreno_dev);
	const struct firmware *fw;
	uint64_t block_size = 0, block_total = 0;
	uint32_t fw_size, *block;
	int ret = -EINVAL;
	u32 lm_major = 1;

	if (!a5xx_core->regfw_name)
		return;

	ret = request_firmware(&fw, a5xx_core->regfw_name, &device->pdev->dev);
	if (ret) {
		dev_err(&device->pdev->dev, "request firmware failed %d, %s\n",
				ret, a5xx_core->regfw_name);
		return;
	}

	/* a530v2 lm_major was 3. a530v3 lm_major was 1 */
	if (adreno_is_a530v2(adreno_dev))
		lm_major = 3;

	fw_size = fw->size / sizeof(uint32_t);
	/* Min valid file of size 6, see file description */
	if (fw_size < 6)
		goto err;
	block = (uint32_t *)fw->data;
	/* All offset numbers calculated from file description */
	while (block_total < fw_size) {
		block_size = block[0];
		if (((block_total + block_size) >= fw_size)
				|| block_size < 5)
			goto err;
		if (block[1] != GPMU_SEQUENCE_ID)
			goto err;

		/* For now ignore blocks other than the LM sequence */
		if (block[4] == LM_SEQUENCE_ID) {
			ret = _read_fw2_block_header(device, &block[2],
				block_size - 2, GPMU_SEQUENCE_ID,
				lm_major, 0);
			if (ret)
				goto err;

			if (block[2] > (block_size - 2))
				goto err;
			adreno_dev->lm_sequence = block + block[2] + 3;
			adreno_dev->lm_size = block_size - block[2] - 2;
		}
		block_total += (block_size + 1);
		block += (block_size + 1);
	}
	if (adreno_dev->lm_sequence)
		return;

err:
	release_firmware(fw);
	dev_err(device->dev,
		     "Register file failed to load sz=%d bsz=%llu header=%d\n",
		     fw_size, block_size, ret);
}

static int _execute_reg_sequence(struct adreno_device *adreno_dev,
			uint32_t *opcode, uint32_t length)
{
	uint32_t *cur = opcode;
	uint64_t reg, val;

	/* todo double check the reg writes */
	while ((cur - opcode) < length) {
		if (cur[0] == 1 && (length - (cur - opcode) >= 4)) {
			/* Write a 32 bit value to a 64 bit reg */
			reg = cur[2];
			reg = (reg << 32) | cur[1];
			kgsl_regwrite(KGSL_DEVICE(adreno_dev), reg, cur[3]);
			cur += 4;
		} else if (cur[0] == 2 && (length - (cur - opcode) >= 5)) {
			/* Write a 64 bit value to a 64 bit reg */
			reg = cur[2];
			reg = (reg << 32) | cur[1];
			val = cur[4];
			val = (val << 32) | cur[3];
			kgsl_regwrite(KGSL_DEVICE(adreno_dev), reg, val);
			cur += 5;
		} else if (cur[0] == 3 && (length - (cur - opcode) >= 2)) {
			/* Delay for X usec */
			udelay(cur[1]);
			cur += 2;
		} else
			return -EINVAL;
	}
	return 0;
}

static uint32_t _write_voltage_table(struct adreno_device *adreno_dev,
			unsigned int addr)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	struct kgsl_pwrctrl *pwr = &device->pwrctrl;
	const struct adreno_a5xx_core *a5xx_core = to_a5xx_core(adreno_dev);
	int i;
	struct dev_pm_opp *opp;
	unsigned int mvolt = 0;

	kgsl_regwrite(device, addr++, a5xx_core->max_power);
	kgsl_regwrite(device, addr++, pwr->num_pwrlevels);

	/* Write voltage in mV and frequency in MHz */
	for (i = 0; i < pwr->num_pwrlevels; i++) {
		opp = dev_pm_opp_find_freq_exact(&device->pdev->dev,
				pwr->pwrlevels[i].gpu_freq, true);
		/* _opp_get returns uV, convert to mV */
		if (!IS_ERR(opp)) {
			mvolt = dev_pm_opp_get_voltage(opp) / 1000;
			dev_pm_opp_put(opp);
		}
		kgsl_regwrite(device, addr++, mvolt);
		kgsl_regwrite(device, addr++,
				pwr->pwrlevels[i].gpu_freq / 1000000);
	}
	return (pwr->num_pwrlevels * 2 + 2);
}

static uint32_t lm_limit(struct adreno_device *adreno_dev)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);

	if (adreno_dev->lm_limit)
		return adreno_dev->lm_limit;

	if (of_property_read_u32(device->pdev->dev.of_node, "qcom,lm-limit",
		&adreno_dev->lm_limit))
		adreno_dev->lm_limit = LM_DEFAULT_LIMIT;

	return adreno_dev->lm_limit;
}
/*
 * a5xx_lm_init() - Initialize LM/DPM on the GPMU
 * @adreno_dev: The adreno device pointer
 */
static void a530_lm_init(struct adreno_device *adreno_dev)
{
	uint32_t length;
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	const struct adreno_a5xx_core *a5xx_core = to_a5xx_core(adreno_dev);

	if (!adreno_dev->lm_enabled)
		return;

	/* If something was wrong with the sequence file, return */
	if (adreno_dev->lm_sequence == NULL)
		return;

	/* Write LM registers including DPM ucode, coefficients, and config */
	if (_execute_reg_sequence(adreno_dev, adreno_dev->lm_sequence,
				adreno_dev->lm_size)) {
		/* If the sequence is invalid, it's not getting better */
		adreno_dev->lm_sequence = NULL;
		dev_warn(device->dev,
				"Invalid LM sequence\n");
		return;
	}

	kgsl_regwrite(device, A5XX_GPMU_TEMP_SENSOR_ID, a5xx_core->gpmu_tsens);
	kgsl_regwrite(device, A5XX_GPMU_DELTA_TEMP_THRESHOLD, 0x1);
	kgsl_regwrite(device, A5XX_GPMU_TEMP_SENSOR_CONFIG, 0x1);

	kgsl_regwrite(device, A5XX_GPMU_GPMU_VOLTAGE,
			(0x80000000 | device->pwrctrl.active_pwrlevel));
	/* use the leakage to set this value at runtime */
	kgsl_regwrite(device, A5XX_GPMU_BASE_LEAKAGE,
		adreno_dev->lm_leakage);

	/* Enable the power threshold and set it to 6000m */
	kgsl_regwrite(device, A5XX_GPMU_GPMU_PWR_THRESHOLD,
		0x80000000 | lm_limit(adreno_dev));

	kgsl_regwrite(device, A5XX_GPMU_BEC_ENABLE, 0x10001FFF);
	kgsl_regwrite(device, A5XX_GDPM_CONFIG1, 0x00201FF1);

	/* Send an initial message to the GPMU with the LM voltage table */
	kgsl_regwrite(device, AGC_MSG_STATE, 1);
	kgsl_regwrite(device, AGC_MSG_COMMAND, AGC_POWER_CONFIG_PRODUCTION_ID);
	length = _write_voltage_table(adreno_dev, AGC_MSG_PAYLOAD);
	kgsl_regwrite(device, AGC_MSG_PAYLOAD_SIZE, length * sizeof(uint32_t));
	kgsl_regwrite(device, AGC_INIT_MSG_MAGIC, AGC_INIT_MSG_VALUE);
}

/*
 * a5xx_lm_enable() - Enable the LM/DPM feature on the GPMU
 * @adreno_dev: The adreno device pointer
 */
static void a530_lm_enable(struct adreno_device *adreno_dev)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);

	if (!adreno_dev->lm_enabled)
		return;

	/* If no sequence properly initialized, return */
	if (adreno_dev->lm_sequence == NULL)
		return;

	kgsl_regwrite(device, A5XX_GDPM_INT_MASK, 0x00000000);
	kgsl_regwrite(device, A5XX_GDPM_INT_EN, 0x0000000A);
	kgsl_regwrite(device, A5XX_GPMU_GPMU_VOLTAGE_INTR_EN_MASK, 0x00000001);
	kgsl_regwrite(device, A5XX_GPMU_TEMP_THRESHOLD_INTR_EN_MASK,
			0x00050000);
	kgsl_regwrite(device, A5XX_GPMU_THROTTLE_UNMASK_FORCE_CTRL,
			0x00030000);

	if (adreno_is_a530(adreno_dev))
		/* Program throttle control, do not enable idle DCS on v3+ */
		kgsl_regwrite(device, A5XX_GPMU_CLOCK_THROTTLE_CTRL,
			adreno_is_a530v2(adreno_dev) ? 0x00060011 : 0x00000011);
}

static void a540_lm_init(struct adreno_device *adreno_dev)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	uint32_t agc_lm_config = AGC_BCL_DISABLED |
		((ADRENO_CHIPID_PATCH(adreno_dev->chipid) & 0x3)
		<< AGC_GPU_VERSION_SHIFT);
	unsigned int r;

	if (!adreno_dev->throttling_enabled)
		agc_lm_config |= AGC_THROTTLE_DISABLE;

	if (adreno_dev->lm_enabled) {
		agc_lm_config |=
			AGC_LM_CONFIG_ENABLE_GPMU_ADAPTIVE |
			AGC_LM_CONFIG_ISENSE_ENABLE;

		kgsl_regread(device, A5XX_GPMU_TEMP_SENSOR_CONFIG, &r);

		if ((r & GPMU_ISENSE_STATUS) == GPMU_ISENSE_END_POINT_CAL_ERR) {
			dev_err(device->dev,
				"GPMU: ISENSE end point calibration failure\n");
			agc_lm_config |= AGC_LM_CONFIG_ENABLE_ERROR;
		}
	}

	kgsl_regwrite(device, AGC_MSG_STATE, 0x80000001);
	kgsl_regwrite(device, AGC_MSG_COMMAND, AGC_POWER_CONFIG_PRODUCTION_ID);
	(void) _write_voltage_table(adreno_dev, AGC_MSG_PAYLOAD);
	kgsl_regwrite(device, AGC_MSG_PAYLOAD + AGC_LM_CONFIG, agc_lm_config);
	kgsl_regwrite(device, AGC_MSG_PAYLOAD + AGC_LEVEL_CONFIG,
		(unsigned int) ~(GENMASK(LM_DCVS_LIMIT, 0) |
				GENMASK(16+LM_DCVS_LIMIT, 16)));

	kgsl_regwrite(device, AGC_MSG_PAYLOAD_SIZE,
		(AGC_LEVEL_CONFIG + 1) * sizeof(uint32_t));
	kgsl_regwrite(device, AGC_INIT_MSG_MAGIC, AGC_INIT_MSG_VALUE);

	kgsl_regwrite(device, A5XX_GPMU_GPMU_VOLTAGE,
		(0x80000000 | device->pwrctrl.active_pwrlevel));

	kgsl_regwrite(device, A5XX_GPMU_GPMU_PWR_THRESHOLD,
		PWR_THRESHOLD_VALID | lm_limit(adreno_dev));

	kgsl_regwrite(device, A5XX_GPMU_GPMU_VOLTAGE_INTR_EN_MASK,
		VOLTAGE_INTR_EN);
}


static void a5xx_lm_enable(struct adreno_device *adreno_dev)
{
	if (adreno_is_a530(adreno_dev))
		a530_lm_enable(adreno_dev);
}

static void a5xx_lm_init(struct adreno_device *adreno_dev)
{
	if (adreno_is_a530(adreno_dev))
		a530_lm_init(adreno_dev);
	else if (adreno_is_a540(adreno_dev))
		a540_lm_init(adreno_dev);
}

static int gpmu_set_level(struct adreno_device *adreno_dev, unsigned int val)
{
	unsigned int reg;
	int retry = 100;

	kgsl_regwrite(KGSL_DEVICE(adreno_dev), A5XX_GPMU_GPMU_VOLTAGE, val);

	do {
		kgsl_regread(KGSL_DEVICE(adreno_dev), A5XX_GPMU_GPMU_VOLTAGE,
			&reg);
	} while ((reg & 0x80000000) && retry--);

	return (reg & 0x80000000) ? -ETIMEDOUT : 0;
}

/*
 * a5xx_pwrlevel_change_settings() - Program the hardware during power level
 * transitions
 * @adreno_dev: The adreno device pointer
 * @prelevel: The previous power level
 * @postlevel: The new power level
 * @post: True if called after the clock change has taken effect
 */
static void a5xx_pwrlevel_change_settings(struct adreno_device *adreno_dev,
				unsigned int prelevel, unsigned int postlevel,
				bool post)
{
	/*
	 * On pre A540 HW only call through if LMx is supported and enabled, and
	 * always call through for a540
	 */
	if (!adreno_is_a540(adreno_dev) && !adreno_dev->lm_enabled)
		return;

	if (!post) {
		if (gpmu_set_level(adreno_dev, (0x80000010 | postlevel)))
			dev_err(KGSL_DEVICE(adreno_dev)->dev,
				"GPMU pre powerlevel did not stabilize\n");
	} else {
		if (gpmu_set_level(adreno_dev, (0x80000000 | postlevel)))
			dev_err(KGSL_DEVICE(adreno_dev)->dev,
				"GPMU post powerlevel did not stabilize\n");
	}
}

/* FW driven idle 10% throttle */
#define IDLE_10PCT 0
/* number of cycles when clock is throttled by 50% (CRC) */
#define CRC_50PCT  1
/* number of cycles when clock is throttled by more than 50% (CRC) */
#define CRC_MORE50PCT 2
/* number of cycles when clock is throttle by less than 50% (CRC) */
#define CRC_LESS50PCT 3

static int64_t a5xx_read_throttling_counters(struct adreno_device *adreno_dev)
{
	int i;
	int64_t adj;
	uint32_t th[ADRENO_GPMU_THROTTLE_COUNTERS];
	struct adreno_busy_data *busy = &adreno_dev->busy_data;

	if (!adreno_dev->throttling_enabled)
		return 0;

	for (i = 0; i < ADRENO_GPMU_THROTTLE_COUNTERS; i++) {
		if (!adreno_dev->gpmu_throttle_counters[i])
			return 0;

		th[i] = counter_delta(KGSL_DEVICE(adreno_dev),
				adreno_dev->gpmu_throttle_counters[i],
				&busy->throttle_cycles[i]);
	}
	adj = th[CRC_MORE50PCT] - th[IDLE_10PCT];
	adj = th[CRC_50PCT] + th[CRC_LESS50PCT] / 3 + (adj < 0 ? 0 : adj) * 3;

	trace_kgsl_clock_throttling(
		th[IDLE_10PCT], th[CRC_50PCT],
		th[CRC_MORE50PCT], th[CRC_LESS50PCT],
		adj);
	return adj;
}

/*
 * a5xx_gpmu_reset() - Re-enable GPMU based power features and restart GPMU
 * @work: Pointer to the work struct for gpmu reset
 *
 * Load the GPMU microcode, set up any features such as hardware clock gating
 * or IFPC, and take the GPMU out of reset.
 */
static void a5xx_gpmu_reset(struct work_struct *work)
{
	struct adreno_device *adreno_dev = container_of(work,
			struct adreno_device, gpmu_work);
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);

	if (test_bit(ADRENO_DEVICE_GPMU_INITIALIZED, &adreno_dev->priv))
		return;

	/*
	 * If GPMU has already experienced a restart or is in the process of it
	 * after the watchdog timeout, then there is no need to reset GPMU
	 * again.
	 */
	if (device->state != KGSL_STATE_AWARE && device->state != KGSL_STATE_ACTIVE)
		return;

	mutex_lock(&device->mutex);

	if (a5xx_regulator_enable(adreno_dev))
		goto out;

	/* Soft reset of the GPMU block */
	kgsl_regwrite(device, A5XX_RBBM_BLOCK_SW_RESET_CMD, BIT(16));

	/* GPU comes up in secured mode, make it unsecured by default */
	if (!ADRENO_FEATURE(adreno_dev, ADRENO_CONTENT_PROTECTION))
		kgsl_regwrite(device, A5XX_RBBM_SECVID_TRUST_CNTL, 0x0);


	a5xx_gpmu_init(adreno_dev);

out:
	mutex_unlock(&device->mutex);
}

static void _setup_throttling_counters(struct adreno_device *adreno_dev)
{
	int i, ret = 0;

	if (!adreno_is_a540(adreno_dev))
		return;

	for (i = 0; i < ADRENO_GPMU_THROTTLE_COUNTERS; i++) {
		/* reset throttled cycles ivalue */
		adreno_dev->busy_data.throttle_cycles[i] = 0;

		/* Throttle countables start at off set 43 */
		ret |= adreno_perfcounter_kernel_get(adreno_dev,
			KGSL_PERFCOUNTER_GROUP_GPMU_PWR, 43 + i,
			&adreno_dev->gpmu_throttle_counters[i], NULL);
	}

	WARN_ONCE(ret, "Unable to get one or more clock throttling registers\n");
}

/*
 * a5xx_start() - Device start
 * @adreno_dev: Pointer to adreno device
 *
 * a5xx device start
 */
static int a5xx_start(struct adreno_device *adreno_dev)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	const struct adreno_a5xx_core *a5xx_core = to_a5xx_core(adreno_dev);
	unsigned int bit;
	int ret;

	ret = kgsl_mmu_start(device);
	if (ret)
		return ret;

	adreno_get_bus_counters(adreno_dev);
	adreno_perfcounter_restore(adreno_dev);

	adreno_dev->irq_mask = A5XX_INT_MASK;

	if (adreno_is_a530(adreno_dev) &&
			ADRENO_FEATURE(adreno_dev, ADRENO_LM))
		adreno_perfcounter_kernel_get(adreno_dev,
			KGSL_PERFCOUNTER_GROUP_GPMU_PWR, 27,
			&adreno_dev->lm_threshold_count, NULL);

	/* Enable 64 bit addressing */
	kgsl_regwrite(device, A5XX_CP_ADDR_MODE_CNTL, 0x1);
	kgsl_regwrite(device, A5XX_VSC_ADDR_MODE_CNTL, 0x1);
	kgsl_regwrite(device, A5XX_GRAS_ADDR_MODE_CNTL, 0x1);
	kgsl_regwrite(device, A5XX_RB_ADDR_MODE_CNTL, 0x1);
	kgsl_regwrite(device, A5XX_PC_ADDR_MODE_CNTL, 0x1);
	kgsl_regwrite(device, A5XX_HLSQ_ADDR_MODE_CNTL, 0x1);
	kgsl_regwrite(device, A5XX_VFD_ADDR_MODE_CNTL, 0x1);
	kgsl_regwrite(device, A5XX_VPC_ADDR_MODE_CNTL, 0x1);
	kgsl_regwrite(device, A5XX_UCHE_ADDR_MODE_CNTL, 0x1);
	kgsl_regwrite(device, A5XX_SP_ADDR_MODE_CNTL, 0x1);
	kgsl_regwrite(device, A5XX_TPL1_ADDR_MODE_CNTL, 0x1);
	kgsl_regwrite(device, A5XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL, 0x1);

	_setup_throttling_counters(adreno_dev);

	/* Set up VBIF registers from the GPU core definition */
	kgsl_regmap_multi_write(&device->regmap, a5xx_core->vbif,
		a5xx_core->vbif_count);

	/* Make all blocks contribute to the GPU BUSY perf counter */
	kgsl_regwrite(device, A5XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xFFFFFFFF);

	/* Program RBBM counter 0 to report GPU busy for frequency scaling */
	kgsl_regwrite(device, A5XX_RBBM_PERFCTR_RBBM_SEL_0, 6);

	/*
	 * Enable the RBBM error reporting bits.  This lets us get
	 * useful information on failure
	 */
	kgsl_regwrite(device, A5XX_RBBM_AHB_CNTL0, 0x00000001);

	if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_FAULT_DETECT_MASK)) {
		/*
		 * We have 4 RB units, and only RB0 activity signals are
		 * working correctly. Mask out RB1-3 activity signals
		 * from the HW hang detection logic as per
		 * recommendation of hardware team.
		 */
		kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL11,
				0xF0000000);
		kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL12,
				0xFFFFFFFF);
		kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL13,
				0xFFFFFFFF);
		kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL14,
				0xFFFFFFFF);
		kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL15,
				0xFFFFFFFF);
		kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL16,
				0xFFFFFFFF);
		kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL17,
				0xFFFFFFFF);
		kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_MASK_CNTL18,
				0xFFFFFFFF);
	}

	/*
	 * Set hang detection threshold to 4 million cycles
	 * (0x3FFFF*16)
	 */
	kgsl_regwrite(device, A5XX_RBBM_INTERFACE_HANG_INT_CNTL,
				  (1 << 30) | 0x3FFFF);

	/* Turn on performance counters */
	kgsl_regwrite(device, A5XX_RBBM_PERFCTR_CNTL, 0x01);

	/*
	 * This is to increase performance by restricting VFD's cache access,
	 * so that LRZ and other data get evicted less.
	 */
	kgsl_regwrite(device, A5XX_UCHE_CACHE_WAYS, 0x02);

	/*
	 * Set UCHE_WRITE_THRU_BASE to the UCHE_TRAP_BASE effectively
	 * disabling L2 bypass
	 */
	kgsl_regwrite(device, A5XX_UCHE_TRAP_BASE_LO, 0xffff0000);
	kgsl_regwrite(device, A5XX_UCHE_TRAP_BASE_HI, 0x0001ffff);
	kgsl_regwrite(device, A5XX_UCHE_WRITE_THRU_BASE_LO, 0xffff0000);
	kgsl_regwrite(device, A5XX_UCHE_WRITE_THRU_BASE_HI, 0x0001ffff);

	/* Program the GMEM VA range for the UCHE path */
	kgsl_regwrite(device, A5XX_UCHE_GMEM_RANGE_MIN_LO,
			adreno_dev->uche_gmem_base);
	kgsl_regwrite(device, A5XX_UCHE_GMEM_RANGE_MIN_HI, 0x0);
	kgsl_regwrite(device, A5XX_UCHE_GMEM_RANGE_MAX_LO,
			adreno_dev->uche_gmem_base +
			adreno_dev->gpucore->gmem_size - 1);
	kgsl_regwrite(device, A5XX_UCHE_GMEM_RANGE_MAX_HI, 0x0);

	/*
	 * Below CP registers are 0x0 by default, program init
	 * values based on a5xx flavor.
	 */
	if (adreno_is_a505_or_a506(adreno_dev) || adreno_is_a508(adreno_dev)) {
		kgsl_regwrite(device, A5XX_CP_MEQ_THRESHOLDS, 0x20);
		kgsl_regwrite(device, A5XX_CP_MERCIU_SIZE, 0x400);
		kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_2, 0x40000030);
		kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_1, 0x20100D0A);
	} else if (adreno_is_a510(adreno_dev)) {
		kgsl_regwrite(device, A5XX_CP_MEQ_THRESHOLDS, 0x20);
		kgsl_regwrite(device, A5XX_CP_MERCIU_SIZE, 0x20);
		kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_2, 0x40000030);
		kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_1, 0x20100D0A);
	} else if (adreno_is_a540(adreno_dev) || adreno_is_a512(adreno_dev)) {
		kgsl_regwrite(device, A5XX_CP_MEQ_THRESHOLDS, 0x40);
		kgsl_regwrite(device, A5XX_CP_MERCIU_SIZE, 0x400);
		kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060);
		kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16);
	} else {
		kgsl_regwrite(device, A5XX_CP_MEQ_THRESHOLDS, 0x40);
		kgsl_regwrite(device, A5XX_CP_MERCIU_SIZE, 0x40);
		kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_2, 0x80000060);
		kgsl_regwrite(device, A5XX_CP_ROQ_THRESHOLDS_1, 0x40201B16);
	}

	/*
	 * vtxFifo and primFifo thresholds default values
	 * are different.
	 */
	if (adreno_is_a505_or_a506(adreno_dev) || adreno_is_a508(adreno_dev))
		kgsl_regwrite(device, A5XX_PC_DBG_ECO_CNTL,
						(0x100 << 11 | 0x100 << 22));
	else if (adreno_is_a510(adreno_dev) || adreno_is_a512(adreno_dev))
		kgsl_regwrite(device, A5XX_PC_DBG_ECO_CNTL,
						(0x200 << 11 | 0x200 << 22));
	else
		kgsl_regwrite(device, A5XX_PC_DBG_ECO_CNTL,
						(0x400 << 11 | 0x300 << 22));

	if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_TWO_PASS_USE_WFI)) {
		/*
		 * Set TWOPASSUSEWFI in A5XX_PC_DBG_ECO_CNTL for
		 * microcodes after v77
		 */
		if ((adreno_compare_pfp_version(adreno_dev, 0x5FF077) >= 0))
			kgsl_regrmw(device, A5XX_PC_DBG_ECO_CNTL, 0, (1 << 8));
	}

	if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_DISABLE_RB_DP2CLOCKGATING)) {
		/*
		 * Disable RB sampler datapath DP2 clock gating
		 * optimization for 1-SP GPU's, by default it is enabled.
		 */
		kgsl_regrmw(device, A5XX_RB_DBG_ECO_CNT, 0, (1 << 9));
	}
	/*
	 * Disable UCHE global filter as SP can invalidate/flush
	 * independently
	 */
	kgsl_regwrite(device, A5XX_UCHE_MODE_CNTL, BIT(29));
	/* Set the USE_RETENTION_FLOPS chicken bit */
	kgsl_regwrite(device, A5XX_CP_CHICKEN_DBG, 0x02000000);

	/* Enable ISDB mode if requested */
	if (test_bit(ADRENO_DEVICE_ISDB_ENABLED, &adreno_dev->priv)) {
		if (!adreno_active_count_get(adreno_dev)) {
			/*
			 * Disable ME/PFP split timeouts when the debugger is
			 * enabled because the CP doesn't know when a shader is
			 * in active debug
			 */
			kgsl_regwrite(device, A5XX_RBBM_AHB_CNTL1, 0x06FFFFFF);

			/* Force the SP0/SP1 clocks on to enable ISDB */
			kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL_SP0, 0x0);
			kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL_SP1, 0x0);
			kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL_SP2, 0x0);
			kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL_SP3, 0x0);
			kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL2_SP0, 0x0);
			kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL2_SP1, 0x0);
			kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL2_SP2, 0x0);
			kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL2_SP3, 0x0);

			/* disable HWCG */
			kgsl_regwrite(device, A5XX_RBBM_CLOCK_CNTL, 0x0);
			kgsl_regwrite(device, A5XX_RBBM_ISDB_CNT, 0x0);
		} else
			dev_err(device->dev,
				"Active count failed while turning on ISDB\n");
	} else {
		/* if not in ISDB mode enable ME/PFP split notification */
		kgsl_regwrite(device, A5XX_RBBM_AHB_CNTL1, 0xA6FFFFFF);
	}

	kgsl_regwrite(device, A5XX_RBBM_AHB_CNTL2, 0x0000003F);
	bit = adreno_dev->highest_bank_bit ?
		(adreno_dev->highest_bank_bit - 13) & 0x03 : 0;
	/*
	 * Program the highest DDR bank bit that was passed in
	 * from the DT in a handful of registers. Some of these
	 * registers will also be written by the UMD, but we
	 * want to program them in case we happen to use the
	 * UCHE before the UMD does
	 */

	kgsl_regwrite(device, A5XX_TPL1_MODE_CNTL, bit << 7);
	kgsl_regwrite(device, A5XX_RB_MODE_CNTL, bit << 1);
	if (adreno_is_a540(adreno_dev) || adreno_is_a512(adreno_dev))
		kgsl_regwrite(device, A5XX_UCHE_DBG_ECO_CNTL_2, bit);

	/* Disable All flat shading optimization */
	kgsl_regrmw(device, A5XX_VPC_DBG_ECO_CNTL, 0, 0x1 << 10);

	/*
	 * VPC corner case with local memory load kill leads to corrupt
	 * internal state. Normal Disable does not work for all a5x chips.
	 * So do the following setting to disable it.
	 */
	if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_DISABLE_LMLOADKILL)) {
		kgsl_regrmw(device, A5XX_VPC_DBG_ECO_CNTL, 0, 0x1 << 23);
		kgsl_regrmw(device, A5XX_HLSQ_DBG_ECO_CNTL, 0x1 << 18, 0);
	}

	if (device->mmu.secured) {
		kgsl_regwrite(device, A5XX_RBBM_SECVID_TSB_CNTL, 0x0);
		kgsl_regwrite(device, A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO,
			lower_32_bits(KGSL_IOMMU_SECURE_BASE32));
		kgsl_regwrite(device, A5XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI,
			upper_32_bits(KGSL_IOMMU_SECURE_BASE32));
		kgsl_regwrite(device, A5XX_RBBM_SECVID_TSB_TRUSTED_SIZE,
			FIELD_PREP(GENMASK(31, 12),
			(KGSL_IOMMU_SECURE_SIZE(&device->mmu) / SZ_4K)));
	}

	a5xx_preemption_start(adreno_dev);
	a5xx_protect_init(adreno_dev);

	return 0;
}

/*
 * Follow the ME_INIT sequence with a preemption yield to allow the GPU to move
 * to a different ringbuffer, if desired
 */
static int _preemption_init(
			struct adreno_device *adreno_dev,
			struct adreno_ringbuffer *rb, unsigned int *cmds,
			struct kgsl_context *context)
{
	unsigned int *cmds_orig = cmds;
	uint64_t gpuaddr = rb->preemption_desc->gpuaddr;

	/* Turn CP protection OFF */
	cmds += cp_protected_mode(adreno_dev, cmds, 0);
	/*
	 * CP during context switch will save context switch info to
	 * a5xx_cp_preemption_record pointed by CONTEXT_SWITCH_SAVE_ADDR
	 */
	*cmds++ = cp_type4_packet(A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_LO, 1);
	*cmds++ = lower_32_bits(gpuaddr);
	*cmds++ = cp_type4_packet(A5XX_CP_CONTEXT_SWITCH_SAVE_ADDR_HI, 1);
	*cmds++ = upper_32_bits(gpuaddr);

	/* Turn CP protection ON */
	cmds += cp_protected_mode(adreno_dev, cmds, 1);

	*cmds++ = cp_type7_packet(CP_PREEMPT_ENABLE_GLOBAL, 1);
	*cmds++ = 0;

	*cmds++ = cp_type7_packet(CP_PREEMPT_ENABLE_LOCAL, 1);
	*cmds++ = 1;

	/* Enable yield in RB only */
	*cmds++ = cp_type7_packet(CP_YIELD_ENABLE, 1);
	*cmds++ = 1;

	*cmds++ = cp_type7_packet(CP_CONTEXT_SWITCH_YIELD, 4);
	cmds += cp_gpuaddr(adreno_dev, cmds, 0x0);
	*cmds++ = 0;
	/* generate interrupt on preemption completion */
	*cmds++ = 1;

	return cmds - cmds_orig;
}

static int a5xx_post_start(struct adreno_device *adreno_dev)
{
	int ret;
	unsigned int *cmds, *start;
	struct adreno_ringbuffer *rb = adreno_dev->cur_rb;

	if (!adreno_is_a530(adreno_dev) &&
		!adreno_is_preemption_enabled(adreno_dev))
		return 0;

	cmds = adreno_ringbuffer_allocspace(rb, 42);
	if (IS_ERR(cmds)) {
		struct kgsl_device *device = KGSL_DEVICE(adreno_dev);

		dev_err(device->dev,
			     "error allocating preemtion init cmds\n");
		return PTR_ERR(cmds);
	}
	start = cmds;

	/*
	 * Send a pipeline stat event whenever the GPU gets powered up
	 * to cause misbehaving perf counters to start ticking
	 */
	if (adreno_is_a530(adreno_dev)) {
		*cmds++ = cp_packet(adreno_dev, CP_EVENT_WRITE, 1);
		*cmds++ = 0xF;
	}

	if (adreno_is_preemption_enabled(adreno_dev)) {
		cmds += _preemption_init(adreno_dev, rb, cmds, NULL);
		rb->_wptr = rb->_wptr - (42 - (cmds - start));
		ret = a5xx_ringbuffer_submit(rb, NULL, false);
	} else {
		rb->_wptr = rb->_wptr - (42 - (cmds - start));
		ret = a5xx_ringbuffer_submit(rb, NULL, true);
	}

	if (!ret) {
		ret = adreno_spin_idle(adreno_dev, 2000);
		if (ret)
			a5xx_spin_idle_debug(adreno_dev,
				"hw initialization failed to idle\n");
	}

	return ret;
}

static int a5xx_gpmu_init(struct adreno_device *adreno_dev)
{
	int ret;

	/* Set up LM before initializing the GPMU */
	a5xx_lm_init(adreno_dev);

	/* Enable SPTP based power collapse before enabling GPMU */
	a5xx_enable_pc(adreno_dev);

	ret = a5xx_gpmu_start(adreno_dev);
	if (ret)
		return ret;

	/* Enable limits management */
	a5xx_lm_enable(adreno_dev);
	return 0;
}

static int a5xx_zap_shader_resume(struct kgsl_device *device)
{
	int ret = qcom_scm_set_remote_state(0, 13);

	if (ret)
		dev_err(device->dev,
			"SCM zap resume call failed: %d\n", ret);

	return ret;
}

/*
 * a5xx_microcode_load() - Load microcode
 * @adreno_dev: Pointer to adreno device
 */
static int a5xx_microcode_load(struct adreno_device *adreno_dev)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	struct adreno_firmware *pm4_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PM4);
	struct adreno_firmware *pfp_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PFP);
	const struct adreno_a5xx_core *a5xx_core = to_a5xx_core(adreno_dev);
	uint64_t gpuaddr;

	gpuaddr = pm4_fw->memdesc->gpuaddr;
	kgsl_regwrite(device, A5XX_CP_PM4_INSTR_BASE_LO,
				lower_32_bits(gpuaddr));
	kgsl_regwrite(device, A5XX_CP_PM4_INSTR_BASE_HI,
				upper_32_bits(gpuaddr));

	gpuaddr = pfp_fw->memdesc->gpuaddr;
	kgsl_regwrite(device, A5XX_CP_PFP_INSTR_BASE_LO,
				lower_32_bits(gpuaddr));
	kgsl_regwrite(device, A5XX_CP_PFP_INSTR_BASE_HI,
				upper_32_bits(gpuaddr));

	/*
	 * Do not invoke to load zap shader if MMU does
	 * not support secure mode.
	 */
	if (!device->mmu.secured)
		return 0;

	if (adreno_dev->zap_loaded && !(ADRENO_FEATURE(adreno_dev,
		ADRENO_CPZ_RETENTION)))
		return a5xx_zap_shader_resume(device);

	return adreno_zap_shader_load(adreno_dev, a5xx_core->zap_name);
}

static int _me_init_ucode_workarounds(struct adreno_device *adreno_dev)
{
	switch (ADRENO_GPUREV(adreno_dev)) {
	case ADRENO_REV_A510:
		return 0x00000001; /* Ucode workaround for token end syncs */
	case ADRENO_REV_A505:
	case ADRENO_REV_A506:
	case ADRENO_REV_A530:
		/*
		 * Ucode workarounds for token end syncs,
		 * WFI after every direct-render 3D mode draw and
		 * WFI after every 2D Mode 3 draw.
		 */
		return 0x0000000B;
	default:
		return 0x00000000; /* No ucode workarounds enabled */
	}
}

/*
 * CP_INIT_MAX_CONTEXT bit tells if the multiple hardware contexts can
 * be used at once of if they should be serialized
 */
#define CP_INIT_MAX_CONTEXT BIT(0)

/* Enables register protection mode */
#define CP_INIT_ERROR_DETECTION_CONTROL BIT(1)

/* Header dump information */
#define CP_INIT_HEADER_DUMP BIT(2) /* Reserved */

/* Default Reset states enabled for PFP and ME */
#define CP_INIT_DEFAULT_RESET_STATE BIT(3)

/* Drawcall filter range */
#define CP_INIT_DRAWCALL_FILTER_RANGE BIT(4)

/* Ucode workaround masks */
#define CP_INIT_UCODE_WORKAROUND_MASK BIT(5)

#define CP_INIT_MASK (CP_INIT_MAX_CONTEXT | \
		CP_INIT_ERROR_DETECTION_CONTROL | \
		CP_INIT_HEADER_DUMP | \
		CP_INIT_DEFAULT_RESET_STATE | \
		CP_INIT_UCODE_WORKAROUND_MASK)

static int a5xx_critical_packet_submit(struct adreno_device *adreno_dev,
					struct adreno_ringbuffer *rb)
{
	unsigned int *cmds;
	int ret;

	if (!critical_packet_constructed)
		return 0;

	cmds = adreno_ringbuffer_allocspace(rb, 4);
	if (IS_ERR(cmds))
		return PTR_ERR(cmds);

	*cmds++ = cp_mem_packet(adreno_dev, CP_INDIRECT_BUFFER_PFE, 2, 1);
	cmds += cp_gpuaddr(adreno_dev, cmds, adreno_dev->critpkts->gpuaddr);
	*cmds++ = crit_pkts_dwords;

	ret = a5xx_ringbuffer_submit(rb, NULL, true);
	if (!ret) {
		ret = adreno_spin_idle(adreno_dev, 20);
		if (ret)
			a5xx_spin_idle_debug(adreno_dev,
				"Critical packet submission failed to idle\n");
	}

	return ret;
}

/*
 * a5xx_send_me_init() - Initialize ringbuffer
 * @adreno_dev: Pointer to adreno device
 * @rb: Pointer to the ringbuffer of device
 *
 * Submit commands for ME initialization,
 */
static int a5xx_send_me_init(struct adreno_device *adreno_dev,
			 struct adreno_ringbuffer *rb)
{
	unsigned int *cmds;
	int i = 0, ret;

	cmds = adreno_ringbuffer_allocspace(rb, 9);
	if (IS_ERR(cmds))
		return PTR_ERR(cmds);

	cmds[i++] = cp_type7_packet(CP_ME_INIT, 8);

	/* Enabled ordinal mask */
	cmds[i++] = CP_INIT_MASK;

	if (CP_INIT_MASK & CP_INIT_MAX_CONTEXT)
		cmds[i++] = 0x00000003;

	if (CP_INIT_MASK & CP_INIT_ERROR_DETECTION_CONTROL)
		cmds[i++] = 0x20000000;

	if (CP_INIT_MASK & CP_INIT_HEADER_DUMP) {
		/* Header dump address */
		cmds[i++] = 0x00000000;
		/* Header dump enable and dump size */
		cmds[i++] = 0x00000000;
	}

	if (CP_INIT_MASK & CP_INIT_DRAWCALL_FILTER_RANGE) {
		/* Start range */
		cmds[i++] = 0x00000000;
		/* End range (inclusive) */
		cmds[i++] = 0x00000000;
	}

	if (CP_INIT_MASK & CP_INIT_UCODE_WORKAROUND_MASK)
		cmds[i++] = _me_init_ucode_workarounds(adreno_dev);

	ret = a5xx_ringbuffer_submit(rb, NULL, true);
	if (!ret) {
		ret = adreno_spin_idle(adreno_dev, 2000);
		if (ret)
			a5xx_spin_idle_debug(adreno_dev,
				"CP initialization failed to idle\n");
	}

	return ret;
}

/*
 * a5xx_rb_start() - Start the ringbuffer
 * @adreno_dev: Pointer to adreno device
 */
static int a5xx_rb_start(struct adreno_device *adreno_dev)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	struct adreno_ringbuffer *rb;
	uint64_t addr;
	unsigned int *cmds;
	int ret, i;

	/* Clear all the ringbuffers */
	FOR_EACH_RINGBUFFER(adreno_dev, rb, i) {
		memset(rb->buffer_desc->hostptr, 0xaa, KGSL_RB_SIZE);
		kgsl_sharedmem_writel(device->scratch,
			SCRATCH_RB_OFFSET(rb->id, rptr), 0);

		rb->wptr = 0;
		rb->_wptr = 0;
		rb->wptr_preempt_end = ~0;
	}

	/* Set up the current ringbuffer */
	rb = ADRENO_CURRENT_RINGBUFFER(adreno_dev);
	addr = SCRATCH_RB_GPU_ADDR(device, rb->id, rptr);

	kgsl_regwrite(device, A5XX_CP_RB_RPTR_ADDR_LO, lower_32_bits(addr));
	kgsl_regwrite(device, A5XX_CP_RB_RPTR_ADDR_HI, upper_32_bits(addr));

	/*
	 * The size of the ringbuffer in the hardware is the log2
	 * representation of the size in quadwords (sizedwords / 2).
	 * Also disable the host RPTR shadow register as it might be unreliable
	 * in certain circumstances.
	 */

	kgsl_regwrite(device, A5XX_CP_RB_CNTL,
		A5XX_CP_RB_CNTL_DEFAULT);

	kgsl_regwrite(device, A5XX_CP_RB_BASE,
		lower_32_bits(rb->buffer_desc->gpuaddr));
	kgsl_regwrite(device, A5XX_CP_RB_BASE_HI,
		upper_32_bits(rb->buffer_desc->gpuaddr));

	ret = a5xx_microcode_load(adreno_dev);
	if (ret)
		return ret;

	/* clear ME_HALT to start micro engine */

	kgsl_regwrite(device, A5XX_CP_ME_CNTL, 0);

	ret = a5xx_send_me_init(adreno_dev, rb);
	if (ret)
		return ret;

	/* Run the critical packets if we need to */
	if (ADRENO_QUIRK(adreno_dev, ADRENO_QUIRK_CRITICAL_PACKETS)) {
		ret = a5xx_critical_packet_submit(adreno_dev, rb);
		if (ret)
			return ret;
	}

	/*
	 * Try to execute the zap shader if it exists, otherwise just try
	 * directly writing to the control register
	 */
	if (!adreno_dev->zap_loaded)
		kgsl_regwrite(device, A5XX_RBBM_SECVID_TRUST_CNTL, 0);
	else {
		cmds = adreno_ringbuffer_allocspace(rb, 2);
		if (IS_ERR(cmds))
			return  PTR_ERR(cmds);

		*cmds++ = cp_packet(adreno_dev, CP_SET_SECURE_MODE, 1);
		*cmds++ = 0;

		ret = a5xx_ringbuffer_submit(rb, NULL, true);
		if (!ret) {
			ret = adreno_spin_idle(adreno_dev, 2000);
			if (ret) {
				a5xx_spin_idle_debug(adreno_dev,
					"Switch to unsecure failed to idle\n");
				return ret;
			}
		}
	}

	ret = a5xx_gpmu_init(adreno_dev);
	if (ret)
		return ret;

	a5xx_post_start(adreno_dev);

	return 0;
}

/*
 * a5xx_microcode_read() - Read microcode
 * @adreno_dev: Pointer to adreno device
 */
static int a5xx_microcode_read(struct adreno_device *adreno_dev)
{
	int ret;
	struct adreno_firmware *pm4_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PM4);
	struct adreno_firmware *pfp_fw = ADRENO_FW(adreno_dev, ADRENO_FW_PFP);
	const struct adreno_a5xx_core *a5xx_core = to_a5xx_core(adreno_dev);

	ret = adreno_get_firmware(adreno_dev, a5xx_core->pm4fw_name, pm4_fw);
	if (ret)
		return ret;

	ret = adreno_get_firmware(adreno_dev, a5xx_core->pfpfw_name, pfp_fw);
	if (ret)
		return ret;

	ret = _load_gpmu_firmware(adreno_dev);
	if (ret)
		return ret;

	_load_regfile(adreno_dev);

	return ret;
}

/* Register offset defines for A5XX, in order of enum adreno_regs */
static unsigned int a5xx_register_offsets[ADRENO_REG_REGISTER_MAX] = {
	ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE, A5XX_CP_RB_BASE),
	ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_BASE_HI, A5XX_CP_RB_BASE_HI),
	ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR_ADDR_LO,
			A5XX_CP_RB_RPTR_ADDR_LO),
	ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR_ADDR_HI,
			A5XX_CP_RB_RPTR_ADDR_HI),
	ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_RPTR, A5XX_CP_RB_RPTR),
	ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_WPTR, A5XX_CP_RB_WPTR),
	ADRENO_REG_DEFINE(ADRENO_REG_CP_ME_CNTL, A5XX_CP_ME_CNTL),
	ADRENO_REG_DEFINE(ADRENO_REG_CP_RB_CNTL, A5XX_CP_RB_CNTL),
	ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BASE, A5XX_CP_IB1_BASE),
	ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BASE_HI, A5XX_CP_IB1_BASE_HI),
	ADRENO_REG_DEFINE(ADRENO_REG_CP_IB1_BUFSZ, A5XX_CP_IB1_BUFSZ),
	ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BASE, A5XX_CP_IB2_BASE),
	ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BASE_HI, A5XX_CP_IB2_BASE_HI),
	ADRENO_REG_DEFINE(ADRENO_REG_CP_IB2_BUFSZ, A5XX_CP_IB2_BUFSZ),
	ADRENO_REG_DEFINE(ADRENO_REG_CP_PROTECT_REG_0, A5XX_CP_PROTECT_REG_0),
	ADRENO_REG_DEFINE(ADRENO_REG_CP_PREEMPT, A5XX_CP_CONTEXT_SWITCH_CNTL),
	ADRENO_REG_DEFINE(ADRENO_REG_CP_PREEMPT_DEBUG, ADRENO_REG_SKIP),
	ADRENO_REG_DEFINE(ADRENO_REG_CP_PREEMPT_DISABLE, ADRENO_REG_SKIP),
	ADRENO_REG_DEFINE(ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_LO,
				A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO),
	ADRENO_REG_DEFINE(ADRENO_REG_CP_CONTEXT_SWITCH_SMMU_INFO_HI,
				A5XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI),
	ADRENO_REG_DEFINE(ADRENO_REG_RBBM_STATUS, A5XX_RBBM_STATUS),
	ADRENO_REG_DEFINE(ADRENO_REG_RBBM_STATUS3, A5XX_RBBM_STATUS3),
	ADRENO_REG_DEFINE(ADRENO_REG_RBBM_INT_0_MASK, A5XX_RBBM_INT_0_MASK),
	ADRENO_REG_DEFINE(ADRENO_REG_RBBM_CLOCK_CTL, A5XX_RBBM_CLOCK_CNTL),
	ADRENO_REG_DEFINE(ADRENO_REG_RBBM_SW_RESET_CMD, A5XX_RBBM_SW_RESET_CMD),
	ADRENO_REG_DEFINE(ADRENO_REG_GPMU_POWER_COUNTER_ENABLE,
				A5XX_GPMU_POWER_COUNTER_ENABLE),
};

static void a5xx_cp_hw_err_callback(struct adreno_device *adreno_dev, int bit)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	unsigned int status1, status2;

	kgsl_regread(device, A5XX_CP_INTERRUPT_STATUS, &status1);

	if (status1 & BIT(A5XX_CP_OPCODE_ERROR)) {
		unsigned int val;

		kgsl_regwrite(device, A5XX_CP_PFP_STAT_ADDR, 0);

		/*
		 * A5XX_CP_PFP_STAT_DATA is indexed, so read it twice to get the
		 * value we want
		 */
		kgsl_regread(device, A5XX_CP_PFP_STAT_DATA, &val);
		kgsl_regread(device, A5XX_CP_PFP_STAT_DATA, &val);

		dev_crit_ratelimited(device->dev,
					"ringbuffer opcode error | possible opcode=0x%8.8X\n",
					val);
	}
	if (status1 & BIT(A5XX_CP_RESERVED_BIT_ERROR))
		dev_crit_ratelimited(device->dev,
					"ringbuffer reserved bit error interrupt\n");
	if (status1 & BIT(A5XX_CP_HW_FAULT_ERROR)) {
		kgsl_regread(device, A5XX_CP_HW_FAULT, &status2);
		dev_crit_ratelimited(device->dev,
					"CP | Ringbuffer HW fault | status=%x\n",
					status2);
	}
	if (status1 & BIT(A5XX_CP_DMA_ERROR))
		dev_crit_ratelimited(device->dev, "CP | DMA error\n");
	if (status1 & BIT(A5XX_CP_REGISTER_PROTECTION_ERROR)) {
		kgsl_regread(device, A5XX_CP_PROTECT_STATUS, &status2);
		dev_crit_ratelimited(device->dev,
					"CP | Protected mode error| %s | addr=%x | status=%x\n",
					status2 & (1 << 24) ? "WRITE" : "READ",
					(status2 & 0xFFFFF) >> 2, status2);
	}
	if (status1 & BIT(A5XX_CP_AHB_ERROR)) {
		kgsl_regread(device, A5XX_CP_AHB_FAULT, &status2);
		dev_crit_ratelimited(device->dev,
					"ringbuffer AHB error interrupt | status=%x\n",
					status2);
	}
}

static void a5xx_err_callback(struct adreno_device *adreno_dev, int bit)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	unsigned int reg;

	switch (bit) {
	case A5XX_INT_RBBM_AHB_ERROR: {
		kgsl_regread(device, A5XX_RBBM_AHB_ERROR_STATUS, &reg);

		/*
		 * Return the word address of the erroring register so that it
		 * matches the register specification
		 */
		dev_crit_ratelimited(device->dev,
					"RBBM | AHB bus error | %s | addr=%x | ports=%x:%x\n",
					reg & (1 << 28) ? "WRITE" : "READ",
					(reg & 0xFFFFF) >> 2,
					(reg >> 20) & 0x3,
					(reg >> 24) & 0xF);

		/* Clear the error */
		kgsl_regwrite(device, A5XX_RBBM_AHB_CMD, (1 << 4));
		break;
	}
	case A5XX_INT_RBBM_TRANSFER_TIMEOUT:
		dev_crit_ratelimited(device->dev,
					"RBBM: AHB transfer timeout\n");
		break;
	case A5XX_INT_RBBM_ME_MS_TIMEOUT:
		kgsl_regread(device, A5XX_RBBM_AHB_ME_SPLIT_STATUS, &reg);
		dev_crit_ratelimited(device->dev,
					"RBBM | ME master split timeout | status=%x\n",
					reg);
		break;
	case A5XX_INT_RBBM_PFP_MS_TIMEOUT:
		kgsl_regread(device, A5XX_RBBM_AHB_PFP_SPLIT_STATUS, &reg);
		dev_crit_ratelimited(device->dev,
					"RBBM | PFP master split timeout | status=%x\n",
					reg);
		break;
	case A5XX_INT_RBBM_ETS_MS_TIMEOUT:
		dev_crit_ratelimited(device->dev,
					"RBBM: ME master split timeout\n");
		break;
	case A5XX_INT_RBBM_ATB_ASYNC_OVERFLOW:
		dev_crit_ratelimited(device->dev,
					"RBBM: ATB ASYNC overflow\n");
		break;
	case A5XX_INT_RBBM_ATB_BUS_OVERFLOW:
		dev_crit_ratelimited(device->dev,
					"RBBM: ATB bus overflow\n");
		break;
	case A5XX_INT_UCHE_OOB_ACCESS:
		dev_crit_ratelimited(device->dev,
					"UCHE: Out of bounds access\n");
		break;
	case A5XX_INT_UCHE_TRAP_INTR:
		dev_crit_ratelimited(device->dev, "UCHE: Trap interrupt\n");
		break;
	case A5XX_INT_GPMU_VOLTAGE_DROOP:
		dev_crit_ratelimited(device->dev, "GPMU: Voltage droop\n");
		break;
	default:
		dev_crit_ratelimited(device->dev, "Unknown interrupt %d\n",
					bit);
	}
}

static void a5xx_irq_storm_worker(struct work_struct *work)
{
	struct adreno_device *adreno_dev = container_of(work,
			struct adreno_device, irq_storm_work);
	struct kgsl_device *device = &adreno_dev->dev;
	unsigned int status;

	mutex_lock(&device->mutex);

	/* Wait for the storm to clear up */
	do {
		kgsl_regwrite(device, A5XX_RBBM_INT_CLEAR_CMD,
				BIT(A5XX_INT_CP_CACHE_FLUSH_TS));
		kgsl_regread(device, A5XX_RBBM_INT_0_STATUS, &status);
	} while (status & BIT(A5XX_INT_CP_CACHE_FLUSH_TS));

	/* Re-enable the interrupt bit in the mask */
	adreno_dev->irq_mask |= BIT(A5XX_INT_CP_CACHE_FLUSH_TS);
	kgsl_regwrite(device, A5XX_RBBM_INT_0_MASK, adreno_dev->irq_mask);
	clear_bit(ADRENO_DEVICE_CACHE_FLUSH_TS_SUSPENDED, &adreno_dev->priv);

	dev_warn(device->dev, "Re-enabled A5XX_INT_CP_CACHE_FLUSH_TS\n");
	mutex_unlock(&device->mutex);

	/* Reschedule just to make sure everything retires */
	adreno_dispatcher_schedule(device);
}

static void a5xx_cp_callback(struct adreno_device *adreno_dev, int bit)
{
	struct kgsl_device *device = &adreno_dev->dev;
	unsigned int cur;
	static unsigned int count;
	static unsigned int prev;

	if (test_bit(ADRENO_DEVICE_CACHE_FLUSH_TS_SUSPENDED, &adreno_dev->priv))
		return;

	kgsl_sharedmem_readl(device->memstore, &cur,
			KGSL_MEMSTORE_OFFSET(KGSL_MEMSTORE_GLOBAL,
				ref_wait_ts));

	/*
	 * prev holds a previously read value
	 * from memory.  It should be changed by the GPU with every
	 * interrupt. If the value we know about and the value we just
	 * read are the same, then we are likely in a storm.
	 * If this happens twice, disable the interrupt in the mask
	 * so the dispatcher can take care of the issue. It is then
	 * up to the dispatcher to re-enable the mask once all work
	 * is done and the storm has ended.
	 */
	if (prev == cur) {
		count++;
		if (count == 2) {
			/* disable interrupt from the mask */
			set_bit(ADRENO_DEVICE_CACHE_FLUSH_TS_SUSPENDED,
					&adreno_dev->priv);

			adreno_dev->irq_mask &=
				~BIT(A5XX_INT_CP_CACHE_FLUSH_TS);

			kgsl_regwrite(device, A5XX_RBBM_INT_0_MASK,
				adreno_dev->irq_mask);

			kgsl_schedule_work(&adreno_dev->irq_storm_work);

			return;
		}
	} else {
		count = 0;
		prev = cur;
	}

	a5xx_preemption_trigger(adreno_dev);
	adreno_dispatcher_schedule(device);
}

static const char *gpmu_int_msg[32] = {
	[FW_INTR_INFO] = "FW_INTR_INFO",
	[LLM_ACK_ERR_INTR] = "LLM_ACK_ERR_INTR",
	[ISENS_TRIM_ERR_INTR] = "ISENS_TRIM_ERR_INTR",
	[ISENS_ERR_INTR] = "ISENS_ERR_INTR",
	[ISENS_IDLE_ERR_INTR] = "ISENS_IDLE_ERR_INTR",
	[ISENS_PWR_ON_ERR_INTR] = "ISENS_PWR_ON_ERR_INTR",
	[6 ... 30] = "",
	[WDOG_EXPITED] = "WDOG_EXPITED"};

static void a5xx_gpmu_int_callback(struct adreno_device *adreno_dev, int bit)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	unsigned int reg, i;

	kgsl_regread(device, A5XX_GPMU_RBBM_INTR_INFO, &reg);

	if (reg & (~VALID_GPMU_IRQ)) {
		dev_crit_ratelimited(device->dev,
					"GPMU: Unknown IRQ mask 0x%08lx in 0x%08x\n",
					reg & (~VALID_GPMU_IRQ), reg);
	}

	for (i = 0; i < 32; i++)
		switch (reg & BIT(i)) {
		case BIT(WDOG_EXPITED):
			if (test_and_clear_bit(ADRENO_DEVICE_GPMU_INITIALIZED,
				&adreno_dev->priv)) {
				/* Stop GPMU */
				kgsl_regwrite(device,
					A5XX_GPMU_CM3_SYSRESET, 1);
				kgsl_schedule_work(&adreno_dev->gpmu_work);
			}
			fallthrough;
		case BIT(FW_INTR_INFO):
			fallthrough;
		case BIT(LLM_ACK_ERR_INTR):
			fallthrough;
		case BIT(ISENS_TRIM_ERR_INTR):
			fallthrough;
		case BIT(ISENS_ERR_INTR):
			fallthrough;
		case BIT(ISENS_IDLE_ERR_INTR):
			fallthrough;
		case BIT(ISENS_PWR_ON_ERR_INTR):
			dev_crit_ratelimited(device->dev,
						"GPMU: interrupt %s(%08lx)\n",
						gpmu_int_msg[i],
						BIT(i));
			break;
	}
}

/*
 * a5x_gpc_err_int_callback() - Isr for GPC error interrupts
 * @adreno_dev: Pointer to device
 * @bit: Interrupt bit
 */
static void a5x_gpc_err_int_callback(struct adreno_device *adreno_dev, int bit)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);

	/*
	 * GPC error is typically the result of mistake SW programming.
	 * Force GPU fault for this interrupt so that we can debug it
	 * with help of register dump.
	 */

	dev_crit(device->dev, "RBBM: GPC error\n");
	adreno_irqctrl(adreno_dev, 0);

	/* Trigger a fault in the dispatcher - this will effect a restart */
	adreno_dispatcher_fault(adreno_dev, ADRENO_SOFT_FAULT);
	adreno_dispatcher_schedule(device);
}

u64 a5xx_read_alwayson(struct adreno_device *adreno_dev)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	u32 lo = 0, hi = 0;

	kgsl_regread(device, A5XX_RBBM_ALWAYSON_COUNTER_LO, &lo);

	/* The upper 32 bits are only reliable on A540 targets */
	if (adreno_is_a540(adreno_dev))
		kgsl_regread(device, A5XX_RBBM_ALWAYSON_COUNTER_HI, &hi);

	return (((u64) hi) << 32) | lo;
}


static const struct adreno_irq_funcs a5xx_irq_funcs[32] = {
	ADRENO_IRQ_CALLBACK(NULL),              /* 0 - RBBM_GPU_IDLE */
	ADRENO_IRQ_CALLBACK(a5xx_err_callback), /* 1 - RBBM_AHB_ERROR */
	ADRENO_IRQ_CALLBACK(a5xx_err_callback), /* 2 - RBBM_TRANSFER_TIMEOUT */
	/* 3 - RBBM_ME_MASTER_SPLIT_TIMEOUT  */
	ADRENO_IRQ_CALLBACK(a5xx_err_callback),
	/* 4 - RBBM_PFP_MASTER_SPLIT_TIMEOUT */
	ADRENO_IRQ_CALLBACK(a5xx_err_callback),
	 /* 5 - RBBM_ETS_MASTER_SPLIT_TIMEOUT */
	ADRENO_IRQ_CALLBACK(a5xx_err_callback),
	/* 6 - RBBM_ATB_ASYNC_OVERFLOW */
	ADRENO_IRQ_CALLBACK(a5xx_err_callback),
	ADRENO_IRQ_CALLBACK(a5x_gpc_err_int_callback), /* 7 - GPC_ERR */
	ADRENO_IRQ_CALLBACK(a5xx_preempt_callback),/* 8 - CP_SW */
	ADRENO_IRQ_CALLBACK(a5xx_cp_hw_err_callback), /* 9 - CP_HW_ERROR */
	/* 10 - CP_CCU_FLUSH_DEPTH_TS */
	ADRENO_IRQ_CALLBACK(NULL),
	 /* 11 - CP_CCU_FLUSH_COLOR_TS */
	ADRENO_IRQ_CALLBACK(NULL),
	 /* 12 - CP_CCU_RESOLVE_TS */
	ADRENO_IRQ_CALLBACK(NULL),
	ADRENO_IRQ_CALLBACK(NULL), /* 13 - CP_IB2_INT */
	ADRENO_IRQ_CALLBACK(NULL), /* 14 - CP_IB1_INT */
	ADRENO_IRQ_CALLBACK(NULL), /* 15 - CP_RB_INT */
	/* 16 - CCP_UNUSED_1 */
	ADRENO_IRQ_CALLBACK(NULL),
	ADRENO_IRQ_CALLBACK(NULL), /* 17 - CP_RB_DONE_TS */
	ADRENO_IRQ_CALLBACK(NULL), /* 18 - CP_WT_DONE_TS */
	ADRENO_IRQ_CALLBACK(NULL), /* 19 - UNKNOWN_1 */
	ADRENO_IRQ_CALLBACK(a5xx_cp_callback), /* 20 - CP_CACHE_FLUSH_TS */
	/* 21 - UNUSED_2 */
	ADRENO_IRQ_CALLBACK(NULL),
	ADRENO_IRQ_CALLBACK(a5xx_err_callback), /* 22 - RBBM_ATB_BUS_OVERFLOW */
	/* 23 - MISC_HANG_DETECT */
	ADRENO_IRQ_CALLBACK(adreno_hang_int_callback),
	ADRENO_IRQ_CALLBACK(a5xx_err_callback), /* 24 - UCHE_OOB_ACCESS */
	ADRENO_IRQ_CALLBACK(a5xx_err_callback), /* 25 - UCHE_TRAP_INTR */
	ADRENO_IRQ_CALLBACK(NULL), /* 26 - DEBBUS_INTR_0 */
	ADRENO_IRQ_CALLBACK(NULL), /* 27 - DEBBUS_INTR_1 */
	ADRENO_IRQ_CALLBACK(a5xx_err_callback), /* 28 - GPMU_VOLTAGE_DROOP */
	ADRENO_IRQ_CALLBACK(a5xx_gpmu_int_callback), /* 29 - GPMU_FIRMWARE */
	ADRENO_IRQ_CALLBACK(NULL), /* 30 - ISDB_CPU_IRQ */
	ADRENO_IRQ_CALLBACK(NULL), /* 31 - ISDB_UNDER_DEBUG */
};

static irqreturn_t a5xx_irq_handler(struct adreno_device *adreno_dev)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	irqreturn_t ret;
	u32 status;

	kgsl_regread(device, A5XX_RBBM_INT_0_STATUS, &status);

	/*
	 * Clear all the interrupt bits except A5XX_INT_RBBM_AHB_ERROR.
	 * The interrupt will stay asserted until it is cleared by the handler
	 * so don't touch it yet to avoid a storm
	 */
	kgsl_regwrite(device, A5XX_RBBM_INT_CLEAR_CMD,
		status & ~A5XX_INT_RBBM_AHB_ERROR);

	/* Call the helper function for callbacks */
	ret = adreno_irq_callbacks(adreno_dev, a5xx_irq_funcs, status);

	trace_kgsl_a5xx_irq_status(adreno_dev, status);

	/* Now chear AHB_ERROR if it was set */
	if (status & A5XX_INT_RBBM_AHB_ERROR)
		kgsl_regwrite(device, A5XX_RBBM_INT_CLEAR_CMD,
			A5XX_INT_RBBM_AHB_ERROR);

	return ret;
}

static bool a5xx_hw_isidle(struct adreno_device *adreno_dev)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	u32 status;

	/*
	 * Due to CRC idle throttling the GPU idle hysteresis on a540 can take
	 * up to 5uS to expire
	 */
	if (adreno_is_a540(adreno_dev))
		udelay(5);

	kgsl_regread(device, A5XX_RBBM_STATUS, &status);

	if (status & 0xfffffffe)
		return false;

	kgsl_regread(device, A5XX_RBBM_INT_0_STATUS, &status);

	/* Return busy if a interrupt is pending */
	return !((status & adreno_dev->irq_mask) ||
		atomic_read(&adreno_dev->pending_irq_refcnt));
}

static int a5xx_clear_pending_transactions(struct adreno_device *adreno_dev)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	u32 mask = A5XX_VBIF_XIN_HALT_CTRL0_MASK;
	int ret;

	kgsl_regwrite(device, A5XX_VBIF_XIN_HALT_CTRL0, mask);
	ret = adreno_wait_for_halt_ack(device, A5XX_VBIF_XIN_HALT_CTRL1, mask);
	kgsl_regwrite(device, A5XX_VBIF_XIN_HALT_CTRL0, 0);

	return ret;
}

static bool a5xx_is_hw_collapsible(struct adreno_device *adreno_dev)
{
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	unsigned int reg;

	if (!adreno_isidle(adreno_dev))
		return false;

	/* If feature is not supported or enabled, no worry */
	if (!adreno_dev->sptp_pc_enabled)
		return true;
	kgsl_regread(device, A5XX_GPMU_SP_PWR_CLK_STATUS, &reg);
	if (reg & BIT(20))
		return false;
	kgsl_regread(device, A5XX_GPMU_RBCCU_PWR_CLK_STATUS, &reg);
	return !(reg & BIT(20));
}

static void a5xx_remove(struct adreno_device *adreno_dev)
{
	if (adreno_preemption_feature_set(adreno_dev))
		del_timer(&adreno_dev->preempt.timer);
}

static void a5xx_power_stats(struct adreno_device *adreno_dev,
		struct kgsl_power_stats *stats)
{
	static u32 rbbm0_hi;
	struct kgsl_device *device = KGSL_DEVICE(adreno_dev);
	s64 freq = kgsl_pwrctrl_active_freq(&device->pwrctrl) / 1000000;
	struct adreno_busy_data *busy = &adreno_dev->busy_data;
	s64 gpu_busy = 0;
	u32 lo, hi;
	s64 adj;

	/* Sometimes this counter can go backwards, so try to detect that */
	kgsl_regread(device, A5XX_RBBM_PERFCTR_RBBM_0_LO, &lo);
	kgsl_regread(device, A5XX_RBBM_PERFCTR_RBBM_0_HI, &hi);

	if (busy->gpu_busy) {
		if (lo < busy->gpu_busy) {
			if (hi == rbbm0_hi) {
				dev_warn_once(device->dev,
					"abmormal value from RBBM_0 perfcounter: %x %x\n",
					lo, busy->gpu_busy);
				gpu_busy = 0;
			} else {
				gpu_busy = (UINT_MAX - busy->gpu_busy) + lo;
				rbbm0_hi = hi;
			}
		} else
			gpu_busy = lo - busy->gpu_busy;
	} else {
		gpu_busy = 0;
		rbbm0_hi = 0;
	}

	busy->gpu_busy = lo;

	adj = a5xx_read_throttling_counters(adreno_dev);
	if (-adj <= gpu_busy)
		gpu_busy += adj;
	else
		gpu_busy = 0;

	stats->busy_time = gpu_busy / freq;

	if (adreno_is_a530(adreno_dev) && adreno_dev->lm_threshold_count)
		kgsl_regread(device, adreno_dev->lm_threshold_count,
			&adreno_dev->lm_threshold_cross);
	else if (adreno_is_a540(adreno_dev))
		adreno_dev->lm_threshold_cross = adj;

	if (!device->pwrctrl.bus_control)
		return;

	stats->ram_time = counter_delta(device, adreno_dev->ram_cycles_lo,
		&busy->bif_ram_cycles);

	stats->ram_wait = counter_delta(device, adreno_dev->starved_ram_lo,
		&busy->bif_starved_ram);
}

static int a5xx_setproperty(struct kgsl_device_private *dev_priv,
		u32 type, void __user *value, u32 sizebytes)
{
	struct kgsl_device *device = dev_priv->device;
	u32 enable;

	if (type != KGSL_PROP_PWRCTRL)
		return -ENODEV;

	if (sizebytes != sizeof(enable))
		return -EINVAL;

	if (copy_from_user(&enable, value, sizeof(enable)))
		return -EFAULT;

	mutex_lock(&device->mutex);

	if (enable) {
		device->pwrctrl.ctrl_flags = 0;
		kgsl_pwrscale_enable(device);
	} else {
		kgsl_pwrctrl_change_state(device, KGSL_STATE_ACTIVE);
		device->pwrctrl.ctrl_flags = KGSL_PWR_ON;
		kgsl_pwrscale_disable(device, true);
	}

	mutex_unlock(&device->mutex);

	return 0;
}

const struct adreno_gpudev adreno_a5xx_gpudev = {
	.reg_offsets = a5xx_register_offsets,
	.probe = a5xx_probe,
	.start = a5xx_start,
	.snapshot = a5xx_snapshot,
	.init = a5xx_init,
	.irq_handler = a5xx_irq_handler,
	.rb_start = a5xx_rb_start,
	.regulator_enable = a5xx_regulator_enable,
	.regulator_disable = a5xx_regulator_disable,
	.pwrlevel_change_settings = a5xx_pwrlevel_change_settings,
	.preemption_schedule = a5xx_preemption_schedule,
	.read_alwayson = a5xx_read_alwayson,
	.hw_isidle = a5xx_hw_isidle,
	.power_ops = &adreno_power_operations,
	.clear_pending_transactions = a5xx_clear_pending_transactions,
	.remove = a5xx_remove,
	.ringbuffer_submitcmd = a5xx_ringbuffer_submitcmd,
	.is_hw_collapsible = a5xx_is_hw_collapsible,
	.power_stats = a5xx_power_stats,
	.setproperty = a5xx_setproperty,
};