drivers/gpu/drm/exynos/exynos_drm_gem.c - kernel/common - Git at Google

 /* exynos_drm_gem.c
  *
  * Copyright (c) 2011 Samsung Electronics Co., Ltd.
  * Author: Inki Dae <[email protected]>
  *
  * This program is free software; you can redistribute  it and/or modify it
  * under  the terms of  the GNU General  Public License as published by the
  * Free Software Foundation;  either version 2 of the  License, or (at your
  * option) any later version.
  */

 #include <drm/drmP.h>
 #include <drm/drm_vma_manager.h>

 #include <linux/shmem_fs.h>
 #include <linux/dma-buf.h>
 #include <linux/pfn_t.h>
 #include <drm/exynos_drm.h>

 #include "exynos_drm_drv.h"
 #include "exynos_drm_gem.h"

 static int exynos_drm_alloc_buf(struct exynos_drm_gem *exynos_gem)
 {
 	struct drm_device *dev = exynos_gem->base.dev;
 	unsigned long attr;
 	unsigned int nr_pages;
 	struct sg_table sgt;
 	int ret = -ENOMEM;

 	if (exynos_gem->dma_addr) {
 		DRM_DEV_DEBUG_KMS(to_dma_dev(dev), "already allocated.\n");
 		return 0;
 	}

 	exynos_gem->dma_attrs = 0;

 	/*
 	 * if EXYNOS_BO_CONTIG, fully physically contiguous memory
 	 * region will be allocated else physically contiguous
 	 * as possible.
 	 */
 	if (!(exynos_gem->flags & EXYNOS_BO_NONCONTIG))
 		exynos_gem->dma_attrs |= DMA_ATTR_FORCE_CONTIGUOUS;

 	/*
 	 * if EXYNOS_BO_WC or EXYNOS_BO_NONCACHABLE, writecombine mapping
 	 * else cachable mapping.
 	 */
 	if (exynos_gem->flags & EXYNOS_BO_WC ||
 			!(exynos_gem->flags & EXYNOS_BO_CACHABLE))
 		attr = DMA_ATTR_WRITE_COMBINE;
 	else
 		attr = DMA_ATTR_NON_CONSISTENT;

 	exynos_gem->dma_attrs |= attr;
 	exynos_gem->dma_attrs |= DMA_ATTR_NO_KERNEL_MAPPING;

 	nr_pages = exynos_gem->size >> PAGE_SHIFT;

 	exynos_gem->pages = kvmalloc_array(nr_pages, sizeof(struct page *),
 			GFP_KERNEL | __GFP_ZERO);
 	if (!exynos_gem->pages) {
 		DRM_DEV_ERROR(to_dma_dev(dev), "failed to allocate pages.\n");
 		return -ENOMEM;
 	}

 	exynos_gem->cookie = dma_alloc_attrs(to_dma_dev(dev), exynos_gem->size,
 					     &exynos_gem->dma_addr, GFP_KERNEL,
 					     exynos_gem->dma_attrs);
 	if (!exynos_gem->cookie) {
 		DRM_DEV_ERROR(to_dma_dev(dev), "failed to allocate buffer.\n");
 		goto err_free;
 	}

 	ret = dma_get_sgtable_attrs(to_dma_dev(dev), &sgt, exynos_gem->cookie,
 				    exynos_gem->dma_addr, exynos_gem->size,
 				    exynos_gem->dma_attrs);
 	if (ret < 0) {
 		DRM_DEV_ERROR(to_dma_dev(dev), "failed to get sgtable.\n");
 		goto err_dma_free;
 	}

 	if (drm_prime_sg_to_page_addr_arrays(&sgt, exynos_gem->pages, NULL,
 					     nr_pages)) {
 		DRM_DEV_ERROR(to_dma_dev(dev), "invalid sgtable.\n");
 		ret = -EINVAL;
 		goto err_sgt_free;
 	}

 	sg_free_table(&sgt);

 	DRM_DEV_DEBUG_KMS(to_dma_dev(dev), "dma_addr(0x%lx), size(0x%lx)\n",
 			(unsigned long)exynos_gem->dma_addr, exynos_gem->size);

 	return 0;

 err_sgt_free:
 	sg_free_table(&sgt);
 err_dma_free:
 	dma_free_attrs(to_dma_dev(dev), exynos_gem->size, exynos_gem->cookie,
 		       exynos_gem->dma_addr, exynos_gem->dma_attrs);
 err_free:
 	kvfree(exynos_gem->pages);

 	return ret;
 }

 static void exynos_drm_free_buf(struct exynos_drm_gem *exynos_gem)
 {
 	struct drm_device *dev = exynos_gem->base.dev;

 	if (!exynos_gem->dma_addr) {
 		DRM_DEV_DEBUG_KMS(dev->dev, "dma_addr is invalid.\n");
 		return;
 	}

 	DRM_DEV_DEBUG_KMS(dev->dev, "dma_addr(0x%lx), size(0x%lx)\n",
 			(unsigned long)exynos_gem->dma_addr, exynos_gem->size);

 	dma_free_attrs(to_dma_dev(dev), exynos_gem->size, exynos_gem->cookie,
 			(dma_addr_t)exynos_gem->dma_addr,
 			exynos_gem->dma_attrs);

 	kvfree(exynos_gem->pages);
 }

 static int exynos_drm_gem_handle_create(struct drm_gem_object *obj,
 					struct drm_file *file_priv,
 					unsigned int *handle)
 {
 	int ret;

 	/*
 	 * allocate a id of idr table where the obj is registered
 	 * and handle has the id what user can see.
 	 */
 	ret = drm_gem_handle_create(file_priv, obj, handle);
 	if (ret)
 		return ret;

 	DRM_DEV_DEBUG_KMS(to_dma_dev(obj->dev), "gem handle = 0x%x\n", *handle);

 	/* drop reference from allocate - handle holds it now. */
 	drm_gem_object_put_unlocked(obj);

 	return 0;
 }

 void exynos_drm_gem_destroy(struct exynos_drm_gem *exynos_gem)
 {
 	struct drm_gem_object *obj = &exynos_gem->base;

 	DRM_DEV_DEBUG_KMS(to_dma_dev(obj->dev), "handle count = %d\n",
 			  obj->handle_count);

 	/*
 	 * do not release memory region from exporter.
 	 *
 	 * the region will be released by exporter
 	 * once dmabuf's refcount becomes 0.
 	 */
 	if (obj->import_attach)
 		drm_prime_gem_destroy(obj, exynos_gem->sgt);
 	else
 		exynos_drm_free_buf(exynos_gem);

 	/* release file pointer to gem object. */
 	drm_gem_object_release(obj);

 	kfree(exynos_gem);
 }

 static struct exynos_drm_gem *exynos_drm_gem_init(struct drm_device *dev,
 						  unsigned long size)
 {
 	struct exynos_drm_gem *exynos_gem;
 	struct drm_gem_object *obj;
 	int ret;

 	exynos_gem = kzalloc(sizeof(*exynos_gem), GFP_KERNEL);
 	if (!exynos_gem)
 		return ERR_PTR(-ENOMEM);

 	exynos_gem->size = size;
 	obj = &exynos_gem->base;

 	ret = drm_gem_object_init(dev, obj, size);
 	if (ret < 0) {
 		DRM_DEV_ERROR(dev->dev, "failed to initialize gem object\n");
 		kfree(exynos_gem);
 		return ERR_PTR(ret);
 	}

 	ret = drm_gem_create_mmap_offset(obj);
 	if (ret < 0) {
 		drm_gem_object_release(obj);
 		kfree(exynos_gem);
 		return ERR_PTR(ret);
 	}

 	DRM_DEV_DEBUG_KMS(dev->dev, "created file object = %pK\n", obj->filp);

 	return exynos_gem;
 }

 struct exynos_drm_gem *exynos_drm_gem_create(struct drm_device *dev,
 					     unsigned int flags,
 					     unsigned long size)
 {
 	struct exynos_drm_gem *exynos_gem;
 	int ret;

 	if (flags & ~(EXYNOS_BO_MASK)) {
 		DRM_DEV_ERROR(dev->dev,
 			      "invalid GEM buffer flags: %u\n", flags);
 		return ERR_PTR(-EINVAL);
 	}

 	if (!size) {
 		DRM_DEV_ERROR(dev->dev, "invalid GEM buffer size: %lu\n", size);
 		return ERR_PTR(-EINVAL);
 	}

 	size = roundup(size, PAGE_SIZE);

 	exynos_gem = exynos_drm_gem_init(dev, size);
 	if (IS_ERR(exynos_gem))
 		return exynos_gem;

 	if (!is_drm_iommu_supported(dev) && (flags & EXYNOS_BO_NONCONTIG)) {
 		/*
 		 * when no IOMMU is available, all allocated buffers are
 		 * contiguous anyway, so drop EXYNOS_BO_NONCONTIG flag
 		 */
 		flags &= ~EXYNOS_BO_NONCONTIG;
 		DRM_WARN("Non-contiguous allocation is not supported without IOMMU, falling back to contiguous buffer\n");
 	}

 	/* set memory type and cache attribute from user side. */
 	exynos_gem->flags = flags;

 	ret = exynos_drm_alloc_buf(exynos_gem);
 	if (ret < 0) {
 		drm_gem_object_release(&exynos_gem->base);
 		kfree(exynos_gem);
 		return ERR_PTR(ret);
 	}

 	return exynos_gem;
 }

 int exynos_drm_gem_create_ioctl(struct drm_device *dev, void *data,
 				struct drm_file *file_priv)
 {
 	struct drm_exynos_gem_create *args = data;
 	struct exynos_drm_gem *exynos_gem;
 	int ret;

 	exynos_gem = exynos_drm_gem_create(dev, args->flags, args->size);
 	if (IS_ERR(exynos_gem))
 		return PTR_ERR(exynos_gem);

 	ret = exynos_drm_gem_handle_create(&exynos_gem->base, file_priv,
 					   &args->handle);
 	if (ret) {
 		exynos_drm_gem_destroy(exynos_gem);
 		return ret;
 	}

 	return 0;
 }

 int exynos_drm_gem_map_ioctl(struct drm_device *dev, void *data,
 			     struct drm_file *file_priv)
 {
 	struct drm_exynos_gem_map *args = data;

 	return drm_gem_dumb_map_offset(file_priv, dev, args->handle,
 				       &args->offset);
 }

 struct exynos_drm_gem *exynos_drm_gem_get(struct drm_file *filp,
 					  unsigned int gem_handle)
 {
 	struct drm_gem_object *obj;

 	obj = drm_gem_object_lookup(filp, gem_handle);
 	if (!obj)
 		return NULL;
 	return to_exynos_gem(obj);
 }

 static int exynos_drm_gem_mmap_buffer(struct exynos_drm_gem *exynos_gem,
 				      struct vm_area_struct *vma)
 {
 	struct drm_device *drm_dev = exynos_gem->base.dev;
 	unsigned long vm_size;
 	int ret;

 	vma->vm_flags &= ~VM_PFNMAP;
 	vma->vm_pgoff = 0;

 	vm_size = vma->vm_end - vma->vm_start;

 	/* check if user-requested size is valid. */
 	if (vm_size > exynos_gem->size)
 		return -EINVAL;

 	ret = dma_mmap_attrs(to_dma_dev(drm_dev), vma, exynos_gem->cookie,
 			     exynos_gem->dma_addr, exynos_gem->size,
 			     exynos_gem->dma_attrs);
 	if (ret < 0) {
 		DRM_ERROR("failed to mmap.\n");
 		return ret;
 	}

 	return 0;
 }

 int exynos_drm_gem_get_ioctl(struct drm_device *dev, void *data,
 				      struct drm_file *file_priv)
 {
 	struct exynos_drm_gem *exynos_gem;
 	struct drm_exynos_gem_info *args = data;
 	struct drm_gem_object *obj;

 	obj = drm_gem_object_lookup(file_priv, args->handle);
 	if (!obj) {
 		DRM_DEV_ERROR(dev->dev, "failed to lookup gem object.\n");
 		return -EINVAL;
 	}

 	exynos_gem = to_exynos_gem(obj);

 	args->flags = exynos_gem->flags;
 	args->size = exynos_gem->size;

 	drm_gem_object_put_unlocked(obj);

 	return 0;
 }

 void exynos_drm_gem_free_object(struct drm_gem_object *obj)
 {
 	exynos_drm_gem_destroy(to_exynos_gem(obj));
 }

 int exynos_drm_gem_dumb_create(struct drm_file *file_priv,
 			       struct drm_device *dev,
 			       struct drm_mode_create_dumb *args)
 {
 	struct exynos_drm_gem *exynos_gem;
 	unsigned int flags;
 	int ret;

 	/*
 	 * allocate memory to be used for framebuffer.
 	 * - this callback would be called by user application
 	 *	with DRM_IOCTL_MODE_CREATE_DUMB command.
 	 */

 	args->pitch = args->width * ((args->bpp + 7) / 8);
 	args->size = args->pitch * args->height;

 	if (is_drm_iommu_supported(dev))
 		flags = EXYNOS_BO_NONCONTIG | EXYNOS_BO_WC;
 	else
 		flags = EXYNOS_BO_CONTIG | EXYNOS_BO_WC;

 	exynos_gem = exynos_drm_gem_create(dev, flags, args->size);
 	if (IS_ERR(exynos_gem)) {
 		dev_warn(dev->dev, "FB allocation failed.\n");
 		return PTR_ERR(exynos_gem);
 	}

 	ret = exynos_drm_gem_handle_create(&exynos_gem->base, file_priv,
 					   &args->handle);
 	if (ret) {
 		exynos_drm_gem_destroy(exynos_gem);
 		return ret;
 	}

 	return 0;
 }

 vm_fault_t exynos_drm_gem_fault(struct vm_fault *vmf)
 {
 	struct vm_area_struct *vma = vmf->vma;
 	struct drm_gem_object *obj = vma->vm_private_data;
 	struct exynos_drm_gem *exynos_gem = to_exynos_gem(obj);
 	unsigned long pfn;
 	pgoff_t page_offset;

 	page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;

 	if (page_offset >= (exynos_gem->size >> PAGE_SHIFT)) {
 		DRM_ERROR("invalid page offset\n");
 		return VM_FAULT_SIGBUS;
 	}

 	pfn = page_to_pfn(exynos_gem->pages[page_offset]);
 	return vmf_insert_mixed(vma, vmf->address,
 			__pfn_to_pfn_t(pfn, PFN_DEV));
 }

 static int exynos_drm_gem_mmap_obj(struct drm_gem_object *obj,
 				   struct vm_area_struct *vma)
 {
 	struct exynos_drm_gem *exynos_gem = to_exynos_gem(obj);
 	int ret;

 	DRM_DEV_DEBUG_KMS(to_dma_dev(obj->dev), "flags = 0x%x\n",
 			  exynos_gem->flags);

 	/* non-cachable as default. */
 	if (exynos_gem->flags & EXYNOS_BO_CACHABLE)
 		vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
 	else if (exynos_gem->flags & EXYNOS_BO_WC)
 		vma->vm_page_prot =
 			pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
 	else
 		vma->vm_page_prot =
 			pgprot_noncached(vm_get_page_prot(vma->vm_flags));

 	ret = exynos_drm_gem_mmap_buffer(exynos_gem, vma);
 	if (ret)
 		goto err_close_vm;

 	return ret;

 err_close_vm:
 	drm_gem_vm_close(vma);

 	return ret;
 }

 int exynos_drm_gem_mmap(struct file *filp, struct vm_area_struct *vma)
 {
 	struct drm_gem_object *obj;
 	int ret;

 	/* set vm_area_struct. */
 	ret = drm_gem_mmap(filp, vma);
 	if (ret < 0) {
 		DRM_ERROR("failed to mmap.\n");
 		return ret;
 	}

 	obj = vma->vm_private_data;

 	if (obj->import_attach)
 		return dma_buf_mmap(obj->dma_buf, vma, 0);

 	return exynos_drm_gem_mmap_obj(obj, vma);
 }

 /* low-level interface prime helpers */
 struct drm_gem_object *exynos_drm_gem_prime_import(struct drm_device *dev,
 					    struct dma_buf *dma_buf)
 {
 	return drm_gem_prime_import_dev(dev, dma_buf, to_dma_dev(dev));
 }

 struct sg_table *exynos_drm_gem_prime_get_sg_table(struct drm_gem_object *obj)
 {
 	struct exynos_drm_gem *exynos_gem = to_exynos_gem(obj);
 	int npages;

 	npages = exynos_gem->size >> PAGE_SHIFT;

 	return drm_prime_pages_to_sg(exynos_gem->pages, npages);
 }

 struct drm_gem_object *
 exynos_drm_gem_prime_import_sg_table(struct drm_device *dev,
 				     struct dma_buf_attachment *attach,
 				     struct sg_table *sgt)
 {
 	struct exynos_drm_gem *exynos_gem;
 	int npages;
 	int ret;

 	exynos_gem = exynos_drm_gem_init(dev, attach->dmabuf->size);
 	if (IS_ERR(exynos_gem)) {
 		ret = PTR_ERR(exynos_gem);
 		return ERR_PTR(ret);
 	}

 	exynos_gem->dma_addr = sg_dma_address(sgt->sgl);

 	npages = exynos_gem->size >> PAGE_SHIFT;
 	exynos_gem->pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
 	if (!exynos_gem->pages) {
 		ret = -ENOMEM;
 		goto err;
 	}

 	ret = drm_prime_sg_to_page_addr_arrays(sgt, exynos_gem->pages, NULL,
 					       npages);
 	if (ret < 0)
 		goto err_free_large;

 	exynos_gem->sgt = sgt;

 	if (sgt->nents == 1) {
 		/* always physically continuous memory if sgt->nents is 1. */
 		exynos_gem->flags |= EXYNOS_BO_CONTIG;
 	} else {
 		/*
 		 * this case could be CONTIG or NONCONTIG type but for now
 		 * sets NONCONTIG.
 		 * TODO. we have to find a way that exporter can notify
 		 * the type of its own buffer to importer.
 		 */
 		exynos_gem->flags |= EXYNOS_BO_NONCONTIG;
 	}

 	return &exynos_gem->base;

 err_free_large:
 	kvfree(exynos_gem->pages);
 err:
 	drm_gem_object_release(&exynos_gem->base);
 	kfree(exynos_gem);
 	return ERR_PTR(ret);
 }

 void *exynos_drm_gem_prime_vmap(struct drm_gem_object *obj)
 {
 	return NULL;
 }

 void exynos_drm_gem_prime_vunmap(struct drm_gem_object *obj, void *vaddr)
 {
 	/* Nothing to do */
 }

 int exynos_drm_gem_prime_mmap(struct drm_gem_object *obj,
 			      struct vm_area_struct *vma)
 {
 	int ret;

 	ret = drm_gem_mmap_obj(obj, obj->size, vma);
 	if (ret < 0)
 		return ret;

 	return exynos_drm_gem_mmap_obj(obj, vma);
 }
	/* exynos_drm_gem.c
	*
	* Copyright (c) 2011 Samsung Electronics Co., Ltd.
	* Author: Inki Dae <[email protected]>
	*
	* This program is free software; you can redistribute it and/or modify it
	* under the terms of the GNU General Public License as published by the
	* Free Software Foundation; either version 2 of the License, or (at your
	* option) any later version.
	*/

	#include <drm/drmP.h>
	#include <drm/drm_vma_manager.h>

	#include <linux/shmem_fs.h>
	#include <linux/dma-buf.h>
	#include <linux/pfn_t.h>
	#include <drm/exynos_drm.h>

	#include "exynos_drm_drv.h"
	#include "exynos_drm_gem.h"

	static int exynos_drm_alloc_buf(struct exynos_drm_gem *exynos_gem)
	{
	struct drm_device *dev = exynos_gem->base.dev;
	unsigned long attr;
	unsigned int nr_pages;
	struct sg_table sgt;
	int ret = -ENOMEM;

	if (exynos_gem->dma_addr) {
	DRM_DEV_DEBUG_KMS(to_dma_dev(dev), "already allocated.\n");
	return 0;
	}

	exynos_gem->dma_attrs = 0;

	/*
	* if EXYNOS_BO_CONTIG, fully physically contiguous memory
	* region will be allocated else physically contiguous
	* as possible.
	*/
	if (!(exynos_gem->flags & EXYNOS_BO_NONCONTIG))
	exynos_gem->dma_attrs \|= DMA_ATTR_FORCE_CONTIGUOUS;

	/*
	* if EXYNOS_BO_WC or EXYNOS_BO_NONCACHABLE, writecombine mapping
	* else cachable mapping.
	*/
	if (exynos_gem->flags & EXYNOS_BO_WC \|\|
	!(exynos_gem->flags & EXYNOS_BO_CACHABLE))
	attr = DMA_ATTR_WRITE_COMBINE;
	else
	attr = DMA_ATTR_NON_CONSISTENT;

	exynos_gem->dma_attrs \|= attr;
	exynos_gem->dma_attrs \|= DMA_ATTR_NO_KERNEL_MAPPING;

	nr_pages = exynos_gem->size >> PAGE_SHIFT;

	exynos_gem->pages = kvmalloc_array(nr_pages, sizeof(struct page *),
	GFP_KERNEL \| __GFP_ZERO);
	if (!exynos_gem->pages) {
	DRM_DEV_ERROR(to_dma_dev(dev), "failed to allocate pages.\n");
	return -ENOMEM;
	}

	exynos_gem->cookie = dma_alloc_attrs(to_dma_dev(dev), exynos_gem->size,
	&exynos_gem->dma_addr, GFP_KERNEL,
	exynos_gem->dma_attrs);
	if (!exynos_gem->cookie) {
	DRM_DEV_ERROR(to_dma_dev(dev), "failed to allocate buffer.\n");
	goto err_free;
	}

	ret = dma_get_sgtable_attrs(to_dma_dev(dev), &sgt, exynos_gem->cookie,
	exynos_gem->dma_addr, exynos_gem->size,
	exynos_gem->dma_attrs);
	if (ret < 0) {
	DRM_DEV_ERROR(to_dma_dev(dev), "failed to get sgtable.\n");
	goto err_dma_free;
	}

	if (drm_prime_sg_to_page_addr_arrays(&sgt, exynos_gem->pages, NULL,
	nr_pages)) {
	DRM_DEV_ERROR(to_dma_dev(dev), "invalid sgtable.\n");
	ret = -EINVAL;
	goto err_sgt_free;
	}

	sg_free_table(&sgt);

	DRM_DEV_DEBUG_KMS(to_dma_dev(dev), "dma_addr(0x%lx), size(0x%lx)\n",
	(unsigned long)exynos_gem->dma_addr, exynos_gem->size);

	return 0;

	err_sgt_free:
	sg_free_table(&sgt);
	err_dma_free:
	dma_free_attrs(to_dma_dev(dev), exynos_gem->size, exynos_gem->cookie,
	exynos_gem->dma_addr, exynos_gem->dma_attrs);
	err_free:
	kvfree(exynos_gem->pages);

	return ret;
	}

	static void exynos_drm_free_buf(struct exynos_drm_gem *exynos_gem)
	{
	struct drm_device *dev = exynos_gem->base.dev;

	if (!exynos_gem->dma_addr) {
	DRM_DEV_DEBUG_KMS(dev->dev, "dma_addr is invalid.\n");
	return;
	}

	DRM_DEV_DEBUG_KMS(dev->dev, "dma_addr(0x%lx), size(0x%lx)\n",
	(unsigned long)exynos_gem->dma_addr, exynos_gem->size);

	dma_free_attrs(to_dma_dev(dev), exynos_gem->size, exynos_gem->cookie,
	(dma_addr_t)exynos_gem->dma_addr,
	exynos_gem->dma_attrs);

	kvfree(exynos_gem->pages);
	}

	static int exynos_drm_gem_handle_create(struct drm_gem_object *obj,
	struct drm_file *file_priv,
	unsigned int *handle)
	{
	int ret;

	/*
	* allocate a id of idr table where the obj is registered
	* and handle has the id what user can see.
	*/
	ret = drm_gem_handle_create(file_priv, obj, handle);
	if (ret)
	return ret;

	DRM_DEV_DEBUG_KMS(to_dma_dev(obj->dev), "gem handle = 0x%x\n", *handle);

	/* drop reference from allocate - handle holds it now. */
	drm_gem_object_put_unlocked(obj);

	return 0;
	}

	void exynos_drm_gem_destroy(struct exynos_drm_gem *exynos_gem)
	{
	struct drm_gem_object *obj = &exynos_gem->base;

	DRM_DEV_DEBUG_KMS(to_dma_dev(obj->dev), "handle count = %d\n",
	obj->handle_count);

	/*
	* do not release memory region from exporter.
	*
	* the region will be released by exporter
	* once dmabuf's refcount becomes 0.
	*/
	if (obj->import_attach)
	drm_prime_gem_destroy(obj, exynos_gem->sgt);
	else
	exynos_drm_free_buf(exynos_gem);

	/* release file pointer to gem object. */
	drm_gem_object_release(obj);

	kfree(exynos_gem);
	}

	static struct exynos_drm_gem exynos_drm_gem_init(struct drm_device dev,
	unsigned long size)
	{
	struct exynos_drm_gem *exynos_gem;
	struct drm_gem_object *obj;
	int ret;

	exynos_gem = kzalloc(sizeof(*exynos_gem), GFP_KERNEL);
	if (!exynos_gem)
	return ERR_PTR(-ENOMEM);

	exynos_gem->size = size;
	obj = &exynos_gem->base;

	ret = drm_gem_object_init(dev, obj, size);
	if (ret < 0) {
	DRM_DEV_ERROR(dev->dev, "failed to initialize gem object\n");
	kfree(exynos_gem);
	return ERR_PTR(ret);
	}

	ret = drm_gem_create_mmap_offset(obj);
	if (ret < 0) {
	drm_gem_object_release(obj);
	kfree(exynos_gem);
	return ERR_PTR(ret);
	}

	DRM_DEV_DEBUG_KMS(dev->dev, "created file object = %pK\n", obj->filp);

	return exynos_gem;
	}

	struct exynos_drm_gem exynos_drm_gem_create(struct drm_device dev,
	unsigned int flags,
	unsigned long size)
	{
	struct exynos_drm_gem *exynos_gem;
	int ret;

	if (flags & ~(EXYNOS_BO_MASK)) {
	DRM_DEV_ERROR(dev->dev,
	"invalid GEM buffer flags: %u\n", flags);
	return ERR_PTR(-EINVAL);
	}

	if (!size) {
	DRM_DEV_ERROR(dev->dev, "invalid GEM buffer size: %lu\n", size);
	return ERR_PTR(-EINVAL);
	}

	size = roundup(size, PAGE_SIZE);

	exynos_gem = exynos_drm_gem_init(dev, size);
	if (IS_ERR(exynos_gem))
	return exynos_gem;

	if (!is_drm_iommu_supported(dev) && (flags & EXYNOS_BO_NONCONTIG)) {
	/*
	* when no IOMMU is available, all allocated buffers are
	* contiguous anyway, so drop EXYNOS_BO_NONCONTIG flag
	*/
	flags &= ~EXYNOS_BO_NONCONTIG;
	DRM_WARN("Non-contiguous allocation is not supported without IOMMU, falling back to contiguous buffer\n");
	}

	/* set memory type and cache attribute from user side. */
	exynos_gem->flags = flags;

	ret = exynos_drm_alloc_buf(exynos_gem);
	if (ret < 0) {
	drm_gem_object_release(&exynos_gem->base);
	kfree(exynos_gem);
	return ERR_PTR(ret);
	}

	return exynos_gem;
	}

	int exynos_drm_gem_create_ioctl(struct drm_device dev, void data,
	struct drm_file *file_priv)
	{
	struct drm_exynos_gem_create *args = data;
	struct exynos_drm_gem *exynos_gem;
	int ret;

	exynos_gem = exynos_drm_gem_create(dev, args->flags, args->size);
	if (IS_ERR(exynos_gem))
	return PTR_ERR(exynos_gem);

	ret = exynos_drm_gem_handle_create(&exynos_gem->base, file_priv,
	&args->handle);
	if (ret) {
	exynos_drm_gem_destroy(exynos_gem);
	return ret;
	}

	return 0;
	}

	int exynos_drm_gem_map_ioctl(struct drm_device dev, void data,
	struct drm_file *file_priv)
	{
	struct drm_exynos_gem_map *args = data;

	return drm_gem_dumb_map_offset(file_priv, dev, args->handle,
	&args->offset);
	}

	struct exynos_drm_gem exynos_drm_gem_get(struct drm_file filp,
	unsigned int gem_handle)
	{
	struct drm_gem_object *obj;

	obj = drm_gem_object_lookup(filp, gem_handle);
	if (!obj)
	return NULL;
	return to_exynos_gem(obj);
	}

	static int exynos_drm_gem_mmap_buffer(struct exynos_drm_gem *exynos_gem,
	struct vm_area_struct *vma)
	{
	struct drm_device *drm_dev = exynos_gem->base.dev;
	unsigned long vm_size;
	int ret;

	vma->vm_flags &= ~VM_PFNMAP;
	vma->vm_pgoff = 0;

	vm_size = vma->vm_end - vma->vm_start;

	/* check if user-requested size is valid. */
	if (vm_size > exynos_gem->size)
	return -EINVAL;

	ret = dma_mmap_attrs(to_dma_dev(drm_dev), vma, exynos_gem->cookie,
	exynos_gem->dma_addr, exynos_gem->size,
	exynos_gem->dma_attrs);
	if (ret < 0) {
	DRM_ERROR("failed to mmap.\n");
	return ret;
	}

	return 0;
	}

	int exynos_drm_gem_get_ioctl(struct drm_device dev, void data,
	struct drm_file *file_priv)
	{
	struct exynos_drm_gem *exynos_gem;
	struct drm_exynos_gem_info *args = data;
	struct drm_gem_object *obj;

	obj = drm_gem_object_lookup(file_priv, args->handle);
	if (!obj) {
	DRM_DEV_ERROR(dev->dev, "failed to lookup gem object.\n");
	return -EINVAL;
	}

	exynos_gem = to_exynos_gem(obj);

	args->flags = exynos_gem->flags;
	args->size = exynos_gem->size;

	drm_gem_object_put_unlocked(obj);

	return 0;
	}

	void exynos_drm_gem_free_object(struct drm_gem_object *obj)
	{
	exynos_drm_gem_destroy(to_exynos_gem(obj));
	}

	int exynos_drm_gem_dumb_create(struct drm_file *file_priv,
	struct drm_device *dev,
	struct drm_mode_create_dumb *args)
	{
	struct exynos_drm_gem *exynos_gem;
	unsigned int flags;
	int ret;

	/*
	* allocate memory to be used for framebuffer.
	* - this callback would be called by user application
	* with DRM_IOCTL_MODE_CREATE_DUMB command.
	*/

	args->pitch = args->width * ((args->bpp + 7) / 8);
	args->size = args->pitch * args->height;

	if (is_drm_iommu_supported(dev))
	flags = EXYNOS_BO_NONCONTIG \| EXYNOS_BO_WC;
	else
	flags = EXYNOS_BO_CONTIG \| EXYNOS_BO_WC;

	exynos_gem = exynos_drm_gem_create(dev, flags, args->size);
	if (IS_ERR(exynos_gem)) {
	dev_warn(dev->dev, "FB allocation failed.\n");
	return PTR_ERR(exynos_gem);
	}

	ret = exynos_drm_gem_handle_create(&exynos_gem->base, file_priv,
	&args->handle);
	if (ret) {
	exynos_drm_gem_destroy(exynos_gem);
	return ret;
	}

	return 0;
	}

	vm_fault_t exynos_drm_gem_fault(struct vm_fault *vmf)
	{
	struct vm_area_struct *vma = vmf->vma;
	struct drm_gem_object *obj = vma->vm_private_data;
	struct exynos_drm_gem *exynos_gem = to_exynos_gem(obj);
	unsigned long pfn;
	pgoff_t page_offset;

	page_offset = (vmf->address - vma->vm_start) >> PAGE_SHIFT;

	if (page_offset >= (exynos_gem->size >> PAGE_SHIFT)) {
	DRM_ERROR("invalid page offset\n");
	return VM_FAULT_SIGBUS;
	}

	pfn = page_to_pfn(exynos_gem->pages[page_offset]);
	return vmf_insert_mixed(vma, vmf->address,
	__pfn_to_pfn_t(pfn, PFN_DEV));
	}

	static int exynos_drm_gem_mmap_obj(struct drm_gem_object *obj,
	struct vm_area_struct *vma)
	{
	struct exynos_drm_gem *exynos_gem = to_exynos_gem(obj);
	int ret;

	DRM_DEV_DEBUG_KMS(to_dma_dev(obj->dev), "flags = 0x%x\n",
	exynos_gem->flags);

	/* non-cachable as default. */
	if (exynos_gem->flags & EXYNOS_BO_CACHABLE)
	vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
	else if (exynos_gem->flags & EXYNOS_BO_WC)
	vma->vm_page_prot =
	pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
	else
	vma->vm_page_prot =
	pgprot_noncached(vm_get_page_prot(vma->vm_flags));

	ret = exynos_drm_gem_mmap_buffer(exynos_gem, vma);
	if (ret)
	goto err_close_vm;

	return ret;

	err_close_vm:
	drm_gem_vm_close(vma);

	return ret;
	}

	int exynos_drm_gem_mmap(struct file filp, struct vm_area_struct vma)
	{
	struct drm_gem_object *obj;
	int ret;

	/* set vm_area_struct. */
	ret = drm_gem_mmap(filp, vma);
	if (ret < 0) {
	DRM_ERROR("failed to mmap.\n");
	return ret;
	}

	obj = vma->vm_private_data;

	if (obj->import_attach)
	return dma_buf_mmap(obj->dma_buf, vma, 0);

	return exynos_drm_gem_mmap_obj(obj, vma);
	}

	/* low-level interface prime helpers */
	struct drm_gem_object exynos_drm_gem_prime_import(struct drm_device dev,
	struct dma_buf *dma_buf)
	{
	return drm_gem_prime_import_dev(dev, dma_buf, to_dma_dev(dev));
	}

	struct sg_table exynos_drm_gem_prime_get_sg_table(struct drm_gem_object obj)
	{
	struct exynos_drm_gem *exynos_gem = to_exynos_gem(obj);
	int npages;

	npages = exynos_gem->size >> PAGE_SHIFT;

	return drm_prime_pages_to_sg(exynos_gem->pages, npages);
	}

	struct drm_gem_object *
	exynos_drm_gem_prime_import_sg_table(struct drm_device *dev,
	struct dma_buf_attachment *attach,
	struct sg_table *sgt)
	{
	struct exynos_drm_gem *exynos_gem;
	int npages;
	int ret;

	exynos_gem = exynos_drm_gem_init(dev, attach->dmabuf->size);
	if (IS_ERR(exynos_gem)) {
	ret = PTR_ERR(exynos_gem);
	return ERR_PTR(ret);
	}

	exynos_gem->dma_addr = sg_dma_address(sgt->sgl);

	npages = exynos_gem->size >> PAGE_SHIFT;
	exynos_gem->pages = kvmalloc_array(npages, sizeof(struct page *), GFP_KERNEL);
	if (!exynos_gem->pages) {
	ret = -ENOMEM;
	goto err;
	}

	ret = drm_prime_sg_to_page_addr_arrays(sgt, exynos_gem->pages, NULL,
	npages);
	if (ret < 0)
	goto err_free_large;

	exynos_gem->sgt = sgt;

	if (sgt->nents == 1) {
	/* always physically continuous memory if sgt->nents is 1. */
	exynos_gem->flags \|= EXYNOS_BO_CONTIG;
	} else {
	/*
	* this case could be CONTIG or NONCONTIG type but for now
	* sets NONCONTIG.
	* TODO. we have to find a way that exporter can notify
	* the type of its own buffer to importer.
	*/
	exynos_gem->flags \|= EXYNOS_BO_NONCONTIG;
	}

	return &exynos_gem->base;

	err_free_large:
	kvfree(exynos_gem->pages);
	err:
	drm_gem_object_release(&exynos_gem->base);
	kfree(exynos_gem);
	return ERR_PTR(ret);
	}

	void exynos_drm_gem_prime_vmap(struct drm_gem_object obj)
	{
	return NULL;
	}

	void exynos_drm_gem_prime_vunmap(struct drm_gem_object obj, void vaddr)
	{
	/* Nothing to do */
	}

	int exynos_drm_gem_prime_mmap(struct drm_gem_object *obj,
	struct vm_area_struct *vma)
	{
	int ret;

	ret = drm_gem_mmap_obj(obj, obj->size, vma);
	if (ret < 0)
	return ret;

	return exynos_drm_gem_mmap_obj(obj, vma);
	}