Initial commit; kernel source import

drivers/video/tegra/nvmap/Makefile

@@ -0,0 +1,8 @@
+GCOV_PROFILE := y
+obj-y += nvmap.o
+obj-y += nvmap_dev.o
+obj-y += nvmap_handle.o
+obj-y += nvmap_heap.o
+obj-y += nvmap_ioctl.o
+obj-y += nvmap_dmabuf.o
+obj-y += nvmap_mm.o

drivers/video/tegra/nvmap/nvmap.c

@@ -0,0 +1,527 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap.c
+ *
+ * Memory manager for Tegra GPU
+ *
+ * Copyright (c) 2009-2013, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#define pr_fmt(fmt)	"nvmap: %s() " fmt, __func__
+
+#include <linux/err.h>
+#include <linux/highmem.h>
+#include <linux/io.h>
+#include <linux/rbtree.h>
+#include <linux/vmalloc.h>
+#include <linux/wait.h>
+#include <linux/slab.h>
+#include <linux/export.h>
+
+#include <asm/pgtable.h>
+#include <asm/tlbflush.h>
+
+#include <linux/nvmap.h>
+#include <trace/events/nvmap.h>
+
+#include "nvmap_priv.h"
+
+/* private nvmap_handle flag for pinning duplicate detection */
+#define NVMAP_HANDLE_VISITED (0x1ul << 31)
+
+static phys_addr_t handle_phys(struct nvmap_handle *h)
+{
+	phys_addr_t addr;
+
+	if (h->heap_pgalloc && h->pgalloc.contig) {
+		addr = page_to_phys(h->pgalloc.pages[0]);
+	} else if (h->heap_pgalloc) {
+		BUG_ON(!h->attachment->priv);
+		addr = sg_dma_address(
+				((struct sg_table *)h->attachment->priv)->sgl);
+	} else {
+		addr = h->carveout->base;
+	}
+
+	return addr;
+}
+
+/*
+ * Do the actual pin. Just calls to the dma_buf code.
+ */
+int __nvmap_pin(struct nvmap_handle_ref *ref, phys_addr_t *phys)
+{
+	struct nvmap_handle *h = ref->handle;
+	struct sg_table *sgt = NULL;
+
+	atomic_inc(&ref->pin);
+
+	/*
+	 * We should not be using a bidirectional mapping here; however, nvmap
+	 * does not really keep track of whether memory is readable, writable,
+	 * or both so this keeps everyone happy.
+	 */
+	sgt = dma_buf_map_attachment(h->attachment, DMA_BIDIRECTIONAL);
+	if (IS_ERR(sgt))
+		goto err;
+	*phys = sg_dma_address(sgt->sgl);
+	trace_nvmap_pin(h->owner, h->owner ? h->owner->name : "unknown", h,
+			atomic_read(&h->pin));
+	return 0;
+
+err:
+	atomic_dec(&ref->pin);
+	return PTR_ERR(sgt);
+}
+
+void __nvmap_unpin(struct nvmap_handle_ref *ref)
+{
+	struct nvmap_handle *h = ref->handle;
+
+	/*
+	 * If the handle has been pinned by other refs it is possible to arrive
+	 * here: the passed ref has a 0 pin count. This is of course invalid.
+	 */
+	if (!atomic_add_unless(&ref->pin, -1, 0))
+		goto done;
+
+	dma_buf_unmap_attachment(h->attachment,
+		h->attachment->priv, DMA_BIDIRECTIONAL);
+
+done:
+	trace_nvmap_unpin(h->owner, h->owner ? h->owner->name : "unknown", h,
+			atomic_read(&h->pin));
+}
+
+int nvmap_pin_ids(struct nvmap_client *client, unsigned int nr,
+		  const unsigned long *ids)
+{
+	int i, err = 0;
+	phys_addr_t phys;
+	struct nvmap_handle_ref *ref;
+
+	/*
+	 * Just try and pin every handle.
+	 */
+	nvmap_ref_lock(client);
+	for (i = 0; i < nr; i++) {
+		if (!ids[i] || !virt_addr_valid(ids[i]))
+			continue;
+
+		ref = __nvmap_validate_id_locked(client, ids[i]);
+		if (!ref) {
+			err = -EPERM;
+			goto err_cleanup;
+		}
+		if (!ref->handle) {
+			err = -EINVAL;
+			goto err_cleanup;
+		}
+
+		err = __nvmap_pin(ref, &phys);
+		if (err)
+			goto err_cleanup;
+	}
+	nvmap_ref_unlock(client);
+	return 0;
+
+err_cleanup:
+	for (--i; i >= 0; i--) {
+		if (!ids[i] || !virt_addr_valid(ids[i]))
+			continue;
+
+		/*
+		 * We will get the ref again - the ref lock has yet to be given
+		 * up so if this worked the first time it will work again.
+		 */
+		ref = __nvmap_validate_id_locked(client, ids[i]);
+		__nvmap_unpin(ref);
+	}
+	nvmap_ref_unlock(client);
+	return err;
+}
+
+/*
+ * This will unpin every handle. If an error occurs on a handle later handles
+ * will still be unpinned.
+ */
+void nvmap_unpin_ids(struct nvmap_client *client, unsigned int nr,
+		     const unsigned long *ids)
+{
+	int i;
+	struct nvmap_handle_ref *ref;
+
+	nvmap_ref_lock(client);
+	for (i = 0; i < nr; i++) {
+		if (!ids[i] || !virt_addr_valid(ids[i]))
+			continue;
+
+		ref = __nvmap_validate_id_locked(client, ids[i]);
+		if (!ref) {
+			pr_info("ref is null during unpin.\n");
+			continue;
+		}
+		if (!ref->handle) {
+			WARN(1, "ref->handle is NULL.\n");
+			continue;
+		}
+
+		__nvmap_unpin(ref);
+	}
+	nvmap_ref_unlock(client);
+}
+
+void *__nvmap_kmap(struct nvmap_handle *h, unsigned int pagenum)
+{
+	phys_addr_t paddr;
+	unsigned long kaddr;
+	pgprot_t prot;
+	pte_t **pte;
+
+	if (!virt_addr_valid(h))
+		return NULL;
+
+	h = nvmap_handle_get(h);
+	if (!h)
+		return NULL;
+
+	if (!h->alloc)
+		goto out;
+
+	if (pagenum >= h->size >> PAGE_SHIFT)
+		goto out;
+	prot = nvmap_pgprot(h, pgprot_kernel);
+	pte = nvmap_alloc_pte(nvmap_dev, (void **)&kaddr);
+	if (!pte)
+		goto out;
+
+	if (h->heap_pgalloc)
+		paddr = page_to_phys(h->pgalloc.pages[pagenum]);
+	else
+		paddr = h->carveout->base + pagenum * PAGE_SIZE;
+
+	set_pte_at(&init_mm, kaddr, *pte,
+				pfn_pte(__phys_to_pfn(paddr), prot));
+	nvmap_flush_tlb_kernel_page(kaddr);
+	return (void *)kaddr;
+out:
+	nvmap_handle_put(h);
+	return NULL;
+}
+
+void __nvmap_kunmap(struct nvmap_handle *h, unsigned int pagenum,
+		  void *addr)
+{
+	phys_addr_t paddr;
+	pte_t **pte;
+
+	if (!h || !h->alloc ||
+	    WARN_ON(!virt_addr_valid(h)) ||
+	    WARN_ON(!addr))
+		return;
+
+	if (WARN_ON(pagenum >= h->size >> PAGE_SHIFT))
+		return;
+
+	if (nvmap_find_cache_maint_op(h->dev, h)) {
+		struct nvmap_share *share = nvmap_get_share_from_dev(h->dev);
+		/* acquire pin lock to ensure maintenance is done before
+		 * handle is pinned */
+		mutex_lock(&share->pin_lock);
+		nvmap_cache_maint_ops_flush(h->dev, h);
+		mutex_unlock(&share->pin_lock);
+	}
+
+	if (h->heap_pgalloc)
+		paddr = page_to_phys(h->pgalloc.pages[pagenum]);
+	else
+		paddr = h->carveout->base + pagenum * PAGE_SIZE;
+
+	if (h->flags != NVMAP_HANDLE_UNCACHEABLE &&
+	    h->flags != NVMAP_HANDLE_WRITE_COMBINE) {
+		dmac_flush_range(addr, addr + PAGE_SIZE);
+#ifndef CONFIG_ARM64
+		outer_flush_range(paddr, paddr + PAGE_SIZE); /* FIXME */
+#endif
+	}
+
+	pte = nvmap_vaddr_to_pte(nvmap_dev, (unsigned long)addr);
+	nvmap_free_pte(nvmap_dev, pte);
+	nvmap_handle_put(h);
+}
+
+void *__nvmap_mmap(struct nvmap_handle *h)
+{
+	pgprot_t prot;
+	unsigned long adj_size;
+	unsigned long offs;
+	struct vm_struct *v;
+	void *p;
+
+	if (!virt_addr_valid(h))
+		return NULL;
+
+	h = nvmap_handle_get(h);
+	if (!h)
+		return NULL;
+
+	if (!h->alloc) {
+		nvmap_handle_put(h);
+		return NULL;
+	}
+
+	prot = nvmap_pgprot(h, pgprot_kernel);
+
+	if (h->heap_pgalloc)
+		return vm_map_ram(h->pgalloc.pages, h->size >> PAGE_SHIFT,
+				  -1, prot);
+
+	/* carveout - explicitly map the pfns into a vmalloc area */
+
+	adj_size = h->carveout->base & ~PAGE_MASK;
+	adj_size += h->size;
+	adj_size = PAGE_ALIGN(adj_size);
+
+	v = alloc_vm_area(adj_size, 0);
+	if (!v) {
+		nvmap_handle_put(h);
+		return NULL;
+	}
+
+	p = v->addr + (h->carveout->base & ~PAGE_MASK);
+
+	for (offs = 0; offs < adj_size; offs += PAGE_SIZE) {
+		unsigned long addr = (unsigned long) v->addr + offs;
+		unsigned int pfn;
+		pgd_t *pgd;
+		pud_t *pud;
+		pmd_t *pmd;
+		pte_t *pte;
+
+		pfn = __phys_to_pfn(h->carveout->base + offs);
+		pgd = pgd_offset_k(addr);
+		pud = pud_alloc(&init_mm, pgd, addr);
+		if (!pud)
+			break;
+		pmd = pmd_alloc(&init_mm, pud, addr);
+		if (!pmd)
+			break;
+		pte = pte_alloc_kernel(pmd, addr);
+		if (!pte)
+			break;
+		set_pte_at(&init_mm, addr, pte, pfn_pte(pfn, prot));
+		nvmap_flush_tlb_kernel_page(addr);
+	}
+
+	if (offs != adj_size) {
+		free_vm_area(v);
+		nvmap_handle_put(h);
+		return NULL;
+	}
+
+	/* leave the handle ref count incremented by 1, so that
+	 * the handle will not be freed while the kernel mapping exists.
+	 * nvmap_handle_put will be called by unmapping this address */
+	return p;
+}
+
+void __nvmap_munmap(struct nvmap_handle *h, void *addr)
+{
+	if (!h || !h->alloc ||
+	    WARN_ON(!virt_addr_valid(h)) ||
+	    WARN_ON(!addr))
+		return;
+
+	if (nvmap_find_cache_maint_op(h->dev, h)) {
+		struct nvmap_share *share = nvmap_get_share_from_dev(h->dev);
+		/* acquire pin lock to ensure maintenance is done before
+		 * handle is pinned */
+		mutex_lock(&share->pin_lock);
+		nvmap_cache_maint_ops_flush(h->dev, h);
+		mutex_unlock(&share->pin_lock);
+	}
+
+	/* Handle can be locked by cache maintenance in
+	 * separate thread */
+	if (h->heap_pgalloc) {
+		vm_unmap_ram(addr, h->size >> PAGE_SHIFT);
+	} else {
+		struct vm_struct *vm;
+		addr -= (h->carveout->base & ~PAGE_MASK);
+		vm = remove_vm_area(addr);
+		BUG_ON(!vm);
+		kfree(vm);
+	}
+	nvmap_handle_put(h);
+}
+
+static struct nvmap_client *nvmap_get_dmabuf_client(void)
+{
+	static struct nvmap_client *client;
+
+	if (!client) {
+		struct nvmap_client *temp;
+
+		temp = __nvmap_create_client(nvmap_dev, "dmabuf_client");
+		if (!temp)
+			return NULL;
+		if (cmpxchg(&client, NULL, temp))
+			nvmap_client_put(temp);
+	}
+	BUG_ON(!client);
+	return client;
+}
+
+static struct nvmap_handle_ref *__nvmap_alloc(struct nvmap_client *client,
+					      size_t size, size_t align,
+					      unsigned int flags,
+					      unsigned int heap_mask)
+{
+	const unsigned int default_heap = NVMAP_HEAP_CARVEOUT_GENERIC;
+	struct nvmap_handle_ref *r = NULL;
+	int err;
+
+	if (!virt_addr_valid(client))
+		return ERR_PTR(-EINVAL);
+
+	if (heap_mask == 0)
+		heap_mask = default_heap;
+
+	r = nvmap_create_handle(client, size);
+	if (IS_ERR(r))
+		return r;
+
+	err = nvmap_alloc_handle_id(client, __nvmap_ref_to_id(r),
+				    heap_mask, align,
+				    0, /* kind n/a */
+				    flags & ~(NVMAP_HANDLE_KIND_SPECIFIED |
+					      NVMAP_HANDLE_COMPR_SPECIFIED));
+
+	if (err) {
+		nvmap_free_handle_id(client, __nvmap_ref_to_id(r));
+		return ERR_PTR(err);
+	}
+
+	return r;
+}
+
+static void __nvmap_free(struct nvmap_client *client,
+			 struct nvmap_handle_ref *r)
+{
+	unsigned long ref_id = __nvmap_ref_to_id(r);
+
+	if (!r ||
+	    WARN_ON(!virt_addr_valid(client)) ||
+	    WARN_ON(!virt_addr_valid(r)) ||
+	    WARN_ON(!virt_addr_valid(ref_id)))
+		return;
+
+	nvmap_free_handle_id(client, ref_id);
+}
+
+struct dma_buf *nvmap_alloc_dmabuf(size_t size, size_t align,
+				   unsigned int flags,
+				   unsigned int heap_mask)
+{
+	struct dma_buf *dmabuf;
+	struct nvmap_handle_ref *ref;
+	struct nvmap_client *client = nvmap_get_dmabuf_client();
+
+	ref = __nvmap_alloc(client, size, align, flags, heap_mask);
+	if (!ref)
+		return ERR_PTR(-ENOMEM);
+	if (IS_ERR(ref))
+		return (struct dma_buf *)ref;
+
+	dmabuf = __nvmap_dmabuf_export_from_ref(ref);
+	__nvmap_free(client, ref);
+	return dmabuf;
+}
+
+void nvmap_handle_put(struct nvmap_handle *h)
+{
+	int cnt;
+
+	if (WARN_ON(!virt_addr_valid(h)))
+		return;
+	cnt = atomic_dec_return(&h->ref);
+
+	if (WARN_ON(cnt < 0)) {
+		pr_err("%s: %s put to negative references\n",
+			__func__, current->comm);
+	} else if (cnt == 0)
+		_nvmap_handle_free(h);
+}
+
+struct sg_table *__nvmap_sg_table(struct nvmap_client *client,
+		struct nvmap_handle *h)
+{
+	struct sg_table *sgt = NULL;
+	int err, npages;
+
+	if (!virt_addr_valid(h))
+		return ERR_PTR(-EINVAL);
+
+	h = nvmap_handle_get(h);
+	if (!h)
+		return ERR_PTR(-EINVAL);
+
+	if (!h->alloc) {
+		nvmap_handle_put(h);
+		return ERR_PTR(-EINVAL);
+	}
+
+	npages = PAGE_ALIGN(h->size) >> PAGE_SHIFT;
+	sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
+	if (!sgt) {
+		err = -ENOMEM;
+		goto err;
+	}
+
+	if (!h->heap_pgalloc) {
+		err = sg_alloc_table(sgt, 1, GFP_KERNEL);
+		if (err)
+			goto err;
+		sg_set_buf(sgt->sgl, phys_to_virt(handle_phys(h)), h->size);
+	} else {
+		err = sg_alloc_table_from_pages(sgt, h->pgalloc.pages,
+				npages, 0, h->size, GFP_KERNEL);
+		if (err)
+			goto err;
+	}
+	if (atomic_read(&h->disable_deferred_cache) <= 1) {
+		/* disable deferred cache maint */
+		atomic_set(&h->disable_deferred_cache, 1);
+		if (nvmap_find_cache_maint_op(nvmap_dev, h))
+			nvmap_cache_maint_ops_flush(nvmap_dev, h);
+		/* avoid unnecessary check for deferred cache maint */
+		atomic_set(&h->disable_deferred_cache, 2);
+	}
+	nvmap_handle_put(h);
+	return sgt;
+
+err:
+	kfree(sgt);
+	nvmap_handle_put(h);
+	return ERR_PTR(err);
+}
+
+void __nvmap_free_sg_table(struct nvmap_client *client,
+		struct nvmap_handle *h, struct sg_table *sgt)
+{
+	sg_free_table(sgt);
+	kfree(sgt);
+}

drivers/video/tegra/nvmap/nvmap_common.h

@@ -0,0 +1,37 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_common.h
+ *
+ * GPU memory management driver for Tegra
+ *
+ * Copyright (c) 2011, NVIDIA Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *'
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include <asm/cacheflush.h>
+
+extern void v7_clean_kern_cache_all(void *);
+
+#define FLUSH_CLEAN_BY_SET_WAY_THRESHOLD (8 * PAGE_SIZE)
+
+static inline void inner_flush_cache_all(void)
+{
+	on_each_cpu((smp_call_func_t)v7_flush_kern_cache_all, NULL, 1);
+}
+
+static inline void inner_clean_cache_all(void)
+{
+	on_each_cpu(v7_clean_kern_cache_all, NULL, 1);
+}

drivers/video/tegra/nvmap/nvmap_heap.c

@@ -0,0 +1,474 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_heap.c
+ *
+ * GPU heap allocator.
+ *
+ * Copyright (c) 2011-2013, NVIDIA Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#define pr_fmt(fmt)	"%s: " fmt, __func__
+
+#include <linux/device.h>
+#include <linux/kernel.h>
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/mutex.h>
+#include <linux/slab.h>
+#include <linux/err.h>
+#include <linux/bug.h>
+#include <linux/stat.h>
+
+#include <linux/nvmap.h>
+#include "nvmap_priv.h"
+#include "nvmap_heap.h"
+
+#include <asm/tlbflush.h>
+#include <asm/cacheflush.h>
+
+#include <linux/dma-mapping.h>
+
+/*
+ * "carveouts" are platform-defined regions of physically contiguous memory
+ * which are not managed by the OS. a platform may specify multiple carveouts,
+ * for either small special-purpose memory regions (like IRAM on Tegra SoCs)
+ * or reserved regions of main system memory.
+ *
+ * the carveout allocator returns allocations which are physically contiguous.
+ */
+
+enum block_type {
+	BLOCK_FIRST_FIT,	/* block was allocated directly from the heap */
+	BLOCK_EMPTY,
+};
+
+struct heap_stat {
+	size_t free;		/* total free size */
+	size_t free_largest;	/* largest free block */
+	size_t free_count;	/* number of free blocks */
+	size_t total;		/* total size */
+	size_t largest;		/* largest unique block */
+	size_t count;		/* total number of blocks */
+	/* fast compaction attempt counter */
+	unsigned int compaction_count_fast;
+	/* full compaction attempt counter */
+	unsigned int compaction_count_full;
+};
+
+struct list_block {
+	struct nvmap_heap_block block;
+	struct list_head all_list;
+	unsigned int mem_prot;
+	phys_addr_t orig_addr;
+	size_t size;
+	size_t align;
+	struct nvmap_heap *heap;
+	struct list_head free_list;
+};
+
+struct nvmap_heap {
+	struct list_head all_list;
+	struct list_head free_list;
+	struct mutex lock;
+	const char *name;
+	void *arg;
+	struct device dev;
+};
+
+static struct kmem_cache *heap_block_cache;
+
+/* returns the free size of the heap (must be called while holding the parent
+ * heap's lock. */
+static phys_addr_t heap_stat(struct nvmap_heap *heap, struct heap_stat *stat)
+{
+	struct list_block *l = NULL;
+	phys_addr_t base = -1ul;
+
+	memset(stat, 0, sizeof(*stat));
+	mutex_lock(&heap->lock);
+	list_for_each_entry(l, &heap->all_list, all_list) {
+		stat->total += l->size;
+		stat->largest = max(l->size, stat->largest);
+		stat->count++;
+		base = min(base, l->orig_addr);
+	}
+
+	list_for_each_entry(l, &heap->free_list, free_list) {
+		stat->free += l->size;
+		stat->free_count++;
+		stat->free_largest = max(l->size, stat->free_largest);
+	}
+	mutex_unlock(&heap->lock);
+
+	return base;
+}
+
+static ssize_t heap_name_show(struct device *dev,
+			      struct device_attribute *attr, char *buf);
+
+static ssize_t heap_stat_show(struct device *dev,
+			      struct device_attribute *attr, char *buf);
+
+static struct device_attribute heap_stat_total_max =
+	__ATTR(total_max, S_IRUGO, heap_stat_show, NULL);
+
+static struct device_attribute heap_stat_total_count =
+	__ATTR(total_count, S_IRUGO, heap_stat_show, NULL);
+
+static struct device_attribute heap_stat_total_size =
+	__ATTR(total_size, S_IRUGO, heap_stat_show, NULL);
+
+static struct device_attribute heap_stat_free_max =
+	__ATTR(free_max, S_IRUGO, heap_stat_show, NULL);
+
+static struct device_attribute heap_stat_free_count =
+	__ATTR(free_count, S_IRUGO, heap_stat_show, NULL);
+
+static struct device_attribute heap_stat_free_size =
+	__ATTR(free_size, S_IRUGO, heap_stat_show, NULL);
+
+static struct device_attribute heap_stat_base =
+	__ATTR(base, S_IRUGO, heap_stat_show, NULL);
+
+static struct device_attribute heap_attr_name =
+	__ATTR(name, S_IRUGO, heap_name_show, NULL);
+
+static struct attribute *heap_stat_attrs[] = {
+	&heap_stat_total_max.attr,
+	&heap_stat_total_count.attr,
+	&heap_stat_total_size.attr,
+	&heap_stat_free_max.attr,
+	&heap_stat_free_count.attr,
+	&heap_stat_free_size.attr,
+	&heap_stat_base.attr,
+	&heap_attr_name.attr,
+	NULL,
+};
+
+static struct attribute_group heap_stat_attr_group = {
+	.attrs	= heap_stat_attrs,
+};
+
+static ssize_t heap_name_show(struct device *dev,
+			      struct device_attribute *attr, char *buf)
+{
+
+	struct nvmap_heap *heap = container_of(dev, struct nvmap_heap, dev);
+	return sprintf(buf, "%s\n", heap->name);
+}
+
+static ssize_t heap_stat_show(struct device *dev,
+			      struct device_attribute *attr, char *buf)
+{
+	struct nvmap_heap *heap = container_of(dev, struct nvmap_heap, dev);
+	struct heap_stat stat;
+	phys_addr_t base;
+
+	base = heap_stat(heap, &stat);
+
+	if (attr == &heap_stat_total_max)
+		return sprintf(buf, "%zu\n", stat.largest);
+	else if (attr == &heap_stat_total_count)
+		return sprintf(buf, "%zu\n", stat.count);
+	else if (attr == &heap_stat_total_size)
+		return sprintf(buf, "%zu\n", stat.total);
+	else if (attr == &heap_stat_free_max)
+		return sprintf(buf, "%zu\n", stat.free_largest);
+	else if (attr == &heap_stat_free_count)
+		return sprintf(buf, "%zu\n", stat.free_count);
+	else if (attr == &heap_stat_free_size)
+		return sprintf(buf, "%zu\n", stat.free);
+	else if (attr == &heap_stat_base)
+		return sprintf(buf, "%08llx\n", (unsigned long long)base);
+	else
+		return -EINVAL;
+}
+
+/*
+ * base_max limits position of allocated chunk in memory.
+ * if base_max is 0 then there is no such limitation.
+ */
+static struct nvmap_heap_block *do_heap_alloc(struct nvmap_heap *heap,
+					      size_t len, size_t align,
+					      unsigned int mem_prot,
+					      phys_addr_t base_max)
+{
+	struct list_block *heap_block = NULL;
+	void *dev_addr = NULL;
+	dma_addr_t dev_base;
+
+	/* since pages are only mappable with one cache attribute,
+	 * and most allocations from carveout heaps are DMA coherent
+	 * (i.e., non-cacheable), round cacheable allocations up to
+	 * a page boundary to ensure that the physical pages will
+	 * only be mapped one way. */
+	if (mem_prot == NVMAP_HANDLE_CACHEABLE ||
+	    mem_prot == NVMAP_HANDLE_INNER_CACHEABLE) {
+		align = max_t(size_t, align, PAGE_SIZE);
+		len = PAGE_ALIGN(len);
+	}
+
+	heap_block = kmem_cache_zalloc(heap_block_cache, GFP_KERNEL);
+	if (!heap_block) {
+		dev_err(&heap->dev, "%s: failed to alloc heap block %s\n",
+			__func__, dev_name(&heap->dev));
+		goto fail_heap_block_alloc;
+	}
+
+	dev_addr = dma_alloc_coherent(&heap->dev, len, &dev_base,
+					DMA_MEMORY_MAP);
+	if (dev_base == DMA_ERROR_CODE) {
+		dev_err(&heap->dev, "%s: failed to alloc DMA coherent mem %s\n",
+			__func__, dev_name(&heap->dev));
+		goto fail_dma_alloc;
+	}
+	pr_debug("dma_alloc_coherent base (%pa) size (%d) heap (%s)\n",
+		&dev_base, len, heap->name);
+
+	heap_block->block.base = dev_base;
+	heap_block->orig_addr = dev_base;
+	heap_block->size = len;
+
+	list_add_tail(&heap_block->all_list, &heap->all_list);
+	heap_block->heap = heap;
+	heap_block->mem_prot = mem_prot;
+	heap_block->align = align;
+	return &heap_block->block;
+
+fail_dma_alloc:
+	kmem_cache_free(heap_block_cache, heap_block);
+fail_heap_block_alloc:
+	return NULL;
+}
+
+#ifdef DEBUG_FREE_LIST
+static void freelist_debug(struct nvmap_heap *heap, const char *title,
+			   struct list_block *token)
+{
+	int i;
+	struct list_block *n;
+
+	dev_debug(&heap->dev, "%s\n", title);
+	i = 0;
+	list_for_each_entry(n, &heap->free_list, free_list) {
+		dev_debug(&heap->dev, "\t%d [%p..%p]%s\n",
+			  i, (void *)n->orig_addr,
+			  (void *)(n->orig_addr + n->size),
+			  (n == token) ? "<--" : "");
+		i++;
+	}
+}
+#else
+#define freelist_debug(_heap, _title, _token)	do { } while (0)
+#endif
+
+static struct list_block *do_heap_free(struct nvmap_heap_block *block)
+{
+	struct list_block *b = container_of(block, struct list_block, block);
+	struct nvmap_heap *heap = b->heap;
+
+	list_del(&b->all_list);
+
+	pr_debug("dma_free_coherent base (0x%pa) size (%d) heap (%s)\n",
+		&block->base, b->size, heap->name);
+	/* assumes dev_alloc_coherent() returns same offset for phys_addr_t */
+	dma_free_coherent(&heap->dev, b->size, (void *)(uintptr_t)block->base,
+				block->base);
+
+	kmem_cache_free(heap_block_cache, b);
+
+	return b;
+}
+
+/* nvmap_heap_alloc: allocates a block of memory of len bytes, aligned to
+ * align bytes. */
+struct nvmap_heap_block *nvmap_heap_alloc(struct nvmap_heap *h,
+					  struct nvmap_handle *handle)
+{
+	struct nvmap_heap_block *b;
+	size_t len        = handle->size;
+	size_t align      = handle->align;
+	unsigned int prot = handle->flags;
+
+	mutex_lock(&h->lock);
+
+	align = max_t(size_t, align, L1_CACHE_BYTES);
+	b = do_heap_alloc(h, len, align, prot, 0);
+
+	if (b) {
+		b->handle = handle;
+		handle->carveout = b;
+	}
+	mutex_unlock(&h->lock);
+	return b;
+}
+
+struct nvmap_heap *nvmap_block_to_heap(struct nvmap_heap_block *b)
+{
+	struct list_block *lb;
+	lb = container_of(b, struct list_block, block);
+	return lb->heap;
+}
+
+/* nvmap_heap_free: frees block b*/
+void nvmap_heap_free(struct nvmap_heap_block *b)
+{
+	struct nvmap_heap *h = nvmap_block_to_heap(b);
+	struct list_block *lb;
+
+	mutex_lock(&h->lock);
+
+	lb = container_of(b, struct list_block, block);
+	nvmap_flush_heap_block(NULL, b, lb->size, lb->mem_prot);
+	do_heap_free(b);
+
+	mutex_unlock(&h->lock);
+}
+
+
+static void heap_release(struct device *heap)
+{
+}
+
+/* nvmap_heap_create: create a heap object of len bytes, starting from
+ * address base.
+ */
+struct nvmap_heap *nvmap_heap_create(struct device *parent, const char *name,
+				     phys_addr_t base, size_t len, void *arg)
+{
+	struct nvmap_heap *h = NULL;
+	int err = 0;
+	DEFINE_DMA_ATTRS(attrs);
+
+	h = kzalloc(sizeof(*h), GFP_KERNEL);
+	if (!h) {
+		dev_err(parent, "%s: out of memory\n", __func__);
+		goto fail_alloc;
+	}
+
+	dev_set_name(&h->dev, "heap-%s", name);
+	h->name = name;
+	h->arg = arg;
+	h->dev.parent = parent;
+	h->dev.driver = NULL;
+	h->dev.release = heap_release;
+
+	if (device_register(&h->dev)) {
+		dev_err(parent, "%s: failed to register %s\n", __func__,
+			dev_name(&h->dev));
+		goto fail_register;
+	}
+
+	if (sysfs_create_group(&h->dev.kobj, &heap_stat_attr_group)) {
+		dev_err(&h->dev, "%s: failed to create attributes\n", __func__);
+		goto fail_sysfs_create_group;
+	}
+
+	INIT_LIST_HEAD(&h->free_list);
+	INIT_LIST_HEAD(&h->all_list);
+	mutex_init(&h->lock);
+
+	err = dma_declare_coherent_memory(&h->dev, 0, base, len,
+		DMA_MEMORY_MAP | DMA_MEMORY_EXCLUSIVE);
+	if (!(err & DMA_MEMORY_MAP) || (base == 0)) {
+		dev_err(&h->dev, "%s: Unable to declare dma coherent memory\n",
+			__func__);
+		goto fail_dma_declare;
+	}
+
+	inner_flush_cache_all();
+	outer_flush_range(base, base + len);
+	wmb();
+
+	dma_set_attr(DMA_ATTR_SKIP_CPU_SYNC, &attrs);
+#ifdef CONFIG_PLATFORM_ENABLE_IOMMU
+	dma_map_linear_attrs(parent->parent, base, len, DMA_TO_DEVICE, &attrs);
+#endif
+
+	return h;
+
+fail_dma_declare:
+	sysfs_remove_group(&h->dev.kobj, &heap_stat_attr_group);
+fail_sysfs_create_group:
+	device_unregister(&h->dev);
+fail_register:
+	kfree(h);
+fail_alloc:
+	return NULL;
+}
+
+void *nvmap_heap_device_to_arg(struct device *dev)
+{
+	struct nvmap_heap *heap = container_of(dev, struct nvmap_heap, dev);
+	return heap->arg;
+}
+
+void *nvmap_heap_to_arg(struct nvmap_heap *heap)
+{
+	return heap->arg;
+}
+
+/* nvmap_heap_destroy: frees all resources in heap */
+void nvmap_heap_destroy(struct nvmap_heap *heap)
+{
+
+	sysfs_remove_group(&heap->dev.kobj, &heap_stat_attr_group);
+	device_unregister(&heap->dev);
+
+	WARN_ON(!list_is_singular(&heap->all_list));
+	while (!list_empty(&heap->all_list)) {
+		struct list_block *l;
+		l = list_first_entry(&heap->all_list, struct list_block,
+				     all_list);
+		list_del(&l->all_list);
+		kmem_cache_free(heap_block_cache, l);
+	}
+
+	kfree(heap);
+}
+
+/* nvmap_heap_create_group: adds the attribute_group grp to the heap kobject */
+int nvmap_heap_create_group(struct nvmap_heap *heap,
+			    const struct attribute_group *grp)
+{
+	return sysfs_create_group(&heap->dev.kobj, grp);
+}
+
+/* nvmap_heap_remove_group: removes the attribute_group grp  */
+void nvmap_heap_remove_group(struct nvmap_heap *heap,
+			     const struct attribute_group *grp)
+{
+	sysfs_remove_group(&heap->dev.kobj, grp);
+}
+
+int nvmap_heap_init(void)
+{
+	heap_block_cache = KMEM_CACHE(list_block, 0);
+	if (!heap_block_cache) {
+		pr_err("%s: unable to create heap block cache\n", __func__);
+		return -ENOMEM;
+	}
+	pr_info("%s: created heap block cache\n", __func__);
+	return 0;
+}
+
+void nvmap_heap_deinit(void)
+{
+	if (heap_block_cache)
+		kmem_cache_destroy(heap_block_cache);
+
+	heap_block_cache = NULL;
+}

drivers/video/tegra/nvmap/nvmap_heap.h

@@ -0,0 +1,65 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_heap.h
+ *
+ * GPU heap allocator.
+ *
+ * Copyright (c) 2010-2013, NVIDIA Corporation. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#ifndef __NVMAP_HEAP_H
+#define __NVMAP_HEAP_H
+
+struct device;
+struct nvmap_heap;
+struct attribute_group;
+
+struct nvmap_heap_block {
+	phys_addr_t	base;
+	unsigned int	type;
+	struct nvmap_handle *handle;
+};
+
+struct nvmap_heap *nvmap_heap_create(struct device *parent, const char *name,
+				phys_addr_t base, size_t len, void *arg);
+
+void nvmap_heap_destroy(struct nvmap_heap *heap);
+
+void *nvmap_heap_device_to_arg(struct device *dev);
+
+void *nvmap_heap_to_arg(struct nvmap_heap *heap);
+
+struct nvmap_heap_block *nvmap_heap_alloc(struct nvmap_heap *heap,
+					  struct nvmap_handle *handle);
+
+struct nvmap_heap *nvmap_block_to_heap(struct nvmap_heap_block *b);
+
+void nvmap_heap_free(struct nvmap_heap_block *block);
+
+int nvmap_heap_create_group(struct nvmap_heap *heap,
+			    const struct attribute_group *grp);
+
+void nvmap_heap_remove_group(struct nvmap_heap *heap,
+			     const struct attribute_group *grp);
+
+int __init nvmap_heap_init(void);
+
+void nvmap_heap_deinit(void);
+
+int nvmap_flush_heap_block(struct nvmap_client *client,
+	struct nvmap_heap_block *block, size_t len, unsigned int prot);
+
+#endif

drivers/video/tegra/nvmap/nvmap_ioctl.h

@@ -0,0 +1,212 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_ioctl.h
+ *
+ * ioctl declarations for nvmap
+ *
+ * Copyright (c) 2010-2013, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#ifndef __VIDEO_TEGRA_NVMAP_IOCTL_H
+#define __VIDEO_TEGRA_NVMAP_IOCTL_H
+
+#include <linux/ioctl.h>
+
+#ifdef __KERNEL__
+#include <linux/file.h>
+#include <linux/nvmap.h>
+#endif
+
+/* Hack for L4T builds.
+ * gstreamer is directly including this header and
+ * looking for ioctls param struct definitions. This hack
+ * is necessary till user space gstreamer is fixed to use
+ * linux/nvmap.h file.
+ *
+ */
+#ifndef __KERNEL__
+enum {
+	NVMAP_HANDLE_PARAM_SIZE = 1,
+	NVMAP_HANDLE_PARAM_ALIGNMENT,
+	NVMAP_HANDLE_PARAM_BASE,
+	NVMAP_HANDLE_PARAM_HEAP,
+	NVMAP_HANDLE_PARAM_KIND,
+	NVMAP_HANDLE_PARAM_COMPR, /* ignored, to be removed */
+};
+
+enum {
+	NVMAP_CACHE_OP_WB = 0,
+	NVMAP_CACHE_OP_INV,
+	NVMAP_CACHE_OP_WB_INV,
+};
+
+struct nvmap_create_handle {
+	union {
+		__u32 key;	/* ClaimPreservedHandle */
+		__u32 id;	/* FromId */
+		__u32 size;	/* CreateHandle */
+		__s32 fd;	/* DmaBufFd or FromFd */
+	};
+	__u32 handle;
+};
+
+struct nvmap_alloc_handle {
+	__u32 handle;
+	__u32 heap_mask;
+	__u32 flags;
+	__u32 align;
+};
+
+struct nvmap_alloc_kind_handle {
+	__u32 handle;
+	__u32 heap_mask;
+	__u32 flags;
+	__u32 align;
+	__u8  kind;
+	__u8  comp_tags;
+};
+
+struct nvmap_map_caller {
+	__u32 handle;		/* hmem */
+	__u32 offset;		/* offset into hmem; should be page-aligned */
+	__u32 length;		/* number of bytes to map */
+	__u32 flags;
+	__u64 addr;		/* user pointer */
+};
+
+struct nvmap_rw_handle {
+	__u64 addr;		/* user pointer */
+	__u32 handle;		/* hmem */
+	__u32 offset;		/* offset into hmem */
+	__u32 elem_size;	/* individual atom size */
+	__u32 hmem_stride;	/* delta in bytes between atoms in hmem */
+	__u32 user_stride;	/* delta in bytes between atoms in user */
+	__u32 count;		/* number of atoms to copy */
+};
+
+struct nvmap_pin_handle {
+	__u64 handles;		/* array of handles to pin/unpin */
+	__u64 addr;		/* array of addresses to return */
+	__u32 count;		/* number of entries in handles */
+};
+
+struct nvmap_handle_param {
+	__u32 handle;
+	__u32 param;
+	__u64 result;
+};
+
+struct nvmap_cache_op {
+	__u64 addr;
+	__u32 handle;
+	__u32 len;
+	__s32 op;
+};
+
+#define NVMAP_IOC_MAGIC 'N'
+
+/* Creates a new memory handle. On input, the argument is the size of the new
+ * handle; on return, the argument is the name of the new handle
+ */
+#define NVMAP_IOC_CREATE  _IOWR(NVMAP_IOC_MAGIC, 0, struct nvmap_create_handle)
+#define NVMAP_IOC_CLAIM   _IOWR(NVMAP_IOC_MAGIC, 1, struct nvmap_create_handle)
+#define NVMAP_IOC_FROM_ID _IOWR(NVMAP_IOC_MAGIC, 2, struct nvmap_create_handle)
+
+/* Actually allocates memory for the specified handle */
+#define NVMAP_IOC_ALLOC    _IOW(NVMAP_IOC_MAGIC, 3, struct nvmap_alloc_handle)
+
+/* Frees a memory handle, unpinning any pinned pages and unmapping any mappings
+ */
+#define NVMAP_IOC_FREE       _IO(NVMAP_IOC_MAGIC, 4)
+
+/* Maps the region of the specified handle into a user-provided virtual address
+ * that was previously created via an mmap syscall on this fd */
+#define NVMAP_IOC_MMAP       _IOWR(NVMAP_IOC_MAGIC, 5, struct nvmap_map_caller)
+
+/* Reads/writes data (possibly strided) from a user-provided buffer into the
+ * hmem at the specified offset */
+#define NVMAP_IOC_WRITE      _IOW(NVMAP_IOC_MAGIC, 6, struct nvmap_rw_handle)
+#define NVMAP_IOC_READ       _IOW(NVMAP_IOC_MAGIC, 7, struct nvmap_rw_handle)
+
+#define NVMAP_IOC_PARAM _IOWR(NVMAP_IOC_MAGIC, 8, struct nvmap_handle_param)
+
+/* Pins a list of memory handles into IO-addressable memory (either IOVMM
+ * space or physical memory, depending on the allocation), and returns the
+ * address. Handles may be pinned recursively. */
+#define NVMAP_IOC_PIN_MULT   _IOWR(NVMAP_IOC_MAGIC, 10, struct nvmap_pin_handle)
+#define NVMAP_IOC_UNPIN_MULT _IOW(NVMAP_IOC_MAGIC, 11, struct nvmap_pin_handle)
+
+#define NVMAP_IOC_CACHE      _IOW(NVMAP_IOC_MAGIC, 12, struct nvmap_cache_op)
+
+/* Returns a global ID usable to allow a remote process to create a handle
+ * reference to the same handle */
+#define NVMAP_IOC_GET_ID  _IOWR(NVMAP_IOC_MAGIC, 13, struct nvmap_create_handle)
+
+/* Returns a dma-buf fd usable to allow a remote process to create a handle
+ * reference to the same handle */
+#define NVMAP_IOC_SHARE  _IOWR(NVMAP_IOC_MAGIC, 14, struct nvmap_create_handle)
+
+/* Returns a file id that allows a remote process to create a handle
+ * reference to the same handle */
+#define NVMAP_IOC_GET_FD  _IOWR(NVMAP_IOC_MAGIC, 15, struct nvmap_create_handle)
+
+/* Create a new memory handle from file id passed */
+#define NVMAP_IOC_FROM_FD _IOWR(NVMAP_IOC_MAGIC, 16, struct nvmap_create_handle)
+
+/* START of T124 IOCTLS */
+/* Actually allocates memory for the specified handle, with kind */
+#define NVMAP_IOC_ALLOC_KIND \
+	_IOW(NVMAP_IOC_MAGIC, 100, struct nvmap_alloc_kind_handle)
+
+#define NVMAP_IOC_MAXNR (_IOC_NR(NVMAP_IOC_ALLOC_KIND))
+#endif
+
+#ifdef __KERNEL__
+int nvmap_ioctl_pinop(struct file *filp, bool is_pin, void __user *arg);
+
+int nvmap_ioctl_get_param(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_getfd(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_alloc(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_alloc_kind(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_free(struct file *filp, unsigned long arg);
+
+int nvmap_ioctl_create(struct file *filp, unsigned int cmd, void __user *arg);
+
+int nvmap_map_into_caller_ptr(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_cache_maint(struct file *filp, void __user *arg);
+
+int nvmap_ioctl_rw_handle(struct file *filp, int is_read, void __user *arg);
+
+int nvmap_ioctl_share_dmabuf(struct file *filp, void __user *arg);
+
+#ifdef CONFIG_NVMAP_USE_FD_FOR_HANDLE
+int nvmap_foreign_dmabuf_add(struct nvmap_handle *h, struct dma_buf *dmabuf,
+		int fd);
+struct nvmap_handle *nvmap_foreign_dmabuf_find_by_fd(int fd);
+struct dma_buf_attachment *nvmap_foreign_dmabuf_get_att(
+		struct dma_buf *dmabuf);
+int nvmap_foreign_dmabuf_get(struct dma_buf *dmabuf);
+int nvmap_foreign_dmabuf_put(struct dma_buf *dmabuf);
+#endif
+
+#endif	/* __KERNEL__ */
+
+#endif	/*  __VIDEO_TEGRA_NVMAP_IOCTL_H */

drivers/video/tegra/nvmap/nvmap_mm.c

@@ -0,0 +1,95 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap_mm.c
+ *
+ * Some MM related functionality specific to nvmap.
+ *
+ * Copyright (c) 2013, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#include "nvmap_priv.h"
+
+void inner_flush_cache_all(void)
+{
+	on_each_cpu((smp_call_func_t)v7_flush_kern_cache_all, NULL, 1);
+}
+
+void inner_clean_cache_all(void)
+{
+	inner_flush_cache_all();
+}
+
+#ifndef CONFIG_NVMAP_CPA
+static void flush_cache(struct page **pages, int numpages)
+{
+	unsigned int i;
+	bool flush_inner = true;
+	unsigned long base;
+
+#if defined(CONFIG_NVMAP_CACHE_MAINT_BY_SET_WAYS)
+	if (numpages >= (cache_maint_inner_threshold >> PAGE_SHIFT)) {
+		inner_flush_cache_all();
+		flush_inner = false;
+	}
+#endif
+
+	for (i = 0; i < numpages; i++) {
+		if (flush_inner)
+			__flush_dcache_page(page_mapping(pages[i]), pages[i]);
+		base = page_to_phys(pages[i]);
+		outer_flush_range(base, base + PAGE_SIZE);
+	}
+}
+#endif
+
+int nvmap_set_pages_array_uc(struct page **pages, int addrinarray)
+{
+#ifdef CONFIG_NVMAP_CPA
+	return set_pages_array_uc(pages, addrinarray);
+#else
+	flush_cache(pages, addrinarray);
+	return 0;
+#endif
+}
+
+int nvmap_set_pages_array_wc(struct page **pages, int addrinarray)
+{
+#ifdef CONFIG_NVMAP_CPA
+	return set_pages_array_wc(pages, addrinarray);
+#else
+	flush_cache(pages, addrinarray);
+	return 0;
+#endif
+}
+
+int nvmap_set_pages_array_iwb(struct page **pages, int addrinarray)
+{
+#ifdef CONFIG_NVMAP_CPA
+	return set_pages_array_iwb(pages, addrinarray);
+#else
+	flush_cache(pages, addrinarray);
+	return 0;
+#endif
+}
+
+int nvmap_set_pages_array_wb(struct page **pages, int addrinarray)
+{
+#ifdef CONFIG_NVMAP_CPA
+	return set_pages_array_wb(pages, addrinarray);
+#else
+	return 0;
+#endif
+}

drivers/video/tegra/nvmap/nvmap_priv.h

@@ -0,0 +1,402 @@
+/*
+ * drivers/video/tegra/nvmap/nvmap.h
+ *
+ * GPU memory management driver for Tegra
+ *
+ * Copyright (c) 2009-2013, NVIDIA CORPORATION. All rights reserved.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *'
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with this program; if not, write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+
+#ifndef __VIDEO_TEGRA_NVMAP_NVMAP_H
+#define __VIDEO_TEGRA_NVMAP_NVMAP_H
+
+#include <linux/list.h>
+#include <linux/mm.h>
+#include <linux/mutex.h>
+#include <linux/rbtree.h>
+#include <linux/sched.h>
+#include <linux/wait.h>
+#include <linux/atomic.h>
+#include <linux/dma-buf.h>
+#include <linux/syscalls.h>
+#include <linux/nvmap.h>
+
+#include <linux/workqueue.h>
+#include <linux/dma-mapping.h>
+#include <linux/dma-direction.h>
+#include <linux/platform_device.h>
+
+#include <asm/tlbflush.h>
+#include <asm/cacheflush.h>
+
+#include "nvmap_heap.h"
+
+struct nvmap_device;
+struct nvmap_share;
+struct page;
+
+extern const struct file_operations nvmap_fd_fops;
+void _nvmap_handle_free(struct nvmap_handle *h);
+extern struct nvmap_share *nvmap_share;
+/* holds max number of handles allocted per process at any time */
+extern u32 nvmap_max_handle_count;
+
+extern struct platform_device *nvmap_pdev;
+
+#if defined(CONFIG_TEGRA_NVMAP)
+#define nvmap_err(_client, _fmt, ...)				\
+	dev_err(nvmap_client_to_device(_client),		\
+		"%s: "_fmt, __func__, ##__VA_ARGS__)
+
+#define nvmap_warn(_client, _fmt, ...)				\
+	dev_warn(nvmap_client_to_device(_client),		\
+		 "%s: "_fmt, __func__, ##__VA_ARGS__)
+
+#define nvmap_debug(_client, _fmt, ...)				\
+	dev_dbg(nvmap_client_to_device(_client),		\
+		"%s: "_fmt, __func__, ##__VA_ARGS__)
+
+#define CACHE_MAINT_IMMEDIATE		0
+#define CACHE_MAINT_ALLOW_DEFERRED	1
+
+struct nvmap_deferred_ops {
+	struct list_head ops_list;
+	spinlock_t deferred_ops_lock;
+	bool enable_deferred_cache_maintenance;
+	u64 deferred_maint_inner_requested;
+	u64 deferred_maint_inner_flushed;
+	u64 deferred_maint_outer_requested;
+	u64 deferred_maint_outer_flushed;
+};
+
+/* handles allocated using shared system memory (either IOVMM- or high-order
+ * page allocations */
+struct nvmap_pgalloc {
+	struct page **pages;
+	bool contig;			/* contiguous system memory */
+	u32 iovm_addr;	/* is non-zero, if client need specific iova mapping */
+};
+
+struct nvmap_handle {
+	struct rb_node node;	/* entry on global handle tree */
+	atomic_t ref;		/* reference count (i.e., # of duplications) */
+	atomic_t pin;		/* pin count */
+	atomic_t disable_deferred_cache;
+	unsigned long flags;
+	size_t size;		/* padded (as-allocated) size */
+	size_t orig_size;	/* original (as-requested) size */
+	size_t align;
+	u8 kind;                /* memory kind (0=pitch, !0 -> blocklinear) */
+	void *map_resources;    /* mapping resources associated with the
+				   buffer */
+	struct nvmap_client *owner;
+	struct nvmap_handle_ref *owner_ref; /* use this ref to avoid spending
+			time on validation in some cases.
+			if handle was duplicated by other client and
+			original client destroy ref, this field
+			has to be set to zero. In this case ref should be
+			obtained through validation */
+
+	/*
+	 * dma_buf necessities. An attachment is made on dma_buf allocation to
+	 * facilitate the nvmap_pin* APIs.
+	 */
+	struct dma_buf *dmabuf;
+	struct dma_buf_attachment *attachment;
+
+	struct nvmap_device *dev;
+	union {
+		struct nvmap_pgalloc pgalloc;
+		struct nvmap_heap_block *carveout;
+	};
+	bool global;		/* handle may be duplicated by other clients */
+	bool secure;		/* zap IOVMM area on unpin */
+	bool heap_pgalloc;	/* handle is page allocated (sysmem / iovmm) */
+	bool alloc;		/* handle has memory allocated */
+	unsigned int userflags;	/* flags passed from userspace */
+	struct mutex lock;
+	void *nvhost_priv;	/* nvhost private data */
+	void (*nvhost_priv_delete)(void *priv);
+};
+
+/* handle_ref objects are client-local references to an nvmap_handle;
+ * they are distinct objects so that handles can be unpinned and
+ * unreferenced the correct number of times when a client abnormally
+ * terminates */
+struct nvmap_handle_ref {
+	struct nvmap_handle *handle;
+	struct rb_node	node;
+	atomic_t	dupes;	/* number of times to free on file close */
+	atomic_t	pin;	/* number of times to unpin on free */
+	bool is_foreign;
+};
+
+#ifdef CONFIG_NVMAP_PAGE_POOLS
+#define NVMAP_UC_POOL NVMAP_HANDLE_UNCACHEABLE
+#define NVMAP_WC_POOL NVMAP_HANDLE_WRITE_COMBINE
+#define NVMAP_IWB_POOL NVMAP_HANDLE_INNER_CACHEABLE
+#define NVMAP_WB_POOL NVMAP_HANDLE_CACHEABLE
+#define NVMAP_NUM_POOLS (NVMAP_HANDLE_CACHEABLE + 1)
+
+struct nvmap_page_pool {
+	struct mutex lock;
+	int npages;
+	struct page **page_array;
+	struct page **shrink_array;
+	int max_pages;
+	int flags;
+};
+
+int nvmap_page_pool_init(struct nvmap_page_pool *pool, int flags);
+#endif
+
+struct nvmap_share {
+	struct tegra_iovmm_client *iovmm;
+	wait_queue_head_t pin_wait;
+	struct mutex pin_lock;
+#ifdef CONFIG_NVMAP_PAGE_POOLS
+	union {
+		struct nvmap_page_pool pools[NVMAP_NUM_POOLS];
+		struct {
+			struct nvmap_page_pool uc_pool;
+			struct nvmap_page_pool wc_pool;
+			struct nvmap_page_pool iwb_pool;
+			struct nvmap_page_pool wb_pool;
+		};
+	};
+#endif
+};
+
+struct nvmap_carveout_commit {
+	size_t commit;
+	struct list_head list;
+};
+
+struct nvmap_client {
+	const char			*name;
+	struct rb_root			handle_refs;
+	atomic_t			iovm_commit;
+	struct mutex			ref_lock;
+	bool				super;
+	bool				kernel_client;
+	atomic_t			count;
+	struct task_struct		*task;
+	struct list_head		list;
+	u32				handle_count;
+	struct nvmap_carveout_commit	carveout_commit[0];
+};
+
+struct nvmap_vma_priv {
+	struct nvmap_handle *handle;
+	size_t		offs;
+	atomic_t	count;	/* number of processes cloning the VMA */
+};
+
+static inline void nvmap_ref_lock(struct nvmap_client *priv)
+{
+	mutex_lock(&priv->ref_lock);
+}
+
+static inline void nvmap_ref_unlock(struct nvmap_client *priv)
+{
+	mutex_unlock(&priv->ref_lock);
+}
+
+/*
+ * NOTE: this does not ensure the continued existence of the underlying
+ * dma_buf. If you want ensure the existence of the dma_buf you must get an
+ * nvmap_handle_ref as that is what tracks the dma_buf refs.
+ */
+static inline struct nvmap_handle *nvmap_handle_get(struct nvmap_handle *h)
+{
+	if (unlikely(atomic_inc_return(&h->ref) <= 1)) {
+		pr_err("%s: %s attempt to get a freed handle\n",
+			__func__, current->group_leader->comm);
+		atomic_dec(&h->ref);
+		return NULL;
+	}
+	return h;
+}
+
+static inline pgprot_t nvmap_pgprot(struct nvmap_handle *h, pgprot_t prot)
+{
+	if (h->flags == NVMAP_HANDLE_UNCACHEABLE)
+		return pgprot_noncached(prot);
+	else if (h->flags == NVMAP_HANDLE_WRITE_COMBINE)
+		return pgprot_writecombine(prot);
+	return prot;
+}
+
+#else /* CONFIG_TEGRA_NVMAP */
+struct nvmap_handle *nvmap_handle_get(struct nvmap_handle *h);
+void nvmap_handle_put(struct nvmap_handle *h);
+pgprot_t nvmap_pgprot(struct nvmap_handle *h, pgprot_t prot);
+
+#endif /* !CONFIG_TEGRA_NVMAP */
+
+struct device *nvmap_client_to_device(struct nvmap_client *client);
+
+pte_t **nvmap_alloc_pte(struct nvmap_device *dev, void **vaddr);
+
+pte_t **nvmap_alloc_pte_irq(struct nvmap_device *dev, void **vaddr);
+
+void nvmap_free_pte(struct nvmap_device *dev, pte_t **pte);
+
+pte_t **nvmap_vaddr_to_pte(struct nvmap_device *dev, unsigned long vaddr);
+
+struct nvmap_heap_block *nvmap_carveout_alloc(struct nvmap_client *dev,
+					      struct nvmap_handle *handle,
+					      unsigned long type);
+
+unsigned long nvmap_carveout_usage(struct nvmap_client *c,
+				   struct nvmap_heap_block *b);
+
+struct nvmap_carveout_node;
+void nvmap_carveout_commit_add(struct nvmap_client *client,
+			       struct nvmap_carveout_node *node, size_t len);
+
+void nvmap_carveout_commit_subtract(struct nvmap_client *client,
+				    struct nvmap_carveout_node *node,
+				    size_t len);
+
+struct nvmap_share *nvmap_get_share_from_dev(struct nvmap_device *dev);
+
+void nvmap_cache_maint_ops_flush(struct nvmap_device *dev,
+		struct nvmap_handle *h);
+
+struct nvmap_deferred_ops *nvmap_get_deferred_ops_from_dev(
+		struct nvmap_device *dev);
+
+int nvmap_find_cache_maint_op(struct nvmap_device *dev,
+		struct nvmap_handle *h);
+
+struct nvmap_handle *nvmap_validate_get(struct nvmap_client *client,
+					unsigned long handle, bool skip_val);
+
+struct nvmap_handle *nvmap_get_handle_id(struct nvmap_client *client,
+					 unsigned long id);
+
+void nvmap_handle_put(struct nvmap_handle *h);
+
+struct nvmap_handle_ref *__nvmap_validate_id_locked(struct nvmap_client *priv,
+						   unsigned long id);
+
+struct nvmap_handle_ref *nvmap_create_handle_dmabuf(struct nvmap_client *client,
+						    struct dma_buf *dmabuf);
+
+struct nvmap_handle_ref *nvmap_create_handle(struct nvmap_client *client,
+					     size_t size);
+
+struct nvmap_handle_ref *nvmap_duplicate_handle_id(struct nvmap_client *client,
+					unsigned long id, bool skip_val);
+
+struct nvmap_handle_ref *nvmap_create_handle_from_fd(
+			struct nvmap_client *client, int fd);
+
+int nvmap_alloc_handle_id(struct nvmap_client *client,
+			  unsigned long id, unsigned int heap_mask,
+			  size_t align, u8 kind,
+			  unsigned int flags);
+
+void nvmap_free_handle_id(struct nvmap_client *c, unsigned long id);
+
+void nvmap_free_handle_user_id(struct nvmap_client *c, unsigned long user_id);
+
+int nvmap_pin_ids(struct nvmap_client *client,
+		  unsigned int nr, const unsigned long *ids);
+
+void nvmap_unpin_ids(struct nvmap_client *priv,
+		     unsigned int nr, const unsigned long *ids);
+
+int nvmap_handle_remove(struct nvmap_device *dev, struct nvmap_handle *h);
+
+void nvmap_handle_add(struct nvmap_device *dev, struct nvmap_handle *h);
+
+int is_nvmap_vma(struct vm_area_struct *vma);
+
+int nvmap_get_dmabuf_fd(struct nvmap_client *client, ulong id);
+int nvmap_install_fd(struct nvmap_client *client,
+	struct nvmap_handle *handle, int fd, void __user *arg,
+	void *op, size_t op_size);
+ulong nvmap_get_id_from_dmabuf(struct nvmap_client *client,
+		struct dma_buf *dmabuf);
+ulong nvmap_get_id_from_dmabuf_fd(struct nvmap_client *client, int fd);
+
+int nvmap_get_handle_param(struct nvmap_client *client,
+		struct nvmap_handle_ref *ref, u32 param, u64 *result);
+
+#ifdef CONFIG_COMPAT
+ulong unmarshal_user_handle(__u32 handle);
+__u32 marshal_kernel_handle(ulong handle);
+ulong unmarshal_user_id(u32 id);
+#else
+ulong unmarshal_user_handle(struct nvmap_handle *handle);
+struct nvmap_handle *marshal_kernel_handle(ulong handle);
+ulong unmarshal_user_id(ulong id);
+#endif
+
+static inline void nvmap_flush_tlb_kernel_page(unsigned long kaddr)
+{
+	flush_tlb_kernel_page(kaddr);
+}
+
+/* MM definitions. */
+extern size_t cache_maint_outer_threshold;
+extern int inner_cache_maint_threshold;
+
+extern void v7_flush_kern_cache_all(void);
+extern void v7_clean_kern_cache_all(void *);
+extern void __flush_dcache_page(struct address_space *, struct page *);
+
+void inner_flush_cache_all(void);
+void inner_clean_cache_all(void);
+int nvmap_set_pages_array_uc(struct page **pages, int addrinarray);
+int nvmap_set_pages_array_wc(struct page **pages, int addrinarray);
+int nvmap_set_pages_array_iwb(struct page **pages, int addrinarray);
+int nvmap_set_pages_array_wb(struct page **pages, int addrinarray);
+
+/* Internal API to support dmabuf */
+struct dma_buf *__nvmap_dmabuf_export(struct nvmap_client *client,
+				 unsigned long id);
+struct dma_buf *__nvmap_make_dmabuf(struct nvmap_client *client,
+				    struct nvmap_handle *handle);
+struct sg_table *__nvmap_sg_table(struct nvmap_client *client,
+				  struct nvmap_handle *h);
+void __nvmap_free_sg_table(struct nvmap_client *client,
+			   struct nvmap_handle *h, struct sg_table *sgt);
+void *__nvmap_kmap(struct nvmap_handle *h, unsigned int pagenum);
+void __nvmap_kunmap(struct nvmap_handle *h, unsigned int pagenum, void *addr);
+void *__nvmap_mmap(struct nvmap_handle *h);
+void __nvmap_munmap(struct nvmap_handle *h, void *addr);
+int __nvmap_map(struct nvmap_handle *h, struct vm_area_struct *vma);
+int __nvmap_get_handle_param(struct nvmap_client *client,
+			     struct nvmap_handle *h, u32 param, u64 *result);
+int __nvmap_cache_maint(struct nvmap_client *client, struct nvmap_handle *h,
+			unsigned long start, unsigned long end,
+			unsigned int op, unsigned int allow_deferred);
+struct nvmap_client *__nvmap_create_client(struct nvmap_device *dev,
+					   const char *name);
+struct dma_buf *__nvmap_dmabuf_export_from_ref(struct nvmap_handle_ref *ref);
+ulong __nvmap_ref_to_id(struct nvmap_handle_ref *ref);
+int __nvmap_pin(struct nvmap_handle_ref *ref, phys_addr_t *phys);
+void __nvmap_unpin(struct nvmap_handle_ref *ref);
+int __nvmap_dmabuf_fd(struct dma_buf *dmabuf, int flags);
+bool nvmap_dmabuf_is_foreign_dmabuf(struct dma_buf *dmabuf);
+
+void nvmap_dmabuf_debugfs_init(struct dentry *nvmap_root);
+int nvmap_dmabuf_stash_init(void);
+
+#endif /* __VIDEO_TEGRA_NVMAP_NVMAP_H */