985 lines
27 KiB
C
985 lines
27 KiB
C
/*
|
|
Copyright (c), 2001-2022, Shenshu Tech. Co., Ltd.
|
|
*/
|
|
|
|
#include <linux/kernel.h>
|
|
#include <linux/version.h>
|
|
#include <linux/module.h>
|
|
#include <linux/types.h>
|
|
#include <linux/errno.h>
|
|
#include <linux/fcntl.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/mman.h>
|
|
#include <linux/miscdevice.h>
|
|
#include <linux/proc_fs.h>
|
|
#include <linux/device.h>
|
|
#include <linux/fs.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/init.h>
|
|
#include <linux/delay.h>
|
|
#include <linux/interrupt.h>
|
|
#include <linux/ioport.h>
|
|
#include <linux/spinlock.h>
|
|
#include <linux/vmalloc.h>
|
|
#include <linux/string.h>
|
|
#include <linux/list.h>
|
|
#include <linux/time.h>
|
|
#include <linux/sched.h>
|
|
#include <linux/dma-mapping.h>
|
|
#include <linux/uaccess.h>
|
|
|
|
#ifdef CONFIG_OT_SYS_SMMU_SUPPORT
|
|
#include <linux/iommu.h>
|
|
#include "linux/vendor/sva_ext.h"
|
|
#endif
|
|
|
|
#include <asm/uaccess.h>
|
|
#include <asm/io.h>
|
|
#include <asm/cacheflush.h>
|
|
#include "securec.h"
|
|
#include "osal_mmz.h"
|
|
#include "ot_osal.h"
|
|
#include "mmz.h"
|
|
|
|
#define error_mmz(s...) \
|
|
do { \
|
|
osal_trace(KERN_ERR "mmz_userdev:%s: ", __FUNCTION__); \
|
|
osal_trace(s); \
|
|
} while (0)
|
|
#define warning(s...) \
|
|
do { \
|
|
osal_trace(KERN_WARNING "mmz_userdev:%s: ", __FUNCTION__); \
|
|
osal_trace(s); \
|
|
} while (0)
|
|
|
|
struct mmz_userdev_info {
|
|
pid_t pid;
|
|
pid_t mmap_pid;
|
|
struct semaphore sem;
|
|
struct osal_list_head list;
|
|
};
|
|
|
|
static void mmz_flush_dcache_mmb_dirty(struct dirty_area *p_area)
|
|
{
|
|
#ifdef CONFIG_64BIT
|
|
__flush_dcache_area(p_area->dirty_virt_start, p_area->dirty_size);
|
|
#else
|
|
/* flush l1 cache, use vir addr */
|
|
__cpuc_flush_dcache_area(p_area->dirty_virt_start, p_area->dirty_size);
|
|
|
|
#ifdef CONFIG_CACHE_L2X0
|
|
/* flush l2 cache, use paddr */
|
|
outer_flush_range(p_area->dirty_phys_start,
|
|
p_area->dirty_phys_start + p_area->dirty_size);
|
|
#endif
|
|
#endif
|
|
}
|
|
|
|
static int mmz_flush_dcache_mmb(const struct mmb_info *pmi)
|
|
{
|
|
ot_mmb_t *mmb = NULL;
|
|
|
|
if (pmi == NULL) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
mmb = pmi->mmb;
|
|
if ((mmb == NULL) || (pmi->map_cached == 0)) {
|
|
osal_trace("%s->%d,error!\n", __func__, __LINE__);
|
|
return -EINVAL;
|
|
}
|
|
|
|
#ifdef CONFIG_64BIT
|
|
__flush_dcache_area(pmi->mapped, (size_t)pmi->size);
|
|
#else
|
|
/* flush l1 cache, use vir addr */
|
|
__cpuc_flush_dcache_area(pmi->mapped, (size_t)pmi->size);
|
|
|
|
#ifdef CONFIG_CACHE_L2X0
|
|
/* flush l2 cache, use paddr */
|
|
outer_flush_range(mmb->phys_addr, mmb->phys_addr + mmb->length);
|
|
#endif
|
|
#endif
|
|
return 0;
|
|
}
|
|
#ifdef CONFIG_64BIT
|
|
#if LINUX_VERSION_CODE < KERNEL_VERSION(5, 10, 0)
|
|
void flush_cache_all(void)
|
|
{
|
|
}
|
|
#endif
|
|
#endif
|
|
|
|
/*
|
|
* this function should never be called with local irq disabled,
|
|
* because on_each_cpu marco will raise ipi interrupt.
|
|
*/
|
|
int mmz_flush_dcache_all(void)
|
|
{
|
|
#ifdef CONFIG_64BIT
|
|
on_each_cpu((smp_call_func_t)flush_cache_all, NULL, 1);
|
|
#else
|
|
on_each_cpu((smp_call_func_t)__cpuc_flush_kern_all, NULL, 1);
|
|
outer_flush_all();
|
|
#endif /* CONFIG_64BIT */
|
|
return 0;
|
|
}
|
|
|
|
static int mmz_userdev_open(struct inode *inode, struct file *file)
|
|
{
|
|
struct mmz_userdev_info *pmu = NULL;
|
|
|
|
if (!capable(CAP_SYS_RAWIO)) {
|
|
return -EPERM;
|
|
}
|
|
|
|
pmu = kmalloc(sizeof(*pmu), GFP_KERNEL);
|
|
if (pmu == NULL) {
|
|
error_mmz("alloc mmz_userdev_info failed!\n");
|
|
return -ENOMEM;
|
|
}
|
|
(void)memset_s(pmu, sizeof(*pmu), 0, sizeof(*pmu));
|
|
pmu->pid = current->pid;
|
|
pmu->mmap_pid = 0;
|
|
sema_init(&pmu->sem, 1);
|
|
OSAL_INIT_LIST_HEAD(&pmu->list);
|
|
|
|
/* This file could be opened just for once */
|
|
file->private_data = (void *)pmu;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int ioctl_mmb_alloc(const struct file *file,
|
|
unsigned int iocmd,
|
|
struct mmb_info *pmi)
|
|
{
|
|
struct mmz_userdev_info *pmu = file->private_data;
|
|
struct mmb_info *new_mmbinfo = NULL;
|
|
ot_mmb_t *mmb = NULL;
|
|
|
|
mmb = ot_mmb_alloc(pmi->mmb_name, pmi->size,
|
|
pmi->align, pmi->gfp, pmi->mmz_name);
|
|
if (mmb == NULL) {
|
|
#if defined(KERNEL_BIT_64) && defined(USER_BIT_32)
|
|
error_mmz("ot_mmb_alloc(%s, %llu, 0x%llx, %lu, %s) failed!\n",
|
|
pmi->mmb_name, pmi->size, pmi->align,
|
|
pmi->gfp, pmi->mmz_name);
|
|
#else
|
|
error_mmz("ot_mmb_alloc(%s, %lu, 0x%lx, %lu, %s) failed!\n",
|
|
pmi->mmb_name, pmi->size, pmi->align,
|
|
pmi->gfp, pmi->mmz_name);
|
|
#endif
|
|
return -ENOMEM;
|
|
}
|
|
|
|
new_mmbinfo = kmalloc(sizeof(*new_mmbinfo), GFP_KERNEL);
|
|
if (new_mmbinfo == NULL) {
|
|
ot_mmb_free(mmb);
|
|
error_mmz("alloc mmb_info failed!\n");
|
|
return -ENOMEM;
|
|
}
|
|
|
|
(void)memcpy_s(new_mmbinfo, sizeof(*new_mmbinfo), pmi, sizeof(*new_mmbinfo));
|
|
new_mmbinfo->phys_addr = ot_mmb_phys(mmb);
|
|
new_mmbinfo->mmb = mmb;
|
|
new_mmbinfo->prot = PROT_READ;
|
|
new_mmbinfo->flags = MAP_SHARED;
|
|
osal_list_add_tail(&new_mmbinfo->list, &pmu->list);
|
|
|
|
pmi->phys_addr = new_mmbinfo->phys_addr;
|
|
|
|
ot_mmb_get(mmb);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int ioctl_mmb_alloc_v2(const struct file *file,
|
|
unsigned int iocmd, struct mmb_info *pmi)
|
|
{
|
|
struct mmz_userdev_info *pmu = file->private_data;
|
|
struct mmb_info *new_mmbinfo = NULL;
|
|
ot_mmb_t *mmb = NULL;
|
|
|
|
mmb = ot_mmb_alloc_v2(pmi->mmb_name, pmi->size, pmi->align,
|
|
pmi->gfp, pmi->mmz_name, pmi->order);
|
|
if (mmb == NULL) {
|
|
#if defined(KERNEL_BIT_64) && defined(USER_BIT_32)
|
|
error_mmz("ot_mmb_alloc(%s, %llu, 0x%llx, %lu, %s) failed!\n",
|
|
pmi->mmb_name, pmi->size, pmi->align,
|
|
pmi->gfp, pmi->mmz_name);
|
|
#else
|
|
error_mmz("ot_mmb_alloc(%s, %lu, 0x%lx, %lu, %s) failed!\n",
|
|
pmi->mmb_name, pmi->size, pmi->align,
|
|
pmi->gfp, pmi->mmz_name);
|
|
#endif
|
|
return -ENOMEM;
|
|
}
|
|
|
|
new_mmbinfo = kmalloc(sizeof(*new_mmbinfo), GFP_KERNEL);
|
|
if (new_mmbinfo == NULL) {
|
|
ot_mmb_free(mmb);
|
|
error_mmz("alloc mmb_info failed!\n");
|
|
return -ENOMEM;
|
|
}
|
|
|
|
(void)memcpy_s(new_mmbinfo, sizeof(*new_mmbinfo), pmi, sizeof(*pmi));
|
|
new_mmbinfo->phys_addr = ot_mmb_phys(mmb);
|
|
new_mmbinfo->mmb = mmb;
|
|
new_mmbinfo->prot = PROT_READ;
|
|
new_mmbinfo->flags = MAP_SHARED;
|
|
osal_list_add_tail(&new_mmbinfo->list, &pmu->list);
|
|
|
|
pmi->phys_addr = new_mmbinfo->phys_addr;
|
|
|
|
ot_mmb_get(mmb);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static struct mmb_info *get_mmbinfo(unsigned long addr, const struct mmz_userdev_info *pmu)
|
|
{
|
|
struct mmb_info *p = NULL;
|
|
|
|
osal_list_for_each_entry(p, &pmu->list, list) {
|
|
if ((addr >= p->phys_addr) && (addr < (p->phys_addr + p->size))) {
|
|
break;
|
|
}
|
|
}
|
|
if (&p->list == &pmu->list) {
|
|
return NULL;
|
|
}
|
|
|
|
return p;
|
|
}
|
|
|
|
static struct mmb_info *get_mmbinfo_safe(unsigned long addr, const struct mmz_userdev_info *pmu)
|
|
{
|
|
struct mmb_info *p = NULL;
|
|
|
|
p = get_mmbinfo(addr, pmu);
|
|
if (p == NULL) {
|
|
error_mmz("mmb(0x%08lX) not found!\n", addr);
|
|
return NULL;
|
|
}
|
|
|
|
return p;
|
|
}
|
|
|
|
static int ioctl_mmb_user_unmap(const struct file *file, unsigned int iocmd, struct mmb_info *pmi);
|
|
|
|
static int _usrdev_mmb_free(struct mmb_info *p)
|
|
{
|
|
int ret;
|
|
|
|
osal_list_del(&p->list);
|
|
ot_mmb_put(p->mmb);
|
|
ret = ot_mmb_free(p->mmb);
|
|
kfree(p);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int ioctl_mmb_free(const struct file *file,
|
|
unsigned int iocmd, const struct mmb_info *pmi)
|
|
{
|
|
int ret;
|
|
struct mmz_userdev_info *pmu = file->private_data;
|
|
struct mmb_info *p = get_mmbinfo_safe(pmi->phys_addr, pmu);
|
|
|
|
if ((p == NULL) || (p->mmb == NULL)) {
|
|
return -EPERM;
|
|
}
|
|
|
|
if (p->delayed_free) {
|
|
warning("mmb<%s> is delayed_free, can not free again!\n", p->mmb->name);
|
|
return -EBUSY;
|
|
}
|
|
|
|
if ((p->map_ref > 0) || (p->mmb_ref > 0)) {
|
|
warning("mmb<%s> is still in use!\n", p->mmb->name);
|
|
p->delayed_free = 1;
|
|
return -EBUSY;
|
|
}
|
|
|
|
ret = _usrdev_mmb_free(p);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int ioctl_mmb_attr(const struct file *file,
|
|
unsigned int iocmd, struct mmb_info *pmi)
|
|
{
|
|
struct mmz_userdev_info *pmu = file->private_data;
|
|
struct mmb_info *p = NULL;
|
|
|
|
if ((p = get_mmbinfo_safe(pmi->phys_addr, pmu)) == NULL) {
|
|
return -EPERM;
|
|
}
|
|
|
|
(void)memcpy_s(pmi, sizeof(*pmi), p, sizeof(*pmi));
|
|
return 0;
|
|
}
|
|
|
|
static int ioctl_mmb_user_remap(struct file *file, unsigned int iocmd, struct mmb_info *pmi, int cached)
|
|
{
|
|
struct mmz_userdev_info *pmu = file->private_data;
|
|
struct mmb_info *p = NULL;
|
|
unsigned long addr, len, prot, flags, pgoff;
|
|
|
|
if ((p = get_mmbinfo_safe(pmi->phys_addr, pmu)) == NULL) {
|
|
return -EPERM;
|
|
}
|
|
/*
|
|
* mmb could be remapped for more than once, but should not
|
|
* be remapped with confusing cache type.
|
|
*/
|
|
if ((p->mapped != NULL) && (p->mmb != NULL) && (p->map_ref > 0)) {
|
|
if (cached != p->map_cached) {
|
|
error_mmz("mmb<%s> already mapped as %s, cannot remap as %s.\n",
|
|
p->mmb->name, p->map_cached ? "cached" : "non-cached", cached ? "cached" : "non-cached");
|
|
return -EINVAL;
|
|
}
|
|
|
|
p->map_ref++;
|
|
p->mmb_ref++;
|
|
|
|
ot_mmb_get(p->mmb);
|
|
|
|
/*
|
|
* pmi->phys may not always start at p->phys,
|
|
* and may start with offset from p->phys.
|
|
* so, we need to calculate with the offset.
|
|
*/
|
|
pmi->mapped = p->mapped + (pmi->phys_addr - p->phys_addr);
|
|
|
|
return 0;
|
|
}
|
|
|
|
if (p->phys_addr & ~PAGE_MASK) {
|
|
return -EINVAL;
|
|
}
|
|
|
|
addr = 0;
|
|
len = PAGE_ALIGN(p->size);
|
|
|
|
prot = pmi->prot;
|
|
flags = pmi->flags;
|
|
if (prot == 0) {
|
|
prot = p->prot;
|
|
}
|
|
if (flags == 0) {
|
|
flags = p->flags;
|
|
}
|
|
|
|
pmu->mmap_pid = current->pid;
|
|
p->map_cached = cached;
|
|
|
|
pgoff = p->phys_addr;
|
|
addr = vm_mmap(file, addr, len, prot, flags, pgoff);
|
|
|
|
pmu->mmap_pid = 0;
|
|
|
|
if (IS_ERR_VALUE((uintptr_t)addr)) {
|
|
error_mmz("vm_mmap(file, 0, %lu, 0x%08lX, 0x%08lX, 0x%08lX) return 0x%08lX\n",
|
|
len, prot, flags, pgoff, addr);
|
|
return addr;
|
|
}
|
|
|
|
p->mapped = (void *)(uintptr_t)addr;
|
|
p->prot = prot;
|
|
p->flags = flags;
|
|
|
|
p->map_ref++;
|
|
p->mmb_ref++;
|
|
ot_mmb_get(p->mmb);
|
|
|
|
/*
|
|
* pmi->phys may not always start at p->phys,
|
|
* and may start with offset from p->phys.
|
|
* so, we need to calculate with the offset.
|
|
*/
|
|
pmi->mapped = p->mapped + (pmi->phys_addr - p->phys_addr);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int ioctl_mmb_user_unmap_with_cache(const struct mmb_info *p, unsigned long addr, unsigned long len)
|
|
{
|
|
struct mm_struct *mm = current->mm;
|
|
|
|
mmz_map_down(mm);
|
|
if (ot_vma_check(addr, addr + len)) {
|
|
error_mmz("mmb<%s> vma is invalid.\n", p->mmb->name);
|
|
mmz_map_up(mm);
|
|
return -EPERM;
|
|
}
|
|
#ifdef CONFIG_64BIT
|
|
__flush_dcache_area((void *)(uintptr_t)addr, (size_t)len);
|
|
#else
|
|
__cpuc_flush_dcache_area((void *)(uintptr_t)addr, (size_t)len);
|
|
#ifdef CONFIG_CACHE_L2X0
|
|
outer_flush_range(p->phys_addr, p->phys_addr + len);
|
|
#endif
|
|
#endif /* CONFIG_64BIT */
|
|
mmz_map_up(mm);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int ioctl_mmb_user_unmap(const struct file *file, unsigned int iocmd, struct mmb_info *pmi)
|
|
{
|
|
int ret;
|
|
unsigned long addr, len;
|
|
struct mmb_info *p = NULL;
|
|
struct mmz_userdev_info *pmu = file->private_data;
|
|
|
|
p = get_mmbinfo_safe(pmi->phys_addr, pmu);
|
|
if (p == NULL) {
|
|
return -EPERM;
|
|
}
|
|
|
|
if (p->mapped == NULL) {
|
|
#if defined(KERNEL_BIT_64) && defined(USER_BIT_32)
|
|
warning("mmb(0x%llx) isn't user-mapped!\n", p->phys_addr);
|
|
#else
|
|
warning("mmb(0x%lx) isn't user-mapped!\n", p->phys_addr);
|
|
#endif
|
|
pmi->mapped = NULL;
|
|
return -EIO;
|
|
}
|
|
|
|
if (!((p->map_ref > 0) && (p->mmb_ref > 0))) {
|
|
error_mmz("mmb<%s> has invalid refer: map_ref=%d, mmb_ref=%d.\n",
|
|
p->mmb->name, p->map_ref, p->mmb_ref);
|
|
return -EIO;
|
|
}
|
|
|
|
p->map_ref--;
|
|
p->mmb_ref--;
|
|
ot_mmb_put(p->mmb);
|
|
|
|
if (p->map_ref > 0) {
|
|
return 0;
|
|
}
|
|
|
|
addr = (unsigned long)(uintptr_t)p->mapped;
|
|
len = PAGE_ALIGN(p->size);
|
|
|
|
/* todo,before unmap,refresh cache manually */
|
|
if (p->map_cached) {
|
|
ret = ioctl_mmb_user_unmap_with_cache(p, addr, len);
|
|
if (ret != 0) {
|
|
return ret;
|
|
}
|
|
}
|
|
|
|
ret = vm_munmap(addr, len);
|
|
if (!IS_ERR_VALUE((uintptr_t)ret)) {
|
|
p->mapped = NULL;
|
|
pmi->mapped = NULL;
|
|
}
|
|
|
|
if (p->delayed_free && (p->map_ref == 0) && (p->mmb_ref == 0)) {
|
|
_usrdev_mmb_free(p);
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int ioctl_mmb_virt2phys(const struct file *file,
|
|
unsigned int iocmd, struct mmb_info *pmi)
|
|
{
|
|
int ret = 0;
|
|
unsigned long virt, phys;
|
|
unsigned long offset = 0;
|
|
|
|
virt = (unsigned long)(uintptr_t)pmi->mapped;
|
|
phys = usr_virt_to_phys(virt);
|
|
if (!phys) {
|
|
ret = -ENOMEM;
|
|
}
|
|
|
|
if (ot_mmb_getby_phys_2(phys, &offset) == NULL) {
|
|
error_mmz("Not mmz alloc memory[0x%lx 0x%lx]! 0x%lx\n", virt, phys, offset);
|
|
return -EINVAL;
|
|
}
|
|
|
|
pmi->phys_addr = phys;
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int mmz_userdev_ioctl_m(struct file *file, unsigned int cmd, struct mmb_info *pmi)
|
|
{
|
|
int ret;
|
|
|
|
switch (_IOC_NR(cmd)) {
|
|
case _IOC_NR(IOC_MMB_ALLOC):
|
|
ret = ioctl_mmb_alloc(file, cmd, pmi);
|
|
break;
|
|
case _IOC_NR(IOC_MMB_ALLOC_V2):
|
|
ret = ioctl_mmb_alloc_v2(file, cmd, pmi);
|
|
break;
|
|
case _IOC_NR(IOC_MMB_ATTR):
|
|
ret = ioctl_mmb_attr(file, cmd, pmi);
|
|
break;
|
|
case _IOC_NR(IOC_MMB_FREE):
|
|
ret = ioctl_mmb_free(file, cmd, pmi);
|
|
break;
|
|
|
|
case _IOC_NR(IOC_MMB_USER_REMAP):
|
|
ret = ioctl_mmb_user_remap(file, cmd, pmi, 0);
|
|
break;
|
|
case _IOC_NR(IOC_MMB_USER_REMAP_CACHED):
|
|
ret = ioctl_mmb_user_remap(file, cmd, pmi, 1);
|
|
break;
|
|
case _IOC_NR(IOC_MMB_USER_UNMAP):
|
|
ret = ioctl_mmb_user_unmap(file, cmd, pmi);
|
|
break;
|
|
case _IOC_NR(IOC_MMB_VIRT_GET_PHYS):
|
|
ret = ioctl_mmb_virt2phys(file, cmd, pmi);
|
|
break;
|
|
default:
|
|
error_mmz("invalid ioctl cmd = %08X\n", cmd);
|
|
ret = -EINVAL;
|
|
break;
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int mmz_userdev_ioctl_r(unsigned int cmd, struct mmb_info *pmi)
|
|
{
|
|
switch (_IOC_NR(cmd)) {
|
|
case _IOC_NR(IOC_MMB_ADD_REF):
|
|
pmi->mmb_ref++;
|
|
ot_mmb_get(pmi->mmb);
|
|
break;
|
|
case _IOC_NR(IOC_MMB_DEC_REF):
|
|
if (pmi->mmb_ref <= 0) {
|
|
error_mmz("mmb<%s> mmb_ref is %d!\n", pmi->mmb->name, pmi->mmb_ref);
|
|
return -EPERM;
|
|
}
|
|
pmi->mmb_ref--;
|
|
ot_mmb_put(pmi->mmb);
|
|
if (pmi->delayed_free && (pmi->mmb_ref == 0) && (pmi->map_ref == 0)) {
|
|
_usrdev_mmb_free(pmi);
|
|
}
|
|
break;
|
|
default:
|
|
return -EINVAL;
|
|
break;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/* just for test */
|
|
static int mmz_userdev_ioctl_t(const struct file *file, unsigned int cmd, const struct mmb_info *pmi);
|
|
|
|
static int mmz_userdev_ioctl_of_m(struct file *file, unsigned int cmd, unsigned long arg)
|
|
{
|
|
struct mmb_info mi = { 0 };
|
|
int ret;
|
|
|
|
if ((_IOC_SIZE(cmd) > sizeof(mi)) || (arg == 0)) {
|
|
error_mmz("_IOC_SIZE(cmd)=%d, arg==0x%08lX\n", _IOC_SIZE(cmd), arg);
|
|
return -EINVAL;
|
|
}
|
|
(void)memset_s(&mi, sizeof(mi), 0, sizeof(mi));
|
|
if (copy_from_user(&mi, (void *)(uintptr_t)arg, _IOC_SIZE(cmd))) {
|
|
osal_trace("\nmmz_userdev_ioctl: copy_from_user error.\n");
|
|
return -EFAULT;
|
|
}
|
|
mi.mmz_name[OT_MMZ_NAME_LEN - 1] = '\0';
|
|
mi.mmb_name[OT_MMB_NAME_LEN - 1] = '\0';
|
|
ret = mmz_userdev_ioctl_m(file, cmd, &mi);
|
|
if (!ret && (cmd & IOC_OUT)) {
|
|
if (copy_to_user((void *)(uintptr_t)arg, &mi, _IOC_SIZE(cmd))) {
|
|
osal_trace("\nmmz_userdev_ioctl: copy_to_user error.\n");
|
|
return -EFAULT;
|
|
}
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
static int mmz_userdev_ioctl_of_r(struct file *file, unsigned int cmd, unsigned long arg,
|
|
struct mmz_userdev_info *pmu)
|
|
{
|
|
struct mmb_info *pmi = NULL;
|
|
|
|
pmi = get_mmbinfo_safe(arg, pmu);
|
|
if (pmi == NULL) {
|
|
return -EPERM;
|
|
}
|
|
|
|
return mmz_userdev_ioctl_r(cmd, pmi);
|
|
}
|
|
|
|
static int mmz_userdev_ioctl_of_c(unsigned int cmd, unsigned long arg, const struct mmz_userdev_info *pmu)
|
|
{
|
|
struct mmb_info *pmi = NULL;
|
|
int ret = 0;
|
|
|
|
if (arg == 0) {
|
|
mmz_flush_dcache_all();
|
|
return ret;
|
|
}
|
|
|
|
pmi = get_mmbinfo_safe(arg, pmu);
|
|
if (pmi == NULL) {
|
|
return -EPERM;
|
|
}
|
|
|
|
switch (_IOC_NR(cmd)) {
|
|
case _IOC_NR(IOC_MMB_FLUSH_DCACHE):
|
|
mmz_flush_dcache_mmb(pmi);
|
|
break;
|
|
default:
|
|
ret = -EINVAL;
|
|
break;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
static int mmz_userdev_check_dirty_area_range(const struct dirty_area *area,
|
|
const struct mmz_userdev_info *pmu)
|
|
{
|
|
ot_mmb_t *mmb = NULL;
|
|
struct mmb_info *pmi = NULL;
|
|
unsigned long offset;
|
|
|
|
mmb = ot_mmb_getby_phys_2(area->dirty_phys_start, &offset);
|
|
if (mmb == NULL) {
|
|
#if defined(KERNEL_BIT_64) && defined(USER_BIT_32)
|
|
error_mmz("dirty_phys_addr=0x%llx\n", area->dirty_phys_start);
|
|
#else
|
|
error_mmz("dirty_phys_addr=0x%lx\n", area->dirty_phys_start);
|
|
#endif
|
|
return -EFAULT;
|
|
}
|
|
|
|
pmi = get_mmbinfo_safe(mmb->phys_addr, pmu);
|
|
if (pmi == NULL) {
|
|
return -EPERM;
|
|
}
|
|
if ((uintptr_t)area->dirty_virt_start != (uintptr_t)pmi->mapped + offset) {
|
|
osal_trace(KERN_WARNING "dirty_virt_start addr was not consistent with dirty_phys_start addr!\n");
|
|
return -EFAULT;
|
|
}
|
|
if (area->dirty_phys_start + area->dirty_size > mmb->phys_addr + mmb->length) {
|
|
osal_trace(KERN_WARNING "\ndirty area overflow!\n");
|
|
return -EFAULT;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int mmz_userdev_ioctl_of_d(unsigned int cmd, unsigned long arg, const struct mmz_userdev_info *pmu)
|
|
{
|
|
struct dirty_area area;
|
|
__phys_addr_type__ orig_addr;
|
|
unsigned long virt_addr;
|
|
struct mm_struct *mm = current->mm;
|
|
int ret;
|
|
|
|
if ((_IOC_SIZE(cmd) != sizeof(area)) || (arg == 0)) {
|
|
error_mmz("_IOC_SIZE(cmd)=%d, arg==0x%08lx\n", _IOC_SIZE(cmd), arg);
|
|
return -EINVAL;
|
|
}
|
|
(void)memset_s(&area, sizeof(area), 0, sizeof(area));
|
|
if (copy_from_user(&area, (void *)(uintptr_t)arg, _IOC_SIZE(cmd))) {
|
|
osal_trace(KERN_WARNING "\nmmz_userdev_ioctl: copy_from_user error.\n");
|
|
return -EFAULT;
|
|
}
|
|
|
|
ret = mmz_userdev_check_dirty_area_range(&area, pmu);
|
|
if (ret != 0) {
|
|
return ret;
|
|
}
|
|
|
|
mmz_map_down(mm);
|
|
|
|
if (ot_vma_check((uintptr_t)area.dirty_virt_start, (uintptr_t)area.dirty_virt_start + area.dirty_size)) {
|
|
osal_trace(KERN_WARNING "\ndirty area[0x%lx,0x%lx] overflow!\n",
|
|
(unsigned long)(uintptr_t)area.dirty_virt_start,
|
|
(unsigned long)(uintptr_t)area.dirty_virt_start + area.dirty_size);
|
|
mmz_map_up(mm);
|
|
return -EFAULT;
|
|
}
|
|
|
|
/* cache line aligned */
|
|
orig_addr = area.dirty_phys_start;
|
|
area.dirty_phys_start &= ~(CACHE_LINE_SIZE - 1);
|
|
virt_addr = (unsigned long)(uintptr_t)area.dirty_virt_start;
|
|
virt_addr &= ~(CACHE_LINE_SIZE - 1);
|
|
area.dirty_virt_start = (void *)(uintptr_t)virt_addr;
|
|
area.dirty_size = (area.dirty_size + (orig_addr - area.dirty_phys_start) +
|
|
(CACHE_LINE_SIZE - 1)) & ~(CACHE_LINE_SIZE - 1);
|
|
|
|
mmz_flush_dcache_mmb_dirty(&area);
|
|
mmz_map_up(mm);
|
|
return 0;
|
|
}
|
|
|
|
static int mmz_userdev_ioctl_of_t(const struct file *file, unsigned int cmd, unsigned long arg,
|
|
const struct mmz_userdev_info *pmu)
|
|
{
|
|
struct mmb_info mi;
|
|
|
|
if ((_IOC_SIZE(cmd) != sizeof(mi)) || (arg == 0)) {
|
|
error_mmz("_IOC_SIZE(cmd)=%d, arg==0x%08lx\n", _IOC_SIZE(cmd), arg);
|
|
return -EINVAL;
|
|
}
|
|
|
|
(void)memset_s(&mi, sizeof(mi), 0, sizeof(mi));
|
|
if (copy_from_user(&mi, (void *)(uintptr_t)arg, sizeof(mi))) {
|
|
osal_trace("\nmmz_userdev_ioctl: copy_from_user error.\n");
|
|
return -EFAULT;
|
|
}
|
|
|
|
if ((get_mmbinfo_safe(mi.phys_addr, pmu)) == NULL) {
|
|
return -EPERM;
|
|
}
|
|
return mmz_userdev_ioctl_t(file, cmd, &mi);
|
|
}
|
|
|
|
static long mmz_userdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
|
|
{
|
|
int ret;
|
|
struct mmz_userdev_info *pmu = file->private_data;
|
|
|
|
down(&pmu->sem);
|
|
|
|
if (_IOC_TYPE(cmd) == 'm') {
|
|
ret = mmz_userdev_ioctl_of_m(file, cmd, arg);
|
|
} else if (_IOC_TYPE(cmd) == 'r') {
|
|
ret = mmz_userdev_ioctl_of_r(file, cmd, arg, pmu);
|
|
} else if (_IOC_TYPE(cmd) == 'c') {
|
|
ret = mmz_userdev_ioctl_of_c(cmd, arg, pmu);
|
|
} else if (_IOC_TYPE(cmd) == 'd') {
|
|
ret = mmz_userdev_ioctl_of_d(cmd, arg, pmu);
|
|
} else if (_IOC_TYPE(cmd) == 't') {
|
|
ret = mmz_userdev_ioctl_of_t(file, cmd, arg, pmu);
|
|
} else {
|
|
ret = -EINVAL;
|
|
}
|
|
|
|
up(&pmu->sem);
|
|
return ret;
|
|
}
|
|
|
|
#ifdef CONFIG_COMPAT
|
|
static long compat_mmz_userdev_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
|
|
{
|
|
return mmz_userdev_ioctl(file, cmd, (unsigned long)(uintptr_t)compat_ptr(arg));
|
|
}
|
|
#endif
|
|
|
|
static void mmz_userdev_mmap_with_file_flag(const struct file *file, struct vm_area_struct *vma,
|
|
int mmb_cached)
|
|
{
|
|
if (file->f_flags & O_SYNC) {
|
|
#ifdef CONFIG_64BIT
|
|
vma->vm_page_prot = __pgprot(pgprot_val(vma->vm_page_prot)
|
|
| PTE_WRITE | PTE_DIRTY);
|
|
#endif
|
|
vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
|
|
} else {
|
|
#ifdef CONFIG_64BIT
|
|
vma->vm_page_prot = __pgprot(pgprot_val(vma->vm_page_prot)
|
|
| PTE_WRITE | PTE_DIRTY);
|
|
#else
|
|
vma->vm_page_prot = __pgprot(pgprot_val(vma->vm_page_prot)
|
|
| L_PTE_PRESENT | L_PTE_YOUNG
|
|
| L_PTE_DIRTY | L_PTE_MT_DEV_CACHED);
|
|
#endif
|
|
if (mmb_cached == 0) {
|
|
/*
|
|
* pagetable property changes from <normal nocache> to
|
|
* <strong order>, for enhance ddr access performance,
|
|
*/
|
|
/* <normal nocache> */
|
|
vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
|
|
}
|
|
}
|
|
}
|
|
|
|
static void mmz_userdev_mmap_page_valid(struct vm_area_struct *vma)
|
|
{
|
|
unsigned long start = vma->vm_start;
|
|
unsigned long pfn = vma->vm_pgoff;
|
|
size_t size = vma->vm_end - vma->vm_start;
|
|
|
|
while (size) {
|
|
if (pfn_valid(pfn)) {
|
|
if (vm_insert_page(vma, start, pfn_to_page(pfn))) {
|
|
error_mmz("insert page failed.\n");
|
|
break;
|
|
}
|
|
} else {
|
|
#ifdef CONFIG_64BIT
|
|
error_mmz("vm map failed for phy address(0x%llx)\n", __pfn_to_phys(pfn));
|
|
#else
|
|
error_mmz("vm map failed for phy address(0x%x)\n", __pfn_to_phys(pfn));
|
|
#endif
|
|
}
|
|
|
|
start += PAGE_SIZE;
|
|
size = (size < PAGE_SIZE) ? 0 : (size - PAGE_SIZE);
|
|
pfn++;
|
|
}
|
|
}
|
|
|
|
static int mmz_userdev_mmap_page_invalid(struct vm_area_struct *vma)
|
|
{
|
|
size_t size = vma->vm_end - vma->vm_start;
|
|
if (size == 0) {
|
|
return -EPERM;
|
|
}
|
|
/*
|
|
* Remap-pfn-range will mark the range
|
|
* as VM_IO and VM_RESERVED
|
|
*/
|
|
if (remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff, vma->vm_end - vma->vm_start,
|
|
vma->vm_page_prot)) {
|
|
return -EAGAIN;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
int mmz_userdev_mmap(struct file *file, struct vm_area_struct *vma)
|
|
{
|
|
struct mmb_info *p = NULL;
|
|
struct mmz_userdev_info *pmu = NULL;
|
|
unsigned long offset;
|
|
ot_mmb_t *mmb = NULL;
|
|
int mmb_cached = 0;
|
|
int ret;
|
|
|
|
if ((file == NULL) || (file->private_data == NULL) || (vma == NULL) || (vma->vm_mm == NULL)) {
|
|
error_mmz("%s - parameter invalid!\n", __FUNCTION__);
|
|
return -1;
|
|
}
|
|
|
|
pmu = file->private_data;
|
|
offset = vma->vm_pgoff << PAGE_SHIFT;
|
|
|
|
p = get_mmbinfo(offset, pmu);
|
|
if (p == NULL) {
|
|
unsigned long mmb_offset;
|
|
mmb = ot_mmb_getby_phys_2(offset, &mmb_offset);
|
|
if (mmb == NULL) {
|
|
/* Allow mmap MMZ allocated by other core. */
|
|
if (ot_map_mmz_check_phys(offset, vma->vm_end - vma->vm_start)) {
|
|
return -EPERM;
|
|
}
|
|
} else {
|
|
mmb_cached = mmb->flags & OT_MMB_MAP2KERN_CACHED;
|
|
}
|
|
} else {
|
|
if (p->mapped != NULL) {
|
|
error_mmz("mmb(0x%08lX) have been mapped already?!\n", offset);
|
|
return -EIO;
|
|
}
|
|
mmb_cached = p->map_cached;
|
|
}
|
|
|
|
mmz_userdev_mmap_with_file_flag(file, vma, mmb_cached);
|
|
|
|
if (pfn_valid(vma->vm_pgoff)) {
|
|
mmz_userdev_mmap_page_valid(vma);
|
|
} else {
|
|
ret = mmz_userdev_mmap_page_invalid(vma);
|
|
if (ret != 0) {
|
|
return ret;
|
|
}
|
|
}
|
|
|
|
#ifdef CONFIG_OT_SYS_SMMU_SUPPORT
|
|
/* flush page table cache to share the data with CBBs that have sMMU */
|
|
svm_flush_cache(vma->vm_mm, vma->vm_start, vma->vm_end - vma->vm_start);
|
|
#endif
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int mmz_userdev_release(struct inode *inode, struct file *file)
|
|
{
|
|
struct mmz_userdev_info *pmu = file->private_data;
|
|
struct mmb_info *p = NULL;
|
|
struct mmb_info *n = NULL;
|
|
|
|
list_for_each_entry_safe(p, n, &pmu->list, list) {
|
|
error_mmz("MMB LEAK(pid=%d): 0x%lX, %lu bytes, '%s'\n",
|
|
pmu->pid, ot_mmb_phys(p->mmb),
|
|
ot_mmb_length(p->mmb),
|
|
ot_mmb_name(p->mmb));
|
|
|
|
/*
|
|
* we do not need to release mapped-area here,
|
|
* system will do it for us
|
|
*/
|
|
if (p->mapped != NULL) {
|
|
#if defined(KERNEL_BIT_64) && defined(USER_BIT_32)
|
|
warning("mmb<0x%llx> mapped to userspace 0x%pK will be unmaped!\n",
|
|
p->phys_addr, p->mapped);
|
|
#else
|
|
warning("mmb<0x%lx> mapped to userspace 0x%pK will be unmaped!\n",
|
|
p->phys_addr, p->mapped);
|
|
#endif
|
|
}
|
|
for (; p->mmb_ref > 0; p->mmb_ref--) {
|
|
ot_mmb_put(p->mmb);
|
|
}
|
|
_usrdev_mmb_free(p);
|
|
}
|
|
|
|
file->private_data = NULL;
|
|
kfree(pmu);
|
|
pmu = NULL;
|
|
return 0;
|
|
}
|
|
|
|
static struct file_operations g_mmz_userdev_fops = {
|
|
.owner = THIS_MODULE,
|
|
.open = mmz_userdev_open,
|
|
.release = mmz_userdev_release,
|
|
#ifdef CONFIG_COMPAT
|
|
.compat_ioctl = compat_mmz_userdev_ioctl,
|
|
#endif
|
|
.unlocked_ioctl = mmz_userdev_ioctl,
|
|
.mmap = mmz_userdev_mmap,
|
|
};
|
|
|
|
static struct miscdevice g_mmz_userdev = {
|
|
.minor = MISC_DYNAMIC_MINOR,
|
|
.fops = &g_mmz_userdev_fops,
|
|
.name = "mmz_userdev"
|
|
};
|
|
|
|
int __init mmz_userdev_init(void)
|
|
{
|
|
int ret;
|
|
ret = misc_register(&g_mmz_userdev);
|
|
if (ret) {
|
|
osal_trace("register mmz dev failure!\n");
|
|
return -1;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
void __exit mmz_userdev_exit(void)
|
|
{
|
|
misc_deregister(&g_mmz_userdev);
|
|
}
|
|
|
|
/* Test func */
|
|
static int mmz_userdev_ioctl_t(const struct file *file, unsigned int cmd, const struct mmb_info *pmi)
|
|
{
|
|
return 0;
|
|
}
|
|
|