/*-
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2011 NetApp, Inc.
 * All rights reserved.
 */

#include <sys/types.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/sx.h>
#include <sys/systm.h>

#include <machine/vmm.h>

#include <vm/vm.h>
#include <vm/vm_param.h>
#include <vm/vm_extern.h>
#include <vm/pmap.h>
#include <vm/vm_map.h>
#include <vm/vm_object.h>
#include <vm/vm_page.h>

#include <dev/vmm/vmm_dev.h>
#include <dev/vmm/vmm_mem.h>
#include <dev/vmm/vmm_vm.h>

static void vm_free_memmap(struct vm *vm, int ident);

int
vm_mem_init(struct vm_mem *mem, vm_offset_t lo, vm_offset_t hi)
{
	mem->mem_vmspace = vmmops_vmspace_alloc(lo, hi);
	if (mem->mem_vmspace == NULL)
		return (ENOMEM);
	sx_init(&mem->mem_segs_lock, "vm_mem_segs");
	return (0);
}

static bool
sysmem_mapping(struct vm_mem *mem, int idx)
{
	if (mem->mem_maps[idx].len != 0 &&
	    mem->mem_segs[mem->mem_maps[idx].segid].sysmem)
		return (true);
	else
		return (false);
}

bool
vm_memseg_sysmem(struct vm *vm, int ident)
{
	struct vm_mem *mem;

	mem = vm_mem(vm);
	vm_assert_memseg_locked(vm);

	if (ident < 0 || ident >= VM_MAX_MEMSEGS)
		return (false);

	return (mem->mem_segs[ident].sysmem);
}

void
vm_mem_cleanup(struct vm *vm)
{
	struct vm_mem *mem;

	mem = vm_mem(vm);

	/*
	 * System memory is removed from the guest address space only when
	 * the VM is destroyed. This is because the mapping remains the same
	 * across VM reset.
	 *
	 * Device memory can be relocated by the guest (e.g. using PCI BARs)
	 * so those mappings are removed on a VM reset.
	 */
	for (int i = 0; i < VM_MAX_MEMMAPS; i++) {
		if (!sysmem_mapping(mem, i))
			vm_free_memmap(vm, i);
	}
}

void
vm_mem_destroy(struct vm *vm)
{
	struct vm_mem *mem;

	mem = vm_mem(vm);
	vm_assert_memseg_xlocked(vm);

	for (int i = 0; i < VM_MAX_MEMMAPS; i++) {
		if (sysmem_mapping(mem, i))
			vm_free_memmap(vm, i);
	}

	for (int i = 0; i < VM_MAX_MEMSEGS; i++)
		vm_free_memseg(vm, i);

	vmmops_vmspace_free(mem->mem_vmspace);

	sx_xunlock(&mem->mem_segs_lock);
	sx_destroy(&mem->mem_segs_lock);
}

struct vmspace *
vm_vmspace(struct vm *vm)
{
	struct vm_mem *mem;

	mem = vm_mem(vm);
	return (mem->mem_vmspace);
}

void
vm_slock_memsegs(struct vm *vm)
{
	sx_slock(&vm_mem(vm)->mem_segs_lock);
}

void
vm_xlock_memsegs(struct vm *vm)
{
	sx_xlock(&vm_mem(vm)->mem_segs_lock);
}

void
vm_unlock_memsegs(struct vm *vm)
{
	sx_unlock(&vm_mem(vm)->mem_segs_lock);
}

void
vm_assert_memseg_locked(struct vm *vm)
{
	sx_assert(&vm_mem(vm)->mem_segs_lock, SX_LOCKED);
}

void
vm_assert_memseg_xlocked(struct vm *vm)
{
	sx_assert(&vm_mem(vm)->mem_segs_lock, SX_XLOCKED);
}

/*
 * Return 'true' if 'gpa' is allocated in the guest address space.
 *
 * This function is called in the context of a running vcpu which acts as
 * an implicit lock on 'vm->mem_maps[]'.
 */
bool
vm_mem_allocated(struct vcpu *vcpu, vm_paddr_t gpa)
{
	struct vm *vm = vcpu_vm(vcpu);
	struct vm_mem_map *mm;
	int i;

#ifdef INVARIANTS
	int hostcpu, state;
	state = vcpu_get_state(vcpu, &hostcpu);
	KASSERT(state == VCPU_RUNNING && hostcpu == curcpu,
	    ("%s: invalid vcpu state %d/%d", __func__, state, hostcpu));
#endif

	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
		mm = &vm_mem(vm)->mem_maps[i];
		if (mm->len != 0 && gpa >= mm->gpa && gpa < mm->gpa + mm->len)
			return (true);		/* 'gpa' is sysmem or devmem */
	}

	return (false);
}

int
vm_alloc_memseg(struct vm *vm, int ident, size_t len, bool sysmem,
    struct domainset *obj_domainset)
{
	struct vm_mem_seg *seg;
	struct vm_mem *mem;
	vm_object_t obj;

	mem = vm_mem(vm);
	vm_assert_memseg_xlocked(vm);

	if (ident < 0 || ident >= VM_MAX_MEMSEGS)
		return (EINVAL);

	if (len == 0 || (len & PAGE_MASK))
		return (EINVAL);

	seg = &mem->mem_segs[ident];
	if (seg->object != NULL) {
		if (seg->len == len && seg->sysmem == sysmem)
			return (EEXIST);
		else
			return (EINVAL);
	}

	/*
	 * When given an impossible policy, signal an
	 * error to the user.
	 */
	if (obj_domainset != NULL && domainset_empty_vm(obj_domainset))
		return (EINVAL);
	obj = vm_object_allocate(OBJT_SWAP, len >> PAGE_SHIFT);
	if (obj == NULL)
		return (ENOMEM);

	seg->len = len;
	seg->object = obj;
	if (obj_domainset != NULL)
		seg->object->domain.dr_policy = obj_domainset;
	seg->sysmem = sysmem;

	return (0);
}

int
vm_get_memseg(struct vm *vm, int ident, size_t *len, bool *sysmem,
    vm_object_t *objptr)
{
	struct vm_mem *mem;
	struct vm_mem_seg *seg;

	mem = vm_mem(vm);

	vm_assert_memseg_locked(vm);

	if (ident < 0 || ident >= VM_MAX_MEMSEGS)
		return (EINVAL);

	seg = &mem->mem_segs[ident];
	if (len)
		*len = seg->len;
	if (sysmem)
		*sysmem = seg->sysmem;
	if (objptr)
		*objptr = seg->object;
	return (0);
}

void
vm_free_memseg(struct vm *vm, int ident)
{
	struct vm_mem_seg *seg;

	KASSERT(ident >= 0 && ident < VM_MAX_MEMSEGS,
	    ("%s: invalid memseg ident %d", __func__, ident));

	seg = &vm_mem(vm)->mem_segs[ident];
	if (seg->object != NULL) {
		vm_object_deallocate(seg->object);
		bzero(seg, sizeof(struct vm_mem_seg));
	}
}

int
vm_mmap_memseg(struct vm *vm, vm_paddr_t gpa, int segid, vm_ooffset_t first,
    size_t len, int prot, int flags)
{
	struct vm_mem *mem;
	struct vm_mem_seg *seg;
	struct vm_mem_map *m, *map;
	struct vm_map *vmmap;
	vm_ooffset_t last;
	int i, error;

	if (prot == 0 || (prot & ~(VM_PROT_ALL)) != 0)
		return (EINVAL);

	if (flags & ~VM_MEMMAP_F_WIRED)
		return (EINVAL);

	if (segid < 0 || segid >= VM_MAX_MEMSEGS)
		return (EINVAL);

	mem = vm_mem(vm);
	seg = &mem->mem_segs[segid];
	if (seg->object == NULL)
		return (EINVAL);

	if (first + len < first || gpa + len < gpa)
		return (EINVAL);
	last = first + len;
	if (first >= last || last > seg->len)
		return (EINVAL);

	if ((gpa | first | last) & PAGE_MASK)
		return (EINVAL);

	map = NULL;
	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
		m = &mem->mem_maps[i];
		if (m->len == 0) {
			map = m;
			break;
		}
	}
	if (map == NULL)
		return (ENOSPC);

	vmmap = &mem->mem_vmspace->vm_map;
	vm_map_lock(vmmap);
	error = vm_map_insert(vmmap, seg->object, first, gpa, gpa + len,
	    prot, prot, 0);
	vm_map_unlock(vmmap);
	if (error != KERN_SUCCESS)
		return (vm_mmap_to_errno(error));
	vm_object_reference(seg->object);

	if (flags & VM_MEMMAP_F_WIRED) {
		error = vm_map_wire(vmmap, gpa, gpa + len,
		    VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES);
		if (error != KERN_SUCCESS) {
			vm_map_remove(vmmap, gpa, gpa + len);
			return (error == KERN_RESOURCE_SHORTAGE ? ENOMEM :
			    EFAULT);
		}
	}

	map->gpa = gpa;
	map->len = len;
	map->segoff = first;
	map->segid = segid;
	map->prot = prot;
	map->flags = flags;
	return (0);
}

int
vm_munmap_memseg(struct vm *vm, vm_paddr_t gpa, size_t len)
{
	struct vm_mem *mem;
	struct vm_mem_map *m;
	int i;

	mem = vm_mem(vm);
	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
		m = &mem->mem_maps[i];
#ifdef VM_MEMMAP_F_IOMMU
		if ((m->flags & VM_MEMMAP_F_IOMMU) != 0)
			continue;
#endif
		if (m->gpa == gpa && m->len == len) {
			vm_free_memmap(vm, i);
			return (0);
		}
	}

	return (EINVAL);
}

int
vm_mmap_getnext(struct vm *vm, vm_paddr_t *gpa, int *segid,
    vm_ooffset_t *segoff, size_t *len, int *prot, int *flags)
{
	struct vm_mem *mem;
	struct vm_mem_map *mm, *mmnext;
	int i;

	mem = vm_mem(vm);

	mmnext = NULL;
	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
		mm = &mem->mem_maps[i];
		if (mm->len == 0 || mm->gpa < *gpa)
			continue;
		if (mmnext == NULL || mm->gpa < mmnext->gpa)
			mmnext = mm;
	}

	if (mmnext != NULL) {
		*gpa = mmnext->gpa;
		if (segid)
			*segid = mmnext->segid;
		if (segoff)
			*segoff = mmnext->segoff;
		if (len)
			*len = mmnext->len;
		if (prot)
			*prot = mmnext->prot;
		if (flags)
			*flags = mmnext->flags;
		return (0);
	} else {
		return (ENOENT);
	}
}

static void
vm_free_memmap(struct vm *vm, int ident)
{
	struct vm_mem_map *mm;
	int error __diagused;

	mm = &vm_mem(vm)->mem_maps[ident];
	if (mm->len) {
		error = vm_map_remove(&vm_vmspace(vm)->vm_map, mm->gpa,
		    mm->gpa + mm->len);
		KASSERT(error == KERN_SUCCESS, ("%s: vm_map_remove error %d",
		    __func__, error));
		bzero(mm, sizeof(struct vm_mem_map));
	}
}

vm_paddr_t
vmm_sysmem_maxaddr(struct vm *vm)
{
	struct vm_mem *mem;
	struct vm_mem_map *mm;
	vm_paddr_t maxaddr;
	int i;

	mem = vm_mem(vm);
	maxaddr = 0;
	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
		mm = &mem->mem_maps[i];
		if (sysmem_mapping(mem, i)) {
			if (maxaddr < mm->gpa + mm->len)
				maxaddr = mm->gpa + mm->len;
		}
	}
	return (maxaddr);
}

static void *
_vm_gpa_hold(struct vm *vm, vm_paddr_t gpa, size_t len, int reqprot,
    void **cookie)
{
	struct vm_mem_map *mm;
	vm_page_t m;
	int i, count, pageoff;

	pageoff = gpa & PAGE_MASK;
	if (len > PAGE_SIZE - pageoff)
		panic("vm_gpa_hold: invalid gpa/len: 0x%016lx/%lu", gpa, len);

	count = 0;
	for (i = 0; i < VM_MAX_MEMMAPS; i++) {
		mm = &vm_mem(vm)->mem_maps[i];
		if (gpa >= mm->gpa && gpa < mm->gpa + mm->len) {
			count = vm_fault_quick_hold_pages(
			    &vm_vmspace(vm)->vm_map, trunc_page(gpa),
			    PAGE_SIZE, reqprot, &m, 1);
			break;
		}
	}

	if (count == 1) {
		*cookie = m;
		return ((void *)(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)) + pageoff));
	} else {
		*cookie = NULL;
		return (NULL);
	}
}

void *
vm_gpa_hold(struct vcpu *vcpu, vm_paddr_t gpa, size_t len, int reqprot,
    void **cookie)
{
#ifdef INVARIANTS
	/*
	 * The current vcpu should be frozen to ensure 'vm_memmap[]'
	 * stability.
	 */
	int state = vcpu_get_state(vcpu, NULL);
	KASSERT(state == VCPU_FROZEN, ("%s: invalid vcpu state %d",
	    __func__, state));
#endif
	return (_vm_gpa_hold(vcpu_vm(vcpu), gpa, len, reqprot, cookie));
}

void *
vm_gpa_hold_global(struct vm *vm, vm_paddr_t gpa, size_t len, int reqprot,
    void **cookie)
{
	vm_assert_memseg_locked(vm);
	return (_vm_gpa_hold(vm, gpa, len, reqprot, cookie));
}

void
vm_gpa_release(void *cookie)
{
	vm_page_t m = cookie;

	vm_page_unwire(m, PQ_ACTIVE);
}