123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510 |
- /*-
- * SPDX-License-Identifier: BSD-2-Clause
- *
- * Copyright (c) 2005, Bosko Milekic <bmilekic@FreeBSD.org>.
- * Copyright (c) 2010 Isilon Systems, Inc. (http://www.isilon.com/)
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice unmodified, this list of conditions, and the following
- * disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
- * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
- * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
- * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
- * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
- * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
- * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
- #include <sys/cdefs.h>
- /*
- * MemGuard is a simple replacement allocator for debugging only
- * which provides ElectricFence-style memory barrier protection on
- * objects being allocated, and is used to detect tampering-after-free
- * scenarios.
- *
- * See the memguard(9) man page for more information on using MemGuard.
- */
- #include "opt_vm.h"
- #include <sys/param.h>
- #include <sys/systm.h>
- #include <sys/kernel.h>
- #include <sys/types.h>
- #include <sys/queue.h>
- #include <sys/lock.h>
- #include <sys/mutex.h>
- #include <sys/malloc.h>
- #include <sys/sysctl.h>
- #include <sys/vmem.h>
- #include <sys/vmmeter.h>
- #include <vm/vm.h>
- #include <vm/uma.h>
- #include <vm/vm_param.h>
- #include <vm/vm_page.h>
- #include <vm/vm_map.h>
- #include <vm/vm_object.h>
- #include <vm/vm_kern.h>
- #include <vm/vm_extern.h>
- #include <vm/uma_int.h>
- #include <vm/memguard.h>
- static SYSCTL_NODE(_vm, OID_AUTO, memguard, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL,
- "MemGuard data");
- /*
- * The vm_memguard_divisor variable controls how much of kernel_arena should be
- * reserved for MemGuard.
- */
- static u_int vm_memguard_divisor;
- SYSCTL_UINT(_vm_memguard, OID_AUTO, divisor, CTLFLAG_RDTUN | CTLFLAG_NOFETCH,
- &vm_memguard_divisor,
- 0, "(kmem_size/memguard_divisor) == memguard submap size");
- /*
- * Short description (ks_shortdesc) of memory type to monitor.
- */
- static char vm_memguard_desc[128] = "";
- static struct malloc_type *vm_memguard_mtype = NULL;
- TUNABLE_STR("vm.memguard.desc", vm_memguard_desc, sizeof(vm_memguard_desc));
- static int
- memguard_sysctl_desc(SYSCTL_HANDLER_ARGS)
- {
- char desc[sizeof(vm_memguard_desc)];
- int error;
- strlcpy(desc, vm_memguard_desc, sizeof(desc));
- error = sysctl_handle_string(oidp, desc, sizeof(desc), req);
- if (error != 0 || req->newptr == NULL)
- return (error);
- mtx_lock(&malloc_mtx);
- /* If mtp is NULL, it will be initialized in memguard_cmp() */
- vm_memguard_mtype = malloc_desc2type(desc);
- strlcpy(vm_memguard_desc, desc, sizeof(vm_memguard_desc));
- mtx_unlock(&malloc_mtx);
- return (error);
- }
- SYSCTL_PROC(_vm_memguard, OID_AUTO, desc,
- CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 0,
- memguard_sysctl_desc, "A", "Short description of memory type to monitor");
- static int
- memguard_sysctl_mapused(SYSCTL_HANDLER_ARGS)
- {
- vmem_size_t size;
- size = vmem_size(memguard_arena, VMEM_ALLOC);
- return (sysctl_handle_long(oidp, &size, sizeof(size), req));
- }
- static vm_offset_t memguard_base;
- static vm_size_t memguard_mapsize;
- static vm_size_t memguard_physlimit;
- static u_long memguard_wasted;
- static u_long memguard_succ;
- static u_long memguard_fail_kva;
- static u_long memguard_fail_pgs;
- SYSCTL_ULONG(_vm_memguard, OID_AUTO, mapsize, CTLFLAG_RD,
- &memguard_mapsize, 0, "MemGuard private arena size");
- SYSCTL_ULONG(_vm_memguard, OID_AUTO, phys_limit, CTLFLAG_RD,
- &memguard_physlimit, 0, "Limit on MemGuard memory consumption");
- SYSCTL_ULONG(_vm_memguard, OID_AUTO, wasted, CTLFLAG_RD,
- &memguard_wasted, 0, "Excess memory used through page promotion");
- SYSCTL_ULONG(_vm_memguard, OID_AUTO, numalloc, CTLFLAG_RD,
- &memguard_succ, 0, "Count of successful MemGuard allocations");
- SYSCTL_ULONG(_vm_memguard, OID_AUTO, fail_kva, CTLFLAG_RD,
- &memguard_fail_kva, 0, "MemGuard failures due to lack of KVA");
- SYSCTL_ULONG(_vm_memguard, OID_AUTO, fail_pgs, CTLFLAG_RD,
- &memguard_fail_pgs, 0, "MemGuard failures due to lack of pages");
- #define MG_GUARD_AROUND 0x001
- #define MG_GUARD_ALLLARGE 0x002
- #define MG_GUARD_NOFREE 0x004
- static int memguard_options = MG_GUARD_AROUND;
- SYSCTL_INT(_vm_memguard, OID_AUTO, options, CTLFLAG_RWTUN,
- &memguard_options, 0,
- "MemGuard options:\n"
- "\t0x001 - add guard pages around each allocation\n"
- "\t0x002 - always use MemGuard for allocations over a page\n"
- "\t0x004 - guard uma(9) zones with UMA_ZONE_NOFREE flag");
- static u_int memguard_minsize;
- static u_long memguard_minsize_reject;
- SYSCTL_UINT(_vm_memguard, OID_AUTO, minsize, CTLFLAG_RW,
- &memguard_minsize, 0, "Minimum size for page promotion");
- SYSCTL_ULONG(_vm_memguard, OID_AUTO, minsize_reject, CTLFLAG_RD,
- &memguard_minsize_reject, 0, "# times rejected for size");
- static u_int memguard_frequency;
- static u_long memguard_frequency_hits;
- SYSCTL_UINT(_vm_memguard, OID_AUTO, frequency, CTLFLAG_RWTUN,
- &memguard_frequency, 0, "Times in 100000 that MemGuard will randomly run");
- SYSCTL_ULONG(_vm_memguard, OID_AUTO, frequency_hits, CTLFLAG_RD,
- &memguard_frequency_hits, 0, "# times MemGuard randomly chose");
- /*
- * Return a fudged value to be used for vm_kmem_size for allocating
- * the kernel_arena.
- */
- unsigned long
- memguard_fudge(unsigned long km_size, const struct vm_map *parent_map)
- {
- u_long mem_pgs, parent_size;
- vm_memguard_divisor = 10;
- /* CTFLAG_RDTUN doesn't work during the early boot process. */
- TUNABLE_INT_FETCH("vm.memguard.divisor", &vm_memguard_divisor);
- parent_size = vm_map_max(parent_map) - vm_map_min(parent_map) +
- PAGE_SIZE;
- /* Pick a conservative value if provided value sucks. */
- if ((vm_memguard_divisor <= 0) ||
- ((parent_size / vm_memguard_divisor) == 0))
- vm_memguard_divisor = 10;
- /*
- * Limit consumption of physical pages to
- * 1/vm_memguard_divisor of system memory. If the KVA is
- * smaller than this then the KVA limit comes into play first.
- * This prevents memguard's page promotions from completely
- * using up memory, since most malloc(9) calls are sub-page.
- */
- mem_pgs = vm_cnt.v_page_count;
- memguard_physlimit = (mem_pgs / vm_memguard_divisor) * PAGE_SIZE;
- /*
- * We want as much KVA as we can take safely. Use at most our
- * allotted fraction of the parent map's size. Limit this to
- * twice the physical memory to avoid using too much memory as
- * pagetable pages (size must be multiple of PAGE_SIZE).
- */
- memguard_mapsize = round_page(parent_size / vm_memguard_divisor);
- if (memguard_mapsize / (2 * PAGE_SIZE) > mem_pgs)
- memguard_mapsize = mem_pgs * 2 * PAGE_SIZE;
- if (km_size + memguard_mapsize > parent_size)
- memguard_mapsize = 0;
- return (km_size + memguard_mapsize);
- }
- /*
- * Initialize the MemGuard mock allocator. All objects from MemGuard come
- * out of a single contiguous chunk of kernel address space that is managed
- * by a vmem arena.
- */
- void
- memguard_init(vmem_t *parent)
- {
- vm_offset_t base;
- vmem_alloc(parent, memguard_mapsize, M_BESTFIT | M_WAITOK, &base);
- vmem_init(memguard_arena, "memguard arena", base, memguard_mapsize,
- PAGE_SIZE, 0, M_WAITOK);
- memguard_base = base;
- printf("MEMGUARD DEBUGGING ALLOCATOR INITIALIZED:\n");
- printf("\tMEMGUARD map base: 0x%lx\n", (u_long)base);
- printf("\tMEMGUARD map size: %jd KBytes\n",
- (uintmax_t)memguard_mapsize >> 10);
- }
- /*
- * Run things that can't be done as early as memguard_init().
- */
- static void
- memguard_sysinit(void)
- {
- struct sysctl_oid_list *parent;
- parent = SYSCTL_STATIC_CHILDREN(_vm_memguard);
- SYSCTL_ADD_UAUTO(NULL, parent, OID_AUTO, "mapstart",
- CTLFLAG_RD, &memguard_base,
- "MemGuard KVA base");
- SYSCTL_ADD_UAUTO(NULL, parent, OID_AUTO, "maplimit",
- CTLFLAG_RD, &memguard_mapsize,
- "MemGuard KVA size");
- SYSCTL_ADD_PROC(NULL, parent, OID_AUTO, "mapused",
- CTLFLAG_RD | CTLFLAG_MPSAFE | CTLTYPE_ULONG, NULL, 0, memguard_sysctl_mapused, "LU",
- "MemGuard KVA used");
- }
- SYSINIT(memguard, SI_SUB_KLD, SI_ORDER_ANY, memguard_sysinit, NULL);
- /*
- * v2sizep() converts a virtual address of the first page allocated for
- * an item to a pointer to u_long recording the size of the original
- * allocation request.
- *
- * This routine is very similar to those defined by UMA in uma_int.h.
- * The difference is that this routine stores the originally allocated
- * size in one of the page's fields that is unused when the page is
- * wired rather than the object field, which is used.
- */
- static u_long *
- v2sizep(vm_offset_t va)
- {
- vm_paddr_t pa;
- struct vm_page *p;
- pa = pmap_kextract(va);
- if (pa == 0)
- panic("MemGuard detected double-free of %p", (void *)va);
- p = PHYS_TO_VM_PAGE(pa);
- KASSERT(vm_page_wired(p) && p->a.queue == PQ_NONE,
- ("MEMGUARD: Expected wired page %p in vtomgfifo!", p));
- return (&p->plinks.memguard.p);
- }
- static u_long *
- v2sizev(vm_offset_t va)
- {
- vm_paddr_t pa;
- struct vm_page *p;
- pa = pmap_kextract(va);
- if (pa == 0)
- panic("MemGuard detected double-free of %p", (void *)va);
- p = PHYS_TO_VM_PAGE(pa);
- KASSERT(vm_page_wired(p) && p->a.queue == PQ_NONE,
- ("MEMGUARD: Expected wired page %p in vtomgfifo!", p));
- return (&p->plinks.memguard.v);
- }
- /*
- * Allocate a single object of specified size with specified flags
- * (either M_WAITOK or M_NOWAIT).
- */
- void *
- memguard_alloc(unsigned long req_size, int flags)
- {
- vm_offset_t addr, origaddr;
- u_long size_p, size_v;
- int do_guard, error, rv;
- size_p = round_page(req_size);
- if (size_p == 0)
- return (NULL);
- /*
- * To ensure there are holes on both sides of the allocation,
- * request 2 extra pages of KVA. Save the value of memguard_options
- * so that we use a consistent value throughout this function.
- */
- size_v = size_p;
- do_guard = (memguard_options & MG_GUARD_AROUND) != 0;
- if (do_guard)
- size_v += 2 * PAGE_SIZE;
- /*
- * When we pass our memory limit, reject sub-page allocations.
- * Page-size and larger allocations will use the same amount
- * of physical memory whether we allocate or hand off to
- * malloc_large(), so keep those.
- */
- if (vmem_size(memguard_arena, VMEM_ALLOC) >= memguard_physlimit &&
- req_size < PAGE_SIZE) {
- addr = (vm_offset_t)NULL;
- memguard_fail_pgs++;
- goto out;
- }
- /*
- * Attempt to avoid address reuse for as long as possible, to increase
- * the likelihood of catching a use-after-free.
- */
- error = vmem_alloc(memguard_arena, size_v, M_NEXTFIT | M_NOWAIT,
- &origaddr);
- if (error != 0) {
- memguard_fail_kva++;
- addr = (vm_offset_t)NULL;
- goto out;
- }
- addr = origaddr;
- if (do_guard)
- addr += PAGE_SIZE;
- rv = kmem_back(kernel_object, addr, size_p, flags);
- if (rv != KERN_SUCCESS) {
- vmem_xfree(memguard_arena, origaddr, size_v);
- memguard_fail_pgs++;
- addr = (vm_offset_t)NULL;
- goto out;
- }
- *v2sizep(trunc_page(addr)) = req_size;
- *v2sizev(trunc_page(addr)) = size_v;
- memguard_succ++;
- if (req_size < PAGE_SIZE) {
- memguard_wasted += (PAGE_SIZE - req_size);
- if (do_guard) {
- /*
- * Align the request to 16 bytes, and return
- * an address near the end of the page, to
- * better detect array overrun.
- */
- req_size = roundup2(req_size, 16);
- addr += (PAGE_SIZE - req_size);
- }
- }
- out:
- return ((void *)addr);
- }
- int
- is_memguard_addr(void *addr)
- {
- vm_offset_t a = (vm_offset_t)(uintptr_t)addr;
- return (a >= memguard_base && a < memguard_base + memguard_mapsize);
- }
- /*
- * Free specified single object.
- */
- void
- memguard_free(void *ptr)
- {
- vm_offset_t addr;
- u_long req_size, size, sizev;
- char *temp;
- int i;
- addr = trunc_page((uintptr_t)ptr);
- req_size = *v2sizep(addr);
- sizev = *v2sizev(addr);
- size = round_page(req_size);
- /*
- * Page should not be guarded right now, so force a write.
- * The purpose of this is to increase the likelihood of
- * catching a double-free, but not necessarily a
- * tamper-after-free (the second thread freeing might not
- * write before freeing, so this forces it to and,
- * subsequently, trigger a fault).
- */
- temp = ptr;
- for (i = 0; i < size; i += PAGE_SIZE)
- temp[i] = 'M';
- /*
- * This requires carnal knowledge of the implementation of
- * kmem_free(), but since we've already replaced kmem_malloc()
- * above, it's not really any worse. We want to use the
- * vm_map lock to serialize updates to memguard_wasted, since
- * we had the lock at increment.
- */
- kmem_unback(kernel_object, addr, size);
- if (sizev > size)
- addr -= PAGE_SIZE;
- vmem_xfree(memguard_arena, addr, sizev);
- if (req_size < PAGE_SIZE)
- memguard_wasted -= (PAGE_SIZE - req_size);
- }
- /*
- * Re-allocate an allocation that was originally guarded.
- */
- void *
- memguard_realloc(void *addr, unsigned long size, struct malloc_type *mtp,
- int flags)
- {
- void *newaddr;
- u_long old_size;
- /*
- * Allocate the new block. Force the allocation to be guarded
- * as the original may have been guarded through random
- * chance, and that should be preserved.
- */
- if ((newaddr = memguard_alloc(size, flags)) == NULL)
- return (NULL);
- /* Copy over original contents. */
- old_size = *v2sizep(trunc_page((uintptr_t)addr));
- bcopy(addr, newaddr, min(size, old_size));
- memguard_free(addr);
- return (newaddr);
- }
- static int
- memguard_cmp(unsigned long size)
- {
- if (size < memguard_minsize) {
- memguard_minsize_reject++;
- return (0);
- }
- if ((memguard_options & MG_GUARD_ALLLARGE) != 0 && size >= PAGE_SIZE)
- return (1);
- if (memguard_frequency > 0 &&
- (random() % 100000) < memguard_frequency) {
- memguard_frequency_hits++;
- return (1);
- }
- return (0);
- }
- int
- memguard_cmp_mtp(struct malloc_type *mtp, unsigned long size)
- {
- if (memguard_cmp(size))
- return(1);
- #if 1
- /*
- * The safest way of comparison is to always compare short description
- * string of memory type, but it is also the slowest way.
- */
- return (strcmp(mtp->ks_shortdesc, vm_memguard_desc) == 0);
- #else
- /*
- * If we compare pointers, there are two possible problems:
- * 1. Memory type was unloaded and new memory type was allocated at the
- * same address.
- * 2. Memory type was unloaded and loaded again, but allocated at a
- * different address.
- */
- if (vm_memguard_mtype != NULL)
- return (mtp == vm_memguard_mtype);
- if (strcmp(mtp->ks_shortdesc, vm_memguard_desc) == 0) {
- vm_memguard_mtype = mtp;
- return (1);
- }
- return (0);
- #endif
- }
- int
- memguard_cmp_zone(uma_zone_t zone)
- {
- if ((memguard_options & MG_GUARD_NOFREE) == 0 &&
- zone->uz_flags & UMA_ZONE_NOFREE)
- return (0);
- if (memguard_cmp(zone->uz_size))
- return (1);
- /*
- * The safest way of comparison is to always compare zone name,
- * but it is also the slowest way.
- */
- return (strcmp(zone->uz_name, vm_memguard_desc) == 0);
- }
- unsigned long
- memguard_get_req_size(const void *addr)
- {
- return (*v2sizep(trunc_page((uintptr_t)addr)));
- }
|