vlibc/src/malloc/malloc.c

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <stddef.h>
#include <stdint.h>

#include <errno.h>

#include <vlibc/features.h>

#include "../internal/malloc.h"
#include "../internal/syscall.h"

/*
 * vlibc — the heap allocator (todo 7).
 *
 * A first-fit allocator with segregated size bins, a brk-grown heap, and an
 * mmap path for large blocks. Layout reference: musl's single-file malloc;
 * this is a clean-room implementation with vlibc's own chunk format.
 *
 * Chunk layout (every chunk address is 16-aligned; every chunk size is a
 * multiple of VLIBC_CHUNK_ALIGN = 32):
 *
 *   struct chunk { size_t psize, csize; }   (16-byte header)
 *   user pointer = chunk + 16
 *
 *   csize low bits (masked with ~3 when reading sizes):
 *     bit 0 VLIBC_CHUNK_INUSE   — chunk is allocated.
 *     bit 1 VLIBC_CHUNK_ALIGNED — chunk is the raw block of an
 *                                 aligned_alloc-style allocation.
 *   psize low bits (masked with ~7 when reading sizes):
 *     bit 0 — the previous chunk is in use (1) or free (0).
 *     bit 2 VLIBC_CHUNK_MMAP — this chunk lives in its own mmap, psize
 *                                 holds the mapping length.
 *
 * Free chunks keep their doubly-linked-list pointers (next/prev) in the
 * first 16 bytes of the user area, which is why the minimum chunk is
 * 16 (header) + 16 (pointers) = 32 bytes. There is never more than one
 * adjacent pair of free chunks: free() coalesces immediately, and the
 * top chunk (the free block at the end of the heap) is kept out of the
 * bins.
 *
 * Allocation: requests whose normalized chunk size reaches
 * VLIBC_MMAP_THRESHOLD (128 KiB) get a private SYS_mmap mapping (freed
 * with SYS_munmap). Everything else searches the bins first-fit (bins 0-3
 * hold the exact sizes 32/64/96/128, bins 4-12 hold log-size ranges up to
 * just below the mmap threshold), then splits the top chunk, growing the
 * heap with SYS_brk as needed. If brk fails, the request falls back to a
 * single-chunk mmap.
 *
 * The brk heap never shrinks below its start address and releases whole
 * trailing pages back to the kernel when the top chunk holds more than a
 * page beyond the minimum chunk. sbrk() is deliberately not provided (see
 * the plan: it conflicts with this allocator).
 *
 * Aligned allocations (alignment > 16): the raw block is a normal chunk
 * marked VLIBC_CHUNK_ALIGNED; the returned pointer sits somewhere inside
 * the block with a two-word descriptor directly before it:
 *
 *   [q-16] = alignment (a power of two, so bit 0 is clear)
 *   [q-8]  = raw pointer (the start of the block's user area)
 *
 * free() tells the two cases apart by the C_INUSE bit of the word at p-16:
 * an allocated chunk always has it set, while a descriptor's alignment word
 * always has it clear. realloc() on an aligned pointer preserves the
 * alignment (via the descriptor) and malloc_usable_size() reports the raw
 * block size minus the alignment offset.
 *
 * malloc(0) returns a unique minimum-size chunk. realloc(p, 0) frees p and
 * returns NULL. calloc overflows and aligned_alloc/posix_memalign argument
 * violations fail with NULL + errno (ENOMEM/EINVAL); posix_memalign returns
 * the error number instead of setting errno and never modifies *memptr on
 * failure. Over-requested brk chunks zero themselves when they come from a
 * fresh page of the heap, which makes calloc cheap in the common case.
 *
 * Single-threaded for now: the pthread todo will add locking around the
 * global allocator state.
 */

#define VLIBC_CHUNK_OVERHEAD 16 /* psize + csize */
#define VLIBC_CHUNK_ALIGN 32    /* chunk-size granularity */
#define VLIBC_CHUNK_MIN 32      /* smallest chunk: header + 2 pointers */

#define VLIBC_CHUNK_INUSE 1UL   /* csize bit 0 */
#define VLIBC_CHUNK_ALIGNED 2UL /* csize bit 1 */
#define VLIBC_CHUNK_MMAP 4UL    /* psize bit 2 (private mmap block) */

#define VLIBC_CSIZE_MASK (~(size_t)3)
#define VLIBC_PSIZE_MASK (~(size_t)7)

/* Requests whose normalized chunk size reaches this go straight to mmap. */
#define VLIBC_MMAP_THRESHOLD ((size_t)(128 * 1024))

/* 4 exact-size fast bins (32/64/96/128) + 9 log-range bins. */
#define VLIBC_BIN_COUNT 13
#define VLIBC_BIN_FAST 4

/* Largest user request that cannot overflow the chunk-size normalization. */
#define VLIBC_CHUNK_MAX_USER ((size_t)-1 - VLIBC_CHUNK_ALIGN - VLIBC_CHUNK_OVERHEAD - 1)

/* Minimum heap growth in bytes, rounded up to a page by the extender. */
#define VLIBC_HEAP_GROW ((size_t)32768)

struct vlibc_chunk
{
    size_t psize;
    size_t csize;
    struct vlibc_chunk *next;
    struct vlibc_chunk *prev;
};

static struct vlibc_chunk *vlibc_bins[VLIBC_BIN_COUNT];
static struct vlibc_chunk *vlibc_top; /* free block at the end of the heap */
static uintptr_t vlibc_heap_start;    /* 0 = heap not yet queried */
static uintptr_t vlibc_heap_end;      /* current brk */
static size_t vlibc_live;             /* allocated chunks on the heap */
static size_t vlibc_live_mmap;        /* allocated mmap chunks */

static void *
vlibc_aligned_core(size_t align, size_t size);

/* Round a user request up to a chunk size (multiple of VLIBC_CHUNK_ALIGN). */
static size_t
vlibc_norm(size_t n)
{
    size_t sz =
        (n + VLIBC_CHUNK_OVERHEAD + VLIBC_CHUNK_ALIGN - 1) & ~(size_t)(VLIBC_CHUNK_ALIGN - 1);

    return sz < VLIBC_CHUNK_MIN ? VLIBC_CHUNK_MIN : sz;
}

/* Index of the bin a free chunk of the given (flag-free) size belongs to. */
static int
vlibc_bin_of(size_t sz)
{
    if (sz <= (size_t)(VLIBC_BIN_FAST * VLIBC_CHUNK_ALIGN))
    {
        return (int)(sz / VLIBC_CHUNK_ALIGN) - 1;
    }
    return (int)(VLIBC_BIN_FAST + 63 - (unsigned)__builtin_clzll(sz / VLIBC_CHUNK_ALIGN) - 3);
}

static void
vlibc_bin_insert(struct vlibc_chunk *c)
{
    int i = vlibc_bin_of(c->csize & VLIBC_CSIZE_MASK);

    c->prev = NULL;
    c->next = vlibc_bins[i];
    if (vlibc_bins[i] != NULL)
    {
        vlibc_bins[i]->prev = c;
    }
    vlibc_bins[i] = c;
}

static void
vlibc_bin_remove(struct vlibc_chunk *c)
{
    int i = vlibc_bin_of(c->csize & VLIBC_CSIZE_MASK);

    if (c->prev != NULL)
    {
        c->prev->next = c->next;
    }
    else
    {
        vlibc_bins[i] = c->next;
    }
    if (c->next != NULL)
    {
        c->next->prev = c->prev;
    }
    c->next = NULL;
    c->prev = NULL;
}

/* Record c (size sz, flag-free) as allocated and update the next header. */
static void
vlibc_mark_alloc(struct vlibc_chunk *c, size_t sz)
{
    struct vlibc_chunk *next = (struct vlibc_chunk *)((char *)c + sz);

    c->csize = sz | VLIBC_CHUNK_INUSE;
    if ((uintptr_t)next < vlibc_heap_end && next != vlibc_top)
    {
        next->psize = c->csize;
    }
}

/*
 * Query the brk once and remember the heap window. Returns 0 on success;
 * -1 leaves the window uninitialized so callers fall back to mmap.
 */
static int
vlibc_heap_init(void)
{
    long r;

    if (vlibc_heap_start != 0)
    {
        return 0;
    }
    r = __syscall1(SYS_brk, 0);
    if (r < 0)
    {
        return -1;
    }
    vlibc_heap_start = ((uintptr_t)r + 15) & ~(uintptr_t)15;
    vlibc_heap_end = vlibc_heap_start;
    return 0;
}

/*
 * Grow the heap by at least need bytes. On success the top chunk covers the
 * new space (created if the heap was fully consumed); on failure the state
 * is unchanged and the caller falls back to a private mmap.
 */
static int
vlibc_heap_extend(size_t need)
{
    size_t want = need > VLIBC_HEAP_GROW ? need : VLIBC_HEAP_GROW;
    uintptr_t new_end;
    long r;

    want = (want + 4095) & ~(size_t)4095;
    new_end = vlibc_heap_end + want;
    r = __syscall1(SYS_brk, (long)new_end);
    if ((uintptr_t)r != new_end)
    {
        return -1;
    }
    if (vlibc_top != NULL)
    {
        vlibc_top->csize += want;
    }
    else
    {
        vlibc_top = (struct vlibc_chunk *)vlibc_heap_end;
        vlibc_top->psize = 0 | VLIBC_CHUNK_INUSE; /* prev size unknown */
        vlibc_top->csize = want;
    }
    vlibc_heap_end = new_end;
    return 0;
}

/*
 * Give whole trailing pages of the top chunk back to the kernel. Always
 * keeps at least a minimum chunk so the heap window never closes.
 */
static void
vlibc_heap_trim(void)
{
    size_t sz = vlibc_top->csize;
    size_t release = (sz - VLIBC_CHUNK_MIN) & ~(size_t)4095;
    uintptr_t new_end;
    long r;

    if (release < 4096)
    {
        return;
    }
    new_end = (uintptr_t)vlibc_top + sz - release;
    r = __syscall1(SYS_brk, (long)new_end);
    if ((uintptr_t)r == new_end)
    {
        vlibc_top->csize = sz - release;
        vlibc_heap_end = new_end;
    }
}

/* Private mmap block of normalized size nn. NULL on failure. */
static void *
vlibc_mmap_chunk(size_t nn)
{
    size_t len;
    long r;
    struct vlibc_chunk *c;

    if (nn > (size_t)-1 - VLIBC_CHUNK_OVERHEAD - 4095)
    {
        return NULL; /* the rounded mapping length would overflow */
    }
    len = (nn + VLIBC_CHUNK_OVERHEAD + 4095) & ~(size_t)4095;
    r = __syscall6(SYS_mmap, 0, (long)len, 3, 0x22, -1, 0);

    if ((uintptr_t)r > (uintptr_t)-4096)
    {
        return NULL; /* kernel returned -errno */
    }
    c = (struct vlibc_chunk *)r;
    c->psize = len | VLIBC_CHUNK_MMAP;
    c->csize = nn | VLIBC_CHUNK_INUSE;
    vlibc_live_mmap++;
    return (char *)c + VLIBC_CHUNK_OVERHEAD;
}

/* First free chunk with size >= nn anywhere in bins [bin_of(nn), end). */
static struct vlibc_chunk *
vlibc_bin_find(size_t nn)
{
    int i;

    for (i = vlibc_bin_of(nn); i < VLIBC_BIN_COUNT; i++)
    {
        struct vlibc_chunk *c;

        for (c = vlibc_bins[i]; c != NULL; c = c->next)
        {
            if ((c->csize & VLIBC_CSIZE_MASK) >= nn)
            {
                return c;
            }
        }
    }
    return NULL;
}

/*
 * Split free chunk c (current size s) into an nn-byte allocation and a free
 * remainder. Requires s >= nn + MIN. The chunk after the original extent
 * gets its psize pointed at the remainder: without that update a later free
 * there would walk back over the stale (too large) size and land on the
 * wrong chunk. When the split chunk reaches the top chunk, the remainder is
 * absorbed into the top instead of entering a bin — a free chunk directly
 * before top would violate the coalescing invariant and leave the top's
 * predecessor size stale.
 */
static void
vlibc_split(struct vlibc_chunk *c, size_t s, size_t nn)
{
    struct vlibc_chunk *rem = (struct vlibc_chunk *)((char *)c + nn);
    struct vlibc_chunk *after = (struct vlibc_chunk *)((char *)c + s);

    rem->psize = nn; /* prev still free at this instant */
    rem->csize = s - nn;
    c->csize = nn;
    if (after == vlibc_top)
    {
        rem->csize += vlibc_top->csize;
        rem->psize = nn | VLIBC_CHUNK_INUSE; /* c becomes allocated below */
        vlibc_top = rem;
    }
    else if (after == (struct vlibc_chunk *)vlibc_heap_end)
    {
        rem->psize = nn | VLIBC_CHUNK_INUSE;
        vlibc_top = rem;
    }
    else
    {
        if ((uintptr_t)after < vlibc_heap_end)
        {
            after->psize = s - nn;
        }
        vlibc_bin_insert(rem);
    }
}

/*
 * Carve nn bytes out of the top chunk and return the user pointer. When the
 * remainder is at least a minimum chunk it becomes the new top; otherwise
 * the whole top is consumed and the next extension starts a fresh one.
 */
static void *
vlibc_carve_top(size_t nn)
{
    struct vlibc_chunk *c = vlibc_top;
    size_t s = c->csize;

    if (s >= nn + VLIBC_CHUNK_MIN)
    {
        struct vlibc_chunk *rem = (struct vlibc_chunk *)((char *)c + nn);

        rem->psize = nn | VLIBC_CHUNK_INUSE;
        rem->csize = s - nn;
        c->csize = nn | VLIBC_CHUNK_INUSE;
        vlibc_top = rem;
    }
    else
    {
        c->csize = s | VLIBC_CHUNK_INUSE;
        vlibc_top = NULL;
    }
    vlibc_live++;
    return (char *)c + VLIBC_CHUNK_OVERHEAD;
}

/*
 * The allocator core (the internal seam src/internal/malloc.h documents).
 * Allocates n bytes, 16-byte aligned; NULL + errno ENOMEM on failure. May
 * return a unique pointer even when n == 0.
 */
hidden void *
__libc_malloc(size_t n) // NOLINT(bugprone-reserved-identifier)
{
    size_t nn;
    struct vlibc_chunk *c;

    if (n > VLIBC_CHUNK_MAX_USER)
    {
        errno = ENOMEM;
        return NULL;
    }
    nn = vlibc_norm(n);
    if (nn >= VLIBC_MMAP_THRESHOLD)
    {
        void *p = vlibc_mmap_chunk(nn);

        if (p == NULL)
        {
            errno = ENOMEM;
        }
        return p;
    }
    c = vlibc_bin_find(nn);
    if (c != NULL)
    {
        size_t s = c->csize & VLIBC_CSIZE_MASK;

        vlibc_bin_remove(c);
        if (s >= nn + VLIBC_CHUNK_MIN)
        {
            vlibc_split(c, s, nn);
            vlibc_mark_alloc(c, nn);
        }
        else
        {
            vlibc_mark_alloc(c, s);
        }
        vlibc_live++;
        return (char *)c + VLIBC_CHUNK_OVERHEAD;
    }
    if (vlibc_heap_init() == 0 && vlibc_heap_extend(nn) == 0)
    {
        return vlibc_carve_top(nn);
    }
    {
        void *p = vlibc_mmap_chunk(nn);

        if (p == NULL)
        {
            errno = ENOMEM;
        }
        return p;
    }
}

/*
 * Release a block returned by __libc_malloc; NULL is a no-op. Coalesces
 * with adjacent free chunks and the top chunk, then trims trailing pages.
 */
hidden void
__libc_free(void *p) // NOLINT(bugprone-reserved-identifier)
{
    struct vlibc_chunk *c;
    size_t sz;

    if (p == NULL)
    {
        return;
    }
    c = (struct vlibc_chunk *)((char *)p - VLIBC_CHUNK_OVERHEAD);
    if ((c->csize & VLIBC_CHUNK_INUSE) == 0)
    {
        /* Aligned allocation: the descriptor sits right before p. */
        void *raw = *(void **)((char *)p - sizeof(void *));

        p = raw;
        c = (struct vlibc_chunk *)((char *)raw - VLIBC_CHUNK_OVERHEAD);
    }
    if ((c->psize & VLIBC_CHUNK_MMAP) != 0)
    {
        (void)__syscall2(SYS_munmap, (long)c, (long)(c->psize & VLIBC_PSIZE_MASK));
        vlibc_live_mmap--;
        return;
    }
    vlibc_live--;
    sz = c->csize & VLIBC_CSIZE_MASK;
    c->csize = sz; /* clear INUSE and ALIGNED */

    /* Coalesce with the following chunk, top chunk first. */
    {
        struct vlibc_chunk *next = (struct vlibc_chunk *)((char *)c + sz);

        if (next == vlibc_top)
        {
            c->csize += vlibc_top->csize;
            vlibc_top = c;
        }
        else if ((uintptr_t)next < vlibc_heap_end && (next->csize & VLIBC_CHUNK_INUSE) == 0)
        {
            struct vlibc_chunk *after;

            vlibc_bin_remove(next);
            c->csize += next->csize;
            after = (struct vlibc_chunk *)((char *)c + c->csize);
            if ((uintptr_t)after < vlibc_heap_end)
            {
                after->psize = c->csize;
            }
        }
    }

    /* Coalesce with the preceding chunk when it is free. */
    if ((c->psize & 1) == 0)
    {
        struct vlibc_chunk *prev =
            (struct vlibc_chunk *)((char *)c - (c->psize & VLIBC_PSIZE_MASK));
        struct vlibc_chunk *after;

        vlibc_bin_remove(prev);
        prev->csize += c->csize;
        if (c == vlibc_top)
        {
            vlibc_top = prev;
        }
        c = prev;
        after = (struct vlibc_chunk *)((char *)c + c->csize);
        if ((uintptr_t)after < vlibc_heap_end)
        {
            after->psize = c->csize;
        }
    }

    /*
     * A free block that ends exactly at the heap end is — or joins — the
     * top chunk. This must also run when the top was fully consumed
     * (vlibc_top == NULL): a freed trailing chunk that merely entered a bin
     * would leave the allocator without a top, and a later extension would
     * then grow over the bin's space while stale top state lets trim shrink
     * the brk across live chunks.
     */
    {
        struct vlibc_chunk *end = (struct vlibc_chunk *)((char *)c + (c->csize & VLIBC_CSIZE_MASK));

        if (end == vlibc_top)
        {
            c->csize += vlibc_top->csize;
            vlibc_top = c;
        }
        else if (end == (struct vlibc_chunk *)vlibc_heap_end)
        {
            vlibc_top = c;
        }
    }

    if (c == vlibc_top)
    {
        vlibc_heap_trim();
    }
    else
    {
        vlibc_bin_insert(c);
    }
}

/*
 * The public malloc: thin forward to the internal seam so that every
 * allocation path in the library shares one implementation.
 */
void *
malloc(size_t n)
{
    return __libc_malloc(n);
}

/* The public free: thin forward to the internal seam. */
void
free(void *p)
{
    __libc_free(p);
}

/*
 * Allocate zeroed memory for nmemb elements of size bytes. The product is
 * overflow-checked; NULL + errno ENOMEM on failure. calloc(n, 0) and
 * calloc(0, n) return a unique zeroed minimum-size chunk like malloc(0).
 */
void *
calloc(size_t nmemb, size_t size)
{
    size_t total;
    void *p;
    struct vlibc_chunk *c;
    size_t usable;
    size_t i;

    if (size != 0 && nmemb > (size_t)-1 / size)
    {
        errno = ENOMEM;
        return NULL;
    }
    total = nmemb * size;
    p = malloc(total);
    if (p == NULL)
    {
        return NULL;
    }
    c = (struct vlibc_chunk *)((char *)p - VLIBC_CHUNK_OVERHEAD);
    if ((c->psize & VLIBC_CHUNK_MMAP) != 0)
    {
        return p; /* a fresh mmap is already zero */
    }
    usable = (c->csize & VLIBC_CSIZE_MASK) - VLIBC_CHUNK_OVERHEAD;
    for (i = 0; i < usable; i++)
    {
        ((unsigned char *)p)[i] = 0;
    }
    return p;
}

/*
 * Resize the block at p to n bytes. realloc(NULL, n) is malloc(n);
 * realloc(p, 0) frees p and returns NULL. Growth first tries to merge the
 * following free chunk (or the top chunk); otherwise the block is copied to
 * a fresh allocation and freed. On failure the original block is untouched
 * and errno is ENOMEM.
 */
void *
realloc(void *p, size_t n)
{
    struct vlibc_chunk *c;
    size_t sz;
    size_t nn;
    void *np;
    size_t old;
    size_t copy;
    size_t i;

    if (p == NULL)
    {
        return malloc(n);
    }
    if (n == 0)
    {
        free(p);
        return NULL;
    }
    if (n > VLIBC_CHUNK_MAX_USER)
    {
        errno = ENOMEM;
        return NULL;
    }
    nn = vlibc_norm(n);

    c = (struct vlibc_chunk *)((char *)p - VLIBC_CHUNK_OVERHEAD);
    if ((c->csize & VLIBC_CHUNK_INUSE) == 0)
    {
        /* Aligned allocation: re-allocate with the original alignment. */
        size_t align = *(size_t *)((char *)p - 2 * sizeof(size_t));
        void *raw = *(void **)((char *)p - sizeof(void *));
        struct vlibc_chunk *rc = (struct vlibc_chunk *)((char *)raw - VLIBC_CHUNK_OVERHEAD);

        old =
            (rc->csize & VLIBC_CSIZE_MASK) - VLIBC_CHUNK_OVERHEAD - ((uintptr_t)p - (uintptr_t)raw);
        if (n % align != 0)
        {
            errno = EINVAL;
            return NULL;
        }
        np = vlibc_aligned_core(align, n);
        if (np == NULL)
        {
            return NULL;
        }
        copy = old < n ? old : n;
        for (i = 0; i < copy; i++)
        {
            ((unsigned char *)np)[i] = ((unsigned char *)p)[i];
        }
        free(p);
        return np;
    }
    if ((c->psize & VLIBC_CHUNK_MMAP) != 0)
    {
        old = (c->csize & VLIBC_CSIZE_MASK) - VLIBC_CHUNK_OVERHEAD;
        np = malloc(n);
        if (np == NULL)
        {
            return NULL;
        }
        copy = old < n ? old : n;
        for (i = 0; i < copy; i++)
        {
            ((unsigned char *)np)[i] = ((unsigned char *)p)[i];
        }
        free(p);
        return np;
    }

    sz = c->csize & VLIBC_CSIZE_MASK;
    if (nn <= sz)
    {
        if (sz >= nn + VLIBC_CHUNK_MIN)
        {
            /* Shrink in place, releasing the tail as a free chunk. */
            vlibc_split(c, sz, nn);
            vlibc_mark_alloc(c, nn);
        }
        return p;
    }

    /* Try to grow into the following free chunk (or the top chunk). */
    {
        struct vlibc_chunk *next = (struct vlibc_chunk *)((char *)c + sz);

        if (next == vlibc_top)
        {
            size_t total = sz + vlibc_top->csize;

            if (total >= nn)
            {
                if (total >= nn + VLIBC_CHUNK_MIN)
                {
                    struct vlibc_chunk *rem = (struct vlibc_chunk *)((char *)c + nn);

                    rem->psize = nn | VLIBC_CHUNK_INUSE;
                    rem->csize = total - nn;
                    c->csize = nn | VLIBC_CHUNK_INUSE;
                    vlibc_top = rem;
                }
                else
                {
                    c->csize = total | VLIBC_CHUNK_INUSE;
                    vlibc_top = NULL;
                }
                return p;
            }
        }
        else if ((uintptr_t)next < vlibc_heap_end && (next->csize & VLIBC_CHUNK_INUSE) == 0)
        {
            size_t total = sz + (next->csize & VLIBC_CSIZE_MASK);

            if (total >= nn)
            {
                vlibc_bin_remove(next);
                if (total >= nn + VLIBC_CHUNK_MIN)
                {
                    struct vlibc_chunk *rem = (struct vlibc_chunk *)((char *)c + nn);
                    struct vlibc_chunk *after;

                    rem->psize = nn | VLIBC_CHUNK_INUSE;
                    rem->csize = total - nn;
                    c->csize = nn | VLIBC_CHUNK_INUSE;
                    after = (struct vlibc_chunk *)((char *)rem + rem->csize);
                    if (after == vlibc_top)
                    {
                        rem->csize += vlibc_top->csize;
                        vlibc_top = rem;
                    }
                    else if (after == (struct vlibc_chunk *)vlibc_heap_end)
                    {
                        vlibc_top = rem;
                    }
                    else
                    {
                        vlibc_bin_insert(rem);
                        if ((uintptr_t)after < vlibc_heap_end)
                        {
                            after->psize = rem->csize;
                        }
                    }
                }
                else
                {
                    struct vlibc_chunk *after = (struct vlibc_chunk *)((char *)c + total);

                    c->csize = total | VLIBC_CHUNK_INUSE;
                    if ((uintptr_t)after < vlibc_heap_end)
                    {
                        after->psize = c->csize;
                    }
                }
                return p;
            }
            /* Keep next linked; fall through to alloc-copy-free. */
        }
    }

    /* Allocate fresh, copy the smaller of the two payloads, free the old. */
    old = sz - VLIBC_CHUNK_OVERHEAD;
    np = malloc(n);
    if (np == NULL)
    {
        return NULL;
    }
    copy = old < n ? old : n;
    for (i = 0; i < copy; i++)
    {
        ((unsigned char *)np)[i] = ((unsigned char *)p)[i];
    }
    free(p);
    return np;
}

/*
 * Core of the aligned family: allocate size bytes aligned to align (a
 * power of two greater than 16). The returned pointer has a two-word
 * descriptor directly before it ([q-16] = align, [q-8] = raw pointer).
 */
static void *
vlibc_aligned_core(size_t align, size_t size)
{
    void *raw;
    void *q;
    struct vlibc_chunk *c;

    raw = malloc(size + align + VLIBC_CHUNK_ALIGN);
    if (raw == NULL)
    {
        return NULL;
    }
    q = (void *)(((uintptr_t)raw + VLIBC_CHUNK_ALIGN + align - 1) & -(uintptr_t)align);
    *(size_t *)((char *)q - 2 * sizeof(size_t)) = align;
    *(void **)((char *)q - sizeof(void *)) = raw;
    c = (struct vlibc_chunk *)((char *)raw - VLIBC_CHUNK_OVERHEAD);
    c->csize |= VLIBC_CHUNK_ALIGNED;
    return q;
}

/*
 * Allocate size bytes aligned to alignment. alignment must be a power of
 * two that is a multiple of sizeof(void *), and size must be a multiple of
 * alignment; a violation fails with NULL + errno EINVAL (a non-power-of-two
 * alignment is undefined behavior in C23, so only well-formed arguments
 * reach the allocator). size 0 returns NULL. The returned pointer is a
 * valid malloc block and is released with free.
 */
void *
aligned_alloc(size_t alignment, size_t size)
{
    if ((alignment & (alignment - 1)) != 0 || alignment % sizeof(void *) != 0 ||
        size % alignment != 0)
    {
        errno = EINVAL;
        return NULL;
    }
    if (size == 0)
    {
        return NULL;
    }
    if (alignment <= VLIBC_CHUNK_OVERHEAD)
    {
        return malloc(size);
    }
    return vlibc_aligned_core(alignment, size);
}

/*
 * Allocate size bytes at address alignment and store the result in
 * *memptr. alignment must be a power of two and a multiple of
 * sizeof(void *). Returns 0 on success, EINVAL for a bad alignment, ENOMEM
 * on allocation failure; never sets errno itself and never modifies
 * *memptr on failure. size 0 returns a unique minimum-size block.
 */
int
posix_memalign(void **memptr, size_t alignment, size_t size)
{
    void *p;

    if (memptr == NULL || (alignment & (alignment - 1)) != 0 || alignment % sizeof(void *) != 0)
    {
        return EINVAL;
    }
    p = vlibc_aligned_core(alignment, size == 0 ? 1 : size);
    if (p == NULL)
    {
        return ENOMEM;
    }
    *memptr = p;
    return 0;
}

#if VLIBC_LEVEL_GE(2)
/*
 * Return the number of bytes actually available in the block at p,
 * including any internal padding. p may be any block returned by the
 * allocator family. NULL returns 0.
 */
size_t
malloc_usable_size(void *p)
{
    struct vlibc_chunk *c;

    if (p == NULL)
    {
        return 0;
    }
    c = (struct vlibc_chunk *)((char *)p - VLIBC_CHUNK_OVERHEAD);
    if ((c->csize & VLIBC_CHUNK_INUSE) == 0)
    {
        /* Aligned allocation: report the raw block minus the offset. */
        void *raw = *(void **)((char *)p - sizeof(void *));
        struct vlibc_chunk *rc = (struct vlibc_chunk *)((char *)raw - VLIBC_CHUNK_OVERHEAD);

        if ((rc->psize & VLIBC_CHUNK_MMAP) != 0)
        {
            return (rc->psize & VLIBC_PSIZE_MASK) - VLIBC_CHUNK_OVERHEAD -
                   ((uintptr_t)p - (uintptr_t)raw);
        }
        return (rc->csize & VLIBC_CSIZE_MASK) - VLIBC_CHUNK_OVERHEAD -
               ((uintptr_t)p - (uintptr_t)raw);
    }
    if ((c->psize & VLIBC_CHUNK_MMAP) != 0)
    {
        return (c->psize & VLIBC_PSIZE_MASK) - VLIBC_CHUNK_OVERHEAD;
    }
    return (c->csize & VLIBC_CSIZE_MASK) - VLIBC_CHUNK_OVERHEAD;
}
#endif /* VLIBC_LEVEL_GE(2) */

/*
 * Internal consistency hook used by the test suite: walk the whole heap
 * chunk-by-chunk (an independent account of every live allocation), sum
 * the in-use chunks, and compare against the allocator's own counters.
 * Returns the total number of live blocks, or (size_t)-1 when the heap
 * walk disagrees with the counters (corruption or an accounting bug).
 */
hidden size_t
__vlibc_malloc_check(void) // NOLINT(bugprone-reserved-identifier)
{
    uintptr_t cur;
    size_t live = 0;

    if (vlibc_heap_start == 0)
    {
        return vlibc_live + vlibc_live_mmap == 0 ? 0 : (size_t)-1;
    }
    cur = vlibc_heap_start;
    while (cur + VLIBC_CHUNK_MIN <= vlibc_heap_end)
    {
        struct vlibc_chunk *c = (struct vlibc_chunk *)cur;
        size_t sz = c->csize & VLIBC_CSIZE_MASK;

        if (sz < VLIBC_CHUNK_MIN || sz % VLIBC_CHUNK_ALIGN != 0 || cur + sz > vlibc_heap_end)
        {
            return (size_t)-1;
        }
        if ((c->csize & VLIBC_CHUNK_INUSE) != 0)
        {
            live++;
        }
        cur += sz;
    }
    if (cur != vlibc_heap_end || live != vlibc_live)
    {
        return (size_t)-1;
    }
    return live + vlibc_live_mmap;
}