vlibc/src/start/tls.c

#include <stdint.h>
#include <string.h>

#include "../internal/syscall.h"
#include "start.h"
#include "tcb.h"

/*
 * vlibc — main-thread TLS bootstrap (todo 3).
 *
 * Makes the TCB-slot errno (and any other TCB state) work from the very
 * first libc call in a STATICALLY linked program. The dynamic case is
 * explicitly out of scope here: when the FS thread pointer is already set
 * (the dynamic loader #62 installed it before transferring control), this
 * function detects that and returns without touching anything.
 *
 * Mechanism:
 *   - The kernel puts the auxiliary vector on the initial stack right after
 *     envp's NULL terminator; AT_PHDR/AT_PHENT/AT_PHNUM locate the ELF
 *     program headers, which are searched for PT_TLS.
 *   - One anonymous private mmap holds the whole static TLS area: the TLS
 *     image at the base, the TCB at the top. The placement formula (tp =
 *     base + round_up(p_memsz, p_align)) is the exact arithmetic the linker
 *     baked into every %fs-relative offset; see tcb.h for the contract.
 *   - The image is copied from memory at p_vaddr: for a static non-PIE
 *     executable GNU ld lays PT_TLS out inside the RW PT_LOAD, so the
 *     template is already mapped (verified empirically). When no PT_LOAD
 *     covers the segment — a layout this project does not produce — the
 *     file fallback reads the segment from /proc/self/exe via pread.
 *   - The TCB's self pointer, errno slot, and fenv defaults are written,
 *     then arch_prctl(ARCH_SET_FS) installs TP. From that instruction on,
 *     __builtin_thread_pointer() (which reads %fs:0) returns TP and errno
 *     works.
 *
 * No errno is touched before arch_prctl: errno lives in the TCB, which does
 * not exist until this function creates it — a pre-TLS errno read would
 * dereference %fs:0 with the kernel's post-exec FS base of 0 and fault on
 * the unmapped NULL page. Failures therefore terminate hard (exit 127 via
 * exit_group) instead of reporting through errno.
 */

/* arch_prctl codes (kernel UAPI asm/prctl.h). */
#define ARCH_SET_FS 0x1002
#define ARCH_GET_FS 0x1003

/* auxv a_type values (kernel UAPI asm/auxvec.h). AT_RANDOM is deliberately
 * not consumed: nothing in this todo needs a per-process random seed. */
#define AT_NULL 0
#define AT_PHDR 3
#define AT_PHENT 4
#define AT_PHNUM 5

/* ELF program header p_type values (kernel UAPI linux/elf.h). */
#define PT_LOAD 1
#define PT_TLS 7

/* mmap/openat/pread constants (kernel UAPI). */
#define PROT_READ 0x1
#define PROT_WRITE 0x2
#define MAP_PRIVATE 0x2
#define MAP_ANONYMOUS 0x20
#define AT_FDCWD (-100)
#define O_RDONLY 0
#define O_CLOEXEC 0x80000

/* ELF64 program header, exactly as laid out in the file/memory image. */
struct elf64_phdr
{
    uint32_t p_type;
    uint32_t p_flags;
    uint64_t p_offset;
    uint64_t p_vaddr;
    uint64_t p_paddr;
    uint64_t p_filesz;
    uint64_t p_memsz;
    uint64_t p_align;
};

/*
 * New-thread TLS geometry, consumed by src/thread/tls.c (todo 44) — see
 * tcb.h for the full contract. __vlibc_static_tls_size is the EXPORTED
 * default-visibility object (no attribute!); the other three are internal
 * and hidden.
 */
size_t __vlibc_static_tls_size = 0;
hidden uintptr_t __vlibc_static_tls_template = 0;
hidden size_t __vlibc_static_tls_filesz = 0;
hidden size_t __vlibc_static_tls_align = 16;

/*
 * Round v up to a multiple of a. a must be a power of two, which ELF
 * p_align of a PT_TLS segment always is.
 */
static size_t
round_up(size_t v, size_t a)
{
    return (v + a - 1) & ~(a - 1);
}

/*
 * Hard-stop for unrecoverable bootstrap failures (mmap ENOMEM, a TLS image
 * that is neither mapped nor readable). exit 127 — errno is unavailable
 * here, because the TCB that would hold it does not exist yet.
 */
static _Noreturn void
tls_fail(void)
{
    __syscall1(SYS_exit_group, 127);
    __builtin_unreachable();
}

/*
 * The helpers below take structurally similar out-parameters (auxv and ELF
 * layout values), so the easily-swappable-parameters check is waived for
 * this section: these are kernel/ELF ABI shapes, not caller-facing APIs.
 */
// NOLINTBEGIN(bugprone-easily-swappable-parameters)

/*
 * Walk the auxiliary vector on the initial stack: envp points at the first
 * environment string, the auxv follows envp's NULL terminator as pairs of
 * (type, value) longs, ending at AT_NULL. The kernel wrote these on the
 * stack at exec time, so the ORIGINAL envp (never the possibly-mutated
 * environ global) must be used.
 */
static void
find_auxv(char **envp, uintptr_t *phdr, uintptr_t *phent, uintptr_t *phnum)
{
    while (*envp != 0)
    {
        envp++;
    }
    envp++; /* step over the NULL terminator */
    for (uintptr_t *aux = (uintptr_t *)envp; aux[0] != AT_NULL; aux += 2)
    {
        if (aux[0] == AT_PHDR)
        {
            *phdr = aux[1];
        }
        else if (aux[0] == AT_PHENT)
        {
            *phent = aux[1];
        }
        else if (aux[0] == AT_PHNUM)
        {
            *phnum = aux[1];
        }
    }
}

/*
 * Search the program headers for PT_TLS and check whether the image is
 * already mapped (some PT_LOAD covers [p_vaddr, p_vaddr + p_filesz)).
 * Returns 1 when PT_TLS exists, filling the out parameters.
 */
static int
find_tls(uintptr_t phdr, uintptr_t phent, uintptr_t phnum, uint64_t *offset, uint64_t *vaddr,
         size_t *filesz, size_t *memsz, size_t *align, int *mapped)
{
    int found = 0;
    uint64_t lo = 0;
    uint64_t hi = 0;

    for (uintptr_t i = 0; i < phnum; i++)
    {
        const struct elf64_phdr *p = (const struct elf64_phdr *)(phdr + i * phent);
        if (p->p_type == PT_TLS)
        {
            *offset = p->p_offset;
            *vaddr = p->p_vaddr;
            *filesz = (size_t)p->p_filesz;
            *memsz = (size_t)p->p_memsz;
            *align = (size_t)p->p_align;
            lo = p->p_vaddr;
            hi = p->p_vaddr + p->p_filesz;
            found = 1;
        }
    }

    *mapped = 0;
    if (found)
    {
        for (uintptr_t i = 0; i < phnum; i++)
        {
            const struct elf64_phdr *p = (const struct elf64_phdr *)(phdr + i * phent);
            if (p->p_type == PT_LOAD && p->p_vaddr <= lo && hi <= p->p_vaddr + p->p_memsz)
            {
                *mapped = 1;
                break;
            }
        }
    }
    return found;
}

/*
 * Copy the initial TLS image (the .tdata template; .tbss is zeroed by the
 * caller). Primary path: memcpy from the mapped image. Fallback: pread the
 * segment out of /proc/self/exe — immune to chdir, works even for deleted
 * executables, no host libc involved.
 */
static void
copy_tls_image(char *dst, uint64_t offset, uint64_t vaddr, size_t filesz, int mapped)
{
    if (mapped)
    {
        memcpy(dst, (const void *)vaddr, filesz);
        return;
    }

    long fd = __syscall3(SYS_openat, AT_FDCWD, (long)"/proc/self/exe", O_RDONLY | O_CLOEXEC);
    if (fd < 0)
    {
        tls_fail();
    }
    size_t done = 0;
    while (done < filesz)
    {
        long r = __syscall4(SYS_pread64, fd, (long)(dst + done), (long)(filesz - done),
                            (long)(offset + done));
        if (r <= 0)
        {
            tls_fail();
        }
        done += (size_t)r;
    }
    __syscall1(SYS_close, fd);
}

// NOLINTEND(bugprone-easily-swappable-parameters)

void
init_main_tls(char **envp)
{
    uintptr_t fs_base = 0;
    uintptr_t phdr = 0, phent = 0, phnum = 0;
    uint64_t offset = 0, vaddr = 0;
    size_t filesz = 0, memsz = 0, align = 16;
    int mapped = 0;
    size_t block, total;
    uintptr_t base, tp;
    char *image;

    /*
     * Static-case guard. arch_prctl(ARCH_GET_FS) is used instead of
     * __builtin_thread_pointer() because the latter dereferences %fs:0,
     * and with the kernel's post-exec FS base of 0 that read faults on the
     * unmapped NULL page. A non-zero FS base means the dynamic loader (#62)
     * already installed a TCB: leave it alone.
     */
    __syscall2(SYS_arch_prctl, ARCH_GET_FS, (long)&fs_base);
    if (fs_base != 0)
    {
        return;
    }

    find_auxv(envp, &phdr, &phent, &phnum);
    if (phdr != 0 && phent != 0 &&
        !find_tls(phdr, phent, phnum, &offset, &vaddr, &filesz, &memsz, &align, &mapped))
    {
        /* No TLS in this program: TCB-only block, arbitrary alignment. */
        filesz = 0;
        memsz = 0;
        align = 16;
    }

    /* Record the geometry for new-thread TLS blocks (#44/#45) BEFORE the
     * template address is known: size/filesz/align are PT_TLS facts, the
     * template address depends on the mmap below. */
    __vlibc_static_tls_size = memsz;
    __vlibc_static_tls_filesz = filesz;
    __vlibc_static_tls_align = align;

    /*
     * Single allocation: TLS image at the base, TCB at tp. tp uses exactly
     * p_align (never bumped): the linker bakes tpoff against
     * round_up(p_memsz, p_align) and any other rounding breaks the
     * %fs-relative offsets (see tcb.h).
     */
    block = round_up(memsz, align);
    total = block + VLIBC_TCB_SIZE;

    base = (uintptr_t)__syscall6(SYS_mmap, 0, (long)total, PROT_READ | PROT_WRITE,
                                 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
    if ((long)base < 0 && (long)base > -4096)
    {
        /* ENOMEM and friends: no errno possible yet (the TCB IS errno). */
        tls_fail();
    }

    tp = base + block;
    image = (char *)base;

    /* The PRIMAL image is the new-thread template: the still-pristine
     * .tdata at p_vaddr when mapped, else this block (the file-fallback
     * path leaves no other copy). Never the main thread's live TLS — its
     * mutations must not leak into threads created later. */
    __vlibc_static_tls_template = mapped ? vaddr : base;

    if (filesz > 0)
    {
        copy_tls_image(image, offset, vaddr, filesz, mapped);
    }
    if (memsz > filesz)
    {
        memset(image + filesz, 0, memsz - filesz);
    }

    /* Build the TCB at TP (authoritative layout in tcb.h). */
    ((uintptr_t *)tp)[VLIBC_TCB_SELF_OFF / sizeof(uintptr_t)] = tp;
    *(int *)(tp + VLIBC_TCB_ERRNO_OFF) = 0;
    *(uint16_t *)(tp + VLIBC_TCB_FENV_X87CW_OFF) = 0x037F; /* x87 reset default */
    *(uint32_t *)(tp + VLIBC_TCB_FENV_MXCSR_OFF) = 0x1F80; /* MXCSR reset default */

    /* From here on the TCB (and errno) is live. Install the thread pointer. */
    __syscall2(SYS_arch_prctl, ARCH_SET_FS, (long)tp);
}