#include #include #include "../internal/syscall.h" #include "start.h" #include "tcb.h" /* * vlibc — main-thread TLS bootstrap (todo 3). * * Makes the TCB-slot errno (and any other TCB state) work from the very * first libc call in a STATICALLY linked program. The dynamic case is * explicitly out of scope here: when the FS thread pointer is already set * (the dynamic loader #62 installed it before transferring control), this * function detects that and returns without touching anything. * * Mechanism: * - The kernel puts the auxiliary vector on the initial stack right after * envp's NULL terminator; AT_PHDR/AT_PHENT/AT_PHNUM locate the ELF * program headers, which are searched for PT_TLS. * - One anonymous private mmap holds the whole static TLS area: the TLS * image at the base, the TCB at the top. The placement formula (tp = * base + round_up(p_memsz, p_align)) is the exact arithmetic the linker * baked into every %fs-relative offset; see tcb.h for the contract. * - The image is copied from memory at p_vaddr: for a static non-PIE * executable GNU ld lays PT_TLS out inside the RW PT_LOAD, so the * template is already mapped (verified empirically). When no PT_LOAD * covers the segment — a layout this project does not produce — the * file fallback reads the segment from /proc/self/exe via pread. * - The TCB's self pointer, errno slot, and fenv defaults are written, * then arch_prctl(ARCH_SET_FS) installs TP. From that instruction on, * __builtin_thread_pointer() (which reads %fs:0) returns TP and errno * works. * * No errno is touched before arch_prctl: errno lives in the TCB, which does * not exist until this function creates it — a pre-TLS errno read would * dereference %fs:0 with the kernel's post-exec FS base of 0 and fault on * the unmapped NULL page. Failures therefore terminate hard (exit 127 via * exit_group) instead of reporting through errno. */ /* arch_prctl codes (kernel UAPI asm/prctl.h). */ #define ARCH_SET_FS 0x1002 #define ARCH_GET_FS 0x1003 /* auxv a_type values (kernel UAPI asm/auxvec.h). AT_RANDOM is deliberately * not consumed: nothing in this todo needs a per-process random seed. */ #define AT_NULL 0 #define AT_PHDR 3 #define AT_PHENT 4 #define AT_PHNUM 5 /* ELF program header p_type values (kernel UAPI linux/elf.h). */ #define PT_LOAD 1 #define PT_TLS 7 /* mmap/openat/pread constants (kernel UAPI). */ #define PROT_READ 0x1 #define PROT_WRITE 0x2 #define MAP_PRIVATE 0x2 #define MAP_ANONYMOUS 0x20 #define AT_FDCWD (-100) #define O_RDONLY 0 #define O_CLOEXEC 0x80000 /* ELF64 program header, exactly as laid out in the file/memory image. */ struct elf64_phdr { uint32_t p_type; uint32_t p_flags; uint64_t p_offset; uint64_t p_vaddr; uint64_t p_paddr; uint64_t p_filesz; uint64_t p_memsz; uint64_t p_align; }; /* * New-thread TLS geometry, consumed by src/thread/tls.c (todo 44) — see * tcb.h for the full contract. __vlibc_static_tls_size is the EXPORTED * default-visibility object (no attribute!); the other three are internal * and hidden. */ size_t __vlibc_static_tls_size = 0; hidden uintptr_t __vlibc_static_tls_template = 0; hidden size_t __vlibc_static_tls_filesz = 0; hidden size_t __vlibc_static_tls_align = 16; /* * Round v up to a multiple of a. a must be a power of two, which ELF * p_align of a PT_TLS segment always is. */ static size_t round_up(size_t v, size_t a) { return (v + a - 1) & ~(a - 1); } /* * Hard-stop for unrecoverable bootstrap failures (mmap ENOMEM, a TLS image * that is neither mapped nor readable). exit 127 — errno is unavailable * here, because the TCB that would hold it does not exist yet. */ static _Noreturn void tls_fail(void) { __syscall1(SYS_exit_group, 127); __builtin_unreachable(); } /* * The helpers below take structurally similar out-parameters (auxv and ELF * layout values), so the easily-swappable-parameters check is waived for * this section: these are kernel/ELF ABI shapes, not caller-facing APIs. */ // NOLINTBEGIN(bugprone-easily-swappable-parameters) /* * Walk the auxiliary vector on the initial stack: envp points at the first * environment string, the auxv follows envp's NULL terminator as pairs of * (type, value) longs, ending at AT_NULL. The kernel wrote these on the * stack at exec time, so the ORIGINAL envp (never the possibly-mutated * environ global) must be used. */ static void find_auxv(char **envp, uintptr_t *phdr, uintptr_t *phent, uintptr_t *phnum) { while (*envp != 0) { envp++; } envp++; /* step over the NULL terminator */ for (uintptr_t *aux = (uintptr_t *)envp; aux[0] != AT_NULL; aux += 2) { if (aux[0] == AT_PHDR) { *phdr = aux[1]; } else if (aux[0] == AT_PHENT) { *phent = aux[1]; } else if (aux[0] == AT_PHNUM) { *phnum = aux[1]; } } } /* * Search the program headers for PT_TLS and check whether the image is * already mapped (some PT_LOAD covers [p_vaddr, p_vaddr + p_filesz)). * Returns 1 when PT_TLS exists, filling the out parameters. */ static int find_tls(uintptr_t phdr, uintptr_t phent, uintptr_t phnum, uint64_t *offset, uint64_t *vaddr, size_t *filesz, size_t *memsz, size_t *align, int *mapped) { int found = 0; uint64_t lo = 0; uint64_t hi = 0; for (uintptr_t i = 0; i < phnum; i++) { const struct elf64_phdr *p = (const struct elf64_phdr *)(phdr + i * phent); if (p->p_type == PT_TLS) { *offset = p->p_offset; *vaddr = p->p_vaddr; *filesz = (size_t)p->p_filesz; *memsz = (size_t)p->p_memsz; *align = (size_t)p->p_align; lo = p->p_vaddr; hi = p->p_vaddr + p->p_filesz; found = 1; } } *mapped = 0; if (found) { for (uintptr_t i = 0; i < phnum; i++) { const struct elf64_phdr *p = (const struct elf64_phdr *)(phdr + i * phent); if (p->p_type == PT_LOAD && p->p_vaddr <= lo && hi <= p->p_vaddr + p->p_memsz) { *mapped = 1; break; } } } return found; } /* * Copy the initial TLS image (the .tdata template; .tbss is zeroed by the * caller). Primary path: memcpy from the mapped image. Fallback: pread the * segment out of /proc/self/exe — immune to chdir, works even for deleted * executables, no host libc involved. */ static void copy_tls_image(char *dst, uint64_t offset, uint64_t vaddr, size_t filesz, int mapped) { if (mapped) { memcpy(dst, (const void *)vaddr, filesz); return; } long fd = __syscall3(SYS_openat, AT_FDCWD, (long)"/proc/self/exe", O_RDONLY | O_CLOEXEC); if (fd < 0) { tls_fail(); } size_t done = 0; while (done < filesz) { long r = __syscall4(SYS_pread64, fd, (long)(dst + done), (long)(filesz - done), (long)(offset + done)); if (r <= 0) { tls_fail(); } done += (size_t)r; } __syscall1(SYS_close, fd); } // NOLINTEND(bugprone-easily-swappable-parameters) void init_main_tls(char **envp) { uintptr_t fs_base = 0; uintptr_t phdr = 0, phent = 0, phnum = 0; uint64_t offset = 0, vaddr = 0; size_t filesz = 0, memsz = 0, align = 16; int mapped = 0; size_t block, total; uintptr_t base, tp; char *image; /* * Static-case guard. arch_prctl(ARCH_GET_FS) is used instead of * __builtin_thread_pointer() because the latter dereferences %fs:0, * and with the kernel's post-exec FS base of 0 that read faults on the * unmapped NULL page. A non-zero FS base means the dynamic loader (#62) * already installed a TCB: leave it alone. */ __syscall2(SYS_arch_prctl, ARCH_GET_FS, (long)&fs_base); if (fs_base != 0) { return; } find_auxv(envp, &phdr, &phent, &phnum); if (phdr != 0 && phent != 0 && !find_tls(phdr, phent, phnum, &offset, &vaddr, &filesz, &memsz, &align, &mapped)) { /* No TLS in this program: TCB-only block, arbitrary alignment. */ filesz = 0; memsz = 0; align = 16; } /* Record the geometry for new-thread TLS blocks (#44/#45) BEFORE the * template address is known: size/filesz/align are PT_TLS facts, the * template address depends on the mmap below. */ __vlibc_static_tls_size = memsz; __vlibc_static_tls_filesz = filesz; __vlibc_static_tls_align = align; /* * Single allocation: TLS image at the base, TCB at tp. tp uses exactly * p_align (never bumped): the linker bakes tpoff against * round_up(p_memsz, p_align) and any other rounding breaks the * %fs-relative offsets (see tcb.h). */ block = round_up(memsz, align); total = block + VLIBC_TCB_SIZE; base = (uintptr_t)__syscall6(SYS_mmap, 0, (long)total, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if ((long)base < 0 && (long)base > -4096) { /* ENOMEM and friends: no errno possible yet (the TCB IS errno). */ tls_fail(); } tp = base + block; image = (char *)base; /* The PRIMAL image is the new-thread template: the still-pristine * .tdata at p_vaddr when mapped, else this block (the file-fallback * path leaves no other copy). Never the main thread's live TLS — its * mutations must not leak into threads created later. */ __vlibc_static_tls_template = mapped ? vaddr : base; if (filesz > 0) { copy_tls_image(image, offset, vaddr, filesz, mapped); } if (memsz > filesz) { memset(image + filesz, 0, memsz - filesz); } /* Build the TCB at TP (authoritative layout in tcb.h). */ ((uintptr_t *)tp)[VLIBC_TCB_SELF_OFF / sizeof(uintptr_t)] = tp; *(int *)(tp + VLIBC_TCB_ERRNO_OFF) = 0; *(uint16_t *)(tp + VLIBC_TCB_FENV_X87CW_OFF) = 0x037F; /* x87 reset default */ *(uint32_t *)(tp + VLIBC_TCB_FENV_MXCSR_OFF) = 0x1F80; /* MXCSR reset default */ /* From here on the TCB (and errno) is live. Install the thread pointer. */ __syscall2(SYS_arch_prctl, ARCH_SET_FS, (long)tp); }