build: rework how sysroots work

/usr/include is now built on the fly, letting me merge include files from
multiple modules, which should be a win for organization later on.

binutils & gcc need to be recompiled.

limits.h shamelessly stolen from heat on #osdev, as gcc stopped providing me
with its own header. which was a hack in the first place

15 files changed, 118 insertions, 41 deletions

diff --git a/configure b/configure
index aec448d..a24548b 100755
--- a/configure
+++ b/configure
@@ -16,7 +16,7 @@ def t(out, deps='', cmds=[], phony=False, verbose=False):
 	else:
 		cmds = ["mkdir -p $(@D)"] + cmds
 	raw(f"{out}: {deps}")
-	v = '' if verbose else '@'
+	v = '' if verbose else '$(V)'
 	for cmd in cmds:
 		raw(f"\t{v}{cmd}")
 
@@ -30,20 +30,24 @@ AR = x86_64-camellia-ar
 AS = x86_64-camellia-as
 CC = x86_64-camellia-gcc
 
+V=@
+
 CFLAGS = -g -std=gnu99 -O2 -fPIC -ftrack-macro-expansion=0 \
  -Wall -Wextra -Wold-style-definition -Wno-address-of-packed-member \
  -Werror=incompatible-pointer-types -Werror=implicit-function-declaration
 
 CFLAGS_KERNEL  = $(CFLAGS) \
 	-ffreestanding -mno-sse -mgeneral-regs-only \
-	--sysroot=src/kernel/sysroot/ -Isrc/ -Isrc/libk/include/
+	--sysroot=out/sysrootk/ -Isrc/
 CFLAGS_LIBC    = $(CFLAGS) -ffreestanding -Isrc/
 
-SPARSEFLAGS = $(CFLAGS_KERNEL) -Wno-non-pointer-null
+SPARSEFLAGS = $(CFLAGS_KERNEL) -Wno-non-pointer-null -Iout/sysrootk/usr/include/
+
+LIB = out/sysrootu/lib/libc.a out/sysrootu/lib/libm.a out/sysrootu/lib/crt0.o
 """)
 
 t('all', 'out/boot.iso out/fs.e2 check', phony=True)
-t('portdeps', 'out/libc.a out/libm.a src/libc/include/__errno.h', phony=True)
+t('portdeps', 'out/sysrootu/usr/include/ $(LIB)', phony=True)
 
 # TODO check userland sources too
 t('check', '', [
@@ -73,15 +77,15 @@ t('out/fs/boot/kernel', 'src/kernel/arch/amd64/linker.ld' + srcobj('src/kernel/'
 	'grub-file --is-x86-multiboot2 $@ || rm $@; test -e $@'
 ])
 
-t('out/libc.a', srcobj('src/libc/') + srcobj('src/libk/'), [
+t('out/sysrootu/lib/libc.a', srcobj('src/libc/') + srcobj('src/libk/'), [
 	'$(AR) rcs $@ $^'
 ])
-t('out/libm.a', '', [
+t('out/sysrootu/lib/libm.a', '', [
 	'$(AR) rcs $@ $^'
 ])
 
 t('out/bootstrap', 'out/bootstrap.elf', ['objcopy -O binary $^ $@'])
-t('out/bootstrap.elf', 'src/bootstrap/linker.ld' + srcobj('src/bootstrap/') + 'out/libc.a', [
+t('out/bootstrap.elf', 'src/bootstrap/linker.ld' + srcobj('src/bootstrap/') + 'out/sysrootu/lib/libc.a', [
 	'$(CC) -nostdlib -Wl,-no-pie -T $^ -o $@'
 ])
 
@@ -95,7 +99,7 @@ userbins = glob('*', root_dir='src/cmd/')
 raw("USERBINS = " + " ".join(userbins))
 
 for cmd in userbins:
-	t(f'out/initrd/bin/amd64/{cmd}', 'out/libc.a' + srcobj(f'src/cmd/{cmd}'), [
+	t(f'out/initrd/bin/amd64/{cmd}', '$(LIB) ' + srcobj(f'src/cmd/{cmd}'), [
 		'$(CC) $^ -o $@'
 	])
 
@@ -119,18 +123,48 @@ t('src/libc/syscall.c',
 	'src/libc/syscall.c.awk src/libk/include/camellia/syscalls.h',
 	['awk -f $^ > $@'])
 
-def cc(rule, args):
-	t(f'out/obj/{rule}.o', f'src/{rule}', [f'$(CC) -c $^ -o $@ {args}'])
+def includes(target, modules, deps='', cmds=[]):
+	dirs = " ".join([f"src/{mod}/include/" for mod in modules])
+	t(target, deps, [
+		"set -eu;"
+		f"for file in $$(find {dirs} -type f,l -name '*.h'); do "
+		"    out=$@$${file#src/*/include/};"
+		"    mkdir -p $$(dirname $$out);"
+		"    ln -rfs $$file $$out;"
+		"done"
+	] + cmds, phony=True)
+
+includes("out/sysrootk/usr/include/", ["libk"])
+# this must work without a toolchain available, as it's required to build
+# the toolchain
+includes("out/sysrootu/usr/include/", ["libc", "libk"], 'src/libc/include/__errno.h', [
+	'ln -rfs src/libc/include/__errno.h $@',
+])
+t('out/sysrootu/lib/crt0.o', 'out/obj/libc/_start.s.o', ['cp $^ $@'])
+
+def cc(rule, args, deps):
+	# The | marks an order-only dependency. This means that sysroot will be
+	# built every time the rule is ran, but without forcing a rebuild if the
+	# rule is fresh.
+	# https://www.gnu.org/software/make/manual/make.html#index-order_002donly-prerequisites
+	t(f'out/obj/{rule}.o', f'src/{rule} | {deps}', [
+		f'$(CC) {args} -c $< -o $@'
+	])
+
+cc("%.c",           "$(CFLAGS)",          "out/sysrootu/usr/include/")
+cc("%.s",           "$(CFLAGS)",          "out/sysrootu/usr/include/")
+cc("%.S",           "$(CFLAGS)",          "out/sysrootu/usr/include/")
+cc("bootstrap/%.c", "$(CFLAGS) -fno-pie", "out/sysrootu/usr/include/")
+cc("libc/%.c",      "$(CFLAGS_LIBC)",     "out/sysrootu/usr/include/")
 
-cc("%.c",           "$(CFLAGS)")
-cc("%.s",           "$(CFLAGS)")
-cc("%.S",           "$(CFLAGS)")
-cc("bootstrap/%.c", "$(CFLAGS) -fno-pie")
-cc("kernel/%.c",    "$(CFLAGS_KERNEL)")
-cc("libc/%.c",      "$(CFLAGS_LIBC)")
-cc("libk/%.c",      "$(CFLAGS_KERNEL)")
+cc("libc/vendor/dlmalloc/%.c", "$(CFLAGS_LIBC) -DMAP_ANONYMOUS -DHAVE_MORECORE=0 -DNO_MALLOC_H -Wno-expansion-to-defined -Wno-old-style-definition", "out/sysrootu/usr/include/")
 
-cc("kernel/arch/amd64/32/%.c", "$(CFLAGS_KERNEL) -fno-pic -m32")
-cc("kernel/arch/amd64/32/%.s", "$(CFLAGS_KERNEL) -fno-pic -m32")
+cc("kernel/%.c",    "$(CFLAGS_KERNEL)", "out/sysrootk/usr/include/")
+cc("kernel/%.s",    "$(CFLAGS_KERNEL)", "out/sysrootk/usr/include/")
+cc("kernel/%.S",    "$(CFLAGS_KERNEL)", "out/sysrootk/usr/include/")
+cc("libk/%.c",      "$(CFLAGS_KERNEL)", "out/sysrootk/usr/include/")
+cc("libk/%.s",      "$(CFLAGS_KERNEL)", "out/sysrootk/usr/include/")
+cc("libk/%.S",      "$(CFLAGS_KERNEL)", "out/sysrootk/usr/include/")
 
-cc("libc/vendor/dlmalloc/%.c", "$(CFLAGS_LIBC) -DMAP_ANONYMOUS -DHAVE_MORECORE=0 -DNO_MALLOC_H -Wno-expansion-to-defined -Wno-old-style-definition")
+cc("kernel/arch/amd64/32/%.c", "$(CFLAGS_KERNEL) -fno-pic -m32", "out/sysrootk/usr/include/")
+cc("kernel/arch/amd64/32/%.s", "$(CFLAGS_KERNEL) -fno-pic -m32", "out/sysrootk/usr/include/")
diff --git a/contrib/dep_builders/binutils b/contrib/dep_builders/binutils
index 42dd04b..7746525 100755
--- a/contrib/dep_builders/binutils
+++ b/contrib/dep_builders/binutils
@@ -10,9 +10,10 @@ fi
 export PREFIX="$(pwd)/toolchain/prefix/"
 export PATH="$(pwd)/toolchain/prefix/bin/:$PATH"
 BUILDDIR="$(pwd)/toolchain/binutils-build/"
-SYSROOT="$(pwd)/sysroot/"
+SYSROOT="$(pwd)/out/sysrootu/"
 mkdir -p $PREFIX
 mkdir -p $BUILDDIR
+make out/sysrootu/usr/include/
 
 cd $BUILDDIR
 
@@ -23,5 +24,5 @@ cd $BUILDDIR
 	--disable-nls \
 	--disable-werror
 mkdir -p gas/doc/ # stupid workaround for the broken build
-make -j4
+make
 make install
diff --git a/contrib/dep_builders/gcc b/contrib/dep_builders/gcc
index 0a37541..defd4d5 100755
--- a/contrib/dep_builders/gcc
+++ b/contrib/dep_builders/gcc
@@ -10,9 +10,10 @@ fi
 export PREFIX="$(pwd)/toolchain/prefix/"
 export PATH="$(pwd)/toolchain/prefix/bin/:$PATH"
 BUILDDIR="$(pwd)/toolchain/gcc-build/"
-SYSROOT="$(pwd)/sysroot/"
+SYSROOT="$(pwd)/out/sysrootu/"
 mkdir -p $PREFIX
 mkdir -p $BUILDDIR
+make out/sysrootu/usr/include/
 
 cd $BUILDDIR
 
@@ -25,7 +26,7 @@ cd $BUILDDIR
 	--disable-nls \
 	--disable-gcov \
 	--disable-werror
-make -j4 all-gcc
-make -j4 all-target-libgcc
+make all-gcc
+make all-target-libgcc
 make install-gcc
 make install-target-libgcc
diff --git a/readme.md b/readme.md
index 7f1d803..7d335b3 100644
--- a/readme.md
+++ b/readme.md
@@ -2,9 +2,10 @@
 an experimental, work-in-progress, microkernel based on some of my ideas for privilege separation.
 
 ## third party code used
-* `src/user/lib/elf.h` from [adachristine](https://github.com/adachristine/sophia/tree/main/api/elf)
-* `src/user/lib/vendor/getopt` from [skeeto](https://github.com/skeeto/getopt)
-* `src/user/lib/vendor/dlmalloc` from [Doug Lea](https://gee.cs.oswego.edu/dl/html/malloc.html)
+* `src/libc/elf.h` from [adachristine](https://github.com/adachristine/sophia/tree/main/api/elf)
+* `src/libc/vendor/getopt` from [skeeto](https://github.com/skeeto/getopt)
+* `src/libc/vendor/dlmalloc` from [Doug Lea](https://gee.cs.oswego.edu/dl/html/malloc.html)
+* `src/libk/include/bits/limits.h` based on [heatd](https://github.com/heatd/)'s code
 * `src/kernel/arch/amd64/3rdparty/multiboot2.h` from the FSF
 
 ## build dependencies
diff --git a/src/libc/include/limits.h b/src/libc/include/limits.h
deleted file mode 100644
index 972553f..0000000
--- a/src/libc/include/limits.h
+++ /dev/null
@@ -1,6 +0,0 @@
-#pragma once
-#include <camellia/path.h> // just for PATH_MAX
-// #include_next <limits.h>
-
-#define _POSIX2_RE_DUP_MAX 255
-#define NAME_MAX 255
diff --git a/src/libc/include/locale.h b/src/libc/include/locale.h
index 1221375..0ea16a6 100644
--- a/src/libc/include/locale.h
+++ b/src/libc/include/locale.h
@@ -1,5 +1,6 @@
 #pragma once
 #include <limits.h>
+#include <stddef.h>
 
 #define LC_ALL 0
 #define LC_COLLATE 1
diff --git a/src/libk/include/bits/limits.h b/src/libk/include/bits/limits.h
new file mode 100644
index 0000000..a64302e
--- /dev/null
+++ b/src/libk/include/bits/limits.h
@@ -0,0 +1,45 @@
+/** Adapted from code (c) 2023 Pedro Falcato, with permission.
+ * Originally licensed under BSD-2-Clause-Patent.
+ */
+
+#if defined (__LP64__)
+#define __LIMITS_64BIT
+#endif
+
+#define CHAR_BIT 8
+
+#define SCHAR_MIN -128
+#define SCHAR_MAX 127
+#define UCHAR_MAX 255
+
+#if '\xff' < 0
+/* char is signed */
+#  define CHAR_MIN  SCHAR_MIN
+#  define CHAR_MAX  SCHAR_MAX
+#else
+/* char is unsigned */
+#  define CHAR_MIN  0
+#  define CHAR_MAX  UCHAR_MAX
+#endif
+
+#define SHRT_MIN   (-1 - 0x7fff)
+#define SHRT_MAX   0x7fff
+#define USHRT_MAX  0xffff
+
+#define INT_MIN   (-1 - 0x7fffffff)
+#define INT_MAX   0x7fffffff
+#define UINT_MAX  0xffffffffU
+
+#ifdef __LIMITS_64BIT
+#  define LONG_MAX   0x7fffffffffffffffL
+#  define LONG_MIN   (-1 - 0x7fffffffffffffffL)
+#  define ULONG_MAX  0xffffffffffffffffUL
+#else
+#  define LONG_MAX   0x7fffffffL
+#  define LONG_MIN   (-1 - 0x7fffffffL)
+#  define ULONG_MAX  0xffffffffUL
+#endif
+
+#define LLONG_MIN   (-1 - 0x7fffffffffffffffLL)
+#define LLONG_MAX   0x7fffffffffffffffLL
+#define ULLONG_MAX  0xffffffffffffffffULL
diff --git a/src/libk/include/camellia/path.h b/src/libk/include/camellia/path.h
index b268595..ee66f7b 100644
--- a/src/libk/include/camellia/path.h
+++ b/src/libk/include/camellia/path.h
@@ -1,8 +1,7 @@
 #pragma once
+#include <limits.h> /* for PATH_MAX */
 #include <stddef.h>
 
-#define PATH_MAX 512
-
 /** Reduce a path to its simplest form.
  * Kinds of invalid paths:
  *  - relative - "" "a" "./a"
diff --git a/src/libk/include/limits.h b/src/libk/include/limits.h
new file mode 100644
index 0000000..4cb972d
--- /dev/null
+++ b/src/libk/include/limits.h
@@ -0,0 +1,5 @@
+#include <bits/limits.h>
+
+#define NAME_MAX 255
+#define PATH_MAX 512
+#define _POSIX2_RE_DUP_MAX 255
diff --git a/sysroot/usr/include b/sysroot/usr/include
deleted file mode 120000
index 3f3830c..0000000
--- a/sysroot/usr/include
+++ /dev/null
@@ -1 +0,0 @@
-../../src/libc/include
\ No newline at end of file
diff --git a/sysroot/usr/lib/crt0.o b/sysroot/usr/lib/crt0.o
deleted file mode 120000
index 648a150..0000000
--- a/sysroot/usr/lib/crt0.o
+++ /dev/null
@@ -1 +0,0 @@
-../../../out/obj/libc/_start.s.o
\ No newline at end of file
diff --git a/sysroot/usr/lib/libc.a b/sysroot/usr/lib/libc.a
deleted file mode 120000
index 135b015..0000000
--- a/sysroot/usr/lib/libc.a
+++ /dev/null
@@ -1 +0,0 @@
-../../../out/libc.a
\ No newline at end of file
diff --git a/sysroot/usr/lib/libm.a b/sysroot/usr/lib/libm.a
deleted file mode 120000
index 1c566f4..0000000
--- a/sysroot/usr/lib/libm.a
+++ /dev/null
@@ -1 +0,0 @@
-../../../out/libm.a
\ No newline at end of file
diff --git a/sysroot/usr/local/include b/sysroot/usr/local/include
deleted file mode 120000
index f4fd256..0000000
--- a/sysroot/usr/local/include
+++ /dev/null
@@ -1 +0,0 @@
-../../../src/libk/include
\ No newline at end of file
diff --git a/toolchain/sysroot b/toolchain/sysroot
new file mode 120000
index 0000000..3862836
--- /dev/null
+++ b/toolchain/sysroot
@@ -0,0 +1 @@
+../out/sysrootu
\ No newline at end of file