clfs

clfs.git
git clone git://git.lenczewski.org/clfs.git
Log | Files | Refs | README
commit c03da04a67715d0bbee7fc0d5fde37f5d853dd73
Author: MikoĊ‚aj Lenczewski <mblenczewski@gmail.com>
Date:   Sun,  7 Apr 2024 21:46:43 +0000

Initial commit

Diffstat:

A.editorconfig | 17+++++++++++++++++


A.gitignore | 5+++++


ATODO.md | 6++++++


Aclean.sh | 7+++++++


Aclfs.sh | 47+++++++++++++++++++++++++++++++++++++++++++++++


Aconfig.sh | 56++++++++++++++++++++++++++++++++++++++++++++++++++++++++


Aenv-clfs.sh | 28++++++++++++++++++++++++++++


Astage1.sh | 24++++++++++++++++++++++++


Astage2.sh | 206+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++


Astage3.sh | 131+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++


10 files changed, 527 insertions(+), 0 deletions(-)

diff --git a/.editorconfig b/.editorconfig
@@ -0,0 +1,17 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+charset = utf-8
+
+guidelines = 80, 120, 160
+
+[*.{c,h,sh}]
+indent_style = tab
+indent_size = 8
+
+[*.{md,txt}]
+indent_style = space
+indent_size = 2
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,5 @@
+sources/
+crosstools/
+clfs/
+
+**/.*.swp
diff --git a/TODO.md b/TODO.md
@@ -0,0 +1,6 @@
+https://gist.github.com/KireinaHoro/282f6c1fef8b155126aaeb0acccf4280
+https://github.com/dslm4515/CMLFS/blob/master/2-Stage1/01-Stage1.LLVM
+https://llvm.org/docs/CMake.html#llvm-related-variables
+https://mcilloni.ovh/2021/02/09/cxx-cross-clang/
+https://clfs.org/view/clfs-embedded/arm/cross-tools/toolchain.html
+https://gist.github.com/mode89/a7a8be6319a3f7b7a68c197801fe13fd
diff --git a/clean.sh b/clean.sh
@@ -0,0 +1,7 @@
+#!/bin/sh
+
+set -ex
+
+rm -rf clfs/
+
+[ "${CLEANALL:-z}" != "z" ] && rm -rf sources/ crosstools/
diff --git a/clfs.sh b/clfs.sh
@@ -0,0 +1,47 @@
+#!/bin/sh
+
+. "$(dirname $0)/config.sh"
+
+# $ROOT now holds the realpath to the directory containing clfs.sh
+
+# clean old clfs build
+rm -rf $CLFS
+
+# fetch sources
+mkdir -p $SOURCES
+
+set -e
+
+# usage: FETCH <src> <pkg> <dir>
+FETCH() {
+	echo "Fetching $1..."
+	[ -f $SOURCES/$2 ] || curl -Lo $SOURCES/$2 $1
+	echo "Unpacking $2..."
+	[ -d $SOURCES/$3 ] || tar xf $SOURCES/$2 -C $SOURCES
+	echo "Unpacked into $SOURCES/$3!"
+}
+
+FETCH $LLVM_SRC $LLVM_PKG $LLVM_DIR
+FETCH $LINUX_SRC $LINUX_PKG $LINUX_DIR
+FETCH $MUSL_SRC $MUSL_PKG $MUSL_DIR
+FETCH $TOYBOX_SRC $TOYBOX_PKG $TOYBOX_DIR
+FETCH $IANA_ETC_SRC $IANA_ETC_PKG $IANA_ETC_DIR
+
+# TODO: do we necessarily want this much noise when running clfs.sh?
+# set -x
+
+mkdir -p $CLFS $WORK $SYSROOT $INITRAMFS $KERNEL
+
+# build our cross-compilation tools (clang + lld)
+cd "$ROOT"
+. "$ROOT/stage1.sh"
+
+# build necessary sysroot to cross-compile c and c++ programs for target
+cd "$ROOT"
+. "$ROOT/stage2.sh"
+
+# cross-compile userland, initramfs, and kernel for target
+cd "$ROOT"
+. "$ROOT/stage3.sh"
+
+echo "Done!"
diff --git a/config.sh b/config.sh
@@ -0,0 +1,56 @@
+#!/bin/sh
+
+ROOT="$(realpath $(dirname $0))"
+
+# clfs target toolchain and kernel definitions
+TARGET="aarch64-linux-musl"
+
+CMAKE_SYSTEM_NAME="Linux"
+CMAKE_SYSTEM_PROCESSOR="aarch64"
+
+MESON_SYSTEM="linux"
+MESON_CPU_FAMILY="aarch64"
+MESON_CPU="aarch64"
+MESON_ENDIAN="little"
+
+LINUX_ARCH="arm64"
+LINUX_HOSTCC_OPTS="LLVM=1" # NOTE: LLVM=1 is needed if your hostcc is clang
+LINUX_CONFIG="$ROOT/kconfig"
+
+# clfs directory definitions
+SOURCES="$ROOT/sources"
+CROSSTOOLS="$ROOT/crosstools"
+CLFS="$ROOT/clfs"
+
+WORK="$CLFS/work"
+SYSROOT="$CLFS/sysroot"
+INITRAMFS="$CLFS/initramfs"
+KERNEL="$CLFS/kernel"
+
+
+# package definitions
+LLVM_VER_MAJOR="18"
+LLVM_VER="18.1.0"
+LLVM_DIR="llvm-project-$LLVM_VER.src"
+LLVM_PKG="$LLVM_DIR.tar.xz"
+LLVM_SRC="https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VER/$LLVM_PKG"
+
+LINUX_VER="6.7.8"
+LINUX_DIR="linux-$LINUX_VER"
+LINUX_PKG="$LINUX_DIR.tar.xz"
+LINUX_SRC="https://www.kernel.org/pub/linux/kernel/v6.x/$LINUX_PKG"
+
+MUSL_VER="1.2.5"
+MUSL_DIR="musl-$MUSL_VER"
+MUSL_PKG="$MUSL_DIR.tar.gz"
+MUSL_SRC="https://musl.libc.org/releases/$MUSL_PKG"
+
+TOYBOX_VER="0.8.9"
+TOYBOX_DIR="toybox-$TOYBOX_VER"
+TOYBOX_PKG="$TOYBOX_DIR.tar.gz"
+TOYBOX_SRC="https://landley.net/toybox/downloads/$TOYBOX_PKG"
+
+IANA_ETC_VER="20240305"
+IANA_ETC_DIR="iana-etc-$IANA_ETC_VER"
+IANA_ETC_PKG="$IANA_ETC_DIR.tar.gz"
+IANA_ETC_SRC="https://github.com/Mic92/iana-etc/releases/download/$IANA_ETC_VER/$IANA_ETC_PKG"
diff --git a/env-clfs.sh b/env-clfs.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+
+. "$(dirname $0)/config.sh"
+
+# switch to cross-toolchain
+export HOSTCC=$(which cc)
+export HOSTCXX=$(which c++)
+export HOSTLD=$(which ld)
+
+export ADDR2LINE=$CROSSTOOLS/bin/llvm-addr2line
+export AR=$CROSSTOOLS/bin/llvm-ar
+export AS=$CROSSTOOLS/bin/llvm-as
+export CC=$TARGET-cc
+export CPP=$CROSSTOOLS/bin/llvm-cpp
+export CXX=$TARGET-c++
+export LD=$TARGET-ld
+export NM=$CROSSTOOLS/bin/llvm-nm
+export OBJCOPY=$CROSSTOOLS/bin/llvm-objcopy
+export OBJDUMP=$CROSSTOOLS/bin/llvm-objdump
+export RANLIB=$CROSSTOOLS/bin/llvm-ranlib
+export READELF=$CROSSTOOLS/bin/llvm-readelf
+export STRIP=$CROSSTOOLS/bin/llvm-strip
+export STRINGS=$CROSSTOOLS/bin/llvm-strings
+
+export PATH="$CROSSTOOLS/bin:$PATH"
+
+# disable command path hashing to pick up new tools
+set +h
diff --git a/stage1.sh b/stage1.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+echo "Building cross-compilation tools..."
+
+if [ ! -e $CROSSTOOLS/.built-crosstools ]; then
+	rm -rf $CROSSTOOLS
+	mkdir -p $CROSSTOOLS
+
+	# build cross-llvm (cland + lld)
+	cmake -S $SOURCES/$LLVM_DIR/llvm -B $CROSSTOOLS/build -GNinja \
+		-DCMAKE_INSTALL_PREFIX=$CROSSTOOLS \
+		-DCMAKE_BUILD_TYPE=MinSizeRel \
+		-DLLVM_ENABLE_PROJECTS="clang;lld" \
+		-DCLANG_DEFAULT_CXX_STDLIB="libc++" \
+		-DCLANG_DEFAULT_RTLIB="compiler-rt" \
+		-DCLANG_DEFAULT_UNWINDLIB="libunwind"
+
+	cmake --build $CROSSTOOLS/build
+	cmake --install $CROSSTOOLS/build
+
+	touch $CROSSTOOLS/.built-crosstools
+fi
+
+echo "Built cross-compilation tools!"
diff --git a/stage2.sh b/stage2.sh
@@ -0,0 +1,206 @@
+#!/bin/sh
+
+. "$(dirname $0)/env-clfs.sh"
+
+echo "Building $TARGET sysroot..."
+
+# cross-compilation wrappers
+cat > $CROSSTOOLS/bin/$TARGET-cc <<EOF
+#!/bin/sh
+exec $CROSSTOOLS/bin/clang \
+	--target=$TARGET \
+	--sysroot=$SYSROOT \
+	-resource-dir=$SYSROOT/usr/lib/clang/$LLVM_VER_MAJOR \
+	\$@
+EOF
+
+cat > $CROSSTOOLS/bin/$TARGET-c++ <<EOF
+#!/bin/sh
+exec $CROSSTOOLS/bin/clang++ \
+	--target=$TARGET \
+	--sysroot=$SYSROOT \
+	-resource-dir=$SYSROOT/usr/lib/clang/$LLVM_VER_MAJOR \
+	\$@
+EOF
+
+cat > $CROSSTOOLS/bin/$TARGET-ld <<EOF
+#!/bin/sh
+exec $CROSSTOOLS/bin/ld.lld \
+	--sysroot=$SYSROOT \
+	\$@
+EOF
+
+chmod +x $CROSSTOOLS/bin/$TARGET-{cc,c++,ld}
+
+cat > $CROSSTOOLS/$TARGET-cmake-toolchain.cmake <<EOF
+set(CMAKE_SYSTEM_NAME $CMAKE_SYSTEM_NAME)
+set(CMAKE_SYSTEM_PROCESSOR $CMAKE_SYSTEM_PROCESSOR)
+
+set(CMAKE_SYSROOT $SYSROOT)
+set(CMAKE_STAGING_PREFIX $SYSROOT)
+
+set(CMAKE_C_COMPILER $CROSSTOOLS/bin/$TARGET-cc)
+set(CMAKE_C_COMPILER_TARGET $TARGET)
+set(CMAKE_CXX_COMPILER $CROSSTOOLS/bin/$TARGET-c++)
+set(CMAKE_CXX_COMPILER_TARGET $TARGET)
+
+set(CMAKE_AR $CROSSTOOLS/bin/llvm-ar)
+set(CMAKE_RANLIB $CROSSTOOLS/bin/llvm-ranlib)
+
+set(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+set(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
+set(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
+set(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY)
+EOF
+
+cat > $CROSSTOOLS/$TARGET-meson-toolchain.txt <<EOF
+[binaries]
+c = '$CROSSTOOLS/bin/$TARGET-cc'
+cpp = '$CROSSTOOLS/bin/$TARGET-c++'
+ld = '$CROSSTOOLS/bin/$TARGET-ld'
+ar = '$CROSSTOOLS/bin/llvm-ar'
+objcopy = '$CROSSTOOLS/bin/llvm-objcopy'
+strip = '$CROSSTOOLS/bin/llvm-strip'
+
+[properties]
+sys_root = '$SYSROOT'
+
+[host_machine]
+system = '$MESON_SYSTEM'
+cpu_family = '$MESON_CPU_FAMILY'
+cpu = '$MESON_CPU'
+endian = '$MESON_ENDIAN'
+EOF
+
+# create FHS-compliant sysroot directories
+mkdir -pv \
+	$SYSROOT/bin \
+	$SYSROOT/dev \
+	$SYSROOT/etc \
+	$SYSROOT/lib \
+	$SYSROOT/mnt \
+	$SYSROOT/opt \
+	$SYSROOT/proc \
+	$SYSROOT/root \
+	$SYSROOT/run \
+	$SYSROOT/sbin \
+	$SYSROOT/srv \
+	$SYSROOT/sys \
+	$SYSROOT/tmp \
+	$SYSROOT/usr \
+	$SYSROOT/var
+
+mkdir -pv \
+	$SYSROOT/etc/opt
+
+mkdir -pv \
+	$SYSROOT/usr/bin \
+	$SYSROOT/usr/include \
+	$SYSROOT/usr/lib \
+	$SYSROOT/usr/local \
+	$SYSROOT/usr/sbin \
+	$SYSROOT/usr/share \
+	$SYSROOT/usr/src
+
+mkdir -pv \
+	$SYSROOT/usr/local/bin \
+	$SYSROOT/usr/local/etc \
+	$SYSROOT/usr/local/games \
+	$SYSROOT/usr/local/include \
+	$SYSROOT/usr/local/lib \
+	$SYSROOT/usr/local/man \
+	$SYSROOT/usr/local/sbin \
+	$SYSROOT/usr/local/share \
+	$SYSROOT/usr/local/src
+
+mkdir -pv \
+	$SYSROOT/var/cache \
+	$SYSROOT/var/lib \
+	$SYSROOT/var/local \
+	$SYSROOT/var/lock \
+	$SYSROOT/var/log \
+	$SYSROOT/var/opt \
+	$SYSROOT/var/run \
+	$SYSROOT/var/spool \
+	$SYSROOT/var/tmp
+
+# build linux-headers (requires HOSTCC)
+cp -r $SOURCES/$LINUX_DIR $WORK/$LINUX_DIR
+cd $WORK/$LINUX_DIR
+
+make HOSTCC=$HOSTCC $LINUX_HOSTCC_OPTS mrproper
+make HOSTCC=$HOSTCC $LINUX_HOSTCC_OPTS ARCH=$LINUX_ARCH \
+	INSTALL_HDR_PATH=$SYSROOT/usr headers_install
+
+# build musl-libc headers
+mkdir -p $WORK/build-libc-headers
+cd $WORK/build-libc-headers
+
+$SOURCES/$MUSL_DIR/configure \
+	CROSS_COMPILER=$TARGET- \
+	--target=$TARGET \
+	--prefix=/usr \
+	--syslibdir=/
+
+make DESTDIR=$SYSROOT install-headers
+
+# build compiler-rt builtins
+mkdir -p $WORK/build-builtins
+cmake -S $SOURCES/$LLVM_DIR/compiler-rt/lib/builtins -B $WORK/build-builtins -GNinja \
+	-DCMAKE_INSTALL_PREFIX=$SYSROOT/usr/lib/clang/$LLVM_VER_MAJOR \
+	-DCOMPILER_RT_DEFAULT_TARGET_TRIPLE=$TARGET
+
+cmake --build $WORK/build-builtins
+cmake --install $WORK/build-builtins
+
+# build musl-libc
+mkdir -p $WORK/build-libc
+cd $WORK/build-libc
+
+$SOURCES/$MUSL_DIR/configure \
+	CROSS_COMPILER=$TARGET- \
+	--target=$TARGET \
+	--prefix=/usr \
+	---syslibdir=/
+
+make DESTDIR=$SYSROOT install -j$(nproc)
+
+cd $WORK
+
+# build libunwind, libcxx, libccxabi
+
+LLVM_LIBUNWIND_FLAGS="
+	-DLIBUNWIND_USE_COMPILER_RT=ON
+"
+
+LLVM_LIBCXXABI_FLAGS="
+	-DLIBCXXABI_INCLUDE_TESTS=OFF
+	-DLIBCXXABI_USE_COMPILER_RT=ON
+	-DLIBCXXABI_USE_LLVM_UNWINDER=ON
+"
+
+LLVM_LIBCXX_FLAGS="
+	-DLIBCXX_INCLUDE_BENCHMARKS=OFF
+	-DLIBCXX_INCLUDE_DOCS=OFF
+	-DLIBCXX_CXX_ABI=libcxxabi
+	-DLIBCXX_USE_COMPILER_RT=ON
+	-DLIBCXX_HAS_MUSL_LIBC=ON
+"
+
+# TODO: because our c/cxx compilers do not yet have libunwind or libc++, we
+# must use -DCMAKE_{C,CXX}_COMPILER_WORKS=ON to pretend we have a working
+# toolchain. -DCMAKE_TRY_COMPILE_TARGET=STATIC_LIBRARY doesn't work, as it
+# leaks information from our host sysroot (__cxa_thread_atexit_impl for libc++)
+cmake -S $SOURCES/$LLVM_DIR/runtimes -B $WORK/build-cxxrt -GNinja \
+	-DCMAKE_INSTALL_PREFIX=$SYSROOT/usr \
+	-DCMAKE_C_COMPILER_WORKS=ON \
+	-DCMAKE_CXX_COMPILER_WORKS=ON \
+	-DLLVM_ENABLE_RUNTIMES="libunwind;libcxxabi;libcxx" \
+	$LLVM_LIBUNWIND_FLAGS \
+	$LLVM_LIBCXXABI_FLAGS \
+	$LLVM_LIBCXX_FLAGS
+
+cmake --build $WORK/build-cxxrt
+cmake --install $WORK/build-cxxrt
+
+echo "Built $TARGET sysroot!"
diff --git a/stage3.sh b/stage3.sh
@@ -0,0 +1,131 @@
+#!/bin/sh
+
+. "$(dirname $0)/env-clfs.sh"
+
+echo "Building $TARGET userland, initramfs, and kernel..."
+
+# build toybox
+cp -r $SOURCES/$TOYBOX_DIR $WORK/$TOYBOX_DIR
+cd $WORK/$TOYBOX_DIR
+
+make LDFLAGS="--static" distclean defconfig toybox
+make PREFIX=$SYSROOT install
+
+# build iana-etc
+cp -r $SOURCES/$IANA_ETC_DIR $WORK/$IANA_ETC_DIR
+cd $WORK/$IANA_ETC_DIR
+
+cp protocols services $SYSROOT/etc/
+
+# fixup sysroot directories
+touch $SYSROOT/var/log/lastlog
+chmod 664 $SYSROOT/var/log/lastlog
+
+# generate default /etc/passwd
+cat > $SYSROOT/etc/passwd <<EOF
+root::0:0:root:/root:/bin/sh
+bin:x:1:1:bin:/bin:/bin/false
+daemon:x:2:6:daemon:/sbin:/bin/false
+adm:x:3:16:adm:/var/adm:/bin/false
+operator:x:50:50:operator:/root:/bin/sh
+nobody:x:65534:65534:nobody:/:/bin/false
+EOF
+
+# generate default /etc/group
+cat > $SYSROOT/etc/group <<EOF
+root:x:0:
+bin:x:1:
+sys:x:2:
+kmem:x:3:
+tty:x:4:
+tape:x:5:
+daemon:x:6:
+floppy:x:7:
+disk:x:8:
+lp:x:9:
+dialout:x:10:
+audio:x:11:
+video:x:12:
+utmp:x:13:
+usb:x:14:
+cdrom:x:15:
+adm:x:16:root,adm,daemon
+console:x:17:
+users:x:100:
+nogroup:x:65533:
+nobody:x:65534:
+EOF
+
+# generate default /etc/inittab
+cat > $SYSROOT/etc/inittab <<EOF
+# /etc/inittab
+# TODO: https://www.linuxfromscratch.org/hints/downloads/files/bsd-init.txt
+
+::sysinit:/etc/rc.d/startup
+
+tty1::respawn:/sbin/getty 38400 tty1
+tty2::respawn:/sbin/getty 38400 tty2
+tty3::respawn:/sbin/getty 38400 tty3
+tty4::respawn:/sbin/getty 38400 tty4
+tty5::respawn:/sbin/getty 38400 tty5
+tty6::respawn:/sbin/getty 38400 tty6
+
+::shutdown:/etc/rc.d/shutdown
+::ctrlaltdel:/sbin/reboot
+EOF
+
+# generate default /etc/fstab
+cat > $SYSROOT/etc/fstab <<EOF
+# file-system	mount-point	type	options		dump	fsck
+EOF
+
+# generate default /etc/hostname
+cat > $SYSROOT/etc/hostname <<EOF
+clfs
+EOF
+
+# generate default /etc/hosts
+cat > $SYSROOT/etc/hosts <<EOF
+# /etc/hosts
+
+127.0.0.1	localhost
+::1		localhost
+EOF
+
+# generate default /etc/profile
+cat > $SYSROOT/etc/profile <<'EOF'
+# /etc/profile
+
+export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
+
+export USER=$(/usr/bin/id -un)
+export LOGNAME=$USER
+export HOSTNAME=$(/bin/hostname)
+export HISTSIZE=1000
+export HISTFILESIZE=1000
+
+#export PAGER=/bin/less
+#export EDITOR=/usr/bin/vim
+EOF
+
+if false; then
+
+mkdir -p $BOOTROOT $INITRAMFS
+
+# build initramfs
+
+
+# build linux-kernel
+cd $SOURCES/$LINUX_DIR
+
+make HOSTCC=$HOSTCC ARCH=$LINUX_ARCH mrproper
+make HOSTCC=$HOSTCC ARCH=$LINUX_ARCH CROSS_COMPILE=$TARGET-
+make HOSTCC=$HOSTCC ARCH=$LINUX_ARCH CROSS_COMPILE=$TARGET- \
+	INSTALL_MOD_PATH=$SYSROOT modules_install
+
+cp -v arch/$LINUX_ARCH/boot/bzImage $BOOTROOT/linuxstub.efi
+cp -rv arch/$LINUX_ARCH/boot/dts $BOOTROOT/
+
+fi
+
+echo "Built $TARGET userland, initramfs, and kernel!"