scripts/mkimage.sh: scripts to build bootable alpine images

scripts/genapkovl-dhcp.sh

+#!/bin/sh -e
+
+HOSTNAME="$1"
+if [ -z "$HOSTNAME" ]; then
+	echo "usage: $0 hostname"
+	exit 1
+fi
+
+cleanup() {
+	rm -rf "$tmp"
+}
+
+makefile() {
+	OWNER="$1"
+	PERMS="$2"
+	FILENAME="$3"
+	cat > "$FILENAME"
+	chown "$OWNER" "$FILENAME"
+	chmod "$PERMS" "$FILENAME"
+}
+
+rc_add() {
+	mkdir -p "$tmp"/etc/runlevels/"$2"
+	ln -sf /etc/init.d/"$1" "$tmp"/etc/runlevels/"$2"/"$1"
+}
+
+tmp="$(mktemp -d)"
+trap cleanup EXIT
+
+mkdir -p "$tmp"/etc
+makefile root:root 0644 "$tmp"/etc/hostname <<EOF
+$HOSTNAME
+EOF
+
+mkdir -p "$tmp"/etc/network
+makefile root:root 0644 "$tmp"/etc/network/interfaces <<EOF
+auto lo
+iface lo inet loopback
+
+auto eth0
+iface eth0 inet dhcp
+EOF
+
+mkdir -p "$tmp"/etc/apk
+makefile root:root 0644 "$tmp"/etc/apk/world <<EOF
+alpine-base
+EOF
+
+rc_add devfs sysinit
+rc_add dmesg sysinit
+rc_add mdev sysinit
+rc_add hwdrivers sysinit
+rc_add modloop sysinit
+
+rc_add hwclock boot
+rc_add modules boot
+rc_add sysctl boot
+rc_add hostname boot
+rc_add bootmisc boot
+rc_add syslog boot
+
+rc_add mount-ro shutdown
+rc_add killprocs shutdown
+rc_add savecache shutdown
+
+tar -c -C "$tmp" etc | gzip -9n > $HOSTNAME.apkovl.tar.gz

scripts/genapkovl-xen.sh

+#!/bin/sh -e
+
+HOSTNAME="$1"
+if [ -z "$HOSTNAME" ]; then
+	echo "usage: $0 hostname"
+	exit 1
+fi
+
+cleanup() {
+	rm -rf "$tmp"
+}
+
+makefile() {
+	OWNER="$1"
+	PERMS="$2"
+	FILENAME="$3"
+	cat > "$FILENAME"
+	chown "$OWNER" "$FILENAME"
+	chmod "$PERMS" "$FILENAME"
+}
+
+rc_add() {
+	mkdir -p "$tmp"/etc/runlevels/"$2"
+	ln -sf /etc/init.d/"$1" "$tmp"/etc/runlevels/"$2"/"$1"
+}
+
+tmp="$(mktemp -d)"
+trap cleanup EXIT
+
+mkdir -p "$tmp"/etc
+makefile root:root 0644 "$tmp"/etc/hostname <<EOF
+$HOSTNAME
+EOF
+
+makefile root:root 0644 "$tmp"/etc/modules <<EOF
+xen_netback
+xen_blkback
+xenfs
+xen-platform-pci
+xen_wdt
+tun
+EOF
+
+mkdir -p "$tmp"/etc/network
+makefile root:root 0644 "$tmp"/etc/network/interfaces <<EOF
+auto lo
+iface lo inet loopback
+EOF
+
+mkdir -p "$tmp"/etc/apk
+
+makefile root:root 0644 "$tmp"/etc/apk/world <<EOF
+xen
+EOF
+
+rc_add devfs sysinit
+rc_add dmesg sysinit
+rc_add udev sysinit
+
+rc_add hwclock boot
+rc_add modules boot
+rc_add sysctl boot
+rc_add hostname boot
+rc_add bootmisc boot
+rc_add syslog boot
+
+rc_add udev-postmount default
+rc_add xenstored default
+rc_add xenconsoled default
+
+rc_add mount-ro shutdown
+rc_add killprocs shutdown
+rc_add savecache shutdown
+
+tar -c -C "$tmp" etc | gzip -9n > $HOSTNAME.apkovl.tar.gz

scripts/mkimage.sh

+#!/bin/sh
+
+# apk add abuild apk-tools alpine-conf busybox fakeroot xorriso
+
+# FIXME: clean workdir out of unneeded sections
+# FIXME: --release: cp/mv images to REPODIR/$ARCH/releases/
+# FIXME: --update-latest: rewrite latest-releases.yaml with this build
+
+set -e
+
+# get abuild configurables
+[ -e /usr/share/abuild/functions.sh ] || (echo "abuild not found" ; exit 1)
+. /usr/share/abuild/functions.sh
+
+# deduce aports directory
+[ -n "$APORTS" ] || APORTS=$(realpath $(dirname $0)/../)
+[ -e "$APORTS/main/build-base" ] || die "Unable to deduce aports base checkout"
+
+# 
+all_sections=""
+all_profiles=""
+all_checksums="sha256 sha512"
+all_arches="armhf x86 x86_64"
+all_dirs=""
+build_date="$(date +%y%m%d)"
+default_arch="$(apk --print-arch)"
+_hostkeys=""
+_simulate=""
+_checksum=""
+
+OUTDIR="$PWD"
+RELEASE="${build_date}"
+
+msg() {
+	if [ -n "$quiet" ]; then return 0; fi
+	local prompt="$GREEN>>>${NORMAL}"
+	local name="${BLUE}mkimage${ARCH+-$ARCH}${NORMAL}"
+	printf "${prompt} ${name}: %s\n" "$1" >&2
+}
+
+list_has() {
+	local needle="$1"
+	local i
+	shift
+	for i in $@; do
+		[ "$needle" != "$i" ] || return 0
+	done
+	return 1
+}
+
+usage() {
+	cat <<EOF
+
+$0	[--tag RELEASE] [--outdir OUTDIR] [--workdir WORKDIR]
+		[--arch ARCH] [--profile PROFILE] [--hostkeys] [--simulate]
+$0	--help
+
+options:
+--arch		Specify which architecture images to build
+		(default: $default_arch)
+--hostkeys	Copy system apk signing keys to created images
+--outdir	Specify directory for the created images
+--profile	Specify which profiles to build
+--simulate	Don't execute commands
+--tag		Build images for tag RELEASE
+--workdir	Specify temporary working directory (cache)
+
+known profiles: $(echo $all_profiles | sort -u)
+
+EOF
+}
+
+# helpers
+load_plugins() {
+	local f
+	[ -e "$1" ] || return 0
+	for f in "$1"/mkimg.*.sh; do
+		[ -e "$f" ] || return 0
+		break
+	done
+	all_profiles="$all_profiles $(sed -n -e 's/^profile_\(.*\)() {$/\1/p' $1/mkimg.*.sh)"
+	all_sections="$all_sections $(sed -n -e 's/^section_\(.*\)() {$/\1/p' $1/mkimg.*.sh)"
+	for f in "$1"/mkimg.*.sh; do
+		. $f
+	done
+}
+
+checksum() {
+	sha1sum | cut -f 1 -d ' '
+}
+
+build_section() {
+	local section="$1"
+	local args="$@"
+	local _dir="${args//[^a-zA-Z0-9]/_}"
+	shift
+	local args="$@"
+
+	if [ -z "$_dir" ]; then
+		_fail="yes"
+		return 1
+	fi
+
+	if [ ! -e "$WORKDIR/${_dir}" ]; then
+		DESTDIR="$WORKDIR/${_dir}.work"
+		msg "--> $section $args"
+		if [ -z "$_simulate" ]; then
+			rm -rf "$DESTDIR"
+			mkdir -p "$DESTDIR"
+			if build_${section} "$@"; then
+				mv "$DESTDIR" "$WORKDIR/${_dir}"
+				_dirty="yes"
+			else
+				rm -rf "$DESTDIR"
+				_fail="yes"
+			fi
+		fi
+	fi
+	unset DESTDIR
+	all_dirs="$all_dirs $_dir"
+	_my_sections="$_my_sections $_dir"
+}
+
+build_profile() {
+	local _id _dir _spec
+	_my_sections=""
+	_dirty="no"
+	_fail="no"
+
+	profile_$PROFILE
+	list_has $ARCH $arch || return 0
+
+	msg "Building $PROFILE"
+
+	# Collect list of needed sections, and make sure they are built
+	for SECTION in $all_sections; do
+		section_$SECTION
+	done
+	[ "$_fail" = "no" ] || return 1
+
+	# Defaults
+	[ -n "$image_name" ] || image_name="alpine-${PROFILE}"
+	[ -n "$output_filename" ] || output_filename="${image_name}-${RELEASE}-${ARCH}.${image_ext}"
+
+	# Construct final image
+	local _imgid=$(echo -n $_my_sections | sort | checksum)
+	DESTDIR=$WORKDIR/image-$_imgid-$ARCH-$PROFILE
+	if [ "_$dirty" = "yes" -o ! -e "$DESTDIR" ]; then
+		msg "Creating $output_filename"
+		if [ -z "$_simulate" ]; then
+			# Merge sections
+			rm -rf "$DESTDIR"
+			mkdir -p "$DESTDIR"
+			for _dir in $_my_sections; do
+				for _fn in $WORKDIR/$_dir/*; do
+					[ ! -e "$_fn" ] || cp -Lrs $_fn $DESTDIR/
+				done
+			done
+			echo "${image_name}-${RELEASE} ${build_date}" > "$DESTDIR"/.alpine-release
+		fi
+	fi
+
+	if [ "_$dirty" = "yes" -o ! -e "$output_filename" ]; then
+		# Create image
+		output_format="${image_ext//[:\.]/}"
+		create_image_${output_format} || _fail="yes"
+
+		if [ "$_checksum" = "yes" ]; then
+			for _c in $all_checksums; do
+				${_c}sum "$output_filename" > "${output_filename}.${_c}"
+			done
+		fi
+	fi
+}
+
+# load plugins
+load_plugins "$(dirname $0)"
+[ -z "$HOME" ] || load_plugins "$HOME/.mkimage"
+
+# parse parameters
+while [ $# -gt 0 ]; do
+	opt="$1"
+	shift
+	case "$opt" in
+	--repository) REPODIR="$1"; shift ;;
+	--workdir) WORKDIR="$1"; shift ;;
+	--outdir) OUTDIR="$1"; shift ;;
+	--tag) RELEASE="$1"; shift ;;
+	--arch) req_arch="$1"; shift ;;
+	--profile) req_profiles="$1"; shift ;;
+	--hostkeys) _hostkeys="--hostkeys";;
+	--simulate) _simulate="yes";;
+	--checksum) _checksum="yes";;
+	--) break ;;
+	-*) usage; exit 1;;
+	esac
+done
+
+# setup defaults
+if [ -z "$WORKDIR" ]; then
+	WORKDIR="$(mktemp -d -t mkimage.XXXXXX)"
+	trap 'rm -rf $WORKDIR' INT
+	mkdir -p "$WORKDIR"
+fi
+req_profiles=${req_profiles:-${all_profiles}}
+req_arch=${req_arch:-${default_arch}}
+[ "$req_arch" != "all" ] || req_arch="${all_arch}"
+[ "$req_profiles" != "all" ] || req_profiles="${all_profiles}"
+
+# create images
+for ARCH in $req_arch; do
+	APKROOT="$WORKDIR/apkroot-$ARCH"
+	if [ ! -e "$APKROOT" ]; then
+		# create root for caching packages
+		mkdir -p "$APKROOT/etc/apk/cache"
+		cp -Pr /etc/apk/keys "$APKROOT/etc/apk/"
+		abuild-apk --arch "$ARCH" --root "$APKROOT" add --initdb
+
+		echo "$REPODIR" > "$APKROOT/etc/apk/repositories"
+	fi
+	abuild-apk update --root "$APKROOT"
+
+	for PROFILE in $req_profiles; do
+		(build_profile)
+	done
+done

scripts/mkimg.arm.sh

+build_rpi_blobs() {
+	local fw
+	for fw in bootcode.bin fixup.dat start.elf ; do
+		curl --remote-time https://raw.githubusercontent.com/raspberrypi/firmware/${rpi_firmware_commit}/boot/${fw} \
+			--output "${DESTDIR}"/${fw} || return 1
+	done
+}
+
+rpi_gen_cmdline() {
+	echo "modules=loop,squashfs,sd-mod,usb-storage quiet ${kernel_cmdline}"
+}
+
+rpi_gen_config() {
+	cat <<EOF
+disable_splash=1
+boot_delay=0
+gpu_mem=256
+gpu_mem_256=64
+[pi0]
+kernel=boot/vmlinuz-rpi
+initramfs boot/initramfs-rpi 0x08000000
+[pi1]
+kernel=boot/vmlinuz-rpi
+initramfs boot/initramfs-rpi 0x08000000
+[pi2]
+kernel=boot/vmlinuz-rpi2
+initramfs boot/initramfs-rpi2 0x08000000
+[pi3]
+kernel=boot/vmlinuz-rpi2
+initramfs boot/initramfs-rpi2 0x08000000
+[all]
+include usercfg.txt
+EOF
+}
+
+build_rpi_config() {
+	rpi_gen_cmdline > "${DESTDIR}"/cmdline.txt
+	rpi_gen_config > "${DESTDIR}"/config.txt
+}
+
+section_rpi_config() {
+	[ -n "$rpi_firmware_commit" ] || return 0
+	build_section rpi_config $( (rpi_gen_cmdline ; rpi_gen_config) | checksum )
+	build_section rpi_blobs "$rpi_firmware_commit"
+}
+
+profile_rpi() {
+	profile_base
+	image_ext="tar.gz"
+	arch="armhf"
+	rpi_firmware_commit="4bf906cdd221c4f6815d0da7dda0cd59d25d945b"
+	kernel_flavors="rpi rpi2"
+	kernel_cmdline="dwc_otg.lpm_enable=0 console=ttyAMA0,115200 console=tty1"
+	initrd_features="base bootchart squashfs ext2 ext3 ext4 f2fs kms mmc raid scsi usb"
+	apkovl="genapkovl-dhcp.sh"
+	hostname="rpi"
+	image_ext="tar.gz"
+}
+
+build_uboot() {
+	# FIXME: Fix apk-tools to extract packages directly
+	local pkg pkgs="$(apk fetch  --simulate --root /tmp/timo/apkroot-armhf/ --recursive u-boot-all | sed -ne "s/^Downloading \([^0-9.]*\)\-.*$/\1/p")"
+	for pkg in $pkgs; do
+		[ "$pkg" == "u-boot-all" ] || apk fetch --root "$APKROOT" --stdout $pkg | tar -C "$DESTDIR" -xz usr
+	done
+	mkdir -p "$DESTDIR"/u-boot
+	mv "$DESTDIR"/usr/sbin/update-u-boot "$DESTDIR"/usr/share/u-boot/* "$DESTDIR"/u-boot
+	rm -rf "$DESTDIR"/usr
+}
+
+section_uboot() {
+	[ -n "$uboot_install" ] || return 0
+	build_section uboot $ARCH $(apk fetch --root "$APKROOT" --simulate --recursive u-boot-all | sort | checksum)
+}
+
+profile_uboot() {
+	profile_base
+	image_ext="tar.gz"
+	arch="armhf armv7"
+	kernel_flavors="grsec"
+	kernel_addons="xtables-addons"
+	initfs_features="base bootchart squashfs ext2 ext3 ext4 kms mmc raid scsi usb"
+	apkovl="genapkovl-dhcp.sh"
+	hostname="alpine"
+	uboot_install="yes"
+}

scripts/mkimg.base.sh

+build_kernel() {
+	local _flavor="$2"
+	shift 3
+	local _pkgs="$@"
+	update-kernel \
+		$_hostkeys \
+		--media \
+		--flavor "$_flavor" \
+		--arch "$ARCH" \
+		--package "$_pkgs" \
+		--feature "$initfs_features" \
+		--repositories-file "$APKROOT/etc/apk/repositories" \
+		"$DESTDIR"
+}
+
+section_kernels() {
+	local _f _a _pkgs
+	for _f in $kernel_flavors; do
+		_pkgs="linux-$_f linux-firmware"
+		for _a in $kernel_addons; do
+			_pkgs="$_pkgs $_a-$_f"
+		done
+		local id=$( (echo "$initfs_features::$_hostkeys" ; apk fetch --root "$APKROOT" --simulate alpine-base $_pkgs | sort) | checksum)
+		build_section kernel $ARCH $_f $id $_pkgs
+	done
+}
+
+build_apks() {
+	local _apksdir="$DESTDIR/apks"
+	local _archdir="$_apksdir/$ARCH"
+	mkdir -p "$_archdir"
+
+	apk fetch --root "$APKROOT" --link --recursive --output "$_archdir" $apks
+	if ! ls "$_archdir"/*.apk >& /dev/null; then
+		return 1
+	fi
+
+	apk index \
+		--description "$RELEASE" \
+		--rewrite-arch "$ARCH" \
+		--index "$_archdir"/APKINDEX.tar.gz \
+		--output "$_archdir"/APKINDEX.tar.gz \
+		"$_archdir"/*.apk
+	abuild-sign "$_archdir"/APKINDEX.tar.gz
+	touch "$_apksdir/.boot_repository"
+}
+
+section_apks() {
+	[ -n "$apks" ] || return 0
+	build_section apks $ARCH $(apk fetch --root "$APKROOT" --simulate --recursive $apks | sort | checksum)
+}
+
+build_apkovl() {
+	local _host="$1"
+	msg "Generating $_host.apkovl.tar.gz"
+	(local _pwd=$PWD; cd "$DESTDIR"; fakeroot "$_pwd"/"$apkovl" "$_host")
+}
+
+section_apkovl() {
+	[ -n "$apkovl" -a -n "$hostname" ] || return 0
+	build_section apkovl $hostname $(checksum < "$apkovl")
+}
+
+build_syslinux() {
+	local _fn
+	mkdir -p "$DESTDIR"/boot/syslinux
+	apk fetch --root "$APKROOT" --stdout syslinux | tar -C "$DESTDIR" -xz usr/share/syslinux
+	for _fn in isolinux.bin ldlinux.c32 libutil.c32 libcom32.c32 mboot.c32; do
+		mv "$DESTDIR"/usr/share/syslinux/$_fn "$DESTDIR"/boot/syslinux/$_fn || return 1
+	done
+	rm -rf "$DESTDIR"/usr
+}
+
+section_syslinux() {
+	[ "$output_format" = "iso" ] || return 0
+	build_section syslinux $(apk fetch --root "$APKROOT" --simulate syslinux | sort | checksum)
+}
+
+syslinux_gen_config() {
+	[ -z "$syslinux_serial" ] || echo "SERIAL $syslinux_serial"
+	echo "TIMEOUT ${syslinux_timeout:-20}"
+	echo "PROMPT ${syslinux_prompt:-1}"
+	echo "DEFAULT ${kernel_flavors%% *}"
+
+	local _f
+	for _f in $kernel_flavors; do
+		if [ -z "${xen_params+set}" ]; then
+			cat <<EOF
+
+LABEL $_f
+	MENU LABEL Linux $_f
+	KERNEL /boot/vmlinuz-$_f
+	INITRD /boot/initramfs-$_f
+	DEVICETREEDIR /boot/dtbs
+	APPEND $initfs_cmdline $kernel_cmdline
+EOF
+		else
+			cat <<EOF
+
+LABEL $_f
+	MENU LABEL Xen/Linux $_f
+	KERNEL /boot/syslinux/mboot.c32
+	APPEND /boot/xen.gz ${xen_params} --- /boot/vmlinuz-$_f $initfs_cmdline $kernel_cmdline --- /boot/initramfs-$_f
+EOF
+		fi
+	done
+}
+
+build_syslinux_cfg() {
+	local syslinux_cfg="$1"
+	mkdir -p "${DESTDIR}/$(dirname $syslinux_cfg)"
+	syslinux_gen_config > "${DESTDIR}"/$syslinux_cfg
+}
+
+section_syslinux_cfg() {
+	syslinux_cfg=""
+	[ ! "$output_format" = "iso" ] || syslinux_cfg="boot/syslinux/syslinux.cfg"
+	[ ! -n "$uboot_install" ] || syslinux_cfg="extlinux/extlinux.conf"
+	[ -n "$syslinux_cfg" ] || return 0
+	build_section syslinux_cfg $syslinux_cfg $(syslinux_gen_config | checksum)
+}
+
+create_image_iso() {
+	local ISO="${OUTDIR}/${output_filename}"
+	xorrisofs \
+		-o ${ISO} -l -J -R \
+		-b boot/syslinux/isolinux.bin		\
+		-c boot/syslinux/boot.cat		\
+		-V "alpine-$PROFILE $RELEASE $ARCH"	\
+		-no-emul-boot		\
+		-boot-load-size 4	\
+		-boot-info-table	\
+		-quiet			\
+		-follow-links		\
+		${iso_opts}		\
+		${DESTDIR}
+	isohybrid ${ISO}
+}
+
+create_image_targz() {
+	tar -C "${DESTDIR}" -chzf ${OUTDIR}/${output_filename} .
+}
+
+profile_base() {
+	kernel_flavors="grsec"
+	initfs_cmdline="modules=loop,squashfs,sd-mod,usb-storage quiet"
+	initfs_features="ata base bootchart cdrom squashfs ext2 ext3 ext4 mmc raid scsi usb virtio"
+	apks="alpine-base alpine-mirrors bkeymaps chrony e2fsprogs network-extras openssl openssh tzdata"
+	apkovl="genapkovl-dhcp.sh"
+	hostname="alpine"
+}

scripts/mkimg.standard.sh

+profile_standard() {
+	profile_base
+	image_ext="iso"
+	arch="x86 x86_64"
+	output_format="iso"
+}
+
+profile_vanilla() {
+	profile_standard
+	kernel_flavors="vanilla"
+}
+
+profile_extended() {
+	profile_standard
+	kernel_addons="dahdi-linux xtables-addons"
+	apks="$apks
+		dahdi-linux dahdi-tools ethtool hwdata lftp links
+		logrotate lua5.3 lsof lm_sensors lxc lxc-templates nano
+		pax-utils paxctl pciutils screen strace sudo tmux
+		usbutils v86d vim xtables-addons
+
+		acct arpon arpwatch awall bridge-utils bwm-ng
+		ca-certificates conntrack-tools cutter cyrus-sasl dhcp
+		dhcpcd dhcrelay dnsmasq email fping fprobe haserl htop
+		igmpproxy ip6tables iproute2 iproute2-qos ipsec-tools
+		iptables iputils irssi ldns-tools links
+		ncurses-terminfo net-snmp net-snmp-tools nrpe nsd
+		opennhrp openvpn openvswitch pingu ppp quagga
+		quagga-nhrp rpcbind sntpc socat ssmtp strongswan
+		sysklogd tcpdump tcpproxy tinyproxy unbound
+		wireless-tools wpa_supplicant zonenotify
+
+		btrfs-progs cksfv dosfstools cryptsetup
+		cciss_vol_status lvm2 mdadm mkinitfs mtools nfs-utils
+		parted rsync sfdisk syslinux unrar util-linux xfsprogs
+		"
+
+	local _k _a
+	for _k in $kernel_flavors; do
+		apks="$apks linux-$_k"
+		for _a in $kernel_addons; do
+			apks="$apks $_a-$_k"
+		done
+	done
+	apks="$apks linux-firmware"
+}
+
+profile_virt() {