From f61f78c719c23da3f534671471f88923371c73cf Mon Sep 17 00:00:00 2001 From: Jake Buchholz Date: Sun, 20 Jan 2019 18:08:51 -0800 Subject: [PATCH 1/2] Allow additional services on the AMI's runlevels I'm using this with my AMIs to add haveged to the boot runlevel to boost the amount of initial entropy on smaller instance types, so sshd can start in under 6s instead of over 2m. add_svcs: boot: - haveged --- Makefile | 4 +++- alpine-ami.yaml | 2 +- make_ami.sh | 13 +++++++++++-- variables.yaml-default | 7 +++++++ 4 files changed, 22 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index f3e4c8a..6a76c0a 100644 --- a/Makefile +++ b/Makefile @@ -24,9 +24,11 @@ build/convert: @echo "#!`pwd`/.py3/bin/python" > build/convert @echo "import yaml, json, sys" >> build/convert @echo "y = yaml.load(open(sys.argv[1]))" >> build/convert - @echo "for k in ['ami_access','deploy_regions','add_repos','add_pkgs']:" >> build/convert + @echo "for k in ['ami_access','deploy_regions','add_repos','add_pkgs','add_svcs']:" >> build/convert @echo " if k in y and isinstance(y[k], list):" >> build/convert @echo " y[k] = ','.join(str(x) for x in y[k])" >> build/convert + @echo " if k in y and isinstance(y[k], dict):" >> build/convert + @echo " y[k] = ':'.join(str(l) + '=' + ','.join(str(s) for s in ss) for l, ss in y[k].items())" >> build/convert @echo "json.dump(y, sys.stdout, indent=4, separators=(',', ': '))" >> build/convert @chmod +x build/convert diff --git a/alpine-ami.yaml b/alpine-ami.yaml index 5991115..4db603f 100644 --- a/alpine-ami.yaml +++ b/alpine-ami.yaml @@ -63,4 +63,4 @@ provisioners: script: "make_ami.sh" environment_vars: - "ALPINE_RELEASE={{user `alpine_release`}}" - execute_command: 'sudo sh -c "{{ .Vars }} {{ .Path }} ''{{user `add_repos`}}'' ''{{user `add_pkgs`}}''"' + execute_command: 'sudo sh -c "{{ .Vars }} {{ .Path }} ''{{user `add_repos`}}'' ''{{user `add_pkgs`}}'' ''{{user `add_svcs`}}''"' diff --git a/make_ami.sh b/make_ami.sh index b5097ab..e891f25 100755 --- a/make_ami.sh +++ b/make_ami.sh @@ -229,11 +229,18 @@ EOF enable_services() { local target="$1" + local add_svcs="$2" rc_add "$target" default sshd chronyd networking tiny-ec2-bootstrap rc_add "$target" sysinit devfs dmesg mdev hwdrivers rc_add "$target" boot modules hwclock swap hostname sysctl bootmisc syslog acpid rc_add "$target" shutdown killprocs savecache mount-ro + + if [ -n "$add_svcs" ]; then + local lvl_svcs; for lvl_svcs in $(echo "$add_svcs" | tr : ' '); do + rc_add "$target" $(echo "$lvl_svcs" | tr =, ' ') + done + fi } create_alpine_user() { @@ -295,13 +302,14 @@ version_sorted() { } main() { - [ "$#" -ne 2 ] && die "Expecting two parameters\nUsage: $0 '[,]' '[,]'" + [ "$#" -ne 3 ] && die "Expecting three parameters\nUsage: $0 '[[,...]]' '[[,...]]' '[=[,...][:...]]'" [ "$ALPINE_RELEASE" != 'edge' ] && { version_sorted $MIN_RELEASE $ALPINE_RELEASE || die "Minimum alpine_release is '$MIN_RELEASE'" } local add_repos="$1" local add_pkgs="$2" + local add_svcs="$3" local device="/dev/xvdf" local target="/mnt/target" @@ -316,6 +324,7 @@ main() { einfo "Creating root filesystem" make_filesystem "$device" "$target" + einfo "Configuring Alpine repositories" setup_repositories "$target" "$add_repos" einfo "Fetching Alpine signing keys" @@ -338,7 +347,7 @@ main() { setup_mdev "$target" setup_fstab "$target" setup_networking "$target" - enable_services "$target" + enable_services "$target" "$add_svcs" create_alpine_user "$target" configure_ntp "$target" diff --git a/variables.yaml-default b/variables.yaml-default index 668233a..15c9bba 100644 --- a/variables.yaml-default +++ b/variables.yaml-default @@ -36,6 +36,13 @@ add_repos: # List of additional packages to add to the AMI. add_pkgs: +# Additional services to start at the specified level. +add_svcs: +# boot: +# - service1 +# default: +# - service2 + # Size of the AMI image (in GiB). volume_size: "1" -- GitLab From 92bf0353e727c1c3184312cfd2306909624e4a6a Mon Sep 17 00:00:00 2001 From: Jake Buchholz Date: Mon, 21 Jan 2019 14:16:25 -0800 Subject: [PATCH 2/2] * fix race condition with nvme-ebs /dev linking * copy nvme stuff to build target in one operation * add eu-north-1 region --- alpine-ami.yaml | 7 ++----- make_ami.sh | 2 +- nvme-ebs-links.sh | 20 -------------------- nvme-ebs-mdev.conf | 3 --- nvme/nvme-ebs-links | 22 ++++++++++++++++++++++ nvme/nvme-ebs-mdev.conf | 3 +++ variables.yaml-default | 1 + 7 files changed, 29 insertions(+), 29 deletions(-) delete mode 100755 nvme-ebs-links.sh delete mode 100644 nvme-ebs-mdev.conf create mode 100755 nvme/nvme-ebs-links create mode 100644 nvme/nvme-ebs-mdev.conf diff --git a/alpine-ami.yaml b/alpine-ami.yaml index 4db603f..a4ba2c0 100644 --- a/alpine-ami.yaml +++ b/alpine-ami.yaml @@ -54,11 +54,8 @@ builders: provisioners: - type: "file" - source: "nvme-ebs-links.sh" - destination: "/tmp/" - - type: "file" - source: "nvme-ebs-mdev.conf" - destination: "/tmp/" + source: "nvme/" + destination: "/tmp" - type: "shell" script: "make_ami.sh" environment_vars: diff --git a/make_ami.sh b/make_ami.sh index e891f25..c8dde78 100755 --- a/make_ami.sh +++ b/make_ami.sh @@ -157,7 +157,7 @@ install_core_packages() { setup_mdev() { local target="$1" - cp -a /tmp/nvme-ebs-links.sh "$target"/lib/mdev + cp /tmp/nvme-ebs-links "$target"/lib/mdev sed -n -i -e '/# fallback/r /tmp/nvme-ebs-mdev.conf' -e 1x -e '2,${x;p}' -e '${x;p}' "$target"/etc/mdev.conf } diff --git a/nvme-ebs-links.sh b/nvme-ebs-links.sh deleted file mode 100755 index ca8c4f6..0000000 --- a/nvme-ebs-links.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/sh - -[ -x /usr/sbin/nvme ] || exit - -case $ACTION in - add|"") - BASE=$(echo $MDEV | sed -re 's/^(nvme[0-9]+n[0-9]+).*/\1/') - PART=$(echo $MDEV | sed -re 's/nvme[0-9]+n[0-9]+p?//g') - EBS=$(/usr/sbin/nvme id-ctrl "/dev/$BASE" -b 2>/dev/null | dd bs=32 skip=96 count=1 2>/dev/null | tr -d ' ') - EBS=${EBS#/dev/}$PART - ln -sf $MDEV ${EBS/xvd/sd} - ln -sf $MDEV ${EBS/sd/xvd} - ;; - remove) - for TARGET in sd* xvd* - do - [ "$(readlink $TARGET 2>/dev/null)" = $MDEV ] && rm -f $TARGET - done - ;; -esac diff --git a/nvme-ebs-mdev.conf b/nvme-ebs-mdev.conf deleted file mode 100644 index 2d60927..0000000 --- a/nvme-ebs-mdev.conf +++ /dev/null @@ -1,3 +0,0 @@ -# ebs nvme links -nvme[0-9]+n[0-9]+.* root:root 0660 */lib/mdev/nvme-ebs-links.sh - diff --git a/nvme/nvme-ebs-links b/nvme/nvme-ebs-links new file mode 100755 index 0000000..c197a73 --- /dev/null +++ b/nvme/nvme-ebs-links @@ -0,0 +1,22 @@ +#!/bin/sh + +[ -x /usr/sbin/nvme ] || exit + +case $ACTION in + add|"") + BASE=$(echo $MDEV | sed -re 's/^(nvme[0-9]+n[0-9]+).*/\1/') + PART=$(echo $MDEV | sed -re 's/nvme[0-9]+n[0-9]+p?//g') + until [ -n "$EBS" ]; do + EBS=$(/usr/sbin/nvme id-ctrl "/dev/$BASE" -b 2>/dev/null | dd bs=32 skip=96 count=1 2>/dev/null | tr -d ' ') + done + EBS=${EBS#/dev/}$PART + ln -sf "$MDEV" "${EBS/xvd/sd}" + ln -sf "$MDEV" "${EBS/sd/xvd}" + ;; + remove) + for TARGET in sd* xvd* + do + [ "$(readlink $TARGET 2>/dev/null)" = "$MDEV" ] && rm -f "$TARGET" + done + ;; +esac diff --git a/nvme/nvme-ebs-mdev.conf b/nvme/nvme-ebs-mdev.conf new file mode 100644 index 0000000..c30b6fd --- /dev/null +++ b/nvme/nvme-ebs-mdev.conf @@ -0,0 +1,3 @@ +# ebs nvme links +nvme[0-9]+n[0-9]+.* root:root 0660 */lib/mdev/nvme-ebs-links + diff --git a/variables.yaml-default b/variables.yaml-default index 15c9bba..d9dff95 100644 --- a/variables.yaml-default +++ b/variables.yaml-default @@ -62,6 +62,7 @@ deploy_regions: - "us-west-2" - "ca-central-1" - "eu-central-1" + - "eu-north-1" - "eu-west-1" - "eu-west-2" - "eu-west-3" -- GitLab