[gentoo-catalyst] catalyst changes for improving automation

[Matt Turner]

The catalyst-auto automation scripts live in a repo separate from
catalyst. That increases the difficulty of changing catalyst's
interface, and it doesn't seem to offer any advantages otherwise.
(Keeping build specs in a separate repo allows them to be updated
independent of catalyst and that is valuable). Additionally, since the
primary way catalyst is used is via this automation, it makes sense to
support this workflow in catalyst directly.

But to get there, there are some changes to catalyst that I think are
improvements on their own and simplify the path to integrating
automation capabilities directly into catalyst. That's what I'd like
to discuss here.

I'd like to:

 1) Replace the custom .spec file format with TOML

 2) Combine .spec file sequences (e.g., stage1 -> stage2 -> stage3 ->
livecd-stage1 -> livecd-stage2) into a single file. I suggest naming
this a ".build" file. This will also allow us to remove the redundant
information that currently has to be specified in stage1.spec,
stage2.spec, stage3.spec, like rel_type, version, profile, etc. It
also means that we remove the nonsensical ability to change settings
from one stage to the next that should not change (e.g., rel_type,
version).

 3) Add ability to denote which stage builds produce artifacts we care
about (and want to save and/or upload) and which are just temporary.
If they're temporary (e.g., a stage1 build) we can delete the artifact
after the build sequence has no further use of it, and we can skip
compressing the result, etc.


To that end, I'm starting by figuring out what I would like the new
spec file format to look like. Below are some open questions and then
a strawman new-style spec file.

• The .spec files in releng.git are really templates that are not
directly usable without sed'ing @REPO_DIR@ and @TIMESTAMP@. It would
be nice if they were directly usable as that would reduce confusion
from users.
  • Can we make them directly usable?
  • Perhaps we can make catalyst handle the replacements directly?
    • Calculating @TIMESTAMP@ is trivially doable—we do it today (see below)
    • We could configure @REPO_DIR@ in catalyst.conf and let catalyst
do the replacement, or we could just make the field relative to some
path specified in catalyst.conf?

• In the current automation scripts, we generate a value for
@TIMESTAMP@ from the git HEAD used in creating the snapshot.
  • Would be nice to remove the dependence on the squashfs snapshot
generation—not difficult to do

• Can we generate and upload a .build file with replacements done to
make stage builds more easily reproducible? Seems easy.

Thoughts? Are there advantages to the current system I'm not considering?

ppc.build:

[build]
arch = "ppc"
subarch = "ppc"

version = "@TIMESTAMP@"
rel_type = "default"
profile = "default/linux/powerpc/ppc32/17.0"
source_subpath = "default/stage3-ppc-latest"
portage_confdir = "@REPO_DIR@/releases/portage/stages"

[build.snapshot]
snapshot = "@TIMESTAMP@"

[build.stage1]
update_seed = "yes"
update_seed_command = "--update --deep --newuse @world"

[build.stage2]
skip = true # Stage3 will be built directly from stage1

[build.stage3]
compression = "xz"

[build.iso]
portage_confdir = "@REPO_DIR@/releases/portage/isos"

[build.iso.stage1]
use = [
    "compile-locales",
    "fbcon",
    "ipv6",
    "livecd",
    "modules",
    "ncurses",
    "nls",
    "nptl",
    "pam",
    "readline",
    "socks5",
    "ssl",
    "static-libs",
    "unicode",
    "xml",
]

packages = [
    "app-admin/pwgen",
    "app-admin/syslog-ng",
    "app-arch/unzip",
    "app-crypt/gnupg",
    "app-laptop/pbbuttonsd",
    "app-misc/livecd-tools",
    "app-misc/screen",
    "app-portage/mirrorselect",
    "app-text/wgetpaste",
    "net-analyzer/tcptraceroute",
    "net-analyzer/traceroute",
    "net-dialup/mingetty",
    "net-dialup/pptpclient",
    "net-dialup/rp-pppoe",
    "net-fs/cifs-utils",
    "net-fs/nfs-utils",
    "net-irc/irssi",
    "net-misc/dhcpcd",
    "net-misc/iputils",
    "net-misc/ntp",
    "net-misc/openssh",
    "net-misc/rdate",
    "net-misc/rsync",
    "net-wireless/wireless-tools",
    "net-wireless/wpa_supplicant",
    "sys-apps/busybox",
    "sys-apps/ethtool",
    "sys-apps/fxload",
    "sys-apps/hdparm",
    "sys-apps/hwsetup",
    "sys-apps/ibm-powerpc-utils",
    "sys-apps/iproute2",
    "sys-apps/lm-sensors",
    "sys-apps/memtester",
    "sys-apps/pcmciautils",
    "sys-apps/powerpc-utils",
    "sys-apps/sdparm",
    "sys-block/parted",
    "sys-boot/grub",
    "sys-firmware/b43-firmware",
    "sys-firmware/b43legacy-firmware",
    "sys-fs/btrfs-progs",
    "sys-fs/dosfstools",
    "sys-fs/e2fsprogs",
    "sys-fs/hfsplusutils",
    "sys-fs/hfsutils",
    "sys-fs/iprutils",
    "sys-fs/jfsutils",
    "sys-fs/lvm2",
    "sys-fs/mac-fdisk",
    "sys-fs/mdadm",
    "sys-fs/ntfs3g",
    "sys-fs/reiserfsprogs",
    "sys-fs/xfsprogs",
    "sys-libs/gpm",
    "www-client/links",
]

[build.iso.stage2]
fstype = "squashfs"
iso = "/var/tmp/catalyst/builds/default/install-powerpc-minimal-@TIMESTAMP@.iso"
type = "gentoo-release-minimal"

rcadd = [
    ["pbbuttonsd", "default"],
]

unmerge = [
    "app-admin/eselect",
    "app-admin/eselect-ctags",
    "app-admin/eselect-vi",
    "app-admin/perl-cleaner",
    "app-admin/python-updater",
    "app-arch/cpio",
    "dev-libs/gmp",
    "dev-libs/libxml2",
    "dev-libs/mpfr",
    "dev-libs/popt",
    "dev-python/pycrypto",
    "dev-util/pkgconfig",
    "perl-core/PodParser",
    "perl-core/Test-Harness",
    "sys-apps/debianutils",
    "sys-apps/diffutils",
    "sys-apps/groff",
    "sys-apps/man-db",
    "sys-apps/man-pages",
    "sys-apps/miscfiles",
    "sys-apps/sandbox",
    "sys-apps/texinfo",
    "sys-devel/autoconf",
    "sys-devel/autoconf-wrapper",
    "sys-devel/automake",
    "sys-devel/automake-wrapper",
    "sys-devel/binutils",
    "sys-devel/binutils-config",
    "sys-devel/bison",
    "sys-devel/flex",
    "sys-devel/gcc",
    "sys-devel/gcc-config",
    "sys-devel/gettext",
    "sys-devel/gnuconfig",
    "sys-devel/libtool",
    "sys-devel/m4",
    "sys-devel/make",
    "sys-devel/patch",
    "sys-libs/db",
    "sys-libs/gdbm",
    "sys-libs/libkudzu",
    "sys-kernel/genkernel",
    "sys-kernel/linux-headers",
]

empty = [
    "/boot",
    "/boot/initr*",
    "/boot/kernel*",
    "/boot/System*",
    "/etc/*-",
    "/etc/cron.daily",
    "/etc/cron.hourly",
    "/etc/cron.monthly",
    "/etc/cron.weekly",
    "/etc/default/audioctl",
    "/etc/dispatch-conf.conf",
    "/etc/env.d/05binutils",
    "/etc/env.d/05gcc",
    "/etc/etc-update.conf",
    "/etc/genkernel.conf",
    "/etc/hosts.bck",
    "/etc/issue*",
    "/etc/logrotate.d",
    "/etc/make.conf*",
    "/etc/make.globals",
    "/etc/make.profile",
    "/etc/man.conf",
    "/etc/modules.autoload.d",
    "/etc/*.old",
    "/etc/resolv.conf",
    "/etc/runlevels/single",
    "/etc/skel",
    "/lib64/dev-state",
    "/lib64/udev-state",
    "/lib*/*.a",
    "/lib*/cpp",
    "/lib/dev-state",
    "/lib*/*.la",
    "/lib/udev-state",
    "/root/.bash_history",
    "/root/.ccache",
    "/root/.viminfo",
    "/sbin/fsck.cramfs",
    "/sbin/fsck.minix",
    "/sbin/mkfs.bfs",
    "/sbin/mkfs.cramfs",
    "/sbin/mkfs.minix",
    "/sbin/*.static",
    "/tmp",
    "/usr/bin/addr2line",
    "/usr/bin/ar",
    "/usr/bin/as",
    "/usr/bin/audioctl",
    "/usr/bin/c++*",
    "/usr/bin/cc",
    "/usr/bin/cjpeg",
    "/usr/bin/cpp",
    "/usr/bin/djpeg",
    "/usr/bin/ebuild",
    "/usr/bin/egencache",
    "/usr/bin/elftoaout",
    "/usr/bin/emerge",
    "/usr/bin/emerge-webrsync",
    "/usr/bin/emirrordist",
    "/usr/bin/f77",
    "/usr/bin/g++*",
    "/usr/bin/g77",
    "/usr/bin/gcc*",
    "/usr/bin/genkernel",
    "/usr/bin/gprof",
    "/usr/bin/jpegtran",
    "/usr/bin/ld",
    "/usr/bin/libpng*",
    "/usr/bin/nm",
    "/usr/bin/objcopy",
    "/usr/bin/objdump",
    "/usr/bin/piggyback*",
    "/usr/bin/portageq",
    "/usr/bin/powerpc64-unknown-linux-gnu-*",
    "/usr/bin/powerpc-unknown-linux-gnu-*",
    "/usr/bin/ranlib",
    "/usr/bin/readelf",
    "/usr/bin/repoman",
    "/usr/bin/size",
    "/usr/bin/strings",
    "/usr/bin/strip",
    "/usr/bin/tbz2tool",
    "/usr/bin/xpak",
    "/usr/bin/yacc",
    "/usr/diet/include",
    "/usr/diet/man",
    "/usr/include",
    "/usr/lib64/awk",
    "/usr/lib64/ccache",
    "/usr/lib64/gcc-config",
    "/usr/lib64/gconv",
    "/usr/lib64/nfs",
    "/usr/lib64/perl5/site_perl",
    "/usr/lib64/portage",
    "/usr/lib64/python*/test",
    "/usr/lib64/X11/config",
    "/usr/lib64/X11/doc",
    "/usr/lib64/X11/etc",
    "/usr/lib*/*.a",
    "/usr/lib*/gcc-lib/*/*/libgcj*",
    "/usr/lib*/*.la",
    "/usr/lib*/perl5/site_perl",
    "/usr/local",
    "/usr/portage",
    "/usr/powerpc64-unknown-linux-gnu",
    "/usr/powerpc-unknown-linux-gnu",
    "/usr/sbin/archive-conf",
    "/usr/sbin/dispatch-conf",
    "/usr/sbin/emaint",
    "/usr/sbin/env-update",
    "/usr/sbin/etc-update",
    "/usr/sbin/fb*",
    "/usr/sbin/fixpackages",
    "/usr/sbin/quickpkg",
    "/usr/sbin/regenworld",
    "/usr/share/aclocal",
    "/usr/share/baselayout",
    "/usr/share/binutils-data",
    "/usr/share/consolefonts/1*",
    "/usr/share/consolefonts/7*",
    "/usr/share/consolefonts/8*",
    "/usr/share/consolefonts/9*",
    "/usr/share/consolefonts/a*",
    "/usr/share/consolefonts/A*",
    "/usr/share/consolefonts/c*",
    "/usr/share/consolefonts/C*",
    "/usr/share/consolefonts/dr*",
    "/usr/share/consolefonts/E*",
    "/usr/share/consolefonts/g*",
    "/usr/share/consolefonts/G*",
    "/usr/share/consolefonts/i*",
    "/usr/share/consolefonts/k*",
    "/usr/share/consolefonts/l*",
    "/usr/share/consolefonts/L*",
    "/usr/share/consolefonts/M*",
    "/usr/share/consolefonts/partialfonts",
    "/usr/share/consolefonts/r*",
    "/usr/share/consolefonts/R*",
    "/usr/share/consolefonts/s*",
    "/usr/share/consolefonts/t*",
    "/usr/share/consolefonts/v*",
    "/usr/share/consoletrans",
    "/usr/share/dict",
    "/usr/share/doc",
    "/usr/share/emacs",
    "/usr/share/et",
    "/usr/share/gcc-data",
    "/usr/share/genkernel",
    "/usr/share/gettext",
    "/usr/share/glib-2.0",
    "/usr/share/gnuconfig",
    "/usr/share/gtk-doc",
    "/usr/share/i18n",
    "/usr/share/info",
    "/usr/share/lcms",
    "/usr/share/libtool",
    "/usr/share/locale",
    "/usr/share/man",
    "/usr/share/misc/*.old",
    "/usr/share/rfc",
    "/usr/share/ss",
    "/usr/share/state",
    "/usr/share/texinfo",
    "/usr/share/unimaps",
    "/usr/share/zoneinfo",
    "/usr/src",
    "/var/cache",
    "/var/empty",
    "/var/lib/portage",
    "/var/log",
    "/var/spool",
    "/var/state",
    "/var/tmp",
]

# Just for bootloader ordering purposes
kernel = ["ppc64", "ppc32", "ibmpower",]

[build.iso.stage2.kernel.ibmpower]
sources = "sys-kernel/gentoo-sources"
config = "../../../kconfig/powerpc/installcd-ibm-4.19.config"
console = ["ttyS0,9600", "hvc0", "hvsi0",]
gk_kernargs = [
    "--kernel-cc='gcc -m64'",
    "--kernel-ld='ld -m elf64ppc'",
    "--kernel-as='as -a64'",
]

[build.iso.stage2.kernel.ppc64]
sources = "sys-kernel/gentoo-sources"
config = "../../../kconfig/powerpc/ppc64.config"
console = ["ttyS0,57600",]
gk_kernargs = [
    "--kernel-cc='gcc -m64'",
    "--kernel-ld='ld -m elf64ppc'",
    "--kernel-as='as -a64'",
]

[build.iso.stage2.kernel.ppc32]
sources = "sys-kernel/gentoo-sources"
config = "../../../kconfig/powerpc/ppc32.config"