[gentoo-releng] [PATCH 2/2] catalyst-auto-x86*.conf: explicitly list files to be transferred

[Ben Kohler]

Previously when we were copying locally there was no (major) problem
with wildcarding a long list of files to be transferred after every
single spec finishes.  Now that we are transferring to a remote host,
this results in a lot of wasted transfer time & bandwidth.

I have adjusted the post_build function to only transfer the new files
known to be produced by each spec (which has files meant to be
published).

Signed-off-by: Ben Kohler <bkohler@g.o>
---
 tools/catalyst-auto-x86-experimental.conf | 71 +++++++++++++----------
 tools/catalyst-auto-x86.conf              | 57 +++++++++---------
 2 files changed, 67 insertions(+), 61 deletions(-)

diff --git a/tools/catalyst-auto-x86-experimental.conf b/tools/catalyst-auto-x86-experimental.conf
index 3fd60287..061cab92 100644
--- a/tools/catalyst-auto-x86-experimental.conf
+++ b/tools/catalyst-auto-x86-experimental.conf
@@ -30,17 +30,27 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
 update_symlinks() {
 	# Symlink the latest stages3 to build from
 	for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do
-		pushd $d >/dev/null
+		pushd "${d}" >/dev/null || exit
 		for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do
-			of=${f/20[0-9][0-9].[0-9]/latest} # for 20XX.Y stuff
-			of=${of/20[0-9][0-9][0-1][0-9][0-9][0-9]/latest} # for 20YYMMDD stuff
-			ln -sf $f $of
+			# 20yymmddThhmmssZ
+			# 20yymmddhhmmss
+			# 20yymmdd
+			# 20yy.n
+			of=$(perl -p \
+				-e 's/20\d{6}T\d{6}Z/latest/g;' \
+				-e 's/20\d{6}\d{6}/latest/g;' \
+				-e 's/20\d{6}/latest/g;' \
+				-e 's/20\d{2}\.\d{2}/latest/g;' \
+				<<<"$f")
+			ln -svf "$f" "$of"
 		done
-		popd >/dev/null
+		popd >/dev/null || exit
 	done
 }
 
 post_build() {
+	local set=$1 spec=$2
+
 	cmd=(
 		rsync
 		-e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no'
@@ -59,34 +69,31 @@ post_build() {
 			DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened
 			;;
 	esac
-	pushd ${BUILD_SRCDIR_BASE}/default >/dev/null
-	mkdir -p ${TMPDIR}/empty
-	"${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT}
-	for file in $(ls stage{3,4}*{${DATESTAMP},${TIMESTAMP}}*${EXTENSIONS} ); do
-		if [ -f $file ]; then
-			"${cmd[@]}" ${file}* ${DEST_DEFAULT}
-		fi
-	done
-	if [ -f *${DATESTAMP}*.iso ]; then
-		"${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT}
-	elif [ -f *${TIMESTAMP}*.iso ]; then
-		"${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT}
-	fi
-	popd >/dev/null
 
-	pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null
-	"${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED}
-	for file in $(ls stage{3,4}*{${DATESTAMP},${TIMESTAMP}}*${EXTENSIONS} ); do
-		if [ -f $file ]; then
-			"${cmd[@]}" $file* ${DEST_HARDENED}
-		fi
-	done
-	if [ -f *${DATESTAMP}*.iso ]; then
-		"${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED}
-	elif [ -f *${TIMESTAMP}*.iso ]; then
-		"${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED}
-	fi
-	popd >/dev/null
+	pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit
+	case ${spec} in
+	stage3.spec)
+		"${cmd[@]}" stage3-i486-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+		;;
+	installcd-stage2-minimal.spec)
+		"${cmd[@]}" install-x86-minimal-${TIMESTAMP}.iso* "${DEST_DEFAULT}"
+		;;
+	i686/stage3.spec)
+		"${cmd[@]}" stage3-i686-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+		;;
+	esac
+	popd >/dev/null || exit
+
+	pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit
+	case ${spec} in
+	hardened/stage3.spec)
+		"${cmd[@]}" stage3-i686-hardened-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+		;;
+	hardened/admincd-stage2.spec)
+		"${cmd[@]}" admincd-x86-${TIMESTAMP}.iso* "${DEST_HARDENED}"
+		;;
+	esac
+	popd >/dev/null || exit
 }
 
 # vim:ft=sh:
diff --git a/tools/catalyst-auto-x86.conf b/tools/catalyst-auto-x86.conf
index 27637e59..e778ce8e 100644
--- a/tools/catalyst-auto-x86.conf
+++ b/tools/catalyst-auto-x86.conf
@@ -30,7 +30,7 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
 update_symlinks() {
 	# Symlink the latest stages3 to build from
 	for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do
-		pushd $d >/dev/null
+		pushd "${d}" >/dev/null || exit
 		for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do
 			# 20yymmddThhmmssZ
 			# 20yymmddhhmmss
@@ -44,11 +44,13 @@ update_symlinks() {
 				<<<"$f")
 			ln -svf "$f" "$of"
 		done
-		popd >/dev/null
+		popd >/dev/null || exit
 	done
 }
 
 post_build() {
+	local set=$1 spec=$2
+
 	cmd=(
 		rsync
 		-e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no'
@@ -67,34 +69,31 @@ post_build() {
 			DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened
 			;;
 	esac
-	pushd ${BUILD_SRCDIR_BASE}/default >/dev/null
-	mkdir -p ${TMPDIR}/empty
-	"${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT}
-	for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
-		if [ -f $file ]; then
-			"${cmd[@]}" ${file}* ${DEST_DEFAULT}
-		fi
-	done
-	if [ -f *${DATESTAMP}*.iso ]; then
-		"${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT}
-	elif [ -f *${TIMESTAMP}*.iso ]; then
-		"${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT}
-	fi
-	popd >/dev/null
 
-	pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null
-	"${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED}
-	for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
-		if [ -f $file ]; then
-			"${cmd[@]}" $file* ${DEST_HARDENED}
-		fi
-	done
-	if [ -f *${DATESTAMP}*.iso ]; then
-		"${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED}
-	elif [ -f *${TIMESTAMP}*.iso ]; then
-		"${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED}
-	fi
-	popd >/dev/null
+	pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit
+	case ${spec} in
+	stage3.spec)
+		"${cmd[@]}" stage3-i486-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+		;;
+	installcd-stage2-minimal.spec)
+		"${cmd[@]}" install-x86-minimal-${TIMESTAMP}.iso* "${DEST_DEFAULT}"
+		;;
+	i686/stage3.spec)
+		"${cmd[@]}" stage3-i686-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
+		;;
+	esac
+	popd >/dev/null || exit
+
+	pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit
+	case ${spec} in
+	hardened/stage3.spec)
+		"${cmd[@]}" stage3-i686-hardened-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
+		;;
+	hardened/admincd-stage2.spec)
+		"${cmd[@]}" admincd-x86-${TIMESTAMP}.iso* "${DEST_HARDENED}"
+		;;
+	esac
+	popd >/dev/null || exit
 }
 
 # vim:ft=sh:
-- 
2.26.2