[gentoo-commits] repo/proj/prefix:master commit in: scripts/auto-bootstraps/

[Fabian Groffen]

commit:     b7c0b89992e7b3673ad3e3ba667b81ce9868b69c
Author:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
AuthorDate: Thu Feb 21 11:37:09 2019 +0000
Commit:     Fabian Groffen <grobian <AT> gentoo <DOT> org>
CommitDate: Thu Feb 21 11:37:09 2019 +0000
URL:        https://gitweb.gentoo.org/repo/proj/prefix.git/commit/?id=b7c0b899

scripts/auto-bootstraps: scripts to perform unattended bootstraps

This includes the scripts that generate the results output of
bootstrap.prefix.bitzolder.nl.

Signed-off-by: Fabian Groffen <grobian <AT> gentoo.org>

 scripts/auto-bootstraps/analyse_result.py   | 178 ++++++++++++++++++++++++++++
 scripts/auto-bootstraps/dobootstrap         | 167 ++++++++++++++++++++++++++
 scripts/auto-bootstraps/process_uploads.sh  |  60 ++++++++++
 scripts/auto-bootstraps/update_distfiles.py |  29 +++++
 4 files changed, 434 insertions(+)

diff --git a/scripts/auto-bootstraps/analyse_result.py b/scripts/auto-bootstraps/analyse_result.py
new file mode 100755
index 0000000000..885c7fc9e7
--- /dev/null
+++ b/scripts/auto-bootstraps/analyse_result.py
@@ -0,0 +1,178 @@
+#!/usr/bin/env python3
+
+import os
+import glob
+import re
+import time
+import html
+
+resultsdir='./results'
+
+def find_last_stage(d):
+    """
+    Returns the last stage worked on.
+    Bootstraps define explicitly stages 1, 2 and 3, we define some more
+    on top of those as follows:
+      0 - bootstrap didn't even start (?!?) or unknown status
+      1 - stage 1 failed
+      2 - stage 2 failed
+      3 - stage 3 failed
+      4 - emerge -e world failed
+      5 - finished successfully
+    """
+
+    def stage_success(stagelog):
+        with open(stagelog, 'rb') as f:
+            line = f.readlines()[-1]
+            res = re.match(r'^\* stage[123] successfully finished',
+                    line.decode('utf-8', 'ignore'))
+            return res is not None
+
+    if not os.path.exists(os.path.join(d, '.stage1-finished')):
+        log = os.path.join(d, 'stage1.log')
+        if not os.path.exists(log):
+            return 0 # nothing exists, assume not started
+        if not stage_success(log):
+            return 1
+
+    if not os.path.exists(os.path.join(d, '.stage2-finished')):
+        log = os.path.join(d, 'stage2.log')
+        if not os.path.exists(log) or not stage_success(log):
+            return 2 # stage1 was success, so 2 must have failed
+
+    if not os.path.exists(os.path.join(d, '.stage3-finished')):
+        log = os.path.join(d, 'stage3.log')
+        if not os.path.exists(log) or not stage_success(log):
+            return 3 # stage2 was success, so 3 must have failed
+
+    # if stage 3 was success, we went onto emerge -e system, if that
+    # failed, portage would have left a build.log behind
+    logs = glob.glob(d + "/portage/*/*/temp/build.log")
+    if len(logs) > 0:
+        return 4
+
+    # ok, so it must have been all good then
+    return 5
+
+def get_err_reason(arch, dte, err):
+    rdir = os.path.join(resultsdir, arch, '%d' % dte)
+
+    if err == 0:
+        return "bootstrap failed to start"
+    if err >= 1 and err <= 3:
+        stagelog = os.path.join(rdir, 'stage%d.log' % err)
+        if os.path.exists(stagelog):
+            line = None
+            with open(stagelog, 'rb') as f:
+                errexp = re.compile(r'^( \* (ERROR:|Fetch failed for)|emerge: there are no) ')
+                for line in f:
+                    res = errexp.match(line.decode('utf-8', 'ignore'))
+                    if res:
+                        break
+            if not line:
+                return '<a href="%s/stage%d.log">stage %d</a> failed' % \
+                        (os.path.join(arch, '%d' % dte), err, err)
+            return '<a href="%s/stage%d.log">stage %d</a> failed<br />%s' % \
+                        (os.path.join(arch, '%d' % dte), err, err, \
+                         html.escape(line.decode('utf-8', 'ignore')))
+        else:
+            return 'stage %d did not start' % err
+    if err == 4:
+        msg = "'emerge -e system' failed while emerging"
+        logs = glob.glob(rdir + "/portage/*/*/temp/build.log")
+        for log in logs:
+            cat, pkg = log.split('/')[-4:-2]
+            msg = msg + ' <a href="%s/temp/build.log">%s/%s</a>' % \
+                    (os.path.join(arch, '%d' % dte, "portage", cat, pkg), \
+                     cat, pkg)
+        return msg
+
+def analyse_arch(d):
+    last_fail = None
+    last_succ = None
+    fail_state = None
+    with os.scandir(d) as it:
+        for f in sorted(it, key=lambda x: (x.is_dir(), x.name), reverse=True):
+            if not f.is_dir(follow_symlinks=False):
+                continue
+            date = int(f.name)
+            res = find_last_stage(os.path.join(d, f.name))
+            if res == 5:
+                if not last_succ:
+                    last_succ = date
+            elif not last_fail:
+                last_fail = date
+                fail_state = res
+            if last_succ and last_fail:
+                break
+
+    return (last_fail, fail_state, last_succ)
+
+archs = {}
+with os.scandir(resultsdir) as it:
+    for f in sorted(it, key=lambda x: (x.is_dir(), x.name)):
+        if not f.is_dir(follow_symlinks=False):
+            continue
+        arch = f.name
+        fail, state, suc = analyse_arch(os.path.join(resultsdir, arch))
+        archs[arch] = (fail, state, suc)
+        if not suc:
+            color = '\033[1;31m'  # red
+        elif fail and suc < fail:
+            color = '\033[1;33m'  # yellow
+        else:
+            color = '\033[1;32m'  # green
+        endc = '\033[0m'
+        print("%s%24s: suc %8s  fail %8s%s" % (color, arch, suc, fail, endc))
+
+# generate html edition
+with open(os.path.join(resultsdir, 'index.html'), "w") as h:
+    h.write("<html>")
+    h.write("<head><title>Gentoo Prefix bootstrap results</title></head>")
+    h.write("<body>")
+    h.write("<h2>Gentoo Prefix bootstraps</h2>")
+    h.write('<table border="1px">')
+    h.write("<th>architecture</th>")
+    h.write("<th>last successful run</th><th>last failed run</th>")
+    h.write("<th>failure</th>")
+    for arch in archs:
+        fail, errcode, suc = archs[arch]
+        if not suc:
+            state = 'red'
+        elif fail and suc < fail:
+            state = 'orange'
+        else:
+            state = 'limegreen'
+
+        h.write('<tr>')
+
+        h.write('<td bgcolor="%s" nowrap="nowrap">' % state)
+        h.write(arch)
+        h.write("</td>")
+
+        h.write("<td>")
+        if suc:
+            h.write('<a href="%s/%s">%s</a>' % (arch, suc, suc))
+        else:
+            h.write('<i>never</i>')
+        h.write("</td>")
+
+        h.write("<td>")
+        if fail:
+            h.write('<a href="%s/%s">%s</a>' % (arch, fail, fail))
+        else:
+            h.write('<i>never</i>')
+        h.write("</td>")
+
+        h.write("<td>")
+        if fail and (not suc or fail > suc):
+            h.write(get_err_reason(arch, fail, errcode))
+        h.write("</td>")
+
+        h.write("</tr>")
+    h.write("</table>")
+    now = time.strftime('%Y-%m-%d %H:%M', time.gmtime())
+    h.write("<p><i>generated: %s</i></p>" % now) 
+    h.write("<p>See also <a href='https://dev.azure.com/12719821/12719821/_build?definitionId=6'>awesomebytes</a></p>")
+    h.write("</body>")
+    h.write("</html>")

diff --git a/scripts/auto-bootstraps/dobootstrap b/scripts/auto-bootstraps/dobootstrap
new file mode 100755
index 0000000000..521f644acf
--- /dev/null
+++ b/scripts/auto-bootstraps/dobootstrap
@@ -0,0 +1,167 @@
+#!/usr/bin/env bash
+
+BOOTSTRAP="${BASH_SOURCE[0]%/*}/bootstrap-prefix.sh"
+BOOTURL="http://rsync.prefix.bitzolder.nl/scripts/bootstrap-prefix.sh"
+UPLOAD="rsync1.prefix.bitzolder.nl::gentoo-portage-bootstraps"
+
+do_fetch() {
+	local FETCHCOMMAND
+	# Try to find a download manager, we only deal with wget,
+	# curl, FreeBSD's fetch and ftp.
+	if [[ x$(type -t wget) == "xfile" ]] ; then
+		FETCH_COMMAND="wget -O -"
+		[[ $(wget -h) == *"--no-check-certificate"* ]] && \
+			FETCH_COMMAND+=" --no-check-certificate"
+	elif [[ x$(type -t curl) == "xfile" ]] ; then
+		FETCH_COMMAND="curl -f -L"
+	else
+		echo "could not download ${1##*/}"
+		exit 1
+	fi
+
+	${FETCH_COMMAND} "${*}" || exit 1
+}
+
+do_prepare() {
+	local bitw=$1
+	local dte=$2
+	local bootstrap
+
+	if [[ -n ${RESUME} && -n ${bitw} && -n ${dte} ]] ; then
+		bootstrap=bootstrap${bitw}-${dte}/bootstrap-prefix.sh
+	elif [[ -n ${DOLOCAL} ]] ; then
+		bootstrap=${BOOTSTRAP}
+	else
+		bootstrap=dobootstrap-do_prepare-$$
+		do_fetch ${BOOTURL} > ${bootstrap}
+	fi
+
+	local chost=$(${BASH} ${bootstrap} chost.guess x)
+	case ${chost} in
+		*86-*)
+			if [[ ${bitw} == 64 ]] ; then
+				chost=x86_64-${chost#*-}
+			else
+				bitw=32
+				chost=i386-${chost#*-}
+			fi
+			;;
+		x86_64-*)
+			if [[ ${bitw} == 32 ]] ; then
+				chost=i386-${chost#*-}
+			else
+				bitw=64
+				chost=x86_64-${chost#*-}
+			fi
+			;;
+		powerpc-*)
+			bitw=32
+			;;
+		sparc-*)
+			if [[ ${bitw} == 64 ]] ; then
+				chost=sparcv9-${chost#*-}
+			else
+				bitw=32
+				chost=sparc-${chost#*-}
+			fi
+			;;
+		sparcv9-*|sparc64-*)
+			if [[ ${bitw} == 32 ]] ; then
+				chost=sparc-${chost#*-}
+			else
+				bitw=64
+				chost=sparcv9-${chost#*-}
+			fi
+			;;
+		*)
+			echo "unhandled CHOST: ${chost}"
+			rm -f dobootstrap-do_prepare-$$
+			exit 1
+			;;
+	esac
+
+	[[ -z ${dte} ]] && dte=$(date "+%Y%m%d")
+	EPREFIX=${PWD}/bootstrap${bitw}-${dte}
+	[[ -n ${OVERRIDE_EPREFIX} ]] && EPREFIX=${OVERRIDE_EPREFIX}
+
+	local bootstrapscript=$(realpath ${BASH_SOURCE[0]} 2>/dev/null)
+	if [[ -z ${bootstrapscript} ]] ; then
+		local b=${BASH_SOURCE[0]}
+		cd "${b%/*}"
+		bootstrapscript=$(pwd -P)/${b##*/}
+	fi
+	echo "EPREFIX=${EPREFIX}"
+	mkdir -p "${EPREFIX}"
+	if [[ ${bootstrap} == dobootstrap-do_prepare-$$ ]] ; then
+		mv "${bootstrap}" "${EPREFIX}"/bootstrap-prefix.sh
+	elif [[ ${bootstrap} != "${EPREFIX}"/bootstrap-prefix.sh ]] ; then
+		cp "${bootstrap}" "${EPREFIX}"/bootstrap-prefix.sh
+	fi
+	cd "${EPREFIX}" || exit 1
+
+	# optional program to keep the machine from sleeping
+	# macOS/BSD: caffeinate
+	keepalive=$(type -P caffeinate)
+	[[ -x ${keepalive} ]] && keepalive+=" -i -m -s" || keepalive=
+
+	env -i \
+		HOME=${EPREFIX} \
+		SHELL=/bin/bash \
+		TERM=${TERM} \
+		USER=${USER} \
+		CHOST=${chost} \
+		GENTOO_MIRRORS="http://distfileslocal/" \
+		EPREFIX=${EPREFIX} \
+		${DOLOCAL+DOLOCAL=1} \
+		${RESUME+RESUME=1} \
+		${LATEST_TREE_YES+LATEST_TREE_YES=1} \
+		${TREE_FROM_SRC+TREE_FROM_SRC=}${TREE_FROM_SRC} \
+		${keepalive} /bin/bash -l -c "${BASH} ${bootstrapscript} bootstrap"
+
+	if [[ -n ${DOPUBLISH} ]] ; then
+		rsync -q /dev/null ${UPLOAD}/${HOSTNAME}-$$/
+		rsync -q /dev/null ${UPLOAD}/${HOSTNAME}-$$/${chost}/
+		rsync -rltv \
+			--exclude=work/ \
+			--exclude=homedir/ \
+			--exclude=files \
+			--exclude=distdir/ \
+			--exclude=image/ \
+			{stage,.stage}* \
+			bootstrap-prefix.sh \
+			startprefix \
+			usr/portage/distfiles \
+			var/tmp/portage \
+			var/log/emerge.log \
+			${UPLOAD}/${HOSTNAME}-$$/${chost}/${dte}/
+		rsync -q /dev/null ${UPLOAD}/${HOSTNAME}-$$/${chost}/${dte}/push-complete/
+	fi
+}
+
+do_bootstrap() {
+	chmod 755 bootstrap-prefix.sh || exit 1
+	${BASH} ./bootstrap-prefix.sh ${EPREFIX} noninteractive
+}
+
+case $1 in
+	bootstrap)
+		do_bootstrap
+		;;
+	local)
+		export DOLOCAL=1
+		do_prepare $2
+		;;
+	resume)
+		export RESUME=1
+		do_prepare "$2" ${3:-${BOOTSTRAP_DATE}}
+		;;
+	*)
+		if [[ ${0} == /net/* ]] ; then
+			echo "internal host, activating local and DOPUBLISH"
+			export DOLOCAL=1
+			export DOPUBLISH=1
+		fi
+		do_prepare $1
+		;;
+esac
+

diff --git a/scripts/auto-bootstraps/process_uploads.sh b/scripts/auto-bootstraps/process_uploads.sh
new file mode 100755
index 0000000000..52bb09ed7f
--- /dev/null
+++ b/scripts/auto-bootstraps/process_uploads.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+UPLOADDIR="./uploads"
+RESULTSDIR="./results"
+
+didsomething=
+for d in ${UPLOADDIR}/* ; do
+	if [[ ! -d "${d}" ]] ; then
+		rm -f "${d}"
+		continue
+	fi
+
+	# structure: randomid/chost/date
+	# chost/date should be the only thing in randomid/ check this
+	set -- "${d}"/*/*
+	if [[ $# -ne 1 ]] || [[ ! -d "$1" ]] ; then
+		rm -Rf "${d}"
+		continue
+	fi
+
+	dir=${1#${d}/}
+	# skip this thing from auto-processing if it is new platform
+	[[ -d ${RESULTSDIR}/${dir%/*} ]] || continue
+	# skip this thing if it already exists
+	[[ -d ${RESULTSDIR}/${dir} ]] && continue
+	# skip this thing if it isn't complete yet
+	[[ -d ${d}/${dir}/push-complete ]] || continue
+
+	# only copy over what we expect, so we leave any uploaded cruft
+	# behind
+	mkdir "${RESULTSDIR}/${dir}"
+	for f in \
+		stage{1,2,3}.log \
+		.stage{1,2,3}-finished \
+		bootstrap-prefix.sh \
+		emerge.log \
+		startprefix \
+		distfiles ;
+	do
+		[[ -e "${d}/${dir}/${f}" ]] && \
+			mv "${d}/${dir}/${f}" "${RESULTSDIR}/${dir}"/
+	done
+	if [[ -e "${d}/${dir}/portage" ]] ; then
+		for pkg in "${d}/${dir}/portage"/*/* ; do
+			w=${pkg#${d}/}
+			mkdir -p "${RESULTSDIR}/${w}"
+			[[ -e "${pkg}"/build-info ]] && \
+				mv "${pkg}"/build-info "${RESULTSDIR}/${w}"/
+			[[ -e "${pkg}"/temp ]] && \
+				mv "${pkg}"/temp "${RESULTSDIR}/${w}"/
+		done
+	fi
+	chmod -R o+rX,go-w "${RESULTSDIR}/${dir}"
+	rm -Rf "${d}"
+
+	[[ -e "${RESULTSDIR}/${dir}"/distfiles ]] && \
+		./update_distfiles.py "${RESULTSDIR}/${dir}"/distfiles > /dev/null
+	didsomething=1
+done
+[[ -n ${didsomething} ]] && ./analyse_result.py > /dev/null

diff --git a/scripts/auto-bootstraps/update_distfiles.py b/scripts/auto-bootstraps/update_distfiles.py
new file mode 100755
index 0000000000..8f44f7fa20
--- /dev/null
+++ b/scripts/auto-bootstraps/update_distfiles.py
@@ -0,0 +1,29 @@
+#!/usr/bin/env python3.6
+
+import hashlib
+import os
+import sys
+
+distfilessrc='./distfiles'
+
+def hash_file(f):
+    hsh = hashlib.new('sha1')
+    with open(f, 'rb') as fle:
+        hsh.update(fle.read())
+    return hsh.hexdigest()
+
+with os.scandir(path=sys.argv[1]) as it:
+    for f in it:
+        if not f.is_file() or f.name.startswith('.'):
+            continue
+        srcfile = os.path.join(sys.argv[1], f.name)
+        h = hash_file(srcfile)
+        distname = os.path.join(distfilessrc,
+                f.name + "@" + h).lower()
+        if os.path.exists(distname):
+            print("DUP %s" % distname.split('/')[-1])
+            os.remove(srcfile)
+            os.link(distname, srcfile, follow_symlinks=False)
+        else:
+            print("NEW %s" % distname.split('/')[-1])
+            os.link(srcfile, distname)