[gentoo-commits] proj/qa-scripts:master commit in: /

[Robin H. Johnson]

commit:     99ceddc02672cbca6e530dbca4cd804e00e4b8d1
Author:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
AuthorDate: Fri May  3 18:26:39 2019 +0000
Commit:     Robin H. Johnson <robbat2 <AT> gentoo <DOT> org>
CommitDate: Fri May  3 18:26:48 2019 +0000
URL:        https://gitweb.gentoo.org/proj/qa-scripts.git/commit/?id=99ceddc0

keyrings: prepare to split out keyring export for faster cycles

Signed-off-by: Robin H. Johnson <robbat2 <AT> gentoo.org>

 create-dev-keyrings.bash                      | 90 +++------------------------
 keyrings-export.bash                          | 33 ++++++++++
 create-dev-keyrings.bash => keyrings.inc.bash | 49 +++------------
 3 files changed, 48 insertions(+), 124 deletions(-)

diff --git a/create-dev-keyrings.bash b/create-dev-keyrings.bash
index 1a9fd76..3f65550 100755
--- a/create-dev-keyrings.bash
+++ b/create-dev-keyrings.bash
@@ -1,91 +1,15 @@
 #!/bin/bash
+# Import key updates from Keyservers
+#
+# TODO:
+# - Turn off export in this script
 
 OUTPUT_DIR=${1:-.}
-
-DEV_BASE='ou=devs,dc=gentoo,dc=org'
-SYSTEM_BASE='ou=system,dc=gentoo,dc=org'
-
-COMMIT_RULE='(&(gentooAccess=git.gentoo.org/repo/gentoo.git)(gentooStatus=active))'
-NONCOMMIT_RULE='(&(!(gentooAccess=git.gentoo.org/repo/gentoo.git))(gentooStatus=active))'
-RETIRED_RULE='(!(gentooStatus=active))'
-
-KS_GENTOO=hkps://keys.gentoo.org/
-KS_SKS=hkps://hkps.pool.sks-keyservers.net/
-
-GPG_TMPDIR=$(mktemp -d)
-clean_tmp() {
-	rm -rf "$GPG_TMPDIR"
-}
-
-# grab_ldap_fingerprints <ldap-rule>
-grab_ldap_fingerprints() {
-	ldapsearch "${@}" -Z gpgfingerprint -LLL |
-		sed -n -e '/^gpgfingerprint: /{s/^.*://;s/ //g;p}' |
-		sort -u |
-		grep -v undefined
-}
-
-# grab_keys <fingerprint>...
-grab_keys() {
-	local retries=0
-	local missing=()
-	local remaining=( "${@}" )
-
-	while :; do
-		timeout 5m  gpg --keyserver $KS_GENTOO -q --recv-keys "${remaining[@]}" || :
-		timeout 20m gpg --keyserver $KS_SKS -q --recv-keys "${remaining[@]}" || :
-		missing=()
-		for key in "${remaining[@]}"; do
-			gpg --list-public "${key}" &>/dev/null || missing+=( "${key}" )
-		done
-
-		[[ ${#missing[@]} -ne 0 ]] || break
-
-		# if we did not make progress, give it a few seconds and retry
-		if [[ ${#missing[@]} -eq ${#remaining[@]} ]]; then
-			if [[ $(( retries++ )) -gt 3 ]]; then
-				echo "Unable to fetch the following keys:"
-				printf '%s\n' "${missing[@]}"
-				break # if we hard-exit, the entire export will fail
-			fi
-			sleep 5
-		fi
-
-		remaining=( "${missing[@]}" )
-	done
-}
-
-# push_keys <fingerprint>...
-push_keys() {
-	# Only send keys that we have
-	local remaining=( $(gpg --with-colon --list-public "${@}" | sed -n '/^pub/{n; /fpr/p }' |cut -d: -f10) )
-	timeout 5m  gpg --keyserver $KS_GENTOO -q --send-keys "${remaining[@]}" || :
-	#timeout 5m  gpg --keyserver $KS_SKS -q --send-keys "${remaining[@]}" || :
-}
-
-export_keys() {
-	DST="$1"
-	TMP="${GPG_TMPDIR}"/$(basename "${DST}")
-	# Must not exist, otherwise GPG will give error
-	[[ -f "${TMP}" ]] && rm -f "${TMP}"
-	# 'gpg --export' returns zero if there was no error with the command itself
-	# If there are no keys in the export set, then it ALSO does not write the destination file
-	# and prints 'gpg: WARNING: nothing exported' to stderr
-	if gpg --output "$TMP" --export "${@}" && test -s "${TMP}"; then
-		chmod a+r "${TMP}"
-		mv "${TMP}" "${DST}"
-	else
-		echo "Unable to export keys to $DST"
-		exit 1
-	fi
-}
+BASEDIR="$(dirname "$0")"
+source "${BASEDIR}"/keyrings.inc.bash
 
 set -e
-
-COMMITTING_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${COMMIT_RULE}") )
-NONCOMMITTING_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${NONCOMMIT_RULE}") )
-RETIRED_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${RETIRED_RULE}") )
-SYSTEM_KEYS=( $(grab_ldap_fingerprints -b "${SYSTEM_BASE}" "${NONCOMMIT_RULE}") )
+export_ldap_data_to_env
 
 grab_keys "${SYSTEM_KEYS[@]}"
 export_keys "${OUTPUT_DIR}"/service-keys.gpg \

diff --git a/keyrings-export.bash b/keyrings-export.bash
new file mode 100755
index 0000000..06f5bab
--- /dev/null
+++ b/keyrings-export.bash
@@ -0,0 +1,33 @@
+#!/bin/bash
+# Export keys to keyrings
+#
+# TODO:
+# - only run the export if there was really a change
+# - requires keeping state to detect changes in keys, there is no usable mtime data in a key itself
+
+OUTPUT_DIR=${1:-.}
+BASEDIR="$(dirname "$0")"
+source "${BASEDIR}"/keyrings.inc.bash
+
+set -e
+export_ldap_data_to_env
+
+export_keys "${OUTPUT_DIR}"/service-keys.gpg \
+	"${SYSTEM_KEYS[@]}"
+
+export_keys "${OUTPUT_DIR}"/committing-devs.gpg \
+	"${COMMITTING_DEVS[@]}"
+
+export_keys "${OUTPUT_DIR}"/active-devs.gpg \
+	"${COMMITTING_DEVS[@]}" \
+	"${NONCOMMITTING_DEVS[@]}"
+
+export_keys "${OUTPUT_DIR}"/retired-devs.gpg \
+	"${RETIRED_DEVS[@]}"
+
+# Everybody together now
+export_keys "${OUTPUT_DIR}"/all-devs.gpg \
+	"${SYSTEM_KEYS[@]}" \
+	"${COMMITTING_DEVS[@]}" \
+	"${NONCOMMITTING_DEVS[@]}" \
+	"${RETIRED_DEVS[@]}"

diff --git a/create-dev-keyrings.bash b/keyrings.inc.bash
old mode 100755
new mode 100644
similarity index 63%
copy from create-dev-keyrings.bash
copy to keyrings.inc.bash
index 1a9fd76..052550d
--- a/create-dev-keyrings.bash
+++ b/keyrings.inc.bash
@@ -1,7 +1,5 @@
 #!/bin/bash
 
-OUTPUT_DIR=${1:-.}
-
 DEV_BASE='ou=devs,dc=gentoo,dc=org'
 SYSTEM_BASE='ou=system,dc=gentoo,dc=org'
 
@@ -16,6 +14,7 @@ GPG_TMPDIR=$(mktemp -d)
 clean_tmp() {
 	rm -rf "$GPG_TMPDIR"
 }
+trap clean_tmp EXIT
 
 # grab_ldap_fingerprints <ldap-rule>
 grab_ldap_fingerprints() {
@@ -80,42 +79,10 @@ export_keys() {
 	fi
 }
 
-set -e
-
-COMMITTING_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${COMMIT_RULE}") )
-NONCOMMITTING_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${NONCOMMIT_RULE}") )
-RETIRED_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${RETIRED_RULE}") )
-SYSTEM_KEYS=( $(grab_ldap_fingerprints -b "${SYSTEM_BASE}" "${NONCOMMIT_RULE}") )
-
-grab_keys "${SYSTEM_KEYS[@]}"
-export_keys "${OUTPUT_DIR}"/service-keys.gpg \
-	"${SYSTEM_KEYS[@]}"
-
-grab_keys "${COMMITTING_DEVS[@]}"
-export_keys "${OUTPUT_DIR}"/committing-devs.gpg \
-	"${COMMITTING_DEVS[@]}"
-
-grab_keys "${NONCOMMITTING_DEVS[@]}"
-export_keys "${OUTPUT_DIR}"/active-devs.gpg \
-	"${COMMITTING_DEVS[@]}" \
-	"${NONCOMMITTING_DEVS[@]}"
-
-# -- not all are on keyservers
-# -- and are unlikely to turn up now
-# -- this needs to fetch from some archive instead
-#grab_keys "${RETIRED_DEVS[@]}"
-export_keys "${OUTPUT_DIR}"/retired-devs.gpg \
-	"${RETIRED_DEVS[@]}"
-
-# Everybody together now
-export_keys "${OUTPUT_DIR}"/all-devs.gpg \
-	"${SYSTEM_KEYS[@]}" \
-	"${COMMITTING_DEVS[@]}" \
-	"${NONCOMMITTING_DEVS[@]}" \
-	"${RETIRED_DEVS[@]}"
-
-# Populate keys.gentoo.org with the keys we have, since they might have come from SKS
-push_keys "${SYSTEM_KEYS[@]}"
-push_keys "${COMMITTING_DEVS[@]}"
-push_keys "${NONCOMMITTING_DEVS[@]}"
-push_keys "${RETIRED_DEVS[@]}"
+# populate common variables
+export_ldap_data_to_env() {
+	export COMMITTING_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${COMMIT_RULE}") )
+	export NONCOMMITTING_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${NONCOMMIT_RULE}") )
+	export RETIRED_DEVS=( $(grab_ldap_fingerprints -b "${DEV_BASE}" "${RETIRED_RULE}") )
+	export SYSTEM_KEYS=( $(grab_ldap_fingerprints -b "${SYSTEM_BASE}" "${NONCOMMIT_RULE}") )
+}