Gentoo Archives: gentoo-releng

From: Ben Kohler <bkohler@g.o>
To: gentoo-releng@l.g.o
Subject: [gentoo-releng] [PATCH 1/2] catalyst-auto-amd64*.conf: explicitly list files to be transferred
Date: Fri, 15 May 2020 13:07:26
Message-Id: 20200515130719.3138513-1-bkohler@gentoo.org
1 Previously when we were copying locally there was no (major) problem
2 with wildcarding a long list of files to be transferred after every
3 single spec finishes. Now that we are transferring to a remote host,
4 this results in a lot of wasted transfer time & bandwidth.
5
6 I have adjusted the post_build function to only transfer the new files
7 known to be produced by each spec (which has files meant to be
8 published).
9
10 Signed-off-by: Ben Kohler <bkohler@g.o>
11 ---
12 tools/catalyst-auto-amd64-experimental.conf | 87 ++++++++++++++-------
13 tools/catalyst-auto-amd64.conf | 86 +++++++++++++-------
14 2 files changed, 113 insertions(+), 60 deletions(-)
15
16 diff --git a/tools/catalyst-auto-amd64-experimental.conf b/tools/catalyst-auto-amd64-experimental.conf
17 index e7b1ea79..2aa91a03 100644
18 --- a/tools/catalyst-auto-amd64-experimental.conf
19 +++ b/tools/catalyst-auto-amd64-experimental.conf
20 @@ -26,6 +26,7 @@ SET_minimal_nomultilib_OPTIONAL_SPECS="stage4-nomultilib-minimal.spec"
21
22 SET_x32_SPECS="stage1-x32.spec stage2-x32.spec stage3-x32.spec"
23
24 +
25 SET_hardened_multilib_SPECS="hardened/stage1.spec hardened/stage2.spec hardened/stage3.spec"
26 SET_hardened_multilib_OPTIONAL_SPECS="hardened/admincd-stage1.spec hardened/admincd-stage2.spec"
27 SET_minimal_hardened_multilib_OPTIONAL_SPECS="hardened/stage4-minimal.spec"
28 @@ -43,7 +44,7 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
29 update_symlinks() {
30 # Symlink the latest stages3 to build from
31 for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do
32 - pushd $d >/dev/null
33 + pushd "${d}" >/dev/null || exit
34 for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do
35 # 20yymmddThhmmssZ
36 # 20yymmddhhmmss
37 @@ -57,11 +58,13 @@ update_symlinks() {
38 <<<"$f")
39 ln -svf "$f" "$of"
40 done
41 - popd >/dev/null
42 + popd >/dev/null || exit
43 done
44 }
45
46 post_build() {
47 + local set=$1 spec=$2
48 +
49 cmd=(
50 rsync
51 -e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no'
52 @@ -80,34 +83,58 @@ post_build() {
53 DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened
54 ;;
55 esac
56 - pushd ${BUILD_SRCDIR_BASE}/default >/dev/null
57 - mkdir -p ${TMPDIR}/empty
58 - "${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT}
59 - for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
60 - if [ -f $file ]; then
61 - "${cmd[@]}" ${file}* ${DEST_DEFAULT}
62 - fi
63 - done
64 - if [ -f *${DATESTAMP}*.iso ]; then
65 - "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT}
66 - elif [ -f *${TIMESTAMP}*.iso ]; then
67 - "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT}
68 - fi
69 - popd >/dev/null
70 -
71 - pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null
72 - "${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED}
73 - for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
74 - if [ -f $file ]; then
75 - "${cmd[@]}" $file* ${DEST_HARDENED}
76 - fi
77 - done
78 - if [ -f *${DATESTAMP}*.iso ]; then
79 - "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED}
80 - elif [ -f *${TIMESTAMP}*.iso ]; then
81 - "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED}
82 - fi
83 - popd >/dev/null
84 +
85 + pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit
86 + case ${spec} in
87 + stage3.spec)
88 + "${cmd[@]}" stage3-amd64-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
89 + ;;
90 + installcd-stage2-minimal.spec)
91 + "${cmd[@]}" install-amd64-minimal-${TIMESTAMP}.iso* "${DEST_DEFAULT}"
92 + ;;
93 + stage4-minimal.spec)
94 + "${cmd[@]}" stage4-amd64-minimal-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
95 + ;;
96 + stage3-nomultilib.spec)
97 + "${cmd[@]}" stage3-amd64-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
98 + ;;
99 + stage4-nomultilib-minimal.spec)
100 + "${cmd[@]}" stage4-amd64-minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
101 + ;;
102 + stage3-x32.spec)
103 + "${cmd[@]}" stage3-x32-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
104 + ;;
105 + esac
106 + popd >/dev/null || exit
107 +
108 + pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit
109 + case ${spec} in
110 + hardened/stage3.spec)
111 + "${cmd[@]}" stage3-amd64-hardened-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
112 + ;;
113 + hardened/admincd-stage2.spec)
114 + "${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}"
115 + ;;
116 + hardened/stage4-minimal.spec)
117 + "${cmd[@]}" stage4-amd64-hardened+minimal-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
118 + ;;
119 + hardened/stage3-nomultilib.spec)
120 + "${cmd[@]}" stage3-amd64-hardened+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
121 + ;;
122 + hardened/stage4-nomultilib-minimal.spec)
123 + "${cmd[@]}" stage4-amd64-hardened+minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
124 + ;;
125 + hardened/stage3-selinux.spec)
126 + "${cmd[@]}" stage3-amd64-hardened-selinux-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
127 + ;;
128 + hardened/admincd-stage2-selinux.spec)
129 + "${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}"
130 + ;;
131 + hardened/stage3-selinux-nomultilib.spec)
132 + "${cmd[@]}" stage3-amd64-hardened-selinux+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
133 + ;;
134 + esac
135 + popd >/dev/null || exit
136 }
137
138 # vim:ft=sh:
139 diff --git a/tools/catalyst-auto-amd64.conf b/tools/catalyst-auto-amd64.conf
140 index 0b7e9b2f..585c22c6 100644
141 --- a/tools/catalyst-auto-amd64.conf
142 +++ b/tools/catalyst-auto-amd64.conf
143 @@ -44,7 +44,7 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]"
144 update_symlinks() {
145 # Symlink the latest stages3 to build from
146 for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do
147 - pushd $d >/dev/null
148 + pushd "${d}" >/dev/null || exit
149 for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do
150 # 20yymmddThhmmssZ
151 # 20yymmddhhmmss
152 @@ -58,11 +58,13 @@ update_symlinks() {
153 <<<"$f")
154 ln -svf "$f" "$of"
155 done
156 - popd >/dev/null
157 + popd >/dev/null || exit
158 done
159 }
160
161 post_build() {
162 + local set=$1 spec=$2
163 +
164 cmd=(
165 rsync
166 -e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no'
167 @@ -81,34 +83,58 @@ post_build() {
168 DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened
169 ;;
170 esac
171 - pushd ${BUILD_SRCDIR_BASE}/default >/dev/null
172 - mkdir -p ${TMPDIR}/empty
173 - "${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT}
174 - for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
175 - if [ -f $file ]; then
176 - "${cmd[@]}" ${file}* ${DEST_DEFAULT}
177 - fi
178 - done
179 - if [ -f *${DATESTAMP}*.iso ]; then
180 - "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT}
181 - elif [ -f *${TIMESTAMP}*.iso ]; then
182 - "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT}
183 - fi
184 - popd >/dev/null
185 -
186 - pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null
187 - "${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED}
188 - for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do
189 - if [ -f $file ]; then
190 - "${cmd[@]}" $file* ${DEST_HARDENED}
191 - fi
192 - done
193 - if [ -f *${DATESTAMP}*.iso ]; then
194 - "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED}
195 - elif [ -f *${TIMESTAMP}*.iso ]; then
196 - "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED}
197 - fi
198 - popd >/dev/null
199 +
200 + pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit
201 + case ${spec} in
202 + stage3.spec)
203 + "${cmd[@]}" stage3-amd64-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
204 + ;;
205 + installcd-stage2-minimal.spec)
206 + "${cmd[@]}" install-amd64-minimal-${TIMESTAMP}.iso* "${DEST_DEFAULT}"
207 + ;;
208 + stage4-minimal.spec)
209 + "${cmd[@]}" stage4-amd64-minimal-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
210 + ;;
211 + stage3-nomultilib.spec)
212 + "${cmd[@]}" stage3-amd64-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
213 + ;;
214 + stage4-nomultilib-minimal.spec)
215 + "${cmd[@]}" stage4-amd64-minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
216 + ;;
217 + stage3-x32.spec)
218 + "${cmd[@]}" stage3-x32-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}"
219 + ;;
220 + esac
221 + popd >/dev/null || exit
222 +
223 + pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit
224 + case ${spec} in
225 + hardened/stage3.spec)
226 + "${cmd[@]}" stage3-amd64-hardened-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
227 + ;;
228 + hardened/admincd-stage2.spec)
229 + "${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}"
230 + ;;
231 + hardened/stage4-minimal.spec)
232 + "${cmd[@]}" stage4-amd64-hardened+minimal-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
233 + ;;
234 + hardened/stage3-nomultilib.spec)
235 + "${cmd[@]}" stage3-amd64-hardened+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
236 + ;;
237 + hardened/stage4-nomultilib-minimal.spec)
238 + "${cmd[@]}" stage4-amd64-hardened+minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
239 + ;;
240 + hardened/stage3-selinux.spec)
241 + "${cmd[@]}" stage3-amd64-hardened-selinux-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
242 + ;;
243 + hardened/admincd-stage2-selinux.spec)
244 + "${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}"
245 + ;;
246 + hardened/stage3-selinux-nomultilib.spec)
247 + "${cmd[@]}" stage3-amd64-hardened-selinux+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}"
248 + ;;
249 + esac
250 + popd >/dev/null || exit
251 }
252
253 # vim:ft=sh:
254 --
255 2.26.2

Replies