1 |
Previously when we were copying locally there was no (major) problem |
2 |
with wildcarding a long list of files to be transferred after every |
3 |
single spec finishes. Now that we are transferring to a remote host, |
4 |
this results in a lot of wasted transfer time & bandwidth. |
5 |
|
6 |
I have adjusted the post_build function to only transfer the new files |
7 |
known to be produced by each spec (which has files meant to be |
8 |
published). |
9 |
|
10 |
Signed-off-by: Ben Kohler <bkohler@g.o> |
11 |
--- |
12 |
tools/catalyst-auto-amd64-experimental.conf | 87 ++++++++++++++------- |
13 |
tools/catalyst-auto-amd64.conf | 86 +++++++++++++------- |
14 |
2 files changed, 113 insertions(+), 60 deletions(-) |
15 |
|
16 |
diff --git a/tools/catalyst-auto-amd64-experimental.conf b/tools/catalyst-auto-amd64-experimental.conf |
17 |
index e7b1ea79..2aa91a03 100644 |
18 |
--- a/tools/catalyst-auto-amd64-experimental.conf |
19 |
+++ b/tools/catalyst-auto-amd64-experimental.conf |
20 |
@@ -26,6 +26,7 @@ SET_minimal_nomultilib_OPTIONAL_SPECS="stage4-nomultilib-minimal.spec" |
21 |
|
22 |
SET_x32_SPECS="stage1-x32.spec stage2-x32.spec stage3-x32.spec" |
23 |
|
24 |
+ |
25 |
SET_hardened_multilib_SPECS="hardened/stage1.spec hardened/stage2.spec hardened/stage3.spec" |
26 |
SET_hardened_multilib_OPTIONAL_SPECS="hardened/admincd-stage1.spec hardened/admincd-stage2.spec" |
27 |
SET_minimal_hardened_multilib_OPTIONAL_SPECS="hardened/stage4-minimal.spec" |
28 |
@@ -43,7 +44,7 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]" |
29 |
update_symlinks() { |
30 |
# Symlink the latest stages3 to build from |
31 |
for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do |
32 |
- pushd $d >/dev/null |
33 |
+ pushd "${d}" >/dev/null || exit |
34 |
for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do |
35 |
# 20yymmddThhmmssZ |
36 |
# 20yymmddhhmmss |
37 |
@@ -57,11 +58,13 @@ update_symlinks() { |
38 |
<<<"$f") |
39 |
ln -svf "$f" "$of" |
40 |
done |
41 |
- popd >/dev/null |
42 |
+ popd >/dev/null || exit |
43 |
done |
44 |
} |
45 |
|
46 |
post_build() { |
47 |
+ local set=$1 spec=$2 |
48 |
+ |
49 |
cmd=( |
50 |
rsync |
51 |
-e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no' |
52 |
@@ -80,34 +83,58 @@ post_build() { |
53 |
DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened |
54 |
;; |
55 |
esac |
56 |
- pushd ${BUILD_SRCDIR_BASE}/default >/dev/null |
57 |
- mkdir -p ${TMPDIR}/empty |
58 |
- "${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT} |
59 |
- for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do |
60 |
- if [ -f $file ]; then |
61 |
- "${cmd[@]}" ${file}* ${DEST_DEFAULT} |
62 |
- fi |
63 |
- done |
64 |
- if [ -f *${DATESTAMP}*.iso ]; then |
65 |
- "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT} |
66 |
- elif [ -f *${TIMESTAMP}*.iso ]; then |
67 |
- "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT} |
68 |
- fi |
69 |
- popd >/dev/null |
70 |
- |
71 |
- pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null |
72 |
- "${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED} |
73 |
- for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do |
74 |
- if [ -f $file ]; then |
75 |
- "${cmd[@]}" $file* ${DEST_HARDENED} |
76 |
- fi |
77 |
- done |
78 |
- if [ -f *${DATESTAMP}*.iso ]; then |
79 |
- "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED} |
80 |
- elif [ -f *${TIMESTAMP}*.iso ]; then |
81 |
- "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED} |
82 |
- fi |
83 |
- popd >/dev/null |
84 |
+ |
85 |
+ pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit |
86 |
+ case ${spec} in |
87 |
+ stage3.spec) |
88 |
+ "${cmd[@]}" stage3-amd64-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" |
89 |
+ ;; |
90 |
+ installcd-stage2-minimal.spec) |
91 |
+ "${cmd[@]}" install-amd64-minimal-${TIMESTAMP}.iso* "${DEST_DEFAULT}" |
92 |
+ ;; |
93 |
+ stage4-minimal.spec) |
94 |
+ "${cmd[@]}" stage4-amd64-minimal-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" |
95 |
+ ;; |
96 |
+ stage3-nomultilib.spec) |
97 |
+ "${cmd[@]}" stage3-amd64-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" |
98 |
+ ;; |
99 |
+ stage4-nomultilib-minimal.spec) |
100 |
+ "${cmd[@]}" stage4-amd64-minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" |
101 |
+ ;; |
102 |
+ stage3-x32.spec) |
103 |
+ "${cmd[@]}" stage3-x32-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" |
104 |
+ ;; |
105 |
+ esac |
106 |
+ popd >/dev/null || exit |
107 |
+ |
108 |
+ pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit |
109 |
+ case ${spec} in |
110 |
+ hardened/stage3.spec) |
111 |
+ "${cmd[@]}" stage3-amd64-hardened-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" |
112 |
+ ;; |
113 |
+ hardened/admincd-stage2.spec) |
114 |
+ "${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}" |
115 |
+ ;; |
116 |
+ hardened/stage4-minimal.spec) |
117 |
+ "${cmd[@]}" stage4-amd64-hardened+minimal-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" |
118 |
+ ;; |
119 |
+ hardened/stage3-nomultilib.spec) |
120 |
+ "${cmd[@]}" stage3-amd64-hardened+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" |
121 |
+ ;; |
122 |
+ hardened/stage4-nomultilib-minimal.spec) |
123 |
+ "${cmd[@]}" stage4-amd64-hardened+minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" |
124 |
+ ;; |
125 |
+ hardened/stage3-selinux.spec) |
126 |
+ "${cmd[@]}" stage3-amd64-hardened-selinux-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" |
127 |
+ ;; |
128 |
+ hardened/admincd-stage2-selinux.spec) |
129 |
+ "${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}" |
130 |
+ ;; |
131 |
+ hardened/stage3-selinux-nomultilib.spec) |
132 |
+ "${cmd[@]}" stage3-amd64-hardened-selinux+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" |
133 |
+ ;; |
134 |
+ esac |
135 |
+ popd >/dev/null || exit |
136 |
} |
137 |
|
138 |
# vim:ft=sh: |
139 |
diff --git a/tools/catalyst-auto-amd64.conf b/tools/catalyst-auto-amd64.conf |
140 |
index 0b7e9b2f..585c22c6 100644 |
141 |
--- a/tools/catalyst-auto-amd64.conf |
142 |
+++ b/tools/catalyst-auto-amd64.conf |
143 |
@@ -44,7 +44,7 @@ EXTENSIONS="[.tar.xz,.tar.bz2,.tar.gz,.tar,.sfs]" |
144 |
update_symlinks() { |
145 |
# Symlink the latest stages3 to build from |
146 |
for d in ${BUILD_SRCDIR_BASE}/{default,hardened} ; do |
147 |
- pushd $d >/dev/null |
148 |
+ pushd "${d}" >/dev/null || exit |
149 |
for f in $(ls stage3*${EXTENSIONS} | grep -v latest | give_latest_from_dates ) ; do |
150 |
# 20yymmddThhmmssZ |
151 |
# 20yymmddhhmmss |
152 |
@@ -58,11 +58,13 @@ update_symlinks() { |
153 |
<<<"$f") |
154 |
ln -svf "$f" "$of" |
155 |
done |
156 |
- popd >/dev/null |
157 |
+ popd >/dev/null || exit |
158 |
done |
159 |
} |
160 |
|
161 |
post_build() { |
162 |
+ local set=$1 spec=$2 |
163 |
+ |
164 |
cmd=( |
165 |
rsync |
166 |
-e 'ssh -i /root/.ssh/id_rsa -o UserKnownHostsFile=/dev/null -o VerifyHostKeyDNS=yes -o StrictHostKeyChecking=no' |
167 |
@@ -81,34 +83,58 @@ post_build() { |
168 |
DEST_HARDENED=${ARCH}@releng-incoming.gentoo.org:${BUILD_DESTDIR_BASE}/hardened |
169 |
;; |
170 |
esac |
171 |
- pushd ${BUILD_SRCDIR_BASE}/default >/dev/null |
172 |
- mkdir -p ${TMPDIR}/empty |
173 |
- "${cmd[@]}" ${TMPDIR}/empty ${DEST_DEFAULT} |
174 |
- for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do |
175 |
- if [ -f $file ]; then |
176 |
- "${cmd[@]}" ${file}* ${DEST_DEFAULT} |
177 |
- fi |
178 |
- done |
179 |
- if [ -f *${DATESTAMP}*.iso ]; then |
180 |
- "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_DEFAULT} |
181 |
- elif [ -f *${TIMESTAMP}*.iso ]; then |
182 |
- "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_DEFAULT} |
183 |
- fi |
184 |
- popd >/dev/null |
185 |
- |
186 |
- pushd ${BUILD_SRCDIR_BASE}/hardened >/dev/null |
187 |
- "${cmd[@]}" ${TMPDIR}/empty ${DEST_HARDENED} |
188 |
- for file in $(ls stage{3,4}*${DATESTAMP}*${EXTENSIONS} stage{3,4}*${TIMESTAMP}*${EXTENSIONS} ); do |
189 |
- if [ -f $file ]; then |
190 |
- "${cmd[@]}" $file* ${DEST_HARDENED} |
191 |
- fi |
192 |
- done |
193 |
- if [ -f *${DATESTAMP}*.iso ]; then |
194 |
- "${cmd[@]}" *${DATESTAMP}*.iso* ${DEST_HARDENED} |
195 |
- elif [ -f *${TIMESTAMP}*.iso ]; then |
196 |
- "${cmd[@]}" *${TIMESTAMP}*.iso* ${DEST_HARDENED} |
197 |
- fi |
198 |
- popd >/dev/null |
199 |
+ |
200 |
+ pushd "${BUILD_SRCDIR_BASE}"/default >/dev/null || exit |
201 |
+ case ${spec} in |
202 |
+ stage3.spec) |
203 |
+ "${cmd[@]}" stage3-amd64-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" |
204 |
+ ;; |
205 |
+ installcd-stage2-minimal.spec) |
206 |
+ "${cmd[@]}" install-amd64-minimal-${TIMESTAMP}.iso* "${DEST_DEFAULT}" |
207 |
+ ;; |
208 |
+ stage4-minimal.spec) |
209 |
+ "${cmd[@]}" stage4-amd64-minimal-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" |
210 |
+ ;; |
211 |
+ stage3-nomultilib.spec) |
212 |
+ "${cmd[@]}" stage3-amd64-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" |
213 |
+ ;; |
214 |
+ stage4-nomultilib-minimal.spec) |
215 |
+ "${cmd[@]}" stage4-amd64-minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" |
216 |
+ ;; |
217 |
+ stage3-x32.spec) |
218 |
+ "${cmd[@]}" stage3-x32-${TIMESTAMP}.tar.xz* "${DEST_DEFAULT}" |
219 |
+ ;; |
220 |
+ esac |
221 |
+ popd >/dev/null || exit |
222 |
+ |
223 |
+ pushd "${BUILD_SRCDIR_BASE}"/hardened >/dev/null || exit |
224 |
+ case ${spec} in |
225 |
+ hardened/stage3.spec) |
226 |
+ "${cmd[@]}" stage3-amd64-hardened-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" |
227 |
+ ;; |
228 |
+ hardened/admincd-stage2.spec) |
229 |
+ "${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}" |
230 |
+ ;; |
231 |
+ hardened/stage4-minimal.spec) |
232 |
+ "${cmd[@]}" stage4-amd64-hardened+minimal-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" |
233 |
+ ;; |
234 |
+ hardened/stage3-nomultilib.spec) |
235 |
+ "${cmd[@]}" stage3-amd64-hardened+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" |
236 |
+ ;; |
237 |
+ hardened/stage4-nomultilib-minimal.spec) |
238 |
+ "${cmd[@]}" stage4-amd64-hardened+minimal-nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" |
239 |
+ ;; |
240 |
+ hardened/stage3-selinux.spec) |
241 |
+ "${cmd[@]}" stage3-amd64-hardened-selinux-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" |
242 |
+ ;; |
243 |
+ hardened/admincd-stage2-selinux.spec) |
244 |
+ "${cmd[@]}" admincd-amd64-${TIMESTAMP}.iso* "${DEST_HARDENED}" |
245 |
+ ;; |
246 |
+ hardened/stage3-selinux-nomultilib.spec) |
247 |
+ "${cmd[@]}" stage3-amd64-hardened-selinux+nomultilib-${TIMESTAMP}.tar.xz* "${DEST_HARDENED}" |
248 |
+ ;; |
249 |
+ esac |
250 |
+ popd >/dev/null || exit |
251 |
} |
252 |
|
253 |
# vim:ft=sh: |
254 |
-- |
255 |
2.26.2 |