1 |
On Sun, 2018-11-18 at 03:37 +0800, Jason Zaman wrote: |
2 |
> Hey all, |
3 |
> |
4 |
> I've been using Bazel (https://bazel.build/) to build TensorFlow for a |
5 |
> while now. Here is a bazel.eclass I'd like to commit to make it easier |
6 |
> for packages that use it to build. It's basically bits that I've |
7 |
> refactored out of the TensorFlow ebuild that would be useful to other |
8 |
> packages as well. I have a bump to sci-libs/tensorflow-1.12.0 prepared |
9 |
> that uses this eclass and have tested a full install. |
10 |
> |
11 |
> -- Jason |
12 |
> |
13 |
> # Copyright 1999-2018 Jason Zaman |
14 |
> # Distributed under the terms of the GNU General Public License v2 |
15 |
> |
16 |
> # @ECLASS: bazel.eclass |
17 |
> # @MAINTAINER: |
18 |
> # Jason Zaman <perfinion@g.o> |
19 |
> # @AUTHOR: |
20 |
> # Jason Zaman <perfinion@g.o> |
21 |
> # @BLURB: Utility functions for packages using Bazel Build |
22 |
> # @DESCRIPTION: |
23 |
> # A utility eclass providing functions to run the Bazel Build system. |
24 |
> # |
25 |
> # This eclass does not export any phase functions. |
26 |
> |
27 |
> case "${EAPI:-0}" in |
28 |
> 0|1|2|3|4|5|6) |
29 |
> die "Unsupported EAPI=${EAPI:-0} (too old) for ${ECLASS}" |
30 |
> ;; |
31 |
> 7) |
32 |
> ;; |
33 |
> *) |
34 |
> die "Unsupported EAPI=${EAPI} (unknown) for ${ECLASS}" |
35 |
> ;; |
36 |
> esac |
37 |
> |
38 |
> if [[ ! ${_BAZEL_ECLASS} ]]; then |
39 |
> |
40 |
> inherit multiprocessing toolchain-funcs |
41 |
> |
42 |
> BDEPEND=">=dev-util/bazel-0.19" |
43 |
> |
44 |
> # @FUNCTION: bazel_get_flags |
45 |
> # @DESCRIPTION: |
46 |
> # Obtain and print the bazel flags for target and host *FLAGS. |
47 |
> # |
48 |
> # To add more flags to this, append the flags to the |
49 |
> # appropriate variable before calling this function |
50 |
> bazel_get_flags() { |
51 |
> local i fs=() |
52 |
> for i in ${CFLAGS}; do |
53 |
> fs+=( "--conlyopt=${i}" ) |
54 |
> done |
55 |
> for i in ${BUILD_CFLAGS}; do |
56 |
> fs+=( "--host_conlyopt=${i}" ) |
57 |
> done |
58 |
> for i in ${CXXFLAGS}; do |
59 |
> fs+=( "--cxxopt=${i}" ) |
60 |
> done |
61 |
> for i in ${BUILD_CXXFLAGS}; do |
62 |
> fs+=( "--host_cxxopt=${i}" ) |
63 |
> done |
64 |
> for i in ${CPPFLAGS}; do |
65 |
> fs+=( "--conlyopt=${i}" "--cxxopt=${i}" ) |
66 |
> done |
67 |
> for i in ${BUILD_CPPFLAGS}; do |
68 |
> fs+=( "--host_conlyopt=${i}" "--host_cxxopt=${i}" ) |
69 |
> done |
70 |
> for i in ${LDFLAGS}; do |
71 |
> fs+=( "--linkopt=${i}" ) |
72 |
> done |
73 |
> for i in ${BUILD_LDFLAGS}; do |
74 |
> fs+=( "--host_linkopt=${i}" ) |
75 |
> done |
76 |
> echo "${fs[*]}" |
77 |
> } |
78 |
> |
79 |
> # @FUNCTION: bazel_setup_bazelrc |
80 |
> # @DESCRIPTION: |
81 |
> # Creates the bazelrc with common options that will be passed |
82 |
> # to bazel. This will be called by ebazel automatically so |
83 |
> # does not need to be called from the ebuild. |
84 |
> bazel_setup_bazelrc() { |
85 |
> if [[ -f "${T}/bazelrc" ]]; then |
86 |
> return |
87 |
> fi |
88 |
> |
89 |
> # F: fopen_wr |
90 |
> # P: /proc/self/setgroups |
91 |
> # Even with standalone enabled, the Bazel sandbox binary is run for feature test: |
92 |
> # https://github.com/bazelbuild/bazel/blob/7b091c1397a82258e26ab5336df6c8dae1d97384/src/main/java/com/google/devtools/build/lib/sandbox/LinuxSandboxedSpawnRunner.java#L61 |
93 |
> # https://github.com/bazelbuild/bazel/blob/76555482873ffcf1d32fb40106f89231b37f850a/src/main/tools/linux-sandbox-pid1.cc#L113 |
94 |
> addpredict /proc |
95 |
> |
96 |
> mkdir -p "${T}/bazel-cache" || die |
97 |
> mkdir -p "${T}/bazel-distdir" || die |
98 |
> |
99 |
> cat > "${T}/bazelrc" <<-EOF || die |
100 |
> startup --batch |
101 |
|
102 |
Maybe indent this stuff to make it stand out from ebuild code. |
103 |
|
104 |
> |
105 |
> # dont strip HOME, portage sets a temp per-package dir |
106 |
> build --action_env HOME |
107 |
> |
108 |
> # make bazel respect MAKEOPTS |
109 |
> build --jobs=$(makeopts_jobs) |
110 |
> build --compilation_mode=opt --host_compilation_mode=opt |
111 |
> |
112 |
> # FLAGS |
113 |
> build $(bazel_get_flags) |
114 |
> |
115 |
> # Use standalone strategy to deactivate the bazel sandbox, since it |
116 |
> # conflicts with FEATURES=sandbox. |
117 |
> build --spawn_strategy=standalone --genrule_strategy=standalone |
118 |
> test --spawn_strategy=standalone --genrule_strategy=standalone |
119 |
> |
120 |
> build --strip=never |
121 |
> build --verbose_failures --noshow_loading_progress |
122 |
> test --verbose_test_summary --verbose_failures --noshow_loading_progress |
123 |
> |
124 |
> # make bazel only fetch distfiles from the cache |
125 |
> fetch --repository_cache="${T}/bazel-cache/" --distdir="${T}/bazel-distdir/" |
126 |
> build --repository_cache="${T}/bazel-cache/" --distdir="${T}/bazel-distdir/" |
127 |
> |
128 |
> build --define=PREFIX=${EPREFIX%/}/usr |
129 |
> build --define=LIBDIR=\$(PREFIX)/$(get_libdir) |
130 |
> |
131 |
> EOF |
132 |
> |
133 |
> tc-is-cross-compiler || \ |
134 |
> echo "build --nodistinct_host_configuration" >> "${T}/bazelrc" || die |
135 |
|
136 |
Don't do || chains, they are unreadable. |
137 |
|
138 |
> } |
139 |
> |
140 |
> # @FUNCTION: ebazel |
141 |
> # @USAGE: [<args>...] |
142 |
> # @DESCRIPTION: |
143 |
> # Run bazel with the bazelrc and output_base. |
144 |
> # |
145 |
> # If $MULTIBUILD_VARIANT is set, this will make an output_base |
146 |
> # specific to that variant. |
147 |
> # bazel_setup_bazelrc will be called and the created bazelrc |
148 |
> # will be passed to bazel. |
149 |
> # |
150 |
> # Will automatically die if bazel does not exit cleanly. |
151 |
> ebazel() { |
152 |
> bazel_setup_bazelrc |
153 |
> |
154 |
> # Use different build folders for each multibuild variant. |
155 |
> local base_suffix="${MULTIBUILD_VARIANT+-}${MULTIBUILD_VARIANT}" |
156 |
|
157 |
Any reason not to use BUILD_DIR instead of reinventing it? |
158 |
|
159 |
> local output_base="${WORKDIR}/bazel-base${base_suffix}" |
160 |
> mkdir -p "${output_base}" || die |
161 |
> |
162 |
> einfo Running: bazel --output_base="${output_base}" "$@" |
163 |
> bazel --bazelrc="${T}/bazelrc" --output_base="${output_base}" $@ || die "ebazel $@" |
164 |
|
165 |
The common practice is to echo >&2 it. Also, you output different |
166 |
arguments than you execute which is going to confuse the hell out of |
167 |
users who'll end up having to debug this. You can use a trick like |
168 |
the following to avoid typing args twice: |
169 |
|
170 |
set -- bazel --bazelrc... |
171 |
echo "${*}" >&2 |
172 |
"${@}" || die ... |
173 |
|
174 |
> } |
175 |
> |
176 |
> # @FUNCTION: bazel_load_distfiles |
177 |
> # @USAGE: <distfiles>... |
178 |
> # @DESCRIPTION: |
179 |
> # Populate the bazel distdir to fetch from since it cannot use |
180 |
> # the network. Bazel looks in distdir but will only look for the |
181 |
> # original filename, not the possibly renamed one that portage |
182 |
> # downloaded. If the line has -> we to rename it back. This also |
183 |
> # handles use-conditionals that SRC_URI does. |
184 |
|
185 |
Why oh why do you have to implement custom parser for the ebuild syntax? |
186 |
That's just asking for horrible failures. |
187 |
|
188 |
> # |
189 |
> # Example: |
190 |
> # @CODE |
191 |
> # bazel_external_uris="http://a/file-2.0.tgz |
192 |
> # python? ( http://b/1.0.tgz -> foo-1.0.tgz )" |
193 |
> # SRC_URI="http://c/${PV}.tgz |
194 |
> # ${bazel_external_uris}" |
195 |
> # |
196 |
> # src_unpack() { |
197 |
> # unpack ${PV}.tgz |
198 |
> # bazel_load_distfiles "${bazel_external_uris}" |
199 |
> # } |
200 |
> # @CODE |
201 |
> bazel_load_distfiles() { |
202 |
> local src dst uri rename |
203 |
> |
204 |
> [[ "$@" ]] || die "Missing args" |
205 |
> mkdir -p "${T}/bazel-distdir" || die |
206 |
> |
207 |
> while read uri rename dst; do |
208 |
> src="${uri##*/}" |
209 |
> [[ -z $src ]] && continue |
210 |
|
211 |
Please use ${foo} syntax in ebuilds, consistently. |
212 |
|
213 |
> if [[ "$rename" != "->" ]]; then |
214 |
> dst="${src}" |
215 |
> fi |
216 |
> |
217 |
> [[ ${A} =~ ${dst} ]] || continue |
218 |
|
219 |
Why are you doing regex match here? Last I checked, we didn't use |
220 |
regular expressions in SRC_URI. |
221 |
|
222 |
> |
223 |
> if [[ "$dst" == "$src" ]]; then |
224 |
> einfo "Copying $dst to bazel distdir ..." |
225 |
> else |
226 |
> einfo "Copying $dst to bazel distdir $src ..." |
227 |
|
228 |
Are you using src and dst to mean the opposite? |
229 |
|
230 |
> fi |
231 |
> dst="$(readlink -f "${DISTDIR}/${dst}")" |
232 |
|
233 |
Looks like you are hardcoding hacks for implementation details which |
234 |
indicates whatever you're doing is a very bad idea, and is going to fail |
235 |
whenever the implementation is subtly different than what you've worked |
236 |
around so far. |
237 |
|
238 |
> ln -s "${dst}" "${T}/bazel-distdir/${src}" || die |
239 |
> done <<< "$(sed -re 's/!?[A-Za-z]+\?\s+\(\s*//g; s/\s+\)//g' <<< "$@")" |
240 |
|
241 |
Please don't use horribly unreadable sed expressions. This just means |
242 |
that whoever will have to touch this eclass in the future will wish you |
243 |
were never recruited. |
244 |
|
245 |
> } |
246 |
> |
247 |
> _BAZEL_ECLASS=1 |
248 |
> fi |
249 |
> |
250 |
> |
251 |
> |
252 |
|
253 |
-- |
254 |
Best regards, |
255 |
Michał Górny |