1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
|
{ stdenv
, bazel
, cacert
, lib
}:
let
bazelPkg = bazel;
in
args@{
name ? "${args.pname}-${args.version}"
, bazel ? bazelPkg
, bazelFlags ? []
, bazelBuildFlags ? []
, bazelFetchFlags ? []
, bazelTarget
, buildAttrs
, fetchAttrs
# Newer versions of Bazel are moving away from built-in rules_cc and instead
# allow fetching it as an external dependency in a WORKSPACE file[1]. If
# removed in the fixed-output fetch phase, building will fail to download it.
# This can be seen e.g. in #73097
#
# This option allows configuring the removal of rules_cc in cases where a
# project depends on it via an external dependency.
#
# [1]: https://github.com/bazelbuild/rules_cc
, removeRulesCC ? true
, removeLocalConfigCc ? true
, removeLocal ? true
# Use build --nobuild instead of fetch. This allows fetching the dependencies
# required for the build as configured, rather than fetching all the dependencies
# which may not work in some situations (e.g. Java code which ends up relying on
# Debian-specific /usr/share/java paths, but doesn't in the configured build).
, fetchConfigured ? false
# Don’t add Bazel --copt and --linkopt from NIX_CFLAGS_COMPILE /
# NIX_LDFLAGS. This is necessary when using a custom toolchain which
# Bazel wants all headers / libraries to come from, like when using
# CROSSTOOL. Weirdly, we can still get the flags through the wrapped
# compiler.
, dontAddBazelOpts ? false
, ...
}:
let
fArgs = removeAttrs args [ "buildAttrs" "fetchAttrs" "removeRulesCC" ];
fBuildAttrs = fArgs // buildAttrs;
fFetchAttrs = fArgs // removeAttrs fetchAttrs [ "sha256" ];
in stdenv.mkDerivation (fBuildAttrs // {
inherit name bazelFlags bazelBuildFlags bazelFetchFlags bazelTarget;
deps = stdenv.mkDerivation (fFetchAttrs // {
name = "${name}-deps.tar.gz";
inherit bazelFlags bazelBuildFlags bazelFetchFlags bazelTarget;
impureEnvVars = lib.fetchers.proxyImpureEnvVars;
nativeBuildInputs = fFetchAttrs.nativeBuildInputs or [] ++ [ bazel ];
preHook = fFetchAttrs.preHook or "" + ''
export bazelOut="$(echo ''${NIX_BUILD_TOP}/output | sed -e 's,//,/,g')"
export bazelUserRoot="$(echo ''${NIX_BUILD_TOP}/tmp | sed -e 's,//,/,g')"
export HOME="$NIX_BUILD_TOP"
export USER="nix"
# This is needed for git_repository with https remotes
export GIT_SSL_CAINFO="${cacert}/etc/ssl/certs/ca-bundle.crt"
# This is needed for Bazel fetchers that are themselves programs (e.g.
# rules_go using the go toolchain)
export SSL_CERT_FILE="${cacert}/etc/ssl/certs/ca-bundle.crt"
'';
buildPhase = fFetchAttrs.buildPhase or ''
runHook preBuild
# Bazel computes the default value of output_user_root before parsing the
# flag. The computation of the default value involves getting the $USER
# from the environment. I don't have that variable when building with
# sandbox enabled. Code here
# https://github.com/bazelbuild/bazel/blob/9323c57607d37f9c949b60e293b573584906da46/src/main/cpp/startup_options.cc#L123-L124
#
# On macOS Bazel will use the system installed Xcode or CLT toolchain instead of the one in the PATH unless we pass BAZEL_USE_CPP_ONLY_TOOLCHAIN
# We disable multithreading for the fetching phase since it can lead to timeouts with many dependencies/threads:
# https://github.com/bazelbuild/bazel/issues/6502
BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 \
USER=homeless-shelter \
bazel \
--output_base="$bazelOut" \
--output_user_root="$bazelUserRoot" \
${if fetchConfigured then "build --nobuild" else "fetch"} \
--loading_phase_threads=1 \
$bazelFlags \
$bazelFetchFlags \
$bazelTarget
runHook postBuild
'';
installPhase = fFetchAttrs.installPhase or ''
runHook preInstall
# Remove all built in external workspaces, Bazel will recreate them when building
rm -rf $bazelOut/external/{bazel_tools,\@bazel_tools.marker}
${if removeRulesCC then "rm -rf $bazelOut/external/{rules_cc,\\@rules_cc.marker}" else ""}
rm -rf $bazelOut/external/{embedded_jdk,\@embedded_jdk.marker}
${if removeLocalConfigCc then "rm -rf $bazelOut/external/{local_config_cc,\@local_config_cc.marker}" else ""}
${if removeLocal then "rm -rf $bazelOut/external/{local_*,\@local_*.marker}" else ""}
# Clear markers
find $bazelOut/external -name '@*\.marker' -exec sh -c 'echo > {}' \;
# Remove all vcs files
rm -rf $(find $bazelOut/external -type d -name .git)
rm -rf $(find $bazelOut/external -type d -name .svn)
rm -rf $(find $bazelOut/external -type d -name .hg)
# Removing top-level symlinks along with their markers.
# This is needed because they sometimes point to temporary paths (?).
# For example, in Tensorflow-gpu build:
# platforms -> NIX_BUILD_TOP/tmp/install/35282f5123611afa742331368e9ae529/_embedded_binaries/platforms
find $bazelOut/external -maxdepth 1 -type l | while read symlink; do
name="$(basename "$symlink")"
rm "$symlink"
test -f "$bazelOut/external/@$name.marker" && rm "$bazelOut/external/@$name.marker"
done
# Patching symlinks to remove build directory reference
find $bazelOut/external -type l | while read symlink; do
new_target="$(readlink "$symlink" | sed "s,$NIX_BUILD_TOP,NIX_BUILD_TOP,")"
rm "$symlink"
ln -sf "$new_target" "$symlink"
done
echo '${bazel.name}' > $bazelOut/external/.nix-bazel-version
(cd $bazelOut/ && tar czf $out --sort=name --mtime='@1' --owner=0 --group=0 --numeric-owner external/)
runHook postInstall
'';
dontFixup = true;
allowedRequisites = [];
outputHashAlgo = "sha256";
outputHash = fetchAttrs.sha256;
});
nativeBuildInputs = fBuildAttrs.nativeBuildInputs or [] ++ [ (bazel.override { enableNixHacks = true; }) ];
preHook = fBuildAttrs.preHook or "" + ''
export bazelOut="$NIX_BUILD_TOP/output"
export bazelUserRoot="$NIX_BUILD_TOP/tmp"
export HOME="$NIX_BUILD_TOP"
'';
preConfigure = ''
mkdir -p "$bazelOut"
(cd $bazelOut && tar xfz $deps)
test "${bazel.name}" = "$(<$bazelOut/external/.nix-bazel-version)" || {
echo "fixed output derivation was built for a different bazel version" >&2
echo " got: $(<$bazelOut/external/.nix-bazel-version)" >&2
echo "expected: ${bazel.name}" >&2
exit 1
}
chmod -R +w $bazelOut
find $bazelOut -type l | while read symlink; do
ln -sf $(readlink "$symlink" | sed "s,NIX_BUILD_TOP,$NIX_BUILD_TOP,") "$symlink"
done
'' + fBuildAttrs.preConfigure or "";
inherit dontAddBazelOpts;
buildPhase = fBuildAttrs.buildPhase or ''
runHook preBuild
# Bazel sandboxes the execution of the tools it invokes, so even though we are
# calling the correct nix wrappers, the values of the environment variables
# the wrappers are expecting will not be set. So instead of relying on the
# wrappers picking them up, pass them in explicitly via `--copt`, `--linkopt`
# and related flags.
#
copts=()
host_copts=()
linkopts=()
host_linkopts=()
if [ -z "''${dontAddBazelOpts:-}" ]; then
for flag in $NIX_CFLAGS_COMPILE; do
copts+=( "--copt=$flag" )
host_copts+=( "--host_copt=$flag" )
done
for flag in $NIX_CXXSTDLIB_COMPILE; do
copts+=( "--copt=$flag" )
host_copts+=( "--host_copt=$flag" )
done
for flag in $NIX_LDFLAGS; do
linkopts+=( "--linkopt=-Wl,$flag" )
host_linkopts+=( "--host_linkopt=-Wl,$flag" )
done
fi
BAZEL_USE_CPP_ONLY_TOOLCHAIN=1 \
USER=homeless-shelter \
bazel \
--output_base="$bazelOut" \
--output_user_root="$bazelUserRoot" \
build \
-j $NIX_BUILD_CORES \
"''${copts[@]}" \
"''${host_copts[@]}" \
"''${linkopts[@]}" \
"''${host_linkopts[@]}" \
$bazelFlags \
$bazelBuildFlags \
$bazelTarget
runHook postBuild
'';
})
|
