Skip to content

Commit 9ef09ec

Browse files
giordanogiordano
committed
[Auditor] Use @spawn instead of @threads for better load balancing
1 parent 9cde1a8 commit 9ef09ec

File tree

1 file changed

+152
-140
lines changed

1 file changed

+152
-140
lines changed

‎src/Auditor.jl

Lines changed: 152 additions & 140 deletions
Original file line numberDiff line numberDiff line change
@@ -80,82 +80,85 @@ function audit(prefix::Prefix, src_name::AbstractString = "";
8080
# Inspect binary files, looking for improper linkage
8181
predicate = f -> (filemode(f) & 0o111) != 0 || valid_library_path(f, platform)
8282
bin_files = collect_files(prefix, predicate; exclude_externalities=false)
83-
Threads.@threads for f in collapse_symlinks(bin_files)
83+
@sync for f in collapse_symlinks(bin_files)
8484
# If `f` is outside of our prefix, ignore it. This happens with files from our dependencies
8585
if !startswith(f, prefix.path)
8686
continue
8787
end
8888

89-
# Peel this binary file open like a delicious tangerine
90-
try
91-
readmeta(f) do ohs
92-
foreach(ohs) do oh
93-
if !is_for_platform(oh, platform)
94-
if verbose
95-
@warn("Skipping binary analysis of $(relpath(f, prefix.path)) (incorrect platform)")
96-
end
97-
else
98-
# Check that the ISA isn't too high
99-
all_ok[] &= check_isa(oh, platform, prefix; verbose, silent)
100-
# Check that the OS ABI is set correctly (often indicates the wrong linker was used)
101-
all_ok[] &= check_os_abi(oh, platform; verbose)
102-
# Make sure all binary files are executables, if libraries aren't
103-
# executables Julia may not be able to dlopen them:
104-
# https://github.com/JuliaLang/julia/issues/38993. In principle this
105-
# should be done when autofix=true, but we have to run this fix on MKL
106-
# for Windows, for which however we have to set autofix=false:
107-
# https://github.com/JuliaPackaging/Yggdrasil/pull/922.
108-
all_ok[] &= ensure_executability(oh; verbose, silent)
109-
110-
# If this is a dynamic object, do the dynamic checks
111-
if isdynamic(oh)
112-
# Check that the libgfortran version matches
113-
all_ok[] &= check_libgfortran_version(oh, platform; verbose, has_csl)
114-
# Check whether the library depends on any of the most common
115-
# libraries provided by `CompilerSupportLibraries_jll`.
116-
all_ok[] &= check_csl_libs(oh, platform; verbose, has_csl)
117-
# Check that the libstdcxx string ABI matches
118-
all_ok[] &= check_cxxstring_abi(oh, platform; verbose)
119-
# Check that this binary file's dynamic linkage works properly. Note to always
120-
# DO THIS ONE LAST as it can actually mutate the file, which causes the previous
121-
# checks to freak out a little bit.
122-
all_ok[] &= check_dynamic_linkage(oh, prefix, bin_files;
123-
platform, silent, verbose, autofix, src_name)
89+
Threads.@spawn let
90+
# Peel this binary file open like a delicious tangerine
91+
try
92+
readmeta(f) do ohs
93+
foreach(ohs) do oh
94+
if !is_for_platform(oh, platform)
95+
if verbose
96+
@warn("Skipping binary analysis of $(relpath(f, prefix.path)) (incorrect platform)")
97+
end
98+
else
99+
# Check that the ISA isn't too high
100+
all_ok[] &= check_isa(oh, platform, prefix; verbose, silent)
101+
# Check that the OS ABI is set correctly (often indicates the wrong linker was used)
102+
all_ok[] &= check_os_abi(oh, platform; verbose)
103+
# Make sure all binary files are executables, if libraries aren't
104+
# executables Julia may not be able to dlopen them:
105+
# https://github.com/JuliaLang/julia/issues/38993. In principle this
106+
# should be done when autofix=true, but we have to run this fix on MKL
107+
# for Windows, for which however we have to set autofix=false:
108+
# https://github.com/JuliaPackaging/Yggdrasil/pull/922.
109+
all_ok[] &= ensure_executability(oh; verbose, silent)
110+
111+
# If this is a dynamic object, do the dynamic checks
112+
if isdynamic(oh)
113+
# Check that the libgfortran version matches
114+
all_ok[] &= check_libgfortran_version(oh, platform; verbose, has_csl)
115+
# Check whether the library depends on any of the most common
116+
# libraries provided by `CompilerSupportLibraries_jll`.
117+
all_ok[] &= check_csl_libs(oh, platform; verbose, has_csl)
118+
# Check that the libstdcxx string ABI matches
119+
all_ok[] &= check_cxxstring_abi(oh, platform; verbose)
120+
# Check that this binary file's dynamic linkage works properly. Note to always
121+
# DO THIS ONE LAST as it can actually mutate the file, which causes the previous
122+
# checks to freak out a little bit.
123+
all_ok[] &= check_dynamic_linkage(oh, prefix, bin_files;
124+
platform, silent, verbose, autofix, src_name)
125+
end
124126
end
125127
end
126128
end
127-
end
128129

129-
# Ensure this file is codesigned (currently only does something on Apple platforms)
130-
all_ok[] &= ensure_codesigned(f, prefix, platform; verbose, subdir=src_name)
131-
catch e
132-
if !isa(e, ObjectFile.MagicMismatch)
133-
rethrow(e)
134-
end
130+
# Ensure this file is codesigned (currently only does something on Apple platforms)
131+
all_ok[] &= ensure_codesigned(f, prefix, platform; verbose, subdir=src_name)
132+
catch e
133+
if !isa(e, ObjectFile.MagicMismatch)
134+
rethrow(e)
135+
end
135136

136-
# If this isn't an actual binary file, skip it
137-
if verbose
138-
@info("Skipping binary analysis of $(relpath(f, prefix.path))")
137+
# If this isn't an actual binary file, skip it
138+
if verbose
139+
@info("Skipping binary analysis of $(relpath(f, prefix.path))")
140+
end
139141
end
140142
end
141143
end
142144

143145
# Find all dynamic libraries
144146
shlib_files = filter(f -> startswith(f, prefix.path) && valid_library_path(f, platform), collapse_symlinks(bin_files))
145147

146-
Threads.@threads for f in shlib_files
147-
# Inspect all shared library files for our platform (but only if we're
148-
# running native, don't try to load library files from other platforms)
149-
if platforms_match(platform, HostPlatform())
150-
if verbose
151-
@info("Checking shared library $(relpath(f, prefix.path))")
152-
end
148+
@sync for f in shlib_files
149+
Threads.@spawn let
150+
# Inspect all shared library files for our platform (but only if we're
151+
# running native, don't try to load library files from other platforms)
152+
if platforms_match(platform, HostPlatform())
153+
if verbose
154+
@info("Checking shared library $(relpath(f, prefix.path))")
155+
end
153156

154-
# dlopen() this library in a separate Julia process so that if we
155-
# try to do something silly like `dlopen()` a .so file that uses
156-
# LLVM in interesting ways on startup, it doesn't kill our main
157-
# Julia process.
158-
dlopen_cmd = """
157+
# dlopen() this library in a separate Julia process so that if we
158+
# try to do something silly like `dlopen()` a .so file that uses
159+
# LLVM in interesting ways on startup, it doesn't kill our main
160+
# Julia process.
161+
dlopen_cmd = """
159162
using Libdl
160163
try
161164
dlopen($(repr(f)))
@@ -167,30 +170,31 @@ function audit(prefix::Prefix, src_name::AbstractString = "";
167170
exit(1)
168171
end
169172
"""
170-
try
171-
p = open(`$(Base.julia_cmd()) -e $dlopen_cmd`)
172-
wait(p)
173-
if p.exitcode != 0
174-
throw("Invalid exit code!")
175-
end
176-
catch
177-
# TODO: Use the relevant ObjFileBase packages to inspect why
178-
# this file is being nasty to us.
179-
if !silent
180-
@warn("$(relpath(f, prefix.path)) cannot be dlopen()'ed")
173+
try
174+
p = open(`$(Base.julia_cmd()) -e $dlopen_cmd`)
175+
wait(p)
176+
if p.exitcode != 0
177+
throw("Invalid exit code!")
178+
end
179+
catch
180+
# TODO: Use the relevant ObjFileBase packages to inspect why
181+
# this file is being nasty to us.
182+
if !silent
183+
@warn("$(relpath(f, prefix.path)) cannot be dlopen()'ed")
184+
end
185+
all_ok[] = false
181186
end
182-
all_ok[] = false
183187
end
184-
end
185188

186-
# Ensure that all libraries have at least some kind of SONAME, if we're
187-
# on that kind of platform
188-
if !Sys.iswindows(platform)
189-
all_ok[] &= ensure_soname(prefix, f, platform; verbose, autofix, subdir=src_name)
190-
end
189+
# Ensure that all libraries have at least some kind of SONAME, if we're
190+
# on that kind of platform
191+
if !Sys.iswindows(platform)
192+
all_ok[] &= ensure_soname(prefix, f, platform; verbose, autofix, subdir=src_name)
193+
end
191194

192-
# Ensure that this library is available at its own SONAME
193-
all_ok[] &= symlink_soname_lib(f; verbose=verbose, autofix=autofix)
195+
# Ensure that this library is available at its own SONAME
196+
all_ok[] &= symlink_soname_lib(f; verbose=verbose, autofix=autofix)
197+
end
194198
end
195199

196200
# remove *.la files generated by GNU libtool
@@ -218,35 +222,37 @@ function audit(prefix::Prefix, src_name::AbstractString = "";
218222

219223
# Make sure that `(exec_)prefix` in pkg-config files use a relative prefix
220224
pc_files = collect_files(prefix, endswith(".pc"))
221-
Threads.@threads for f in pc_files
222-
for var in ["prefix", "exec_prefix"]
223-
pc_re = Regex("^$var=(/.*)\$")
224-
# We want to replace every instance of `prefix=...` with
225-
# `prefix=${pcfiledir}/../..`
226-
changed = false
227-
buf = IOBuffer()
228-
for l in readlines(f)
229-
m = match(pc_re, l)
230-
if m !== nothing
231-
# dealing with an absolute path we need to relativize;
232-
# determine how many directories we need to go up.
233-
dir = m.captures[1]
234-
f_rel = relpath(f, prefix.path)
235-
ndirs = count('/', f_rel)
236-
prefix_rel = join([".." for _ in 1:ndirs], "/")
237-
l = "$var=\${pcfiledir}/$prefix_rel"
238-
changed = true
225+
@sync for f in pc_files
226+
Threads.@spawn let
227+
for var in ["prefix", "exec_prefix"]
228+
pc_re = Regex("^$var=(/.*)\$")
229+
# We want to replace every instance of `prefix=...` with
230+
# `prefix=${pcfiledir}/../..`
231+
changed = false
232+
buf = IOBuffer()
233+
for l in readlines(f)
234+
m = match(pc_re, l)
235+
if m !== nothing
236+
# dealing with an absolute path we need to relativize;
237+
# determine how many directories we need to go up.
238+
dir = m.captures[1]
239+
f_rel = relpath(f, prefix.path)
240+
ndirs = count('/', f_rel)
241+
prefix_rel = join([".." for _ in 1:ndirs], "/")
242+
l = "$var=\${pcfiledir}/$prefix_rel"
243+
changed = true
244+
end
245+
println(buf, l)
239246
end
240-
println(buf, l)
241-
end
242-
str = String(take!(buf))
247+
str = String(take!(buf))
243248

244-
if changed
245-
# Overwrite file
246-
if verbose
247-
@info("Relocatize pkg-config file $f")
249+
if changed
250+
# Overwrite file
251+
if verbose
252+
@info("Relocatize pkg-config file $f")
253+
end
254+
write(f, str)
248255
end
249-
write(f, str)
250256
end
251257
end
252258
end
@@ -255,14 +261,16 @@ function audit(prefix::Prefix, src_name::AbstractString = "";
255261
# We also cannot allow any symlinks in Windows because it requires
256262
# Admin privileges to create them. Orz
257263
symlinks = collect_files(prefix, islink, exclude_dirs = false)
258-
Threads.@threads for f in symlinks
259-
try
260-
src_path = realpath(f)
261-
if isfile(src_path) || isdir(src_path)
262-
rm(f; force=true)
263-
cp(src_path, f, follow_symlinks = true)
264+
@sync for f in symlinks
265+
Threads.@spawn let
266+
try
267+
src_path = realpath(f)
268+
if isfile(src_path) || isdir(src_path)
269+
rm(f; force=true)
270+
cp(src_path, f, follow_symlinks = true)
271+
end
272+
catch
264273
end
265-
catch
266274
end
267275
end
268276

@@ -294,11 +302,13 @@ function audit(prefix::Prefix, src_name::AbstractString = "";
294302

295303
# Normalise timestamp of Windows import libraries.
296304
import_libraries = collect_files(prefix, endswith(".dll.a"))
297-
Threads.@threads for implib in import_libraries
298-
if verbose
299-
@info("Normalising timestamps in import library $(implib)")
305+
@sync for implib in import_libraries
306+
Threads.@spawn let
307+
if verbose
308+
@info("Normalising timestamps in import library $(implib)")
309+
end
310+
normalise_implib_timestamp(implib)
300311
end
301-
normalise_implib_timestamp(implib)
302312
end
303313

304314
end
@@ -404,7 +414,7 @@ function check_dynamic_linkage(oh, prefix, bin_files;
404414
# Look at every dynamic link, and see if we should do anything about that link...
405415
libs = find_libraries(oh)
406416
ignored_libraries = String[]
407-
Threads.@threads for libname in collect(keys(libs))
417+
@sync for libname in collect(keys(libs))
408418
if should_ignore_lib(libname, oh, platform)
409419
push!(ignored_libraries, libname)
410420
continue
@@ -421,40 +431,42 @@ function check_dynamic_linkage(oh, prefix, bin_files;
421431
continue
422432
end
423433

424-
if !isfile(libs[libname])
425-
# If we couldn't resolve this library, let's try autofixing,
426-
# if we're allowed to by the user
427-
if autofix
428-
# First, is this a library that we already know about?
429-
known_bins = lowercase.(basename.(bin_files))
430-
kidx = findfirst(known_bins .== lowercase(basename(libname)))
431-
if kidx !== nothing
432-
# If it is, point to that file instead!
433-
new_link = update_linkage(prefix, platform, path(oh), libs[libname], bin_files[kidx]; verbose, subdir=src_name)
434-
435-
if verbose && new_link !== nothing
436-
@info("$(filename): Linked library $(libname) has been auto-mapped to $(new_link)")
434+
Threads.@spawn let
435+
if !isfile(libs[libname])
436+
# If we couldn't resolve this library, let's try autofixing,
437+
# if we're allowed to by the user
438+
if autofix
439+
# First, is this a library that we already know about?
440+
known_bins = lowercase.(basename.(bin_files))
441+
kidx = findfirst(known_bins .== lowercase(basename(libname)))
442+
if kidx !== nothing
443+
# If it is, point to that file instead!
444+
new_link = update_linkage(prefix, platform, path(oh), libs[libname], bin_files[kidx]; verbose, subdir=src_name)
445+
446+
if verbose && new_link !== nothing
447+
@info("$(filename): Linked library $(libname) has been auto-mapped to $(new_link)")
448+
end
449+
else
450+
if !silent
451+
@warn("$(filename): Linked library $(libname) could not be resolved and could not be auto-mapped")
452+
if is_troublesome_library_link(libname, platform)
453+
@warn("$(filename): Depending on $(libname) is known to cause problems at runtime, make sure to link against the JLL library instead")
454+
end
455+
end
456+
all_ok[] = false
437457
end
438458
else
439459
if !silent
440-
@warn("$(filename): Linked library $(libname) could not be resolved and could not be auto-mapped")
441-
if is_troublesome_library_link(libname, platform)
442-
@warn("$(filename): Depending on $(libname) is known to cause problems at runtime, make sure to link against the JLL library instead")
443-
end
460+
@warn("$(filename): Linked library $(libname) could not be resolved within the given prefix")
444461
end
445462
all_ok[] = false
446463
end
447-
else
464+
elseif !startswith(libs[libname], prefix.path)
448465
if !silent
449-
@warn("$(filename): Linked library $(libname) could not be resolved within the given prefix")
466+
@warn("$(filename): Linked library $(libname) (resolved path $(libs[libname])) is not within the given prefix")
450467
end
451468
all_ok[] = false
452469
end
453-
elseif !startswith(libs[libname], prefix.path)
454-
if !silent
455-
@warn("$(filename): Linked library $(libname) (resolved path $(libs[libname])) is not within the given prefix")
456-
end
457-
all_ok[] = false
458470
end
459471
end
460472

0 commit comments

Comments
 (0)