From 436101ae49b77dac0a7fceed4588ef28e695d24e Mon Sep 17 00:00:00 2001 From: theogf Date: Wed, 13 Mar 2024 17:47:27 +0100 Subject: [PATCH 1/6] Add RemoteLogger --- Manifest.toml | 22 ++++++++++++++++++++++ Project.toml | 1 + src/Distributed.jl | 3 +++ src/logger.jl | 27 +++++++++++++++++++++++++++ test/distributed_exec.jl | 16 +++++++++++++++- 5 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 Manifest.toml create mode 100644 src/logger.jl diff --git a/Manifest.toml b/Manifest.toml new file mode 100644 index 0000000..c7778f9 --- /dev/null +++ b/Manifest.toml @@ -0,0 +1,22 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.10.2" +manifest_format = "2.0" +project_hash = "018bb01ed144ca156f5b8524b61e9dc7c0db49e4" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" diff --git a/Project.toml b/Project.toml index bb30760..48d85f2 100644 --- a/Project.toml +++ b/Project.toml @@ -3,6 +3,7 @@ uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" version = "1.11.0" [deps] +Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Serialization = "9e88b42a-f829-5b0c-bbe9-9e923198166b" Sockets = "6462fe0b-24de-5631-8697-dd941f90decc" diff --git a/src/Distributed.jl b/src/Distributed.jl index a7c5b17..58ab879 100644 --- a/src/Distributed.jl +++ b/src/Distributed.jl @@ -21,6 +21,7 @@ using Base.Threads: Event using Serialization, Sockets import Serialization: serialize, deserialize import Sockets: connect, wait_connected +import Logging # NOTE: clusterserialize.jl imports additional symbols from Serialization for use @@ -60,6 +61,7 @@ export WorkerConfig, RemoteException, ProcessExitedException, + RemoteLogger, process_messages, remoteref_id, @@ -107,6 +109,7 @@ include("messages.jl") include("process_messages.jl") # process incoming messages include("remotecall.jl") # the remotecall* api include("macros.jl") # @spawn and friends +include("logger.jl") include("workerpool.jl") include("pmap.jl") include("managers.jl") # LocalManager and SSHManager diff --git a/src/logger.jl b/src/logger.jl new file mode 100644 index 0000000..9b40f97 --- /dev/null +++ b/src/logger.jl @@ -0,0 +1,27 @@ +""" + RemoteLogger(pid=1, min_level=Info) + +Logger that forward all logging to worker `pid` via `remote_do` along with +adding the current worker `id` as a `pid` kwarg. +""" +struct RemoteLogger <: Logging.AbstractLogger + pid::Int + min_level::Logging.LogLevel +end +function RemoteLogger(pid=1) + RemoteLogger(pid, Logging.Info) +end + +Logging.min_enabled_level(logger::RemoteLogger) = logger.min_level +Logging.shouldlog(logger::RemoteLogger, level, _module, group, id) = true + +# TODO: probably should live in base/logging.jl? +function logmsg(level::Logging.LogLevel, message, _module, _group, _id, _file, _line; kwargs...) + Logging.@logmsg level message _module=_module _group=_group _id=_id _file=_file _line=_line kwargs... +end + +function Logging.handle_message(logger::RemoteLogger, level::Logging.LogLevel, message, _module, _group, _id, + _file, _line; kwargs...) + @nospecialize + remote_do(logmsg_code, logger.pid, level, message, _module, _group, _id, _file, _line; pid=myid(), kwargs...) +end diff --git a/test/distributed_exec.jl b/test/distributed_exec.jl index 7b5c983..14939c8 100644 --- a/test/distributed_exec.jl +++ b/test/distributed_exec.jl @@ -1,6 +1,6 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -using Test, Distributed, Random, Serialization, Sockets +using Test, Distributed, Random, Serialization, Sockets, Logging import Distributed: launch, manage sharedir = normpath(joinpath(Sys.BINDIR, "..", "share")) @@ -1927,6 +1927,20 @@ begin end end +# test logging +w = only(addprocs(1)) +@test_logs (:info, "from pid $w") begin + prev_logger = global_logger(current_logger()) + try + wait(@spawnat w with_env(RemoteLogger(1)) do + @info("from pid $(myid())") + end) + finally + global_logger(prev_logger) + end +end +wait(rmprocs([w])) + # Run topology tests last after removing all workers, since a given # cluster at any time only supports a single topology. nprocs() > 1 && rmprocs(workers()) From 903dfb14e59d7b3ff13b8cb47197738cb7ad3ad2 Mon Sep 17 00:00:00 2001 From: theogf Date: Wed, 13 Mar 2024 17:58:00 +0100 Subject: [PATCH 2/6] fix tests --- src/logger.jl | 2 +- test/distributed_exec.jl | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/logger.jl b/src/logger.jl index 9b40f97..babda97 100644 --- a/src/logger.jl +++ b/src/logger.jl @@ -23,5 +23,5 @@ end function Logging.handle_message(logger::RemoteLogger, level::Logging.LogLevel, message, _module, _group, _id, _file, _line; kwargs...) @nospecialize - remote_do(logmsg_code, logger.pid, level, message, _module, _group, _id, _file, _line; pid=myid(), kwargs...) + remote_do(logmsg, logger.pid, level, message, _module, _group, _id, _file, _line; pid=myid(), kwargs...) end diff --git a/test/distributed_exec.jl b/test/distributed_exec.jl index 14939c8..c4ca6e5 100644 --- a/test/distributed_exec.jl +++ b/test/distributed_exec.jl @@ -1929,10 +1929,11 @@ end # test logging w = only(addprocs(1)) +@everywhere using Logging @test_logs (:info, "from pid $w") begin prev_logger = global_logger(current_logger()) try - wait(@spawnat w with_env(RemoteLogger(1)) do + wait(@spawnat w with_logger(RemoteLogger(1)) do @info("from pid $(myid())") end) finally From 28980db67a7bae97e5d246d159e3f761931f2d7a Mon Sep 17 00:00:00 2001 From: theogf Date: Thu, 14 Mar 2024 10:05:01 +0100 Subject: [PATCH 3/6] add to README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 76f6355..4269c63 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,7 @@ For controlling other processes via RPC: For communicating between processes in the style of a channel or stream: - `RemoteChannel` - a `Channel`-like object that can be `put!` to or `take!` from any process +- `RemoteLogger` - an `AbstractLogger` forwarding logs to a given worker For controlling multiple processes at once: From 70426ee4dd2fddedc70679bd65d4b920cdf3c368 Mon Sep 17 00:00:00 2001 From: theogf Date: Thu, 14 Mar 2024 10:08:09 +0100 Subject: [PATCH 4/6] rm Manifest --- Manifest.toml | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 Manifest.toml diff --git a/Manifest.toml b/Manifest.toml deleted file mode 100644 index c7778f9..0000000 --- a/Manifest.toml +++ /dev/null @@ -1,22 +0,0 @@ -# This file is machine-generated - editing it directly is not advised - -julia_version = "1.10.2" -manifest_format = "2.0" -project_hash = "018bb01ed144ca156f5b8524b61e9dc7c0db49e4" - -[[deps.Logging]] -uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" - -[[deps.Random]] -deps = ["SHA"] -uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" - -[[deps.SHA]] -uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" -version = "0.7.0" - -[[deps.Serialization]] -uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" - -[[deps.Sockets]] -uuid = "6462fe0b-24de-5631-8697-dd941f90decc" From 882f55d82ee771aac470f333432fa32944b79827 Mon Sep 17 00:00:00 2001 From: theogf Date: Sat, 20 Jul 2024 15:43:41 +0200 Subject: [PATCH 5/6] Add missing doc reference Co-authored-by: simonbyrne --- docs/src/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/src/index.md b/docs/src/index.md index 22d63ce..88fccd9 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -48,6 +48,7 @@ Distributed.channel_from_id Distributed.worker_id_from_socket Distributed.cluster_cookie() Distributed.cluster_cookie(::Any) +Distributed.RemoteLogger ``` ## Cluster Manager Interface From e9b4d9fa1e7d45cb2997ba33a87cff09221a646e Mon Sep 17 00:00:00 2001 From: theogf Date: Sat, 20 Jul 2024 15:44:58 +0200 Subject: [PATCH 6/6] Fix typo --- src/logger.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/logger.jl b/src/logger.jl index babda97..3673cd4 100644 --- a/src/logger.jl +++ b/src/logger.jl @@ -1,7 +1,7 @@ """ RemoteLogger(pid=1, min_level=Info) -Logger that forward all logging to worker `pid` via `remote_do` along with +Logger that forwards all logging to worker `pid` via `remote_do` along with adding the current worker `id` as a `pid` kwarg. """ struct RemoteLogger <: Logging.AbstractLogger @@ -17,7 +17,7 @@ Logging.shouldlog(logger::RemoteLogger, level, _module, group, id) = true # TODO: probably should live in base/logging.jl? function logmsg(level::Logging.LogLevel, message, _module, _group, _id, _file, _line; kwargs...) - Logging.@logmsg level message _module=_module _group=_group _id=_id _file=_file _line=_line kwargs... + Logging.@logmsg level message _module = _module _group = _group _id = _id _file = _file _line = _line kwargs... end function Logging.handle_message(logger::RemoteLogger, level::Logging.LogLevel, message, _module, _group, _id,