diff --git a/dd-java-agent/agent-bootstrap/build.gradle b/dd-java-agent/agent-bootstrap/build.gradle index bf9f1742443..99334877866 100644 --- a/dd-java-agent/agent-bootstrap/build.gradle +++ b/dd-java-agent/agent-bootstrap/build.gradle @@ -23,6 +23,7 @@ dependencies { api project(':dd-java-agent:agent-logging') api project(':dd-java-agent:agent-debugger:debugger-bootstrap') api project(':components:json') + api libs.slf4j // ^ Generally a bad idea for libraries, but we're shadowing. diff --git a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java index 00b54848832..f2d3e044dd0 100644 --- a/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java +++ b/dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java @@ -20,6 +20,7 @@ import datadog.trace.api.appsec.AppSecEventTracker; import datadog.trace.api.config.AppSecConfig; import datadog.trace.api.config.CiVisibilityConfig; +import datadog.trace.api.config.CrashTrackingConfig; import datadog.trace.api.config.CwsConfig; import datadog.trace.api.config.DebuggerConfig; import datadog.trace.api.config.GeneralConfig; @@ -54,6 +55,7 @@ import java.net.URISyntaxException; import java.net.URL; import java.security.CodeSource; +import java.util.Arrays; import java.util.EnumSet; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -95,6 +97,9 @@ private enum AgentFeature { TRACING(TraceInstrumentationConfig.TRACE_ENABLED, true), JMXFETCH(JmxFetchConfig.JMX_FETCH_ENABLED, true), STARTUP_LOGS(GeneralConfig.STARTUP_LOGS_ENABLED, DEFAULT_STARTUP_LOGS_ENABLED), + CRASH_TRACKING( + CrashTrackingConfig.CRASH_TRACKING_ENABLED, + CrashTrackingConfig.CRASH_TRACKING_ENABLED_DEFAULT), PROFILING(ProfilingConfig.PROFILING_ENABLED, false), APPSEC(AppSecConfig.APPSEC_ENABLED, false), IAST(IastConfig.IAST_ENABLED, false), @@ -146,9 +151,11 @@ public boolean isEnabledByDefault() { private static ClassLoader AGENT_CLASSLOADER = null; private static volatile Runnable PROFILER_INIT_AFTER_JMX = null; + private static volatile Runnable CRASHTRACKER_INIT_AFTER_JMX = null; private static boolean jmxFetchEnabled = true; private static boolean profilingEnabled = false; + private static boolean crashTrackingEnabled = false; private static boolean appSecEnabled; private static boolean appSecFullyDisabled; private static boolean remoteConfigEnabled = true; @@ -276,6 +283,7 @@ public static void start( jmxFetchEnabled = isFeatureEnabled(AgentFeature.JMXFETCH); profilingEnabled = isFeatureEnabled(AgentFeature.PROFILING); + crashTrackingEnabled = isFeatureEnabled(AgentFeature.CRASH_TRACKING); usmEnabled = isFeatureEnabled(AgentFeature.USM); appSecEnabled = isFeatureEnabled(AgentFeature.APPSEC); appSecFullyDisabled = isFullyDisabled(AgentFeature.APPSEC); @@ -293,6 +301,18 @@ public static void start( patchJPSAccess(inst); + // We need to run the crashtracking initialization after all the config has been resolved + if (crashTrackingEnabled) { + if (Platform.isJavaVersionAtLeast(9)) { + // it is safe to initialize crashtracking 'in-line' + initializeCrashTracking(); + } else { + // for Java 8 we are relying on JMX to give us the process PID + // we need to delay the crash tracking initialization until JMX is available + CRASHTRACKER_INIT_AFTER_JMX = Agent::initializeCrashTracking; + } + } + if (profilingEnabled) { if (!isOracleJDK8()) { // Profiling agent startup code is written in a way to allow `startProfilingAgent` be called @@ -305,13 +325,7 @@ public static void start( // Profiling can not run early on Oracle JDK 8 because it will cause JFR initialization // deadlock. // Oracle JDK 8 JFR controller requires JMX so register an 'after-jmx-initialized' callback. - PROFILER_INIT_AFTER_JMX = - new Runnable() { - @Override - public void run() { - startProfilingAgent(false, inst); - } - }; + PROFILER_INIT_AFTER_JMX = () -> startProfilingAgent(false, inst); } } @@ -757,25 +771,33 @@ private static synchronized void startJmx() { if (jmxStarting.getAndSet(true)) { return; // another thread is already in startJmx } - // error tracking initialization relies on JMX being available - initializeErrorTracking(); if (jmxFetchEnabled) { startJmxFetch(); } initializeJmxSystemAccessProvider(AGENT_CLASSLOADER); + if (crashTrackingEnabled && CRASHTRACKER_INIT_AFTER_JMX != null) { + try { + CRASHTRACKER_INIT_AFTER_JMX.run(); + } finally { + CRASHTRACKER_INIT_AFTER_JMX = null; + } + } if (profilingEnabled) { registerDeadlockDetectionEvent(); registerSmapEntryEvent(); if (PROFILER_INIT_AFTER_JMX != null) { - if (getJmxStartDelay() == 0) { - log.debug("Waiting for profiler initialization"); - AgentTaskScheduler.INSTANCE.scheduleWithJitter( - PROFILER_INIT_AFTER_JMX, 500, TimeUnit.MILLISECONDS); - } else { - log.debug("Initializing profiler"); - PROFILER_INIT_AFTER_JMX.run(); + try { + if (getJmxStartDelay() == 0) { + log.debug("Waiting for profiler initialization"); + AgentTaskScheduler.INSTANCE.scheduleWithJitter( + PROFILER_INIT_AFTER_JMX, 500, TimeUnit.MILLISECONDS); + } else { + log.debug("Initializing profiler"); + PROFILER_INIT_AFTER_JMX.run(); + } + } finally { + PROFILER_INIT_AFTER_JMX = null; } - PROFILER_INIT_AFTER_JMX = null; } } } @@ -1017,16 +1039,38 @@ private static void stopTelemetry() { } } - private static void initializeErrorTracking() { + private static void initializeCrashTracking() { + initializeCrashTracking(true); + } + + private static void initializeCrashTracking(boolean delayed) { if (Platform.isJ9()) { // TODO currently crash tracking is supported only for HotSpot based JVMs return; } + log.debug("Initializing crashtracking"); try { - Class clz = AGENT_CLASSLOADER.loadClass("com.datadog.crashtracking.ScriptInitializer"); - clz.getMethod("initialize").invoke(null); + Class clz = AGENT_CLASSLOADER.loadClass("datadog.crashtracking.Initializer"); + // first try to use the JVMAccess using the native library + boolean rslt = (boolean) clz.getMethod("initialize", boolean.class).invoke(null, false); + if (!rslt) { + if (delayed) { + // already delayed initialization, so no need to reschedule it again + // just call initialize and force JMX + rslt = (boolean) clz.getMethod("initialize", boolean.class).invoke(null, true); + } else { + // delayed initialization, so we need to reschedule it and mark as delayed + CRASHTRACKER_INIT_AFTER_JMX = Agent::initializeCrashTracking; + } + } + if (rslt) { + log.debug("Crashtracking initialized"); + } else { + log.debug( + SEND_TELEMETRY, "Crashtracking failed to initialize. No additional details available."); + } } catch (Throwable t) { - log.debug("Unable to initialize crash uploader", t); + log.debug(SEND_TELEMETRY, "Unable to initialize crashtracking", t); } } @@ -1125,8 +1169,11 @@ public void withTracer(TracerAPI tracer) { } }); } - } catch (final Throwable ex) { - log.error("Throwable thrown while starting profiling agent", ex); + } catch (final Throwable t) { + log.error( + SEND_TELEMETRY, + "Throwable thrown while starting profiling agent " + + Arrays.toString(t.getCause().getStackTrace())); } finally { Thread.currentThread().setContextClassLoader(contextLoader); } diff --git a/dd-java-agent/agent-crashtracking/build.gradle b/dd-java-agent/agent-crashtracking/build.gradle index 9b4947aeb75..60a2873dd21 100644 --- a/dd-java-agent/agent-crashtracking/build.gradle +++ b/dd-java-agent/agent-crashtracking/build.gradle @@ -16,6 +16,7 @@ dependencies { implementation project(':internal-api') implementation project(':utils:container-utils') implementation project(':utils:version-utils') + implementation project(path: ':dd-java-agent:ddprof-lib', configuration: 'shadow') implementation libs.okhttp implementation libs.moshi diff --git a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/ScriptInitializer.java b/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/ScriptInitializer.java deleted file mode 100644 index e07be4b7e3f..00000000000 --- a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/ScriptInitializer.java +++ /dev/null @@ -1,165 +0,0 @@ -package com.datadog.crashtracking; - -import static datadog.trace.util.AgentThreadFactory.AGENT_THREAD_GROUP; -import static java.util.Comparator.reverseOrder; -import static java.util.Locale.ROOT; - -import com.sun.management.HotSpotDiagnosticMXBean; -import datadog.trace.api.Platform; -import datadog.trace.util.PidHelper; -import java.io.BufferedWriter; -import java.io.IOException; -import java.io.InputStream; -import java.lang.management.ManagementFactory; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.function.Predicate; -import java.util.stream.Stream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public final class ScriptInitializer { - static final Logger LOG = LoggerFactory.getLogger(ScriptInitializer.class); - static final String PID_PREFIX = "_pid"; - static final String RWXRWXRWX = "rwxrwxrwx"; - static final String R_XR_XR_X = "r-xr-xr-x"; - - public static void initialize() { - // this is HotSpot specific implementation (eg. will not work for IBM J9) - HotSpotDiagnosticMXBean diagBean = - ManagementFactory.getPlatformMXBean(HotSpotDiagnosticMXBean.class); - - initializeCrashUploader(diagBean); - initializeOOMENotifier(diagBean); - } - - static InputStream getCrashUploaderTemplate() { - String name = Platform.isWindows() ? "upload_crash.bat" : "upload_crash.sh"; - return CrashUploader.class.getResourceAsStream(name); - } - - static InputStream getOomeNotifierTemplate() { - String name = Platform.isWindows() ? "notify_oome.bat" : "notify_oome.sh"; - return OOMENotifier.class.getResourceAsStream(name); - } - - static String findAgentJar() { - String agentPath = null; - String classResourceName = CrashUploader.class.getName().replace('.', '/') + ".class"; - URL classResource = CrashUploader.class.getClassLoader().getResource(classResourceName); - String selfClass = classResource == null ? "null" : classResource.toString(); - if (selfClass.startsWith("jar:file:")) { - int idx = selfClass.lastIndexOf(".jar"); - if (idx > -1) { - agentPath = selfClass.substring(9, idx + 4); - } - } - // test harness env is different; use the known project structure to locate the agent jar - else if (selfClass.startsWith("file:")) { - int idx = selfClass.lastIndexOf("dd-java-agent"); - if (idx > -1) { - Path libsPath = Paths.get(selfClass.substring(5, idx + 13), "build", "libs"); - try (Stream files = Files.walk(libsPath)) { - Predicate isJarFile = - p -> p.getFileName().toString().toLowerCase(ROOT).endsWith(".jar"); - agentPath = - files - .sorted(reverseOrder()) - .filter(isJarFile) - .findFirst() - .map(Path::toString) - .orElse(null); - } catch (IOException ignored) { - // Ignore failure to get agent path - } - } - } - return agentPath; - } - - static void writeConfig(Path scriptPath, String... entries) { - String cfgFileName = getBaseName(scriptPath) + PID_PREFIX + PidHelper.getPid() + ".cfg"; - Path cfgPath = scriptPath.resolveSibling(cfgFileName); - LOG.debug("Writing config file: {}", cfgPath); - try (BufferedWriter bw = Files.newBufferedWriter(cfgPath)) { - for (int i = 0; i < entries.length; i += 2) { - bw.write(entries[i]); - bw.write('='); - bw.write(entries[i + 1]); - bw.newLine(); - } - bw.write("java_home=" + System.getProperty("java.home")); - bw.newLine(); - - Runtime.getRuntime() - .addShutdownHook( - new Thread( - AGENT_THREAD_GROUP, - () -> { - try { - LOG.debug("Deleting config file: {}", cfgPath); - Files.deleteIfExists(cfgPath); - } catch (IOException e) { - LOG.warn("Failed deleting config file: {}", cfgPath, e); - } - })); - LOG.debug("Config file written: {}", cfgPath); - } catch (IOException e) { - LOG.warn("Failed writing config file: {}", cfgPath); - try { - Files.deleteIfExists(cfgPath); - } catch (IOException ignored) { - // ignore - } - } - } - - private static String getBaseName(Path path) { - String filename = path.getFileName().toString(); - int dotIndex = filename.lastIndexOf('.'); - if (dotIndex == -1) { - return filename; - } - return filename.substring(0, dotIndex); - } - - /** - * If the value of `-XX:OnError` JVM argument is referring to `dd_crash_uploader.sh` or - * `dd_crash_uploader.bat` and the script does not exist it will be created and prefilled with - * code ensuring the error log upload will be triggered on JVM crash. - */ - private static void initializeCrashUploader(HotSpotDiagnosticMXBean diagBean) { - try { - String onErrorVal = diagBean.getVMOption("OnError").getValue(); - String onErrorFile = diagBean.getVMOption("ErrorFile").getValue(); - CrashUploaderScriptInitializer.initialize(onErrorVal, onErrorFile); - } catch (Throwable t) { - logInitializationError( - "Unexpected exception while creating custom crash upload script. Crash tracking will not work properly.", - t); - } - } - - private static void initializeOOMENotifier(HotSpotDiagnosticMXBean diagBean) { - try { - String onOutOfMemoryVal = diagBean.getVMOption("OnOutOfMemoryError").getValue(); - OOMENotifierScriptInitializer.initialize(onOutOfMemoryVal); - } catch (Throwable t) { - logInitializationError( - "Unexpected exception while initializing OOME notifier. OOMEs will not be tracked.", t); - } - } - - private static void logInitializationError(String msg, Throwable t) { - if (LOG.isDebugEnabled()) { - LOG.warn("{}", msg, t); - } else { - LOG.warn( - "{} [{}] (Change the logging level to debug to see the full stacktrace)", - msg, - t.getMessage()); - } - } -} diff --git a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/CrashLogParser.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashLogParser.java similarity index 52% rename from dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/CrashLogParser.java rename to dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashLogParser.java index caa5c9468ea..3fecb528759 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/CrashLogParser.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashLogParser.java @@ -1,7 +1,7 @@ -package com.datadog.crashtracking; +package datadog.crashtracking; -import com.datadog.crashtracking.dto.CrashLog; -import com.datadog.crashtracking.parsers.HotspotCrashLogParser; +import datadog.crashtracking.dto.CrashLog; +import datadog.crashtracking.parsers.HotspotCrashLogParser; public final class CrashLogParser { public static CrashLog fromHotspotCrashLog(String logText) { diff --git a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/CrashUploader.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java similarity index 99% rename from dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/CrashUploader.java rename to dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java index 6354106dd0e..b8cf20dd8f9 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/CrashUploader.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java @@ -1,4 +1,4 @@ -package com.datadog.crashtracking; +package datadog.crashtracking; import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_PROXY_HOST; import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_PROXY_PASSWORD; @@ -7,11 +7,11 @@ import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_UPLOAD_TIMEOUT; import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_UPLOAD_TIMEOUT_DEFAULT; -import com.datadog.crashtracking.dto.CrashLog; import com.squareup.moshi.JsonWriter; import datadog.common.container.ContainerInfo; import datadog.common.version.VersionInfo; import datadog.communication.http.OkHttpUtils; +import datadog.crashtracking.dto.CrashLog; import datadog.trace.api.Config; import datadog.trace.api.DDTags; import datadog.trace.bootstrap.config.provider.ConfigProvider; diff --git a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/CrashUploaderScriptInitializer.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploaderScriptInitializer.java similarity index 82% rename from dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/CrashUploaderScriptInitializer.java rename to dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploaderScriptInitializer.java index 6579c151f74..e3283fd96af 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/CrashUploaderScriptInitializer.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploaderScriptInitializer.java @@ -1,11 +1,12 @@ -package com.datadog.crashtracking; +package datadog.crashtracking; -import static com.datadog.crashtracking.ScriptInitializer.LOG; -import static com.datadog.crashtracking.ScriptInitializer.RWXRWXRWX; -import static com.datadog.crashtracking.ScriptInitializer.R_XR_XR_X; -import static com.datadog.crashtracking.ScriptInitializer.findAgentJar; -import static com.datadog.crashtracking.ScriptInitializer.getCrashUploaderTemplate; -import static com.datadog.crashtracking.ScriptInitializer.writeConfig; +import static datadog.crashtracking.Initializer.LOG; +import static datadog.crashtracking.Initializer.RWXRWXRWX; +import static datadog.crashtracking.Initializer.R_XR_XR_X; +import static datadog.crashtracking.Initializer.findAgentJar; +import static datadog.crashtracking.Initializer.getCrashUploaderTemplate; +import static datadog.crashtracking.Initializer.writeConfig; +import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY; import static java.nio.file.attribute.PosixFilePermissions.asFileAttribute; import static java.nio.file.attribute.PosixFilePermissions.fromString; import static java.util.Locale.ROOT; @@ -30,7 +31,8 @@ private CrashUploaderScriptInitializer() {} // @VisibleForTests static void initialize(String onErrorVal, String onErrorFile) { if (onErrorVal == null || onErrorVal.isEmpty()) { - LOG.debug("'-XX:OnError' argument was not provided. Crash tracking is disabled."); + LOG.debug( + SEND_TELEMETRY, "'-XX:OnError' argument was not provided. Crash tracking is disabled."); return; } if (onErrorFile == null || onErrorFile.isEmpty()) { @@ -42,7 +44,7 @@ static void initialize(String onErrorVal, String onErrorFile) { String agentJar = findAgentJar(); if (agentJar == null) { - LOG.warn("Unable to locate the agent jar. {}", SETUP_FAILURE_MESSAGE); + LOG.warn(SEND_TELEMETRY, "Unable to locate the agent jar. {}", SETUP_FAILURE_MESSAGE); return; } @@ -63,6 +65,7 @@ private static boolean copyCrashUploaderScript( Files.createDirectories(scriptDirectory, asFileAttribute(fromString(RWXRWXRWX))); } catch (UnsupportedOperationException e) { LOG.warn( + SEND_TELEMETRY, "Unsupported permissions {} for {}. {}", RWXRWXRWX, scriptDirectory, @@ -71,11 +74,13 @@ private static boolean copyCrashUploaderScript( } catch (FileAlreadyExistsException ignored) { // can be safely ignored; if the folder exists we will just reuse it if (!Files.isWritable(scriptDirectory)) { - LOG.warn("Read only directory {}. {}", scriptDirectory, SETUP_FAILURE_MESSAGE); + LOG.warn( + SEND_TELEMETRY, "Read only directory {}. {}", scriptDirectory, SETUP_FAILURE_MESSAGE); return false; } } catch (IOException e) { LOG.warn( + SEND_TELEMETRY, "Failed to create writable crash tracking script folder {}. {}", scriptDirectory, SETUP_FAILURE_MESSAGE); diff --git a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/Initializer.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/Initializer.java new file mode 100644 index 00000000000..0346902af54 --- /dev/null +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/Initializer.java @@ -0,0 +1,310 @@ +package datadog.crashtracking; + +import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY; +import static datadog.trace.util.AgentThreadFactory.AGENT_THREAD_GROUP; +import static java.util.Comparator.reverseOrder; +import static java.util.Locale.ROOT; + +import com.datadoghq.profiler.JVMAccess; +import com.sun.management.HotSpotDiagnosticMXBean; +import datadog.libs.ddprof.DdprofLibraryLoader; +import datadog.trace.api.Platform; +import datadog.trace.util.PidHelper; +import datadog.trace.util.TempLocationManager; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.InputStream; +import java.lang.management.ManagementFactory; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.StringTokenizer; +import java.util.function.Predicate; +import java.util.stream.Stream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public final class Initializer { + static final Logger LOG = LoggerFactory.getLogger(Initializer.class); + static final String PID_PREFIX = "_pid"; + static final String RWXRWXRWX = "rwxrwxrwx"; + static final String R_XR_XR_X = "r-xr-xr-x"; + + private interface FlagAccess { + String getValue(String flagName); + + boolean setValue(String flagName, String value); + } + + private static final class JVMFlagAccess implements FlagAccess { + private final JVMAccess.Flags flags; + + JVMFlagAccess(JVMAccess.Flags flags) { + this.flags = flags; + } + + @Override + public String getValue(String flagName) { + return flags.getStringFlag(flagName); + } + + @Override + public boolean setValue(String flagName, String value) { + flags.setStringFlag(flagName, value); + return flags.getStringFlag(flagName).equals(value); + } + } + + private static final class JMXFlagAccess implements FlagAccess { + private final HotSpotDiagnosticMXBean diagBean; + + JMXFlagAccess(HotSpotDiagnosticMXBean diagBean) { + this.diagBean = diagBean; + } + + @Override + public String getValue(String flagName) { + return diagBean.getVMOption(flagName).getValue(); + } + + @Override + public boolean setValue(String flagName, String value) { + // can not really set the underlying JVM flag value + // let's pretend everything went just fine + return true; + } + } + + public static boolean requiresJMX() { + return DdprofLibraryLoader.jvmAccess().getReasonNotLoaded() != null; + } + + public static boolean initialize(boolean forceJmx) { + try { + FlagAccess access = null; + if (forceJmx) { + access = + new JMXFlagAccess(ManagementFactory.getPlatformMXBean(HotSpotDiagnosticMXBean.class)); + } else { + DdprofLibraryLoader.JVMAccessHolder jvmAccessHolder = DdprofLibraryLoader.jvmAccess(); + + if (jvmAccessHolder.getReasonNotLoaded() != null) { + LOG.debug( + SEND_TELEMETRY, + "Failed to load JVM access library: {}. Crash tracking will need to rely on user provided JVM arguments.", + jvmAccessHolder.getReasonNotLoaded().getMessage()); + return false; + } else { + JVMAccess.Flags flags = jvmAccessHolder.getComponent().flags(); + access = new JVMFlagAccess(flags); + } + } + + initializeCrashUploader(access); + initializeOOMENotifier(access); + return true; + } catch (Throwable t) { + LOG.debug(SEND_TELEMETRY, "Failed to initialize crash tracking: {}", t.getMessage(), t); + } + return false; + } + + static InputStream getCrashUploaderTemplate() { + String name = Platform.isWindows() ? "upload_crash.bat" : "upload_crash.sh"; + return CrashUploader.class.getResourceAsStream(name); + } + + static InputStream getOomeNotifierTemplate() { + String name = Platform.isWindows() ? "notify_oome.bat" : "notify_oome.sh"; + return OOMENotifier.class.getResourceAsStream(name); + } + + static String findAgentJar() { + String agentPath = null; + String classResourceName = CrashUploader.class.getName().replace('.', '/') + ".class"; + URL classResource = CrashUploader.class.getClassLoader().getResource(classResourceName); + String selfClass = classResource == null ? "null" : classResource.toString(); + if (selfClass.startsWith("jar:file:")) { + int idx = selfClass.lastIndexOf(".jar"); + if (idx > -1) { + agentPath = selfClass.substring(9, idx + 4); + } + } + // test harness env is different; use the known project structure to locate the agent jar + else if (selfClass.startsWith("file:")) { + int idx = selfClass.lastIndexOf("dd-java-agent"); + if (idx > -1) { + Path libsPath = Paths.get(selfClass.substring(5, idx + 13), "build", "libs"); + try (Stream files = Files.walk(libsPath)) { + Predicate isJarFile = + p -> p.getFileName().toString().toLowerCase(ROOT).endsWith(".jar"); + agentPath = + files + .sorted(reverseOrder()) + .filter(isJarFile) + .findFirst() + .map(Path::toString) + .orElse(null); + } catch (IOException ignored) { + // Ignore failure to get agent path + } + } + } + return agentPath; + } + + static void writeConfig(Path scriptPath, String... entries) { + String cfgFileName = getBaseName(scriptPath) + PID_PREFIX + PidHelper.getPid() + ".cfg"; + Path cfgPath = scriptPath.resolveSibling(cfgFileName); + LOG.debug("Writing config file: {}", cfgPath); + try (BufferedWriter bw = Files.newBufferedWriter(cfgPath)) { + for (int i = 0; i < entries.length; i += 2) { + bw.write(entries[i]); + bw.write('='); + bw.write(entries[i + 1]); + bw.newLine(); + } + bw.write("java_home=" + System.getProperty("java.home")); + bw.newLine(); + + Runtime.getRuntime() + .addShutdownHook( + new Thread( + AGENT_THREAD_GROUP, + () -> { + try { + LOG.debug("Deleting config file: {}", cfgPath); + Files.deleteIfExists(cfgPath); + } catch (IOException e) { + LOG.warn(SEND_TELEMETRY, "Failed deleting config file: {}", cfgPath, e); + } + })); + LOG.debug("Config file written: {}", cfgPath); + } catch (IOException e) { + LOG.warn(SEND_TELEMETRY, "Failed writing config file: {}", cfgPath); + try { + Files.deleteIfExists(cfgPath); + } catch (IOException ignored) { + // ignore + } + } + } + + private static String getBaseName(Path path) { + String filename = path.getFileName().toString(); + int dotIndex = filename.lastIndexOf('.'); + if (dotIndex == -1) { + return filename; + } + return filename.substring(0, dotIndex); + } + + /** + * If the value of `-XX:OnError` JVM argument is referring to `dd_crash_uploader.sh` or + * `dd_crash_uploader.bat` and the script does not exist it will be created and prefilled with + * code ensuring the error log upload will be triggered on JVM crash. + */ + private static void initializeCrashUploader(FlagAccess flags) { + try { + String onErrorVal = flags.getValue("OnError"); + String onErrorFile = flags.getValue("ErrorFile"); + + String uploadScript = getScript("dd_crash_uploader"); + if (onErrorVal == null || onErrorVal.isEmpty()) { + onErrorVal = uploadScript; + } else if (!onErrorVal.contains("dd_crash_uploader")) { + // we can chain scripts so let's preserve the original value in addition to our crash + // uploader + onErrorVal = uploadScript + "; " + onErrorVal; + } else { + StringTokenizer st = new StringTokenizer(onErrorVal, ";"); + while (st.hasMoreTokens()) { + String part = st.nextToken(); + if (part.trim().contains("dd_crash_uploader")) { + // reuse the existing script name + uploadScript = part.trim().replace(" %p", ""); + break; + } + } + } + + // set the JVM flag + boolean rslt = flags.setValue("OnError", onErrorVal); + if (!rslt && LOG.isDebugEnabled()) { + LOG.debug( + SEND_TELEMETRY, + "Unable to set OnError flag to {}. Crash-tracking may not work.", + onErrorVal); + } + + CrashUploaderScriptInitializer.initialize(uploadScript, onErrorFile); + } catch (Throwable t) { + logInitializationError( + "Unexpected exception while creating custom crash upload script. Crash tracking will not work properly.", + t); + } + } + + private static void initializeOOMENotifier(FlagAccess flags) { + try { + String onOutOfMemoryVal = flags.getValue("OnOutOfMemoryError"); + + String notifierScript = getScript("dd_oome_notifier"); + + if (onOutOfMemoryVal == null || onOutOfMemoryVal.isEmpty()) { + onOutOfMemoryVal = notifierScript; + } else if (!onOutOfMemoryVal.contains("dd_oome_notifier")) { + // we can chain scripts so let's preserve the original value in addition to our oome tracker + onOutOfMemoryVal = notifierScript + "; " + onOutOfMemoryVal; + } else { + StringTokenizer st = new StringTokenizer(onOutOfMemoryVal, ";"); + while (st.hasMoreTokens()) { + String part = st.nextToken(); + if (part.trim().contains("dd_oome_notifier")) { + // reuse the existing script name + notifierScript = part.trim(); + break; + } + } + } + + // set the JVM flag + boolean rslt = flags.setValue("OnOutOfMemoryError", onOutOfMemoryVal); + if (!rslt && LOG.isDebugEnabled()) { + LOG.debug( + SEND_TELEMETRY, + "Unable to set OnOutOfMemoryError flag to {}. OOME tracking may not work.", + onOutOfMemoryVal); + } + + OOMENotifierScriptInitializer.initialize(notifierScript); + } catch (Throwable t) { + logInitializationError( + "Unexpected exception while initializing OOME notifier. OOMEs will not be tracked.", t); + } + } + + private static String getScript(String scriptName) { + return TempLocationManager.getInstance().getTempDir().toString() + + "/" + + getScriptFileName(scriptName) + + " %p"; + } + + private static String getScriptFileName(String scriptName) { + return scriptName + "." + (Platform.isWindows() ? "bat" : "sh"); + } + + private static void logInitializationError(String msg, Throwable t) { + if (LOG.isDebugEnabled()) { + LOG.warn(SEND_TELEMETRY, "{}", msg, t); + } else { + LOG.warn( + SEND_TELEMETRY, + "{} [{}] (Change the logging level to debug to see the full stacktrace)", + msg, + t.getMessage()); + } + } +} diff --git a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/OOMENotifier.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/OOMENotifier.java similarity index 96% rename from dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/OOMENotifier.java rename to dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/OOMENotifier.java index c9728426a98..4e8486c1c39 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/OOMENotifier.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/OOMENotifier.java @@ -1,4 +1,4 @@ -package com.datadog.crashtracking; +package datadog.crashtracking; import static datadog.communication.monitor.DDAgentStatsDClientManager.statsDClientManager; diff --git a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/OOMENotifierScriptInitializer.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/OOMENotifierScriptInitializer.java similarity index 84% rename from dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/OOMENotifierScriptInitializer.java rename to dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/OOMENotifierScriptInitializer.java index 66fc6fb5274..f3655787767 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/OOMENotifierScriptInitializer.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/OOMENotifierScriptInitializer.java @@ -1,12 +1,13 @@ -package com.datadog.crashtracking; - -import static com.datadog.crashtracking.ScriptInitializer.LOG; -import static com.datadog.crashtracking.ScriptInitializer.PID_PREFIX; -import static com.datadog.crashtracking.ScriptInitializer.RWXRWXRWX; -import static com.datadog.crashtracking.ScriptInitializer.R_XR_XR_X; -import static com.datadog.crashtracking.ScriptInitializer.findAgentJar; -import static com.datadog.crashtracking.ScriptInitializer.getOomeNotifierTemplate; -import static com.datadog.crashtracking.ScriptInitializer.writeConfig; +package datadog.crashtracking; + +import static datadog.crashtracking.Initializer.LOG; +import static datadog.crashtracking.Initializer.PID_PREFIX; +import static datadog.crashtracking.Initializer.RWXRWXRWX; +import static datadog.crashtracking.Initializer.R_XR_XR_X; +import static datadog.crashtracking.Initializer.findAgentJar; +import static datadog.crashtracking.Initializer.getOomeNotifierTemplate; +import static datadog.crashtracking.Initializer.writeConfig; +import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY; import static java.nio.file.FileVisitResult.CONTINUE; import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; import static java.nio.file.attribute.PosixFilePermissions.asFileAttribute; @@ -37,19 +38,24 @@ private OOMENotifierScriptInitializer() {} // @VisibleForTests static void initialize(String onOutOfMemoryVal) { if (onOutOfMemoryVal == null || onOutOfMemoryVal.isEmpty()) { - LOG.debug("'-XX:OnOutOfMemoryError' argument was not provided. OOME tracking is disabled."); + LOG.debug( + SEND_TELEMETRY, + "'-XX:OnOutOfMemoryError' argument was not provided. OOME tracking is disabled."); return; } Path scriptPath = getOOMEScripPath(onOutOfMemoryVal); if (scriptPath == null) { LOG.debug( + SEND_TELEMETRY, "OOME notifier script value ({}) does not follow the expected format: /dd_oome_notifier.(sh|bat) %p. OOME tracking is disabled.", onOutOfMemoryVal); return; } String agentJar = findAgentJar(); if (agentJar == null) { - LOG.warn("Unable to locate the agent jar. OOME notification will not work properly."); + LOG.warn( + SEND_TELEMETRY, + "Unable to locate the agent jar. OOME notification will not work properly."); return; } if (!copyOOMEscript(scriptPath)) { @@ -84,6 +90,7 @@ private static boolean copyOOMEscript(Path scriptPath) { Files.createDirectories(scriptDirectory, asFileAttribute(fromString(RWXRWXRWX))); } catch (UnsupportedOperationException e) { LOG.warn( + SEND_TELEMETRY, "Unsupported permissions {} for {}. OOME notification will not work properly.", RWXRWXRWX, scriptDirectory); @@ -92,11 +99,14 @@ private static boolean copyOOMEscript(Path scriptPath) { // can be safely ignored; if the folder exists we will just reuse it if (!Files.isWritable(scriptDirectory)) { LOG.warn( - "Read only directory {}. OOME notification will not work properly.", scriptDirectory); + SEND_TELEMETRY, + "Read only directory {}. OOME notification will not work properly.", + scriptDirectory); return false; } } catch (IOException e) { LOG.warn( + SEND_TELEMETRY, "Failed to create writable OOME script folder {}. OOME notification will not work properly.", scriptDirectory); return false; @@ -107,7 +117,9 @@ private static boolean copyOOMEscript(Path scriptPath) { Files.setPosixFilePermissions(scriptPath, fromString(R_XR_XR_X)); } catch (IOException e) { LOG.warn( - "Failed to copy OOME script {}. OOME notification will not work properly.", scriptPath); + SEND_TELEMETRY, + "Failed to copy OOME script {}. OOME notification will not work properly.", + scriptPath); return false; } return true; diff --git a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/CrashLog.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/CrashLog.java similarity index 98% rename from dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/CrashLog.java rename to dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/CrashLog.java index adcc327719e..f410ac1c938 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/CrashLog.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/CrashLog.java @@ -1,4 +1,4 @@ -package com.datadog.crashtracking.dto; +package datadog.crashtracking.dto; import com.squareup.moshi.Json; import com.squareup.moshi.JsonAdapter; diff --git a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/ErrorData.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/ErrorData.java similarity index 96% rename from dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/ErrorData.java rename to dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/ErrorData.java index d79deed3e57..9ccb9a96e0d 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/ErrorData.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/ErrorData.java @@ -1,4 +1,4 @@ -package com.datadog.crashtracking.dto; +package datadog.crashtracking.dto; import com.squareup.moshi.Json; import java.util.Objects; diff --git a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/Metadata.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/Metadata.java similarity index 96% rename from dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/Metadata.java rename to dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/Metadata.java index 62e49398990..963c03954bd 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/Metadata.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/Metadata.java @@ -1,4 +1,4 @@ -package com.datadog.crashtracking.dto; +package datadog.crashtracking.dto; import com.squareup.moshi.Json; import java.util.Collections; diff --git a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/OSInfo.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/OSInfo.java similarity index 96% rename from dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/OSInfo.java rename to dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/OSInfo.java index 39963d0e9a4..793ec27aa4f 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/OSInfo.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/OSInfo.java @@ -1,4 +1,4 @@ -package com.datadog.crashtracking.dto; +package datadog.crashtracking.dto; import com.squareup.moshi.Json; import java.util.Objects; diff --git a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/ProcInfo.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/ProcInfo.java similarity index 92% rename from dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/ProcInfo.java rename to dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/ProcInfo.java index 13f9c05933a..36dd8f5b3b3 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/ProcInfo.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/ProcInfo.java @@ -1,4 +1,4 @@ -package com.datadog.crashtracking.dto; +package datadog.crashtracking.dto; import java.util.Objects; diff --git a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/SemanticVersion.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/SemanticVersion.java similarity index 98% rename from dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/SemanticVersion.java rename to dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/SemanticVersion.java index b39cfd56873..d604859829e 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/SemanticVersion.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/SemanticVersion.java @@ -1,4 +1,4 @@ -package com.datadog.crashtracking.dto; +package datadog.crashtracking.dto; import com.squareup.moshi.FromJson; import com.squareup.moshi.JsonReader; diff --git a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/StackFrame.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/StackFrame.java similarity index 94% rename from dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/StackFrame.java rename to dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/StackFrame.java index e3f14ca48ba..e4232936935 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/StackFrame.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/StackFrame.java @@ -1,4 +1,4 @@ -package com.datadog.crashtracking.dto; +package datadog.crashtracking.dto; import java.util.Objects; diff --git a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/StackTrace.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/StackTrace.java similarity index 94% rename from dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/StackTrace.java rename to dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/StackTrace.java index aa65b475e24..23a12ca2c69 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/dto/StackTrace.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/StackTrace.java @@ -1,4 +1,4 @@ -package com.datadog.crashtracking.dto; +package datadog.crashtracking.dto; import java.util.Arrays; import java.util.Objects; diff --git a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/parsers/HotspotCrashLogParser.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/parsers/HotspotCrashLogParser.java similarity index 94% rename from dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/parsers/HotspotCrashLogParser.java rename to dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/parsers/HotspotCrashLogParser.java index e869981ff61..bd9dc47bfaf 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/com/datadog/crashtracking/parsers/HotspotCrashLogParser.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/parsers/HotspotCrashLogParser.java @@ -1,16 +1,16 @@ -package com.datadog.crashtracking.parsers; +package datadog.crashtracking.parsers; import static java.time.format.DateTimeFormatter.ISO_OFFSET_DATE_TIME; -import com.datadog.crashtracking.dto.CrashLog; -import com.datadog.crashtracking.dto.ErrorData; -import com.datadog.crashtracking.dto.Metadata; -import com.datadog.crashtracking.dto.OSInfo; -import com.datadog.crashtracking.dto.ProcInfo; -import com.datadog.crashtracking.dto.SemanticVersion; -import com.datadog.crashtracking.dto.StackFrame; -import com.datadog.crashtracking.dto.StackTrace; import datadog.common.version.VersionInfo; +import datadog.crashtracking.dto.CrashLog; +import datadog.crashtracking.dto.ErrorData; +import datadog.crashtracking.dto.Metadata; +import datadog.crashtracking.dto.OSInfo; +import datadog.crashtracking.dto.ProcInfo; +import datadog.crashtracking.dto.SemanticVersion; +import datadog.crashtracking.dto.StackFrame; +import datadog.crashtracking.dto.StackTrace; import java.time.OffsetDateTime; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; diff --git a/dd-java-agent/agent-crashtracking/src/main/resources/com/datadog/crashtracking/notify_oome.bat b/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/notify_oome.bat similarity index 100% rename from dd-java-agent/agent-crashtracking/src/main/resources/com/datadog/crashtracking/notify_oome.bat rename to dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/notify_oome.bat diff --git a/dd-java-agent/agent-crashtracking/src/main/resources/com/datadog/crashtracking/notify_oome.sh b/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/notify_oome.sh similarity index 100% rename from dd-java-agent/agent-crashtracking/src/main/resources/com/datadog/crashtracking/notify_oome.sh rename to dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/notify_oome.sh diff --git a/dd-java-agent/agent-crashtracking/src/main/resources/com/datadog/crashtracking/upload_crash.bat b/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/upload_crash.bat similarity index 100% rename from dd-java-agent/agent-crashtracking/src/main/resources/com/datadog/crashtracking/upload_crash.bat rename to dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/upload_crash.bat diff --git a/dd-java-agent/agent-crashtracking/src/main/resources/com/datadog/crashtracking/upload_crash.sh b/dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/upload_crash.sh similarity index 100% rename from dd-java-agent/agent-crashtracking/src/main/resources/com/datadog/crashtracking/upload_crash.sh rename to dd-java-agent/agent-crashtracking/src/main/resources/datadog/crashtracking/upload_crash.sh diff --git a/dd-java-agent/agent-crashtracking/src/test/java/com/datadog/crashtracking/CrashUploaderTest.java b/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/CrashUploaderTest.java similarity index 99% rename from dd-java-agent/agent-crashtracking/src/test/java/com/datadog/crashtracking/CrashUploaderTest.java rename to dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/CrashUploaderTest.java index 2f5255a0440..8aef67b2301 100644 --- a/dd-java-agent/agent-crashtracking/src/test/java/com/datadog/crashtracking/CrashUploaderTest.java +++ b/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/CrashUploaderTest.java @@ -1,13 +1,13 @@ -package com.datadog.crashtracking; +package datadog.crashtracking; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; -import com.datadog.crashtracking.dto.CrashLog; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import datadog.common.version.VersionInfo; +import datadog.crashtracking.dto.CrashLog; import datadog.trace.api.Config; import datadog.trace.bootstrap.config.provider.ConfigProvider; import java.io.BufferedReader; diff --git a/dd-java-agent/agent-crashtracking/src/test/java/com/datadog/crashtracking/ScriptInitializerTest.java b/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/ScriptInitializerTest.java similarity index 99% rename from dd-java-agent/agent-crashtracking/src/test/java/com/datadog/crashtracking/ScriptInitializerTest.java rename to dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/ScriptInitializerTest.java index 0c7ffabe7c7..20c517ef077 100644 --- a/dd-java-agent/agent-crashtracking/src/test/java/com/datadog/crashtracking/ScriptInitializerTest.java +++ b/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/ScriptInitializerTest.java @@ -1,4 +1,4 @@ -package com.datadog.crashtracking; +package datadog.crashtracking; import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertFalse; diff --git a/dd-java-agent/agent-crashtracking/src/test/java/com/datadog/crashtracking/parsers/HotspotCrashLogParserTest.java b/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/parsers/HotspotCrashLogParserTest.java similarity index 95% rename from dd-java-agent/agent-crashtracking/src/test/java/com/datadog/crashtracking/parsers/HotspotCrashLogParserTest.java rename to dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/parsers/HotspotCrashLogParserTest.java index bcb46b821b9..3fb1111f235 100644 --- a/dd-java-agent/agent-crashtracking/src/test/java/com/datadog/crashtracking/parsers/HotspotCrashLogParserTest.java +++ b/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/parsers/HotspotCrashLogParserTest.java @@ -1,4 +1,4 @@ -package com.datadog.crashtracking.parsers; +package datadog.crashtracking.parsers; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/dd-java-agent/agent-profiling/build.gradle b/dd-java-agent/agent-profiling/build.gradle index 64d39485b19..06b22dbd10c 100644 --- a/dd-java-agent/agent-profiling/build.gradle +++ b/dd-java-agent/agent-profiling/build.gradle @@ -17,7 +17,7 @@ dependencies { api libs.slf4j api project(':internal-api') - implementation project(path: ':dd-java-agent:agent-profiling:profiling-ddprof', configuration: 'shadow') + api project(':dd-java-agent:agent-profiling:profiling-ddprof') api project(':dd-java-agent:agent-profiling:profiling-uploader') api project(':dd-java-agent:agent-profiling:profiling-controller') api project(':dd-java-agent:agent-profiling:profiling-controller-jfr') diff --git a/dd-java-agent/agent-profiling/profiling-controller-ddprof/build.gradle b/dd-java-agent/agent-profiling/profiling-controller-ddprof/build.gradle index 225a953ba80..10c47351a42 100644 --- a/dd-java-agent/agent-profiling/profiling-controller-ddprof/build.gradle +++ b/dd-java-agent/agent-profiling/profiling-controller-ddprof/build.gradle @@ -23,7 +23,7 @@ excludedClassesCoverage += [ dependencies { api libs.slf4j api project(':internal-api') - implementation project(path: ':dd-java-agent:agent-profiling:profiling-ddprof', configuration: 'shadow') + api project(':dd-java-agent:agent-profiling:profiling-ddprof') api project(':dd-java-agent:agent-profiling:profiling-controller') api project(':dd-java-agent:agent-profiling:profiling-utils') diff --git a/dd-java-agent/agent-profiling/profiling-controller-ddprof/src/test/java/com/datadog/profiling/controller/ddprof/DatadogProfilerOngoingRecordingTest.java b/dd-java-agent/agent-profiling/profiling-controller-ddprof/src/test/java/com/datadog/profiling/controller/ddprof/DatadogProfilerOngoingRecordingTest.java index f4143f56ea6..16a9cc33177 100644 --- a/dd-java-agent/agent-profiling/profiling-controller-ddprof/src/test/java/com/datadog/profiling/controller/ddprof/DatadogProfilerOngoingRecordingTest.java +++ b/dd-java-agent/agent-profiling/profiling-controller-ddprof/src/test/java/com/datadog/profiling/controller/ddprof/DatadogProfilerOngoingRecordingTest.java @@ -3,7 +3,7 @@ import static org.junit.jupiter.api.Assertions.*; import com.datadog.profiling.ddprof.DatadogProfiler; -import com.datadog.profiling.ddprof.JavaProfilerLoader; +import datadog.libs.ddprof.DdprofLibraryLoader; import datadog.trace.api.profiling.RecordingData; import java.time.Instant; import org.junit.Assume; @@ -34,7 +34,8 @@ public class DatadogProfilerOngoingRecordingTest { public static void setupAll() { // If the profiler couldn't be loaded, the reason why is saved. // This test assumes the profiler could be loaded. - Assume.assumeNoException("profiler not available", JavaProfilerLoader.REASON_NOT_LOADED); + Assume.assumeNoException( + "profiler not available", DdprofLibraryLoader.javaProfiler().getReasonNotLoaded()); } @BeforeEach diff --git a/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/OpenJdkController.java b/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/OpenJdkController.java index d3dc087ae91..3a8eda88c06 100644 --- a/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/OpenJdkController.java +++ b/dd-java-agent/agent-profiling/profiling-controller-openjdk/src/main/java/com/datadog/profiling/controller/openjdk/OpenJdkController.java @@ -31,7 +31,6 @@ import com.datadog.profiling.controller.ConfigurationException; import com.datadog.profiling.controller.Controller; import com.datadog.profiling.controller.ControllerContext; -import com.datadog.profiling.controller.TempLocationManager; import com.datadog.profiling.controller.jfr.JFRAccess; import com.datadog.profiling.controller.jfr.JfpUtils; import com.datadog.profiling.controller.openjdk.events.AvailableProcessorCoresEvent; @@ -41,6 +40,7 @@ import datadog.trace.bootstrap.config.provider.ConfigProvider; import datadog.trace.bootstrap.instrumentation.jfr.backpressure.BackpressureProfiling; import datadog.trace.bootstrap.instrumentation.jfr.exceptions.ExceptionProfiling; +import datadog.trace.util.TempLocationManager; import de.thetaphi.forbiddenapis.SuppressForbidden; import java.io.IOException; import java.nio.file.Files; diff --git a/dd-java-agent/agent-profiling/profiling-controller/build.gradle b/dd-java-agent/agent-profiling/profiling-controller/build.gradle index 6f8f3314f64..e880bf542c2 100644 --- a/dd-java-agent/agent-profiling/profiling-controller/build.gradle +++ b/dd-java-agent/agent-profiling/profiling-controller/build.gradle @@ -25,6 +25,5 @@ dependencies { testImplementation libs.guava testImplementation libs.bundles.mockito testImplementation group: 'org.awaitility', name: 'awaitility', version: '4.0.1' - testImplementation project(':utils:test-utils') // @Flaky annotation } diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/build.gradle b/dd-java-agent/agent-profiling/profiling-ddprof/build.gradle index 8263d06c63d..c576e4767b4 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/build.gradle +++ b/dd-java-agent/agent-profiling/profiling-ddprof/build.gradle @@ -29,7 +29,7 @@ excludedClassesCoverage += [ dependencies { api project(':dd-java-agent:agent-profiling:profiling-controller') api project(':dd-java-agent:agent-profiling:profiling-utils') - implementation project.hasProperty('ddprof.jar') ? files(project.getProperty('ddprof.jar')) : libs.ddprof + api project(path: ':dd-java-agent:ddprof-lib', configuration: 'shadow') annotationProcessor libs.autoservice.processor compileOnly libs.autoservice.annotation @@ -40,20 +40,6 @@ dependencies { testImplementation libs.bundles.junit5 } -shadowJar { - archiveClassifier = '' - include { - def rslt = false - rslt |= it.path == "com" || it.path == "com/datadog" - || it.path.startsWith("com/datadog/") || it.path == "com/datadoghq" || it.path == "com/datadoghq/profiler" - || it.path.startsWith("com/datadoghq/profiler") - rslt |= it.path == "META-INF" || it.path == "META-INF/services" || it.path.startsWith("META-INF/services/") || it.path.startsWith("META-INF/native-libs/") - rslt |= (it.path.contains("ddprof") && it.path.endsWith(".jar")) - return rslt - } -} - -build.dependsOn shadowJar configurations.all { resolutionStrategy.cacheChangingModulesFor 0, 'seconds' diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java index 21aa4564d58..fc7f109db87 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/DatadogProfiler.java @@ -33,15 +33,16 @@ import static datadog.trace.api.config.ProfilingConfig.PROFILING_QUEUEING_TIME_THRESHOLD_MILLIS_DEFAULT; import com.datadog.profiling.controller.OngoingRecording; -import com.datadog.profiling.controller.TempLocationManager; import com.datadog.profiling.utils.ProfilingMode; import com.datadoghq.profiler.ContextSetter; import com.datadoghq.profiler.JavaProfiler; +import datadog.libs.ddprof.DdprofLibraryLoader; import datadog.trace.api.Platform; import datadog.trace.api.config.ProfilingConfig; import datadog.trace.api.profiling.RecordingData; import datadog.trace.bootstrap.config.provider.ConfigProvider; import datadog.trace.bootstrap.instrumentation.api.TaskWrapper; +import datadog.trace.util.TempLocationManager; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; @@ -118,13 +119,14 @@ private DatadogProfiler(ConfigProvider configProvider) { // visible for testing DatadogProfiler(ConfigProvider configProvider, Set contextAttributes) { this.configProvider = configProvider; - this.profiler = JavaProfilerLoader.PROFILER; + this.profiler = DdprofLibraryLoader.javaProfiler().getComponent(); this.detailedDebugLogging = configProvider.getBoolean( PROFILING_DETAILED_DEBUG_LOGGING, PROFILING_DETAILED_DEBUG_LOGGING_DEFAULT); - if (JavaProfilerLoader.REASON_NOT_LOADED != null) { + Throwable reasonNotLoaded = DdprofLibraryLoader.javaProfiler().getReasonNotLoaded(); + if (reasonNotLoaded != null) { throw new UnsupportedOperationException( - "Unable to instantiate datadog profiler", JavaProfilerLoader.REASON_NOT_LOADED); + "Unable to instantiate datadog profiler", reasonNotLoaded); } // TODO enable/disable events by name (e.g. datadog.ExecutionSample), not flag, so configuration diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/JavaProfilerLoader.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/JavaProfilerLoader.java deleted file mode 100644 index 654bd63f904..00000000000 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/main/java/com/datadog/profiling/ddprof/JavaProfilerLoader.java +++ /dev/null @@ -1,47 +0,0 @@ -package com.datadog.profiling.ddprof; - -import com.datadog.profiling.controller.TempLocationManager; -import com.datadoghq.profiler.JavaProfiler; -import datadog.trace.api.config.ProfilingConfig; -import datadog.trace.bootstrap.config.provider.ConfigProvider; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.attribute.PosixFilePermissions; - -/** - * Only loading the profiler itself needs to be protected as a singleton. Separating the loading - * logic minimises the amount of config which needs to be early loaded to just the library location. - */ -public class JavaProfilerLoader { - static final JavaProfiler PROFILER; - public static final Throwable REASON_NOT_LOADED; - - static { - JavaProfiler profiler; - Throwable reasonNotLoaded = null; - try { - ConfigProvider configProvider = ConfigProvider.getInstance(); - String scratch = configProvider.getString(ProfilingConfig.PROFILING_DATADOG_PROFILER_SCRATCH); - if (scratch == null) { - Path scratchPath = TempLocationManager.getInstance().getTempDir().resolve("scratch"); - if (!Files.exists(scratchPath)) { - Files.createDirectories( - scratchPath, - PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxr-xr-x"))); - } - scratch = scratchPath.toString(); - } - profiler = - JavaProfiler.getInstance( - configProvider.getString(ProfilingConfig.PROFILING_DATADOG_PROFILER_LIBPATH), - scratch); - // sanity test - force load Datadog profiler to catch it not being available early - profiler.execute("status"); - } catch (Throwable t) { - reasonNotLoaded = t; - profiler = null; - } - PROFILER = profiler; - REASON_NOT_LOADED = reasonNotLoaded; - } -} diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerRecordingTest.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerRecordingTest.java index 00afb2899c7..e8a6b087fdd 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerRecordingTest.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerRecordingTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.*; +import datadog.libs.ddprof.DdprofLibraryLoader; import datadog.trace.api.Platform; import datadog.trace.api.profiling.RecordingData; import datadog.trace.bootstrap.config.provider.ConfigProvider; @@ -21,7 +22,8 @@ class DatadogProfilerRecordingTest { @BeforeEach void setup() throws Exception { Assume.assumeTrue(Platform.isLinux()); - Assume.assumeNoException("Profiler not available", JavaProfilerLoader.REASON_NOT_LOADED); + Assume.assumeNoException( + "Profiler not available", DdprofLibraryLoader.jvmAccess().getReasonNotLoaded()); profiler = DatadogProfiler.newInstance(ConfigProvider.getInstance()); Assume.assumeFalse(profiler.isActive()); recording = (DatadogProfilerRecording) profiler.start(); @@ -39,7 +41,8 @@ void shutdown() throws Exception { @Test void testClose() throws Exception { - Assume.assumeNoException("Profiler not available", JavaProfilerLoader.REASON_NOT_LOADED); + Assume.assumeNoException( + "Profiler not available", DdprofLibraryLoader.javaProfiler().getReasonNotLoaded()); assertTrue(Files.exists(recording.getRecordingFile())); recording.close(); assertFalse(Files.exists(recording.getRecordingFile())); @@ -47,7 +50,8 @@ void testClose() throws Exception { @Test void testStop() throws Exception { - Assume.assumeNoException("Profiler not available", JavaProfilerLoader.REASON_NOT_LOADED); + Assume.assumeNoException( + "Profiler not available", DdprofLibraryLoader.javaProfiler().getReasonNotLoaded()); RecordingData data = recording.stop(); assertNotNull(data); assertTrue(Files.exists(recording.getRecordingFile())); @@ -55,7 +59,8 @@ void testStop() throws Exception { @Test void testSnapshot() throws Exception { - Assume.assumeNoException("Profiler not available", JavaProfilerLoader.REASON_NOT_LOADED); + Assume.assumeNoException( + "Profiler not available", DdprofLibraryLoader.javaProfiler().getReasonNotLoaded()); RecordingData data = recording.snapshot(Instant.now()); assertNotNull(data); assertTrue(Files.exists(recording.getRecordingFile())); diff --git a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java index ae144425ca5..61dee010073 100644 --- a/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java +++ b/dd-java-agent/agent-profiling/profiling-ddprof/src/test/java/com/datadog/profiling/ddprof/DatadogProfilerTest.java @@ -8,6 +8,7 @@ import com.datadog.profiling.controller.OngoingRecording; import com.datadog.profiling.controller.UnsupportedEnvironmentException; import com.datadog.profiling.utils.ProfilingMode; +import datadog.libs.ddprof.DdprofLibraryLoader; import datadog.trace.api.Platform; import datadog.trace.api.config.ProfilingConfig; import datadog.trace.api.profiling.ProfilingScope; @@ -43,7 +44,8 @@ public void setup() { @Test void test() throws Exception { - Assume.assumeNoException("Profiler not available", JavaProfilerLoader.REASON_NOT_LOADED); + Assume.assumeNoException( + "Profiler not available", DdprofLibraryLoader.javaProfiler().getReasonNotLoaded()); DatadogProfiler profiler = DatadogProfiler.newInstance(ConfigProvider.getInstance()); assertFalse(profiler.enabledModes().isEmpty()); @@ -76,7 +78,8 @@ void test() throws Exception { @ParameterizedTest @MethodSource("profilingModes") void testStartCmd(boolean cpu, boolean wall, boolean alloc, boolean memleak) throws Exception { - Assume.assumeNoException("Profiler not available", JavaProfilerLoader.REASON_NOT_LOADED); + Assume.assumeNoException( + "Profiler not available", DdprofLibraryLoader.javaProfiler().getReasonNotLoaded()); DatadogProfiler profiler = DatadogProfiler.newInstance(configProvider(cpu, wall, alloc, memleak)); diff --git a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/AgentCLI.java b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/AgentCLI.java index 664bd780d53..ffb7c48f46f 100644 --- a/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/AgentCLI.java +++ b/dd-java-agent/agent-tooling/src/main/java/datadog/trace/agent/tooling/AgentCLI.java @@ -1,7 +1,7 @@ package datadog.trace.agent.tooling; -import com.datadog.crashtracking.CrashUploader; -import com.datadog.crashtracking.OOMENotifier; +import datadog.crashtracking.CrashUploader; +import datadog.crashtracking.OOMENotifier; import datadog.trace.agent.tooling.bytebuddy.SharedTypePools; import datadog.trace.agent.tooling.bytebuddy.matcher.HierarchyMatchers; import datadog.trace.agent.tooling.profiler.EnvironmentChecker; diff --git a/dd-java-agent/build.gradle b/dd-java-agent/build.gradle index 0278a24cd9f..3c988fb5264 100644 --- a/dd-java-agent/build.gradle +++ b/dd-java-agent/build.gradle @@ -276,6 +276,9 @@ dependencies { sharedShadowInclude project(':dd-java-agent:agent-crashtracking'), { transitive = false } + sharedShadowInclude project(path: ':dd-java-agent:ddprof-lib', configuration: 'shadow'), { + transitive = false + } traceShadowInclude project(':dd-trace-core') } @@ -306,7 +309,7 @@ tasks.register('checkAgentJarSize').configure { doLast { // Arbitrary limit to prevent unintentional increases to the agent jar size // Raise or lower as required - assert shadowJar.archiveFile.get().getAsFile().length() <= 32 * 1024 * 1024 + assert shadowJar.archiveFile.get().getAsFile().length() <= 35 * 1024 * 1024 } dependsOn "shadowJar" diff --git a/dd-java-agent/ddprof-lib/build.gradle b/dd-java-agent/ddprof-lib/build.gradle new file mode 100644 index 00000000000..642c4a8f632 --- /dev/null +++ b/dd-java-agent/ddprof-lib/build.gradle @@ -0,0 +1,30 @@ +plugins { + id "com.gradleup.shadow" +} + +apply from: "$rootDir/gradle/java.gradle" + +dependencies { + // This module provides the ddprof library as an api dependency + // so that other modules can easily depend on it. + implementation project.hasProperty('ddprof.jar') ? files(project.getProperty('ddprof.jar')) : libs.ddprof + api project(':internal-api') + api project(':dd-trace-api') +} + +shadowJar { + dependencies deps.excludeShared + archiveClassifier = 'all' + include { + def rslt = false + rslt |= it.path == 'datadog' || it.path == "com" || it.path == "com/datadog" + || it.path.startsWith("datadog/") || it.path == "com/datadoghq" || it.path == "com/datadoghq/profiler" + || it.path.startsWith("com/datadoghq/profiler") + rslt |= it.path == "META-INF" || it.path == "META-INF/services" || it.path.startsWith("META-INF/services/") || it.path.startsWith("META-INF/native-libs/") + rslt |= (it.path.contains("ddprof") && it.path.endsWith(".jar")) + println("Checking ${it.path} = ${rslt}") + return rslt + } +} + +build.dependsOn shadowJar diff --git a/dd-java-agent/ddprof-lib/src/main/java/datadog/libs/ddprof/DdprofLibraryLoader.java b/dd-java-agent/ddprof-lib/src/main/java/datadog/libs/ddprof/DdprofLibraryLoader.java new file mode 100644 index 00000000000..b73698512e2 --- /dev/null +++ b/dd-java-agent/ddprof-lib/src/main/java/datadog/libs/ddprof/DdprofLibraryLoader.java @@ -0,0 +1,156 @@ +package datadog.libs.ddprof; + +import com.datadoghq.profiler.JVMAccess; +import com.datadoghq.profiler.JavaProfiler; +import datadog.trace.api.config.ProfilingConfig; +import datadog.trace.bootstrap.config.provider.ConfigProvider; +import datadog.trace.util.TempLocationManager; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Supplier; + +/** + * A wrapper around unified loading of the Datadog profiler and JVM access. It exposes {@linkplain + * JavaProfiler} and {@linkplain JVMAccess} components which are lazily loaded. The wrapper makes + * sure the underlying library is loaded only once. Each component is returned within a 'holder' + * which provides the component as well as the reason why a component could not have been + * constructed, if that's the case. + */ +public final class DdprofLibraryLoader { + public abstract static class ComponentHolder { + private volatile boolean loaded = false; + + private T component; + private Throwable reasonNotLoaded; + + private final Supplier> initializer; + + protected ComponentHolder(T component, Throwable reasonNotLoaded) { + assert component != null || reasonNotLoaded != null; + this.component = component; + this.reasonNotLoaded = reasonNotLoaded; + this.initializer = null; // no need to initialize again + this.loaded = true; // already loaded + } + + protected ComponentHolder(Supplier> initializer) { + assert initializer != null; + this.initializer = initializer; + } + + private void initialize() { + if (!loaded) { + // all holders need to be synchronized on the same class to ensure that only one is loading + // the native library at a time + synchronized (DdprofLibraryLoader.class) { + if (!loaded) { + ComponentHolder h = initializer.get(); + component = h.getComponent(); + reasonNotLoaded = h.getReasonNotLoaded(); + loaded = true; + } + } + } + } + + public final T getComponent() { + initialize(); + return component; + } + + public final Throwable getReasonNotLoaded() { + initialize(); + return reasonNotLoaded; + } + } + + public static final class JavaProfilerHolder extends ComponentHolder { + public JavaProfilerHolder(Supplier> initializer) { + super(initializer); + } + + JavaProfilerHolder(JavaProfiler component, Throwable reasonNotLoaded) { + super(component, reasonNotLoaded); + } + } + + public static final class JVMAccessHolder extends ComponentHolder { + public JVMAccessHolder(Supplier> initializer) { + super(initializer); + } + + JVMAccessHolder(JVMAccess component, Throwable reasonNotLoaded) { + super(component, reasonNotLoaded); + } + } + + private static final JavaProfilerHolder PROFILER_HOLDER = + new JavaProfilerHolder(DdprofLibraryLoader::initJavaProfiler); + + private static final JVMAccessHolder JVM_ACCESS_HOLDER = + new JVMAccessHolder(DdprofLibraryLoader::initJVMAccess); + + public static JavaProfilerHolder javaProfiler() { + return PROFILER_HOLDER; + } + + public static JVMAccessHolder jvmAccess() { + return JVM_ACCESS_HOLDER; + } + + private static JavaProfilerHolder initJavaProfiler() { + JavaProfiler profiler; + Throwable reasonNotLoaded = null; + try { + ConfigProvider configProvider = ConfigProvider.getInstance(); + String scratch = getScratchDir(configProvider); + profiler = + JavaProfiler.getInstance( + configProvider.getString(ProfilingConfig.PROFILING_DATADOG_PROFILER_LIBPATH), + scratch); + // sanity test - force load Datadog profiler to catch it not being available early + profiler.execute("status"); + } catch (Throwable t) { + reasonNotLoaded = t; + profiler = null; + } + return new JavaProfilerHolder(profiler, reasonNotLoaded); + } + + static JVMAccessHolder initJVMAccess() { + ConfigProvider configProvider = ConfigProvider.getInstance(); + AtomicReference reasonNotLoaded = new AtomicReference<>(); + JVMAccess jvmAccess = null; + try { + String scratchDir = getScratchDir(configProvider); + jvmAccess = new JVMAccess(null, scratchDir, reasonNotLoaded::set); + } catch (Throwable t) { + if (reasonNotLoaded.get() == null) { + reasonNotLoaded.set(t); + } else { + // if we already have a reason, don't overwrite it + // this can happen if the JVMAccess constructor throws an exception + // and then the execute method throws another one + } + jvmAccess = null; + } + return new JVMAccessHolder(jvmAccess, reasonNotLoaded.get()); + } + + private static String getScratchDir(ConfigProvider configProvider) throws IOException { + String scratch = configProvider.getString(ProfilingConfig.PROFILING_DATADOG_PROFILER_SCRATCH); + if (scratch == null) { + Path scratchPath = TempLocationManager.getInstance().getTempDir().resolve("scratch"); + if (!Files.exists(scratchPath)) { + Files.createDirectories( + scratchPath, + PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxr-xr-x"))); + } + scratch = scratchPath.toString(); + } + return scratch; + } +} diff --git a/dd-smoke-tests/crashtracking/build.gradle b/dd-smoke-tests/crashtracking/build.gradle index 12916dac735..87a76203349 100644 --- a/dd-smoke-tests/crashtracking/build.gradle +++ b/dd-smoke-tests/crashtracking/build.gradle @@ -19,6 +19,7 @@ jar { dependencies { testImplementation project(':dd-smoke-tests') testImplementation project(':dd-java-agent:agent-profiling:profiling-testing') + testImplementation project(path: ':dd-java-agent:ddprof-lib', configuration: 'shadow') testImplementation libs.bundles.junit5 testImplementation group: 'org.hamcrest', name: 'hamcrest', version: '2.1' @@ -28,5 +29,10 @@ dependencies { tasks.withType(Test).configureEach { dependsOn "shadowJar" jvmArgs "-Ddatadog.smoketest.app.shadowJar.path=${tasks.shadowJar.archiveFile.get()}" + + testLogging { + events "started", "passed", "skipped", "failed" + showStandardStreams = true + } } diff --git a/dd-smoke-tests/crashtracking/src/main/java/datadog/smoketest/crashtracking/CrashtrackingTestApplication.java b/dd-smoke-tests/crashtracking/src/main/java/datadog/smoketest/crashtracking/CrashtrackingTestApplication.java index 5e8be98ea6d..5b8301ba5af 100644 --- a/dd-smoke-tests/crashtracking/src/main/java/datadog/smoketest/crashtracking/CrashtrackingTestApplication.java +++ b/dd-smoke-tests/crashtracking/src/main/java/datadog/smoketest/crashtracking/CrashtrackingTestApplication.java @@ -1,9 +1,12 @@ package datadog.smoketest.crashtracking; +import com.sun.management.HotSpotDiagnosticMXBean; +import java.lang.management.ManagementFactory; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -11,16 +14,59 @@ public class CrashtrackingTestApplication { public static void main(String[] args) throws Exception { - if (args.length == 0) { - throw new RuntimeException("Expecting the control file as an argument"); + HotSpotDiagnosticMXBean diagBean = + ManagementFactory.getPlatformMXBean(HotSpotDiagnosticMXBean.class); + String onError = null; + if (diagBean == null) { + System.err.println("HotSpotDiagnosticMXBean is not available"); + System.exit(-1); } - System.out.println("=== CrashtrackingTestApplication ==="); + + String onOutOfMemoryError = diagBean.getVMOption("OnOutOfMemoryError").getValue(); + long ts = System.nanoTime(); + // There is a delay between the JVM startup and the moment when the OnError option is set due to + // possible + // dependency on JMX - let's wait for it to be set. + while ((onError == null || onError.isEmpty()) && (System.nanoTime() - ts) < 5_000_000_000L) { + onError = diagBean.getVMOption("OnError").getValue(); + if (onOutOfMemoryError == null) { + onOutOfMemoryError = diagBean.getVMOption("OnOutOfMemoryError").getValue(); + } + LockSupport.parkNanos(100_00_000L); // 100ms + } + + if (onError == null && onOutOfMemoryError == null) { + System.err.println("Neither OnError nor OnOutOfMemoryError is specified"); + System.exit(-1); + } + + String crashUploaderScript = + Arrays.stream(onError.split(";")) + .filter(s -> s.trim().contains("dd_crash_uploader")) + .findFirst() + .map(s -> s.replace(" %p", "")) + .orElse(null); + String oomeNotifierScript = + Arrays.stream(onOutOfMemoryError.split(";")) + .filter(s -> s.trim().contains("dd_oome_notifier")) + .findFirst() + .map(s -> s.replace(" %p", "")) + .orElse(null); + if (crashUploaderScript == null && oomeNotifierScript == null) { + System.err.println("Neither OnError nor OnOutOfMemoryError contains the expected value"); + System.exit(-1); + } + + System.out.println("===> Waiting..."); + System.out.flush(); + CountDownLatch latch = new CountDownLatch(1); Thread t = new Thread( () -> { - Path scriptPath = Paths.get(args[0]); + Path scriptPath = + Paths.get(crashUploaderScript != null ? crashUploaderScript : oomeNotifierScript); while (!Files.exists(scriptPath)) { System.out.println("Waiting for the script " + scriptPath + " to be created..."); LockSupport.parkNanos(1_000_000_000L); diff --git a/dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/CrashtrackingSmokeTest.java b/dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/CrashtrackingSmokeTest.java index e3d8b38659a..5aaa41bc7b4 100644 --- a/dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/CrashtrackingSmokeTest.java +++ b/dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/CrashtrackingSmokeTest.java @@ -2,9 +2,10 @@ import static org.hamcrest.CoreMatchers.containsString; import static org.hamcrest.MatcherAssert.assertThat; -import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assumptions.assumeFalse; +import static org.junit.jupiter.api.Assumptions.assumeTrue; import com.squareup.moshi.JsonAdapter; import com.squareup.moshi.Moshi; @@ -16,8 +17,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; +import java.util.List; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; @@ -50,6 +53,11 @@ public class CrashtrackingSmokeTest { static void setupAll() { // Only Hotspot based implementation are supported assumeFalse(Platform.isJ9()); + // // Currently, we require the ddprof java library for crash-tracking; bail out if not + // supported + // assumeTrue( + // DdprofLibraryLoader.jvmAccess().getReasonNotLoaded() == null, + // "JVM Access is not available"); } private Path tempDir; @@ -130,15 +138,9 @@ private static String getExtension() { return Platform.isWindows() ? "bat" : "sh"; } - /* - * NOTE: The current implementation of crash tracking doesn't work with ancient version of bash - * that ships with OS X by default. - */ @Test - void testCrashTracking() throws Exception { - Path script = tempDir.resolve("dd_crash_uploader." + getExtension()); - String onErrorValue = script + " %p"; - String errorFile = tempDir.resolve("hs_err.log").toString(); + void testAutoInjection() throws Exception { + assumeTrue(Platform.isLinux()); // we support only linux ATM ProcessBuilder pb = new ProcessBuilder( @@ -147,21 +149,59 @@ void testCrashTracking() throws Exception { "-javaagent:" + agentShadowJar(), "-Xmx96m", "-Xms96m", - "-XX:OnError=" + onErrorValue, - "-XX:ErrorFile=" + errorFile, "-XX:+CrashOnOutOfMemoryError", // Use OOME to trigger crash "-Ddd.dogstatsd.start-delay=0", // Minimize the delay to initialize JMX and create - // the scripts + // no need to specify the scripts "-Ddd.trace.enabled=false", "-jar", - appShadowJar(), - script.toString())); + appShadowJar())); + + pb.environment().put("DD_TRACE_AGENT_PORT", String.valueOf(tracingServer.getPort())); + + Process p = pb.start(); + OUTPUT.captureOutput( + p, LOG_FILE_DIR.resolve("testProcess.testCrashTrackingInjected.log").toFile()); + + assertExpectedCrash(p); + } + + /* + * NOTE: The current implementation of crash tracking doesn't work with ancient version of bash + * that ships with OS X by default. + */ + @Test + void testCrashTracking() throws Exception { + Path script = tempDir.resolve("dd_crash_uploader." + getExtension()); + String onErrorValue = script + " %p"; + String errorFile = tempDir.resolve("hs_err.log").toString(); + + String onErrorArg = + !Platform.isLinux() + ? "-XX:OnError=" + onErrorValue + : ""; // on Linux we can automatically inject the arg + List processArgs = new ArrayList<>(); + processArgs.add(javaPath()); + processArgs.add("-javaagent:" + agentShadowJar()); + processArgs.add("-Xmx96m"); + processArgs.add("-Xms96m"); + if (!onErrorArg.isEmpty()) { + processArgs.add(onErrorArg); + } + processArgs.add("-XX:ErrorFile=" + errorFile); + processArgs.add("-XX:+CrashOnOutOfMemoryError"); // Use OOME to trigger crash + processArgs.add( + "-Ddd.dogstatsd.start-delay=0"); // Minimize the delay to initialize JMX and create the + // scripts + processArgs.add("-Ddd.trace.enabled=false"); + processArgs.add("-jar"); + processArgs.add(appShadowJar()); + ProcessBuilder pb = new ProcessBuilder(processArgs); pb.environment().put("DD_TRACE_AGENT_PORT", String.valueOf(tracingServer.getPort())); Process p = pb.start(); OUTPUT.captureOutput(p, LOG_FILE_DIR.resolve("testProcess.testCrashTracking.log").toFile()); - assertNotEquals(0, p.waitFor(), "Application should have crashed"); + assertExpectedCrash(p); assertCrashData(); } @@ -189,15 +229,14 @@ void testCrashTrackingLegacy() throws Exception { // the scripts "-Ddd.trace.enabled=false", "-jar", - appShadowJar(), - script.toString())); + appShadowJar())); pb.environment().put("DD_TRACE_AGENT_PORT", String.valueOf(tracingServer.getPort())); Process p = pb.start(); OUTPUT.captureOutput( p, LOG_FILE_DIR.resolve("testProcess.testCrashTrackingLegacy.log").toFile()); - assertNotEquals(0, p.waitFor(), "Application should have crashed"); + assertExpectedCrash(p); assertCrashData(); } @@ -212,28 +251,37 @@ void testOomeTracking() throws Exception { String onErrorValue = script + " %p"; String errorFile = tempDir.resolve("hs_err_pid%p.log").toString(); - ProcessBuilder pb = - new ProcessBuilder( - Arrays.asList( - javaPath(), - "-javaagent:" + agentShadowJar(), - "-Xmx96m", - "-Xms96m", - "-XX:OnOutOfMemoryError=" + onErrorValue, - "-XX:ErrorFile=" + errorFile, - "-XX:+CrashOnOutOfMemoryError", // Use OOME to trigger crash - "-Ddd.dogstatsd.start-delay=0", // Minimize the delay to initialize JMX and create - // the scripts - "-Ddd.trace.enabled=false", - "-jar", - appShadowJar(), - script.toString())); + String onOOMEArg = + !Platform.isLinux() + ? "-XX:OnOutOfMemoryError=" + onErrorValue + : ""; // on Linux we can automatically inject the arg + + List processArgs = new ArrayList<>(); + processArgs.add(javaPath()); + processArgs.add("-javaagent:" + agentShadowJar()); + processArgs.add("-Xmx96m"); + processArgs.add("-Xms96m"); + if (!onOOMEArg.isEmpty()) { + processArgs.add(onOOMEArg); + } + processArgs.add("-XX:ErrorFile=" + errorFile); + processArgs.add("-XX:+CrashOnOutOfMemoryError"); // Use OOME to trigger crash + processArgs.add( + "-Ddd.dogstatsd.start-delay=0"); // Minimize the delay to initialize JMX and create the + // scripts + processArgs.add("-Ddd.trace.enabled=false"); + processArgs.add("-jar"); + processArgs.add(appShadowJar()); + + ProcessBuilder pb = new ProcessBuilder(processArgs); pb.environment().put("DD_DOGSTATSD_PORT", String.valueOf(udpServer.getPort())); + System.out.println("==> Process args: " + pb.command()); + Process p = pb.start(); OUTPUT.captureOutput(p, LOG_FILE_DIR.resolve("testProcess.testOomeTracking.log").toFile()); - assertNotEquals(0, p.waitFor(), "Application should have crashed"); + assertExpectedCrash(p); assertOOMEvent(); } @@ -260,20 +308,24 @@ void testCombineTracking() throws Exception { // the scripts "-Ddd.trace.enabled=false", "-jar", - appShadowJar(), - oomeScript.toString())); + appShadowJar())); pb.environment().put("DD_TRACE_AGENT_PORT", String.valueOf(tracingServer.getPort())); pb.environment().put("DD_DOGSTATSD_PORT", String.valueOf(udpServer.getPort())); Process p = pb.start(); OUTPUT.captureOutput(p, LOG_FILE_DIR.resolve("testProcess.testCombineTracking.log").toFile()); - assertNotEquals(0, p.waitFor(), "Application should have crashed"); - + assertExpectedCrash(p); assertCrashData(); assertOOMEvent(); } + private static void assertExpectedCrash(Process p) throws InterruptedException { + // exit code -1 means the test application exited prematurely + // exit code > 0 means the test application crashed, as expected + assertTrue(p.waitFor() > 0, "Application should have crashed"); + } + private void assertCrashData() throws InterruptedException { CrashTelemetryData crashData = crashEvents.poll(DATA_TIMEOUT_MS, TimeUnit.MILLISECONDS); assertNotNull(crashData, "Crash data not uploaded"); diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/CrashTrackingConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/CrashTrackingConfig.java index f4bc484f642..945ed868b66 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/CrashTrackingConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/CrashTrackingConfig.java @@ -7,6 +7,8 @@ * documentation for details. */ public final class CrashTrackingConfig { + public static final String CRASH_TRACKING_ENABLED = "crashtracking.enabled"; + public static final boolean CRASH_TRACKING_ENABLED_DEFAULT = true; public static final String CRASH_TRACKING_TAGS = "crashtracking.tags"; diff --git a/gradle/dependencies.gradle b/gradle/dependencies.gradle index cb46e079d9a..4f3a1147821 100644 --- a/gradle/dependencies.gradle +++ b/gradle/dependencies.gradle @@ -28,6 +28,7 @@ final class CachedData { exclude(project(':utils:time-utils')) exclude(project(':utils:version-utils')) exclude(project(':dd-java-agent:agent-crashtracking')) + exclude(project(':dd-java-agent:ddprof-lib')) exclude(dependency('org.slf4j::')) // okhttp and its transitives (both fork and non-fork) diff --git a/internal-api/build.gradle b/internal-api/build.gradle index fbf1916a4af..5e27fa142c7 100644 --- a/internal-api/build.gradle +++ b/internal-api/build.gradle @@ -224,12 +224,16 @@ excludedClassesCoverage += [ // Trivial holder and no-op 'datadog.trace.bootstrap.instrumentation.api.SpanPostProcessor.Holder', 'datadog.trace.bootstrap.instrumentation.api.SpanPostProcessor.NoOpSpanPostProcessor', + 'datadog.trace.util.TempLocationManager', + 'datadog.trace.util.TempLocationManager.*', ] excludedClassesBranchCoverage = [ 'datadog.trace.api.ProductActivationConfig', 'datadog.trace.api.ClassloaderConfigurationOverrides.Lazy', 'datadog.trace.util.stacktrace.HotSpotStackWalker', - 'datadog.trace.util.stacktrace.StackWalkerFactory' + 'datadog.trace.util.stacktrace.StackWalkerFactory', + 'datadog.trace.util.TempLocationManager', + 'datadog.trace.util.TempLocationManager.*' ] excludedClassesInstructionCoverage = [ 'datadog.trace.bootstrap.config.provider.EnvironmentConfigSource', diff --git a/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/TempLocationManager.java b/internal-api/src/main/java/datadog/trace/util/TempLocationManager.java similarity index 96% rename from dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/TempLocationManager.java rename to internal-api/src/main/java/datadog/trace/util/TempLocationManager.java index 900d05805f2..bf71f4971f6 100644 --- a/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/TempLocationManager.java +++ b/internal-api/src/main/java/datadog/trace/util/TempLocationManager.java @@ -1,11 +1,9 @@ -package com.datadog.profiling.controller; +package datadog.trace.util; import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY; import datadog.trace.api.config.ProfilingConfig; import datadog.trace.bootstrap.config.provider.ConfigProvider; -import datadog.trace.util.AgentTaskScheduler; -import datadog.trace.util.PidHelper; import java.io.IOException; import java.nio.file.FileSystems; import java.nio.file.FileVisitResult; @@ -19,9 +17,7 @@ import java.nio.file.attribute.PosixFilePermissions; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -222,8 +218,6 @@ boolean await(long timeout, TimeUnit unit) throws Throwable { private final CleanupTask cleanupTask = new CleanupTask(); private final CleanupHook cleanupTestHook; - private final Map ignoredPaths = new ConcurrentHashMap<>(); - /** * Get the singleton instance of the TempLocationManager. It will run the cleanup task in the * background. @@ -356,18 +350,6 @@ public Path getTempDir(Path subPath, boolean create) { return rslt; } - public void ignore(Path path) { - if (path.startsWith(baseTempDir)) { - // ignore the path if it is a child of the base temp directory - ignoredPaths.put(path, path); - } else { - log.debug( - "Path {} which is not a child of the base temp directory {} can not be ignored", - path, - baseTempDir); - } - } - /** * Walk the base temp directory recursively and remove all inactive per-process entries. No * timeout is applied. diff --git a/dd-java-agent/agent-profiling/profiling-controller/src/test/java/com/datadog/profiling/controller/TempLocationManagerTest.java b/internal-api/src/test/java/datadog/trace/util/TempLocationManagerTest.java similarity index 97% rename from dd-java-agent/agent-profiling/profiling-controller/src/test/java/com/datadog/profiling/controller/TempLocationManagerTest.java rename to internal-api/src/test/java/datadog/trace/util/TempLocationManagerTest.java index 6ba440e4d05..af9580f8981 100644 --- a/dd-java-agent/agent-profiling/profiling-controller/src/test/java/com/datadog/profiling/controller/TempLocationManagerTest.java +++ b/internal-api/src/test/java/datadog/trace/util/TempLocationManagerTest.java @@ -1,11 +1,14 @@ -package com.datadog.profiling.controller; +package datadog.trace.util; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; import datadog.trace.api.config.ProfilingConfig; import datadog.trace.bootstrap.config.provider.ConfigProvider; import datadog.trace.test.util.Flaky; -import datadog.trace.util.PidHelper; import java.io.IOException; import java.nio.file.FileVisitResult; import java.nio.file.Files; diff --git a/settings.gradle b/settings.gradle index db584c29b13..3464126bf75 100644 --- a/settings.gradle +++ b/settings.gradle @@ -80,6 +80,7 @@ include ':dd-java-agent:agent-debugger:debugger-test-scala' include ':dd-java-agent:agent-debugger:debugger-el' include ':dd-java-agent:agent-crashtracking' +include ':dd-java-agent:ddprof-lib' include ':dd-java-agent:agent-otel:otel-bootstrap' include ':dd-java-agent:agent-otel:otel-shim'