diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index efcbec882..f7af8ed81 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -32,3 +32,6 @@ packages/plugins/custom-hooks @yoannmoin # True End packages/plugins/true-end @yoannmoinet + +# Ci Visibility +packages/plugins/ci-visibility @yoannmoinet diff --git a/README.md b/README.md index 632b24f0a..00b31ffe8 100644 --- a/README.md +++ b/README.md @@ -24,10 +24,12 @@ To interact with Datadog directly from your builds. - [Configuration](#configuration) - [`auth.apiKey`](#authapikey) - [`auth.appKey`](#authappkey) + - [`customPlugins`](#customplugins) - [`disableGit`](#disablegit) - [`logLevel`](#loglevel) - - [`customPlugins`](#customplugins) + - [`metadata.name`](#metadataname) - [Features](#features) + - [Ci Visibility](#ci-visibility-----) - [Error Tracking](#error-tracking-----) - [Telemetry](#telemetry-----) - [Contributing](#contributing) @@ -90,6 +92,9 @@ Follow the specific documentation for each bundler: }; customPlugins?: (arg: GetPluginsArg) => UnpluginPlugin[]; logLevel?: 'debug' | 'info' | 'warn' | 'error' | 'none'; + ciVisibility?: { + disabled?: boolean; + }; errorTracking?: { disabled?: boolean; sourcemaps?: { @@ -137,19 +142,6 @@ In order to interact with Datadog, you have to use [your own API Key](https://ap In order to interact with Datadog, you have to use [your own Application Key](https://app.datadoghq.com/organization-settings/application-keys). -### `disableGit` - -> default: `false` - -Disable the [Git plugin](/packages/plugins/git#readme) if you don't want to use it.
-For instance if you see a `Error: No git remotes available` error. - -### `logLevel` - -> default: `'warn'` - -Which level of log do you want to show. - ### `customPlugins` > default: `[]` @@ -184,6 +176,9 @@ Your function will receive three arguments: The `context` is a shared object that is mutated during the build process. +Your function has to return an array of [Unplugin Plugins definitions](https://unplugin.unjs.io/guide/#supported-hooks).
+You can also use our own [custom hooks](/packages/plugins/custom-hooks#existing-hooks). +
Full context object @@ -191,20 +186,34 @@ The `context` is a shared object that is mutated during the build process. 
 type GlobalContext = {
+    // Trigger an asynchronous custom hook.
+    asyncHook: async (name: string, ...args: any[]) => Promise;
     // Mirror of the user's config.
     auth?: {
         apiKey?: string;
+        appKey?: string;
     };
-    // More details on the currently running bundler.
-    bundler: BundlerReport
-    // Added in `writeBundle`.
-    build: BuildReport
+    // Available in the `buildReport` hook.
+    build: BuildReport;
+    // Available in the `bundlerReport` hook.
+    bundler: BundlerReport;
     cwd: string;
-    getLogger: (name: string) => Logger
-    // Added in `buildStart`.
-    git?: Git
-    inject: Injection
+    env: string;
+    getLogger: (name: string) => Logger;
+    // Available in the `git` hook.
+    git?: Git;
+    // Trigger a synchronous custom hook.
+    hook: (name: string, ...args: any[]) => void;
+    inject: Injection;
+    // The list of all the plugin names that are currently running in the ecosystem.
+    pluginNames: string[];
+    // The list of all the plugin instances that are currently running in the ecosystem.
+    plugins: Plugin[];
+    // Send a log to Datadog.
+    sendLog: (message: string, context?: Record) => Promise;
+    // The start time of the build.
     start: number;
+    // The version of the plugin.
     version: string;
 }
@@ -214,9 +223,49 @@ type GlobalContext = { #### [📝 Full documentation ➡️](/packages/factory#global-context) + +### `disableGit` + +> default: `false` + +Disable the [Git plugin](/packages/plugins/git#readme) if you don't want to use it.
+For instance if you see a `Error: No git remotes available` error. + +### `logLevel` + +> default: `'warn'` + +Which level of log do you want to show. + +### `metadata.name` +> default: `null` + +The name of the build.
+This is used to identify the build in logs, metrics and spans. + ## Features +### Ci Visibility ESBuild Rollup Rspack Vite Webpack + +> Interact with CI Visibility directly from your build system. + +#### [📝 Full documentation ➡️](/packages/plugins/ci-visibility#readme) + +
+ +Configuration + +```typescript +datadogWebpackPlugin({ + ciVisibility?: { + disabled?: boolean, + } +}); +``` + +
+ ### Error Tracking ESBuild Rollup Rspack Vite Webpack > Interact with Error Tracking directly from your build system. diff --git a/packages/core/src/helpers/plugins.test.ts b/packages/core/src/helpers/plugins.test.ts index f2bce4342..be379ab57 100644 --- a/packages/core/src/helpers/plugins.test.ts +++ b/packages/core/src/helpers/plugins.test.ts @@ -3,8 +3,11 @@ // Copyright 2019-Present Datadog, Inc. import { INJECTED_FILE } from '@dd/core/constants'; +import { defaultPluginOptions, getSourcemapsConfiguration } from '@dd/tests/_jest/helpers/mocks'; -import { cleanPluginName, isInjectionFile } from './plugins'; +import type { Options } from '../types'; + +import { cleanPluginName, isInjectionFile, shouldGetGitInfo } from './plugins'; describe('plugins', () => { describe('cleanPluginName', () => { @@ -80,4 +83,38 @@ describe('plugins', () => { expect(isInjectionFile(input)).toBe(result); }); }); + + describe('shouldGetGitInfo', () => { + const pluginOptions: Options = { + ...defaultPluginOptions, + }; + const sourcemapsOptions = getSourcemapsConfiguration(); + test.each([ + { + description: 'true with sourcemaps', + input: { ...pluginOptions, errorTracking: { sourcemaps: sourcemapsOptions } }, + result: true, + }, + { + description: 'false with no sourcemaps', + input: { ...pluginOptions }, + result: false, + }, + { + description: 'false if disabled globaly', + input: { ...pluginOptions, disableGit: true }, + result: false, + }, + { + description: 'false if disabled localy', + input: { + ...pluginOptions, + errorTracking: { sourcemaps: { ...sourcemapsOptions, disableGit: true } }, + }, + result: false, + }, + ])('Should be $description', ({ input, result }) => { + expect(shouldGetGitInfo(input)).toBe(result); + }); + }); }); diff --git a/packages/core/src/helpers/plugins.ts b/packages/core/src/helpers/plugins.ts index ee1160802..1f685cb35 100644 --- a/packages/core/src/helpers/plugins.ts +++ b/packages/core/src/helpers/plugins.ts @@ -12,6 +12,7 @@ import type { GlobalContext, Input, IterableElement, + Options, Output, SerializedBuildReport, SerializedEntry, @@ -37,6 +38,7 @@ export const serializeBuildReport = (report: BuildReport): SerializedBuildReport const jsonReport: SerializedBuildReport = { bundler: report.bundler, errors: report.errors, + metadata: report.metadata, warnings: report.warnings, logs: report.logs, timings: report.timings, @@ -92,6 +94,7 @@ export const unserializeBuildReport = (report: SerializedBuildReport): BuildRepo const buildReport: BuildReport = { bundler: report.bundler, errors: report.errors, + metadata: report.metadata, warnings: report.warnings, logs: report.logs, timings: report.timings, @@ -250,3 +253,13 @@ export const debugFilesPlugins = (context: GlobalContext): CustomPlugins => { }, ]; }; + +// Verify that we should get the git information based on the options. +// Only get git information if sourcemaps are enabled and git is not disabled. +export const shouldGetGitInfo = (options: Options): boolean => { + return ( + !!options.errorTracking?.sourcemaps && + options.errorTracking?.sourcemaps.disableGit !== true && + options.disableGit !== true + ); +}; diff --git a/packages/core/src/helpers/strings.test.ts b/packages/core/src/helpers/strings.test.ts index 37d46eb44..fa6d37f10 100644 --- a/packages/core/src/helpers/strings.test.ts +++ b/packages/core/src/helpers/strings.test.ts @@ -56,4 +56,80 @@ describe('Strings Helpers', () => { }, ); }); + + describe('filterSensitiveInfoFromRepositoryUrl', () => { + test.each([ + { + description: 'return empty string when input is empty', + input: '', + expected: '', + }, + { + description: 'not modify git@ URLs', + input: 'git@github.com:user/repository.git', + expected: 'git@github.com:user/repository.git', + }, + { + description: 'strip username and password from https URLs', + input: 'https://user:password@github.com/user/repository.git', + expected: 'https://github.com/user/repository.git', + }, + { + description: 'strip username and password from ssh URLs', + input: 'ssh://user:password@github.com/user/repository.git', + expected: 'ssh://github.com/user/repository.git', + }, + { + description: 'strip username and password from ftp URLs', + input: 'ftp://user:password@github.com/user/repository.git', + expected: 'ftp://github.com/user/repository.git', + }, + { + description: 'handle URLs with no credentials', + input: 'https://github.com/user/repository.git', + expected: 'https://github.com/user/repository.git', + }, + { + description: 'handle URLs with port', + input: 'https://github.com:8080/user/repository.git', + expected: 'https://github.com:8080/user/repository.git', + }, + { + description: 'remove root pathname', + input: 'https://github.com/', + expected: 'https://github.com', + }, + { + description: 'handle URLs with only host', + input: 'github.com', + expected: 'github.com', + }, + { + description: 'keep invalid URLs unchanged', + input: 'invalid-url', + expected: 'invalid-url', + }, + ])('Should $description', async ({ input, expected }) => { + const { filterSensitiveInfoFromRepositoryUrl } = await import( + '@dd/core/helpers/strings' + ); + expect(filterSensitiveInfoFromRepositoryUrl(input)).toBe(expected); + }); + }); + + describe('capitalize', () => { + test.each([ + ['hello world', 'Hello World'], + ['hello', 'Hello'], + ['HELLO', 'Hello'], + ['hELLO', 'Hello'], + ['hELLO wORLD', 'Hello World'], + ['hELLO wORLD!', 'Hello World!'], + ['hELLO wORLD! 123', 'Hello World! 123'], + ['', ''], + ])('Should capitalize "%s" => "%s"', async (str, expected) => { + const { capitalize } = await import('@dd/core/helpers/strings'); + expect(capitalize(str)).toBe(expected); + }); + }); }); diff --git a/packages/core/src/helpers/strings.ts b/packages/core/src/helpers/strings.ts index f9066a231..8f275c3d2 100644 --- a/packages/core/src/helpers/strings.ts +++ b/packages/core/src/helpers/strings.ts @@ -41,5 +41,31 @@ export const truncateString = ( return `${str.slice(0, leftStop)}${placeholder}${str.slice(-rightStop)}`; }; +// Remove the sensitive information from a repository URL. +export const filterSensitiveInfoFromRepositoryUrl = (repositoryUrl: string = '') => { + try { + // Keep empty strings and git@ URLs as they are. + if (!repositoryUrl || repositoryUrl.startsWith('git@')) { + return repositoryUrl; + } + + const url = new URL(repositoryUrl); + + // Construct clean URL with protocol, host and pathname (if not root) + const cleanPath = url.pathname === '/' ? '' : url.pathname; + const protocol = url.protocol ? `${url.protocol}//` : ''; + return `${protocol}${url.host}${cleanPath}`; + } catch { + return repositoryUrl; + } +}; + +// Capitalize the first letter of each word in a string. +export const capitalize = (str: string) => + str + .split(' ') + .map((st) => st.charAt(0).toUpperCase() + st.slice(1).toLowerCase()) + .join(' '); + let index = 0; export const getUniqueId = () => `${Date.now()}.${performance.now()}.${++index}`; diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts index 756c73b2f..df5f085dc 100644 --- a/packages/core/src/types.ts +++ b/packages/core/src/types.ts @@ -9,6 +9,8 @@ import type { TrackedFilesMatcher } from '@dd/internal-git-plugin/trackedFilesMatcher'; /* eslint-disable arca/import-ordering */ // #imports-injection-marker +import type { CiVisibilityOptions } from '@dd/ci-visibility-plugin/types'; +import type * as ciVisibility from '@dd/ci-visibility-plugin'; import type { ErrorTrackingOptions } from '@dd/error-tracking-plugin/types'; import type * as errorTracking from '@dd/error-tracking-plugin'; import type { RumOptions } from '@dd/rum-plugin/types'; @@ -28,7 +30,21 @@ export type IterableElement> = IterableType extends Iterable ? ElementType : never; export interface RepositoryData { - hash: string; + commit: { + hash: string; + message: string; + author: { + name: string; + email: string; + date: string; + }; + committer: { + name: string; + email: string; + date: string; + }; + }; + branch: string; remote: string; trackedFilesMatcher: TrackedFilesMatcher; } @@ -42,14 +58,20 @@ export type SerializedEntry = Assign; export type SerializedOutput = Assign; +export type LogTags = string[]; export type Timer = { label: string; pluginName: string; - spans: { start: number; end?: number; tags?: string[] }[]; - tags: string[]; + spans: { start: number; end?: number; tags: LogTags }[]; + tags: LogTags; total: number; logLevel: LogLevel; }; + +export type BuildMetadata = { + name?: string; +}; + export type BuildReport = { bundler: Omit; errors: string[]; @@ -61,6 +83,7 @@ export type BuildReport = { message: string; time: number; }[]; + metadata: BuildMetadata; timings: Timer[]; entries?: Entry[]; inputs?: Input[]; @@ -107,16 +130,16 @@ export type ToInjectItem = { export type TimeLogger = { timer: Timer; - resume: () => void; - end: () => void; - pause: () => void; - tag: (tags: string[], opts?: { span?: boolean }) => void; + resume: (startTime?: number) => void; + end: (endTime?: number) => void; + pause: (pauseTime?: number) => void; + tag: (tags: LogTags, opts?: { span?: boolean }) => void; }; // The rest parameter is a LogLevel or a boolean to auto start the timer. export type TimeLog = ( label: string, - opts?: { level?: LogLevel; start?: boolean; log?: boolean; tags?: string[] }, + opts?: { level?: LogLevel; start?: boolean | number; log?: boolean; tags?: LogTags }, ) => TimeLogger; export type GetLogger = (name: string) => Logger; export type Logger = { @@ -133,18 +156,18 @@ export type TriggerHook = ( ...args: Parameters> ) => R; export type GlobalContext = { + asyncHook: TriggerHook>; auth?: AuthOptions; - inject: (item: ToInjectItem) => void; - bundler: BundlerReport; build: BuildReport; + bundler: BundlerReport; cwd: string; env: Env; getLogger: GetLogger; git?: RepositoryData; - asyncHook: TriggerHook>; hook: TriggerHook; - plugins: (PluginOptions | CustomPluginOptions)[]; + inject: (item: ToInjectItem) => void; pluginNames: string[]; + plugins: (PluginOptions | CustomPluginOptions)[]; sendLog: (message: string, ctx?: any) => Promise; start: number; version: string; @@ -200,6 +223,7 @@ export type AuthOptions = { export interface BaseOptions { auth?: AuthOptions; + metadata?: BuildMetadata; disableGit?: boolean; logLevel?: LogLevel; } @@ -207,6 +231,7 @@ export interface BaseOptions { export interface Options extends BaseOptions { // Each product should have a unique entry. // #types-injection-marker + [ciVisibility.CONFIG_KEY]?: CiVisibilityOptions; [errorTracking.CONFIG_KEY]?: ErrorTrackingOptions; [rum.CONFIG_KEY]?: RumOptions; [telemetry.CONFIG_KEY]?: TelemetryOptions; diff --git a/packages/factory/README.md b/packages/factory/README.md index de1c87f16..485bab3c8 100644 --- a/packages/factory/README.md +++ b/packages/factory/README.md @@ -141,29 +141,22 @@ export const getMyPlugins = (context: GlobalContext) => { The time logger is a helper to log/report the duration of a task. It is useful to debug performance issues. -It can be found in your logger. - -```typescript -const log = context.getLogger('my-plugin'); -const timer = log.time('my-task'); -// [... do stuff ...] -timer.end(); -``` +It can be found on your logger's instance. ### Options -- `start`: Whether to start the timer immediately. Defaults to `true`. -- `log`: Whether to log the timer. Defaults to `true`. -- `level`: The log level to use. Defaults to `debug`. -- `tags`: Initial tags to associate with the timer. Defaults to `[]`. - ```typescript -{ - start: boolean, +const logger = context.getLogger('my-plugin'); +const timer = logger.time('my-task', { + // Whether to start the timer immediately, at the given timestamp or not at all. Defaults to `true`. + start: boolean | number, + // Whether to log the timer or not when it starts and finishes. Defaults to `true`. log: boolean, + // The log level to use. Defaults to `debug`. level: LogLevel, - tags: string[] -} + // Tags to associate with the timer. Defaults to `[]`. + tags: string[], +}); ``` ### Features @@ -171,11 +164,11 @@ timer.end(); Pause/resume the timer. ```typescript -timer.pause(); +timer.pause(timeOverride?: number); // [... do stuff ...] -timer.resume(); +timer.resume(timeOverride?: number); // [... do more stuff ...] -timer.end(); +timer.end(timeOverride?: number); ``` Add tags to the timer or to active spans. @@ -238,7 +231,7 @@ All the timers will be reported in `context.build.timings`, with all their spans "tags": ["step:initialize"] } ], - "tags": ["feature:upload", "operation:compress"], + "tags": ["feature:upload", "operation:compress", "plugin:my-plugin", "level:debug"], "total": 1000, "logLevel": "debug" } @@ -254,38 +247,52 @@ It is passed to your plugin's initialization, and **is mutated during the build 
 type GlobalContext = {
+    // Trigger an asynchronous custom hook.
+    asyncHook: async (name: string, ...args: any[]) => Promise;
     // Mirror of the user's config.
     auth?: {
         apiKey?: string;
+        appKey?: string;
     };
-    // More details on the currently running bundler.
-    bundler: BundlerReport
-    // Added in `writeBundle`.
-    build: BuildReport
+    // Available in the `buildReport` hook.
+    build: BuildReport;
+    // Available in the `bundlerReport` hook.
+    bundler: BundlerReport;
     cwd: string;
-    getLogger: (name: string) => Logger
-    // Added in `buildStart`.
-    git?: Git
-    inject: Injection
+    env: string;
+    getLogger: (name: string) => Logger;
+    // Available in the `git` hook.
+    git?: Git;
+    // Trigger a synchronous custom hook.
+    hook: (name: string, ...args: any[]) => void;
+    inject: Injection;
+    // The list of all the plugin names that are currently running in the ecosystem.
+    pluginNames: string[];
+    // The list of all the plugin instances that are currently running in the ecosystem.
+    plugins: Plugin[];
+    // Send a log to Datadog.
+    sendLog: (message: string, context?: Record) => Promise;
+    // The start time of the build.
     start: number;
+    // The version of the plugin.
     version: string;
 }
> [!NOTE] > Some parts of the context are only available after certain hooks: -> - `context.bundler.rawConfig` is added in the `buildStart` hook. -> - `context.build.*` is populated in the `writeBundle` hook. -> - `context.git.*` is populated in the `buildStart` hook. - -Your function will need to return an array of [Unplugin Plugins definitions](https://unplugin.unjs.io/guide/#supported-hooks). +> - all the helper functions, `asyncHook`, `getLogger`, `hook`, `inject`, `sendLog`, are available in the `init` hook. +> - `cwd` is available in the `cwd` hook. +> - `context.bundler.rawConfig` is available in the `bundlerReport` hook. +> - `context.build.*` is available in the `buildReport` hook. +> - `context.git.*` is available in the `git` hook. ## Hooks ### `init` This hook is called when the factory is done initializing.
-It is useful to initialise some global dependencies. +It is useful to initialise some global dependencies and start using helper functions that are attached to the context. Happens before any other hook. ```typescript diff --git a/packages/factory/package.json b/packages/factory/package.json index 29a06ad91..3bfd27a4e 100644 --- a/packages/factory/package.json +++ b/packages/factory/package.json @@ -16,6 +16,7 @@ "./*": "./src/*.ts" }, "dependencies": { + "@dd/ci-visibility-plugin": "workspace:*", "@dd/core": "workspace:*", "@dd/error-tracking-plugin": "workspace:*", "@dd/internal-analytics-plugin": "workspace:*", diff --git a/packages/factory/src/helpers/context.ts b/packages/factory/src/helpers/context.ts index 4189b9cc3..85d8ba65a 100644 --- a/packages/factory/src/helpers/context.ts +++ b/packages/factory/src/helpers/context.ts @@ -32,6 +32,7 @@ export const getContext = ({ errors: [], warnings: [], logs: [], + metadata: options.metadata || {}, timings: [], bundler: { name: bundlerName, diff --git a/packages/factory/src/helpers/logger.test.ts b/packages/factory/src/helpers/logger.test.ts index fed2f6e50..37cb7a4d4 100644 --- a/packages/factory/src/helpers/logger.test.ts +++ b/packages/factory/src/helpers/logger.test.ts @@ -210,13 +210,15 @@ describe('logger', () => { { start: expect.any(Number), end: expect.any(Number), + tags: ['plugin:testLogger'], }, { start: expect.any(Number), end: expect.any(Number), + tags: ['plugin:testLogger'], }, ], - tags: [], + tags: ['plugin:testLogger', 'level:debug'], total: 300, logLevel: 'debug', }); @@ -230,7 +232,8 @@ describe('logger', () => { timer.tag(['tag1', 'tag2']); timer.end(); - expect(timer.timer.tags).toEqual(['tag1', 'tag2']); + expect(timer.timer.tags).toContain('tag1'); + expect(timer.timer.tags).toContain('tag2'); }); test('Should tag the spans.', () => { @@ -239,7 +242,8 @@ describe('logger', () => { timer.tag(['tag1', 'tag2'], { span: true }); timer.end(); - expect(timer.timer.spans[0].tags).toEqual(['tag1', 'tag2']); + expect(timer.timer.spans[0].tags).toContain('tag1'); + expect(timer.timer.spans[0].tags).toContain('tag2'); }); }); diff --git a/packages/factory/src/helpers/logger.ts b/packages/factory/src/helpers/logger.ts index a3c219c71..a1ac2960c 100644 --- a/packages/factory/src/helpers/logger.ts +++ b/packages/factory/src/helpers/logger.ts @@ -43,13 +43,14 @@ export const getLoggerFactory = logFn = console.log; } - const prefix = `[${type}|${build.bundler.fullName}|${cleanedName}]`; + const buildName = build.metadata?.name ? `${build.metadata.name}|` : ''; + const prefix = `[${buildName}${type}|${build.bundler.fullName}|${cleanedName}]`; // Keep a trace of the log in the build report. const content = typeof text === 'string' ? text : JSON.stringify(text, null, 2); build.logs.push({ bundler: build.bundler.fullName, - pluginName: cleanedName, + pluginName: name, type, message: content, time: Date.now(), @@ -71,10 +72,10 @@ export const getLoggerFactory = const time: TimeLog = (label, opts = {}) => { const { level = 'debug', start = true, log: toLog = true, tags = [] } = opts; const timer: Timer = { - pluginName: cleanedName, + pluginName: name, label, spans: [], - tags, + tags: [...tags, `plugin:${name}`, `level:${level}`], logLevel: level, total: 0, }; @@ -85,16 +86,27 @@ export const getLoggerFactory = const getUncompleteSpans = () => timer.spans.filter((span) => !span.end); // Push a new span. - const resume: TimeLogger['resume'] = () => { + const resume: TimeLogger['resume'] = (startTime?: number) => { + // Ignore if there is already an ongoing span. + const uncompleteSpans = getUncompleteSpans(); + if (uncompleteSpans.length) { + return; + } + // Log the start if it's the first span. if (!timer.spans.length && toLog) { log(c.dim(`[${c.cyan(label)}] : start`), 'debug'); } - timer.spans.push({ start: Date.now() }); + + // Add the new span. + timer.spans.push({ + start: startTime || Date.now(), + tags: [`plugin:${name}`], + }); }; // Complete all the uncompleted spans. - const pause: TimeLogger['pause'] = () => { + const pause: TimeLogger['pause'] = (pauseTime?: number) => { const uncompleteSpans = getUncompleteSpans(); if (!uncompleteSpans?.length) { @@ -107,13 +119,13 @@ export const getLoggerFactory = } for (const span of uncompleteSpans) { - span.end = Date.now(); + span.end = pauseTime || Date.now(); } }; // End the timer and add it to the build report. - const end: TimeLogger['end'] = () => { - pause(); + const end: TimeLogger['end'] = (endTime?: number) => { + pause(endTime); const duration = timer.spans.reduce( (acc, span) => acc + (span.end! - span.start), 0, @@ -130,7 +142,6 @@ export const getLoggerFactory = if (span) { const uncompleteSpans = getUncompleteSpans(); for (const uncompleteSpan of uncompleteSpans) { - uncompleteSpan.tags = uncompleteSpan.tags || []; uncompleteSpan.tags.push(...tagsToAdd); } } else { @@ -140,7 +151,11 @@ export const getLoggerFactory = // Auto start the timer. if (start) { - resume(); + let param: number | undefined; + if (typeof start === 'number') { + param = start; + } + resume(param); } const timeLogger: TimeLogger = { diff --git a/packages/factory/src/helpers/wrapPlugins.test.ts b/packages/factory/src/helpers/wrapPlugins.test.ts index 513767018..47586a20a 100644 --- a/packages/factory/src/helpers/wrapPlugins.test.ts +++ b/packages/factory/src/helpers/wrapPlugins.test.ts @@ -49,10 +49,8 @@ describe('profilePlugins', () => { expect(mockTimer.end).toHaveBeenCalledTimes(1); // Verify the timer got tagged with the plugin names. - expect(mockTimer.timer.tags).toEqual([ - 'plugin:datadog-test-1-plugin', - 'plugin:datadog-test-2-plugin', - ]); + expect(mockTimer.timer.tags).toContain('plugin:datadog-test-1-plugin'); + expect(mockTimer.timer.tags).toContain('plugin:datadog-test-2-plugin'); // Verify the result contains the expected plugins expect(result).toHaveLength(2); diff --git a/packages/factory/src/helpers/wrapPlugins.ts b/packages/factory/src/helpers/wrapPlugins.ts index 7acdba830..199c2af60 100644 --- a/packages/factory/src/helpers/wrapPlugins.ts +++ b/packages/factory/src/helpers/wrapPlugins.ts @@ -3,6 +3,7 @@ // Copyright 2019-Present Datadog, Inc. import { HOST_NAME } from '@dd/core/constants'; +import { cleanPluginName } from '@dd/core/helpers/plugins'; import type { CustomPluginOptions, GetCustomPlugins, @@ -49,7 +50,10 @@ type HookFn = NonNullable { return (...args: Parameters) => { - const timer = log.time(`hook | ${pluginName} | ${hookName}`, { log: false }); + const timer = log.time(`${pluginName} | ${hookName}`, { + log: false, + tags: ['type:hook', `hook:${hookName}`], + }); // @ts-expect-error, can't type "args" correctly: "A spread argument must either have a tuple type or be passed to a rest parameter." const result = hook(...args); @@ -68,12 +72,13 @@ export const wrapPlugin = (plugin: PluginOptions | CustomPluginOptions, log: Log const wrappedPlugin: PluginOptions | CustomPluginOptions = { ...plugin, }; + const name = cleanPluginName(plugin.name); // Wrap all the hooks that we want to trace. for (const hookName of HOOKS_TO_TRACE) { const hook = plugin[hookName as PluginHookName]; if (hook) { - wrappedPlugin[hookName as PluginHookName] = wrapHook(plugin.name, hookName, hook, log); + wrappedPlugin[hookName as PluginHookName] = wrapHook(name, hookName, hook, log); } } diff --git a/packages/factory/src/index.ts b/packages/factory/src/index.ts index 4a38e6980..46c9b2b9d 100644 --- a/packages/factory/src/index.ts +++ b/packages/factory/src/index.ts @@ -31,6 +31,7 @@ import { getContext } from './helpers/context'; import { wrapGetPlugins } from './helpers/wrapPlugins'; import { HOST_NAME } from '@dd/core/constants'; // #imports-injection-marker +import * as ciVisibility from '@dd/ci-visibility-plugin'; import * as errorTracking from '@dd/error-tracking-plugin'; import * as rum from '@dd/rum-plugin'; import * as telemetry from '@dd/telemetry-plugin'; @@ -43,6 +44,7 @@ import { getInjectionPlugins } from '@dd/internal-injection-plugin'; import { getTrueEndPlugins } from '@dd/internal-true-end-plugin'; // #imports-injection-marker // #types-export-injection-marker +export type { types as CiVisibilityTypes } from '@dd/ci-visibility-plugin'; export type { types as ErrorTrackingTypes } from '@dd/error-tracking-plugin'; export type { types as RumTypes } from '@dd/rum-plugin'; export type { types as TelemetryTypes } from '@dd/telemetry-plugin'; @@ -107,6 +109,7 @@ export const buildPluginFactory = ({ // Add the customer facing plugins. pluginsToAdd.push( // #configs-injection-marker + ['ci-visibility', ciVisibility.getPlugins], ['error-tracking', errorTracking.getPlugins], ['rum', rum.getPlugins], ['telemetry', telemetry.getPlugins], diff --git a/packages/factory/src/validate.ts b/packages/factory/src/validate.ts index 80cd28e51..78da01066 100644 --- a/packages/factory/src/validate.ts +++ b/packages/factory/src/validate.ts @@ -9,6 +9,7 @@ export const validateOptions = (options: Options = {}): OptionsWithDefaults => { auth: {}, disableGit: false, logLevel: 'warn', + metadata: {}, ...options, }; }; diff --git a/packages/plugins/analytics/src/index.ts b/packages/plugins/analytics/src/index.ts index fb1503450..70cc09215 100644 --- a/packages/plugins/analytics/src/index.ts +++ b/packages/plugins/analytics/src/index.ts @@ -13,7 +13,7 @@ export const getAnalyticsPlugins: GetInternalPlugins = (arg: GetPluginsArg) => { const { context } = arg; const log = arg.context.getLogger(PLUGIN_NAME); - context.sendLog = async (message: string, overrides: any = {}) => { + context.sendLog = async (message: string, overrides: Record = {}) => { // Only send logs in production. if (context.env !== 'production') { return; @@ -39,6 +39,7 @@ export const getAnalyticsPlugins: GetInternalPlugins = (arg: GetPluginsArg) => { message, service: 'build-plugins', bundler, + metadata: context.build.metadata, plugins: context.pluginNames, version: context.version, team: 'language-foundations', diff --git a/packages/plugins/ci-visibility/README.md b/packages/plugins/ci-visibility/README.md new file mode 100644 index 000000000..a93ceabba --- /dev/null +++ b/packages/plugins/ci-visibility/README.md @@ -0,0 +1,21 @@ +# Ci Visibility Plugin + +Interact with CI Visibility directly from your build system. + + + +## Table of content + + + + +- [Configuration](#configuration) + + +## Configuration + +```ts +ciVisibility?: { + disabled?: boolean; +} +``` \ No newline at end of file diff --git a/packages/plugins/ci-visibility/package.json b/packages/plugins/ci-visibility/package.json new file mode 100644 index 000000000..00a516e5a --- /dev/null +++ b/packages/plugins/ci-visibility/package.json @@ -0,0 +1,30 @@ +{ + "name": "@dd/ci-visibility-plugin", + "packageManager": "yarn@4.0.2", + "license": "MIT", + "private": true, + "author": "Datadog", + "description": "Interact with CI Visibility directly from your build system.", + "homepage": "https://github.com/DataDog/build-plugins/tree/main/packages/plugins/ci-visibility#readme", + "repository": { + "type": "git", + "url": "https://github.com/DataDog/build-plugins", + "directory": "packages/plugins/ci-visibility" + }, + "exports": { + ".": "./src/index.ts", + "./*": "./src/*.ts" + }, + "scripts": { + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@dd/core": "workspace:*", + "@dd/internal-build-report-plugin": "workspace:*", + "chalk": "2.3.1", + "p-queue": "6.6.2" + }, + "devDependencies": { + "typescript": "5.4.3" + } +} diff --git a/packages/plugins/ci-visibility/src/constants.ts b/packages/plugins/ci-visibility/src/constants.ts new file mode 100644 index 000000000..7076371bd --- /dev/null +++ b/packages/plugins/ci-visibility/src/constants.ts @@ -0,0 +1,79 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import type { PluginName } from '@dd/core/types'; + +export const CONFIG_KEY = 'ciVisibility' as const; +export const PLUGIN_NAME: PluginName = 'datadog-ci-visibility-plugin' as const; + +export const CI_ENGINES = { + APPVEYOR: 'appveyor', + AWSCODEPIPELINE: 'awscodepipeline', + AZURE: 'azurepipelines', + BITBUCKET: 'bitbucket', + BITRISE: 'bitrise', + BUDDY: 'buddy', + BUILDKITE: 'buildkite', + CIRCLECI: 'circleci', + CODEFRESH: 'codefresh', + GITHUB: 'github', + GITLAB: 'gitlab', + JENKINS: 'jenkins', + TRAVIS: 'travisci', + TEAMCITY: 'teamcity', + UNKNOWN: 'unknown', +}; + +export const SUPPORTED_PROVIDERS = [ + CI_ENGINES.GITHUB, + CI_ENGINES.GITLAB, + CI_ENGINES.JENKINS, + CI_ENGINES.CIRCLECI, + CI_ENGINES.AWSCODEPIPELINE, + CI_ENGINES.AZURE, + CI_ENGINES.BUILDKITE, +] as const; + +// Tags +// For the CI provider. +export const CI_PIPELINE_URL = 'ci.pipeline.url'; +export const CI_PROVIDER_NAME = 'ci.provider.name'; +export const CI_PIPELINE_ID = 'ci.pipeline.id'; +export const CI_PIPELINE_NAME = 'ci.pipeline.name'; +export const CI_PIPELINE_NUMBER = 'ci.pipeline.number'; +export const CI_WORKSPACE_PATH = 'ci.workspace_path'; +export const GIT_REPOSITORY_URL = 'git.repository_url'; +export const CI_JOB_URL = 'ci.job.url'; +export const CI_JOB_NAME = 'ci.job.name'; +export const CI_STAGE_NAME = 'ci.stage.name'; +export const CI_NODE_NAME = 'ci.node.name'; +export const CI_NODE_LABELS = 'ci.node.labels'; +export const CI_ENV_VARS = '_dd.ci.env_vars'; + +// For Git. +export const GIT_BRANCH = 'git.branch'; +export const GIT_COMMIT_AUTHOR_DATE = 'git.commit.author.date'; +export const GIT_COMMIT_AUTHOR_EMAIL = 'git.commit.author.email'; +export const GIT_COMMIT_AUTHOR_NAME = 'git.commit.author.name'; +export const GIT_COMMIT_COMMITTER_DATE = 'git.commit.committer.date'; +export const GIT_COMMIT_COMMITTER_EMAIL = 'git.commit.committer.email'; +export const GIT_COMMIT_COMMITTER_NAME = 'git.commit.committer.name'; +export const GIT_COMMIT_MESSAGE = 'git.commit.message'; +export const GIT_SHA = 'git.commit.sha'; +export const GIT_TAG = 'git.tag'; +export const GIT_HEAD_SHA = 'git.commit.head_sha'; +export const GIT_BASE_REF = 'git.commit.base_ref'; +export const GIT_PULL_REQUEST_BASE_BRANCH_SHA = 'git.pull_request.base_branch_sha'; +export const GIT_PULL_REQUEST_BASE_BRANCH = 'git.pull_request.base_branch'; + +// For the plugin. +export const BUILD_PLUGIN_VERSION = 'build.plugin.version'; +export const BUILD_PLUGIN_ENV = 'build.plugin.env'; +export const BUILD_PLUGIN_BUNDLER_NAME = 'build.bundler.name'; +export const BUILD_PLUGIN_BUNDLER_VERSION = 'build.bundler.version'; +export const BUILD_PLUGIN_SPAN_PREFIX = 'build.span'; + +// Intake +export const INTAKE_HOST = 'app.datadoghq.com'; +export const INTAKE_PATH = 'api/intake/ci/custom_spans'; diff --git a/packages/plugins/ci-visibility/src/helpers/buildSpansPlugin.ts b/packages/plugins/ci-visibility/src/helpers/buildSpansPlugin.ts new file mode 100644 index 000000000..36fdcdd98 --- /dev/null +++ b/packages/plugins/ci-visibility/src/helpers/buildSpansPlugin.ts @@ -0,0 +1,99 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { shouldGetGitInfo } from '@dd/core/helpers/plugins'; +import type { GlobalContext, Options, PluginName, PluginOptions } from '@dd/core/types'; +import { PLUGIN_NAME as BUILD_REPORT_PLUGIN_NAME } from '@dd/internal-build-report-plugin'; + +export const BUILD_SPANS_PLUGIN_NAME: PluginName = 'datadog-ci-visibility-build-spans-plugin'; + +export const getBuildSpansPlugin = (context: GlobalContext, options: Options): PluginOptions => { + const log = context.getLogger(BUILD_SPANS_PLUGIN_NAME); + + const timeBuildReport = log.time('Build report', { start: false }); + const timeGit = log.time('Git', { start: false }); + const timeHold = log.time('Hold', { start: context.start }); + const timeTotal = log.time('Total time', { start: context.start }); + const timeInit = log.time('Datadog plugins initialization', { start: context.start }); + const timeBuild = log.time('Build', { start: false }); + const timeWrite = log.time('Write', { start: false }); + const timeLoad = log.time('Load', { start: false }); + const timeTransform = log.time('Transform', { start: false }); + + let lastTransformTime = context.start; + let lastWriteTime = context.start; + + return { + name: BUILD_SPANS_PLUGIN_NAME, + enforce: 'pre', + init() { + timeInit.end(); + }, + buildStart() { + timeHold.end(); + timeBuild.resume(); + if (shouldGetGitInfo(options)) { + timeGit.resume(); + } + }, + git() { + timeGit.end(); + }, + loadInclude() { + return true; + }, + load() { + timeLoad.resume(); + return null; + }, + transformInclude() { + return true; + }, + transform() { + timeTransform.resume(); + lastTransformTime = Date.now(); + return null; + }, + buildEnd() { + timeLoad.end(); + timeTransform.end(); + timeBuild.end(); + timeWrite.resume(); + }, + writeBundle() { + lastWriteTime = Date.now(); + }, + buildReport() { + for (const timing of context.build.timings) { + if ( + timing.pluginName !== BUILD_REPORT_PLUGIN_NAME || + timing.label !== 'build report' + ) { + continue; + } + + // Copy build report spans to our own logger. + for (const span of timing.spans) { + const end = span.end || Date.now(); + timeBuildReport.resume(span.start); + timeBuildReport.pause(end); + } + } + }, + asyncTrueEnd() { + // esbuild may not call buildEnd in time to define the write phase. + // So lets simulate this from the last transform time. + // This is a bit of a hack, but it's better than nothing. + if (context.bundler.fullName === 'esbuild') { + if (!timeWrite.timer.spans.length) { + timeWrite.resume(lastTransformTime); + } + } + }, + syncTrueEnd() { + timeWrite.end(lastWriteTime); + timeTotal.end(); + }, + }; +}; diff --git a/packages/plugins/ci-visibility/src/helpers/ciSpanTags.ts b/packages/plugins/ci-visibility/src/helpers/ciSpanTags.ts new file mode 100644 index 000000000..9593edd41 --- /dev/null +++ b/packages/plugins/ci-visibility/src/helpers/ciSpanTags.ts @@ -0,0 +1,763 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { readJsonSync } from '@dd/core/helpers/fs'; +import { filterSensitiveInfoFromRepositoryUrl } from '@dd/core/helpers/strings'; + +import { + GIT_COMMIT_MESSAGE, + GIT_COMMIT_AUTHOR_EMAIL, + GIT_COMMIT_AUTHOR_NAME, + GIT_BRANCH, + GIT_TAG, + CI_WORKSPACE_PATH, + GIT_SHA, + CI_JOB_NAME, + CI_NODE_LABELS, + CI_NODE_NAME, + CI_ENV_VARS, + CI_PIPELINE_URL, + GIT_REPOSITORY_URL, + CI_PROVIDER_NAME, + CI_PIPELINE_NUMBER, + CI_STAGE_NAME, + CI_JOB_URL, + CI_PIPELINE_ID, + CI_PIPELINE_NAME, + CI_ENGINES, + GIT_COMMIT_AUTHOR_DATE, + GIT_PULL_REQUEST_BASE_BRANCH, + GIT_BASE_REF, + GIT_HEAD_SHA, + GIT_PULL_REQUEST_BASE_BRANCH_SHA, + GIT_COMMIT_COMMITTER_EMAIL, + GIT_COMMIT_COMMITTER_NAME, +} from '../constants'; +import type { SpanTags } from '../types'; + +// Receives a string with the form 'John Doe ' +// and returns { name: 'John Doe', email: 'john.doe@gmail.com' } +// Exported for testing purposes. +export const parseEmailAndName = (emailAndName: string | undefined) => { + if (!emailAndName) { + return { name: '', email: '' }; + } + let name = ''; + let email = ''; + const matchNameAndEmail = emailAndName.match(/(?:"?([^"]*)"?\s)?(?:]+)>?)/); + if (matchNameAndEmail) { + name = matchNameAndEmail[1]; + email = matchNameAndEmail[2]; + } + + return { name, email }; +}; + +type GitHubWebhookPayload = { + pull_request?: { + head?: { + sha: string; + }; + base?: { + sha: string; + }; + }; +}; +const getGitHubEventPayload = () => { + if (!process.env.GITHUB_EVENT_PATH) { + return; + } + + return readJsonSync(process.env.GITHUB_EVENT_PATH) as GitHubWebhookPayload; +}; + +// Normalize a ref. +export const normalizeRef = (ref: string | undefined) => { + if (!ref) { + return ref; + } + + return ref.replace(/origin\/|refs\/heads\/|tags\//gm, ''); +}; + +// Resolve a tilde in a file path. +const resolveTilde = (filePath: string | undefined) => { + if (!filePath || typeof filePath !== 'string') { + return ''; + } + // '~/folder/path' or '~' + if (filePath[0] === '~' && (filePath[1] === '/' || filePath.length === 1)) { + return filePath.replace('~', process.env.HOME ?? ''); + } + + return filePath; +}; + +// Remove empty values from the tags object. +const removeEmptyValues = (tags: SpanTags) => { + return Object.fromEntries( + Object.entries(tags).filter( + ([_, value]) => value !== null && value !== undefined && value !== '', + ), + ); +}; + +export const getCIProvider = (): string => { + if (process.env.CIRCLECI) { + return CI_ENGINES.CIRCLECI; + } + + if (process.env.GITLAB_CI) { + return CI_ENGINES.GITLAB; + } + + if (process.env.GITHUB_ACTIONS || process.env.GITHUB_ACTION) { + return CI_ENGINES.GITHUB; + } + + if (process.env.BUILDKITE) { + return CI_ENGINES.BUILDKITE; + } + + if (process.env.BUDDY) { + return CI_ENGINES.BUDDY; + } + + if (process.env.TEAMCITY_VERSION) { + return CI_ENGINES.TEAMCITY; + } + + if (process.env.JENKINS_URL) { + return CI_ENGINES.JENKINS; + } + + if (process.env.TF_BUILD) { + return CI_ENGINES.AZURE; + } + + if (process.env.CF_BUILD_ID) { + return CI_ENGINES.CODEFRESH; + } + + if (process.env.APPVEYOR) { + return CI_ENGINES.APPVEYOR; + } + + if (process.env.BITBUCKET_COMMIT) { + return CI_ENGINES.BITBUCKET; + } + + if (process.env.BITRISE_BUILD_SLUG) { + return CI_ENGINES.BITRISE; + } + + if (process.env.CODEBUILD_INITIATOR?.startsWith('codepipeline')) { + return CI_ENGINES.AWSCODEPIPELINE; + } + + return CI_ENGINES.UNKNOWN; +}; + +export const getCISpanTags = (): SpanTags => { + const env = process.env; + let tags: SpanTags = {}; + + if (env.DRONE) { + const { + DRONE_BUILD_NUMBER, + DRONE_BUILD_LINK, + DRONE_STEP_NAME, + DRONE_STAGE_NAME, + DRONE_WORKSPACE, + DRONE_GIT_HTTP_URL, + DRONE_COMMIT_SHA, + DRONE_BRANCH, + DRONE_TAG, + DRONE_COMMIT_AUTHOR_NAME, + DRONE_COMMIT_AUTHOR_EMAIL, + DRONE_COMMIT_MESSAGE, + } = env; + tags = { + [CI_PROVIDER_NAME]: 'drone', + [CI_PIPELINE_NUMBER]: DRONE_BUILD_NUMBER, + [CI_PIPELINE_URL]: DRONE_BUILD_LINK, + [CI_JOB_NAME]: DRONE_STEP_NAME, + [CI_STAGE_NAME]: DRONE_STAGE_NAME, + [CI_WORKSPACE_PATH]: DRONE_WORKSPACE, + [GIT_REPOSITORY_URL]: DRONE_GIT_HTTP_URL, + [GIT_SHA]: DRONE_COMMIT_SHA, + [GIT_BRANCH]: DRONE_BRANCH, + [GIT_TAG]: DRONE_TAG, + [GIT_COMMIT_AUTHOR_NAME]: DRONE_COMMIT_AUTHOR_NAME, + [GIT_COMMIT_AUTHOR_EMAIL]: DRONE_COMMIT_AUTHOR_EMAIL, + [GIT_COMMIT_MESSAGE]: DRONE_COMMIT_MESSAGE, + }; + } + + if (env.CIRCLECI) { + const { + CIRCLE_BUILD_NUM, + CIRCLE_WORKFLOW_ID, + CIRCLE_PROJECT_REPONAME, + CIRCLE_BUILD_URL, + CIRCLE_WORKING_DIRECTORY, + CIRCLE_BRANCH, + CIRCLE_TAG, + CIRCLE_SHA1, + CIRCLE_REPOSITORY_URL, + CIRCLE_JOB, + } = env; + + const pipelineUrl = `https://app.circleci.com/pipelines/workflows/${CIRCLE_WORKFLOW_ID}`; + + tags = { + [CI_JOB_URL]: CIRCLE_BUILD_URL, + [CI_PIPELINE_ID]: CIRCLE_WORKFLOW_ID, + [CI_PIPELINE_NAME]: CIRCLE_PROJECT_REPONAME, + [CI_PIPELINE_URL]: pipelineUrl, + [CI_JOB_NAME]: CIRCLE_JOB, + [CI_PROVIDER_NAME]: CI_ENGINES.CIRCLECI, + [CI_WORKSPACE_PATH]: CIRCLE_WORKING_DIRECTORY, + [GIT_SHA]: CIRCLE_SHA1, + [GIT_REPOSITORY_URL]: CIRCLE_REPOSITORY_URL, + [GIT_TAG]: CIRCLE_TAG, + [GIT_BRANCH]: CIRCLE_BRANCH, + [CI_ENV_VARS]: JSON.stringify({ + CIRCLE_WORKFLOW_ID, + // Snapshots are generated automatically and are sort sensitive + CIRCLE_BUILD_NUM, + }), + }; + } + + if (env.TRAVIS) { + const { + TRAVIS_PULL_REQUEST_BRANCH, + TRAVIS_BRANCH, + TRAVIS_COMMIT, + TRAVIS_REPO_SLUG, + TRAVIS_TAG, + TRAVIS_JOB_WEB_URL, + TRAVIS_BUILD_ID, + TRAVIS_BUILD_NUMBER, + TRAVIS_BUILD_WEB_URL, + TRAVIS_BUILD_DIR, + TRAVIS_COMMIT_MESSAGE, + } = env; + tags = { + [CI_JOB_URL]: TRAVIS_JOB_WEB_URL, + [CI_PIPELINE_ID]: TRAVIS_BUILD_ID, + [CI_PIPELINE_NAME]: TRAVIS_REPO_SLUG, + [CI_PIPELINE_NUMBER]: TRAVIS_BUILD_NUMBER, + [CI_PIPELINE_URL]: TRAVIS_BUILD_WEB_URL, + [CI_PROVIDER_NAME]: CI_ENGINES.TRAVIS, + [CI_WORKSPACE_PATH]: TRAVIS_BUILD_DIR, + [GIT_SHA]: TRAVIS_COMMIT, + [GIT_TAG]: TRAVIS_TAG, + [GIT_BRANCH]: TRAVIS_PULL_REQUEST_BRANCH || TRAVIS_BRANCH, + [GIT_REPOSITORY_URL]: `https://github.com/${TRAVIS_REPO_SLUG}.git`, + [GIT_COMMIT_MESSAGE]: TRAVIS_COMMIT_MESSAGE, + }; + } + + if (env.GITLAB_CI) { + const { + CI_PIPELINE_ID: GITLAB_CI_PIPELINE_ID, + CI_PROJECT_PATH, + CI_PIPELINE_IID, + CI_PIPELINE_URL: GITLAB_CI_PIPELINE_URL, + CI_PROJECT_DIR, + CI_COMMIT_REF_NAME, + CI_COMMIT_TAG, + CI_COMMIT_SHA, + CI_REPOSITORY_URL, + CI_JOB_URL: GITLAB_CI_JOB_URL, + CI_JOB_STAGE, + CI_JOB_NAME: GITLAB_CI_JOB_NAME, + CI_COMMIT_MESSAGE, + CI_COMMIT_TIMESTAMP, + CI_COMMIT_AUTHOR, + CI_JOB_ID: GITLAB_CI_JOB_ID, + CI_PROJECT_URL: GITLAB_CI_PROJECT_URL, + CI_RUNNER_ID, + CI_RUNNER_TAGS, + } = env; + + const { name, email } = parseEmailAndName(CI_COMMIT_AUTHOR); + + tags = { + [CI_JOB_NAME]: GITLAB_CI_JOB_NAME, + [CI_JOB_URL]: GITLAB_CI_JOB_URL, + [CI_PIPELINE_ID]: GITLAB_CI_PIPELINE_ID, + [CI_PIPELINE_NAME]: CI_PROJECT_PATH, + [CI_PIPELINE_NUMBER]: CI_PIPELINE_IID, + [CI_PIPELINE_URL]: GITLAB_CI_PIPELINE_URL, + [CI_PROVIDER_NAME]: CI_ENGINES.GITLAB, + [CI_WORKSPACE_PATH]: CI_PROJECT_DIR, + [CI_STAGE_NAME]: CI_JOB_STAGE, + [GIT_BRANCH]: CI_COMMIT_REF_NAME, + [GIT_SHA]: CI_COMMIT_SHA, + [GIT_REPOSITORY_URL]: CI_REPOSITORY_URL, + [GIT_TAG]: CI_COMMIT_TAG, + [GIT_COMMIT_MESSAGE]: CI_COMMIT_MESSAGE, + [GIT_COMMIT_AUTHOR_NAME]: name, + [GIT_COMMIT_AUTHOR_EMAIL]: email, + [GIT_COMMIT_AUTHOR_DATE]: CI_COMMIT_TIMESTAMP, + [CI_ENV_VARS]: JSON.stringify({ + CI_PROJECT_URL: GITLAB_CI_PROJECT_URL, + // Snapshots are generated automatically and are sort sensitive + CI_PIPELINE_ID: GITLAB_CI_PIPELINE_ID, + CI_JOB_ID: GITLAB_CI_JOB_ID, + }), + [CI_NODE_LABELS]: CI_RUNNER_TAGS, + [CI_NODE_NAME]: CI_RUNNER_ID, + }; + } + + if (env.GITHUB_ACTIONS || env.GITHUB_ACTION) { + const { + GITHUB_RUN_ID, + GITHUB_WORKFLOW, + GITHUB_RUN_NUMBER, + GITHUB_WORKSPACE, + GITHUB_HEAD_REF, + GITHUB_JOB, + GITHUB_REF, + GITHUB_SHA, + GITHUB_REPOSITORY, + GITHUB_SERVER_URL, + GITHUB_RUN_ATTEMPT, + DD_GITHUB_JOB_NAME, + GITHUB_BASE_REF, + } = env; + const repositoryUrl = `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git`; + let pipelineURL = `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}`; + + // Some older versions of enterprise might not have this yet. + if (GITHUB_RUN_ATTEMPT) { + pipelineURL += `/attempts/${GITHUB_RUN_ATTEMPT}`; + } + + tags = { + [CI_JOB_NAME]: GITHUB_JOB, + [CI_JOB_URL]: filterSensitiveInfoFromRepositoryUrl( + `${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/commit/${GITHUB_SHA}/checks`, + ), + [CI_PIPELINE_ID]: GITHUB_RUN_ID, + [CI_PIPELINE_NAME]: GITHUB_WORKFLOW, + [CI_PIPELINE_NUMBER]: GITHUB_RUN_NUMBER, + [CI_PIPELINE_URL]: filterSensitiveInfoFromRepositoryUrl(pipelineURL), + [CI_PROVIDER_NAME]: CI_ENGINES.GITHUB, + [CI_WORKSPACE_PATH]: GITHUB_WORKSPACE, + [GIT_SHA]: GITHUB_SHA, + [GIT_REPOSITORY_URL]: repositoryUrl, + [GIT_BRANCH]: GITHUB_HEAD_REF || GITHUB_REF || '', + [CI_ENV_VARS]: JSON.stringify({ + GITHUB_SERVER_URL: filterSensitiveInfoFromRepositoryUrl(GITHUB_SERVER_URL), + // Snapshots are generated automatically and are sort sensitive + GITHUB_REPOSITORY, + GITHUB_RUN_ID, + GITHUB_RUN_ATTEMPT, + DD_GITHUB_JOB_NAME, + }), + }; + + if (GITHUB_BASE_REF) { + // GITHUB_BASE_REF is defined if it's a pull_request or pull_request_target trigger + tags[GIT_BASE_REF] = GITHUB_BASE_REF; + tags[GIT_PULL_REQUEST_BASE_BRANCH] = GITHUB_BASE_REF; + try { + const eventPayload = getGitHubEventPayload(); + tags[GIT_HEAD_SHA] = eventPayload?.pull_request?.head?.sha; + tags[GIT_PULL_REQUEST_BASE_BRANCH_SHA] = eventPayload?.pull_request?.base?.sha; + } catch (e) { + // ignore malformed event content + } + } + } + + if (env.JENKINS_URL) { + const { + WORKSPACE, + BUILD_TAG, + JOB_NAME, + BUILD_NUMBER, + BUILD_URL, + GIT_BRANCH: JENKINS_GIT_BRANCH, + GIT_COMMIT, + GIT_URL, + GIT_URL_1, + DD_CUSTOM_TRACE_ID, + DD_CUSTOM_PARENT_ID, + NODE_NAME, + NODE_LABELS, + } = env; + + tags = { + [CI_PIPELINE_ID]: BUILD_TAG, + [CI_PIPELINE_NUMBER]: BUILD_NUMBER, + [CI_PIPELINE_URL]: BUILD_URL, + [CI_PROVIDER_NAME]: CI_ENGINES.JENKINS, + [CI_WORKSPACE_PATH]: WORKSPACE, + [GIT_SHA]: GIT_COMMIT, + [GIT_REPOSITORY_URL]: GIT_URL || GIT_URL_1, + [GIT_BRANCH]: JENKINS_GIT_BRANCH, + [CI_NODE_NAME]: NODE_NAME, + [CI_ENV_VARS]: JSON.stringify({ + DD_CUSTOM_TRACE_ID, + DD_CUSTOM_PARENT_ID, + }), + }; + + if (NODE_LABELS) { + let nodeLabels; + try { + nodeLabels = JSON.stringify(NODE_LABELS.split(' ')); + tags[CI_NODE_LABELS] = nodeLabels; + } catch (e) { + // ignore errors + } + } + + let finalPipelineName = ''; + if (JOB_NAME) { + // Job names can contain parameters, e.g. jobName/KEY1=VALUE1,KEY2=VALUE2/branchName + const jobNameAndParams = JOB_NAME.split('/'); + if (jobNameAndParams.length > 1 && jobNameAndParams[1].includes('=')) { + finalPipelineName = jobNameAndParams[0]; + } else { + const normalizedBranch = normalizeRef(JENKINS_GIT_BRANCH); + finalPipelineName = JOB_NAME.replace(`/${normalizedBranch}`, ''); + } + tags[CI_PIPELINE_NAME] = finalPipelineName; + } + } + + if (env.BUILDKITE) { + const { + BUILDKITE_AGENT_ID, + BUILDKITE_BRANCH, + BUILDKITE_COMMIT, + BUILDKITE_REPO, + BUILDKITE_TAG, + BUILDKITE_BUILD_ID, + BUILDKITE_PIPELINE_SLUG, + BUILDKITE_BUILD_NUMBER, + BUILDKITE_BUILD_URL, + BUILDKITE_JOB_ID, + BUILDKITE_BUILD_CHECKOUT_PATH, + BUILDKITE_BUILD_AUTHOR, + BUILDKITE_BUILD_AUTHOR_EMAIL, + BUILDKITE_MESSAGE, + } = env; + + const extraTags = Object.keys(env) + .filter((envVar) => envVar.startsWith('BUILDKITE_AGENT_META_DATA_')) + .map((metadataKey) => { + const key = metadataKey.replace('BUILDKITE_AGENT_META_DATA_', '').toLowerCase(); + + return `${key}:${env[metadataKey]}`; + }); + + tags = { + [CI_NODE_NAME]: BUILDKITE_AGENT_ID, + [CI_PROVIDER_NAME]: CI_ENGINES.BUILDKITE, + [CI_PIPELINE_ID]: BUILDKITE_BUILD_ID, + [CI_PIPELINE_NAME]: BUILDKITE_PIPELINE_SLUG, + [CI_PIPELINE_NUMBER]: BUILDKITE_BUILD_NUMBER, + [CI_PIPELINE_URL]: BUILDKITE_BUILD_URL, + [CI_JOB_URL]: `${BUILDKITE_BUILD_URL}#${BUILDKITE_JOB_ID}`, + [GIT_SHA]: BUILDKITE_COMMIT, + [CI_WORKSPACE_PATH]: BUILDKITE_BUILD_CHECKOUT_PATH, + [GIT_REPOSITORY_URL]: BUILDKITE_REPO, + [GIT_TAG]: BUILDKITE_TAG, + [GIT_BRANCH]: BUILDKITE_BRANCH, + [GIT_COMMIT_AUTHOR_NAME]: BUILDKITE_BUILD_AUTHOR, + [GIT_COMMIT_AUTHOR_EMAIL]: BUILDKITE_BUILD_AUTHOR_EMAIL, + [GIT_COMMIT_MESSAGE]: BUILDKITE_MESSAGE, + [CI_ENV_VARS]: JSON.stringify({ + BUILDKITE_BUILD_ID, + BUILDKITE_JOB_ID, + }), + }; + if (extraTags.length) { + tags[CI_NODE_LABELS] = JSON.stringify(extraTags); + } + } + + if (env.BITRISE_BUILD_SLUG) { + const { + BITRISE_GIT_COMMIT, + GIT_CLONE_COMMIT_HASH, + BITRISEIO_GIT_BRANCH_DEST, + BITRISE_GIT_BRANCH, + BITRISE_BUILD_SLUG, + BITRISE_TRIGGERED_WORKFLOW_ID, + BITRISE_BUILD_NUMBER, + BITRISE_BUILD_URL, + BITRISE_SOURCE_DIR, + GIT_REPOSITORY_URL: BITRISE_GIT_REPOSITORY_URL, + BITRISE_GIT_TAG, + BITRISE_GIT_MESSAGE, + } = env; + + tags = { + [CI_PROVIDER_NAME]: CI_ENGINES.BITRISE, + [CI_PIPELINE_ID]: BITRISE_BUILD_SLUG, + [CI_PIPELINE_NAME]: BITRISE_TRIGGERED_WORKFLOW_ID, + [CI_PIPELINE_NUMBER]: BITRISE_BUILD_NUMBER, + [CI_PIPELINE_URL]: BITRISE_BUILD_URL, + [GIT_SHA]: BITRISE_GIT_COMMIT || GIT_CLONE_COMMIT_HASH, + [GIT_REPOSITORY_URL]: BITRISE_GIT_REPOSITORY_URL, + [CI_WORKSPACE_PATH]: BITRISE_SOURCE_DIR, + [GIT_TAG]: BITRISE_GIT_TAG, + [GIT_BRANCH]: BITRISEIO_GIT_BRANCH_DEST || BITRISE_GIT_BRANCH, + [GIT_COMMIT_MESSAGE]: BITRISE_GIT_MESSAGE, + }; + } + + if (env.BITBUCKET_COMMIT) { + const { + BITBUCKET_REPO_FULL_NAME, + BITBUCKET_BUILD_NUMBER, + BITBUCKET_BRANCH, + BITBUCKET_COMMIT, + BITBUCKET_GIT_SSH_ORIGIN, + BITBUCKET_GIT_HTTP_ORIGIN, + BITBUCKET_TAG, + BITBUCKET_PIPELINE_UUID, + BITBUCKET_CLONE_DIR, + } = env; + + const url = `https://bitbucket.org/${BITBUCKET_REPO_FULL_NAME}/addon/pipelines/home#!/results/${BITBUCKET_BUILD_NUMBER}`; + + tags = { + [CI_PROVIDER_NAME]: CI_ENGINES.BITBUCKET, + [GIT_SHA]: BITBUCKET_COMMIT, + [CI_PIPELINE_NUMBER]: BITBUCKET_BUILD_NUMBER, + [CI_PIPELINE_NAME]: BITBUCKET_REPO_FULL_NAME, + [CI_JOB_URL]: url, + [CI_PIPELINE_URL]: url, + [GIT_BRANCH]: BITBUCKET_BRANCH, + [GIT_TAG]: BITBUCKET_TAG, + [GIT_REPOSITORY_URL]: BITBUCKET_GIT_SSH_ORIGIN || BITBUCKET_GIT_HTTP_ORIGIN, + [CI_WORKSPACE_PATH]: BITBUCKET_CLONE_DIR, + [CI_PIPELINE_ID]: + BITBUCKET_PIPELINE_UUID && BITBUCKET_PIPELINE_UUID.replace(/{|}/gm, ''), + }; + } + + if (env.CF_BUILD_ID) { + const { CF_BUILD_ID, CF_PIPELINE_NAME, CF_BUILD_URL, CF_STEP_NAME, CF_BRANCH } = env; + + tags = { + [CI_PROVIDER_NAME]: CI_ENGINES.CODEFRESH, + [CI_PIPELINE_ID]: CF_BUILD_ID, + [CI_PIPELINE_URL]: CF_BUILD_URL, + [CI_PIPELINE_NAME]: CF_PIPELINE_NAME, + [CI_JOB_NAME]: CF_STEP_NAME, + [GIT_BRANCH]: CF_BRANCH, + [CI_ENV_VARS]: JSON.stringify({ CF_BUILD_ID }), + }; + } + + if (env.TEAMCITY_VERSION) { + const { BUILD_URL, TEAMCITY_BUILDCONF_NAME } = env; + + tags = { + [CI_PROVIDER_NAME]: CI_ENGINES.TEAMCITY, + [CI_JOB_URL]: BUILD_URL, + [CI_JOB_NAME]: TEAMCITY_BUILDCONF_NAME, + }; + } + + if (env.TF_BUILD) { + const { + BUILD_SOURCESDIRECTORY, + BUILD_BUILDID, + BUILD_DEFINITIONNAME, + SYSTEM_TEAMFOUNDATIONSERVERURI, + SYSTEM_TEAMPROJECTID, + SYSTEM_JOBID, + SYSTEM_TASKINSTANCEID, + SYSTEM_PULLREQUEST_SOURCEBRANCH, + BUILD_SOURCEBRANCH, + BUILD_SOURCEBRANCHNAME, + SYSTEM_PULLREQUEST_SOURCECOMMITID, + SYSTEM_PULLREQUEST_SOURCEREPOSITORYURI, + BUILD_REPOSITORY_URI, + BUILD_SOURCEVERSION, + BUILD_REQUESTEDFORID, + BUILD_REQUESTEDFOREMAIL, + BUILD_SOURCEVERSIONMESSAGE, + SYSTEM_STAGEDISPLAYNAME, + SYSTEM_JOBDISPLAYNAME, + } = env; + + tags = { + [CI_PROVIDER_NAME]: CI_ENGINES.AZURE, + [CI_PIPELINE_ID]: BUILD_BUILDID, + [CI_PIPELINE_NAME]: BUILD_DEFINITIONNAME, + [CI_PIPELINE_NUMBER]: BUILD_BUILDID, + [GIT_SHA]: SYSTEM_PULLREQUEST_SOURCECOMMITID || BUILD_SOURCEVERSION, + [CI_WORKSPACE_PATH]: BUILD_SOURCESDIRECTORY, + [GIT_REPOSITORY_URL]: SYSTEM_PULLREQUEST_SOURCEREPOSITORYURI || BUILD_REPOSITORY_URI, + [GIT_BRANCH]: + SYSTEM_PULLREQUEST_SOURCEBRANCH || BUILD_SOURCEBRANCH || BUILD_SOURCEBRANCHNAME, + [GIT_COMMIT_AUTHOR_NAME]: BUILD_REQUESTEDFORID, + [GIT_COMMIT_AUTHOR_EMAIL]: BUILD_REQUESTEDFOREMAIL, + [GIT_COMMIT_MESSAGE]: BUILD_SOURCEVERSIONMESSAGE, + [CI_STAGE_NAME]: SYSTEM_STAGEDISPLAYNAME, + [CI_JOB_NAME]: SYSTEM_JOBDISPLAYNAME, + [CI_ENV_VARS]: JSON.stringify({ + SYSTEM_TEAMPROJECTID, + BUILD_BUILDID, + SYSTEM_JOBID, + }), + }; + + if (SYSTEM_TEAMFOUNDATIONSERVERURI && SYSTEM_TEAMPROJECTID && BUILD_BUILDID) { + const baseUrl = `${SYSTEM_TEAMFOUNDATIONSERVERURI}${SYSTEM_TEAMPROJECTID}/_build/results?buildId=${BUILD_BUILDID}`; + const pipelineUrl = baseUrl; + const jobUrl = `${baseUrl}&view=logs&j=${SYSTEM_JOBID}&t=${SYSTEM_TASKINSTANCEID}`; + + tags = { + ...tags, + [CI_PIPELINE_URL]: pipelineUrl, + [CI_JOB_URL]: jobUrl, + }; + } + } + + if (env.APPVEYOR) { + const { + APPVEYOR_REPO_NAME, + APPVEYOR_REPO_PROVIDER, + APPVEYOR_BUILD_FOLDER, + APPVEYOR_BUILD_ID, + APPVEYOR_BUILD_NUMBER, + APPVEYOR_REPO_COMMIT, + APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH, + APPVEYOR_REPO_BRANCH, + APPVEYOR_REPO_TAG_NAME, + APPVEYOR_REPO_COMMIT_AUTHOR, + APPVEYOR_REPO_COMMIT_AUTHOR_EMAIL, + APPVEYOR_REPO_COMMIT_MESSAGE, + APPVEYOR_REPO_COMMIT_MESSAGE_EXTENDED, + } = env; + + const pipelineUrl = `https://ci.appveyor.com/project/${APPVEYOR_REPO_NAME}/builds/${APPVEYOR_BUILD_ID}`; + + tags = { + [CI_PROVIDER_NAME]: CI_ENGINES.APPVEYOR, + [CI_PIPELINE_URL]: pipelineUrl, + [CI_PIPELINE_ID]: APPVEYOR_BUILD_ID, + [CI_PIPELINE_NAME]: APPVEYOR_REPO_NAME, + [CI_PIPELINE_NUMBER]: APPVEYOR_BUILD_NUMBER, + [CI_JOB_URL]: pipelineUrl, + [CI_WORKSPACE_PATH]: APPVEYOR_BUILD_FOLDER, + [GIT_COMMIT_AUTHOR_NAME]: APPVEYOR_REPO_COMMIT_AUTHOR, + [GIT_COMMIT_AUTHOR_EMAIL]: APPVEYOR_REPO_COMMIT_AUTHOR_EMAIL, + [GIT_COMMIT_MESSAGE]: `${APPVEYOR_REPO_COMMIT_MESSAGE || ''}\n${APPVEYOR_REPO_COMMIT_MESSAGE_EXTENDED || ''}`, + }; + + if (APPVEYOR_REPO_PROVIDER === 'github') { + tags = { + ...tags, + [GIT_REPOSITORY_URL]: `https://github.com/${APPVEYOR_REPO_NAME}.git`, + [GIT_SHA]: APPVEYOR_REPO_COMMIT, + [GIT_TAG]: APPVEYOR_REPO_TAG_NAME, + [GIT_BRANCH]: APPVEYOR_PULL_REQUEST_HEAD_REPO_BRANCH || APPVEYOR_REPO_BRANCH, + }; + } + } + + if (env.BUDDY) { + const { + BUDDY_PIPELINE_NAME, + BUDDY_PIPELINE_ID, + BUDDY_EXECUTION_ID, + BUDDY_SCM_URL, + BUDDY_EXECUTION_BRANCH, + BUDDY_EXECUTION_TAG, + BUDDY_EXECUTION_REVISION, + BUDDY_EXECUTION_URL, + BUDDY_EXECUTION_REVISION_MESSAGE, + BUDDY_EXECUTION_REVISION_COMMITTER_NAME, + BUDDY_EXECUTION_REVISION_COMMITTER_EMAIL, + } = env; + + tags = { + [CI_PROVIDER_NAME]: CI_ENGINES.BUDDY, + [CI_PIPELINE_ID]: `${BUDDY_PIPELINE_ID || ''}/${BUDDY_EXECUTION_ID || ''}`, + [CI_PIPELINE_NAME]: BUDDY_PIPELINE_NAME, + [CI_PIPELINE_NUMBER]: `${BUDDY_EXECUTION_ID || ''}`, // gets parsed to int again later using parsePipelineNumber + [CI_PIPELINE_URL]: BUDDY_EXECUTION_URL, + [GIT_SHA]: BUDDY_EXECUTION_REVISION, + [GIT_BRANCH]: BUDDY_EXECUTION_BRANCH, + [GIT_TAG]: BUDDY_EXECUTION_TAG, + [GIT_REPOSITORY_URL]: BUDDY_SCM_URL, + [GIT_COMMIT_MESSAGE]: BUDDY_EXECUTION_REVISION_MESSAGE, + [GIT_COMMIT_COMMITTER_EMAIL]: BUDDY_EXECUTION_REVISION_COMMITTER_EMAIL, + [GIT_COMMIT_COMMITTER_NAME]: BUDDY_EXECUTION_REVISION_COMMITTER_NAME, + }; + } + + if (env.CF_BUILD_ID) { + const { CF_BUILD_ID, CF_PIPELINE_NAME, CF_BUILD_URL, CF_STEP_NAME, CF_BRANCH } = env; + tags = { + [CI_PROVIDER_NAME]: 'codefresh', + [CI_PIPELINE_ID]: CF_BUILD_ID, + [CI_PIPELINE_NAME]: CF_PIPELINE_NAME, + [CI_PIPELINE_URL]: CF_BUILD_URL, + [CI_JOB_NAME]: CF_STEP_NAME, + [CI_ENV_VARS]: JSON.stringify({ CF_BUILD_ID }), + }; + + const isTag = CF_BRANCH && CF_BRANCH.includes('tags/'); + const refKey = isTag ? GIT_TAG : GIT_BRANCH; + const ref = normalizeRef(CF_BRANCH); + + tags[refKey] = ref; + } + + if (env.CODEBUILD_INITIATOR?.startsWith('codepipeline')) { + const { CODEBUILD_BUILD_ARN, DD_ACTION_EXECUTION_ID, DD_PIPELINE_EXECUTION_ID } = env; + tags = { + [CI_PROVIDER_NAME]: CI_ENGINES.AWSCODEPIPELINE, + [CI_PIPELINE_ID]: DD_PIPELINE_EXECUTION_ID, + [CI_ENV_VARS]: JSON.stringify({ + CODEBUILD_BUILD_ARN, + DD_PIPELINE_EXECUTION_ID, + DD_ACTION_EXECUTION_ID, + }), + }; + } + + if (tags[CI_WORKSPACE_PATH]) { + tags[CI_WORKSPACE_PATH] = resolveTilde(tags[CI_WORKSPACE_PATH]); + } + if (tags[GIT_REPOSITORY_URL]) { + tags[GIT_REPOSITORY_URL] = filterSensitiveInfoFromRepositoryUrl(tags[GIT_REPOSITORY_URL]); + } + + if (tags[GIT_TAG]) { + tags[GIT_TAG] = normalizeRef(tags[GIT_TAG]); + } + + if (tags[GIT_BRANCH]) { + // Here we handle the case where GIT_BRANCH actually contains a tag + const branch = tags[GIT_BRANCH] || ''; + if (branch.startsWith('tags/') || branch.includes('/tags/')) { + if (!tags[GIT_TAG]) { + tags[GIT_TAG] = normalizeRef(branch); + } + tags[GIT_BRANCH] = ''; + } else { + tags[GIT_BRANCH] = normalizeRef(branch); + } + } + + return removeEmptyValues(tags); +}; diff --git a/packages/plugins/ci-visibility/src/helpers/customSpans.ts b/packages/plugins/ci-visibility/src/helpers/customSpans.ts new file mode 100644 index 000000000..28e06aa46 --- /dev/null +++ b/packages/plugins/ci-visibility/src/helpers/customSpans.ts @@ -0,0 +1,56 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { capitalize } from '@dd/core/helpers/strings'; +import type { GlobalContext } from '@dd/core/types'; +import crypto from 'crypto'; + +import type { CustomSpan } from '../types'; + +import { BUILD_SPANS_PLUGIN_NAME } from './buildSpansPlugin'; + +type SimpleSpan = Omit< + CustomSpan, + 'ci_provider' | 'span_id' | 'error_message' | 'exit_code' | 'measures' +>; + +export const getBuildName = (context: GlobalContext): string => { + return context.build.metadata?.name ? `"${context.build.metadata.name}"` : '"unknown build"'; +}; + +export const getCustomSpan = (provider: string, overrides: SimpleSpan): CustomSpan => ({ + ci_provider: provider, + span_id: crypto.randomBytes(5).toString('hex'), + error_message: '', + exit_code: 0, + measures: {}, + ...overrides, +}); + +export const getCustomSpans = (provider: string, context: GlobalContext): CustomSpan[] => { + const buildName = getBuildName(context); + const name = `Build of ${buildName} with ${capitalize(context.bundler.fullName)}`; + const spans: SimpleSpan[] = []; + + // Add all the spans from the time loggers. + for (const timing of context.build.timings) { + // Only add spans that are coming from our own plugin. + if (timing.pluginName !== BUILD_SPANS_PLUGIN_NAME) { + continue; + } + + for (const span of timing.spans) { + const end = span.end || Date.now(); + spans.push({ + command: `${name} | ${timing.pluginName} | ${capitalize(timing.label)}`, + name: `${capitalize(timing.label)}`, + start_time: new Date(span.start).toISOString(), + end_time: new Date(end).toISOString(), + tags: [`buildName:${buildName}`, ...timing.tags, ...span.tags], + }); + } + } + + return spans.map((span) => getCustomSpan(provider, span)); +}; diff --git a/packages/plugins/ci-visibility/src/helpers/parseTags.test.ts b/packages/plugins/ci-visibility/src/helpers/parseTags.test.ts new file mode 100644 index 000000000..6f86407cc --- /dev/null +++ b/packages/plugins/ci-visibility/src/helpers/parseTags.test.ts @@ -0,0 +1,84 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { BUILD_PLUGIN_SPAN_PREFIX } from '../constants'; + +import { parseTags } from './parseTags'; + +const testCases = [ + { + description: 'return an empty object when no tags are provided', + input: { spanTags: {}, tags: [] }, + expected: {}, + }, + { + description: 'add prefix to tags without it', + input: { spanTags: {}, tags: ['tag:value'] }, + expected: { [`${BUILD_PLUGIN_SPAN_PREFIX}.tag`]: 'value' }, + }, + { + description: 'not add prefix to tags that already have it', + input: { spanTags: {}, tags: [`${BUILD_PLUGIN_SPAN_PREFIX}.tag:value`] }, + expected: { [`${BUILD_PLUGIN_SPAN_PREFIX}.tag`]: 'value' }, + }, + { + description: 'merge values for the same tag', + input: { spanTags: {}, tags: ['tag:value1', 'tag:value2'] }, + expected: { [`${BUILD_PLUGIN_SPAN_PREFIX}.tag`]: 'value1,value2' }, + }, + { + description: 'deduplicate values for the same tag', + input: { spanTags: {}, tags: ['tag:value', 'tag:value'] }, + expected: { [`${BUILD_PLUGIN_SPAN_PREFIX}.tag`]: 'value' }, + }, + { + description: 'handle tags with multiple colons in value', + input: { spanTags: {}, tags: ['tag:value:with:colons'] }, + expected: { [`${BUILD_PLUGIN_SPAN_PREFIX}.tag`]: 'value:with:colons' }, + }, + { + description: 'preserve existing tags from spanTags', + input: { + spanTags: { existing: 'existingValue' }, + tags: ['tag:value'], + }, + expected: { + existing: 'existingValue', + [`${BUILD_PLUGIN_SPAN_PREFIX}.tag`]: 'value', + }, + }, + { + description: 'merge new tag values with existing tag values', + input: { + spanTags: { tag: 'existingValue' }, + tags: ['tag:newValue'], + }, + expected: { tag: 'existingValue,newValue' }, + }, + { + description: 'handle tags with spaces around the separator', + input: { spanTags: {}, tags: ['tag : value'] }, + expected: { [`${BUILD_PLUGIN_SPAN_PREFIX}.tag`]: 'value' }, + }, + { + description: 'skip empty values when converting sets to string', + input: { spanTags: { [`${BUILD_PLUGIN_SPAN_PREFIX}.empty`]: '' }, tags: [] }, + expected: {}, + }, + { + description: 'parse comma-separated values in existing tags', + input: { + spanTags: { [`${BUILD_PLUGIN_SPAN_PREFIX}.tag`]: 'value1,value2' }, + tags: ['tag:value3'], + }, + expected: { [`${BUILD_PLUGIN_SPAN_PREFIX}.tag`]: 'value1,value2,value3' }, + }, +]; + +describe('parseTags', () => { + test.each(testCases)('Should $description', ({ input, expected }) => { + const result = parseTags(input.spanTags, input.tags); + expect(result).toEqual(expected); + }); +}); diff --git a/packages/plugins/ci-visibility/src/helpers/parseTags.ts b/packages/plugins/ci-visibility/src/helpers/parseTags.ts new file mode 100644 index 000000000..4e294b793 --- /dev/null +++ b/packages/plugins/ci-visibility/src/helpers/parseTags.ts @@ -0,0 +1,54 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import type { LogTags } from '@dd/core/types'; + +import { BUILD_PLUGIN_SPAN_PREFIX } from '../constants'; +import type { SpanTag, SpanTags } from '../types'; + +export const parseTags = (spanTags: SpanTags, tags: LogTags): SpanTags => { + const parsedTags: SpanTags = {}; + const allTagsWithUniqueValues: Record> = {}; + + // Add the default tags to the temporary tags Sets. + for (const [key, value] of Object.entries(spanTags)) { + if (value) { + allTagsWithUniqueValues[key] = new Set(value.split(/ *, */g)); + } + } + + // Get all the tags and their (unique) values. + for (const tag of tags) { + const [key, ...rest] = tag.split(/ *: */g); + const verifiedKey = + key.startsWith(BUILD_PLUGIN_SPAN_PREFIX) || allTagsWithUniqueValues[key] + ? key + : `${BUILD_PLUGIN_SPAN_PREFIX}.${key}`; + const value = rest.join(':'); + + // If the value is already in the set, skip it. + if (allTagsWithUniqueValues[verifiedKey]?.has(value)) { + continue; + } + + // If the key doesn't exist, create a new set. + if (!allTagsWithUniqueValues[verifiedKey]) { + allTagsWithUniqueValues[verifiedKey] = new Set(); + } + + allTagsWithUniqueValues[verifiedKey].add(value); + } + + // Convert the sets into SpanTags. + for (const [key, value] of Object.entries(allTagsWithUniqueValues)) { + const stringValue = Array.from(value).join(','); + if (!stringValue) { + continue; + } + + parsedTags[key as SpanTag] = stringValue; + } + + return parsedTags; +}; diff --git a/packages/plugins/ci-visibility/src/helpers/sendSpans.ts b/packages/plugins/ci-visibility/src/helpers/sendSpans.ts new file mode 100644 index 000000000..46286e177 --- /dev/null +++ b/packages/plugins/ci-visibility/src/helpers/sendSpans.ts @@ -0,0 +1,92 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { doRequest, NB_RETRIES } from '@dd/core/helpers/request'; +import type { AuthOptions, Logger } from '@dd/core/types'; +import chalk from 'chalk'; +import PQueue from 'p-queue'; + +import { INTAKE_PATH, INTAKE_HOST } from '../constants'; +import type { CustomSpan, CustomSpanPayload, SpanTags } from '../types'; + +import { parseTags } from './parseTags'; + +const green = chalk.green.bold; +const yellow = chalk.yellow.bold; + +export const sendSpans = async ( + auth: AuthOptions, + payloads: CustomSpan[], + spanTags: SpanTags, + log: Logger, +) => { + const errors: string[] = []; + const warnings: string[] = []; + + if (!auth.apiKey) { + errors.push('No authentication token provided.'); + return { errors, warnings }; + } + + if (payloads.length === 0) { + warnings.push('No spans to submit.'); + return { errors, warnings }; + } + + // @ts-expect-error PQueue's default isn't typed. + const Queue = PQueue.default ? PQueue.default : PQueue; + const queue = new Queue({ concurrency: 20 }); + const addPromises = []; + + log.debug( + `Submitting ${green(payloads.length.toString())} span${payloads.length <= 1 ? '' : 's'}.`, + ); + for (const span of payloads) { + log.debug(`Queuing span ${green(span.name)}.`); + const spanToSubmit: CustomSpanPayload = { + ...span, + tags: parseTags(spanTags, span.tags), + }; + + addPromises.push( + queue.add(async () => { + try { + await doRequest({ + url: `https://${INTAKE_HOST}/${INTAKE_PATH}`, + auth: { apiKey: auth.apiKey }, + method: 'POST', + getData: () => { + const data = { + data: { + type: 'ci_app_custom_span', + attributes: spanToSubmit, + }, + }; + + return { + data: JSON.stringify(data), + headers: { + 'Content-Type': 'application/json', + }, + }; + }, + // On retry we store the error as a warning. + onRetry: (error: Error, attempt: number) => { + const warningMessage = `Failed to submit span ${yellow(span.name)}:\n ${error.message}\nRetrying ${attempt}/${NB_RETRIES}`; + // This will be logged at the end of the process. + warnings.push(warningMessage); + }, + }); + log.debug(`Submitted span ${green(span.name)}.`); + } catch (e: any) { + errors.push(`Failed to submit span ${yellow(span.name)}:\n ${e.message}`); + } + }), + ); + } + + await Promise.all(addPromises); + await queue.onIdle(); + return { warnings, errors }; +}; diff --git a/packages/plugins/ci-visibility/src/index.test.ts b/packages/plugins/ci-visibility/src/index.test.ts new file mode 100644 index 000000000..a2d1d64e4 --- /dev/null +++ b/packages/plugins/ci-visibility/src/index.test.ts @@ -0,0 +1,69 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import { getPlugins } from '@dd/ci-visibility-plugin'; +import { getContextMock } from '@dd/tests/_jest/helpers/mocks'; +import { runBundlers } from '@dd/tests/_jest/helpers/runBundlers'; +import nock from 'nock'; + +import { INTAKE_PATH, INTAKE_HOST } from './constants'; + +describe('Ci Visibility Plugin', () => { + describe('getPlugins', () => { + test('Should not initialize the plugin if disabled', async () => { + expect( + getPlugins({ + options: { ciVisibility: { disabled: true } }, + context: getContextMock(), + bundler: {}, + }), + ).toHaveLength(0); + expect( + getPlugins({ options: {}, context: getContextMock(), bundler: {} }), + ).toHaveLength(0); + }); + + test('Should initialize the plugin if enabled', async () => { + expect( + getPlugins({ + options: { ciVisibility: {} }, + context: getContextMock(), + bundler: {}, + }).length, + ).toBeGreaterThan(0); + }); + }); + + describe('With a supported CI provider', () => { + const replyMock = jest.fn(() => ({})); + beforeAll(async () => { + nock(`https://${INTAKE_HOST}`) + // Intercept logs submissions. + .post(`/${INTAKE_PATH}`) + .reply(200, replyMock) + .persist(); + }); + + afterAll(async () => { + nock.cleanAll(); + }); + + test('Should send spans to Datadog', async () => { + // Spoof a github action. + process.env.GITHUB_ACTIONS = 'true'; + + const { errors } = await runBundlers({ + auth: { + apiKey: 'test', + }, + ciVisibility: {}, + }); + + expect(errors).toHaveLength(0); + expect(replyMock).toHaveBeenCalled(); + + delete process.env.GITHUB_ACTIONS; + }); + }); +}); diff --git a/packages/plugins/ci-visibility/src/index.ts b/packages/plugins/ci-visibility/src/index.ts new file mode 100644 index 000000000..7ce40e369 --- /dev/null +++ b/packages/plugins/ci-visibility/src/index.ts @@ -0,0 +1,134 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import type { GetPlugins, Options } from '@dd/core/types'; + +import { + CONFIG_KEY, + GIT_BRANCH, + GIT_COMMIT_AUTHOR_DATE, + GIT_COMMIT_AUTHOR_EMAIL, + GIT_COMMIT_AUTHOR_NAME, + GIT_COMMIT_COMMITTER_DATE, + GIT_COMMIT_COMMITTER_EMAIL, + GIT_COMMIT_COMMITTER_NAME, + GIT_COMMIT_MESSAGE, + GIT_REPOSITORY_URL, + GIT_SHA, + BUILD_PLUGIN_ENV, + BUILD_PLUGIN_VERSION, + BUILD_PLUGIN_BUNDLER_NAME, + BUILD_PLUGIN_BUNDLER_VERSION, + PLUGIN_NAME, + SUPPORTED_PROVIDERS, +} from './constants'; +import { getBuildSpansPlugin } from './helpers/buildSpansPlugin'; +import { getCIProvider, getCISpanTags } from './helpers/ciSpanTags'; +import { getCustomSpans } from './helpers/customSpans'; +import { sendSpans } from './helpers/sendSpans'; +import type { CiVisibilityOptions, CiVisibilityOptionsWithDefaults, SpanTags } from './types'; + +export { CONFIG_KEY, PLUGIN_NAME }; + +export const helpers = { + // Add the helpers you'd like to expose here. +}; + +export type types = { + // Add the types you'd like to expose here. + CiVisibilityOptions: CiVisibilityOptions; +}; + +// Deal with validation and defaults here. +export const validateOptions = (options: Options): CiVisibilityOptionsWithDefaults => { + const validatedOptions: CiVisibilityOptionsWithDefaults = { + disabled: !options[CONFIG_KEY], + ...options[CONFIG_KEY], + }; + + return validatedOptions; +}; + +export const getPlugins: GetPlugins = ({ options, context }) => { + const log = context.getLogger(PLUGIN_NAME); + // Verify configuration. + const validatedOptions = validateOptions(options); + + // If the plugin is disabled, return an empty array. + if (validatedOptions.disabled) { + return []; + } + + // Will populate with more tags as we get them. + const spanTags: SpanTags = getCISpanTags(); + + // Add basic tags. + spanTags[BUILD_PLUGIN_VERSION] = context.version; + spanTags[BUILD_PLUGIN_ENV] = context.env; + + return [ + getBuildSpansPlugin(context, options), + { + name: PLUGIN_NAME, + enforce: 'post', + git: (gitData) => { + // Add tags from git data. + spanTags[GIT_REPOSITORY_URL] = gitData.remote; + spanTags[GIT_BRANCH] = gitData.branch; + spanTags[GIT_SHA] = gitData.commit.hash; + spanTags[GIT_COMMIT_MESSAGE] = gitData.commit.message; + spanTags[GIT_COMMIT_AUTHOR_NAME] = gitData.commit.author.name; + spanTags[GIT_COMMIT_AUTHOR_EMAIL] = gitData.commit.author.email; + spanTags[GIT_COMMIT_AUTHOR_DATE] = gitData.commit.author.date; + spanTags[GIT_COMMIT_COMMITTER_NAME] = gitData.commit.committer.name; + spanTags[GIT_COMMIT_COMMITTER_EMAIL] = gitData.commit.committer.email; + spanTags[GIT_COMMIT_COMMITTER_DATE] = gitData.commit.committer.date; + }, + bundlerReport: (bundlerReport) => { + // Add custom tags from the bundler report. + spanTags[BUILD_PLUGIN_BUNDLER_NAME] = bundlerReport.name; + spanTags[BUILD_PLUGIN_BUNDLER_VERSION] = bundlerReport.version; + }, + // NOTE: This is a bit off for esbuild because of its "trueEnd" implementation. + async asyncTrueEnd() { + if (!options.auth?.apiKey) { + log.info('No auth options, skipping.'); + return; + } + + const ci_provider = getCIProvider(); + // Only run if we're on a supported provider. + if (!SUPPORTED_PROVIDERS.includes(ci_provider)) { + log.info( + `"${ci_provider}" is not a supported provider, skipping spans submission`, + ); + return; + } + + const spansToSubmit = getCustomSpans(ci_provider, context); + + try { + const { errors, warnings } = await sendSpans( + options.auth, + spansToSubmit, + spanTags, + log, + ); + + if (warnings.length > 0) { + log.warn( + `Warnings while submitting spans:\n - ${warnings.join('\n - ')}`, + ); + } + + if (errors.length) { + log.warn(`Error submitting some spans:\n - ${errors.join('\n - ')}`); + } + } catch (error) { + log.warn(`Error submitting spans: ${error}`); + } + }, + }, + ]; +}; diff --git a/packages/plugins/ci-visibility/src/types.ts b/packages/plugins/ci-visibility/src/types.ts new file mode 100644 index 000000000..acb10b606 --- /dev/null +++ b/packages/plugins/ci-visibility/src/types.ts @@ -0,0 +1,100 @@ +// Unless explicitly stated otherwise all files in this repository are licensed under the MIT License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2019-Present Datadog, Inc. + +import type { Assign, LogTags } from '@dd/core/types'; + +import type { + GIT_COMMIT_AUTHOR_EMAIL, + GIT_COMMIT_AUTHOR_NAME, + GIT_COMMIT_AUTHOR_DATE, + GIT_COMMIT_MESSAGE, + GIT_COMMIT_COMMITTER_DATE, + GIT_COMMIT_COMMITTER_EMAIL, + GIT_COMMIT_COMMITTER_NAME, + CI_ENV_VARS, + CI_NODE_NAME, + CI_NODE_LABELS, + GIT_BASE_REF, + GIT_HEAD_SHA, + GIT_PULL_REQUEST_BASE_BRANCH, + GIT_PULL_REQUEST_BASE_BRANCH_SHA, + SUPPORTED_PROVIDERS, + GIT_TAG, + CI_JOB_NAME, + CI_JOB_URL, + CI_PIPELINE_ID, + CI_PIPELINE_NAME, + CI_PIPELINE_NUMBER, + CI_PIPELINE_URL, + CI_PROVIDER_NAME, + CI_STAGE_NAME, + CI_WORKSPACE_PATH, + GIT_BRANCH, + GIT_REPOSITORY_URL, + GIT_SHA, + BUILD_PLUGIN_VERSION, + BUILD_PLUGIN_ENV, + BUILD_PLUGIN_BUNDLER_NAME, + BUILD_PLUGIN_BUNDLER_VERSION, + BUILD_PLUGIN_SPAN_PREFIX, +} from './constants'; + +export type CiVisibilityOptions = { + disabled?: boolean; +}; + +export interface CustomSpan { + ci_provider: string; + span_id: string; + command: string; + name: string; + start_time: string; + end_time: string; + error_message: string; + exit_code: number; + tags: LogTags; + measures: Partial>; +} + +export type CustomSpanPayload = Assign; + +export type Provider = (typeof SUPPORTED_PROVIDERS)[number]; + +export type SpanTag = + | typeof CI_ENV_VARS + | typeof CI_JOB_NAME + | typeof CI_JOB_URL + | typeof CI_NODE_LABELS + | typeof CI_NODE_NAME + | typeof CI_PIPELINE_ID + | typeof CI_PIPELINE_NAME + | typeof CI_PIPELINE_NUMBER + | typeof CI_PIPELINE_URL + | typeof CI_PROVIDER_NAME + | typeof CI_STAGE_NAME + | typeof CI_WORKSPACE_PATH + | typeof GIT_BASE_REF + | typeof GIT_BRANCH + | typeof GIT_COMMIT_AUTHOR_DATE + | typeof GIT_COMMIT_AUTHOR_EMAIL + | typeof GIT_COMMIT_AUTHOR_NAME + | typeof GIT_COMMIT_COMMITTER_DATE + | typeof GIT_COMMIT_COMMITTER_EMAIL + | typeof GIT_COMMIT_COMMITTER_NAME + | typeof GIT_COMMIT_MESSAGE + | typeof GIT_HEAD_SHA + | typeof GIT_PULL_REQUEST_BASE_BRANCH + | typeof GIT_PULL_REQUEST_BASE_BRANCH_SHA + | typeof GIT_REPOSITORY_URL + | typeof GIT_SHA + | typeof GIT_TAG + | typeof BUILD_PLUGIN_VERSION + | typeof BUILD_PLUGIN_ENV + | typeof BUILD_PLUGIN_BUNDLER_NAME + | typeof BUILD_PLUGIN_BUNDLER_VERSION + | `${typeof BUILD_PLUGIN_SPAN_PREFIX}.${string}`; + +export type SpanTags = Partial>; + +export type CiVisibilityOptionsWithDefaults = Required; diff --git a/packages/plugins/ci-visibility/tsconfig.json b/packages/plugins/ci-visibility/tsconfig.json new file mode 100644 index 000000000..6c1d3065e --- /dev/null +++ b/packages/plugins/ci-visibility/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "baseUrl": "./", + "rootDir": "./", + "outDir": "./dist" + }, + "include": ["**/*"], + "exclude": ["dist", "node_modules"] +} \ No newline at end of file diff --git a/packages/plugins/error-tracking/src/sourcemaps/payload.ts b/packages/plugins/error-tracking/src/sourcemaps/payload.ts index 08b201389..196683ce0 100644 --- a/packages/plugins/error-tracking/src/sourcemaps/payload.ts +++ b/packages/plugins/error-tracking/src/sourcemaps/payload.ts @@ -178,7 +178,7 @@ export const getPayload = async ( ); }, ), - hash: git.hash, + hash: git.commit.hash, repository_url: git.remote, }, ], diff --git a/packages/plugins/error-tracking/src/sourcemaps/sender.ts b/packages/plugins/error-tracking/src/sourcemaps/sender.ts index 8c5422142..3d327c683 100644 --- a/packages/plugins/error-tracking/src/sourcemaps/sender.ts +++ b/packages/plugins/error-tracking/src/sourcemaps/sender.ts @@ -164,7 +164,7 @@ export const sendSourcemaps = async ( const metadata: Metadata = { git_repository_url: context.git?.remote, - git_commit_sha: context.git?.hash, + git_commit_sha: context.git?.commit?.hash, plugin_version: context.version, project_path: context.bundler.outDir, service: options.service, diff --git a/packages/plugins/git/src/helpers.test.ts b/packages/plugins/git/src/helpers.test.ts index c79df297e..096938e01 100644 --- a/packages/plugins/git/src/helpers.test.ts +++ b/packages/plugins/git/src/helpers.test.ts @@ -8,7 +8,7 @@ import { vol } from 'memfs'; jest.mock('fs', () => require('memfs').fs); describe('Git Plugin helpers', () => { - describe('GetRepositoryData', () => { + describe('getRepositoryData', () => { beforeEach(() => { // Emulate some fixtures. vol.fromJSON({ @@ -31,12 +31,22 @@ describe('Git Plugin helpers', () => { getRemotes: (arg: boolean) => [ { refs: { push: 'git@github.com:user/repository.git' } }, ], + branch: () => ({ current: 'main' }), + show: ([, format]: [string, string]) => { + if (format === '--format=%s') { + return 'test message '; + } + if (format === '--format=%an,%ae,%aI,%cn,%ce,%cI') { + return 'John Doe ,john.doe@example.com,2021-01-01 ,Jane Smith,jane.smith@example.com,2021-01-02'; + } + return ''; + }, raw: (arg: string) => 'src/core/plugins/git/helpers.test.ts', revparse: (arg: string) => '25da22df90210a40b919debe3f7ebfb0c1811898', }); test('Should return the relevant data from git', async () => { - const data = await getRepositoryData(createMockSimpleGit() as any, ''); + const data = await getRepositoryData(createMockSimpleGit() as any); if (!data) { fail('data should not be undefined'); } @@ -46,24 +56,15 @@ describe('Git Plugin helpers', () => { () => undefined, ); expect(data.remote).toBe('git@github.com:user/repository.git'); - expect(data.hash).toBe('25da22df90210a40b919debe3f7ebfb0c1811898'); - expect(files).toStrictEqual(['src/core/plugins/git/helpers.test.ts']); - }); - - test('Should return the relevant data from git with a different remote', async () => { - const data = await getRepositoryData( - createMockSimpleGit() as any, - 'git@github.com:user/other.git', - ); - if (!data) { - fail('data should not be undefined'); - } - const files = data.trackedFilesMatcher.matchSourcemap( - 'fixtures/common.min.js.map', - () => undefined, - ); - expect(data.remote).toBe('git@github.com:user/other.git'); - expect(data.hash).toBe('25da22df90210a40b919debe3f7ebfb0c1811898'); + expect(data.commit.hash).toBe('25da22df90210a40b919debe3f7ebfb0c1811898'); + expect(data.commit.message).toBe('test message'); + expect(data.commit.author.name).toBe('John Doe'); + expect(data.commit.author.email).toBe('john.doe@example.com'); + expect(data.commit.author.date).toBe('2021-01-01'); + expect(data.commit.committer.name).toBe('Jane Smith'); + expect(data.commit.committer.email).toBe('jane.smith@example.com'); + expect(data.commit.committer.date).toBe('2021-01-02'); + expect(data.branch).toBe('main'); expect(files).toStrictEqual(['src/core/plugins/git/helpers.test.ts']); }); }); diff --git a/packages/plugins/git/src/helpers.ts b/packages/plugins/git/src/helpers.ts index 601d7af7d..2d7a9cedb 100644 --- a/packages/plugins/git/src/helpers.ts +++ b/packages/plugins/git/src/helpers.ts @@ -2,10 +2,10 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. +import { filterSensitiveInfoFromRepositoryUrl } from '@dd/core/helpers/strings'; import type { RepositoryData } from '@dd/core/types'; import type { SimpleGit, BranchSummary } from 'simple-git'; import { simpleGit } from 'simple-git'; -import { URL } from 'url'; import { TrackedFilesMatcher } from './trackedFilesMatcher'; @@ -40,12 +40,12 @@ export const gitRemote = async (git: SimpleGit): Promise => { for (const remote of remotes) { if (remote.name === defaultRemote) { - return stripCredentials(remote.refs.push); + return filterSensitiveInfoFromRepositoryUrl(remote.refs.push); } } // Falling back to picking the first remote in the list if the default remote is not found. - return stripCredentials(remotes[0].refs.push); + return filterSensitiveInfoFromRepositoryUrl(remotes[0].refs.push); }; export const getDefaultRemoteName = async (git: SimpleGit): Promise => { @@ -56,19 +56,6 @@ export const getDefaultRemoteName = async (git: SimpleGit): Promise => { } }; -// StripCredentials removes credentials from a remote HTTP url. -export const stripCredentials = (remote: string) => { - try { - const url = new URL(remote); - url.username = ''; - url.password = ''; - - return url.toString(); - } catch { - return remote; - } -}; - // Returns the hash of the current repository. export const gitHash = async (git: SimpleGit): Promise => git.revparse('HEAD'); @@ -93,31 +80,50 @@ export const gitRepositoryURL = async (git: SimpleGit): Promise => // Returns the current hash and remote as well as a TrackedFilesMatcher. // // To obtain the list of tracked files paths tied to a specific sourcemap, invoke the 'matchSourcemap' method. -export const getRepositoryData = async ( - git: SimpleGit, - repositoryURL?: string | undefined, -): Promise => { - // Invoke git commands to retrieve the remote, hash and tracked files. +export const getRepositoryData = async (git: SimpleGit): Promise => { + // Invoke git commands to retrieve some informations and tracked files. // We're using Promise.all instead of Promise.allSettled since we want to fail early if // any of the promises fails. - let remote: string; - let hash: string; - let trackedFiles: string[]; - - if (repositoryURL) { - [hash, trackedFiles] = await Promise.all([gitHash(git), gitTrackedFiles(git)]); - remote = repositoryURL; - } else { - [remote, hash, trackedFiles] = await Promise.all([ - gitRemote(git), - gitHash(git), - gitTrackedFiles(git), - ]); - } - const data = { - hash, - remote, + const proms: [ + ReturnType, + ReturnType, + ReturnType, + ReturnType, + ReturnType, + ReturnType, + ] = [ + gitHash(git), + gitBranch(git), + gitMessage(git), + gitAuthorAndCommitter(git), + gitTrackedFiles(git), + gitRemote(git), + ]; + + const [hash, branch, message, authorAndCommitter, trackedFiles, remote] = + await Promise.all(proms); + + const [authorName, authorEmail, authorDate, committerName, committerEmail, committerDate] = + authorAndCommitter.split(',').map((item) => item.trim()); + + const data: RepositoryData = { + commit: { + author: { + name: authorName, + email: authorEmail, + date: authorDate, + }, + committer: { + name: committerName, + email: committerEmail, + date: committerDate, + }, + message: message.trim(), + hash, + }, + branch: branch.current, + remote: remote.trim(), trackedFilesMatcher: new TrackedFilesMatcher(trackedFiles), }; diff --git a/packages/plugins/git/src/index.test.ts b/packages/plugins/git/src/index.test.ts index ccf40d7ae..57988d4cb 100644 --- a/packages/plugins/git/src/index.test.ts +++ b/packages/plugins/git/src/index.test.ts @@ -5,11 +5,11 @@ import type { Options, RepositoryData } from '@dd/core/types'; import { uploadSourcemaps } from '@dd/error-tracking-plugin/sourcemaps/index'; import { getRepositoryData } from '@dd/internal-git-plugin/helpers'; -import { TrackedFilesMatcher } from '@dd/internal-git-plugin/trackedFilesMatcher'; import { API_PATH, FAKE_URL, defaultPluginOptions, + getRepositoryDataMock, getSourcemapsConfiguration, } from '@dd/tests/_jest/helpers/mocks'; import { BUNDLERS, runBundlers } from '@dd/tests/_jest/helpers/runBundlers'; @@ -50,11 +50,7 @@ describe('Git Plugin', () => { }); describe('Enabled', () => { - const mockGitData: RepositoryData = { - hash: 'hash', - remote: 'remote', - trackedFilesMatcher: new TrackedFilesMatcher([]), - }; + const mockGitData = getRepositoryDataMock(); // Intercept contexts to verify it at the moment they're used. const gitReports: Record = {}; diff --git a/packages/plugins/git/src/index.ts b/packages/plugins/git/src/index.ts index 44b2fd9ae..f580ecfbe 100644 --- a/packages/plugins/git/src/index.ts +++ b/packages/plugins/git/src/index.ts @@ -2,6 +2,7 @@ // This product includes software developed at Datadog (https://www.datadoghq.com/). // Copyright 2019-Present Datadog, Inc. +import { shouldGetGitInfo } from '@dd/core/helpers/plugins'; import type { GetInternalPlugins, GetPluginsArg } from '@dd/core/types'; import { getRepositoryData, newSimpleGit } from './helpers'; @@ -16,14 +17,7 @@ export const getGitPlugins: GetInternalPlugins = (arg: GetPluginsArg) => { name: PLUGIN_NAME, enforce: 'pre', async buildStart() { - // Verify that we should get the git information based on the options. - // Only get git information if sourcemaps are enabled and git is not disabled. - const shouldGetGitInfo = - options.errorTracking?.sourcemaps && - options.errorTracking?.sourcemaps.disableGit !== true && - options.disableGit !== true; - - if (!shouldGetGitInfo) { + if (!shouldGetGitInfo(options)) { return; } diff --git a/packages/published/esbuild-plugin/src/index.ts b/packages/published/esbuild-plugin/src/index.ts index 6ff523630..9939d404c 100644 --- a/packages/published/esbuild-plugin/src/index.ts +++ b/packages/published/esbuild-plugin/src/index.ts @@ -9,6 +9,7 @@ import type { Options } from '@dd/core/types'; import type { // #types-export-injection-marker + CiVisibilityTypes, ErrorTrackingTypes, RumTypes, TelemetryTypes, @@ -22,6 +23,7 @@ import pkg from '../package.json'; export type EsbuildPluginOptions = Options; export type { // #types-export-injection-marker + CiVisibilityTypes, ErrorTrackingTypes, RumTypes, TelemetryTypes, diff --git a/packages/published/rollup-plugin/src/index.ts b/packages/published/rollup-plugin/src/index.ts index 59c46cba7..ae889b6ee 100644 --- a/packages/published/rollup-plugin/src/index.ts +++ b/packages/published/rollup-plugin/src/index.ts @@ -9,6 +9,7 @@ import type { Options } from '@dd/core/types'; import type { // #types-export-injection-marker + CiVisibilityTypes, ErrorTrackingTypes, RumTypes, TelemetryTypes, @@ -22,6 +23,7 @@ import pkg from '../package.json'; export type RollupPluginOptions = Options; export type { // #types-export-injection-marker + CiVisibilityTypes, ErrorTrackingTypes, RumTypes, TelemetryTypes, diff --git a/packages/published/rspack-plugin/src/index.ts b/packages/published/rspack-plugin/src/index.ts index aec0805e7..2f3deb5de 100644 --- a/packages/published/rspack-plugin/src/index.ts +++ b/packages/published/rspack-plugin/src/index.ts @@ -9,6 +9,7 @@ import type { Options } from '@dd/core/types'; import type { // #types-export-injection-marker + CiVisibilityTypes, ErrorTrackingTypes, RumTypes, TelemetryTypes, @@ -22,6 +23,7 @@ import pkg from '../package.json'; export type RspackPluginOptions = Options; export type { // #types-export-injection-marker + CiVisibilityTypes, ErrorTrackingTypes, RumTypes, TelemetryTypes, diff --git a/packages/published/vite-plugin/src/index.ts b/packages/published/vite-plugin/src/index.ts index 7ea5cfbf6..2b270e789 100644 --- a/packages/published/vite-plugin/src/index.ts +++ b/packages/published/vite-plugin/src/index.ts @@ -9,6 +9,7 @@ import type { Options } from '@dd/core/types'; import type { // #types-export-injection-marker + CiVisibilityTypes, ErrorTrackingTypes, RumTypes, TelemetryTypes, @@ -22,6 +23,7 @@ import pkg from '../package.json'; export type VitePluginOptions = Options; export type { // #types-export-injection-marker + CiVisibilityTypes, ErrorTrackingTypes, RumTypes, TelemetryTypes, diff --git a/packages/published/webpack-plugin/src/index.ts b/packages/published/webpack-plugin/src/index.ts index 24a29bbfd..04c55cb94 100644 --- a/packages/published/webpack-plugin/src/index.ts +++ b/packages/published/webpack-plugin/src/index.ts @@ -9,6 +9,7 @@ import type { Options } from '@dd/core/types'; import type { // #types-export-injection-marker + CiVisibilityTypes, ErrorTrackingTypes, RumTypes, TelemetryTypes, @@ -22,6 +23,7 @@ import pkg from '../package.json'; export type WebpackPluginOptions = Options; export type { // #types-export-injection-marker + CiVisibilityTypes, ErrorTrackingTypes, RumTypes, TelemetryTypes, diff --git a/packages/tests/src/_jest/helpers/mocks.ts b/packages/tests/src/_jest/helpers/mocks.ts index fabfb8ea8..daae282cd 100644 --- a/packages/tests/src/_jest/helpers/mocks.ts +++ b/packages/tests/src/_jest/helpers/mocks.ts @@ -56,6 +56,7 @@ export const defaultPluginOptions: GetPluginsOptions = { auth: defaultAuth, disableGit: false, logLevel: 'debug', + metadata: {}, }; export const getMockTimer = (overrides: Partial = {}): TimeLogger => { @@ -142,6 +143,7 @@ export const getEsbuildMock = (overrides: Partial = {}): PluginBuil export const getMockBuildReport = (overrides: Partial = {}): BuildReport => ({ errors: [], warnings: [], + metadata: {}, logs: [], timings: [], ...overrides, @@ -487,7 +489,21 @@ export const getMetadataMock = (options: Partial = {}): Metadata => { export const getRepositoryDataMock = (options: Partial = {}): RepositoryData => { return { - hash: 'hash', + commit: { + hash: 'hash', + message: 'message', + author: { + name: 'author', + email: 'author@example.com', + date: '2021-01-01', + }, + committer: { + name: 'committer', + email: 'committer@example.com', + date: '2021-01-01', + }, + }, + branch: 'branch', remote: 'remote', trackedFilesMatcher: new TrackedFilesMatcher(['/path/to/minified.min.js']), ...options, diff --git a/yarn.lock b/yarn.lock index 9a8ebf44b..6827a31d7 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1650,6 +1650,18 @@ __metadata: languageName: unknown linkType: soft +"@dd/ci-visibility-plugin@workspace:*, @dd/ci-visibility-plugin@workspace:packages/plugins/ci-visibility": + version: 0.0.0-use.local + resolution: "@dd/ci-visibility-plugin@workspace:packages/plugins/ci-visibility" + dependencies: + "@dd/core": "workspace:*" + "@dd/internal-build-report-plugin": "workspace:*" + chalk: "npm:2.3.1" + p-queue: "npm:6.6.2" + typescript: "npm:5.4.3" + languageName: unknown + linkType: soft + "@dd/core@workspace:*, @dd/core@workspace:packages/core": version: 0.0.0-use.local resolution: "@dd/core@workspace:packages/core" @@ -1681,6 +1693,7 @@ __metadata: version: 0.0.0-use.local resolution: "@dd/factory@workspace:packages/factory" dependencies: + "@dd/ci-visibility-plugin": "workspace:*" "@dd/core": "workspace:*" "@dd/error-tracking-plugin": "workspace:*" "@dd/internal-analytics-plugin": "workspace:*"