feat: decouple log configuration from apifylog

docs/examples/file_download_stream.ts

@@ -1,9 +1,9 @@
 import { pipeline, Transform } from 'stream';
 
-import { FileDownload, type Log } from 'crawlee';
+import { FileDownload, type CrawleeLogger } from 'crawlee';
 
 // A sample Transform stream logging the download progress.
-function createProgressTracker({ url, log, totalBytes }: { url: URL; log: Log; totalBytes: number }) {
+function createProgressTracker({ url, log, totalBytes }: { url: URL; log: CrawleeLogger; totalBytes: number }) {
     let downloadedBytes = 0;
 
     return new Transform({

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -6,6 +6,7 @@ import type {
     AddRequestsBatchedResult,
     AutoscaledPoolOptions,
     Configuration,
+    CrawleeLogger,
     CrawlingContext,
     DatasetExportOptions,
     EnqueueLinksOptions,
@@ -40,6 +41,7 @@ import {
     EnqueueStrategy,
     EventType,
     KeyValueStore,
+    LogLevel,
     mergeCookies,
     NonRetryableError,
     purgeDefaultStorages,
@@ -77,8 +79,6 @@ import { getDomain } from 'tldts';
 import type { ReadonlyDeep, SetRequired } from 'type-fest';
 
 import { LruCache } from '@apify/datastructures';
-import type { Log } from '@apify/log';
-import defaultLog, { LogLevel } from '@apify/log';
 import { addTimeoutToPromise, TimeoutError, tryCancel } from '@apify/timeout';
 import { cryptoRandomObjectId } from '@apify/utilities';
 
@@ -371,7 +371,7 @@ export interface BasicCrawlerOptions<
     onSkippedRequest?: SkippedRequestCallback;
 
     /** @internal */
-    log?: Log;
+    log?: CrawleeLogger;
 
     /**
      * Enables experimental features of Crawlee, which can alter the behavior of the crawler.
@@ -415,6 +415,12 @@ export interface BasicCrawlerOptions<
      */
     eventManager?: EventManager;
 
+    /**
+     * Custom logger to use for this crawler.
+     * If provided, the crawler will use its own ServiceLocator instance instead of the global one.
+     */
+    logger?: CrawleeLogger;
+
     /**
      * A unique identifier for the crawler instance. This ID is used to isolate the state returned by
      * {@apilink BasicCrawler.useState|`crawler.useState()`} from other crawler instances.
@@ -586,7 +592,7 @@ export class BasicCrawler<
     running = false;
     hasFinishedBefore = false;
 
-    readonly log: Log;
+    readonly log: CrawleeLogger;
     protected requestHandler!: RequestHandler<ExtendedContext>;
     protected errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
     protected failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
@@ -651,6 +657,7 @@ export class BasicCrawler<
         configuration: ow.optional.object,
         storageClient: ow.optional.object,
         eventManager: ow.optional.object,
+        logger: ow.optional.object,
 
         // AutoscaledPool shorthands
         minConcurrency: ow.optional.number,
@@ -695,6 +702,7 @@ export class BasicCrawler<
             configuration,
             storageClient,
             eventManager,
+            logger,
 
             // AutoscaledPool shorthands
             minConcurrency,
@@ -714,7 +722,7 @@ export class BasicCrawler<
             httpClient,
 
             // internal
-            log = defaultLog.child({ prefix: this.constructor.name }),
+            log: logOverride,
             experiments = {},
 
             id,
@@ -730,15 +738,18 @@ export class BasicCrawler<
         if (
             storageClient ||
             eventManager ||
+            logger ||
             (configuration !== undefined && configuration !== serviceLocator.getConfiguration())
         ) {
-            const scopedServiceLocator = new ServiceLocator(configuration, eventManager, storageClient);
+            const scopedServiceLocator = new ServiceLocator(configuration, eventManager, storageClient, logger);
             serviceLocatorScope = bindMethodsToServiceLocator(scopedServiceLocator, this);
         }
 
         try {
             serviceLocatorScope.enterScope();
 
+            const log = logOverride ?? serviceLocator.getLogger().child({ prefix: this.constructor.name });
+
             // Store whether the user explicitly provided an ID
             this.hasExplicitId = id !== undefined;
             // Store the user-provided ID, or generate a unique one for tracking purposes (not for state key)
@@ -833,7 +844,7 @@ export class BasicCrawler<
             this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
             this.maxSessionRotations = maxSessionRotations;
             this.stats = new Statistics({
-                logMessage: `${log.getOptions().prefix} request statistics:`,
+                logMessage: `${log.getOptions().prefix ?? this.constructor.name} request statistics:`,
                 log,
                 ...(this.hasExplicitId ? { id: this.crawlerId } : {}),
                 ...statisticsOptions,
@@ -944,7 +955,7 @@ export class BasicCrawler<
     async setStatusMessage(message: string, options: SetStatusMessageOptions = {}) {
         const data =
             options.isStatusMessageTerminal != null ? { terminal: options.isStatusMessageTerminal } : undefined;
-        this.log.internal(LogLevel[(options.level as 'DEBUG') ?? 'DEBUG'], message, data);
+        this.log.logWithLevel(LogLevel[(options.level as 'DEBUG') ?? 'DEBUG'], message, data);
 
         const client = serviceLocator.getStorageClient();
 
@@ -1099,7 +1110,7 @@ export class BasicCrawler<
                 retryHistogram: this.stats.requestRetryHistogram,
                 ...finalStats,
             };
-            this.log.info('Final request statistics:', stats);
+            this.log.info('Final request statistics:', stats as unknown as Record<string, unknown>);
 
             if (this.stats.errorTracker.total !== 0) {
                 const prettify = ([count, info]: [number, string[]]) =>
@@ -1193,12 +1204,14 @@ export class BasicCrawler<
         BasicCrawler.useStateCrawlerIds.add(this.crawlerId);
 
         if (BasicCrawler.useStateCrawlerIds.size > 1) {
-            defaultLog.warningOnce(
-                'Multiple crawler instances are calling useState() without an explicit `id` option. \n' +
-                    'This means they will share the same state object, which is likely unintended. \n' +
-                    'To fix this, provide a unique `id` option to each crawler instance. \n' +
-                    'Example: new BasicCrawler({ id: "my-crawler-1", ... })',
-            );
+            serviceLocator
+                .getLogger()
+                .warningOnce(
+                    'Multiple crawler instances are calling useState() without an explicit `id` option. \n' +
+                        'This means they will share the same state object, which is likely unintended. \n' +
+                        'To fix this, provide a unique `id` option to each crawler instance. \n' +
+                        'Example: new BasicCrawler({ id: "my-crawler-1", ... })',
+                );
         }
 
         return kvs.getAutoSavedValue<State>(BasicCrawler.CRAWLEE_STATE_KEY, defaultValue);

packages/browser-pool/src/abstract-classes/browser-controller.ts

@@ -1,3 +1,4 @@
+import { type CrawleeLogger, serviceLocator } from '@crawlee/core';
 import type { Cookie, Dictionary } from '@crawlee/types';
 import { nanoid } from 'nanoid';
 import { TypedEmitter } from 'tiny-typed-emitter';
@@ -6,7 +7,6 @@ import { tryCancel } from '@apify/timeout';
 
 import { BROWSER_CONTROLLER_EVENTS } from '../events.js';
 import type { LaunchContext } from '../launch-context.js';
-import { log } from '../logger.js';
 import type { UnwrapPromise } from '../utils.js';
 import type { BrowserPlugin, CommonBrowser, CommonLibrary } from './browser-plugin.js';
 
@@ -40,6 +40,7 @@ export abstract class BrowserController<
     NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,
 > extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> {
     id = nanoid();
+    protected log!: CrawleeLogger;
 
     /**
      * The `BrowserPlugin` instance used to launch the browser.
@@ -90,6 +91,7 @@ export abstract class BrowserController<
 
     constructor(browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>) {
         super();
+        this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
         this.browserPlugin = browserPlugin;
     }
 
@@ -136,14 +138,14 @@ export abstract class BrowserController<
             // TODO: shouldn't this go in a finally instead?
             this.isActive = false;
         } catch (error) {
-            log.debug(`Could not close browser.\nCause: ${(error as Error).message}`, { id: this.id });
+            this.log.debug(`Could not close browser.\nCause: ${(error as Error).message}`, { id: this.id });
         }
 
         this.emit(BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED, this);
 
         setTimeout(() => {
             this._kill().catch((err) => {
-                log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
+                this.log.debug(`Could not kill browser.\nCause: ${err.message}`, { id: this.id });
             });
         }, PROCESS_KILL_TIMEOUT_MILLIS);
     }

packages/browser-pool/src/abstract-classes/browser-plugin.ts

@@ -1,4 +1,4 @@
-import { CriticalError } from '@crawlee/core';
+import { type CrawleeLogger, CriticalError, serviceLocator } from '@crawlee/core';
 import type { Dictionary } from '@crawlee/types';
 import merge from 'lodash.merge';
 
@@ -105,6 +105,7 @@ export abstract class BrowserPlugin<
     NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,
 > {
     name = this.constructor.name;
+    protected log!: CrawleeLogger;
     library: Library;
     launchOptions: LibraryOptions;
     proxyUrl?: string;
@@ -121,6 +122,7 @@ export abstract class BrowserPlugin<
             browserPerProxy = false,
         } = options;
 
+        this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
         this.library = library;
         this.launchOptions = launchOptions;
         this.proxyUrl = proxyUrl && new URL(proxyUrl).href.slice(0, -1);

packages/browser-pool/src/browser-pool.ts

@@ -1,4 +1,4 @@
-import type { TieredProxy } from '@crawlee/core';
+import { type CrawleeLogger, serviceLocator, type TieredProxy } from '@crawlee/core';
 import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator';
 import { FingerprintGenerator } from 'fingerprint-generator';
 import { FingerprintInjector } from 'fingerprint-injector';
@@ -20,7 +20,6 @@ import {
 } from './fingerprinting/hooks.js';
 import type { FingerprintGeneratorOptions } from './fingerprinting/types.js';
 import type { LaunchContext } from './launch-context.js';
-import { log } from './logger.js';
 import type { InferBrowserPluginArray, UnwrapPromise } from './utils.js';
 
 const PAGE_CLOSE_KILL_TIMEOUT_MILLIS = 1000;
@@ -334,9 +333,11 @@ export class BrowserPool<
     private browserRetireInterval?: NodeJS.Timeout;
 
     private limiter = pLimit(1);
+    private log!: CrawleeLogger;
 
     constructor(options: Options & BrowserPoolHooks<BrowserControllerReturn, LaunchContextReturn, PageReturn>) {
         super();
+        this.log = serviceLocator.getLogger().child({ prefix: 'BrowserPool' });
 
         this.browserKillerInterval!.unref();
 
@@ -708,7 +709,7 @@ export class BrowserPool<
             throw err;
         }
 
-        log.debug('Launched new browser.', { id: browserController.id });
+        this.log.debug('Launched new browser.', { id: browserController.id });
         browserController.proxyTier = proxyTier;
         browserController.proxyUrl = proxyUrl;
 
@@ -719,7 +720,7 @@ export class BrowserPool<
         } catch (err) {
             this.startingBrowserControllers.delete(browserController);
             browserController.close().catch((closeErr) => {
-                log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
+                this.log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
                     id: browserController.id,
                 });
             });
@@ -774,15 +775,15 @@ export class BrowserPool<
 
             if (isBrowserIdle || isBrowserEmpty) {
                 const { id } = controller;
-                log.debug('Closing retired browser.', { id });
+                this.log.debug('Closing retired browser.', { id });
                 await controller.close();
                 this.retiredBrowserControllers.delete(controller);
                 closedBrowserIds.push(id);
             }
         }
 
         if (closedBrowserIds.length) {
-            log.debug('Closed retired browsers.', {
+            this.log.debug('Closed retired browsers.', {
                 count: closedBrowserIds.length,
                 closedBrowserIds,
             });
@@ -798,7 +799,7 @@ export class BrowserPool<
             await this._executeHooks(this.prePageCloseHooks, page, browserController);
 
             await originalPageClose.apply(page, args).catch((err: Error) => {
-                log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
+                this.log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
             });
 
             await this._executeHooks(this.postPageCloseHooks, pageId, browserController);
@@ -821,7 +822,7 @@ export class BrowserPool<
             // Run this with a delay, otherwise page.close()
             // might fail with "Protocol error (Target.closeTarget): Target closed."
             setTimeout(() => {
-                log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
+                this.log.debug('Closing retired browser because it has no active pages', { id: browserController.id });
                 void browserController.close().finally(() => {
                     this.retiredBrowserControllers.delete(browserController);
                 });

packages/browser-pool/src/playwright/playwright-plugin.ts

@@ -6,7 +6,6 @@ import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
 import { anonymizeProxySugar } from '../anonymize-proxy.js';
 import type { createProxyServerForContainers } from '../container-proxy-server.js';
 import type { LaunchContext } from '../launch-context.js';
-import { log } from '../logger.js';
 import { getLocalProxyAddress } from '../proxy-server.js';
 import type { SafeParameters } from '../utils.js';
 import { PlaywrightBrowser as PlaywrightBrowserWithPersistentContext } from './playwright-browser.js';
@@ -82,7 +81,7 @@ export class PlaywrightPlugin extends BrowserPlugin<
                     this._browserVersion = inactiveBrowser.version();
 
                     inactiveBrowser.close().catch((error) => {
-                        log.exception(error, 'Failed to close browser.');
+                        this.log.exception(error, 'Failed to close browser.');
                     });
                 }
 

packages/browser-pool/src/puppeteer/puppeteer-controller.ts

@@ -6,7 +6,6 @@ import { tryCancel } from '@apify/timeout';
 
 import { BrowserController } from '../abstract-classes/browser-controller.js';
 import { anonymizeProxySugar } from '../anonymize-proxy.js';
-import { log } from '../logger.js';
 
 export interface PuppeteerNewPageOptions extends PuppeteerTypes.BrowserContextOptions {
     proxyUsername?: string;
@@ -88,7 +87,7 @@ export class PuppeteerController extends BrowserController<
                     try {
                         await context.close();
                     } catch (error: any) {
-                        log.exception(error, 'Failed to close context.');
+                        this.log.exception(error, 'Failed to close context.');
                     } finally {
                         await close();
                     }
@@ -120,7 +119,7 @@ export class PuppeteerController extends BrowserController<
         const browserProcess = this.browser.process();
 
         if (!browserProcess) {
-            log.debug('Browser was connected using the `puppeteer.connect` method no browser to kill.');
+            this.log.debug('Browser was connected using the `puppeteer.connect` method no browser to kill.');
             return;
         }
 
@@ -135,7 +134,7 @@ export class PuppeteerController extends BrowserController<
             await this.browser.close();
             clearTimeout(timeout);
         } catch (error) {
-            log.debug('Browser was already killed.', { error });
+            this.log.debug('Browser was already killed.', { error });
         }
     }
 

packages/browser-pool/src/puppeteer/puppeteer-plugin.ts

@@ -7,7 +7,6 @@ import type * as PuppeteerTypes from 'puppeteer';
 import { BrowserPlugin } from '../abstract-classes/browser-plugin.js';
 import { anonymizeProxySugar } from '../anonymize-proxy.js';
 import type { LaunchContext } from '../launch-context.js';
-import { log } from '../logger.js';
 import { noop } from '../utils.js';
 import type { PuppeteerNewPageOptions } from './puppeteer-controller.js';
 import { PuppeteerController } from './puppeteer-controller.js';
@@ -95,12 +94,12 @@ export class PuppeteerPlugin extends BrowserPlugin<
 
                 if (page) {
                     page.on('error', (error) => {
-                        log.exception(error, 'Page crashed.');
+                        this.log.exception(error, 'Page crashed.');
                         page.close().catch(noop);
                     });
                 }
             } catch (error: any) {
-                log.exception(error, 'Failed to retrieve page from target.');
+                this.log.exception(error, 'Failed to retrieve page from target.');
             }
         });