From e5562614236d0b703d6a1f55e0cf8474cf03b071 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 9 Apr 2025 18:56:35 +0530 Subject: [PATCH 01/70] change system_var to config_variable and user_var to user_variable and add migration file --- ...743085000787-updateKeyValuePairTypeEnum.ts | 69 +++++++++++++++++++ .../key-value-pair/key-value-pair.entity.ts | 6 +- .../user-vars/services/user-vars.service.ts | 16 ++--- 3 files changed, 80 insertions(+), 11 deletions(-) create mode 100644 packages/twenty-server/src/database/typeorm/core/migrations/common/1743085000787-updateKeyValuePairTypeEnum.ts diff --git a/packages/twenty-server/src/database/typeorm/core/migrations/common/1743085000787-updateKeyValuePairTypeEnum.ts b/packages/twenty-server/src/database/typeorm/core/migrations/common/1743085000787-updateKeyValuePairTypeEnum.ts new file mode 100644 index 000000000000..7e862202ea2c --- /dev/null +++ b/packages/twenty-server/src/database/typeorm/core/migrations/common/1743085000787-updateKeyValuePairTypeEnum.ts @@ -0,0 +1,69 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class UpdateKeyValuePairTypeEnum1743085000787 + implements MigrationInterface +{ + name = 'UpdateKeyValuePairTypeEnum1743085000787'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "core"."keyValuePair" ALTER COLUMN "type" DROP DEFAULT`, + ); + + await queryRunner.query( + `ALTER TABLE "core"."keyValuePair" ALTER COLUMN "type" TYPE text USING "type"::text`, + ); + + await queryRunner.query( + `UPDATE "core"."keyValuePair" SET "type" = 'USER_VARIABLE' WHERE "type" = 'USER_VAR'`, + ); + await queryRunner.query( + `UPDATE "core"."keyValuePair" SET "type" = 'CONFIG_VARIABLE' WHERE "type" = 'SYSTEM_VAR'`, + ); + + await queryRunner.query(`DROP TYPE "core"."keyValuePair_type_enum"`); + + await queryRunner.query( + `CREATE TYPE "core"."keyValuePair_type_enum" AS ENUM('USER_VARIABLE', 'FEATURE_FLAG', 'CONFIG_VARIABLE')`, + ); + + await queryRunner.query( + `ALTER TABLE "core"."keyValuePair" ALTER COLUMN "type" TYPE "core"."keyValuePair_type_enum" USING "type"::"core"."keyValuePair_type_enum"`, + ); + + await queryRunner.query( + `ALTER TABLE "core"."keyValuePair" ALTER COLUMN "type" SET DEFAULT 'USER_VARIABLE'`, + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "core"."keyValuePair" ALTER COLUMN "type" DROP DEFAULT`, + ); + + await queryRunner.query( + `ALTER TABLE "core"."keyValuePair" ALTER COLUMN "type" TYPE text USING "type"::text`, + ); + + await queryRunner.query( + `UPDATE "core"."keyValuePair" SET "type" = 'USER_VAR' WHERE "type" = 'USER_VARIABLE'`, + ); + await queryRunner.query( + `UPDATE "core"."keyValuePair" SET "type" = 'SYSTEM_VAR' WHERE "type" = 'CONFIG_VARIABLE'`, + ); + + await queryRunner.query(`DROP TYPE "core"."keyValuePair_type_enum"`); + + await queryRunner.query( + `CREATE TYPE "core"."keyValuePair_type_enum" AS ENUM('USER_VAR', 'FEATURE_FLAG', 'SYSTEM_VAR')`, + ); + + await queryRunner.query( + `ALTER TABLE "core"."keyValuePair" ALTER COLUMN "type" TYPE "core"."keyValuePair_type_enum" USING "type"::"core"."keyValuePair_type_enum"`, + ); + + await queryRunner.query( + `ALTER TABLE "core"."keyValuePair" ALTER COLUMN "type" SET DEFAULT 'USER_VAR'`, + ); + } +} diff --git a/packages/twenty-server/src/engine/core-modules/key-value-pair/key-value-pair.entity.ts b/packages/twenty-server/src/engine/core-modules/key-value-pair/key-value-pair.entity.ts index 68629572eb64..805a398e6902 100644 --- a/packages/twenty-server/src/engine/core-modules/key-value-pair/key-value-pair.entity.ts +++ b/packages/twenty-server/src/engine/core-modules/key-value-pair/key-value-pair.entity.ts @@ -19,9 +19,9 @@ import { User } from 'src/engine/core-modules/user/user.entity'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; export enum KeyValuePairType { - USER_VAR = 'USER_VAR', + USER_VARIABLE = 'USER_VARIABLE', FEATURE_FLAG = 'FEATURE_FLAG', - SYSTEM_VAR = 'SYSTEM_VAR', + CONFIG_VARIABLE = 'CONFIG_VARIABLE', } @Entity({ name: 'keyValuePair', schema: 'core' }) @@ -75,7 +75,7 @@ export class KeyValuePair { type: 'enum', enum: Object.values(KeyValuePairType), nullable: false, - default: KeyValuePairType.USER_VAR, + default: KeyValuePairType.USER_VARIABLE, }) type: KeyValuePairType; diff --git a/packages/twenty-server/src/engine/core-modules/user/user-vars/services/user-vars.service.ts b/packages/twenty-server/src/engine/core-modules/user/user-vars/services/user-vars.service.ts index 2d60cf030b08..d3eff0b9ca98 100644 --- a/packages/twenty-server/src/engine/core-modules/user/user-vars/services/user-vars.service.ts +++ b/packages/twenty-server/src/engine/core-modules/user/user-vars/services/user-vars.service.ts @@ -23,7 +23,7 @@ export class UserVarsService< if (workspaceId) { userVarWorkspaceLevel = await this.keyValuePairService.get({ - type: KeyValuePairType.USER_VAR, + type: KeyValuePairType.USER_VARIABLE, userId: null, workspaceId, key, @@ -40,7 +40,7 @@ export class UserVarsService< if (userId) { userVarUserLevel = await this.keyValuePairService.get({ - type: KeyValuePairType.USER_VAR, + type: KeyValuePairType.USER_VARIABLE, userId, workspaceId: null, key, @@ -55,7 +55,7 @@ export class UserVarsService< if (userId && workspaceId) { userVarWorkspaceAndUserLevel = await this.keyValuePairService.get({ - type: KeyValuePairType.USER_VAR, + type: KeyValuePairType.USER_VARIABLE, userId, workspaceId, key, @@ -88,7 +88,7 @@ export class UserVarsService< result = [ ...result, ...(await this.keyValuePairService.get({ - type: KeyValuePairType.USER_VAR, + type: KeyValuePairType.USER_VARIABLE, userId, workspaceId: null, })), @@ -99,7 +99,7 @@ export class UserVarsService< result = [ ...result, ...(await this.keyValuePairService.get({ - type: KeyValuePairType.USER_VAR, + type: KeyValuePairType.USER_VARIABLE, userId: null, workspaceId, })), @@ -110,7 +110,7 @@ export class UserVarsService< result = [ ...result, ...(await this.keyValuePairService.get({ - type: KeyValuePairType.USER_VAR, + type: KeyValuePairType.USER_VARIABLE, userId, workspaceId, })), @@ -136,7 +136,7 @@ export class UserVarsService< workspaceId, key: key, value, - type: KeyValuePairType.USER_VAR, + type: KeyValuePairType.USER_VARIABLE, }); } @@ -153,7 +153,7 @@ export class UserVarsService< userId, workspaceId, key, - type: KeyValuePairType.USER_VAR, + type: KeyValuePairType.USER_VARIABLE, }); } } From 3ad7cb2441c1b5a277304760ca08789e46920b4d Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 9 Apr 2025 19:02:13 +0530 Subject: [PATCH 02/70] add IS_CONFIG_VAR_IN_DB_ENABLED --- packages/twenty-server/.env.example | 3 ++- .../core-modules/twenty-config/config-variables.ts | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/packages/twenty-server/.env.example b/packages/twenty-server/.env.example index ff6d0b8b7a6e..0daef0329396 100644 --- a/packages/twenty-server/.env.example +++ b/packages/twenty-server/.env.example @@ -75,4 +75,5 @@ FRONTEND_URL=http://localhost:3001 # SSL_CERT_PATH="./certs/your-cert.crt" # CLOUDFLARE_API_KEY= # CLOUDFLARE_ZONE_ID= -# CLOUDFLARE_WEBHOOK_SECRET= \ No newline at end of file +# CLOUDFLARE_WEBHOOK_SECRET= +# IS_CONFIG_VAR_IN_DB_ENABLED=false \ No newline at end of file diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts index ad9fd467edb7..cb88a1fe47d6 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts @@ -693,6 +693,15 @@ export class ConfigVariables { @IsOptional() PG_SSL_ALLOW_SELF_SIGNED = false; + @ConfigVariablesMetadata({ + group: ConfigVariablesGroup.ServerConfig, + description: 'Enable configuration variables to be stored in the database', + }) + @CastToBoolean() + @IsBoolean() + @IsOptional() + IS_CONFIG_VAR_IN_DB_ENABLED = false; + @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Time-to-live for cache storage in seconds', From f61b830067bb727c257cc461821c99bf170ff758 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 9 Apr 2025 19:07:38 +0530 Subject: [PATCH 03/70] lint --- packages/twenty-server/.env.example | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/twenty-server/.env.example b/packages/twenty-server/.env.example index 0daef0329396..70abf02ac714 100644 --- a/packages/twenty-server/.env.example +++ b/packages/twenty-server/.env.example @@ -76,4 +76,4 @@ FRONTEND_URL=http://localhost:3001 # CLOUDFLARE_API_KEY= # CLOUDFLARE_ZONE_ID= # CLOUDFLARE_WEBHOOK_SECRET= -# IS_CONFIG_VAR_IN_DB_ENABLED=false \ No newline at end of file +# IS_CONFIG_VAR_IN_DB_ENABLED=false From e58ef317f108d815ae6dbe3474b61d7198f31c9a Mon Sep 17 00:00:00 2001 From: ehconitin Date: Thu, 10 Apr 2025 13:53:54 +0530 Subject: [PATCH 04/70] rename service spec --- ...{environment.service.spec.ts => twenty-config.service.spec.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename packages/twenty-server/src/engine/core-modules/twenty-config/{environment.service.spec.ts => twenty-config.service.spec.ts} (100%) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/environment.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts similarity index 100% rename from packages/twenty-server/src/engine/core-modules/twenty-config/environment.service.spec.ts rename to packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts From 31ed7e7ee02d956f4090d8f45ee2b87c0387af09 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Thu, 10 Apr 2025 14:41:48 +0530 Subject: [PATCH 05/70] Copy over from POC --- .../config-variables-cache.constants.ts | 10 + .../drivers/database-config.driver.ts | 433 ++++++++++++++++++ .../drivers/environment-config.driver.ts | 15 + .../enums/initialization-state.enum.ts | 6 + .../config-var-cache-entry.interface.ts | 5 + .../database-config-driver.interface.ts | 27 ++ .../convert-config-var-to-app-type.util.ts | 32 ++ ...convert-config-var-to-storage-type.util.ts | 32 ++ .../utils/is-env-only-config-var.util.ts | 13 + 9 files changed, 573 insertions(+) create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache.constants.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/drivers/environment-config.driver.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/enums/initialization-state.enum.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/interfaces/config-var-cache-entry.interface.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/interfaces/database-config-driver.interface.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache.constants.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache.constants.ts new file mode 100644 index 000000000000..19a39ff8d7e4 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache.constants.ts @@ -0,0 +1,10 @@ +// TTL values in milliseconds +// TODO: seperate file for each constant +export const POSITIVE_CACHE_TTL = 5 * 60 * 1000; // 5 minutes +export const NEGATIVE_CACHE_TTL = 60 * 1000; // 1 minute +export const CACHE_SCAVENGE_INTERVAL = 5 * 60 * 1000; // 5 minutes +export const MAX_CACHE_ENTRIES = 1000; // Maximum number of entries in cache + +// Retry configuration +export const INITIAL_RETRY_DELAY = 1000; // 1 second +export const MAX_RETRY_ATTEMPTS = 3; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts new file mode 100644 index 000000000000..935bf0b5fd80 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -0,0 +1,433 @@ +import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { IsNull, Repository } from 'typeorm'; + +import { ConfigVarCacheEntry } from 'src/engine/core-modules/twenty-config/interfaces/config-var-cache-entry.interface'; +import { DatabaseConfigDriverInterface } from 'src/engine/core-modules/twenty-config/interfaces/database-config-driver.interface'; + +import { + KeyValuePair, + KeyValuePairType, +} from 'src/engine/core-modules/key-value-pair/key-value-pair.entity'; +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { + CACHE_SCAVENGE_INTERVAL, + INITIAL_RETRY_DELAY, + MAX_CACHE_ENTRIES, + MAX_RETRY_ATTEMPTS, + NEGATIVE_CACHE_TTL, + POSITIVE_CACHE_TTL, +} from 'src/engine/core-modules/twenty-config/constants/config-variables-cache.constants'; +import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; +import { InitializationState } from 'src/engine/core-modules/twenty-config/enums/initialization-state.enum'; +import { convertConfigVarToAppType } from 'src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util'; +import { convertConfigVarToStorageType } from 'src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util'; +import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; + +@Injectable() +export class DatabaseConfigDriver + implements DatabaseConfigDriverInterface, OnModuleDestroy +{ + private readonly valueCache: Map>; + private readonly negativeLookupCache: Map< + string, + ConfigVarCacheEntry + >; + private initializationState = InitializationState.NOT_INITIALIZED; + private initializationPromise: Promise | null = null; + private retryAttempts = 0; + private cacheScavengeInterval: NodeJS.Timeout; + private readonly logger = new Logger(DatabaseConfigDriver.name); + + constructor( + private readonly environmentDriver: EnvironmentConfigDriver, + @InjectRepository(KeyValuePair, 'core') + private readonly keyValuePairRepository: Repository, + ) { + this.valueCache = new Map(); + this.negativeLookupCache = new Map(); + this.startCacheScavenging(); + } + + /** + * Initialize the database driver by loading all config variables from DB + */ + async initialize(): Promise { + if (this.initializationPromise) { + return this.initializationPromise; + } + + this.initializationPromise = new Promise((resolve) => { + this.loadAllConfigVarsFromDb() + .then(() => { + this.initializationState = InitializationState.INITIALIZED; + resolve(); + }) + .catch((error) => { + this.logger.error('Failed to initialize database driver', error); + this.initializationState = InitializationState.FAILED; + this.scheduleRetry(); + // We still resolve the promise to prevent bootstrap from failing + // The driver will fallback to environment variables when in FAILED state + resolve(); + }); + }); + + return this.initializationPromise; + } + + get(key: T): ConfigVariables[T] { + // 1. If in not-initialized or initializing state, use environment variable + if (this.initializationState !== InitializationState.INITIALIZED) { + this.logger.debug( + `[Cache:${key}] Using env due to initialization state: ${this.initializationState}`, + ); + + return this.environmentDriver.get(key); + } + + // 2. Check if this is an environment-only variable + if (isEnvOnlyConfigVar(key)) { + this.logger.debug(`[Cache:${key}] Using env due to isEnvOnly flag`); + + return this.environmentDriver.get(key); + } + + // 3. Check negative lookup cache first (quick rejection) + const negativeCacheEntry = this.negativeLookupCache.get(key as string); + + if (negativeCacheEntry && !this.isCacheExpired(negativeCacheEntry)) { + this.logger.debug(`[Cache:${key}] Negative cache hit - using env`); + console.log(`🔴 CACHE: Negative cache hit for ${key as string}`); + + return this.environmentDriver.get(key); + } + + // 4. Check value cache + const valueCacheEntry = this.valueCache.get(key as string); + + if (valueCacheEntry && !this.isCacheExpired(valueCacheEntry)) { + this.logger.debug( + `[Cache:${key}] Positive cache hit - using cached value`, + ); + console.log(`🟢 CACHE: Positive cache hit for ${key as string}`); + + // Convert the value to the appropriate type before returning + return convertConfigVarToAppType(valueCacheEntry.value, key); + } + + // 5. Schedule background refresh - Cache miss + this.logger.debug(`[Cache:${key}] Cache miss - scheduling refresh`); + console.log( + `🟡 CACHE: Cache miss for ${key as string} - scheduling refresh`, + ); + + this.scheduleRefresh(key).catch((error) => { + this.logger.error(`Failed to refresh config for ${key as string}`, error); + }); + + // 6. Return environment value immediately + return this.environmentDriver.get(key); + } + + async refreshConfig(key: keyof ConfigVariables): Promise { + try { + this.logger.debug(`[Cache:${key}] Refreshing from database`); + console.log(`🔄 CACHE: Refreshing ${key as string} from database`); + + const result = await this.queryDatabase(key as string); + + if (result !== undefined) { + // Store the original value in the cache + this.valueCache.set(key as string, { + value: result, + timestamp: Date.now(), + ttl: POSITIVE_CACHE_TTL, + }); + this.negativeLookupCache.delete(key as string); + this.logger.debug( + `[Cache:${key}] Updated positive cache with value from DB`, + ); + console.log(`✅ CACHE: Updated positive cache for ${key as string}`); + } else { + this.negativeLookupCache.set(key as string, { + value: true, + timestamp: Date.now(), + ttl: NEGATIVE_CACHE_TTL, + }); + this.valueCache.delete(key as string); + this.logger.debug( + `[Cache:${key}] Updated negative cache (not found in DB)`, + ); + console.log( + `❌ CACHE: Updated negative cache for ${key as string} (not found in DB)`, + ); + } + } catch (error) { + this.logger.error(`Failed to refresh config for ${key as string}`, error); + } + } + + async update( + key: T, + value: ConfigVariables[T], + ): Promise { + try { + // Convert the value to JSON storage format using the converter + // TODO: same here. More clean way would be to have a non json table for config vars, which means a new table just for the config vars + // TODO: and then we can just store the value as is, without converting to json + // or can we store the type of value win the json? + const processedValue = convertConfigVarToStorageType(value); + + this.logger.debug(`[Cache:${key}] Updating in database`, { + originalType: typeof value, + processedType: typeof processedValue, + isArray: Array.isArray(processedValue), + }); + console.log(`🔵 CACHE: Updating ${key as string} in database`); + + // Check if the record exists + const existingRecord = await this.keyValuePairRepository.findOne({ + where: { + key: key as string, + userId: IsNull(), + workspaceId: IsNull(), + type: KeyValuePairType.CONFIG_VARIABLE, + }, + }); + + if (existingRecord) { + // Update existing record + this.logger.debug(`[Cache:${key}] Updating existing record in DB`); + await this.keyValuePairRepository.update( + { + id: existingRecord.id, + }, + { + value: processedValue, + }, + ); + } else { + // Insert new record + this.logger.debug(`[Cache:${key}] Inserting new record in DB`); + await this.keyValuePairRepository.insert({ + key: key as string, + value: processedValue, + userId: null, + workspaceId: null, + type: KeyValuePairType.CONFIG_VARIABLE, + }); + } + + // Update cache immediately with the properly converted value + this.valueCache.set(key as string, { + value: processedValue, + timestamp: Date.now(), + ttl: POSITIVE_CACHE_TTL, + }); + this.negativeLookupCache.delete(key as string); + this.logger.debug(`[Cache:${key}] Updated cache with new value`); + console.log( + `✅ CACHE: Updated cache for ${key as string} with new value`, + ); + } catch (error) { + this.logger.error(`Failed to update config for ${key as string}`, error); + throw error; + } + } + + clearCache(key: keyof ConfigVariables): void { + this.valueCache.delete(key as string); + this.negativeLookupCache.delete(key as string); + } + + clearAllCache(): void { + this.valueCache.clear(); + this.negativeLookupCache.clear(); + } + + onModuleDestroy() { + if (this.cacheScavengeInterval) { + clearInterval(this.cacheScavengeInterval); + } + } + + private async scheduleRefresh(key: keyof ConfigVariables): Promise { + // Log when a refresh is scheduled but not yet executed + this.logger.debug(`[Cache:${key}] Scheduling background refresh`); + console.log(`🕒 CACHE: Scheduling background refresh for ${key as string}`); + + setImmediate(() => { + this.logger.debug(`[Cache:${key}] Executing background refresh`); + console.log( + `⏳ CACHE: Executing background refresh for ${key as string}`, + ); + + this.refreshConfig(key).catch((error) => { + this.logger.error( + `Failed to refresh config for ${key as string}`, + error, + ); + }); + }); + + return Promise.resolve(); + } + + private scheduleRetry(): void { + if (this.retryAttempts >= MAX_RETRY_ATTEMPTS) { + this.logger.error('Max retry attempts reached, giving up initialization'); + + return; + } + + const delay = INITIAL_RETRY_DELAY * Math.pow(2, this.retryAttempts); + + this.retryAttempts++; + + setTimeout(() => { + this.initializationPromise = null; + this.initialize().catch((error) => { + this.logger.error('Retry initialization failed', error); + }); + }, delay); + } + + private async loadAllConfigVarsFromDb(): Promise { + try { + const configVars = await this.keyValuePairRepository.find({ + where: { + type: KeyValuePairType.CONFIG_VARIABLE, + userId: IsNull(), + workspaceId: IsNull(), + }, + }); + + if (!configVars.length) { + return; + } + + const now = Date.now(); + + for (const configVar of configVars) { + if (configVar.value !== null) { + this.valueCache.set(configVar.key, { + value: configVar.value, + timestamp: now, + ttl: POSITIVE_CACHE_TTL, + }); + } + } + + this.logger.log( + `Loaded ${configVars.length} config variables from database`, + ); + } catch (error) { + this.logger.error('Failed to load config variables from database', error); + throw error; + } + } + + private async queryDatabase(key: string): Promise { + try { + const result = await this.keyValuePairRepository.findOne({ + where: { + type: KeyValuePairType.CONFIG_VARIABLE, + key, + userId: IsNull(), + workspaceId: IsNull(), + }, + }); + + return result?.value; + } catch (error) { + this.logger.error(`Failed to query database for ${key}`, error); + + return undefined; + } + } + + private isCacheExpired(entry: ConfigVarCacheEntry): boolean { + return Date.now() - entry.timestamp > entry.ttl; + } + + private startCacheScavenging(): void { + this.cacheScavengeInterval = setInterval(() => { + this.scavengeCache(); + }, CACHE_SCAVENGE_INTERVAL); + } + + private scavengeCache(): void { + const now = Date.now(); + + for (const [key, entry] of this.valueCache.entries()) { + if (now - entry.timestamp > entry.ttl) { + this.valueCache.delete(key); + } + } + + for (const [key, entry] of this.negativeLookupCache.entries()) { + if (now - entry.timestamp > entry.ttl) { + this.negativeLookupCache.delete(key); + } + } + + // THis is some thing I added later not in the design doc + // makes sense tho, because we don't want to have too many cache entries + if (this.valueCache.size > MAX_CACHE_ENTRIES) { + const entriesToDelete = this.valueCache.size - MAX_CACHE_ENTRIES; + const keysToDelete = Array.from(this.valueCache.keys()).slice( + 0, + entriesToDelete, + ); + + keysToDelete.forEach((key) => this.valueCache.delete(key)); + } + + if (this.negativeLookupCache.size > MAX_CACHE_ENTRIES) { + const entriesToDelete = this.negativeLookupCache.size - MAX_CACHE_ENTRIES; + const keysToDelete = Array.from(this.negativeLookupCache.keys()).slice( + 0, + entriesToDelete, + ); + + keysToDelete.forEach((key) => this.negativeLookupCache.delete(key)); + } + } + + getFromValueCache(key: string): any { + const entry = this.valueCache.get(key); + + if (entry && !this.isCacheExpired(entry)) { + // again same type conversion -- I don't like it + return convertConfigVarToAppType( + entry.value, + key as keyof ConfigVariables, + ); + } + + return undefined; + } + + // Helper method to get cache information for debugging + getCacheInfo(): { + positiveEntries: number; + negativeEntries: number; + cacheKeys: string[]; + } { + const validPositiveEntries = Array.from(this.valueCache.entries()).filter( + ([_, entry]) => !this.isCacheExpired(entry), + ); + + const validNegativeEntries = Array.from( + this.negativeLookupCache.entries(), + ).filter(([_, entry]) => !this.isCacheExpired(entry)); + + return { + positiveEntries: validPositiveEntries.length, + negativeEntries: validNegativeEntries.length, + cacheKeys: validPositiveEntries.map(([key]) => key), + }; + } +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/environment-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/environment-config.driver.ts new file mode 100644 index 000000000000..e18059e53c0e --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/environment-config.driver.ts @@ -0,0 +1,15 @@ +import { Injectable } from '@nestjs/common'; +import { ConfigService } from '@nestjs/config'; + +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +@Injectable() +export class EnvironmentConfigDriver { + constructor(private readonly configService: ConfigService) {} + + get(key: T): ConfigVariables[T] { + return this.configService.get( + key, + new ConfigVariables()[key], + ); + } +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/enums/initialization-state.enum.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/enums/initialization-state.enum.ts new file mode 100644 index 000000000000..af938681403a --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/enums/initialization-state.enum.ts @@ -0,0 +1,6 @@ +export enum InitializationState { + NOT_INITIALIZED = 'NOT_INITIALIZED', + INITIALIZING = 'INITIALIZING', + INITIALIZED = 'INITIALIZED', + FAILED = 'FAILED', +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/interfaces/config-var-cache-entry.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/interfaces/config-var-cache-entry.interface.ts new file mode 100644 index 000000000000..9ee4f990b25b --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/interfaces/config-var-cache-entry.interface.ts @@ -0,0 +1,5 @@ +export interface ConfigVarCacheEntry { + value: T; + timestamp: number; + ttl: number; +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/interfaces/database-config-driver.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/interfaces/database-config-driver.interface.ts new file mode 100644 index 000000000000..c558d7c72e09 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/interfaces/database-config-driver.interface.ts @@ -0,0 +1,27 @@ +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; + +/** + * Interface for drivers that support database-backed configuration + * with caching and initialization capabilities + */ +export interface DatabaseConfigDriverInterface { + /** + * Get a configuration value + */ + get(key: T): ConfigVariables[T]; + + /** + * Initialize the driver + */ + initialize(): Promise; + + /** + * Clear a specific key from cache + */ + clearCache(key: keyof ConfigVariables): void; + + /** + * Refresh a specific configuration from its source + */ + refreshConfig(key: keyof ConfigVariables): Promise; +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util.ts new file mode 100644 index 000000000000..f22e6f105764 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util.ts @@ -0,0 +1,32 @@ +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; + +/** + * Convert a database value to the appropriate type for the environment variable + */ +export const convertConfigVarToAppType = ( + dbValue: unknown, + key: T, +): ConfigVariables[T] => { + if (dbValue === null || dbValue === undefined) { + return dbValue as ConfigVariables[T]; + } + + // Get the default value to determine the expected type + const defaultValue = new ConfigVariables()[key]; + const valueType = typeof defaultValue; + + if (valueType === 'boolean' && typeof dbValue === 'string') { + return (dbValue === 'true') as unknown as ConfigVariables[T]; + } + + if (valueType === 'number' && typeof dbValue === 'string') { + return Number(dbValue) as unknown as ConfigVariables[T]; + } + + // Handle arrays and other complex types + if (Array.isArray(defaultValue) && typeof dbValue === 'object') { + return dbValue as ConfigVariables[T]; + } + + return dbValue as ConfigVariables[T]; +}; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util.ts new file mode 100644 index 000000000000..c6143da87079 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util.ts @@ -0,0 +1,32 @@ +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; + +/** + * Convert an environment variable value to a format suitable for database storage + */ +export const convertConfigVarToStorageType = ( + appValue: ConfigVariables[T], +): JSON | undefined => { + if (appValue === undefined) { + return undefined; + } + + // Convert to the appropriate type for JSON storage + if (typeof appValue === 'boolean') { + return appValue as unknown as JSON; + } + + if (typeof appValue === 'number') { + return appValue as unknown as JSON; + } + + if (typeof appValue === 'string') { + return appValue as unknown as JSON; + } + + // For arrays and objects, ensure they're stored as JSON-compatible values + if (Array.isArray(appValue) || typeof appValue === 'object') { + return appValue as unknown as JSON; + } + + return appValue as unknown as JSON; +}; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util.ts new file mode 100644 index 000000000000..717636f7cff9 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util.ts @@ -0,0 +1,13 @@ +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { TypedReflect } from 'src/utils/typed-reflect'; + +/** + * Checks if the environment variable is environment-only based on metadata + */ +export const isEnvOnlyConfigVar = (key: keyof ConfigVariables): boolean => { + const metadata = + TypedReflect.getMetadata('config-variables', ConfigVariables) ?? {}; + const envMetadata = metadata[key]; + + return !!envMetadata?.isEnvOnly; +}; From 46ba298a492eaf9aa0a3a7d33b99d25730eae85a Mon Sep 17 00:00:00 2001 From: ehconitin Date: Thu, 10 Apr 2025 14:53:19 +0530 Subject: [PATCH 06/70] continue --- .../twenty-config/config-variables.ts | 5 + .../config-variables-metadata.decorator.ts | 1 + .../twenty-config/twenty-config.module.ts | 13 +- .../twenty-config/twenty-config.service.ts | 216 +++++++++++++++++- 4 files changed, 222 insertions(+), 13 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts index cb88a1fe47d6..4be643f5b448 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts @@ -673,6 +673,7 @@ export class ConfigVariables { group: ConfigVariablesGroup.ServerConfig, isSensitive: true, description: 'Database connection URL', + isEnvOnly: true, }) @IsDefined() @IsUrl({ @@ -687,6 +688,7 @@ export class ConfigVariables { group: ConfigVariablesGroup.ServerConfig, description: 'Allow connections to a database with self-signed certificates', + isEnvOnly: true, }) @CastToBoolean() @IsBoolean() @@ -696,6 +698,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerConfig, description: 'Enable configuration variables to be stored in the database', + isEnvOnly: true, }) @CastToBoolean() @IsBoolean() @@ -713,6 +716,7 @@ export class ConfigVariables { group: ConfigVariablesGroup.ServerConfig, isSensitive: true, description: 'URL for cache storage (e.g., Redis connection URL)', + isEnvOnly: true, }) @IsOptional() @IsUrl({ @@ -751,6 +755,7 @@ export class ConfigVariables { group: ConfigVariablesGroup.ServerConfig, isSensitive: true, description: 'Secret key for the application', + isEnvOnly: true, }) @IsString() APP_SECRET: string; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator.ts index 5f84c9da83f2..0ba4449dbadf 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator.ts @@ -7,6 +7,7 @@ export interface ConfigVariablesMetadataOptions { group: ConfigVariablesGroup; description: string; isSensitive?: boolean; + isEnvOnly?: boolean; } export type ConfigVariablesMetadataMap = { diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts index 2083625d83f0..1ae686f6073e 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts @@ -1,7 +1,11 @@ import { Global, Module } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; +import { TypeOrmModule } from '@nestjs/typeorm'; +import { KeyValuePair } from 'src/engine/core-modules/key-value-pair/key-value-pair.entity'; import { validate } from 'src/engine/core-modules/twenty-config/config-variables'; +import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; +import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; import { ConfigurableModuleClass } from 'src/engine/core-modules/twenty-config/twenty-config.module-definition'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; @@ -14,8 +18,13 @@ import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twent validate, envFilePath: process.env.NODE_ENV === 'test' ? '.env.test' : '.env', }), + TypeOrmModule.forFeature([KeyValuePair], 'core'), ], - providers: [TwentyConfigService], - exports: [TwentyConfigService], + providers: [ + TwentyConfigService, + EnvironmentConfigDriver, + DatabaseConfigDriver, + ], + exports: [TwentyConfigService, EnvironmentConfigDriver, DatabaseConfigDriver], }) export class TwentyConfigModule extends ConfigurableModuleClass {} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts index 6dd527991c2a..dd77afb4c981 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts @@ -1,22 +1,155 @@ -import { Injectable } from '@nestjs/common'; +import { + Injectable, + Logger, + OnApplicationBootstrap, + OnModuleInit, +} from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; import { CONFIG_VARIABLES_MASKING_CONFIG } from 'src/engine/core-modules/twenty-config/constants/config-variables-masking-config'; import { ConfigVariablesMetadataOptions } from 'src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator'; +import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; +import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; import { ConfigVariablesMaskingStrategies } from 'src/engine/core-modules/twenty-config/enums/config-variables-masking-strategies.enum'; +import { InitializationState } from 'src/engine/core-modules/twenty-config/enums/initialization-state.enum'; import { configVariableMaskSensitiveData } from 'src/engine/core-modules/twenty-config/utils/config-variable-mask-sensitive-data.util'; import { TypedReflect } from 'src/utils/typed-reflect'; @Injectable() -export class TwentyConfigService { - constructor(private readonly configService: ConfigService) {} +export class TwentyConfigService + implements OnModuleInit, OnApplicationBootstrap +{ + private driver: DatabaseConfigDriver | EnvironmentConfigDriver; + private initializationState = InitializationState.NOT_INITIALIZED; + private readonly isConfigVarInDbEnabled: boolean; + private readonly logger = new Logger(TwentyConfigService.name); + + constructor( + private readonly configService: ConfigService, + private readonly databaseConfigDriver: DatabaseConfigDriver, + private readonly environmentConfigDriver: EnvironmentConfigDriver, + ) { + // Always start with environment driver during construction + this.driver = this.environmentConfigDriver; + + const configVarInDb = this.configService.get('IS_CONFIG_VAR_IN_DB_ENABLED'); + + // Handle both string and boolean values for IS_CONFIG_VAR_IN_DB_ENABLED + this.isConfigVarInDbEnabled = configVarInDb === true; + + this.logger.log( + `Database configuration is ${this.isConfigVarInDbEnabled ? 'enabled' : 'disabled'}`, + ); + } + + async onModuleInit() { + // During module init, only use the environment driver + // and mark as initialized if not using DB driver + if (!this.isConfigVarInDbEnabled) { + this.logger.log( + 'Database configuration is disabled, using environment variables only', + ); + this.initializationState = InitializationState.INITIALIZED; + + return; + } + + // If we're using DB driver, mark as not initialized yet + // Will be fully initialized in onApplicationBootstrap + this.logger.log( + 'Database configuration is enabled, will initialize after application bootstrap', + ); + } + + async onApplicationBootstrap() { + if (!this.isConfigVarInDbEnabled) { + return; + } + + try { + this.logger.log('Initializing database driver for configuration'); + this.initializationState = InitializationState.INITIALIZING; + + // Try to initialize the database driver + await this.databaseConfigDriver.initialize(); + + // If initialization succeeds, switch to the database driver + this.driver = this.databaseConfigDriver; + this.initializationState = InitializationState.INITIALIZED; + this.logger.log('Database driver initialized successfully'); + this.logger.log(`Active driver: DatabaseDriver`); + } catch (error) { + // If initialization fails for any reason, log the error and keep using the environment driver + this.logger.error( + 'Failed to initialize database driver, falling back to environment variables', + error, + ); + this.initializationState = InitializationState.FAILED; + + // Ensure we're still using the environment driver + this.driver = this.environmentConfigDriver; + this.logger.log(`Active driver: EnvironmentDriver (fallback)`); + } + } get(key: T): ConfigVariables[T] { - return this.configService.get( - key, - new ConfigVariables()[key], + const value = this.driver.get(key); + + this.logger.debug( + `Getting value for '${key}' from ${this.driver.constructor.name}`, + { + valueType: typeof value, + isArray: Array.isArray(value), + value: typeof value === 'object' ? JSON.stringify(value) : value, + }, ); + + return value; + } + + async update( + key: T, + value: ConfigVariables[T], + ): Promise { + if (!this.isConfigVarInDbEnabled) { + throw new Error( + 'Database configuration is disabled, cannot update configuration', + ); + } + + if (this.initializationState !== InitializationState.INITIALIZED) { + throw new Error( + 'Environment service not initialized, cannot update configuration', + ); + } + + const metadata = + TypedReflect.getMetadata('config-variables', ConfigVariables) ?? {}; + const envMetadata = metadata[key]; + + if (envMetadata?.isEnvOnly) { + throw new Error( + `Cannot update environment-only variable: ${key as string}`, + ); + } + + if (this.driver === this.databaseConfigDriver) { + await this.databaseConfigDriver.update(key, value); + } else { + throw new Error( + 'Database driver not active, cannot update configuration', + ); + } + } + + getMetadata( + key: keyof ConfigVariables, + ): ConfigVariablesMetadataOptions | undefined { + const metadata = + TypedReflect.getMetadata('config-variables', ConfigVariables) ?? {}; + + return metadata[key]; } getAll(): Record< @@ -24,6 +157,7 @@ export class TwentyConfigService { { value: ConfigVariables[keyof ConfigVariables]; metadata: ConfigVariablesMetadataOptions; + source: string; } > { const result: Record< @@ -31,18 +165,35 @@ export class TwentyConfigService { { value: ConfigVariables[keyof ConfigVariables]; metadata: ConfigVariablesMetadataOptions; + source: string; } > = {}; - const configVars = new ConfigVariables(); + const envVars = new ConfigVariables(); const metadata = TypedReflect.getMetadata('config-variables', ConfigVariables) ?? {}; + const isUsingDatabaseDriver = + this.driver === this.databaseConfigDriver && + this.isConfigVarInDbEnabled && + this.initializationState === InitializationState.INITIALIZED; + Object.entries(metadata).forEach(([key, envMetadata]) => { - let value = - this.configService.get(key) ?? - configVars[key as keyof ConfigVariables] ?? - ''; + let value = this.get(key as keyof ConfigVariables) ?? ''; + let source = 'ENVIRONMENT'; + + if (isUsingDatabaseDriver && !envMetadata.isEnvOnly) { + const valueCacheEntry = + this.databaseConfigDriver.getFromValueCache(key); + + if (valueCacheEntry) { + source = 'DATABASE'; + } else if (value === envVars[key as keyof ConfigVariables]) { + source = 'DEFAULT'; + } + } else if (value === envVars[key as keyof ConfigVariables]) { + source = 'DEFAULT'; + } if (typeof value === 'string' && key in CONFIG_VARIABLES_MASKING_CONFIG) { const varMaskingConfig = @@ -65,9 +216,52 @@ export class TwentyConfigService { result[key] = { value, metadata: envMetadata, + source, }; }); return result; } + + clearCache(key: keyof ConfigVariables): void { + if (this.driver === this.databaseConfigDriver) { + this.databaseConfigDriver.clearCache(key); + } + } + + async refreshConfig(key: keyof ConfigVariables): Promise { + if (this.driver === this.databaseConfigDriver) { + await this.databaseConfigDriver.refreshConfig(key); + } + } + + // Get cache information for debugging + getCacheInfo(): { + usingDatabaseDriver: boolean; + initializationState: string; + cacheStats?: { + positiveEntries: number; + negativeEntries: number; + cacheKeys: string[]; + }; + } { + const isUsingDatabaseDriver = + this.driver === this.databaseConfigDriver && + this.isConfigVarInDbEnabled && + this.initializationState === InitializationState.INITIALIZED; + + const result = { + usingDatabaseDriver: isUsingDatabaseDriver, + initializationState: InitializationState[this.initializationState], + }; + + if (isUsingDatabaseDriver) { + return { + ...result, + cacheStats: this.databaseConfigDriver.getCacheInfo(), + }; + } + + return result; + } } From d5c6038ccad81c390e206067f0091b43ab1f49e9 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Thu, 10 Apr 2025 15:19:28 +0530 Subject: [PATCH 07/70] twenty config module is global? --- .../engine/core-modules/file/file-upload/file-upload.module.ts | 3 +-- .../twenty-server/src/engine/core-modules/file/file.module.ts | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/file/file-upload/file-upload.module.ts b/packages/twenty-server/src/engine/core-modules/file/file-upload/file-upload.module.ts index f2d109931453..1f18acaf4b47 100644 --- a/packages/twenty-server/src/engine/core-modules/file/file-upload/file-upload.module.ts +++ b/packages/twenty-server/src/engine/core-modules/file/file-upload/file-upload.module.ts @@ -3,11 +3,10 @@ import { Module } from '@nestjs/common'; import { FileUploadResolver } from 'src/engine/core-modules/file/file-upload/resolvers/file-upload.resolver'; import { FileUploadService } from 'src/engine/core-modules/file/file-upload/services/file-upload.service'; import { FileModule } from 'src/engine/core-modules/file/file.module'; -import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; @Module({ imports: [FileModule], - providers: [FileUploadService, FileUploadResolver, TwentyConfigService], + providers: [FileUploadService, FileUploadResolver], exports: [FileUploadService, FileUploadResolver], }) export class FileUploadModule {} diff --git a/packages/twenty-server/src/engine/core-modules/file/file.module.ts b/packages/twenty-server/src/engine/core-modules/file/file.module.ts index b8a3b356bf0b..e48f16c2495f 100644 --- a/packages/twenty-server/src/engine/core-modules/file/file.module.ts +++ b/packages/twenty-server/src/engine/core-modules/file/file.module.ts @@ -6,7 +6,6 @@ import { FileWorkspaceFolderDeletionJob } from 'src/engine/core-modules/file/job import { FileAttachmentListener } from 'src/engine/core-modules/file/listeners/file-attachment.listener'; import { FileWorkspaceMemberListener } from 'src/engine/core-modules/file/listeners/file-workspace-member.listener'; import { JwtModule } from 'src/engine/core-modules/jwt/jwt.module'; -import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { FileController } from './controllers/file.controller'; import { FileService } from './services/file.service'; @@ -15,7 +14,6 @@ import { FileService } from './services/file.service'; imports: [JwtModule], providers: [ FileService, - TwentyConfigService, FilePathGuard, FileAttachmentListener, FileWorkspaceMemberListener, From eeb744821472f1e4f8501dde104bb6e2aed136a9 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Thu, 10 Apr 2025 18:18:20 +0530 Subject: [PATCH 08/70] wip: const renaming --- .../config-variables-cache-max-entries.ts | 1 + ...onfig-variables-cache-scavenge-interval.ts | 1 + .../constants/config-variables-cache-ttl.ts | 1 + .../config-variables-cache.constants.ts | 10 -- ...abase-config-driver-initial-retry-delay.ts | 1 + ...onfig-driver-initialization-max-retries.ts | 1 + .../drivers/database-config.driver.ts | 95 +++++++++---------- 7 files changed, 51 insertions(+), 59 deletions(-) create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-max-entries.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl.ts delete mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache.constants.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-initial-retry-delay.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-initialization-max-retries.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-max-entries.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-max-entries.ts new file mode 100644 index 000000000000..613b6cf5dda2 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-max-entries.ts @@ -0,0 +1 @@ +export const CONFIG_VARIABLES_CACHE_MAX_ENTRIES = 1000; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval.ts new file mode 100644 index 000000000000..6c12a4c7b438 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval.ts @@ -0,0 +1 @@ +export const CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL = 5 * 60 * 1000; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl.ts new file mode 100644 index 000000000000..cf0739190a51 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl.ts @@ -0,0 +1 @@ +export const CONFIG_VARIABLES_CACHE_TTL = 30 * 1000; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache.constants.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache.constants.ts deleted file mode 100644 index 19a39ff8d7e4..000000000000 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache.constants.ts +++ /dev/null @@ -1,10 +0,0 @@ -// TTL values in milliseconds -// TODO: seperate file for each constant -export const POSITIVE_CACHE_TTL = 5 * 60 * 1000; // 5 minutes -export const NEGATIVE_CACHE_TTL = 60 * 1000; // 1 minute -export const CACHE_SCAVENGE_INTERVAL = 5 * 60 * 1000; // 5 minutes -export const MAX_CACHE_ENTRIES = 1000; // Maximum number of entries in cache - -// Retry configuration -export const INITIAL_RETRY_DELAY = 1000; // 1 second -export const MAX_RETRY_ATTEMPTS = 3; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-initial-retry-delay.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-initial-retry-delay.ts new file mode 100644 index 000000000000..4f89c083adcf --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-initial-retry-delay.ts @@ -0,0 +1 @@ +export const DATABASE_CONFIG_DRIVER_INITIAL_RETRY_DELAY = 1000; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-initialization-max-retries.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-initialization-max-retries.ts new file mode 100644 index 000000000000..5c4dc5e16209 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-initialization-max-retries.ts @@ -0,0 +1 @@ +export const DATABASE_CONFIG_DRIVER_INITIALIZATION_MAX_RETRIES = 3; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index 935bf0b5fd80..b49ce7529e37 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -11,14 +11,11 @@ import { KeyValuePairType, } from 'src/engine/core-modules/key-value-pair/key-value-pair.entity'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; -import { - CACHE_SCAVENGE_INTERVAL, - INITIAL_RETRY_DELAY, - MAX_CACHE_ENTRIES, - MAX_RETRY_ATTEMPTS, - NEGATIVE_CACHE_TTL, - POSITIVE_CACHE_TTL, -} from 'src/engine/core-modules/twenty-config/constants/config-variables-cache.constants'; +import { CONFIG_VARIABLES_CACHE_MAX_ENTRIES } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-max-entries'; +import { CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval'; +import { CONFIG_VARIABLES_CACHE_TTL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl'; +import { DATABASE_CONFIG_DRIVER_INITIAL_RETRY_DELAY } from 'src/engine/core-modules/twenty-config/constants/database-config-driver-initial-retry-delay'; +import { DATABASE_CONFIG_DRIVER_INITIALIZATION_MAX_RETRIES } from 'src/engine/core-modules/twenty-config/constants/database-config-driver-initialization-max-retries'; import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; import { InitializationState } from 'src/engine/core-modules/twenty-config/enums/initialization-state.enum'; import { convertConfigVarToAppType } from 'src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util'; @@ -98,8 +95,7 @@ export class DatabaseConfigDriver const negativeCacheEntry = this.negativeLookupCache.get(key as string); if (negativeCacheEntry && !this.isCacheExpired(negativeCacheEntry)) { - this.logger.debug(`[Cache:${key}] Negative cache hit - using env`); - console.log(`🔴 CACHE: Negative cache hit for ${key as string}`); + this.logger.debug(`🔴 [Cache:${key}] Negative cache hit - using env`); return this.environmentDriver.get(key); } @@ -109,19 +105,15 @@ export class DatabaseConfigDriver if (valueCacheEntry && !this.isCacheExpired(valueCacheEntry)) { this.logger.debug( - `[Cache:${key}] Positive cache hit - using cached value`, + `🟢 [Cache:${key}] Positive cache hit - using cached value`, ); - console.log(`🟢 CACHE: Positive cache hit for ${key as string}`); // Convert the value to the appropriate type before returning return convertConfigVarToAppType(valueCacheEntry.value, key); } // 5. Schedule background refresh - Cache miss - this.logger.debug(`[Cache:${key}] Cache miss - scheduling refresh`); - console.log( - `🟡 CACHE: Cache miss for ${key as string} - scheduling refresh`, - ); + this.logger.debug(`🟡 [Cache:${key}] Cache miss - scheduling refresh`); this.scheduleRefresh(key).catch((error) => { this.logger.error(`Failed to refresh config for ${key as string}`, error); @@ -133,8 +125,7 @@ export class DatabaseConfigDriver async refreshConfig(key: keyof ConfigVariables): Promise { try { - this.logger.debug(`[Cache:${key}] Refreshing from database`); - console.log(`🔄 CACHE: Refreshing ${key as string} from database`); + this.logger.debug(`🔄 [Cache:${key}] Refreshing from database`); const result = await this.queryDatabase(key as string); @@ -143,25 +134,21 @@ export class DatabaseConfigDriver this.valueCache.set(key as string, { value: result, timestamp: Date.now(), - ttl: POSITIVE_CACHE_TTL, + ttl: CONFIG_VARIABLES_CACHE_TTL, }); this.negativeLookupCache.delete(key as string); this.logger.debug( - `[Cache:${key}] Updated positive cache with value from DB`, + `✅ [Cache:${key}] Updated positive cache with value from DB`, ); - console.log(`✅ CACHE: Updated positive cache for ${key as string}`); } else { this.negativeLookupCache.set(key as string, { value: true, timestamp: Date.now(), - ttl: NEGATIVE_CACHE_TTL, + ttl: CONFIG_VARIABLES_CACHE_TTL, }); this.valueCache.delete(key as string); this.logger.debug( - `[Cache:${key}] Updated negative cache (not found in DB)`, - ); - console.log( - `❌ CACHE: Updated negative cache for ${key as string} (not found in DB)`, + `❌ [Cache:${key}] Updated negative cache (not found in DB)`, ); } } catch (error) { @@ -180,12 +167,11 @@ export class DatabaseConfigDriver // or can we store the type of value win the json? const processedValue = convertConfigVarToStorageType(value); - this.logger.debug(`[Cache:${key}] Updating in database`, { + this.logger.debug(`🔵 [Cache:${key}] Updating in database`, { originalType: typeof value, processedType: typeof processedValue, isArray: Array.isArray(processedValue), }); - console.log(`🔵 CACHE: Updating ${key as string} in database`); // Check if the record exists const existingRecord = await this.keyValuePairRepository.findOne({ @@ -199,7 +185,7 @@ export class DatabaseConfigDriver if (existingRecord) { // Update existing record - this.logger.debug(`[Cache:${key}] Updating existing record in DB`); + this.logger.debug(`🔄 [Cache:${key}] Updating existing record in DB`); await this.keyValuePairRepository.update( { id: existingRecord.id, @@ -210,7 +196,7 @@ export class DatabaseConfigDriver ); } else { // Insert new record - this.logger.debug(`[Cache:${key}] Inserting new record in DB`); + this.logger.debug(`🔄 [Cache:${key}] Inserting new record in DB`); await this.keyValuePairRepository.insert({ key: key as string, value: processedValue, @@ -224,13 +210,10 @@ export class DatabaseConfigDriver this.valueCache.set(key as string, { value: processedValue, timestamp: Date.now(), - ttl: POSITIVE_CACHE_TTL, + ttl: CONFIG_VARIABLES_CACHE_TTL, }); this.negativeLookupCache.delete(key as string); - this.logger.debug(`[Cache:${key}] Updated cache with new value`); - console.log( - `✅ CACHE: Updated cache for ${key as string} with new value`, - ); + this.logger.debug(`✅ [Cache:${key}] Updated cache with new value`); } catch (error) { this.logger.error(`Failed to update config for ${key as string}`, error); throw error; @@ -255,14 +238,10 @@ export class DatabaseConfigDriver private async scheduleRefresh(key: keyof ConfigVariables): Promise { // Log when a refresh is scheduled but not yet executed - this.logger.debug(`[Cache:${key}] Scheduling background refresh`); - console.log(`🕒 CACHE: Scheduling background refresh for ${key as string}`); + this.logger.debug(`🕒 [Cache:${key}] Scheduling background refresh`); setImmediate(() => { - this.logger.debug(`[Cache:${key}] Executing background refresh`); - console.log( - `⏳ CACHE: Executing background refresh for ${key as string}`, - ); + this.logger.debug(`⏳ [Cache:${key}] Executing background refresh`); this.refreshConfig(key).catch((error) => { this.logger.error( @@ -276,13 +255,17 @@ export class DatabaseConfigDriver } private scheduleRetry(): void { - if (this.retryAttempts >= MAX_RETRY_ATTEMPTS) { + if ( + this.retryAttempts >= DATABASE_CONFIG_DRIVER_INITIALIZATION_MAX_RETRIES + ) { this.logger.error('Max retry attempts reached, giving up initialization'); return; } - const delay = INITIAL_RETRY_DELAY * Math.pow(2, this.retryAttempts); + const delay = + DATABASE_CONFIG_DRIVER_INITIAL_RETRY_DELAY * + Math.pow(2, this.retryAttempts); this.retryAttempts++; @@ -315,7 +298,7 @@ export class DatabaseConfigDriver this.valueCache.set(configVar.key, { value: configVar.value, timestamp: now, - ttl: POSITIVE_CACHE_TTL, + ttl: CONFIG_VARIABLES_CACHE_TTL, }); } } @@ -355,44 +338,58 @@ export class DatabaseConfigDriver private startCacheScavenging(): void { this.cacheScavengeInterval = setInterval(() => { this.scavengeCache(); - }, CACHE_SCAVENGE_INTERVAL); + }, CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL); } private scavengeCache(): void { const now = Date.now(); + let expiredCount = 0; + let sizeLimitedCount = 0; for (const [key, entry] of this.valueCache.entries()) { if (now - entry.timestamp > entry.ttl) { this.valueCache.delete(key); + expiredCount++; } } for (const [key, entry] of this.negativeLookupCache.entries()) { if (now - entry.timestamp > entry.ttl) { this.negativeLookupCache.delete(key); + expiredCount++; } } // THis is some thing I added later not in the design doc // makes sense tho, because we don't want to have too many cache entries - if (this.valueCache.size > MAX_CACHE_ENTRIES) { - const entriesToDelete = this.valueCache.size - MAX_CACHE_ENTRIES; + if (this.valueCache.size > CONFIG_VARIABLES_CACHE_MAX_ENTRIES) { + const entriesToDelete = + this.valueCache.size - CONFIG_VARIABLES_CACHE_MAX_ENTRIES; const keysToDelete = Array.from(this.valueCache.keys()).slice( 0, entriesToDelete, ); keysToDelete.forEach((key) => this.valueCache.delete(key)); + sizeLimitedCount += entriesToDelete; } - if (this.negativeLookupCache.size > MAX_CACHE_ENTRIES) { - const entriesToDelete = this.negativeLookupCache.size - MAX_CACHE_ENTRIES; + if (this.negativeLookupCache.size > CONFIG_VARIABLES_CACHE_MAX_ENTRIES) { + const entriesToDelete = + this.negativeLookupCache.size - CONFIG_VARIABLES_CACHE_MAX_ENTRIES; const keysToDelete = Array.from(this.negativeLookupCache.keys()).slice( 0, entriesToDelete, ); keysToDelete.forEach((key) => this.negativeLookupCache.delete(key)); + sizeLimitedCount += entriesToDelete; + } + + if (expiredCount > 0 || sizeLimitedCount > 0) { + this.logger.debug( + `Cache scavenging completed: ${expiredCount} expired entries removed, ${sizeLimitedCount} entries removed due to size limits`, + ); } } From 07744afa38fd361a0fe169639c00bebb16c8a9b8 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Thu, 10 Apr 2025 22:31:52 +0530 Subject: [PATCH 09/70] hasty commit, needs rework :) --- .../drivers/database-config.driver.ts | 104 +++++++++--------- .../config-var-cache-entry.interface.ts | 9 +- .../twenty-config/twenty-config.service.ts | 18 +-- 3 files changed, 64 insertions(+), 67 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index b49ce7529e37..46764aa22905 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -3,7 +3,11 @@ import { InjectRepository } from '@nestjs/typeorm'; import { IsNull, Repository } from 'typeorm'; -import { ConfigVarCacheEntry } from 'src/engine/core-modules/twenty-config/interfaces/config-var-cache-entry.interface'; +import { + ConfigKey, + ConfigValue, + ConfigVarCacheEntry, +} from 'src/engine/core-modules/twenty-config/interfaces/config-var-cache-entry.interface'; import { DatabaseConfigDriverInterface } from 'src/engine/core-modules/twenty-config/interfaces/database-config-driver.interface'; import { @@ -22,14 +26,20 @@ import { convertConfigVarToAppType } from 'src/engine/core-modules/twenty-config import { convertConfigVarToStorageType } from 'src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util'; import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; +interface NegativeLookupCacheEntry { + value: boolean; + timestamp: number; + ttl: number; +} + @Injectable() export class DatabaseConfigDriver implements DatabaseConfigDriverInterface, OnModuleDestroy { - private readonly valueCache: Map>; + private readonly valueCache: Map>; private readonly negativeLookupCache: Map< - string, - ConfigVarCacheEntry + ConfigKey, + NegativeLookupCacheEntry >; private initializationState = InitializationState.NOT_INITIALIZED; private initializationPromise: Promise | null = null; @@ -47,9 +57,6 @@ export class DatabaseConfigDriver this.startCacheScavenging(); } - /** - * Initialize the database driver by loading all config variables from DB - */ async initialize(): Promise { if (this.initializationPromise) { return this.initializationPromise; @@ -75,7 +82,6 @@ export class DatabaseConfigDriver } get(key: T): ConfigVariables[T] { - // 1. If in not-initialized or initializing state, use environment variable if (this.initializationState !== InitializationState.INITIALIZED) { this.logger.debug( `[Cache:${key}] Using env due to initialization state: ${this.initializationState}`, @@ -84,15 +90,13 @@ export class DatabaseConfigDriver return this.environmentDriver.get(key); } - // 2. Check if this is an environment-only variable if (isEnvOnlyConfigVar(key)) { this.logger.debug(`[Cache:${key}] Using env due to isEnvOnly flag`); return this.environmentDriver.get(key); } - // 3. Check negative lookup cache first (quick rejection) - const negativeCacheEntry = this.negativeLookupCache.get(key as string); + const negativeCacheEntry = this.negativeLookupCache.get(key as ConfigKey); if (negativeCacheEntry && !this.isCacheExpired(negativeCacheEntry)) { this.logger.debug(`🔴 [Cache:${key}] Negative cache hit - using env`); @@ -100,26 +104,22 @@ export class DatabaseConfigDriver return this.environmentDriver.get(key); } - // 4. Check value cache - const valueCacheEntry = this.valueCache.get(key as string); + const valueCacheEntry = this.valueCache.get(key as ConfigKey); if (valueCacheEntry && !this.isCacheExpired(valueCacheEntry)) { this.logger.debug( `🟢 [Cache:${key}] Positive cache hit - using cached value`, ); - // Convert the value to the appropriate type before returning return convertConfigVarToAppType(valueCacheEntry.value, key); } - // 5. Schedule background refresh - Cache miss this.logger.debug(`🟡 [Cache:${key}] Cache miss - scheduling refresh`); this.scheduleRefresh(key).catch((error) => { this.logger.error(`Failed to refresh config for ${key as string}`, error); }); - // 6. Return environment value immediately return this.environmentDriver.get(key); } @@ -130,23 +130,22 @@ export class DatabaseConfigDriver const result = await this.queryDatabase(key as string); if (result !== undefined) { - // Store the original value in the cache - this.valueCache.set(key as string, { + this.valueCache.set(key as ConfigKey, { value: result, timestamp: Date.now(), ttl: CONFIG_VARIABLES_CACHE_TTL, }); - this.negativeLookupCache.delete(key as string); + this.negativeLookupCache.delete(key as ConfigKey); this.logger.debug( `✅ [Cache:${key}] Updated positive cache with value from DB`, ); } else { - this.negativeLookupCache.set(key as string, { + this.negativeLookupCache.set(key as ConfigKey, { value: true, timestamp: Date.now(), ttl: CONFIG_VARIABLES_CACHE_TTL, }); - this.valueCache.delete(key as string); + this.valueCache.delete(key as ConfigKey); this.logger.debug( `❌ [Cache:${key}] Updated negative cache (not found in DB)`, ); @@ -161,10 +160,6 @@ export class DatabaseConfigDriver value: ConfigVariables[T], ): Promise { try { - // Convert the value to JSON storage format using the converter - // TODO: same here. More clean way would be to have a non json table for config vars, which means a new table just for the config vars - // TODO: and then we can just store the value as is, without converting to json - // or can we store the type of value win the json? const processedValue = convertConfigVarToStorageType(value); this.logger.debug(`🔵 [Cache:${key}] Updating in database`, { @@ -173,7 +168,6 @@ export class DatabaseConfigDriver isArray: Array.isArray(processedValue), }); - // Check if the record exists const existingRecord = await this.keyValuePairRepository.findOne({ where: { key: key as string, @@ -184,7 +178,6 @@ export class DatabaseConfigDriver }); if (existingRecord) { - // Update existing record this.logger.debug(`🔄 [Cache:${key}] Updating existing record in DB`); await this.keyValuePairRepository.update( { @@ -195,7 +188,6 @@ export class DatabaseConfigDriver }, ); } else { - // Insert new record this.logger.debug(`🔄 [Cache:${key}] Inserting new record in DB`); await this.keyValuePairRepository.insert({ key: key as string, @@ -206,13 +198,12 @@ export class DatabaseConfigDriver }); } - // Update cache immediately with the properly converted value - this.valueCache.set(key as string, { - value: processedValue, + this.valueCache.set(key, { + value, timestamp: Date.now(), ttl: CONFIG_VARIABLES_CACHE_TTL, }); - this.negativeLookupCache.delete(key as string); + this.negativeLookupCache.delete(key); this.logger.debug(`✅ [Cache:${key}] Updated cache with new value`); } catch (error) { this.logger.error(`Failed to update config for ${key as string}`, error); @@ -221,8 +212,8 @@ export class DatabaseConfigDriver } clearCache(key: keyof ConfigVariables): void { - this.valueCache.delete(key as string); - this.negativeLookupCache.delete(key as string); + this.valueCache.delete(key as ConfigKey); + this.negativeLookupCache.delete(key as ConfigKey); } clearAllCache(): void { @@ -237,7 +228,6 @@ export class DatabaseConfigDriver } private async scheduleRefresh(key: keyof ConfigVariables): Promise { - // Log when a refresh is scheduled but not yet executed this.logger.debug(`🕒 [Cache:${key}] Scheduling background refresh`); setImmediate(() => { @@ -295,8 +285,11 @@ export class DatabaseConfigDriver for (const configVar of configVars) { if (configVar.value !== null) { - this.valueCache.set(configVar.key, { - value: configVar.value, + const key = configVar.key as keyof ConfigVariables; + const value = convertConfigVarToAppType(configVar.value, key); + + this.valueCache.set(key, { + value, timestamp: now, ttl: CONFIG_VARIABLES_CACHE_TTL, }); @@ -312,7 +305,9 @@ export class DatabaseConfigDriver } } - private async queryDatabase(key: string): Promise { + private async queryDatabase( + key: string, + ): Promise | undefined> { try { const result = await this.keyValuePairRepository.findOne({ where: { @@ -323,7 +318,11 @@ export class DatabaseConfigDriver }, }); - return result?.value; + if (!result?.value) { + return undefined; + } + + return convertConfigVarToAppType(result.value, key as ConfigKey); } catch (error) { this.logger.error(`Failed to query database for ${key}`, error); @@ -331,7 +330,9 @@ export class DatabaseConfigDriver } } - private isCacheExpired(entry: ConfigVarCacheEntry): boolean { + private isCacheExpired( + entry: ConfigVarCacheEntry | NegativeLookupCacheEntry, + ): boolean { return Date.now() - entry.timestamp > entry.ttl; } @@ -360,8 +361,6 @@ export class DatabaseConfigDriver } } - // THis is some thing I added later not in the design doc - // makes sense tho, because we don't want to have too many cache entries if (this.valueCache.size > CONFIG_VARIABLES_CACHE_MAX_ENTRIES) { const entriesToDelete = this.valueCache.size - CONFIG_VARIABLES_CACHE_MAX_ENTRIES; @@ -393,21 +392,22 @@ export class DatabaseConfigDriver } } - getFromValueCache(key: string): any { - const entry = this.valueCache.get(key); + getFromValueCache( + key: T, + ): ConfigVariables[T] | undefined { + const entry = this.valueCache.get(key as ConfigKey); - if (entry && !this.isCacheExpired(entry)) { - // again same type conversion -- I don't like it - return convertConfigVarToAppType( - entry.value, - key as keyof ConfigVariables, - ); + if (!entry) return undefined; + + if (Date.now() - entry.timestamp > entry.ttl) { + this.valueCache.delete(key as ConfigKey); + + return undefined; } - return undefined; + return entry.value as ConfigVariables[T]; } - // Helper method to get cache information for debugging getCacheInfo(): { positiveEntries: number; negativeEntries: number; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/interfaces/config-var-cache-entry.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/interfaces/config-var-cache-entry.interface.ts index 9ee4f990b25b..92ca3492ac88 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/interfaces/config-var-cache-entry.interface.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/interfaces/config-var-cache-entry.interface.ts @@ -1,5 +1,10 @@ -export interface ConfigVarCacheEntry { - value: T; +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; + +export type ConfigKey = keyof ConfigVariables; +export type ConfigValue = ConfigVariables[T]; + +export interface ConfigVarCacheEntry { + value: ConfigValue; timestamp: number; ttl: number; } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts index dd77afb4c981..59e19aee34be 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts @@ -6,6 +6,8 @@ import { } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; +import { ConfigKey } from 'src/engine/core-modules/twenty-config/interfaces/config-var-cache-entry.interface'; + import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; import { CONFIG_VARIABLES_MASKING_CONFIG } from 'src/engine/core-modules/twenty-config/constants/config-variables-masking-config'; import { ConfigVariablesMetadataOptions } from 'src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator'; @@ -30,12 +32,10 @@ export class TwentyConfigService private readonly databaseConfigDriver: DatabaseConfigDriver, private readonly environmentConfigDriver: EnvironmentConfigDriver, ) { - // Always start with environment driver during construction this.driver = this.environmentConfigDriver; const configVarInDb = this.configService.get('IS_CONFIG_VAR_IN_DB_ENABLED'); - // Handle both string and boolean values for IS_CONFIG_VAR_IN_DB_ENABLED this.isConfigVarInDbEnabled = configVarInDb === true; this.logger.log( @@ -44,8 +44,6 @@ export class TwentyConfigService } async onModuleInit() { - // During module init, only use the environment driver - // and mark as initialized if not using DB driver if (!this.isConfigVarInDbEnabled) { this.logger.log( 'Database configuration is disabled, using environment variables only', @@ -55,8 +53,6 @@ export class TwentyConfigService return; } - // If we're using DB driver, mark as not initialized yet - // Will be fully initialized in onApplicationBootstrap this.logger.log( 'Database configuration is enabled, will initialize after application bootstrap', ); @@ -71,23 +67,19 @@ export class TwentyConfigService this.logger.log('Initializing database driver for configuration'); this.initializationState = InitializationState.INITIALIZING; - // Try to initialize the database driver await this.databaseConfigDriver.initialize(); - // If initialization succeeds, switch to the database driver this.driver = this.databaseConfigDriver; this.initializationState = InitializationState.INITIALIZED; this.logger.log('Database driver initialized successfully'); this.logger.log(`Active driver: DatabaseDriver`); } catch (error) { - // If initialization fails for any reason, log the error and keep using the environment driver this.logger.error( 'Failed to initialize database driver, falling back to environment variables', error, ); this.initializationState = InitializationState.FAILED; - // Ensure we're still using the environment driver this.driver = this.environmentConfigDriver; this.logger.log(`Active driver: EnvironmentDriver (fallback)`); } @@ -183,8 +175,9 @@ export class TwentyConfigService let source = 'ENVIRONMENT'; if (isUsingDatabaseDriver && !envMetadata.isEnvOnly) { - const valueCacheEntry = - this.databaseConfigDriver.getFromValueCache(key); + const valueCacheEntry = this.databaseConfigDriver.getFromValueCache( + key as ConfigKey, + ); if (valueCacheEntry) { source = 'DATABASE'; @@ -235,7 +228,6 @@ export class TwentyConfigService } } - // Get cache information for debugging getCacheInfo(): { usingDatabaseDriver: boolean; initializationState: string; From 7c072cbf67259a12b761cb20dcb92ee0231c90bd Mon Sep 17 00:00:00 2001 From: ehconitin Date: Sat, 12 Apr 2025 17:38:37 +0530 Subject: [PATCH 10/70] refactor --- .../cache/config-cache.service.ts | 173 +++++++ .../config-cache-entry.interface.ts | 16 + .../drivers/database-config.driver.ts | 424 ++++-------------- .../storage/config-storage.service.ts | 154 +++++++ .../interfaces/config-storage.interface.ts | 16 + .../twenty-config/twenty-config.module.ts | 6 +- .../twenty-config/twenty-config.service.ts | 10 +- ...convert-config-var-to-storage-type.util.ts | 31 +- 8 files changed, 481 insertions(+), 349 deletions(-) create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/storage/interfaces/config-storage.interface.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts new file mode 100644 index 000000000000..2ec7ae325853 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts @@ -0,0 +1,173 @@ +import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; + +import { CONFIG_VARIABLES_CACHE_MAX_ENTRIES } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-max-entries'; +import { CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval'; +import { CONFIG_VARIABLES_CACHE_TTL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl'; + +import { + ConfigCacheEntry, + ConfigKey, + ConfigNegativeCacheEntry, + ConfigValue, +} from './interfaces/config-cache-entry.interface'; + +@Injectable() +export class ConfigCacheService implements OnModuleDestroy { + private readonly valueCache: Map>; + private readonly negativeLookupCache: Map< + ConfigKey, + ConfigNegativeCacheEntry + >; + private cacheScavengeInterval: NodeJS.Timeout; + private readonly logger = new Logger(ConfigCacheService.name); + + constructor() { + this.valueCache = new Map(); + this.negativeLookupCache = new Map(); + this.startCacheScavenging(); + } + + get(key: T): ConfigValue | undefined { + const entry = this.valueCache.get(key); + + if (entry && !this.isCacheExpired(entry)) { + this.logger.debug(`🟢 [Cache:${key}] Positive cache hit`); + + return entry.value as ConfigValue; + } + + return undefined; + } + + getNegativeLookup(key: ConfigKey): boolean { + const entry = this.negativeLookupCache.get(key); + + if (entry && !this.isCacheExpired(entry)) { + this.logger.debug(`🔴 [Cache:${key}] Negative cache hit`); + + return true; + } + + return false; + } + + set(key: T, value: ConfigValue): void { + this.valueCache.set(key, { + value, + timestamp: Date.now(), + ttl: CONFIG_VARIABLES_CACHE_TTL, + }); + this.negativeLookupCache.delete(key); + this.logger.debug(`✅ [Cache:${key}] Updated positive cache`); + } + + setNegativeLookup(key: ConfigKey): void { + this.negativeLookupCache.set(key, { + value: true, + timestamp: Date.now(), + ttl: CONFIG_VARIABLES_CACHE_TTL, + }); + this.valueCache.delete(key); + this.logger.debug(`❌ [Cache:${key}] Updated negative cache`); + } + + clear(key: ConfigKey): void { + this.valueCache.delete(key); + this.negativeLookupCache.delete(key); + this.logger.debug(`🧹 [Cache:${key}] Cleared cache entries`); + } + + clearAll(): void { + this.valueCache.clear(); + this.negativeLookupCache.clear(); + this.logger.debug('🧹 Cleared all cache entries'); + } + + getCacheInfo(): { + positiveEntries: number; + negativeEntries: number; + cacheKeys: string[]; + } { + const validPositiveEntries = Array.from(this.valueCache.entries()).filter( + ([_, entry]) => !this.isCacheExpired(entry), + ); + + const validNegativeEntries = Array.from( + this.negativeLookupCache.entries(), + ).filter(([_, entry]) => !this.isCacheExpired(entry)); + + return { + positiveEntries: validPositiveEntries.length, + negativeEntries: validNegativeEntries.length, + cacheKeys: validPositiveEntries.map(([key]) => key), + }; + } + + onModuleDestroy() { + if (this.cacheScavengeInterval) { + clearInterval(this.cacheScavengeInterval); + } + } + + private isCacheExpired( + entry: ConfigCacheEntry | ConfigNegativeCacheEntry, + ): boolean { + return Date.now() - entry.timestamp > entry.ttl; + } + + private startCacheScavenging(): void { + this.cacheScavengeInterval = setInterval(() => { + this.scavengeCache(); + }, CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL); + } + + private scavengeCache(): void { + const now = Date.now(); + let expiredCount = 0; + let sizeLimitedCount = 0; + + for (const [key, entry] of this.valueCache.entries()) { + if (now - entry.timestamp > entry.ttl) { + this.valueCache.delete(key); + expiredCount++; + } + } + + for (const [key, entry] of this.negativeLookupCache.entries()) { + if (now - entry.timestamp > entry.ttl) { + this.negativeLookupCache.delete(key); + expiredCount++; + } + } + + if (this.valueCache.size > CONFIG_VARIABLES_CACHE_MAX_ENTRIES) { + const entriesToDelete = + this.valueCache.size - CONFIG_VARIABLES_CACHE_MAX_ENTRIES; + const keysToDelete = Array.from(this.valueCache.keys()).slice( + 0, + entriesToDelete, + ); + + keysToDelete.forEach((key) => this.valueCache.delete(key)); + sizeLimitedCount += entriesToDelete; + } + + if (this.negativeLookupCache.size > CONFIG_VARIABLES_CACHE_MAX_ENTRIES) { + const entriesToDelete = + this.negativeLookupCache.size - CONFIG_VARIABLES_CACHE_MAX_ENTRIES; + const keysToDelete = Array.from(this.negativeLookupCache.keys()).slice( + 0, + entriesToDelete, + ); + + keysToDelete.forEach((key) => this.negativeLookupCache.delete(key)); + sizeLimitedCount += entriesToDelete; + } + + if (expiredCount > 0 || sizeLimitedCount > 0) { + this.logger.debug( + `🧹 Cache scavenging completed: ${expiredCount} expired entries removed, ${sizeLimitedCount} entries removed due to size limits`, + ); + } + } +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts new file mode 100644 index 000000000000..2aead2390d1f --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts @@ -0,0 +1,16 @@ +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; + +export type ConfigKey = keyof ConfigVariables; +export type ConfigValue = ConfigVariables[T]; + +export interface ConfigCacheEntry { + value: ConfigValue; + timestamp: number; + ttl: number; +} + +export interface ConfigNegativeCacheEntry { + value: boolean; + timestamp: number; + ttl: number; +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index 46764aa22905..b0146c6a9681 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -1,247 +1,170 @@ import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; -import { InjectRepository } from '@nestjs/typeorm'; -import { IsNull, Repository } from 'typeorm'; - -import { - ConfigKey, - ConfigValue, - ConfigVarCacheEntry, -} from 'src/engine/core-modules/twenty-config/interfaces/config-var-cache-entry.interface'; import { DatabaseConfigDriverInterface } from 'src/engine/core-modules/twenty-config/interfaces/database-config-driver.interface'; -import { - KeyValuePair, - KeyValuePairType, -} from 'src/engine/core-modules/key-value-pair/key-value-pair.entity'; +import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; -import { CONFIG_VARIABLES_CACHE_MAX_ENTRIES } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-max-entries'; -import { CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval'; -import { CONFIG_VARIABLES_CACHE_TTL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl'; import { DATABASE_CONFIG_DRIVER_INITIAL_RETRY_DELAY } from 'src/engine/core-modules/twenty-config/constants/database-config-driver-initial-retry-delay'; import { DATABASE_CONFIG_DRIVER_INITIALIZATION_MAX_RETRIES } from 'src/engine/core-modules/twenty-config/constants/database-config-driver-initialization-max-retries'; -import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; import { InitializationState } from 'src/engine/core-modules/twenty-config/enums/initialization-state.enum'; -import { convertConfigVarToAppType } from 'src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util'; -import { convertConfigVarToStorageType } from 'src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util'; +import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; -interface NegativeLookupCacheEntry { - value: boolean; - timestamp: number; - ttl: number; -} +import { EnvironmentConfigDriver } from './environment-config.driver'; @Injectable() export class DatabaseConfigDriver implements DatabaseConfigDriverInterface, OnModuleDestroy { - private readonly valueCache: Map>; - private readonly negativeLookupCache: Map< - ConfigKey, - NegativeLookupCacheEntry - >; private initializationState = InitializationState.NOT_INITIALIZED; private initializationPromise: Promise | null = null; private retryAttempts = 0; - private cacheScavengeInterval: NodeJS.Timeout; + private retryTimer?: NodeJS.Timeout; private readonly logger = new Logger(DatabaseConfigDriver.name); constructor( + private readonly configCache: ConfigCacheService, + private readonly configStorage: ConfigStorageService, private readonly environmentDriver: EnvironmentConfigDriver, - @InjectRepository(KeyValuePair, 'core') - private readonly keyValuePairRepository: Repository, - ) { - this.valueCache = new Map(); - this.negativeLookupCache = new Map(); - this.startCacheScavenging(); - } + ) {} async initialize(): Promise { if (this.initializationPromise) { return this.initializationPromise; } - this.initializationPromise = new Promise((resolve) => { - this.loadAllConfigVarsFromDb() - .then(() => { - this.initializationState = InitializationState.INITIALIZED; - resolve(); - }) - .catch((error) => { - this.logger.error('Failed to initialize database driver', error); - this.initializationState = InitializationState.FAILED; - this.scheduleRetry(); - // We still resolve the promise to prevent bootstrap from failing - // The driver will fallback to environment variables when in FAILED state - resolve(); - }); - }); + this.initializationPromise = this.doInitialize(); return this.initializationPromise; } + private async doInitialize(): Promise { + try { + await this.loadAllConfigVarsFromDb(); + this.initializationState = InitializationState.INITIALIZED; + } catch (error) { + this.logger.error('Failed to initialize database driver', error); + this.initializationState = InitializationState.FAILED; + this.scheduleRetry(); + } + } + get(key: T): ConfigVariables[T] { - if (this.initializationState !== InitializationState.INITIALIZED) { + if (this.shouldUseEnvironment(key)) { this.logger.debug( - `[Cache:${key}] Using env due to initialization state: ${this.initializationState}`, + `[Config:${key}] Using env due to ${ + this.initializationState !== InitializationState.INITIALIZED + ? 'initialization state' + : 'isEnvOnly flag' + }`, ); return this.environmentDriver.get(key); } - if (isEnvOnlyConfigVar(key)) { - this.logger.debug(`[Cache:${key}] Using env due to isEnvOnly flag`); + const cachedValue = this.configCache.get(key); - return this.environmentDriver.get(key); + if (cachedValue !== undefined) { + return cachedValue; } - const negativeCacheEntry = this.negativeLookupCache.get(key as ConfigKey); - - if (negativeCacheEntry && !this.isCacheExpired(negativeCacheEntry)) { - this.logger.debug(`🔴 [Cache:${key}] Negative cache hit - using env`); - + if (this.configCache.getNegativeLookup(key)) { return this.environmentDriver.get(key); } - const valueCacheEntry = this.valueCache.get(key as ConfigKey); + this.scheduleRefresh(key); - if (valueCacheEntry && !this.isCacheExpired(valueCacheEntry)) { - this.logger.debug( - `🟢 [Cache:${key}] Positive cache hit - using cached value`, - ); + return this.environmentDriver.get(key); + } - return convertConfigVarToAppType(valueCacheEntry.value, key); + async update( + key: T, + value: ConfigVariables[T], + ): Promise { + if (this.shouldUseEnvironment(key)) { + throw new Error( + `Cannot update environment-only variable: ${key as string}`, + ); } - this.logger.debug(`🟡 [Cache:${key}] Cache miss - scheduling refresh`); + await this.configStorage.set(key, value); + this.configCache.set(key, value); + } - this.scheduleRefresh(key).catch((error) => { - this.logger.error(`Failed to refresh config for ${key as string}`, error); - }); + clearCache(key: keyof ConfigVariables): void { + this.configCache.clear(key); + } - return this.environmentDriver.get(key); + clearAllCache(): void { + this.configCache.clearAll(); } async refreshConfig(key: keyof ConfigVariables): Promise { try { - this.logger.debug(`🔄 [Cache:${key}] Refreshing from database`); - - const result = await this.queryDatabase(key as string); - - if (result !== undefined) { - this.valueCache.set(key as ConfigKey, { - value: result, - timestamp: Date.now(), - ttl: CONFIG_VARIABLES_CACHE_TTL, - }); - this.negativeLookupCache.delete(key as ConfigKey); - this.logger.debug( - `✅ [Cache:${key}] Updated positive cache with value from DB`, - ); + const value = await this.configStorage.get(key); + + if (value !== undefined) { + this.configCache.set(key, value); } else { - this.negativeLookupCache.set(key as ConfigKey, { - value: true, - timestamp: Date.now(), - ttl: CONFIG_VARIABLES_CACHE_TTL, - }); - this.valueCache.delete(key as ConfigKey); - this.logger.debug( - `❌ [Cache:${key}] Updated negative cache (not found in DB)`, - ); + this.configCache.setNegativeLookup(key); } } catch (error) { this.logger.error(`Failed to refresh config for ${key as string}`, error); + this.configCache.setNegativeLookup(key); } } - async update( - key: T, - value: ConfigVariables[T], - ): Promise { - try { - const processedValue = convertConfigVarToStorageType(value); + getCacheInfo(): { + positiveEntries: number; + negativeEntries: number; + cacheKeys: string[]; + } { + return this.configCache.getCacheInfo(); + } - this.logger.debug(`🔵 [Cache:${key}] Updating in database`, { - originalType: typeof value, - processedType: typeof processedValue, - isArray: Array.isArray(processedValue), - }); + private shouldUseEnvironment(key: keyof ConfigVariables): boolean { + return ( + this.initializationState !== InitializationState.INITIALIZED || + isEnvOnlyConfigVar(key) + ); + } - const existingRecord = await this.keyValuePairRepository.findOne({ - where: { - key: key as string, - userId: IsNull(), - workspaceId: IsNull(), - type: KeyValuePairType.CONFIG_VARIABLE, - }, - }); + private async loadAllConfigVarsFromDb(): Promise { + try { + const configVars = await this.configStorage.loadAll(); - if (existingRecord) { - this.logger.debug(`🔄 [Cache:${key}] Updating existing record in DB`); - await this.keyValuePairRepository.update( - { - id: existingRecord.id, - }, - { - value: processedValue, - }, - ); - } else { - this.logger.debug(`🔄 [Cache:${key}] Inserting new record in DB`); - await this.keyValuePairRepository.insert({ - key: key as string, - value: processedValue, - userId: null, - workspaceId: null, - type: KeyValuePairType.CONFIG_VARIABLE, - }); + for (const [key, value] of configVars.entries()) { + this.configCache.set(key, value); } - this.valueCache.set(key, { - value, - timestamp: Date.now(), - ttl: CONFIG_VARIABLES_CACHE_TTL, - }); - this.negativeLookupCache.delete(key); - this.logger.debug(`✅ [Cache:${key}] Updated cache with new value`); + this.logger.log( + `Loaded ${configVars.size} config variables from database`, + ); } catch (error) { - this.logger.error(`Failed to update config for ${key as string}`, error); + this.logger.error('Failed to load config variables from database', error); throw error; } } - clearCache(key: keyof ConfigVariables): void { - this.valueCache.delete(key as ConfigKey); - this.negativeLookupCache.delete(key as ConfigKey); - } - - clearAllCache(): void { - this.valueCache.clear(); - this.negativeLookupCache.clear(); - } + private async scheduleRefresh(key: keyof ConfigVariables): Promise { + if (this.initializationState !== InitializationState.INITIALIZED) { + this.logger.debug( + `[Config:${key}] Skipping refresh due to initialization state`, + ); - onModuleDestroy() { - if (this.cacheScavengeInterval) { - clearInterval(this.cacheScavengeInterval); + return; } - } - private async scheduleRefresh(key: keyof ConfigVariables): Promise { - this.logger.debug(`🕒 [Cache:${key}] Scheduling background refresh`); - - setImmediate(() => { - this.logger.debug(`⏳ [Cache:${key}] Executing background refresh`); + this.logger.debug(`🕒 [Config:${key}] Scheduling background refresh`); - this.refreshConfig(key).catch((error) => { - this.logger.error( - `Failed to refresh config for ${key as string}`, - error, - ); + try { + await Promise.resolve().then(async () => { + this.logger.debug(`⏳ [Config:${key}] Executing background refresh`); + await this.refreshConfig(key); }); - }); - - return Promise.resolve(); + } catch (error) { + this.logger.error(`Failed to refresh config for ${key as string}`, error); + } } private scheduleRetry(): void { @@ -259,7 +182,11 @@ export class DatabaseConfigDriver this.retryAttempts++; - setTimeout(() => { + if (this.retryTimer) { + clearTimeout(this.retryTimer); + } + + this.retryTimer = setTimeout(() => { this.initializationPromise = null; this.initialize().catch((error) => { this.logger.error('Retry initialization failed', error); @@ -267,164 +194,9 @@ export class DatabaseConfigDriver }, delay); } - private async loadAllConfigVarsFromDb(): Promise { - try { - const configVars = await this.keyValuePairRepository.find({ - where: { - type: KeyValuePairType.CONFIG_VARIABLE, - userId: IsNull(), - workspaceId: IsNull(), - }, - }); - - if (!configVars.length) { - return; - } - - const now = Date.now(); - - for (const configVar of configVars) { - if (configVar.value !== null) { - const key = configVar.key as keyof ConfigVariables; - const value = convertConfigVarToAppType(configVar.value, key); - - this.valueCache.set(key, { - value, - timestamp: now, - ttl: CONFIG_VARIABLES_CACHE_TTL, - }); - } - } - - this.logger.log( - `Loaded ${configVars.length} config variables from database`, - ); - } catch (error) { - this.logger.error('Failed to load config variables from database', error); - throw error; - } - } - - private async queryDatabase( - key: string, - ): Promise | undefined> { - try { - const result = await this.keyValuePairRepository.findOne({ - where: { - type: KeyValuePairType.CONFIG_VARIABLE, - key, - userId: IsNull(), - workspaceId: IsNull(), - }, - }); - - if (!result?.value) { - return undefined; - } - - return convertConfigVarToAppType(result.value, key as ConfigKey); - } catch (error) { - this.logger.error(`Failed to query database for ${key}`, error); - - return undefined; - } - } - - private isCacheExpired( - entry: ConfigVarCacheEntry | NegativeLookupCacheEntry, - ): boolean { - return Date.now() - entry.timestamp > entry.ttl; - } - - private startCacheScavenging(): void { - this.cacheScavengeInterval = setInterval(() => { - this.scavengeCache(); - }, CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL); - } - - private scavengeCache(): void { - const now = Date.now(); - let expiredCount = 0; - let sizeLimitedCount = 0; - - for (const [key, entry] of this.valueCache.entries()) { - if (now - entry.timestamp > entry.ttl) { - this.valueCache.delete(key); - expiredCount++; - } - } - - for (const [key, entry] of this.negativeLookupCache.entries()) { - if (now - entry.timestamp > entry.ttl) { - this.negativeLookupCache.delete(key); - expiredCount++; - } - } - - if (this.valueCache.size > CONFIG_VARIABLES_CACHE_MAX_ENTRIES) { - const entriesToDelete = - this.valueCache.size - CONFIG_VARIABLES_CACHE_MAX_ENTRIES; - const keysToDelete = Array.from(this.valueCache.keys()).slice( - 0, - entriesToDelete, - ); - - keysToDelete.forEach((key) => this.valueCache.delete(key)); - sizeLimitedCount += entriesToDelete; - } - - if (this.negativeLookupCache.size > CONFIG_VARIABLES_CACHE_MAX_ENTRIES) { - const entriesToDelete = - this.negativeLookupCache.size - CONFIG_VARIABLES_CACHE_MAX_ENTRIES; - const keysToDelete = Array.from(this.negativeLookupCache.keys()).slice( - 0, - entriesToDelete, - ); - - keysToDelete.forEach((key) => this.negativeLookupCache.delete(key)); - sizeLimitedCount += entriesToDelete; - } - - if (expiredCount > 0 || sizeLimitedCount > 0) { - this.logger.debug( - `Cache scavenging completed: ${expiredCount} expired entries removed, ${sizeLimitedCount} entries removed due to size limits`, - ); - } - } - - getFromValueCache( - key: T, - ): ConfigVariables[T] | undefined { - const entry = this.valueCache.get(key as ConfigKey); - - if (!entry) return undefined; - - if (Date.now() - entry.timestamp > entry.ttl) { - this.valueCache.delete(key as ConfigKey); - - return undefined; + onModuleDestroy() { + if (this.retryTimer) { + clearTimeout(this.retryTimer); } - - return entry.value as ConfigVariables[T]; - } - - getCacheInfo(): { - positiveEntries: number; - negativeEntries: number; - cacheKeys: string[]; - } { - const validPositiveEntries = Array.from(this.valueCache.entries()).filter( - ([_, entry]) => !this.isCacheExpired(entry), - ); - - const validNegativeEntries = Array.from( - this.negativeLookupCache.entries(), - ).filter(([_, entry]) => !this.isCacheExpired(entry)); - - return { - positiveEntries: validPositiveEntries.length, - negativeEntries: validNegativeEntries.length, - cacheKeys: validPositiveEntries.map(([key]) => key), - }; } } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts new file mode 100644 index 000000000000..1ebb2d7c407f --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts @@ -0,0 +1,154 @@ +import { Injectable, Logger } from '@nestjs/common'; +import { InjectRepository } from '@nestjs/typeorm'; + +import { IsNull, Repository } from 'typeorm'; + +import { + KeyValuePair, + KeyValuePairType, +} from 'src/engine/core-modules/key-value-pair/key-value-pair.entity'; +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { convertConfigVarToAppType } from 'src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util'; +import { convertConfigVarToStorageType } from 'src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util'; + +import { ConfigStorageInterface } from './interfaces/config-storage.interface'; + +@Injectable() +export class ConfigStorageService implements ConfigStorageInterface { + private readonly logger = new Logger(ConfigStorageService.name); + + constructor( + @InjectRepository(KeyValuePair, 'core') + private readonly keyValuePairRepository: Repository, + ) {} + + async get( + key: T, + ): Promise { + try { + const result = await this.keyValuePairRepository.findOne({ + where: { + type: KeyValuePairType.CONFIG_VARIABLE, + key: key as string, + userId: IsNull(), + workspaceId: IsNull(), + }, + }); + + if (!result?.value) { + return undefined; + } + + try { + return convertConfigVarToAppType(result.value, key); + } catch (error) { + this.logger.error( + `Failed to convert value to app type for key ${key as string}`, + error, + ); + throw error; + } + } catch (error) { + this.logger.error(`Failed to get config for ${key as string}`, error); + throw error; + } + } + + async set( + key: T, + value: ConfigVariables[T], + ): Promise { + try { + let processedValue; + + try { + processedValue = convertConfigVarToStorageType(value); + } catch (error) { + this.logger.error( + `Failed to convert value to storage type for key ${key as string}`, + error, + ); + throw error; + } + + const existingRecord = await this.keyValuePairRepository.findOne({ + where: { + key: key as string, + userId: IsNull(), + workspaceId: IsNull(), + type: KeyValuePairType.CONFIG_VARIABLE, + }, + }); + + if (existingRecord) { + await this.keyValuePairRepository.update( + { id: existingRecord.id }, + { value: processedValue }, + ); + } else { + await this.keyValuePairRepository.insert({ + key: key as string, + value: processedValue, + userId: null, + workspaceId: null, + type: KeyValuePairType.CONFIG_VARIABLE, + }); + } + } catch (error) { + this.logger.error(`Failed to set config for ${key as string}`, error); + throw error; + } + } + + async delete(key: T): Promise { + try { + await this.keyValuePairRepository.delete({ + type: KeyValuePairType.CONFIG_VARIABLE, + key: key as string, + userId: IsNull(), + workspaceId: IsNull(), + }); + } catch (error) { + this.logger.error(`Failed to delete config for ${key as string}`, error); + throw error; + } + } + + async loadAll(): Promise> { + try { + const configVars = await this.keyValuePairRepository.find({ + where: { + type: KeyValuePairType.CONFIG_VARIABLE, + userId: IsNull(), + workspaceId: IsNull(), + }, + }); + + const result = new Map(); + + for (const configVar of configVars) { + if (configVar.value !== null) { + const key = configVar.key as keyof ConfigVariables; + + try { + const value = convertConfigVarToAppType(configVar.value, key); + + result.set(key, value); + } catch (error) { + this.logger.error( + `Failed to convert value to app type for key ${key as string}`, + error, + ); + // Skip this value but continue processing others + continue; + } + } + } + + return result; + } catch (error) { + this.logger.error('Failed to load all config variables', error); + throw error; + } + } +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/interfaces/config-storage.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/interfaces/config-storage.interface.ts new file mode 100644 index 000000000000..23238791c102 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/interfaces/config-storage.interface.ts @@ -0,0 +1,16 @@ +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; + +export interface ConfigStorageInterface { + get( + key: T, + ): Promise; + + set( + key: T, + value: ConfigVariables[T], + ): Promise; + + delete(key: T): Promise; + + loadAll(): Promise>; +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts index 1ae686f6073e..b15d4b4a361e 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts @@ -3,9 +3,11 @@ import { ConfigModule } from '@nestjs/config'; import { TypeOrmModule } from '@nestjs/typeorm'; import { KeyValuePair } from 'src/engine/core-modules/key-value-pair/key-value-pair.entity'; +import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; import { validate } from 'src/engine/core-modules/twenty-config/config-variables'; import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; +import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; import { ConfigurableModuleClass } from 'src/engine/core-modules/twenty-config/twenty-config.module-definition'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; @@ -24,7 +26,9 @@ import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twent TwentyConfigService, EnvironmentConfigDriver, DatabaseConfigDriver, + ConfigCacheService, + ConfigStorageService, ], - exports: [TwentyConfigService, EnvironmentConfigDriver, DatabaseConfigDriver], + exports: [TwentyConfigService], }) export class TwentyConfigModule extends ConfigurableModuleClass {} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts index 59e19aee34be..058eb8df122e 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts @@ -6,8 +6,6 @@ import { } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; -import { ConfigKey } from 'src/engine/core-modules/twenty-config/interfaces/config-var-cache-entry.interface'; - import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; import { CONFIG_VARIABLES_MASKING_CONFIG } from 'src/engine/core-modules/twenty-config/constants/config-variables-masking-config'; import { ConfigVariablesMetadataOptions } from 'src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator'; @@ -175,13 +173,11 @@ export class TwentyConfigService let source = 'ENVIRONMENT'; if (isUsingDatabaseDriver && !envMetadata.isEnvOnly) { - const valueCacheEntry = this.databaseConfigDriver.getFromValueCache( - key as ConfigKey, - ); + const value = this.get(key as keyof ConfigVariables); - if (valueCacheEntry) { + if (value !== envVars[key as keyof ConfigVariables]) { source = 'DATABASE'; - } else if (value === envVars[key as keyof ConfigVariables]) { + } else { source = 'DEFAULT'; } } else if (value === envVars[key as keyof ConfigVariables]) { diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util.ts index c6143da87079..438a866121a9 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util.ts @@ -5,28 +5,29 @@ import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-va */ export const convertConfigVarToStorageType = ( appValue: ConfigVariables[T], -): JSON | undefined => { +): unknown => { if (appValue === undefined) { - return undefined; + return null; } - // Convert to the appropriate type for JSON storage - if (typeof appValue === 'boolean') { - return appValue as unknown as JSON; + if ( + typeof appValue === 'string' || + typeof appValue === 'number' || + typeof appValue === 'boolean' || + appValue === null + ) { + return appValue; } - if (typeof appValue === 'number') { - return appValue as unknown as JSON; + if (Array.isArray(appValue)) { + return appValue.map((item) => convertConfigVarToStorageType(item)); } - if (typeof appValue === 'string') { - return appValue as unknown as JSON; + if (typeof appValue === 'object') { + return JSON.parse(JSON.stringify(appValue)); } - // For arrays and objects, ensure they're stored as JSON-compatible values - if (Array.isArray(appValue) || typeof appValue === 'object') { - return appValue as unknown as JSON; - } - - return appValue as unknown as JSON; + throw new Error( + `Cannot convert value of type ${typeof appValue} to storage format`, + ); }; From 3a4c3176429352220070383f8c32f55319657e2f Mon Sep 17 00:00:00 2001 From: ehconitin Date: Tue, 15 Apr 2025 16:14:41 +0530 Subject: [PATCH 11/70] grep review --- .../drivers/database-config.driver.ts | 17 ++-- .../drivers/environment-config.driver.ts | 9 +- .../storage/config-storage.service.ts | 11 ++- .../interfaces/config-storage.interface.ts | 4 +- .../twenty-config.service.spec.ts | 82 ------------------- .../twenty-config/twenty-config.service.ts | 8 +- .../convert-config-var-to-app-type.util.ts | 19 ++++- ...convert-config-var-to-storage-type.util.ts | 6 +- 8 files changed, 53 insertions(+), 103 deletions(-) delete mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index b0146c6a9681..d1f46043dfb8 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -40,6 +40,7 @@ export class DatabaseConfigDriver private async doInitialize(): Promise { try { + this.initializationState = InitializationState.INITIALIZING; await this.loadAllConfigVarsFromDb(); this.initializationState = InitializationState.INITIALIZED; } catch (error) { @@ -158,9 +159,11 @@ export class DatabaseConfigDriver this.logger.debug(`🕒 [Config:${key}] Scheduling background refresh`); try { - await Promise.resolve().then(async () => { - this.logger.debug(`⏳ [Config:${key}] Executing background refresh`); - await this.refreshConfig(key); + await new Promise((resolve, reject) => { + setImmediate(async () => { + this.logger.debug(`⏳ [Config:${key}] Executing background refresh`); + await this.refreshConfig(key).then(resolve).catch(reject); + }); }); } catch (error) { this.logger.error(`Failed to refresh config for ${key as string}`, error); @@ -168,8 +171,10 @@ export class DatabaseConfigDriver } private scheduleRetry(): void { + this.retryAttempts++; + if ( - this.retryAttempts >= DATABASE_CONFIG_DRIVER_INITIALIZATION_MAX_RETRIES + this.retryAttempts > DATABASE_CONFIG_DRIVER_INITIALIZATION_MAX_RETRIES ) { this.logger.error('Max retry attempts reached, giving up initialization'); @@ -178,9 +183,7 @@ export class DatabaseConfigDriver const delay = DATABASE_CONFIG_DRIVER_INITIAL_RETRY_DELAY * - Math.pow(2, this.retryAttempts); - - this.retryAttempts++; + Math.pow(2, this.retryAttempts - 1); if (this.retryTimer) { clearTimeout(this.retryTimer); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/environment-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/environment-config.driver.ts index e18059e53c0e..894e20090885 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/environment-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/environment-config.driver.ts @@ -2,14 +2,19 @@ import { Injectable } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; + @Injectable() export class EnvironmentConfigDriver { - constructor(private readonly configService: ConfigService) {} + private readonly defaultConfigVariables: ConfigVariables; + + constructor(private readonly configService: ConfigService) { + this.defaultConfigVariables = new ConfigVariables(); + } get(key: T): ConfigVariables[T] { return this.configService.get( key, - new ConfigVariables()[key], + this.defaultConfigVariables[key], ); } } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts index 1ebb2d7c407f..a25ba0eb091e 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts @@ -35,7 +35,7 @@ export class ConfigStorageService implements ConfigStorageInterface { }, }); - if (!result?.value) { + if (result === null) { return undefined; } @@ -114,7 +114,9 @@ export class ConfigStorageService implements ConfigStorageInterface { } } - async loadAll(): Promise> { + async loadAll(): Promise< + Map + > { try { const configVars = await this.keyValuePairRepository.find({ where: { @@ -124,7 +126,10 @@ export class ConfigStorageService implements ConfigStorageInterface { }, }); - const result = new Map(); + const result = new Map< + keyof ConfigVariables, + ConfigVariables[keyof ConfigVariables] + >(); for (const configVar of configVars) { if (configVar.value !== null) { diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/interfaces/config-storage.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/interfaces/config-storage.interface.ts index 23238791c102..cab1fa21d633 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/interfaces/config-storage.interface.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/interfaces/config-storage.interface.ts @@ -12,5 +12,7 @@ export interface ConfigStorageInterface { delete(key: T): Promise; - loadAll(): Promise>; + loadAll(): Promise< + Map + >; } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts deleted file mode 100644 index 977c67c78189..000000000000 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts +++ /dev/null @@ -1,82 +0,0 @@ -import { ConfigService } from '@nestjs/config'; -import { Test, TestingModule } from '@nestjs/testing'; - -import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; -import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; - -describe('TwentyConfigService', () => { - let service: TwentyConfigService; - let configService: ConfigService; - - beforeEach(async () => { - const module: TestingModule = await Test.createTestingModule({ - providers: [ - TwentyConfigService, - { - provide: ConfigService, - useValue: { - get: jest.fn(), - }, - }, - ], - }).compile(); - - service = module.get(TwentyConfigService); - configService = module.get(ConfigService); - - Reflect.defineMetadata('config-variables', {}, ConfigVariables); - }); - - it('should be defined', () => { - expect(service).toBeDefined(); - }); - - describe('getAll()', () => { - it('should return empty object when no config variables are defined', () => { - const result = service.getAll(); - - expect(result).toEqual({}); - }); - - it('should return config variables with their metadata', () => { - const mockMetadata = { - TEST_VAR: { - title: 'Test Var', - description: 'Test Description', - }, - }; - - Reflect.defineMetadata('config-variables', mockMetadata, ConfigVariables); - - jest.spyOn(configService, 'get').mockReturnValue('test-value'); - - const result = service.getAll(); - - expect(result).toEqual({ - TEST_VAR: { - value: 'test-value', - metadata: mockMetadata.TEST_VAR, - }, - }); - }); - - it('should mask sensitive data according to masking config', () => { - const mockMetadata = { - APP_SECRET: { - title: 'App Secret', - description: 'Application secret key', - sensitive: true, - }, - }; - - Reflect.defineMetadata('config-variables', mockMetadata, ConfigVariables); - - jest.spyOn(configService, 'get').mockReturnValue('super-secret-value'); - - const result = service.getAll(); - - expect(result.APP_SECRET.value).not.toBe('super-secret-value'); - expect(result.APP_SECRET.value).toMatch(/^\*+[a-zA-Z0-9]{5}$/); - }); - }); -}); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts index 058eb8df122e..5eaba1a9f2bb 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts @@ -110,7 +110,7 @@ export class TwentyConfigService if (this.initializationState !== InitializationState.INITIALIZED) { throw new Error( - 'Environment service not initialized, cannot update configuration', + 'TwentyConfigService not initialized, cannot update configuration', ); } @@ -128,7 +128,7 @@ export class TwentyConfigService await this.databaseConfigDriver.update(key, value); } else { throw new Error( - 'Database driver not active, cannot update configuration', + 'Database driver not initialized, cannot update configuration', ); } } @@ -173,9 +173,9 @@ export class TwentyConfigService let source = 'ENVIRONMENT'; if (isUsingDatabaseDriver && !envMetadata.isEnvOnly) { - const value = this.get(key as keyof ConfigVariables); + const dbValue = this.get(key as keyof ConfigVariables); - if (value !== envVars[key as keyof ConfigVariables]) { + if (dbValue !== envVars[key as keyof ConfigVariables]) { source = 'DATABASE'; } else { source = 'DEFAULT'; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util.ts index f22e6f105764..57e119a1a49f 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util.ts @@ -20,11 +20,24 @@ export const convertConfigVarToAppType = ( } if (valueType === 'number' && typeof dbValue === 'string') { - return Number(dbValue) as unknown as ConfigVariables[T]; + const parsedNumber = parseFloat(dbValue); + + if (isNaN(parsedNumber)) { + throw new Error( + `Invalid number value for config variable ${key}: ${dbValue}`, + ); + } + + return parsedNumber as unknown as ConfigVariables[T]; } - // Handle arrays and other complex types - if (Array.isArray(defaultValue) && typeof dbValue === 'object') { + if (Array.isArray(defaultValue)) { + if (!Array.isArray(dbValue)) { + throw new Error( + `Expected array value for config variable ${key}, got ${typeof dbValue}`, + ); + } + return dbValue as ConfigVariables[T]; } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util.ts index 438a866121a9..05bbd3e41dff 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util.ts @@ -20,7 +20,11 @@ export const convertConfigVarToStorageType = ( } if (Array.isArray(appValue)) { - return appValue.map((item) => convertConfigVarToStorageType(item)); + return appValue.map((item) => + convertConfigVarToStorageType( + item as ConfigVariables[keyof ConfigVariables], + ), + ); } if (typeof appValue === 'object') { From 1eece63dd10f9ecc8fe129f72224af3e1f7cc8ef Mon Sep 17 00:00:00 2001 From: ehconitin Date: Tue, 15 Apr 2025 17:01:12 +0530 Subject: [PATCH 12/70] dont need this overenginnered code, max entries would always be same as the number of env vars in config variables file --- .../cache/config-cache.service.ts | 30 ++----------------- .../config-variables-cache-max-entries.ts | 1 - 2 files changed, 2 insertions(+), 29 deletions(-) delete mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-max-entries.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts index 2ec7ae325853..3d42be007824 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts @@ -1,6 +1,5 @@ import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; -import { CONFIG_VARIABLES_CACHE_MAX_ENTRIES } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-max-entries'; import { CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval'; import { CONFIG_VARIABLES_CACHE_TTL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl'; @@ -124,7 +123,6 @@ export class ConfigCacheService implements OnModuleDestroy { private scavengeCache(): void { const now = Date.now(); let expiredCount = 0; - let sizeLimitedCount = 0; for (const [key, entry] of this.valueCache.entries()) { if (now - entry.timestamp > entry.ttl) { @@ -140,33 +138,9 @@ export class ConfigCacheService implements OnModuleDestroy { } } - if (this.valueCache.size > CONFIG_VARIABLES_CACHE_MAX_ENTRIES) { - const entriesToDelete = - this.valueCache.size - CONFIG_VARIABLES_CACHE_MAX_ENTRIES; - const keysToDelete = Array.from(this.valueCache.keys()).slice( - 0, - entriesToDelete, - ); - - keysToDelete.forEach((key) => this.valueCache.delete(key)); - sizeLimitedCount += entriesToDelete; - } - - if (this.negativeLookupCache.size > CONFIG_VARIABLES_CACHE_MAX_ENTRIES) { - const entriesToDelete = - this.negativeLookupCache.size - CONFIG_VARIABLES_CACHE_MAX_ENTRIES; - const keysToDelete = Array.from(this.negativeLookupCache.keys()).slice( - 0, - entriesToDelete, - ); - - keysToDelete.forEach((key) => this.negativeLookupCache.delete(key)); - sizeLimitedCount += entriesToDelete; - } - - if (expiredCount > 0 || sizeLimitedCount > 0) { + if (expiredCount > 0) { this.logger.debug( - `🧹 Cache scavenging completed: ${expiredCount} expired entries removed, ${sizeLimitedCount} entries removed due to size limits`, + `🧹 Cache scavenging completed: ${expiredCount} expired entries removed`, ); } } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-max-entries.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-max-entries.ts deleted file mode 100644 index 613b6cf5dda2..000000000000 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-max-entries.ts +++ /dev/null @@ -1 +0,0 @@ -export const CONFIG_VARIABLES_CACHE_MAX_ENTRIES = 1000; From be2bf6456445fc00105b8b81dd1d131f3a1d9010 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Tue, 15 Apr 2025 17:18:09 +0530 Subject: [PATCH 13/70] WIP: tests - add cache test --- .../cache/config-cache.service.spec.ts | 288 ++++++++++++++++++ 1 file changed, 288 insertions(+) create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts new file mode 100644 index 000000000000..9e69e8ddec9b --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts @@ -0,0 +1,288 @@ +import { Test, TestingModule } from '@nestjs/testing'; + +import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { CONFIG_VARIABLES_CACHE_TTL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl'; + +describe('ConfigCacheService', () => { + let service: ConfigCacheService; + let originalDateNow: () => number; + + beforeAll(() => { + originalDateNow = Date.now; + }); + + afterAll(() => { + global.Date.now = originalDateNow; + }); + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ConfigCacheService], + }).compile(); + + service = module.get(ConfigCacheService); + jest.clearAllMocks(); + }); + + afterEach(() => { + jest.clearAllMocks(); + service.onModuleDestroy(); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('get and set', () => { + it('should set and get a value from cache', () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const value = true; + + service.set(key, value); + const result = service.get(key); + + expect(result).toBe(value); + }); + + it('should return undefined for non-existent key', () => { + const result = service.get( + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ); + + expect(result).toBeUndefined(); + }); + + it('should handle different value types', () => { + const booleanKey = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const stringKey = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + const numberKey = 'NODE_PORT' as keyof ConfigVariables; + + service.set(booleanKey, true); + service.set(stringKey, 'test@example.com'); + service.set(numberKey, 3000); + + expect(service.get(booleanKey)).toBe(true); + expect(service.get(stringKey)).toBe('test@example.com'); + expect(service.get(numberKey)).toBe(3000); + }); + }); + + describe('negative lookup cache', () => { + it('should set and get negative lookup', () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + + service.setNegativeLookup(key); + const result = service.getNegativeLookup(key); + + expect(result).toBe(true); + }); + + it('should return false for non-existent negative lookup', () => { + const result = service.getNegativeLookup( + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ); + + expect(result).toBe(false); + }); + + it('should clear negative lookup when setting a value', () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + + service.setNegativeLookup(key); + service.set(key, true); + + expect(service.getNegativeLookup(key)).toBe(false); + expect(service.get(key)).toBe(true); + }); + }); + + describe('clear operations', () => { + it('should clear specific key', () => { + const key1 = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const key2 = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + + service.set(key1, true); + service.set(key2, 'test@example.com'); + service.clear(key1); + + expect(service.get(key1)).toBeUndefined(); + expect(service.get(key2)).toBe('test@example.com'); + }); + + it('should clear all entries', () => { + const key1 = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const key2 = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + + service.set(key1, true); + service.set(key2, 'test@example.com'); + service.clearAll(); + + expect(service.get(key1)).toBeUndefined(); + expect(service.get(key2)).toBeUndefined(); + }); + }); + + describe('cache expiration', () => { + it('should expire entries after TTL', () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const value = true; + + service.set(key, value); + + // Mock Date.now to simulate time passing + const originalNow = Date.now; + + Date.now = jest.fn(() => originalNow() + CONFIG_VARIABLES_CACHE_TTL + 1); + + const result = service.get(key); + + expect(result).toBeUndefined(); + + // Restore Date.now + Date.now = originalNow; + }); + + it('should not expire entries before TTL', () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const value = true; + + service.set(key, value); + + // Mock Date.now to simulate time passing but less than TTL + const originalNow = Date.now; + + Date.now = jest.fn(() => originalNow() + CONFIG_VARIABLES_CACHE_TTL - 1); + + const result = service.get(key); + + expect(result).toBe(value); + + // Restore Date.now + Date.now = originalNow; + }); + }); + + describe('getCacheInfo', () => { + it('should return correct cache information', () => { + const key1 = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const key2 = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + const key3 = 'NODE_PORT' as keyof ConfigVariables; + + service.set(key1, true); + service.set(key2, 'test@example.com'); + service.setNegativeLookup(key3); + + const info = service.getCacheInfo(); + + expect(info.positiveEntries).toBe(2); + expect(info.negativeEntries).toBe(1); + expect(info.cacheKeys).toContain(key1); + expect(info.cacheKeys).toContain(key2); + expect(info.cacheKeys).not.toContain(key3); + }); + + it('should not include expired entries in cache info', () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + + service.set(key, true); + + // Mock Date.now to simulate time passing + const originalNow = Date.now; + + Date.now = jest.fn(() => originalNow() + CONFIG_VARIABLES_CACHE_TTL + 1); + + const info = service.getCacheInfo(); + + expect(info.positiveEntries).toBe(0); + expect(info.negativeEntries).toBe(0); + expect(info.cacheKeys).toHaveLength(0); + + // Restore Date.now + Date.now = originalNow; + }); + }); + + describe('module lifecycle', () => { + it('should clean up interval on module destroy', () => { + const clearIntervalSpy = jest.spyOn(global, 'clearInterval'); + + service.onModuleDestroy(); + expect(clearIntervalSpy).toHaveBeenCalled(); + }); + }); + + describe('cache scavenging', () => { + it('should remove expired entries during scavenging', () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + + service.set(key, true); + + // Mock Date.now to simulate time passing + const currentTime = Date.now(); + + jest + .spyOn(global.Date, 'now') + .mockImplementation(() => currentTime + CONFIG_VARIABLES_CACHE_TTL + 1); + + // Trigger scavenging by getting the value + const result = service.get(key); + + expect(result).toBeUndefined(); + }); + + it('should not remove non-expired entries during scavenging', () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + + service.set(key, true); + + // Mock Date.now to simulate time passing but still within TTL + const currentTime = Date.now(); + + jest + .spyOn(global.Date, 'now') + .mockImplementation(() => currentTime + CONFIG_VARIABLES_CACHE_TTL - 1); + + // Trigger scavenging by getting the value + const result = service.get(key); + + expect(result).toBe(true); + }); + }); + + describe('edge cases', () => { + it('should handle empty string values', () => { + const key = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + + service.set(key, ''); + expect(service.get(key)).toBe(''); + }); + + it('should handle zero values', () => { + const key = 'NODE_PORT' as keyof ConfigVariables; + + service.set(key, 0); + expect(service.get(key)).toBe(0); + }); + + it('should handle clearing non-existent keys', () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + + expect(() => service.clear(key)).not.toThrow(); + + const otherKey = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + + service.set(otherKey, 'test@example.com'); + service.clear(key); + expect(service.get(otherKey)).toBe('test@example.com'); + }); + + it('should handle empty cache operations', () => { + expect(service.getCacheInfo()).toEqual({ + positiveEntries: 0, + negativeEntries: 0, + cacheKeys: [], + }); + }); + }); +}); From 071b1473dd23e14d9c7f56040eb04d6a538b3607 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Tue, 15 Apr 2025 17:36:49 +0530 Subject: [PATCH 14/70] WIP: tests -- add tests for drivers --- .../__tests__/database-config.driver.spec.ts | 340 ++++++++++++++++++ .../environment-config.driver.spec.ts | 88 +++++ 2 files changed, 428 insertions(+) create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts new file mode 100644 index 000000000000..c3d223ff87ea --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -0,0 +1,340 @@ +import { Test, TestingModule } from '@nestjs/testing'; + +import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; +import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; +import { InitializationState } from 'src/engine/core-modules/twenty-config/enums/initialization-state.enum'; +import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; +import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; + +jest.mock( + 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util', + () => ({ + isEnvOnlyConfigVar: jest.fn(), + }), +); + +describe('DatabaseConfigDriver', () => { + let driver: DatabaseConfigDriver; + let configCache: ConfigCacheService; + let configStorage: ConfigStorageService; + let environmentDriver: EnvironmentConfigDriver; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + DatabaseConfigDriver, + { + provide: ConfigCacheService, + useValue: { + get: jest.fn(), + set: jest.fn(), + clear: jest.fn(), + clearAll: jest.fn(), + getNegativeLookup: jest.fn(), + setNegativeLookup: jest.fn(), + getCacheInfo: jest.fn(), + }, + }, + { + provide: ConfigStorageService, + useValue: { + get: jest.fn(), + set: jest.fn(), + loadAll: jest.fn(), + }, + }, + { + provide: EnvironmentConfigDriver, + useValue: { + get: jest.fn(), + }, + }, + ], + }).compile(); + + driver = module.get(DatabaseConfigDriver); + configCache = module.get(ConfigCacheService); + configStorage = module.get(ConfigStorageService); + environmentDriver = module.get( + EnvironmentConfigDriver, + ); + + jest.clearAllMocks(); + (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(false); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should be defined', () => { + expect(driver).toBeDefined(); + }); + + describe('initialization', () => { + it('should initialize successfully', async () => { + const configVars = new Map< + keyof ConfigVariables, + ConfigVariables[keyof ConfigVariables] + >([ + ['AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, true], + ['EMAIL_FROM_ADDRESS' as keyof ConfigVariables, 'test@example.com'], + ]); + + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(configVars); + + await driver.initialize(); + + expect(configStorage.loadAll).toHaveBeenCalled(); + expect(configCache.set).toHaveBeenCalledTimes(2); + }); + + it('should handle initialization failure and retry', async () => { + const error = new Error('DB error'); + + jest.spyOn(configStorage, 'loadAll').mockRejectedValue(error); + + await driver.initialize(); + + expect((driver as any).initializationState).toBe( + InitializationState.FAILED, + ); + expect(configStorage.loadAll).toHaveBeenCalled(); + }); + + it('should retry initialization after failure', async () => { + const error = new Error('DB error'); + + jest + .spyOn(configStorage, 'loadAll') + .mockRejectedValueOnce(error) + .mockResolvedValueOnce(new Map()); + + await driver.initialize(); + + expect((driver as any).initializationState).toBe( + InitializationState.FAILED, + ); + expect((driver as any).retryAttempts).toBeGreaterThan(0); + }); + + it('should handle concurrent initialization', async () => { + const configVars = new Map([ + ['AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, true], + ]); + + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(configVars); + + const promises = [ + driver.initialize(), + driver.initialize(), + driver.initialize(), + ]; + + await Promise.all(promises); + + expect(configStorage.loadAll).toHaveBeenCalledTimes(1); + }); + }); + + describe('get', () => { + beforeEach(async () => { + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(new Map()); + await driver.initialize(); + }); + + it('should use environment driver when not initialized', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const envValue = true; + + jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); + (driver as any).initializationState = InitializationState.NOT_INITIALIZED; + + const result = driver.get(key); + + expect(result).toBe(envValue); + expect(environmentDriver.get).toHaveBeenCalledWith(key); + }); + + it('should use environment driver for env-only variables', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const envValue = true; + + (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(true); + jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); + + const result = driver.get(key); + + expect(result).toBe(envValue); + expect(environmentDriver.get).toHaveBeenCalledWith(key); + expect(isEnvOnlyConfigVar).toHaveBeenCalledWith(key); + }); + + it('should return cached value when available', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const cachedValue = true; + + jest.spyOn(configCache, 'get').mockReturnValue(cachedValue); + + const result = driver.get(key); + + expect(result).toBe(cachedValue); + expect(configCache.get).toHaveBeenCalledWith(key); + }); + + it('should use environment driver when negative lookup is cached', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const envValue = true; + + jest.spyOn(configCache, 'get').mockReturnValue(undefined); + jest.spyOn(configCache, 'getNegativeLookup').mockReturnValue(true); + jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); + + const result = driver.get(key); + + expect(result).toBe(envValue); + expect(environmentDriver.get).toHaveBeenCalledWith(key); + }); + + it('should handle different config variable types correctly', () => { + const stringKey = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + const booleanKey = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const numberKey = 'NODE_PORT' as keyof ConfigVariables; + + const stringValue = 'test@example.com'; + const booleanValue = true; + const numberValue = 3000; + + jest.spyOn(configCache, 'get').mockImplementation((key) => { + switch (key) { + case stringKey: + return stringValue; + case booleanKey: + return booleanValue; + case numberKey: + return numberValue; + default: + return undefined; + } + }); + + expect(driver.get(stringKey)).toBe(stringValue); + expect(driver.get(booleanKey)).toBe(booleanValue); + expect(driver.get(numberKey)).toBe(numberValue); + }); + }); + + describe('update', () => { + beforeEach(async () => { + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(new Map()); + await driver.initialize(); + (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(false); + }); + + it('should update config in storage and cache', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const value = true; + + await driver.update(key, value); + + expect(configStorage.set).toHaveBeenCalledWith(key, value); + expect(configCache.set).toHaveBeenCalledWith(key, value); + }); + + it('should throw error when updating env-only variable', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const value = true; + + (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(true); + + await expect(driver.update(key, value)).rejects.toThrow(); + }); + + it('should throw error when updating before initialization', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const value = true; + + (driver as any).initializationState = InitializationState.NOT_INITIALIZED; + + await expect(driver.update(key, value)).rejects.toThrow(); + }); + }); + + describe('cache operations', () => { + it('should clear specific key from cache', () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + + driver.clearCache(key); + + expect(configCache.clear).toHaveBeenCalledWith(key); + }); + + it('should clear all cache', () => { + driver.clearAllCache(); + + expect(configCache.clearAll).toHaveBeenCalled(); + }); + + it('should return cache info', () => { + const cacheInfo = { + positiveEntries: 2, + negativeEntries: 1, + cacheKeys: ['AUTH_PASSWORD_ENABLED', 'EMAIL_FROM_ADDRESS'], + }; + + jest.spyOn(configCache, 'getCacheInfo').mockReturnValue(cacheInfo); + + const result = driver.getCacheInfo(); + + expect(result).toEqual(cacheInfo); + }); + }); + + describe('error handling', () => { + beforeEach(async () => { + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(new Map()); + await driver.initialize(); + (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(false); + }); + + it('should handle storage service errors during update', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const value = true; + const error = new Error('Storage error'); + + jest.spyOn(configStorage, 'set').mockRejectedValue(error); + + await expect(driver.update(key, value)).rejects.toThrow(); + expect(configCache.set).not.toHaveBeenCalled(); + }); + + it('should handle cache service errors', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const envValue = true; + + jest.spyOn(configCache, 'get').mockImplementation(() => { + throw new Error('Cache error'); + }); + + jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); + + expect(() => driver.get(key)).toThrow('Cache error'); + }); + + it('should handle environment driver errors', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + + jest.spyOn(configCache, 'get').mockImplementation(() => { + throw new Error('Cache error'); + }); + + jest.spyOn(environmentDriver, 'get').mockImplementation(() => { + throw new Error('Environment error'); + }); + + expect(() => driver.get(key)).toThrow('Cache error'); + }); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts new file mode 100644 index 000000000000..951410c5e94f --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts @@ -0,0 +1,88 @@ +import { ConfigService } from '@nestjs/config'; +import { Test, TestingModule } from '@nestjs/testing'; + +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; + +describe('EnvironmentConfigDriver', () => { + let driver: EnvironmentConfigDriver; + let configService: ConfigService; + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + EnvironmentConfigDriver, + { + provide: ConfigService, + useValue: { + get: jest.fn(), + }, + }, + ], + }).compile(); + + driver = module.get(EnvironmentConfigDriver); + configService = module.get(ConfigService); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should be defined', () => { + expect(driver).toBeDefined(); + }); + + describe('get', () => { + it('should return value from config service when available', () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const expectedValue = true; + + jest.spyOn(configService, 'get').mockReturnValue(expectedValue); + + const result = driver.get(key); + + expect(result).toBe(expectedValue); + expect(configService.get).toHaveBeenCalledWith(key, expect.any(Boolean)); + }); + + it('should return default value when config service returns undefined', () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const defaultValue = new ConfigVariables()[key]; + + jest + .spyOn(configService, 'get') + .mockImplementation((_, defaultVal) => defaultVal); + + const result = driver.get(key); + + expect(result).toBe(defaultValue); + expect(configService.get).toHaveBeenCalledWith(key, defaultValue); + }); + + it('should handle different config variable types', () => { + const booleanKey = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const stringKey = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + const numberKey = 'NODE_PORT' as keyof ConfigVariables; + + jest + .spyOn(configService, 'get') + .mockImplementation((key: keyof ConfigVariables) => { + switch (key) { + case booleanKey: + return true; + case stringKey: + return 'test@example.com'; + case numberKey: + return 3000; + default: + return undefined; + } + }); + + expect(driver.get(booleanKey)).toBe(true); + expect(driver.get(stringKey)).toBe('test@example.com'); + expect(driver.get(numberKey)).toBe(3000); + }); + }); +}); From f0fe2d9ed79876d5892d3a1aa63b728a46547d59 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Tue, 15 Apr 2025 17:58:17 +0530 Subject: [PATCH 15/70] WIP: tests -- add test for storage service --- .../storage/config-storage.service.spec.ts | 460 ++++++++++++++++++ 1 file changed, 460 insertions(+) create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.spec.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.spec.ts new file mode 100644 index 000000000000..0b9568a006ed --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.spec.ts @@ -0,0 +1,460 @@ +import { Test, TestingModule } from '@nestjs/testing'; +import { getRepositoryToken } from '@nestjs/typeorm'; + +import { IsNull, Repository } from 'typeorm'; + +import { + KeyValuePair, + KeyValuePairType, +} from 'src/engine/core-modules/key-value-pair/key-value-pair.entity'; +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; +import { convertConfigVarToAppType } from 'src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util'; +import { convertConfigVarToStorageType } from 'src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util'; +import { User } from 'src/engine/core-modules/user/user.entity'; +import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; + +jest.mock( + 'src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util', + () => ({ + convertConfigVarToAppType: jest.fn(), + }), +); + +jest.mock( + 'src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util', + () => ({ + convertConfigVarToStorageType: jest.fn(), + }), +); + +describe('ConfigStorageService', () => { + let service: ConfigStorageService; + let keyValuePairRepository: Repository; + + const createMockKeyValuePair = ( + key: string, + value: string, + ): KeyValuePair => ({ + id: '1', + key, + value: value as unknown as JSON, + type: KeyValuePairType.CONFIG_VARIABLE, + userId: null, + workspaceId: null, + user: null as unknown as User, + workspace: null as unknown as Workspace, + createdAt: new Date(), + updatedAt: new Date(), + textValueDeprecated: null, + deletedAt: null, + }); + + beforeEach(async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + ConfigStorageService, + { + provide: getRepositoryToken(KeyValuePair, 'core'), + useValue: { + findOne: jest.fn(), + find: jest.fn(), + update: jest.fn(), + insert: jest.fn(), + delete: jest.fn(), + }, + }, + ], + }).compile(); + + service = module.get(ConfigStorageService); + keyValuePairRepository = module.get>( + getRepositoryToken(KeyValuePair, 'core'), + ); + + jest.clearAllMocks(); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('get', () => { + it('should return undefined when key not found', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + + jest.spyOn(keyValuePairRepository, 'findOne').mockResolvedValue(null); + + const result = await service.get(key); + + expect(result).toBeUndefined(); + expect(keyValuePairRepository.findOne).toHaveBeenCalledWith({ + where: { + type: KeyValuePairType.CONFIG_VARIABLE, + key: key as string, + userId: IsNull(), + workspaceId: IsNull(), + }, + }); + }); + + it('should return converted value when key found', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const storedValue = 'true'; + const convertedValue = true; + + const mockRecord = createMockKeyValuePair(key as string, storedValue); + + jest + .spyOn(keyValuePairRepository, 'findOne') + .mockResolvedValue(mockRecord); + + (convertConfigVarToAppType as jest.Mock).mockReturnValue(convertedValue); + + const result = await service.get(key); + + expect(result).toBe(convertedValue); + expect(convertConfigVarToAppType).toHaveBeenCalledWith(storedValue, key); + }); + + it('should handle conversion errors', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const error = new Error('Conversion error'); + + const mockRecord = createMockKeyValuePair(key as string, 'invalid'); + + jest + .spyOn(keyValuePairRepository, 'findOne') + .mockResolvedValue(mockRecord); + + (convertConfigVarToAppType as jest.Mock).mockImplementation(() => { + throw error; + }); + + await expect(service.get(key)).rejects.toThrow('Conversion error'); + }); + }); + + describe('set', () => { + it('should update existing record', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const value = true; + const storedValue = 'true'; + + const mockRecord = createMockKeyValuePair(key as string, 'false'); + + jest + .spyOn(keyValuePairRepository, 'findOne') + .mockResolvedValue(mockRecord); + + (convertConfigVarToStorageType as jest.Mock).mockReturnValue(storedValue); + + await service.set(key, value); + + expect(keyValuePairRepository.update).toHaveBeenCalledWith( + { id: '1' }, + { value: storedValue }, + ); + }); + + it('should insert new record', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const value = true; + const storedValue = 'true'; + + jest.spyOn(keyValuePairRepository, 'findOne').mockResolvedValue(null); + + (convertConfigVarToStorageType as jest.Mock).mockReturnValue(storedValue); + + await service.set(key, value); + + expect(keyValuePairRepository.insert).toHaveBeenCalledWith({ + key: key as string, + value: storedValue, + userId: null, + workspaceId: null, + type: KeyValuePairType.CONFIG_VARIABLE, + }); + }); + + it('should handle conversion errors', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const value = true; + const error = new Error('Conversion error'); + + (convertConfigVarToStorageType as jest.Mock).mockImplementation(() => { + throw error; + }); + + await expect(service.set(key, value)).rejects.toThrow('Conversion error'); + }); + }); + + describe('delete', () => { + it('should delete record', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + + await service.delete(key); + + expect(keyValuePairRepository.delete).toHaveBeenCalledWith({ + type: KeyValuePairType.CONFIG_VARIABLE, + key: key as string, + userId: IsNull(), + workspaceId: IsNull(), + }); + }); + + it('should handle delete errors', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const error = new Error('Delete error'); + + jest.spyOn(keyValuePairRepository, 'delete').mockRejectedValue(error); + + await expect(service.delete(key)).rejects.toThrow('Delete error'); + }); + }); + + describe('loadAll', () => { + it('should load and convert all config variables', async () => { + const configVars: KeyValuePair[] = [ + createMockKeyValuePair('AUTH_PASSWORD_ENABLED', 'true'), + createMockKeyValuePair('EMAIL_FROM_ADDRESS', 'test@example.com'), + ]; + + jest.spyOn(keyValuePairRepository, 'find').mockResolvedValue(configVars); + + (convertConfigVarToAppType as jest.Mock).mockImplementation( + (value, key) => { + if (key === 'AUTH_PASSWORD_ENABLED') return true; + if (key === 'EMAIL_FROM_ADDRESS') return 'test@example.com'; + + return value; + }, + ); + + const result = await service.loadAll(); + + expect(result.size).toBe(2); + expect(result.get('AUTH_PASSWORD_ENABLED' as keyof ConfigVariables)).toBe( + true, + ); + expect(result.get('EMAIL_FROM_ADDRESS' as keyof ConfigVariables)).toBe( + 'test@example.com', + ); + }); + + it('should skip invalid values but continue processing', async () => { + const configVars: KeyValuePair[] = [ + createMockKeyValuePair('AUTH_PASSWORD_ENABLED', 'invalid'), + createMockKeyValuePair('EMAIL_FROM_ADDRESS', 'test@example.com'), + ]; + + jest.spyOn(keyValuePairRepository, 'find').mockResolvedValue(configVars); + + (convertConfigVarToAppType as jest.Mock) + .mockImplementationOnce(() => { + throw new Error('Invalid value'); + }) + .mockImplementationOnce((value) => value); + + const result = await service.loadAll(); + + expect(result.size).toBe(1); + expect(result.get('EMAIL_FROM_ADDRESS' as keyof ConfigVariables)).toBe( + 'test@example.com', + ); + }); + + it('should handle find errors', async () => { + const error = new Error('Find error'); + + jest.spyOn(keyValuePairRepository, 'find').mockRejectedValue(error); + + await expect(service.loadAll()).rejects.toThrow('Find error'); + }); + + describe('Null Value Handling', () => { + it('should handle null values in loadAll', async () => { + const configVars: KeyValuePair[] = [ + { + ...createMockKeyValuePair('AUTH_PASSWORD_ENABLED', 'true'), + value: null as unknown as JSON, + }, + createMockKeyValuePair('EMAIL_FROM_ADDRESS', 'test@example.com'), + ]; + + jest + .spyOn(keyValuePairRepository, 'find') + .mockResolvedValue(configVars); + + (convertConfigVarToAppType as jest.Mock).mockImplementation((value) => { + if (value === null) throw new Error('Null value'); + + return value; + }); + + const result = await service.loadAll(); + + expect(result.size).toBe(1); + expect(result.get('EMAIL_FROM_ADDRESS' as keyof ConfigVariables)).toBe( + 'test@example.com', + ); + expect(convertConfigVarToAppType).toHaveBeenCalledTimes(1); // Only called for non-null value + }); + }); + }); + + describe('Edge Cases and Additional Scenarios', () => { + describe('Type Safety', () => { + it('should enforce correct types for boolean config variables', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const invalidValue = 'not-a-boolean'; + + const mockRecord = createMockKeyValuePair(key as string, invalidValue); + + jest + .spyOn(keyValuePairRepository, 'findOne') + .mockResolvedValue(mockRecord); + + (convertConfigVarToAppType as jest.Mock).mockImplementation(() => { + throw new Error('Invalid boolean value'); + }); + + await expect(service.get(key)).rejects.toThrow('Invalid boolean value'); + }); + + it('should enforce correct types for string config variables', async () => { + const key = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + const invalidValue = '123'; // Not a valid email + + const mockRecord = createMockKeyValuePair(key as string, invalidValue); + + jest + .spyOn(keyValuePairRepository, 'findOne') + .mockResolvedValue(mockRecord); + + (convertConfigVarToAppType as jest.Mock).mockImplementation(() => { + throw new Error('Invalid string value'); + }); + + await expect(service.get(key)).rejects.toThrow('Invalid string value'); + }); + }); + + describe('Concurrent Operations', () => { + it('should handle concurrent get/set operations', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const initialValue = true; + const newValue = false; + + const initialRecord = createMockKeyValuePair(key as string, 'true'); + const updatedRecord = createMockKeyValuePair(key as string, 'false'); + + jest + .spyOn(keyValuePairRepository, 'findOne') + .mockResolvedValueOnce(initialRecord) + .mockResolvedValueOnce(initialRecord) + .mockResolvedValueOnce(updatedRecord); + + (convertConfigVarToAppType as jest.Mock) + .mockReturnValueOnce(initialValue) + .mockReturnValueOnce(newValue); + + (convertConfigVarToStorageType as jest.Mock).mockReturnValueOnce( + 'false', + ); + + const firstGet = service.get(key); + const setOperation = service.set(key, newValue); + const secondGet = service.get(key); + + const [firstResult, , secondResult] = await Promise.all([ + firstGet, + setOperation, + secondGet, + ]); + + expect(firstResult).toBe(initialValue); + expect(secondResult).toBe(newValue); + }); + + it('should handle concurrent delete operations', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + + jest + .spyOn(keyValuePairRepository, 'delete') + .mockResolvedValueOnce({ affected: 1 } as any) + .mockResolvedValueOnce({ affected: 0 } as any); + + const firstDelete = service.delete(key); + const secondDelete = service.delete(key); + + await Promise.all([firstDelete, secondDelete]); + + expect(keyValuePairRepository.delete).toHaveBeenCalledTimes(2); + }); + }); + + describe('Database Connection Issues', () => { + it('should handle database connection failures in get', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const error = new Error('Database connection failed'); + + jest.spyOn(keyValuePairRepository, 'findOne').mockRejectedValue(error); + + await expect(service.get(key)).rejects.toThrow( + 'Database connection failed', + ); + }); + + it('should handle database connection failures in set', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const value = true; + const error = new Error('Database connection failed'); + + (convertConfigVarToStorageType as jest.Mock).mockReturnValue('true'); + jest.spyOn(keyValuePairRepository, 'findOne').mockRejectedValue(error); + + await expect(service.set(key, value)).rejects.toThrow( + 'Database connection failed', + ); + }); + + it('should handle database connection failures in delete', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const error = new Error('Database connection failed'); + + jest.spyOn(keyValuePairRepository, 'delete').mockRejectedValue(error); + + await expect(service.delete(key)).rejects.toThrow( + 'Database connection failed', + ); + }); + + it('should handle database connection failures in loadAll', async () => { + const error = new Error('Database connection failed'); + + jest.spyOn(keyValuePairRepository, 'find').mockRejectedValue(error); + + await expect(service.loadAll()).rejects.toThrow( + 'Database connection failed', + ); + }); + + it('should handle database operation timeouts', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const error = new Error('Database operation timed out'); + + jest.useFakeTimers(); + jest.spyOn(keyValuePairRepository, 'findOne').mockRejectedValue(error); + + const promise = service.get(key); + + jest.advanceTimersByTime(1000); + + await expect(promise).rejects.toThrow('Database operation timed out'); + jest.useRealTimers(); + }, 10000); + }); + }); +}); From c52cecfb17ca61e89d2d974e48750cdbfdbf8617 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Tue, 15 Apr 2025 18:27:45 +0530 Subject: [PATCH 16/70] fix test --- .../services/microsoft-get-messages.service.spec.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts index afe372e39bb1..5d7aba2033a1 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.spec.ts @@ -3,7 +3,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ConnectedAccountProvider } from 'twenty-shared/types'; -import { TwentyConfigModule } from 'src/engine/core-modules/twenty-config/twenty-config.module'; +import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { MicrosoftOAuth2ClientManagerService } from 'src/modules/connected-account/oauth2-client-manager/drivers/microsoft/microsoft-oauth2-client-manager.service'; import { microsoftGraphBatchWithHtmlMessagesResponse, @@ -21,7 +21,6 @@ describe('Microsoft get messages service', () => { beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ - imports: [TwentyConfigModule.forRoot({})], providers: [ MicrosoftGetMessagesService, MicrosoftHandleErrorService, @@ -29,6 +28,10 @@ describe('Microsoft get messages service', () => { MicrosoftOAuth2ClientManagerService, MicrosoftFetchByBatchService, ConfigService, + { + provide: TwentyConfigService, + useValue: {}, + }, ], }).compile(); @@ -37,6 +40,10 @@ describe('Microsoft get messages service', () => { ); }); + afterEach(() => { + jest.clearAllMocks(); + }); + it('Should be defined', () => { expect(service).toBeDefined(); }); From 0f8012bc7785b37cc519ed3822f04e2fd060b4b9 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 16 Apr 2025 12:18:21 +0530 Subject: [PATCH 17/70] WIP: tests -- fix config cache test --- .../cache/config-cache.service.spec.ts | 96 ++++++------------- 1 file changed, 31 insertions(+), 65 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts index 9e69e8ddec9b..dbda586d1519 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts @@ -6,15 +6,16 @@ import { CONFIG_VARIABLES_CACHE_TTL } from 'src/engine/core-modules/twenty-confi describe('ConfigCacheService', () => { let service: ConfigCacheService; - let originalDateNow: () => number; - beforeAll(() => { - originalDateNow = Date.now; - }); - - afterAll(() => { - global.Date.now = originalDateNow; - }); + const withMockedDate = (timeOffset: number, callback: () => void) => { + const originalNow = Date.now; + try { + Date.now = jest.fn(() => originalNow() + timeOffset); + callback(); + } finally { + Date.now = originalNow; + } + }; beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ @@ -130,17 +131,10 @@ describe('ConfigCacheService', () => { service.set(key, value); - // Mock Date.now to simulate time passing - const originalNow = Date.now; - - Date.now = jest.fn(() => originalNow() + CONFIG_VARIABLES_CACHE_TTL + 1); - - const result = service.get(key); - - expect(result).toBeUndefined(); - - // Restore Date.now - Date.now = originalNow; + withMockedDate(CONFIG_VARIABLES_CACHE_TTL + 1, () => { + const result = service.get(key); + expect(result).toBeUndefined(); + }); }); it('should not expire entries before TTL', () => { @@ -149,17 +143,10 @@ describe('ConfigCacheService', () => { service.set(key, value); - // Mock Date.now to simulate time passing but less than TTL - const originalNow = Date.now; - - Date.now = jest.fn(() => originalNow() + CONFIG_VARIABLES_CACHE_TTL - 1); - - const result = service.get(key); - - expect(result).toBe(value); - - // Restore Date.now - Date.now = originalNow; + withMockedDate(CONFIG_VARIABLES_CACHE_TTL - 1, () => { + const result = service.get(key); + expect(result).toBe(value); + }); }); }); @@ -187,19 +174,12 @@ describe('ConfigCacheService', () => { service.set(key, true); - // Mock Date.now to simulate time passing - const originalNow = Date.now; - - Date.now = jest.fn(() => originalNow() + CONFIG_VARIABLES_CACHE_TTL + 1); - - const info = service.getCacheInfo(); - - expect(info.positiveEntries).toBe(0); - expect(info.negativeEntries).toBe(0); - expect(info.cacheKeys).toHaveLength(0); - - // Restore Date.now - Date.now = originalNow; + withMockedDate(CONFIG_VARIABLES_CACHE_TTL + 1, () => { + const info = service.getCacheInfo(); + expect(info.positiveEntries).toBe(0); + expect(info.negativeEntries).toBe(0); + expect(info.cacheKeys).toHaveLength(0); + }); }); }); @@ -218,17 +198,10 @@ describe('ConfigCacheService', () => { service.set(key, true); - // Mock Date.now to simulate time passing - const currentTime = Date.now(); - - jest - .spyOn(global.Date, 'now') - .mockImplementation(() => currentTime + CONFIG_VARIABLES_CACHE_TTL + 1); - - // Trigger scavenging by getting the value - const result = service.get(key); - - expect(result).toBeUndefined(); + withMockedDate(CONFIG_VARIABLES_CACHE_TTL + 1, () => { + const result = service.get(key); + expect(result).toBeUndefined(); + }); }); it('should not remove non-expired entries during scavenging', () => { @@ -236,17 +209,10 @@ describe('ConfigCacheService', () => { service.set(key, true); - // Mock Date.now to simulate time passing but still within TTL - const currentTime = Date.now(); - - jest - .spyOn(global.Date, 'now') - .mockImplementation(() => currentTime + CONFIG_VARIABLES_CACHE_TTL - 1); - - // Trigger scavenging by getting the value - const result = service.get(key); - - expect(result).toBe(true); + withMockedDate(CONFIG_VARIABLES_CACHE_TTL - 1, () => { + const result = service.get(key); + expect(result).toBe(true); + }); }); }); From 337ff08f5587416c77c77ceb832ce79dcb41f275 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 16 Apr 2025 12:22:23 +0530 Subject: [PATCH 18/70] WIP: tests -- add more test cases in db config driver --- .../__tests__/database-config.driver.spec.ts | 57 ++++++++++++++++--- 1 file changed, 50 insertions(+), 7 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index c3d223ff87ea..ea33ee26ee74 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -310,31 +310,74 @@ describe('DatabaseConfigDriver', () => { expect(configCache.set).not.toHaveBeenCalled(); }); - it('should handle cache service errors', async () => { + it('should use environment driver when not initialized', async () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const envValue = true; - jest.spyOn(configCache, 'get').mockImplementation(() => { - throw new Error('Cache error'); - }); + (driver as any).initializationState = InitializationState.NOT_INITIALIZED; + jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); + + const result = driver.get(key); + + expect(result).toBe(envValue); + expect(environmentDriver.get).toHaveBeenCalledWith(key); + expect(configCache.get).not.toHaveBeenCalled(); + }); + it('should use environment driver for env-only variables', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const envValue = true; + + (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(true); jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); - expect(() => driver.get(key)).toThrow('Cache error'); + const result = driver.get(key); + + expect(result).toBe(envValue); + expect(environmentDriver.get).toHaveBeenCalledWith(key); + expect(configCache.get).not.toHaveBeenCalled(); + expect(isEnvOnlyConfigVar).toHaveBeenCalledWith(key); }); - it('should handle environment driver errors', async () => { + it('should use environment driver when negative lookup is cached', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const envValue = true; + + jest.spyOn(configCache, 'get').mockReturnValue(undefined); + jest.spyOn(configCache, 'getNegativeLookup').mockReturnValue(true); + jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); + + const result = driver.get(key); + + expect(result).toBe(envValue); + expect(environmentDriver.get).toHaveBeenCalledWith(key); + expect(configCache.get).toHaveBeenCalledWith(key); + expect(configCache.getNegativeLookup).toHaveBeenCalledWith(key); + }); + + it('should propagate cache service errors when no fallback conditions are met', async () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; jest.spyOn(configCache, 'get').mockImplementation(() => { throw new Error('Cache error'); }); + expect(() => driver.get(key)).toThrow('Cache error'); + expect(configCache.get).toHaveBeenCalledWith(key); + expect(environmentDriver.get).not.toHaveBeenCalled(); + }); + + it('should propagate environment driver errors when using environment driver', async () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + + // Force use of environment driver + (driver as any).initializationState = InitializationState.NOT_INITIALIZED; jest.spyOn(environmentDriver, 'get').mockImplementation(() => { throw new Error('Environment error'); }); - expect(() => driver.get(key)).toThrow('Cache error'); + expect(() => driver.get(key)).toThrow('Environment error'); + expect(environmentDriver.get).toHaveBeenCalledWith(key); }); }); }); From b0c80214d9b6e20cf83810b63e318bd09ed791da Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 16 Apr 2025 12:23:37 +0530 Subject: [PATCH 19/70] WIP: tests -- fix incorrect env driver test --- .../__tests__/environment-config.driver.spec.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts index 951410c5e94f..53d90f60ceba 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts @@ -37,13 +37,14 @@ describe('EnvironmentConfigDriver', () => { it('should return value from config service when available', () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const expectedValue = true; + const defaultValue = new ConfigVariables()[key]; jest.spyOn(configService, 'get').mockReturnValue(expectedValue); const result = driver.get(key); expect(result).toBe(expectedValue); - expect(configService.get).toHaveBeenCalledWith(key, expect.any(Boolean)); + expect(configService.get).toHaveBeenCalledWith(key, defaultValue); }); it('should return default value when config service returns undefined', () => { @@ -65,6 +66,8 @@ describe('EnvironmentConfigDriver', () => { const stringKey = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; const numberKey = 'NODE_PORT' as keyof ConfigVariables; + const defaultValues = new ConfigVariables(); + jest .spyOn(configService, 'get') .mockImplementation((key: keyof ConfigVariables) => { @@ -81,8 +84,13 @@ describe('EnvironmentConfigDriver', () => { }); expect(driver.get(booleanKey)).toBe(true); + expect(configService.get).toHaveBeenCalledWith(booleanKey, defaultValues[booleanKey]); + expect(driver.get(stringKey)).toBe('test@example.com'); + expect(configService.get).toHaveBeenCalledWith(stringKey, defaultValues[stringKey]); + expect(driver.get(numberKey)).toBe(3000); + expect(configService.get).toHaveBeenCalledWith(numberKey, defaultValues[numberKey]); }); }); }); From 0c79d81792b2d614d0043369a5c243d787c155ec Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 16 Apr 2025 12:27:26 +0530 Subject: [PATCH 20/70] WIP: tests -- fix test --- .../storage/config-storage.service.spec.ts | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.spec.ts index 0b9568a006ed..e861812a19c6 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.spec.ts @@ -445,16 +445,23 @@ describe('ConfigStorageService', () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const error = new Error('Database operation timed out'); - jest.useFakeTimers(); - jest.spyOn(keyValuePairRepository, 'findOne').mockRejectedValue(error); + let rejectPromise: ((error: Error) => void) | undefined; + const timeoutPromise = new Promise((_, reject) => { + rejectPromise = reject; + }); - const promise = service.get(key); + jest.spyOn(keyValuePairRepository, 'findOne').mockReturnValue(timeoutPromise); - jest.advanceTimersByTime(1000); + const promise = service.get(key); + + // Simulate timeout by rejecting the promise + if (!rejectPromise) { + throw new Error('Reject function not assigned'); + } + rejectPromise(error); await expect(promise).rejects.toThrow('Database operation timed out'); - jest.useRealTimers(); - }, 10000); + }); }); }); }); From 4f96a722f7cc4a2b0358b664e09810af80da3d10 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 16 Apr 2025 12:33:47 +0530 Subject: [PATCH 21/70] change IS_CONFIG_VAR_IN_DB_ENABLED to IS_CONFIG_VARIABLES_IN_DB_ENABLED --- packages/twenty-server/.env.example | 2 +- .../src/engine/core-modules/twenty-config/config-variables.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/twenty-server/.env.example b/packages/twenty-server/.env.example index 70abf02ac714..7841fa7c4672 100644 --- a/packages/twenty-server/.env.example +++ b/packages/twenty-server/.env.example @@ -76,4 +76,4 @@ FRONTEND_URL=http://localhost:3001 # CLOUDFLARE_API_KEY= # CLOUDFLARE_ZONE_ID= # CLOUDFLARE_WEBHOOK_SECRET= -# IS_CONFIG_VAR_IN_DB_ENABLED=false +# IS_CONFIG_VARIABLES_IN_DB_ENABLED=false diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts index 02b71c5a4aa2..8824f19e9045 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts @@ -727,7 +727,7 @@ export class ConfigVariables { @CastToBoolean() @IsBoolean() @IsOptional() - IS_CONFIG_VAR_IN_DB_ENABLED = false; + IS_CONFIG_VARIABLES_IN_DB_ENABLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, From df938fadf2e4ab4960db0bd95ad908a72bdb33ea Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 16 Apr 2025 12:34:43 +0530 Subject: [PATCH 22/70] lint --- packages/twenty-server/.env.example | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/twenty-server/.env.example b/packages/twenty-server/.env.example index 7841fa7c4672..e81da3a79219 100644 --- a/packages/twenty-server/.env.example +++ b/packages/twenty-server/.env.example @@ -77,3 +77,4 @@ FRONTEND_URL=http://localhost:3001 # CLOUDFLARE_ZONE_ID= # CLOUDFLARE_WEBHOOK_SECRET= # IS_CONFIG_VARIABLES_IN_DB_ENABLED=false + From e0bd022890214cfdf8e9c9cf95d72462b0760900 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 16 Apr 2025 12:39:31 +0530 Subject: [PATCH 23/70] lint --- .../cache/config-cache.service.spec.ts | 6 ++++++ .../environment-config.driver.spec.ts | 19 ++++++++++++++----- .../storage/config-storage.service.spec.ts | 6 ++++-- 3 files changed, 24 insertions(+), 7 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts index dbda586d1519..aad6e9c18b78 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts @@ -9,6 +9,7 @@ describe('ConfigCacheService', () => { const withMockedDate = (timeOffset: number, callback: () => void) => { const originalNow = Date.now; + try { Date.now = jest.fn(() => originalNow() + timeOffset); callback(); @@ -133,6 +134,7 @@ describe('ConfigCacheService', () => { withMockedDate(CONFIG_VARIABLES_CACHE_TTL + 1, () => { const result = service.get(key); + expect(result).toBeUndefined(); }); }); @@ -145,6 +147,7 @@ describe('ConfigCacheService', () => { withMockedDate(CONFIG_VARIABLES_CACHE_TTL - 1, () => { const result = service.get(key); + expect(result).toBe(value); }); }); @@ -176,6 +179,7 @@ describe('ConfigCacheService', () => { withMockedDate(CONFIG_VARIABLES_CACHE_TTL + 1, () => { const info = service.getCacheInfo(); + expect(info.positiveEntries).toBe(0); expect(info.negativeEntries).toBe(0); expect(info.cacheKeys).toHaveLength(0); @@ -200,6 +204,7 @@ describe('ConfigCacheService', () => { withMockedDate(CONFIG_VARIABLES_CACHE_TTL + 1, () => { const result = service.get(key); + expect(result).toBeUndefined(); }); }); @@ -211,6 +216,7 @@ describe('ConfigCacheService', () => { withMockedDate(CONFIG_VARIABLES_CACHE_TTL - 1, () => { const result = service.get(key); + expect(result).toBe(true); }); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts index 53d90f60ceba..e3d187835871 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts @@ -84,13 +84,22 @@ describe('EnvironmentConfigDriver', () => { }); expect(driver.get(booleanKey)).toBe(true); - expect(configService.get).toHaveBeenCalledWith(booleanKey, defaultValues[booleanKey]); - + expect(configService.get).toHaveBeenCalledWith( + booleanKey, + defaultValues[booleanKey], + ); + expect(driver.get(stringKey)).toBe('test@example.com'); - expect(configService.get).toHaveBeenCalledWith(stringKey, defaultValues[stringKey]); - + expect(configService.get).toHaveBeenCalledWith( + stringKey, + defaultValues[stringKey], + ); + expect(driver.get(numberKey)).toBe(3000); - expect(configService.get).toHaveBeenCalledWith(numberKey, defaultValues[numberKey]); + expect(configService.get).toHaveBeenCalledWith( + numberKey, + defaultValues[numberKey], + ); }); }); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.spec.ts index e861812a19c6..a3d0a5cca623 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.spec.ts @@ -450,10 +450,12 @@ describe('ConfigStorageService', () => { rejectPromise = reject; }); - jest.spyOn(keyValuePairRepository, 'findOne').mockReturnValue(timeoutPromise); + jest + .spyOn(keyValuePairRepository, 'findOne') + .mockReturnValue(timeoutPromise); const promise = service.get(key); - + // Simulate timeout by rejecting the promise if (!rejectPromise) { throw new Error('Reject function not assigned'); From 2462f0e10c121fc153a97cc37edbc9ad381f4346 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 16 Apr 2025 13:11:09 +0530 Subject: [PATCH 24/70] WIP: tests -- scavenging test cases --- .../cache/config-cache.service.spec.ts | 74 +++++++++++++++---- 1 file changed, 60 insertions(+), 14 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts index aad6e9c18b78..29a118b922c6 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts @@ -2,10 +2,13 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval'; import { CONFIG_VARIABLES_CACHE_TTL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl'; describe('ConfigCacheService', () => { let service: ConfigCacheService; + let setIntervalSpy: jest.SpyInstance; + let clearIntervalSpy: jest.SpyInstance; const withMockedDate = (timeOffset: number, callback: () => void) => { const originalNow = Date.now; @@ -19,17 +22,22 @@ describe('ConfigCacheService', () => { }; beforeEach(async () => { + jest.useFakeTimers({ doNotFake: ['setInterval', 'clearInterval'] }); + setIntervalSpy = jest.spyOn(global, 'setInterval'); + clearIntervalSpy = jest.spyOn(global, 'clearInterval'); + const module: TestingModule = await Test.createTestingModule({ providers: [ConfigCacheService], }).compile(); service = module.get(ConfigCacheService); - jest.clearAllMocks(); }); afterEach(() => { - jest.clearAllMocks(); service.onModuleDestroy(); + setIntervalSpy.mockRestore(); + clearIntervalSpy.mockRestore(); + jest.useRealTimers(); }); it('should be defined', () => { @@ -197,28 +205,66 @@ describe('ConfigCacheService', () => { }); describe('cache scavenging', () => { - it('should remove expired entries during scavenging', () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + beforeEach(() => { + jest.useFakeTimers({ doNotFake: ['setInterval', 'clearInterval'] }); + }); - service.set(key, true); + afterEach(() => { + jest.useRealTimers(); + }); - withMockedDate(CONFIG_VARIABLES_CACHE_TTL + 1, () => { - const result = service.get(key); + it('should set up scavenging interval on initialization', () => { + expect(setIntervalSpy).toHaveBeenCalledWith( + expect.any(Function), + CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL, + ); + }); - expect(result).toBeUndefined(); - }); + it('should clean up interval on module destroy', () => { + const intervalId = setIntervalSpy.mock.results[0].value; + + service.onModuleDestroy(); + expect(clearIntervalSpy).toHaveBeenCalledWith(intervalId); }); - it('should not remove non-expired entries during scavenging', () => { + it('should automatically scavenge expired entries after interval', () => { + const key1 = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const key2 = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + + service.set(key1, true); + service.set(key2, 'test@example.com'); + + jest.advanceTimersByTime(CONFIG_VARIABLES_CACHE_TTL + 1); + jest.advanceTimersByTime(CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL); + + expect(service.get(key1)).toBeUndefined(); + expect(service.get(key2)).toBeUndefined(); + }); + + it('should not remove non-expired entries after interval', () => { + const key1 = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const key2 = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + + service.set(key1, true); + service.set(key2, 'test@example.com'); + + jest.advanceTimersByTime(CONFIG_VARIABLES_CACHE_TTL - 1); + + expect(service.get(key1)).toBe(true); + expect(service.get(key2)).toBe('test@example.com'); + }); + + it('should scavenge multiple times when multiple intervals pass', () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; service.set(key, true); - withMockedDate(CONFIG_VARIABLES_CACHE_TTL - 1, () => { - const result = service.get(key); + jest.advanceTimersByTime( + CONFIG_VARIABLES_CACHE_TTL + + CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL * 3, + ); - expect(result).toBe(true); - }); + expect(service.get(key)).toBeUndefined(); }); }); From b259e41625e13e2426ad45eba86c14e099f32ecd Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 16 Apr 2025 13:12:04 +0530 Subject: [PATCH 25/70] fix --- .../twenty-config/cache/config-cache.service.spec.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts index 29a118b922c6..53aa000b9f0b 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts @@ -178,6 +178,7 @@ describe('ConfigCacheService', () => { expect(info.cacheKeys).toContain(key1); expect(info.cacheKeys).toContain(key2); expect(info.cacheKeys).not.toContain(key3); + expect(service.getNegativeLookup(key3)).toBe(true); }); it('should not include expired entries in cache info', () => { From 1b9d524759e038c3728670a1ef464ff3e2d5e594 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 16 Apr 2025 14:13:08 +0530 Subject: [PATCH 26/70] WIP: continue --- .../config-cache.service.spec.ts | 0 .../cache/config-cache.service.ts | 5 ++- .../config-cache-entry.interface.ts | 1 - .../__tests__/database-config.driver.spec.ts | 31 ++++++++++---- .../drivers/database-config.driver.ts | 39 ++++++++++------- .../database-config-driver.interface.ts | 5 --- .../config-storage.service.spec.ts | 6 +-- .../storage/config-storage.service.ts | 42 ++++++++----------- .../twenty-config/twenty-config.service.ts | 8 +--- 9 files changed, 71 insertions(+), 66 deletions(-) rename packages/twenty-server/src/engine/core-modules/twenty-config/cache/{ => __tests__}/config-cache.service.spec.ts (100%) rename packages/twenty-server/src/engine/core-modules/twenty-config/{ => drivers}/interfaces/database-config-driver.interface.ts (85%) rename packages/twenty-server/src/engine/core-modules/twenty-config/storage/{ => __tests__}/config-storage.service.spec.ts (98%) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts similarity index 100% rename from packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.spec.ts rename to packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts index 3d42be007824..c30db088c153 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts @@ -62,7 +62,6 @@ export class ConfigCacheService implements OnModuleDestroy { setNegativeLookup(key: ConfigKey): void { this.negativeLookupCache.set(key, { - value: true, timestamp: Date.now(), ttl: CONFIG_VARIABLES_CACHE_TTL, }); @@ -111,7 +110,9 @@ export class ConfigCacheService implements OnModuleDestroy { private isCacheExpired( entry: ConfigCacheEntry | ConfigNegativeCacheEntry, ): boolean { - return Date.now() - entry.timestamp > entry.ttl; + const now = Date.now(); + + return now - entry.timestamp > entry.ttl; } private startCacheScavenging(): void { diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts index 2aead2390d1f..d7fcc4a2ac93 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts @@ -10,7 +10,6 @@ export interface ConfigCacheEntry { } export interface ConfigNegativeCacheEntry { - value: boolean; timestamp: number; ttl: number; } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index ea33ee26ee74..ca65689f3f9f 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -263,14 +263,6 @@ describe('DatabaseConfigDriver', () => { }); describe('cache operations', () => { - it('should clear specific key from cache', () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - - driver.clearCache(key); - - expect(configCache.clear).toHaveBeenCalledWith(key); - }); - it('should clear all cache', () => { driver.clearAllCache(); @@ -380,4 +372,27 @@ describe('DatabaseConfigDriver', () => { expect(environmentDriver.get).toHaveBeenCalledWith(key); }); }); + + it('should refresh config from storage', async () => { + const key = 'AUTH_PASSWORD_ENABLED'; + const value = true; + + jest.spyOn(configStorage, 'get').mockResolvedValue(value); + + await driver.refreshConfig(key); + + expect(configStorage.get).toHaveBeenCalledWith(key); + expect(configCache.set).toHaveBeenCalledWith(key, value); + }); + + it('should handle refresh when value is undefined', async () => { + const key = 'AUTH_PASSWORD_ENABLED'; + + jest.spyOn(configStorage, 'get').mockResolvedValue(undefined); + + await driver.refreshConfig(key); + + expect(configStorage.get).toHaveBeenCalledWith(key); + expect(configCache.set).not.toHaveBeenCalled(); + }); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index d1f46043dfb8..adb1033fab10 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -1,6 +1,6 @@ import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; -import { DatabaseConfigDriverInterface } from 'src/engine/core-modules/twenty-config/interfaces/database-config-driver.interface'; +import { DatabaseConfigDriverInterface } from 'src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface'; import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; @@ -29,10 +29,17 @@ export class DatabaseConfigDriver ) {} async initialize(): Promise { + if (this.initializationState === InitializationState.INITIALIZED) { + return Promise.resolve(); + } if (this.initializationPromise) { return this.initializationPromise; } + if (this.initializationState === InitializationState.FAILED) { + this.initializationState = InitializationState.NOT_INITIALIZED; + } + this.initializationPromise = this.doInitialize(); return this.initializationPromise; @@ -88,12 +95,13 @@ export class DatabaseConfigDriver ); } - await this.configStorage.set(key, value); - this.configCache.set(key, value); - } - - clearCache(key: keyof ConfigVariables): void { - this.configCache.clear(key); + try { + await this.configStorage.set(key, value); + this.configCache.set(key, value); + } catch (error) { + this.logger.error(`Failed to update config for ${key as string}`, error); + throw error; + } } clearAllCache(): void { @@ -158,16 +166,15 @@ export class DatabaseConfigDriver this.logger.debug(`🕒 [Config:${key}] Scheduling background refresh`); - try { - await new Promise((resolve, reject) => { - setImmediate(async () => { - this.logger.debug(`⏳ [Config:${key}] Executing background refresh`); - await this.refreshConfig(key).then(resolve).catch(reject); - }); + setImmediate(async () => { + this.logger.debug(`⏳ [Config:${key}] Executing background refresh`); + await this.refreshConfig(key).catch((error) => { + this.logger.error( + `Failed to refresh config for ${key as string}`, + error, + ); }); - } catch (error) { - this.logger.error(`Failed to refresh config for ${key as string}`, error); - } + }); } private scheduleRetry(): void { diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/interfaces/database-config-driver.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts similarity index 85% rename from packages/twenty-server/src/engine/core-modules/twenty-config/interfaces/database-config-driver.interface.ts rename to packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts index c558d7c72e09..3c77cc4eb36f 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/interfaces/database-config-driver.interface.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts @@ -15,11 +15,6 @@ export interface DatabaseConfigDriverInterface { */ initialize(): Promise; - /** - * Clear a specific key from cache - */ - clearCache(key: keyof ConfigVariables): void; - /** * Refresh a specific configuration from its source */ diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts similarity index 98% rename from packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.spec.ts rename to packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts index a3d0a5cca623..551c591f6e4a 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts @@ -1,7 +1,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { getRepositoryToken } from '@nestjs/typeorm'; -import { IsNull, Repository } from 'typeorm'; +import { DeleteResult, IsNull, Repository } from 'typeorm'; import { KeyValuePair, @@ -383,8 +383,8 @@ describe('ConfigStorageService', () => { jest .spyOn(keyValuePairRepository, 'delete') - .mockResolvedValueOnce({ affected: 1 } as any) - .mockResolvedValueOnce({ affected: 0 } as any); + .mockResolvedValueOnce({ affected: 1 } as DeleteResult) + .mockResolvedValueOnce({ affected: 0 } as DeleteResult); const firstDelete = service.delete(key); const secondDelete = service.delete(key); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts index a25ba0eb091e..1892be69b6aa 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts @@ -1,7 +1,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { IsNull, Repository } from 'typeorm'; +import { FindOptionsWhere, IsNull, Repository } from 'typeorm'; import { KeyValuePair, @@ -22,17 +22,23 @@ export class ConfigStorageService implements ConfigStorageInterface { private readonly keyValuePairRepository: Repository, ) {} + private getConfigVariableWhereClause( + key?: string, + ): FindOptionsWhere { + return { + type: KeyValuePairType.CONFIG_VARIABLE, + ...(key ? { key } : {}), + userId: IsNull(), + workspaceId: IsNull(), + }; + } + async get( key: T, ): Promise { try { const result = await this.keyValuePairRepository.findOne({ - where: { - type: KeyValuePairType.CONFIG_VARIABLE, - key: key as string, - userId: IsNull(), - workspaceId: IsNull(), - }, + where: this.getConfigVariableWhereClause(key as string), }); if (result === null) { @@ -72,12 +78,7 @@ export class ConfigStorageService implements ConfigStorageInterface { } const existingRecord = await this.keyValuePairRepository.findOne({ - where: { - key: key as string, - userId: IsNull(), - workspaceId: IsNull(), - type: KeyValuePairType.CONFIG_VARIABLE, - }, + where: this.getConfigVariableWhereClause(key as string), }); if (existingRecord) { @@ -102,12 +103,9 @@ export class ConfigStorageService implements ConfigStorageInterface { async delete(key: T): Promise { try { - await this.keyValuePairRepository.delete({ - type: KeyValuePairType.CONFIG_VARIABLE, - key: key as string, - userId: IsNull(), - workspaceId: IsNull(), - }); + await this.keyValuePairRepository.delete( + this.getConfigVariableWhereClause(key as string), + ); } catch (error) { this.logger.error(`Failed to delete config for ${key as string}`, error); throw error; @@ -119,11 +117,7 @@ export class ConfigStorageService implements ConfigStorageInterface { > { try { const configVars = await this.keyValuePairRepository.find({ - where: { - type: KeyValuePairType.CONFIG_VARIABLE, - userId: IsNull(), - workspaceId: IsNull(), - }, + where: this.getConfigVariableWhereClause(), }); const result = new Map< diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts index 5eaba1a9f2bb..9ae415621f21 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts @@ -173,7 +173,7 @@ export class TwentyConfigService let source = 'ENVIRONMENT'; if (isUsingDatabaseDriver && !envMetadata.isEnvOnly) { - const dbValue = this.get(key as keyof ConfigVariables); + const dbValue = value; if (dbValue !== envVars[key as keyof ConfigVariables]) { source = 'DATABASE'; @@ -212,12 +212,6 @@ export class TwentyConfigService return result; } - clearCache(key: keyof ConfigVariables): void { - if (this.driver === this.databaseConfigDriver) { - this.databaseConfigDriver.clearCache(key); - } - } - async refreshConfig(key: keyof ConfigVariables): Promise { if (this.driver === this.databaseConfigDriver) { await this.databaseConfigDriver.refreshConfig(key); From 62c93a6138ee2bb032a6a90c89732f62303135ab Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 16 Apr 2025 14:43:31 +0530 Subject: [PATCH 27/70] fix --- nx | 0 .../core-modules/twenty-config/twenty-config.service.ts | 4 +++- 2 files changed, 3 insertions(+), 1 deletion(-) mode change 100644 => 100755 nx diff --git a/nx b/nx old mode 100644 new mode 100755 diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts index 9ae415621f21..2fe8f2a59fff 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts @@ -32,7 +32,9 @@ export class TwentyConfigService ) { this.driver = this.environmentConfigDriver; - const configVarInDb = this.configService.get('IS_CONFIG_VAR_IN_DB_ENABLED'); + const configVarInDb = this.configService.get( + 'IS_CONFIG_VARIABLES_IN_DB_ENABLED', + ); this.isConfigVarInDbEnabled = configVarInDb === true; From 9b58af464cbc6d8abc1a143148907e9f1615bd07 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 16 Apr 2025 14:54:05 +0530 Subject: [PATCH 28/70] materialize config source into a enum --- .../cache/config-cache.service.ts | 17 --------- .../drivers/database-config.driver.ts | 15 -------- .../twenty-config/enums/config-source.enum.ts | 5 +++ .../twenty-config/twenty-config.service.ts | 37 ++++++++----------- 4 files changed, 21 insertions(+), 53 deletions(-) create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/enums/config-source.enum.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts index c30db088c153..2cfcfd1378df 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts @@ -30,8 +30,6 @@ export class ConfigCacheService implements OnModuleDestroy { const entry = this.valueCache.get(key); if (entry && !this.isCacheExpired(entry)) { - this.logger.debug(`🟢 [Cache:${key}] Positive cache hit`); - return entry.value as ConfigValue; } @@ -42,8 +40,6 @@ export class ConfigCacheService implements OnModuleDestroy { const entry = this.negativeLookupCache.get(key); if (entry && !this.isCacheExpired(entry)) { - this.logger.debug(`🔴 [Cache:${key}] Negative cache hit`); - return true; } @@ -57,7 +53,6 @@ export class ConfigCacheService implements OnModuleDestroy { ttl: CONFIG_VARIABLES_CACHE_TTL, }); this.negativeLookupCache.delete(key); - this.logger.debug(`✅ [Cache:${key}] Updated positive cache`); } setNegativeLookup(key: ConfigKey): void { @@ -66,19 +61,16 @@ export class ConfigCacheService implements OnModuleDestroy { ttl: CONFIG_VARIABLES_CACHE_TTL, }); this.valueCache.delete(key); - this.logger.debug(`❌ [Cache:${key}] Updated negative cache`); } clear(key: ConfigKey): void { this.valueCache.delete(key); this.negativeLookupCache.delete(key); - this.logger.debug(`🧹 [Cache:${key}] Cleared cache entries`); } clearAll(): void { this.valueCache.clear(); this.negativeLookupCache.clear(); - this.logger.debug('🧹 Cleared all cache entries'); } getCacheInfo(): { @@ -123,26 +115,17 @@ export class ConfigCacheService implements OnModuleDestroy { private scavengeCache(): void { const now = Date.now(); - let expiredCount = 0; for (const [key, entry] of this.valueCache.entries()) { if (now - entry.timestamp > entry.ttl) { this.valueCache.delete(key); - expiredCount++; } } for (const [key, entry] of this.negativeLookupCache.entries()) { if (now - entry.timestamp > entry.ttl) { this.negativeLookupCache.delete(key); - expiredCount++; } } - - if (expiredCount > 0) { - this.logger.debug( - `🧹 Cache scavenging completed: ${expiredCount} expired entries removed`, - ); - } } } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index adb1033fab10..cf169efe7dc4 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -59,14 +59,6 @@ export class DatabaseConfigDriver get(key: T): ConfigVariables[T] { if (this.shouldUseEnvironment(key)) { - this.logger.debug( - `[Config:${key}] Using env due to ${ - this.initializationState !== InitializationState.INITIALIZED - ? 'initialization state' - : 'isEnvOnly flag' - }`, - ); - return this.environmentDriver.get(key); } @@ -157,17 +149,10 @@ export class DatabaseConfigDriver private async scheduleRefresh(key: keyof ConfigVariables): Promise { if (this.initializationState !== InitializationState.INITIALIZED) { - this.logger.debug( - `[Config:${key}] Skipping refresh due to initialization state`, - ); - return; } - this.logger.debug(`🕒 [Config:${key}] Scheduling background refresh`); - setImmediate(async () => { - this.logger.debug(`⏳ [Config:${key}] Executing background refresh`); await this.refreshConfig(key).catch((error) => { this.logger.error( `Failed to refresh config for ${key as string}`, diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/enums/config-source.enum.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/enums/config-source.enum.ts new file mode 100644 index 000000000000..2dbc408ae092 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/enums/config-source.enum.ts @@ -0,0 +1,5 @@ +export enum ConfigSource { + ENVIRONMENT = 'ENVIRONMENT', + DATABASE = 'DATABASE', + DEFAULT = 'DEFAULT', +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts index 2fe8f2a59fff..5bb1fcb49d9c 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts @@ -6,11 +6,14 @@ import { } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; +import { isString } from 'class-validator'; + import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; import { CONFIG_VARIABLES_MASKING_CONFIG } from 'src/engine/core-modules/twenty-config/constants/config-variables-masking-config'; import { ConfigVariablesMetadataOptions } from 'src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator'; import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; +import { ConfigSource } from 'src/engine/core-modules/twenty-config/enums/config-source.enum'; import { ConfigVariablesMaskingStrategies } from 'src/engine/core-modules/twenty-config/enums/config-variables-masking-strategies.enum'; import { InitializationState } from 'src/engine/core-modules/twenty-config/enums/initialization-state.enum'; import { configVariableMaskSensitiveData } from 'src/engine/core-modules/twenty-config/utils/config-variable-mask-sensitive-data.util'; @@ -88,15 +91,6 @@ export class TwentyConfigService get(key: T): ConfigVariables[T] { const value = this.driver.get(key); - this.logger.debug( - `Getting value for '${key}' from ${this.driver.constructor.name}`, - { - valueType: typeof value, - isArray: Array.isArray(value), - value: typeof value === 'object' ? JSON.stringify(value) : value, - }, - ); - return value; } @@ -149,7 +143,7 @@ export class TwentyConfigService { value: ConfigVariables[keyof ConfigVariables]; metadata: ConfigVariablesMetadataOptions; - source: string; + source: ConfigSource; } > { const result: Record< @@ -157,7 +151,7 @@ export class TwentyConfigService { value: ConfigVariables[keyof ConfigVariables]; metadata: ConfigVariablesMetadataOptions; - source: string; + source: ConfigSource; } > = {}; @@ -172,21 +166,22 @@ export class TwentyConfigService Object.entries(metadata).forEach(([key, envMetadata]) => { let value = this.get(key as keyof ConfigVariables) ?? ''; - let source = 'ENVIRONMENT'; + let source = ConfigSource.ENVIRONMENT; - if (isUsingDatabaseDriver && !envMetadata.isEnvOnly) { + if (!isUsingDatabaseDriver || envMetadata.isEnvOnly) { + if (value === envVars[key as keyof ConfigVariables]) { + source = ConfigSource.DEFAULT; + } + } else { const dbValue = value; - if (dbValue !== envVars[key as keyof ConfigVariables]) { - source = 'DATABASE'; - } else { - source = 'DEFAULT'; - } - } else if (value === envVars[key as keyof ConfigVariables]) { - source = 'DEFAULT'; + source = + dbValue !== envVars[key as keyof ConfigVariables] + ? ConfigSource.DATABASE + : ConfigSource.DEFAULT; } - if (typeof value === 'string' && key in CONFIG_VARIABLES_MASKING_CONFIG) { + if (isString(value) && key in CONFIG_VARIABLES_MASKING_CONFIG) { const varMaskingConfig = CONFIG_VARIABLES_MASKING_CONFIG[ key as keyof typeof CONFIG_VARIABLES_MASKING_CONFIG From 44b1a0052b3563baa2142c0162426698127d4cb1 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 16 Apr 2025 15:00:10 +0530 Subject: [PATCH 29/70] small improvements --- .../interfaces/config-var-cache-entry.interface.ts | 10 ---------- .../twenty-config/twenty-config.service.ts | 6 +++--- 2 files changed, 3 insertions(+), 13 deletions(-) delete mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/interfaces/config-var-cache-entry.interface.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/interfaces/config-var-cache-entry.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/interfaces/config-var-cache-entry.interface.ts deleted file mode 100644 index 92ca3492ac88..000000000000 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/interfaces/config-var-cache-entry.interface.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; - -export type ConfigKey = keyof ConfigVariables; -export type ConfigValue = ConfigVariables[T]; - -export interface ConfigVarCacheEntry { - value: ConfigValue; - timestamp: number; - ttl: number; -} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts index 5bb1fcb49d9c..d90a6d4346e8 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts @@ -155,7 +155,7 @@ export class TwentyConfigService } > = {}; - const envVars = new ConfigVariables(); + const configVars = new ConfigVariables(); const metadata = TypedReflect.getMetadata('config-variables', ConfigVariables) ?? {}; @@ -169,14 +169,14 @@ export class TwentyConfigService let source = ConfigSource.ENVIRONMENT; if (!isUsingDatabaseDriver || envMetadata.isEnvOnly) { - if (value === envVars[key as keyof ConfigVariables]) { + if (value === configVars[key as keyof ConfigVariables]) { source = ConfigSource.DEFAULT; } } else { const dbValue = value; source = - dbValue !== envVars[key as keyof ConfigVariables] + dbValue !== configVars[key as keyof ConfigVariables] ? ConfigSource.DATABASE : ConfigSource.DEFAULT; } From fac8f309186d21474bfcb5a63a20c7b4e57970c4 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 16 Apr 2025 15:13:53 +0530 Subject: [PATCH 30/70] rename --- .../twenty-config/drivers/database-config.driver.ts | 11 ++++------- .../interfaces/database-config-driver.interface.ts | 4 ++-- .../twenty-config/twenty-config.service.ts | 6 ------ 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index cf169efe7dc4..8651a1bc83fa 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -100,7 +100,7 @@ export class DatabaseConfigDriver this.configCache.clearAll(); } - async refreshConfig(key: keyof ConfigVariables): Promise { + async fetchAndCacheConfig(key: keyof ConfigVariables): Promise { try { const value = await this.configStorage.get(key); @@ -110,7 +110,7 @@ export class DatabaseConfigDriver this.configCache.setNegativeLookup(key); } } catch (error) { - this.logger.error(`Failed to refresh config for ${key as string}`, error); + this.logger.error(`Failed to fetch config for ${key as string}`, error); this.configCache.setNegativeLookup(key); } } @@ -153,11 +153,8 @@ export class DatabaseConfigDriver } setImmediate(async () => { - await this.refreshConfig(key).catch((error) => { - this.logger.error( - `Failed to refresh config for ${key as string}`, - error, - ); + await this.fetchAndCacheConfig(key).catch((error) => { + this.logger.error(`Failed to fetch config for ${key as string}`, error); }); }); } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts index 3c77cc4eb36f..8924a6b65217 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts @@ -16,7 +16,7 @@ export interface DatabaseConfigDriverInterface { initialize(): Promise; /** - * Refresh a specific configuration from its source + * Fetch and cache a specific configuration from its source */ - refreshConfig(key: keyof ConfigVariables): Promise; + fetchAndCacheConfig(key: keyof ConfigVariables): Promise; } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts index d90a6d4346e8..ad8ee96f397d 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts @@ -209,12 +209,6 @@ export class TwentyConfigService return result; } - async refreshConfig(key: keyof ConfigVariables): Promise { - if (this.driver === this.databaseConfigDriver) { - await this.databaseConfigDriver.refreshConfig(key); - } - } - getCacheInfo(): { usingDatabaseDriver: boolean; initializationState: string; From c3b61e94594cee0a87a1fd399ae3b3dcd6ac3e6a Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 16 Apr 2025 15:17:54 +0530 Subject: [PATCH 31/70] fix --- .../drivers/__tests__/database-config.driver.spec.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index ca65689f3f9f..e35dcfb67b6e 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -379,7 +379,7 @@ describe('DatabaseConfigDriver', () => { jest.spyOn(configStorage, 'get').mockResolvedValue(value); - await driver.refreshConfig(key); + await driver.fetchAndCacheConfig(key); expect(configStorage.get).toHaveBeenCalledWith(key); expect(configCache.set).toHaveBeenCalledWith(key, value); @@ -390,7 +390,7 @@ describe('DatabaseConfigDriver', () => { jest.spyOn(configStorage, 'get').mockResolvedValue(undefined); - await driver.refreshConfig(key); + await driver.fetchAndCacheConfig(key); expect(configStorage.get).toHaveBeenCalledWith(key); expect(configCache.set).not.toHaveBeenCalled(); From 2182804c9d6966487988e0ea11f6cdd2511ccb3d Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 16 Apr 2025 16:01:21 +0530 Subject: [PATCH 32/70] cleaning --- .../drivers/__tests__/database-config.driver.spec.ts | 6 ------ .../twenty-config/drivers/database-config.driver.ts | 4 ---- .../interfaces/database-config-driver.interface.ts | 12 ++++++++++-- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index e35dcfb67b6e..fccd6f8e0668 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -263,12 +263,6 @@ describe('DatabaseConfigDriver', () => { }); describe('cache operations', () => { - it('should clear all cache', () => { - driver.clearAllCache(); - - expect(configCache.clearAll).toHaveBeenCalled(); - }); - it('should return cache info', () => { const cacheInfo = { positiveEntries: 2, diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index 8651a1bc83fa..b311b5792229 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -96,10 +96,6 @@ export class DatabaseConfigDriver } } - clearAllCache(): void { - this.configCache.clearAll(); - } - async fetchAndCacheConfig(key: keyof ConfigVariables): Promise { try { const value = await this.configStorage.get(key); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts index 8924a6b65217..03605e2acf23 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts @@ -5,15 +5,23 @@ import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-va * with caching and initialization capabilities */ export interface DatabaseConfigDriverInterface { + /** + * Initialize the driver + */ + initialize(): Promise; + /** * Get a configuration value */ get(key: T): ConfigVariables[T]; /** - * Initialize the driver + * Update a configuration value in the database and cache */ - initialize(): Promise; + update( + key: T, + value: ConfigVariables[T], + ): Promise; /** * Fetch and cache a specific configuration from its source From ee2131b78a714e4b30e3d40c9f684714d2d08cef Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 16 Apr 2025 16:43:23 +0530 Subject: [PATCH 33/70] fix initialization bug --- .../__tests__/database-config.driver.spec.ts | 55 +++++++++++++++++-- .../drivers/database-config.driver.ts | 27 +++++++-- 2 files changed, 74 insertions(+), 8 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index fccd6f8e0668..a46b7a479244 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -109,15 +109,31 @@ describe('DatabaseConfigDriver', () => { jest .spyOn(configStorage, 'loadAll') - .mockRejectedValueOnce(error) - .mockResolvedValueOnce(new Map()); + .mockRejectedValueOnce(error) // First call fails + .mockResolvedValueOnce(new Map()); // Second call succeeds + const originalScheduleRetry = (driver as any).scheduleRetry; + + (driver as any).scheduleRetry = jest.fn(function () { + this.initializationPromise = null; + this.initializationState = InitializationState.NOT_INITIALIZED; + this.initialize(); + }); + + // First initialization attempt (will fail and trigger retry) await driver.initialize(); + // The mock scheduleRetry should have been called + expect((driver as any).scheduleRetry).toHaveBeenCalled(); + + // Since our mock immediately calls initialize again and we've mocked + // the second attempt to succeed, the state should now be INITIALIZED expect((driver as any).initializationState).toBe( - InitializationState.FAILED, + InitializationState.INITIALIZED, ); - expect((driver as any).retryAttempts).toBeGreaterThan(0); + + // Restore original method + (driver as any).scheduleRetry = originalScheduleRetry; }); it('should handle concurrent initialization', async () => { @@ -137,6 +153,37 @@ describe('DatabaseConfigDriver', () => { expect(configStorage.loadAll).toHaveBeenCalledTimes(1); }); + + it('should reset retry attempts after successful initialization', async () => { + (driver as any).retryAttempts = 3; + + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(new Map()); + + await driver.initialize(); + + expect((driver as any).retryAttempts).toBe(0); + expect((driver as any).initializationState).toBe( + InitializationState.INITIALIZED, + ); + }); + + it('should increment retry attempts when initialization fails', async () => { + const error = new Error('DB error'); + + (driver as any).retryAttempts = 0; + + const originalScheduleRetry = (driver as any).scheduleRetry; + + (driver as any).scheduleRetry = jest.fn(); + + jest.spyOn(configStorage, 'loadAll').mockRejectedValue(error); + + await driver.initialize(); + + expect((driver as any).retryAttempts).toBe(1); + + (driver as any).scheduleRetry = originalScheduleRetry; + }); }); describe('get', () => { diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index b311b5792229..49d7ce04f603 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -50,10 +50,20 @@ export class DatabaseConfigDriver this.initializationState = InitializationState.INITIALIZING; await this.loadAllConfigVarsFromDb(); this.initializationState = InitializationState.INITIALIZED; + // Reset retry attempts on successful initialization + this.retryAttempts = 0; } catch (error) { - this.logger.error('Failed to initialize database driver', error); + this.retryAttempts++; + + this.logger.error( + `Failed to initialize database driver (attempt ${this.retryAttempts})`, + error instanceof Error ? error.stack : error, + ); this.initializationState = InitializationState.FAILED; this.scheduleRetry(); + } finally { + // Reset the promise to allow future initialization attempts + this.initializationPromise = null; } } @@ -161,7 +171,9 @@ export class DatabaseConfigDriver if ( this.retryAttempts > DATABASE_CONFIG_DRIVER_INITIALIZATION_MAX_RETRIES ) { - this.logger.error('Max retry attempts reached, giving up initialization'); + this.logger.error( + `Max retry attempts (${DATABASE_CONFIG_DRIVER_INITIALIZATION_MAX_RETRIES}) reached, giving up initialization`, + ); return; } @@ -170,14 +182,21 @@ export class DatabaseConfigDriver DATABASE_CONFIG_DRIVER_INITIAL_RETRY_DELAY * Math.pow(2, this.retryAttempts - 1); + this.logger.log( + `Scheduling retry attempt ${this.retryAttempts} in ${delay}ms`, + ); + if (this.retryTimer) { clearTimeout(this.retryTimer); } this.retryTimer = setTimeout(() => { - this.initializationPromise = null; + this.logger.log(`Executing retry attempt ${this.retryAttempts}`); this.initialize().catch((error) => { - this.logger.error('Retry initialization failed', error); + this.logger.error( + `Retry initialization attempt ${this.retryAttempts} failed`, + error instanceof Error ? error.stack : error, + ); }); }, delay); } From a409d2104a03606b1b18bfbea9d26d8a6d07be1d Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 16 Apr 2025 17:06:08 +0530 Subject: [PATCH 34/70] add service test --- .../twenty-config.service.spec.ts | 497 ++++++++++++++++++ 1 file changed, 497 insertions(+) create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts new file mode 100644 index 000000000000..2ede06811fed --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts @@ -0,0 +1,497 @@ +import { ConfigService } from '@nestjs/config'; +import { Test, TestingModule } from '@nestjs/testing'; + +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; +import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; +import { ConfigSource } from 'src/engine/core-modules/twenty-config/enums/config-source.enum'; +import { ConfigVariablesGroup } from 'src/engine/core-modules/twenty-config/enums/config-variables-group.enum'; +import { InitializationState } from 'src/engine/core-modules/twenty-config/enums/initialization-state.enum'; +import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; +import { TypedReflect } from 'src/utils/typed-reflect'; + +jest.mock('src/utils/typed-reflect', () => ({ + TypedReflect: { + getMetadata: jest.fn(), + defineMetadata: jest.fn(), + }, +})); + +jest.mock( + 'src/engine/core-modules/twenty-config/constants/config-variables-masking-config', + () => ({ + CONFIG_VARIABLES_MASKING_CONFIG: { + SENSITIVE_VAR: { + strategy: 'LAST_N_CHARS', + chars: 5, + }, + }, + }), +); + +type TwentyConfigServicePrivateProps = { + driver: DatabaseConfigDriver | EnvironmentConfigDriver; + isConfigVarInDbEnabled: boolean; + initializationState: InitializationState; +}; + +const mockConfigVarMetadata = { + TEST_VAR: { + group: ConfigVariablesGroup.GoogleAuth, + description: 'Test variable', + isEnvOnly: false, + }, + ENV_ONLY_VAR: { + group: ConfigVariablesGroup.StorageConfig, + description: 'Environment only variable', + isEnvOnly: true, + }, + SENSITIVE_VAR: { + group: ConfigVariablesGroup.Logging, + description: 'Sensitive variable', + isSensitive: true, + }, +}; + +const setupTestModule = async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + TwentyConfigService, + { + provide: ConfigService, + useValue: { + get: jest.fn(), + }, + }, + { + provide: DatabaseConfigDriver, + useValue: { + initialize: jest.fn().mockResolvedValue(undefined), + get: jest.fn(), + update: jest.fn(), + getCacheInfo: jest.fn(), + }, + }, + { + provide: EnvironmentConfigDriver, + useValue: { + get: jest.fn(), + }, + }, + ], + }).compile(); + + return { + service: module.get(TwentyConfigService), + configService: module.get(ConfigService), + databaseConfigDriver: + module.get(DatabaseConfigDriver), + environmentConfigDriver: module.get( + EnvironmentConfigDriver, + ), + }; +}; + +const setPrivateProps = ( + service: TwentyConfigService, + props: Partial, +) => { + Object.entries(props).forEach(([key, value]) => { + Object.defineProperty(service, key, { + value, + writable: true, + }); + }); +}; + +describe('TwentyConfigService', () => { + let service: TwentyConfigService; + let configService: ConfigService; + let databaseConfigDriver: DatabaseConfigDriver; + let environmentConfigDriver: EnvironmentConfigDriver; + + beforeEach(async () => { + const testModule = await setupTestModule(); + + service = testModule.service; + configService = testModule.configService; + databaseConfigDriver = testModule.databaseConfigDriver; + environmentConfigDriver = testModule.environmentConfigDriver; + + (TypedReflect.getMetadata as jest.Mock).mockReturnValue( + mockConfigVarMetadata, + ); + }); + + afterEach(() => { + jest.clearAllMocks(); + }); + + it('should be defined', () => { + expect(service).toBeDefined(); + }); + + describe('constructor', () => { + it('should initialize with environment driver when database config is disabled', () => { + jest.spyOn(configService, 'get').mockReturnValue(false); + + const newService = new TwentyConfigService( + configService, + databaseConfigDriver, + environmentConfigDriver, + ); + + const privateProps = + newService as unknown as TwentyConfigServicePrivateProps; + + expect(privateProps.driver).toBe(environmentConfigDriver); + expect(privateProps.isConfigVarInDbEnabled).toBe(false); + }); + + it('should initialize with environment driver initially when database config is enabled', () => { + jest.spyOn(configService, 'get').mockReturnValue(true); + + const newService = new TwentyConfigService( + configService, + databaseConfigDriver, + environmentConfigDriver, + ); + + const privateProps = + newService as unknown as TwentyConfigServicePrivateProps; + + expect(privateProps.driver).toBe(environmentConfigDriver); + expect(privateProps.isConfigVarInDbEnabled).toBe(true); + }); + }); + + describe('onModuleInit', () => { + it('should set initialization state to INITIALIZED when db config is disabled', async () => { + jest.spyOn(configService, 'get').mockReturnValue(false); + + const newService = new TwentyConfigService( + configService, + databaseConfigDriver, + environmentConfigDriver, + ); + + await newService.onModuleInit(); + + const privateProps = + newService as unknown as TwentyConfigServicePrivateProps; + + expect(privateProps.initializationState).toBe( + InitializationState.INITIALIZED, + ); + }); + + it('should not change initialization state when db config is enabled', async () => { + jest.spyOn(configService, 'get').mockReturnValue(true); + + const newService = new TwentyConfigService( + configService, + databaseConfigDriver, + environmentConfigDriver, + ); + + await newService.onModuleInit(); + + const privateProps = + newService as unknown as TwentyConfigServicePrivateProps; + + expect(privateProps.initializationState).toBe( + InitializationState.NOT_INITIALIZED, + ); + }); + }); + + describe('onApplicationBootstrap', () => { + it('should do nothing when db config is disabled', async () => { + jest.spyOn(configService, 'get').mockReturnValue(false); + + const newService = new TwentyConfigService( + configService, + databaseConfigDriver, + environmentConfigDriver, + ); + + await newService.onApplicationBootstrap(); + + expect(databaseConfigDriver.initialize).not.toHaveBeenCalled(); + + const privateProps = + newService as unknown as TwentyConfigServicePrivateProps; + + expect(privateProps.driver).toBe(environmentConfigDriver); + }); + + it('should initialize database driver when db config is enabled', async () => { + jest.spyOn(configService, 'get').mockReturnValue(true); + + const newService = new TwentyConfigService( + configService, + databaseConfigDriver, + environmentConfigDriver, + ); + + await newService.onApplicationBootstrap(); + + expect(databaseConfigDriver.initialize).toHaveBeenCalled(); + + const privateProps = + newService as unknown as TwentyConfigServicePrivateProps; + + expect(privateProps.driver).toBe(databaseConfigDriver); + expect(privateProps.initializationState).toBe( + InitializationState.INITIALIZED, + ); + }); + + it('should fall back to environment driver when database initialization fails', async () => { + jest.spyOn(configService, 'get').mockReturnValue(true); + jest + .spyOn(databaseConfigDriver, 'initialize') + .mockRejectedValue(new Error('DB initialization failed')); + + const newService = new TwentyConfigService( + configService, + databaseConfigDriver, + environmentConfigDriver, + ); + + await newService.onApplicationBootstrap(); + + expect(databaseConfigDriver.initialize).toHaveBeenCalled(); + + const privateProps = + newService as unknown as TwentyConfigServicePrivateProps; + + expect(privateProps.driver).toBe(environmentConfigDriver); + expect(privateProps.initializationState).toBe(InitializationState.FAILED); + }); + }); + + describe('get', () => { + it('should delegate to the active driver', () => { + const key = 'TEST_VAR' as keyof ConfigVariables; + const expectedValue = 'test value'; + + jest.spyOn(databaseConfigDriver, 'get').mockReturnValue(expectedValue); + + setPrivateProps(service, { driver: databaseConfigDriver }); + + const result = service.get(key); + + expect(result).toBe(expectedValue); + expect(databaseConfigDriver.get).toHaveBeenCalledWith(key); + }); + }); + + describe('update', () => { + const setupUpdateTest = ( + props: Partial, + ) => { + setPrivateProps(service, { + isConfigVarInDbEnabled: true, + initializationState: InitializationState.INITIALIZED, + driver: databaseConfigDriver, + ...props, + }); + }; + + it('should throw error when database config is disabled', async () => { + setupUpdateTest({ isConfigVarInDbEnabled: false }); + + await expect( + service.update('TEST_VAR' as keyof ConfigVariables, 'new value'), + ).rejects.toThrow( + 'Database configuration is disabled, cannot update configuration', + ); + }); + + it('should throw error when not initialized', async () => { + setupUpdateTest({ + initializationState: InitializationState.NOT_INITIALIZED, + }); + + await expect( + service.update('TEST_VAR' as keyof ConfigVariables, 'new value'), + ).rejects.toThrow( + 'TwentyConfigService not initialized, cannot update configuration', + ); + }); + + it('should throw error when updating environment-only variable', async () => { + setupUpdateTest({}); + + await expect( + service.update('ENV_ONLY_VAR' as keyof ConfigVariables, 'new value'), + ).rejects.toThrow( + 'Cannot update environment-only variable: ENV_ONLY_VAR', + ); + }); + + it('should throw error when driver is not DatabaseConfigDriver', async () => { + setupUpdateTest({ driver: environmentConfigDriver }); + + await expect( + service.update('TEST_VAR' as keyof ConfigVariables, 'new value'), + ).rejects.toThrow( + 'Database driver not initialized, cannot update configuration', + ); + }); + + it('should update config when all conditions are met', async () => { + const key = 'TEST_VAR' as keyof ConfigVariables; + const newValue = 'new value'; + + setupUpdateTest({}); + jest.spyOn(databaseConfigDriver, 'update').mockResolvedValue(undefined); + + await service.update(key, newValue); + + expect(databaseConfigDriver.update).toHaveBeenCalledWith(key, newValue); + }); + }); + + describe('getMetadata', () => { + it('should return metadata for a config variable', () => { + const result = service.getMetadata('TEST_VAR' as keyof ConfigVariables); + + expect(result).toEqual(mockConfigVarMetadata.TEST_VAR); + }); + + it('should return undefined when metadata does not exist', () => { + const result = service.getMetadata( + 'UNKNOWN_VAR' as keyof ConfigVariables, + ); + + expect(result).toBeUndefined(); + }); + }); + + describe('getAll', () => { + const setupDriverMocks = () => { + jest + .spyOn(environmentConfigDriver, 'get') + .mockImplementation((key: keyof ConfigVariables) => { + const keyStr = String(key); + const values = { + TEST_VAR: 'env test value', + ENV_ONLY_VAR: 'env only value', + SENSITIVE_VAR: 'sensitive_data_123', + }; + + return values[keyStr] || ''; + }); + + jest + .spyOn(databaseConfigDriver, 'get') + .mockImplementation((key: keyof ConfigVariables) => { + const keyStr = String(key); + const values = { + TEST_VAR: 'db test value', + SENSITIVE_VAR: 'sensitive_data_123', + }; + + return values[keyStr] || ''; + }); + }; + + beforeEach(() => { + setupDriverMocks(); + }); + + it('should return all config variables with environment source when using environment driver', () => { + setPrivateProps(service, { + driver: environmentConfigDriver, + isConfigVarInDbEnabled: false, + initializationState: InitializationState.INITIALIZED, + }); + + const result = service.getAll(); + + expect(result).toEqual({ + TEST_VAR: { + value: 'env test value', + metadata: mockConfigVarMetadata.TEST_VAR, + source: ConfigSource.ENVIRONMENT, + }, + ENV_ONLY_VAR: { + value: 'env only value', + metadata: mockConfigVarMetadata.ENV_ONLY_VAR, + source: ConfigSource.ENVIRONMENT, + }, + SENSITIVE_VAR: { + value: expect.any(String), + metadata: mockConfigVarMetadata.SENSITIVE_VAR, + source: ConfigSource.ENVIRONMENT, + }, + }); + + expect(result.SENSITIVE_VAR.value).not.toBe('sensitive_data_123'); + }); + + it('should return config variables with database source when using database driver', () => { + setPrivateProps(service, { + driver: databaseConfigDriver, + isConfigVarInDbEnabled: true, + initializationState: InitializationState.INITIALIZED, + }); + + const result = service.getAll(); + + expect(result.TEST_VAR.source).toBe(ConfigSource.DATABASE); + expect(result.ENV_ONLY_VAR.source).toBe(ConfigSource.ENVIRONMENT); + }); + }); + + describe('getCacheInfo', () => { + const setupCacheInfoTest = ( + props: Partial, + ) => { + setPrivateProps(service, { + driver: environmentConfigDriver, + isConfigVarInDbEnabled: false, + initializationState: InitializationState.INITIALIZED, + ...props, + }); + }; + + it('should return basic info when not using database driver', () => { + setupCacheInfoTest({}); + + const result = service.getCacheInfo(); + + expect(result).toEqual({ + usingDatabaseDriver: false, + initializationState: 'INITIALIZED', + }); + }); + + it('should return cache stats when using database driver', () => { + const cacheStats = { + positiveEntries: 2, + negativeEntries: 1, + cacheKeys: ['TEST_VAR', 'SENSITIVE_VAR'], + }; + + setupCacheInfoTest({ + driver: databaseConfigDriver, + isConfigVarInDbEnabled: true, + }); + + jest + .spyOn(databaseConfigDriver, 'getCacheInfo') + .mockReturnValue(cacheStats); + + const result = service.getCacheInfo(); + + expect(result).toEqual({ + usingDatabaseDriver: true, + initializationState: 'INITIALIZED', + cacheStats, + }); + }); + }); +}); From 51fdd5c5cd35d428f2841dbcf05c1f98b8ec76da Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 16 Apr 2025 17:37:05 +0530 Subject: [PATCH 35/70] grep review --- .../__tests__/config-cache.service.spec.ts | 2 -- .../cache/config-cache.service.ts | 3 +-- .../__tests__/database-config.driver.spec.ts | 8 ++++-- .../drivers/database-config.driver.ts | 13 +++++++--- .../twenty-config.service.spec.ts | 25 ++++++++++++++++--- 5 files changed, 39 insertions(+), 12 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts index 53aa000b9f0b..b5266a90a834 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts @@ -198,8 +198,6 @@ describe('ConfigCacheService', () => { describe('module lifecycle', () => { it('should clean up interval on module destroy', () => { - const clearIntervalSpy = jest.spyOn(global, 'clearInterval'); - service.onModuleDestroy(); expect(clearIntervalSpy).toHaveBeenCalled(); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts index 2cfcfd1378df..0608de41551a 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts @@ -1,4 +1,4 @@ -import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; +import { Injectable, OnModuleDestroy } from '@nestjs/common'; import { CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval'; import { CONFIG_VARIABLES_CACHE_TTL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl'; @@ -18,7 +18,6 @@ export class ConfigCacheService implements OnModuleDestroy { ConfigNegativeCacheEntry >; private cacheScavengeInterval: NodeJS.Timeout; - private readonly logger = new Logger(ConfigCacheService.name); constructor() { this.valueCache = new Map(); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index a46b7a479244..eb138acc40cf 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -172,15 +172,19 @@ describe('DatabaseConfigDriver', () => { (driver as any).retryAttempts = 0; + const scheduleRetrySpy = jest.fn(); const originalScheduleRetry = (driver as any).scheduleRetry; - (driver as any).scheduleRetry = jest.fn(); + (driver as any).scheduleRetry = scheduleRetrySpy; jest.spyOn(configStorage, 'loadAll').mockRejectedValue(error); await driver.initialize(); - expect((driver as any).retryAttempts).toBe(1); + expect(scheduleRetrySpy).toHaveBeenCalled(); + expect((driver as any).initializationState).toBe( + InitializationState.FAILED, + ); (driver as any).scheduleRetry = originalScheduleRetry; }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index 49d7ce04f603..f4df0d3ed175 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -53,10 +53,8 @@ export class DatabaseConfigDriver // Reset retry attempts on successful initialization this.retryAttempts = 0; } catch (error) { - this.retryAttempts++; - this.logger.error( - `Failed to initialize database driver (attempt ${this.retryAttempts})`, + `Failed to initialize database driver (attempt ${this.retryAttempts + 1})`, error instanceof Error ? error.stack : error, ); this.initializationState = InitializationState.FAILED; @@ -192,6 +190,15 @@ export class DatabaseConfigDriver this.retryTimer = setTimeout(() => { this.logger.log(`Executing retry attempt ${this.retryAttempts}`); + + if (this.initializationState === InitializationState.INITIALIZING) { + this.logger.log( + 'Skipping retry attempt as initialization is already in progress', + ); + + return; + } + this.initialize().catch((error) => { this.logger.error( `Retry initialization attempt ${this.retryAttempts} failed`, diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts index 2ede06811fed..5f38ed8a148f 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts @@ -389,6 +389,10 @@ describe('TwentyConfigService', () => { .spyOn(databaseConfigDriver, 'get') .mockImplementation((key: keyof ConfigVariables) => { const keyStr = String(key); + + if (mockConfigVarMetadata[keyStr]?.isEnvOnly) { + return environmentConfigDriver.get(key); + } const values = { TEST_VAR: 'db test value', SENSITIVE_VAR: 'sensitive_data_123', @@ -429,7 +433,7 @@ describe('TwentyConfigService', () => { }, }); - expect(result.SENSITIVE_VAR.value).not.toBe('sensitive_data_123'); + expect(result.SENSITIVE_VAR.value).toBe('********a_123'); }); it('should return config variables with database source when using database driver', () => { @@ -441,8 +445,23 @@ describe('TwentyConfigService', () => { const result = service.getAll(); - expect(result.TEST_VAR.source).toBe(ConfigSource.DATABASE); - expect(result.ENV_ONLY_VAR.source).toBe(ConfigSource.ENVIRONMENT); + expect(result.TEST_VAR).toEqual({ + value: 'db test value', + metadata: mockConfigVarMetadata.TEST_VAR, + source: ConfigSource.DATABASE, + }); + + expect(result.ENV_ONLY_VAR).toEqual({ + value: 'env only value', + metadata: mockConfigVarMetadata.ENV_ONLY_VAR, + source: ConfigSource.ENVIRONMENT, + }); + + expect(result.SENSITIVE_VAR).toEqual({ + value: '********a_123', + metadata: mockConfigVarMetadata.SENSITIVE_VAR, + source: ConfigSource.DATABASE, + }); }); }); From 0ca0bf1b051ab9d1537cceb0b47eb78242155976 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Thu, 17 Apr 2025 00:06:25 +0530 Subject: [PATCH 36/70] rename --- .../__tests__/config-cache.service.spec.ts | 52 +++++++++++------ .../cache/config-cache.service.ts | 57 ++++++++++--------- .../config-cache-entry.interface.ts | 2 +- .../__tests__/database-config.driver.spec.ts | 45 ++++++++------- .../drivers/database-config.driver.ts | 50 +++++++++------- ...ts => config-initialization-state.enum.ts} | 2 +- .../twenty-config.service.spec.ts | 30 +++++----- .../twenty-config/twenty-config.service.ts | 23 ++++---- 8 files changed, 151 insertions(+), 110 deletions(-) rename packages/twenty-server/src/engine/core-modules/twenty-config/enums/{initialization-state.enum.ts => config-initialization-state.enum.ts} (75%) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts index b5266a90a834..900af29fd688 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts @@ -79,31 +79,32 @@ describe('ConfigCacheService', () => { }); describe('negative lookup cache', () => { - it('should set and get negative lookup', () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + it('should check if a negative cache entry exists', () => { + const key = 'TEST_KEY' as any; - service.setNegativeLookup(key); - const result = service.getNegativeLookup(key); + service.markKeyAsMissing(key); + const result = service.isKeyKnownMissing(key); expect(result).toBe(true); }); - it('should return false for non-existent negative lookup', () => { - const result = service.getNegativeLookup( - 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, - ); + it('should return false for negative cache entry check when not in cache', () => { + const key = 'NON_EXISTENT_KEY' as any; + + const result = service.isKeyKnownMissing(key); expect(result).toBe(false); }); - it('should clear negative lookup when setting a value', () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + it('should return false for negative cache entry check when expired', () => { + const key = 'TEST_KEY' as any; - service.setNegativeLookup(key); - service.set(key, true); + service.markKeyAsMissing(key); - expect(service.getNegativeLookup(key)).toBe(false); - expect(service.get(key)).toBe(true); + // Mock a date beyond the TTL + jest.spyOn(Date, 'now').mockReturnValueOnce(Date.now() + 1000000); + + expect(service.isKeyKnownMissing(key)).toBe(false); }); }); @@ -169,7 +170,7 @@ describe('ConfigCacheService', () => { service.set(key1, true); service.set(key2, 'test@example.com'); - service.setNegativeLookup(key3); + service.markKeyAsMissing(key3); const info = service.getCacheInfo(); @@ -178,7 +179,7 @@ describe('ConfigCacheService', () => { expect(info.cacheKeys).toContain(key1); expect(info.cacheKeys).toContain(key2); expect(info.cacheKeys).not.toContain(key3); - expect(service.getNegativeLookup(key3)).toBe(true); + expect(service.isKeyKnownMissing(key3)).toBe(true); }); it('should not include expired entries in cache info', () => { @@ -194,6 +195,25 @@ describe('ConfigCacheService', () => { expect(info.cacheKeys).toHaveLength(0); }); }); + + it('should properly count cache entries', () => { + const key1 = 'KEY1' as any; + const key2 = 'KEY2' as any; + const key3 = 'KEY3' as any; + + // Add some values to the cache + service.set(key1, 'value1'); + service.set(key2, 'value2'); + service.markKeyAsMissing(key3); + + const cacheInfo = service.getCacheInfo(); + + expect(cacheInfo.positiveEntries).toBe(2); + expect(cacheInfo.negativeEntries).toBe(1); + expect(cacheInfo.cacheKeys).toContain(key1); + expect(cacheInfo.cacheKeys).toContain(key2); + expect(service.isKeyKnownMissing(key3)).toBe(true); + }); }); describe('module lifecycle', () => { diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts index 0608de41551a..f4cd9be320e1 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts @@ -6,27 +6,30 @@ import { CONFIG_VARIABLES_CACHE_TTL } from 'src/engine/core-modules/twenty-confi import { ConfigCacheEntry, ConfigKey, - ConfigNegativeCacheEntry, + ConfigKnownMissingEntry, ConfigValue, } from './interfaces/config-cache-entry.interface'; @Injectable() export class ConfigCacheService implements OnModuleDestroy { - private readonly valueCache: Map>; - private readonly negativeLookupCache: Map< + private readonly foundConfigValuesCache: Map< ConfigKey, - ConfigNegativeCacheEntry + ConfigCacheEntry + >; + private readonly knownMissingKeysCache: Map< + ConfigKey, + ConfigKnownMissingEntry >; private cacheScavengeInterval: NodeJS.Timeout; constructor() { - this.valueCache = new Map(); - this.negativeLookupCache = new Map(); + this.foundConfigValuesCache = new Map(); + this.knownMissingKeysCache = new Map(); this.startCacheScavenging(); } get(key: T): ConfigValue | undefined { - const entry = this.valueCache.get(key); + const entry = this.foundConfigValuesCache.get(key); if (entry && !this.isCacheExpired(entry)) { return entry.value as ConfigValue; @@ -35,8 +38,8 @@ export class ConfigCacheService implements OnModuleDestroy { return undefined; } - getNegativeLookup(key: ConfigKey): boolean { - const entry = this.negativeLookupCache.get(key); + isKeyKnownMissing(key: ConfigKey): boolean { + const entry = this.knownMissingKeysCache.get(key); if (entry && !this.isCacheExpired(entry)) { return true; @@ -46,30 +49,30 @@ export class ConfigCacheService implements OnModuleDestroy { } set(key: T, value: ConfigValue): void { - this.valueCache.set(key, { + this.foundConfigValuesCache.set(key, { value, timestamp: Date.now(), ttl: CONFIG_VARIABLES_CACHE_TTL, }); - this.negativeLookupCache.delete(key); + this.knownMissingKeysCache.delete(key); } - setNegativeLookup(key: ConfigKey): void { - this.negativeLookupCache.set(key, { + markKeyAsMissing(key: ConfigKey): void { + this.knownMissingKeysCache.set(key, { timestamp: Date.now(), ttl: CONFIG_VARIABLES_CACHE_TTL, }); - this.valueCache.delete(key); + this.foundConfigValuesCache.delete(key); } clear(key: ConfigKey): void { - this.valueCache.delete(key); - this.negativeLookupCache.delete(key); + this.foundConfigValuesCache.delete(key); + this.knownMissingKeysCache.delete(key); } clearAll(): void { - this.valueCache.clear(); - this.negativeLookupCache.clear(); + this.foundConfigValuesCache.clear(); + this.knownMissingKeysCache.clear(); } getCacheInfo(): { @@ -77,12 +80,12 @@ export class ConfigCacheService implements OnModuleDestroy { negativeEntries: number; cacheKeys: string[]; } { - const validPositiveEntries = Array.from(this.valueCache.entries()).filter( - ([_, entry]) => !this.isCacheExpired(entry), - ); + const validPositiveEntries = Array.from( + this.foundConfigValuesCache.entries(), + ).filter(([_, entry]) => !this.isCacheExpired(entry)); const validNegativeEntries = Array.from( - this.negativeLookupCache.entries(), + this.knownMissingKeysCache.entries(), ).filter(([_, entry]) => !this.isCacheExpired(entry)); return { @@ -99,7 +102,7 @@ export class ConfigCacheService implements OnModuleDestroy { } private isCacheExpired( - entry: ConfigCacheEntry | ConfigNegativeCacheEntry, + entry: ConfigCacheEntry | ConfigKnownMissingEntry, ): boolean { const now = Date.now(); @@ -115,15 +118,15 @@ export class ConfigCacheService implements OnModuleDestroy { private scavengeCache(): void { const now = Date.now(); - for (const [key, entry] of this.valueCache.entries()) { + for (const [key, entry] of this.foundConfigValuesCache.entries()) { if (now - entry.timestamp > entry.ttl) { - this.valueCache.delete(key); + this.foundConfigValuesCache.delete(key); } } - for (const [key, entry] of this.negativeLookupCache.entries()) { + for (const [key, entry] of this.knownMissingKeysCache.entries()) { if (now - entry.timestamp > entry.ttl) { - this.negativeLookupCache.delete(key); + this.knownMissingKeysCache.delete(key); } } } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts index d7fcc4a2ac93..1f6a9418e23c 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts @@ -9,7 +9,7 @@ export interface ConfigCacheEntry { ttl: number; } -export interface ConfigNegativeCacheEntry { +export interface ConfigKnownMissingEntry { timestamp: number; ttl: number; } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index eb138acc40cf..9cf4b293c4a6 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -4,7 +4,7 @@ import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/ import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; -import { InitializationState } from 'src/engine/core-modules/twenty-config/enums/initialization-state.enum'; +import { ConfigInitializationState } from 'src/engine/core-modules/twenty-config/enums/config-initialization-state.enum'; import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; @@ -32,8 +32,8 @@ describe('DatabaseConfigDriver', () => { set: jest.fn(), clear: jest.fn(), clearAll: jest.fn(), - getNegativeLookup: jest.fn(), - setNegativeLookup: jest.fn(), + isKeyKnownMissing: jest.fn(), + markKeyAsMissing: jest.fn(), getCacheInfo: jest.fn(), }, }, @@ -98,8 +98,8 @@ describe('DatabaseConfigDriver', () => { await driver.initialize(); - expect((driver as any).initializationState).toBe( - InitializationState.FAILED, + expect((driver as any).configInitializationState).toBe( + ConfigInitializationState.FAILED, ); expect(configStorage.loadAll).toHaveBeenCalled(); }); @@ -115,8 +115,9 @@ describe('DatabaseConfigDriver', () => { const originalScheduleRetry = (driver as any).scheduleRetry; (driver as any).scheduleRetry = jest.fn(function () { - this.initializationPromise = null; - this.initializationState = InitializationState.NOT_INITIALIZED; + this.configInitializationPromise = null; + this.configInitializationState = + ConfigInitializationState.NOT_INITIALIZED; this.initialize(); }); @@ -128,8 +129,8 @@ describe('DatabaseConfigDriver', () => { // Since our mock immediately calls initialize again and we've mocked // the second attempt to succeed, the state should now be INITIALIZED - expect((driver as any).initializationState).toBe( - InitializationState.INITIALIZED, + expect((driver as any).configInitializationState).toBe( + ConfigInitializationState.INITIALIZED, ); // Restore original method @@ -162,8 +163,8 @@ describe('DatabaseConfigDriver', () => { await driver.initialize(); expect((driver as any).retryAttempts).toBe(0); - expect((driver as any).initializationState).toBe( - InitializationState.INITIALIZED, + expect((driver as any).configInitializationState).toBe( + ConfigInitializationState.INITIALIZED, ); }); @@ -182,8 +183,8 @@ describe('DatabaseConfigDriver', () => { await driver.initialize(); expect(scheduleRetrySpy).toHaveBeenCalled(); - expect((driver as any).initializationState).toBe( - InitializationState.FAILED, + expect((driver as any).configInitializationState).toBe( + ConfigInitializationState.FAILED, ); (driver as any).scheduleRetry = originalScheduleRetry; @@ -201,7 +202,8 @@ describe('DatabaseConfigDriver', () => { const envValue = true; jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); - (driver as any).initializationState = InitializationState.NOT_INITIALIZED; + (driver as any).configInitializationState = + ConfigInitializationState.NOT_INITIALIZED; const result = driver.get(key); @@ -240,7 +242,7 @@ describe('DatabaseConfigDriver', () => { const envValue = true; jest.spyOn(configCache, 'get').mockReturnValue(undefined); - jest.spyOn(configCache, 'getNegativeLookup').mockReturnValue(true); + jest.spyOn(configCache, 'isKeyKnownMissing').mockReturnValue(true); jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); const result = driver.get(key); @@ -307,7 +309,8 @@ describe('DatabaseConfigDriver', () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const value = true; - (driver as any).initializationState = InitializationState.NOT_INITIALIZED; + (driver as any).configInitializationState = + ConfigInitializationState.NOT_INITIALIZED; await expect(driver.update(key, value)).rejects.toThrow(); }); @@ -351,7 +354,8 @@ describe('DatabaseConfigDriver', () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const envValue = true; - (driver as any).initializationState = InitializationState.NOT_INITIALIZED; + (driver as any).configInitializationState = + ConfigInitializationState.NOT_INITIALIZED; jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); const result = driver.get(key); @@ -381,7 +385,7 @@ describe('DatabaseConfigDriver', () => { const envValue = true; jest.spyOn(configCache, 'get').mockReturnValue(undefined); - jest.spyOn(configCache, 'getNegativeLookup').mockReturnValue(true); + jest.spyOn(configCache, 'isKeyKnownMissing').mockReturnValue(true); jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); const result = driver.get(key); @@ -389,7 +393,7 @@ describe('DatabaseConfigDriver', () => { expect(result).toBe(envValue); expect(environmentDriver.get).toHaveBeenCalledWith(key); expect(configCache.get).toHaveBeenCalledWith(key); - expect(configCache.getNegativeLookup).toHaveBeenCalledWith(key); + expect(configCache.isKeyKnownMissing).toHaveBeenCalledWith(key); }); it('should propagate cache service errors when no fallback conditions are met', async () => { @@ -408,7 +412,8 @@ describe('DatabaseConfigDriver', () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; // Force use of environment driver - (driver as any).initializationState = InitializationState.NOT_INITIALIZED; + (driver as any).configInitializationState = + ConfigInitializationState.NOT_INITIALIZED; jest.spyOn(environmentDriver, 'get').mockImplementation(() => { throw new Error('Environment error'); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index f4df0d3ed175..93d7982abdad 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -6,7 +6,7 @@ import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/ import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; import { DATABASE_CONFIG_DRIVER_INITIAL_RETRY_DELAY } from 'src/engine/core-modules/twenty-config/constants/database-config-driver-initial-retry-delay'; import { DATABASE_CONFIG_DRIVER_INITIALIZATION_MAX_RETRIES } from 'src/engine/core-modules/twenty-config/constants/database-config-driver-initialization-max-retries'; -import { InitializationState } from 'src/engine/core-modules/twenty-config/enums/initialization-state.enum'; +import { ConfigInitializationState } from 'src/engine/core-modules/twenty-config/enums/config-initialization-state.enum'; import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; @@ -16,8 +16,8 @@ import { EnvironmentConfigDriver } from './environment-config.driver'; export class DatabaseConfigDriver implements DatabaseConfigDriverInterface, OnModuleDestroy { - private initializationState = InitializationState.NOT_INITIALIZED; - private initializationPromise: Promise | null = null; + private configInitializationState = ConfigInitializationState.NOT_INITIALIZED; + private configInitializationPromise: Promise | null = null; private retryAttempts = 0; private retryTimer?: NodeJS.Timeout; private readonly logger = new Logger(DatabaseConfigDriver.name); @@ -29,27 +29,30 @@ export class DatabaseConfigDriver ) {} async initialize(): Promise { - if (this.initializationState === InitializationState.INITIALIZED) { + if ( + this.configInitializationState === ConfigInitializationState.INITIALIZED + ) { return Promise.resolve(); } - if (this.initializationPromise) { - return this.initializationPromise; + if (this.configInitializationPromise) { + return this.configInitializationPromise; } - if (this.initializationState === InitializationState.FAILED) { - this.initializationState = InitializationState.NOT_INITIALIZED; + if (this.configInitializationState === ConfigInitializationState.FAILED) { + this.configInitializationState = + ConfigInitializationState.NOT_INITIALIZED; } - this.initializationPromise = this.doInitialize(); + this.configInitializationPromise = this.doInitialize(); - return this.initializationPromise; + return this.configInitializationPromise; } private async doInitialize(): Promise { try { - this.initializationState = InitializationState.INITIALIZING; + this.configInitializationState = ConfigInitializationState.INITIALIZING; await this.loadAllConfigVarsFromDb(); - this.initializationState = InitializationState.INITIALIZED; + this.configInitializationState = ConfigInitializationState.INITIALIZED; // Reset retry attempts on successful initialization this.retryAttempts = 0; } catch (error) { @@ -57,11 +60,11 @@ export class DatabaseConfigDriver `Failed to initialize database driver (attempt ${this.retryAttempts + 1})`, error instanceof Error ? error.stack : error, ); - this.initializationState = InitializationState.FAILED; + this.configInitializationState = ConfigInitializationState.FAILED; this.scheduleRetry(); } finally { // Reset the promise to allow future initialization attempts - this.initializationPromise = null; + this.configInitializationPromise = null; } } @@ -76,7 +79,7 @@ export class DatabaseConfigDriver return cachedValue; } - if (this.configCache.getNegativeLookup(key)) { + if (this.configCache.isKeyKnownMissing(key)) { return this.environmentDriver.get(key); } @@ -111,11 +114,11 @@ export class DatabaseConfigDriver if (value !== undefined) { this.configCache.set(key, value); } else { - this.configCache.setNegativeLookup(key); + this.configCache.markKeyAsMissing(key); } } catch (error) { this.logger.error(`Failed to fetch config for ${key as string}`, error); - this.configCache.setNegativeLookup(key); + this.configCache.markKeyAsMissing(key); } } @@ -129,8 +132,8 @@ export class DatabaseConfigDriver private shouldUseEnvironment(key: keyof ConfigVariables): boolean { return ( - this.initializationState !== InitializationState.INITIALIZED || - isEnvOnlyConfigVar(key) + this.configInitializationState !== + ConfigInitializationState.INITIALIZED || isEnvOnlyConfigVar(key) ); } @@ -152,7 +155,9 @@ export class DatabaseConfigDriver } private async scheduleRefresh(key: keyof ConfigVariables): Promise { - if (this.initializationState !== InitializationState.INITIALIZED) { + if ( + this.configInitializationState !== ConfigInitializationState.INITIALIZED + ) { return; } @@ -191,7 +196,10 @@ export class DatabaseConfigDriver this.retryTimer = setTimeout(() => { this.logger.log(`Executing retry attempt ${this.retryAttempts}`); - if (this.initializationState === InitializationState.INITIALIZING) { + if ( + this.configInitializationState === + ConfigInitializationState.INITIALIZING + ) { this.logger.log( 'Skipping retry attempt as initialization is already in progress', ); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/enums/initialization-state.enum.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/enums/config-initialization-state.enum.ts similarity index 75% rename from packages/twenty-server/src/engine/core-modules/twenty-config/enums/initialization-state.enum.ts rename to packages/twenty-server/src/engine/core-modules/twenty-config/enums/config-initialization-state.enum.ts index af938681403a..476ef28d106a 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/enums/initialization-state.enum.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/enums/config-initialization-state.enum.ts @@ -1,4 +1,4 @@ -export enum InitializationState { +export enum ConfigInitializationState { NOT_INITIALIZED = 'NOT_INITIALIZED', INITIALIZING = 'INITIALIZING', INITIALIZED = 'INITIALIZED', diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts index 5f38ed8a148f..192a9986c834 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts @@ -4,9 +4,9 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; +import { ConfigInitializationState } from 'src/engine/core-modules/twenty-config/enums/config-initialization-state.enum'; import { ConfigSource } from 'src/engine/core-modules/twenty-config/enums/config-source.enum'; import { ConfigVariablesGroup } from 'src/engine/core-modules/twenty-config/enums/config-variables-group.enum'; -import { InitializationState } from 'src/engine/core-modules/twenty-config/enums/initialization-state.enum'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; import { TypedReflect } from 'src/utils/typed-reflect'; @@ -32,7 +32,7 @@ jest.mock( type TwentyConfigServicePrivateProps = { driver: DatabaseConfigDriver | EnvironmentConfigDriver; isConfigVarInDbEnabled: boolean; - initializationState: InitializationState; + configInitializationState: ConfigInitializationState; }; const mockConfigVarMetadata = { @@ -180,8 +180,8 @@ describe('TwentyConfigService', () => { const privateProps = newService as unknown as TwentyConfigServicePrivateProps; - expect(privateProps.initializationState).toBe( - InitializationState.INITIALIZED, + expect(privateProps.configInitializationState).toBe( + ConfigInitializationState.INITIALIZED, ); }); @@ -199,8 +199,8 @@ describe('TwentyConfigService', () => { const privateProps = newService as unknown as TwentyConfigServicePrivateProps; - expect(privateProps.initializationState).toBe( - InitializationState.NOT_INITIALIZED, + expect(privateProps.configInitializationState).toBe( + ConfigInitializationState.NOT_INITIALIZED, ); }); }); @@ -242,8 +242,8 @@ describe('TwentyConfigService', () => { newService as unknown as TwentyConfigServicePrivateProps; expect(privateProps.driver).toBe(databaseConfigDriver); - expect(privateProps.initializationState).toBe( - InitializationState.INITIALIZED, + expect(privateProps.configInitializationState).toBe( + ConfigInitializationState.INITIALIZED, ); }); @@ -267,7 +267,9 @@ describe('TwentyConfigService', () => { newService as unknown as TwentyConfigServicePrivateProps; expect(privateProps.driver).toBe(environmentConfigDriver); - expect(privateProps.initializationState).toBe(InitializationState.FAILED); + expect(privateProps.configInitializationState).toBe( + ConfigInitializationState.FAILED, + ); }); }); @@ -293,7 +295,7 @@ describe('TwentyConfigService', () => { ) => { setPrivateProps(service, { isConfigVarInDbEnabled: true, - initializationState: InitializationState.INITIALIZED, + configInitializationState: ConfigInitializationState.INITIALIZED, driver: databaseConfigDriver, ...props, }); @@ -311,7 +313,7 @@ describe('TwentyConfigService', () => { it('should throw error when not initialized', async () => { setupUpdateTest({ - initializationState: InitializationState.NOT_INITIALIZED, + configInitializationState: ConfigInitializationState.NOT_INITIALIZED, }); await expect( @@ -410,7 +412,7 @@ describe('TwentyConfigService', () => { setPrivateProps(service, { driver: environmentConfigDriver, isConfigVarInDbEnabled: false, - initializationState: InitializationState.INITIALIZED, + configInitializationState: ConfigInitializationState.INITIALIZED, }); const result = service.getAll(); @@ -440,7 +442,7 @@ describe('TwentyConfigService', () => { setPrivateProps(service, { driver: databaseConfigDriver, isConfigVarInDbEnabled: true, - initializationState: InitializationState.INITIALIZED, + configInitializationState: ConfigInitializationState.INITIALIZED, }); const result = service.getAll(); @@ -472,7 +474,7 @@ describe('TwentyConfigService', () => { setPrivateProps(service, { driver: environmentConfigDriver, isConfigVarInDbEnabled: false, - initializationState: InitializationState.INITIALIZED, + configInitializationState: ConfigInitializationState.INITIALIZED, ...props, }); }; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts index ad8ee96f397d..098f818a9e2c 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts @@ -13,9 +13,9 @@ import { CONFIG_VARIABLES_MASKING_CONFIG } from 'src/engine/core-modules/twenty- import { ConfigVariablesMetadataOptions } from 'src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator'; import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; +import { ConfigInitializationState } from 'src/engine/core-modules/twenty-config/enums/config-initialization-state.enum'; import { ConfigSource } from 'src/engine/core-modules/twenty-config/enums/config-source.enum'; import { ConfigVariablesMaskingStrategies } from 'src/engine/core-modules/twenty-config/enums/config-variables-masking-strategies.enum'; -import { InitializationState } from 'src/engine/core-modules/twenty-config/enums/initialization-state.enum'; import { configVariableMaskSensitiveData } from 'src/engine/core-modules/twenty-config/utils/config-variable-mask-sensitive-data.util'; import { TypedReflect } from 'src/utils/typed-reflect'; @@ -24,7 +24,7 @@ export class TwentyConfigService implements OnModuleInit, OnApplicationBootstrap { private driver: DatabaseConfigDriver | EnvironmentConfigDriver; - private initializationState = InitializationState.NOT_INITIALIZED; + private configInitializationState = ConfigInitializationState.NOT_INITIALIZED; private readonly isConfigVarInDbEnabled: boolean; private readonly logger = new Logger(TwentyConfigService.name); @@ -51,7 +51,7 @@ export class TwentyConfigService this.logger.log( 'Database configuration is disabled, using environment variables only', ); - this.initializationState = InitializationState.INITIALIZED; + this.configInitializationState = ConfigInitializationState.INITIALIZED; return; } @@ -68,12 +68,12 @@ export class TwentyConfigService try { this.logger.log('Initializing database driver for configuration'); - this.initializationState = InitializationState.INITIALIZING; + this.configInitializationState = ConfigInitializationState.INITIALIZING; await this.databaseConfigDriver.initialize(); this.driver = this.databaseConfigDriver; - this.initializationState = InitializationState.INITIALIZED; + this.configInitializationState = ConfigInitializationState.INITIALIZED; this.logger.log('Database driver initialized successfully'); this.logger.log(`Active driver: DatabaseDriver`); } catch (error) { @@ -81,7 +81,7 @@ export class TwentyConfigService 'Failed to initialize database driver, falling back to environment variables', error, ); - this.initializationState = InitializationState.FAILED; + this.configInitializationState = ConfigInitializationState.FAILED; this.driver = this.environmentConfigDriver; this.logger.log(`Active driver: EnvironmentDriver (fallback)`); @@ -104,7 +104,9 @@ export class TwentyConfigService ); } - if (this.initializationState !== InitializationState.INITIALIZED) { + if ( + this.configInitializationState !== ConfigInitializationState.INITIALIZED + ) { throw new Error( 'TwentyConfigService not initialized, cannot update configuration', ); @@ -162,7 +164,7 @@ export class TwentyConfigService const isUsingDatabaseDriver = this.driver === this.databaseConfigDriver && this.isConfigVarInDbEnabled && - this.initializationState === InitializationState.INITIALIZED; + this.configInitializationState === ConfigInitializationState.INITIALIZED; Object.entries(metadata).forEach(([key, envMetadata]) => { let value = this.get(key as keyof ConfigVariables) ?? ''; @@ -221,11 +223,12 @@ export class TwentyConfigService const isUsingDatabaseDriver = this.driver === this.databaseConfigDriver && this.isConfigVarInDbEnabled && - this.initializationState === InitializationState.INITIALIZED; + this.configInitializationState === ConfigInitializationState.INITIALIZED; const result = { usingDatabaseDriver: isUsingDatabaseDriver, - initializationState: InitializationState[this.initializationState], + initializationState: + ConfigInitializationState[this.configInitializationState], }; if (isUsingDatabaseDriver) { From 49b532c87181f254964f384506b3c788dc555ad2 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Thu, 17 Apr 2025 19:33:18 +0530 Subject: [PATCH 37/70] rethink conversion --- .../config-value-converter.service.spec.ts | 190 ++++++++++++++++++ .../config-value-converter.service.ts | 177 ++++++++++++++++ .../decorators/cast-to-boolean.decorator.ts | 23 ++- .../cast-to-log-level-array.decorator.ts | 19 +- .../cast-to-meter-driver.decorator.ts | 18 +- .../cast-to-positive-number.decorator.ts | 19 +- .../__tests__/config-storage.service.spec.ts | 88 ++++---- .../storage/config-storage.service.ts | 18 +- .../twenty-config/twenty-config.module.ts | 8 +- .../convert-config-var-to-app-type.util.ts | 45 ----- ...convert-config-var-to-storage-type.util.ts | 37 ---- .../utils/is-env-only-config-var.util.ts | 3 - .../twenty-server/src/utils/typed-reflect.ts | 2 + 13 files changed, 510 insertions(+), 137 deletions(-) create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.spec.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts delete mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util.ts delete mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.spec.ts new file mode 100644 index 000000000000..606aa89b7329 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.spec.ts @@ -0,0 +1,190 @@ +import { LogLevel } from '@nestjs/common'; +import { Test, TestingModule } from '@nestjs/testing'; + +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { TypedReflect } from 'src/utils/typed-reflect'; + +import { ConfigValueConverterService } from './config-value-converter.service'; + +describe('ConfigValueConverterService', () => { + let service: ConfigValueConverterService; + + beforeEach(async () => { + const mockConfigVariables = { + NODE_PORT: 3000, + }; + + const module: TestingModule = await Test.createTestingModule({ + providers: [ + ConfigValueConverterService, + { + provide: ConfigVariables, + useValue: mockConfigVariables, + }, + ], + }).compile(); + + service = module.get( + ConfigValueConverterService, + ); + }); + + describe('convertToAppValue', () => { + it('should convert string to boolean based on metadata', () => { + // Mock the metadata + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce('boolean'); + + expect( + service.convertToAppValue( + 'true', + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ), + ).toBe(true); + expect( + service.convertToAppValue( + 'True', + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ), + ).toBe(true); + expect( + service.convertToAppValue( + 'yes', + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ), + ).toBe(true); + expect( + service.convertToAppValue( + '1', + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ), + ).toBe(true); + + expect( + service.convertToAppValue( + 'false', + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ), + ).toBe(false); + expect( + service.convertToAppValue( + 'False', + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ), + ).toBe(false); + expect( + service.convertToAppValue( + 'no', + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ), + ).toBe(false); + expect( + service.convertToAppValue( + '0', + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ), + ).toBe(false); + }); + + it('should convert string to number based on metadata', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce('number'); + + expect( + service.convertToAppValue('42', 'NODE_PORT' as keyof ConfigVariables), + ).toBe(42); + expect( + service.convertToAppValue('3.14', 'NODE_PORT' as keyof ConfigVariables), + ).toBe(3.14); + + expect(() => { + service.convertToAppValue( + 'not-a-number', + 'NODE_PORT' as keyof ConfigVariables, + ); + }).toThrow(); + }); + + it('should convert string to array based on metadata', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce('array'); + + expect( + service.convertToAppValue( + 'log,error,warn', + 'LOG_LEVELS' as keyof ConfigVariables, + ), + ).toEqual(['log', 'error', 'warn']); + + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce('array'); + expect( + service.convertToAppValue( + '["log","error","warn"]', + 'LOG_LEVELS' as keyof ConfigVariables, + ), + ).toEqual(['log', 'error', 'warn']); + }); + + it('should handle enum values as strings', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce(undefined); + + expect( + service.convertToAppValue( + 'development', + 'NODE_ENV' as keyof ConfigVariables, + ), + ).toBe('development'); + }); + + it('should handle various input types', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce('boolean'); + expect( + service.convertToAppValue( + true, + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ), + ).toBe(true); + + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce('number'); + expect( + service.convertToAppValue(42, 'NODE_PORT' as keyof ConfigVariables), + ).toBe(42); + + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce('array'); + expect( + service.convertToAppValue( + ['log', 'error'] as LogLevel[], + 'LOG_LEVELS' as keyof ConfigVariables, + ), + ).toEqual(['log', 'error']); + }); + + it('should fall back to default value approach when no metadata', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce(undefined); + + expect( + service.convertToAppValue('42', 'NODE_PORT' as keyof ConfigVariables), + ).toBe(42); + }); + }); + + describe('convertToStorageValue', () => { + it('should handle primitive types directly', () => { + expect(service.convertToStorageValue('string-value' as any)).toBe( + 'string-value', + ); + expect(service.convertToStorageValue(42 as any)).toBe(42); + expect(service.convertToStorageValue(true as any)).toBe(true); + expect(service.convertToStorageValue(undefined as any)).toBe(null); + }); + + it('should handle arrays', () => { + const array = ['log', 'error', 'warn'] as LogLevel[]; + + expect(service.convertToStorageValue(array as any)).toEqual(array); + }); + + it('should handle objects', () => { + const obj = { key: 'value' }; + + expect(service.convertToStorageValue(obj as any)).toEqual(obj); + }); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts new file mode 100644 index 000000000000..d2e4914c57f6 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts @@ -0,0 +1,177 @@ +import { Injectable } from '@nestjs/common'; + +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { TypedReflect } from 'src/utils/typed-reflect'; + +export type ConfigVariableType = + | 'boolean' + | 'number' + | 'array' + | 'string' + | 'enum'; + +@Injectable() +export class ConfigValueConverterService { + constructor(private readonly configVariables: ConfigVariables) {} + + convertToAppValue( + dbValue: unknown, + key: T, + ): ConfigVariables[T] | undefined { + if (dbValue === null || dbValue === undefined) { + return undefined; + } + + const configType = this.getConfigVarType(key); + + try { + switch (configType) { + case 'boolean': + return this.convertToBoolean(dbValue) as ConfigVariables[T]; + + case 'number': + return this.convertToNumber( + dbValue, + key as string, + ) as ConfigVariables[T]; + + case 'array': + return this.convertToArray( + dbValue, + key as string, + ) as ConfigVariables[T]; + + case 'string': + case 'enum': + default: + return dbValue as ConfigVariables[T]; + } + } catch (error) { + throw new Error( + `Failed to convert ${key as string} to app value: ${(error as Error).message}`, + ); + } + } + + convertToStorageValue( + appValue: ConfigVariables[T], + ): unknown { + if (appValue === undefined) { + return null; + } + + if ( + typeof appValue === 'string' || + typeof appValue === 'number' || + typeof appValue === 'boolean' || + appValue === null + ) { + return appValue; + } + + if (Array.isArray(appValue)) { + return appValue; + } + + if (typeof appValue === 'object') { + return JSON.parse(JSON.stringify(appValue)); + } + + throw new Error( + `Cannot convert value of type ${typeof appValue} to storage format`, + ); + } + + private getConfigVarType( + key: T, + ): ConfigVariableType { + const metadataType = TypedReflect.getMetadata( + 'config-variable:type', + ConfigVariables.prototype.constructor, + key as string, + ) as ConfigVariableType | undefined; + + if (metadataType) { + return metadataType; + } + + const defaultValue = this.configVariables[key]; + + if (typeof defaultValue === 'boolean') return 'boolean'; + if (typeof defaultValue === 'number') return 'number'; + if (Array.isArray(defaultValue)) return 'array'; + + return 'string'; + } + + private convertToBoolean(value: unknown): boolean { + if (typeof value === 'boolean') { + return value; + } + + if (typeof value === 'number') { + return value !== 0; + } + + if (typeof value === 'string') { + const lowerValue = value.toLowerCase(); + + if (['true', 'on', 'yes', '1'].includes(lowerValue)) { + return true; + } + + if (['false', 'off', 'no', '0'].includes(lowerValue)) { + return false; + } + } + + return false; + } + + private convertToNumber(value: unknown, key: string): number { + if (typeof value === 'number') { + return value; + } + + if (typeof value === 'string') { + const parsedNumber = parseFloat(value); + + if (isNaN(parsedNumber)) { + throw new Error( + `Invalid number value for config variable ${key}: ${value}`, + ); + } + + return parsedNumber; + } + + throw new Error( + `Cannot convert ${typeof value} to number for config variable ${key}`, + ); + } + + private convertToArray(value: unknown, key: string): unknown[] { + if (Array.isArray(value)) { + return value; + } + + if (typeof value === 'string') { + try { + const parsedArray = JSON.parse(value); + + if (Array.isArray(parsedArray)) { + return parsedArray; + } + } catch { + // JSON parsing failed - normal for comma-separated values + // TODO: Handle this better + } + + return value.split(',').map((item) => item.trim()); + } + + throw new Error( + `Expected array value for config variable ${key}, got ${typeof value}`, + ); + } +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-boolean.decorator.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-boolean.decorator.ts index 8b1942a1084c..2b6f37487e3f 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-boolean.decorator.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-boolean.decorator.ts @@ -1,16 +1,31 @@ import { Transform } from 'class-transformer'; -export const CastToBoolean = () => - Transform(({ value }: { value: string }) => toBoolean(value)); +import { TypedReflect } from 'src/utils/typed-reflect'; + +export const CastToBoolean = + () => + (target: T, propertyKey: keyof T & string) => { + Transform(({ value }: { value: string }) => toBoolean(value))( + target, + propertyKey, + ); + + TypedReflect.defineMetadata( + 'config-variable:type', + 'boolean', + target.constructor, + propertyKey, + ); + }; const toBoolean = (value: any) => { if (typeof value === 'boolean') { return value; } - if (['true', 'on', 'yes', '1'].includes(value.toLowerCase())) { + if (['true', 'on', 'yes', '1'].includes(value?.toLowerCase())) { return true; } - if (['false', 'off', 'no', '0'].includes(value.toLowerCase())) { + if (['false', 'off', 'no', '0'].includes(value?.toLowerCase())) { return false; } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-log-level-array.decorator.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-log-level-array.decorator.ts index 35afda2f3312..773163cf8dd2 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-log-level-array.decorator.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-log-level-array.decorator.ts @@ -1,7 +1,22 @@ import { Transform } from 'class-transformer'; -export const CastToLogLevelArray = () => - Transform(({ value }: { value: string }) => toLogLevelArray(value)); +import { TypedReflect } from 'src/utils/typed-reflect'; + +export const CastToLogLevelArray = + () => + (target: T, propertyKey: keyof T & string) => { + Transform(({ value }: { value: string }) => toLogLevelArray(value))( + target, + propertyKey, + ); + + TypedReflect.defineMetadata( + 'config-variable:type', + 'array', + target.constructor, + propertyKey, + ); + }; const toLogLevelArray = (value: any) => { if (typeof value === 'string') { diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-meter-driver.decorator.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-meter-driver.decorator.ts index e8d2512eeff1..c9883f6062ad 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-meter-driver.decorator.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-meter-driver.decorator.ts @@ -1,9 +1,23 @@ import { Transform } from 'class-transformer'; import { MeterDriver } from 'src/engine/core-modules/metrics/types/meter-driver.type'; +import { TypedReflect } from 'src/utils/typed-reflect'; -export const CastToMeterDriverArray = () => - Transform(({ value }: { value: string }) => toMeterDriverArray(value)); +export const CastToMeterDriverArray = + () => + (target: T, propertyKey: keyof T & string) => { + Transform(({ value }: { value: string }) => toMeterDriverArray(value))( + target, + propertyKey, + ); + + TypedReflect.defineMetadata( + 'config-variable:type', + 'array', + target.constructor, + propertyKey, + ); + }; const toMeterDriverArray = (value: string | undefined) => { if (typeof value === 'string') { diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-positive-number.decorator.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-positive-number.decorator.ts index 1f532d9661b1..8664a023ff39 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-positive-number.decorator.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-positive-number.decorator.ts @@ -1,7 +1,22 @@ import { Transform } from 'class-transformer'; -export const CastToPositiveNumber = () => - Transform(({ value }: { value: string }) => toNumber(value)); +import { TypedReflect } from 'src/utils/typed-reflect'; + +export const CastToPositiveNumber = + () => + (target: T, propertyKey: keyof T & string) => { + Transform(({ value }: { value: string }) => toNumber(value))( + target, + propertyKey, + ); + + TypedReflect.defineMetadata( + 'config-variable:type', + 'number', + target.constructor, + propertyKey, + ); + }; const toNumber = (value: any) => { if (typeof value === 'number') { diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts index 551c591f6e4a..d143f09d6a80 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts @@ -8,29 +8,15 @@ import { KeyValuePairType, } from 'src/engine/core-modules/key-value-pair/key-value-pair.entity'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { ConfigValueConverterService } from 'src/engine/core-modules/twenty-config/conversion/config-value-converter.service'; import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; -import { convertConfigVarToAppType } from 'src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util'; -import { convertConfigVarToStorageType } from 'src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util'; import { User } from 'src/engine/core-modules/user/user.entity'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; -jest.mock( - 'src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util', - () => ({ - convertConfigVarToAppType: jest.fn(), - }), -); - -jest.mock( - 'src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util', - () => ({ - convertConfigVarToStorageType: jest.fn(), - }), -); - describe('ConfigStorageService', () => { let service: ConfigStorageService; let keyValuePairRepository: Repository; + let configValueConverter: ConfigValueConverterService; const createMockKeyValuePair = ( key: string, @@ -54,6 +40,14 @@ describe('ConfigStorageService', () => { const module: TestingModule = await Test.createTestingModule({ providers: [ ConfigStorageService, + { + provide: ConfigValueConverterService, + useValue: { + convertToAppValue: jest.fn(), + convertToStorageValue: jest.fn(), + }, + }, + ConfigVariables, { provide: getRepositoryToken(KeyValuePair, 'core'), useValue: { @@ -71,6 +65,9 @@ describe('ConfigStorageService', () => { keyValuePairRepository = module.get>( getRepositoryToken(KeyValuePair, 'core'), ); + configValueConverter = module.get( + ConfigValueConverterService, + ); jest.clearAllMocks(); }); @@ -109,12 +106,17 @@ describe('ConfigStorageService', () => { .spyOn(keyValuePairRepository, 'findOne') .mockResolvedValue(mockRecord); - (convertConfigVarToAppType as jest.Mock).mockReturnValue(convertedValue); + (configValueConverter.convertToAppValue as jest.Mock).mockReturnValue( + convertedValue, + ); const result = await service.get(key); expect(result).toBe(convertedValue); - expect(convertConfigVarToAppType).toHaveBeenCalledWith(storedValue, key); + expect(configValueConverter.convertToAppValue).toHaveBeenCalledWith( + storedValue, + key, + ); }); it('should handle conversion errors', async () => { @@ -127,9 +129,11 @@ describe('ConfigStorageService', () => { .spyOn(keyValuePairRepository, 'findOne') .mockResolvedValue(mockRecord); - (convertConfigVarToAppType as jest.Mock).mockImplementation(() => { - throw error; - }); + (configValueConverter.convertToAppValue as jest.Mock).mockImplementation( + () => { + throw error; + }, + ); await expect(service.get(key)).rejects.toThrow('Conversion error'); }); @@ -147,7 +151,9 @@ describe('ConfigStorageService', () => { .spyOn(keyValuePairRepository, 'findOne') .mockResolvedValue(mockRecord); - (convertConfigVarToStorageType as jest.Mock).mockReturnValue(storedValue); + (configValueConverter.convertToStorageValue as jest.Mock).mockReturnValue( + storedValue, + ); await service.set(key, value); @@ -164,7 +170,9 @@ describe('ConfigStorageService', () => { jest.spyOn(keyValuePairRepository, 'findOne').mockResolvedValue(null); - (convertConfigVarToStorageType as jest.Mock).mockReturnValue(storedValue); + (configValueConverter.convertToStorageValue as jest.Mock).mockReturnValue( + storedValue, + ); await service.set(key, value); @@ -182,7 +190,9 @@ describe('ConfigStorageService', () => { const value = true; const error = new Error('Conversion error'); - (convertConfigVarToStorageType as jest.Mock).mockImplementation(() => { + ( + configValueConverter.convertToStorageValue as jest.Mock + ).mockImplementation(() => { throw error; }); @@ -223,7 +233,7 @@ describe('ConfigStorageService', () => { jest.spyOn(keyValuePairRepository, 'find').mockResolvedValue(configVars); - (convertConfigVarToAppType as jest.Mock).mockImplementation( + (configValueConverter.convertToAppValue as jest.Mock).mockImplementation( (value, key) => { if (key === 'AUTH_PASSWORD_ENABLED') return true; if (key === 'EMAIL_FROM_ADDRESS') return 'test@example.com'; @@ -251,7 +261,7 @@ describe('ConfigStorageService', () => { jest.spyOn(keyValuePairRepository, 'find').mockResolvedValue(configVars); - (convertConfigVarToAppType as jest.Mock) + (configValueConverter.convertToAppValue as jest.Mock) .mockImplementationOnce(() => { throw new Error('Invalid value'); }) @@ -287,7 +297,9 @@ describe('ConfigStorageService', () => { .spyOn(keyValuePairRepository, 'find') .mockResolvedValue(configVars); - (convertConfigVarToAppType as jest.Mock).mockImplementation((value) => { + ( + configValueConverter.convertToAppValue as jest.Mock + ).mockImplementation((value) => { if (value === null) throw new Error('Null value'); return value; @@ -299,7 +311,7 @@ describe('ConfigStorageService', () => { expect(result.get('EMAIL_FROM_ADDRESS' as keyof ConfigVariables)).toBe( 'test@example.com', ); - expect(convertConfigVarToAppType).toHaveBeenCalledTimes(1); // Only called for non-null value + expect(configValueConverter.convertToAppValue).toHaveBeenCalledTimes(1); // Only called for non-null value }); }); }); @@ -316,7 +328,9 @@ describe('ConfigStorageService', () => { .spyOn(keyValuePairRepository, 'findOne') .mockResolvedValue(mockRecord); - (convertConfigVarToAppType as jest.Mock).mockImplementation(() => { + ( + configValueConverter.convertToAppValue as jest.Mock + ).mockImplementation(() => { throw new Error('Invalid boolean value'); }); @@ -333,7 +347,9 @@ describe('ConfigStorageService', () => { .spyOn(keyValuePairRepository, 'findOne') .mockResolvedValue(mockRecord); - (convertConfigVarToAppType as jest.Mock).mockImplementation(() => { + ( + configValueConverter.convertToAppValue as jest.Mock + ).mockImplementation(() => { throw new Error('Invalid string value'); }); @@ -356,13 +372,13 @@ describe('ConfigStorageService', () => { .mockResolvedValueOnce(initialRecord) .mockResolvedValueOnce(updatedRecord); - (convertConfigVarToAppType as jest.Mock) + (configValueConverter.convertToAppValue as jest.Mock) .mockReturnValueOnce(initialValue) .mockReturnValueOnce(newValue); - (convertConfigVarToStorageType as jest.Mock).mockReturnValueOnce( - 'false', - ); + ( + configValueConverter.convertToStorageValue as jest.Mock + ).mockReturnValueOnce('false'); const firstGet = service.get(key); const setOperation = service.set(key, newValue); @@ -412,7 +428,9 @@ describe('ConfigStorageService', () => { const value = true; const error = new Error('Database connection failed'); - (convertConfigVarToStorageType as jest.Mock).mockReturnValue('true'); + ( + configValueConverter.convertToStorageValue as jest.Mock + ).mockReturnValue('true'); jest.spyOn(keyValuePairRepository, 'findOne').mockRejectedValue(error); await expect(service.set(key, value)).rejects.toThrow( diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts index 1892be69b6aa..c4834fcbd281 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts @@ -8,8 +8,7 @@ import { KeyValuePairType, } from 'src/engine/core-modules/key-value-pair/key-value-pair.entity'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; -import { convertConfigVarToAppType } from 'src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util'; -import { convertConfigVarToStorageType } from 'src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util'; +import { ConfigValueConverterService } from 'src/engine/core-modules/twenty-config/conversion/config-value-converter.service'; import { ConfigStorageInterface } from './interfaces/config-storage.interface'; @@ -20,6 +19,7 @@ export class ConfigStorageService implements ConfigStorageInterface { constructor( @InjectRepository(KeyValuePair, 'core') private readonly keyValuePairRepository: Repository, + private readonly configValueConverter: ConfigValueConverterService, ) {} private getConfigVariableWhereClause( @@ -46,7 +46,7 @@ export class ConfigStorageService implements ConfigStorageInterface { } try { - return convertConfigVarToAppType(result.value, key); + return this.configValueConverter.convertToAppValue(result.value, key); } catch (error) { this.logger.error( `Failed to convert value to app type for key ${key as string}`, @@ -68,7 +68,7 @@ export class ConfigStorageService implements ConfigStorageInterface { let processedValue; try { - processedValue = convertConfigVarToStorageType(value); + processedValue = this.configValueConverter.convertToStorageValue(value); } catch (error) { this.logger.error( `Failed to convert value to storage type for key ${key as string}`, @@ -130,9 +130,15 @@ export class ConfigStorageService implements ConfigStorageInterface { const key = configVar.key as keyof ConfigVariables; try { - const value = convertConfigVarToAppType(configVar.value, key); + const value = this.configValueConverter.convertToAppValue( + configVar.value, + key, + ); - result.set(key, value); + // Only set values that are defined + if (value !== undefined) { + result.set(key, value); + } } catch (error) { this.logger.error( `Failed to convert value to app type for key ${key as string}`, diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts index b15d4b4a361e..00475c8a9156 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts @@ -4,7 +4,11 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { KeyValuePair } from 'src/engine/core-modules/key-value-pair/key-value-pair.entity'; import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; -import { validate } from 'src/engine/core-modules/twenty-config/config-variables'; +import { + ConfigVariables, + validate, +} from 'src/engine/core-modules/twenty-config/config-variables'; +import { ConfigValueConverterService } from 'src/engine/core-modules/twenty-config/conversion/config-value-converter.service'; import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; @@ -28,6 +32,8 @@ import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twent DatabaseConfigDriver, ConfigCacheService, ConfigStorageService, + ConfigValueConverterService, + ConfigVariables, ], exports: [TwentyConfigService], }) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util.ts deleted file mode 100644 index 57e119a1a49f..000000000000 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-app-type.util.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; - -/** - * Convert a database value to the appropriate type for the environment variable - */ -export const convertConfigVarToAppType = ( - dbValue: unknown, - key: T, -): ConfigVariables[T] => { - if (dbValue === null || dbValue === undefined) { - return dbValue as ConfigVariables[T]; - } - - // Get the default value to determine the expected type - const defaultValue = new ConfigVariables()[key]; - const valueType = typeof defaultValue; - - if (valueType === 'boolean' && typeof dbValue === 'string') { - return (dbValue === 'true') as unknown as ConfigVariables[T]; - } - - if (valueType === 'number' && typeof dbValue === 'string') { - const parsedNumber = parseFloat(dbValue); - - if (isNaN(parsedNumber)) { - throw new Error( - `Invalid number value for config variable ${key}: ${dbValue}`, - ); - } - - return parsedNumber as unknown as ConfigVariables[T]; - } - - if (Array.isArray(defaultValue)) { - if (!Array.isArray(dbValue)) { - throw new Error( - `Expected array value for config variable ${key}, got ${typeof dbValue}`, - ); - } - - return dbValue as ConfigVariables[T]; - } - - return dbValue as ConfigVariables[T]; -}; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util.ts deleted file mode 100644 index 05bbd3e41dff..000000000000 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/convert-config-var-to-storage-type.util.ts +++ /dev/null @@ -1,37 +0,0 @@ -import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; - -/** - * Convert an environment variable value to a format suitable for database storage - */ -export const convertConfigVarToStorageType = ( - appValue: ConfigVariables[T], -): unknown => { - if (appValue === undefined) { - return null; - } - - if ( - typeof appValue === 'string' || - typeof appValue === 'number' || - typeof appValue === 'boolean' || - appValue === null - ) { - return appValue; - } - - if (Array.isArray(appValue)) { - return appValue.map((item) => - convertConfigVarToStorageType( - item as ConfigVariables[keyof ConfigVariables], - ), - ); - } - - if (typeof appValue === 'object') { - return JSON.parse(JSON.stringify(appValue)); - } - - throw new Error( - `Cannot convert value of type ${typeof appValue} to storage format`, - ); -}; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util.ts index 717636f7cff9..ccd782d758cf 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util.ts @@ -1,9 +1,6 @@ import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; import { TypedReflect } from 'src/utils/typed-reflect'; -/** - * Checks if the environment variable is environment-only based on metadata - */ export const isEnvOnlyConfigVar = (key: keyof ConfigVariables): boolean => { const metadata = TypedReflect.getMetadata('config-variables', ConfigVariables) ?? {}; diff --git a/packages/twenty-server/src/utils/typed-reflect.ts b/packages/twenty-server/src/utils/typed-reflect.ts index a871540f90bd..58ec53f95ba9 100644 --- a/packages/twenty-server/src/utils/typed-reflect.ts +++ b/packages/twenty-server/src/utils/typed-reflect.ts @@ -3,6 +3,7 @@ import 'reflect-metadata'; import { Gate } from 'src/engine/twenty-orm/interfaces/gate.interface'; import { WorkspaceEntityDuplicateCriteria } from 'src/engine/api/graphql/workspace-query-builder/types/workspace-entity-duplicate-criteria.type'; +import { ConfigVariableType } from 'src/engine/core-modules/twenty-config/conversion/config-value-converter.service'; import { ConfigVariablesMetadataMap } from 'src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator'; export interface ReflectMetadataTypeMap { @@ -16,6 +17,7 @@ export interface ReflectMetadataTypeMap { ['workspace:duplicate-criteria-metadata-args']: WorkspaceEntityDuplicateCriteria[]; ['config-variables']: ConfigVariablesMetadataMap; ['workspace:is-searchable-metadata-args']: boolean; + ['config-variable:type']: ConfigVariableType; } export class TypedReflect { From ebc03b1c9b6e7efd3c84ca254c44981429325faf Mon Sep 17 00:00:00 2001 From: ehconitin Date: Fri, 18 Apr 2025 15:46:22 +0530 Subject: [PATCH 38/70] improvements --- .../twenty-config/cache/config-cache.service.ts | 1 + .../twenty-config/decorators/cast-to-boolean.decorator.ts | 2 +- .../core-modules/twenty-config/twenty-config.module.ts | 6 +++++- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts index f4cd9be320e1..66fd04337aeb 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts @@ -99,6 +99,7 @@ export class ConfigCacheService implements OnModuleDestroy { if (this.cacheScavengeInterval) { clearInterval(this.cacheScavengeInterval); } + this.clearAll(); } private isCacheExpired( diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-boolean.decorator.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-boolean.decorator.ts index 2b6f37487e3f..d037b741353e 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-boolean.decorator.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-boolean.decorator.ts @@ -5,7 +5,7 @@ import { TypedReflect } from 'src/utils/typed-reflect'; export const CastToBoolean = () => (target: T, propertyKey: keyof T & string) => { - Transform(({ value }: { value: string }) => toBoolean(value))( + Transform(({ value }: { value: any }) => toBoolean(value))( target, propertyKey, ); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts index 00475c8a9156..9942bb8b0faa 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts @@ -26,6 +26,7 @@ import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twent }), TypeOrmModule.forFeature([KeyValuePair], 'core'), ], + providers: [ TwentyConfigService, EnvironmentConfigDriver, @@ -33,7 +34,10 @@ import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twent ConfigCacheService, ConfigStorageService, ConfigValueConverterService, - ConfigVariables, + { + provide: ConfigVariables, + useValue: new ConfigVariables(), + }, ], exports: [TwentyConfigService], }) From 96502818521c5361ee56c1bf17e60e0681e4aeca Mon Sep 17 00:00:00 2001 From: ehconitin Date: Mon, 21 Apr 2025 13:11:19 +0530 Subject: [PATCH 39/70] automate simple validation and transformers --- .../twenty-config/config-variables.ts | 215 ++++++++++-------- .../config-value-converter.service.spec.ts | 127 ++++++++--- .../config-value-converter.service.ts | 164 ++++++------- .../decorators/cast-to-boolean.decorator.ts | 33 --- .../cast-to-log-level-array.decorator.ts | 19 +- .../cast-to-meter-driver.decorator.ts | 18 +- .../cast-to-positive-number.decorator.ts | 19 +- .../cast-to-string-array.decorator.ts | 12 - .../config-variables-metadata.decorator.ts | 26 ++- .../__tests__/config-storage.service.spec.ts | 73 +++--- .../storage/config-storage.service.ts | 10 +- .../types/config-variable-options.type.ts | 3 + .../types/config-variable-type.type.ts | 6 + .../utils/apply-basic-validators.util.ts | 55 +++++ .../utils/config-transformers.util.ts | 79 +++++++ .../twenty-server/src/utils/typed-reflect.ts | 2 - 16 files changed, 508 insertions(+), 353 deletions(-) delete mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-boolean.decorator.ts delete mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-string-array.decorator.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/types/config-variable-options.type.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/types/config-variable-type.type.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/utils/config-transformers.util.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts index e6e476c6b850..a47b2d0347ef 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts @@ -2,10 +2,7 @@ import { LogLevel, Logger } from '@nestjs/common'; import { plainToClass } from 'class-transformer'; import { - IsBoolean, IsDefined, - IsEnum, - IsNumber, IsOptional, IsString, IsUrl, @@ -28,7 +25,6 @@ import { StorageDriverType } from 'src/engine/core-modules/file-storage/interfac import { LoggerDriverType } from 'src/engine/core-modules/logger/interfaces'; import { MeterDriver } from 'src/engine/core-modules/metrics/types/meter-driver.type'; import { ServerlessDriverType } from 'src/engine/core-modules/serverless/serverless.interface'; -import { CastToBoolean } from 'src/engine/core-modules/twenty-config/decorators/cast-to-boolean.decorator'; import { CastToLogLevelArray } from 'src/engine/core-modules/twenty-config/decorators/cast-to-log-level-array.decorator'; import { CastToMeterDriverArray } from 'src/engine/core-modules/twenty-config/decorators/cast-to-meter-driver.decorator'; import { CastToPositiveNumber } from 'src/engine/core-modules/twenty-config/decorators/cast-to-positive-number.decorator'; @@ -44,35 +40,33 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Enable or disable password authentication for users', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() AUTH_PASSWORD_ENABLED = true; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Prefills tim@apple.dev in the login form, used in local development for quicker sign-in', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() @ValidateIf((env) => env.AUTH_PASSWORD_ENABLED) SIGN_IN_PREFILLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Require email verification for user accounts', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() IS_EMAIL_VERIFICATION_REQUIRED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Duration for which the email verification token is valid', + type: 'string', }) @IsDuration() @IsOptional() @@ -81,6 +75,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Duration for which the password reset token is valid', + type: 'string', }) @IsDuration() @IsOptional() @@ -89,31 +84,31 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.GoogleAuth, description: 'Enable or disable the Google Calendar integration', + type: 'boolean', }) - @CastToBoolean() CALENDAR_PROVIDER_GOOGLE_ENABLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.GoogleAuth, description: 'Callback URL for Google Auth APIs', + type: 'string', }) AUTH_GOOGLE_APIS_CALLBACK_URL: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.GoogleAuth, description: 'Enable or disable Google Single Sign-On (SSO)', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() AUTH_GOOGLE_ENABLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.GoogleAuth, isSensitive: true, description: 'Client ID for Google authentication', + type: 'string', }) - @IsString() @ValidateIf((env) => env.AUTH_GOOGLE_ENABLED) AUTH_GOOGLE_CLIENT_ID: string; @@ -121,8 +116,8 @@ export class ConfigVariables { group: ConfigVariablesGroup.GoogleAuth, isSensitive: true, description: 'Client secret for Google authentication', + type: 'string', }) - @IsString() @ValidateIf((env) => env.AUTH_GOOGLE_ENABLED) AUTH_GOOGLE_CLIENT_SECRET: string; @@ -130,6 +125,7 @@ export class ConfigVariables { group: ConfigVariablesGroup.GoogleAuth, isSensitive: true, description: 'Callback URL for Google authentication', + type: 'string', }) @IsUrl({ require_tld: false, require_protocol: true }) @ValidateIf((env) => env.AUTH_GOOGLE_ENABLED) @@ -138,25 +134,24 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.GoogleAuth, description: 'Enable or disable the Gmail messaging integration', + type: 'boolean', }) - @CastToBoolean() MESSAGING_PROVIDER_GMAIL_ENABLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.MicrosoftAuth, description: 'Enable or disable Microsoft authentication', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() AUTH_MICROSOFT_ENABLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.MicrosoftAuth, isSensitive: true, description: 'Client ID for Microsoft authentication', + type: 'string', }) - @IsString() @ValidateIf((env) => env.AUTH_MICROSOFT_ENABLED) AUTH_MICROSOFT_CLIENT_ID: string; @@ -164,8 +159,8 @@ export class ConfigVariables { group: ConfigVariablesGroup.MicrosoftAuth, isSensitive: true, description: 'Client secret for Microsoft authentication', + type: 'string', }) - @IsString() @ValidateIf((env) => env.AUTH_MICROSOFT_ENABLED) AUTH_MICROSOFT_CLIENT_SECRET: string; @@ -173,6 +168,7 @@ export class ConfigVariables { group: ConfigVariablesGroup.MicrosoftAuth, isSensitive: true, description: 'Callback URL for Microsoft authentication', + type: 'string', }) @IsUrl({ require_tld: false, require_protocol: true }) @ValidateIf((env) => env.AUTH_MICROSOFT_ENABLED) @@ -182,6 +178,7 @@ export class ConfigVariables { group: ConfigVariablesGroup.MicrosoftAuth, isSensitive: true, description: 'Callback URL for Microsoft APIs', + type: 'string', }) @IsUrl({ require_tld: false, require_protocol: true }) @ValidateIf((env) => env.AUTH_MICROSOFT_ENABLED) @@ -190,15 +187,15 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.MicrosoftAuth, description: 'Enable or disable the Microsoft messaging integration', + type: 'boolean', }) - @CastToBoolean() MESSAGING_PROVIDER_MICROSOFT_ENABLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.MicrosoftAuth, description: 'Enable or disable the Microsoft Calendar integration', + type: 'boolean', }) - @CastToBoolean() CALENDAR_PROVIDER_MICROSOFT_ENABLED = false; @ConfigVariablesMetadata({ @@ -206,14 +203,15 @@ export class ConfigVariables { isSensitive: true, description: 'Legacy variable to be deprecated when all API Keys expire. Replaced by APP_KEY', + type: 'string', }) @IsOptional() - @IsString() ACCESS_TOKEN_SECRET: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Duration for which the access token is valid', + type: 'string', }) @IsDuration() @IsOptional() @@ -222,6 +220,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Duration for which the refresh token is valid', + type: 'string', }) @IsOptional() REFRESH_TOKEN_EXPIRES_IN = '60d'; @@ -229,6 +228,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Cooldown period for refreshing tokens', + type: 'string', }) @IsDuration() @IsOptional() @@ -237,6 +237,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Duration for which the login token is valid', + type: 'string', }) @IsDuration() @IsOptional() @@ -245,6 +246,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Duration for which the file token is valid', + type: 'string', }) @IsDuration() @IsOptional() @@ -253,6 +255,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Duration for which the invitation token is valid', + type: 'string', }) @IsDuration() @IsOptional() @@ -261,51 +264,57 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Duration for which the short-term token is valid', + type: 'string', }) SHORT_TERM_TOKEN_EXPIRES_IN = '5m'; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.EmailSettings, description: 'Email address used as the sender for outgoing emails', + type: 'string', }) EMAIL_FROM_ADDRESS = 'noreply@yourdomain.com'; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.EmailSettings, description: 'Email address used for system notifications', + type: 'string', }) EMAIL_SYSTEM_ADDRESS = 'system@yourdomain.com'; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.EmailSettings, description: 'Name used in the From header for outgoing emails', + type: 'string', }) EMAIL_FROM_NAME = 'Felix from Twenty'; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.EmailSettings, description: 'Email driver to use for sending emails', + type: 'string', }) EMAIL_DRIVER: EmailDriver = EmailDriver.Logger; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.EmailSettings, description: 'SMTP host for sending emails', + type: 'string', }) EMAIL_SMTP_HOST: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.EmailSettings, description: 'Use unsecure connection for SMTP', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() EMAIL_SMTP_NO_TLS = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.EmailSettings, description: 'SMTP port for sending emails', + type: 'number', }) @CastToPositiveNumber() EMAIL_SMTP_PORT = 587; @@ -313,6 +322,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.EmailSettings, description: 'SMTP user for authentication', + type: 'string', }) EMAIL_SMTP_USER: string; @@ -320,28 +330,31 @@ export class ConfigVariables { group: ConfigVariablesGroup.EmailSettings, isSensitive: true, description: 'SMTP password for authentication', + type: 'string', }) EMAIL_SMTP_PASSWORD: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.StorageConfig, description: 'Type of storage to use (local or S3)', + type: 'enum', + options: Object.values(StorageDriverType), }) - @IsEnum(StorageDriverType) @IsOptional() STORAGE_TYPE: StorageDriverType = StorageDriverType.Local; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.StorageConfig, description: 'Local path for storage when using local storage type', + type: 'string', }) - @IsString() @ValidateIf((env) => env.STORAGE_TYPE === StorageDriverType.Local) STORAGE_LOCAL_PATH = '.local-storage'; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.StorageConfig, description: 'S3 region for storage when using S3 storage type', + type: 'string', }) @ValidateIf((env) => env.STORAGE_TYPE === StorageDriverType.S3) @IsAWSRegion() @@ -350,17 +363,17 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.StorageConfig, description: 'S3 bucket name for storage when using S3 storage type', + type: 'string', }) @ValidateIf((env) => env.STORAGE_TYPE === StorageDriverType.S3) - @IsString() STORAGE_S3_NAME: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.StorageConfig, description: 'S3 endpoint for storage when using S3 storage type', + type: 'string', }) @ValidateIf((env) => env.STORAGE_TYPE === StorageDriverType.S3) - @IsString() @IsOptional() STORAGE_S3_ENDPOINT: string; @@ -369,9 +382,9 @@ export class ConfigVariables { isSensitive: true, description: 'S3 access key ID for authentication when using S3 storage type', + type: 'string', }) @ValidateIf((env) => env.STORAGE_TYPE === StorageDriverType.S3) - @IsString() @IsOptional() STORAGE_S3_ACCESS_KEY_ID: string; @@ -380,23 +393,25 @@ export class ConfigVariables { isSensitive: true, description: 'S3 secret access key for authentication when using S3 storage type', + type: 'string', }) @ValidateIf((env) => env.STORAGE_TYPE === StorageDriverType.S3) - @IsString() @IsOptional() STORAGE_S3_SECRET_ACCESS_KEY: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerlessConfig, description: 'Type of serverless execution (local or Lambda)', + type: 'enum', + options: Object.values(ServerlessDriverType), }) - @IsEnum(ServerlessDriverType) @IsOptional() SERVERLESS_TYPE: ServerlessDriverType = ServerlessDriverType.Local; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerlessConfig, description: 'Throttle limit for serverless function execution', + type: 'number', }) @CastToPositiveNumber() SERVERLESS_FUNCTION_EXEC_THROTTLE_LIMIT = 10; @@ -405,6 +420,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerlessConfig, description: 'Time-to-live for serverless function execution throttle', + type: 'number', }) @CastToPositiveNumber() SERVERLESS_FUNCTION_EXEC_THROTTLE_TTL = 1000; @@ -412,6 +428,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerlessConfig, description: 'Region for AWS Lambda functions', + type: 'string', }) @ValidateIf((env) => env.SERVERLESS_TYPE === ServerlessDriverType.Lambda) @IsAWSRegion() @@ -420,17 +437,17 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerlessConfig, description: 'IAM role for AWS Lambda functions', + type: 'string', }) @ValidateIf((env) => env.SERVERLESS_TYPE === ServerlessDriverType.Lambda) - @IsString() SERVERLESS_LAMBDA_ROLE: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerlessConfig, description: 'Role to assume when hosting lambdas in dedicated AWS account', + type: 'string', }) @ValidateIf((env) => env.SERVERLESS_TYPE === ServerlessDriverType.Lambda) - @IsString() @IsOptional() SERVERLESS_LAMBDA_SUBHOSTING_ROLE?: string; @@ -438,9 +455,9 @@ export class ConfigVariables { group: ConfigVariablesGroup.ServerlessConfig, isSensitive: true, description: 'Access key ID for AWS Lambda functions', + type: 'string', }) @ValidateIf((env) => env.SERVERLESS_TYPE === ServerlessDriverType.Lambda) - @IsString() @IsOptional() SERVERLESS_LAMBDA_ACCESS_KEY_ID: string; @@ -448,24 +465,24 @@ export class ConfigVariables { group: ConfigVariablesGroup.ServerlessConfig, isSensitive: true, description: 'Secret access key for AWS Lambda functions', + type: 'string', }) @ValidateIf((env) => env.SERVERLESS_TYPE === ServerlessDriverType.Lambda) - @IsString() @IsOptional() SERVERLESS_LAMBDA_SECRET_ACCESS_KEY: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.AnalyticsConfig, description: 'Enable or disable analytics for telemetry', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() ANALYTICS_ENABLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.AnalyticsConfig, description: 'Clickhouse host for analytics', + type: 'string', }) @IsOptional() @IsUrl({ @@ -478,34 +495,32 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Logging, description: 'Enable or disable telemetry logging', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() TELEMETRY_ENABLED = true; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.BillingConfig, description: 'Enable or disable billing features', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() IS_BILLING_ENABLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.BillingConfig, description: 'Link required for billing plan', + type: 'string', }) - @IsString() @ValidateIf((env) => env.IS_BILLING_ENABLED === true) BILLING_PLAN_REQUIRED_LINK: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.BillingConfig, description: 'Duration of free trial with credit card in days', + type: 'number', }) - @IsNumber() @CastToPositiveNumber() @IsOptional() @ValidateIf((env) => env.IS_BILLING_ENABLED === true) @@ -514,8 +529,8 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.BillingConfig, description: 'Duration of free trial without credit card in days', + type: 'number', }) - @IsNumber() @CastToPositiveNumber() @IsOptional() @ValidateIf((env) => env.IS_BILLING_ENABLED === true) @@ -524,8 +539,8 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.BillingConfig, description: 'Amount of money in cents to trigger a billing threshold', + type: 'number', }) - @IsNumber() @CastToPositiveNumber() @ValidateIf((env) => env.IS_BILLING_ENABLED === true) BILLING_SUBSCRIPTION_THRESHOLD_AMOUNT = 10000; @@ -533,8 +548,8 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.BillingConfig, description: 'Amount of credits for the free trial without credit card', + type: 'number', }) - @IsNumber() @CastToPositiveNumber() @ValidateIf((env) => env.IS_BILLING_ENABLED === true) BILLING_FREE_WORKFLOW_CREDITS_FOR_TRIAL_PERIOD_WITHOUT_CREDIT_CARD = 5000; @@ -542,8 +557,8 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.BillingConfig, description: 'Amount of credits for the free trial with credit card', + type: 'number', }) - @IsNumber() @CastToPositiveNumber() @ValidateIf((env) => env.IS_BILLING_ENABLED === true) BILLING_FREE_WORKFLOW_CREDITS_FOR_TRIAL_PERIOD_WITH_CREDIT_CARD = 10000; @@ -552,8 +567,8 @@ export class ConfigVariables { group: ConfigVariablesGroup.BillingConfig, isSensitive: true, description: 'Stripe API key for billing', + type: 'string', }) - @IsString() @ValidateIf((env) => env.IS_BILLING_ENABLED === true) BILLING_STRIPE_API_KEY: string; @@ -561,14 +576,15 @@ export class ConfigVariables { group: ConfigVariablesGroup.BillingConfig, isSensitive: true, description: 'Stripe webhook secret for billing', + type: 'string', }) - @IsString() @ValidateIf((env) => env.IS_BILLING_ENABLED === true) BILLING_STRIPE_WEBHOOK_SECRET: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerConfig, description: 'Url for the frontend application', + type: 'string', }) @IsUrl({ require_tld: false, require_protocol: true }) @IsOptional() @@ -578,33 +594,33 @@ export class ConfigVariables { group: ConfigVariablesGroup.ServerConfig, description: 'Default subdomain for the frontend when multi-workspace is enabled', + type: 'string', }) - @IsString() @ValidateIf((env) => env.IS_MULTIWORKSPACE_ENABLED) DEFAULT_SUBDOMAIN = 'app'; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'ID for the Chrome extension', + type: 'string', }) - @IsString() @IsOptional() CHROME_EXTENSION_ID: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Logging, description: 'Enable or disable buffering for logs before sending', + type: 'boolean', }) - @CastToBoolean() - @IsBoolean() @IsOptional() LOGGER_IS_BUFFER_ENABLED = true; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Logging, description: 'Driver used for handling exceptions (Console or Sentry)', + type: 'enum', + options: Object.values(ExceptionHandlerDriver), }) - @IsEnum(ExceptionHandlerDriver) @IsOptional() EXCEPTION_HANDLER_DRIVER: ExceptionHandlerDriver = ExceptionHandlerDriver.Console; @@ -612,6 +628,8 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Logging, description: 'Levels of logging to be captured', + type: 'array', + options: ['log', 'error', 'warn'], }) @CastToLogLevelArray() @IsOptional() @@ -620,6 +638,8 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Metering, description: 'Driver used for collect metrics (OpenTelemetry or Console)', + type: 'array', + options: ['OpenTelemetry', 'Console'], }) @CastToMeterDriverArray() @IsOptional() @@ -628,6 +648,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Metering, description: 'Endpoint URL for the OpenTelemetry collector', + type: 'string', }) @IsOptional() OTLP_COLLECTOR_ENDPOINT_URL: string; @@ -635,58 +656,60 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ExceptionHandler, description: 'Driver used for logging (only console for now)', + type: 'enum', + options: Object.values(LoggerDriverType), }) - @IsEnum(LoggerDriverType) @IsOptional() LOGGER_DRIVER: LoggerDriverType = LoggerDriverType.Console; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ExceptionHandler, description: 'Data Source Name (DSN) for Sentry logging', + type: 'string', }) @ValidateIf( (env) => env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.Sentry, ) - @IsString() SENTRY_DSN: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ExceptionHandler, description: 'Front-end DSN for Sentry logging', + type: 'string', }) @ValidateIf( (env) => env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.Sentry, ) - @IsString() SENTRY_FRONT_DSN: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ExceptionHandler, description: 'Release version for Sentry logging', + type: 'string', }) @ValidateIf( (env) => env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.Sentry, ) - @IsString() @IsOptional() SENTRY_RELEASE: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ExceptionHandler, description: 'Environment name for Sentry logging', + type: 'string', }) @ValidateIf( (env) => env.EXCEPTION_HANDLER_DRIVER === ExceptionHandlerDriver.Sentry, ) - @IsString() @IsOptional() SENTRY_ENVIRONMENT: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.SupportChatConfig, description: 'Driver used for support chat integration', + type: 'enum', + options: Object.values(SupportDriver), }) - @IsEnum(SupportDriver) @IsOptional() SUPPORT_DRIVER: SupportDriver = SupportDriver.None; @@ -694,24 +717,25 @@ export class ConfigVariables { group: ConfigVariablesGroup.SupportChatConfig, isSensitive: true, description: 'Chat ID for the support front integration', + type: 'string', }) @ValidateIf((env) => env.SUPPORT_DRIVER === SupportDriver.Front) - @IsString() SUPPORT_FRONT_CHAT_ID: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.SupportChatConfig, isSensitive: true, description: 'HMAC key for the support front integration', + type: 'string', }) @ValidateIf((env) => env.SUPPORT_DRIVER === SupportDriver.Front) - @IsString() SUPPORT_FRONT_HMAC_KEY: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerConfig, isSensitive: true, description: 'Database connection URL', + type: 'string', isEnvOnly: true, }) @IsDefined() @@ -728,9 +752,8 @@ export class ConfigVariables { description: 'Allow connections to a database with self-signed certificates', isEnvOnly: true, + type: 'boolean', }) - @CastToBoolean() - @IsBoolean() @IsOptional() PG_SSL_ALLOW_SELF_SIGNED = false; @@ -738,15 +761,15 @@ export class ConfigVariables { group: ConfigVariablesGroup.ServerConfig, description: 'Enable configuration variables to be stored in the database', isEnvOnly: true, + type: 'boolean', }) - @CastToBoolean() - @IsBoolean() @IsOptional() IS_CONFIG_VARIABLES_IN_DB_ENABLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.TokensDuration, description: 'Time-to-live for cache storage in seconds', + type: 'number', }) @CastToPositiveNumber() CACHE_STORAGE_TTL: number = 3600 * 24 * 7; @@ -756,6 +779,7 @@ export class ConfigVariables { isSensitive: true, description: 'URL for cache storage (e.g., Redis connection URL)', isEnvOnly: true, + type: 'string', }) @IsOptional() @IsUrl({ @@ -768,23 +792,24 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerConfig, description: 'Node environment (development, production, etc.)', + type: 'enum', + options: Object.values(NodeEnvironment), }) - @IsEnum(NodeEnvironment) - @IsString() NODE_ENV: NodeEnvironment = NodeEnvironment.production; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerConfig, description: 'Port for the node server', + type: 'number', }) @CastToPositiveNumber() - @IsNumber() @IsOptional() NODE_PORT = 3000; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerConfig, description: 'Base URL for the server', + type: 'string', }) @IsUrl({ require_tld: false, require_protocol: true }) @IsOptional() @@ -795,22 +820,23 @@ export class ConfigVariables { isSensitive: true, description: 'Secret key for the application', isEnvOnly: true, + type: 'string', }) - @IsString() APP_SECRET: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.RateLimiting, description: 'Maximum number of records affected by mutations', + type: 'number', }) @CastToPositiveNumber() @IsOptional() - @IsNumber() MUTATION_MAXIMUM_AFFECTED_RECORDS = 100; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.RateLimiting, description: 'Time-to-live for API rate limiting in milliseconds', + type: 'number', }) @CastToPositiveNumber() API_RATE_LIMITING_TTL = 100; @@ -819,6 +845,7 @@ export class ConfigVariables { group: ConfigVariablesGroup.RateLimiting, description: 'Maximum number of requests allowed in the rate limiting window', + type: 'number', }) @CastToPositiveNumber() API_RATE_LIMITING_LIMIT = 500; @@ -826,8 +853,8 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.SSL, description: 'Path to the SSL key for enabling HTTPS in local development', + type: 'string', }) - @IsString() @IsOptional() SSL_KEY_PATH: string; @@ -835,8 +862,8 @@ export class ConfigVariables { group: ConfigVariablesGroup.SSL, description: 'Path to the SSL certificate for enabling HTTPS in local development', + type: 'string', }) - @IsString() @IsOptional() SSL_CERT_PATH: string; @@ -844,6 +871,7 @@ export class ConfigVariables { group: ConfigVariablesGroup.CloudflareConfig, isSensitive: true, description: 'API key for Cloudflare integration', + type: 'string', }) @IsString() @ValidateIf((env) => env.CLOUDFLARE_ZONE_ID) @@ -852,22 +880,24 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.CloudflareConfig, description: 'Zone ID for Cloudflare integration', + type: 'string', }) - @IsString() @ValidateIf((env) => env.CLOUDFLARE_API_KEY) CLOUDFLARE_ZONE_ID: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Random string to validate queries from Cloudflare', + type: 'string', }) - @IsString() @IsOptional() CLOUDFLARE_WEBHOOK_SECRET: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.LLM, description: 'Driver for the LLM chat model', + type: 'enum', + options: Object.values(LLMChatModelDriver), }) LLM_CHAT_MODEL_DRIVER: LLMChatModelDriver; @@ -875,6 +905,7 @@ export class ConfigVariables { group: ConfigVariablesGroup.LLM, isSensitive: true, description: 'API key for OpenAI integration', + type: 'string', }) OPENAI_API_KEY: string; @@ -882,37 +913,40 @@ export class ConfigVariables { group: ConfigVariablesGroup.LLM, isSensitive: true, description: 'Secret key for Langfuse integration', + type: 'string', }) LANGFUSE_SECRET_KEY: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.LLM, description: 'Public key for Langfuse integration', + type: 'string', }) LANGFUSE_PUBLIC_KEY: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.LLM, description: 'Driver for LLM tracing', + type: 'enum', + options: Object.values(LLMTracingDriver), }) LLM_TRACING_DRIVER: LLMTracingDriver = LLMTracingDriver.Console; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerConfig, description: 'Enable or disable multi-workspace support', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() IS_MULTIWORKSPACE_ENABLED = false; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Number of inactive days before sending a deletion warning for workspaces. Used in the workspace deletion cron job to determine when to send warning emails.', + type: 'number', }) @CastToPositiveNumber() - @IsNumber() @IsStrictlyLowerThan('WORKSPACE_INACTIVE_DAYS_BEFORE_SOFT_DELETION', { message: '"WORKSPACE_INACTIVE_DAYS_BEFORE_NOTIFICATION" should be strictly lower than "WORKSPACE_INACTIVE_DAYS_BEFORE_SOFT_DELETION"', @@ -922,9 +956,9 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Number of inactive days before soft deleting workspaces', + type: 'number', }) @CastToPositiveNumber() - @IsNumber() @IsStrictlyLowerThan('WORKSPACE_INACTIVE_DAYS_BEFORE_DELETION', { message: '"WORKSPACE_INACTIVE_DAYS_BEFORE_SOFT_DELETION" should be strictly lower than "WORKSPACE_INACTIVE_DAYS_BEFORE_DELETION"', @@ -934,24 +968,25 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Number of inactive days before deleting workspaces', + type: 'number', }) @CastToPositiveNumber() - @IsNumber() WORKSPACE_INACTIVE_DAYS_BEFORE_DELETION = 21; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Maximum number of workspaces that can be deleted in a single execution', + type: 'number', }) @CastToPositiveNumber() - @IsNumber() @ValidateIf((env) => env.MAX_NUMBER_OF_WORKSPACES_DELETED_PER_EXECUTION > 0) MAX_NUMBER_OF_WORKSPACES_DELETED_PER_EXECUTION = 5; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.RateLimiting, description: 'Throttle limit for workflow execution', + type: 'number', }) @CastToPositiveNumber() WORKFLOW_EXEC_THROTTLE_LIMIT = 500; @@ -959,6 +994,7 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.RateLimiting, description: 'Time-to-live for workflow execution throttle in milliseconds', + type: 'number', }) @CastToPositiveNumber() WORKFLOW_EXEC_THROTTLE_TTL = 1000; @@ -966,8 +1002,9 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.CaptchaConfig, description: 'Driver for captcha integration', + type: 'enum', + options: Object.values(CaptchaDriverType), }) - @IsEnum(CaptchaDriverType) @IsOptional() CAPTCHA_DRIVER?: CaptchaDriverType; @@ -975,8 +1012,8 @@ export class ConfigVariables { group: ConfigVariablesGroup.CaptchaConfig, isSensitive: true, description: 'Site key for captcha integration', + type: 'string', }) - @IsString() @IsOptional() CAPTCHA_SITE_KEY?: string; @@ -984,8 +1021,8 @@ export class ConfigVariables { group: ConfigVariablesGroup.CaptchaConfig, isSensitive: true, description: 'Secret key for captcha integration', + type: 'string', }) - @IsString() @IsOptional() CAPTCHA_SECRET_KEY?: string; @@ -993,16 +1030,16 @@ export class ConfigVariables { group: ConfigVariablesGroup.ServerConfig, isSensitive: true, description: 'License key for the Enterprise version', + type: 'string', }) - @IsString() @IsOptional() ENTERPRISE_KEY: string; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Health monitoring time window in minutes', + type: 'number', }) - @IsNumber() @CastToPositiveNumber() @IsOptional() HEALTH_METRICS_TIME_WINDOW_IN_MINUTES = 5; @@ -1010,15 +1047,15 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.Other, description: 'Enable or disable the attachment preview feature', + type: 'boolean', }) - @CastToBoolean() @IsOptional() - @IsBoolean() IS_ATTACHMENT_PREVIEW_ENABLED = true; @ConfigVariablesMetadata({ group: ConfigVariablesGroup.ServerConfig, description: 'Twenty server version', + type: 'string', }) @IsOptionalOrEmptyString() @IsTwentySemVer() diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.spec.ts index 606aa89b7329..f8f992deb0de 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.spec.ts @@ -2,6 +2,7 @@ import { LogLevel } from '@nestjs/common'; import { Test, TestingModule } from '@nestjs/testing'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { ConfigVariablesGroup } from 'src/engine/core-modules/twenty-config/enums/config-variables-group.enum'; import { TypedReflect } from 'src/utils/typed-reflect'; import { ConfigValueConverterService } from './config-value-converter.service'; @@ -29,56 +30,62 @@ describe('ConfigValueConverterService', () => { ); }); - describe('convertToAppValue', () => { + describe('convertDbValueToAppValue', () => { it('should convert string to boolean based on metadata', () => { // Mock the metadata - jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce('boolean'); + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + AUTH_PASSWORD_ENABLED: { + type: 'boolean', + group: ConfigVariablesGroup.Other, + description: 'Enable or disable password authentication for users', + }, + }); expect( - service.convertToAppValue( + service.convertDbValueToAppValue( 'true', 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, ), ).toBe(true); expect( - service.convertToAppValue( + service.convertDbValueToAppValue( 'True', 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, ), ).toBe(true); expect( - service.convertToAppValue( + service.convertDbValueToAppValue( 'yes', 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, ), ).toBe(true); expect( - service.convertToAppValue( + service.convertDbValueToAppValue( '1', 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, ), ).toBe(true); expect( - service.convertToAppValue( + service.convertDbValueToAppValue( 'false', 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, ), ).toBe(false); expect( - service.convertToAppValue( + service.convertDbValueToAppValue( 'False', 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, ), ).toBe(false); expect( - service.convertToAppValue( + service.convertDbValueToAppValue( 'no', 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, ), ).toBe(false); expect( - service.convertToAppValue( + service.convertDbValueToAppValue( '0', 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, ), @@ -86,36 +93,60 @@ describe('ConfigValueConverterService', () => { }); it('should convert string to number based on metadata', () => { - jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce('number'); + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + NODE_PORT: { + type: 'number', + group: ConfigVariablesGroup.ServerConfig, + description: 'Port for the node server', + }, + }); expect( - service.convertToAppValue('42', 'NODE_PORT' as keyof ConfigVariables), + service.convertDbValueToAppValue( + '42', + 'NODE_PORT' as keyof ConfigVariables, + ), ).toBe(42); expect( - service.convertToAppValue('3.14', 'NODE_PORT' as keyof ConfigVariables), + service.convertDbValueToAppValue( + '3.14', + 'NODE_PORT' as keyof ConfigVariables, + ), ).toBe(3.14); - expect(() => { - service.convertToAppValue( + expect( + service.convertDbValueToAppValue( 'not-a-number', 'NODE_PORT' as keyof ConfigVariables, - ); - }).toThrow(); + ), + ).toBeUndefined(); }); it('should convert string to array based on metadata', () => { - jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce('array'); + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + LOG_LEVELS: { + type: 'array', + group: ConfigVariablesGroup.Logging, + description: 'Levels of logging to be captured', + }, + }); expect( - service.convertToAppValue( + service.convertDbValueToAppValue( 'log,error,warn', 'LOG_LEVELS' as keyof ConfigVariables, ), ).toEqual(['log', 'error', 'warn']); - jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce('array'); + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + LOG_LEVELS: { + type: 'array', + group: ConfigVariablesGroup.Logging, + description: 'Levels of logging to be captured', + }, + }); expect( - service.convertToAppValue( + service.convertDbValueToAppValue( '["log","error","warn"]', 'LOG_LEVELS' as keyof ConfigVariables, ), @@ -126,7 +157,7 @@ describe('ConfigValueConverterService', () => { jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce(undefined); expect( - service.convertToAppValue( + service.convertDbValueToAppValue( 'development', 'NODE_ENV' as keyof ConfigVariables, ), @@ -134,22 +165,43 @@ describe('ConfigValueConverterService', () => { }); it('should handle various input types', () => { - jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce('boolean'); + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + AUTH_PASSWORD_ENABLED: { + type: 'boolean', + group: ConfigVariablesGroup.Other, + description: 'Enable or disable password authentication for users', + }, + }); expect( - service.convertToAppValue( + service.convertDbValueToAppValue( true, 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, ), ).toBe(true); - jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce('number'); + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + NODE_PORT: { + type: 'number', + group: ConfigVariablesGroup.ServerConfig, + description: 'Port for the node server', + }, + }); expect( - service.convertToAppValue(42, 'NODE_PORT' as keyof ConfigVariables), + service.convertDbValueToAppValue( + 42, + 'NODE_PORT' as keyof ConfigVariables, + ), ).toBe(42); - jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce('array'); + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + LOG_LEVELS: { + type: 'array', + group: ConfigVariablesGroup.Logging, + description: 'Levels of logging to be captured', + }, + }); expect( - service.convertToAppValue( + service.convertDbValueToAppValue( ['log', 'error'] as LogLevel[], 'LOG_LEVELS' as keyof ConfigVariables, ), @@ -160,31 +212,34 @@ describe('ConfigValueConverterService', () => { jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce(undefined); expect( - service.convertToAppValue('42', 'NODE_PORT' as keyof ConfigVariables), + service.convertDbValueToAppValue( + '42', + 'NODE_PORT' as keyof ConfigVariables, + ), ).toBe(42); }); }); - describe('convertToStorageValue', () => { + describe('convertAppValueToDbValue', () => { it('should handle primitive types directly', () => { - expect(service.convertToStorageValue('string-value' as any)).toBe( + expect(service.convertAppValueToDbValue('string-value' as any)).toBe( 'string-value', ); - expect(service.convertToStorageValue(42 as any)).toBe(42); - expect(service.convertToStorageValue(true as any)).toBe(true); - expect(service.convertToStorageValue(undefined as any)).toBe(null); + expect(service.convertAppValueToDbValue(42 as any)).toBe(42); + expect(service.convertAppValueToDbValue(true as any)).toBe(true); + expect(service.convertAppValueToDbValue(undefined as any)).toBe(null); }); it('should handle arrays', () => { const array = ['log', 'error', 'warn'] as LogLevel[]; - expect(service.convertToStorageValue(array as any)).toEqual(array); + expect(service.convertAppValueToDbValue(array as any)).toEqual(array); }); it('should handle objects', () => { const obj = { key: 'value' }; - expect(service.convertToStorageValue(obj as any)).toEqual(obj); + expect(service.convertAppValueToDbValue(obj as any)).toEqual(obj); }); }); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts index d2e4914c57f6..e444a9222db7 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts @@ -1,20 +1,17 @@ import { Injectable } from '@nestjs/common'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { ConfigVariablesMetadataMap } from 'src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator'; +import { ConfigVariableOptions } from 'src/engine/core-modules/twenty-config/types/config-variable-options.type'; +import { ConfigVariableType } from 'src/engine/core-modules/twenty-config/types/config-variable-type.type'; +import { configTransformers } from 'src/engine/core-modules/twenty-config/utils/config-transformers.util'; import { TypedReflect } from 'src/utils/typed-reflect'; -export type ConfigVariableType = - | 'boolean' - | 'number' - | 'array' - | 'string' - | 'enum'; - @Injectable() export class ConfigValueConverterService { constructor(private readonly configVariables: ConfigVariables) {} - convertToAppValue( + convertDbValueToAppValue( dbValue: unknown, key: T, ): ConfigVariables[T] | undefined { @@ -22,27 +19,33 @@ export class ConfigValueConverterService { return undefined; } - const configType = this.getConfigVarType(key); + const metadata = this.getConfigVariableMetadata(key); + const configType = metadata?.type || this.inferTypeFromValue(key); + const options = metadata?.options; try { switch (configType) { case 'boolean': - return this.convertToBoolean(dbValue) as ConfigVariables[T]; + return configTransformers.boolean(dbValue) as ConfigVariables[T]; case 'number': - return this.convertToNumber( - dbValue, - key as string, - ) as ConfigVariables[T]; - - case 'array': - return this.convertToArray( - dbValue, - key as string, - ) as ConfigVariables[T]; + return configTransformers.number(dbValue) as ConfigVariables[T]; case 'string': - case 'enum': + return configTransformers.string(dbValue) as ConfigVariables[T]; + + case 'array': { + const result = this.convertToArray(dbValue, options); + + return result as ConfigVariables[T]; + } + + case 'enum': { + const result = this.convertToEnum(dbValue, options); + + return result as ConfigVariables[T]; + } + default: return dbValue as ConfigVariables[T]; } @@ -53,7 +56,7 @@ export class ConfigValueConverterService { } } - convertToStorageValue( + convertAppValueToDbValue( appValue: ConfigVariables[T], ): unknown { if (appValue === undefined) { @@ -82,96 +85,75 @@ export class ConfigValueConverterService { ); } - private getConfigVarType( - key: T, - ): ConfigVariableType { - const metadataType = TypedReflect.getMetadata( - 'config-variable:type', - ConfigVariables.prototype.constructor, - key as string, - ) as ConfigVariableType | undefined; - - if (metadataType) { - return metadataType; - } - - const defaultValue = this.configVariables[key]; - - if (typeof defaultValue === 'boolean') return 'boolean'; - if (typeof defaultValue === 'number') return 'number'; - if (Array.isArray(defaultValue)) return 'array'; - - return 'string'; - } - - private convertToBoolean(value: unknown): boolean { - if (typeof value === 'boolean') { - return value; - } - - if (typeof value === 'number') { - return value !== 0; + private convertToArray( + value: unknown, + options?: ConfigVariableOptions, + ): unknown[] { + if (Array.isArray(value)) { + return this.validateArrayAgainstOptions(value, options); } if (typeof value === 'string') { - const lowerValue = value.toLowerCase(); + try { + const parsedArray = JSON.parse(value); - if (['true', 'on', 'yes', '1'].includes(lowerValue)) { - return true; - } + if (Array.isArray(parsedArray)) { + return this.validateArrayAgainstOptions(parsedArray, options); + } + } catch { + const splitArray = value.split(',').map((item) => item.trim()); - if (['false', 'off', 'no', '0'].includes(lowerValue)) { - return false; + return this.validateArrayAgainstOptions(splitArray, options); } } - return false; + return []; } - private convertToNumber(value: unknown, key: string): number { - if (typeof value === 'number') { - return value; + private validateArrayAgainstOptions( + array: unknown[], + options?: ConfigVariableOptions, + ): unknown[] { + if (!options || !Array.isArray(options) || options.length === 0) { + return array; } - if (typeof value === 'string') { - const parsedNumber = parseFloat(value); + return array.filter((item) => options.includes(item as any)); + } - if (isNaN(parsedNumber)) { - throw new Error( - `Invalid number value for config variable ${key}: ${value}`, - ); - } + private convertToEnum( + value: unknown, + options?: ConfigVariableOptions, + ): unknown { + if (!options || !Array.isArray(options) || options.length === 0) { + return value; + } - return parsedNumber; + if (options.includes(value as any)) { + return value; } - throw new Error( - `Cannot convert ${typeof value} to number for config variable ${key}`, - ); + return undefined; } - private convertToArray(value: unknown, key: string): unknown[] { - if (Array.isArray(value)) { - return value; - } + private getConfigVariableMetadata(key: T) { + const allMetadata = TypedReflect.getMetadata( + 'config-variables', + ConfigVariables.prototype.constructor, + ) as ConfigVariablesMetadataMap | undefined; - if (typeof value === 'string') { - try { - const parsedArray = JSON.parse(value); + return allMetadata?.[key as string]; + } - if (Array.isArray(parsedArray)) { - return parsedArray; - } - } catch { - // JSON parsing failed - normal for comma-separated values - // TODO: Handle this better - } + private inferTypeFromValue( + key: T, + ): ConfigVariableType { + const defaultValue = this.configVariables[key]; - return value.split(',').map((item) => item.trim()); - } + if (typeof defaultValue === 'boolean') return 'boolean'; + if (typeof defaultValue === 'number') return 'number'; + if (Array.isArray(defaultValue)) return 'array'; - throw new Error( - `Expected array value for config variable ${key}, got ${typeof value}`, - ); + return 'string'; } } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-boolean.decorator.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-boolean.decorator.ts deleted file mode 100644 index d037b741353e..000000000000 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-boolean.decorator.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { Transform } from 'class-transformer'; - -import { TypedReflect } from 'src/utils/typed-reflect'; - -export const CastToBoolean = - () => - (target: T, propertyKey: keyof T & string) => { - Transform(({ value }: { value: any }) => toBoolean(value))( - target, - propertyKey, - ); - - TypedReflect.defineMetadata( - 'config-variable:type', - 'boolean', - target.constructor, - propertyKey, - ); - }; - -const toBoolean = (value: any) => { - if (typeof value === 'boolean') { - return value; - } - if (['true', 'on', 'yes', '1'].includes(value?.toLowerCase())) { - return true; - } - if (['false', 'off', 'no', '0'].includes(value?.toLowerCase())) { - return false; - } - - return undefined; -}; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-log-level-array.decorator.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-log-level-array.decorator.ts index 773163cf8dd2..35afda2f3312 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-log-level-array.decorator.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-log-level-array.decorator.ts @@ -1,22 +1,7 @@ import { Transform } from 'class-transformer'; -import { TypedReflect } from 'src/utils/typed-reflect'; - -export const CastToLogLevelArray = - () => - (target: T, propertyKey: keyof T & string) => { - Transform(({ value }: { value: string }) => toLogLevelArray(value))( - target, - propertyKey, - ); - - TypedReflect.defineMetadata( - 'config-variable:type', - 'array', - target.constructor, - propertyKey, - ); - }; +export const CastToLogLevelArray = () => + Transform(({ value }: { value: string }) => toLogLevelArray(value)); const toLogLevelArray = (value: any) => { if (typeof value === 'string') { diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-meter-driver.decorator.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-meter-driver.decorator.ts index c9883f6062ad..e8d2512eeff1 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-meter-driver.decorator.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-meter-driver.decorator.ts @@ -1,23 +1,9 @@ import { Transform } from 'class-transformer'; import { MeterDriver } from 'src/engine/core-modules/metrics/types/meter-driver.type'; -import { TypedReflect } from 'src/utils/typed-reflect'; -export const CastToMeterDriverArray = - () => - (target: T, propertyKey: keyof T & string) => { - Transform(({ value }: { value: string }) => toMeterDriverArray(value))( - target, - propertyKey, - ); - - TypedReflect.defineMetadata( - 'config-variable:type', - 'array', - target.constructor, - propertyKey, - ); - }; +export const CastToMeterDriverArray = () => + Transform(({ value }: { value: string }) => toMeterDriverArray(value)); const toMeterDriverArray = (value: string | undefined) => { if (typeof value === 'string') { diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-positive-number.decorator.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-positive-number.decorator.ts index 8664a023ff39..1f532d9661b1 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-positive-number.decorator.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-positive-number.decorator.ts @@ -1,22 +1,7 @@ import { Transform } from 'class-transformer'; -import { TypedReflect } from 'src/utils/typed-reflect'; - -export const CastToPositiveNumber = - () => - (target: T, propertyKey: keyof T & string) => { - Transform(({ value }: { value: string }) => toNumber(value))( - target, - propertyKey, - ); - - TypedReflect.defineMetadata( - 'config-variable:type', - 'number', - target.constructor, - propertyKey, - ); - }; +export const CastToPositiveNumber = () => + Transform(({ value }: { value: string }) => toNumber(value)); const toNumber = (value: any) => { if (typeof value === 'number') { diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-string-array.decorator.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-string-array.decorator.ts deleted file mode 100644 index b70af4e4a8ac..000000000000 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/cast-to-string-array.decorator.ts +++ /dev/null @@ -1,12 +0,0 @@ -import { Transform } from 'class-transformer'; - -export const CastToStringArray = () => - Transform(({ value }: { value: string }) => toStringArray(value)); - -const toStringArray = (value: any) => { - if (typeof value === 'string') { - return value.split(',').map((item) => item.trim()); - } - - return undefined; -}; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator.ts index 0ba4449dbadf..693ec130e6d1 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator.ts @@ -1,6 +1,13 @@ -import { registerDecorator, ValidationOptions } from 'class-validator'; +import { + IsOptional, + registerDecorator, + ValidationOptions, +} from 'class-validator'; import { ConfigVariablesGroup } from 'src/engine/core-modules/twenty-config/enums/config-variables-group.enum'; +import { ConfigVariableOptions } from 'src/engine/core-modules/twenty-config/types/config-variable-options.type'; +import { ConfigVariableType } from 'src/engine/core-modules/twenty-config/types/config-variable-type.type'; +import { applyBasicValidators } from 'src/engine/core-modules/twenty-config/utils/apply-basic-validators.util'; import { TypedReflect } from 'src/utils/typed-reflect'; export interface ConfigVariablesMetadataOptions { @@ -8,6 +15,8 @@ export interface ConfigVariablesMetadataOptions { description: string; isSensitive?: boolean; isEnvOnly?: boolean; + type?: ConfigVariableType; + options?: ConfigVariableOptions; } export type ConfigVariablesMetadataMap = { @@ -31,6 +40,21 @@ export function ConfigVariablesMetadata( target.constructor, ); + const propertyDescriptor = Object.getOwnPropertyDescriptor( + target.constructor.prototype, + propertyKey, + ); + const hasDefaultValue = + propertyDescriptor && propertyDescriptor.value !== undefined; + + if (!hasDefaultValue) { + IsOptional()(target, propertyKey); + } + + if (options.type) { + applyBasicValidators(options.type, target, propertyKey.toString()); + } + registerDecorator({ name: propertyKey.toString(), target: target.constructor, diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts index d143f09d6a80..e0c45a920fa5 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts @@ -43,8 +43,8 @@ describe('ConfigStorageService', () => { { provide: ConfigValueConverterService, useValue: { - convertToAppValue: jest.fn(), - convertToStorageValue: jest.fn(), + convertDbValueToAppValue: jest.fn(), + convertAppValueToDbValue: jest.fn(), }, }, ConfigVariables, @@ -106,17 +106,16 @@ describe('ConfigStorageService', () => { .spyOn(keyValuePairRepository, 'findOne') .mockResolvedValue(mockRecord); - (configValueConverter.convertToAppValue as jest.Mock).mockReturnValue( - convertedValue, - ); + ( + configValueConverter.convertDbValueToAppValue as jest.Mock + ).mockReturnValue(convertedValue); const result = await service.get(key); expect(result).toBe(convertedValue); - expect(configValueConverter.convertToAppValue).toHaveBeenCalledWith( - storedValue, - key, - ); + expect( + configValueConverter.convertDbValueToAppValue, + ).toHaveBeenCalledWith(storedValue, key); }); it('should handle conversion errors', async () => { @@ -129,11 +128,11 @@ describe('ConfigStorageService', () => { .spyOn(keyValuePairRepository, 'findOne') .mockResolvedValue(mockRecord); - (configValueConverter.convertToAppValue as jest.Mock).mockImplementation( - () => { - throw error; - }, - ); + ( + configValueConverter.convertDbValueToAppValue as jest.Mock + ).mockImplementation(() => { + throw error; + }); await expect(service.get(key)).rejects.toThrow('Conversion error'); }); @@ -151,9 +150,9 @@ describe('ConfigStorageService', () => { .spyOn(keyValuePairRepository, 'findOne') .mockResolvedValue(mockRecord); - (configValueConverter.convertToStorageValue as jest.Mock).mockReturnValue( - storedValue, - ); + ( + configValueConverter.convertAppValueToDbValue as jest.Mock + ).mockReturnValue(storedValue); await service.set(key, value); @@ -170,9 +169,9 @@ describe('ConfigStorageService', () => { jest.spyOn(keyValuePairRepository, 'findOne').mockResolvedValue(null); - (configValueConverter.convertToStorageValue as jest.Mock).mockReturnValue( - storedValue, - ); + ( + configValueConverter.convertAppValueToDbValue as jest.Mock + ).mockReturnValue(storedValue); await service.set(key, value); @@ -191,7 +190,7 @@ describe('ConfigStorageService', () => { const error = new Error('Conversion error'); ( - configValueConverter.convertToStorageValue as jest.Mock + configValueConverter.convertAppValueToDbValue as jest.Mock ).mockImplementation(() => { throw error; }); @@ -233,14 +232,14 @@ describe('ConfigStorageService', () => { jest.spyOn(keyValuePairRepository, 'find').mockResolvedValue(configVars); - (configValueConverter.convertToAppValue as jest.Mock).mockImplementation( - (value, key) => { - if (key === 'AUTH_PASSWORD_ENABLED') return true; - if (key === 'EMAIL_FROM_ADDRESS') return 'test@example.com'; + ( + configValueConverter.convertDbValueToAppValue as jest.Mock + ).mockImplementation((value, key) => { + if (key === 'AUTH_PASSWORD_ENABLED') return true; + if (key === 'EMAIL_FROM_ADDRESS') return 'test@example.com'; - return value; - }, - ); + return value; + }); const result = await service.loadAll(); @@ -261,7 +260,7 @@ describe('ConfigStorageService', () => { jest.spyOn(keyValuePairRepository, 'find').mockResolvedValue(configVars); - (configValueConverter.convertToAppValue as jest.Mock) + (configValueConverter.convertDbValueToAppValue as jest.Mock) .mockImplementationOnce(() => { throw new Error('Invalid value'); }) @@ -298,7 +297,7 @@ describe('ConfigStorageService', () => { .mockResolvedValue(configVars); ( - configValueConverter.convertToAppValue as jest.Mock + configValueConverter.convertDbValueToAppValue as jest.Mock ).mockImplementation((value) => { if (value === null) throw new Error('Null value'); @@ -311,7 +310,9 @@ describe('ConfigStorageService', () => { expect(result.get('EMAIL_FROM_ADDRESS' as keyof ConfigVariables)).toBe( 'test@example.com', ); - expect(configValueConverter.convertToAppValue).toHaveBeenCalledTimes(1); // Only called for non-null value + expect( + configValueConverter.convertDbValueToAppValue, + ).toHaveBeenCalledTimes(1); // Only called for non-null value }); }); }); @@ -329,7 +330,7 @@ describe('ConfigStorageService', () => { .mockResolvedValue(mockRecord); ( - configValueConverter.convertToAppValue as jest.Mock + configValueConverter.convertDbValueToAppValue as jest.Mock ).mockImplementation(() => { throw new Error('Invalid boolean value'); }); @@ -348,7 +349,7 @@ describe('ConfigStorageService', () => { .mockResolvedValue(mockRecord); ( - configValueConverter.convertToAppValue as jest.Mock + configValueConverter.convertDbValueToAppValue as jest.Mock ).mockImplementation(() => { throw new Error('Invalid string value'); }); @@ -372,12 +373,12 @@ describe('ConfigStorageService', () => { .mockResolvedValueOnce(initialRecord) .mockResolvedValueOnce(updatedRecord); - (configValueConverter.convertToAppValue as jest.Mock) + (configValueConverter.convertDbValueToAppValue as jest.Mock) .mockReturnValueOnce(initialValue) .mockReturnValueOnce(newValue); ( - configValueConverter.convertToStorageValue as jest.Mock + configValueConverter.convertAppValueToDbValue as jest.Mock ).mockReturnValueOnce('false'); const firstGet = service.get(key); @@ -429,7 +430,7 @@ describe('ConfigStorageService', () => { const error = new Error('Database connection failed'); ( - configValueConverter.convertToStorageValue as jest.Mock + configValueConverter.convertAppValueToDbValue as jest.Mock ).mockReturnValue('true'); jest.spyOn(keyValuePairRepository, 'findOne').mockRejectedValue(error); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts index c4834fcbd281..f3fa02f27aad 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts @@ -46,7 +46,10 @@ export class ConfigStorageService implements ConfigStorageInterface { } try { - return this.configValueConverter.convertToAppValue(result.value, key); + return this.configValueConverter.convertDbValueToAppValue( + result.value, + key, + ); } catch (error) { this.logger.error( `Failed to convert value to app type for key ${key as string}`, @@ -68,7 +71,8 @@ export class ConfigStorageService implements ConfigStorageInterface { let processedValue; try { - processedValue = this.configValueConverter.convertToStorageValue(value); + processedValue = + this.configValueConverter.convertAppValueToDbValue(value); } catch (error) { this.logger.error( `Failed to convert value to storage type for key ${key as string}`, @@ -130,7 +134,7 @@ export class ConfigStorageService implements ConfigStorageInterface { const key = configVar.key as keyof ConfigVariables; try { - const value = this.configValueConverter.convertToAppValue( + const value = this.configValueConverter.convertDbValueToAppValue( configVar.value, key, ); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/types/config-variable-options.type.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/types/config-variable-options.type.ts new file mode 100644 index 000000000000..067ebdc14662 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/types/config-variable-options.type.ts @@ -0,0 +1,3 @@ +export type ConfigVariableOptions = + | readonly (string | number | boolean)[] + | Record; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/types/config-variable-type.type.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/types/config-variable-type.type.ts new file mode 100644 index 000000000000..0313be6ed1da --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/types/config-variable-type.type.ts @@ -0,0 +1,6 @@ +export type ConfigVariableType = + | 'boolean' + | 'number' + | 'array' + | 'string' + | 'enum'; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts new file mode 100644 index 000000000000..19670172d188 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts @@ -0,0 +1,55 @@ +import { Transform } from 'class-transformer'; +import { + IsArray, + IsBoolean, + IsEnum, + IsNumber, + IsString, +} from 'class-validator'; + +import { ConfigVariableOptions } from 'src/engine/core-modules/twenty-config/types/config-variable-options.type'; +import { ConfigVariableType } from 'src/engine/core-modules/twenty-config/types/config-variable-type.type'; +import { configTransformers } from 'src/engine/core-modules/twenty-config/utils/config-transformers.util'; + +// TODO: Add support for custom validators +// Not sure about tailored custom validators for single variables +// Maybe just a single validate method in the decorator that accepts a list of validators +// and applies them to the variable +// also not sure about the file placement/naming -- should this be a util or a service? factory? + +export function applyBasicValidators( + type: ConfigVariableType, + target: object, + propertyKey: string, + options?: ConfigVariableOptions, +): void { + switch (type) { + case 'boolean': + IsBoolean()(target, propertyKey); + Transform(({ value }) => configTransformers.boolean(value))( + target, + propertyKey, + ); + break; + case 'number': + IsNumber()(target, propertyKey); + Transform(({ value }) => configTransformers.number(value))( + target, + propertyKey, + ); + break; + case 'string': + IsString()(target, propertyKey); + break; + case 'enum': + if (options && Array.isArray(options)) { + IsEnum(options)(target, propertyKey); + } + break; + case 'array': + IsArray()(target, propertyKey); + break; + default: + break; + } +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/config-transformers.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/config-transformers.util.ts new file mode 100644 index 000000000000..5dbacf93c996 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/config-transformers.util.ts @@ -0,0 +1,79 @@ +export const configTransformers = { + boolean: (value: unknown): boolean | undefined => { + if (value === null || value === undefined) { + return undefined; + } + + if (typeof value === 'boolean') { + return value; + } + + if (typeof value === 'number') { + return value !== 0; + } + + if (typeof value === 'string') { + const lowerValue = value.toLowerCase(); + + if (['true', 'on', 'yes', '1'].includes(lowerValue)) { + return true; + } + + if (['false', 'off', 'no', '0'].includes(lowerValue)) { + return false; + } + } + + return undefined; + }, + + number: (value: unknown): number | undefined => { + if (value === null || value === undefined) { + return undefined; + } + + if (typeof value === 'number') { + return value; + } + + if (typeof value === 'string') { + const parsedNumber = parseFloat(value); + + if (isNaN(parsedNumber)) { + return undefined; + } + + return parsedNumber; + } + + if (typeof value === 'boolean') { + return value ? 1 : 0; + } + + return undefined; + }, + + string: (value: unknown): string | undefined => { + if (value === null || value === undefined) { + return undefined; + } + + if (typeof value === 'string') { + return value; + } + + if (typeof value === 'number' || typeof value === 'boolean') { + return String(value); + } + + if (Array.isArray(value) || typeof value === 'object') { + try { + return JSON.stringify(value); + } catch { + return undefined; + } + } + + return undefined; + }, +}; diff --git a/packages/twenty-server/src/utils/typed-reflect.ts b/packages/twenty-server/src/utils/typed-reflect.ts index 58ec53f95ba9..a871540f90bd 100644 --- a/packages/twenty-server/src/utils/typed-reflect.ts +++ b/packages/twenty-server/src/utils/typed-reflect.ts @@ -3,7 +3,6 @@ import 'reflect-metadata'; import { Gate } from 'src/engine/twenty-orm/interfaces/gate.interface'; import { WorkspaceEntityDuplicateCriteria } from 'src/engine/api/graphql/workspace-query-builder/types/workspace-entity-duplicate-criteria.type'; -import { ConfigVariableType } from 'src/engine/core-modules/twenty-config/conversion/config-value-converter.service'; import { ConfigVariablesMetadataMap } from 'src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator'; export interface ReflectMetadataTypeMap { @@ -17,7 +16,6 @@ export interface ReflectMetadataTypeMap { ['workspace:duplicate-criteria-metadata-args']: WorkspaceEntityDuplicateCriteria[]; ['config-variables']: ConfigVariablesMetadataMap; ['workspace:is-searchable-metadata-args']: boolean; - ['config-variable:type']: ConfigVariableType; } export class TypedReflect { From afa4f730e5215f4f7f55b4c4b488c75c418d52dd Mon Sep 17 00:00:00 2001 From: ehconitin Date: Mon, 21 Apr 2025 18:49:38 +0530 Subject: [PATCH 40/70] WIP: use p retry instead of custom retry mechanism --- packages/twenty-server/package.json | 1 + .../database-config-driver-retry-options.ts | 7 ++ .../drivers/database-config.driver.ts | 95 +++++++------------ yarn.lock | 3 +- 4 files changed, 45 insertions(+), 61 deletions(-) create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-retry-options.ts diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json index e264b3978cf5..74cf990d2ed2 100644 --- a/packages/twenty-server/package.json +++ b/packages/twenty-server/package.json @@ -52,6 +52,7 @@ "monaco-editor-auto-typings": "^0.4.5", "openid-client": "^5.7.0", "otplib": "^12.0.1", + "p-retry": "4.6.2", "passport": "^0.7.0", "psl": "^1.9.0", "redis": "^4.7.0", diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-retry-options.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-retry-options.ts new file mode 100644 index 000000000000..ada926967f61 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-retry-options.ts @@ -0,0 +1,7 @@ +export const DATABASE_CONFIG_DRIVER_RETRY_OPTIONS = { + retries: 5, + factor: 2, + minTimeout: 1000, + maxTimeout: 30000, + randomize: true, // adds jitter +}; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index 93d7982abdad..80324028a656 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -1,11 +1,12 @@ import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; +import pRetry from 'p-retry'; + import { DatabaseConfigDriverInterface } from 'src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface'; import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; -import { DATABASE_CONFIG_DRIVER_INITIAL_RETRY_DELAY } from 'src/engine/core-modules/twenty-config/constants/database-config-driver-initial-retry-delay'; -import { DATABASE_CONFIG_DRIVER_INITIALIZATION_MAX_RETRIES } from 'src/engine/core-modules/twenty-config/constants/database-config-driver-initialization-max-retries'; +import { DATABASE_CONFIG_DRIVER_RETRY_OPTIONS } from 'src/engine/core-modules/twenty-config/constants/database-config-driver-retry-options'; import { ConfigInitializationState } from 'src/engine/core-modules/twenty-config/enums/config-initialization-state.enum'; import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; @@ -18,8 +19,7 @@ export class DatabaseConfigDriver { private configInitializationState = ConfigInitializationState.NOT_INITIALIZED; private configInitializationPromise: Promise | null = null; - private retryAttempts = 0; - private retryTimer?: NodeJS.Timeout; + private abortController: AbortController | null = null; private readonly logger = new Logger(DatabaseConfigDriver.name); constructor( @@ -49,22 +49,43 @@ export class DatabaseConfigDriver } private async doInitialize(): Promise { + this.abortController = new AbortController(); + try { this.configInitializationState = ConfigInitializationState.INITIALIZING; - await this.loadAllConfigVarsFromDb(); + + await pRetry( + async () => { + if (this.abortController?.signal.aborted) { + throw new pRetry.AbortError('Initialization aborted'); + } + + await this.loadAllConfigVarsFromDb(); + + return true; + }, + { + ...DATABASE_CONFIG_DRIVER_RETRY_OPTIONS, + onFailedAttempt: (error) => { + this.logger.error( + `Failed to initialize database driver (attempt ${error.attemptNumber}/${DATABASE_CONFIG_DRIVER_RETRY_OPTIONS.retries})`, + error instanceof Error ? error.stack : error, + ); + }, + }, + ); + this.configInitializationState = ConfigInitializationState.INITIALIZED; - // Reset retry attempts on successful initialization - this.retryAttempts = 0; + this.logger.log('Database config driver successfully initialized'); } catch (error) { this.logger.error( - `Failed to initialize database driver (attempt ${this.retryAttempts + 1})`, + 'Failed to initialize database driver after maximum retries', error instanceof Error ? error.stack : error, ); this.configInitializationState = ConfigInitializationState.FAILED; - this.scheduleRetry(); } finally { - // Reset the promise to allow future initialization attempts this.configInitializationPromise = null; + this.abortController = null; } } @@ -168,57 +189,11 @@ export class DatabaseConfigDriver }); } - private scheduleRetry(): void { - this.retryAttempts++; - - if ( - this.retryAttempts > DATABASE_CONFIG_DRIVER_INITIALIZATION_MAX_RETRIES - ) { - this.logger.error( - `Max retry attempts (${DATABASE_CONFIG_DRIVER_INITIALIZATION_MAX_RETRIES}) reached, giving up initialization`, - ); - - return; - } - - const delay = - DATABASE_CONFIG_DRIVER_INITIAL_RETRY_DELAY * - Math.pow(2, this.retryAttempts - 1); - - this.logger.log( - `Scheduling retry attempt ${this.retryAttempts} in ${delay}ms`, - ); - - if (this.retryTimer) { - clearTimeout(this.retryTimer); - } - - this.retryTimer = setTimeout(() => { - this.logger.log(`Executing retry attempt ${this.retryAttempts}`); - - if ( - this.configInitializationState === - ConfigInitializationState.INITIALIZING - ) { - this.logger.log( - 'Skipping retry attempt as initialization is already in progress', - ); - - return; - } - - this.initialize().catch((error) => { - this.logger.error( - `Retry initialization attempt ${this.retryAttempts} failed`, - error instanceof Error ? error.stack : error, - ); - }); - }, delay); - } - onModuleDestroy() { - if (this.retryTimer) { - clearTimeout(this.retryTimer); + // Abort any in-progress initialization + if (this.abortController) { + this.abortController.abort(); + this.abortController = null; } } } diff --git a/yarn.lock b/yarn.lock index cc5ec7f55c04..42418ae6b768 100644 --- a/yarn.lock +++ b/yarn.lock @@ -46722,7 +46722,7 @@ __metadata: languageName: node linkType: hard -"p-retry@npm:4": +"p-retry@npm:4, p-retry@npm:4.6.2": version: 4.6.2 resolution: "p-retry@npm:4.6.2" dependencies: @@ -55084,6 +55084,7 @@ __metadata: monaco-editor-auto-typings: "npm:^0.4.5" openid-client: "npm:^5.7.0" otplib: "npm:^12.0.1" + p-retry: "npm:4.6.2" passport: "npm:^0.7.0" psl: "npm:^1.9.0" redis: "npm:^4.7.0" From 1504a20005d5101e59aa7228742e1448833e3ff1 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Mon, 21 Apr 2025 18:49:49 +0530 Subject: [PATCH 41/70] fix tests --- .../__tests__/database-config.driver.spec.ts | 255 +++++++++++++----- 1 file changed, 181 insertions(+), 74 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index 9cf4b293c4a6..b9d3ca803cd0 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -8,6 +8,14 @@ import { ConfigInitializationState } from 'src/engine/core-modules/twenty-config import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; +jest.mock('p-retry', () => { + return { + __esModule: true, + default: jest.fn().mockImplementation(async (fn) => fn()), + AbortError: jest.requireActual('p-retry').AbortError, + }; +}); + jest.mock( 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util', () => ({ @@ -67,6 +75,7 @@ describe('DatabaseConfigDriver', () => { afterEach(() => { jest.clearAllMocks(); + jest.useRealTimers(); }); it('should be defined', () => { @@ -79,8 +88,8 @@ describe('DatabaseConfigDriver', () => { keyof ConfigVariables, ConfigVariables[keyof ConfigVariables] >([ - ['AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, true], - ['EMAIL_FROM_ADDRESS' as keyof ConfigVariables, 'test@example.com'], + ['AUTH_PASSWORD_ENABLED', true], + ['EMAIL_FROM_ADDRESS', 'test@example.com'], ]); jest.spyOn(configStorage, 'loadAll').mockResolvedValue(configVars); @@ -89,60 +98,60 @@ describe('DatabaseConfigDriver', () => { expect(configStorage.loadAll).toHaveBeenCalled(); expect(configCache.set).toHaveBeenCalledTimes(2); + expect(driver['configInitializationState']).toBe( + ConfigInitializationState.INITIALIZED, + ); }); - it('should handle initialization failure and retry', async () => { + it('should handle initialization failure', async () => { const error = new Error('DB error'); jest.spyOn(configStorage, 'loadAll').mockRejectedValue(error); await driver.initialize(); - expect((driver as any).configInitializationState).toBe( + expect(driver['configInitializationState']).toBe( ConfigInitializationState.FAILED, ); - expect(configStorage.loadAll).toHaveBeenCalled(); }); - it('should retry initialization after failure', async () => { + it('should succeed after multiple failures', async () => { + const pRetryMock = jest.requireMock('p-retry').default; + + pRetryMock.mockImplementationOnce(async (fn) => { + try { + return await fn(); + } catch (error) { + try { + return await fn(); + } catch (error) { + return await fn(); + } + } + }); + const error = new Error('DB error'); jest .spyOn(configStorage, 'loadAll') - .mockRejectedValueOnce(error) // First call fails - .mockResolvedValueOnce(new Map()); // Second call succeeds + .mockRejectedValueOnce(error) + .mockRejectedValueOnce(error) + .mockResolvedValueOnce(new Map()); - const originalScheduleRetry = (driver as any).scheduleRetry; + jest.spyOn(driver['logger'], 'error'); + jest.spyOn(driver['logger'], 'log'); - (driver as any).scheduleRetry = jest.fn(function () { - this.configInitializationPromise = null; - this.configInitializationState = - ConfigInitializationState.NOT_INITIALIZED; - this.initialize(); - }); - - // First initialization attempt (will fail and trigger retry) await driver.initialize(); - // The mock scheduleRetry should have been called - expect((driver as any).scheduleRetry).toHaveBeenCalled(); - - // Since our mock immediately calls initialize again and we've mocked - // the second attempt to succeed, the state should now be INITIALIZED - expect((driver as any).configInitializationState).toBe( + expect(driver['configInitializationState']).toBe( ConfigInitializationState.INITIALIZED, ); - // Restore original method - (driver as any).scheduleRetry = originalScheduleRetry; + expect(configStorage.loadAll).toHaveBeenCalledTimes(3); }); - it('should handle concurrent initialization', async () => { - const configVars = new Map([ - ['AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, true], - ]); - - jest.spyOn(configStorage, 'loadAll').mockResolvedValue(configVars); + it('should handle concurrent initialization calls', async () => { + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(new Map()); const promises = [ driver.initialize(), @@ -155,40 +164,32 @@ describe('DatabaseConfigDriver', () => { expect(configStorage.loadAll).toHaveBeenCalledTimes(1); }); - it('should reset retry attempts after successful initialization', async () => { - (driver as any).retryAttempts = 3; - - jest.spyOn(configStorage, 'loadAll').mockResolvedValue(new Map()); + it('should abort initialization when destroyed', async () => { + jest.useFakeTimers(); - await driver.initialize(); - - expect((driver as any).retryAttempts).toBe(0); - expect((driver as any).configInitializationState).toBe( - ConfigInitializationState.INITIALIZED, - ); - }); - - it('should increment retry attempts when initialization fails', async () => { - const error = new Error('DB error'); + let rejectFn!: (reason?: any) => void; + const pendingPromise = new Promise< + Map + >((_, reject) => { + rejectFn = reject; + }); - (driver as any).retryAttempts = 0; + jest.spyOn(configStorage, 'loadAll').mockReturnValue(pendingPromise); - const scheduleRetrySpy = jest.fn(); - const originalScheduleRetry = (driver as any).scheduleRetry; + const initPromise = driver.initialize(); - (driver as any).scheduleRetry = scheduleRetrySpy; + driver.onModuleDestroy(); - jest.spyOn(configStorage, 'loadAll').mockRejectedValue(error); + rejectFn(new Error('Aborted')); - await driver.initialize(); + await initPromise; - expect(scheduleRetrySpy).toHaveBeenCalled(); - expect((driver as any).configInitializationState).toBe( + expect(driver['configInitializationState']).toBe( ConfigInitializationState.FAILED, ); - (driver as any).scheduleRetry = originalScheduleRetry; - }); + jest.useRealTimers(); + }, 10000); }); describe('get', () => { @@ -202,7 +203,7 @@ describe('DatabaseConfigDriver', () => { const envValue = true; jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); - (driver as any).configInitializationState = + driver['configInitializationState'] = ConfigInitializationState.NOT_INITIALIZED; const result = driver.get(key); @@ -251,6 +252,30 @@ describe('DatabaseConfigDriver', () => { expect(environmentDriver.get).toHaveBeenCalledWith(key); }); + it('should schedule refresh when cache misses and not known missing', async () => { + jest.useFakeTimers(); + + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const envValue = true; + + jest.spyOn(configCache, 'get').mockReturnValue(undefined); + jest.spyOn(configCache, 'isKeyKnownMissing').mockReturnValue(false); + jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); + + const fetchSpy = jest + .spyOn(driver, 'fetchAndCacheConfig') + .mockResolvedValue(); + + const result = driver.get(key); + + expect(result).toBe(envValue); + + // Run immediate callbacks + jest.runAllTimers(); + + expect(fetchSpy).toHaveBeenCalledWith(key); + }); + it('should handle different config variable types correctly', () => { const stringKey = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; const booleanKey = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; @@ -309,13 +334,58 @@ describe('DatabaseConfigDriver', () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const value = true; - (driver as any).configInitializationState = + driver['configInitializationState'] = ConfigInitializationState.NOT_INITIALIZED; await expect(driver.update(key, value)).rejects.toThrow(); }); }); + describe('fetchAndCacheConfig', () => { + it('should refresh config from storage', async () => { + const key = 'AUTH_PASSWORD_ENABLED'; + const value = true; + + jest.spyOn(configStorage, 'get').mockResolvedValue(value); + + await driver.fetchAndCacheConfig(key); + + expect(configStorage.get).toHaveBeenCalledWith(key); + expect(configCache.set).toHaveBeenCalledWith(key, value); + }); + + it('should mark key as missing when value is undefined', async () => { + const key = 'AUTH_PASSWORD_ENABLED'; + + jest.spyOn(configStorage, 'get').mockResolvedValue(undefined); + + await driver.fetchAndCacheConfig(key); + + expect(configStorage.get).toHaveBeenCalledWith(key); + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(key); + expect(configCache.set).not.toHaveBeenCalled(); + }); + + it('should mark key as missing when storage fetch fails', async () => { + const key = 'AUTH_PASSWORD_ENABLED'; + const error = new Error('Storage error'); + + jest.spyOn(configStorage, 'get').mockRejectedValue(error); + const loggerSpy = jest + .spyOn(driver['logger'], 'error') + .mockImplementation(); + + await driver.fetchAndCacheConfig(key); + + expect(configStorage.get).toHaveBeenCalledWith(key); + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(key); + expect(loggerSpy).toHaveBeenCalledWith( + expect.stringContaining('Failed to fetch config'), + error, + ); + }); + }); + describe('cache operations', () => { it('should return cache info', () => { const cacheInfo = { @@ -345,16 +415,23 @@ describe('DatabaseConfigDriver', () => { const error = new Error('Storage error'); jest.spyOn(configStorage, 'set').mockRejectedValue(error); + const loggerSpy = jest + .spyOn(driver['logger'], 'error') + .mockImplementation(); await expect(driver.update(key, value)).rejects.toThrow(); expect(configCache.set).not.toHaveBeenCalled(); + expect(loggerSpy).toHaveBeenCalledWith( + expect.stringContaining('Failed to update config'), + error, + ); }); it('should use environment driver when not initialized', async () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const envValue = true; - (driver as any).configInitializationState = + driver['configInitializationState'] = ConfigInitializationState.NOT_INITIALIZED; jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); @@ -411,8 +488,7 @@ describe('DatabaseConfigDriver', () => { it('should propagate environment driver errors when using environment driver', async () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - // Force use of environment driver - (driver as any).configInitializationState = + driver['configInitializationState'] = ConfigInitializationState.NOT_INITIALIZED; jest.spyOn(environmentDriver, 'get').mockImplementation(() => { throw new Error('Environment error'); @@ -423,26 +499,57 @@ describe('DatabaseConfigDriver', () => { }); }); - it('should refresh config from storage', async () => { - const key = 'AUTH_PASSWORD_ENABLED'; - const value = true; + describe('background operations', () => { + it('should schedule refresh in background', async () => { + jest.useFakeTimers(); - jest.spyOn(configStorage, 'get').mockResolvedValue(value); + const fetchPromise = Promise.resolve(); + const fetchSpy = jest + .spyOn(driver, 'fetchAndCacheConfig') + .mockReturnValue(fetchPromise); - await driver.fetchAndCacheConfig(key); + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(new Map()); + await driver.initialize(); - expect(configStorage.get).toHaveBeenCalledWith(key); - expect(configCache.set).toHaveBeenCalledWith(key, value); - }); + const key = 'MISSING_KEY' as keyof ConfigVariables; + + jest.spyOn(configCache, 'get').mockReturnValue(undefined); + jest.spyOn(configCache, 'isKeyKnownMissing').mockReturnValue(false); + + driver.get(key); - it('should handle refresh when value is undefined', async () => { - const key = 'AUTH_PASSWORD_ENABLED'; + jest.runAllTimers(); - jest.spyOn(configStorage, 'get').mockResolvedValue(undefined); + await fetchPromise; - await driver.fetchAndCacheConfig(key); + expect(fetchSpy).toHaveBeenCalledWith(key); - expect(configStorage.get).toHaveBeenCalledWith(key); - expect(configCache.set).not.toHaveBeenCalled(); + jest.useRealTimers(); + }); + + it('should correctly populate cache during initialization', async () => { + const configVars = new Map< + keyof ConfigVariables, + ConfigVariables[keyof ConfigVariables] + >([ + ['AUTH_PASSWORD_ENABLED', true], + ['EMAIL_FROM_ADDRESS', 'test@example.com'], + ['NODE_PORT', 3000], + ]); + + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(configVars); + + await driver.initialize(); + + expect(configCache.set).toHaveBeenCalledWith( + 'AUTH_PASSWORD_ENABLED', + true, + ); + expect(configCache.set).toHaveBeenCalledWith( + 'EMAIL_FROM_ADDRESS', + 'test@example.com', + ); + expect(configCache.set).toHaveBeenCalledWith('NODE_PORT', 3000); + }); }); }); From 8ac0b15e581a1fd957a5ddf3d9443902d4a10b82 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Tue, 22 Apr 2025 17:15:33 +0530 Subject: [PATCH 42/70] hasty commit --- .../core-modules/twenty-config/config-variables.ts | 5 ++--- .../conversion/config-value-converter.service.ts | 4 ++-- .../config-variables-metadata.decorator.ts | 7 ++++++- .../utils/apply-basic-validators.util.ts | 12 ++++++------ 4 files changed, 16 insertions(+), 12 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts index 5dd8269d7980..2637fa6c8399 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/config-variables.ts @@ -4,7 +4,6 @@ import { plainToClass } from 'class-transformer'; import { IsDefined, IsOptional, - IsString, IsUrl, ValidateIf, ValidationError, @@ -292,7 +291,8 @@ export class ConfigVariables { @ConfigVariablesMetadata({ group: ConfigVariablesGroup.EmailSettings, description: 'Email driver to use for sending emails', - type: 'string', + type: 'enum', + options: Object.values(EmailDriver), }) EMAIL_DRIVER: EmailDriver = EmailDriver.Logger; @@ -861,7 +861,6 @@ export class ConfigVariables { description: 'API key for Cloudflare integration', type: 'string', }) - @IsString() @ValidateIf((env) => env.CLOUDFLARE_ZONE_ID) CLOUDFLARE_API_KEY: string; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts index e444a9222db7..d445376fa85e 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts @@ -118,7 +118,7 @@ export class ConfigValueConverterService { return array; } - return array.filter((item) => options.includes(item as any)); + return array.filter((item) => options.includes(item as string)); } private convertToEnum( @@ -129,7 +129,7 @@ export class ConfigValueConverterService { return value; } - if (options.includes(value as any)) { + if (options.includes(value as string)) { return value; } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator.ts index 693ec130e6d1..914d79558957 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator.ts @@ -52,7 +52,12 @@ export function ConfigVariablesMetadata( } if (options.type) { - applyBasicValidators(options.type, target, propertyKey.toString()); + applyBasicValidators( + options.type, + target, + propertyKey.toString(), + options.options, + ); } registerDecorator({ diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts index 19670172d188..2bce625c4bee 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts @@ -1,10 +1,10 @@ import { Transform } from 'class-transformer'; import { - IsArray, - IsBoolean, - IsEnum, - IsNumber, - IsString, + IsArray, + IsBoolean, + IsEnum, + IsNumber, + IsString, } from 'class-validator'; import { ConfigVariableOptions } from 'src/engine/core-modules/twenty-config/types/config-variable-options.type'; @@ -50,6 +50,6 @@ export function applyBasicValidators( IsArray()(target, propertyKey); break; default: - break; + throw new Error(`Unsupported config variable type: ${type}`); } } From 35535a8f1bc370666649fae474703b133d396e6d Mon Sep 17 00:00:00 2001 From: ehconitin Date: Tue, 22 Apr 2025 23:46:36 +0530 Subject: [PATCH 43/70] remove p retry --- .../cache/config-cache.service.ts | 4 + .../database-config-driver-retry-options.ts | 7 - .../__tests__/database-config.driver.spec.ts | 121 +--------------- .../drivers/database-config.driver.ts | 129 ++++++------------ .../twenty-config.service.spec.ts | 46 +------ .../twenty-config/twenty-config.service.ts | 28 +--- .../utils/apply-basic-validators.util.ts | 10 +- 7 files changed, 57 insertions(+), 288 deletions(-) delete mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-retry-options.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts index 66fd04337aeb..c315e0dae7e9 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts @@ -110,6 +110,10 @@ export class ConfigCacheService implements OnModuleDestroy { return now - entry.timestamp > entry.ttl; } + // Should this process be done using @nestjs/schedule? + // setInterval is not a good idea because it will not be able to run in a cluster? + // Maybe it should be a part of the DatabaseConfigDriver? -- for now kept it here to have a single place to control the cache + private startCacheScavenging(): void { this.cacheScavengeInterval = setInterval(() => { this.scavengeCache(); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-retry-options.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-retry-options.ts deleted file mode 100644 index ada926967f61..000000000000 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-retry-options.ts +++ /dev/null @@ -1,7 +0,0 @@ -export const DATABASE_CONFIG_DRIVER_RETRY_OPTIONS = { - retries: 5, - factor: 2, - minTimeout: 1000, - maxTimeout: 30000, - randomize: true, // adds jitter -}; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index b9d3ca803cd0..da10092a46bb 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -4,18 +4,9 @@ import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/ import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; -import { ConfigInitializationState } from 'src/engine/core-modules/twenty-config/enums/config-initialization-state.enum'; import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; -jest.mock('p-retry', () => { - return { - __esModule: true, - default: jest.fn().mockImplementation(async (fn) => fn()), - AbortError: jest.requireActual('p-retry').AbortError, - }; -}); - jest.mock( 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util', () => ({ @@ -98,56 +89,17 @@ describe('DatabaseConfigDriver', () => { expect(configStorage.loadAll).toHaveBeenCalled(); expect(configCache.set).toHaveBeenCalledTimes(2); - expect(driver['configInitializationState']).toBe( - ConfigInitializationState.INITIALIZED, - ); }); it('should handle initialization failure', async () => { const error = new Error('DB error'); jest.spyOn(configStorage, 'loadAll').mockRejectedValue(error); - - await driver.initialize(); - - expect(driver['configInitializationState']).toBe( - ConfigInitializationState.FAILED, - ); - }); - - it('should succeed after multiple failures', async () => { - const pRetryMock = jest.requireMock('p-retry').default; - - pRetryMock.mockImplementationOnce(async (fn) => { - try { - return await fn(); - } catch (error) { - try { - return await fn(); - } catch (error) { - return await fn(); - } - } - }); - - const error = new Error('DB error'); - - jest - .spyOn(configStorage, 'loadAll') - .mockRejectedValueOnce(error) - .mockRejectedValueOnce(error) - .mockResolvedValueOnce(new Map()); - jest.spyOn(driver['logger'], 'error'); - jest.spyOn(driver['logger'], 'log'); - await driver.initialize(); + await expect(driver.initialize()).rejects.toThrow(error); - expect(driver['configInitializationState']).toBe( - ConfigInitializationState.INITIALIZED, - ); - - expect(configStorage.loadAll).toHaveBeenCalledTimes(3); + expect(driver['logger'].error).toHaveBeenCalled(); }); it('should handle concurrent initialization calls', async () => { @@ -163,33 +115,6 @@ describe('DatabaseConfigDriver', () => { expect(configStorage.loadAll).toHaveBeenCalledTimes(1); }); - - it('should abort initialization when destroyed', async () => { - jest.useFakeTimers(); - - let rejectFn!: (reason?: any) => void; - const pendingPromise = new Promise< - Map - >((_, reject) => { - rejectFn = reject; - }); - - jest.spyOn(configStorage, 'loadAll').mockReturnValue(pendingPromise); - - const initPromise = driver.initialize(); - - driver.onModuleDestroy(); - - rejectFn(new Error('Aborted')); - - await initPromise; - - expect(driver['configInitializationState']).toBe( - ConfigInitializationState.FAILED, - ); - - jest.useRealTimers(); - }, 10000); }); describe('get', () => { @@ -198,20 +123,6 @@ describe('DatabaseConfigDriver', () => { await driver.initialize(); }); - it('should use environment driver when not initialized', async () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - const envValue = true; - - jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); - driver['configInitializationState'] = - ConfigInitializationState.NOT_INITIALIZED; - - const result = driver.get(key); - - expect(result).toBe(envValue); - expect(environmentDriver.get).toHaveBeenCalledWith(key); - }); - it('should use environment driver for env-only variables', async () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const envValue = true; @@ -329,16 +240,6 @@ describe('DatabaseConfigDriver', () => { await expect(driver.update(key, value)).rejects.toThrow(); }); - - it('should throw error when updating before initialization', async () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - const value = true; - - driver['configInitializationState'] = - ConfigInitializationState.NOT_INITIALIZED; - - await expect(driver.update(key, value)).rejects.toThrow(); - }); }); describe('fetchAndCacheConfig', () => { @@ -427,21 +328,6 @@ describe('DatabaseConfigDriver', () => { ); }); - it('should use environment driver when not initialized', async () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - const envValue = true; - - driver['configInitializationState'] = - ConfigInitializationState.NOT_INITIALIZED; - jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); - - const result = driver.get(key); - - expect(result).toBe(envValue); - expect(environmentDriver.get).toHaveBeenCalledWith(key); - expect(configCache.get).not.toHaveBeenCalled(); - }); - it('should use environment driver for env-only variables', async () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const envValue = true; @@ -488,8 +374,7 @@ describe('DatabaseConfigDriver', () => { it('should propagate environment driver errors when using environment driver', async () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - driver['configInitializationState'] = - ConfigInitializationState.NOT_INITIALIZED; + (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(true); jest.spyOn(environmentDriver, 'get').mockImplementation(() => { throw new Error('Environment error'); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index 80324028a656..d65d9c05d9fd 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -1,25 +1,17 @@ -import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; - -import pRetry from 'p-retry'; +import { Injectable, Logger } from '@nestjs/common'; import { DatabaseConfigDriverInterface } from 'src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface'; import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; -import { DATABASE_CONFIG_DRIVER_RETRY_OPTIONS } from 'src/engine/core-modules/twenty-config/constants/database-config-driver-retry-options'; -import { ConfigInitializationState } from 'src/engine/core-modules/twenty-config/enums/config-initialization-state.enum'; import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; import { EnvironmentConfigDriver } from './environment-config.driver'; @Injectable() -export class DatabaseConfigDriver - implements DatabaseConfigDriverInterface, OnModuleDestroy -{ - private configInitializationState = ConfigInitializationState.NOT_INITIALIZED; - private configInitializationPromise: Promise | null = null; - private abortController: AbortController | null = null; +export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { + private initializationPromise: Promise | null = null; private readonly logger = new Logger(DatabaseConfigDriver.name); constructor( @@ -29,68 +21,38 @@ export class DatabaseConfigDriver ) {} async initialize(): Promise { - if ( - this.configInitializationState === ConfigInitializationState.INITIALIZED - ) { - return Promise.resolve(); - } - if (this.configInitializationPromise) { - return this.configInitializationPromise; - } + if (this.initializationPromise) { + this.logger.verbose('Config database initialization already in progress'); - if (this.configInitializationState === ConfigInitializationState.FAILED) { - this.configInitializationState = - ConfigInitializationState.NOT_INITIALIZED; + return this.initializationPromise; } - this.configInitializationPromise = this.doInitialize(); - - return this.configInitializationPromise; - } - - private async doInitialize(): Promise { - this.abortController = new AbortController(); + this.logger.debug('Starting database config initialization'); + + const promise = this.loadAllConfigVarsFromDb() + .then((loadedCount) => { + this.logger.log( + `Database config ready: loaded ${loadedCount} variables`, + ); + }) + .catch((error) => { + this.logger.error( + 'Failed to load database config: unable to connect to database or fetch config', + error instanceof Error ? error.stack : error, + ); + throw error; + }) + .finally(() => { + this.initializationPromise = null; + }); - try { - this.configInitializationState = ConfigInitializationState.INITIALIZING; - - await pRetry( - async () => { - if (this.abortController?.signal.aborted) { - throw new pRetry.AbortError('Initialization aborted'); - } - - await this.loadAllConfigVarsFromDb(); - - return true; - }, - { - ...DATABASE_CONFIG_DRIVER_RETRY_OPTIONS, - onFailedAttempt: (error) => { - this.logger.error( - `Failed to initialize database driver (attempt ${error.attemptNumber}/${DATABASE_CONFIG_DRIVER_RETRY_OPTIONS.retries})`, - error instanceof Error ? error.stack : error, - ); - }, - }, - ); + this.initializationPromise = promise; - this.configInitializationState = ConfigInitializationState.INITIALIZED; - this.logger.log('Database config driver successfully initialized'); - } catch (error) { - this.logger.error( - 'Failed to initialize database driver after maximum retries', - error instanceof Error ? error.stack : error, - ); - this.configInitializationState = ConfigInitializationState.FAILED; - } finally { - this.configInitializationPromise = null; - this.abortController = null; - } + return promise; } get(key: T): ConfigVariables[T] { - if (this.shouldUseEnvironment(key)) { + if (isEnvOnlyConfigVar(key)) { return this.environmentDriver.get(key); } @@ -113,7 +75,7 @@ export class DatabaseConfigDriver key: T, value: ConfigVariables[T], ): Promise { - if (this.shouldUseEnvironment(key)) { + if (isEnvOnlyConfigVar(key)) { throw new Error( `Cannot update environment-only variable: ${key as string}`, ); @@ -122,6 +84,7 @@ export class DatabaseConfigDriver try { await this.configStorage.set(key, value); this.configCache.set(key, value); + this.logger.debug(`Updated config variable: ${key as string}`); } catch (error) { this.logger.error(`Failed to update config for ${key as string}`, error); throw error; @@ -134,8 +97,14 @@ export class DatabaseConfigDriver if (value !== undefined) { this.configCache.set(key, value); + this.logger.debug( + `Refreshed config variable in cache: ${key as string}`, + ); } else { this.configCache.markKeyAsMissing(key); + this.logger.debug( + `Marked config variable as missing: ${key as string}`, + ); } } catch (error) { this.logger.error(`Failed to fetch config for ${key as string}`, error); @@ -151,24 +120,16 @@ export class DatabaseConfigDriver return this.configCache.getCacheInfo(); } - private shouldUseEnvironment(key: keyof ConfigVariables): boolean { - return ( - this.configInitializationState !== - ConfigInitializationState.INITIALIZED || isEnvOnlyConfigVar(key) - ); - } - - private async loadAllConfigVarsFromDb(): Promise { + private async loadAllConfigVarsFromDb(): Promise { try { + this.logger.debug('Fetching all config variables from database'); const configVars = await this.configStorage.loadAll(); for (const [key, value] of configVars.entries()) { this.configCache.set(key, value); } - this.logger.log( - `Loaded ${configVars.size} config variables from database`, - ); + return configVars.size; } catch (error) { this.logger.error('Failed to load config variables from database', error); throw error; @@ -176,24 +137,10 @@ export class DatabaseConfigDriver } private async scheduleRefresh(key: keyof ConfigVariables): Promise { - if ( - this.configInitializationState !== ConfigInitializationState.INITIALIZED - ) { - return; - } - setImmediate(async () => { await this.fetchAndCacheConfig(key).catch((error) => { this.logger.error(`Failed to fetch config for ${key as string}`, error); }); }); } - - onModuleDestroy() { - // Abort any in-progress initialization - if (this.abortController) { - this.abortController.abort(); - this.abortController = null; - } - } } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts index 192a9986c834..3491b0390672 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts @@ -166,46 +166,6 @@ describe('TwentyConfigService', () => { }); describe('onModuleInit', () => { - it('should set initialization state to INITIALIZED when db config is disabled', async () => { - jest.spyOn(configService, 'get').mockReturnValue(false); - - const newService = new TwentyConfigService( - configService, - databaseConfigDriver, - environmentConfigDriver, - ); - - await newService.onModuleInit(); - - const privateProps = - newService as unknown as TwentyConfigServicePrivateProps; - - expect(privateProps.configInitializationState).toBe( - ConfigInitializationState.INITIALIZED, - ); - }); - - it('should not change initialization state when db config is enabled', async () => { - jest.spyOn(configService, 'get').mockReturnValue(true); - - const newService = new TwentyConfigService( - configService, - databaseConfigDriver, - environmentConfigDriver, - ); - - await newService.onModuleInit(); - - const privateProps = - newService as unknown as TwentyConfigServicePrivateProps; - - expect(privateProps.configInitializationState).toBe( - ConfigInitializationState.NOT_INITIALIZED, - ); - }); - }); - - describe('onApplicationBootstrap', () => { it('should do nothing when db config is disabled', async () => { jest.spyOn(configService, 'get').mockReturnValue(false); @@ -215,7 +175,7 @@ describe('TwentyConfigService', () => { environmentConfigDriver, ); - await newService.onApplicationBootstrap(); + await newService.onModuleInit(); expect(databaseConfigDriver.initialize).not.toHaveBeenCalled(); @@ -234,7 +194,7 @@ describe('TwentyConfigService', () => { environmentConfigDriver, ); - await newService.onApplicationBootstrap(); + await newService.onModuleInit(); expect(databaseConfigDriver.initialize).toHaveBeenCalled(); @@ -259,7 +219,7 @@ describe('TwentyConfigService', () => { environmentConfigDriver, ); - await newService.onApplicationBootstrap(); + await newService.onModuleInit(); expect(databaseConfigDriver.initialize).toHaveBeenCalled(); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts index 098f818a9e2c..4c9cb2f208df 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts @@ -1,9 +1,4 @@ -import { - Injectable, - Logger, - OnApplicationBootstrap, - OnModuleInit, -} from '@nestjs/common'; +import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { isString } from 'class-validator'; @@ -20,9 +15,7 @@ import { configVariableMaskSensitiveData } from 'src/engine/core-modules/twenty- import { TypedReflect } from 'src/utils/typed-reflect'; @Injectable() -export class TwentyConfigService - implements OnModuleInit, OnApplicationBootstrap -{ +export class TwentyConfigService implements OnModuleInit { private driver: DatabaseConfigDriver | EnvironmentConfigDriver; private configInitializationState = ConfigInitializationState.NOT_INITIALIZED; private readonly isConfigVarInDbEnabled: boolean; @@ -35,11 +28,8 @@ export class TwentyConfigService ) { this.driver = this.environmentConfigDriver; - const configVarInDb = this.configService.get( - 'IS_CONFIG_VARIABLES_IN_DB_ENABLED', - ); - - this.isConfigVarInDbEnabled = configVarInDb === true; + this.isConfigVarInDbEnabled = + this.configService.get('IS_CONFIG_VARIABLES_IN_DB_ENABLED') === true; this.logger.log( `Database configuration is ${this.isConfigVarInDbEnabled ? 'enabled' : 'disabled'}`, @@ -56,16 +46,6 @@ export class TwentyConfigService return; } - this.logger.log( - 'Database configuration is enabled, will initialize after application bootstrap', - ); - } - - async onApplicationBootstrap() { - if (!this.isConfigVarInDbEnabled) { - return; - } - try { this.logger.log('Initializing database driver for configuration'); this.configInitializationState = ConfigInitializationState.INITIALIZING; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts index 2bce625c4bee..8f442d28d9fc 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts @@ -1,10 +1,10 @@ import { Transform } from 'class-transformer'; import { - IsArray, - IsBoolean, - IsEnum, - IsNumber, - IsString, + IsArray, + IsBoolean, + IsEnum, + IsNumber, + IsString, } from 'class-validator'; import { ConfigVariableOptions } from 'src/engine/core-modules/twenty-config/types/config-variable-options.type'; From c1cf699124cbccad7dac26364571f1c67418b64b Mon Sep 17 00:00:00 2001 From: ehconitin Date: Tue, 22 Apr 2025 23:53:57 +0530 Subject: [PATCH 44/70] remove lib --- packages/twenty-server/package.json | 1 - yarn.lock | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json index 74cf990d2ed2..e264b3978cf5 100644 --- a/packages/twenty-server/package.json +++ b/packages/twenty-server/package.json @@ -52,7 +52,6 @@ "monaco-editor-auto-typings": "^0.4.5", "openid-client": "^5.7.0", "otplib": "^12.0.1", - "p-retry": "4.6.2", "passport": "^0.7.0", "psl": "^1.9.0", "redis": "^4.7.0", diff --git a/yarn.lock b/yarn.lock index 42418ae6b768..cc5ec7f55c04 100644 --- a/yarn.lock +++ b/yarn.lock @@ -46722,7 +46722,7 @@ __metadata: languageName: node linkType: hard -"p-retry@npm:4, p-retry@npm:4.6.2": +"p-retry@npm:4": version: 4.6.2 resolution: "p-retry@npm:4.6.2" dependencies: @@ -55084,7 +55084,6 @@ __metadata: monaco-editor-auto-typings: "npm:^0.4.5" openid-client: "npm:^5.7.0" otplib: "npm:^12.0.1" - p-retry: "npm:4.6.2" passport: "npm:^0.7.0" psl: "npm:^1.9.0" redis: "npm:^4.7.0" From 5ce55be0cfee94f120ba3414187d30ea60df01ad Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 23 Apr 2025 12:45:59 +0530 Subject: [PATCH 45/70] add @nestjs/schedule package --- packages/twenty-server/package.json | 1 + yarn.lock | 37 +++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json index e264b3978cf5..fa6c37ef9b67 100644 --- a/packages/twenty-server/package.json +++ b/packages/twenty-server/package.json @@ -24,6 +24,7 @@ "@nestjs/cache-manager": "^2.2.1", "@nestjs/devtools-integration": "^0.1.6", "@nestjs/graphql": "patch:@nestjs/graphql@12.1.1#./patches/@nestjs+graphql+12.1.1.patch", + "@nestjs/schedule": "^6.0.0", "@node-saml/passport-saml": "^5.0.0", "@opentelemetry/api": "^1.9.0", "@opentelemetry/exporter-metrics-otlp-http": "^0.200.0", diff --git a/yarn.lock b/yarn.lock index 6e5a7dce3de5..602189afc4cf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10561,6 +10561,18 @@ __metadata: languageName: node linkType: hard +"@nestjs/schedule@npm:^6.0.0": + version: 6.0.0 + resolution: "@nestjs/schedule@npm:6.0.0" + dependencies: + cron: "npm:4.3.0" + peerDependencies: + "@nestjs/common": ^10.0.0 || ^11.0.0 + "@nestjs/core": ^10.0.0 || ^11.0.0 + checksum: 10c0/406d301b5a6e4e7c01eddb39865fb812a477bc01685699e3b85185c190bd3228d22b7523b4018990aa2b055b791defbaed0b9abdc368493c7acc667f63951d87 + languageName: node + linkType: hard + "@nestjs/schematics@npm:^10.0.1": version: 10.1.3 resolution: "@nestjs/schematics@npm:10.1.3" @@ -22593,6 +22605,13 @@ __metadata: languageName: node linkType: hard +"@types/luxon@npm:~3.6.0": + version: 3.6.2 + resolution: "@types/luxon@npm:3.6.2" + checksum: 10c0/7572ee52b3d3e9dd10464b90561a728b90f34b9a257751cc3ce23762693dd1d14fa98b7f103e2efe2c6f49033331f91de5681ffd65cca88618cefe555be326db + languageName: node + linkType: hard + "@types/markdown-it@npm:12.2.3": version: 12.2.3 resolution: "@types/markdown-it@npm:12.2.3" @@ -30201,6 +30220,16 @@ __metadata: languageName: node linkType: hard +"cron@npm:4.3.0": + version: 4.3.0 + resolution: "cron@npm:4.3.0" + dependencies: + "@types/luxon": "npm:~3.6.0" + luxon: "npm:~3.6.0" + checksum: 10c0/859805a56ce6af19177a0abff3688e917d2124c0fa086234333d20fd4ca6efe333af2ab4ba6ba06c3aa68c6b2eec30e375c795e97e735c70fcac126057332f10 + languageName: node + linkType: hard + "cross-env@npm:^7.0.3": version: 7.0.3 resolution: "cross-env@npm:7.0.3" @@ -42038,6 +42067,13 @@ __metadata: languageName: node linkType: hard +"luxon@npm:~3.6.0": + version: 3.6.1 + resolution: "luxon@npm:3.6.1" + checksum: 10c0/906d57a9dc4d1de9383f2e9223e378c298607c1b4d17b6657b836a3cd120feb1c1de3b5d06d846a3417e1ca764de8476e8c23b3cd4083b5cdb870adcb06a99d5 + languageName: node + linkType: hard + "lz-string@npm:^1.4.4, lz-string@npm:^1.5.0": version: 1.5.0 resolution: "lz-string@npm:1.5.0" @@ -55047,6 +55083,7 @@ __metadata: "@nestjs/cli": "npm:10.3.0" "@nestjs/devtools-integration": "npm:^0.1.6" "@nestjs/graphql": "patch:@nestjs/graphql@12.1.1#./patches/@nestjs+graphql+12.1.1.patch" + "@nestjs/schedule": "npm:^6.0.0" "@node-saml/passport-saml": "npm:^5.0.0" "@nx/js": "npm:18.3.3" "@opentelemetry/api": "npm:^1.9.0" From edbe5e2940c76f6a72189659f07fc35676de47d5 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 23 Apr 2025 14:05:04 +0530 Subject: [PATCH 46/70] continue --- packages/twenty-server/package.json | 2 +- .../__tests__/config-cache.service.spec.ts | 110 +++++++++--------- .../cache/config-cache.service.ts | 40 ++++--- ...onfig-variables-cache-scavenge-interval.ts | 2 +- .../twenty-config/twenty-config.module.ts | 2 + yarn.lock | 46 ++++---- 6 files changed, 103 insertions(+), 99 deletions(-) diff --git a/packages/twenty-server/package.json b/packages/twenty-server/package.json index fa6c37ef9b67..4cb9219f8461 100644 --- a/packages/twenty-server/package.json +++ b/packages/twenty-server/package.json @@ -24,7 +24,7 @@ "@nestjs/cache-manager": "^2.2.1", "@nestjs/devtools-integration": "^0.1.6", "@nestjs/graphql": "patch:@nestjs/graphql@12.1.1#./patches/@nestjs+graphql+12.1.1.patch", - "@nestjs/schedule": "^6.0.0", + "@nestjs/schedule": "^3.0.0", "@node-saml/passport-saml": "^5.0.0", "@opentelemetry/api": "^1.9.0", "@opentelemetry/exporter-metrics-otlp-http": "^0.200.0", diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts index 900af29fd688..eb7d4fb004b5 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts @@ -1,14 +1,12 @@ +import { ScheduleModule } from '@nestjs/schedule'; import { Test, TestingModule } from '@nestjs/testing'; import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; -import { CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval'; import { CONFIG_VARIABLES_CACHE_TTL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl'; describe('ConfigCacheService', () => { let service: ConfigCacheService; - let setIntervalSpy: jest.SpyInstance; - let clearIntervalSpy: jest.SpyInstance; const withMockedDate = (timeOffset: number, callback: () => void) => { const originalNow = Date.now; @@ -22,11 +20,10 @@ describe('ConfigCacheService', () => { }; beforeEach(async () => { - jest.useFakeTimers({ doNotFake: ['setInterval', 'clearInterval'] }); - setIntervalSpy = jest.spyOn(global, 'setInterval'); - clearIntervalSpy = jest.spyOn(global, 'clearInterval'); + jest.useFakeTimers(); const module: TestingModule = await Test.createTestingModule({ + imports: [ScheduleModule.forRoot()], providers: [ConfigCacheService], }).compile(); @@ -35,8 +32,6 @@ describe('ConfigCacheService', () => { afterEach(() => { service.onModuleDestroy(); - setIntervalSpy.mockRestore(); - clearIntervalSpy.mockRestore(); jest.useRealTimers(); }); @@ -80,7 +75,7 @@ describe('ConfigCacheService', () => { describe('negative lookup cache', () => { it('should check if a negative cache entry exists', () => { - const key = 'TEST_KEY' as any; + const key = 'TEST_KEY' as keyof ConfigVariables; service.markKeyAsMissing(key); const result = service.isKeyKnownMissing(key); @@ -89,7 +84,7 @@ describe('ConfigCacheService', () => { }); it('should return false for negative cache entry check when not in cache', () => { - const key = 'NON_EXISTENT_KEY' as any; + const key = 'NON_EXISTENT_KEY' as keyof ConfigVariables; const result = service.isKeyKnownMissing(key); @@ -97,7 +92,7 @@ describe('ConfigCacheService', () => { }); it('should return false for negative cache entry check when expired', () => { - const key = 'TEST_KEY' as any; + const key = 'TEST_KEY' as keyof ConfigVariables; service.markKeyAsMissing(key); @@ -197,9 +192,9 @@ describe('ConfigCacheService', () => { }); it('should properly count cache entries', () => { - const key1 = 'KEY1' as any; - const key2 = 'KEY2' as any; - const key3 = 'KEY3' as any; + const key1 = 'KEY1' as keyof ConfigVariables; + const key2 = 'KEY2' as keyof ConfigVariables; + const key3 = 'KEY3' as keyof ConfigVariables; // Add some values to the cache service.set(key1, 'value1'); @@ -217,73 +212,72 @@ describe('ConfigCacheService', () => { }); describe('module lifecycle', () => { - it('should clean up interval on module destroy', () => { + it('should clear cache on module destroy', () => { + const key = 'TEST_KEY' as keyof ConfigVariables; + + service.set(key, 'test'); + service.onModuleDestroy(); - expect(clearIntervalSpy).toHaveBeenCalled(); + + expect(service.get(key)).toBeUndefined(); }); }); describe('cache scavenging', () => { - beforeEach(() => { - jest.useFakeTimers({ doNotFake: ['setInterval', 'clearInterval'] }); - }); + it('should have a public scavengeCache method', () => { + expect(typeof service.scavengeCache).toBe('function'); - afterEach(() => { - jest.useRealTimers(); - }); + const prototype = Object.getPrototypeOf(service); - it('should set up scavenging interval on initialization', () => { - expect(setIntervalSpy).toHaveBeenCalledWith( - expect.any(Function), - CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL, - ); + expect( + Object.getOwnPropertyDescriptor(prototype, 'scavengeCache'), + ).toBeDefined(); }); - it('should clean up interval on module destroy', () => { - const intervalId = setIntervalSpy.mock.results[0].value; + it('should remove expired entries when scavengeCache is called', () => { + const key = 'TEST_KEY' as keyof ConfigVariables; - service.onModuleDestroy(); - expect(clearIntervalSpy).toHaveBeenCalledWith(intervalId); + service.set(key, 'test'); + + withMockedDate(CONFIG_VARIABLES_CACHE_TTL + 1, () => { + service.scavengeCache(); + + expect(service.get(key)).toBeUndefined(); + }); }); - it('should automatically scavenge expired entries after interval', () => { - const key1 = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - const key2 = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + it('should not remove non-expired entries when scavengeCache is called', () => { + const key = 'TEST_KEY' as keyof ConfigVariables; - service.set(key1, true); - service.set(key2, 'test@example.com'); + service.set(key, 'test'); - jest.advanceTimersByTime(CONFIG_VARIABLES_CACHE_TTL + 1); - jest.advanceTimersByTime(CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL); + withMockedDate(CONFIG_VARIABLES_CACHE_TTL - 1, () => { + service.scavengeCache(); - expect(service.get(key1)).toBeUndefined(); - expect(service.get(key2)).toBeUndefined(); + expect(service.get(key)).toBe('test'); + }); }); - it('should not remove non-expired entries after interval', () => { - const key1 = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - const key2 = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + it('should handle multiple entries with different expiration times', () => { + const expiredKey = 'EXPIRED_KEY' as keyof ConfigVariables; + const validKey = 'VALID_KEY' as keyof ConfigVariables; - service.set(key1, true); - service.set(key2, 'test@example.com'); + withMockedDate(-CONFIG_VARIABLES_CACHE_TTL * 2, () => { + service.set(expiredKey, 'expired-value'); + }); - jest.advanceTimersByTime(CONFIG_VARIABLES_CACHE_TTL - 1); + service.set(validKey, 'valid-value'); - expect(service.get(key1)).toBe(true); - expect(service.get(key2)).toBe('test@example.com'); - }); - - it('should scavenge multiple times when multiple intervals pass', () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + service.scavengeCache(); - service.set(key, true); + expect(service.get(expiredKey)).toBeUndefined(), + expect(service.get(validKey)).toBe('valid-value'); + }); - jest.advanceTimersByTime( - CONFIG_VARIABLES_CACHE_TTL + - CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL * 3, - ); + it('should handle empty cache when scavenging', () => { + service.clearAll(); - expect(service.get(key)).toBeUndefined(); + expect(() => service.scavengeCache()).not.toThrow(); }); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts index c315e0dae7e9..c0658cb46479 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts @@ -1,6 +1,7 @@ -import { Injectable, OnModuleDestroy } from '@nestjs/common'; +import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; +import { Cron } from '@nestjs/schedule'; -import { CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval'; +import { CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL_MINUTES } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval'; import { CONFIG_VARIABLES_CACHE_TTL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl'; import { @@ -12,6 +13,7 @@ import { @Injectable() export class ConfigCacheService implements OnModuleDestroy { + private readonly logger = new Logger(ConfigCacheService.name); private readonly foundConfigValuesCache: Map< ConfigKey, ConfigCacheEntry @@ -20,12 +22,10 @@ export class ConfigCacheService implements OnModuleDestroy { ConfigKey, ConfigKnownMissingEntry >; - private cacheScavengeInterval: NodeJS.Timeout; constructor() { this.foundConfigValuesCache = new Map(); this.knownMissingKeysCache = new Map(); - this.startCacheScavenging(); } get(key: T): ConfigValue | undefined { @@ -96,9 +96,6 @@ export class ConfigCacheService implements OnModuleDestroy { } onModuleDestroy() { - if (this.cacheScavengeInterval) { - clearInterval(this.cacheScavengeInterval); - } this.clearAll(); } @@ -110,29 +107,38 @@ export class ConfigCacheService implements OnModuleDestroy { return now - entry.timestamp > entry.ttl; } - // Should this process be done using @nestjs/schedule? - // setInterval is not a good idea because it will not be able to run in a cluster? - // Maybe it should be a part of the DatabaseConfigDriver? -- for now kept it here to have a single place to control the cache - - private startCacheScavenging(): void { - this.cacheScavengeInterval = setInterval(() => { - this.scavengeCache(); - }, CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL); - } + /** + * Scavenge the cache to remove expired entries. + * + * Note: While we're using @nestjs/schedule for this task, the recommended way to run + * distributed cron jobs is still BullMQ. This is used here because we want to have + * one execution per pod with direct access to memory. + */ + @Cron(`0 */${CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL_MINUTES} * * * *`) + public scavengeCache(): void { + this.logger.log('Starting cache scavenging process'); - private scavengeCache(): void { const now = Date.now(); + let removedCount = 0; for (const [key, entry] of this.foundConfigValuesCache.entries()) { if (now - entry.timestamp > entry.ttl) { this.foundConfigValuesCache.delete(key); + removedCount++; } } + let removedMissingCount = 0; + for (const [key, entry] of this.knownMissingKeysCache.entries()) { if (now - entry.timestamp > entry.ttl) { this.knownMissingKeysCache.delete(key); + removedMissingCount++; } } + + this.logger.log( + `Cache scavenging completed: removed ${removedCount} expired entries and ${removedMissingCount} expired missing entries`, + ); } } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval.ts index 6c12a4c7b438..f5440ed34492 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval.ts @@ -1 +1 @@ -export const CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL = 5 * 60 * 1000; +export const CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL_MINUTES = 5; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts index 9942bb8b0faa..4d6d09b65fca 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts @@ -1,5 +1,6 @@ import { Global, Module } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; +import { ScheduleModule } from '@nestjs/schedule'; import { TypeOrmModule } from '@nestjs/typeorm'; import { KeyValuePair } from 'src/engine/core-modules/key-value-pair/key-value-pair.entity'; @@ -25,6 +26,7 @@ import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twent envFilePath: process.env.NODE_ENV === 'test' ? '.env.test' : '.env', }), TypeOrmModule.forFeature([KeyValuePair], 'core'), + ScheduleModule.forRoot(), ], providers: [ diff --git a/yarn.lock b/yarn.lock index 602189afc4cf..36e6a027beb2 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10561,15 +10561,17 @@ __metadata: languageName: node linkType: hard -"@nestjs/schedule@npm:^6.0.0": - version: 6.0.0 - resolution: "@nestjs/schedule@npm:6.0.0" +"@nestjs/schedule@npm:^3.0.0": + version: 3.0.4 + resolution: "@nestjs/schedule@npm:3.0.4" dependencies: - cron: "npm:4.3.0" + cron: "npm:2.4.3" + uuid: "npm:9.0.1" peerDependencies: - "@nestjs/common": ^10.0.0 || ^11.0.0 - "@nestjs/core": ^10.0.0 || ^11.0.0 - checksum: 10c0/406d301b5a6e4e7c01eddb39865fb812a477bc01685699e3b85185c190bd3228d22b7523b4018990aa2b055b791defbaed0b9abdc368493c7acc667f63951d87 + "@nestjs/common": ^8.0.0 || ^9.0.0 || ^10.0.0 + "@nestjs/core": ^8.0.0 || ^9.0.0 || ^10.0.0 + reflect-metadata: ^0.1.12 + checksum: 10c0/10af832f611139b586bd85714da1697276ddf51513c91cb20c82eb481c9b1741ec85560f7c55d6e560c8ab7f4665f95cb9fd77241a84f2b432b03e2819ba0f86 languageName: node linkType: hard @@ -22605,10 +22607,10 @@ __metadata: languageName: node linkType: hard -"@types/luxon@npm:~3.6.0": - version: 3.6.2 - resolution: "@types/luxon@npm:3.6.2" - checksum: 10c0/7572ee52b3d3e9dd10464b90561a728b90f34b9a257751cc3ce23762693dd1d14fa98b7f103e2efe2c6f49033331f91de5681ffd65cca88618cefe555be326db +"@types/luxon@npm:~3.3.0": + version: 3.3.8 + resolution: "@types/luxon@npm:3.3.8" + checksum: 10c0/f2ffa31364eb94ca0474a196f533d301025a203bb2758ce0cf209f338cece0af169edea230b5c0b1a68a71adb02f369faa5ec0bd824deb8f0a08cac6803b1b06 languageName: node linkType: hard @@ -30220,13 +30222,13 @@ __metadata: languageName: node linkType: hard -"cron@npm:4.3.0": - version: 4.3.0 - resolution: "cron@npm:4.3.0" +"cron@npm:2.4.3": + version: 2.4.3 + resolution: "cron@npm:2.4.3" dependencies: - "@types/luxon": "npm:~3.6.0" - luxon: "npm:~3.6.0" - checksum: 10c0/859805a56ce6af19177a0abff3688e917d2124c0fa086234333d20fd4ca6efe333af2ab4ba6ba06c3aa68c6b2eec30e375c795e97e735c70fcac126057332f10 + "@types/luxon": "npm:~3.3.0" + luxon: "npm:~3.3.0" + checksum: 10c0/3112d4cb0aa1c1129c0bb742eec205e38948806c907e21a0680d1aa83a1270bfade9fcf090c1604529684d21a64d74eb89075e25c16d48d95cf3c5b5d032f316 languageName: node linkType: hard @@ -42067,10 +42069,10 @@ __metadata: languageName: node linkType: hard -"luxon@npm:~3.6.0": - version: 3.6.1 - resolution: "luxon@npm:3.6.1" - checksum: 10c0/906d57a9dc4d1de9383f2e9223e378c298607c1b4d17b6657b836a3cd120feb1c1de3b5d06d846a3417e1ca764de8476e8c23b3cd4083b5cdb870adcb06a99d5 +"luxon@npm:~3.3.0": + version: 3.3.0 + resolution: "luxon@npm:3.3.0" + checksum: 10c0/47f8e1e96b25441c799b8aa833b3f007fb1854713bcffc8c3384eda8e61fc9af1f038474d137274d2d386492f341c8a8c992fc78c213adfb3143780feba2776c languageName: node linkType: hard @@ -55083,7 +55085,7 @@ __metadata: "@nestjs/cli": "npm:10.3.0" "@nestjs/devtools-integration": "npm:^0.1.6" "@nestjs/graphql": "patch:@nestjs/graphql@12.1.1#./patches/@nestjs+graphql+12.1.1.patch" - "@nestjs/schedule": "npm:^6.0.0" + "@nestjs/schedule": "npm:^3.0.0" "@node-saml/passport-saml": "npm:^5.0.0" "@nx/js": "npm:18.3.3" "@opentelemetry/api": "npm:^1.9.0" From 970fbbfcfd11bd654e75a1db13677225f766d9d8 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 23 Apr 2025 16:28:03 +0530 Subject: [PATCH 47/70] fix --- .../__tests__/config-cache.service.spec.ts | 60 ++++++++++--------- .../cache/config-cache.service.ts | 21 ++++--- .../__tests__/database-config.driver.spec.ts | 56 +++++++++++------ .../drivers/database-config.driver.ts | 14 +++-- .../database-config-driver.interface.ts | 2 +- 5 files changed, 95 insertions(+), 58 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts index eb7d4fb004b5..e070717cccfd 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts @@ -47,7 +47,8 @@ describe('ConfigCacheService', () => { service.set(key, value); const result = service.get(key); - expect(result).toBe(value); + expect(result.value).toBe(value); + expect(result.isStale).toBe(false); }); it('should return undefined for non-existent key', () => { @@ -55,7 +56,8 @@ describe('ConfigCacheService', () => { 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, ); - expect(result).toBeUndefined(); + expect(result.value).toBeUndefined(); + expect(result.isStale).toBe(false); }); it('should handle different value types', () => { @@ -67,9 +69,9 @@ describe('ConfigCacheService', () => { service.set(stringKey, 'test@example.com'); service.set(numberKey, 3000); - expect(service.get(booleanKey)).toBe(true); - expect(service.get(stringKey)).toBe('test@example.com'); - expect(service.get(numberKey)).toBe(3000); + expect(service.get(booleanKey).value).toBe(true); + expect(service.get(stringKey).value).toBe('test@example.com'); + expect(service.get(numberKey).value).toBe(3000); }); }); @@ -82,7 +84,7 @@ describe('ConfigCacheService', () => { expect(result).toBe(true); }); - + it('should return false for negative cache entry check when not in cache', () => { const key = 'NON_EXISTENT_KEY' as keyof ConfigVariables; @@ -112,8 +114,8 @@ describe('ConfigCacheService', () => { service.set(key2, 'test@example.com'); service.clear(key1); - expect(service.get(key1)).toBeUndefined(); - expect(service.get(key2)).toBe('test@example.com'); + expect(service.get(key1).value).toBeUndefined(); + expect(service.get(key2).value).toBe('test@example.com'); }); it('should clear all entries', () => { @@ -124,13 +126,13 @@ describe('ConfigCacheService', () => { service.set(key2, 'test@example.com'); service.clearAll(); - expect(service.get(key1)).toBeUndefined(); - expect(service.get(key2)).toBeUndefined(); + expect(service.get(key1).value).toBeUndefined(); + expect(service.get(key2).value).toBeUndefined(); }); }); describe('cache expiration', () => { - it('should expire entries after TTL', () => { + it('should mark entries as stale after TTL', () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const value = true; @@ -139,11 +141,12 @@ describe('ConfigCacheService', () => { withMockedDate(CONFIG_VARIABLES_CACHE_TTL + 1, () => { const result = service.get(key); - expect(result).toBeUndefined(); + expect(result.value).toBe(value); + expect(result.isStale).toBe(true); }); }); - it('should not expire entries before TTL', () => { + it('should not mark entries as stale before TTL', () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const value = true; @@ -152,7 +155,8 @@ describe('ConfigCacheService', () => { withMockedDate(CONFIG_VARIABLES_CACHE_TTL - 1, () => { const result = service.get(key); - expect(result).toBe(value); + expect(result.value).toBe(value); + expect(result.isStale).toBe(false); }); }); }); @@ -219,18 +223,18 @@ describe('ConfigCacheService', () => { service.onModuleDestroy(); - expect(service.get(key)).toBeUndefined(); + expect(service.get(key).value).toBeUndefined(); }); }); describe('cache scavenging', () => { it('should have a public scavengeCache method', () => { - expect(typeof service.scavengeCache).toBe('function'); + expect(typeof service.scavengeConfigVariablesCache).toBe('function'); const prototype = Object.getPrototypeOf(service); expect( - Object.getOwnPropertyDescriptor(prototype, 'scavengeCache'), + Object.getOwnPropertyDescriptor(prototype, 'scavengeConfigVariablesCache'), ).toBeDefined(); }); @@ -240,9 +244,9 @@ describe('ConfigCacheService', () => { service.set(key, 'test'); withMockedDate(CONFIG_VARIABLES_CACHE_TTL + 1, () => { - service.scavengeCache(); + service.scavengeConfigVariablesCache(); - expect(service.get(key)).toBeUndefined(); + expect(service.get(key).value).toBeUndefined(); }); }); @@ -252,9 +256,9 @@ describe('ConfigCacheService', () => { service.set(key, 'test'); withMockedDate(CONFIG_VARIABLES_CACHE_TTL - 1, () => { - service.scavengeCache(); + service.scavengeConfigVariablesCache(); - expect(service.get(key)).toBe('test'); + expect(service.get(key).value).toBe('test'); }); }); @@ -268,16 +272,16 @@ describe('ConfigCacheService', () => { service.set(validKey, 'valid-value'); - service.scavengeCache(); + service.scavengeConfigVariablesCache(); - expect(service.get(expiredKey)).toBeUndefined(), - expect(service.get(validKey)).toBe('valid-value'); + expect(service.get(expiredKey).value).toBeUndefined(), + expect(service.get(validKey).value).toBe('valid-value'); }); it('should handle empty cache when scavenging', () => { service.clearAll(); - expect(() => service.scavengeCache()).not.toThrow(); + expect(() => service.scavengeConfigVariablesCache()).not.toThrow(); }); }); @@ -286,14 +290,14 @@ describe('ConfigCacheService', () => { const key = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; service.set(key, ''); - expect(service.get(key)).toBe(''); + expect(service.get(key).value).toBe(''); }); it('should handle zero values', () => { const key = 'NODE_PORT' as keyof ConfigVariables; service.set(key, 0); - expect(service.get(key)).toBe(0); + expect(service.get(key).value).toBe(0); }); it('should handle clearing non-existent keys', () => { @@ -305,7 +309,7 @@ describe('ConfigCacheService', () => { service.set(otherKey, 'test@example.com'); service.clear(key); - expect(service.get(otherKey)).toBe('test@example.com'); + expect(service.get(otherKey).value).toBe('test@example.com'); }); it('should handle empty cache operations', () => { diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts index c0658cb46479..01b853cb7baa 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts @@ -28,14 +28,21 @@ export class ConfigCacheService implements OnModuleDestroy { this.knownMissingKeysCache = new Map(); } - get(key: T): ConfigValue | undefined { + get( + key: T, + ): { value: ConfigValue | undefined; isStale: boolean } { const entry = this.foundConfigValuesCache.get(key); - if (entry && !this.isCacheExpired(entry)) { - return entry.value as ConfigValue; + if (!entry) { + return { value: undefined, isStale: false }; } - return undefined; + const isStale = this.isCacheExpired(entry); + + return { + value: entry.value as ConfigValue, + isStale, + }; } isKeyKnownMissing(key: ConfigKey): boolean { @@ -115,8 +122,8 @@ export class ConfigCacheService implements OnModuleDestroy { * one execution per pod with direct access to memory. */ @Cron(`0 */${CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL_MINUTES} * * * *`) - public scavengeCache(): void { - this.logger.log('Starting cache scavenging process'); + public scavengeConfigVariablesCache(): void { + this.logger.log('Starting config variables cache scavenging process'); const now = Date.now(); let removedCount = 0; @@ -138,7 +145,7 @@ export class ConfigCacheService implements OnModuleDestroy { } this.logger.log( - `Cache scavenging completed: removed ${removedCount} expired entries and ${removedMissingCount} expired missing entries`, + `Config variables cache scavenging completed: removed ${removedCount} expired entries and ${removedMissingCount} expired missing entries`, ); } } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index da10092a46bb..46d0ff4f16d0 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -137,11 +137,11 @@ describe('DatabaseConfigDriver', () => { expect(isEnvOnlyConfigVar).toHaveBeenCalledWith(key); }); - it('should return cached value when available', async () => { + it('should return cached value when available and not stale', async () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const cachedValue = true; - jest.spyOn(configCache, 'get').mockReturnValue(cachedValue); + jest.spyOn(configCache, 'get').mockReturnValue({ value: cachedValue, isStale: false }); const result = driver.get(key); @@ -149,11 +149,33 @@ describe('DatabaseConfigDriver', () => { expect(configCache.get).toHaveBeenCalledWith(key); }); + it('should return cached value and schedule refresh when stale', async () => { + jest.useFakeTimers(); + + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const cachedValue = true; + + jest.spyOn(configCache, 'get').mockReturnValue({ value: cachedValue, isStale: true }); + + const fetchSpy = jest + .spyOn(driver, 'fetchAndCacheConfigVariable') + .mockResolvedValue(); + + const result = driver.get(key); + + expect(result).toBe(cachedValue); + + // Run immediate callbacks + jest.runAllTimers(); + + expect(fetchSpy).toHaveBeenCalledWith(key); + }); + it('should use environment driver when negative lookup is cached', async () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const envValue = true; - jest.spyOn(configCache, 'get').mockReturnValue(undefined); + jest.spyOn(configCache, 'get').mockReturnValue({ value: undefined, isStale: false }); jest.spyOn(configCache, 'isKeyKnownMissing').mockReturnValue(true); jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); @@ -169,12 +191,12 @@ describe('DatabaseConfigDriver', () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const envValue = true; - jest.spyOn(configCache, 'get').mockReturnValue(undefined); + jest.spyOn(configCache, 'get').mockReturnValue({ value: undefined, isStale: false }); jest.spyOn(configCache, 'isKeyKnownMissing').mockReturnValue(false); jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); const fetchSpy = jest - .spyOn(driver, 'fetchAndCacheConfig') + .spyOn(driver, 'fetchAndCacheConfigVariable') .mockResolvedValue(); const result = driver.get(key); @@ -199,13 +221,13 @@ describe('DatabaseConfigDriver', () => { jest.spyOn(configCache, 'get').mockImplementation((key) => { switch (key) { case stringKey: - return stringValue; + return { value: stringValue, isStale: false }; case booleanKey: - return booleanValue; + return { value: booleanValue, isStale: false }; case numberKey: - return numberValue; + return { value: numberValue, isStale: false }; default: - return undefined; + return { value: undefined, isStale: false }; } }); @@ -242,14 +264,14 @@ describe('DatabaseConfigDriver', () => { }); }); - describe('fetchAndCacheConfig', () => { - it('should refresh config from storage', async () => { + describe('fetchAndCacheConfigVariable', () => { + it('should refresh config variable from storage', async () => { const key = 'AUTH_PASSWORD_ENABLED'; const value = true; jest.spyOn(configStorage, 'get').mockResolvedValue(value); - await driver.fetchAndCacheConfig(key); + await driver.fetchAndCacheConfigVariable(key); expect(configStorage.get).toHaveBeenCalledWith(key); expect(configCache.set).toHaveBeenCalledWith(key, value); @@ -260,7 +282,7 @@ describe('DatabaseConfigDriver', () => { jest.spyOn(configStorage, 'get').mockResolvedValue(undefined); - await driver.fetchAndCacheConfig(key); + await driver.fetchAndCacheConfigVariable(key); expect(configStorage.get).toHaveBeenCalledWith(key); expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(key); @@ -276,7 +298,7 @@ describe('DatabaseConfigDriver', () => { .spyOn(driver['logger'], 'error') .mockImplementation(); - await driver.fetchAndCacheConfig(key); + await driver.fetchAndCacheConfigVariable(key); expect(configStorage.get).toHaveBeenCalledWith(key); expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(key); @@ -347,7 +369,7 @@ describe('DatabaseConfigDriver', () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const envValue = true; - jest.spyOn(configCache, 'get').mockReturnValue(undefined); + jest.spyOn(configCache, 'get').mockReturnValue({ value: undefined, isStale: false }); jest.spyOn(configCache, 'isKeyKnownMissing').mockReturnValue(true); jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); @@ -390,7 +412,7 @@ describe('DatabaseConfigDriver', () => { const fetchPromise = Promise.resolve(); const fetchSpy = jest - .spyOn(driver, 'fetchAndCacheConfig') + .spyOn(driver, 'fetchAndCacheConfigVariable') .mockReturnValue(fetchPromise); jest.spyOn(configStorage, 'loadAll').mockResolvedValue(new Map()); @@ -398,7 +420,7 @@ describe('DatabaseConfigDriver', () => { const key = 'MISSING_KEY' as keyof ConfigVariables; - jest.spyOn(configCache, 'get').mockReturnValue(undefined); + jest.spyOn(configCache, 'get').mockReturnValue({ value: undefined, isStale: false }); jest.spyOn(configCache, 'isKeyKnownMissing').mockReturnValue(false); driver.get(key); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index d65d9c05d9fd..d440ad556b3f 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -56,10 +56,14 @@ export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { return this.environmentDriver.get(key); } - const cachedValue = this.configCache.get(key); + const { value, isStale } = this.configCache.get(key); - if (cachedValue !== undefined) { - return cachedValue; + if (value !== undefined) { + if (isStale) { + this.scheduleRefresh(key); + } + + return value; } if (this.configCache.isKeyKnownMissing(key)) { @@ -91,7 +95,7 @@ export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { } } - async fetchAndCacheConfig(key: keyof ConfigVariables): Promise { + async fetchAndCacheConfigVariable(key: keyof ConfigVariables): Promise { try { const value = await this.configStorage.get(key); @@ -138,7 +142,7 @@ export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { private async scheduleRefresh(key: keyof ConfigVariables): Promise { setImmediate(async () => { - await this.fetchAndCacheConfig(key).catch((error) => { + await this.fetchAndCacheConfigVariable(key).catch((error) => { this.logger.error(`Failed to fetch config for ${key as string}`, error); }); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts index 03605e2acf23..d01a9f424206 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts @@ -26,5 +26,5 @@ export interface DatabaseConfigDriverInterface { /** * Fetch and cache a specific configuration from its source */ - fetchAndCacheConfig(key: keyof ConfigVariables): Promise; + fetchAndCacheConfigVariable(key: keyof ConfigVariables): Promise; } From 42d7061d4c3c6ada165096317b2de53ea9249d9d Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 23 Apr 2025 17:42:58 +0530 Subject: [PATCH 48/70] imrove --- .../__tests__/config-cache.service.spec.ts | 118 ++++++------------ .../cache/config-cache.service.ts | 29 +---- ...onfig-variables-cache-scavenge-interval.ts | 1 - .../config-variables-refresh-interval.ts | 1 + ...abase-config-driver-initial-retry-delay.ts | 1 - ...onfig-driver-initialization-max-retries.ts | 1 - .../__tests__/database-config.driver.spec.ts | 98 +++++++++++++++ .../drivers/database-config.driver.ts | 67 ++++++++++ .../database-config-driver.interface.ts | 5 + .../__tests__/config-storage.service.spec.ts | 76 +++++++++++ .../storage/config-storage.service.ts | 65 +++++++++- 11 files changed, 353 insertions(+), 109 deletions(-) delete mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-refresh-interval.ts delete mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-initial-retry-delay.ts delete mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-initialization-max-retries.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts index e070717cccfd..2077a2513885 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts @@ -227,96 +227,58 @@ describe('ConfigCacheService', () => { }); }); - describe('cache scavenging', () => { - it('should have a public scavengeCache method', () => { - expect(typeof service.scavengeConfigVariablesCache).toBe('function'); - - const prototype = Object.getPrototypeOf(service); - - expect( - Object.getOwnPropertyDescriptor(prototype, 'scavengeConfigVariablesCache'), - ).toBeDefined(); - }); - - it('should remove expired entries when scavengeCache is called', () => { - const key = 'TEST_KEY' as keyof ConfigVariables; - - service.set(key, 'test'); - + describe('getExpiredKeys', () => { + it('should return expired keys from both positive and negative caches', () => { + const expiredKey1 = 'EXPIRED_KEY1' as keyof ConfigVariables; + const expiredKey2 = 'EXPIRED_KEY2' as keyof ConfigVariables; + const expiredNegativeKey = 'EXPIRED_NEGATIVE_KEY' as keyof ConfigVariables; + + // Set up keys that will expire + service.set(expiredKey1, 'value1'); + service.set(expiredKey2, 'value2'); + service.markKeyAsMissing(expiredNegativeKey); + + // Make the above keys expire withMockedDate(CONFIG_VARIABLES_CACHE_TTL + 1, () => { - service.scavengeConfigVariablesCache(); - - expect(service.get(key).value).toBeUndefined(); - }); - }); - - it('should not remove non-expired entries when scavengeCache is called', () => { - const key = 'TEST_KEY' as keyof ConfigVariables; - - service.set(key, 'test'); - - withMockedDate(CONFIG_VARIABLES_CACHE_TTL - 1, () => { - service.scavengeConfigVariablesCache(); - - expect(service.get(key).value).toBe('test'); - }); - }); - - it('should handle multiple entries with different expiration times', () => { - const expiredKey = 'EXPIRED_KEY' as keyof ConfigVariables; - const validKey = 'VALID_KEY' as keyof ConfigVariables; - - withMockedDate(-CONFIG_VARIABLES_CACHE_TTL * 2, () => { - service.set(expiredKey, 'expired-value'); + // Add a fresh key after the time change + const freshKey = 'FRESH_KEY' as keyof ConfigVariables; + service.set(freshKey, 'value3'); + + const expiredKeys = service.getExpiredKeys(); + + expect(expiredKeys).toContain(expiredKey1); + expect(expiredKeys).toContain(expiredKey2); + expect(expiredKeys).toContain(expiredNegativeKey); + expect(expiredKeys).not.toContain(freshKey); }); - - service.set(validKey, 'valid-value'); - - service.scavengeConfigVariablesCache(); - - expect(service.get(expiredKey).value).toBeUndefined(), - expect(service.get(validKey).value).toBe('valid-value'); }); - it('should handle empty cache when scavenging', () => { - service.clearAll(); - - expect(() => service.scavengeConfigVariablesCache()).not.toThrow(); - }); - }); - - describe('edge cases', () => { - it('should handle empty string values', () => { - const key = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + it('should return empty array when no keys are expired', () => { + const key1 = 'KEY1' as keyof ConfigVariables; + const key2 = 'KEY2' as keyof ConfigVariables; + const negativeKey = 'NEGATIVE_KEY' as keyof ConfigVariables; - service.set(key, ''); - expect(service.get(key).value).toBe(''); - }); + service.set(key1, 'value1'); + service.set(key2, 'value2'); + service.markKeyAsMissing(negativeKey); - it('should handle zero values', () => { - const key = 'NODE_PORT' as keyof ConfigVariables; + const expiredKeys = service.getExpiredKeys(); - service.set(key, 0); - expect(service.get(key).value).toBe(0); + expect(expiredKeys).toHaveLength(0); }); - it('should handle clearing non-existent keys', () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - - expect(() => service.clear(key)).not.toThrow(); + it('should not have duplicates if a key is in both caches', () => { + const key = 'DUPLICATE_KEY' as keyof ConfigVariables; - const otherKey = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; + // Manually manipulate the caches to simulate a key in both caches + service.set(key, 'value'); + service.markKeyAsMissing(key); - service.set(otherKey, 'test@example.com'); - service.clear(key); - expect(service.get(otherKey).value).toBe('test@example.com'); - }); + withMockedDate(CONFIG_VARIABLES_CACHE_TTL + 1, () => { + const expiredKeys = service.getExpiredKeys(); - it('should handle empty cache operations', () => { - expect(service.getCacheInfo()).toEqual({ - positiveEntries: 0, - negativeEntries: 0, - cacheKeys: [], + // Should only appear once in the result + expect(expiredKeys.filter(k => k === key)).toHaveLength(1); }); }); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts index 01b853cb7baa..124a41e08d9b 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts @@ -1,7 +1,5 @@ import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; -import { Cron } from '@nestjs/schedule'; -import { CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL_MINUTES } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval'; import { CONFIG_VARIABLES_CACHE_TTL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl'; import { @@ -114,38 +112,23 @@ export class ConfigCacheService implements OnModuleDestroy { return now - entry.timestamp > entry.ttl; } - /** - * Scavenge the cache to remove expired entries. - * - * Note: While we're using @nestjs/schedule for this task, the recommended way to run - * distributed cron jobs is still BullMQ. This is used here because we want to have - * one execution per pod with direct access to memory. - */ - @Cron(`0 */${CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL_MINUTES} * * * *`) - public scavengeConfigVariablesCache(): void { - this.logger.log('Starting config variables cache scavenging process'); - + getExpiredKeys(): ConfigKey[] { const now = Date.now(); - let removedCount = 0; + const expiredPositiveKeys: ConfigKey[] = []; + const expiredNegativeKeys: ConfigKey[] = []; for (const [key, entry] of this.foundConfigValuesCache.entries()) { if (now - entry.timestamp > entry.ttl) { - this.foundConfigValuesCache.delete(key); - removedCount++; + expiredPositiveKeys.push(key); } } - let removedMissingCount = 0; - for (const [key, entry] of this.knownMissingKeysCache.entries()) { if (now - entry.timestamp > entry.ttl) { - this.knownMissingKeysCache.delete(key); - removedMissingCount++; + expiredNegativeKeys.push(key); } } - this.logger.log( - `Config variables cache scavenging completed: removed ${removedCount} expired entries and ${removedMissingCount} expired missing entries`, - ); + return [...new Set([...expiredPositiveKeys, ...expiredNegativeKeys])]; } } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval.ts deleted file mode 100644 index f5440ed34492..000000000000 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-scavenge-interval.ts +++ /dev/null @@ -1 +0,0 @@ -export const CONFIG_VARIABLES_CACHE_SCAVENGE_INTERVAL_MINUTES = 5; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-refresh-interval.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-refresh-interval.ts new file mode 100644 index 000000000000..028c47e2c2f1 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-refresh-interval.ts @@ -0,0 +1 @@ +export const CONFIG_VARIABLES_REFRESH_INTERVAL_MINUTES = 5; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-initial-retry-delay.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-initial-retry-delay.ts deleted file mode 100644 index 4f89c083adcf..000000000000 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-initial-retry-delay.ts +++ /dev/null @@ -1 +0,0 @@ -export const DATABASE_CONFIG_DRIVER_INITIAL_RETRY_DELAY = 1000; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-initialization-max-retries.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-initialization-max-retries.ts deleted file mode 100644 index 5c4dc5e16209..000000000000 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/database-config-driver-initialization-max-retries.ts +++ /dev/null @@ -1 +0,0 @@ -export const DATABASE_CONFIG_DRIVER_INITIALIZATION_MAX_RETRIES = 3; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index 46d0ff4f16d0..88b6885cfee6 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -34,6 +34,7 @@ describe('DatabaseConfigDriver', () => { isKeyKnownMissing: jest.fn(), markKeyAsMissing: jest.fn(), getCacheInfo: jest.fn(), + getExpiredKeys: jest.fn(), }, }, { @@ -42,6 +43,7 @@ describe('DatabaseConfigDriver', () => { get: jest.fn(), set: jest.fn(), loadAll: jest.fn(), + loadByKeys: jest.fn(), }, }, { @@ -459,4 +461,100 @@ describe('DatabaseConfigDriver', () => { expect(configCache.set).toHaveBeenCalledWith('NODE_PORT', 3000); }); }); + + describe('refreshExpiredCache', () => { + beforeEach(() => { + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(new Map()); + // Add loadByKeys mock + jest.spyOn(configStorage, 'loadByKeys').mockResolvedValue(new Map()); + // Add getExpiredKeys mock + jest.spyOn(configCache, 'getExpiredKeys').mockReturnValue([]); + }); + + it('should do nothing when there are no expired keys', async () => { + jest.spyOn(configCache, 'getExpiredKeys').mockReturnValue([]); + + await driver.refreshExpiredCache(); + + expect(configStorage.loadByKeys).not.toHaveBeenCalled(); + }); + + it('should filter out env-only variables from expired keys', async () => { + const expiredKeys = [ + 'AUTH_PASSWORD_ENABLED', + 'ENV_ONLY_VAR', + 'EMAIL_FROM_ADDRESS', + ] as Array; + + jest.spyOn(configCache, 'getExpiredKeys').mockReturnValue(expiredKeys); + + // Simulate one env-only var + (isEnvOnlyConfigVar as jest.Mock).mockImplementation((key) => { + return key === 'ENV_ONLY_VAR'; + }); + + await driver.refreshExpiredCache(); + + expect(configStorage.loadByKeys).toHaveBeenCalledWith([ + 'AUTH_PASSWORD_ENABLED', + 'EMAIL_FROM_ADDRESS', + ]); + }); + + it('should mark all keys as missing when no values are found', async () => { + const expiredKeys = [ + 'AUTH_PASSWORD_ENABLED', + 'EMAIL_FROM_ADDRESS', + ] as Array; + + jest.spyOn(configCache, 'getExpiredKeys').mockReturnValue(expiredKeys); + jest.spyOn(configStorage, 'loadByKeys').mockResolvedValue(new Map()); + + await driver.refreshExpiredCache(); + + expect(configCache.markKeyAsMissing).toHaveBeenCalledTimes(2); + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith('AUTH_PASSWORD_ENABLED'); + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith('EMAIL_FROM_ADDRESS'); + }); + + it('should update cache with refreshed values', async () => { + const expiredKeys = [ + 'AUTH_PASSWORD_ENABLED', + 'EMAIL_FROM_ADDRESS', + 'MISSING_KEY', + ] as Array; + + const refreshedValues = new Map< + keyof ConfigVariables, + ConfigVariables[keyof ConfigVariables] + >([ + ['AUTH_PASSWORD_ENABLED', true], + ['EMAIL_FROM_ADDRESS', 'test@example.com'], + ]); + + jest.spyOn(configCache, 'getExpiredKeys').mockReturnValue(expiredKeys); + jest.spyOn(configStorage, 'loadByKeys').mockResolvedValue(refreshedValues); + + await driver.refreshExpiredCache(); + + // Should set found values + expect(configCache.set).toHaveBeenCalledWith('AUTH_PASSWORD_ENABLED', true); + expect(configCache.set).toHaveBeenCalledWith('EMAIL_FROM_ADDRESS', 'test@example.com'); + + // Should mark missing values as missing + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith('MISSING_KEY'); + }); + + it('should handle errors gracefully', async () => { + const error = new Error('Database error'); + jest.spyOn(configCache, 'getExpiredKeys').mockReturnValue(['AUTH_PASSWORD_ENABLED'] as Array); + jest.spyOn(configStorage, 'loadByKeys').mockRejectedValue(error); + jest.spyOn(driver['logger'], 'error'); + + await driver.refreshExpiredCache(); + + // Should log error and not throw + expect(driver['logger'].error).toHaveBeenCalled(); + }); + }); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index d440ad556b3f..4fc57ccd3cfe 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -1,9 +1,11 @@ import { Injectable, Logger } from '@nestjs/common'; +import { Cron } from '@nestjs/schedule'; import { DatabaseConfigDriverInterface } from 'src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface'; import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { CONFIG_VARIABLES_REFRESH_INTERVAL_MINUTES } from 'src/engine/core-modules/twenty-config/constants/config-variables-refresh-interval'; import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; @@ -147,4 +149,69 @@ export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { }); }); } + + /** + * Proactively refreshes expired entries in the config cache. + * This method runs on a schedule and fetches all expired keys in one database query, + * then updates the cache with fresh values. + */ + @Cron(`0 */${CONFIG_VARIABLES_REFRESH_INTERVAL_MINUTES} * * * *`) + async refreshExpiredCache(): Promise { + try { + this.logger.log('Starting proactive refresh of expired config variables'); + + const expiredKeys = this.configCache.getExpiredKeys(); + + if (expiredKeys.length === 0) { + this.logger.debug('No expired config variables to refresh'); + + return; + } + + this.logger.debug( + `Found ${expiredKeys.length} expired config variables to refresh`, + ); + + const refreshableKeys = expiredKeys.filter( + (key) => !isEnvOnlyConfigVar(key), + ) as Array; + + if (refreshableKeys.length === 0) { + this.logger.debug( + 'No refreshable config variables found (all are env-only)', + ); + + return; + } + + const refreshedValues = + await this.configStorage.loadByKeys(refreshableKeys); + + if (refreshedValues.size === 0) { + this.logger.debug('No values found for expired keys'); + + for (const key of refreshableKeys) { + this.configCache.markKeyAsMissing(key); + } + + return; + } + + for (const [key, value] of refreshedValues.entries()) { + this.configCache.set(key, value); + } + + for (const key of refreshableKeys) { + if (!refreshedValues.has(key)) { + this.configCache.markKeyAsMissing(key); + } + } + + this.logger.log( + `Refreshed ${refreshedValues.size} config variables in cache`, + ); + } catch (error) { + this.logger.error('Failed to refresh expired config variables', error); + } + } } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts index d01a9f424206..e440f62543fc 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts @@ -27,4 +27,9 @@ export interface DatabaseConfigDriverInterface { * Fetch and cache a specific configuration from its source */ fetchAndCacheConfigVariable(key: keyof ConfigVariables): Promise; + + /** + * Proactively refreshes expired entries in the config cache + */ + refreshExpiredCache(): Promise; } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts index e0c45a920fa5..f52a560695d9 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts @@ -12,6 +12,7 @@ import { ConfigValueConverterService } from 'src/engine/core-modules/twenty-conf import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; import { User } from 'src/engine/core-modules/user/user.entity'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; +import { In } from 'typeorm'; describe('ConfigStorageService', () => { let service: ConfigStorageService; @@ -485,4 +486,79 @@ describe('ConfigStorageService', () => { }); }); }); + + describe('loadByKeys', () => { + it('should load and convert specified config variables', async () => { + const keys = ['AUTH_PASSWORD_ENABLED', 'EMAIL_FROM_ADDRESS'] as Array; + const configVars: KeyValuePair[] = [ + createMockKeyValuePair('AUTH_PASSWORD_ENABLED', 'true'), + createMockKeyValuePair('EMAIL_FROM_ADDRESS', 'test@example.com'), + ]; + + jest.spyOn(keyValuePairRepository, 'find').mockResolvedValue(configVars); + + ( + configValueConverter.convertDbValueToAppValue as jest.Mock + ).mockImplementation((value, key) => { + if (key === 'AUTH_PASSWORD_ENABLED') return true; + if (key === 'EMAIL_FROM_ADDRESS') return 'test@example.com'; + + return value; + }); + + const result = await service.loadByKeys(keys); + + expect(result.size).toBe(2); + expect(result.get('AUTH_PASSWORD_ENABLED' as keyof ConfigVariables)).toBe( + true, + ); + expect(result.get('EMAIL_FROM_ADDRESS' as keyof ConfigVariables)).toBe( + 'test@example.com', + ); + expect(keyValuePairRepository.find).toHaveBeenCalledWith({ + where: { + type: KeyValuePairType.CONFIG_VARIABLE, + key: In(keys), + userId: IsNull(), + workspaceId: IsNull(), + }, + }); + }); + + it('should return empty map when no keys are provided', async () => { + const keys: Array = []; + + const result = await service.loadByKeys(keys); + + expect(result.size).toBe(0); + expect(keyValuePairRepository.find).not.toHaveBeenCalled(); + }); + + it('should handle conversion errors and skip problematic entries', async () => { + const keys = ['AUTH_PASSWORD_ENABLED', 'PROBLEMATIC_KEY'] as Array; + const configVars: KeyValuePair[] = [ + createMockKeyValuePair('AUTH_PASSWORD_ENABLED', 'true'), + createMockKeyValuePair('PROBLEMATIC_KEY', 'bad-value'), + ]; + + jest.spyOn(keyValuePairRepository, 'find').mockResolvedValue(configVars); + + ( + configValueConverter.convertDbValueToAppValue as jest.Mock + ).mockImplementation((value, key) => { + if (key === 'AUTH_PASSWORD_ENABLED') return true; + if (key === 'PROBLEMATIC_KEY') throw new Error('Conversion error'); + + return value; + }); + + const result = await service.loadByKeys(keys); + + expect(result.size).toBe(1); + expect(result.get('AUTH_PASSWORD_ENABLED' as keyof ConfigVariables)).toBe( + true, + ); + expect(result.has('PROBLEMATIC_KEY' as keyof ConfigVariables)).toBe(false); + }); + }); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts index f3fa02f27aad..2e52ac22662c 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts @@ -1,7 +1,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { FindOptionsWhere, IsNull, Repository } from 'typeorm'; +import { FindOptionsWhere, In, IsNull, Repository } from 'typeorm'; import { KeyValuePair, @@ -23,11 +23,20 @@ export class ConfigStorageService implements ConfigStorageInterface { ) {} private getConfigVariableWhereClause( - key?: string, + key?: string | string[], ): FindOptionsWhere { + if (Array.isArray(key) && key.length > 0) { + return { + type: KeyValuePairType.CONFIG_VARIABLE, + key: In(key), + userId: IsNull(), + workspaceId: IsNull(), + }; + } + return { type: KeyValuePairType.CONFIG_VARIABLE, - ...(key ? { key } : {}), + ...(key && !Array.isArray(key) ? { key } : {}), userId: IsNull(), workspaceId: IsNull(), }; @@ -46,6 +55,10 @@ export class ConfigStorageService implements ConfigStorageInterface { } try { + this.logger.debug( + `Fetching config for ${key as string} in database: ${result?.value}`, + ); + return this.configValueConverter.convertDbValueToAppValue( result.value, key, @@ -139,7 +152,6 @@ export class ConfigStorageService implements ConfigStorageInterface { key, ); - // Only set values that are defined if (value !== undefined) { result.set(key, value); } @@ -148,7 +160,6 @@ export class ConfigStorageService implements ConfigStorageInterface { `Failed to convert value to app type for key ${key as string}`, error, ); - // Skip this value but continue processing others continue; } } @@ -160,4 +171,48 @@ export class ConfigStorageService implements ConfigStorageInterface { throw error; } } + + async loadByKeys( + keys: T[], + ): Promise> { + try { + if (keys.length === 0) { + return new Map(); + } + + const configVars = await this.keyValuePairRepository.find({ + where: this.getConfigVariableWhereClause(keys as string[]), + }); + + const result = new Map(); + + for (const configVar of configVars) { + if (configVar.value !== null) { + const key = configVar.key as T; + + try { + const value = this.configValueConverter.convertDbValueToAppValue( + configVar.value, + key, + ); + + if (value !== undefined) { + result.set(key, value); + } + } catch (error) { + this.logger.error( + `Failed to convert value to app type for key ${key as string}`, + error, + ); + continue; + } + } + } + + return result; + } catch (error) { + this.logger.error('Failed to load config variables by keys', error); + throw error; + } + } } From 2ffce8e1937a4002affc7bc02125a0d9b0fc44a0 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 23 Apr 2025 17:46:56 +0530 Subject: [PATCH 49/70] rename timestamp to registered at --- .../twenty-config/cache/config-cache.service.ts | 10 +++++----- .../cache/interfaces/config-cache-entry.interface.ts | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts index 124a41e08d9b..c940902c0eb5 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts @@ -56,7 +56,7 @@ export class ConfigCacheService implements OnModuleDestroy { set(key: T, value: ConfigValue): void { this.foundConfigValuesCache.set(key, { value, - timestamp: Date.now(), + registeredAt: Date.now(), ttl: CONFIG_VARIABLES_CACHE_TTL, }); this.knownMissingKeysCache.delete(key); @@ -64,7 +64,7 @@ export class ConfigCacheService implements OnModuleDestroy { markKeyAsMissing(key: ConfigKey): void { this.knownMissingKeysCache.set(key, { - timestamp: Date.now(), + registeredAt: Date.now(), ttl: CONFIG_VARIABLES_CACHE_TTL, }); this.foundConfigValuesCache.delete(key); @@ -109,7 +109,7 @@ export class ConfigCacheService implements OnModuleDestroy { ): boolean { const now = Date.now(); - return now - entry.timestamp > entry.ttl; + return now - entry.registeredAt > entry.ttl; } getExpiredKeys(): ConfigKey[] { @@ -118,13 +118,13 @@ export class ConfigCacheService implements OnModuleDestroy { const expiredNegativeKeys: ConfigKey[] = []; for (const [key, entry] of this.foundConfigValuesCache.entries()) { - if (now - entry.timestamp > entry.ttl) { + if (now - entry.registeredAt > entry.ttl) { expiredPositiveKeys.push(key); } } for (const [key, entry] of this.knownMissingKeysCache.entries()) { - if (now - entry.timestamp > entry.ttl) { + if (now - entry.registeredAt > entry.ttl) { expiredNegativeKeys.push(key); } } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts index 1f6a9418e23c..65f5e447fc7f 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts @@ -5,11 +5,11 @@ export type ConfigValue = ConfigVariables[T]; export interface ConfigCacheEntry { value: ConfigValue; - timestamp: number; + registeredAt: number; ttl: number; } export interface ConfigKnownMissingEntry { - timestamp: number; + registeredAt: number; ttl: number; } From f0a06bac622a8065d1b12f9528da565dd5a8e1c7 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 23 Apr 2025 17:56:50 +0530 Subject: [PATCH 50/70] lint --- .../__tests__/config-cache.service.spec.ts | 12 ++- .../__tests__/database-config.driver.spec.ts | 93 ++++++++++++------- .../__tests__/config-storage.service.spec.ts | 15 ++- 3 files changed, 78 insertions(+), 42 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts index 2077a2513885..e8ff3189ecd9 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts @@ -84,7 +84,7 @@ describe('ConfigCacheService', () => { expect(result).toBe(true); }); - + it('should return false for negative cache entry check when not in cache', () => { const key = 'NON_EXISTENT_KEY' as keyof ConfigVariables; @@ -231,19 +231,21 @@ describe('ConfigCacheService', () => { it('should return expired keys from both positive and negative caches', () => { const expiredKey1 = 'EXPIRED_KEY1' as keyof ConfigVariables; const expiredKey2 = 'EXPIRED_KEY2' as keyof ConfigVariables; - const expiredNegativeKey = 'EXPIRED_NEGATIVE_KEY' as keyof ConfigVariables; + const expiredNegativeKey = + 'EXPIRED_NEGATIVE_KEY' as keyof ConfigVariables; // Set up keys that will expire service.set(expiredKey1, 'value1'); service.set(expiredKey2, 'value2'); service.markKeyAsMissing(expiredNegativeKey); - + // Make the above keys expire withMockedDate(CONFIG_VARIABLES_CACHE_TTL + 1, () => { // Add a fresh key after the time change const freshKey = 'FRESH_KEY' as keyof ConfigVariables; + service.set(freshKey, 'value3'); - + const expiredKeys = service.getExpiredKeys(); expect(expiredKeys).toContain(expiredKey1); @@ -278,7 +280,7 @@ describe('ConfigCacheService', () => { const expiredKeys = service.getExpiredKeys(); // Should only appear once in the result - expect(expiredKeys.filter(k => k === key)).toHaveLength(1); + expect(expiredKeys.filter((k) => k === key)).toHaveLength(1); }); }); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index 88b6885cfee6..aafdad3d1439 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -143,7 +143,9 @@ describe('DatabaseConfigDriver', () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const cachedValue = true; - jest.spyOn(configCache, 'get').mockReturnValue({ value: cachedValue, isStale: false }); + jest + .spyOn(configCache, 'get') + .mockReturnValue({ value: cachedValue, isStale: false }); const result = driver.get(key); @@ -153,12 +155,14 @@ describe('DatabaseConfigDriver', () => { it('should return cached value and schedule refresh when stale', async () => { jest.useFakeTimers(); - + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const cachedValue = true; - jest.spyOn(configCache, 'get').mockReturnValue({ value: cachedValue, isStale: true }); - + jest + .spyOn(configCache, 'get') + .mockReturnValue({ value: cachedValue, isStale: true }); + const fetchSpy = jest .spyOn(driver, 'fetchAndCacheConfigVariable') .mockResolvedValue(); @@ -166,10 +170,10 @@ describe('DatabaseConfigDriver', () => { const result = driver.get(key); expect(result).toBe(cachedValue); - + // Run immediate callbacks jest.runAllTimers(); - + expect(fetchSpy).toHaveBeenCalledWith(key); }); @@ -177,7 +181,9 @@ describe('DatabaseConfigDriver', () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const envValue = true; - jest.spyOn(configCache, 'get').mockReturnValue({ value: undefined, isStale: false }); + jest + .spyOn(configCache, 'get') + .mockReturnValue({ value: undefined, isStale: false }); jest.spyOn(configCache, 'isKeyKnownMissing').mockReturnValue(true); jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); @@ -193,7 +199,9 @@ describe('DatabaseConfigDriver', () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const envValue = true; - jest.spyOn(configCache, 'get').mockReturnValue({ value: undefined, isStale: false }); + jest + .spyOn(configCache, 'get') + .mockReturnValue({ value: undefined, isStale: false }); jest.spyOn(configCache, 'isKeyKnownMissing').mockReturnValue(false); jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); @@ -371,7 +379,9 @@ describe('DatabaseConfigDriver', () => { const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const envValue = true; - jest.spyOn(configCache, 'get').mockReturnValue({ value: undefined, isStale: false }); + jest + .spyOn(configCache, 'get') + .mockReturnValue({ value: undefined, isStale: false }); jest.spyOn(configCache, 'isKeyKnownMissing').mockReturnValue(true); jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); @@ -422,7 +432,9 @@ describe('DatabaseConfigDriver', () => { const key = 'MISSING_KEY' as keyof ConfigVariables; - jest.spyOn(configCache, 'get').mockReturnValue({ value: undefined, isStale: false }); + jest + .spyOn(configCache, 'get') + .mockReturnValue({ value: undefined, isStale: false }); jest.spyOn(configCache, 'isKeyKnownMissing').mockReturnValue(false); driver.get(key); @@ -473,9 +485,9 @@ describe('DatabaseConfigDriver', () => { it('should do nothing when there are no expired keys', async () => { jest.spyOn(configCache, 'getExpiredKeys').mockReturnValue([]); - + await driver.refreshExpiredCache(); - + expect(configStorage.loadByKeys).not.toHaveBeenCalled(); }); @@ -485,16 +497,16 @@ describe('DatabaseConfigDriver', () => { 'ENV_ONLY_VAR', 'EMAIL_FROM_ADDRESS', ] as Array; - + jest.spyOn(configCache, 'getExpiredKeys').mockReturnValue(expiredKeys); - + // Simulate one env-only var (isEnvOnlyConfigVar as jest.Mock).mockImplementation((key) => { return key === 'ENV_ONLY_VAR'; }); - + await driver.refreshExpiredCache(); - + expect(configStorage.loadByKeys).toHaveBeenCalledWith([ 'AUTH_PASSWORD_ENABLED', 'EMAIL_FROM_ADDRESS', @@ -506,15 +518,19 @@ describe('DatabaseConfigDriver', () => { 'AUTH_PASSWORD_ENABLED', 'EMAIL_FROM_ADDRESS', ] as Array; - + jest.spyOn(configCache, 'getExpiredKeys').mockReturnValue(expiredKeys); jest.spyOn(configStorage, 'loadByKeys').mockResolvedValue(new Map()); - + await driver.refreshExpiredCache(); - + expect(configCache.markKeyAsMissing).toHaveBeenCalledTimes(2); - expect(configCache.markKeyAsMissing).toHaveBeenCalledWith('AUTH_PASSWORD_ENABLED'); - expect(configCache.markKeyAsMissing).toHaveBeenCalledWith('EMAIL_FROM_ADDRESS'); + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith( + 'AUTH_PASSWORD_ENABLED', + ); + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith( + 'EMAIL_FROM_ADDRESS', + ); }); it('should update cache with refreshed values', async () => { @@ -523,7 +539,7 @@ describe('DatabaseConfigDriver', () => { 'EMAIL_FROM_ADDRESS', 'MISSING_KEY', ] as Array; - + const refreshedValues = new Map< keyof ConfigVariables, ConfigVariables[keyof ConfigVariables] @@ -531,28 +547,41 @@ describe('DatabaseConfigDriver', () => { ['AUTH_PASSWORD_ENABLED', true], ['EMAIL_FROM_ADDRESS', 'test@example.com'], ]); - + jest.spyOn(configCache, 'getExpiredKeys').mockReturnValue(expiredKeys); - jest.spyOn(configStorage, 'loadByKeys').mockResolvedValue(refreshedValues); - + jest + .spyOn(configStorage, 'loadByKeys') + .mockResolvedValue(refreshedValues); + await driver.refreshExpiredCache(); - + // Should set found values - expect(configCache.set).toHaveBeenCalledWith('AUTH_PASSWORD_ENABLED', true); - expect(configCache.set).toHaveBeenCalledWith('EMAIL_FROM_ADDRESS', 'test@example.com'); - + expect(configCache.set).toHaveBeenCalledWith( + 'AUTH_PASSWORD_ENABLED', + true, + ); + expect(configCache.set).toHaveBeenCalledWith( + 'EMAIL_FROM_ADDRESS', + 'test@example.com', + ); + // Should mark missing values as missing expect(configCache.markKeyAsMissing).toHaveBeenCalledWith('MISSING_KEY'); }); it('should handle errors gracefully', async () => { const error = new Error('Database error'); - jest.spyOn(configCache, 'getExpiredKeys').mockReturnValue(['AUTH_PASSWORD_ENABLED'] as Array); + + jest + .spyOn(configCache, 'getExpiredKeys') + .mockReturnValue(['AUTH_PASSWORD_ENABLED'] as Array< + keyof ConfigVariables + >); jest.spyOn(configStorage, 'loadByKeys').mockRejectedValue(error); jest.spyOn(driver['logger'], 'error'); - + await driver.refreshExpiredCache(); - + // Should log error and not throw expect(driver['logger'].error).toHaveBeenCalled(); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts index f52a560695d9..030e7ea0feb9 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts @@ -1,7 +1,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { getRepositoryToken } from '@nestjs/typeorm'; -import { DeleteResult, IsNull, Repository } from 'typeorm'; +import { DeleteResult, IsNull, Repository, In } from 'typeorm'; import { KeyValuePair, @@ -12,7 +12,6 @@ import { ConfigValueConverterService } from 'src/engine/core-modules/twenty-conf import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; import { User } from 'src/engine/core-modules/user/user.entity'; import { Workspace } from 'src/engine/core-modules/workspace/workspace.entity'; -import { In } from 'typeorm'; describe('ConfigStorageService', () => { let service: ConfigStorageService; @@ -489,7 +488,9 @@ describe('ConfigStorageService', () => { describe('loadByKeys', () => { it('should load and convert specified config variables', async () => { - const keys = ['AUTH_PASSWORD_ENABLED', 'EMAIL_FROM_ADDRESS'] as Array; + const keys = ['AUTH_PASSWORD_ENABLED', 'EMAIL_FROM_ADDRESS'] as Array< + keyof ConfigVariables + >; const configVars: KeyValuePair[] = [ createMockKeyValuePair('AUTH_PASSWORD_ENABLED', 'true'), createMockKeyValuePair('EMAIL_FROM_ADDRESS', 'test@example.com'), @@ -535,7 +536,9 @@ describe('ConfigStorageService', () => { }); it('should handle conversion errors and skip problematic entries', async () => { - const keys = ['AUTH_PASSWORD_ENABLED', 'PROBLEMATIC_KEY'] as Array; + const keys = ['AUTH_PASSWORD_ENABLED', 'PROBLEMATIC_KEY'] as Array< + keyof ConfigVariables + >; const configVars: KeyValuePair[] = [ createMockKeyValuePair('AUTH_PASSWORD_ENABLED', 'true'), createMockKeyValuePair('PROBLEMATIC_KEY', 'bad-value'), @@ -558,7 +561,9 @@ describe('ConfigStorageService', () => { expect(result.get('AUTH_PASSWORD_ENABLED' as keyof ConfigVariables)).toBe( true, ); - expect(result.has('PROBLEMATIC_KEY' as keyof ConfigVariables)).toBe(false); + expect(result.has('PROBLEMATIC_KEY' as keyof ConfigVariables)).toBe( + false, + ); }); }); }); From ba2a645908fd781fbbcb85373f0f3b282262e425 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 23 Apr 2025 23:20:25 +0530 Subject: [PATCH 51/70] add debounce mechanism to prevent duplicate config refreshes --- .../__tests__/database-config.driver.spec.ts | 91 +++++++++++++++++-- .../drivers/database-config.driver.ts | 18 +++- 2 files changed, 100 insertions(+), 9 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index aafdad3d1439..bf60f421143c 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -171,7 +171,6 @@ describe('DatabaseConfigDriver', () => { expect(result).toBe(cachedValue); - // Run immediate callbacks jest.runAllTimers(); expect(fetchSpy).toHaveBeenCalledWith(key); @@ -213,7 +212,6 @@ describe('DatabaseConfigDriver', () => { expect(result).toBe(envValue); - // Run immediate callbacks jest.runAllTimers(); expect(fetchSpy).toHaveBeenCalledWith(key); @@ -448,6 +446,90 @@ describe('DatabaseConfigDriver', () => { jest.useRealTimers(); }); + it('should not schedule multiple refreshes for the same key', async () => { + const originalScheduleRefresh = driver['scheduleRefresh']; + const refreshSpyFn = jest.fn(); + + const refreshingKeys = new Set(); + + driver['scheduleRefresh'] = jest.fn().mockImplementation(async (key) => { + if (refreshingKeys.has(key)) { + return; + } + + refreshingKeys.add(key); + refreshSpyFn(key); + + refreshingKeys.delete(key); + }); + + const key = 'MISSING_KEY' as keyof ConfigVariables; + + // Call multiple times in sequence + await driver['scheduleRefresh'](key); + await driver['scheduleRefresh'](key); + await driver['scheduleRefresh'](key); + + // refreshSpyFn should be called three times because we're not keeping + // the key in refreshingKeys across calls (our mock clears it immediately) + expect(refreshSpyFn).toHaveBeenCalledTimes(3); + + refreshSpyFn.mockClear(); + + let isRefreshing = false; + driver['scheduleRefresh'] = jest.fn().mockImplementation(async (key) => { + if (isRefreshing) { + return; + } + + isRefreshing = true; + refreshSpyFn(key); + + }); + + // Make concurrent calls + const promise1 = driver['scheduleRefresh'](key); + const promise2 = driver['scheduleRefresh'](key); + const promise3 = driver['scheduleRefresh'](key); + + await Promise.all([promise1, promise2, promise3]); + + expect(refreshSpyFn).toHaveBeenCalledTimes(1); + expect(refreshSpyFn).toHaveBeenCalledWith(key); + + driver['scheduleRefresh'] = originalScheduleRefresh; + }); + + it('should handle refresh failures and clean up', async () => { + // Mock the internal scheduleRefresh implementation to directly test error handling + const originalScheduleRefresh = driver['scheduleRefresh']; + const error = new Error('Fetch error'); + + driver['scheduleRefresh'] = jest.fn().mockImplementation(async (key) => { + if (driver['refreshingKeys'].has(key)) { + return; + } + + driver['refreshingKeys'].add(key); + + try { + // Simulate a fetch error + throw error; + } catch (e) { + driver['refreshingKeys'].delete(key); + throw e; + } + }); + + const key = 'MISSING_KEY' as keyof ConfigVariables; + + await expect(driver['scheduleRefresh'](key)).rejects.toThrow(error); + + expect(driver['refreshingKeys'].has(key)).toBe(false); + + driver['scheduleRefresh'] = originalScheduleRefresh; + }); + it('should correctly populate cache during initialization', async () => { const configVars = new Map< keyof ConfigVariables, @@ -477,9 +559,7 @@ describe('DatabaseConfigDriver', () => { describe('refreshExpiredCache', () => { beforeEach(() => { jest.spyOn(configStorage, 'loadAll').mockResolvedValue(new Map()); - // Add loadByKeys mock jest.spyOn(configStorage, 'loadByKeys').mockResolvedValue(new Map()); - // Add getExpiredKeys mock jest.spyOn(configCache, 'getExpiredKeys').mockReturnValue([]); }); @@ -555,7 +635,6 @@ describe('DatabaseConfigDriver', () => { await driver.refreshExpiredCache(); - // Should set found values expect(configCache.set).toHaveBeenCalledWith( 'AUTH_PASSWORD_ENABLED', true, @@ -565,7 +644,6 @@ describe('DatabaseConfigDriver', () => { 'test@example.com', ); - // Should mark missing values as missing expect(configCache.markKeyAsMissing).toHaveBeenCalledWith('MISSING_KEY'); }); @@ -582,7 +660,6 @@ describe('DatabaseConfigDriver', () => { await driver.refreshExpiredCache(); - // Should log error and not throw expect(driver['logger'].error).toHaveBeenCalled(); }); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index 4fc57ccd3cfe..8640eee97b3f 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -15,6 +15,7 @@ import { EnvironmentConfigDriver } from './environment-config.driver'; export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { private initializationPromise: Promise | null = null; private readonly logger = new Logger(DatabaseConfigDriver.name); + private readonly refreshingKeys = new Set(); constructor( private readonly configCache: ConfigCacheService, @@ -143,10 +144,23 @@ export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { } private async scheduleRefresh(key: keyof ConfigVariables): Promise { + if (this.refreshingKeys.has(key)) { + this.logger.debug(`Refresh for key ${key as string} already in progress`); + + return; + } + + this.refreshingKeys.add(key); + setImmediate(async () => { - await this.fetchAndCacheConfigVariable(key).catch((error) => { + try { + await this.fetchAndCacheConfigVariable(key); + this.logger.debug(`Completed refresh for key ${key as string}`); + } catch (error) { this.logger.error(`Failed to fetch config for ${key as string}`, error); - }); + } finally { + this.refreshingKeys.delete(key); + } }); } From 2d2de8622ddb562bf46a974f8f2bc24a121ab02b Mon Sep 17 00:00:00 2001 From: ehconitin Date: Wed, 23 Apr 2025 23:21:11 +0530 Subject: [PATCH 52/70] lint --- .../__tests__/database-config.driver.spec.ts | 46 +++++++++---------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index bf60f421143c..eacd6a60f3fc 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -449,69 +449,69 @@ describe('DatabaseConfigDriver', () => { it('should not schedule multiple refreshes for the same key', async () => { const originalScheduleRefresh = driver['scheduleRefresh']; const refreshSpyFn = jest.fn(); - + const refreshingKeys = new Set(); - + driver['scheduleRefresh'] = jest.fn().mockImplementation(async (key) => { if (refreshingKeys.has(key)) { return; } - + refreshingKeys.add(key); refreshSpyFn(key); - + refreshingKeys.delete(key); }); - + const key = 'MISSING_KEY' as keyof ConfigVariables; - + // Call multiple times in sequence await driver['scheduleRefresh'](key); await driver['scheduleRefresh'](key); await driver['scheduleRefresh'](key); - + // refreshSpyFn should be called three times because we're not keeping // the key in refreshingKeys across calls (our mock clears it immediately) expect(refreshSpyFn).toHaveBeenCalledTimes(3); - + refreshSpyFn.mockClear(); - + let isRefreshing = false; + driver['scheduleRefresh'] = jest.fn().mockImplementation(async (key) => { if (isRefreshing) { return; } - + isRefreshing = true; refreshSpyFn(key); - }); - + // Make concurrent calls const promise1 = driver['scheduleRefresh'](key); const promise2 = driver['scheduleRefresh'](key); const promise3 = driver['scheduleRefresh'](key); - + await Promise.all([promise1, promise2, promise3]); - + expect(refreshSpyFn).toHaveBeenCalledTimes(1); expect(refreshSpyFn).toHaveBeenCalledWith(key); - + driver['scheduleRefresh'] = originalScheduleRefresh; }); - + it('should handle refresh failures and clean up', async () => { // Mock the internal scheduleRefresh implementation to directly test error handling const originalScheduleRefresh = driver['scheduleRefresh']; const error = new Error('Fetch error'); - + driver['scheduleRefresh'] = jest.fn().mockImplementation(async (key) => { if (driver['refreshingKeys'].has(key)) { return; } - + driver['refreshingKeys'].add(key); - + try { // Simulate a fetch error throw error; @@ -520,13 +520,13 @@ describe('DatabaseConfigDriver', () => { throw e; } }); - + const key = 'MISSING_KEY' as keyof ConfigVariables; - + await expect(driver['scheduleRefresh'](key)).rejects.toThrow(error); - + expect(driver['refreshingKeys'].has(key)).toBe(false); - + driver['scheduleRefresh'] = originalScheduleRefresh; }); From c62f6775bc9c2b4970d931f229883adb90a5369e Mon Sep 17 00:00:00 2001 From: ehconitin Date: Thu, 24 Apr 2025 16:00:35 +0530 Subject: [PATCH 53/70] simplify --- .../__tests__/config-cache.service.spec.ts | 157 ++--- .../cache/config-cache.service.ts | 85 +-- .../config-cache-entry.interface.ts | 7 - .../constants/config-variables-cache-ttl.ts | 1 - ...riables-refresh-cron-interval.constants.ts | 1 + .../config-variables-refresh-interval.ts | 1 - .../__tests__/database-config.driver.spec.ts | 541 +++++------------- .../drivers/database-config.driver.ts | 153 +++-- .../database-config-driver.interface.ts | 4 +- .../__tests__/config-storage.service.spec.ts | 83 +-- .../storage/config-storage.service.ts | 59 +- 11 files changed, 256 insertions(+), 836 deletions(-) delete mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-refresh-cron-interval.constants.ts delete mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-refresh-interval.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts index e8ff3189ecd9..b6f6355cf424 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts @@ -3,22 +3,10 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; -import { CONFIG_VARIABLES_CACHE_TTL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl'; describe('ConfigCacheService', () => { let service: ConfigCacheService; - const withMockedDate = (timeOffset: number, callback: () => void) => { - const originalNow = Date.now; - - try { - Date.now = jest.fn(() => originalNow() + timeOffset); - callback(); - } finally { - Date.now = originalNow; - } - }; - beforeEach(async () => { jest.useFakeTimers(); @@ -47,8 +35,7 @@ describe('ConfigCacheService', () => { service.set(key, value); const result = service.get(key); - expect(result.value).toBe(value); - expect(result.isStale).toBe(false); + expect(result).toBe(value); }); it('should return undefined for non-existent key', () => { @@ -56,8 +43,7 @@ describe('ConfigCacheService', () => { 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, ); - expect(result.value).toBeUndefined(); - expect(result.isStale).toBe(false); + expect(result).toBeUndefined(); }); it('should handle different value types', () => { @@ -69,9 +55,9 @@ describe('ConfigCacheService', () => { service.set(stringKey, 'test@example.com'); service.set(numberKey, 3000); - expect(service.get(booleanKey).value).toBe(true); - expect(service.get(stringKey).value).toBe('test@example.com'); - expect(service.get(numberKey).value).toBe(3000); + expect(service.get(booleanKey)).toBe(true); + expect(service.get(stringKey)).toBe('test@example.com'); + expect(service.get(numberKey)).toBe(3000); }); }); @@ -92,17 +78,6 @@ describe('ConfigCacheService', () => { expect(result).toBe(false); }); - - it('should return false for negative cache entry check when expired', () => { - const key = 'TEST_KEY' as keyof ConfigVariables; - - service.markKeyAsMissing(key); - - // Mock a date beyond the TTL - jest.spyOn(Date, 'now').mockReturnValueOnce(Date.now() + 1000000); - - expect(service.isKeyKnownMissing(key)).toBe(false); - }); }); describe('clear operations', () => { @@ -114,8 +89,8 @@ describe('ConfigCacheService', () => { service.set(key2, 'test@example.com'); service.clear(key1); - expect(service.get(key1).value).toBeUndefined(); - expect(service.get(key2).value).toBe('test@example.com'); + expect(service.get(key1)).toBeUndefined(); + expect(service.get(key2)).toBe('test@example.com'); }); it('should clear all entries', () => { @@ -126,38 +101,8 @@ describe('ConfigCacheService', () => { service.set(key2, 'test@example.com'); service.clearAll(); - expect(service.get(key1).value).toBeUndefined(); - expect(service.get(key2).value).toBeUndefined(); - }); - }); - - describe('cache expiration', () => { - it('should mark entries as stale after TTL', () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - const value = true; - - service.set(key, value); - - withMockedDate(CONFIG_VARIABLES_CACHE_TTL + 1, () => { - const result = service.get(key); - - expect(result.value).toBe(value); - expect(result.isStale).toBe(true); - }); - }); - - it('should not mark entries as stale before TTL', () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - const value = true; - - service.set(key, value); - - withMockedDate(CONFIG_VARIABLES_CACHE_TTL - 1, () => { - const result = service.get(key); - - expect(result.value).toBe(value); - expect(result.isStale).toBe(false); - }); + expect(service.get(key1)).toBeUndefined(); + expect(service.get(key2)).toBeUndefined(); }); }); @@ -181,20 +126,6 @@ describe('ConfigCacheService', () => { expect(service.isKeyKnownMissing(key3)).toBe(true); }); - it('should not include expired entries in cache info', () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - - service.set(key, true); - - withMockedDate(CONFIG_VARIABLES_CACHE_TTL + 1, () => { - const info = service.getCacheInfo(); - - expect(info.positiveEntries).toBe(0); - expect(info.negativeEntries).toBe(0); - expect(info.cacheKeys).toHaveLength(0); - }); - }); - it('should properly count cache entries', () => { const key1 = 'KEY1' as keyof ConfigVariables; const key2 = 'KEY2' as keyof ConfigVariables; @@ -223,65 +154,47 @@ describe('ConfigCacheService', () => { service.onModuleDestroy(); - expect(service.get(key).value).toBeUndefined(); + expect(service.get(key)).toBeUndefined(); }); }); - describe('getExpiredKeys', () => { - it('should return expired keys from both positive and negative caches', () => { - const expiredKey1 = 'EXPIRED_KEY1' as keyof ConfigVariables; - const expiredKey2 = 'EXPIRED_KEY2' as keyof ConfigVariables; - const expiredNegativeKey = - 'EXPIRED_NEGATIVE_KEY' as keyof ConfigVariables; - - // Set up keys that will expire - service.set(expiredKey1, 'value1'); - service.set(expiredKey2, 'value2'); - service.markKeyAsMissing(expiredNegativeKey); - - // Make the above keys expire - withMockedDate(CONFIG_VARIABLES_CACHE_TTL + 1, () => { - // Add a fresh key after the time change - const freshKey = 'FRESH_KEY' as keyof ConfigVariables; - - service.set(freshKey, 'value3'); - - const expiredKeys = service.getExpiredKeys(); - - expect(expiredKeys).toContain(expiredKey1); - expect(expiredKeys).toContain(expiredKey2); - expect(expiredKeys).toContain(expiredNegativeKey); - expect(expiredKeys).not.toContain(freshKey); - }); - }); - - it('should return empty array when no keys are expired', () => { - const key1 = 'KEY1' as keyof ConfigVariables; - const key2 = 'KEY2' as keyof ConfigVariables; + describe('getAllKeys', () => { + it('should return all keys from both positive and negative caches', () => { + const positiveKey1 = 'POSITIVE_KEY1' as keyof ConfigVariables; + const positiveKey2 = 'POSITIVE_KEY2' as keyof ConfigVariables; const negativeKey = 'NEGATIVE_KEY' as keyof ConfigVariables; - service.set(key1, 'value1'); - service.set(key2, 'value2'); + // Set up keys + service.set(positiveKey1, 'value1'); + service.set(positiveKey2, 'value2'); service.markKeyAsMissing(negativeKey); - const expiredKeys = service.getExpiredKeys(); + const allKeys = service.getAllKeys(); - expect(expiredKeys).toHaveLength(0); + expect(allKeys).toContain(positiveKey1); + expect(allKeys).toContain(positiveKey2); + expect(allKeys).toContain(negativeKey); }); - it('should not have duplicates if a key is in both caches', () => { + it('should return empty array when no keys exist', () => { + const allKeys = service.getAllKeys(); + expect(allKeys).toHaveLength(0); + }); + + it('should not have duplicates if a key somehow exists in both caches', () => { const key = 'DUPLICATE_KEY' as keyof ConfigVariables; - // Manually manipulate the caches to simulate a key in both caches + // First add to positive cache service.set(key, 'value'); - service.markKeyAsMissing(key); + + // Then force it into negative cache (normally this would remove from positive) + // We're bypassing normal behavior for testing edge cases + service['knownMissingKeysCache'].add(key); - withMockedDate(CONFIG_VARIABLES_CACHE_TTL + 1, () => { - const expiredKeys = service.getExpiredKeys(); + const allKeys = service.getAllKeys(); - // Should only appear once in the result - expect(expiredKeys.filter((k) => k === key)).toHaveLength(1); - }); + // Should only appear once in the result + expect(allKeys.filter((k) => k === key)).toHaveLength(1); }); }); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts index c940902c0eb5..8b539569c5c1 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts @@ -1,11 +1,8 @@ import { Injectable, Logger, OnModuleDestroy } from '@nestjs/common'; -import { CONFIG_VARIABLES_CACHE_TTL } from 'src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl'; - import { ConfigCacheEntry, ConfigKey, - ConfigKnownMissingEntry, ConfigValue, } from './interfaces/config-cache-entry.interface'; @@ -16,57 +13,34 @@ export class ConfigCacheService implements OnModuleDestroy { ConfigKey, ConfigCacheEntry >; - private readonly knownMissingKeysCache: Map< - ConfigKey, - ConfigKnownMissingEntry - >; + private readonly knownMissingKeysCache: Set; constructor() { this.foundConfigValuesCache = new Map(); - this.knownMissingKeysCache = new Map(); + this.knownMissingKeysCache = new Set(); } - get( - key: T, - ): { value: ConfigValue | undefined; isStale: boolean } { + get(key: T): ConfigValue | undefined { const entry = this.foundConfigValuesCache.get(key); if (!entry) { - return { value: undefined, isStale: false }; + return undefined; } - const isStale = this.isCacheExpired(entry); - - return { - value: entry.value as ConfigValue, - isStale, - }; + return entry.value as ConfigValue; } isKeyKnownMissing(key: ConfigKey): boolean { - const entry = this.knownMissingKeysCache.get(key); - - if (entry && !this.isCacheExpired(entry)) { - return true; - } - - return false; + return this.knownMissingKeysCache.has(key); } set(key: T, value: ConfigValue): void { - this.foundConfigValuesCache.set(key, { - value, - registeredAt: Date.now(), - ttl: CONFIG_VARIABLES_CACHE_TTL, - }); + this.foundConfigValuesCache.set(key, { value }); this.knownMissingKeysCache.delete(key); } markKeyAsMissing(key: ConfigKey): void { - this.knownMissingKeysCache.set(key, { - registeredAt: Date.now(), - ttl: CONFIG_VARIABLES_CACHE_TTL, - }); + this.knownMissingKeysCache.add(key); this.foundConfigValuesCache.delete(key); } @@ -85,18 +59,10 @@ export class ConfigCacheService implements OnModuleDestroy { negativeEntries: number; cacheKeys: string[]; } { - const validPositiveEntries = Array.from( - this.foundConfigValuesCache.entries(), - ).filter(([_, entry]) => !this.isCacheExpired(entry)); - - const validNegativeEntries = Array.from( - this.knownMissingKeysCache.entries(), - ).filter(([_, entry]) => !this.isCacheExpired(entry)); - return { - positiveEntries: validPositiveEntries.length, - negativeEntries: validNegativeEntries.length, - cacheKeys: validPositiveEntries.map(([key]) => key), + positiveEntries: this.foundConfigValuesCache.size, + negativeEntries: this.knownMissingKeysCache.size, + cacheKeys: Array.from(this.foundConfigValuesCache.keys()), }; } @@ -104,31 +70,10 @@ export class ConfigCacheService implements OnModuleDestroy { this.clearAll(); } - private isCacheExpired( - entry: ConfigCacheEntry | ConfigKnownMissingEntry, - ): boolean { - const now = Date.now(); - - return now - entry.registeredAt > entry.ttl; - } - - getExpiredKeys(): ConfigKey[] { - const now = Date.now(); - const expiredPositiveKeys: ConfigKey[] = []; - const expiredNegativeKeys: ConfigKey[] = []; - - for (const [key, entry] of this.foundConfigValuesCache.entries()) { - if (now - entry.registeredAt > entry.ttl) { - expiredPositiveKeys.push(key); - } - } - - for (const [key, entry] of this.knownMissingKeysCache.entries()) { - if (now - entry.registeredAt > entry.ttl) { - expiredNegativeKeys.push(key); - } - } + getAllKeys(): ConfigKey[] { + const positiveKeys = Array.from(this.foundConfigValuesCache.keys()); + const negativeKeys = Array.from(this.knownMissingKeysCache); - return [...new Set([...expiredPositiveKeys, ...expiredNegativeKeys])]; + return [...new Set([...positiveKeys, ...negativeKeys])]; } } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts index 65f5e447fc7f..988e39cc73bf 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/interfaces/config-cache-entry.interface.ts @@ -5,11 +5,4 @@ export type ConfigValue = ConfigVariables[T]; export interface ConfigCacheEntry { value: ConfigValue; - registeredAt: number; - ttl: number; -} - -export interface ConfigKnownMissingEntry { - registeredAt: number; - ttl: number; } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl.ts deleted file mode 100644 index cf0739190a51..000000000000 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-cache-ttl.ts +++ /dev/null @@ -1 +0,0 @@ -export const CONFIG_VARIABLES_CACHE_TTL = 30 * 1000; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-refresh-cron-interval.constants.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-refresh-cron-interval.constants.ts new file mode 100644 index 000000000000..7dcf53a62c4d --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-refresh-cron-interval.constants.ts @@ -0,0 +1 @@ +export const CONFIG_VARIABLES_REFRESH_CRON_INTERVAL = '*/15 * * * * *'; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-refresh-interval.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-refresh-interval.ts deleted file mode 100644 index 028c47e2c2f1..000000000000 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-refresh-interval.ts +++ /dev/null @@ -1 +0,0 @@ -export const CONFIG_VARIABLES_REFRESH_INTERVAL_MINUTES = 5; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index eacd6a60f3fc..c4398d82c002 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -14,16 +14,51 @@ jest.mock( }), ); +const CONFIG_PASSWORD_KEY = 'AUTH_PASSWORD_ENABLED'; +const CONFIG_EMAIL_KEY = 'EMAIL_FROM_ADDRESS'; +const CONFIG_ENV_ONLY_KEY = 'ENV_ONLY_VAR'; +const CONFIG_PORT_KEY = 'NODE_PORT'; + +class TestDatabaseConfigDriver extends DatabaseConfigDriver { + // Expose the protected/private property for testing + public get testAllPossibleConfigKeys(): Array { + return this['allPossibleConfigKeys']; + } + + // Override Object.keys usage in constructor with our test keys + constructor( + configCache: ConfigCacheService, + configStorage: ConfigStorageService, + environmentDriver: EnvironmentConfigDriver, + ) { + super(configCache, configStorage, environmentDriver); + + + Object.defineProperty(this, 'allPossibleConfigKeys', { + value: [CONFIG_PASSWORD_KEY, CONFIG_EMAIL_KEY, CONFIG_PORT_KEY], + writable: false, + configurable: true, + }); + } +} + describe('DatabaseConfigDriver', () => { - let driver: DatabaseConfigDriver; + let driver: TestDatabaseConfigDriver; let configCache: ConfigCacheService; let configStorage: ConfigStorageService; let environmentDriver: EnvironmentConfigDriver; beforeEach(async () => { + (isEnvOnlyConfigVar as jest.Mock).mockImplementation((key) => { + return key === CONFIG_ENV_ONLY_KEY; + }); + const module: TestingModule = await Test.createTestingModule({ providers: [ - DatabaseConfigDriver, + { + provide: DatabaseConfigDriver, + useClass: TestDatabaseConfigDriver, + }, { provide: ConfigCacheService, useValue: { @@ -34,7 +69,7 @@ describe('DatabaseConfigDriver', () => { isKeyKnownMissing: jest.fn(), markKeyAsMissing: jest.fn(), getCacheInfo: jest.fn(), - getExpiredKeys: jest.fn(), + getAllKeys: jest.fn(), }, }, { @@ -43,7 +78,6 @@ describe('DatabaseConfigDriver', () => { get: jest.fn(), set: jest.fn(), loadAll: jest.fn(), - loadByKeys: jest.fn(), }, }, { @@ -55,7 +89,7 @@ describe('DatabaseConfigDriver', () => { ], }).compile(); - driver = module.get(DatabaseConfigDriver); + driver = module.get(DatabaseConfigDriver) as TestDatabaseConfigDriver; configCache = module.get(ConfigCacheService); configStorage = module.get(ConfigStorageService); environmentDriver = module.get( @@ -63,7 +97,6 @@ describe('DatabaseConfigDriver', () => { ); jest.clearAllMocks(); - (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(false); }); afterEach(() => { @@ -76,28 +109,34 @@ describe('DatabaseConfigDriver', () => { }); describe('initialization', () => { - it('should initialize successfully', async () => { - const configVars = new Map< - keyof ConfigVariables, - ConfigVariables[keyof ConfigVariables] - >([ - ['AUTH_PASSWORD_ENABLED', true], - ['EMAIL_FROM_ADDRESS', 'test@example.com'], - ]); + it('should have allPossibleConfigKeys properly set', () => { + expect(driver.testAllPossibleConfigKeys).toContain(CONFIG_PASSWORD_KEY); + expect(driver.testAllPossibleConfigKeys).toContain(CONFIG_EMAIL_KEY); + expect(driver.testAllPossibleConfigKeys).not.toContain(CONFIG_ENV_ONLY_KEY); + }); + + it('should initialize successfully with DB values and mark missing keys', async () => { + const configVars = new Map(); + configVars.set(CONFIG_PASSWORD_KEY, true); jest.spyOn(configStorage, 'loadAll').mockResolvedValue(configVars); await driver.initialize(); expect(configStorage.loadAll).toHaveBeenCalled(); - expect(configCache.set).toHaveBeenCalledTimes(2); + + expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, true); + + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(CONFIG_EMAIL_KEY); + + expect(configCache.markKeyAsMissing).not.toHaveBeenCalledWith(CONFIG_ENV_ONLY_KEY); }); it('should handle initialization failure', async () => { const error = new Error('DB error'); jest.spyOn(configStorage, 'loadAll').mockRejectedValue(error); - jest.spyOn(driver['logger'], 'error'); + jest.spyOn(driver['logger'], 'error').mockImplementation(); await expect(driver.initialize()).rejects.toThrow(error); @@ -126,122 +165,76 @@ describe('DatabaseConfigDriver', () => { }); it('should use environment driver for env-only variables', async () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const envValue = true; (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(true); jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); - const result = driver.get(key); + const result = driver.get(CONFIG_PASSWORD_KEY); expect(result).toBe(envValue); - expect(environmentDriver.get).toHaveBeenCalledWith(key); - expect(isEnvOnlyConfigVar).toHaveBeenCalledWith(key); - }); - - it('should return cached value when available and not stale', async () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - const cachedValue = true; - - jest - .spyOn(configCache, 'get') - .mockReturnValue({ value: cachedValue, isStale: false }); - - const result = driver.get(key); - - expect(result).toBe(cachedValue); - expect(configCache.get).toHaveBeenCalledWith(key); + expect(environmentDriver.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); + expect(isEnvOnlyConfigVar).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); }); - it('should return cached value and schedule refresh when stale', async () => { - jest.useFakeTimers(); - - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + it('should return cached value when available', async () => { const cachedValue = true; - jest - .spyOn(configCache, 'get') - .mockReturnValue({ value: cachedValue, isStale: true }); - - const fetchSpy = jest - .spyOn(driver, 'fetchAndCacheConfigVariable') - .mockResolvedValue(); + jest.spyOn(configCache, 'get').mockReturnValue(cachedValue); - const result = driver.get(key); + const result = driver.get(CONFIG_PASSWORD_KEY); expect(result).toBe(cachedValue); - - jest.runAllTimers(); - - expect(fetchSpy).toHaveBeenCalledWith(key); + expect(configCache.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); }); it('should use environment driver when negative lookup is cached', async () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const envValue = true; - jest - .spyOn(configCache, 'get') - .mockReturnValue({ value: undefined, isStale: false }); + jest.spyOn(configCache, 'get').mockReturnValue(undefined); jest.spyOn(configCache, 'isKeyKnownMissing').mockReturnValue(true); jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); - const result = driver.get(key); + const result = driver.get(CONFIG_PASSWORD_KEY); expect(result).toBe(envValue); - expect(environmentDriver.get).toHaveBeenCalledWith(key); + expect(environmentDriver.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); }); - it('should schedule refresh when cache misses and not known missing', async () => { - jest.useFakeTimers(); - - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + it('should fallback to environment when not in cache and not known missing', async () => { const envValue = true; - jest - .spyOn(configCache, 'get') - .mockReturnValue({ value: undefined, isStale: false }); + jest.spyOn(configCache, 'get').mockReturnValue(undefined); jest.spyOn(configCache, 'isKeyKnownMissing').mockReturnValue(false); jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); - const fetchSpy = jest - .spyOn(driver, 'fetchAndCacheConfigVariable') - .mockResolvedValue(); - - const result = driver.get(key); + const result = driver.get(CONFIG_PASSWORD_KEY); expect(result).toBe(envValue); - - jest.runAllTimers(); - - expect(fetchSpy).toHaveBeenCalledWith(key); + expect(environmentDriver.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); }); it('should handle different config variable types correctly', () => { - const stringKey = 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables; - const booleanKey = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - const numberKey = 'NODE_PORT' as keyof ConfigVariables; - const stringValue = 'test@example.com'; const booleanValue = true; const numberValue = 3000; jest.spyOn(configCache, 'get').mockImplementation((key) => { switch (key) { - case stringKey: - return { value: stringValue, isStale: false }; - case booleanKey: - return { value: booleanValue, isStale: false }; - case numberKey: - return { value: numberValue, isStale: false }; + case CONFIG_EMAIL_KEY: + return stringValue; + case CONFIG_PASSWORD_KEY: + return booleanValue; + case CONFIG_PORT_KEY: + return numberValue; default: - return { value: undefined, isStale: false }; + return undefined; } }); - expect(driver.get(stringKey)).toBe(stringValue); - expect(driver.get(booleanKey)).toBe(booleanValue); - expect(driver.get(numberKey)).toBe(numberValue); + expect(driver.get(CONFIG_EMAIL_KEY)).toBe(stringValue); + expect(driver.get(CONFIG_PASSWORD_KEY)).toBe(booleanValue); + expect(driver.get(CONFIG_PORT_KEY)).toBe(numberValue); }); }); @@ -253,52 +246,46 @@ describe('DatabaseConfigDriver', () => { }); it('should update config in storage and cache', async () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const value = true; - await driver.update(key, value); + await driver.update(CONFIG_PASSWORD_KEY, value); - expect(configStorage.set).toHaveBeenCalledWith(key, value); - expect(configCache.set).toHaveBeenCalledWith(key, value); + expect(configStorage.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, value); + expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, value); }); it('should throw error when updating env-only variable', async () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; const value = true; (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(true); - await expect(driver.update(key, value)).rejects.toThrow(); + await expect(driver.update(CONFIG_PASSWORD_KEY, value)).rejects.toThrow(); }); }); describe('fetchAndCacheConfigVariable', () => { it('should refresh config variable from storage', async () => { - const key = 'AUTH_PASSWORD_ENABLED'; const value = true; jest.spyOn(configStorage, 'get').mockResolvedValue(value); - await driver.fetchAndCacheConfigVariable(key); + await driver.fetchAndCacheConfigVariable(CONFIG_PASSWORD_KEY); - expect(configStorage.get).toHaveBeenCalledWith(key); - expect(configCache.set).toHaveBeenCalledWith(key, value); + expect(configStorage.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); + expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, value); }); it('should mark key as missing when value is undefined', async () => { - const key = 'AUTH_PASSWORD_ENABLED'; - jest.spyOn(configStorage, 'get').mockResolvedValue(undefined); - await driver.fetchAndCacheConfigVariable(key); + await driver.fetchAndCacheConfigVariable(CONFIG_PASSWORD_KEY); - expect(configStorage.get).toHaveBeenCalledWith(key); - expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(key); + expect(configStorage.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); expect(configCache.set).not.toHaveBeenCalled(); }); it('should mark key as missing when storage fetch fails', async () => { - const key = 'AUTH_PASSWORD_ENABLED'; const error = new Error('Storage error'); jest.spyOn(configStorage, 'get').mockRejectedValue(error); @@ -306,10 +293,10 @@ describe('DatabaseConfigDriver', () => { .spyOn(driver['logger'], 'error') .mockImplementation(); - await driver.fetchAndCacheConfigVariable(key); + await driver.fetchAndCacheConfigVariable(CONFIG_PASSWORD_KEY); - expect(configStorage.get).toHaveBeenCalledWith(key); - expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(key); + expect(configStorage.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); expect(loggerSpy).toHaveBeenCalledWith( expect.stringContaining('Failed to fetch config'), error, @@ -322,7 +309,7 @@ describe('DatabaseConfigDriver', () => { const cacheInfo = { positiveEntries: 2, negativeEntries: 1, - cacheKeys: ['AUTH_PASSWORD_ENABLED', 'EMAIL_FROM_ADDRESS'], + cacheKeys: [CONFIG_PASSWORD_KEY, CONFIG_EMAIL_KEY], }; jest.spyOn(configCache, 'getCacheInfo').mockReturnValue(cacheInfo); @@ -333,332 +320,66 @@ describe('DatabaseConfigDriver', () => { }); }); - describe('error handling', () => { - beforeEach(async () => { - jest.spyOn(configStorage, 'loadAll').mockResolvedValue(new Map()); - await driver.initialize(); - (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(false); - }); - - it('should handle storage service errors during update', async () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - const value = true; - const error = new Error('Storage error'); - - jest.spyOn(configStorage, 'set').mockRejectedValue(error); - const loggerSpy = jest - .spyOn(driver['logger'], 'error') - .mockImplementation(); - - await expect(driver.update(key, value)).rejects.toThrow(); - expect(configCache.set).not.toHaveBeenCalled(); - expect(loggerSpy).toHaveBeenCalledWith( - expect.stringContaining('Failed to update config'), - error, - ); - }); - - it('should use environment driver for env-only variables', async () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - const envValue = true; - - (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(true); - jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); - - const result = driver.get(key); - - expect(result).toBe(envValue); - expect(environmentDriver.get).toHaveBeenCalledWith(key); - expect(configCache.get).not.toHaveBeenCalled(); - expect(isEnvOnlyConfigVar).toHaveBeenCalledWith(key); - }); - - it('should use environment driver when negative lookup is cached', async () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - const envValue = true; - - jest - .spyOn(configCache, 'get') - .mockReturnValue({ value: undefined, isStale: false }); - jest.spyOn(configCache, 'isKeyKnownMissing').mockReturnValue(true); - jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); - - const result = driver.get(key); - - expect(result).toBe(envValue); - expect(environmentDriver.get).toHaveBeenCalledWith(key); - expect(configCache.get).toHaveBeenCalledWith(key); - expect(configCache.isKeyKnownMissing).toHaveBeenCalledWith(key); - }); - - it('should propagate cache service errors when no fallback conditions are met', async () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - - jest.spyOn(configCache, 'get').mockImplementation(() => { - throw new Error('Cache error'); - }); - - expect(() => driver.get(key)).toThrow('Cache error'); - expect(configCache.get).toHaveBeenCalledWith(key); - expect(environmentDriver.get).not.toHaveBeenCalled(); - }); - - it('should propagate environment driver errors when using environment driver', async () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - - (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(true); - jest.spyOn(environmentDriver, 'get').mockImplementation(() => { - throw new Error('Environment error'); - }); - - expect(() => driver.get(key)).toThrow('Environment error'); - expect(environmentDriver.get).toHaveBeenCalledWith(key); - }); - }); - - describe('background operations', () => { - it('should schedule refresh in background', async () => { - jest.useFakeTimers(); - - const fetchPromise = Promise.resolve(); - const fetchSpy = jest - .spyOn(driver, 'fetchAndCacheConfigVariable') - .mockReturnValue(fetchPromise); - - jest.spyOn(configStorage, 'loadAll').mockResolvedValue(new Map()); - await driver.initialize(); - - const key = 'MISSING_KEY' as keyof ConfigVariables; - - jest - .spyOn(configCache, 'get') - .mockReturnValue({ value: undefined, isStale: false }); - jest.spyOn(configCache, 'isKeyKnownMissing').mockReturnValue(false); - - driver.get(key); - - jest.runAllTimers(); - - await fetchPromise; - - expect(fetchSpy).toHaveBeenCalledWith(key); - - jest.useRealTimers(); - }); - - it('should not schedule multiple refreshes for the same key', async () => { - const originalScheduleRefresh = driver['scheduleRefresh']; - const refreshSpyFn = jest.fn(); - - const refreshingKeys = new Set(); - - driver['scheduleRefresh'] = jest.fn().mockImplementation(async (key) => { - if (refreshingKeys.has(key)) { - return; - } - - refreshingKeys.add(key); - refreshSpyFn(key); - - refreshingKeys.delete(key); - }); - - const key = 'MISSING_KEY' as keyof ConfigVariables; - - // Call multiple times in sequence - await driver['scheduleRefresh'](key); - await driver['scheduleRefresh'](key); - await driver['scheduleRefresh'](key); - - // refreshSpyFn should be called three times because we're not keeping - // the key in refreshingKeys across calls (our mock clears it immediately) - expect(refreshSpyFn).toHaveBeenCalledTimes(3); - - refreshSpyFn.mockClear(); - - let isRefreshing = false; - - driver['scheduleRefresh'] = jest.fn().mockImplementation(async (key) => { - if (isRefreshing) { - return; - } - - isRefreshing = true; - refreshSpyFn(key); - }); - - // Make concurrent calls - const promise1 = driver['scheduleRefresh'](key); - const promise2 = driver['scheduleRefresh'](key); - const promise3 = driver['scheduleRefresh'](key); - - await Promise.all([promise1, promise2, promise3]); - - expect(refreshSpyFn).toHaveBeenCalledTimes(1); - expect(refreshSpyFn).toHaveBeenCalledWith(key); + describe('refreshAllCache', () => { + it('should load all config values from DB', async () => { + const dbValues = new Map(); + dbValues.set(CONFIG_PASSWORD_KEY, true); + dbValues.set(CONFIG_EMAIL_KEY, 'test@example.com'); + + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(dbValues); - driver['scheduleRefresh'] = originalScheduleRefresh; - }); - - it('should handle refresh failures and clean up', async () => { - // Mock the internal scheduleRefresh implementation to directly test error handling - const originalScheduleRefresh = driver['scheduleRefresh']; - const error = new Error('Fetch error'); - - driver['scheduleRefresh'] = jest.fn().mockImplementation(async (key) => { - if (driver['refreshingKeys'].has(key)) { - return; - } - - driver['refreshingKeys'].add(key); - - try { - // Simulate a fetch error - throw error; - } catch (e) { - driver['refreshingKeys'].delete(key); - throw e; - } - }); - - const key = 'MISSING_KEY' as keyof ConfigVariables; + await driver.refreshAllCache(); - await expect(driver['scheduleRefresh'](key)).rejects.toThrow(error); - - expect(driver['refreshingKeys'].has(key)).toBe(false); - - driver['scheduleRefresh'] = originalScheduleRefresh; + expect(configStorage.loadAll).toHaveBeenCalled(); + expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, true); + expect(configCache.set).toHaveBeenCalledWith(CONFIG_EMAIL_KEY, 'test@example.com'); }); - it('should correctly populate cache during initialization', async () => { - const configVars = new Map< - keyof ConfigVariables, - ConfigVariables[keyof ConfigVariables] - >([ - ['AUTH_PASSWORD_ENABLED', true], - ['EMAIL_FROM_ADDRESS', 'test@example.com'], - ['NODE_PORT', 3000], - ]); + it('should not affect env-only variables when found in DB', async () => { + const dbValues = new Map(); + dbValues.set(CONFIG_PASSWORD_KEY, true); + dbValues.set(CONFIG_ENV_ONLY_KEY, 'env-value'); + + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(dbValues); - jest.spyOn(configStorage, 'loadAll').mockResolvedValue(configVars); + await driver.refreshAllCache(); - await driver.initialize(); - - expect(configCache.set).toHaveBeenCalledWith( - 'AUTH_PASSWORD_ENABLED', - true, - ); - expect(configCache.set).toHaveBeenCalledWith( - 'EMAIL_FROM_ADDRESS', - 'test@example.com', - ); - expect(configCache.set).toHaveBeenCalledWith('NODE_PORT', 3000); + expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, true); + + expect(configCache.set).not.toHaveBeenCalledWith(CONFIG_ENV_ONLY_KEY, 'env-value'); }); - }); - describe('refreshExpiredCache', () => { - beforeEach(() => { + it('should mark keys as missing when not found in DB', async () => { jest.spyOn(configStorage, 'loadAll').mockResolvedValue(new Map()); - jest.spyOn(configStorage, 'loadByKeys').mockResolvedValue(new Map()); - jest.spyOn(configCache, 'getExpiredKeys').mockReturnValue([]); - }); - - it('should do nothing when there are no expired keys', async () => { - jest.spyOn(configCache, 'getExpiredKeys').mockReturnValue([]); - await driver.refreshExpiredCache(); + await driver.refreshAllCache(); - expect(configStorage.loadByKeys).not.toHaveBeenCalled(); + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(CONFIG_EMAIL_KEY); + + expect(configCache.markKeyAsMissing).not.toHaveBeenCalledWith(CONFIG_ENV_ONLY_KEY); }); - it('should filter out env-only variables from expired keys', async () => { - const expiredKeys = [ - 'AUTH_PASSWORD_ENABLED', - 'ENV_ONLY_VAR', - 'EMAIL_FROM_ADDRESS', - ] as Array; + it('should properly handle mix of found and missing keys', async () => { + const dbValues = new Map(); + dbValues.set(CONFIG_PASSWORD_KEY, true); - jest.spyOn(configCache, 'getExpiredKeys').mockReturnValue(expiredKeys); + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(dbValues); - // Simulate one env-only var - (isEnvOnlyConfigVar as jest.Mock).mockImplementation((key) => { - return key === 'ENV_ONLY_VAR'; - }); + await driver.refreshAllCache(); - await driver.refreshExpiredCache(); - - expect(configStorage.loadByKeys).toHaveBeenCalledWith([ - 'AUTH_PASSWORD_ENABLED', - 'EMAIL_FROM_ADDRESS', - ]); - }); - - it('should mark all keys as missing when no values are found', async () => { - const expiredKeys = [ - 'AUTH_PASSWORD_ENABLED', - 'EMAIL_FROM_ADDRESS', - ] as Array; - - jest.spyOn(configCache, 'getExpiredKeys').mockReturnValue(expiredKeys); - jest.spyOn(configStorage, 'loadByKeys').mockResolvedValue(new Map()); - - await driver.refreshExpiredCache(); - - expect(configCache.markKeyAsMissing).toHaveBeenCalledTimes(2); - expect(configCache.markKeyAsMissing).toHaveBeenCalledWith( - 'AUTH_PASSWORD_ENABLED', - ); - expect(configCache.markKeyAsMissing).toHaveBeenCalledWith( - 'EMAIL_FROM_ADDRESS', - ); - }); - - it('should update cache with refreshed values', async () => { - const expiredKeys = [ - 'AUTH_PASSWORD_ENABLED', - 'EMAIL_FROM_ADDRESS', - 'MISSING_KEY', - ] as Array; - - const refreshedValues = new Map< - keyof ConfigVariables, - ConfigVariables[keyof ConfigVariables] - >([ - ['AUTH_PASSWORD_ENABLED', true], - ['EMAIL_FROM_ADDRESS', 'test@example.com'], - ]); - - jest.spyOn(configCache, 'getExpiredKeys').mockReturnValue(expiredKeys); - jest - .spyOn(configStorage, 'loadByKeys') - .mockResolvedValue(refreshedValues); - - await driver.refreshExpiredCache(); - - expect(configCache.set).toHaveBeenCalledWith( - 'AUTH_PASSWORD_ENABLED', - true, - ); - expect(configCache.set).toHaveBeenCalledWith( - 'EMAIL_FROM_ADDRESS', - 'test@example.com', - ); - - expect(configCache.markKeyAsMissing).toHaveBeenCalledWith('MISSING_KEY'); + expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, true); + + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(CONFIG_EMAIL_KEY); }); it('should handle errors gracefully', async () => { const error = new Error('Database error'); - jest - .spyOn(configCache, 'getExpiredKeys') - .mockReturnValue(['AUTH_PASSWORD_ENABLED'] as Array< - keyof ConfigVariables - >); - jest.spyOn(configStorage, 'loadByKeys').mockRejectedValue(error); - jest.spyOn(driver['logger'], 'error'); + jest.spyOn(configStorage, 'loadAll').mockRejectedValue(error); + jest.spyOn(driver['logger'], 'error').mockImplementation(); - await driver.refreshExpiredCache(); + await driver.refreshAllCache(); expect(driver['logger'].error).toHaveBeenCalled(); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index 8640eee97b3f..ffbaf6a19bbb 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -5,7 +5,7 @@ import { DatabaseConfigDriverInterface } from 'src/engine/core-modules/twenty-co import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; -import { CONFIG_VARIABLES_REFRESH_INTERVAL_MINUTES } from 'src/engine/core-modules/twenty-config/constants/config-variables-refresh-interval'; +import { CONFIG_VARIABLES_REFRESH_CRON_INTERVAL } from 'src/engine/core-modules/twenty-config/constants/config-variables-refresh-cron-interval.constants'; import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; @@ -15,32 +15,42 @@ import { EnvironmentConfigDriver } from './environment-config.driver'; export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { private initializationPromise: Promise | null = null; private readonly logger = new Logger(DatabaseConfigDriver.name); - private readonly refreshingKeys = new Set(); + private readonly allPossibleConfigKeys: Array; constructor( private readonly configCache: ConfigCacheService, private readonly configStorage: ConfigStorageService, private readonly environmentDriver: EnvironmentConfigDriver, - ) {} + ) { + const allKeys = Object.keys(new ConfigVariables()) as Array< + keyof ConfigVariables + >; + + this.allPossibleConfigKeys = allKeys.filter( + (key) => !isEnvOnlyConfigVar(key), + ); + } async initialize(): Promise { if (this.initializationPromise) { - this.logger.verbose('Config database initialization already in progress'); + this.logger.verbose( + '[INIT] Config database initialization already in progress', + ); return this.initializationPromise; } - this.logger.debug('Starting database config initialization'); + this.logger.debug('[INIT] Starting database config initialization'); const promise = this.loadAllConfigVarsFromDb() .then((loadedCount) => { this.logger.log( - `Database config ready: loaded ${loadedCount} variables`, + `[INIT] Database config ready: loaded ${loadedCount} variables`, ); }) .catch((error) => { this.logger.error( - 'Failed to load database config: unable to connect to database or fetch config', + '[INIT] Failed to load database config: unable to connect to database or fetch config', error instanceof Error ? error.stack : error, ); throw error; @@ -59,13 +69,9 @@ export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { return this.environmentDriver.get(key); } - const { value, isStale } = this.configCache.get(key); + const value = this.configCache.get(key); if (value !== undefined) { - if (isStale) { - this.scheduleRefresh(key); - } - return value; } @@ -73,8 +79,6 @@ export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { return this.environmentDriver.get(key); } - this.scheduleRefresh(key); - return this.environmentDriver.get(key); } @@ -91,9 +95,12 @@ export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { try { await this.configStorage.set(key, value); this.configCache.set(key, value); - this.logger.debug(`Updated config variable: ${key as string}`); + this.logger.debug(`[UPDATE] Updated config variable: ${key as string}`); } catch (error) { - this.logger.error(`Failed to update config for ${key as string}`, error); + this.logger.error( + `[UPDATE] Failed to update config for ${key as string}`, + error, + ); throw error; } } @@ -105,16 +112,19 @@ export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { if (value !== undefined) { this.configCache.set(key, value); this.logger.debug( - `Refreshed config variable in cache: ${key as string}`, + `[FETCH] Refreshed config variable in cache: ${key as string}`, ); } else { this.configCache.markKeyAsMissing(key); this.logger.debug( - `Marked config variable as missing: ${key as string}`, + `[FETCH] Marked config variable as missing: ${key as string}`, ); } } catch (error) { - this.logger.error(`Failed to fetch config for ${key as string}`, error); + this.logger.error( + `[FETCH] Failed to fetch config for ${key as string}`, + error, + ); this.configCache.markKeyAsMissing(key); } } @@ -129,103 +139,76 @@ export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { private async loadAllConfigVarsFromDb(): Promise { try { - this.logger.debug('Fetching all config variables from database'); + this.logger.debug('[INIT] Fetching all config variables from database'); const configVars = await this.configStorage.loadAll(); + this.logger.debug( + `[INIT] Checking ${this.allPossibleConfigKeys.length} possible config variables`, + ); + for (const [key, value] of configVars.entries()) { this.configCache.set(key, value); } + for (const key of this.allPossibleConfigKeys) { + if (!configVars.has(key)) { + this.configCache.markKeyAsMissing(key); + } + } + + const missingKeysCount = + this.allPossibleConfigKeys.length - configVars.size; + + this.logger.debug( + `[INIT] Initial cache state: ${configVars.size} found values, ${missingKeysCount} missing values`, + ); + return configVars.size; } catch (error) { - this.logger.error('Failed to load config variables from database', error); + this.logger.error( + '[INIT] Failed to load config variables from database', + error, + ); throw error; } } - private async scheduleRefresh(key: keyof ConfigVariables): Promise { - if (this.refreshingKeys.has(key)) { - this.logger.debug(`Refresh for key ${key as string} already in progress`); - - return; - } - - this.refreshingKeys.add(key); - - setImmediate(async () => { - try { - await this.fetchAndCacheConfigVariable(key); - this.logger.debug(`Completed refresh for key ${key as string}`); - } catch (error) { - this.logger.error(`Failed to fetch config for ${key as string}`, error); - } finally { - this.refreshingKeys.delete(key); - } - }); - } - /** - * Proactively refreshes expired entries in the config cache. - * This method runs on a schedule and fetches all expired keys in one database query, + * Refreshes all database-backed config variables. + * This method runs on a schedule and fetches all configs in one database query, * then updates the cache with fresh values. */ - @Cron(`0 */${CONFIG_VARIABLES_REFRESH_INTERVAL_MINUTES} * * * *`) - async refreshExpiredCache(): Promise { + @Cron(CONFIG_VARIABLES_REFRESH_CRON_INTERVAL) + async refreshAllCache(): Promise { try { - this.logger.log('Starting proactive refresh of expired config variables'); - - const expiredKeys = this.configCache.getExpiredKeys(); + this.logger.log('[REFRESH] Starting refresh of all config variables'); - if (expiredKeys.length === 0) { - this.logger.debug('No expired config variables to refresh'); - - return; - } + const dbValues = await this.configStorage.loadAll(); this.logger.debug( - `Found ${expiredKeys.length} expired config variables to refresh`, + `[REFRESH] Checking ${this.allPossibleConfigKeys.length} possible config variables`, ); - const refreshableKeys = expiredKeys.filter( - (key) => !isEnvOnlyConfigVar(key), - ) as Array; - - if (refreshableKeys.length === 0) { - this.logger.debug( - 'No refreshable config variables found (all are env-only)', - ); - - return; - } - - const refreshedValues = - await this.configStorage.loadByKeys(refreshableKeys); - - if (refreshedValues.size === 0) { - this.logger.debug('No values found for expired keys'); - - for (const key of refreshableKeys) { - this.configCache.markKeyAsMissing(key); + for (const [key, value] of dbValues.entries()) { + if (!isEnvOnlyConfigVar(key)) { + this.configCache.set(key, value); } - - return; } - for (const [key, value] of refreshedValues.entries()) { - this.configCache.set(key, value); - } - - for (const key of refreshableKeys) { - if (!refreshedValues.has(key)) { + for (const key of this.allPossibleConfigKeys) { + if (!dbValues.has(key)) { this.configCache.markKeyAsMissing(key); } } + const missingKeysCount = + this.allPossibleConfigKeys.length - dbValues.size; + this.logger.log( - `Refreshed ${refreshedValues.size} config variables in cache`, + `[REFRESH] Refreshed config cache: ${dbValues.size} found values, ${missingKeysCount} missing values`, ); } catch (error) { - this.logger.error('Failed to refresh expired config variables', error); + this.logger.error('[REFRESH] Failed to refresh config variables', error); } } } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts index e440f62543fc..f2c895c489bb 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts @@ -29,7 +29,7 @@ export interface DatabaseConfigDriverInterface { fetchAndCacheConfigVariable(key: keyof ConfigVariables): Promise; /** - * Proactively refreshes expired entries in the config cache + * Refreshes all entries in the config cache */ - refreshExpiredCache(): Promise; + refreshAllCache(): Promise; } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts index 030e7ea0feb9..e0c45a920fa5 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/__tests__/config-storage.service.spec.ts @@ -1,7 +1,7 @@ import { Test, TestingModule } from '@nestjs/testing'; import { getRepositoryToken } from '@nestjs/typeorm'; -import { DeleteResult, IsNull, Repository, In } from 'typeorm'; +import { DeleteResult, IsNull, Repository } from 'typeorm'; import { KeyValuePair, @@ -485,85 +485,4 @@ describe('ConfigStorageService', () => { }); }); }); - - describe('loadByKeys', () => { - it('should load and convert specified config variables', async () => { - const keys = ['AUTH_PASSWORD_ENABLED', 'EMAIL_FROM_ADDRESS'] as Array< - keyof ConfigVariables - >; - const configVars: KeyValuePair[] = [ - createMockKeyValuePair('AUTH_PASSWORD_ENABLED', 'true'), - createMockKeyValuePair('EMAIL_FROM_ADDRESS', 'test@example.com'), - ]; - - jest.spyOn(keyValuePairRepository, 'find').mockResolvedValue(configVars); - - ( - configValueConverter.convertDbValueToAppValue as jest.Mock - ).mockImplementation((value, key) => { - if (key === 'AUTH_PASSWORD_ENABLED') return true; - if (key === 'EMAIL_FROM_ADDRESS') return 'test@example.com'; - - return value; - }); - - const result = await service.loadByKeys(keys); - - expect(result.size).toBe(2); - expect(result.get('AUTH_PASSWORD_ENABLED' as keyof ConfigVariables)).toBe( - true, - ); - expect(result.get('EMAIL_FROM_ADDRESS' as keyof ConfigVariables)).toBe( - 'test@example.com', - ); - expect(keyValuePairRepository.find).toHaveBeenCalledWith({ - where: { - type: KeyValuePairType.CONFIG_VARIABLE, - key: In(keys), - userId: IsNull(), - workspaceId: IsNull(), - }, - }); - }); - - it('should return empty map when no keys are provided', async () => { - const keys: Array = []; - - const result = await service.loadByKeys(keys); - - expect(result.size).toBe(0); - expect(keyValuePairRepository.find).not.toHaveBeenCalled(); - }); - - it('should handle conversion errors and skip problematic entries', async () => { - const keys = ['AUTH_PASSWORD_ENABLED', 'PROBLEMATIC_KEY'] as Array< - keyof ConfigVariables - >; - const configVars: KeyValuePair[] = [ - createMockKeyValuePair('AUTH_PASSWORD_ENABLED', 'true'), - createMockKeyValuePair('PROBLEMATIC_KEY', 'bad-value'), - ]; - - jest.spyOn(keyValuePairRepository, 'find').mockResolvedValue(configVars); - - ( - configValueConverter.convertDbValueToAppValue as jest.Mock - ).mockImplementation((value, key) => { - if (key === 'AUTH_PASSWORD_ENABLED') return true; - if (key === 'PROBLEMATIC_KEY') throw new Error('Conversion error'); - - return value; - }); - - const result = await service.loadByKeys(keys); - - expect(result.size).toBe(1); - expect(result.get('AUTH_PASSWORD_ENABLED' as keyof ConfigVariables)).toBe( - true, - ); - expect(result.has('PROBLEMATIC_KEY' as keyof ConfigVariables)).toBe( - false, - ); - }); - }); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts index 2e52ac22662c..0254d9f17f20 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/storage/config-storage.service.ts @@ -1,7 +1,7 @@ import { Injectable, Logger } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; -import { FindOptionsWhere, In, IsNull, Repository } from 'typeorm'; +import { FindOptionsWhere, IsNull, Repository } from 'typeorm'; import { KeyValuePair, @@ -23,20 +23,11 @@ export class ConfigStorageService implements ConfigStorageInterface { ) {} private getConfigVariableWhereClause( - key?: string | string[], + key?: string, ): FindOptionsWhere { - if (Array.isArray(key) && key.length > 0) { - return { - type: KeyValuePairType.CONFIG_VARIABLE, - key: In(key), - userId: IsNull(), - workspaceId: IsNull(), - }; - } - return { type: KeyValuePairType.CONFIG_VARIABLE, - ...(key && !Array.isArray(key) ? { key } : {}), + ...(key ? { key } : {}), userId: IsNull(), workspaceId: IsNull(), }; @@ -171,48 +162,4 @@ export class ConfigStorageService implements ConfigStorageInterface { throw error; } } - - async loadByKeys( - keys: T[], - ): Promise> { - try { - if (keys.length === 0) { - return new Map(); - } - - const configVars = await this.keyValuePairRepository.find({ - where: this.getConfigVariableWhereClause(keys as string[]), - }); - - const result = new Map(); - - for (const configVar of configVars) { - if (configVar.value !== null) { - const key = configVar.key as T; - - try { - const value = this.configValueConverter.convertDbValueToAppValue( - configVar.value, - key, - ); - - if (value !== undefined) { - result.set(key, value); - } - } catch (error) { - this.logger.error( - `Failed to convert value to app type for key ${key as string}`, - error, - ); - continue; - } - } - } - - return result; - } catch (error) { - this.logger.error('Failed to load config variables by keys', error); - throw error; - } - } } From 61aae916b6aa141347b001145984074304413d3a Mon Sep 17 00:00:00 2001 From: ehconitin Date: Thu, 24 Apr 2025 16:06:16 +0530 Subject: [PATCH 54/70] lint --- .../__tests__/config-cache.service.spec.ts | 3 +- .../__tests__/database-config.driver.spec.ts | 86 +++++++++++++------ 2 files changed, 61 insertions(+), 28 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts index b6f6355cf424..63981c0a675e 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts @@ -178,6 +178,7 @@ describe('ConfigCacheService', () => { it('should return empty array when no keys exist', () => { const allKeys = service.getAllKeys(); + expect(allKeys).toHaveLength(0); }); @@ -186,7 +187,7 @@ describe('ConfigCacheService', () => { // First add to positive cache service.set(key, 'value'); - + // Then force it into negative cache (normally this would remove from positive) // We're bypassing normal behavior for testing edge cases service['knownMissingKeysCache'].add(key); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index c4398d82c002..6c9099a2c240 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -16,7 +16,7 @@ jest.mock( const CONFIG_PASSWORD_KEY = 'AUTH_PASSWORD_ENABLED'; const CONFIG_EMAIL_KEY = 'EMAIL_FROM_ADDRESS'; -const CONFIG_ENV_ONLY_KEY = 'ENV_ONLY_VAR'; +const CONFIG_ENV_ONLY_KEY = 'ENV_ONLY_VAR'; const CONFIG_PORT_KEY = 'NODE_PORT'; class TestDatabaseConfigDriver extends DatabaseConfigDriver { @@ -24,7 +24,7 @@ class TestDatabaseConfigDriver extends DatabaseConfigDriver { public get testAllPossibleConfigKeys(): Array { return this['allPossibleConfigKeys']; } - + // Override Object.keys usage in constructor with our test keys constructor( configCache: ConfigCacheService, @@ -32,8 +32,7 @@ class TestDatabaseConfigDriver extends DatabaseConfigDriver { environmentDriver: EnvironmentConfigDriver, ) { super(configCache, configStorage, environmentDriver); - - + Object.defineProperty(this, 'allPossibleConfigKeys', { value: [CONFIG_PASSWORD_KEY, CONFIG_EMAIL_KEY, CONFIG_PORT_KEY], writable: false, @@ -52,7 +51,7 @@ describe('DatabaseConfigDriver', () => { (isEnvOnlyConfigVar as jest.Mock).mockImplementation((key) => { return key === CONFIG_ENV_ONLY_KEY; }); - + const module: TestingModule = await Test.createTestingModule({ providers: [ { @@ -89,7 +88,9 @@ describe('DatabaseConfigDriver', () => { ], }).compile(); - driver = module.get(DatabaseConfigDriver) as TestDatabaseConfigDriver; + driver = module.get( + DatabaseConfigDriver, + ) as TestDatabaseConfigDriver; configCache = module.get(ConfigCacheService); configStorage = module.get(ConfigStorageService); environmentDriver = module.get( @@ -112,11 +113,14 @@ describe('DatabaseConfigDriver', () => { it('should have allPossibleConfigKeys properly set', () => { expect(driver.testAllPossibleConfigKeys).toContain(CONFIG_PASSWORD_KEY); expect(driver.testAllPossibleConfigKeys).toContain(CONFIG_EMAIL_KEY); - expect(driver.testAllPossibleConfigKeys).not.toContain(CONFIG_ENV_ONLY_KEY); + expect(driver.testAllPossibleConfigKeys).not.toContain( + CONFIG_ENV_ONLY_KEY, + ); }); - + it('should initialize successfully with DB values and mark missing keys', async () => { const configVars = new Map(); + configVars.set(CONFIG_PASSWORD_KEY, true); jest.spyOn(configStorage, 'loadAll').mockResolvedValue(configVars); @@ -124,12 +128,16 @@ describe('DatabaseConfigDriver', () => { await driver.initialize(); expect(configStorage.loadAll).toHaveBeenCalled(); - + expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, true); - - expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(CONFIG_EMAIL_KEY); - - expect(configCache.markKeyAsMissing).not.toHaveBeenCalledWith(CONFIG_ENV_ONLY_KEY); + + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith( + CONFIG_EMAIL_KEY, + ); + + expect(configCache.markKeyAsMissing).not.toHaveBeenCalledWith( + CONFIG_ENV_ONLY_KEY, + ); }); it('should handle initialization failure', async () => { @@ -250,7 +258,10 @@ describe('DatabaseConfigDriver', () => { await driver.update(CONFIG_PASSWORD_KEY, value); - expect(configStorage.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, value); + expect(configStorage.set).toHaveBeenCalledWith( + CONFIG_PASSWORD_KEY, + value, + ); expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, value); }); @@ -281,7 +292,9 @@ describe('DatabaseConfigDriver', () => { await driver.fetchAndCacheConfigVariable(CONFIG_PASSWORD_KEY); expect(configStorage.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); - expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith( + CONFIG_PASSWORD_KEY, + ); expect(configCache.set).not.toHaveBeenCalled(); }); @@ -296,7 +309,9 @@ describe('DatabaseConfigDriver', () => { await driver.fetchAndCacheConfigVariable(CONFIG_PASSWORD_KEY); expect(configStorage.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); - expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith( + CONFIG_PASSWORD_KEY, + ); expect(loggerSpy).toHaveBeenCalledWith( expect.stringContaining('Failed to fetch config'), error, @@ -323,30 +338,38 @@ describe('DatabaseConfigDriver', () => { describe('refreshAllCache', () => { it('should load all config values from DB', async () => { const dbValues = new Map(); + dbValues.set(CONFIG_PASSWORD_KEY, true); dbValues.set(CONFIG_EMAIL_KEY, 'test@example.com'); - + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(dbValues); await driver.refreshAllCache(); expect(configStorage.loadAll).toHaveBeenCalled(); expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, true); - expect(configCache.set).toHaveBeenCalledWith(CONFIG_EMAIL_KEY, 'test@example.com'); + expect(configCache.set).toHaveBeenCalledWith( + CONFIG_EMAIL_KEY, + 'test@example.com', + ); }); it('should not affect env-only variables when found in DB', async () => { const dbValues = new Map(); + dbValues.set(CONFIG_PASSWORD_KEY, true); dbValues.set(CONFIG_ENV_ONLY_KEY, 'env-value'); - + jest.spyOn(configStorage, 'loadAll').mockResolvedValue(dbValues); await driver.refreshAllCache(); expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, true); - - expect(configCache.set).not.toHaveBeenCalledWith(CONFIG_ENV_ONLY_KEY, 'env-value'); + + expect(configCache.set).not.toHaveBeenCalledWith( + CONFIG_ENV_ONLY_KEY, + 'env-value', + ); }); it('should mark keys as missing when not found in DB', async () => { @@ -354,14 +377,21 @@ describe('DatabaseConfigDriver', () => { await driver.refreshAllCache(); - expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); - expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(CONFIG_EMAIL_KEY); - - expect(configCache.markKeyAsMissing).not.toHaveBeenCalledWith(CONFIG_ENV_ONLY_KEY); + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith( + CONFIG_PASSWORD_KEY, + ); + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith( + CONFIG_EMAIL_KEY, + ); + + expect(configCache.markKeyAsMissing).not.toHaveBeenCalledWith( + CONFIG_ENV_ONLY_KEY, + ); }); it('should properly handle mix of found and missing keys', async () => { const dbValues = new Map(); + dbValues.set(CONFIG_PASSWORD_KEY, true); jest.spyOn(configStorage, 'loadAll').mockResolvedValue(dbValues); @@ -369,8 +399,10 @@ describe('DatabaseConfigDriver', () => { await driver.refreshAllCache(); expect(configCache.set).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY, true); - - expect(configCache.markKeyAsMissing).toHaveBeenCalledWith(CONFIG_EMAIL_KEY); + + expect(configCache.markKeyAsMissing).toHaveBeenCalledWith( + CONFIG_EMAIL_KEY, + ); }); it('should handle errors gracefully', async () => { From 0393cfc72febf8851b3262c8c56f818ea69507f9 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Fri, 25 Apr 2025 01:15:57 +0530 Subject: [PATCH 55/70] refactor --- .../__tests__/config-cache.service.spec.ts | 25 +++++++++ .../cache/config-cache.service.ts | 13 +++++ .../__tests__/database-config.driver.spec.ts | 51 ++--------------- .../drivers/database-config.driver.ts | 21 +------ .../database-config-driver.interface.ts | 5 +- .../twenty-config.service.spec.ts | 55 +++++++++++++++++-- .../twenty-config/twenty-config.service.ts | 21 ++++++- 7 files changed, 118 insertions(+), 73 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts index 63981c0a675e..5ec394b6b6d4 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts @@ -61,6 +61,31 @@ describe('ConfigCacheService', () => { }); }); + describe('getOrFallback', () => { + it('should return cached value when available', () => { + const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; + const value = true; + const fallbackFn = jest.fn().mockReturnValue(false); + + service.set(key, value); + const result = service.getOrFallback(key, fallbackFn); + + expect(result).toBe(value); + expect(fallbackFn).not.toHaveBeenCalled(); + }); + + it('should call fallback function when value not in cache', () => { + const key = 'NON_EXISTENT_KEY' as keyof ConfigVariables; + const fallbackValue = 'fallback value'; + const fallbackFn = jest.fn().mockReturnValue(fallbackValue); + + const result = service.getOrFallback(key, fallbackFn); + + expect(result).toBe(fallbackValue); + expect(fallbackFn).toHaveBeenCalledTimes(1); + }); + }); + describe('negative lookup cache', () => { it('should check if a negative cache entry exists', () => { const key = 'TEST_KEY' as keyof ConfigVariables; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts index 8b539569c5c1..eb60ee21c516 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts @@ -30,6 +30,19 @@ export class ConfigCacheService implements OnModuleDestroy { return entry.value as ConfigValue; } + getOrFallback( + key: T, + fallbackFn: () => R, + ): ConfigValue | R { + const value = this.get(key); + + if (value !== undefined) { + return value; + } + + return fallbackFn(); + } + isKeyKnownMissing(key: ConfigKey): boolean { return this.knownMissingKeysCache.has(key); } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index 6c9099a2c240..8dec7cbb9152 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -3,7 +3,6 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; -import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; @@ -29,9 +28,8 @@ class TestDatabaseConfigDriver extends DatabaseConfigDriver { constructor( configCache: ConfigCacheService, configStorage: ConfigStorageService, - environmentDriver: EnvironmentConfigDriver, ) { - super(configCache, configStorage, environmentDriver); + super(configCache, configStorage); Object.defineProperty(this, 'allPossibleConfigKeys', { value: [CONFIG_PASSWORD_KEY, CONFIG_EMAIL_KEY, CONFIG_PORT_KEY], @@ -45,7 +43,6 @@ describe('DatabaseConfigDriver', () => { let driver: TestDatabaseConfigDriver; let configCache: ConfigCacheService; let configStorage: ConfigStorageService; - let environmentDriver: EnvironmentConfigDriver; beforeEach(async () => { (isEnvOnlyConfigVar as jest.Mock).mockImplementation((key) => { @@ -69,6 +66,7 @@ describe('DatabaseConfigDriver', () => { markKeyAsMissing: jest.fn(), getCacheInfo: jest.fn(), getAllKeys: jest.fn(), + getOrFallback: jest.fn(), }, }, { @@ -79,12 +77,6 @@ describe('DatabaseConfigDriver', () => { loadAll: jest.fn(), }, }, - { - provide: EnvironmentConfigDriver, - useValue: { - get: jest.fn(), - }, - }, ], }).compile(); @@ -93,9 +85,6 @@ describe('DatabaseConfigDriver', () => { ) as TestDatabaseConfigDriver; configCache = module.get(ConfigCacheService); configStorage = module.get(ConfigStorageService); - environmentDriver = module.get( - EnvironmentConfigDriver, - ); jest.clearAllMocks(); }); @@ -172,19 +161,6 @@ describe('DatabaseConfigDriver', () => { await driver.initialize(); }); - it('should use environment driver for env-only variables', async () => { - const envValue = true; - - (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(true); - jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); - - const result = driver.get(CONFIG_PASSWORD_KEY); - - expect(result).toBe(envValue); - expect(environmentDriver.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); - expect(isEnvOnlyConfigVar).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); - }); - it('should return cached value when available', async () => { const cachedValue = true; @@ -196,30 +172,13 @@ describe('DatabaseConfigDriver', () => { expect(configCache.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); }); - it('should use environment driver when negative lookup is cached', async () => { - const envValue = true; - - jest.spyOn(configCache, 'get').mockReturnValue(undefined); - jest.spyOn(configCache, 'isKeyKnownMissing').mockReturnValue(true); - jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); - - const result = driver.get(CONFIG_PASSWORD_KEY); - - expect(result).toBe(envValue); - expect(environmentDriver.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); - }); - - it('should fallback to environment when not in cache and not known missing', async () => { - const envValue = true; - + it('should return undefined when value is not in cache', async () => { jest.spyOn(configCache, 'get').mockReturnValue(undefined); - jest.spyOn(configCache, 'isKeyKnownMissing').mockReturnValue(false); - jest.spyOn(environmentDriver, 'get').mockReturnValue(envValue); const result = driver.get(CONFIG_PASSWORD_KEY); - expect(result).toBe(envValue); - expect(environmentDriver.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); + expect(result).toBeUndefined(); + expect(configCache.get).toHaveBeenCalledWith(CONFIG_PASSWORD_KEY); }); it('should handle different config variable types correctly', () => { diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index ffbaf6a19bbb..b26dff174c8d 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -9,8 +9,6 @@ import { CONFIG_VARIABLES_REFRESH_CRON_INTERVAL } from 'src/engine/core-modules/ import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; -import { EnvironmentConfigDriver } from './environment-config.driver'; - @Injectable() export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { private initializationPromise: Promise | null = null; @@ -20,7 +18,6 @@ export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { constructor( private readonly configCache: ConfigCacheService, private readonly configStorage: ConfigStorageService, - private readonly environmentDriver: EnvironmentConfigDriver, ) { const allKeys = Object.keys(new ConfigVariables()) as Array< keyof ConfigVariables @@ -64,22 +61,8 @@ export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { return promise; } - get(key: T): ConfigVariables[T] { - if (isEnvOnlyConfigVar(key)) { - return this.environmentDriver.get(key); - } - - const value = this.configCache.get(key); - - if (value !== undefined) { - return value; - } - - if (this.configCache.isKeyKnownMissing(key)) { - return this.environmentDriver.get(key); - } - - return this.environmentDriver.get(key); + get(key: T): ConfigVariables[T] | undefined { + return this.configCache.get(key); } async update( diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts index f2c895c489bb..9e7cd5e1ce2a 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts @@ -11,9 +11,10 @@ export interface DatabaseConfigDriverInterface { initialize(): Promise; /** - * Get a configuration value + * Get a configuration value from cache + * Returns undefined if not in cache */ - get(key: T): ConfigVariables[T]; + get(key: T): ConfigVariables[T] | undefined; /** * Update a configuration value in the database and cache diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts index 3491b0390672..db5b18717699 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts @@ -8,6 +8,7 @@ import { ConfigInitializationState } from 'src/engine/core-modules/twenty-config import { ConfigSource } from 'src/engine/core-modules/twenty-config/enums/config-source.enum'; import { ConfigVariablesGroup } from 'src/engine/core-modules/twenty-config/enums/config-variables-group.enum'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; +import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; import { TypedReflect } from 'src/utils/typed-reflect'; jest.mock('src/utils/typed-reflect', () => ({ @@ -29,6 +30,13 @@ jest.mock( }), ); +jest.mock( + 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util', + () => ({ + isEnvOnlyConfigVar: jest.fn(), + }), +); + type TwentyConfigServicePrivateProps = { driver: DatabaseConfigDriver | EnvironmentConfigDriver; isConfigVarInDbEnabled: boolean; @@ -234,18 +242,57 @@ describe('TwentyConfigService', () => { }); describe('get', () => { - it('should delegate to the active driver', () => { - const key = 'TEST_VAR' as keyof ConfigVariables; - const expectedValue = 'test value'; + const key = 'TEST_VAR' as keyof ConfigVariables; + const expectedValue = 'test value'; - jest.spyOn(databaseConfigDriver, 'get').mockReturnValue(expectedValue); + beforeEach(() => { + (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(false); + }); + + it('should use environment driver for environment-only variables', () => { + (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(true); + jest.spyOn(environmentConfigDriver, 'get').mockReturnValue(expectedValue); + + const result = service.get(key); + + expect(result).toBe(expectedValue); + expect(environmentConfigDriver.get).toHaveBeenCalledWith(key); + }); + it('should use database driver when it is active and value is found', () => { + jest.spyOn(databaseConfigDriver, 'get').mockReturnValue(expectedValue); setPrivateProps(service, { driver: databaseConfigDriver }); const result = service.get(key); expect(result).toBe(expectedValue); expect(databaseConfigDriver.get).toHaveBeenCalledWith(key); + expect(environmentConfigDriver.get).not.toHaveBeenCalled(); + }); + + it('should fall back to environment driver when database driver is active but value is not found', () => { + const envValue = 'env value'; + + jest.spyOn(databaseConfigDriver, 'get').mockReturnValue(undefined); + jest.spyOn(environmentConfigDriver, 'get').mockReturnValue(envValue); + setPrivateProps(service, { driver: databaseConfigDriver }); + + const result = service.get(key); + + expect(result).toBe(envValue); + expect(databaseConfigDriver.get).toHaveBeenCalledWith(key); + expect(environmentConfigDriver.get).toHaveBeenCalledWith(key); + }); + + it('should use environment driver when it is active', () => { + jest.spyOn(environmentConfigDriver, 'get').mockReturnValue(expectedValue); + setPrivateProps(service, { driver: environmentConfigDriver }); + + const result = service.get(key); + + expect(result).toBe(expectedValue); + expect(environmentConfigDriver.get).toHaveBeenCalledWith(key); + expect(databaseConfigDriver.get).not.toHaveBeenCalled(); }); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts index 4c9cb2f208df..0e59b8838455 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts @@ -12,6 +12,7 @@ import { ConfigInitializationState } from 'src/engine/core-modules/twenty-config import { ConfigSource } from 'src/engine/core-modules/twenty-config/enums/config-source.enum'; import { ConfigVariablesMaskingStrategies } from 'src/engine/core-modules/twenty-config/enums/config-variables-masking-strategies.enum'; import { configVariableMaskSensitiveData } from 'src/engine/core-modules/twenty-config/utils/config-variable-mask-sensitive-data.util'; +import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; import { TypedReflect } from 'src/utils/typed-reflect'; @Injectable() @@ -69,9 +70,25 @@ export class TwentyConfigService implements OnModuleInit { } get(key: T): ConfigVariables[T] { - const value = this.driver.get(key); + // Environment-only variables always come from the environment driver + if (isEnvOnlyConfigVar(key)) { + return this.environmentConfigDriver.get(key); + } + + // If we're using the database driver, check it first then fall back to environment + if (this.driver === this.databaseConfigDriver) { + const dbValue = this.databaseConfigDriver.get(key); + + if (dbValue !== undefined) { + return dbValue; + } + + // Fall back to environment if not in database + return this.environmentConfigDriver.get(key); + } - return value; + // If we're not using the database driver, use the environment driver directly + return this.environmentConfigDriver.get(key); } async update( From c8fdaca9c68962db1fb8480ef0c8a25ee9997cc3 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Fri, 25 Apr 2025 01:36:40 +0530 Subject: [PATCH 56/70] converters improvements --- .../config-value-converter.service.ts | 87 +++++++++++++++---- .../utils/apply-basic-validators.util.ts | 29 ++++--- 2 files changed, 86 insertions(+), 30 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts index d445376fa85e..69f96e4c285e 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts @@ -1,4 +1,4 @@ -import { Injectable } from '@nestjs/common'; +import { Injectable, Logger } from '@nestjs/common'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; import { ConfigVariablesMetadataMap } from 'src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator'; @@ -9,6 +9,8 @@ import { TypedReflect } from 'src/utils/typed-reflect'; @Injectable() export class ConfigValueConverterService { + private readonly logger = new Logger(ConfigValueConverterService.name); + constructor(private readonly configVariables: ConfigVariables) {} convertDbValueToAppValue( @@ -25,14 +27,39 @@ export class ConfigValueConverterService { try { switch (configType) { - case 'boolean': - return configTransformers.boolean(dbValue) as ConfigVariables[T]; + case 'boolean': { + const result = configTransformers.boolean(dbValue); + + if (result === undefined) { + throw new Error( + `Value '${String(dbValue)}' cannot be converted to boolean`, + ); + } + + return result as ConfigVariables[T]; + } + + case 'number': { + const result = configTransformers.number(dbValue); + + if (result === undefined) { + throw new Error( + `Value '${String(dbValue)}' cannot be converted to number`, + ); + } + + return result as ConfigVariables[T]; + } + + case 'string': { + const result = configTransformers.string(dbValue); - case 'number': - return configTransformers.number(dbValue) as ConfigVariables[T]; + if (result === undefined) { + throw new Error(`Value cannot be converted to string`); + } - case 'string': - return configTransformers.string(dbValue) as ConfigVariables[T]; + return result as ConfigVariables[T]; + } case 'array': { const result = this.convertToArray(dbValue, options); @@ -43,31 +70,43 @@ export class ConfigValueConverterService { case 'enum': { const result = this.convertToEnum(dbValue, options); + if (result === undefined) { + this.logger.warn( + `Enum value '${String(dbValue)}' not found in options for key ${String(key)}`, + ); + } + return result as ConfigVariables[T]; } default: + this.logger.warn( + `No specific conversion for type '${configType}' of key '${String(key)}'`, + ); + return dbValue as ConfigVariables[T]; } } catch (error) { + const errorMessage = + error instanceof Error ? error.message : String(error); + throw new Error( - `Failed to convert ${key as string} to app value: ${(error as Error).message}`, + `Failed to convert value for key '${String(key)}': ${errorMessage}`, ); } } convertAppValueToDbValue( - appValue: ConfigVariables[T], + appValue: ConfigVariables[T] | null | undefined, ): unknown { - if (appValue === undefined) { + if (appValue === undefined || appValue === null) { return null; } if ( typeof appValue === 'string' || typeof appValue === 'number' || - typeof appValue === 'boolean' || - appValue === null + typeof appValue === 'boolean' ) { return appValue; } @@ -77,7 +116,13 @@ export class ConfigValueConverterService { } if (typeof appValue === 'object') { - return JSON.parse(JSON.stringify(appValue)); + try { + return JSON.parse(JSON.stringify(appValue)); + } catch (error) { + throw new Error( + `Failed to serialize object value: ${error instanceof Error ? error.message : String(error)}`, + ); + } } throw new Error( @@ -107,7 +152,7 @@ export class ConfigValueConverterService { } } - return []; + return this.validateArrayAgainstOptions([value], options); } private validateArrayAgainstOptions( @@ -118,13 +163,23 @@ export class ConfigValueConverterService { return array; } - return array.filter((item) => options.includes(item as string)); + return array.filter((item) => { + const included = options.includes(item as string); + + if (!included) { + this.logger.debug( + `Filtered out array item '${String(item)}' not in allowed options`, + ); + } + + return included; + }); } private convertToEnum( value: unknown, options?: ConfigVariableOptions, - ): unknown { + ): unknown | undefined { if (!options || !Array.isArray(options) || options.length === 0) { return value; } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts index 8f442d28d9fc..e517a4052f55 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts @@ -11,12 +11,6 @@ import { ConfigVariableOptions } from 'src/engine/core-modules/twenty-config/typ import { ConfigVariableType } from 'src/engine/core-modules/twenty-config/types/config-variable-type.type'; import { configTransformers } from 'src/engine/core-modules/twenty-config/utils/config-transformers.util'; -// TODO: Add support for custom validators -// Not sure about tailored custom validators for single variables -// Maybe just a single validate method in the decorator that accepts a list of validators -// and applies them to the variable -// also not sure about the file placement/naming -- should this be a util or a service? factory? - export function applyBasicValidators( type: ConfigVariableType, target: object, @@ -25,30 +19,37 @@ export function applyBasicValidators( ): void { switch (type) { case 'boolean': + Transform(({ value }) => { + const result = configTransformers.boolean(value); + + return result !== undefined ? result : value; + })(target, propertyKey); IsBoolean()(target, propertyKey); - Transform(({ value }) => configTransformers.boolean(value))( - target, - propertyKey, - ); break; + case 'number': + Transform(({ value }) => { + const result = configTransformers.number(value); + + return result !== undefined ? result : value; + })(target, propertyKey); IsNumber()(target, propertyKey); - Transform(({ value }) => configTransformers.number(value))( - target, - propertyKey, - ); break; + case 'string': IsString()(target, propertyKey); break; + case 'enum': if (options && Array.isArray(options)) { IsEnum(options)(target, propertyKey); } break; + case 'array': IsArray()(target, propertyKey); break; + default: throw new Error(`Unsupported config variable type: ${type}`); } From 2af30202f4234b216ed6b9ab6eae23d2376f28bc Mon Sep 17 00:00:00 2001 From: ehconitin Date: Fri, 25 Apr 2025 01:45:37 +0530 Subject: [PATCH 57/70] add safe guards on conversion --- .../config-value-converter.service.ts | 38 +++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts index 69f96e4c285e..ded35c5635e3 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts @@ -35,6 +35,11 @@ export class ConfigValueConverterService { `Value '${String(dbValue)}' cannot be converted to boolean`, ); } + if (typeof result !== 'boolean') { + throw new Error( + `Expected boolean for key ${key}, got ${typeof result}`, + ); + } return result as ConfigVariables[T]; } @@ -47,6 +52,11 @@ export class ConfigValueConverterService { `Value '${String(dbValue)}' cannot be converted to number`, ); } + if (typeof result !== 'number') { + throw new Error( + `Expected number for key ${key}, got ${typeof result}`, + ); + } return result as ConfigVariables[T]; } @@ -55,7 +65,14 @@ export class ConfigValueConverterService { const result = configTransformers.string(dbValue); if (result === undefined) { - throw new Error(`Value cannot be converted to string`); + throw new Error( + `Value '${String(dbValue)}' cannot be converted to string`, + ); + } + if (typeof result !== 'string') { + throw new Error( + `Expected string for key ${key}, got ${typeof result}`, + ); } return result as ConfigVariables[T]; @@ -64,6 +81,12 @@ export class ConfigValueConverterService { case 'array': { const result = this.convertToArray(dbValue, options); + if (!Array.isArray(result)) { + throw new Error( + `Expected array for key ${key}, got ${typeof result}`, + ); + } + return result as ConfigVariables[T]; } @@ -71,8 +94,8 @@ export class ConfigValueConverterService { const result = this.convertToEnum(dbValue, options); if (result === undefined) { - this.logger.warn( - `Enum value '${String(dbValue)}' not found in options for key ${String(key)}`, + throw new Error( + `Invalid enum value for key ${key}: ${String(dbValue)}`, ); } @@ -80,18 +103,11 @@ export class ConfigValueConverterService { } default: - this.logger.warn( - `No specific conversion for type '${configType}' of key '${String(key)}'`, - ); - return dbValue as ConfigVariables[T]; } } catch (error) { - const errorMessage = - error instanceof Error ? error.message : String(error); - throw new Error( - `Failed to convert value for key '${String(key)}': ${errorMessage}`, + `Failed to convert ${key as string} to app value: ${(error as Error).message}`, ); } } From 9f267e7a43d6b6d8bd131992a3a2f016ad06704b Mon Sep 17 00:00:00 2001 From: ehconitin Date: Fri, 25 Apr 2025 02:06:17 +0530 Subject: [PATCH 58/70] add tests and more improvements --- .../__tests__/config-cache.service.spec.ts | 25 -- .../cache/config-cache.service.ts | 13 - .../config-value-converter.service.spec.ts | 227 ++++++++++++++++++ .../config-value-converter.service.ts | 29 +-- .../__tests__/database-config.driver.spec.ts | 1 - .../twenty-config/twenty-config.service.ts | 10 +- .../apply-basic-validators.util.spec.ts | 148 ++++++++++++ .../config-transformers.util.spec.ts | 100 ++++++++ 8 files changed, 482 insertions(+), 71 deletions(-) create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/apply-basic-validators.util.spec.ts create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/config-transformers.util.spec.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts index 5ec394b6b6d4..63981c0a675e 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts @@ -61,31 +61,6 @@ describe('ConfigCacheService', () => { }); }); - describe('getOrFallback', () => { - it('should return cached value when available', () => { - const key = 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables; - const value = true; - const fallbackFn = jest.fn().mockReturnValue(false); - - service.set(key, value); - const result = service.getOrFallback(key, fallbackFn); - - expect(result).toBe(value); - expect(fallbackFn).not.toHaveBeenCalled(); - }); - - it('should call fallback function when value not in cache', () => { - const key = 'NON_EXISTENT_KEY' as keyof ConfigVariables; - const fallbackValue = 'fallback value'; - const fallbackFn = jest.fn().mockReturnValue(fallbackValue); - - const result = service.getOrFallback(key, fallbackFn); - - expect(result).toBe(fallbackValue); - expect(fallbackFn).toHaveBeenCalledTimes(1); - }); - }); - describe('negative lookup cache', () => { it('should check if a negative cache entry exists', () => { const key = 'TEST_KEY' as keyof ConfigVariables; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts index eb60ee21c516..8b539569c5c1 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts @@ -30,19 +30,6 @@ export class ConfigCacheService implements OnModuleDestroy { return entry.value as ConfigValue; } - getOrFallback( - key: T, - fallbackFn: () => R, - ): ConfigValue | R { - const value = this.get(key); - - if (value !== undefined) { - return value; - } - - return fallbackFn(); - } - isKeyKnownMissing(key: ConfigKey): boolean { return this.knownMissingKeysCache.has(key); } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.spec.ts index f8f992deb0de..8a167c5f3723 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.spec.ts @@ -3,10 +3,31 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; import { ConfigVariablesGroup } from 'src/engine/core-modules/twenty-config/enums/config-variables-group.enum'; +import { configTransformers } from 'src/engine/core-modules/twenty-config/utils/config-transformers.util'; import { TypedReflect } from 'src/utils/typed-reflect'; import { ConfigValueConverterService } from './config-value-converter.service'; +// Mock configTransformers for type validation tests +jest.mock( + 'src/engine/core-modules/twenty-config/utils/config-transformers.util', + () => { + const originalModule = jest.requireActual( + 'src/engine/core-modules/twenty-config/utils/config-transformers.util', + ); + + return { + configTransformers: { + ...originalModule.configTransformers, + // These mocked versions can be overridden in specific tests + _mockedBoolean: jest.fn(), + _mockedNumber: jest.fn(), + _mockedString: jest.fn(), + }, + }; + }, +); + describe('ConfigValueConverterService', () => { let service: ConfigValueConverterService; @@ -218,6 +239,189 @@ describe('ConfigValueConverterService', () => { ), ).toBe(42); }); + + it('should handle null and undefined values', () => { + expect( + service.convertDbValueToAppValue( + null, + 'NODE_PORT' as keyof ConfigVariables, + ), + ).toBeUndefined(); + + expect( + service.convertDbValueToAppValue( + undefined, + 'NODE_PORT' as keyof ConfigVariables, + ), + ).toBeUndefined(); + }); + + it('should throw error if boolean converter returns non-boolean', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + AUTH_PASSWORD_ENABLED: { + type: 'boolean', + group: ConfigVariablesGroup.Other, + description: 'Test boolean', + }, + }); + + const originalBoolean = configTransformers.boolean; + + configTransformers.boolean = jest + .fn() + .mockImplementation(() => 'not-a-boolean'); + + expect(() => { + service.convertDbValueToAppValue( + 'true', + 'AUTH_PASSWORD_ENABLED' as keyof ConfigVariables, + ); + }).toThrow(/Expected boolean for key AUTH_PASSWORD_ENABLED/); + + configTransformers.boolean = originalBoolean; + }); + + it('should throw error if number converter returns non-number', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + NODE_PORT: { + type: 'number', + group: ConfigVariablesGroup.ServerConfig, + description: 'Test number', + }, + }); + + const originalNumber = configTransformers.number; + + configTransformers.number = jest + .fn() + .mockImplementation(() => 'not-a-number'); + + expect(() => { + service.convertDbValueToAppValue( + '42', + 'NODE_PORT' as keyof ConfigVariables, + ); + }).toThrow(/Expected number for key NODE_PORT/); + + configTransformers.number = originalNumber; + }); + + it('should throw error if string converter returns non-string', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + EMAIL_FROM_ADDRESS: { + type: 'string', + group: ConfigVariablesGroup.EmailSettings, + description: 'Test string', + }, + }); + + const originalString = configTransformers.string; + + configTransformers.string = jest.fn().mockImplementation(() => 42); + + expect(() => { + service.convertDbValueToAppValue( + 'test@example.com', + 'EMAIL_FROM_ADDRESS' as keyof ConfigVariables, + ); + }).toThrow(/Expected string for key EMAIL_FROM_ADDRESS/); + + configTransformers.string = originalString; + }); + + it('should throw error if array conversion produces non-array', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + LOG_LEVELS: { + type: 'array', + group: ConfigVariablesGroup.Logging, + description: 'Test array', + }, + }); + + const convertToArraySpy = jest + .spyOn( + service as any, // Cast to any to access private method + 'convertToArray', + ) + .mockReturnValueOnce('not-an-array'); + + expect(() => { + service.convertDbValueToAppValue( + 'log,error,warn', + 'LOG_LEVELS' as keyof ConfigVariables, + ); + }).toThrow(/Expected array for key LOG_LEVELS/); + + convertToArraySpy.mockRestore(); + }); + + it('should handle array with option validation', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + LOG_LEVELS: { + type: 'array', + group: ConfigVariablesGroup.Logging, + description: 'Test array with options', + options: ['log', 'error', 'warn', 'debug'], + }, + }); + + expect( + service.convertDbValueToAppValue( + 'log,error,warn', + 'LOG_LEVELS' as keyof ConfigVariables, + ), + ).toEqual(['log', 'error', 'warn']); + + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + LOG_LEVELS: { + type: 'array', + group: ConfigVariablesGroup.Logging, + description: 'Test array with options', + options: ['log', 'error', 'warn', 'debug'], + }, + }); + + expect( + service.convertDbValueToAppValue( + 'log,invalid,warn', + 'LOG_LEVELS' as keyof ConfigVariables, + ), + ).toEqual(['log', 'warn']); + }); + + it('should properly handle enum with options', () => { + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + LOG_LEVEL: { + type: 'enum', + group: ConfigVariablesGroup.Logging, + description: 'Test enum', + options: ['log', 'error', 'warn', 'debug'], + }, + }); + + expect( + service.convertDbValueToAppValue( + 'error', + 'LOG_LEVEL' as keyof ConfigVariables, + ), + ).toBe('error'); + + jest.spyOn(TypedReflect, 'getMetadata').mockReturnValueOnce({ + LOG_LEVEL: { + type: 'enum', + group: ConfigVariablesGroup.Logging, + description: 'Test enum', + options: ['log', 'error', 'warn', 'debug'], + }, + }); + + expect( + service.convertDbValueToAppValue( + 'invalid', + 'LOG_LEVEL' as keyof ConfigVariables, + ), + ).toBeUndefined(); + }); }); describe('convertAppValueToDbValue', () => { @@ -241,5 +445,28 @@ describe('ConfigValueConverterService', () => { expect(service.convertAppValueToDbValue(obj as any)).toEqual(obj); }); + + it('should convert null to null', () => { + expect(service.convertAppValueToDbValue(null as any)).toBe(null); + }); + + it('should throw error for unsupported types', () => { + const symbol = Symbol('test'); + + expect(() => { + service.convertAppValueToDbValue(symbol as any); + }).toThrow(/Cannot convert value of type symbol/); + }); + + it('should handle serialization errors', () => { + // Create an object with circular reference + const circular: any = {}; + + circular.self = circular; + + expect(() => { + service.convertAppValueToDbValue(circular as any); + }).toThrow(/Failed to serialize object value/); + }); }); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts index ded35c5635e3..6080354e16a1 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts @@ -30,12 +30,7 @@ export class ConfigValueConverterService { case 'boolean': { const result = configTransformers.boolean(dbValue); - if (result === undefined) { - throw new Error( - `Value '${String(dbValue)}' cannot be converted to boolean`, - ); - } - if (typeof result !== 'boolean') { + if (result !== undefined && typeof result !== 'boolean') { throw new Error( `Expected boolean for key ${key}, got ${typeof result}`, ); @@ -47,12 +42,7 @@ export class ConfigValueConverterService { case 'number': { const result = configTransformers.number(dbValue); - if (result === undefined) { - throw new Error( - `Value '${String(dbValue)}' cannot be converted to number`, - ); - } - if (typeof result !== 'number') { + if (result !== undefined && typeof result !== 'number') { throw new Error( `Expected number for key ${key}, got ${typeof result}`, ); @@ -64,12 +54,7 @@ export class ConfigValueConverterService { case 'string': { const result = configTransformers.string(dbValue); - if (result === undefined) { - throw new Error( - `Value '${String(dbValue)}' cannot be converted to string`, - ); - } - if (typeof result !== 'string') { + if (result !== undefined && typeof result !== 'string') { throw new Error( `Expected string for key ${key}, got ${typeof result}`, ); @@ -81,7 +66,7 @@ export class ConfigValueConverterService { case 'array': { const result = this.convertToArray(dbValue, options); - if (!Array.isArray(result)) { + if (result !== undefined && !Array.isArray(result)) { throw new Error( `Expected array for key ${key}, got ${typeof result}`, ); @@ -93,12 +78,6 @@ export class ConfigValueConverterService { case 'enum': { const result = this.convertToEnum(dbValue, options); - if (result === undefined) { - throw new Error( - `Invalid enum value for key ${key}: ${String(dbValue)}`, - ); - } - return result as ConfigVariables[T]; } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index 8dec7cbb9152..6774f16e1d27 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -66,7 +66,6 @@ describe('DatabaseConfigDriver', () => { markKeyAsMissing: jest.fn(), getCacheInfo: jest.fn(), getAllKeys: jest.fn(), - getOrFallback: jest.fn(), }, }, { diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts index 0e59b8838455..d33bd645d111 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts @@ -70,24 +70,20 @@ export class TwentyConfigService implements OnModuleInit { } get(key: T): ConfigVariables[T] { - // Environment-only variables always come from the environment driver if (isEnvOnlyConfigVar(key)) { return this.environmentConfigDriver.get(key); } - // If we're using the database driver, check it first then fall back to environment if (this.driver === this.databaseConfigDriver) { - const dbValue = this.databaseConfigDriver.get(key); + const cachedValueFromDb = this.databaseConfigDriver.get(key); - if (dbValue !== undefined) { - return dbValue; + if (cachedValueFromDb !== undefined) { + return cachedValueFromDb; } - // Fall back to environment if not in database return this.environmentConfigDriver.get(key); } - // If we're not using the database driver, use the environment driver directly return this.environmentConfigDriver.get(key); } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/apply-basic-validators.util.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/apply-basic-validators.util.spec.ts new file mode 100644 index 000000000000..a44b45d534e0 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/apply-basic-validators.util.spec.ts @@ -0,0 +1,148 @@ +import { Transform } from 'class-transformer'; +import { + IsArray, + IsBoolean, + IsEnum, + IsNumber, + IsString, +} from 'class-validator'; + +import { applyBasicValidators } from 'src/engine/core-modules/twenty-config/utils/apply-basic-validators.util'; +import { configTransformers } from 'src/engine/core-modules/twenty-config/utils/config-transformers.util'; + +jest.mock('class-transformer', () => ({ + Transform: jest.fn(), +})); + +jest.mock('class-validator', () => ({ + IsBoolean: jest.fn().mockReturnValue(jest.fn()), + IsNumber: jest.fn().mockReturnValue(jest.fn()), + IsString: jest.fn().mockReturnValue(jest.fn()), + IsEnum: jest.fn().mockReturnValue(jest.fn()), + IsArray: jest.fn().mockReturnValue(jest.fn()), +})); + +jest.mock( + 'src/engine/core-modules/twenty-config/utils/config-transformers.util', + () => ({ + configTransformers: { + boolean: jest.fn(), + number: jest.fn(), + }, + }), +); + +describe('applyBasicValidators', () => { + const mockTarget = {}; + const mockPropertyKey = 'testProperty'; + + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('boolean type', () => { + it('should apply boolean transformers and validators', () => { + let capturedTransformFn; + + (Transform as jest.Mock).mockImplementation((transformFn) => { + capturedTransformFn = transformFn; + + return jest.fn(); + }); + + applyBasicValidators('boolean', mockTarget, mockPropertyKey); + + expect(Transform).toHaveBeenCalled(); + expect(IsBoolean).toHaveBeenCalled(); + + const transformFn = capturedTransformFn; + const mockTransformParams = { value: 'true' }; + + (configTransformers.boolean as jest.Mock).mockReturnValueOnce(true); + const result1 = transformFn(mockTransformParams); + + expect(configTransformers.boolean).toHaveBeenCalledWith('true'); + expect(result1).toBe(true); + + (configTransformers.boolean as jest.Mock).mockReturnValueOnce(undefined); + const result2 = transformFn(mockTransformParams); + + expect(result2).toBe('true'); + }); + }); + + describe('number type', () => { + it('should apply number transformers and validators', () => { + let capturedTransformFn; + + (Transform as jest.Mock).mockImplementation((transformFn) => { + capturedTransformFn = transformFn; + + return jest.fn(); + }); + + applyBasicValidators('number', mockTarget, mockPropertyKey); + + expect(Transform).toHaveBeenCalled(); + expect(IsNumber).toHaveBeenCalled(); + + const transformFn = capturedTransformFn; + const mockTransformParams = { value: '42' }; + + (configTransformers.number as jest.Mock).mockReturnValueOnce(42); + const result1 = transformFn(mockTransformParams); + + expect(configTransformers.number).toHaveBeenCalledWith('42'); + expect(result1).toBe(42); + + (configTransformers.number as jest.Mock).mockReturnValueOnce(undefined); + const result2 = transformFn(mockTransformParams); + + expect(result2).toBe('42'); + }); + }); + + describe('string type', () => { + it('should apply string validator', () => { + applyBasicValidators('string', mockTarget, mockPropertyKey); + + expect(IsString).toHaveBeenCalled(); + expect(Transform).not.toHaveBeenCalled(); // String doesn't need a transform + }); + }); + + describe('enum type', () => { + it('should apply enum validator with options', () => { + const enumOptions = ['option1', 'option2', 'option3']; + + applyBasicValidators('enum', mockTarget, mockPropertyKey, enumOptions); + + expect(IsEnum).toHaveBeenCalledWith(enumOptions); + expect(Transform).not.toHaveBeenCalled(); // Enum doesn't need a transform + }); + + it('should not apply enum validator without options', () => { + applyBasicValidators('enum', mockTarget, mockPropertyKey); + + expect(IsEnum).not.toHaveBeenCalled(); + expect(Transform).not.toHaveBeenCalled(); + }); + }); + + describe('array type', () => { + it('should apply array validator', () => { + applyBasicValidators('array', mockTarget, mockPropertyKey); + + expect(IsArray).toHaveBeenCalled(); + expect(Transform).not.toHaveBeenCalled(); // Array doesn't need a transform + }); + }); + + describe('unsupported type', () => { + it('should throw error for unsupported types', () => { + expect(() => { + applyBasicValidators('unsupported' as any, mockTarget, mockPropertyKey); + }).toThrow('Unsupported config variable type: unsupported'); + }); + }); +}); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/config-transformers.util.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/config-transformers.util.spec.ts new file mode 100644 index 000000000000..ee43dc02770b --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/config-transformers.util.spec.ts @@ -0,0 +1,100 @@ +import { configTransformers } from 'src/engine/core-modules/twenty-config/utils/config-transformers.util'; + +describe('configTransformers', () => { + describe('boolean', () => { + it('should handle true values correctly', () => { + expect(configTransformers.boolean(true)).toBe(true); + expect(configTransformers.boolean('true')).toBe(true); + expect(configTransformers.boolean('True')).toBe(true); + expect(configTransformers.boolean('yes')).toBe(true); + expect(configTransformers.boolean('on')).toBe(true); + expect(configTransformers.boolean('1')).toBe(true); + expect(configTransformers.boolean(1)).toBe(true); + }); + + it('should handle false values correctly', () => { + expect(configTransformers.boolean(false)).toBe(false); + expect(configTransformers.boolean('false')).toBe(false); + expect(configTransformers.boolean('False')).toBe(false); + expect(configTransformers.boolean('no')).toBe(false); + expect(configTransformers.boolean('off')).toBe(false); + expect(configTransformers.boolean('0')).toBe(false); + expect(configTransformers.boolean(0)).toBe(false); + }); + + it('should return undefined for invalid values', () => { + expect(configTransformers.boolean('invalid')).toBeUndefined(); + expect(configTransformers.boolean('random_string')).toBeUndefined(); + expect(configTransformers.boolean({})).toBeUndefined(); + expect(configTransformers.boolean([])).toBeUndefined(); + }); + + it('should handle null and undefined', () => { + expect(configTransformers.boolean(null)).toBeUndefined(); + expect(configTransformers.boolean(undefined)).toBeUndefined(); + }); + }); + + describe('number', () => { + it('should handle valid number values', () => { + expect(configTransformers.number(42)).toBe(42); + expect(configTransformers.number('42')).toBe(42); + expect(configTransformers.number('-42')).toBe(-42); + expect(configTransformers.number('3.14')).toBe(3.14); + expect(configTransformers.number('0')).toBe(0); + }); + + it('should handle boolean values', () => { + expect(configTransformers.number(true)).toBe(1); + expect(configTransformers.number(false)).toBe(0); + }); + + it('should return undefined for invalid values', () => { + expect(configTransformers.number('invalid')).toBeUndefined(); + expect(configTransformers.number('forty-two')).toBeUndefined(); + expect(configTransformers.number({})).toBeUndefined(); + expect(configTransformers.number([])).toBeUndefined(); + }); + + it('should handle null and undefined', () => { + expect(configTransformers.number(null)).toBeUndefined(); + expect(configTransformers.number(undefined)).toBeUndefined(); + }); + }); + + describe('string', () => { + it('should handle string values', () => { + expect(configTransformers.string('test')).toBe('test'); + expect(configTransformers.string('')).toBe(''); + }); + + it('should convert numbers to strings', () => { + expect(configTransformers.string(42)).toBe('42'); + expect(configTransformers.string(0)).toBe('0'); + expect(configTransformers.string(3.14)).toBe('3.14'); + }); + + it('should convert booleans to strings', () => { + expect(configTransformers.string(true)).toBe('true'); + expect(configTransformers.string(false)).toBe('false'); + }); + + it('should convert arrays and objects to JSON strings', () => { + expect(configTransformers.string(['a', 'b', 'c'])).toBe('["a","b","c"]'); + expect(configTransformers.string({ a: 1, b: 2 })).toBe('{"a":1,"b":2}'); + }); + + it('should handle null and undefined', () => { + expect(configTransformers.string(null)).toBeUndefined(); + expect(configTransformers.string(undefined)).toBeUndefined(); + }); + + it('should handle failed JSON stringification', () => { + const circular: any = {}; + + circular.self = circular; + + expect(configTransformers.string(circular)).toBeUndefined(); + }); + }); +}); From 883f50a5df8d2e095412d71d8dc266c586cbdb3a Mon Sep 17 00:00:00 2001 From: ehconitin Date: Fri, 25 Apr 2025 02:07:00 +0530 Subject: [PATCH 59/70] lint --- .../__tests__/apply-basic-validators.util.spec.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/apply-basic-validators.util.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/apply-basic-validators.util.spec.ts index a44b45d534e0..153062016f89 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/apply-basic-validators.util.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/apply-basic-validators.util.spec.ts @@ -1,10 +1,10 @@ import { Transform } from 'class-transformer'; import { - IsArray, - IsBoolean, - IsEnum, - IsNumber, - IsString, + IsArray, + IsBoolean, + IsEnum, + IsNumber, + IsString, } from 'class-validator'; import { applyBasicValidators } from 'src/engine/core-modules/twenty-config/utils/apply-basic-validators.util'; From a273216a05df255eaf9529b9a6ea6331038467db Mon Sep 17 00:00:00 2001 From: ehconitin Date: Sat, 26 Apr 2025 00:59:33 +0530 Subject: [PATCH 60/70] Refactor config system: Eliminate custom state management for NestJS patterns --- .../engine/core-modules/core-engine.module.ts | 6 +- .../__tests__/database-config.driver.spec.ts | 29 +- .../drivers/database-config.driver.ts | 84 +++--- .../drivers/database-config.module.ts | 34 +++ .../database-config-driver.interface.ts | 16 +- .../enums/config-initialization-state.enum.ts | 6 - .../twenty-config/twenty-config.module.ts | 69 ++--- .../twenty-config.service.spec.ts | 275 +++++++----------- .../twenty-config/twenty-config.service.ts | 101 ++----- ...osoft-get-message-list.service.dev.spec.ts | 6 +- ...microsoft-get-messages.service.dev.spec.ts | 2 +- 11 files changed, 261 insertions(+), 367 deletions(-) create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.module.ts delete mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/enums/config-initialization-state.enum.ts diff --git a/packages/twenty-server/src/engine/core-modules/core-engine.module.ts b/packages/twenty-server/src/engine/core-modules/core-engine.module.ts index e5e2d2b87bb2..7782ade9fa58 100644 --- a/packages/twenty-server/src/engine/core-modules/core-engine.module.ts +++ b/packages/twenty-server/src/engine/core-modules/core-engine.module.ts @@ -2,6 +2,7 @@ import { Module } from '@nestjs/common'; import { HttpAdapterHost } from '@nestjs/core'; import { EventEmitterModule } from '@nestjs/event-emitter'; +import { WorkspaceQueryRunnerModule } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-runner.module'; import { ActorModule } from 'src/engine/core-modules/actor/actor.module'; import { AdminPanelModule } from 'src/engine/core-modules/admin-panel/admin-panel.module'; import { AppTokenModule } from 'src/engine/core-modules/app-token/app-token.module'; @@ -47,9 +48,8 @@ import { WorkflowApiModule } from 'src/engine/core-modules/workflow/workflow-api import { WorkspaceInvitationModule } from 'src/engine/core-modules/workspace-invitation/workspace-invitation.module'; import { WorkspaceModule } from 'src/engine/core-modules/workspace/workspace.module'; import { RoleModule } from 'src/engine/metadata-modules/role/role.module'; -import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/workspace-event-emitter.module'; -import { WorkspaceQueryRunnerModule } from 'src/engine/api/graphql/workspace-query-runner/workspace-query-runner.module'; import { SubscriptionsModule } from 'src/engine/subscriptions/subscriptions.module'; +import { WorkspaceEventEmitterModule } from 'src/engine/workspace-event-emitter/workspace-event-emitter.module'; import { AnalyticsModule } from './analytics/analytics.module'; import { ClientConfigModule } from './client-config/client-config.module'; @@ -81,7 +81,7 @@ import { FileModule } from './file/file.module'; AdminPanelModule, LabModule, RoleModule, - TwentyConfigModule, + TwentyConfigModule.forRoot(), RedisClientModule, WorkspaceQueryRunnerModule, SubscriptionsModule, diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index 6774f16e1d27..3558cedba234 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -113,7 +113,7 @@ describe('DatabaseConfigDriver', () => { jest.spyOn(configStorage, 'loadAll').mockResolvedValue(configVars); - await driver.initialize(); + await driver.onModuleInit(); expect(configStorage.loadAll).toHaveBeenCalled(); @@ -128,38 +128,21 @@ describe('DatabaseConfigDriver', () => { ); }); - it('should handle initialization failure', async () => { + it('should handle initialization failure gracefully', async () => { const error = new Error('DB error'); jest.spyOn(configStorage, 'loadAll').mockRejectedValue(error); jest.spyOn(driver['logger'], 'error').mockImplementation(); - await expect(driver.initialize()).rejects.toThrow(error); + // Should not throw because we're handling errors internally now + await driver.onModuleInit(); expect(driver['logger'].error).toHaveBeenCalled(); - }); - - it('should handle concurrent initialization calls', async () => { - jest.spyOn(configStorage, 'loadAll').mockResolvedValue(new Map()); - - const promises = [ - driver.initialize(), - driver.initialize(), - driver.initialize(), - ]; - - await Promise.all(promises); - - expect(configStorage.loadAll).toHaveBeenCalledTimes(1); + expect(configStorage.loadAll).toHaveBeenCalled(); }); }); describe('get', () => { - beforeEach(async () => { - jest.spyOn(configStorage, 'loadAll').mockResolvedValue(new Map()); - await driver.initialize(); - }); - it('should return cached value when available', async () => { const cachedValue = true; @@ -206,8 +189,6 @@ describe('DatabaseConfigDriver', () => { describe('update', () => { beforeEach(async () => { - jest.spyOn(configStorage, 'loadAll').mockResolvedValue(new Map()); - await driver.initialize(); (isEnvOnlyConfigVar as jest.Mock).mockReturnValue(false); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index b26dff174c8d..e4394c922cce 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -1,4 +1,4 @@ -import { Injectable, Logger } from '@nestjs/common'; +import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; import { Cron } from '@nestjs/schedule'; import { DatabaseConfigDriverInterface } from 'src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface'; @@ -10,8 +10,9 @@ import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/stor import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/is-env-only-config-var.util'; @Injectable() -export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { - private initializationPromise: Promise | null = null; +export class DatabaseConfigDriver + implements DatabaseConfigDriverInterface, OnModuleInit +{ private readonly logger = new Logger(DatabaseConfigDriver.name); private readonly allPossibleConfigKeys: Array; @@ -26,39 +27,29 @@ export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { this.allPossibleConfigKeys = allKeys.filter( (key) => !isEnvOnlyConfigVar(key), ); + + this.logger.debug( + '[INIT] Database config driver created, monitoring keys: ' + + this.allPossibleConfigKeys.length, + ); } - async initialize(): Promise { - if (this.initializationPromise) { - this.logger.verbose( - '[INIT] Config database initialization already in progress', - ); + async onModuleInit(): Promise { + try { + this.logger.log('[INIT] Loading initial config variables from database'); + const loadedCount = await this.loadAllConfigVarsFromDb(); - return this.initializationPromise; + this.logger.log( + `[INIT] Config variables loaded: ${loadedCount} values found, ${this.allPossibleConfigKeys.length - loadedCount} missing`, + ); + } catch (error) { + this.logger.error( + '[INIT] Failed to load config variables from database, falling back to environment variables', + error instanceof Error ? error.stack : error, + ); + // Don't rethrow to allow the application to continue + // The driver's cache will be empty but the service will fall back to env vars } - - this.logger.debug('[INIT] Starting database config initialization'); - - const promise = this.loadAllConfigVarsFromDb() - .then((loadedCount) => { - this.logger.log( - `[INIT] Database config ready: loaded ${loadedCount} variables`, - ); - }) - .catch((error) => { - this.logger.error( - '[INIT] Failed to load database config: unable to connect to database or fetch config', - error instanceof Error ? error.stack : error, - ); - throw error; - }) - .finally(() => { - this.initializationPromise = null; - }); - - this.initializationPromise = promise; - - return promise; } get(key: T): ConfigVariables[T] | undefined { @@ -78,10 +69,12 @@ export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { try { await this.configStorage.set(key, value); this.configCache.set(key, value); - this.logger.debug(`[UPDATE] Updated config variable: ${key as string}`); + this.logger.debug( + `[UPDATE] Config variable ${key as string} updated successfully`, + ); } catch (error) { this.logger.error( - `[UPDATE] Failed to update config for ${key as string}`, + `[UPDATE] Failed to update config variable ${key as string}`, error, ); throw error; @@ -95,17 +88,17 @@ export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { if (value !== undefined) { this.configCache.set(key, value); this.logger.debug( - `[FETCH] Refreshed config variable in cache: ${key as string}`, + `[FETCH] Config variable ${key as string} loaded from database`, ); } else { this.configCache.markKeyAsMissing(key); this.logger.debug( - `[FETCH] Marked config variable as missing: ${key as string}`, + `[FETCH] Config variable ${key as string} not found in database, marked as missing`, ); } } catch (error) { this.logger.error( - `[FETCH] Failed to fetch config for ${key as string}`, + `[FETCH] Failed to fetch config variable ${key as string} from database`, error, ); this.configCache.markKeyAsMissing(key); @@ -122,11 +115,11 @@ export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { private async loadAllConfigVarsFromDb(): Promise { try { - this.logger.debug('[INIT] Fetching all config variables from database'); + this.logger.debug('[LOAD] Fetching all config variables from database'); const configVars = await this.configStorage.loadAll(); this.logger.debug( - `[INIT] Checking ${this.allPossibleConfigKeys.length} possible config variables`, + `[LOAD] Processing ${this.allPossibleConfigKeys.length} possible config variables`, ); for (const [key, value] of configVars.entries()) { @@ -143,13 +136,13 @@ export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { this.allPossibleConfigKeys.length - configVars.size; this.logger.debug( - `[INIT] Initial cache state: ${configVars.size} found values, ${missingKeysCount} missing values`, + `[LOAD] Cached ${configVars.size} config variables, marked ${missingKeysCount} keys as missing`, ); return configVars.size; } catch (error) { this.logger.error( - '[INIT] Failed to load config variables from database', + '[LOAD] Failed to load config variables from database', error, ); throw error; @@ -164,12 +157,14 @@ export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { @Cron(CONFIG_VARIABLES_REFRESH_CRON_INTERVAL) async refreshAllCache(): Promise { try { - this.logger.log('[REFRESH] Starting refresh of all config variables'); + this.logger.debug( + '[REFRESH] Starting scheduled refresh of config variables', + ); const dbValues = await this.configStorage.loadAll(); this.logger.debug( - `[REFRESH] Checking ${this.allPossibleConfigKeys.length} possible config variables`, + `[REFRESH] Processing ${this.allPossibleConfigKeys.length} possible config variables`, ); for (const [key, value] of dbValues.entries()) { @@ -188,10 +183,11 @@ export class DatabaseConfigDriver implements DatabaseConfigDriverInterface { this.allPossibleConfigKeys.length - dbValues.size; this.logger.log( - `[REFRESH] Refreshed config cache: ${dbValues.size} found values, ${missingKeysCount} missing values`, + `[REFRESH] Config variables refreshed: ${dbValues.size} values updated, ${missingKeysCount} marked as missing`, ); } catch (error) { this.logger.error('[REFRESH] Failed to refresh config variables', error); + // Error is caught and logged but not rethrown to prevent the cron job from crashing } } } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.module.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.module.ts new file mode 100644 index 000000000000..a17a25958b79 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.module.ts @@ -0,0 +1,34 @@ +import { DynamicModule, Module } from '@nestjs/common'; +import { ScheduleModule } from '@nestjs/schedule'; +import { TypeOrmModule } from '@nestjs/typeorm'; + +import { KeyValuePair } from 'src/engine/core-modules/key-value-pair/key-value-pair.entity'; +import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; +import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { ConfigValueConverterService } from 'src/engine/core-modules/twenty-config/conversion/config-value-converter.service'; +import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; +import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; + +@Module({}) +export class DatabaseConfigModule { + static forRoot(): DynamicModule { + return { + module: DatabaseConfigModule, + imports: [ + TypeOrmModule.forFeature([KeyValuePair], 'core'), + ScheduleModule.forRoot(), + ], + providers: [ + DatabaseConfigDriver, + ConfigCacheService, + ConfigStorageService, + ConfigValueConverterService, + { + provide: ConfigVariables, + useValue: new ConfigVariables(), + }, + ], + exports: [DatabaseConfigDriver], + }; + } +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts index 9e7cd5e1ce2a..620f902e70ef 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts @@ -2,14 +2,9 @@ import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-va /** * Interface for drivers that support database-backed configuration - * with caching and initialization capabilities + * with caching capabilities */ export interface DatabaseConfigDriverInterface { - /** - * Initialize the driver - */ - initialize(): Promise; - /** * Get a configuration value from cache * Returns undefined if not in cache @@ -33,4 +28,13 @@ export interface DatabaseConfigDriverInterface { * Refreshes all entries in the config cache */ refreshAllCache(): Promise; + + /** + * Get information about the cache state + */ + getCacheInfo(): { + positiveEntries: number; + negativeEntries: number; + cacheKeys: string[]; + }; } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/enums/config-initialization-state.enum.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/enums/config-initialization-state.enum.ts deleted file mode 100644 index 476ef28d106a..000000000000 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/enums/config-initialization-state.enum.ts +++ /dev/null @@ -1,6 +0,0 @@ -export enum ConfigInitializationState { - NOT_INITIALIZED = 'NOT_INITIALIZED', - INITIALIZING = 'INITIALIZING', - INITIALIZED = 'INITIALIZED', - FAILED = 'FAILED', -} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts index 4d6d09b65fca..4b6ba8c0fbe8 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts @@ -1,46 +1,47 @@ -import { Global, Module } from '@nestjs/common'; +import { DynamicModule, Global, Module } from '@nestjs/common'; import { ConfigModule } from '@nestjs/config'; -import { ScheduleModule } from '@nestjs/schedule'; -import { TypeOrmModule } from '@nestjs/typeorm'; -import { KeyValuePair } from 'src/engine/core-modules/key-value-pair/key-value-pair.entity'; -import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; import { ConfigVariables, validate, } from 'src/engine/core-modules/twenty-config/config-variables'; -import { ConfigValueConverterService } from 'src/engine/core-modules/twenty-config/conversion/config-value-converter.service'; -import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; +import { DatabaseConfigModule } from 'src/engine/core-modules/twenty-config/drivers/database-config.module'; import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; -import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; import { ConfigurableModuleClass } from 'src/engine/core-modules/twenty-config/twenty-config.module-definition'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; @Global() -@Module({ - imports: [ - ConfigModule.forRoot({ - isGlobal: true, - expandVariables: true, - validate, - envFilePath: process.env.NODE_ENV === 'test' ? '.env.test' : '.env', - }), - TypeOrmModule.forFeature([KeyValuePair], 'core'), - ScheduleModule.forRoot(), - ], +@Module({}) +export class TwentyConfigModule extends ConfigurableModuleClass { + static forRoot(): DynamicModule { + const isConfigVariablesInDbEnabled = + process.env.IS_CONFIG_VARIABLES_IN_DB_ENABLED === 'true'; - providers: [ - TwentyConfigService, - EnvironmentConfigDriver, - DatabaseConfigDriver, - ConfigCacheService, - ConfigStorageService, - ConfigValueConverterService, - { - provide: ConfigVariables, - useValue: new ConfigVariables(), - }, - ], - exports: [TwentyConfigService], -}) -export class TwentyConfigModule extends ConfigurableModuleClass {} + const imports = [ + ConfigModule.forRoot({ + isGlobal: true, + expandVariables: true, + validate, + envFilePath: process.env.NODE_ENV === 'test' ? '.env.test' : '.env', + }), + ]; + + if (isConfigVariablesInDbEnabled) { + imports.push(DatabaseConfigModule.forRoot()); + } + + return { + module: TwentyConfigModule, + imports, + providers: [ + TwentyConfigService, + EnvironmentConfigDriver, + { + provide: ConfigVariables, + useValue: new ConfigVariables(), + }, + ], + exports: [TwentyConfigService], + }; + } +} diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts index db5b18717699..fab9c79eed4c 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts @@ -4,7 +4,6 @@ import { Test, TestingModule } from '@nestjs/testing'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; -import { ConfigInitializationState } from 'src/engine/core-modules/twenty-config/enums/config-initialization-state.enum'; import { ConfigSource } from 'src/engine/core-modules/twenty-config/enums/config-source.enum'; import { ConfigVariablesGroup } from 'src/engine/core-modules/twenty-config/enums/config-variables-group.enum'; import { TwentyConfigService } from 'src/engine/core-modules/twenty-config/twenty-config.service'; @@ -38,9 +37,7 @@ jest.mock( ); type TwentyConfigServicePrivateProps = { - driver: DatabaseConfigDriver | EnvironmentConfigDriver; - isConfigVarInDbEnabled: boolean; - configInitializationState: ConfigInitializationState; + isDatabaseDriverActive: boolean; }; const mockConfigVarMetadata = { @@ -61,20 +58,14 @@ const mockConfigVarMetadata = { }, }; -const setupTestModule = async () => { +// Setup with database driver +const setupTestModule = async (isDatabaseConfigEnabled = true) => { const module: TestingModule = await Test.createTestingModule({ providers: [ TwentyConfigService, - { - provide: ConfigService, - useValue: { - get: jest.fn(), - }, - }, { provide: DatabaseConfigDriver, useValue: { - initialize: jest.fn().mockResolvedValue(undefined), get: jest.fn(), update: jest.fn(), getCacheInfo: jest.fn(), @@ -86,17 +77,64 @@ const setupTestModule = async () => { get: jest.fn(), }, }, + { + provide: ConfigService, + useValue: { + get: jest.fn().mockImplementation((key) => { + if (key === 'IS_CONFIG_VARIABLES_IN_DB_ENABLED') { + return isDatabaseConfigEnabled ? 'true' : 'false'; + } + + return null; + }), + }, + }, ], }).compile(); return { service: module.get(TwentyConfigService), - configService: module.get(ConfigService), databaseConfigDriver: module.get(DatabaseConfigDriver), environmentConfigDriver: module.get( EnvironmentConfigDriver, ), + configService: module.get(ConfigService), + }; +}; + +// Setup without database driver +const setupTestModuleWithoutDb = async () => { + const module: TestingModule = await Test.createTestingModule({ + providers: [ + TwentyConfigService, + { + provide: EnvironmentConfigDriver, + useValue: { + get: jest.fn(), + }, + }, + { + provide: ConfigService, + useValue: { + get: jest.fn().mockImplementation((key) => { + if (key === 'IS_CONFIG_VARIABLES_IN_DB_ENABLED') { + return 'false'; + } + + return null; + }), + }, + }, + ], + }).compile(); + + return { + service: module.get(TwentyConfigService), + environmentConfigDriver: module.get( + EnvironmentConfigDriver, + ), + configService: module.get(ConfigService), }; }; @@ -114,15 +152,13 @@ const setPrivateProps = ( describe('TwentyConfigService', () => { let service: TwentyConfigService; - let configService: ConfigService; let databaseConfigDriver: DatabaseConfigDriver; let environmentConfigDriver: EnvironmentConfigDriver; beforeEach(async () => { - const testModule = await setupTestModule(); + const testModule = await setupTestModule(true); service = testModule.service; - configService = testModule.configService; databaseConfigDriver = testModule.databaseConfigDriver; environmentConfigDriver = testModule.environmentConfigDriver; @@ -140,104 +176,28 @@ describe('TwentyConfigService', () => { }); describe('constructor', () => { - it('should initialize with environment driver when database config is disabled', () => { - jest.spyOn(configService, 'get').mockReturnValue(false); - - const newService = new TwentyConfigService( - configService, - databaseConfigDriver, - environmentConfigDriver, - ); - - const privateProps = - newService as unknown as TwentyConfigServicePrivateProps; - - expect(privateProps.driver).toBe(environmentConfigDriver); - expect(privateProps.isConfigVarInDbEnabled).toBe(false); - }); - - it('should initialize with environment driver initially when database config is enabled', () => { - jest.spyOn(configService, 'get').mockReturnValue(true); - - const newService = new TwentyConfigService( - configService, - databaseConfigDriver, - environmentConfigDriver, - ); - - const privateProps = - newService as unknown as TwentyConfigServicePrivateProps; + it('should set isDatabaseDriverActive to false when database config is disabled', async () => { + const { service, configService } = await setupTestModuleWithoutDb(); - expect(privateProps.driver).toBe(environmentConfigDriver); - expect(privateProps.isConfigVarInDbEnabled).toBe(true); - }); - }); - - describe('onModuleInit', () => { - it('should do nothing when db config is disabled', async () => { - jest.spyOn(configService, 'get').mockReturnValue(false); - - const newService = new TwentyConfigService( - configService, - databaseConfigDriver, - environmentConfigDriver, + // Verify ConfigService was called with the right parameter + expect(configService.get).toHaveBeenCalledWith( + 'IS_CONFIG_VARIABLES_IN_DB_ENABLED', ); - await newService.onModuleInit(); - - expect(databaseConfigDriver.initialize).not.toHaveBeenCalled(); - - const privateProps = - newService as unknown as TwentyConfigServicePrivateProps; - - expect(privateProps.driver).toBe(environmentConfigDriver); + // Test behavior that results from the configuration + expect(service.getCacheInfo().usingDatabaseDriver).toBe(false); }); - it('should initialize database driver when db config is enabled', async () => { - jest.spyOn(configService, 'get').mockReturnValue(true); + it('should set isDatabaseDriverActive to true when database config is enabled and driver is available', async () => { + const { service, configService } = await setupTestModule(true); - const newService = new TwentyConfigService( - configService, - databaseConfigDriver, - environmentConfigDriver, + // Verify ConfigService was called with the right parameter + expect(configService.get).toHaveBeenCalledWith( + 'IS_CONFIG_VARIABLES_IN_DB_ENABLED', ); - await newService.onModuleInit(); - - expect(databaseConfigDriver.initialize).toHaveBeenCalled(); - - const privateProps = - newService as unknown as TwentyConfigServicePrivateProps; - - expect(privateProps.driver).toBe(databaseConfigDriver); - expect(privateProps.configInitializationState).toBe( - ConfigInitializationState.INITIALIZED, - ); - }); - - it('should fall back to environment driver when database initialization fails', async () => { - jest.spyOn(configService, 'get').mockReturnValue(true); - jest - .spyOn(databaseConfigDriver, 'initialize') - .mockRejectedValue(new Error('DB initialization failed')); - - const newService = new TwentyConfigService( - configService, - databaseConfigDriver, - environmentConfigDriver, - ); - - await newService.onModuleInit(); - - expect(databaseConfigDriver.initialize).toHaveBeenCalled(); - - const privateProps = - newService as unknown as TwentyConfigServicePrivateProps; - - expect(privateProps.driver).toBe(environmentConfigDriver); - expect(privateProps.configInitializationState).toBe( - ConfigInitializationState.FAILED, - ); + // Test behavior that results from the configuration + expect(service.getCacheInfo().usingDatabaseDriver).toBe(true); }); }); @@ -259,9 +219,9 @@ describe('TwentyConfigService', () => { expect(environmentConfigDriver.get).toHaveBeenCalledWith(key); }); - it('should use database driver when it is active and value is found', () => { + it('should use database driver when isDatabaseDriverActive is true and value is found', () => { jest.spyOn(databaseConfigDriver, 'get').mockReturnValue(expectedValue); - setPrivateProps(service, { driver: databaseConfigDriver }); + setPrivateProps(service, { isDatabaseDriverActive: true }); const result = service.get(key); @@ -275,7 +235,7 @@ describe('TwentyConfigService', () => { jest.spyOn(databaseConfigDriver, 'get').mockReturnValue(undefined); jest.spyOn(environmentConfigDriver, 'get').mockReturnValue(envValue); - setPrivateProps(service, { driver: databaseConfigDriver }); + setPrivateProps(service, { isDatabaseDriverActive: true }); const result = service.get(key); @@ -284,9 +244,9 @@ describe('TwentyConfigService', () => { expect(environmentConfigDriver.get).toHaveBeenCalledWith(key); }); - it('should use environment driver when it is active', () => { + it('should use environment driver when isDatabaseDriverActive is false', () => { jest.spyOn(environmentConfigDriver, 'get').mockReturnValue(expectedValue); - setPrivateProps(service, { driver: environmentConfigDriver }); + setPrivateProps(service, { isDatabaseDriverActive: false }); const result = service.get(key); @@ -297,41 +257,21 @@ describe('TwentyConfigService', () => { }); describe('update', () => { - const setupUpdateTest = ( - props: Partial, - ) => { - setPrivateProps(service, { - isConfigVarInDbEnabled: true, - configInitializationState: ConfigInitializationState.INITIALIZED, - driver: databaseConfigDriver, - ...props, - }); - }; - - it('should throw error when database config is disabled', async () => { - setupUpdateTest({ isConfigVarInDbEnabled: false }); + it('should throw error when database driver is not active', async () => { + setPrivateProps(service, { isDatabaseDriverActive: false }); await expect( service.update('TEST_VAR' as keyof ConfigVariables, 'new value'), ).rejects.toThrow( - 'Database configuration is disabled, cannot update configuration', - ); - }); - - it('should throw error when not initialized', async () => { - setupUpdateTest({ - configInitializationState: ConfigInitializationState.NOT_INITIALIZED, - }); - - await expect( - service.update('TEST_VAR' as keyof ConfigVariables, 'new value'), - ).rejects.toThrow( - 'TwentyConfigService not initialized, cannot update configuration', + 'Database configuration is disabled or unavailable, cannot update configuration', ); }); it('should throw error when updating environment-only variable', async () => { - setupUpdateTest({}); + setPrivateProps(service, { isDatabaseDriverActive: true }); + (TypedReflect.getMetadata as jest.Mock).mockReturnValue({ + ENV_ONLY_VAR: { isEnvOnly: true }, + }); await expect( service.update('ENV_ONLY_VAR' as keyof ConfigVariables, 'new value'), @@ -340,27 +280,28 @@ describe('TwentyConfigService', () => { ); }); - it('should throw error when driver is not DatabaseConfigDriver', async () => { - setupUpdateTest({ driver: environmentConfigDriver }); - - await expect( - service.update('TEST_VAR' as keyof ConfigVariables, 'new value'), - ).rejects.toThrow( - 'Database driver not initialized, cannot update configuration', - ); - }); - - it('should update config when all conditions are met', async () => { + it('should update config when database driver is active', async () => { const key = 'TEST_VAR' as keyof ConfigVariables; const newValue = 'new value'; - setupUpdateTest({}); + setPrivateProps(service, { isDatabaseDriverActive: true }); jest.spyOn(databaseConfigDriver, 'update').mockResolvedValue(undefined); await service.update(key, newValue); expect(databaseConfigDriver.update).toHaveBeenCalledWith(key, newValue); }); + + it('should propagate errors from database driver', async () => { + const error = new Error('Database error'); + + setPrivateProps(service, { isDatabaseDriverActive: true }); + jest.spyOn(databaseConfigDriver, 'update').mockRejectedValue(error); + + await expect( + service.update('TEST_VAR' as keyof ConfigVariables, 'new value'), + ).rejects.toThrow(error); + }); }); describe('getMetadata', () => { @@ -415,11 +356,9 @@ describe('TwentyConfigService', () => { setupDriverMocks(); }); - it('should return all config variables with environment source when using environment driver', () => { + it('should return all config variables with environment source when database driver is not active', () => { setPrivateProps(service, { - driver: environmentConfigDriver, - isConfigVarInDbEnabled: false, - configInitializationState: ConfigInitializationState.INITIALIZED, + isDatabaseDriverActive: false, }); const result = service.getAll(); @@ -445,11 +384,9 @@ describe('TwentyConfigService', () => { expect(result.SENSITIVE_VAR.value).toBe('********a_123'); }); - it('should return config variables with database source when using database driver', () => { + it('should return config variables with database source when database driver is active', () => { setPrivateProps(service, { - driver: databaseConfigDriver, - isConfigVarInDbEnabled: true, - configInitializationState: ConfigInitializationState.INITIALIZED, + isDatabaseDriverActive: true, }); const result = service.getAll(); @@ -475,38 +412,27 @@ describe('TwentyConfigService', () => { }); describe('getCacheInfo', () => { - const setupCacheInfoTest = ( - props: Partial, - ) => { + it('should return basic info when database driver is not active', () => { setPrivateProps(service, { - driver: environmentConfigDriver, - isConfigVarInDbEnabled: false, - configInitializationState: ConfigInitializationState.INITIALIZED, - ...props, + isDatabaseDriverActive: false, }); - }; - - it('should return basic info when not using database driver', () => { - setupCacheInfoTest({}); const result = service.getCacheInfo(); expect(result).toEqual({ usingDatabaseDriver: false, - initializationState: 'INITIALIZED', }); }); - it('should return cache stats when using database driver', () => { + it('should return cache stats when database driver is active', () => { const cacheStats = { positiveEntries: 2, negativeEntries: 1, cacheKeys: ['TEST_VAR', 'SENSITIVE_VAR'], }; - setupCacheInfoTest({ - driver: databaseConfigDriver, - isConfigVarInDbEnabled: true, + setPrivateProps(service, { + isDatabaseDriverActive: true, }); jest @@ -517,7 +443,6 @@ describe('TwentyConfigService', () => { expect(result).toEqual({ usingDatabaseDriver: true, - initializationState: 'INITIALIZED', cacheStats, }); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts index d33bd645d111..17e669edd62e 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts @@ -1,4 +1,4 @@ -import { Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { Injectable, Logger, Optional } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { isString } from 'class-validator'; @@ -8,7 +8,6 @@ import { CONFIG_VARIABLES_MASKING_CONFIG } from 'src/engine/core-modules/twenty- import { ConfigVariablesMetadataOptions } from 'src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator'; import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; -import { ConfigInitializationState } from 'src/engine/core-modules/twenty-config/enums/config-initialization-state.enum'; import { ConfigSource } from 'src/engine/core-modules/twenty-config/enums/config-source.enum'; import { ConfigVariablesMaskingStrategies } from 'src/engine/core-modules/twenty-config/enums/config-variables-masking-strategies.enum'; import { configVariableMaskSensitiveData } from 'src/engine/core-modules/twenty-config/utils/config-variable-mask-sensitive-data.util'; @@ -16,56 +15,37 @@ import { isEnvOnlyConfigVar } from 'src/engine/core-modules/twenty-config/utils/ import { TypedReflect } from 'src/utils/typed-reflect'; @Injectable() -export class TwentyConfigService implements OnModuleInit { - private driver: DatabaseConfigDriver | EnvironmentConfigDriver; - private configInitializationState = ConfigInitializationState.NOT_INITIALIZED; - private readonly isConfigVarInDbEnabled: boolean; +export class TwentyConfigService { private readonly logger = new Logger(TwentyConfigService.name); + private readonly isDatabaseDriverActive: boolean; constructor( - private readonly configService: ConfigService, - private readonly databaseConfigDriver: DatabaseConfigDriver, private readonly environmentConfigDriver: EnvironmentConfigDriver, + @Optional() private readonly databaseConfigDriver: DatabaseConfigDriver, + private readonly configService: ConfigService, ) { - this.driver = this.environmentConfigDriver; + const isConfigVariablesInDbEnabled = + this.configService.get('IS_CONFIG_VARIABLES_IN_DB_ENABLED') === 'true'; - this.isConfigVarInDbEnabled = - this.configService.get('IS_CONFIG_VARIABLES_IN_DB_ENABLED') === true; + this.isDatabaseDriverActive = + isConfigVariablesInDbEnabled && !!this.databaseConfigDriver; this.logger.log( - `Database configuration is ${this.isConfigVarInDbEnabled ? 'enabled' : 'disabled'}`, + `Database configuration is ${isConfigVariablesInDbEnabled ? 'enabled' : 'disabled'}`, ); - } - async onModuleInit() { - if (!this.isConfigVarInDbEnabled) { - this.logger.log( - 'Database configuration is disabled, using environment variables only', + if (isConfigVariablesInDbEnabled && !this.databaseConfigDriver) { + this.logger.warn( + 'Database config is enabled but driver is not available. Using environment variables only.', ); - this.configInitializationState = ConfigInitializationState.INITIALIZED; - - return; } - try { - this.logger.log('Initializing database driver for configuration'); - this.configInitializationState = ConfigInitializationState.INITIALIZING; - - await this.databaseConfigDriver.initialize(); - - this.driver = this.databaseConfigDriver; - this.configInitializationState = ConfigInitializationState.INITIALIZED; - this.logger.log('Database driver initialized successfully'); - this.logger.log(`Active driver: DatabaseDriver`); - } catch (error) { - this.logger.error( - 'Failed to initialize database driver, falling back to environment variables', - error, - ); - this.configInitializationState = ConfigInitializationState.FAILED; - - this.driver = this.environmentConfigDriver; - this.logger.log(`Active driver: EnvironmentDriver (fallback)`); + if (this.isDatabaseDriverActive) { + this.logger.log('Using database configuration driver'); + // The database driver will load config variables asynchronously via its onModuleInit lifecycle hook + // In the meantime, we'll use the environment driver -- fallback + } else { + this.logger.log('Using environment variables only for configuration'); } } @@ -74,7 +54,7 @@ export class TwentyConfigService implements OnModuleInit { return this.environmentConfigDriver.get(key); } - if (this.driver === this.databaseConfigDriver) { + if (this.isDatabaseDriverActive) { const cachedValueFromDb = this.databaseConfigDriver.get(key); if (cachedValueFromDb !== undefined) { @@ -91,17 +71,9 @@ export class TwentyConfigService implements OnModuleInit { key: T, value: ConfigVariables[T], ): Promise { - if (!this.isConfigVarInDbEnabled) { + if (!this.isDatabaseDriverActive) { throw new Error( - 'Database configuration is disabled, cannot update configuration', - ); - } - - if ( - this.configInitializationState !== ConfigInitializationState.INITIALIZED - ) { - throw new Error( - 'TwentyConfigService not initialized, cannot update configuration', + 'Database configuration is disabled or unavailable, cannot update configuration', ); } @@ -115,12 +87,12 @@ export class TwentyConfigService implements OnModuleInit { ); } - if (this.driver === this.databaseConfigDriver) { + try { await this.databaseConfigDriver.update(key, value); - } else { - throw new Error( - 'Database driver not initialized, cannot update configuration', - ); + this.logger.debug(`Updated config variable: ${key as string}`); + } catch (error) { + this.logger.error(`Failed to update config for ${key as string}`, error); + throw error; } } @@ -154,16 +126,11 @@ export class TwentyConfigService implements OnModuleInit { const metadata = TypedReflect.getMetadata('config-variables', ConfigVariables) ?? {}; - const isUsingDatabaseDriver = - this.driver === this.databaseConfigDriver && - this.isConfigVarInDbEnabled && - this.configInitializationState === ConfigInitializationState.INITIALIZED; - Object.entries(metadata).forEach(([key, envMetadata]) => { let value = this.get(key as keyof ConfigVariables) ?? ''; let source = ConfigSource.ENVIRONMENT; - if (!isUsingDatabaseDriver || envMetadata.isEnvOnly) { + if (!this.isDatabaseDriverActive || envMetadata.isEnvOnly) { if (value === configVars[key as keyof ConfigVariables]) { source = ConfigSource.DEFAULT; } @@ -206,25 +173,17 @@ export class TwentyConfigService implements OnModuleInit { getCacheInfo(): { usingDatabaseDriver: boolean; - initializationState: string; cacheStats?: { positiveEntries: number; negativeEntries: number; cacheKeys: string[]; }; } { - const isUsingDatabaseDriver = - this.driver === this.databaseConfigDriver && - this.isConfigVarInDbEnabled && - this.configInitializationState === ConfigInitializationState.INITIALIZED; - const result = { - usingDatabaseDriver: isUsingDatabaseDriver, - initializationState: - ConfigInitializationState[this.configInitializationState], + usingDatabaseDriver: this.isDatabaseDriverActive, }; - if (isUsingDatabaseDriver) { + if (this.isDatabaseDriverActive) { return { ...result, cacheStats: this.databaseConfigDriver.getCacheInfo(), diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts index 88bfa5a53ec5..836586d576eb 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-message-list.service.dev.spec.ts @@ -28,7 +28,7 @@ xdescribe('Microsoft dev tests : get message list service', () => { beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ - imports: [TwentyConfigModule.forRoot({})], + imports: [TwentyConfigModule.forRoot()], providers: [ MicrosoftGetMessageListService, MicrosoftClientProvider, @@ -118,7 +118,7 @@ xdescribe('Microsoft dev tests : get full message list service for folders', () beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ - imports: [TwentyConfigModule.forRoot({})], + imports: [TwentyConfigModule.forRoot()], providers: [ MicrosoftGetMessageListService, MicrosoftClientProvider, @@ -207,7 +207,7 @@ xdescribe('Microsoft dev tests : get partial message list service for folders', beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ - imports: [TwentyConfigModule.forRoot({})], + imports: [TwentyConfigModule.forRoot()], providers: [ MicrosoftGetMessageListService, MicrosoftClientProvider, diff --git a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.dev.spec.ts b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.dev.spec.ts index 1010eb3f768e..b9e577a0fb15 100644 --- a/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.dev.spec.ts +++ b/packages/twenty-server/src/modules/messaging/message-import-manager/drivers/microsoft/services/microsoft-get-messages.service.dev.spec.ts @@ -23,7 +23,7 @@ xdescribe('Microsoft dev tests : get messages service', () => { beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ - imports: [TwentyConfigModule.forRoot({})], + imports: [TwentyConfigModule.forRoot()], providers: [ MicrosoftGetMessagesService, MicrosoftHandleErrorService, From 496b4f3fb0f3e3c3e009d8ce9828fd2c0fc88dce Mon Sep 17 00:00:00 2001 From: ehconitin Date: Sat, 26 Apr 2025 01:29:50 +0530 Subject: [PATCH 61/70] rename lingo --- .../cache/__tests__/config-cache.service.spec.ts | 8 ++++---- .../twenty-config/cache/config-cache.service.ts | 14 +++++++------- .../__tests__/database-config.driver.spec.ts | 4 ++-- .../drivers/database-config.driver.ts | 4 ++-- .../interfaces/database-config-driver.interface.ts | 4 ++-- .../twenty-config/twenty-config.service.spec.ts | 4 ++-- .../twenty-config/twenty-config.service.ts | 4 ++-- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts index 63981c0a675e..f137e145271a 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts @@ -118,8 +118,8 @@ describe('ConfigCacheService', () => { const info = service.getCacheInfo(); - expect(info.positiveEntries).toBe(2); - expect(info.negativeEntries).toBe(1); + expect(info.foundConfigValues).toBe(2); + expect(info.knownMissingKeys).toBe(1); expect(info.cacheKeys).toContain(key1); expect(info.cacheKeys).toContain(key2); expect(info.cacheKeys).not.toContain(key3); @@ -138,8 +138,8 @@ describe('ConfigCacheService', () => { const cacheInfo = service.getCacheInfo(); - expect(cacheInfo.positiveEntries).toBe(2); - expect(cacheInfo.negativeEntries).toBe(1); + expect(cacheInfo.foundConfigValues).toBe(2); + expect(cacheInfo.knownMissingKeys).toBe(1); expect(cacheInfo.cacheKeys).toContain(key1); expect(cacheInfo.cacheKeys).toContain(key2); expect(service.isKeyKnownMissing(key3)).toBe(true); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts index 8b539569c5c1..d6679d80c000 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts @@ -55,13 +55,13 @@ export class ConfigCacheService implements OnModuleDestroy { } getCacheInfo(): { - positiveEntries: number; - negativeEntries: number; + foundConfigValues: number; + knownMissingKeys: number; cacheKeys: string[]; } { return { - positiveEntries: this.foundConfigValuesCache.size, - negativeEntries: this.knownMissingKeysCache.size, + foundConfigValues: this.foundConfigValuesCache.size, + knownMissingKeys: this.knownMissingKeysCache.size, cacheKeys: Array.from(this.foundConfigValuesCache.keys()), }; } @@ -71,9 +71,9 @@ export class ConfigCacheService implements OnModuleDestroy { } getAllKeys(): ConfigKey[] { - const positiveKeys = Array.from(this.foundConfigValuesCache.keys()); - const negativeKeys = Array.from(this.knownMissingKeysCache); + const foundKeys = Array.from(this.foundConfigValuesCache.keys()); + const missingKeys = Array.from(this.knownMissingKeysCache); - return [...new Set([...positiveKeys, ...negativeKeys])]; + return [...new Set([...foundKeys, ...missingKeys])]; } } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index 3558cedba234..f0bdd63c27d0 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -261,8 +261,8 @@ describe('DatabaseConfigDriver', () => { describe('cache operations', () => { it('should return cache info', () => { const cacheInfo = { - positiveEntries: 2, - negativeEntries: 1, + foundConfigValues: 2, + knownMissingKeys: 1, cacheKeys: [CONFIG_PASSWORD_KEY, CONFIG_EMAIL_KEY], }; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts index e4394c922cce..8c8be48316b8 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.driver.ts @@ -106,8 +106,8 @@ export class DatabaseConfigDriver } getCacheInfo(): { - positiveEntries: number; - negativeEntries: number; + foundConfigValues: number; + knownMissingKeys: number; cacheKeys: string[]; } { return this.configCache.getCacheInfo(); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts index 620f902e70ef..a61a80a19b7b 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/interfaces/database-config-driver.interface.ts @@ -33,8 +33,8 @@ export interface DatabaseConfigDriverInterface { * Get information about the cache state */ getCacheInfo(): { - positiveEntries: number; - negativeEntries: number; + foundConfigValues: number; + knownMissingKeys: number; cacheKeys: string[]; }; } diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts index fab9c79eed4c..c9e3b9c34b5f 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts @@ -426,8 +426,8 @@ describe('TwentyConfigService', () => { it('should return cache stats when database driver is active', () => { const cacheStats = { - positiveEntries: 2, - negativeEntries: 1, + foundConfigValues: 2, + knownMissingKeys: 1, cacheKeys: ['TEST_VAR', 'SENSITIVE_VAR'], }; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts index 17e669edd62e..bedc42412d0a 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts @@ -174,8 +174,8 @@ export class TwentyConfigService { getCacheInfo(): { usingDatabaseDriver: boolean; cacheStats?: { - positiveEntries: number; - negativeEntries: number; + foundConfigValues: number; + knownMissingKeys: number; cacheKeys: string[]; }; } { From 79c8e3a0cfba4f7932372245ab4ab62518fd1d0b Mon Sep 17 00:00:00 2001 From: ehconitin Date: Sat, 26 Apr 2025 01:39:57 +0530 Subject: [PATCH 62/70] continue --- .../twenty-server/src/database/typeorm/typeorm.module.ts | 2 +- .../src/engine/core-modules/core-engine.module.ts | 2 +- .../cache/__tests__/config-cache.service.spec.ts | 5 +---- .../twenty-config/cache/config-cache.service.ts | 7 +++++++ 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/packages/twenty-server/src/database/typeorm/typeorm.module.ts b/packages/twenty-server/src/database/typeorm/typeorm.module.ts index 88a9b26e390c..11a590405757 100644 --- a/packages/twenty-server/src/database/typeorm/typeorm.module.ts +++ b/packages/twenty-server/src/database/typeorm/typeorm.module.ts @@ -20,6 +20,7 @@ const coreTypeORMFactory = async (): Promise => ({ @Module({ imports: [ + TwentyConfigModule, TypeOrmModule.forRootAsync({ useFactory: metadataTypeORMFactory, name: 'metadata', @@ -28,7 +29,6 @@ const coreTypeORMFactory = async (): Promise => ({ useFactory: coreTypeORMFactory, name: 'core', }), - TwentyConfigModule, ], providers: [TypeORMService], exports: [TypeORMService], diff --git a/packages/twenty-server/src/engine/core-modules/core-engine.module.ts b/packages/twenty-server/src/engine/core-modules/core-engine.module.ts index 7782ade9fa58..d5ef3c18c53c 100644 --- a/packages/twenty-server/src/engine/core-modules/core-engine.module.ts +++ b/packages/twenty-server/src/engine/core-modules/core-engine.module.ts @@ -57,6 +57,7 @@ import { FileModule } from './file/file.module'; @Module({ imports: [ + TwentyConfigModule.forRoot(), HealthModule, AnalyticsModule, AuthModule, @@ -81,7 +82,6 @@ import { FileModule } from './file/file.module'; AdminPanelModule, LabModule, RoleModule, - TwentyConfigModule.forRoot(), RedisClientModule, WorkspaceQueryRunnerModule, SubscriptionsModule, diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts index f137e145271a..730b41edc317 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts @@ -8,8 +8,6 @@ describe('ConfigCacheService', () => { let service: ConfigCacheService; beforeEach(async () => { - jest.useFakeTimers(); - const module: TestingModule = await Test.createTestingModule({ imports: [ScheduleModule.forRoot()], providers: [ConfigCacheService], @@ -20,7 +18,6 @@ describe('ConfigCacheService', () => { afterEach(() => { service.onModuleDestroy(); - jest.useRealTimers(); }); it('should be defined', () => { @@ -190,7 +187,7 @@ describe('ConfigCacheService', () => { // Then force it into negative cache (normally this would remove from positive) // We're bypassing normal behavior for testing edge cases - service['knownMissingKeysCache'].add(key); + service.addToMissingKeysForTesting(key); const allKeys = service.getAllKeys(); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts index d6679d80c000..58fbae2b432b 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/config-cache.service.ts @@ -76,4 +76,11 @@ export class ConfigCacheService implements OnModuleDestroy { return [...new Set([...foundKeys, ...missingKeys])]; } + + /** + * Helper method for testing edge cases + */ + addToMissingKeysForTesting(key: ConfigKey): void { + this.knownMissingKeysCache.add(key); + } } From 44660cb43718541406aa3ef4d6a45780e40c2744 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Sat, 26 Apr 2025 02:24:26 +0530 Subject: [PATCH 63/70] grep --- .../__tests__/config-cache.service.spec.ts | 2 -- .../config-value-converter.service.spec.ts | 3 +-- .../__tests__/database-config.driver.spec.ts | 18 ++++++++++++- .../drivers/database-config.module.ts | 4 ++- .../twenty-config.service.spec.ts | 18 +++++++++++-- .../apply-basic-validators.util.spec.ts | 25 ++++++++++++++----- .../utils/apply-basic-validators.util.ts | 2 +- 7 files changed, 57 insertions(+), 15 deletions(-) rename packages/twenty-server/src/engine/core-modules/twenty-config/conversion/{ => __tests__}/config-value-converter.service.spec.ts (99%) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts index 730b41edc317..33a219a80c97 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/cache/__tests__/config-cache.service.spec.ts @@ -1,4 +1,3 @@ -import { ScheduleModule } from '@nestjs/schedule'; import { Test, TestingModule } from '@nestjs/testing'; import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; @@ -9,7 +8,6 @@ describe('ConfigCacheService', () => { beforeEach(async () => { const module: TestingModule = await Test.createTestingModule({ - imports: [ScheduleModule.forRoot()], providers: [ConfigCacheService], }).compile(); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/__tests__/config-value-converter.service.spec.ts similarity index 99% rename from packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.spec.ts rename to packages/twenty-server/src/engine/core-modules/twenty-config/conversion/__tests__/config-value-converter.service.spec.ts index 8a167c5f3723..705c0509ab8c 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/__tests__/config-value-converter.service.spec.ts @@ -2,12 +2,11 @@ import { LogLevel } from '@nestjs/common'; import { Test, TestingModule } from '@nestjs/testing'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { ConfigValueConverterService } from 'src/engine/core-modules/twenty-config/conversion/config-value-converter.service'; import { ConfigVariablesGroup } from 'src/engine/core-modules/twenty-config/enums/config-variables-group.enum'; import { configTransformers } from 'src/engine/core-modules/twenty-config/utils/config-transformers.util'; import { TypedReflect } from 'src/utils/typed-reflect'; -import { ConfigValueConverterService } from './config-value-converter.service'; - // Mock configTransformers for type validation tests jest.mock( 'src/engine/core-modules/twenty-config/utils/config-transformers.util', diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts index f0bdd63c27d0..9398e033ad9d 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/database-config.driver.spec.ts @@ -344,15 +344,31 @@ describe('DatabaseConfigDriver', () => { ); }); - it('should handle errors gracefully', async () => { + it('should handle errors gracefully and verify cache remains unchanged', async () => { const error = new Error('Database error'); jest.spyOn(configStorage, 'loadAll').mockRejectedValue(error); jest.spyOn(driver['logger'], 'error').mockImplementation(); + const mockCacheState = new Map(); + + mockCacheState.set(CONFIG_PASSWORD_KEY, false); + jest + .spyOn(configCache, 'getAllKeys') + .mockReturnValue([CONFIG_PASSWORD_KEY]); + jest + .spyOn(configCache, 'get') + .mockImplementation((key) => mockCacheState.get(key)); + await driver.refreshAllCache(); expect(driver['logger'].error).toHaveBeenCalled(); + expect(configStorage.loadAll).toHaveBeenCalled(); + + expect(configCache.set).not.toHaveBeenCalled(); + expect(configCache.markKeyAsMissing).not.toHaveBeenCalled(); + expect(configCache.clear).not.toHaveBeenCalled(); + expect(configCache.clearAll).not.toHaveBeenCalled(); }); }); }); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.module.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.module.ts index a17a25958b79..b55f767ba14c 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.module.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.module.ts @@ -9,6 +9,8 @@ import { ConfigValueConverterService } from 'src/engine/core-modules/twenty-conf import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; +const CONFIG_VARIABLES_INSTANCE = new ConfigVariables(); + @Module({}) export class DatabaseConfigModule { static forRoot(): DynamicModule { @@ -25,7 +27,7 @@ export class DatabaseConfigModule { ConfigValueConverterService, { provide: ConfigVariables, - useValue: new ConfigVariables(), + useValue: CONFIG_VARIABLES_INSTANCE, }, ], exports: [DatabaseConfigDriver], diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts index c9e3b9c34b5f..3346a3b56bcf 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts @@ -219,6 +219,20 @@ describe('TwentyConfigService', () => { expect(environmentConfigDriver.get).toHaveBeenCalledWith(key); }); + it('should return undefined when key does not exist in any driver', () => { + const nonExistentKey = 'NON_EXISTENT_KEY' as keyof ConfigVariables; + + jest.spyOn(databaseConfigDriver, 'get').mockReturnValue(undefined); + jest.spyOn(environmentConfigDriver, 'get').mockReturnValue(undefined); + setPrivateProps(service, { isDatabaseDriverActive: true }); + + const result = service.get(nonExistentKey); + + expect(result).toBeUndefined(); + expect(databaseConfigDriver.get).toHaveBeenCalledWith(nonExistentKey); + expect(environmentConfigDriver.get).toHaveBeenCalledWith(nonExistentKey); + }); + it('should use database driver when isDatabaseDriverActive is true and value is found', () => { jest.spyOn(databaseConfigDriver, 'get').mockReturnValue(expectedValue); setPrivateProps(service, { isDatabaseDriverActive: true }); @@ -332,7 +346,7 @@ describe('TwentyConfigService', () => { SENSITIVE_VAR: 'sensitive_data_123', }; - return values[keyStr] || ''; + return values[keyStr] || undefined; }); jest @@ -348,7 +362,7 @@ describe('TwentyConfigService', () => { SENSITIVE_VAR: 'sensitive_data_123', }; - return values[keyStr] || ''; + return values[keyStr] || undefined; }); }; diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/apply-basic-validators.util.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/apply-basic-validators.util.spec.ts index 153062016f89..b290b4ca45d8 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/apply-basic-validators.util.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/apply-basic-validators.util.spec.ts @@ -1,10 +1,10 @@ import { Transform } from 'class-transformer'; import { - IsArray, - IsBoolean, - IsEnum, - IsNumber, - IsString, + IsArray, + IsBoolean, + IsEnum, + IsNumber, + IsString, } from 'class-validator'; import { applyBasicValidators } from 'src/engine/core-modules/twenty-config/utils/apply-basic-validators.util'; @@ -112,7 +112,7 @@ describe('applyBasicValidators', () => { }); describe('enum type', () => { - it('should apply enum validator with options', () => { + it('should apply enum validator with string array options', () => { const enumOptions = ['option1', 'option2', 'option3']; applyBasicValidators('enum', mockTarget, mockPropertyKey, enumOptions); @@ -121,6 +121,19 @@ describe('applyBasicValidators', () => { expect(Transform).not.toHaveBeenCalled(); // Enum doesn't need a transform }); + it('should apply enum validator with enum object options', () => { + enum TestEnum { + Option1 = 'value1', + Option2 = 'value2', + Option3 = 'value3', + } + + applyBasicValidators('enum', mockTarget, mockPropertyKey, TestEnum); + + expect(IsEnum).toHaveBeenCalledWith(TestEnum); + expect(Transform).not.toHaveBeenCalled(); // Enum doesn't need a transform + }); + it('should not apply enum validator without options', () => { applyBasicValidators('enum', mockTarget, mockPropertyKey); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts index e517a4052f55..38a7bb5dd13c 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts @@ -41,7 +41,7 @@ export function applyBasicValidators( break; case 'enum': - if (options && Array.isArray(options)) { + if (options) { IsEnum(options)(target, propertyKey); } break; From a887bf87d3f63661aa7463af3abf94182bda996c Mon Sep 17 00:00:00 2001 From: ehconitin Date: Sat, 26 Apr 2025 02:25:24 +0530 Subject: [PATCH 64/70] lint --- .../__tests__/apply-basic-validators.util.spec.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/apply-basic-validators.util.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/apply-basic-validators.util.spec.ts index b290b4ca45d8..ed0f390b6d4b 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/apply-basic-validators.util.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/__tests__/apply-basic-validators.util.spec.ts @@ -1,10 +1,10 @@ import { Transform } from 'class-transformer'; import { - IsArray, - IsBoolean, - IsEnum, - IsNumber, - IsString, + IsArray, + IsBoolean, + IsEnum, + IsNumber, + IsString, } from 'class-validator'; import { applyBasicValidators } from 'src/engine/core-modules/twenty-config/utils/apply-basic-validators.util'; From 7ff696a316c473e8cc3135b2099756c7b2837048 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Sat, 26 Apr 2025 02:54:57 +0530 Subject: [PATCH 65/70] last one I promise --- .../config-variables-instance-tokens.constants.ts | 3 +++ .../conversion/config-value-converter.service.ts | 8 ++++++-- .../twenty-config/drivers/database-config.module.ts | 7 +++---- .../drivers/environment-config.driver.ts | 13 +++++++------ .../twenty-config/twenty-config.module.ts | 3 ++- .../twenty-config/twenty-config.service.ts | 7 +++---- 6 files changed, 24 insertions(+), 17 deletions(-) create mode 100644 packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants.ts diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants.ts new file mode 100644 index 000000000000..014a29c18497 --- /dev/null +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants.ts @@ -0,0 +1,3 @@ +export const CONFIG_VARIABLES_INSTANCE_TOKEN = Symbol( + 'CONFIG_VARIABLES_INSTANCE_TOKEN', +); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts index 6080354e16a1..33bb46f58bc9 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/config-value-converter.service.ts @@ -1,6 +1,7 @@ -import { Injectable, Logger } from '@nestjs/common'; +import { Inject, Injectable, Logger } from '@nestjs/common'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { CONFIG_VARIABLES_INSTANCE_TOKEN } from 'src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants'; import { ConfigVariablesMetadataMap } from 'src/engine/core-modules/twenty-config/decorators/config-variables-metadata.decorator'; import { ConfigVariableOptions } from 'src/engine/core-modules/twenty-config/types/config-variable-options.type'; import { ConfigVariableType } from 'src/engine/core-modules/twenty-config/types/config-variable-type.type'; @@ -11,7 +12,10 @@ import { TypedReflect } from 'src/utils/typed-reflect'; export class ConfigValueConverterService { private readonly logger = new Logger(ConfigValueConverterService.name); - constructor(private readonly configVariables: ConfigVariables) {} + constructor( + @Inject(CONFIG_VARIABLES_INSTANCE_TOKEN) + private readonly configVariables: ConfigVariables, + ) {} convertDbValueToAppValue( dbValue: unknown, diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.module.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.module.ts index b55f767ba14c..d22af8cafeb2 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.module.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/database-config.module.ts @@ -5,12 +5,11 @@ import { TypeOrmModule } from '@nestjs/typeorm'; import { KeyValuePair } from 'src/engine/core-modules/key-value-pair/key-value-pair.entity'; import { ConfigCacheService } from 'src/engine/core-modules/twenty-config/cache/config-cache.service'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { CONFIG_VARIABLES_INSTANCE_TOKEN } from 'src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants'; import { ConfigValueConverterService } from 'src/engine/core-modules/twenty-config/conversion/config-value-converter.service'; import { DatabaseConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/database-config.driver'; import { ConfigStorageService } from 'src/engine/core-modules/twenty-config/storage/config-storage.service'; -const CONFIG_VARIABLES_INSTANCE = new ConfigVariables(); - @Module({}) export class DatabaseConfigModule { static forRoot(): DynamicModule { @@ -26,8 +25,8 @@ export class DatabaseConfigModule { ConfigStorageService, ConfigValueConverterService, { - provide: ConfigVariables, - useValue: CONFIG_VARIABLES_INSTANCE, + provide: CONFIG_VARIABLES_INSTANCE_TOKEN, + useValue: new ConfigVariables(), }, ], exports: [DatabaseConfigDriver], diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/environment-config.driver.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/environment-config.driver.ts index 894e20090885..b87381f574e2 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/environment-config.driver.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/environment-config.driver.ts @@ -1,15 +1,16 @@ -import { Injectable } from '@nestjs/common'; +import { Inject, Injectable } from '@nestjs/common'; import { ConfigService } from '@nestjs/config'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { CONFIG_VARIABLES_INSTANCE_TOKEN } from 'src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants'; @Injectable() export class EnvironmentConfigDriver { - private readonly defaultConfigVariables: ConfigVariables; - - constructor(private readonly configService: ConfigService) { - this.defaultConfigVariables = new ConfigVariables(); - } + constructor( + private readonly configService: ConfigService, + @Inject(CONFIG_VARIABLES_INSTANCE_TOKEN) + private readonly defaultConfigVariables: ConfigVariables, + ) {} get(key: T): ConfigVariables[T] { return this.configService.get( diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts index 4b6ba8c0fbe8..d650e8d62789 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.module.ts @@ -5,6 +5,7 @@ import { ConfigVariables, validate, } from 'src/engine/core-modules/twenty-config/config-variables'; +import { CONFIG_VARIABLES_INSTANCE_TOKEN } from 'src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants'; import { DatabaseConfigModule } from 'src/engine/core-modules/twenty-config/drivers/database-config.module'; import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; import { ConfigurableModuleClass } from 'src/engine/core-modules/twenty-config/twenty-config.module-definition'; @@ -37,7 +38,7 @@ export class TwentyConfigModule extends ConfigurableModuleClass { TwentyConfigService, EnvironmentConfigDriver, { - provide: ConfigVariables, + provide: CONFIG_VARIABLES_INSTANCE_TOKEN, useValue: new ConfigVariables(), }, ], diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts index bedc42412d0a..b5ed1783f088 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.ts @@ -1,5 +1,4 @@ import { Injectable, Logger, Optional } from '@nestjs/common'; -import { ConfigService } from '@nestjs/config'; import { isString } from 'class-validator'; @@ -22,10 +21,10 @@ export class TwentyConfigService { constructor( private readonly environmentConfigDriver: EnvironmentConfigDriver, @Optional() private readonly databaseConfigDriver: DatabaseConfigDriver, - private readonly configService: ConfigService, ) { - const isConfigVariablesInDbEnabled = - this.configService.get('IS_CONFIG_VARIABLES_IN_DB_ENABLED') === 'true'; + const isConfigVariablesInDbEnabled = this.environmentConfigDriver.get( + 'IS_CONFIG_VARIABLES_IN_DB_ENABLED', + ); this.isDatabaseDriverActive = isConfigVariablesInDbEnabled && !!this.databaseConfigDriver; From af26e740c0566d2e18cbee10faaa1cbde86a6840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?F=C3=A9lix=20Malfait?= Date: Sat, 26 Apr 2025 08:19:06 +0200 Subject: [PATCH 66/70] Fix one tesdt --- .../__tests__/config-value-converter.service.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/__tests__/config-value-converter.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/__tests__/config-value-converter.service.spec.ts index 705c0509ab8c..e10a890c113a 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/__tests__/config-value-converter.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/conversion/__tests__/config-value-converter.service.spec.ts @@ -2,6 +2,7 @@ import { LogLevel } from '@nestjs/common'; import { Test, TestingModule } from '@nestjs/testing'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { CONFIG_VARIABLES_INSTANCE_TOKEN } from 'src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants'; import { ConfigValueConverterService } from 'src/engine/core-modules/twenty-config/conversion/config-value-converter.service'; import { ConfigVariablesGroup } from 'src/engine/core-modules/twenty-config/enums/config-variables-group.enum'; import { configTransformers } from 'src/engine/core-modules/twenty-config/utils/config-transformers.util'; @@ -39,7 +40,7 @@ describe('ConfigValueConverterService', () => { providers: [ ConfigValueConverterService, { - provide: ConfigVariables, + provide: CONFIG_VARIABLES_INSTANCE_TOKEN, useValue: mockConfigVariables, }, ], From f3c1690266efb0c0c4113d8d93413a1581407349 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Sat, 26 Apr 2025 12:01:42 +0530 Subject: [PATCH 67/70] fix --- .../drivers/__tests__/environment-config.driver.spec.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts index e3d187835871..1efc5fe90484 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/drivers/__tests__/environment-config.driver.spec.ts @@ -2,6 +2,7 @@ import { ConfigService } from '@nestjs/config'; import { Test, TestingModule } from '@nestjs/testing'; import { ConfigVariables } from 'src/engine/core-modules/twenty-config/config-variables'; +import { CONFIG_VARIABLES_INSTANCE_TOKEN } from 'src/engine/core-modules/twenty-config/constants/config-variables-instance-tokens.constants'; import { EnvironmentConfigDriver } from 'src/engine/core-modules/twenty-config/drivers/environment-config.driver'; describe('EnvironmentConfigDriver', () => { @@ -18,6 +19,10 @@ describe('EnvironmentConfigDriver', () => { get: jest.fn(), }, }, + { + provide: CONFIG_VARIABLES_INSTANCE_TOKEN, + useValue: new ConfigVariables(), + }, ], }).compile(); From e2344f029df43437c5c71f065eab9284e498eda4 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Sat, 26 Apr 2025 12:20:08 +0530 Subject: [PATCH 68/70] fix test --- .../twenty-config.service.spec.ts | 63 ++++++++++--------- .../utils/apply-basic-validators.util.ts | 12 ++-- 2 files changed, 41 insertions(+), 34 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts index 3346a3b56bcf..f8e2ab94becb 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/twenty-config.service.spec.ts @@ -60,6 +60,16 @@ const mockConfigVarMetadata = { // Setup with database driver const setupTestModule = async (isDatabaseConfigEnabled = true) => { + const configServiceMock = { + get: jest.fn().mockImplementation((key) => { + if (key === 'IS_CONFIG_VARIABLES_IN_DB_ENABLED') { + return isDatabaseConfigEnabled ? 'true' : 'false'; + } + + return undefined; + }), + }; + const module: TestingModule = await Test.createTestingModule({ providers: [ TwentyConfigService, @@ -74,20 +84,14 @@ const setupTestModule = async (isDatabaseConfigEnabled = true) => { { provide: EnvironmentConfigDriver, useValue: { - get: jest.fn(), + get: jest.fn().mockImplementation((key) => { + return configServiceMock.get(key); + }), }, }, { provide: ConfigService, - useValue: { - get: jest.fn().mockImplementation((key) => { - if (key === 'IS_CONFIG_VARIABLES_IN_DB_ENABLED') { - return isDatabaseConfigEnabled ? 'true' : 'false'; - } - - return null; - }), - }, + useValue: configServiceMock, }, ], }).compile(); @@ -105,26 +109,30 @@ const setupTestModule = async (isDatabaseConfigEnabled = true) => { // Setup without database driver const setupTestModuleWithoutDb = async () => { + const configServiceMock = { + get: jest.fn().mockImplementation((key) => { + if (key === 'IS_CONFIG_VARIABLES_IN_DB_ENABLED') { + return 'false'; + } + + return undefined; + }), + }; + const module: TestingModule = await Test.createTestingModule({ providers: [ TwentyConfigService, { provide: EnvironmentConfigDriver, useValue: { - get: jest.fn(), + get: jest.fn().mockImplementation((key) => { + return configServiceMock.get(key); + }), }, }, { provide: ConfigService, - useValue: { - get: jest.fn().mockImplementation((key) => { - if (key === 'IS_CONFIG_VARIABLES_IN_DB_ENABLED') { - return 'false'; - } - - return null; - }), - }, + useValue: configServiceMock, }, ], }).compile(); @@ -177,26 +185,23 @@ describe('TwentyConfigService', () => { describe('constructor', () => { it('should set isDatabaseDriverActive to false when database config is disabled', async () => { - const { service, configService } = await setupTestModuleWithoutDb(); + const { service, environmentConfigDriver } = + await setupTestModuleWithoutDb(); - // Verify ConfigService was called with the right parameter - expect(configService.get).toHaveBeenCalledWith( + expect(environmentConfigDriver.get).toHaveBeenCalledWith( 'IS_CONFIG_VARIABLES_IN_DB_ENABLED', ); - // Test behavior that results from the configuration expect(service.getCacheInfo().usingDatabaseDriver).toBe(false); }); it('should set isDatabaseDriverActive to true when database config is enabled and driver is available', async () => { - const { service, configService } = await setupTestModule(true); + const { service, environmentConfigDriver } = await setupTestModule(true); - // Verify ConfigService was called with the right parameter - expect(configService.get).toHaveBeenCalledWith( + expect(environmentConfigDriver.get).toHaveBeenCalledWith( 'IS_CONFIG_VARIABLES_IN_DB_ENABLED', ); - // Test behavior that results from the configuration expect(service.getCacheInfo().usingDatabaseDriver).toBe(true); }); }); @@ -237,6 +242,8 @@ describe('TwentyConfigService', () => { jest.spyOn(databaseConfigDriver, 'get').mockReturnValue(expectedValue); setPrivateProps(service, { isDatabaseDriverActive: true }); + jest.clearAllMocks(); + const result = service.get(key); expect(result).toBe(expectedValue); diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts index 38a7bb5dd13c..ec02318c6d8b 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts @@ -1,10 +1,10 @@ import { Transform } from 'class-transformer'; import { - IsArray, - IsBoolean, - IsEnum, - IsNumber, - IsString, + IsArray, + IsBoolean, + IsEnum, + IsNumber, + IsString, } from 'class-validator'; import { ConfigVariableOptions } from 'src/engine/core-modules/twenty-config/types/config-variable-options.type'; @@ -41,7 +41,7 @@ export function applyBasicValidators( break; case 'enum': - if (options) { + if (options && Array.isArray(options)) { IsEnum(options)(target, propertyKey); } break; From 649a68ee5973b89160c7a6d28bded7b61ca61409 Mon Sep 17 00:00:00 2001 From: ehconitin Date: Sat, 26 Apr 2025 12:20:41 +0530 Subject: [PATCH 69/70] lint --- .../twenty-config/utils/apply-basic-validators.util.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts index ec02318c6d8b..e517a4052f55 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts @@ -1,10 +1,10 @@ import { Transform } from 'class-transformer'; import { - IsArray, - IsBoolean, - IsEnum, - IsNumber, - IsString, + IsArray, + IsBoolean, + IsEnum, + IsNumber, + IsString, } from 'class-validator'; import { ConfigVariableOptions } from 'src/engine/core-modules/twenty-config/types/config-variable-options.type'; From b37d278d37bf48019c1c00a41be9271ef1eb793b Mon Sep 17 00:00:00 2001 From: ehconitin Date: Sat, 26 Apr 2025 12:22:07 +0530 Subject: [PATCH 70/70] fix test --- .../twenty-config/utils/apply-basic-validators.util.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts index e517a4052f55..38a7bb5dd13c 100644 --- a/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts +++ b/packages/twenty-server/src/engine/core-modules/twenty-config/utils/apply-basic-validators.util.ts @@ -41,7 +41,7 @@ export function applyBasicValidators( break; case 'enum': - if (options && Array.isArray(options)) { + if (options) { IsEnum(options)(target, propertyKey); } break;