From 3f8a09569fd79d3273081bc5c0b370a00e782aa8 Mon Sep 17 00:00:00 2001 From: David Kellner <52860029+kellnerd@users.noreply.github.com> Date: Mon, 12 Aug 2024 18:50:55 +0200 Subject: [PATCH 01/14] refactor(imports): Adapt to the new schema where imports have BBIDs Mostly column, property and table names had to be updated to follow the schema. A few other identifiers have been renamed as well for consistency, although this was not strictly necessary. --- src/func/imports/approve-import.ts | 8 ++-- src/func/imports/create-import.ts | 59 +++++++++++------------- src/func/imports/delete-import.ts | 17 +++---- src/func/imports/discard.ts | 9 ++-- src/func/imports/index.ts | 4 +- src/func/imports/misc.ts | 59 ++++++++++++------------ src/func/imports/recent-imports.ts | 56 +++++++++++----------- src/models/imports/authorImport.ts | 2 +- src/models/imports/editionGroupImport.ts | 2 +- src/models/imports/editionImport.ts | 2 +- src/models/imports/publisherImport.ts | 2 +- src/models/imports/seriesImport.ts | 2 +- src/models/imports/workImport.ts | 2 +- src/types/entity.ts | 1 + src/types/imports.ts | 28 +++++------ src/types/parser.ts | 8 ++-- 16 files changed, 130 insertions(+), 131 deletions(-) diff --git a/src/func/imports/approve-import.ts b/src/func/imports/approve-import.ts index f6be42e0..2d8abfbb 100644 --- a/src/func/imports/approve-import.ts +++ b/src/func/imports/approve-import.ts @@ -37,7 +37,7 @@ interface approveEntityPropsType { export async function approveImport( {orm, transacting, importEntity, editorId}: approveEntityPropsType ): Promise> { - const {source, importId, type: entityType, disambiguationId, aliasSet, + const {source, bbid: pendingEntityBbid, type: entityType, disambiguationId, aliasSet, identifierSetId, annotationId} = importEntity; const {id: aliasSetId} = aliasSet; @@ -78,11 +78,11 @@ export async function approveImport( const newEntity = await new Entity({type: entityType}) .save(null, {transacting}); - const bbid = newEntity.get('bbid'); + const acceptedEntityBbid = newEntity.get('bbid'); const propsToSet = _.extend({ aliasSetId, annotationId, - bbid, + bbid: acceptedEntityBbid, disambiguationId, identifierSetId, revisionId @@ -101,7 +101,7 @@ export async function approveImport( withRelated: ['defaultAlias'] }); - await deleteImport(transacting, importId, bbid); + await deleteImport(transacting, pendingEntityBbid, acceptedEntityBbid); return entity; } diff --git a/src/func/imports/create-import.ts b/src/func/imports/create-import.ts index 20a9b952..8ac191ad 100644 --- a/src/func/imports/create-import.ts +++ b/src/func/imports/create-import.ts @@ -18,8 +18,8 @@ */ -import {ENTITY_TYPES, type EntityTypeString} from '../../types/entity'; -import type {ImportHeaderT, ImportMetadataT, _ImportT} from '../../types/imports'; +import {ENTITY_TYPES, EntityT, type EntityTypeString} from '../../types/entity'; +import type {ImportHeaderT, ImportMetadataT} from '../../types/imports'; import type {ParsedEdition, ParsedEntity, QueuedEntity} from '../../types/parser'; import type {ORM} from '../..'; @@ -27,7 +27,7 @@ import type {Transaction} from '../types'; import _ from 'lodash'; import {camelToSnake} from '../../util'; import {getAdditionalEntityProps} from '../entity'; -import {getOriginSourceId} from './misc'; +import {getExternalSourceId} from './misc'; import {updateAliasSet} from '../alias'; import {updateAnnotation} from '../annotation'; import {updateDisambiguation} from '../disambiguation'; @@ -36,20 +36,18 @@ import {updateLanguageSet} from '../language'; import {updateReleaseEventSet} from '../releaseEvent'; -function createImportRecord(transacting: Transaction, data: _ImportT) { - return transacting.insert(camelToSnake(data)).into('bookbrainz.import').returning('id'); +function createEntityRecord(transacting: Transaction, data: EntityT) { + return transacting.insert(camelToSnake(data)).into('bookbrainz.entity').returning('bbid'); } function createOrUpdateImportMetadata(transacting: Transaction, record: ImportMetadataT) { - return transacting.insert(camelToSnake(record)).into('bookbrainz.link_import') - .onConflict(['origin_source_id', 'origin_id']).merge(); + return transacting.insert(camelToSnake(record)).into('bookbrainz.import_metadata') + .onConflict(['external_source_id', 'external_identifier']).merge(); } function getImportMetadata(transacting: Transaction, externalSourceId: number, externalIdentifier: string) { - return transacting.select('import_id', 'entity_id').from('bookbrainz.link_import').where(camelToSnake({ - originId: externalIdentifier, - originSourceId: externalSourceId - })); + return transacting.select('pending_entity_bbid', 'accepted_entity_bbid').from('bookbrainz.import_metadata') + .where(camelToSnake({externalIdentifier, externalSourceId})); } /** IDs of extra data sets which not all entity types have. */ @@ -93,7 +91,7 @@ function createImportDataRecord(transacting: Transaction, dataSets: DataSetIds, function createOrUpdateImportHeader(transacting: Transaction, record: ImportHeaderT, entityType: EntityTypeString) { const table = `bookbrainz.${_.snakeCase(entityType)}_import_header`; return transacting.insert(camelToSnake(record)).into(table) - .onConflict('import_id').merge(); + .onConflict('bbid').merge(); } async function updateEntityExtraDataSets( @@ -140,8 +138,8 @@ export type ImportStatus = export type ImportResult = { - /** ID of the imported entity (numeric for now, will be a BBID in a future version). */ - importId: number | string; + /** BBID of the pending imported entity. */ + importId: string; /** Import status of the processed entity. */ status: ImportStatus; @@ -156,25 +154,24 @@ export function createImport(orm: ORM, importData: QueuedEntity, { return orm.bookshelf.transaction(async (transacting) => { const {entityType} = importData; - const {alias, annotation, identifiers, disambiguation, source} = importData.data; + const {alias, annotation, identifiers, disambiguation, externalSource} = importData.data; - // Get origin_source - let originSourceId: number = null; + let externalSourceId: number = null; try { - originSourceId = await getOriginSourceId(transacting, source); + externalSourceId = await getExternalSourceId(transacting, externalSource); } catch (err) { // TODO: useless, we are only catching our self-thrown errors here throw new Error(`Error during getting source id - ${err}`); } - const [existingImport] = await getImportMetadata(transacting, originSourceId, importData.originId); + const [existingImport] = await getImportMetadata(transacting, externalSourceId, importData.externalIdentifier); if (existingImport) { - const isPendingImport = !existingImport.entity_id; + const isPendingImport = !existingImport.accepted_entity_bbid; if (existingImportAction === 'skip') { return { - importId: existingImport.import_id, + importId: existingImport.pending_entity_bbid, status: isPendingImport ? 'skipped pending' : 'skipped accepted' }; } @@ -186,14 +183,14 @@ export function createImport(orm: ORM, importData: QueuedEntity, { if (existingImportAction === 'update pending') { // We only want to update pending, but not accepted entities return { - importId: existingImport.import_id, + importId: existingImport.pending_entity_bbid, status: 'skipped accepted' }; } // We also want to create updates for already accepted entities ('update pending and accepted') // TODO: implement this feature in a later version and drop the following temporary return statement return { - importId: existingImport.import_id, + importId: existingImport.pending_entity_bbid, status: 'skipped accepted' }; } @@ -229,11 +226,11 @@ export function createImport(orm: ORM, importData: QueuedEntity, { } // Create import entity (if it is not already existing from a previous import attempt) - let importId: number = existingImport?.import_id; + let importId: string = existingImport?.pending_entity_bbid; if (!importId) { try { - const [idObj] = await createImportRecord(transacting, {type: entityType}); - importId = _.get(idObj, 'id'); + const [idObj] = await createEntityRecord(transacting, {isImport: true, type: entityType}); + importId = _.get(idObj, 'bbid'); } catch (err) { throw new Error(`Failed to create a new import ID: ${err}`); @@ -241,12 +238,12 @@ export function createImport(orm: ORM, importData: QueuedEntity, { } const importMetadata: ImportMetadataT = { - importId, - importMetadata: importData.data.metadata, + additionalData: importData.data.metadata, + externalIdentifier: importData.externalIdentifier, + externalSourceId, importedAt: transacting.raw("timezone('UTC'::TEXT, now())"), lastEdited: importData.lastEdited, - originId: importData.originId, - originSourceId + pendingEntityBbid: importId }; try { @@ -257,7 +254,7 @@ export function createImport(orm: ORM, importData: QueuedEntity, { } try { - await createOrUpdateImportHeader(transacting, {dataId, importId}, entityType); + await createOrUpdateImportHeader(transacting, {bbid: importId, dataId}, entityType); } catch (err) { throw new Error(`Failed to upsert import header: ${err}`); diff --git a/src/func/imports/delete-import.ts b/src/func/imports/delete-import.ts index 89fec124..f6e8221b 100644 --- a/src/func/imports/delete-import.ts +++ b/src/func/imports/delete-import.ts @@ -21,19 +21,20 @@ import {camelToSnake, snakeToCamel} from '../../util'; import type {Transaction} from '../types'; +// TODO: Do we actually want to delete any data of discarded imports? export async function deleteImport( - transacting: Transaction, importId: number, entityId?: string | null | undefined + transacting: Transaction, importId: string, entityId?: string | null | undefined ) { // Get the type of the import const [typeObj] = await transacting.select('type') - .from('bookbrainz.import').where('id', importId); + .from('bookbrainz.entity').where('bbid', importId); const {type: importType} = typeObj; // Get the dataId of the import const [dataIdObj] = await transacting.select('data_id') .from(`bookbrainz.${_.snakeCase(importType)}_import_header`) - .where('import_id', importId); + .where('bbid', importId); const {dataId}: {dataId: number} = snakeToCamel(dataIdObj); // Update link table arguments - if entityId present add it to the args obj @@ -43,23 +44,23 @@ export async function deleteImport( await Promise.all([ // Delete the import header and entity data table records transacting(`bookbrainz.${_.snakeCase(importType)}_import_header`) - .where('import_id', importId).del() + .where('bbid', importId).del() .then(() => transacting(`bookbrainz.${_.snakeCase(importType)}_data`) .where('id', dataId).del()), // Delete the discard votes transacting('bookbrainz.discard_votes') - .where('import_id', importId).del(), + .where('import_bbid', importId).del(), /* Update the link import record: -> set importId as null -> if entityId provided, update it */ - transacting('bookbrainz.link_import') - .where('import_id', importId) + transacting('bookbrainz.import_metadata') + .where('pending_entity_bbid', importId) .update(camelToSnake(linkUpdateObj)) ]); // Finally delete the import table record - return transacting('bookbrainz.import').where('id', importId).del(); + return transacting('bookbrainz.entity').where('bbid', importId).del(); } diff --git a/src/func/imports/discard.ts b/src/func/imports/discard.ts index cd33b6ac..68691065 100644 --- a/src/func/imports/discard.ts +++ b/src/func/imports/discard.ts @@ -25,11 +25,11 @@ import {deleteImport} from './delete-import'; export const DISCARD_LIMIT = 1; export async function discardVotesCast( - transacting: Transaction, importId: number + transacting: Transaction, importId: string ): Promise> { const votes = await transacting.select('*') .from('bookbrainz.discard_votes') - .where('import_id', importId); + .where('import_bbid', importId); return votes.map(snakeToCamel); } @@ -39,19 +39,20 @@ export async function discardVotesCast( * it returns a Promise that resolves to true, else it returns an promise that * resolves to false. * @param {Transaction} transacting - The knex Transacting object - * @param {number} importId - Id of the import + * @param {string} importId - BBID of the import * @param {number} editorId - Id of the user casting the vote * @returns {Promise} - Promise if records has been deleted or * Promise if the record is still present */ export async function castDiscardVote( - transacting: Transaction, importId: number, editorId: number + transacting: Transaction, importId: string, editorId: number ): Promise { const votesCast = await discardVotesCast(transacting, importId); // If editor has already cast the vote, reject the vote for (const vote of votesCast) { if (vote.editor_id === editorId) { + // TODO: This property can't exist since we are using `snakeToCamel`? throw new Error('Already cast the vote'); } } diff --git a/src/func/imports/index.ts b/src/func/imports/index.ts index 6c073187..2a536823 100644 --- a/src/func/imports/index.ts +++ b/src/func/imports/index.ts @@ -19,8 +19,8 @@ export {DISCARD_LIMIT, castDiscardVote, discardVotesCast} from './discard'; export { - getImportDetails, getOriginSourceFromId, getOriginSourceId, - originSourceMapping + getImportMetadata, getExternalSourceFromId, getExternalSourceId, + getExternalSourceMapping } from './misc'; export {getRecentImports, getTotalImports} from './recent-imports'; export {approveImport} from './approve-import'; diff --git a/src/func/imports/misc.ts b/src/func/imports/misc.ts index 107be615..553ef198 100644 --- a/src/func/imports/misc.ts +++ b/src/func/imports/misc.ts @@ -17,16 +17,17 @@ */ import * as _ from 'lodash'; +import type {ImportMetadataWithSourceT} from '../../types/imports'; import type {Transaction} from '../types'; import {snakeToCamel} from '../../util'; -export async function originSourceMapping( +export async function getExternalSourceMapping( transacting: Transaction, idAsKey?: boolean ) { - const mappingRecord = await transacting.select('*') - .from('bookbrainz.origin_source'); - return mappingRecord.reduce( + const externalSources = await transacting.select('*') + .from('bookbrainz.external_source'); + return externalSources.reduce( (mapping, {id, name}) => { if (idAsKey) { return _.assign(mapping, {[id]: name}); @@ -36,75 +37,75 @@ export async function originSourceMapping( ); } -export async function getOriginSourceId( - transacting: Transaction, source: string +export async function getExternalSourceId( + transacting: Transaction, sourceName: string ): Promise { - let originSourceId: number | null | undefined = null; + let externalSourceId: number | null | undefined = null; try { const [idObj] = await transacting.select('id') - .from('bookbrainz.origin_source') - .where('name', '=', source); - originSourceId = _.get(idObj, 'id'); + .from('bookbrainz.external_source') + .where('name', '=', sourceName); + externalSourceId = _.get(idObj, 'id'); } catch (err) { // Should error loudly if anything goes wrong throw new Error( - `Error while extracting origin source using ${source} - ${err}` + `Error while extracting external source using ${sourceName} - ${err}` ); } // Create the data source if it does not exist - if (!originSourceId) { + if (!externalSourceId) { try { - const [idObj] = await transacting.insert([{name: source}]) - .into('bookbrainz.origin_source') + const [idObj] = await transacting.insert([{name: sourceName}]) + .into('bookbrainz.external_source') .returning('id'); - originSourceId = _.get(idObj, 'id'); + externalSourceId = _.get(idObj, 'id'); } catch (err) { // Should error loudly if anything goes wrong throw new Error( - `Error while creating a new source ${source} - ${err}` + `Error while creating a new source ${sourceName} - ${err}` ); } } // Returning the {id} of the origin source - return originSourceId || null; + return externalSourceId || null; } -export async function getOriginSourceFromId( - transacting: Transaction, originSourceId: number +export async function getExternalSourceFromId( + transacting: Transaction, externalSourceId: number ): Promise { // Should error loudly if anything goes wrong const [nameObj] = await transacting.select('name') - .from('bookbrainz.origin_source') - .where('id', originSourceId); + .from('bookbrainz.external_source') + .where('id', externalSourceId); if (!nameObj || !nameObj.name) { throw new Error( - `No source found with the given source Id ${originSourceId}` + `No source found with the given source ID ${externalSourceId}` ); } return nameObj.name; } -export async function getImportDetails( - transacting: Transaction, importId: number -): Promise> { +export async function getImportMetadata( + transacting: Transaction, importId: string +): Promise { // Should error loudly if anything goes wrong const [details] = await transacting.select('*') - .from('bookbrainz.link_import') - .where('import_id', importId); + .from('bookbrainz.import_metadata') + .where('pending_entity_bbid', importId); if (!details) { throw new Error(`Details for the import ${importId} not found`); } - details.source = await getOriginSourceFromId( - transacting, details.origin_source_id + details.source = await getExternalSourceFromId( + transacting, details.external_source_id ); return snakeToCamel(details); diff --git a/src/func/imports/recent-imports.ts b/src/func/imports/recent-imports.ts index 3c45c657..25d0d807 100644 --- a/src/func/imports/recent-imports.ts +++ b/src/func/imports/recent-imports.ts @@ -21,8 +21,8 @@ import {ENTITY_TYPES, type EntityTypeString} from '../../types/entity'; import type {ORM} from '../..'; import type {Transaction} from '../types'; import {getAliasByIds} from '../alias'; +import {getExternalSourceMapping} from './misc'; import moment from 'moment'; -import {originSourceMapping} from './misc'; import {snakeToCamel} from '../../util'; /** getRecentImportIdsByType - @@ -40,24 +40,24 @@ async function getRecentImportUtilData( // Extract recent imports, types and import timeStamp const recentImportUtilsData = await transacting.select( - 'link.imported_at', - 'link.origin_source_id', - 'bookbrainz.import.id', - 'bookbrainz.import.type' + 'meta.imported_at', + 'meta.external_source_id', + 'bookbrainz.entity.bbid', + 'bookbrainz.entity.type' ) - .from('bookbrainz.import') + .from('bookbrainz.entity') .join( transacting.select( - 'import_id', 'imported_at', 'origin_source_id' + 'pending_entity_bbid', 'imported_at', 'external_source_id' ) - .from('bookbrainz.link_import') + .from('bookbrainz.import_metadata') .orderBy('imported_at') - .whereNot('import_id', null) + .whereNot('pending_entity_bbid', null) .limit(limit) .offset(offset) - .as('link'), - 'link.import_id', - 'bookbrainz.import.id' + .as('meta'), + 'meta.pending_entity_bbid', + 'bookbrainz.entity.bbid' ); /* Construct importHolder object (holds importIds classified by their types) @@ -72,21 +72,21 @@ async function getRecentImportUtilData( imports into the their respective types => timestampMap: Object{importId: imported_at} This holds a mapping of all imports and their imported_at timestamps - => originIdMap: Object{originId: origin_id} - This holds a mapping of all imports and their origin_ids + => externalSourceIdMap: Object{importId: external_source_id} + This holds a mapping of all imports and their external_source_ids */ return recentImportUtilsData.reduce((holder: Record, data: any) => { holder.importHolder[data.type].push(data.id); - holder.originIdMap[data.id] = data.origin_source_id; + holder.externalSourceIdMap[data.id] = data.external_source_id; holder.timeStampMap[data.id] = data.imported_at; return holder; - }, {importHolder, originIdMap: {}, timeStampMap: {}}); + }, {externalSourceIdMap: {}, importHolder, timeStampMap: {}}); } function getRecentImportsByType(transacting: Transaction, type: EntityTypeString, importIds: string[]) { return transacting.select('*') .from(`bookbrainz.${_.snakeCase(type)}_import`) - .whereIn('import_id', importIds); + .whereIn('bbid', importIds); } export async function getRecentImports( @@ -96,10 +96,10 @@ export async function getRecentImports( => importHolder - holds recentImports classified by entity type => timeStampMap - holds value importedAt value in object with importId as key - => originIdMap - holds value originId value in object with importId as + => externalSourceIdMap - holds value originId value in object with importId as key */ - const {importHolder: recentImportIdsByType, timeStampMap, originIdMap} = + const {importHolder: recentImportIdsByType, timeStampMap, externalSourceIdMap} = await getRecentImportUtilData(transacting, limit, offset); /* Fetch imports for each entity type using their importIds @@ -125,19 +125,19 @@ export async function getRecentImports( /* Add timestamp, source and defaultAlias to the recentImports Previously while getting utils data we fetched a mapping of importId to - timestamp and originId. + timestamp and externalSourceId. We also fetched map of aliasId to alias object. Now using those we populate our final object */ - // First, get the origin mapping => {originId: name} - const sourceMapping = await originSourceMapping(transacting, true); + // First, get the external source mapping => {id: name} + const sourceMapping = await getExternalSourceMapping(transacting, true); return recentImports.map(recentImport => { // Add timestamp recentImport.importedAt = - moment(timeStampMap[recentImport.import_id]).format('YYYY-MM-DD'); + moment(timeStampMap[recentImport.bbid]).format('YYYY-MM-DD'); - // Add origin source - const originId = originIdMap[recentImport.import_id]; - recentImport.source = sourceMapping[originId]; + // Add name of external source + const externalSourceId = externalSourceIdMap[recentImport.bbid]; + recentImport.source = sourceMapping[externalSourceId]; // Add default alias const defaultAliasId = _.get(recentImport, 'default_alias_id'); @@ -147,8 +147,10 @@ export async function getRecentImports( }); } +// TODO: Besides being expensive to compute, this will no longer be accurate once +// we keep the pending entity data column populated for potential source updates. export async function getTotalImports(transacting: Transaction) { const [{count}] = - await transacting('bookbrainz.link_import').count('import_id'); + await transacting('bookbrainz.import_metadata').count('pending_entity_bbid'); return parseInt(count as string, 10); } diff --git a/src/models/imports/authorImport.ts b/src/models/imports/authorImport.ts index 7a603671..ab1cc594 100644 --- a/src/models/imports/authorImport.ts +++ b/src/models/imports/authorImport.ts @@ -26,7 +26,7 @@ export default function author(bookshelf: Bookshelf) { defaultAlias() { return this.belongsTo('Alias', 'default_alias_id'); }, - idAttribute: 'import_id', + idAttribute: 'bbid', tableName: 'bookbrainz.author_import' }); diff --git a/src/models/imports/editionGroupImport.ts b/src/models/imports/editionGroupImport.ts index 99c44e85..b2d2778a 100644 --- a/src/models/imports/editionGroupImport.ts +++ b/src/models/imports/editionGroupImport.ts @@ -26,7 +26,7 @@ export default function editionGroup(bookshelf: Bookshelf) { defaultAlias() { return this.belongsTo('Alias', 'default_alias_id'); }, - idAttribute: 'import_id', + idAttribute: 'bbid', tableName: 'bookbrainz.edition_group_import' }); diff --git a/src/models/imports/editionImport.ts b/src/models/imports/editionImport.ts index 11d8647c..23d729a3 100644 --- a/src/models/imports/editionImport.ts +++ b/src/models/imports/editionImport.ts @@ -26,7 +26,7 @@ export default function edition(bookshelf: Bookshelf) { defaultAlias() { return this.belongsTo('Alias', 'default_alias_id'); }, - idAttribute: 'import_id', + idAttribute: 'bbid', tableName: 'bookbrainz.edition_import' }); diff --git a/src/models/imports/publisherImport.ts b/src/models/imports/publisherImport.ts index 0401c90a..000908e2 100644 --- a/src/models/imports/publisherImport.ts +++ b/src/models/imports/publisherImport.ts @@ -26,7 +26,7 @@ export default function publisher(bookshelf: Bookshelf) { defaultAlias() { return this.belongsTo('Alias', 'default_alias_id'); }, - idAttribute: 'import_id', + idAttribute: 'bbid', tableName: 'bookbrainz.publisher_import' }); diff --git a/src/models/imports/seriesImport.ts b/src/models/imports/seriesImport.ts index 3b76bdce..6c1e4627 100644 --- a/src/models/imports/seriesImport.ts +++ b/src/models/imports/seriesImport.ts @@ -26,7 +26,7 @@ export default function series(bookshelf: Bookshelf) { defaultAlias() { return this.belongsTo('Alias', 'default_alias_id'); }, - idAttribute: 'import_id', + idAttribute: 'bbid', tableName: 'bookbrainz.series_import' }); diff --git a/src/models/imports/workImport.ts b/src/models/imports/workImport.ts index 2cbf9461..e1732428 100644 --- a/src/models/imports/workImport.ts +++ b/src/models/imports/workImport.ts @@ -25,7 +25,7 @@ export default function work(bookshelf: Bookshelf) { defaultAlias() { return this.belongsTo('Alias', 'default_alias_id'); }, - idAttribute: 'import_id', + idAttribute: 'bbid', tableName: 'bookbrainz.work_import' }); diff --git a/src/types/entity.ts b/src/types/entity.ts index 30dc7691..4d2b0f9c 100644 --- a/src/types/entity.ts +++ b/src/types/entity.ts @@ -34,6 +34,7 @@ export type EntityTypeString = typeof ENTITY_TYPES[number]; // TODO: incomplete export type EntityT = { + isImport?: boolean, type: EntityTypeString, }; diff --git a/src/types/imports.ts b/src/types/imports.ts index 5fdd2c05..e9b344a7 100644 --- a/src/types/imports.ts +++ b/src/types/imports.ts @@ -16,21 +16,12 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ -import {EntityTypeString} from './entity'; import {IdentifierT} from './identifiers'; import type {Knex} from 'knex'; -import {WithId} from './utils'; -// TODO: Drop type once we merge the `import` table into the `entity` table -export type _ImportT = { - type: EntityTypeString; -}; - -export type _ImportWithIdT = WithId<_ImportT>; - export type ImportHeaderT = { - importId: number; + bbid: string; dataId: number; }; @@ -49,11 +40,10 @@ export type AdditionalImportDataT = { [custom: string]: any; }; -/** Type for the `link_import` table, which should be renamed for clarity (TODO). */ export type ImportMetadataT = { - importId: number; - originSourceId: number; - originId: string; + pendingEntityBbid: string; + externalSourceId: number; + externalIdentifier: string; /** TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT timezone('UTC'::TEXT, now()) */ importedAt?: Knex.Raw; @@ -62,8 +52,14 @@ export type ImportMetadataT = { lastEdited: string; /** UUID */ - entityId?: string; + acceptedEntityBbid?: string; /** JSONB */ - importMetadata: AdditionalImportDataT; + additionalData: AdditionalImportDataT; +}; + +export type ImportMetadataWithSourceT = ImportMetadataT & { + + /** Name of the external source. */ + source: string; }; diff --git a/src/types/parser.ts b/src/types/parser.ts index 8d914f1c..4191249d 100644 --- a/src/types/parser.ts +++ b/src/types/parser.ts @@ -35,9 +35,9 @@ type ParsedBaseEntity = { disambiguation?: string; identifiers: IdentifierT[]; metadata: AdditionalImportDataT; - source: string; + externalSource: string; + externalIdentifier?: string; lastEdited?: string; - originId?: string; }; export type ParsedAuthor = ParsedBaseEntity & { @@ -100,7 +100,7 @@ export type ParsedEntity = export type QueuedEntity = { data: T; entityType: EntityTypeString; - source: string; + externalSource: string; + externalIdentifier?: string; lastEdited?: string; - originId?: string; }; From 2578a5ebda064f6d0ce751736b0540f5914a44b1 Mon Sep 17 00:00:00 2001 From: David Kellner <52860029+kellnerd@users.noreply.github.com> Date: Mon, 19 Aug 2024 17:30:01 +0200 Subject: [PATCH 02/14] fix(imports): Remove unused property from function parameter type It is never used and never passed, but causes a TS error. --- src/func/create-entity.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/func/create-entity.ts b/src/func/create-entity.ts index d17fcbb1..11894dda 100644 --- a/src/func/create-entity.ts +++ b/src/func/create-entity.ts @@ -54,8 +54,7 @@ interface CreateEntityPropsType { orm: ORM, transacting: Transaction, editorId: string, - entityData: ExtraEntityDataType, - entityType: EntityTypeString + entityData: ExtraEntityDataType } // TODO: function is only used to approve imports, check whether type error below is critical From 39f0c4386ee772c869e983d3c3ebe17dd0ee4c0d Mon Sep 17 00:00:00 2001 From: David Kellner <52860029+kellnerd@users.noreply.github.com> Date: Mon, 19 Aug 2024 17:31:54 +0200 Subject: [PATCH 03/14] fix(imports): Properly load recent imports after schema change --- src/func/imports/recent-imports.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/func/imports/recent-imports.ts b/src/func/imports/recent-imports.ts index 25d0d807..ab8447d1 100644 --- a/src/func/imports/recent-imports.ts +++ b/src/func/imports/recent-imports.ts @@ -46,6 +46,7 @@ async function getRecentImportUtilData( 'bookbrainz.entity.type' ) .from('bookbrainz.entity') + .where('is_import', true) .join( transacting.select( 'pending_entity_bbid', 'imported_at', 'external_source_id' @@ -76,9 +77,9 @@ async function getRecentImportUtilData( This holds a mapping of all imports and their external_source_ids */ return recentImportUtilsData.reduce((holder: Record, data: any) => { - holder.importHolder[data.type].push(data.id); - holder.externalSourceIdMap[data.id] = data.external_source_id; - holder.timeStampMap[data.id] = data.imported_at; + holder.importHolder[data.type].push(data.bbid); + holder.externalSourceIdMap[data.bbid] = data.external_source_id; + holder.timeStampMap[data.bbid] = data.imported_at; return holder; }, {externalSourceIdMap: {}, importHolder, timeStampMap: {}}); } From 56df87e5130ca099b8169ae790a517372f7a82fe Mon Sep 17 00:00:00 2001 From: David Kellner <52860029+kellnerd@users.noreply.github.com> Date: Mon, 19 Aug 2024 17:35:22 +0200 Subject: [PATCH 04/14] refactor(imports): Avoid re-throwing our own errors The error message is already specific enough, no need to make it longer. --- src/func/imports/create-import.ts | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/func/imports/create-import.ts b/src/func/imports/create-import.ts index 8ac191ad..a9539dfb 100644 --- a/src/func/imports/create-import.ts +++ b/src/func/imports/create-import.ts @@ -156,15 +156,7 @@ export function createImport(orm: ORM, importData: QueuedEntity, { const {entityType} = importData; const {alias, annotation, identifiers, disambiguation, externalSource} = importData.data; - let externalSourceId: number = null; - - try { - externalSourceId = await getExternalSourceId(transacting, externalSource); - } - catch (err) { - // TODO: useless, we are only catching our self-thrown errors here - throw new Error(`Error during getting source id - ${err}`); - } + const externalSourceId: number = await getExternalSourceId(transacting, externalSource); const [existingImport] = await getImportMetadata(transacting, externalSourceId, importData.externalIdentifier); if (existingImport) { From f1b2cf356851968053f1a2b2be76baeedac8828b Mon Sep 17 00:00:00 2001 From: David Kellner <52860029+kellnerd@users.noreply.github.com> Date: Mon, 19 Aug 2024 18:38:19 +0200 Subject: [PATCH 05/14] fix(imports): Adapt deletion process to the new schema In the end we of course want to reuse `bbid` and `data_id` of accepted entities instead of duplicating the data and deleting the original. --- src/func/imports/delete-import.ts | 10 +++++----- src/func/imports/discard.ts | 14 +++++++------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/func/imports/delete-import.ts b/src/func/imports/delete-import.ts index f6e8221b..5622e971 100644 --- a/src/func/imports/delete-import.ts +++ b/src/func/imports/delete-import.ts @@ -21,13 +21,13 @@ import {camelToSnake, snakeToCamel} from '../../util'; import type {Transaction} from '../types'; -// TODO: Do we actually want to delete any data of discarded imports? +// TODO: Don't call this function on approval, we want to reuse BBID and data of approved imports! export async function deleteImport( transacting: Transaction, importId: string, entityId?: string | null | undefined ) { // Get the type of the import const [typeObj] = await transacting.select('type') - .from('bookbrainz.entity').where('bbid', importId); + .from('bookbrainz.entity').where(camelToSnake({bbid: importId, isImport: true})); const {type: importType} = typeObj; // Get the dataId of the import @@ -38,8 +38,8 @@ export async function deleteImport( const {dataId}: {dataId: number} = snakeToCamel(dataIdObj); // Update link table arguments - if entityId present add it to the args obj - const linkUpdateObj: {importId: null, entityId?: string} = - entityId ? {entityId, importId: null} : {importId: null}; + const metadataUpdate: {pendingEntityBbid: null, acceptedEntityBbid?: string} = + entityId ? {acceptedEntityBbid: entityId, pendingEntityBbid: null} : {pendingEntityBbid: null}; await Promise.all([ // Delete the import header and entity data table records @@ -58,7 +58,7 @@ export async function deleteImport( -> if entityId provided, update it */ transacting('bookbrainz.import_metadata') .where('pending_entity_bbid', importId) - .update(camelToSnake(linkUpdateObj)) + .update(camelToSnake(metadataUpdate)) ]); // Finally delete the import table record diff --git a/src/func/imports/discard.ts b/src/func/imports/discard.ts index 68691065..f33dcc5f 100644 --- a/src/func/imports/discard.ts +++ b/src/func/imports/discard.ts @@ -25,11 +25,11 @@ import {deleteImport} from './delete-import'; export const DISCARD_LIMIT = 1; export async function discardVotesCast( - transacting: Transaction, importId: string + transacting: Transaction, importBbid: string ): Promise> { const votes = await transacting.select('*') .from('bookbrainz.discard_votes') - .where('import_bbid', importId); + .where('import_bbid', importBbid); return votes.map(snakeToCamel); } @@ -39,15 +39,15 @@ export async function discardVotesCast( * it returns a Promise that resolves to true, else it returns an promise that * resolves to false. * @param {Transaction} transacting - The knex Transacting object - * @param {string} importId - BBID of the import + * @param {string} importBbid - BBID of the import * @param {number} editorId - Id of the user casting the vote * @returns {Promise} - Promise if records has been deleted or * Promise if the record is still present */ export async function castDiscardVote( - transacting: Transaction, importId: string, editorId: number + transacting: Transaction, importBbid: string, editorId: number ): Promise { - const votesCast = await discardVotesCast(transacting, importId); + const votesCast = await discardVotesCast(transacting, importBbid); // If editor has already cast the vote, reject the vote for (const vote of votesCast) { @@ -59,13 +59,13 @@ export async function castDiscardVote( // If cast vote is decisive one, delete the records if (votesCast.length === DISCARD_LIMIT) { - await deleteImport(transacting, importId); + await deleteImport(transacting, importBbid); // The record been deleted return true; } else if (votesCast.length < DISCARD_LIMIT) { // Cast vote if it's below the limit - await transacting.insert(camelToSnake({editorId, importId})) + await transacting.insert(camelToSnake({editorId, importBbid})) .into('bookbrainz.discard_votes'); } else { From 532da0491137d4a37b535cc73bcc3fd4aa3caa77 Mon Sep 17 00:00:00 2001 From: David Kellner <52860029+kellnerd@users.noreply.github.com> Date: Thu, 12 Sep 2024 19:25:17 +0200 Subject: [PATCH 06/14] refactor(imports): Adapt approval to the new way imports are loaded Also make `ImportMetadataT` more versatile/useful. --- src/func/imports/approve-import.ts | 6 ++++-- src/types/imports.ts | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/func/imports/approve-import.ts b/src/func/imports/approve-import.ts index 2d8abfbb..b6c06baf 100644 --- a/src/func/imports/approve-import.ts +++ b/src/func/imports/approve-import.ts @@ -20,6 +20,7 @@ import * as _ from 'lodash'; import { getAdditionalEntityProps, getEntityModelByType, getEntitySetMetadataByType } from '../entity'; +import type {ImportMetadataWithSourceT} from '../../types/imports'; import type {ORM} from '../..'; import type {Transaction} from '../types'; import {createNote} from '../note'; @@ -37,8 +38,9 @@ interface approveEntityPropsType { export async function approveImport( {orm, transacting, importEntity, editorId}: approveEntityPropsType ): Promise> { - const {source, bbid: pendingEntityBbid, type: entityType, disambiguationId, aliasSet, + const {bbid: pendingEntityBbid, type: entityType, disambiguationId, aliasSet, identifierSetId, annotationId} = importEntity; + const metadata: ImportMetadataWithSourceT = importEntity.importMetadata; const {id: aliasSetId} = aliasSet; const {Annotation, Entity, Revision} = orm; @@ -59,7 +61,7 @@ export async function approveImport( .save({lastRevisionId: revisionId}, {transacting}); } - const note = `Approved from automatically imported record of ${source}`; + const note = `Approved from automatically imported record of ${metadata.source}`; // Create a new note promise const notePromise = createNote(orm, note, editorId, revision, transacting); diff --git a/src/types/imports.ts b/src/types/imports.ts index e9b344a7..f3a4e8cd 100644 --- a/src/types/imports.ts +++ b/src/types/imports.ts @@ -41,18 +41,18 @@ export type AdditionalImportDataT = { }; export type ImportMetadataT = { - pendingEntityBbid: string; + pendingEntityBbid: string | null; externalSourceId: number; externalIdentifier: string; /** TIMESTAMP WITHOUT TIME ZONE NOT NULL DEFAULT timezone('UTC'::TEXT, now()) */ - importedAt?: Knex.Raw; + importedAt?: string; /** TIMESTAMP WITHOUT TIME ZONE */ lastEdited: string; /** UUID */ - acceptedEntityBbid?: string; + acceptedEntityBbid?: string | null; /** JSONB */ additionalData: AdditionalImportDataT; From b38e35877db398b2c1e82b5df665ce9957b7d1a4 Mon Sep 17 00:00:00 2001 From: David Kellner <52860029+kellnerd@users.noreply.github.com> Date: Fri, 13 Sep 2024 11:30:01 +0200 Subject: [PATCH 07/14] fix(imports): Remove raw SQL default value It is not needed and it is causing a type error since the last commit. --- src/func/imports/create-import.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/func/imports/create-import.ts b/src/func/imports/create-import.ts index a9539dfb..7c5372d6 100644 --- a/src/func/imports/create-import.ts +++ b/src/func/imports/create-import.ts @@ -233,7 +233,6 @@ export function createImport(orm: ORM, importData: QueuedEntity, { additionalData: importData.data.metadata, externalIdentifier: importData.externalIdentifier, externalSourceId, - importedAt: transacting.raw("timezone('UTC'::TEXT, now())"), lastEdited: importData.lastEdited, pendingEntityBbid: importId }; From d1f5c4839982dcb4f312a1c6b7b83dd4a9e4346e Mon Sep 17 00:00:00 2001 From: David Kellner <52860029+kellnerd@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:50:33 +0100 Subject: [PATCH 08/14] chore(deps): Install kysely --- package.json | 1 + yarn.lock | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/package.json b/package.json index 9b2e2dcb..7a137baa 100644 --- a/package.json +++ b/package.json @@ -71,6 +71,7 @@ "deep-diff": "^1.0.2", "immutable": "^3.8.2", "knex": "^2.4.2", + "kysely": "^0.27.4", "lodash": "^4.17.21", "moment": "^2.29.1", "pg": "^8.6.0", diff --git a/yarn.lock b/yarn.lock index d55df505..fb85179e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2795,6 +2795,11 @@ knex@^2.4.2: tarn "^3.0.2" tildify "2.0.0" +kysely@^0.27.4: + version "0.27.4" + resolved "https://registry.yarnpkg.com/kysely/-/kysely-0.27.4.tgz#96a0285467b380948b4de03b20d87e82d797449b" + integrity sha512-dyNKv2KRvYOQPLCAOCjjQuCk4YFd33BvGdf/o5bC7FiW+BB6snA81Zt+2wT9QDFzKqxKa5rrOmvlK/anehCcgA== + levn@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade" From 290920129c270a7ecbf1f546ba3f06e2ef770b54 Mon Sep 17 00:00:00 2001 From: David Kellner <52860029+kellnerd@users.noreply.github.com> Date: Mon, 28 Oct 2024 14:54:10 +0100 Subject: [PATCH 09/14] chore(eslint): Drop comma-dangle rule The rule to disallow trailing commas often forces you to pointlessly touch neighboring lines when you add or remove items. If we want to enforce consistent dangling commas, we should rather force them to always be present. But since our whole codebase follows this rule, it would be better to have a grace period to gradually add commas. https://github.com/metabrainz/bookbrainz-site/commit/2fc11a22d2750b320fe48dc6a15ccab393137472 --- .eslintrc.js | 1 - 1 file changed, 1 deletion(-) diff --git a/.eslintrc.js b/.eslintrc.js index e5d8d615..606596b0 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -200,7 +200,6 @@ const stylisticIssuesRules = { properties: 'always' } ], - 'comma-dangle': ERROR, 'comma-spacing': ERROR, 'comma-style': ERROR, 'computed-property-spacing': ERROR, From 05446cd86c3f1dcde3378aeab9e248d04f4e6284 Mon Sep 17 00:00:00 2001 From: David Kellner <52860029+kellnerd@users.noreply.github.com> Date: Mon, 28 Oct 2024 15:37:01 +0100 Subject: [PATCH 10/14] feat: Generate types for DB schema and setup kysely Run the kysely-codegen task to regenerate the types. Don't forget to reformat the generated code to follow our linting rules. Since this is using DB introspection, you have to specify a DB connection string via the --url param or the DATABASE_URL env var: https://github.com/RobinBlomberg/kysely-codegen --- package.json | 2 + src/index.ts | 13 + src/types/schema.ts | 1034 +++++++++++++++++++++++++++++++++++++++++++ yarn.lock | 140 +++++- 4 files changed, 1178 insertions(+), 11 deletions(-) create mode 100644 src/types/schema.ts diff --git a/package.json b/package.json index 7a137baa..b9568034 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "scripts": { "build": "rimraf lib/* && babel src --out-dir lib --extensions .js,.ts && tsc", "build-js-for-test": "rimraf lib/* && babel src --out-dir lib --source-maps inline --extensions .js,.ts", + "kysely-codegen": "kysely-codegen --out-file src/types/schema.ts --schema bookbrainz --camel-case", "lint": "eslint .", "lint-errors": "eslint --quiet .", "lint-staged": "lint-staged", @@ -100,6 +101,7 @@ "glob": "^7.1.2", "husky": "^8.0.0", "jsinspect": "^0.12.7", + "kysely-codegen": "^0.17.0", "lint-staged": "^13.1.0", "mocha": "^10.2.0", "node-uuid": "^1.4.8", diff --git a/src/index.ts b/src/index.ts index 7da0158d..75bbb6ed 100644 --- a/src/index.ts +++ b/src/index.ts @@ -19,9 +19,12 @@ import * as func from './func'; import * as util from './util'; // eslint-disable-line import/no-namespace +import {CamelCasePlugin, Kysely, PostgresDialect} from 'kysely'; import {type Knex, knex} from 'knex'; import Bookshelf from '@metabrainz/bookshelf'; +import type {DB} from './types/schema'; +import {Pool} from 'pg'; import achievementType from './models/achievementType'; import achievementUnlock from './models/achievementUnlock'; import adminLog from './models/adminLog'; @@ -117,6 +120,15 @@ export default function init(config: Knex.Config) { const SeriesData = seriesData(bookshelf); const WorkData = workData(bookshelf); + const kysely = new Kysely({ + dialect: new PostgresDialect({ + pool: new Pool(config.connection as Knex.ConnectionConfig), + }), + plugins: [ + new CamelCasePlugin(), + ], + }); + return { AchievementType: achievementType(bookshelf), AchievementUnlock: achievementUnlock(bookshelf), @@ -197,6 +209,7 @@ export default function init(config: Knex.Config) { WorkType: workType(bookshelf), bookshelf, func, + kysely, util }; } diff --git a/src/types/schema.ts b/src/types/schema.ts new file mode 100644 index 00000000..3a185653 --- /dev/null +++ b/src/types/schema.ts @@ -0,0 +1,1034 @@ +/** + * This file was generated by kysely-codegen. + * Please do not edit it manually. + */ + +import type {ColumnType} from 'kysely'; + + +export type AdminActionType = 'Change Privileges'; + +export type EntityType = 'Author' | 'Edition' | 'EditionGroup' | 'Publisher' | 'Series' | 'Work'; + +export type ExternalServiceOauthType = 'critiquebrainz'; + +export type Generated = T extends ColumnType + ? ColumnType + : ColumnType; + +export type Json = JsonValue; + +export type JsonArray = JsonValue[]; + +export type JsonObject = { + [x: string]: JsonValue | undefined; +}; + +export type JsonPrimitive = boolean | number | string | null; + +export type JsonValue = JsonArray | JsonObject | JsonPrimitive; + +export type LangProficiency = 'ADVANCED' | 'BASIC' | 'INTERMEDIATE' | 'NATIVE'; + +export type Timestamp = ColumnType; + +export interface AchievementType { + badgeUrl: string | null; + description: string; + id: Generated; + name: string; +} + +export interface AchievementUnlock { + achievementId: number; + editorId: number; + id: Generated; + profileRank: number | null; + unlockedAt: Generated; +} + +export interface AdminLog { + actionType: AdminActionType; + adminId: number; + id: Generated; + newPrivs: number | null; + note: string; + oldPrivs: number | null; + targetUserId: number; + time: Generated; +} + +export interface Alias { + id: Generated; + languageId: number | null; + name: string; + primary: Generated; + sortName: string; +} + +export interface AliasSet { + defaultAliasId: number | null; + id: Generated; +} + +export interface AliasSetAlias { + aliasId: number; + setId: number; +} + +export interface Annotation { + content: string; + id: Generated; + lastRevisionId: number | null; +} + +export interface Author { + aliasSetId: number | null; + annotationId: number | null; + areaId: number | null; + authorType: string | null; + bbid: string | null; + beginAreaId: number | null; + beginDay: number | null; + beginMonth: number | null; + beginYear: number | null; + dataId: number | null; + defaultAliasId: number | null; + disambiguation: string | null; + disambiguationId: number | null; + endAreaId: number | null; + endDay: number | null; + ended: boolean | null; + endMonth: number | null; + endYear: number | null; + genderId: number | null; + identifierSetId: number | null; + master: boolean | null; + name: string | null; + relationshipSetId: number | null; + revisionId: number | null; + sortName: string | null; + type: EntityType | null; + typeId: number | null; +} + +export interface AuthorCredit { + authorCount: number; + beginPhrase: Generated; + id: Generated; + refCount: Generated; +} + +export interface AuthorCreditName { + authorBbid: string; + authorCreditId: number; + joinPhrase: string; + name: string; + position: number; +} + +export interface AuthorData { + aliasSetId: number; + annotationId: number | null; + areaId: number | null; + beginAreaId: number | null; + beginDay: number | null; + beginMonth: number | null; + beginYear: number | null; + disambiguationId: number | null; + endAreaId: number | null; + endDay: number | null; + ended: Generated; + endMonth: number | null; + endYear: number | null; + genderId: number | null; + id: Generated; + identifierSetId: number | null; + relationshipSetId: number | null; + typeId: number | null; +} + +export interface AuthorHeader { + bbid: string; + masterRevisionId: number | null; +} + +export interface AuthorImport { + aliasSetId: number | null; + annotationId: number | null; + areaId: number | null; + authorType: string | null; + bbid: string | null; + beginAreaId: number | null; + beginDay: number | null; + beginMonth: number | null; + beginYear: number | null; + dataId: number | null; + defaultAliasId: number | null; + disambiguation: string | null; + disambiguationId: number | null; + endAreaId: number | null; + endDay: number | null; + ended: boolean | null; + endMonth: number | null; + endYear: number | null; + genderId: number | null; + identifierSetId: number | null; + master: boolean | null; + name: string | null; + relationshipSetId: number | null; + revisionId: number | null; + sortName: string | null; + type: EntityType | null; + typeId: number | null; +} + +export interface AuthorImportHeader { + bbid: string; + dataId: number; +} + +export interface AuthorRevision { + bbid: string; + dataId: number | null; + id: number; + isMerge: Generated; +} + +export interface AuthorType { + id: Generated; + label: string; +} + +export interface Disambiguation { + comment: string; + id: Generated; +} + +export interface DiscardVotes { + editorId: number; + importBbid: string; + votedAt: Generated; +} + +export interface Edition { + aliasSetId: number | null; + annotationId: number | null; + authorCreditId: number | null; + bbid: string | null; + dataId: Generated; + defaultAliasId: number | null; + depth: number | null; + disambiguation: string | null; + disambiguationId: number | null; + editionGroupBbid: string | null; + formatId: Generated; + height: number | null; + identifierSetId: number | null; + languageSetId: number | null; + master: boolean | null; + name: string | null; + pages: number | null; + publisherSetId: number | null; + relationshipSetId: number | null; + releaseEventSetId: number | null; + revisionId: number | null; + sortName: string | null; + statusId: Generated; + type: EntityType | null; + weight: number | null; + width: number | null; +} + +export interface EditionData { + aliasSetId: number; + annotationId: number | null; + authorCreditId: number | null; + depth: number | null; + disambiguationId: number | null; + editionGroupBbid: string | null; + formatId: number | null; + height: number | null; + id: Generated; + identifierSetId: number | null; + languageSetId: number | null; + pages: number | null; + publisherSetId: number | null; + relationshipSetId: number | null; + releaseEventSetId: number | null; + statusId: number | null; + weight: number | null; + width: number | null; +} + +export interface EditionFormat { + id: Generated; + label: string; +} + +export interface EditionGroup { + aliasSetId: number | null; + annotationId: number | null; + authorCreditId: number | null; + bbid: string | null; + dataId: number | null; + defaultAliasId: number | null; + disambiguation: string | null; + disambiguationId: number | null; + editionGroupType: string | null; + identifierSetId: number | null; + master: boolean | null; + name: string | null; + relationshipSetId: number | null; + revisionId: number | null; + sortName: string | null; + type: EntityType | null; + typeId: number | null; +} + +export interface EditionGroupData { + aliasSetId: number; + annotationId: number | null; + authorCreditId: number | null; + disambiguationId: number | null; + id: Generated; + identifierSetId: number | null; + relationshipSetId: number | null; + typeId: number | null; +} + +export interface EditionGroupHeader { + bbid: string; + masterRevisionId: number | null; +} + +export interface EditionGroupImport { + aliasSetId: number | null; + annotationId: number | null; + authorCreditId: number | null; + bbid: string | null; + dataId: number | null; + defaultAliasId: number | null; + disambiguation: string | null; + disambiguationId: number | null; + editionGroupType: string | null; + identifierSetId: number | null; + master: boolean | null; + name: string | null; + relationshipSetId: number | null; + revisionId: number | null; + sortName: string | null; + type: EntityType | null; + typeId: number | null; +} + +export interface EditionGroupImportHeader { + bbid: string; + dataId: number; +} + +export interface EditionGroupRevision { + bbid: string; + dataId: number | null; + id: number; + isMerge: Generated; +} + +export interface EditionGroupType { + id: Generated; + label: string; +} + +export interface EditionHeader { + bbid: string; + masterRevisionId: number | null; +} + +export interface EditionImport { + aliasSetId: number | null; + annotationId: number | null; + authorCreditId: number | null; + bbid: string | null; + dataId: number | null; + defaultAliasId: number | null; + depth: number | null; + disambiguation: string | null; + disambiguationId: number | null; + editionGroupBbid: string | null; + formatId: number | null; + height: number | null; + identifierSetId: number | null; + languageSetId: number | null; + master: boolean | null; + name: string | null; + pages: number | null; + publisherSetId: number | null; + relationshipSetId: number | null; + releaseEventSetId: number | null; + revisionId: number | null; + sortName: string | null; + statusId: number | null; + type: EntityType | null; + weight: number | null; + width: number | null; +} + +export interface EditionImportHeader { + bbid: string; + dataId: number; +} + +export interface EditionRevision { + bbid: string; + dataId: number | null; + id: number; + isMerge: Generated; +} + +export interface EditionStatus { + id: Generated; + label: string; +} + +export interface Editor { + activeAt: Generated; + areaId: number | null; + bio: Generated; + cachedMetabrainzName: string | null; + createdAt: Generated; + genderId: number | null; + id: Generated; + metabrainzUserId: number | null; + name: string; + privs: Generated; + reputation: Generated; + revisionsApplied: Generated; + revisionsReverted: Generated; + titleUnlockId: number | null; + totalRevisions: Generated; + typeId: Generated; +} + +export interface EditorLanguage { + editorId: number; + languageId: number; + proficiency: LangProficiency; +} + +export interface EditorType { + id: Generated; + label: string; +} + +export interface Entity { + bbid: Generated; + isImport: Generated; + type: EntityType; +} + +export interface EntityRedirect { + sourceBbid: string; + targetBbid: string; +} + +export interface ExternalServiceOauth { + accessToken: string; + editorId: number; + id: Generated; + refreshToken: string | null; + scopes: string[] | null; + service: ExternalServiceOauthType; + tokenExpires: Timestamp | null; +} + +export interface ExternalSource { + id: Generated; + name: string; +} + +export interface Identifier { + id: Generated; + typeId: Generated; + value: string; +} + +export interface IdentifierSet { + id: Generated; +} + +export interface IdentifierSetIdentifier { + identifierId: number; + setId: number; +} + +export interface IdentifierType { + childOrder: Generated; + deprecated: Generated; + description: string; + detectionRegex: string | null; + displayTemplate: string; + entityType: EntityType; + id: Generated; + label: string; + parentId: number | null; + validationRegex: string; +} + +export interface ImportMetadata { + acceptedEntityBbid: string | null; + additionalData: Json | null; + externalIdentifier: string; + externalSourceId: number; + importedAt: Generated; + lastEdited: Timestamp | null; + pendingEntityBbid: string | null; +} + +export interface LanguageSet { + id: Generated; +} + +export interface LanguageSetLanguage { + languageId: number; + setId: number; +} + +export interface MusicbrainzArea { + beginDateDay: number | null; + beginDateMonth: number | null; + beginDateYear: number | null; + comment: Generated; + editsPending: Generated; + endDateDay: number | null; + endDateMonth: number | null; + endDateYear: number | null; + ended: Generated; + gid: string; + id: Generated; + lastUpdated: Generated; + name: string; + type: number | null; +} + +export interface MusicbrainzAreaType { + childOrder: Generated; + description: string | null; + id: Generated; + name: string; + parent: number | null; +} + +export interface MusicbrainzCountryArea { + area: number; +} + +export interface MusicbrainzGender { + childOrder: Generated; + description: string | null; + id: Generated; + name: string; + parent: number | null; +} + +export interface MusicbrainzLanguage { + frequency: Generated; + id: Generated; + isoCode1: string | null; + isoCode2b: string | null; + isoCode2t: string | null; + isoCode3: string | null; + name: string; +} + +export interface MusicbrainzReplicationControl { + currentReplicationSequence: number | null; + currentSchemaSequence: number; + id: Generated; + lastReplicationDate: Timestamp | null; +} + +export interface Note { + authorId: number; + content: string; + id: Generated; + postedAt: Generated; + revisionId: number; +} + +export interface Publisher { + aliasSetId: number | null; + annotationId: number | null; + areaId: number | null; + bbid: string | null; + beginDay: number | null; + beginMonth: number | null; + beginYear: number | null; + dataId: Generated; + defaultAliasId: number | null; + disambiguation: string | null; + disambiguationId: number | null; + endDay: number | null; + ended: boolean | null; + endMonth: number | null; + endYear: number | null; + identifierSetId: number | null; + master: boolean | null; + name: string | null; + publisherType: string | null; + relationshipSetId: number | null; + revisionId: number | null; + sortName: string | null; + type: EntityType | null; + typeId: Generated; +} + +export interface PublisherData { + aliasSetId: number; + annotationId: number | null; + areaId: number | null; + beginDay: number | null; + beginMonth: number | null; + beginYear: number | null; + disambiguationId: number | null; + endDay: number | null; + ended: Generated; + endMonth: number | null; + endYear: number | null; + id: Generated; + identifierSetId: number | null; + relationshipSetId: number | null; + typeId: number | null; +} + +export interface PublisherHeader { + bbid: string; + masterRevisionId: number | null; +} + +export interface PublisherImport { + aliasSetId: number | null; + annotationId: number | null; + areaId: number | null; + bbid: string | null; + beginDay: number | null; + beginMonth: number | null; + beginYear: number | null; + dataId: number | null; + defaultAliasId: number | null; + disambiguation: string | null; + disambiguationId: number | null; + endDay: number | null; + ended: boolean | null; + endMonth: number | null; + endYear: number | null; + identifierSetId: number | null; + master: boolean | null; + name: string | null; + publisherType: string | null; + relationshipSetId: number | null; + revisionId: number | null; + sortName: string | null; + type: EntityType | null; + typeId: number | null; +} + +export interface PublisherImportHeader { + bbid: string; + dataId: number; +} + +export interface PublisherRevision { + bbid: string; + dataId: number | null; + id: number; + isMerge: Generated; +} + +export interface PublisherSet { + id: Generated; +} + +export interface PublisherSetPublisher { + publisherBbid: string; + setId: number; +} + +export interface PublisherType { + id: Generated; + label: string; +} + +export interface Relationship { + attributeSetId: Generated; + id: Generated; + sourceBbid: string; + targetBbid: string; + typeId: Generated; +} + +export interface RelationshipAttribute { + attributeType: number; + id: Generated; +} + +export interface RelationshipAttributeSet { + id: Generated; +} + +export interface RelationshipAttributeSetRelationshipAttribute { + attributeId: number; + setId: number; +} + +export interface RelationshipAttributeTextValue { + attributeId: number; + textValue: string | null; +} + +export interface RelationshipAttributeType { + childOrder: Generated; + description: string | null; + id: Generated; + lastUpdated: Generated; + name: string; + parent: number | null; + root: number; +} + +export interface RelationshipSet { + id: Generated; +} + +export interface RelationshipSetRelationship { + relationshipId: number; + setId: number; +} + +export interface RelationshipType { + childOrder: Generated; + deprecated: Generated; + description: string; + id: Generated; + label: string; + linkPhrase: string; + parentId: number | null; + reverseLinkPhrase: string; + sourceEntityType: EntityType; + targetEntityType: EntityType; +} + +export interface RelationshipTypeAttributeType { + attributeType: number; + lastUpdated: Generated; + max: number | null; + min: number | null; + relationshipType: number; +} + +export interface ReleaseEvent { + areaId: number | null; + day: number | null; + id: Generated; + month: number | null; + year: number | null; +} + +export interface ReleaseEventSet { + id: Generated; +} + +export interface ReleaseEventSetReleaseEvent { + releaseEventId: number; + setId: number; +} + +export interface Revision { + authorId: number; + createdAt: Generated; + id: Generated; + isMerge: Generated; +} + +export interface RevisionParent { + childId: number; + parentId: number; +} + +export interface Series { + aliasSetId: number | null; + annotationId: number | null; + bbid: string | null; + dataId: Generated; + defaultAliasId: number | null; + disambiguation: string | null; + disambiguationId: number | null; + entityType: EntityType | null; + identifierSetId: number | null; + master: boolean | null; + name: string | null; + orderingTypeId: Generated; + relationshipSetId: number | null; + revisionId: number | null; + sortName: string | null; + type: EntityType | null; +} + +export interface SeriesData { + aliasSetId: number; + annotationId: number | null; + disambiguationId: number | null; + entityType: EntityType; + id: Generated; + identifierSetId: number | null; + orderingTypeId: number; + relationshipSetId: number | null; +} + +export interface SeriesHeader { + bbid: string; + masterRevisionId: number | null; +} + +export interface SeriesImport { + aliasSetId: number | null; + annotationId: number | null; + bbid: string | null; + dataId: number | null; + defaultAliasId: number | null; + disambiguation: string | null; + disambiguationId: number | null; + entityType: EntityType | null; + identifierSetId: number | null; + master: boolean | null; + name: string | null; + orderingTypeId: number | null; + relationshipSetId: number | null; + revisionId: number | null; + sortName: string | null; + type: EntityType | null; +} + +export interface SeriesImportHeader { + bbid: string; + dataId: number; +} + +export interface SeriesOrderingType { + id: Generated; + label: string; +} + +export interface SeriesRevision { + bbid: string; + dataId: number | null; + id: number; + isMerge: Generated; +} + +export interface TitleType { + description: string; + id: Generated; + title: string; +} + +export interface TitleUnlock { + editorId: number; + id: Generated; + titleId: number; + unlockedAt: Generated; +} + +export interface UserCollection { + createdAt: Generated; + description: Generated; + entityType: EntityType; + id: Generated; + lastModified: Generated; + name: string; + ownerId: number; + public: Generated; +} + +export interface UserCollectionCollaborator { + collaboratorId: number; + collectionId: string; +} + +export interface UserCollectionItem { + addedAt: Generated; + bbid: string; + collectionId: string; +} + +export interface Work { + aliasSetId: number | null; + annotationId: number | null; + bbid: string | null; + dataId: Generated; + defaultAliasId: number | null; + disambiguation: string | null; + disambiguationId: number | null; + identifierSetId: number | null; + languageSetId: number | null; + master: boolean | null; + name: string | null; + relationshipSetId: number | null; + revisionId: number | null; + sortName: string | null; + type: EntityType | null; + typeId: Generated; + workType: string | null; +} + +export interface WorkData { + aliasSetId: number; + annotationId: number | null; + disambiguationId: number | null; + id: Generated; + identifierSetId: number | null; + languageSetId: number | null; + relationshipSetId: number | null; + typeId: number | null; +} + +export interface WorkHeader { + bbid: string; + masterRevisionId: number | null; +} + +export interface WorkImport { + aliasSetId: number | null; + annotationId: number | null; + bbid: string | null; + dataId: number | null; + defaultAliasId: number | null; + disambiguation: string | null; + disambiguationId: number | null; + identifierSetId: number | null; + languageSetId: number | null; + master: boolean | null; + name: string | null; + relationshipSetId: number | null; + revisionId: number | null; + sortName: string | null; + type: EntityType | null; + typeId: number | null; + workType: string | null; +} + +export interface WorkImportHeader { + bbid: string; + dataId: number; +} + +export interface WorkRevision { + bbid: string; + dataId: number | null; + id: number; + isMerge: Generated; +} + +export interface WorkType { + id: Generated; + label: string; +} + +export interface DB { + achievementType: AchievementType; + achievementUnlock: AchievementUnlock; + adminLog: AdminLog; + alias: Alias; + aliasSet: AliasSet; + aliasSetAlias: AliasSetAlias; + annotation: Annotation; + author: Author; + authorCredit: AuthorCredit; + authorCreditName: AuthorCreditName; + authorData: AuthorData; + authorHeader: AuthorHeader; + authorImport: AuthorImport; + authorImportHeader: AuthorImportHeader; + authorRevision: AuthorRevision; + authorType: AuthorType; + disambiguation: Disambiguation; + discardVotes: DiscardVotes; + edition: Edition; + editionData: EditionData; + editionFormat: EditionFormat; + editionGroup: EditionGroup; + editionGroupData: EditionGroupData; + editionGroupHeader: EditionGroupHeader; + editionGroupImport: EditionGroupImport; + editionGroupImportHeader: EditionGroupImportHeader; + editionGroupRevision: EditionGroupRevision; + editionGroupType: EditionGroupType; + editionHeader: EditionHeader; + editionImport: EditionImport; + editionImportHeader: EditionImportHeader; + editionRevision: EditionRevision; + editionStatus: EditionStatus; + editor: Editor; + editorLanguage: EditorLanguage; + editorType: EditorType; + entity: Entity; + entityRedirect: EntityRedirect; + externalServiceOauth: ExternalServiceOauth; + externalSource: ExternalSource; + identifier: Identifier; + identifierSet: IdentifierSet; + identifierSetIdentifier: IdentifierSetIdentifier; + identifierType: IdentifierType; + importMetadata: ImportMetadata; + languageSet: LanguageSet; + languageSetLanguage: LanguageSetLanguage; + 'musicbrainz.area': MusicbrainzArea; + 'musicbrainz.areaType': MusicbrainzAreaType; + 'musicbrainz.countryArea': MusicbrainzCountryArea; + 'musicbrainz.gender': MusicbrainzGender; + 'musicbrainz.language': MusicbrainzLanguage; + 'musicbrainz.replicationControl': MusicbrainzReplicationControl; + note: Note; + publisher: Publisher; + publisherData: PublisherData; + publisherHeader: PublisherHeader; + publisherImport: PublisherImport; + publisherImportHeader: PublisherImportHeader; + publisherRevision: PublisherRevision; + publisherSet: PublisherSet; + publisherSetPublisher: PublisherSetPublisher; + publisherType: PublisherType; + relationship: Relationship; + relationshipAttribute: RelationshipAttribute; + relationshipAttributeSet: RelationshipAttributeSet; + relationshipAttributeSetRelationshipAttribute: RelationshipAttributeSetRelationshipAttribute; + relationshipAttributeTextValue: RelationshipAttributeTextValue; + relationshipAttributeType: RelationshipAttributeType; + relationshipSet: RelationshipSet; + relationshipSetRelationship: RelationshipSetRelationship; + relationshipType: RelationshipType; + relationshipTypeAttributeType: RelationshipTypeAttributeType; + releaseEvent: ReleaseEvent; + releaseEventSet: ReleaseEventSet; + releaseEventSetReleaseEvent: ReleaseEventSetReleaseEvent; + revision: Revision; + revisionParent: RevisionParent; + series: Series; + seriesData: SeriesData; + seriesHeader: SeriesHeader; + seriesImport: SeriesImport; + seriesImportHeader: SeriesImportHeader; + seriesOrderingType: SeriesOrderingType; + seriesRevision: SeriesRevision; + titleType: TitleType; + titleUnlock: TitleUnlock; + userCollection: UserCollection; + userCollectionCollaborator: UserCollectionCollaborator; + userCollectionItem: UserCollectionItem; + work: Work; + workData: WorkData; + workHeader: WorkHeader; + workImport: WorkImport; + workImportHeader: WorkImportHeader; + workRevision: WorkRevision; + workType: WorkType; +} diff --git a/yarn.lock b/yarn.lock index fb85179e..180a56bb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1503,7 +1503,7 @@ brace-expansion@^2.0.1: dependencies: balanced-match "^1.0.0" -braces@^3.0.2, braces@~3.0.2: +braces@^3.0.2, braces@^3.0.3, braces@~3.0.2: version "3.0.3" resolved "https://registry.yarnpkg.com/braces/-/braces-3.0.3.tgz#490332f40919452272d55a8480adc0c441358789" integrity sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA== @@ -1578,7 +1578,15 @@ chai@^4.1.2: pathval "^1.1.1" type-detect "^4.0.5" -chalk@^2.0.0, chalk@^2.1.0, chalk@^2.4.2: +chalk@4.1.2, chalk@^4.0.0, chalk@^4.1.0: + version "4.1.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" + integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== + dependencies: + ansi-styles "^4.1.0" + supports-color "^7.1.0" + +chalk@^2.0.0, chalk@^2.1.0, chalk@^2.3.2, chalk@^2.4.2: version "2.4.2" resolved "https://registry.yarnpkg.com/chalk/-/chalk-2.4.2.tgz#cd42541677a54333cf541a49108c1432b44c9424" integrity sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ== @@ -1587,14 +1595,6 @@ chalk@^2.0.0, chalk@^2.1.0, chalk@^2.4.2: escape-string-regexp "^1.0.5" supports-color "^5.3.0" -chalk@^4.0.0, chalk@^4.1.0: - version "4.1.2" - resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" - integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== - dependencies: - ansi-styles "^4.1.0" - supports-color "^7.1.0" - check-error@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/check-error/-/check-error-1.0.2.tgz#574d312edd88bb5dd8912e9286dd6c0aed4aac82" @@ -1797,6 +1797,11 @@ diff@5.0.0: resolved "https://registry.yarnpkg.com/diff/-/diff-5.0.0.tgz#7ed6ad76d859d030787ec35855f5b1daf31d852b" integrity sha512-/VTCrvm5Z0JGty/BWHljh+BAiw3IK+2j87NGMu8Nwc/f48WoDAC395uomO9ZD117ZOBaHmkX1oyLvkVM/aIT3w== +diff@^3.5.0: + version "3.5.0" + resolved "https://registry.yarnpkg.com/diff/-/diff-3.5.0.tgz#800c0dd1e0a8bfbc95835c202ad220fe317e5a12" + integrity sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA== + dir-glob@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f" @@ -1818,6 +1823,18 @@ doctrine@^3.0.0: dependencies: esutils "^2.0.2" +dotenv-expand@^11.0.6: + version "11.0.6" + resolved "https://registry.yarnpkg.com/dotenv-expand/-/dotenv-expand-11.0.6.tgz#f2c840fd924d7c77a94eff98f153331d876882d3" + integrity sha512-8NHi73otpWsZGBSZwwknTXS5pqMOrk9+Ssrna8xCaxkzEpU9OTf9R5ArQGVw03//Zmk9MOwLPng9WwndvpAJ5g== + dependencies: + dotenv "^16.4.4" + +dotenv@^16.4.4, dotenv@^16.4.5: + version "16.4.5" + resolved "https://registry.yarnpkg.com/dotenv/-/dotenv-16.4.5.tgz#cdd3b3b604cb327e286b4762e13502f717cb099f" + integrity sha512-ZmdL2rui+eB2YwhsWzjInR8LldtZHGDoQ1ugH85ppHKwpUHL7j7rN0Ti9NCnGiQbhaZ11FpR+7ao1dNsmduNUg== + eastasianwidth@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/eastasianwidth/-/eastasianwidth-0.2.0.tgz#696ce2ec0aa0e6ea93a397ffcf24aa7840c827cb" @@ -2256,6 +2273,11 @@ function-bind@^1.1.1: resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" integrity sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A== +function-bind@^1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.2.tgz#2c02d864d97f3ea6c8830c464cbd11ab6eab7a1c" + integrity sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA== + function.prototype.name@^1.1.5: version "1.1.5" resolved "https://registry.yarnpkg.com/function.prototype.name/-/function.prototype.name-1.1.5.tgz#cce0505fe1ffb80503e6f9e46cc64e46a12a9621" @@ -2323,6 +2345,17 @@ getopts@2.3.0: resolved "https://registry.yarnpkg.com/getopts/-/getopts-2.3.0.tgz#71e5593284807e03e2427449d4f6712a268666f4" integrity sha512-5eDf9fuSXwxBL6q5HX+dhDj+dslFGWzU5thZ9kNKUkcPtaPdatmUFKwHFrLb/uf/WpA4BHET+AX3Scl56cAjpA== +git-diff@^2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/git-diff/-/git-diff-2.0.6.tgz#4a8ece670d64d1f9f4e68191ad8b1013900f6c1e" + integrity sha512-/Iu4prUrydE3Pb3lCBMbcSNIf81tgGt0W1ZwknnyF62t3tHmtiJTRj0f+1ZIhp3+Rh0ktz1pJVoa7ZXUCskivA== + dependencies: + chalk "^2.3.2" + diff "^3.5.0" + loglevel "^1.6.1" + shelljs "^0.8.1" + shelljs.exec "^1.1.7" + glob-parent@^5.1.2, glob-parent@~5.1.2: version "5.1.2" resolved "https://registry.yarnpkg.com/glob-parent/-/glob-parent-5.1.2.tgz#869832c58034fe68a4093c17dc15e8340d8401c4" @@ -2349,7 +2382,7 @@ glob@7.2.0: once "^1.3.0" path-is-absolute "^1.0.0" -glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.2.0: +glob@^7.0.0, glob@^7.1.1, glob@^7.1.2, glob@^7.1.3, glob@^7.2.0: version "7.2.3" resolved "https://registry.yarnpkg.com/glob/-/glob-7.2.3.tgz#b8df0fb802bbfa8e89bd1d938b4e16578ed44f2b" integrity sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q== @@ -2450,6 +2483,13 @@ has@^1.0.3: dependencies: function-bind "^1.1.1" +hasown@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/hasown/-/hasown-2.0.2.tgz#003eaf91be7adc372e84ec59dc37252cedb80003" + integrity sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ== + dependencies: + function-bind "^1.1.2" + he@1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" @@ -2530,6 +2570,11 @@ internal-slot@^1.0.4: has "^1.0.3" side-channel "^1.0.4" +interpret@^1.0.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.4.0.tgz#665ab8bc4da27a774a40584e812e3e0fa45b1a1e" + integrity sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA== + interpret@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/interpret/-/interpret-2.2.0.tgz#1a78a0b5965c40a5416d007ad6f50ad27c417df9" @@ -2571,6 +2616,13 @@ is-callable@^1.1.3, is-callable@^1.1.4, is-callable@^1.2.7: resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.7.tgz#3bc2a85ea742d9e36205dcacdd72ca1fdc51b055" integrity sha512-1BC0BVFhS/p0qtw6enp8e+8OD0UrK0oFLztSjNzhcKA3WDuJxxAPXzPuPtKkjEY9UUoEWlX/8fgKeu2S8i9JTA== +is-core-module@^2.13.0: + version "2.15.1" + resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.15.1.tgz#a7363a25bee942fefab0de13bf6aa372c82dcc37" + integrity sha512-z0vtXSwucUJtANQWldhbtbt7BnL0vxiFjIdDLAatwhDYty2bad6s+rijD6Ri4YuYJubLzIJLUidCh09e1djEVQ== + dependencies: + hasown "^2.0.2" + is-core-module@^2.8.1, is-core-module@^2.9.0: version "2.11.0" resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.11.0.tgz#ad4cb3e3863e814523c96f3f58d26cc570ff0144" @@ -2795,6 +2847,19 @@ knex@^2.4.2: tarn "^3.0.2" tildify "2.0.0" +kysely-codegen@^0.17.0: + version "0.17.0" + resolved "https://registry.yarnpkg.com/kysely-codegen/-/kysely-codegen-0.17.0.tgz#07bb2182ce2f315953c2407a52c99ee1ee942f91" + integrity sha512-C36g6epial8cIOSBEWGI9sRfkKSsEzTcivhjPivtYFQnhMdXnrVFaUe7UMZHeSdXaHiWDqDOkReJgWLD8nPKdg== + dependencies: + chalk "4.1.2" + dotenv "^16.4.5" + dotenv-expand "^11.0.6" + git-diff "^2.0.6" + micromatch "^4.0.8" + minimist "^1.2.8" + pluralize "^8.0.0" + kysely@^0.27.4: version "0.27.4" resolved "https://registry.yarnpkg.com/kysely/-/kysely-0.27.4.tgz#96a0285467b380948b4de03b20d87e82d797449b" @@ -2894,6 +2959,11 @@ log-update@^4.0.0: slice-ansi "^4.0.0" wrap-ansi "^6.2.0" +loglevel@^1.6.1: + version "1.9.2" + resolved "https://registry.yarnpkg.com/loglevel/-/loglevel-1.9.2.tgz#c2e028d6c757720107df4e64508530db6621ba08" + integrity sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg== + loupe@^2.3.1: version "2.3.6" resolved "https://registry.yarnpkg.com/loupe/-/loupe-2.3.6.tgz#76e4af498103c532d1ecc9be102036a21f787b53" @@ -2941,6 +3011,14 @@ micromatch@^4.0.4, micromatch@^4.0.5: braces "^3.0.2" picomatch "^2.3.1" +micromatch@^4.0.8: + version "4.0.8" + resolved "https://registry.yarnpkg.com/micromatch/-/micromatch-4.0.8.tgz#d66fa18f3a47076789320b9b1af32bd86d9fa202" + integrity sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA== + dependencies: + braces "^3.0.3" + picomatch "^2.3.1" + mimic-fn@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" @@ -2970,6 +3048,11 @@ minimist@^1.2.0, minimist@^1.2.6: resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.7.tgz#daa1c4d91f507390437c6a8bc01078e7000c4d18" integrity sha512-bzfL1YUZsP41gmu/qjrEk0Q6i2ix/cVeAhbCbqH9u3zYutS1cLg00qhrD0M2MVdCcx4Sc0UpP2eBWo9rotpq6g== +minimist@^1.2.8: + version "1.2.8" + resolved "https://registry.yarnpkg.com/minimist/-/minimist-1.2.8.tgz#c1a464e7693302e082a075cee0c057741ac4772c" + integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== + mocha@^10.2.0: version "10.2.0" resolved "https://registry.yarnpkg.com/mocha/-/mocha-10.2.0.tgz#1fd4a7c32ba5ac372e03a17eef435bd00e5c68b8" @@ -3350,6 +3433,11 @@ pkg-dir@^3.0.0: dependencies: find-up "^3.0.0" +pluralize@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/pluralize/-/pluralize-8.0.0.tgz#1a6fa16a38d12a1901e0320fa017051c539ce3b1" + integrity sha512-Nc3IT5yHzflTfbjgqWcCPpo7DaKy4FnpB0l/zCAW0Tc7jxAiuqSxHasntB3D7887LSrA93kDJ9IXovxJYxyLCA== + postgres-array@~2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/postgres-array/-/postgres-array-2.0.0.tgz#48f8fce054fbc69671999329b8834b772652d82e" @@ -3438,6 +3526,13 @@ readdirp@~3.6.0: dependencies: picomatch "^2.2.1" +rechoir@^0.6.2: + version "0.6.2" + resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.6.2.tgz#85204b54dba82d5742e28c96756ef43af50e3384" + integrity sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw== + dependencies: + resolve "^1.1.6" + rechoir@^0.8.0: version "0.8.0" resolved "https://registry.yarnpkg.com/rechoir/-/rechoir-0.8.0.tgz#49f866e0d32146142da3ad8f0eff352b3215ff22" @@ -3527,6 +3622,15 @@ resolve-from@^5.0.0: resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-5.0.0.tgz#c35225843df8f776df21c57557bc087e9dfdfc69" integrity sha512-qYg9KP24dD5qka9J47d0aVky0N+b4fTU89LN9iDnjB5waksiC49rvMB0PrUJQGoTmH50XPiqOvAjDfaijGxYZw== +resolve@^1.1.6: + version "1.22.8" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.8.tgz#b6c87a9f2aa06dfab52e3d70ac8cde321fa5a48d" + integrity sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw== + dependencies: + is-core-module "^2.13.0" + path-parse "^1.0.7" + supports-preserve-symlinks-flag "^1.0.0" + resolve@^1.10.1, resolve@^1.14.2, resolve@^1.20.0, resolve@^1.22.0: version "1.22.1" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.1.tgz#27cb2ebb53f91abb49470a928bba7558066ac177" @@ -3637,6 +3741,20 @@ shebang-regex@^3.0.0: resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== +shelljs.exec@^1.1.7: + version "1.1.8" + resolved "https://registry.yarnpkg.com/shelljs.exec/-/shelljs.exec-1.1.8.tgz#6f3c8dd017cb96d2dea82e712b758eab4fc2f68c" + integrity sha512-vFILCw+lzUtiwBAHV8/Ex8JsFjelFMdhONIsgKNLgTzeRckp2AOYRQtHJE/9LhNvdMmE27AGtzWx0+DHpwIwSw== + +shelljs@^0.8.1: + version "0.8.5" + resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.8.5.tgz#de055408d8361bed66c669d2f000538ced8ee20c" + integrity sha512-TiwcRcrkhHvbrZbnRcFYMLl30Dfov3HKqzp5tO5b4pt6G/SezKcYhmDg15zXVBswHmctSAQKznqNW2LO5tTDow== + dependencies: + glob "^7.0.0" + interpret "^1.0.0" + rechoir "^0.6.2" + side-channel@^1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/side-channel/-/side-channel-1.0.4.tgz#efce5c8fdc104ee751b25c58d4290011fa5ea2cf" From fb8fd5a3ab72c7bdade5d6a5bc314c97f6c7dfdc Mon Sep 17 00:00:00 2001 From: David Kellner <52860029+kellnerd@users.noreply.github.com> Date: Mon, 28 Oct 2024 22:36:57 +0100 Subject: [PATCH 11/14] feat(imports): Preserve BBID and DB rows of entities during approval Rather than collecting all properties of the pending entity data and creating a new entity (with new BBID), we simply create an entity header and a revision which link to the existing data. This is a complete rewrite using Kysely instead of Bookshelf/Knex. The final loading of the entity model (for search indexing) is out of scope for this function and has been moved into bookbrainz-site. --- src/func/imports/approve-import.ts | 165 ++++++++++++++--------------- src/func/imports/delete-import.ts | 1 - src/util.ts | 4 + 3 files changed, 85 insertions(+), 85 deletions(-) diff --git a/src/func/imports/approve-import.ts b/src/func/imports/approve-import.ts index b6c06baf..a52816ee 100644 --- a/src/func/imports/approve-import.ts +++ b/src/func/imports/approve-import.ts @@ -1,5 +1,5 @@ /* - * Copyright (C) 2018 Shivam Tripathi + * Copyright (C) 2024 David Kellner * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -16,94 +16,91 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ -import * as _ from 'lodash'; -import { - getAdditionalEntityProps, getEntityModelByType, getEntitySetMetadataByType -} from '../entity'; +import type {EntityType} from '../../types/schema'; import type {ImportMetadataWithSourceT} from '../../types/imports'; import type {ORM} from '../..'; -import type {Transaction} from '../types'; -import {createNote} from '../note'; -import {deleteImport} from './delete-import'; -import {incrementEditorEditCountById} from '../editor'; +import {uncapitalize} from '../../util'; -interface approveEntityPropsType { - orm: ORM, - transacting: Transaction, +export async function approveImport({editorId, importEntity, orm}: { + editorId: number, importEntity: any, - editorId: string -} - -export async function approveImport( - {orm, transacting, importEntity, editorId}: approveEntityPropsType -): Promise> { - const {bbid: pendingEntityBbid, type: entityType, disambiguationId, aliasSet, - identifierSetId, annotationId} = importEntity; + orm: ORM, +}) { + const {bbid, type, annotationId} = importEntity; const metadata: ImportMetadataWithSourceT = importEntity.importMetadata; - const {id: aliasSetId} = aliasSet; - - const {Annotation, Entity, Revision} = orm; - - // Increase user edit count - const editorUpdatePromise = - incrementEditorEditCountById(orm, editorId, transacting); - - // Create a new revision record - const revision = await new Revision({ - authorId: editorId - }).save(null, {transacting}); - const revisionId = revision.get('id'); - - if (annotationId) { - // Set revision of our annotation which is NULL for imports - await new Annotation({id: annotationId}) - .save({lastRevisionId: revisionId}, {transacting}); - } - - const note = `Approved from automatically imported record of ${metadata.source}`; - // Create a new note promise - const notePromise = createNote(orm, note, editorId, revision, transacting); - - // Get additional props - const additionalProps = getAdditionalEntityProps(importEntity, entityType); - - // Collect the entity sets from the importEntity - const entitySetMetadata = getEntitySetMetadataByType(entityType); - const entitySets = entitySetMetadata.reduce( - (set, {entityIdField}) => - _.assign(set, {[entityIdField]: importEntity[entityIdField]}) - , {} - ); - - await Promise.all([notePromise, editorUpdatePromise]); - - const newEntity = await new Entity({type: entityType}) - .save(null, {transacting}); - const acceptedEntityBbid = newEntity.get('bbid'); - const propsToSet = _.extend({ - aliasSetId, - annotationId, - bbid: acceptedEntityBbid, - disambiguationId, - identifierSetId, - revisionId - }, entitySets, additionalProps); - - const Model = getEntityModelByType(orm, entityType); - - const entityModel = await new Model(propsToSet) - .save(null, { - method: 'insert', - transacting - }); - - const entity = await entityModel.refresh({ - transacting, - withRelated: ['defaultAlias'] + const entityType = uncapitalize(type as EntityType); + + await orm.kysely.transaction().execute(async (trx) => { + const pendingUpdates = [ + // Mark the pending entity as accepted + trx.updateTable('entity') + .set('isImport', false) + .where((eb) => eb.and({bbid, isImport: true})) + .executeTakeFirst(), + // Indicate approval of the entity by setting the accepted BBID + trx.updateTable('importMetadata') + .set('acceptedEntityBbid', bbid) + .where('pendingEntityBbid', '=', bbid) + .executeTakeFirst(), + // Increment revision count of the active editor + trx.updateTable('editor') + .set((eb) => ({ + revisionsApplied: eb('revisionsApplied', '+', 1), + totalRevisions: eb('totalRevisions', '+', 1), + })) + .where('id', '=', editorId) + .executeTakeFirst(), + ]; + + // Create a new revision and an entity header + const revision = await trx.insertInto('revision') + .values({authorId: editorId}) + .returning('id') + .executeTakeFirstOrThrow(); + await trx.insertInto(`${entityType}Header`) + .values({bbid}) + .executeTakeFirstOrThrow(); + + // Create initial entity revision using the entity data from the import + await trx.insertInto(`${entityType}Revision`) + .values((eb) => ({ + bbid, + dataId: eb.selectFrom(`${entityType}ImportHeader`) + .select('dataId') + .where('bbid', '=', bbid), + id: revision.id + })) + .executeTakeFirstOrThrow(); + + // Update the entity header with the revision, doing this earlier causes a FK constraint violation + pendingUpdates.push(trx.updateTable(`${entityType}Header`) + .set('masterRevisionId', revision.id) + .where('bbid', '=', bbid) + .executeTakeFirst()); + + if (annotationId) { + // Set revision of our annotation which is NULL for pending imports + pendingUpdates.push(trx.updateTable('annotation') + .set('lastRevisionId', revision.id) + .where('id', '=', annotationId) + .executeTakeFirst()); + } + + // Create edit note + await trx.insertInto('note') + .values({ + authorId: editorId, + content: `Approved automatically imported record ${metadata.externalIdentifier} from ${metadata.source}`, + revisionId: revision.id, + }) + .executeTakeFirstOrThrow(); + + return Promise.all(pendingUpdates.map(async (update) => { + const {numUpdatedRows} = await update; + if (Number(numUpdatedRows) !== 1) { + throw new Error(`Failed to approve import of ${bbid}`); + } + })); }); - - await deleteImport(transacting, pendingEntityBbid, acceptedEntityBbid); - - return entity; } diff --git a/src/func/imports/delete-import.ts b/src/func/imports/delete-import.ts index 5622e971..60cf6024 100644 --- a/src/func/imports/delete-import.ts +++ b/src/func/imports/delete-import.ts @@ -21,7 +21,6 @@ import {camelToSnake, snakeToCamel} from '../../util'; import type {Transaction} from '../types'; -// TODO: Don't call this function on approval, we want to reuse BBID and data of approved imports! export async function deleteImport( transacting: Transaction, importId: string, entityId?: string | null | undefined ) { diff --git a/src/util.ts b/src/util.ts index da59d13a..cc0e7c5e 100644 --- a/src/util.ts +++ b/src/util.ts @@ -79,6 +79,10 @@ export function camelToSnake(attrs: C) { {} as S); } +export function uncapitalize(word: T): Uncapitalize { + return word.replace(/^./, (first) => first.toLowerCase()) as Uncapitalize; +} + export class EntityTypeError extends Error { constructor(message: string) { super(message); From 1ee3d1abf5ead65b3e74c37b5d758707e7b3b60d Mon Sep 17 00:00:00 2001 From: David Kellner <52860029+kellnerd@users.noreply.github.com> Date: Fri, 1 Nov 2024 23:52:42 +0100 Subject: [PATCH 12/14] fix(imports): Mark `ParsedEdition.editionGroupBbid` as optional From an older schema, there are still editions without edition group. --- src/types/parser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types/parser.ts b/src/types/parser.ts index 4191249d..1c957410 100644 --- a/src/types/parser.ts +++ b/src/types/parser.ts @@ -53,7 +53,7 @@ export type ParsedAuthor = ParsedBaseEntity & { }; export type ParsedEdition = ParsedBaseEntity & { - editionGroupBbid: string; + editionGroupBbid?: string; width?: number; height?: number; depth?: number; From fdc7cbb5f895f825b51e9ba8f7090c3a8865e15e Mon Sep 17 00:00:00 2001 From: David Kellner <52860029+kellnerd@users.noreply.github.com> Date: Sun, 3 Nov 2024 12:03:32 +0100 Subject: [PATCH 13/14] fix(imports): Update recent import queries for new approval behavior The `pending_entity_bbid` column is no longer set to NULL on approval, we have to additionally check for NULL in accepted_entity_bbid to find pending imports. --- src/func/imports/recent-imports.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/func/imports/recent-imports.ts b/src/func/imports/recent-imports.ts index ab8447d1..82fb0bc1 100644 --- a/src/func/imports/recent-imports.ts +++ b/src/func/imports/recent-imports.ts @@ -53,7 +53,8 @@ async function getRecentImportUtilData( ) .from('bookbrainz.import_metadata') .orderBy('imported_at') - .whereNot('pending_entity_bbid', null) + .whereNotNull('pending_entity_bbid') + .whereNull('accepted_entity_bbid') .limit(limit) .offset(offset) .as('meta'), @@ -148,10 +149,9 @@ export async function getRecentImports( }); } -// TODO: Besides being expensive to compute, this will no longer be accurate once -// we keep the pending entity data column populated for potential source updates. +// TODO: This will become expensive/pointless to compute, refactor the entire module. export async function getTotalImports(transacting: Transaction) { const [{count}] = - await transacting('bookbrainz.import_metadata').count('pending_entity_bbid'); + await transacting('bookbrainz.import_metadata').count('pending_entity_bbid').whereNull('accepted_entity_bbid'); return parseInt(count as string, 10); } From f860e4e8ad8c66e1efb8149479c36659e3e34fe2 Mon Sep 17 00:00:00 2001 From: David Kellner <52860029+kellnerd@users.noreply.github.com> Date: Mon, 9 Dec 2024 14:40:29 +0100 Subject: [PATCH 14/14] chore: Delete unused `createEntity` and its exclusive dependencies It is no longer needed to edit imported entities during approval. They are now approved before being edited, just like regular entities. Last usage was removed in https://github.com/metabrainz/bookbrainz-site/commit/d9830e5792759037e8601f66a37b2c2bcdfda571 --- src/func/create-entity.ts | 151 -------------------------------------- src/func/editor.ts | 28 ------- src/func/entity-sets.ts | 105 -------------------------- src/func/entity.ts | 53 ------------- src/func/index.ts | 1 - src/func/note.ts | 53 ------------- src/func/set.ts | 22 ------ src/types/imports.ts | 1 - 8 files changed, 414 deletions(-) delete mode 100644 src/func/create-entity.ts delete mode 100644 src/func/entity-sets.ts delete mode 100644 src/func/note.ts diff --git a/src/func/create-entity.ts b/src/func/create-entity.ts deleted file mode 100644 index 11894dda..00000000 --- a/src/func/create-entity.ts +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Adapted from bookbrainz-site - * Copyright (C) 2016 Sean Burke - * 2016 Ben Ockmore - * 2018 Shivam Tripathi - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -import * as _ from 'lodash'; -import { - getAdditionalEntityProps, getEntityModelByType, getEntitySetMetadataByType -} from './entity'; -import type {AliasWithDefaultT} from '../types/aliases'; -import type {EntityTypeString} from '../types/entity'; -import type {IdentifierT} from '../types/identifiers'; -import type {ORM} from '..'; -import type {Transaction} from './types'; -import {createNote} from './note'; -import {incrementEditorEditCountById} from './editor'; -import {updateAliasSet} from './alias'; -import {updateAnnotation} from './annotation'; -import {updateDisambiguation} from './disambiguation'; -import {updateEntitySets} from './entity-sets'; -import {updateIdentifierSet} from './identifier'; - - -interface EntityDataType { - aliases: Array, - annotation: string, - disambiguation: string, - identifiers: Array, - note: string, - type: EntityTypeString -} - -interface ExtraEntityDataType extends EntityDataType { - [propName: string]: any; -} - -interface CreateEntityPropsType { - orm: ORM, - transacting: Transaction, - editorId: string, - entityData: ExtraEntityDataType -} - -// TODO: function is only used to approve imports, check whether type error below is critical -export async function createEntity({ - editorId, entityData, orm, transacting -}: CreateEntityPropsType) { - const {Entity, Revision} = orm; - - const {aliases, annotation, disambiguation, identifiers, note, - type: entityType, ...entitySetData} = entityData; - - // Increase user edit count - const editorUpdatePromise = - incrementEditorEditCountById(orm, editorId, transacting); - - // Create a new revision record - const revisionPromise = new Revision({ - authorId: editorId - }).save(null, {transacting}); - - // Create a new note promise - const notePromise = revisionPromise - .then((revision) => createNote( - orm, note, editorId, revision, transacting - )); - - // Create a new aliasSet with aliases obtained - const aliasSetPromise = updateAliasSet( - orm, transacting, null, null, aliases || [] - ); - - // Create identifier set using the identifiers obtained - const identSetPromise = updateIdentifierSet( - orm, transacting, null, identifiers || [] - ); - - // Create a new annotation using the revision - const annotationPromise = revisionPromise.then( - (revision) => updateAnnotation( - orm, transacting, null, annotation, revision - ) - ); - - // Create a new disambiguation for the entity - const disambiguationPromise = updateDisambiguation( - orm, transacting, null, disambiguation - ); - - // Get additional props - // @ts-expect-error Not sure why we have this error - const additionalProps = getAdditionalEntityProps(entityData, entityType); - - // Create entitySets - const entitySetMetadata = getEntitySetMetadataByType(entityType); - const entitySetsPromise = updateEntitySets( - entitySetMetadata, null, entitySetData, transacting, orm - ); - - const [ - revisionRecord, aliasSetRecord, identSetRecord, annotationRecord, - disambiguationRecord, entitySets - ] = await Promise.all([ - revisionPromise, aliasSetPromise, identSetPromise, - annotationPromise, disambiguationPromise, entitySetsPromise, - editorUpdatePromise, notePromise - ]); - - const newEntity = await new Entity({type: entityType}) - .save(null, {transacting}); - const propsToSet = _.extend({ - aliasSetId: aliasSetRecord && aliasSetRecord.get('id'), - annotationId: annotationRecord && annotationRecord.get('id'), - bbid: newEntity.get('bbid'), - disambiguationId: - disambiguationRecord && disambiguationRecord.get('id'), - identifierSetId: identSetRecord && identSetRecord.get('id'), - revisionId: revisionRecord && revisionRecord.get('id') - }, entitySets, additionalProps); - - const Model = getEntityModelByType(orm, entityType); - - const entityModel = await new Model(propsToSet) - .save(null, { - method: 'insert', - transacting - }); - - const entity = await entityModel.refresh({ - transacting, - withRelated: ['defaultAlias'] - }); - - return entity; -} diff --git a/src/func/editor.ts b/src/func/editor.ts index 3771a94b..443e2bcf 100644 --- a/src/func/editor.ts +++ b/src/func/editor.ts @@ -2,7 +2,6 @@ * Copied from bookbrainz-site * Copyright (C) 2016 Sean Burke * 2016 Ben Ockmore - * 2018 Shivam Tripathi * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,33 +18,6 @@ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. */ -import {ORM} from '..'; -import type {Transaction} from './types'; - -/** - * Adds 1 to the edit count of the specified editor - * - * @param {ORM} orm - the BookBrainz ORM, initialized during app setup - * @param {string} id - row ID of editor to be updated - * @param {Transaction} transacting - Bookshelf transaction object (must be in - * progress) - * @returns {Promise} - Resolves to the updated editor model - */ -export function incrementEditorEditCountById( - orm: ORM, - id: string, - transacting: Transaction -): Promise { - const {Editor} = orm; - return new Editor({id}) - .fetch({transacting}) - .then((editor) => { - // @ts-expect-error -- Types for custom methods of Bookshelf models are lacking - editor.incrementEditCount(); - return editor.save(null, {transacting}); - }); -} - /* eslint-disable camelcase */ function getEditorIDByMetaBrainzID(trx, metabrainzUserID) { return trx('bookbrainz.editor') diff --git a/src/func/entity-sets.ts b/src/func/entity-sets.ts deleted file mode 100644 index a87b6067..00000000 --- a/src/func/entity-sets.ts +++ /dev/null @@ -1,105 +0,0 @@ -/* Adapted from bookbrainz-site - * Copyright (C) 2016 Sean Burke - * 2016 Ben Ockmore - * 2018 Shivam Tripathi - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -import * as _ from 'lodash'; -import type {SetItemT, Transaction} from './types'; -import { - createNewSetWithItems, getAddedItems, getComparisonFunc, getRemovedItems, - getUnchangedItems -} from './set'; -import type {EntitySetMetadataT} from './entity'; -import type {ORM} from '..'; - - -function updateEntitySet( - transacting: Transaction, oldSet: any, newItems: Array, - derivedSet: EntitySetMetadataT, orm: ORM -): Promise { - const oldItems = - oldSet ? oldSet.related(derivedSet.propName).toJSON() : []; - - const comparisonFunc = getComparisonFunc( - [derivedSet.idField, ...derivedSet.mutableFields || []] - ); - - if (_.isEmpty(oldItems) && _.isEmpty(newItems)) { - return Promise.resolve(oldSet); - } - - const addedItems = getAddedItems(oldItems, newItems, comparisonFunc); - const removedItems = - getRemovedItems(oldItems, newItems, comparisonFunc); - const unchangedItems = - getUnchangedItems(oldItems, newItems, comparisonFunc); - - const isSetUnmodified = _.isEmpty(addedItems) && _.isEmpty(removedItems); - if (isSetUnmodified) { - // No action - set has not changed - return Promise.resolve(oldSet); - } - return createNewSetWithItems( - // @ts-expect-error We don't know why this `model` property is expected, - // even though it does not exist on Entity Set metadata - orm, transacting, derivedSet.model, [...unchangedItems, ...addedItems], - [], derivedSet.propName, derivedSet.idField - ); -} - - -export async function updateEntitySets( - derivedSets: EntitySetMetadataT[] | null | undefined, currentEntity: any, - entityData: any, transacting: Transaction, orm: ORM -): Promise | null | undefined> { - // If no entity sets, return null - if (!derivedSets) { - return null; - } - - // Process each entitySet - const newProps = await Promise.all(derivedSets.map(async (derivedSet) => { - const newItems = entityData[derivedSet.propName]; - - if (!(currentEntity && currentEntity[derivedSet.name])) { - return Promise.resolve(null); - } - - // TODO: Find out why we expect a non-existing `model` property here!? - // @ts-expect-error We don't know why this `model` property is expected, - // even though it does not exist on Entity Set metadata - const oldSetRecord = await derivedSet.model.forge({ - id: currentEntity[derivedSet.name].id - }).fetch({ - require: false, - transacting, - withRelated: [derivedSet.propName] - }); - - const newSetRecord = await updateEntitySet( - transacting, oldSetRecord, newItems, derivedSet, orm - ); - - return { - [derivedSet.entityIdField]: - newSetRecord ? newSetRecord.get('id') : null - }; - })); - - return newProps.reduce((result, value) => _.assign(result, value), {}); -} diff --git a/src/func/entity.ts b/src/func/entity.ts index dbad68e8..00653a89 100644 --- a/src/func/entity.ts +++ b/src/func/entity.ts @@ -79,59 +79,6 @@ export function getAdditionalEntityProps( } } -export type EntitySetMetadataT = { - entityIdField: string; - idField: string; - mutableFields?: string[]; - name: string; - propName: string; -}; - -/** - * @param {string} entityType - Entity type string - * @returns {Object} - Returns entitySetMetadata (derivedSets) -*/ -export function getEntitySetMetadataByType(entityType: EntityTypeString): EntitySetMetadataT[] { - if (entityType === 'Edition') { - return [ - { - entityIdField: 'languageSetId', - idField: 'id', - name: 'languageSet', - propName: 'languages' - }, - { - entityIdField: 'publisherSetId', - idField: 'bbid', - name: 'publisherSet', - propName: 'publishers' - }, - { - entityIdField: 'releaseEventSetId', - idField: 'id', - mutableFields: [ - 'date', - 'areaId' - ], - name: 'releaseEventSet', - propName: 'releaseEvents' - } - ]; - } - else if (entityType === 'Work') { - return [ - { - entityIdField: 'languageSetId', - idField: 'id', - name: 'languageSet', - propName: 'languages' - } - ]; - } - - return []; -} - /** * Returns all entity models defined in bookbrainz-data-js * diff --git a/src/func/index.ts b/src/func/index.ts index 10d43cee..06ee8736 100644 --- a/src/func/index.ts +++ b/src/func/index.ts @@ -33,4 +33,3 @@ export * as relationshipAttributes from './relationshipAttributes'; export * as releaseEvent from './releaseEvent'; export * as set from './set'; export * as work from './work'; -export {createEntity} from './create-entity'; diff --git a/src/func/note.ts b/src/func/note.ts deleted file mode 100644 index f9bc90d5..00000000 --- a/src/func/note.ts +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copied from bookbrainz-site - * Copyright (C) 2016 Sean Burke - * 2016 Ben Ockmore - * 2017 Daniel Hsing - * 2018 Shivam Tripathi - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License along - * with this program; if not, write to the Free Software Foundation, Inc., - * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - */ - -import type {ORM} from '..'; -import type {Transaction} from './types'; - -/** - * @param {ORM} orm - Bookbrainz orm wrapper containing all models - * @param {string} content - Note content - * @param {string} editorId - Editor's Id - * @param {Object} revision - Revision object created using orm.Revision model - * @param {Transaction} transacting - The transaction model - * @returns {Object | null} Returns the created Note object or returns null - */ -export function createNote( - orm: ORM, - content: string, - editorId: string, - revision: any, - transacting: Transaction -) { - const {Note} = orm; - if (content) { - const revisionId = revision.get('id'); - return new Note({ - authorId: editorId, - content, - revisionId - }) - .save(null, {transacting}); - } - - return null; -} diff --git a/src/func/set.ts b/src/func/set.ts index 58d37f20..d3575825 100644 --- a/src/func/set.ts +++ b/src/func/set.ts @@ -21,28 +21,6 @@ import type {FormRelationshipAttributesT as RelationshipAttributeT, SetItemT, Tr import type Bookshelf from '@metabrainz/bookshelf'; import type {ORM} from '..'; -/** - * Returns a function which compares two object provided to it using the - * comparison fields mentioned - * @param {Array} compareFields - Comparison fields of two objects - * @returns {Function} - Returns a comparison function - */ -export function getComparisonFunc(compareFields: Array) { - /** - * @param {any} obj - Object for comparison - * @param {any} other - Object for comparison - * @returns {boolean} Boolean value denoting objects are equal on fields - */ - return function Cmp(obj: any, other: any): boolean { - for (const field of compareFields) { - if (obj[field] !== other[field]) { - return false; - } - } - return true; - }; -} - /** * Get the intersection of two arrays of objects using a custom comparison * function. The two arrays represent two versions of a single set - one array diff --git a/src/types/imports.ts b/src/types/imports.ts index f3a4e8cd..a599a525 100644 --- a/src/types/imports.ts +++ b/src/types/imports.ts @@ -17,7 +17,6 @@ */ import {IdentifierT} from './identifiers'; -import type {Knex} from 'knex'; export type ImportHeaderT = {