Make it work

package.json

@@ -73,5 +73,10 @@
     "printWidth": 110,
     "arrowParens": "avoid"
   },
-  "name": "ldes2service"
+  "name": "ldes2service",
+  "dependencies": {
+    "@treecg/ldes-types": "0.0.4-alpha.0",
+    "execa": "^6.0.0",
+    "node-fetch": "^3.1.0"
+  }
 }

packages/ldes-manager/bin/server.ts

@@ -1,4 +1,4 @@
-import type { IGeneratorPluginOptions } from '@ldes/types';
+import type { IGeneratorPluginOptions } from '@treecg/ldes-types';
 import type { FastifyInstance } from 'fastify';
 import Fastify from 'fastify';
 import FastifyCors from 'fastify-cors';

packages/ldes-replicator/README.md

@@ -25,7 +25,7 @@ async function run(): Promise<void> {
 
   const streams = [LDESClient.createReadStream('LDES.example', streamOptions)];
 
-  const orchestrator = new Orchestrator(connectors, state, streams);
+  const orchestrator = new Orchestrator(state, connectors, streams);
 
   await orchestrator.provision();
   await orchestrator.run();

packages/ldes-replicator/bin/ldes-replicator.ts

@@ -5,8 +5,8 @@
 import * as fs from 'fs/promises';
 import { Command, flags } from '@oclif/command';
 import { newEngine } from '@treecg/actor-init-ldes-client';
-import type { IRedisStateConfig } from '@treecg/ldes-redis-state';
-import { RedisState } from '@treecg/ldes-redis-state';
+import type { IRedisStateConfig } from '../../ldes-redis-state';
+import { RedisState } from '../../ldes-redis-state';
 import type { ConnectorConfigs, LdesObjects, LdesShape } from '@treecg/ldes-types';
 import type { Quad } from 'n3';
 import { DataFactory, Store } from 'n3';
@@ -137,6 +137,7 @@ class LdesReplicator extends Command {
 
     const options = {
       pollingInterval: config.replicator.polling_interval,
+      representation: 'Object',
     };
 
     const streams: LdesObjects = Object.fromEntries(

packages/ldes-replicator/lib/Orchestrator.ts

@@ -37,8 +37,8 @@ export class Orchestrator {
 
       return new Promise<void>((resolve, reject) => {
         ldesObject.stream
-          .on('readable', async () => {
-            await this.processData(ldesObject, connectors);
+          .on('data', async member => {
+            await this.processMember(member.object, connectors);
           })
           .on('error', (error: any) => reject(error));
 
@@ -53,28 +53,29 @@ export class Orchestrator {
     const promises: Promise<void>[] = [];
     const state = this.stateStore.provision();
 
-    Object.values(this.ldesObjects).forEach(ldesObject => {
-      const ldesConnectors: IWritableConnector[] = Object.values(this.connectorsConfig).map(
-        (con: IConnectorConfig) => {
-          const config = con.settings || {};
-          const Connector = require(con.type || '@ldes/ldes-dummy-connector');
-
-          const connectorName = Object.keys(Connector).find(key => key.endsWith('Connector'));
-
-          if (!connectorName) {
-            throw new Error(`The connector ${con.type} couldn't be loaded correctly!`);
-          }
-
-          const connector = new Connector[connectorName](config, ldesObject.shape, ldesObject.name);
-
-          promises.push(connector.provision());
+    for (const ldesObject of Object.values(this.ldesObjects)) {
+      const ldesConnectors: IWritableConnector[] = [];
+      for (const con of Object.values(this.connectorsConfig)) {
+        const config = con.settings || {};
+        let Connector;
+        if (con.type) {
+          Connector = await import(`${con.type}`);
+        } else {
+          throw new Error('connector not found');
+        }
+        const connectorName = Object.keys(Connector).find(key => key.endsWith('Connector'));
 
-          return connector;
+        if (!connectorName) {
+          throw new Error(`The connector ${con.type} couldn't be loaded correctly!`);
         }
-      );
 
+        const connector = new Connector[connectorName](config, ldesObject.shape, ldesObject.name);
+
+        promises.push(connector.provision());
+        ldesConnectors.push(connector);
+      }
       this.ldesConnectors.set(ldesObject, ldesConnectors);
-    });
+    }
 
     await Promise.all([state, ...promises]);
 
@@ -99,4 +100,8 @@ export class Orchestrator {
       member = ldesObject.stream.read();
     }
   }
+
+  protected async processMember(member: string, connectors: IWritableConnector[]): Promise<void> {
+    await Promise.all(connectors.map(con => con.writeVersion(member)));
+  }
 }

packages/ldes-replicator/lib/dependenciesSetup.ts

@@ -1,5 +1,5 @@
 import type { IConnectorConfig } from '@treecg/ldes-types';
-import * as execa from 'execa';
+import execa = require('execa');
 
 const LDES_CONNECTORS = new Set(['@treecg/ldes-postgres-connector', '@treecg/ldes-mongodb-connector']);
 

packages/ldes-replicator/package.json

@@ -1,7 +1,7 @@
 {
   "private": "true",
   "name": "@treecg/ldes-replicator",
-  "version": "0.0.4-alpha.0",
+    "version": "0.0.4-alpha.0",
   "description": "> Allows you to replicate any amount of LDES into different backends via connectors.",
   "homepage": "https://github.com/Informatievlaanderen/ldes2service#readme",
   "license": "ISC",
@@ -28,12 +28,24 @@
     "build:ts": "tsc"
   },
   "dependencies": {
+    "@comunica/actor-abstract-mediatyped": "^1.22.0",
+    "@comunica/actor-http-native": "^1.22.1",
+    "@comunica/actor-rdf-parse-html": "^1.22.0",
+    "@comunica/actor-rdf-parse-html-microdata": "^1.22.0",
+    "@comunica/actor-rdf-parse-html-rdfa": "^1.22.0",
+    "@comunica/actor-rdf-parse-rdfxml": "^1.22.0",
+    "@comunica/actor-rdf-parse-xml-rdfa": "^1.22.0",
+    "@comunica/actor-rdf-serialize-jsonld": "^1.22.0",
+    "@comunica/actor-rdf-serialize-n3": "^1.22.0",
+    "@comunica/bus-rdf-serialize": "^1.22.0",
     "@comunica/core": "~1.19.2",
     "@oclif/command": "^1.8.0",
     "@oclif/errors": "^1.3.5",
     "@treecg/actor-init-ldes-client": "^2.3.4",
-    "@treecg/ldes-postgres-connector": "^0.0.4-alpha.0",
-    "@treecg/ldes-redis-state": "^0.0.4-alpha.0",
+    "@treecg/ldes-ngsild-connector": "^0.0.4",
+    "@treecg/ldes-postgres-connector": "link:../ldes-postgres-connector",
+    "@treecg/ldes-redis-state": "link:../ldes-redis-state",
+    "@treecg/ldes-dummy-connector": "link:../ldes-dummy-connector",
     "@treecg/ldes-types": "^0.0.4-alpha.0",
     "@types/node": "^16.4.3",
     "dotenv": "^10.0.0",

tsconfig.json

@@ -4,6 +4,7 @@
     "module": "commonjs",
     "lib": ["es6", "es2019", "esnext.asynciterable"],
     "target": "es2017",
+    "outDir": "./",
     "preserveConstEnums": true,
     "downlevelIteration": true,
     "skipLibCheck": true,
@@ -14,6 +15,7 @@
     "experimentalDecorators": true,
     "emitDecoratorMetadata": true,
     "allowSyntheticDefaultImports": true,
+    "esModuleInterop": true
   },
   "include": [
     "packages/*/index.ts",