adding replciator tests

adding nyc for code coverage

adding replicator tests

linting

Fix replications tests, add more debug output

Fix linter

fixing ipfs config in tests

fix: linting

Author: aphelionz

.gitignore

@@ -3,3 +3,5 @@ node_modules/
 test/browser/bundle.js
 test/browser/bundle.js.map
 dist/orbit-db-store.min.js.map
+.nyc_output
+orbitdb/

package-lock.json

@@ -5,7 +5,7 @@
   "main": "src/Store.js",
   "scripts": {
     "test": "npm run test:node && npm run test:browser",
-    "test:node": "TEST=all mocha",
+    "test:node": "TEST=go mocha; TEST=js nyc mocha",
     "test:browser": "npm run build:tests && mocha-headless-chrome -f ./test/browser/index.html -a no-sandbox",
     "build": "npm run build:dist",
     "build:dist": "webpack --config ./conf/webpack.config.js --display-modules --sort-modules-by size --mode production",
@@ -49,11 +49,12 @@
     "markdown-toc": "^1.2.0",
     "mocha": "^8.1.1",
     "mocha-headless-chrome": "^3.1.0",
+    "nyc": "^15.1.0",
     "orbit-db-cache": "~0.3.0",
     "orbit-db-identity-provider": "~0.3.1",
     "orbit-db-keystore": "~0.3.5",
     "orbit-db-storage-adapter": "^0.5.3",
-    "orbit-db-test-utils": "^0.11.0",
+    "orbit-db-test-utils": "^0.11.1",
     "rimraf": "^3.0.0",
     "standard": "^14.0.2",
     "webpack": "^4.44.1",

package.json

@@ -165,11 +165,13 @@ class Replicator extends EventEmitter {
     this._stats.tasksStarted += 1
 
     const exclude = []
+    // console.log('>', hash)
     const log = await Log.fromEntryHash(this._store._ipfs, this._store.identity, hash, { logId: this._store._oplog.id, access: this._store.access, length: batchSize, exclude })
     this._buffer.push(log)
 
     const latest = log.values[0]
     delete this._queue[hash]
+    // console.log('>>', latest.payload)
 
     // Mark this task as processed
     this._stats.tasksProcessed += 1

src/Replicator.js

@@ -38,7 +38,7 @@ Object.keys(testAPIs).forEach((IPFS) => {
       const cache = new Cache(cacheStore)
 
       testIdentity = await IdentityProvider.createIdentity({ id: 'userA', keystore })
-      ipfsd = await startIpfs(IPFS, ipfsConfig)
+      ipfsd = await startIpfs(IPFS, ipfsConfig.daemon1)
       ipfs = ipfsd.api
 
       const address = 'test-address'

test/add.spec.js

@@ -38,7 +38,7 @@ Object.keys(testAPIs).forEach((IPFS) => {
       const cache = new Cache(cacheStore)
 
       testIdentity = await IdentityProvider.createIdentity({ id: 'userA', keystore })
-      ipfs = await startIpfs(IPFS, ipfsConfig)
+      ipfs = await startIpfs(IPFS, ipfsConfig.daemon1)
 
       const address = 'test-address'
       store = new Store(ipfs, testIdentity, address, DefaultOptions)

test/constructor.spec.js

@@ -50,7 +50,7 @@ Object.keys(testAPIs).forEach((IPFS) => {
       const cache = new Cache(cacheStore)
 
       testIdentity = await IdentityProvider.createIdentity({ id: 'userA', keystore })
-      ipfsd = await startIpfs(IPFS, ipfsConfig)
+      ipfsd = await startIpfs(IPFS, ipfsConfig.daemon1)
       ipfs = ipfsd.api
 
       const address = 'test-address'

test/events.spec.js

@@ -0,0 +1,126 @@
+const assert = require('assert')
+const Log = require('ipfs-log')
+
+const Keystore = require('orbit-db-keystore')
+const IdentityProvider = require('orbit-db-identity-provider')
+const Replicator = require('../src/Replicator')
+
+const {
+  connectPeers,
+  config,
+  testAPIs,
+  startIpfs,
+  stopIpfs
+} = require('orbit-db-test-utils')
+
+// Tests timeout
+const timeout = 30000
+
+class DummyStore {
+  constructor (log, ipfs, identity) {
+    this._oplog = log
+    this._ipfs = ipfs
+    this.identity = identity
+  }
+
+  async close () {}
+}
+
+Object.keys(testAPIs).forEach((IPFS) => {
+  describe(`Replicator, ${IPFS}`, function () {
+    this.timeout(timeout)
+
+    let log, ipfsd, ipfs, replicator, store, keystore, signingKeystore
+
+    const { identityKeysPath } = config
+
+    before(async () => {
+      keystore = new Keystore(identityKeysPath)
+
+      ipfsd = await startIpfs(IPFS, config.daemon1)
+      ipfs = ipfsd.api
+      const id = await ipfsd.api.id()
+
+      const testIdentity = await IdentityProvider.createIdentity({ id, keystore })
+      log = new Log(ipfs, testIdentity)
+
+      store = new DummyStore(log, ipfs, testIdentity)
+      replicator = new Replicator(store, 123)
+    })
+
+    after(async () => {
+      await store.close()
+      await stopIpfs(ipfsd)
+      await replicator.stop()
+      await keystore.close()
+    })
+
+    it('default options', async () => {
+      assert.deepStrictEqual(replicator._buffer, [])
+    })
+
+    describe('concurrency = 1', function () {
+      let ipfsd2, ipfs2, log2, store2
+
+      this.timeout(timeout)
+
+      const logLength = 100
+
+      before(async () => {
+        ipfsd2 = await startIpfs(IPFS, config.daemon2)
+        ipfs2 = ipfsd2.api
+        await connectPeers(ipfs, ipfs2)
+
+        const testIdentity = await IdentityProvider.createIdentity({ id: 'userB', keystore, signingKeystore })
+        log2 = new Log(ipfs2, testIdentity, { logId: log.id })
+
+        console.log(`writing ${logLength} entries to the log`)
+        for (let i = 0; i < logLength; i++) {
+          await log2.append(`entry${i}`)
+        }
+        assert(log2.values.length, logLength)
+
+        store2 = new DummyStore(log2, ipfs2, testIdentity)
+      })
+
+      after(async () => {
+        await store2.close()
+        await stopIpfs(ipfsd2)
+      })
+
+      it('loads', (done) => {
+        let replicated = 0
+
+        assert.strictEqual(log.id, log2.id)
+
+        replicator.load(log2.heads)
+
+        assert.strictEqual(replicator._buffer.length, 0)
+        assert.deepStrictEqual(replicator.getQueue()[0], log2.heads[0])
+        assert.strictEqual(replicator.tasksQueued, 1)
+        assert.strictEqual(replicator.tasksRequested, 1)
+        assert.strictEqual(replicator.tasksStarted, 0) // ??
+
+        replicator.on('load.end', async (replicatedLogs) => {
+          replicated++
+          assert.strictEqual(replicator.tasksStarted, replicated) // ??
+          assert.strictEqual(replicator.tasksQueued, 0)
+          assert.strictEqual(replicator.tasksFinished, replicated)
+          // console.log(replicatedLogs.length)
+          for (const replicatedLog of replicatedLogs) {
+            // console.log(replicatedLog.values.length, log.values.length, replicatedLog.values[0])
+            await log.join(replicatedLog)
+          }
+          // console.log(log.values.length)
+          // console.log(log.values[0].payload)
+          // console.log(log.values.map(e => e.payload).join('\n'))
+
+          if (log.values.length === logLength) {
+            assert.deepStrictEqual(log.values, log2.values)
+            done()
+          }
+        })
+      })
+    })
+  })
+})

test/replicator.spec.js

@@ -38,7 +38,7 @@ Object.keys(testAPIs).forEach((IPFS) => {
       const cache = new Cache(cacheStore)
 
       testIdentity = await IdentityProvider.createIdentity({ id: 'userA', keystore })
-      ipfsd = await startIpfs(IPFS, ipfsConfig)
+      ipfsd = await startIpfs(IPFS, ipfsConfig.daemon1)
       ipfs = ipfsd.api
 
       const address = 'test-address'