feat: upgrade deps and add experimental query resolver (#14)

* feat: upgrade deps and add experimental query resolver

* chore: bump version

Author: tchardin

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@dcdn/graphsync",
-  "version": "0.2.4",
+  "version": "0.2.5",
   "description": "JS implementation of GraphSync and DataTransfer protocols",
   "main": "dist/src/index.js",
   "files": [
@@ -27,9 +27,9 @@
   },
   "type": "module",
   "devDependencies": {
-    "@chainsafe/libp2p-noise": "^9.0.0",
-    "@libp2p/interface-mocks": "^7.0.3",
-    "@libp2p/mplex": "^7.0.0",
+    "@chainsafe/libp2p-noise": "^10.0.1",
+    "@libp2p/interface-mocks": "^8.0.1",
+    "@libp2p/mplex": "^7.1.0",
     "@types/bl": "^5.0.2",
     "@types/mime": "^2.0.3",
     "@types/uuid": "^8.3.4",
@@ -43,9 +43,9 @@
   "dependencies": {
     "@ipld/dag-cbor": "^7.0.1",
     "@ipld/dag-pb": "^2.1.16",
-    "@libp2p/interface-connection": "^3.0.2",
-    "@libp2p/interface-peer-id": "^1.0.5",
-    "@libp2p/interface-registrar": "^2.0.3",
+    "@libp2p/interface-connection": "^3.0.3",
+    "@libp2p/interface-peer-id": "^1.0.6",
+    "@libp2p/interface-registrar": "^2.0.4",
     "@libp2p/peer-id": "^1.1.16",
     "@libp2p/tcp": "^5.0.1",
     "@multiformats/multiaddr": "^11.0.5",

src/index.ts

@@ -1,4 +1,4 @@
 export * from "./traversal.js";
 export {GraphSync, Request} from "./graphsync.js";
-export {fetch, unixfsPathSelector, getPeer} from "./resolver.js";
+export {fetch, unixfsPathSelector, getPeer, resolveQuery} from "./resolver.js";
 export {push} from "./push.js";

src/resolver.ts

@@ -11,6 +11,10 @@ import {
   parseContext,
   walkBlocks,
   SelectorNode,
+  Selector,
+  Node,
+  PathSegment,
+  ExploreInterpretAs,
 } from "./traversal.js";
 import mime from "mime/lite.js";
 import type {GraphSync} from "./graphsync.js";
@@ -193,3 +197,79 @@ export async function fetch(url: string, init: FetchInit): Promise<Response> {
     });
   }
 }
+
+export async function resolveQuery(
+  nd: Node,
+  sel: Selector,
+  loader: LinkLoader
+): Promise<any> {
+  if (sel instanceof ExploreInterpretAs) {
+    const reify = loader.reifier(sel.adl);
+    if (reify) {
+      const rnd = await reify(nd, loader);
+      const next = sel.explore(nd, new PathSegment(""));
+      if (next) {
+        sel = next;
+      }
+      nd = rnd;
+    }
+  }
+
+  let results: {[key: string]: any} | any[];
+
+  switch (nd.kind) {
+    case Kind.Map:
+      results = {};
+      break;
+    case Kind.List:
+      results = [];
+      break;
+    default:
+      return nd.value;
+  }
+
+  const attn = sel.interests();
+  if (attn.length) {
+    for (const ps of attn) {
+      let value = await nd.lookupBySegment(ps);
+      if (value === null) {
+        break;
+      }
+      const sNext = sel.explore(nd.value, ps);
+      if (sNext !== null) {
+        if (value.kind === Kind.Link) {
+          const blk = await loader.load(value.value);
+
+          value = new BasicNode(blk.value);
+        }
+
+        const val = await resolveQuery(value, sNext, loader);
+        if (Array.isArray(results)) {
+          results.push(val);
+        } else {
+          results[ps.value] = val;
+        }
+      }
+    }
+  } else {
+    // visit everything
+    for await (let {pathSegment, value} of nd.entries()) {
+      const sNext = sel.explore(nd.value, pathSegment);
+      if (sNext !== null) {
+        if (value.kind === Kind.Link) {
+          const blk = await loader.load(value.value);
+
+          value = new BasicNode(blk.value);
+        }
+
+        const val = await resolveQuery(value, sNext, loader);
+        if (Array.isArray(results)) {
+          results.push(val);
+        } else {
+          results[pathSegment.value] = val;
+        }
+      }
+    }
+  }
+  return results;
+}

src/traversal.ts

@@ -155,7 +155,7 @@ export class BasicNode implements Node {
   async lookupBySegment(seg: PathSegment): Promise<Node | null> {
     const val = this.value[seg.value];
     if (val) {
-      return val;
+      return new BasicNode(val);
     }
     return null;
   }
@@ -374,6 +374,8 @@ export function parseContext() {
           return this.parseExploreInterpretAs(node[key]);
         case ".":
           return this.parseMatcher(node[key]);
+        case "i":
+          return this.parseExploreIndex(node[key]);
         default:
           throw new Error("unknown selector");
       }
@@ -415,6 +417,21 @@ export function parseContext() {
       }
       return new ExploreFields(selections, interests);
     },
+    parseExploreIndex(node: any): Selector {
+      const index = node["i"];
+      const next = this.parseSelector(node[">"]);
+      if (typeof index !== "number") {
+        throw new Error(
+          "selector spec parse rejected: index field must be present in ExploreIndex selector"
+        );
+      }
+      if (!next) {
+        throw new Error(
+          "selector spec parse rejected: next field must be present in ExploreIndex selector"
+        );
+      }
+      return new ExploreIndex(next, index);
+    },
     parseExploreInterpretAs(node: any): Selector {
       const adl = node["as"];
       const next: SelectorNode = node[">"];
@@ -499,6 +516,30 @@ export class ExploreFields implements Selector {
   }
 }
 
+export class ExploreIndex implements Selector {
+  next: Selector;
+  interest: [PathSegment];
+  constructor(next: Selector, index: number) {
+    this.next = next;
+    this.interest = [new PathSegment(index)];
+  }
+  interests(): PathSegment[] {
+    return this.interest;
+  }
+  explore(node: any, path: PathSegment): Selector | null {
+    if (!Array.isArray(node)) {
+      return null;
+    }
+    if (path.toIndex() === this.interest[0].toIndex()) {
+      return this.next;
+    }
+    return null;
+  }
+  decide(node: any): boolean {
+    return false;
+  }
+}
+
 export class ExploreInterpretAs implements Selector {
   next: Selector;
   adl: string;
@@ -656,9 +697,9 @@ export async function unixfsReifier(
     try {
       const unixfs = UnixFS.unmarshal(node.value.Data);
       if (unixfs.isDirectory()) {
-        const dir: {[key: string]: Node} = {};
+        const dir: {[key: string]: CID} = {};
         for (const link of node.value.Links) {
-          dir[link.Name] = new BasicNode(link.Hash);
+          dir[link.Name] = link.Hash;
         }
         return new BasicNode(dir);
       }

test/listening.node.ts

@@ -2,7 +2,7 @@ import {expect} from "aegir/chai";
 import {createLibp2p, Libp2p} from "libp2p";
 import {mplex} from "@libp2p/mplex";
 import {tcp} from "@libp2p/tcp";
-import {Noise} from "@chainsafe/libp2p-noise";
+import {noise} from "@chainsafe/libp2p-noise";
 import {MemoryBlockstore} from "blockstore-core/memory";
 import {importer} from "ipfs-unixfs-importer";
 import {GraphSync} from "../src/graphsync.js";
@@ -16,7 +16,7 @@ async function createNode(): Promise<Libp2p> {
     },
     streamMuxers: [mplex()],
     transports: [tcp()],
-    connectionEncryption: [() => new Noise()],
+    connectionEncryption: [noise()],
   });
   await node.start();
   return node;

test/resolver.spec.ts

@@ -1,10 +1,24 @@
 import {expect} from "aegir/chai";
-import {resolve, unixfsPathSelector, getPeer} from "../src/resolver.js";
+import {
+  resolve,
+  unixfsPathSelector,
+  getPeer,
+  resolveQuery,
+} from "../src/resolver.js";
+import {
+  selectorBuilder as sb,
+  BasicNode,
+  parseContext,
+  LinkSystem,
+} from "../src/traversal.js";
 import {MemoryBlockstore} from "blockstore-core/memory";
 import {MockLibp2p, concatChunkIterator} from "./mock-libp2p.js";
 import {peerIdFromString} from "@libp2p/peer-id";
 import {GraphSync} from "../src/graphsync.js";
 import {importer} from "ipfs-unixfs-importer";
+import {encode} from "multiformats/block";
+import * as codec from "@ipld/dag-cbor";
+import {sha256 as hasher} from "multiformats/hashes/sha2";
 
 describe("resolver", () => {
   it("parse a unixfs path", () => {
@@ -68,4 +82,74 @@ describe("resolver", () => {
     );
     expect(multiaddrs[0].protos()[0].name).to.equal("ip4");
   });
+
+  it("resolves a query", async () => {
+    const blocks = new MemoryBlockstore();
+
+    const account1 = {
+      balance: 300,
+      lastUpdated: "yesterday",
+    };
+    const account1Block = await encode({value: account1, codec, hasher});
+    await blocks.put(account1Block.cid, account1Block.bytes);
+
+    const account2 = {
+      balance: 100,
+      lastUpdated: "now",
+    };
+    const account2Block = await encode({value: account2, codec, hasher});
+    await blocks.put(account2Block.cid, account2Block.bytes);
+
+    const state = {
+      "0x01": account1Block.cid,
+      "0x02": account2Block.cid,
+    };
+    const stateBlock = await encode({value: state, codec, hasher});
+    await blocks.put(stateBlock.cid, stateBlock.bytes);
+
+    const msg = {
+      from: "0x01",
+      to: "0x02",
+      amount: 100,
+    };
+    const msgBlock = await encode({value: msg, codec, hasher});
+    await blocks.put(msgBlock.cid, msgBlock.bytes);
+
+    const root = {
+      state: stateBlock.cid,
+      epoch: Date.now(),
+      messages: [msgBlock.cid],
+    };
+    const rootBlock = await encode({value: root, codec, hasher});
+    await blocks.put(rootBlock.cid, rootBlock.bytes);
+
+    const selector = sb.exploreFields({
+      state: sb.exploreFields({
+        "0x02": sb.exploreFields({
+          balance: sb.match(),
+          lastUpdated: sb.match(),
+        }),
+      }),
+      messages: sb.exploreIndex(
+        0,
+        sb.exploreFields({
+          amount: sb.match(),
+        })
+      ),
+    });
+    const sel = parseContext().parseSelector(selector);
+    const ls = new LinkSystem(blocks);
+
+    const result = await resolveQuery(new BasicNode(root), sel, ls);
+
+    expect(result).to.deep.equal({
+      state: {
+        "0x02": {
+          balance: 100,
+          lastUpdated: "now",
+        },
+      },
+      messages: [{amount: 100}],
+    });
+  });
 });

test/traversal.spec.ts

@@ -225,4 +225,61 @@ describe("traversal", () => {
     );
     expect(selector3).to.deep.equal(unixfsSelector);
   });
+
+  it("selects an index", async () => {
+    const bs = new MemoryBlockstore();
+
+    const first = new Uint8Array(5 * 256);
+    const second = new Uint8Array(3 * 256);
+    const third = new Uint8Array(2 * 256);
+    const forth = new Uint8Array(4 * 256);
+
+    // chunk and dagify it then get the root cid
+    let cid;
+    for await (const chunk of importer(
+      [
+        {path: "first", content: first},
+        {path: "second", content: second},
+        {path: "/children/third", content: third},
+        {path: "/children/forth", content: forth},
+      ],
+      bs,
+      {
+        cidVersion: 1,
+        maxChunkSize: 256,
+        rawLeaves: true,
+        wrapWithDirectory: true,
+      }
+    )) {
+      if (chunk.path === "") {
+        cid = chunk.cid;
+      }
+    }
+    const selector = builder.exploreFields({
+      Links: builder.exploreIndex(
+        0,
+        builder.exploreRecursive(
+          builder.exploreAll(builder.edge()),
+          builder.depth(2)
+        )
+      ),
+    });
+
+    const sel = parseContext().parseSelector(selector);
+
+    const source = new LinkSystem(bs);
+
+    let last;
+    for await (const blk of walkBlocks(new BasicNode(cid), sel, source)) {
+      last = blk;
+    }
+
+    if (!last) {
+      throw new Error("failed traversal");
+    }
+
+    expect(last.cid.toString()).to.equal(
+      "bafybeiepvdqmdakhtwotvykxujrmt5fcq4xca5jmoo6wzxhjk3q3pqe4te"
+    );
+  });
 });