diff --git a/cid-vs-multihash.js b/cid-vs-multihash.js new file mode 100644 index 0000000..9139cd9 --- /dev/null +++ b/cid-vs-multihash.js @@ -0,0 +1,25 @@ +import { CID } from 'multiformats/cid' +import * as multihash from 'multiformats/hashes/digest' + +/* +HUMAN READABLE CID +base32 - cidv1 - raw - (sha2-256 : 256 : FAE8D07F50DBBCD3FDD65610ADCD6C3DF2171B31AE97041B1E222506E78DFD94) +MULTIBASE - VERSION - MULTICODEC - MULTIHASH (NAME : SIZE : DIGEST IN HEX) +*/ +const MY_CID = 'bafkreih25dih6ug3xtj73vswccw423b56ilrwmnos4cbwhrceudopdp5sq' +console.log('CID string', MY_CID) + +// Found via https://cid.contact/cid/bafkreih25dih6ug3xtj73vswccw423b56ilrwmnos4cbwhrceudopdp5sq +const MULTIHASH = 'EiD66NB/UNu80/3WVhCtzWw98hcbMa6XBBseIiUG5439lA==' +console.log('IPNI Multihash (base64)', MULTIHASH) +console.log('IPNI multihash (hex)', Buffer.from(MULTIHASH, 'base64').toString('hex')) + +const cid = CID.parse(MY_CID) +console.log('CID multihash (hex) ', Buffer.from(cid.multihash.bytes).toString('hex')) +console.log('CID codec', '0x' + cid.code.toString(16)) + +const cid2 = CID.create(cid.version, 0x70 /* dag-pb */, cid.multihash) +console.log('dag-pb CID', cid2) + +console.log('CID FROM MULTIHASH', CID.create(1, 0x55 /* raw */, multihash.decode(Buffer.from(MULTIHASH, 'base64')))) +console.log('ORIGINAL CID ', cid) diff --git a/package-lock.json b/package-lock.json index 4d76240..c6a03d4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,13 +1,302 @@ { - "name": "piece-indexer", + "name": "@filecoin-station/piece-indexer", "version": "0.1.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "piece-indexer", + "name": "@filecoin-station/piece-indexer", "version": "0.1.0", - "license": "(Apache-2.0 AND MIT)" + "license": "(Apache-2.0 AND MIT)", + "dependencies": { + "@ipld/dag-cbor": "^9.2.1", + "@multiformats/multiaddr-to-uri": "^10.1.0", + "multiformats": "^13.1.3" + } + }, + "node_modules/@chainsafe/is-ip": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/@chainsafe/is-ip/-/is-ip-2.0.2.tgz", + "integrity": "sha512-ndGqEMG1W5WkGagaqOZHpPU172AGdxr+LD15sv3WIUvT5oCFUrG1Y0CW/v2Egwj4JXEvSibaIIIqImsm98y1nA==" + }, + "node_modules/@chainsafe/netmask": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@chainsafe/netmask/-/netmask-2.0.0.tgz", + "integrity": "sha512-I3Z+6SWUoaljh3TBzCnCxjlUyN8tA+NAk5L6m9IxvCf1BENQTePzPMis97CoN/iMW1St3WN+AWCCRp+TTBRiDg==", + "dependencies": { + "@chainsafe/is-ip": "^2.0.1" + } + }, + "node_modules/@ipld/dag-cbor": { + "version": "9.2.1", + "resolved": "https://registry.npmjs.org/@ipld/dag-cbor/-/dag-cbor-9.2.1.tgz", + "integrity": "sha512-nyY48yE7r3dnJVlxrdaimrbloh4RokQaNRdI//btfTkcTEZbpmSrbYcBQ4VKTf8ZxXAOUJy4VsRpkJo+y9RTnA==", + "dependencies": { + "cborg": "^4.0.0", + "multiformats": "^13.1.0" + }, + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/@leichtgewicht/ip-codec": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@leichtgewicht/ip-codec/-/ip-codec-2.0.5.tgz", + "integrity": "sha512-Vo+PSpZG2/fmgmiNzYK9qWRh8h/CHrwD0mo1h1DzL4yzHNSfWYujGTYsWGreD000gcgmZ7K4Ys6Tx9TxtsKdDw==" + }, + "node_modules/@libp2p/interface": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@libp2p/interface/-/interface-1.5.0.tgz", + "integrity": "sha512-SivVvZ+7r7Vgnv+Y88nGZTpG449PYZAPgfLhVqTXn6T4NAFt47InzC7UMFnVqSQuT21YmI9DoeaVXH73CsHNrg==", + "dependencies": { + "@multiformats/multiaddr": "^12.2.3", + "it-pushable": "^3.2.3", + "it-stream-types": "^2.0.1", + "multiformats": "^13.1.0", + "progress-events": "^1.0.0", + "uint8arraylist": "^2.4.8" + } + }, + "node_modules/@multiformats/dns": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/@multiformats/dns/-/dns-1.0.6.tgz", + "integrity": "sha512-nt/5UqjMPtyvkG9BQYdJ4GfLK3nMqGpFZOzf4hAmIa0sJh2LlS9YKXZ4FgwBDsaHvzZqR/rUFIywIc7pkHNNuw==", + "dependencies": { + "@types/dns-packet": "^5.6.5", + "buffer": "^6.0.3", + "dns-packet": "^5.6.1", + "hashlru": "^2.3.0", + "p-queue": "^8.0.1", + "progress-events": "^1.0.0", + "uint8arrays": "^5.0.2" + } + }, + "node_modules/@multiformats/multiaddr": { + "version": "12.3.0", + "resolved": "https://registry.npmjs.org/@multiformats/multiaddr/-/multiaddr-12.3.0.tgz", + "integrity": "sha512-JQ8Gc/jgucqqvEaDTFN/AvxlYDHEE7lgEWLMYW7hKZkWggER+GvG/tVxUgUxIP8M0vFpvEHKKHE0lKzyMsgi8Q==", + "dependencies": { + "@chainsafe/is-ip": "^2.0.1", + "@chainsafe/netmask": "^2.0.0", + "@libp2p/interface": "^1.0.0", + "@multiformats/dns": "^1.0.3", + "multiformats": "^13.0.0", + "uint8-varint": "^2.0.1", + "uint8arrays": "^5.0.0" + } + }, + "node_modules/@multiformats/multiaddr-to-uri": { + "version": "10.1.0", + "resolved": "https://registry.npmjs.org/@multiformats/multiaddr-to-uri/-/multiaddr-to-uri-10.1.0.tgz", + "integrity": "sha512-ZNwSAx3ssBWwd4y0LKrOsq9xG7LBHboQxnUdSduNc2fTh/NS1UjA2slgUy6KHxH5k9S2DSus0iU2CoyJyN0/pg==", + "dependencies": { + "@multiformats/multiaddr": "^12.3.0" + } + }, + "node_modules/@types/dns-packet": { + "version": "5.6.5", + "resolved": "https://registry.npmjs.org/@types/dns-packet/-/dns-packet-5.6.5.tgz", + "integrity": "sha512-qXOC7XLOEe43ehtWJCMnQXvgcIpv6rPmQ1jXT98Ad8A3TB1Ue50jsCbSSSyuazScEuZ/Q026vHbrOTVkmwA+7Q==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/node": { + "version": "20.14.9", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.9.tgz", + "integrity": "sha512-06OCtnTXtWOZBJlRApleWndH4JsRVs1pDCc8dLSQp+7PpUpX3ePdHyeNSFTeSe7FtKyQkrlPvHwJOW3SLd8Oyg==", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/buffer": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", + "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.2.1" + } + }, + "node_modules/cborg": { + "version": "4.2.2", + "resolved": "https://registry.npmjs.org/cborg/-/cborg-4.2.2.tgz", + "integrity": "sha512-A0z7WhnY4HDLrVdnQI4i/OLG3kANHotk5NzDpr2iauf4xrmQPwJCxlbCnIXkVrFtsr8G3omfvvr5oF50i1Zt8g==", + "bin": { + "cborg": "lib/bin.js" + } + }, + "node_modules/dns-packet": { + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.6.1.tgz", + "integrity": "sha512-l4gcSouhcgIKRvyy99RNVOgxXiicE+2jZoNmaNmZ6JXiGajBOJAesk1OBlJuM5k2c+eudGdLxDqXuPCKIj6kpw==", + "dependencies": { + "@leichtgewicht/ip-codec": "^2.0.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/eventemitter3": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", + "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==" + }, + "node_modules/hashlru": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/hashlru/-/hashlru-2.3.0.tgz", + "integrity": "sha512-0cMsjjIC8I+D3M44pOQdsy0OHXGLVz6Z0beRuufhKa0KfaD2wGwAev6jILzXsd3/vpnNQJmWyZtIILqM1N+n5A==" + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ] + }, + "node_modules/it-pushable": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/it-pushable/-/it-pushable-3.2.3.tgz", + "integrity": "sha512-gzYnXYK8Y5t5b/BnJUr7glfQLO4U5vyb05gPx/TyTw+4Bv1zM9gFk4YsOrnulWefMewlphCjKkakFvj1y99Tcg==", + "dependencies": { + "p-defer": "^4.0.0" + } + }, + "node_modules/it-stream-types": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/it-stream-types/-/it-stream-types-2.0.1.tgz", + "integrity": "sha512-6DmOs5r7ERDbvS4q8yLKENcj6Yecr7QQTqWApbZdfAUTEC947d+PEha7PCqhm//9oxaLYL7TWRekwhoXl2s6fg==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/multiformats": { + "version": "13.1.3", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-13.1.3.tgz", + "integrity": "sha512-CZPi9lFZCM/+7oRolWYsvalsyWQGFo+GpdaTmjxXXomC+nP/W1Rnxb9sUgjvmNmRZ5bOPqRAl4nuK+Ydw/4tGw==" + }, + "node_modules/p-defer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/p-defer/-/p-defer-4.0.1.tgz", + "integrity": "sha512-Mr5KC5efvAK5VUptYEIopP1bakB85k2IWXaRC0rsh1uwn1L6M0LVml8OIQ4Gudg4oyZakf7FmeRLkMMtZW1i5A==", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-queue": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-8.0.1.tgz", + "integrity": "sha512-NXzu9aQJTAzbBqOt2hwsR63ea7yvxJc0PwN/zobNAudYfb1B7R08SzB4TsLeSbUCuG467NhnoT0oO6w1qRO+BA==", + "dependencies": { + "eventemitter3": "^5.0.1", + "p-timeout": "^6.1.2" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-6.1.2.tgz", + "integrity": "sha512-UbD77BuZ9Bc9aABo74gfXhNvzC9Tx7SxtHSh1fxvx3jTLLYvmVhiQZZrJzqqU0jKbN32kb5VOKiLEQI/3bIjgQ==", + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/progress-events": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/progress-events/-/progress-events-1.0.0.tgz", + "integrity": "sha512-zIB6QDrSbPfRg+33FZalluFIowkbV5Xh1xSuetjG+rlC5he6u2dc6VQJ0TbMdlN3R1RHdpOqxEFMKTnQ+itUwA==", + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + } + }, + "node_modules/uint8-varint": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/uint8-varint/-/uint8-varint-2.0.4.tgz", + "integrity": "sha512-FwpTa7ZGA/f/EssWAb5/YV6pHgVF1fViKdW8cWaEarjB8t7NyofSWBdOTyFPaGuUG4gx3v1O3PQ8etsiOs3lcw==", + "dependencies": { + "uint8arraylist": "^2.0.0", + "uint8arrays": "^5.0.0" + } + }, + "node_modules/uint8arraylist": { + "version": "2.4.8", + "resolved": "https://registry.npmjs.org/uint8arraylist/-/uint8arraylist-2.4.8.tgz", + "integrity": "sha512-vc1PlGOzglLF0eae1M8mLRTBivsvrGsdmJ5RbK3e+QRvRLOZfZhQROTwH/OfyF3+ZVUg9/8hE8bmKP2CvP9quQ==", + "dependencies": { + "uint8arrays": "^5.0.1" + } + }, + "node_modules/uint8arrays": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/uint8arrays/-/uint8arrays-5.1.0.tgz", + "integrity": "sha512-vA6nFepEmlSKkMBnLBaUMVvAC4G3CTmO58C12y4sq6WPDOR7mOFYOi7GlrQ4djeSbP6JG9Pv9tJDM97PedRSww==", + "dependencies": { + "multiformats": "^13.0.0" + } + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" } } } diff --git a/package.json b/package.json index 0e9b370..e324438 100644 --- a/package.json +++ b/package.json @@ -16,5 +16,10 @@ "bugs": { "url": "https://github.com/filecoin-station/piece-indexer/issues" }, - "homepage": "https://github.com/filecoin-station/piece-indexer#readme" + "homepage": "https://github.com/filecoin-station/piece-indexer#readme", + "dependencies": { + "@ipld/dag-cbor": "^9.2.1", + "@multiformats/multiaddr-to-uri": "^10.1.0", + "multiformats": "^13.1.3" + } } diff --git a/parse-metada.js b/parse-metada.js new file mode 100644 index 0000000..0013fd1 --- /dev/null +++ b/parse-metada.js @@ -0,0 +1,31 @@ +import { varint } from 'multiformats' +import * as cbor from '@ipld/dag-cbor' + +// console.log(parseMetadata( +// 'kBKjaFBpZWNlQ0lE2CpYKAABgeIDkiAgieDL6/pbxDBAJtdZ19ZsvGh1NE77nBxbFwesoG1S/jJsVmVyaWZpZWREZWFs9W1GYXN0UmV0cmlldmFs9Q' +// )) + +export function parseMetadata (meta) { + const bytes = Buffer.from(meta, 'base64') + const [protocolCode, nextOffset] = varint.decode(bytes) + + const protocol = { + 0x900: 'bitswap', + 0x910: 'graphsync', + 0x0920: 'http' + }[protocolCode] ?? '0x' + protocolCode.toString(16) + + if (protocol === 'graphsync') { + // console.log(bytes.subarray(nextOffset).toString('hex')) + + /** @type {{ + PieceCID: import('multiformats/cid').CID, + VerifiedDeal: boolean, + FastRetrieval: boolean + }} */ + const deal = cbor.decode(bytes.subarray(nextOffset)) + return { protocol, deal } + } else { + return { protocol } + } +} diff --git a/poc.js b/poc.js new file mode 100644 index 0000000..896460a --- /dev/null +++ b/poc.js @@ -0,0 +1,104 @@ +import { multiaddrToUri } from '@multiformats/multiaddr-to-uri' +import { CID } from 'multiformats/cid' +import * as multihash from 'multiformats/hashes/digest' +import assert from 'node:assert' +import { parseMetadata } from './parse-metada.js' + +// IPNI specs: https://github.com/ipni/specs + +// TODO: list all index providers +// https://cid.contact/providers + +// https://github.com/filecoin-station/frisbii-on-fly +// This index provider does not set Graphsync metadata +// const providerPeerId = '12D3KooWHge6fZmx6fMsizP9YYpbJPNZjiWz7Ye1WLCvmj6VTnjq' +// +// Doesn't work - cannot fetch Entries CID +// Error: Cannot GET http://183.60.90.198:3104/ipni/v1/ad/bafkreehdwdcefgh4dqkjv67uzcmw7oje: 500 +// unable to load data for cid +// const providerPeerId = '12D3KooWFpNqyFpqujkMXeKrbtasvKkbUSL8ipN5vXtNyfVo7n4f' + +// This one works: +const providerPeerId = '12D3KooWDYiKtcxTrjNFtR6UqKRkJpESYHmmFznQAAkDX2ZHQ49t' + +// Get the latest announcement +const res = await fetch(`https://cid.contact/providers/${encodeURIComponent(providerPeerId)}`) +assert(res.ok) + +/** @type {{ +AddrInfo:{ID:string,Addrs:string[]}, +LastAdvertisement:{"/":string}, +LastAdvertisementTime: string, +Publisher:{ID:string,Addrs:string[]}, +FrozenAt:null + * }} + */ +const providerMetadata = await res.json() + +console.log('Provider metadata:', providerMetadata) +const providerBaseUrl = multiaddrToUri(providerMetadata.Publisher.Addrs[0], { assumeHttp: false }) +if (!providerBaseUrl.startsWith('http')) { + throw new Error(`Unsupported URI scheme: ${providerBaseUrl}`) +} + +// TODO: handle libp2p index providers, e.g. +// Publisher: { +// ID: '12D3KooWCmLYzfYU2fWVnWJDuEkjt7d8pq7PHYAkXoUFVCPogTA9', +// Addrs: [ '/ip4/103.9.208.54/tcp/48080' ] +// } + +/** @type {{ + Addresses: string[], + ContextID: { '/': { bytes: string } },, + Entries: { + '/': string + }, + IsRm: false, + Metadata: { '/': { bytes: string } }, + Provider: string + Signature: { + '/': { + bytes: string + } + } + }} */ +const head = await fetchCid(providerBaseUrl, providerMetadata.LastAdvertisement['/']) +console.log('HEAD', head) + +const entries = await fetchCid(providerBaseUrl, head.Entries['/']) +// console.log('ENTRIES', entries.Entries) + +const entryHash = entries.Entries[0]['/'].bytes +console.log('FIRST ENTRY:', entryHash) +const payloadCid = CID.create(1, 0x55 /* raw */, multihash.decode(Buffer.from(entryHash, 'base64'))) +console.log('PAYLOAD CID', payloadCid) + +// const provRes = await fetch(`https://cid.contact/cid/${encodeURIComponent(payloadCid)}`) +// const provBody = await provRes.json() +// const allProviderResults = provBody.MultihashResults[0].ProviderResults +// // console.log('filtering Provider.ID === %s && ContextID === %s', providerPeerId,head.ContextID['/'].bytes) +// const provider = allProviderResults.find(r => r.Provider.ID === providerPeerId && r.ContextID === head.ContextID['/'].bytes) +// console.log('Retrieval provider', provider) +// console.log('You can fetch the block via lassie') +// console.log(' lassie fetch -o /dev/null -v --dag-scope block --providers %s/p2p/%s %s', provider.Provider.Addrs[0], providerPeerId, payloadCid) + +console.log('Metadata bytes', head.Metadata['/'].bytes) +const meta = parseMetadata(head.Metadata['/'].bytes) +console.log('Metadata', meta) + +if (meta.deal) { + console.log('\n=====') + console.log('PieceCID ', meta.deal.PieceCID) + console.log('PayloadCID', payloadCid) +} + +async function fetchCid (providerBaseUrl, adCid) { + const url = new URL(adCid, new URL('/ipni/v1/ad/_', providerBaseUrl)) + console.log('fetching %s', url) + const res = await fetch(url) + console.log('status', res.status) + if (!res.ok) { + throw new Error(`Cannot GET ${url}: ${res.status}\n${await res.text()}`) + } + return await res.json() +}