From 6796de90b2a4ea186bc1f3e98d3fe396bbdbafa5 Mon Sep 17 00:00:00 2001 From: wayne Date: Tue, 14 Oct 2025 20:40:26 +0100 Subject: [PATCH 1/7] pending ES allTokens, SDK price+mcap fetchers, error handling, remove WIP work --- defi/l2/adapters/thirdParty.ts | 97 ++---- defi/l2/tvl.ts | 2 +- defi/l2/utils.ts | 3 +- defi/l2/v2/index.ts | 462 +++++++++++++++++++++----- defi/l2/{test.ts => verifyChanges.ts} | 0 5 files changed, 410 insertions(+), 154 deletions(-) rename defi/l2/{test.ts => verifyChanges.ts} (100%) diff --git a/defi/l2/adapters/thirdParty.ts b/defi/l2/adapters/thirdParty.ts index b6828f5114..eb7a2597b2 100644 --- a/defi/l2/adapters/thirdParty.ts +++ b/defi/l2/adapters/thirdParty.ts @@ -7,12 +7,10 @@ import fetch from "node-fetch"; import { additional, excluded } from "./manual"; import axios from "axios"; import PromisePool from "@supercharge/promise-pool"; +import _ from "lodash"; -let bridgePromises: { [bridge: string]: Promise } = {}; const addresses: { [chain: Chain]: Address[] } = {}; allChainKeys.map((c: string) => (addresses[c] = [])); -let doneAdapters: string[] = []; -let mappingDone: boolean = false; const chainMap: { [chain: string]: string } = { binance: "bsc", @@ -25,24 +23,15 @@ Object.keys(providers).map((c: string) => { }); const hyperlane = async (): Promise => { - const bridge = "hyperlane"; - if (doneAdapters.includes(bridge)) return; - if (!(bridge in bridgePromises)) - bridgePromises[bridge] = fetch( - "https://raw.githubusercontent.com/Eclipse-Laboratories-Inc/gist/refs/heads/main/hyperlane-assets.json" - ).then((r) => r.json()); - const data = await bridgePromises[bridge]; + const data = await fetch( + "https://raw.githubusercontent.com/Eclipse-Laboratories-Inc/gist/refs/heads/main/hyperlane-assets.json" + ).then((r) => r.json()); if (!addresses.eclipse) addresses.eclipse = []; data.map(({ address }: any) => addresses.eclipse.push(address)); - doneAdapters.push(bridge); }; const axelar = async (): Promise => { - const bridge = "axelar"; - if (doneAdapters.includes(bridge)) return; - if (!(bridge in bridgePromises)) - bridgePromises[bridge] = fetch("https://api.axelarscan.io/api/getAssets").then((r) => r.json()); - const data = await bridgePromises[bridge]; + const data = await fetch("https://api.axelarscan.io/api/getAssets").then((r) => r.json()); data.map((token: any) => { if (!token.addresses) return; Object.keys(token.addresses).map((chain: string) => { @@ -51,22 +40,16 @@ const axelar = async (): Promise => { if (!allChainKeys.includes(normalizedChain)) return; if (!("address" in token.addresses[chain] && "symbol" in token.addresses[chain])) return; if (!token.addresses[chain].symbol.startsWith("axl")) return; - addresses[normalizedChain].push(token.addresses[chain].address.toLowerCase()); + addresses[normalizedChain].push(token.addresses[chain].address); }); }); - doneAdapters.push(bridge); }; const wormhole = async (): Promise => { - const bridge = "wormhole"; + const data = await axios.get( + "https://raw.githubusercontent.com/wormhole-foundation/wormhole-token-list/main/content/by_dest.csv" + ); - if (doneAdapters.includes(bridge)) return; - if (!(bridge in bridgePromises)) - bridgePromises[bridge] = axios.get( - "https://raw.githubusercontent.com/wormhole-foundation/wormhole-token-list/main/content/by_dest.csv" - ); - - const data = (await bridgePromises[bridge]).data; const chainMap: { [ticker: string]: string } = { sol: "solana", eth: "ethereum", @@ -86,7 +69,7 @@ const wormhole = async (): Promise => { base: "base", }; - const lines = data.split("\n"); + const lines = data.data.split("\n"); lines.shift(); lines.map((l: string) => { const rows = l.split(","); @@ -95,43 +78,32 @@ const wormhole = async (): Promise => { if (!addresses[chain]) addresses[chain] = []; addresses[chain].push(rows[3]); }); - doneAdapters.push(bridge); }; const celer = async (): Promise => { - const bridge = "celer"; - if (doneAdapters.includes(bridge)) return; - if (!(bridge in bridgePromises)) - bridgePromises[bridge] = fetch("https://cbridge-prod2.celer.app/v2/getTransferConfigsForAll").then((r) => r.json()); - const data = await bridgePromises[bridge]; + const data = await fetch("https://cbridge-prod2.celer.app/v2/getTransferConfigsForAll").then((r) => r.json()); data.pegged_pair_configs.map((pp: any) => { const chain = chainIdMap[pp.org_chain_id]; let normalizedChain: string = chain; if (chain in chainMap) normalizedChain = chainMap[chain]; if (!allChainKeys.includes(normalizedChain)) return; if (!addresses[normalizedChain]) addresses[normalizedChain] = []; - addresses[normalizedChain].push(pp.pegged_token.token.address.toLowerCase()); + addresses[normalizedChain].push(pp.pegged_token.token.address); }); - doneAdapters.push(bridge); }; const layerzero = async (): Promise => { - const bridge = "layerzero"; - if (doneAdapters.includes(bridge)) return; - if (!(bridge in bridgePromises)) { - bridgePromises[bridge] = Promise.all([ - fetch( - "https://gist.githubusercontent.com/vrtnd/02b1125edf1afe2baddbf1027157aa31/raw/5cab2009357b1acb8982e6a80e66b64ab7ea1251/mappings.json" - ).then((r) => r.json()), - fetch("https://metadata.layerzero-api.com/v1/metadata").then((r) => r.json()), - ]); - } - const data = await bridgePromises[bridge]; + const data = await Promise.all([ + fetch( + "https://gist.githubusercontent.com/vrtnd/02b1125edf1afe2baddbf1027157aa31/raw/5cab2009357b1acb8982e6a80e66b64ab7ea1251/mappings.json" + ).then((r) => r.json()), + fetch("https://metadata.layerzero-api.com/v1/metadata").then((r) => r.json()), + ]); data[0].map(({ to }: any) => { const [chain, address] = to.split(":"); if (!(chain in addresses)) addresses[chain] = []; - if (!(address in addresses[chain])) addresses[chain].push(address.toLowerCase()); + if (!(address in addresses[chain])) addresses[chain].push(address); }); const nonEvmMapping: { [key: string]: string } = { @@ -173,18 +145,12 @@ const layerzero = async (): Promise => { if (!(chain in addresses)) addresses[chain] = []; addresses[chain].push(...staticTokens[chain]); }); - - doneAdapters.push(bridge); }; const flow = async (): Promise => { - const bridge = "flow"; - if (doneAdapters.includes(bridge)) return; - if (!(bridge in bridgePromises)) - bridgePromises[bridge] = fetch( - "https://raw.githubusercontent.com/onflow/assets/refs/heads/main/tokens/outputs/mainnet/token-list.json" - ).then((r) => r.json()); - const data = await bridgePromises[bridge]; + const data = await fetch( + "https://raw.githubusercontent.com/onflow/assets/refs/heads/main/tokens/outputs/mainnet/token-list.json" + ).then((r) => r.json()); data.tokens.map(({ chainId, address, tags }: any) => { const chain = chainIdMap[chainId]; if (!allChainKeys.includes(chain)) return; @@ -192,14 +158,9 @@ const flow = async (): Promise => { if (!(chain in addresses)) addresses[chain] = []; addresses[chain].push(address); }); - - doneAdapters.push(bridge); }; const unit = async (): Promise => { - const bridge = "unit"; - if (doneAdapters.includes(bridge)) return; - const staticTokens: { [chain: string]: string[] } = { hyperliquid: [ "0x9FDBdA0A5e284c32744D2f17Ee5c74B284993463", @@ -213,27 +174,24 @@ const unit = async (): Promise => { if (!(chain in addresses)) addresses[chain] = []; addresses[chain].push(...staticTokens[chain]); }); - - doneAdapters.push(bridge); }; const adapters = { axelar, wormhole, celer, hyperlane, layerzero, flow, unit }; const filteredAddresses: { [chain: Chain]: Address[] } = {}; -const tokenAddresses = async (): Promise<{ [chain: Chain]: Address[] }> => { +const tokenAddresses = _.once(async (): Promise<{ [chain: Chain]: Address[] }> => { await PromisePool.withConcurrency(5) .for(Object.entries(adapters)) .process(async ([key, adapter]: any) => { try { await adapter(); } catch (e: any) { - throw new Error(`${key} fails with ${e.message}`); + throw new Error(`${key} fails with ${e.message}`); } }); - if (Object.keys(adapters).length == doneAdapters.length && mappingDone) return filteredAddresses; - + // remove excluded assets and add additional assets, normalize case Object.keys(addresses).map((chain: string) => { let chainAddresses = chain in excluded ? addresses[chain].filter((t: string) => !excluded[chain].includes(t)) : addresses[chain]; @@ -248,8 +206,7 @@ const tokenAddresses = async (): Promise<{ [chain: Chain]: Address[] }> => { filteredAddresses[chain] = [...new Set([...chainAddresses, ...additionalTokens])]; }); - mappingDone = true; return filteredAddresses; -}; +}); -export default tokenAddresses; \ No newline at end of file +export default tokenAddresses; diff --git a/defi/l2/tvl.ts b/defi/l2/tvl.ts index c9134dd8d4..6909b7d24f 100644 --- a/defi/l2/tvl.ts +++ b/defi/l2/tvl.ts @@ -8,7 +8,7 @@ import { Chain } from "@defillama/sdk/build/general"; import { getMcaps } from "./utils"; import { getCurrentUnixTimestamp } from "../src/utils/date"; import { getChainDisplayName } from "../src/utils/normalizeChain"; -import { verifyChanges } from "./test"; +import { verifyChanges } from "./verifyChanges"; import { getExcludedTvl } from "./excluded"; import { saveRawBridgedTvls } from "./raw"; diff --git a/defi/l2/utils.ts b/defi/l2/utils.ts index 570dcc29ad..e0686cf524 100644 --- a/defi/l2/utils.ts +++ b/defi/l2/utils.ts @@ -380,9 +380,8 @@ export async function fetchSupplies( } } export async function fetchBridgeTokenList(chain: Chain): Promise { - const j = Object.keys(incomingAssets).indexOf(chain); try { - const tokens: Address[] = j == -1 ? [] : await Object.values(incomingAssets)[j](); + const tokens: Address[] = incomingAssets[chain as keyof typeof incomingAssets] ? await incomingAssets[chain as keyof typeof incomingAssets]() : [] tokens.push(...((await fetchThirdPartyTokenList())[chain] ?? [])); let filteredTokens: Address[] = chain in excluded ? tokens.filter((t: string) => !excluded[chain].includes(t)) : tokens; diff --git a/defi/l2/v2/index.ts b/defi/l2/v2/index.ts index cf62ef7d93..985781644f 100644 --- a/defi/l2/v2/index.ts +++ b/defi/l2/v2/index.ts @@ -1,12 +1,9 @@ import { canonicalBridgeIds, excludedTvlKeys, protocolBridgeIds, zero, ownTokens, allChainKeys } from "../constants"; -import getTVLOfRecordClosestToTimestamp from "../../src/utils/shared/getRecordClosestToTimestamp"; import { getCurrentUnixTimestamp } from "../../src/utils/date"; -import { fetchAllTokens } from "../../src/utils/shared/bridgedTvlPostgres"; import { Chain } from "@defillama/sdk/build/types"; import { getMcaps, getPrices, fetchBridgeTokenList, fetchSupplies } from "../utils"; import { fetchAdaTokens } from "../adapters/ada"; import { withTimeout } from "../../src/utils/shared/withTimeout"; -import PromisePool from "@supercharge/promise-pool"; import { CoinsApiData, FinalData, FinalChainData } from "../types"; import { McapsApiData } from "../types"; import { getBlock } from "@defillama/sdk/build/util/blocks"; @@ -16,12 +13,300 @@ import { bridgedTvlMixedCaseChains, chainsThatShouldNotBeLowerCased } from "../. import { getR2JSONString, storeR2JSONString } from "../../src/utils/r2"; import { additional, excluded } from "../adapters/manual"; import { storeHistoricalToDB } from "./storeToDb"; -import { stablecoins } from "../../src/getProtocols"; +// import { stablecoins } from "../../src/getProtocols"; import { metadata as rwaMetadata } from "../../src/rwa/protocols"; -import { verifyChanges } from "../test"; - +import { verifyChanges } from "../verifyChanges"; +import { initializePriceQueryFilter, whitelistedTokenSet } from "../../src/storeTvlInterval/computeTVL"; +import { getClosestProtocolItem, getLatestProtocolItems, initializeTVLCacheDB } from "../../src/api2/db"; +import { hourlyRawTokensTvl } from "../../src/utils/getLastRecord"; +import { Balances } from "@defillama/sdk"; +import runInPromisePool from "@defillama/sdk/build/util/promisePool"; +import setEnvSecrets from "../../src/utils/shared/setEnvSecrets"; + +const stablecoins = [ + "USDT", + "USDC", + "DAI", + "FRAX", + "AUSDC", + "AMMUSDC", + "AETHUSDC", + "ADAI", + "AMMDAI", + "AETHDAI", + "AUSDT", + "AMMUSDT", + "LUSD", + "ALUSD", + "GUSD", + "AGUSD", + "TUSD", + "ATUSD", + "USDP", + "AUSDP", + "FEI", + "AFEI", + "BUSD", + "YYDAI+YUSDC+YUSDT+YTUSD", + "CDAI", + "BSC-USD", + "USD+", + "SUSD", + "DOLA", + "AMUSDC", + "AMDAI", + "AVUSDC", + "AVDAI", + "AAVAUSDC", + "AMUSDT", + "AAVAUSDT", + "AAVADAI", + "AVUSDT", + "AOPTUSDC", + "SUSDE", + "USDY", + "USTC", + "MIM", + "USDN", + "YUSD", + "USDD", + "PAI", + "HUSD", + "NUSD", + "FLEXUSD", + "OUSD", + "CUSD", + "RSV", + "MUSD", + "USDK", + "VAI", + "TOR", + "DOC", + "USDS", + "USDB", + "USDJ", + "STBL", + "VOLT", + "RAI", + "FLOAT", + "USDX", + "ZUSD", + "USX", + "ASEED", + "BAI", + "EURT", + "EURC", + "EURS", + "CEUR", + "SEUR", + "USN", + "EURA", + "PAR", + "USH", + "3USD", + "SIGUSD", + "HOME", + "FIAT", + "PUSD", + "FUSD", + "UXD", + "USDH", + "FPI", + "BEAN", + "USDL", + "DUSD", + "VST", + "KUSD", + "USDTZ", + "MONEY", + "UUSD", + "USDI", + "NOTE", + "LISUSD", + "USK", + "ARUSD", + "USDW", + "BOB", + "USDR", + "IUSD", + "XAI", + "RUSD", + "IBEUR", + "PINA", + "DJED", + "BAOUSD", + "USP", + "EUROE", + "CASH", + "DSU", + "EURE", + "ANONUSD", + "NXUSD", + "DCHF", + "EUSD", + "CZUSD", + "D2O", + "CRVUSD", + "DAI+", + "USDT+", + "SILK", + "CLEVUSD", + "R", + "GRAI", + "ERN", + "GHO", + "FDUSD", + "PYUSD", + "SLSD", + "GYEN", + "STAR", + "PEUSD", + "EUSD(V2)", + "MKUSD", + "LCNY", + "NEX", + "SVUSD", + "UAHT", + "USDM", + "NARS", + "IST", + "CDT", + "EEUR", + "EGBP", + "HYDT", + "USDV", + "HYUSD", + "CADC", + "USDE", + "AEUR", + "MYUSD", + "SCB", + "ZKUSD", + "BUCK", + "USDGLO", + "VCHF", + "VEUR", + "USDRIF", + "DLLR", + "EURD", + "XUSD", + "ULTRA", + "USDCB", + "AUDD", + "CGUSD", + "FETH", + "FXUSD", + "GAI", + "EURO3", + "HAI", + "BUIDL", + "PXDC", + "FXD", + "UNO", + "USD3", + "CJPY", + "BREAD", + "ZUNUSD", + "ZUNETH", + "BTCUSD", + "WEN", + "GYD", + "ISC", + "KNOX", + "USC", + "RGUSD", + "BITUSD", + "USDA", + "USD0", + "USR", + "AUSD", + "CREAL", + "HCHF", + "HEXDC", + "USDZ", + "BNUSD", + "DYAD", + "DCKUSD", + "DEUSD", + "THUSD", + "MOD", + "M", + "SATUSD", + "USDF", + "USDTB", + "PAUSD", + "XY", + "ZEUSD", + "ZCHF", + "BOLD", + "USDQ", + "LVLUSD", + "HONEY", + "PINTO", + "WUSD", + "FRXUSD", + "SYUSD", + "USYC", + "SCUSD", + "EURR", + "USDO", + "CSUSDL", + "EUROP", + "USDFC", + "BRZ", + "RLUSD", + "FEUSD", + "USBD", + "EURCV", + "REUSD", + "TBILL", + "A7A5", + "MSD", + "VUSD", + "USD1", + "MEAD", + "YU", + "BENJI", + "AVUSD", + "YLDS", + "USDAF", + "EURQ", + "WEUSD", + "PARAUSD", + "CNHT", + "MXNT", + "USDU", + "MNEE", + "GGUSD", + "USDG", + "USND", + "EBUSD", + "XSGD", + "VGBP", + "BNBUSD", + "USDA+", + "MSUSD", +]; const searchWidth = 10800; // 3hr +const allTokens: { [chain: Chain]: string[] } = {}; + +// fetch a list of all token addresses from ES +async function fetchAllTokens() { + await initializePriceQueryFilter(); + + whitelistedTokenSet.forEach((t) => { + const seperater = t.indexOf(":"); + const chain = t.substring(0, seperater); + const address = t.substring(seperater + 1); + + if (!allTokens[chain]) allTokens[chain] = []; + allTokens[chain].push(address); + }); + return allTokens; +} + +// find the prices, mcaps and supplies of all tokens on all chains async function fetchNativeAndMcaps( timestamp: number, override: boolean = false @@ -47,20 +332,21 @@ async function fetchNativeAndMcaps( const allPrices: { [token: string]: CoinsApiData } = {}; const allMcaps: { [token: string]: McapsApiData } = {}; - await PromisePool.withConcurrency(5) - .for(allChainKeys) - .process(async (chain: Chain) => { + await runInPromisePool({ + items: allChainKeys, + concurrency: 5, + processor: async (chain: Chain) => { + // protocol bridge IDs are closed and have no native tvl if (Object.values(protocolBridgeIds).includes(chain)) return; - await withTimeout(1000 * 60 * (override ? 120 : 120), minted(chain)).catch(() => { + await withTimeout(1000 * 60 * (override ? 20 : 120), minted(chain)).catch(() => { throw new Error(`fetchMinted() timed out for ${chain}`); }); async function minted(chain: Chain) { try { const start = new Date().getTime(); - let storedTokens = await fetchAllTokens(chain); - if (chain == "cardano") storedTokens = await fetchAdaTokens(); + const storedTokens = chain == "cardano" ? await fetchAdaTokens() : allTokens[chain]; const ownTokenCgid: string | undefined = ownTokens[chain]?.address.startsWith("coingecko:") ? ownTokens[chain].address @@ -119,23 +405,29 @@ async function fetchNativeAndMcaps( const time = end - start; if (time > 60 * 1000) console.log(`${chain}: ${time}`); } catch (e) { - console.error(`fetchNativeAndMcaps() failed for ${chain} with ${e}`); + throw new Error(`fetchNativeAndMcaps() failed for ${chain} with ${e}`); } } - }); + }, + }); return { chainData, allPrices, allMcaps }; } +// incoming asset lists are fetched from canonical bridge token mappings async function fetchIncomingAssetsList(): Promise<{ [chain: Chain]: string[] }> { const incomingAssets: { [chain: Chain]: string[] } = {}; - await PromisePool.withConcurrency(5) - .for(allChainKeys) - .process(async (chain: Chain) => { + // fetch all canonical bridge incoming assets + await runInPromisePool({ + items: allChainKeys, + concurrency: 5, + processor: async (chain: Chain) => { incomingAssets[chain] = await fetchBridgeTokenList(chain); - }); + }, + }); + // add any additional hard coded Object.keys(additional).map((chain) => { if (!incomingAssets[chain]) incomingAssets[chain] = []; const additionalTokens = bridgedTvlMixedCaseChains.includes(chain) @@ -144,6 +436,7 @@ async function fetchIncomingAssetsList(): Promise<{ [chain: Chain]: string[] }> incomingAssets[chain].push(...additionalTokens); }); + // exclude any unwanted assets const filteredIncomingAssets: { [chain: Chain]: string[] } = {}; Object.keys(incomingAssets).filter((chain: Chain) => { filteredIncomingAssets[chain] = @@ -155,40 +448,43 @@ async function fetchIncomingAssetsList(): Promise<{ [chain: Chain]: string[] }> return incomingAssets; } +// outgoing amounts from chains are derived from balances of canonical bridges of each chain async function fetchOutgoingAmountsFromDB(timestamp: number): Promise<{ sourceChainAmounts: { [chain: Chain]: { [token: string]: BigNumber } }; protocolAmounts: { [protocolSlug: string]: { [token: string]: BigNumber } }; destinationChainAmounts: { [chain: Chain]: { [token: string]: BigNumber } }; }> { const ids: string[] = [...Object.keys(canonicalBridgeIds), ...Object.keys(protocolBridgeIds)]; - const tvls: any[] = await Promise.all( - ids.map((i: string) => - getTVLOfRecordClosestToTimestamp( - `hourlyRawTokensTvl#${i}`, - timestamp == 0 ? getCurrentUnixTimestamp() : timestamp, - searchWidth - ) - ) - ); + + await initializeTVLCacheDB(); + const tvls = + timestamp == 0 + ? await getLatestProtocolItems(hourlyRawTokensTvl, { filterLast24Hours: true }) + : await runInPromisePool({ + items: ids, + concurrency: 5, + processor: (i: string) => + getClosestProtocolItem(hourlyRawTokensTvl, i, getCurrentUnixTimestamp(), { searchWidth }), + }); const sourceChainAmounts: { [chain: Chain]: { [token: string]: BigNumber } } = {}; const protocolAmounts: { [chain: Chain]: { [token: string]: BigNumber } } = {}; const destinationChainAmounts: { [chain: Chain]: { [token: string]: BigNumber } } = {}; - tvls.map((b: any, i: number) => { - Object.keys(b).map((chain: string) => { + tvls.map(({ data, id }: any, i: number) => { + if (!ids.includes(id)) return; + Object.keys(data).map((chain: string) => { if (excludedTvlKeys.includes(chain)) return; if (!sourceChainAmounts[chain]) sourceChainAmounts[chain] = {}; - Object.keys(b[chain]).map((token: string) => { + Object.keys(data[chain]).map((token: string) => { const key = normalizeKey(token); if (!sourceChainAmounts[chain][key]) sourceChainAmounts[chain][key] = zero; - sourceChainAmounts[chain][key] = sourceChainAmounts[chain][key].plus(b[chain][token]); + sourceChainAmounts[chain][key] = sourceChainAmounts[chain][key].plus(data[chain][token]); - const protocolId = b.PK.substring(b.PK.indexOf("#") + 1); - if (Object.keys(protocolBridgeIds).includes(protocolId)) { - const protocolSlug = protocolBridgeIds[protocolId]; + if (Object.keys(protocolBridgeIds).includes(id)) { + const protocolSlug = protocolBridgeIds[id]; if (!protocolAmounts[protocolSlug]) protocolAmounts[protocolSlug] = {}; if (!protocolAmounts[protocolSlug][key]) protocolAmounts[protocolSlug][key] = zero; - protocolAmounts[protocolSlug][key] = protocolAmounts[protocolSlug][key].plus(b[chain][token]); + protocolAmounts[protocolSlug][key] = protocolAmounts[protocolSlug][key].plus(data[chain][token]); return; } @@ -196,7 +492,7 @@ async function fetchOutgoingAmountsFromDB(timestamp: number): Promise<{ if (!destinationChainAmounts[destinationChain]) destinationChainAmounts[destinationChain] = {}; if (!destinationChainAmounts[destinationChain][key]) destinationChainAmounts[destinationChain][key] = zero; destinationChainAmounts[destinationChain][key] = destinationChainAmounts[destinationChain][key].plus( - b[chain][token] + data[chain][token] ); }); }); @@ -205,6 +501,7 @@ async function fetchOutgoingAmountsFromDB(timestamp: number): Promise<{ return { sourceChainAmounts, protocolAmounts, destinationChainAmounts }; } +// fetch amounts in hardcoded excluded wallets - eg pre mines etc async function fetchExcludedAmounts(timestamp: number) { const excludedTokensAndOwners: { [chain: string]: [string, string][] } = { base: [ @@ -223,9 +520,11 @@ async function fetchExcludedAmounts(timestamp: number) { ], }; - const excludedAmounts: { [chain: string]: { [token: string]: BigNumber } } = {}; - await Promise.all( - Object.keys(excludedTokensAndOwners).map(async (chain: string) => { + const excludedAmounts: { [chain: string]: Balances } = {}; + await runInPromisePool({ + items: Object.keys(excludedTokensAndOwners), + concurrency: 5, + processor: async (chain: string) => { const block = await getBlock(chain, (timestamp == 0 ? getCurrentUnixTimestamp() : timestamp) - 10); const balances = await multiCall({ @@ -235,17 +534,17 @@ async function fetchExcludedAmounts(timestamp: number) { block: block.number, }); + excludedAmounts[chain] = new Balances({ chain }); excludedTokensAndOwners[chain].map(([token, _], i) => { - if (!excludedAmounts[chain]) excludedAmounts[chain] = {}; - if (!excludedAmounts[chain][token]) excludedAmounts[chain][token] = zero; - excludedAmounts[chain][token] = excludedAmounts[chain][token].plus(balances[i]); + excludedAmounts[chain].add(token, BigNumber(balances[i])); }); - }) - ); + }, + }); return excludedAmounts; } +// fetch stablecoin symbols async function fetchStablecoinSymbols() { const { peggedAssets } = await fetch("https://stablecoins.llama.fi/stablecoins").then((r) => r.json()); const symbols = peggedAssets.map((s: any) => s.symbol); @@ -253,29 +552,33 @@ async function fetchStablecoinSymbols() { return allSymbols; } +// fetch lst symbols async function fetchLstSymbols() { const assets = await fetch("https://yields.llama.fi/lsdRates").then((r) => r.json()); const symbols = assets.map((s: any) => s.symbol.toUpperCase()); return symbols; } +// fetch rwa symbols function fetchRwaSymbols() { const allSymbols: { [symbol: string]: boolean } = {}; Object.values(rwaMetadata).map(({ matchExact, symbols }: { matchExact: boolean; symbols: string[] }) => { - symbols.map((symbol) => allSymbols[symbol] = matchExact) + symbols.map((symbol) => (allSymbols[symbol] = matchExact)); }); return allSymbols; } +// check if a symbol is an rwa symbol function isRwaSymbol(symbol: string, rwaSymbols: { [symbol: string]: boolean }) { if (rwaSymbols[symbol]) return true; Object.keys(rwaSymbols).map((s) => { if (!rwaSymbols[s] && symbol.startsWith(s)) return true; }); - return false + return false; } +// normalize a key to a standard format function normalizeKey(key: string) { if (key.startsWith("0x")) return `ethereum:${key.toLowerCase()}`; const [chain, address] = key.split(":"); @@ -284,6 +587,7 @@ function normalizeKey(key: string) { return key.toLowerCase(); } +// check if a symbol is an own token function isOwnToken(chain: string, symbol: string) { const ownToken = ownTokens[chain]; if (!ownToken) return false; @@ -296,8 +600,23 @@ function isOwnToken(chain: string, symbol: string) { return false; } +// create a new chain assets object +const newChainAssets = () => ({ + canonical: { breakdown: {} }, + thirdParty: { breakdown: {} }, + native: { breakdown: {} }, + ownTokens: { breakdown: {} }, + total: { breakdown: {} }, + rwa: { breakdown: {} }, + lst: { breakdown: {} }, + stablecoins: { breakdown: {} }, +}); + +// main function async function main() { + await setEnvSecrets(); const timestamp = 0; + await fetchAllTokens(); const { sourceChainAmounts, protocolAmounts, destinationChainAmounts } = await fetchOutgoingAmountsFromDB(timestamp); const incomingAssets = await fetchIncomingAssetsList(); const excludedAmounts = await fetchExcludedAmounts(timestamp); @@ -306,6 +625,7 @@ async function main() { const lstSymbols = await fetchLstSymbols(); const rwaSymbols = fetchRwaSymbols(); + // adjust native asset balances by excluded and outgoing amounts const nativeDataAfterDeductions: { [chain: Chain]: { [token: string]: BigNumber } } = {}; allChainKeys.map((chain: Chain) => { if (Object.values(protocolBridgeIds).includes(chain)) return; @@ -317,7 +637,7 @@ async function main() { const key = normalizeKey(token); const coinData = allPrices[key]; if (!coinData || !coinData.price) return; - const excludedAmount = excludedAmounts[chain]?.[key] ?? zero; + const excludedAmount = excludedAmounts[chain]?._balances[key] ?? zero; const sourceChainAmount = sourceChainAmounts[chain]?.[key] ?? zero; const nativeAmount = BigNumber(supplies[token]); @@ -342,6 +662,7 @@ async function main() { } }); + // adjust balances for mcaps const nativeDataAfterMcaps: { [chain: Chain]: { [token: string]: BigNumber } } = {}; Object.keys(nativeDataAfterDeductions).map((chain: Chain) => { nativeDataAfterMcaps[chain] = {}; @@ -352,18 +673,10 @@ async function main() { }); }); + // split all chain balances into sections (canonical, thirdParty, native, ownTokens, stablecoins, rwa, lst) const rawData: FinalData = {}; allChainKeys.map((chain: Chain) => { - rawData[chain] = { - canonical: { breakdown: {} }, - thirdParty: { breakdown: {} }, - native: { breakdown: {} }, - ownTokens: { breakdown: {} }, - total: { breakdown: {} }, - rwa: { breakdown: {} }, - lst: { breakdown: {} }, - stablecoins: { breakdown: {} }, - }; + rawData[chain] = newChainAssets(); if (Object.values(protocolBridgeIds).includes(chain)) { const protocolAmount = protocolAmounts[chain]; if (!protocolAmount) { @@ -409,19 +722,11 @@ async function main() { }); }); + // create a symbol map and symbol data for human readable data const symbolMap: { [key: string]: string } = (await getR2JSONString("chainAssetsSymbolMap")) ?? {}; const symbolData: FinalData = {}; Object.keys(rawData).map((chain: Chain) => { - symbolData[chain] = { - canonical: { breakdown: {} }, - thirdParty: { breakdown: {} }, - native: { breakdown: {} }, - ownTokens: { breakdown: {} }, - total: { breakdown: {} }, - rwa: { breakdown: {} }, - lst: { breakdown: {} }, - stablecoins: { breakdown: {} }, - }; + symbolData[chain] = newChainAssets(); Object.keys(rawData[chain]).map((section: string) => { Object.keys(rawData[chain][section as keyof FinalChainData].breakdown).map((key: string) => { const symbol = allPrices[key].symbol.toUpperCase(); @@ -432,6 +737,7 @@ async function main() { }); }); + // create symbol key data const symbolMapPromise = storeR2JSONString("chainAssetsSymbolMap", JSON.stringify(symbolMap)); [rawData, symbolData].map((allData) => { Object.keys(allData).map((chain: Chain) => { @@ -448,18 +754,10 @@ async function main() { }); }); + // sort symbol data for manual verifcation const sortedSymbolData: FinalData = {}; Object.keys(symbolData).map((chain: Chain) => { - sortedSymbolData[chain] = { - canonical: { breakdown: {} }, - thirdParty: { breakdown: {} }, - native: { breakdown: {} }, - ownTokens: { breakdown: {} }, - total: { breakdown: {} }, - rwa: { breakdown: {} }, - lst: { breakdown: {} }, - stablecoins: { breakdown: {} }, - }; + sortedSymbolData[chain] = newChainAssets(); Object.keys(symbolData[chain]).map((section: string) => { const a = Object.entries(symbolData[chain][section as keyof FinalChainData].breakdown).sort((a: any, b: any) => b[1].minus(a[1]) @@ -485,7 +783,9 @@ async function main() { ]); } -main().catch((e) => { - console.error(e); - process.exit(1); -}).then(() => process.exit(0)); // ts-node defi/l2/v2/index.ts +main() + .catch((e) => { + console.error(e); + process.exit(1); + }) + .then(() => process.exit(0)); // ts-node defi/l2/v2/index.ts diff --git a/defi/l2/test.ts b/defi/l2/verifyChanges.ts similarity index 100% rename from defi/l2/test.ts rename to defi/l2/verifyChanges.ts From c8ad3f706aa4b34b6bdef67101ee3d09a51ee5b7 Mon Sep 17 00:00:00 2001 From: wayne Date: Wed, 15 Oct 2025 13:09:28 +0100 Subject: [PATCH 2/7] error logging, ES alltokens pending --- defi/l2/excluded.ts | 2 +- defi/l2/incoming.ts | 3 +- defi/l2/native.ts | 3 +- defi/l2/tvl.ts | 2 +- defi/l2/utils.ts | 109 +----------------------- defi/l2/v2/index.ts | 27 +++--- defi/src/storeTvlInterval/computeTVL.ts | 4 +- 7 files changed, 23 insertions(+), 127 deletions(-) diff --git a/defi/l2/excluded.ts b/defi/l2/excluded.ts index 0778e1c71b..a4ca96439c 100644 --- a/defi/l2/excluded.ts +++ b/defi/l2/excluded.ts @@ -1,4 +1,4 @@ -import { getPrices } from "./utils"; +import { getPrices } from "@defillama/sdk/build/util/coinsApi"; import { excludedTvlId } from "./constants"; import { multiCall } from "@defillama/sdk/build/abi/abi2"; import { getBlock } from "@defillama/sdk/build/util/blocks"; diff --git a/defi/l2/incoming.ts b/defi/l2/incoming.ts index f1764ea1d7..0199069ea8 100644 --- a/defi/l2/incoming.ts +++ b/defi/l2/incoming.ts @@ -3,7 +3,8 @@ import { Chain } from "@defillama/sdk/build/general"; import BigNumber from "bignumber.js"; import { DollarValues, TokenTvlData } from "./types"; import { geckoSymbols, zero } from "./constants"; -import { fetchBridgeTokenList, fetchSupplies, getPrices } from "./utils"; +import { fetchBridgeTokenList, fetchSupplies } from "./utils"; +import { getPrices } from "@defillama/sdk/build/util/coinsApi"; export async function fetchIncoming(params: { canonical: TokenTvlData; diff --git a/defi/l2/native.ts b/defi/l2/native.ts index 801dda7f5f..b0afd693b9 100644 --- a/defi/l2/native.ts +++ b/defi/l2/native.ts @@ -5,11 +5,12 @@ import { Chain } from "@defillama/sdk/build/general"; import BigNumber from "bignumber.js"; import { Address } from "@defillama/sdk/build/types"; import { geckoSymbols, ownTokens, zero } from "./constants"; -import { getMcaps, getPrices, fetchBridgeTokenList, fetchSupplies } from "./utils"; +import { fetchBridgeTokenList, fetchSupplies } from "./utils"; import { fetchAdaTokens } from "./adapters/ada"; import { nativeWhitelist } from "./adapters/manual"; import { withTimeout } from "../src/utils/shared/withTimeout"; import PromisePool from "@supercharge/promise-pool"; +import { getPrices, getMcaps } from "@defillama/sdk/build/util/coinsApi"; export async function fetchMinted(params: { chains: TokenTvlData; diff --git a/defi/l2/tvl.ts b/defi/l2/tvl.ts index 6909b7d24f..a2209d6d73 100644 --- a/defi/l2/tvl.ts +++ b/defi/l2/tvl.ts @@ -5,12 +5,12 @@ import { ChainData, DollarValues, FinalData } from "./types"; import BigNumber from "bignumber.js"; import { allChainKeys, ownTokens, tokenFlowCategories, zero } from "./constants"; import { Chain } from "@defillama/sdk/build/general"; -import { getMcaps } from "./utils"; import { getCurrentUnixTimestamp } from "../src/utils/date"; import { getChainDisplayName } from "../src/utils/normalizeChain"; import { verifyChanges } from "./verifyChanges"; import { getExcludedTvl } from "./excluded"; import { saveRawBridgedTvls } from "./raw"; +import { getMcaps } from "@defillama/sdk/build/util/coinsApi"; export default async function main(override?: boolean, timestamp?: number) { let symbolMap: { [pk: string]: string | null } = {}; diff --git a/defi/l2/utils.ts b/defi/l2/utils.ts index e0686cf524..68f9398088 100644 --- a/defi/l2/utils.ts +++ b/defi/l2/utils.ts @@ -1,5 +1,5 @@ import BigNumber from "bignumber.js"; -import { AllProtocols, CoinsApiData, McapsApiData, TokenTvlData } from "./types"; +import { AllProtocols, TokenTvlData } from "./types"; import { canonicalBridgeIds, excludedTvlKeys, geckoSymbols, protocolBridgeIds, zero } from "./constants"; import fetch from "node-fetch"; import { bridgedTvlMixedCaseChains } from "../src/utils/shared/constants"; @@ -60,113 +60,6 @@ async function restCallWrapper(request: () => Promise, retries: number = 8, } throw new Error(`couldnt work ${name} call after retries!`); } -export async function getPrices( - readKeys: string[], - timestamp: number | "now" -): Promise<{ [address: string]: CoinsApiData }> { - if (!readKeys.length) return {}; - const bodies: string[] = []; - for (let i = 0; i < readKeys.length; i += 100) { - const body = { - coins: readKeys.slice(i, i + 100), - } as any; - if (timestamp !== "now") { - body.timestamp = timestamp; - } - bodies.push(JSON.stringify(body)); - } - - const tokenData: any[] = []; - await PromisePool.withConcurrency(10) - .for(bodies) - .process(async (body) => { - const res = await restCallWrapper( - () => - fetch( - `https://coins.llama.fi/prices?source=internal${ - process.env.COINS_KEY ? `?apikey=${process.env.COINS_KEY}` : "" - }`, - { - method: "POST", - body, - headers: { "Content-Type": "application/json" }, - } - ) - .then((r) => r.json()) - .then((r) => - Object.entries(r.coins).map(([PK, value]) => ({ - ...(value as any), - PK, - })) - ), - undefined, - "coin prices" - ); - tokenData.push(res); - }) - .catch((e) => { - throw new Error(`coin prices call failed with ${e}`); - }); - - const aggregatedRes: { [address: string]: CoinsApiData } = {}; - const normalizedReadKeys = readKeys.map((k: string) => k.toLowerCase()); - tokenData.map((batch: CoinsApiData[]) => { - batch.map((a: CoinsApiData) => { - if (!a.PK) return; - const i = normalizedReadKeys.indexOf(a.PK.toLowerCase()); - aggregatedRes[readKeys[i]] = a; - }); - }); - - return aggregatedRes; -} -export async function getMcaps( - readKeys: string[], - timestamp: number | "now" -): Promise<{ [address: string]: McapsApiData }> { - if (!readKeys.length) return {}; - const bodies: string[] = []; - for (let i = 0; i < readKeys.length; i += 100) { - const body = { - coins: readKeys.slice(i, i + 100), - } as any; - if (timestamp !== "now") { - body.timestamp = timestamp; - } - bodies.push(JSON.stringify(body)); - } - - const tokenData: any[] = []; - await PromisePool.withConcurrency(10) - .for(bodies) - .process(async (body) => { - const res = await restCallWrapper( - () => - fetch(`https://coins.llama.fi/mcaps${process.env.COINS_KEY ? `?apikey=${process.env.COINS_KEY}` : ""}`, { - method: "POST", - body, - headers: { "Content-Type": "application/json" }, - }).then((r) => r.json()), - undefined, - "mcaps" - ); - tokenData.push(res); - }) - .catch((e) => { - throw new Error(`coin mcaps call failed with ${e}`); - }); - - const aggregatedRes: { [address: string]: any } = {}; - const normalizedReadKeys = readKeys.map((k: string) => k.toLowerCase()); - tokenData.map((batch: { [address: string]: McapsApiData }[]) => { - Object.keys(batch).map((a: any) => { - if (!batch[a].mcap) return; - const i = normalizedReadKeys.indexOf(a.toLowerCase()); - aggregatedRes[readKeys[i]] = batch[a]; - }); - }); - return aggregatedRes; -} async function getOsmosisSupplies(tokens: string[], timestamp?: number): Promise<{ [token: string]: number }> { if (timestamp) throw new Error(`timestamp incompatible with Osmosis adapter!`); const supplies: { [token: string]: number } = {}; diff --git a/defi/l2/v2/index.ts b/defi/l2/v2/index.ts index 985781644f..ff82de4711 100644 --- a/defi/l2/v2/index.ts +++ b/defi/l2/v2/index.ts @@ -1,7 +1,7 @@ import { canonicalBridgeIds, excludedTvlKeys, protocolBridgeIds, zero, ownTokens, allChainKeys } from "../constants"; import { getCurrentUnixTimestamp } from "../../src/utils/date"; import { Chain } from "@defillama/sdk/build/types"; -import { getMcaps, getPrices, fetchBridgeTokenList, fetchSupplies } from "../utils"; +import { fetchBridgeTokenList, fetchSupplies } from "../utils"; import { fetchAdaTokens } from "../adapters/ada"; import { withTimeout } from "../../src/utils/shared/withTimeout"; import { CoinsApiData, FinalData, FinalChainData } from "../types"; @@ -22,6 +22,7 @@ import { hourlyRawTokensTvl } from "../../src/utils/getLastRecord"; import { Balances } from "@defillama/sdk"; import runInPromisePool from "@defillama/sdk/build/util/promisePool"; import setEnvSecrets from "../../src/utils/shared/setEnvSecrets"; +import { getPrices, getMcaps } from "@defillama/sdk/build/util/coinsApi"; const stablecoins = [ "USDT", @@ -354,16 +355,16 @@ async function fetchNativeAndMcaps( if (ownTokenCgid) storedTokens.push(ownTokenCgid); let prices: { [token: string]: CoinsApiData } = {}; - try { - prices = await getR2JSONString(`prices/${chain}.json`); - } catch (e) { - console.log(`${chain} prices not cached, fetching`); + // try { + // prices = await getR2JSONString(`prices/${chain}.json`); + // } catch (e) { + // console.log(`${chain} prices not cached, fetching`); prices = await getPrices( storedTokens.map((t: string) => normalizeKey(t.startsWith("coingecko:") ? t : `${chain}:${t}`)), timestamp ); await storeR2JSONString(`prices/${chain}.json`, JSON.stringify(prices)); - } + // } Object.keys(prices).map((p: string) => { if (p.startsWith("coingecko:")) prices[p].decimals = 0; @@ -371,13 +372,13 @@ async function fetchNativeAndMcaps( }); let mcaps: { [token: string]: McapsApiData } = {}; - try { - mcaps = await getR2JSONString(`mcaps/${chain}.json`); - } catch (e) { - console.log(`${chain} mcaps not cached, fetching`); + // try { + // mcaps = await getR2JSONString(`mcaps/${chain}.json`); + // } catch (e) { + // console.log(`${chain} mcaps not cached, fetching`); mcaps = await getMcaps(Object.keys(prices), timestamp); - await storeR2JSONString(`mcaps/${chain}.json`, JSON.stringify(mcaps)); - } + // await storeR2JSONString(`mcaps/${chain}.json`, JSON.stringify(mcaps)); + // } Object.keys(mcaps).map((m: string) => { allMcaps[m] = mcaps[m]; }); @@ -743,7 +744,7 @@ async function main() { Object.keys(allData).map((chain: Chain) => { let totalTotal = zero; Object.keys(allData[chain]).map((section: string) => { - if (section == "total") return; + if (['total', 'ownTokens'].includes(section)) return; const amounts = Object.values(allData[chain][section as keyof FinalChainData].breakdown); const total = amounts.length ? (amounts.reduce((p: any, c: any) => c.plus(p), zero) as BigNumber) : zero; allData[chain][section as keyof FinalChainData].total = total; diff --git a/defi/src/storeTvlInterval/computeTVL.ts b/defi/src/storeTvlInterval/computeTVL.ts index 117806d72b..5ae7aa7fa7 100644 --- a/defi/src/storeTvlInterval/computeTVL.ts +++ b/defi/src/storeTvlInterval/computeTVL.ts @@ -226,10 +226,10 @@ interface Counter { // some protocols have a lot of tokens, but we dont have price support for most of them, so we first fetch a list of tokens for which we do have price, then filter out the rest const priceQueryFilterCoins = !!process.env.PRICE_QUERY_FILTER_FOR_KNOWN_COINS -const whitelistedTokenSet = new Set() as Set; +export const whitelistedTokenSet = new Set() as Set; let priceQueryFilterInitializedPromise: any -async function initializePriceQueryFilter() { +export async function initializePriceQueryFilter() { if (!priceQueryFilterInitializedPromise) priceQueryFilterInitializedPromise = _initializePriceQueryFilter() return priceQueryFilterInitializedPromise async function _initializePriceQueryFilter() { From 5a3a6f4bcbfdee22dee6da92c1fe3db168e22858 Mon Sep 17 00:00:00 2001 From: wayne Date: Wed, 15 Oct 2025 14:56:24 +0100 Subject: [PATCH 3/7] L2 allToken ES --- defi/l2/v2/index.ts | 317 ++---------------------- defi/src/storeTvlInterval/computeTVL.ts | 4 +- 2 files changed, 30 insertions(+), 291 deletions(-) diff --git a/defi/l2/v2/index.ts b/defi/l2/v2/index.ts index ff82de4711..2ef7914930 100644 --- a/defi/l2/v2/index.ts +++ b/defi/l2/v2/index.ts @@ -13,10 +13,10 @@ import { bridgedTvlMixedCaseChains, chainsThatShouldNotBeLowerCased } from "../. import { getR2JSONString, storeR2JSONString } from "../../src/utils/r2"; import { additional, excluded } from "../adapters/manual"; import { storeHistoricalToDB } from "./storeToDb"; -// import { stablecoins } from "../../src/getProtocols"; +import { stablecoins } from "../../src/getProtocols"; import { metadata as rwaMetadata } from "../../src/rwa/protocols"; import { verifyChanges } from "../verifyChanges"; -import { initializePriceQueryFilter, whitelistedTokenSet } from "../../src/storeTvlInterval/computeTVL"; +import { initializePriceQueryFilter, whitelistedTokenSetRawPids } from "../../src/storeTvlInterval/computeTVL"; import { getClosestProtocolItem, getLatestProtocolItems, initializeTVLCacheDB } from "../../src/api2/db"; import { hourlyRawTokensTvl } from "../../src/utils/getLastRecord"; import { Balances } from "@defillama/sdk"; @@ -24,270 +24,6 @@ import runInPromisePool from "@defillama/sdk/build/util/promisePool"; import setEnvSecrets from "../../src/utils/shared/setEnvSecrets"; import { getPrices, getMcaps } from "@defillama/sdk/build/util/coinsApi"; -const stablecoins = [ - "USDT", - "USDC", - "DAI", - "FRAX", - "AUSDC", - "AMMUSDC", - "AETHUSDC", - "ADAI", - "AMMDAI", - "AETHDAI", - "AUSDT", - "AMMUSDT", - "LUSD", - "ALUSD", - "GUSD", - "AGUSD", - "TUSD", - "ATUSD", - "USDP", - "AUSDP", - "FEI", - "AFEI", - "BUSD", - "YYDAI+YUSDC+YUSDT+YTUSD", - "CDAI", - "BSC-USD", - "USD+", - "SUSD", - "DOLA", - "AMUSDC", - "AMDAI", - "AVUSDC", - "AVDAI", - "AAVAUSDC", - "AMUSDT", - "AAVAUSDT", - "AAVADAI", - "AVUSDT", - "AOPTUSDC", - "SUSDE", - "USDY", - "USTC", - "MIM", - "USDN", - "YUSD", - "USDD", - "PAI", - "HUSD", - "NUSD", - "FLEXUSD", - "OUSD", - "CUSD", - "RSV", - "MUSD", - "USDK", - "VAI", - "TOR", - "DOC", - "USDS", - "USDB", - "USDJ", - "STBL", - "VOLT", - "RAI", - "FLOAT", - "USDX", - "ZUSD", - "USX", - "ASEED", - "BAI", - "EURT", - "EURC", - "EURS", - "CEUR", - "SEUR", - "USN", - "EURA", - "PAR", - "USH", - "3USD", - "SIGUSD", - "HOME", - "FIAT", - "PUSD", - "FUSD", - "UXD", - "USDH", - "FPI", - "BEAN", - "USDL", - "DUSD", - "VST", - "KUSD", - "USDTZ", - "MONEY", - "UUSD", - "USDI", - "NOTE", - "LISUSD", - "USK", - "ARUSD", - "USDW", - "BOB", - "USDR", - "IUSD", - "XAI", - "RUSD", - "IBEUR", - "PINA", - "DJED", - "BAOUSD", - "USP", - "EUROE", - "CASH", - "DSU", - "EURE", - "ANONUSD", - "NXUSD", - "DCHF", - "EUSD", - "CZUSD", - "D2O", - "CRVUSD", - "DAI+", - "USDT+", - "SILK", - "CLEVUSD", - "R", - "GRAI", - "ERN", - "GHO", - "FDUSD", - "PYUSD", - "SLSD", - "GYEN", - "STAR", - "PEUSD", - "EUSD(V2)", - "MKUSD", - "LCNY", - "NEX", - "SVUSD", - "UAHT", - "USDM", - "NARS", - "IST", - "CDT", - "EEUR", - "EGBP", - "HYDT", - "USDV", - "HYUSD", - "CADC", - "USDE", - "AEUR", - "MYUSD", - "SCB", - "ZKUSD", - "BUCK", - "USDGLO", - "VCHF", - "VEUR", - "USDRIF", - "DLLR", - "EURD", - "XUSD", - "ULTRA", - "USDCB", - "AUDD", - "CGUSD", - "FETH", - "FXUSD", - "GAI", - "EURO3", - "HAI", - "BUIDL", - "PXDC", - "FXD", - "UNO", - "USD3", - "CJPY", - "BREAD", - "ZUNUSD", - "ZUNETH", - "BTCUSD", - "WEN", - "GYD", - "ISC", - "KNOX", - "USC", - "RGUSD", - "BITUSD", - "USDA", - "USD0", - "USR", - "AUSD", - "CREAL", - "HCHF", - "HEXDC", - "USDZ", - "BNUSD", - "DYAD", - "DCKUSD", - "DEUSD", - "THUSD", - "MOD", - "M", - "SATUSD", - "USDF", - "USDTB", - "PAUSD", - "XY", - "ZEUSD", - "ZCHF", - "BOLD", - "USDQ", - "LVLUSD", - "HONEY", - "PINTO", - "WUSD", - "FRXUSD", - "SYUSD", - "USYC", - "SCUSD", - "EURR", - "USDO", - "CSUSDL", - "EUROP", - "USDFC", - "BRZ", - "RLUSD", - "FEUSD", - "USBD", - "EURCV", - "REUSD", - "TBILL", - "A7A5", - "MSD", - "VUSD", - "USD1", - "MEAD", - "YU", - "BENJI", - "AVUSD", - "YLDS", - "USDAF", - "EURQ", - "WEUSD", - "PARAUSD", - "CNHT", - "MXNT", - "USDU", - "MNEE", - "GGUSD", - "USDG", - "USND", - "EBUSD", - "XSGD", - "VGBP", - "BNBUSD", - "USDA+", - "MSUSD", -]; const searchWidth = 10800; // 3hr const allTokens: { [chain: Chain]: string[] } = {}; @@ -295,10 +31,12 @@ const allTokens: { [chain: Chain]: string[] } = {}; async function fetchAllTokens() { await initializePriceQueryFilter(); - whitelistedTokenSet.forEach((t) => { + whitelistedTokenSetRawPids.forEach((t) => { const seperater = t.indexOf(":"); const chain = t.substring(0, seperater); + const address = t.substring(seperater + 1); + if (address == 'undefined') return; if (!allTokens[chain]) allTokens[chain] = []; allTokens[chain].push(address); @@ -363,7 +101,7 @@ async function fetchNativeAndMcaps( storedTokens.map((t: string) => normalizeKey(t.startsWith("coingecko:") ? t : `${chain}:${t}`)), timestamp ); - await storeR2JSONString(`prices/${chain}.json`, JSON.stringify(prices)); + // await storeR2JSONString(`prices/${chain}.json`, JSON.stringify(prices)); // } Object.keys(prices).map((p: string) => { @@ -375,7 +113,7 @@ async function fetchNativeAndMcaps( // try { // mcaps = await getR2JSONString(`mcaps/${chain}.json`); // } catch (e) { - // console.log(`${chain} mcaps not cached, fetching`); + // console.log(`${chain} mcaps not cached, fetching`); mcaps = await getMcaps(Object.keys(prices), timestamp); // await storeR2JSONString(`mcaps/${chain}.json`, JSON.stringify(mcaps)); // } @@ -384,12 +122,12 @@ async function fetchNativeAndMcaps( }); let supplies; - try { - supplies = await getR2JSONString(`supplies/${chain}.json`); - if (ownTokenCgid && mcaps[ownTokenCgid]) - supplies[ownTokenCgid] = mcaps[ownTokenCgid].mcap / prices[ownTokenCgid].price; - } catch (e) { - console.log(`${chain} supplies not cached, fetching`); + // try { + // supplies = await getR2JSONString(`supplies/${chain}.json`); + // if (ownTokenCgid && mcaps[ownTokenCgid]) + // supplies[ownTokenCgid] = mcaps[ownTokenCgid].mcap / prices[ownTokenCgid].price; + // } catch (e) { + // console.log(`${chain} supplies not cached, fetching`); supplies = await fetchSupplies( chain, Object.keys(prices).map((t: string) => t.substring(t.indexOf(":") + 1)), @@ -397,8 +135,8 @@ async function fetchNativeAndMcaps( ); if (ownTokenCgid && mcaps[ownTokenCgid]) supplies[ownTokenCgid] = mcaps[ownTokenCgid].mcap / prices[ownTokenCgid].price; - await storeR2JSONString(`supplies/${chain}.json`, JSON.stringify(supplies)); - } + // await storeR2JSONString(`supplies/${chain}.json`, JSON.stringify(supplies)); + // } chainData[chain] = { prices, mcaps, supplies }; @@ -471,7 +209,7 @@ async function fetchOutgoingAmountsFromDB(timestamp: number): Promise<{ const sourceChainAmounts: { [chain: Chain]: { [token: string]: BigNumber } } = {}; const protocolAmounts: { [chain: Chain]: { [token: string]: BigNumber } } = {}; const destinationChainAmounts: { [chain: Chain]: { [token: string]: BigNumber } } = {}; - tvls.map(({ data, id }: any, i: number) => { + tvls.map(({ data, id }: any) => { if (!ids.includes(id)) return; Object.keys(data).map((chain: string) => { if (excludedTvlKeys.includes(chain)) return; @@ -489,7 +227,8 @@ async function fetchOutgoingAmountsFromDB(timestamp: number): Promise<{ return; } - const destinationChain = [...Object.values(canonicalBridgeIds), ...Object.values(protocolBridgeIds)][i]; + const destinationChain = canonicalBridgeIds[id] + if (!destinationChain) return; if (!destinationChainAmounts[destinationChain]) destinationChainAmounts[destinationChain] = {}; if (!destinationChainAmounts[destinationChain][key]) destinationChainAmounts[destinationChain][key] = zero; destinationChainAmounts[destinationChain][key] = destinationChainAmounts[destinationChain][key].plus( @@ -615,7 +354,6 @@ const newChainAssets = () => ({ // main function async function main() { - await setEnvSecrets(); const timestamp = 0; await fetchAllTokens(); const { sourceChainAmounts, protocolAmounts, destinationChainAmounts } = await fetchOutgoingAmountsFromDB(timestamp); @@ -690,7 +428,6 @@ async function main() { if (!coinData || !coinData.price) return; const amount = BigNumber(coinData.price).times(protocolAmount[token]).div(BigNumber(10).pow(coinData.decimals)); - const symbol = allPrices[key].symbol.toUpperCase(); // filter for rwa, lst, stablecoins let section = "canonical"; @@ -701,8 +438,6 @@ async function main() { rawData[chain][section as keyof FinalChainData].breakdown[key] = amount; if (!isOwnToken(chain, symbol)) rawData[chain].total.breakdown[key] = amount; }); - - return; } Object.keys(nativeDataAfterMcaps[chain]).map((key: string) => { @@ -744,10 +479,10 @@ async function main() { Object.keys(allData).map((chain: Chain) => { let totalTotal = zero; Object.keys(allData[chain]).map((section: string) => { - if (['total', 'ownTokens'].includes(section)) return; const amounts = Object.values(allData[chain][section as keyof FinalChainData].breakdown); const total = amounts.length ? (amounts.reduce((p: any, c: any) => c.plus(p), zero) as BigNumber) : zero; allData[chain][section as keyof FinalChainData].total = total; + if (section == 'ownTokens') return; totalTotal = totalTotal.plus(total); }); @@ -760,15 +495,17 @@ async function main() { Object.keys(symbolData).map((chain: Chain) => { sortedSymbolData[chain] = newChainAssets(); Object.keys(symbolData[chain]).map((section: string) => { - const a = Object.entries(symbolData[chain][section as keyof FinalChainData].breakdown).sort((a: any, b: any) => + const orderedTokenAmounts = Object.entries(symbolData[chain][section as keyof FinalChainData].breakdown).sort((a: any, b: any) => b[1].minus(a[1]) ); - const b = a.slice(0, 100); - const c: { [key: string]: string } = {}; - b.map(([key, value]: any) => { - c[key] = value.toString(); + + const top100Tokens = orderedTokenAmounts.slice(0, 100); + const topTokensObject: { [key: string]: string } = {}; + top100Tokens.map(([key, value]: any) => { + topTokensObject[key] = value.toString(); }); - sortedSymbolData[chain][section as keyof FinalChainData].breakdown = c; + + sortedSymbolData[chain][section as keyof FinalChainData].breakdown = topTokensObject; sortedSymbolData[chain][section as keyof FinalChainData].total = symbolData[chain][section as keyof FinalChainData].total.toString(); sortedSymbolData[chain].total.total = symbolData[chain].total.total.toString(); diff --git a/defi/src/storeTvlInterval/computeTVL.ts b/defi/src/storeTvlInterval/computeTVL.ts index 5ae7aa7fa7..a19bae69d1 100644 --- a/defi/src/storeTvlInterval/computeTVL.ts +++ b/defi/src/storeTvlInterval/computeTVL.ts @@ -226,7 +226,8 @@ interface Counter { // some protocols have a lot of tokens, but we dont have price support for most of them, so we first fetch a list of tokens for which we do have price, then filter out the rest const priceQueryFilterCoins = !!process.env.PRICE_QUERY_FILTER_FOR_KNOWN_COINS -export const whitelistedTokenSet = new Set() as Set; +const whitelistedTokenSet = new Set() as Set; +export const whitelistedTokenSetRawPids = new Set() as Set; let priceQueryFilterInitializedPromise: any export async function initializePriceQueryFilter() { @@ -251,6 +252,7 @@ export async function initializePriceQueryFilter() { while (response.hits.hits.length) { response.hits.hits.map((i: any) => { + whitelistedTokenSetRawPids.add(`${i._source.chain}:${i._source.address}`); whitelistedTokenSet.add(normalizeCoinId(i._source.pid)); }) sdk.log(`Fetched ${whitelistedTokenSet.size} records, ${response.hits.hits.length} in batch`); From 046984c2c398d53b7a42d439b793cbecd9d9f75a Mon Sep 17 00:00:00 2001 From: wayne Date: Thu, 16 Oct 2025 13:09:48 +0100 Subject: [PATCH 4/7] cached fetches --- defi/l2/adapters/ada.ts | 4 +- defi/l2/adapters/thirdParty.ts | 69 ++++++++++++++++++++-------------- defi/l2/v2/index.ts | 53 ++++++++++++++------------ 3 files changed, 71 insertions(+), 55 deletions(-) diff --git a/defi/l2/adapters/ada.ts b/defi/l2/adapters/ada.ts index f6f9144e1f..700d32d3e7 100644 --- a/defi/l2/adapters/ada.ts +++ b/defi/l2/adapters/ada.ts @@ -1,7 +1,7 @@ -import fetch from "node-fetch"; +import { cachedFetch } from "@defillama/sdk/build/util/cache"; export async function fetchAdaTokens(): Promise { - const res = await fetch(`https://api.muesliswap.com/token-list`).then((r) => r.json()); + const res = await cachedFetch({key: "muesliswap-token-list", endpoint: "https://api.muesliswap.com/token-list"}) const coins = res .filter((c: any) => c.supply.circulating != null) .map((c: any) => ({ diff --git a/defi/l2/adapters/thirdParty.ts b/defi/l2/adapters/thirdParty.ts index eb7a2597b2..796033f0d2 100644 --- a/defi/l2/adapters/thirdParty.ts +++ b/defi/l2/adapters/thirdParty.ts @@ -3,11 +3,10 @@ import providers from "@defillama/sdk/build/providers.json"; import { Address } from "@defillama/sdk/build/types"; import { allChainKeys } from "../constants"; import { bridgedTvlMixedCaseChains } from "../../src/utils/shared/constants"; -import fetch from "node-fetch"; import { additional, excluded } from "./manual"; -import axios from "axios"; -import PromisePool from "@supercharge/promise-pool"; import _ from "lodash"; +import { cachedFetch } from "@defillama/sdk/build/util/cache"; +import runInPromisePool from "@defillama/sdk/build/util/promisePool"; const addresses: { [chain: Chain]: Address[] } = {}; allChainKeys.map((c: string) => (addresses[c] = [])); @@ -23,15 +22,18 @@ Object.keys(providers).map((c: string) => { }); const hyperlane = async (): Promise => { - const data = await fetch( - "https://raw.githubusercontent.com/Eclipse-Laboratories-Inc/gist/refs/heads/main/hyperlane-assets.json" - ).then((r) => r.json()); + const data = await cachedFetch({ + key: "hyperlane-assets", + endpoint: "https://raw.githubusercontent.com/Eclipse-Laboratories-Inc/gist/refs/heads/main/hyperlane-assets.json", + }); + if (Object.keys(data).length == 0) throw new Error("No data or cache found for hyperlane third party"); if (!addresses.eclipse) addresses.eclipse = []; data.map(({ address }: any) => addresses.eclipse.push(address)); }; const axelar = async (): Promise => { - const data = await fetch("https://api.axelarscan.io/api/getAssets").then((r) => r.json()); + const data = await cachedFetch({ key: "axelar-assets", endpoint: "https://api.axelarscan.io/api/getAssets" }); + if (Object.keys(data).length == 0) throw new Error("No data or cache found for axelar third party"); data.map((token: any) => { if (!token.addresses) return; Object.keys(token.addresses).map((chain: string) => { @@ -46,9 +48,11 @@ const axelar = async (): Promise => { }; const wormhole = async (): Promise => { - const data = await axios.get( - "https://raw.githubusercontent.com/wormhole-foundation/wormhole-token-list/main/content/by_dest.csv" - ); + const data = await cachedFetch({ + key: "wormhole-token-list", + endpoint: "https://raw.githubusercontent.com/wormhole-foundation/wormhole-token-list/main/content/by_dest.csv", + }); + if (data.length == 0) throw new Error("No data or cache found for wormhole third party"); const chainMap: { [ticker: string]: string } = { sol: "solana", @@ -69,7 +73,7 @@ const wormhole = async (): Promise => { base: "base", }; - const lines = data.data.split("\n"); + const lines = data.split("\n"); lines.shift(); lines.map((l: string) => { const rows = l.split(","); @@ -81,7 +85,11 @@ const wormhole = async (): Promise => { }; const celer = async (): Promise => { - const data = await fetch("https://cbridge-prod2.celer.app/v2/getTransferConfigsForAll").then((r) => r.json()); + const data = await cachedFetch({ + key: "celer-transfer-configs", + endpoint: "https://cbridge-prod2.celer.app/v2/getTransferConfigsForAll", + }); + if (Object.keys(data).length == 0) throw new Error("No data or cache found for celer third party"); data.pegged_pair_configs.map((pp: any) => { const chain = chainIdMap[pp.org_chain_id]; let normalizedChain: string = chain; @@ -94,11 +102,15 @@ const celer = async (): Promise => { const layerzero = async (): Promise => { const data = await Promise.all([ - fetch( - "https://gist.githubusercontent.com/vrtnd/02b1125edf1afe2baddbf1027157aa31/raw/5cab2009357b1acb8982e6a80e66b64ab7ea1251/mappings.json" - ).then((r) => r.json()), - fetch("https://metadata.layerzero-api.com/v1/metadata").then((r) => r.json()), + cachedFetch({ + key: "layerzero-mappings", + endpoint: + "https://gist.githubusercontent.com/vrtnd/02b1125edf1afe2baddbf1027157aa31/raw/5cab2009357b1acb8982e6a80e66b64ab7ea1251/mappings.json", + }), + cachedFetch({ key: "layerzero-metadata", endpoint: "https://metadata.layerzero-api.com/v1/metadata" }), ]); + if (data[0].length == 0 || Object.keys(data[1]).length == 0) + throw new Error("No data or cache found for layerzero third party"); data[0].map(({ to }: any) => { const [chain, address] = to.split(":"); @@ -148,9 +160,11 @@ const layerzero = async (): Promise => { }; const flow = async (): Promise => { - const data = await fetch( - "https://raw.githubusercontent.com/onflow/assets/refs/heads/main/tokens/outputs/mainnet/token-list.json" - ).then((r) => r.json()); + const data = await cachedFetch({ + key: "flow-token-list", + endpoint: "https://raw.githubusercontent.com/onflow/assets/refs/heads/main/tokens/outputs/mainnet/token-list.json", + }); + if (Object.keys(data).length == 0) throw new Error("No data or cache found for flow third party"); data.tokens.map(({ chainId, address, tags }: any) => { const chain = chainIdMap[chainId]; if (!allChainKeys.includes(chain)) return; @@ -181,15 +195,14 @@ const adapters = { axelar, wormhole, celer, hyperlane, layerzero, flow, unit }; const filteredAddresses: { [chain: Chain]: Address[] } = {}; const tokenAddresses = _.once(async (): Promise<{ [chain: Chain]: Address[] }> => { - await PromisePool.withConcurrency(5) - .for(Object.entries(adapters)) - .process(async ([key, adapter]: any) => { - try { - await adapter(); - } catch (e: any) { - throw new Error(`${key} fails with ${e.message}`); - } - }); + await runInPromisePool({ + items: Object.entries(adapters), + concurrency: 5, + processor: async ([key, adapter]: any) => { + await adapter().catch((e: any) => { + throw new Error(`${key} fails with ${e.message}`); + }); + }}) // remove excluded assets and add additional assets, normalize case Object.keys(addresses).map((chain: string) => { diff --git a/defi/l2/v2/index.ts b/defi/l2/v2/index.ts index 2ef7914930..0511dccd6c 100644 --- a/defi/l2/v2/index.ts +++ b/defi/l2/v2/index.ts @@ -21,7 +21,7 @@ import { getClosestProtocolItem, getLatestProtocolItems, initializeTVLCacheDB } import { hourlyRawTokensTvl } from "../../src/utils/getLastRecord"; import { Balances } from "@defillama/sdk"; import runInPromisePool from "@defillama/sdk/build/util/promisePool"; -import setEnvSecrets from "../../src/utils/shared/setEnvSecrets"; +import { cachedFetch } from "@defillama/sdk/build/util/cache"; import { getPrices, getMcaps } from "@defillama/sdk/build/util/coinsApi"; const searchWidth = 10800; // 3hr @@ -36,7 +36,7 @@ async function fetchAllTokens() { const chain = t.substring(0, seperater); const address = t.substring(seperater + 1); - if (address == 'undefined') return; + if (address == "undefined") return; if (!allTokens[chain]) allTokens[chain] = []; allTokens[chain].push(address); @@ -92,15 +92,15 @@ async function fetchNativeAndMcaps( : undefined; if (ownTokenCgid) storedTokens.push(ownTokenCgid); - let prices: { [token: string]: CoinsApiData } = {}; + // let prices: { [token: string]: CoinsApiData } = {}; // try { // prices = await getR2JSONString(`prices/${chain}.json`); // } catch (e) { // console.log(`${chain} prices not cached, fetching`); - prices = await getPrices( - storedTokens.map((t: string) => normalizeKey(t.startsWith("coingecko:") ? t : `${chain}:${t}`)), - timestamp - ); + const prices = await getPrices( + storedTokens.map((t: string) => normalizeKey(t.startsWith("coingecko:") ? t : `${chain}:${t}`)), + timestamp + ); // await storeR2JSONString(`prices/${chain}.json`, JSON.stringify(prices)); // } @@ -109,33 +109,33 @@ async function fetchNativeAndMcaps( allPrices[p] = prices[p]; }); - let mcaps: { [token: string]: McapsApiData } = {}; + // let mcaps: { [token: string]: McapsApiData } = {}; // try { // mcaps = await getR2JSONString(`mcaps/${chain}.json`); // } catch (e) { - // console.log(`${chain} mcaps not cached, fetching`); - mcaps = await getMcaps(Object.keys(prices), timestamp); + // console.log(`${chain} mcaps not cached, fetching`); + const mcaps = await getMcaps(Object.keys(prices), timestamp); // await storeR2JSONString(`mcaps/${chain}.json`, JSON.stringify(mcaps)); // } Object.keys(mcaps).map((m: string) => { allMcaps[m] = mcaps[m]; }); - let supplies; + // let supplies; // try { // supplies = await getR2JSONString(`supplies/${chain}.json`); // if (ownTokenCgid && mcaps[ownTokenCgid]) // supplies[ownTokenCgid] = mcaps[ownTokenCgid].mcap / prices[ownTokenCgid].price; // } catch (e) { // console.log(`${chain} supplies not cached, fetching`); - supplies = await fetchSupplies( - chain, - Object.keys(prices).map((t: string) => t.substring(t.indexOf(":") + 1)), - timestamp - ); - if (ownTokenCgid && mcaps[ownTokenCgid]) - supplies[ownTokenCgid] = mcaps[ownTokenCgid].mcap / prices[ownTokenCgid].price; - // await storeR2JSONString(`supplies/${chain}.json`, JSON.stringify(supplies)); + const supplies = await fetchSupplies( + chain, + Object.keys(prices).map((t: string) => t.substring(t.indexOf(":") + 1)), + timestamp + ); + if (ownTokenCgid && mcaps[ownTokenCgid]) + supplies[ownTokenCgid] = mcaps[ownTokenCgid].mcap / prices[ownTokenCgid].price; + // await storeR2JSONString(`supplies/${chain}.json`, JSON.stringify(supplies)); // } chainData[chain] = { prices, mcaps, supplies }; @@ -227,7 +227,7 @@ async function fetchOutgoingAmountsFromDB(timestamp: number): Promise<{ return; } - const destinationChain = canonicalBridgeIds[id] + const destinationChain = canonicalBridgeIds[id]; if (!destinationChain) return; if (!destinationChainAmounts[destinationChain]) destinationChainAmounts[destinationChain] = {}; if (!destinationChainAmounts[destinationChain][key]) destinationChainAmounts[destinationChain][key] = zero; @@ -286,7 +286,10 @@ async function fetchExcludedAmounts(timestamp: number) { // fetch stablecoin symbols async function fetchStablecoinSymbols() { - const { peggedAssets } = await fetch("https://stablecoins.llama.fi/stablecoins").then((r) => r.json()); + const { peggedAssets } = await cachedFetch({ + key: "stablecoin-symbols", + endpoint: "https://stablecoins.llama.fi/stablecoins", + }); const symbols = peggedAssets.map((s: any) => s.symbol); const allSymbols = [...new Set([...symbols, ...stablecoins].map((t) => t.toUpperCase()))]; return allSymbols; @@ -294,7 +297,7 @@ async function fetchStablecoinSymbols() { // fetch lst symbols async function fetchLstSymbols() { - const assets = await fetch("https://yields.llama.fi/lsdRates").then((r) => r.json()); + const assets = await cachedFetch({ key: "lst-symbols", endpoint: "https://yields.llama.fi/lsdRates" }); const symbols = assets.map((s: any) => s.symbol.toUpperCase()); return symbols; } @@ -482,7 +485,7 @@ async function main() { const amounts = Object.values(allData[chain][section as keyof FinalChainData].breakdown); const total = amounts.length ? (amounts.reduce((p: any, c: any) => c.plus(p), zero) as BigNumber) : zero; allData[chain][section as keyof FinalChainData].total = total; - if (section == 'ownTokens') return; + if (section == "ownTokens") return; totalTotal = totalTotal.plus(total); }); @@ -495,8 +498,8 @@ async function main() { Object.keys(symbolData).map((chain: Chain) => { sortedSymbolData[chain] = newChainAssets(); Object.keys(symbolData[chain]).map((section: string) => { - const orderedTokenAmounts = Object.entries(symbolData[chain][section as keyof FinalChainData].breakdown).sort((a: any, b: any) => - b[1].minus(a[1]) + const orderedTokenAmounts = Object.entries(symbolData[chain][section as keyof FinalChainData].breakdown).sort( + (a: any, b: any) => b[1].minus(a[1]) ); const top100Tokens = orderedTokenAmounts.slice(0, 100); From a0233f8fc7d519143f91c84fde363311619070d1 Mon Sep 17 00:00:00 2001 From: wayne Date: Thu, 23 Oct 2025 15:36:08 +0100 Subject: [PATCH 5/7] coins sdk integration --- defi/l2/adapters/ada.ts | 4 ++-- defi/l2/excluded.ts | 4 ++-- defi/l2/incoming.ts | 4 ++-- defi/l2/native.ts | 8 ++++---- defi/l2/tvl.ts | 4 ++-- defi/l2/v2/index.ts | 6 +++--- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/defi/l2/adapters/ada.ts b/defi/l2/adapters/ada.ts index 700d32d3e7..fa1da99afc 100644 --- a/defi/l2/adapters/ada.ts +++ b/defi/l2/adapters/ada.ts @@ -1,7 +1,7 @@ -import { cachedFetch } from "@defillama/sdk/build/util/cache"; +import { cache } from "@defillama/sdk"; export async function fetchAdaTokens(): Promise { - const res = await cachedFetch({key: "muesliswap-token-list", endpoint: "https://api.muesliswap.com/token-list"}) + const res = await cache.cachedFetch({key: "muesliswap-token-list", endpoint: "https://api.muesliswap.com/token-list"}) const coins = res .filter((c: any) => c.supply.circulating != null) .map((c: any) => ({ diff --git a/defi/l2/excluded.ts b/defi/l2/excluded.ts index a4ca96439c..0cd6c3497c 100644 --- a/defi/l2/excluded.ts +++ b/defi/l2/excluded.ts @@ -1,4 +1,4 @@ -import { getPrices } from "@defillama/sdk/build/util/coinsApi"; +import { coins } from "@defillama/sdk"; import { excludedTvlId } from "./constants"; import { multiCall } from "@defillama/sdk/build/abi/abi2"; import { getBlock } from "@defillama/sdk/build/util/blocks"; @@ -54,7 +54,7 @@ export async function getExcludedTvl(timestamp: number) { block: block.number, withMetadata: true, }), - getPrices( + coins.getPrices( uniqueTokens.map((token) => `${chain}:${token}`), timestamp ), diff --git a/defi/l2/incoming.ts b/defi/l2/incoming.ts index 0199069ea8..15e87eb8b5 100644 --- a/defi/l2/incoming.ts +++ b/defi/l2/incoming.ts @@ -4,7 +4,7 @@ import BigNumber from "bignumber.js"; import { DollarValues, TokenTvlData } from "./types"; import { geckoSymbols, zero } from "./constants"; import { fetchBridgeTokenList, fetchSupplies } from "./utils"; -import { getPrices } from "@defillama/sdk/build/util/coinsApi"; +import { coins } from "@defillama/sdk"; export async function fetchIncoming(params: { canonical: TokenTvlData; @@ -24,7 +24,7 @@ export async function fetchIncoming(params: { return; } - const prices = await getPrices( + const prices = await coins.getPrices( tokens.map((t: string) => `${chain}:${t}`), timestamp ); diff --git a/defi/l2/native.ts b/defi/l2/native.ts index b0afd693b9..b26c2ac4f4 100644 --- a/defi/l2/native.ts +++ b/defi/l2/native.ts @@ -1,6 +1,6 @@ import { getCurrentUnixTimestamp } from "../src/utils/date"; import { fetchAllTokens } from "../src/utils/shared/bridgedTvlPostgres"; -import { McapData, TokenTvlData, DollarValues, CoinsApiData } from "./types"; +import { McapData, TokenTvlData, DollarValues } from "./types"; import { Chain } from "@defillama/sdk/build/general"; import BigNumber from "bignumber.js"; import { Address } from "@defillama/sdk/build/types"; @@ -10,7 +10,7 @@ import { fetchAdaTokens } from "./adapters/ada"; import { nativeWhitelist } from "./adapters/manual"; import { withTimeout } from "../src/utils/shared/withTimeout"; import PromisePool from "@supercharge/promise-pool"; -import { getPrices, getMcaps } from "@defillama/sdk/build/util/coinsApi"; +import { coins } from "@defillama/sdk"; export async function fetchMinted(params: { chains: TokenTvlData; @@ -52,7 +52,7 @@ export async function fetchMinted(params: { console.log(`DBUG start for ${chain}`); // do these in order to lighten rpc, rest load - const prices = await getPrices( + const prices = await coins.getPrices( storedTokens.map((t: string) => (t.startsWith("coingecko:") ? t : `${chain}:${t}`)), timestamp ); @@ -62,7 +62,7 @@ export async function fetchMinted(params: { Object.keys(prices).map((p: string) => { if (p.startsWith("coingecko:")) prices[p].decimals = 0; }); - const mcaps = await getMcaps(Object.keys(prices), timestamp); + const mcaps = await coins.getMcaps(Object.keys(prices), timestamp); console.log(`DBUG mcaps done for ${chain}`); diff --git a/defi/l2/tvl.ts b/defi/l2/tvl.ts index a2209d6d73..629f8edeed 100644 --- a/defi/l2/tvl.ts +++ b/defi/l2/tvl.ts @@ -10,7 +10,7 @@ import { getChainDisplayName } from "../src/utils/normalizeChain"; import { verifyChanges } from "./verifyChanges"; import { getExcludedTvl } from "./excluded"; import { saveRawBridgedTvls } from "./raw"; -import { getMcaps } from "@defillama/sdk/build/util/coinsApi"; +import { coins } from "@defillama/sdk"; export default async function main(override?: boolean, timestamp?: number) { let symbolMap: { [pk: string]: string | null } = {}; @@ -86,7 +86,7 @@ async function translateToChainData( : `${chain}:${ownTokens[chain].address}` ); const nativeTokenSymbols = Object.keys(ownTokens).map((chain: string) => ownTokens[chain].ticker); - const mcapsPromise = getMcaps(nativeTokenKeys, timestamp); + const mcapsPromise = coins.getMcaps(nativeTokenKeys, timestamp); const nativeTokenTotalValues: any = {}; let translatedData: any = {}; diff --git a/defi/l2/v2/index.ts b/defi/l2/v2/index.ts index 0511dccd6c..b34ebeda21 100644 --- a/defi/l2/v2/index.ts +++ b/defi/l2/v2/index.ts @@ -22,7 +22,7 @@ import { hourlyRawTokensTvl } from "../../src/utils/getLastRecord"; import { Balances } from "@defillama/sdk"; import runInPromisePool from "@defillama/sdk/build/util/promisePool"; import { cachedFetch } from "@defillama/sdk/build/util/cache"; -import { getPrices, getMcaps } from "@defillama/sdk/build/util/coinsApi"; +import { coins } from "@defillama/sdk"; const searchWidth = 10800; // 3hr const allTokens: { [chain: Chain]: string[] } = {}; @@ -97,7 +97,7 @@ async function fetchNativeAndMcaps( // prices = await getR2JSONString(`prices/${chain}.json`); // } catch (e) { // console.log(`${chain} prices not cached, fetching`); - const prices = await getPrices( + const prices = await coins.getPrices( storedTokens.map((t: string) => normalizeKey(t.startsWith("coingecko:") ? t : `${chain}:${t}`)), timestamp ); @@ -114,7 +114,7 @@ async function fetchNativeAndMcaps( // mcaps = await getR2JSONString(`mcaps/${chain}.json`); // } catch (e) { // console.log(`${chain} mcaps not cached, fetching`); - const mcaps = await getMcaps(Object.keys(prices), timestamp); + const mcaps = await coins.getMcaps(Object.keys(prices), timestamp); // await storeR2JSONString(`mcaps/${chain}.json`, JSON.stringify(mcaps)); // } Object.keys(mcaps).map((m: string) => { From d6c65773daf2093dac00f1f29acacba919264734 Mon Sep 17 00:00:00 2001 From: wayne Date: Fri, 31 Oct 2025 11:35:48 +0000 Subject: [PATCH 6/7] patch own tokens, tron, sol, totals --- defi/l2/adapters/index.ts | 52 +++++++++++++++--------------- defi/l2/constants.ts | 32 +++++++++---------- defi/l2/utils.ts | 8 ++--- defi/l2/v2/index.ts | 66 +++++++++++++++++++++++++-------------- defi/src/getProtocols.ts | 2 ++ 5 files changed, 90 insertions(+), 70 deletions(-) diff --git a/defi/l2/adapters/index.ts b/defi/l2/adapters/index.ts index 946151338f..6044d8fe58 100644 --- a/defi/l2/adapters/index.ts +++ b/defi/l2/adapters/index.ts @@ -131,19 +131,19 @@ export const starknet = async (): Promise => { addresses.starknet = data.map((t: any) => t.l2_token_address?.toLowerCase()).filter((t: any) => t != null); return addresses.starknet; }; -export const era = async (): Promise => { - if (addresses.era) return addresses.era; - const { - data: { result: data }, - } = await axios.post("https://mainnet.era.zksync.io", { - method: "zks_getConfirmedTokens", - params: [0, 255], - id: 1, - jsonrpc: "2.0", - }); - addresses.era = data.map((d: any) => d.l2Address.toLowerCase()); - return addresses.era; -}; +// export const era = async (): Promise => { +// if (addresses.era) return addresses.era; +// const { +// data: { result: data }, +// } = await axios.post("https://mainnet.era.zksync.io", { +// method: "zks_getConfirmedTokens", +// params: [0, 255], +// id: 1, +// jsonrpc: "2.0", +// }); +// addresses.era = data.map((d: any) => d.l2Address.toLowerCase()); +// return addresses.era; +// }; export const tron = async (): Promise => { if (!("tron" in addresses)) addresses.tron = []; addresses.tron.push( @@ -177,19 +177,19 @@ export const mode = async (): Promise => { ); return addresses.mode; }; -export const zklink = async (): Promise => { - if (addresses.zklink) return addresses.zklink; - const allTokens = []; - let page = 1; - do { - const { items, meta } = await fetch(`https://explorer-api.zklink.io/tokens?limit=200&page=${page}&key=`); - allTokens.push(...items); - page++; - if (page >= meta.totalPages) break; - } while (page < 100); - addresses.zklink = allTokens.map((d: any) => d.l2Address.toLowerCase()); - return addresses.zklink; -}; +// export const zklink = async (): Promise => { +// if (addresses.zklink) return addresses.zklink; +// const allTokens = []; +// let page = 1; +// do { +// const { items, meta } = await fetch(`https://explorer-api.zklink.io/tokens?limit=200&page=${page}&key=`); +// allTokens.push(...items); +// page++; +// if (page >= meta.totalPages) break; +// } while (page < 100); +// addresses.zklink = allTokens.map((d: any) => d.l2Address.toLowerCase()); +// return addresses.zklink; +// }; export const manta = async (): Promise => { if (addresses.manta) return addresses.manta; const bridge = ( diff --git a/defi/l2/constants.ts b/defi/l2/constants.ts index 5007b0f11a..752e7ec8bd 100644 --- a/defi/l2/constants.ts +++ b/defi/l2/constants.ts @@ -29,7 +29,7 @@ export const chainsWithoutCanonicalBridges: string[] = [ // "aurora", "berachain", "flow", - "somnia", + "somnia", "plasma" ]; @@ -57,7 +57,7 @@ export const canonicalBridgeIds: { [id: string]: Chain } = { "3785": "polygon_zkevm", "3786": "scroll", "3787": "starknet", - "3788": "era", + // "3788": "era", "3813": "alephium", "3861": "rsk", // "3866": "near", @@ -65,18 +65,17 @@ export const canonicalBridgeIds: { [id: string]: Chain } = { "3935": "boba", "3936": "zksync", "4032": "manta", - "4141": "BSquared", + "4141": "bsquared", "4236": "blast", "4237": "mode", - "4335": "zklink", + // "4335": "zklink", "4336": "kinto", "4384": "rss3_vsl", "4438": "degen", "4439": "pulsechain", "4440": "ronin", - "4690": "lorenzo", "4692": "taiko", - "4558": "BOB", + "4558": "bob", "4796": "sanko", "4797": "xai", "4124": "merlin", @@ -95,36 +94,27 @@ export const canonicalBridgeIds: { [id: string]: Chain } = { "5513": "ink", "5538": "swellchain", "5552": "cronos_zkevm", - "5564": "eclipse", + // "5564": "eclipse", "5565": "shape", "5566": "zora", - "5609": "sophon", "5624": "soneium", "5683": "sonic", "5691": "abstract", "5692": "ancient8", - "5693": "cyber", "5694": "fraxtal", "5695": "gravity", "5696": "karak", "5697": "kroma", - "5698": "orderly", "5699": "rari", "5700": "redstone", "5701": "wc", "5702": "zero_network", // "5703": "zkfair", "5732": "sxr", - "5735": "sorare", "5772": "unichain", // "5833": "formnetwork", - "5854": "hemi-l2", "6063": "mxczkevm", "6148": "lens", - "6149": "openzk", - "6150": "treasure", - "6151": "zkcandy", - "6284": "ao", "6424": "soon", "6468": "btnx", "6498": "eventum", @@ -140,8 +130,18 @@ export const protocolBridgeIds: { [chain: string]: Chain } = { "344": "zkswap", "5130": "polynomial", // "5323": "exSat", + "5609": "sophon", + "5693": "cyber", + "5735": "sorare", + "5854": "hemi-l2", "4947": "ignition-fbtc", + "4690": "lorenzo", "4702": "immutable zkevm", + "5698": "orderly", + "6149": "openzk", + "6150": "treasure", + "6151": "zkcandy", + "6284": "ao", "6401": "embr", "6414": "xion", "6438": "echelon_initia", diff --git a/defi/l2/utils.ts b/defi/l2/utils.ts index 68f9398088..4eeb9b6f4f 100644 --- a/defi/l2/utils.ts +++ b/defi/l2/utils.ts @@ -225,15 +225,15 @@ async function getEVMSupplies( })), abi: "erc20:totalSupply", permitFailure: true, - block: block.block, + block: block?.block, }); contracts.slice(i, i + step).map((c: Address, i: number) => { if (res[i]) supplies[`${chain}:${bridgedTvlMixedCaseChains.includes(chain) ? c : c.toLowerCase()}`] = res[i]; }); - } catch { + } catch (e) { try { process.env.TRON_RPC = process.env.TRON_RPC?.substring(process.env.TRON_RPC.indexOf(",") + 1); - await PromisePool.withConcurrency(2) + await PromisePool.withConcurrency(5) .for(contracts.slice(i, i + step)) .process(async (target) => { const res = await call({ @@ -242,7 +242,7 @@ async function getEVMSupplies( abi: "erc20:totalSupply", block, }).catch(async (e) => { - await sleep(2000); + await sleep(1000); if (chain == "tron") console.log(`${target}:: \t ${e.message}`); }); if (res) diff --git a/defi/l2/v2/index.ts b/defi/l2/v2/index.ts index b34ebeda21..32fe10e4d9 100644 --- a/defi/l2/v2/index.ts +++ b/defi/l2/v2/index.ts @@ -3,13 +3,18 @@ import { getCurrentUnixTimestamp } from "../../src/utils/date"; import { Chain } from "@defillama/sdk/build/types"; import { fetchBridgeTokenList, fetchSupplies } from "../utils"; import { fetchAdaTokens } from "../adapters/ada"; +import { fetchAllTokens as fetchAllTokensFromDB } from "../../src/utils/shared/bridgedTvlPostgres"; import { withTimeout } from "../../src/utils/shared/withTimeout"; import { CoinsApiData, FinalData, FinalChainData } from "../types"; import { McapsApiData } from "../types"; import { getBlock } from "@defillama/sdk/build/util/blocks"; import { multiCall } from "@defillama/sdk/build/abi/abi2"; import BigNumber from "bignumber.js"; -import { bridgedTvlMixedCaseChains, chainsThatShouldNotBeLowerCased } from "../../src/utils/shared/constants"; +import { + bridgedTvlMixedCaseChains, + chainsThatShouldNotBeLowerCased, + chainsWithCaseSensitiveDataProviders, +} from "../../src/utils/shared/constants"; import { getR2JSONString, storeR2JSONString } from "../../src/utils/r2"; import { additional, excluded } from "../adapters/manual"; import { storeHistoricalToDB } from "./storeToDb"; @@ -85,7 +90,12 @@ async function fetchNativeAndMcaps( try { const start = new Date().getTime(); - const storedTokens = chain == "cardano" ? await fetchAdaTokens() : allTokens[chain]; + const storedTokens = + chain == "cardano" + ? await fetchAdaTokens() + : [...chainsThatShouldNotBeLowerCased, ...chainsWithCaseSensitiveDataProviders].includes(chain) + ? await fetchAllTokensFromDB(chain) + : allTokens[chain]; const ownTokenCgid: string | undefined = ownTokens[chain]?.address.startsWith("coingecko:") ? ownTokens[chain].address @@ -327,6 +337,7 @@ function normalizeKey(key: string) { const [chain, address] = key.split(":"); if (!address) return `coingecko:${key}`; if (chainsThatShouldNotBeLowerCased.includes(chain)) return key; + if (chainsWithCaseSensitiveDataProviders.includes(chain)) return key; return key.toLowerCase(); } @@ -434,31 +445,35 @@ async function main() { const symbol = allPrices[key].symbol.toUpperCase(); // filter for rwa, lst, stablecoins let section = "canonical"; - if (isOwnToken(chain, symbol)) section = "ownTokens"; - else if (stablecoinSymbols.includes(symbol)) section = "stablecoins"; + if (isOwnToken(chain, symbol)) { + if (ownTokens[chain].address.startsWith("coingecko:") && !key.startsWith("coingecko:")) return; + section = "ownTokens"; + } else if (stablecoinSymbols.includes(symbol)) section = "stablecoins"; else if (isRwaSymbol(symbol, rwaSymbols)) section = "rwa"; else if (lstSymbols.includes(symbol)) section = "lst"; rawData[chain][section as keyof FinalChainData].breakdown[key] = amount; if (!isOwnToken(chain, symbol)) rawData[chain].total.breakdown[key] = amount; }); - } + } else { + Object.keys(nativeDataAfterMcaps[chain]).map((key: string) => { + if (!allPrices[key]) return; + const symbol = allPrices[key].symbol.toUpperCase(); - Object.keys(nativeDataAfterMcaps[chain]).map((key: string) => { - if (!allPrices[key]) return; - const symbol = allPrices[key].symbol.toUpperCase(); - - let section = "native"; - if (isOwnToken(chain, symbol)) section = "ownTokens"; - else if (stablecoinSymbols.includes(symbol)) section = "stablecoins"; - else if (isRwaSymbol(symbol, rwaSymbols)) section = "rwa"; - else if (lstSymbols.includes(symbol)) section = "lst"; - else if (incomingAssets[chain] && incomingAssets[chain].includes(key.substring(key.indexOf(":") + 1))) - section = "thirdParty"; - else if (!key.startsWith("coingecko:") && !key.startsWith(chain)) section = "canonical"; - const amount = nativeDataAfterMcaps[chain][key]; - rawData[chain][section as keyof FinalChainData].breakdown[key] = amount; - if (section != "ownTokens") rawData[chain].total.breakdown[key] = amount; - }); + let section = "native"; + if (isOwnToken(chain, symbol)) { + if (ownTokens[chain].address.startsWith("coingecko:") && !key.startsWith("coingecko:")) return; + section = "ownTokens"; + } else if (stablecoinSymbols.includes(symbol)) section = "stablecoins"; + else if (isRwaSymbol(symbol, rwaSymbols)) section = "rwa"; + else if (lstSymbols.includes(symbol)) section = "lst"; + else if (incomingAssets[chain] && incomingAssets[chain].includes(key.substring(key.indexOf(":") + 1))) + section = "thirdParty"; + else if (!key.startsWith("coingecko:") && !key.startsWith(chain)) section = "canonical"; + const amount = nativeDataAfterMcaps[chain][key]; + rawData[chain][section as keyof FinalChainData].breakdown[key] = amount; + if (section != "ownTokens") rawData[chain].total.breakdown[key] = amount; + }); + } }); // create a symbol map and symbol data for human readable data @@ -469,8 +484,11 @@ async function main() { Object.keys(rawData[chain]).map((section: string) => { Object.keys(rawData[chain][section as keyof FinalChainData].breakdown).map((key: string) => { const symbol = allPrices[key].symbol.toUpperCase(); - symbolData[chain][section as keyof FinalChainData].breakdown[symbol] = - rawData[chain][section as keyof FinalChainData].breakdown[key]; + if (!symbolData[chain][section as keyof FinalChainData].breakdown[symbol]) + symbolData[chain][section as keyof FinalChainData].breakdown[symbol] = zero; + symbolData[chain][section as keyof FinalChainData].breakdown[symbol] = symbolData[chain][ + section as keyof FinalChainData + ].breakdown[symbol].plus(rawData[chain][section as keyof FinalChainData].breakdown[key]); if (!symbolMap[key]) symbolMap[key] = symbol; }); }); @@ -485,7 +503,7 @@ async function main() { const amounts = Object.values(allData[chain][section as keyof FinalChainData].breakdown); const total = amounts.length ? (amounts.reduce((p: any, c: any) => c.plus(p), zero) as BigNumber) : zero; allData[chain][section as keyof FinalChainData].total = total; - if (section == "ownTokens") return; + if (["ownTokens", "total"].includes(section)) return; totalTotal = totalTotal.plus(total); }); diff --git a/defi/src/getProtocols.ts b/defi/src/getProtocols.ts index f52ab52610..98d6a0e1ee 100644 --- a/defi/src/getProtocols.ts +++ b/defi/src/getProtocols.ts @@ -51,6 +51,8 @@ const majors = [ "mETH", ].map((t) => t.toUpperCase()); export const stablecoins = [ + "USD₮0", + "USD₮", "USDT", "USDC", "DAI", From 080fc6276377e898e942c16ec6dcc655f8966f30 Mon Sep 17 00:00:00 2001 From: wayne Date: Fri, 31 Oct 2025 15:21:03 +0000 Subject: [PATCH 7/7] v2 in job --- defi/l2/index.ts | 3 +++ defi/l2/v2/index.ts | 18 ++++-------------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/defi/l2/index.ts b/defi/l2/index.ts index de91263723..e296897c90 100644 --- a/defi/l2/index.ts +++ b/defi/l2/index.ts @@ -2,6 +2,7 @@ import chainAssets from "../l2/tvl"; import { storeR2JSONString } from "../src/utils/r2"; import { getCurrentUnixTimestamp } from "../src/utils/date"; import storeHistorical from "../l2/storeToDb"; +import { storeChainAssetsV2 } from "./v2"; export default async function storeChainAssets(override: boolean) { const res: any = await chainAssets(override); @@ -11,5 +12,7 @@ export default async function storeChainAssets(override: boolean) { await storeR2JSONString("chainAssets", JSON.stringify(res)); await storeHistorical(res); console.log("chain assets stored"); + await storeChainAssetsV2(override); + console.log("chain assets v2 stored"); process.exit(); } diff --git a/defi/l2/v2/index.ts b/defi/l2/v2/index.ts index 32fe10e4d9..e7e089cc4a 100644 --- a/defi/l2/v2/index.ts +++ b/defi/l2/v2/index.ts @@ -51,10 +51,7 @@ async function fetchAllTokens() { } // find the prices, mcaps and supplies of all tokens on all chains -async function fetchNativeAndMcaps( - timestamp: number, - override: boolean = false -): Promise<{ +async function fetchNativeAndMcaps(timestamp: number): Promise<{ chainData: { [chain: Chain]: { prices: { [token: string]: CoinsApiData }; @@ -82,7 +79,7 @@ async function fetchNativeAndMcaps( processor: async (chain: Chain) => { // protocol bridge IDs are closed and have no native tvl if (Object.values(protocolBridgeIds).includes(chain)) return; - await withTimeout(1000 * 60 * (override ? 20 : 120), minted(chain)).catch(() => { + await withTimeout(1000 * 60 * 20, minted(chain)).catch(() => { throw new Error(`fetchMinted() timed out for ${chain}`); }); @@ -367,7 +364,7 @@ const newChainAssets = () => ({ }); // main function -async function main() { +export async function storeChainAssetsV2(override: boolean = false) { const timestamp = 0; await fetchAllTokens(); const { sourceChainAmounts, protocolAmounts, destinationChainAmounts } = await fetchOutgoingAmountsFromDB(timestamp); @@ -533,7 +530,7 @@ async function main() { }); }); - await verifyChanges(symbolData); + if (!override) await verifyChanges(symbolData); await Promise.all([ symbolMapPromise, @@ -541,10 +538,3 @@ async function main() { storeR2JSONString("chainAssets2", JSON.stringify({ timestamp: getCurrentUnixTimestamp(), value: symbolData })), ]); } - -main() - .catch((e) => { - console.error(e); - process.exit(1); - }) - .then(() => process.exit(0)); // ts-node defi/l2/v2/index.ts