// Coin metadata (picture, description, socials) from each coin's metadata link, and a small picture cache. // // Metadata links are chosen by whoever launched the coin, so every fetch is checked first: only http(s), and // never an address on this machine or a private network (a link like http://127.0.0.1/... must not reach // services running next to OScope). import dns from 'node:dns/promises'; import net from 'node:net'; import { config } from './config.mjs'; const MAX_JSON = 256 * 1024; const MAX_IMG = 6 * 1024 * 1024; const IMG_CACHE_BYTES = 64 * 1024 * 1024; function privateIp(ip) { if (net.isIPv4(ip)) { const [a, b] = ip.split('.').map(Number); return a === 10 || a === 127 || a === 0 || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31) || (a === 192 && b === 168) || (a === 100 && b >= 64 && b <= 127) || a >= 224; } const v = ip.toLowerCase(); if (v.startsWith('::ffff:')) return privateIp(v.slice(7)); return v === '::1' || v === '::' || v.startsWith('fc') || v.startsWith('fd') || v.startsWith('fe80'); } async function safeUrl(raw) { let u; try { u = new URL(raw); } catch { return null; } if (u.protocol !== 'https:' && u.protocol !== 'http:') return null; if (u.username || u.password) return null; const host = u.hostname.replace(/^\[|\]$/g, ''); if (host === 'localhost' || host.endsWith('.localhost') || host.endsWith('.local') || host.endsWith('.internal')) return null; try { const addrs = net.isIP(host) ? [{ address: host }] : await dns.lookup(host, { all: true }); if (!addrs.length || addrs.some(a => privateIp(a.address))) return null; } catch { return null; } return u; } async function get(raw, maxBytes, accept, hops = 0) { const u = await safeUrl(raw); if (!u) throw new Error('blocked url'); const res = await fetch(u, { redirect: 'manual', signal: AbortSignal.timeout(8000), headers: { accept, 'user-agent': 'OScope' } }); if (res.status >= 300 && res.status < 400 && res.headers.get('location') && hops < 3) { return get(new URL(res.headers.get('location'), u).href, maxBytes, accept, hops + 1); } if (!res.ok) throw new Error('http ' + res.status); const len = +res.headers.get('content-length') || 0; if (len > maxBytes) throw new Error('too big'); const chunks = []; let size = 0; for await (const ch of res.body) { size += ch.length; if (size > maxBytes) throw new Error('too big'); chunks.push(ch); } return { type: (res.headers.get('content-type') || '').split(';')[0].trim().toLowerCase(), body: Buffer.concat(chunks) }; } // Most coins keep their metadata and picture on IPFS and name one gateway in the link. Gateways come and go // (several big ones no longer answer programs at all), so a link with /ipfs/ is asked for on the gateways // in IPFS_GATEWAYS. The one that answered last is asked first. const DEAD_FOR_PROGRAMS = /^https?:\/\/(ipfs\.io|dweb\.link|w3s\.link|nftstorage\.link|gateway\.pinata\.cloud|cloudflare-ipfs\.com)\//i; const gateways = [...config.ipfsGateways]; function candidates(raw) { const m = String(raw).match(/^(?:https?:\/\/[^/]+\/ipfs\/|ipfs:\/\/(?:ipfs\/)?)([A-Za-z0-9]{20,}(?:[/?].*)?)$/); if (!m) return [raw]; const own = DEAD_FOR_PROGRAMS.test(raw) || raw.startsWith('ipfs:') ? [] : [raw]; return [...own, ...gateways.filter(g => !raw.startsWith(g)).map(g => g + m[1])]; } async function getAny(raw, maxBytes, accept) { let last; for (const url of candidates(raw)) { try { const r = await get(url, maxBytes, accept); const i = gateways.findIndex(g => url.startsWith(g)); if (i > 0) gateways.unshift(...gateways.splice(i, 1)); return r; } catch (e) { last = e; } } throw last || new Error('no address'); } const cleanLink = v => { if (typeof v !== 'string') return ''; v = v.trim(); if (!v) return ''; if (!/^https?:\/\//i.test(v)) { if (/^@?[A-Za-z0-9_]{1,15}$/.test(v)) return 'https://x.com/' + v.replace(/^@/, ''); if (/^[\w.-]+\.[a-z]{2,}(\/\S*)?$/i.test(v)) v = 'https://' + v; else return ''; } try { const u = new URL(v); return u.protocol === 'https:' || u.protocol === 'http:' ? u.href.slice(0, 300) : ''; } catch { return ''; } }; export class Meta { constructor(store) { this.store = store; this.queue = []; this.active = 0; this.limit = 12; this.imgs = new Map(); // mint -> { type, body } this.imgBytes = 0; this.imgWait = new Map(); this.imgBad = new Map(); // mint -> when its picture last failed: not asked for again for 5 minutes } want(c) { if (c.metaState) return; c.metaState = 1; this.queue.push(c); this.pump(); } // a page is looking at this coin: fetch it before the rest urgent(c) { if (c.metaState === 1) { const i = this.queue.indexOf(c); if (i > 0) { this.queue.splice(i, 1); this.queue.unshift(c); } } else if (!c.metaState) { c.metaState = 1; this.queue.unshift(c); this.pump(); } } pump() { while (this.active < this.limit && this.queue.length) { // newest first: a coin launched a second ago matters more than one from the backlog const c = this.queue.length > 400 ? this.queue.pop() : this.queue.shift(); this.active++; this.load(c).finally(() => { this.active--; this.pump(); }); } if (this.queue.length > 4000) this.queue.splice(0, this.queue.length - 4000); } async load(c) { if (!c.uri) { c.metaState = 3; return; } try { const r = await getAny(c.uri, MAX_JSON, 'application/json,*/*'); if (r.type.startsWith('image/')) { c.image = c.uri; } else { const j = JSON.parse(r.body.toString('utf8')); const img = typeof j.image === 'string' ? j.image : ''; c.image = img; c.description = typeof j.description === 'string' ? j.description.slice(0, 600) : ''; const ext = j.extensions || {}; c.twitter = cleanLink(j.twitter || ext.twitter || ''); c.website = cleanLink(j.website || ext.website || ''); c.telegram = cleanLink(j.telegram || ext.telegram || ''); if (!c.name && typeof j.name === 'string') c.name = j.name.slice(0, 60); if (!c.symbol && typeof j.symbol === 'string') c.symbol = j.symbol.slice(0, 20); } c.metaState = 2; } catch { // a link is often not reachable yet in the first seconds after a launch: try again a few times c.metaTries = (c.metaTries || 0) + 1; c.metaState = 3; const wait = [3000, 15000, 60000][c.metaTries - 1]; if (wait) setTimeout(() => { if (c.metaState === 3) { c.metaState = 0; this.want(c); } }, wait).unref(); } c.ver++; } async image(mint) { const hit = this.imgs.get(mint); if (hit) { this.imgs.delete(mint); this.imgs.set(mint, hit); return hit; } if (this.imgWait.has(mint)) return this.imgWait.get(mint); const c = this.store.coins.get(mint); if (!c || !c.image) return null; const bad = this.imgBad.get(mint); if (bad && Date.now() - bad < 300000) return null; const p = getAny(c.image, MAX_IMG, 'image/*').then(r => { if (!r.type.startsWith('image/')) { this.imgBad.set(mint, Date.now()); return null; } this.imgBad.delete(mint); const v = { type: r.type, body: r.body }; this.imgs.set(mint, v); this.imgBytes += r.body.length; for (const [k, old] of this.imgs) { if (this.imgBytes <= IMG_CACHE_BYTES) break; this.imgs.delete(k); this.imgBytes -= old.body.length; } return v; }).catch(() => { this.imgBad.set(mint, Date.now()); if (this.imgBad.size > 5000) this.imgBad.clear(); return null; }).finally(() => this.imgWait.delete(mint)); this.imgWait.set(mint, p); return p; } }