Back to OScope
oscope/src/meta.mjs
OScope OScope: open source memecoin terminal for Solana f8ff931·Oct 20, 2026 2 Commits
Code
183 lines (170 loc) · 7.54 KB
Raw
// Coin metadata (picture, description, socials) from each coin's metadata link, and a small picture cache.
//
// Metadata links are chosen by whoever launched the coin, so every fetch is checked first: only http(s), and
// never an address on this machine or a private network (a link like http://127.0.0.1/... must not reach
// services running next to OScope).
import dns from 'node:dns/promises';
import net from 'node:net';
import { config } from './config.mjs';
const MAX_JSON = 256 * 1024;
const MAX_IMG = 6 * 1024 * 1024;
const IMG_CACHE_BYTES = 64 * 1024 * 1024;
function privateIp(ip) {
if (net.isIPv4(ip)) {
const [a, b] = ip.split('.').map(Number);
return a === 10 || a === 127 || a === 0 || (a === 169 && b === 254) || (a === 172 && b >= 16 && b <= 31) ||
(a === 192 && b === 168) || (a === 100 && b >= 64 && b <= 127) || a >= 224;
}
const v = ip.toLowerCase();
if (v.startsWith('::ffff:')) return privateIp(v.slice(7));
return v === '::1' || v === '::' || v.startsWith('fc') || v.startsWith('fd') || v.startsWith('fe80');
}
async function safeUrl(raw) {
let u;
try { u = new URL(raw); } catch { return null; }
if (u.protocol !== 'https:' && u.protocol !== 'http:') return null;
if (u.username || u.password) return null;
const host = u.hostname.replace(/^\[|\]$/g, '');
if (host === 'localhost' || host.endsWith('.localhost') || host.endsWith('.local') || host.endsWith('.internal')) return null;
try {
const addrs = net.isIP(host) ? [{ address: host }] : await dns.lookup(host, { all: true });
if (!addrs.length || addrs.some(a => privateIp(a.address))) return null;
} catch { return null; }
return u;
}
async function get(raw, maxBytes, accept, hops = 0) {
const u = await safeUrl(raw);
if (!u) throw new Error('blocked url');
const res = await fetch(u, { redirect: 'manual', signal: AbortSignal.timeout(8000), headers: { accept, 'user-agent': 'OScope' } });
if (res.status >= 300 && res.status < 400 && res.headers.get('location') && hops < 3) {
return get(new URL(res.headers.get('location'), u).href, maxBytes, accept, hops + 1);
}
if (!res.ok) throw new Error('http ' + res.status);
const len = +res.headers.get('content-length') || 0;
if (len > maxBytes) throw new Error('too big');
const chunks = [];
let size = 0;
for await (const ch of res.body) {
size += ch.length;
if (size > maxBytes) throw new Error('too big');
chunks.push(ch);
}
return { type: (res.headers.get('content-type') || '').split(';')[0].trim().toLowerCase(), body: Buffer.concat(chunks) };
}
// Most coins keep their metadata and picture on IPFS and name one gateway in the link. Gateways come and go
// (several big ones no longer answer programs at all), so a link with /ipfs/<id> is asked for on the gateways
// in IPFS_GATEWAYS. The one that answered last is asked first.
const DEAD_FOR_PROGRAMS = /^https?:\/\/(ipfs\.io|dweb\.link|w3s\.link|nftstorage\.link|gateway\.pinata\.cloud|cloudflare-ipfs\.com)\//i;
const gateways = [...config.ipfsGateways];
function candidates(raw) {
const m = String(raw).match(/^(?:https?:\/\/[^/]+\/ipfs\/|ipfs:\/\/(?:ipfs\/)?)([A-Za-z0-9]{20,}(?:[/?].*)?)$/);
if (!m) return [raw];
const own = DEAD_FOR_PROGRAMS.test(raw) || raw.startsWith('ipfs:') ? [] : [raw];
return [...own, ...gateways.filter(g => !raw.startsWith(g)).map(g => g + m[1])];
}
async function getAny(raw, maxBytes, accept) {
let last;
for (const url of candidates(raw)) {
try {
const r = await get(url, maxBytes, accept);
const i = gateways.findIndex(g => url.startsWith(g));
if (i > 0) gateways.unshift(...gateways.splice(i, 1));
return r;
} catch (e) { last = e; }
}
throw last || new Error('no address');
}
const cleanLink = v => {
if (typeof v !== 'string') return '';
v = v.trim();
if (!v) return '';
if (!/^https?:\/\//i.test(v)) {
if (/^@?[A-Za-z0-9_]{1,15}$/.test(v)) return 'https://x.com/' + v.replace(/^@/, '');
if (/^[\w.-]+\.[a-z]{2,}(\/\S*)?$/i.test(v)) v = 'https://' + v;
else return '';
}
try { const u = new URL(v); return u.protocol === 'https:' || u.protocol === 'http:' ? u.href.slice(0, 300) : ''; } catch { return ''; }
};
export class Meta {
constructor(store) {
this.store = store;
this.queue = [];
this.active = 0;
this.limit = 12;
this.imgs = new Map(); // mint -> { type, body }
this.imgBytes = 0;
this.imgWait = new Map();
this.imgBad = new Map(); // mint -> when its picture last failed: not asked for again for 5 minutes
}
want(c) {
if (c.metaState) return;
c.metaState = 1;
this.queue.push(c);
this.pump();
}
// a page is looking at this coin: fetch it before the rest
urgent(c) {
if (c.metaState === 1) { const i = this.queue.indexOf(c); if (i > 0) { this.queue.splice(i, 1); this.queue.unshift(c); } }
else if (!c.metaState) { c.metaState = 1; this.queue.unshift(c); this.pump(); }
}
pump() {
while (this.active < this.limit && this.queue.length) {
// newest first: a coin launched a second ago matters more than one from the backlog
const c = this.queue.length > 400 ? this.queue.pop() : this.queue.shift();
this.active++;
this.load(c).finally(() => { this.active--; this.pump(); });
}
if (this.queue.length > 4000) this.queue.splice(0, this.queue.length - 4000);
}
async load(c) {
if (!c.uri) { c.metaState = 3; return; }
try {
const r = await getAny(c.uri, MAX_JSON, 'application/json,*/*');
if (r.type.startsWith('image/')) { c.image = c.uri; }
else {
const j = JSON.parse(r.body.toString('utf8'));
const img = typeof j.image === 'string' ? j.image : '';
c.image = img;
c.description = typeof j.description === 'string' ? j.description.slice(0, 600) : '';
const ext = j.extensions || {};
c.twitter = cleanLink(j.twitter || ext.twitter || '');
c.website = cleanLink(j.website || ext.website || '');
c.telegram = cleanLink(j.telegram || ext.telegram || '');
if (!c.name && typeof j.name === 'string') c.name = j.name.slice(0, 60);
if (!c.symbol && typeof j.symbol === 'string') c.symbol = j.symbol.slice(0, 20);
}
c.metaState = 2;
} catch {
// a link is often not reachable yet in the first seconds after a launch: try again a few times
c.metaTries = (c.metaTries || 0) + 1;
c.metaState = 3;
const wait = [3000, 15000, 60000][c.metaTries - 1];
if (wait) setTimeout(() => { if (c.metaState === 3) { c.metaState = 0; this.want(c); } }, wait).unref();
}
c.ver++;
}
async image(mint) {
const hit = this.imgs.get(mint);
if (hit) { this.imgs.delete(mint); this.imgs.set(mint, hit); return hit; }
if (this.imgWait.has(mint)) return this.imgWait.get(mint);
const c = this.store.coins.get(mint);
if (!c || !c.image) return null;
const bad = this.imgBad.get(mint);
if (bad && Date.now() - bad < 300000) return null;
const p = getAny(c.image, MAX_IMG, 'image/*').then(r => {
if (!r.type.startsWith('image/')) { this.imgBad.set(mint, Date.now()); return null; }
this.imgBad.delete(mint);
const v = { type: r.type, body: r.body };
this.imgs.set(mint, v);
this.imgBytes += r.body.length;
for (const [k, old] of this.imgs) {
if (this.imgBytes <= IMG_CACHE_BYTES) break;
this.imgs.delete(k);
this.imgBytes -= old.body.length;
}
return v;
}).catch(() => { this.imgBad.set(mint, Date.now()); if (this.imgBad.size > 5000) this.imgBad.clear(); return null; }).finally(() => this.imgWait.delete(mint));
this.imgWait.set(mint, p);
return p;
}
}