PronounsPage/server/analyseLinks.ts

import './setup.ts';

import SQL from 'sql-template-strings';

import dbConnection from './db.ts';
import copyImage from './imageCopy.ts';

import { newDate, random } from '#shared/helpers.ts';
import { LinkAnalyser } from '#shared/links.ts';

const timer = (ms: number) => new Promise((res) => setTimeout(res, ms));

(async () => {
    const analyser = new LinkAnalyser();
    const db = await dbConnection();
    while (true) {
        const chunk = await db.all(
            SQL`SELECT url
            FROM links
            WHERE (expiresAt IS NULL OR expiresAt <= ${newDate().getTime() / 1000})
            ORDER BY RANDOM()
            LIMIT 64`,
        );
        console.log(`Fetching ${chunk.length} links: (${chunk.map((l) => l.url).join(', ')})`);
        if (chunk.length === 0) {
            await timer(1000);
            continue;
        }
        const results = await Promise.all(
            chunk.map(({ url }) => Promise.race([
                analyser.analyse(url).then(async (result) => ({
                    ...result,
                    faviconCache: result.favicon ? await copyImage('favicon-cache', result.favicon!, 30) : null,
                })),
                new Promise((resolve) => setTimeout(() => resolve({ url, error: new Error('timeout') }), 12000)),
            ])),
        ) as { [k: string]: string }[];
        for (const result of results) {
            // random TTL (0-30 days) in order to spread out the legacy load
            const expireAt = (newDate().getTime() / 1000) + Math.floor(30 * 24 * 60 * 60 * random());
            if (result.error) {
                console.error(result);
                await db.get(SQL`UPDATE links SET expiresAt = ${expireAt} WHERE url=${result.url}`);
            } else {
                await db.get(SQL`UPDATE links
                    SET expiresAt = ${expireAt},
                        favicon = ${result.favicon},
                        faviconCache = ${result.faviconCache},
                        relMe = ${JSON.stringify(result.relMe)},
                        nodeinfo = ${JSON.stringify(result.nodeinfo)}
                    WHERE url=${result.url}`);
            }
        }
    }
})();