TheCloud/PronounsPage
1
0
Fork 0
mirror of https://gitlab.com/PronounsPage/PronounsPage.git synced 2025-09-26 14:32:04 -04:00
Code Issues Packages Projects Releases Wiki Activity

[profile] #290 generic rel=me (links table) - fixes

Browse Source
This commit is contained in:
Andrea Vos 2023-01-25 22:15:48 +01:00
parent bfda73dd46
commit ebad259da5
2 changed files with 8 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
server/analyseLinks.js
View File

@ -20,11 +20,13 @@ const timer = ms => new Promise( res => setTimeout(res, ms));
const results = await Promise.all(chunk.map(({url}) => analyser.analyse(url))); const results = await Promise.all(chunk.map(({url}) => analyser.analyse(url)));
for (let result of results) { for (let result of results) {
// random TTL (0-30 days) in order to spread out the legacy load // random TTL (0-30 days) in order to spread out the legacy load
const expireAt = parseInt(new Date() / 1000) + parseInt(30*24*60*60*Math.random());
if (result.error) { if (result.error) {
await db.get(SQL`UPDATE links SET expiresAt = ${parseInt(new Date() / 1000) + parseInt(30*24*60*60*Math.random())} WHERE url=${result.url}`); console.error(result);
await db.get(SQL`UPDATE links SET expiresAt = ${expireAt} WHERE url=${result.url}`);
} else { } else {
await db.get(SQL`UPDATE links await db.get(SQL`UPDATE links
SET expiresAt = ${parseInt(new Date() / 1000) + parseInt(30*24*60*60*Math.random())}, SET expiresAt = ${expireAt},
favicon = ${result.favicon}, favicon = ${result.favicon},
relMe = ${JSON.stringify(result.relMe)}, relMe = ${JSON.stringify(result.relMe)},
nodeinfo = ${JSON.stringify(result.nodeinfo)} nodeinfo = ${JSON.stringify(result.nodeinfo)}

7
src/links.js
View File

@ -1,6 +1,7 @@
const AbortController = require('abort-controller'); const AbortController = require('abort-controller');
const fetch = require('node-fetch'); const fetch = require('node-fetch');
const jsdom = require('jsdom'); const {JSDOM} = require('jsdom');
const VirtualConsole = require('jsdom/lib/jsdom/virtual-console');
const SQL = require("sql-template-strings"); const SQL = require("sql-template-strings");
const normaliseUrl = (url) => { const normaliseUrl = (url) => {
@ -18,9 +19,9 @@ class LinkAnalyser {
let $document; let $document;
try { try {
const controller = new AbortController(); const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 3000); const timeout = setTimeout(() => controller.abort(), 10000);
const htmlString = await (await fetch(url, {signal: controller.signal})).text(); const htmlString = await (await fetch(url, {signal: controller.signal})).text();
$document = new jsdom.JSDOM(htmlString, { virtualConsole: new jsdom.virtualConsole() }); $document = new JSDOM(htmlString, { virtualConsole: new VirtualConsole() });
} catch (e) { } catch (e) {
return { return {
url: url.toString(), url: url.toString(),