import { Router } from 'express'; import type { Request } from 'express'; import { getH3Event } from 'h3-express'; import Papa from 'papaparse'; import sha1 from 'sha1'; import SQL from 'sql-template-strings'; import { ulid } from 'ulid'; import { groupBy, handleErrorAsync } from '../../src/helpers.ts'; import { intersection, difference } from '../../src/sets.ts'; import { buildChart } from '../../src/stats.ts'; import { auditLog } from '../audit.ts'; import type { Aggregate, Config } from '~/locale/config.ts'; import { getLocale, loadConfig } from '~/server/data.ts'; interface CensusRow { id: string; locale: string; edition: string; userId: string | null; fingerprint: string | null; answers: string; writins: string; ip: string | null; userAgent: string | null; suspicious: number; troll: number | null; relevant: number | null; } const getIp = (req: Request): string | undefined => { try { return req.headers['x-forwarded-for'] as string || req.connection.remoteAddress || req.ips.join(',') || req.ip; } catch { return ''; } }; const buildFingerprint = (req: Request): string => sha1(` ${getIp(req)} ${req.headers['user-agent']} ${req.headers['accept-language']} `); const hasFinished = async (req: Request, config: Config): Promise => { if (req.user) { const byUser = await req.db.get(SQL` SELECT * FROM census_deduplication WHERE locale = ${config.locale} AND edition = ${config.census.edition} AND userId = ${req.user.id} `); return !!byUser; } const fingerprint = buildFingerprint(req); const byFingerprint = await req.db.get(SQL` SELECT * FROM census_deduplication WHERE locale = ${config.locale} AND edition = ${config.census.edition} AND fingerprint = ${fingerprint} AND userId IS NULL `); return !!byFingerprint; }; const isRelevant = (config: Config, answers: Record): boolean => { return config.census.relevant!.includes(answers['0']); }; const isTroll = (config: Config, answers: Record, writins: Record): boolean | null => { if (Object.values(writins).filter((x) => !!x).length) { return null; // unknown, send to moderation } const hasAnswersToTextAnswers = config.census.questions!.some((question, index) => { return question.type === 'textarea' && answers[index.toString()]; }); if (hasAnswersToTextAnswers) { return null; // unknown, send to moderation } return false; // no free-text provided }; const boolToInt = (value: unknown): number | null => { if (value === null) { return null; } if (value === true) { return 1; } if (value === false) { return 0; } throw `Invalid value ${value}`; }; const router = Router(); router.get('/census/finished', handleErrorAsync(async (req, res) => { const locale = getLocale(getH3Event(req)); const config = await loadConfig(locale); return res.json(await hasFinished(req, config)); })); router.post('/census/submit', handleErrorAsync(async (req, res) => { const locale = getLocale(getH3Event(req)); const config = await loadConfig(locale); const answers = JSON.parse(req.body.answers); const writins = JSON.parse(req.body.writins); const id = ulid(); await req.db.get(SQL` INSERT INTO census (id, locale, edition, answers, writins, suspicious, relevant, troll) VALUES ( ${id}, ${config.locale}, ${config.census.edition}, ${req.body.answers}, ${req.body.writins}, ${await hasFinished(req, config)}, ${boolToInt(isRelevant(config, answers))}, ${boolToInt(isTroll(config, answers, writins))} ) `); await req.db.get(SQL`INSERT INTO census_deduplication (locale, edition, userId, fingerprint) VALUES ( ${config.locale}, ${config.census.edition}, ${req.user ? req.user.id : null}, ${buildFingerprint(req)} )`); await auditLog(req, 'census/submitted_answer'); return res.json(id); })); const normaliseCensusGraph = (graph: Record): Record => { const newGraph: Record = {}; Object.entries(graph).forEach(([date, count]) => { date = date.substring(5); // remove year // only accept February (other months might appear because of a timezone bug, dismiss them) if (date.startsWith('02')) { newGraph[date] = count; } }); return newGraph; }; interface Count { c: number; } router.get('/census/count', handleErrorAsync(async (req, res) => { if (!req.isGranted('census')) { return res.json({ all: 0, nonbinary: 0, usable: 0, awaiting: 0, graphs: {}, }); } const locale = getLocale(getH3Event(req)); const config = await loadConfig(locale); // duplication reason: https://github.com/felixfbecker/node-sql-template-strings/issues/71 return res.json({ all: (await req.db.get(SQL` SELECT COUNT(*) as c FROM census WHERE locale = ${config.locale} AND edition = ${config.census.edition} `))!.c, nonbinary: (await req.db.get(SQL` SELECT COUNT(*) as c FROM census WHERE locale = ${config.locale} AND edition = ${config.census.edition} AND relevant = 1 `))!.c, usable: (await req.db.get(SQL` SELECT COUNT(*) as c FROM census WHERE locale = ${config.locale} AND edition = ${config.census.edition} AND relevant = 1 AND troll = 0 `))!.c, awaiting: (await req.db.get(SQL` SELECT COUNT(*) as c FROM census WHERE locale = ${config.locale} AND edition = ${config.census.edition} AND troll IS NULL `))!.c, graphs: Object.fromEntries(Object.entries(groupBy( await req.db.all>(SQL` SELECT edition, id FROM census WHERE locale = ${config.locale} AND relevant = 1 AND troll = 0 `), (r) => r.edition, )).map(([edition, rows]) => { return [edition, normaliseCensusGraph(buildChart(rows, false))]; })), }); })); const calculateAggregate = (config: Aggregate, answer: Set): boolean => { const expected = new Set(config.values); if (config.exclusive && difference(answer, expected).size > 0) { return false; } switch (config.operation) { case 'OR': return intersection(expected, answer).size > 0; case 'AND': return intersection(expected, answer).size === expected.size; default: throw new Error(`Operation "${config.operation} not supported"`); } }; router.get('/census/export', handleErrorAsync(async (req, res) => { if (!req.isGranted('census')) { return res.status(401).json({ error: 'Unauthorised' }); } const locale = getLocale(getH3Event(req)); const config = await loadConfig(locale); const report = []; for (const { answers: answersRaw, writins: writinsRaw } of await req.db .all>(SQL` SELECT answers, writins FROM census WHERE locale = ${config.locale} AND edition = ${config.census.edition} AND suspicious = 0 AND troll = 0 AND relevant = 1 `)) { const answers = JSON.parse(answersRaw) as Record; const writins = JSON.parse(writinsRaw); const answer: Record = {}; let i = 0; for (const question of config.census.questions!) { if (question.type === 'checkbox') { const answerForAggregate: Set = new Set(); for (const [option, _comment] of [...(question.optionsFirst || []), ...question.options, ...(question.optionsLast || [])]) { const checked = (answers[i.toString()] || [] as string[]).includes(option); answer[`${i}_${option}`] = checked ? 1 : ''; if (checked) { answerForAggregate.add(option); } } for (const [key, aggr] of Object.entries(question.aggregates || {})) { answer[`${i}_aggr_${key}`] = calculateAggregate(aggr, answerForAggregate) ? 1 : ''; } } else { answer[`${i}_`] = (answers[i.toString()] || '').toString().replace(/\n/g, ' | '); for (const [key, aggr] of Object.entries(question.aggregates || {})) { answer[`${i}_aggr_${key}`] = calculateAggregate(aggr, new Set([answer[`${i}_`]])) ? 1 : ''; } } if (question.writein) { answer[`${i}__writein`] = (writins[i.toString()] || '').replace(/\n/g, ' | '); } i++; } report.push(answer); } return res.set('content-type', 'text/csv').send(Papa.unparse(report)); })); router.get('/census/moderation/queue', handleErrorAsync(async (req, res) => { if (!req.isGranted('census')) { return res.status(401).json({ error: 'Unauthorised' }); } const locale = getLocale(getH3Event(req)); const config = await loadConfig(locale); const queue = await req.db.all(SQL` SELECT id, answers, writins FROM census WHERE locale = ${config.locale} AND edition = ${config.census.edition} AND troll IS NULL ORDER BY RANDOM() `); return res.json({ count: queue.length, next: queue.length ? queue[0] : null, }); })); router.post('/census/moderation/decide', handleErrorAsync(async (req, res) => { if (!req.isGranted('census')) { return res.status(401).json({ error: 'Unauthorised' }); } await req.db.get(SQL` UPDATE census SET troll = ${parseInt(req.body.decision)} WHERE id = ${req.body.id} `); await auditLog(req, 'census/moderated_answer', { id: req.body.id, decision: req.body.decision, }); return res.json('ok'); })); export default router;