mirror of
https://gitlab.com/PronounsPage/PronounsPage.git
synced 2025-08-03 19:17:07 -04:00
323 lines
10 KiB
TypeScript
323 lines
10 KiB
TypeScript
import { Router } from 'express';
|
|
import type { Request } from 'express';
|
|
import { getH3Event } from 'h3-express';
|
|
import Papa from 'papaparse';
|
|
import sha1 from 'sha1';
|
|
import SQL from 'sql-template-strings';
|
|
import { ulid } from 'ulid';
|
|
|
|
import { groupBy, handleErrorAsync } from '../../src/helpers.ts';
|
|
import { intersection, difference } from '../../src/sets.ts';
|
|
import { buildChart } from '../../src/stats.ts';
|
|
import { auditLog } from '../audit.ts';
|
|
|
|
import type { Aggregate, Config } from '~/locale/config.ts';
|
|
import { getLocale, loadConfig } from '~/server/data.ts';
|
|
|
|
interface CensusRow {
|
|
id: string;
|
|
locale: string;
|
|
edition: string;
|
|
userId: string | null;
|
|
fingerprint: string | null;
|
|
answers: string;
|
|
writins: string;
|
|
ip: string | null;
|
|
userAgent: string | null;
|
|
suspicious: number;
|
|
troll: number | null;
|
|
relevant: number | null;
|
|
}
|
|
|
|
const getIp = (req: Request): string | undefined => {
|
|
try {
|
|
return req.headers['x-forwarded-for'] as string || req.connection.remoteAddress || req.ips.join(',') || req.ip;
|
|
} catch {
|
|
return '';
|
|
}
|
|
};
|
|
|
|
const buildFingerprint = (req: Request): string => sha1(`
|
|
${getIp(req)}
|
|
${req.headers['user-agent']}
|
|
${req.headers['accept-language']}
|
|
`);
|
|
|
|
const hasFinished = async (req: Request, config: Config): Promise<boolean> => {
|
|
if (req.user) {
|
|
const byUser = await req.db.get(SQL`
|
|
SELECT * FROM census_deduplication
|
|
WHERE locale = ${config.locale}
|
|
AND edition = ${config.census.edition}
|
|
AND userId = ${req.user.id}
|
|
`);
|
|
return !!byUser;
|
|
}
|
|
|
|
const fingerprint = buildFingerprint(req);
|
|
const byFingerprint = await req.db.get(SQL`
|
|
SELECT * FROM census_deduplication
|
|
WHERE locale = ${config.locale}
|
|
AND edition = ${config.census.edition}
|
|
AND fingerprint = ${fingerprint}
|
|
AND userId IS NULL
|
|
`);
|
|
return !!byFingerprint;
|
|
};
|
|
|
|
const isRelevant = (config: Config, answers: Record<string, string>): boolean => {
|
|
return config.census.relevant!.includes(answers['0']);
|
|
};
|
|
|
|
const isTroll = (config: Config, answers: Record<string, string>, writins: Record<string, string>): boolean | null => {
|
|
if (Object.values(writins).filter((x) => !!x).length) {
|
|
return null; // unknown, send to moderation
|
|
}
|
|
|
|
const hasAnswersToTextAnswers = config.census.questions!.some((question, index) => {
|
|
return question.type === 'textarea' && answers[index.toString()];
|
|
});
|
|
if (hasAnswersToTextAnswers) {
|
|
return null; // unknown, send to moderation
|
|
}
|
|
|
|
return false; // no free-text provided
|
|
};
|
|
|
|
const boolToInt = (value: unknown): number | null => {
|
|
if (value === null) {
|
|
return null;
|
|
}
|
|
if (value === true) {
|
|
return 1;
|
|
}
|
|
if (value === false) {
|
|
return 0;
|
|
}
|
|
throw `Invalid value ${value}`;
|
|
};
|
|
|
|
const router = Router();
|
|
|
|
router.get('/census/finished', handleErrorAsync(async (req, res) => {
|
|
const locale = getLocale(getH3Event(req));
|
|
const config = await loadConfig(locale);
|
|
return res.json(await hasFinished(req, config));
|
|
}));
|
|
|
|
router.post('/census/submit', handleErrorAsync(async (req, res) => {
|
|
const locale = getLocale(getH3Event(req));
|
|
const config = await loadConfig(locale);
|
|
|
|
const answers = JSON.parse(req.body.answers);
|
|
const writins = JSON.parse(req.body.writins);
|
|
|
|
const id = ulid();
|
|
await req.db.get(SQL`
|
|
INSERT INTO census (id, locale, edition, answers, writins, suspicious, relevant, troll)
|
|
VALUES (
|
|
${id},
|
|
${config.locale},
|
|
${config.census.edition},
|
|
${req.body.answers},
|
|
${req.body.writins},
|
|
${await hasFinished(req, config)},
|
|
${boolToInt(isRelevant(config, answers))},
|
|
${boolToInt(isTroll(config, answers, writins))}
|
|
)
|
|
`);
|
|
|
|
await req.db.get(SQL`INSERT INTO census_deduplication (locale, edition, userId, fingerprint) VALUES (
|
|
${config.locale},
|
|
${config.census.edition},
|
|
${req.user ? req.user.id : null},
|
|
${buildFingerprint(req)}
|
|
)`);
|
|
|
|
await auditLog(req, 'census/submitted_answer');
|
|
|
|
return res.json(id);
|
|
}));
|
|
|
|
const normaliseCensusGraph = (graph: Record<string, number>): Record<string, number> => {
|
|
const newGraph: Record<string, number> = {};
|
|
Object.entries(graph).forEach(([date, count]) => {
|
|
date = date.substring(5); // remove year
|
|
// only accept February (other months might appear because of a timezone bug, dismiss them)
|
|
if (date.startsWith('02')) {
|
|
newGraph[date] = count;
|
|
}
|
|
});
|
|
return newGraph;
|
|
};
|
|
|
|
interface Count {
|
|
c: number;
|
|
}
|
|
|
|
router.get('/census/count', handleErrorAsync(async (req, res) => {
|
|
if (!req.isGranted('census')) {
|
|
return res.json({
|
|
all: 0,
|
|
nonbinary: 0,
|
|
usable: 0,
|
|
awaiting: 0,
|
|
graphs: {},
|
|
});
|
|
}
|
|
|
|
const locale = getLocale(getH3Event(req));
|
|
const config = await loadConfig(locale);
|
|
|
|
// duplication reason: https://github.com/felixfbecker/node-sql-template-strings/issues/71
|
|
|
|
return res.json({
|
|
all: (await req.db.get<Count>(SQL`
|
|
SELECT COUNT(*) as c FROM census
|
|
WHERE locale = ${config.locale}
|
|
AND edition = ${config.census.edition}
|
|
`))!.c,
|
|
nonbinary: (await req.db.get<Count>(SQL`
|
|
SELECT COUNT(*) as c FROM census
|
|
WHERE locale = ${config.locale}
|
|
AND edition = ${config.census.edition}
|
|
AND relevant = 1
|
|
`))!.c,
|
|
usable: (await req.db.get<Count>(SQL`
|
|
SELECT COUNT(*) as c FROM census
|
|
WHERE locale = ${config.locale}
|
|
AND edition = ${config.census.edition}
|
|
AND relevant = 1
|
|
AND troll = 0
|
|
`))!.c,
|
|
awaiting: (await req.db.get<Count>(SQL`
|
|
SELECT COUNT(*) as c FROM census
|
|
WHERE locale = ${config.locale}
|
|
AND edition = ${config.census.edition}
|
|
AND troll IS NULL
|
|
`))!.c,
|
|
graphs: Object.fromEntries(Object.entries(groupBy(
|
|
await req.db.all<Pick<CensusRow, 'edition' | 'id'>>(SQL`
|
|
SELECT edition, id
|
|
FROM census
|
|
WHERE locale = ${config.locale}
|
|
AND relevant = 1
|
|
AND troll = 0
|
|
`),
|
|
(r) => r.edition,
|
|
)).map(([edition, rows]) => {
|
|
return [edition, normaliseCensusGraph(buildChart(rows, false))];
|
|
})),
|
|
});
|
|
}));
|
|
|
|
const calculateAggregate = (config: Aggregate, answer: Set<number | string>): boolean => {
|
|
const expected = new Set(config.values);
|
|
if (config.exclusive && difference(answer, expected).size > 0) {
|
|
return false;
|
|
}
|
|
switch (config.operation) {
|
|
case 'OR':
|
|
return intersection(expected, answer).size > 0;
|
|
case 'AND':
|
|
return intersection(expected, answer).size === expected.size;
|
|
default:
|
|
throw new Error(`Operation "${config.operation} not supported"`);
|
|
}
|
|
};
|
|
|
|
router.get('/census/export', handleErrorAsync(async (req, res) => {
|
|
if (!req.isGranted('census')) {
|
|
return res.status(401).json({ error: 'Unauthorised' });
|
|
}
|
|
|
|
const locale = getLocale(getH3Event(req));
|
|
const config = await loadConfig(locale);
|
|
|
|
const report = [];
|
|
for (const { answers: answersRaw, writins: writinsRaw } of await req.db
|
|
.all<Pick<CensusRow, 'answers' | 'writins'>>(SQL`
|
|
SELECT answers, writins FROM census
|
|
WHERE locale = ${config.locale}
|
|
AND edition = ${config.census.edition}
|
|
AND suspicious = 0
|
|
AND troll = 0
|
|
AND relevant = 1
|
|
`)) {
|
|
const answers = JSON.parse(answersRaw) as Record<string, string>;
|
|
const writins = JSON.parse(writinsRaw);
|
|
|
|
const answer: Record<string, number | string> = {};
|
|
let i = 0;
|
|
for (const question of config.census.questions!) {
|
|
if (question.type === 'checkbox') {
|
|
const answerForAggregate: Set<string> = new Set();
|
|
for (const [option, _comment] of [...(question.optionsFirst || []), ...question.options, ...(question.optionsLast || [])]) {
|
|
const checked = (answers[i.toString()] || [] as string[]).includes(option);
|
|
answer[`${i}_${option}`] = checked ? 1 : '';
|
|
if (checked) {
|
|
answerForAggregate.add(option);
|
|
}
|
|
}
|
|
for (const [key, aggr] of Object.entries(question.aggregates || {})) {
|
|
answer[`${i}_aggr_${key}`] = calculateAggregate(aggr, answerForAggregate) ? 1 : '';
|
|
}
|
|
} else {
|
|
answer[`${i}_`] = (answers[i.toString()] || '').toString().replace(/\n/g, ' | ');
|
|
for (const [key, aggr] of Object.entries(question.aggregates || {})) {
|
|
answer[`${i}_aggr_${key}`] = calculateAggregate(aggr, new Set([answer[`${i}_`]])) ? 1 : '';
|
|
}
|
|
}
|
|
if (question.writein) {
|
|
answer[`${i}__writein`] = (writins[i.toString()] || '').replace(/\n/g, ' | ');
|
|
}
|
|
i++;
|
|
}
|
|
|
|
report.push(answer);
|
|
}
|
|
|
|
return res.set('content-type', 'text/csv').send(Papa.unparse(report));
|
|
}));
|
|
|
|
router.get('/census/moderation/queue', handleErrorAsync(async (req, res) => {
|
|
if (!req.isGranted('census')) {
|
|
return res.status(401).json({ error: 'Unauthorised' });
|
|
}
|
|
|
|
const locale = getLocale(getH3Event(req));
|
|
const config = await loadConfig(locale);
|
|
|
|
const queue = await req.db.all(SQL`
|
|
SELECT id, answers, writins FROM census
|
|
WHERE locale = ${config.locale}
|
|
AND edition = ${config.census.edition}
|
|
AND troll IS NULL
|
|
ORDER BY RANDOM()
|
|
`);
|
|
|
|
return res.json({
|
|
count: queue.length,
|
|
next: queue.length ? queue[0] : null,
|
|
});
|
|
}));
|
|
|
|
router.post('/census/moderation/decide', handleErrorAsync(async (req, res) => {
|
|
if (!req.isGranted('census')) {
|
|
return res.status(401).json({ error: 'Unauthorised' });
|
|
}
|
|
|
|
await req.db.get(SQL`
|
|
UPDATE census SET troll = ${parseInt(req.body.decision)} WHERE id = ${req.body.id}
|
|
`);
|
|
|
|
await auditLog(req, 'census/moderated_answer', {
|
|
id: req.body.id,
|
|
decision: req.body.decision,
|
|
});
|
|
|
|
return res.json('ok');
|
|
}));
|
|
|
|
export default router;
|