Merge branch 'split-audit' into 'main'

split audit log into a separate database and compress payload

See merge request PronounsPage/PronounsPage!547

.gitignore

@@ -6,6 +6,8 @@
 /db.sqlite-wal
 /db.sqlite-*
 
+/audit.sqlite
+
 /*.sqlite
 
 /daemonise.json

pages/admin/audit-log/[username]/[id].vue

@@ -65,7 +65,7 @@
                                 <strong>{{ logEntry.event.split('/')[0] }}</strong>/{{ logEntry.event.split('/')[1] }}
                             </td>
                             <td>
-                                <pre v-if="logEntry.payload">{{ JSON.stringify(JSON.parse(logEntry.payload), null, 4) }}</pre>
+                                <pre v-if="logEntry.payload">{{ JSON.stringify(logEntry.payload, null, 4) }}</pre>
                             </td>
                         </tr>
                     </tbody>

server/audit.ts

@@ -1,26 +1,94 @@
+import zlib from 'node:zlib';
+import { promisify } from 'util';
+
 import * as Sentry from '@sentry/node';
 import SQL from 'sql-template-strings';
+import * as sqlite from 'sqlite';
+import sqlite3 from 'sqlite3';
 import { ulid } from 'ulid';
 
-import type { Database } from './db.ts';
 import type { UserRow } from './express/user.ts';
 
+import { rootDir } from '~/server/paths.ts';
+
 interface Payload {
     userId?: string;
     [key: string]: unknown;
 }
 
-export default async (
-    req: { db: Database; user?: Pick<UserRow, 'id' | 'username'> | null; rawUser?: Pick<UserRow, 'id' | 'username'> | undefined },
+let connection: sqlite.Database | null = null;
+const connect = async (): Promise<sqlite.Database> => {
+    if (!connection) {
+        connection = await sqlite.open({
+            filename: `${rootDir}/audit.sqlite`,
+            driver: sqlite3.Database,
+        });
+
+        // we don't want to replicate the migration setup just for this, so a little workaround here.
+        // it's already executed on live db, so only creating the table on dev
+        if (process.env.NODE_ENV === 'development') {
+            await connection.exec(`
+                CREATE TABLE IF NOT EXISTS audit_log
+                (
+                    id          TEXT NOT NULL PRIMARY KEY,
+                    userId      TEXT,
+                    username    TEXT,
+                    aboutUserId TEXT NULL,
+                    event       TEXT NOT NULL,
+                    payload     BLOB NULL
+                );
+                CREATE INDEX IF NOT EXISTS "audit_log_userId" ON "audit_log" ("userId");
+                CREATE INDEX IF NOT EXISTS "audit_log_username" ON "audit_log" ("username");
+                CREATE INDEX IF NOT EXISTS "audit_log_aboutUserId" ON "audit_log" ("aboutUserId");
+            `);
+        }
+    }
+
+    return connection;
+};
+
+export const closeAuditLogConnection = async () => {
+    if (connection) {
+        await connection.close();
+        connection = null;
+    }
+};
+
+const gzip = promisify(zlib.gzip);
+const gunzip = promisify(zlib.gunzip);
+
+export const auditLog = async (
+    req: { user?: Pick<UserRow, 'id' | 'username'> | null; rawUser?: Pick<UserRow, 'id' | 'username'> | undefined },
     event: string,
     payload: Payload | null = null,
 ): Promise<void> => {
+    const db = await connect();
+
     try {
+        const compressedPayload = payload
+            ? await gzip(Buffer.from(JSON.stringify(payload)) as Uint8Array)
+            : null;
+
         const user = req.user || req.rawUser || { id: null, username: null };
-        await req.db.get(SQL`INSERT INTO audit_log (id, userId, aboutUserId, username, event, payload) VALUES (
-            ${ulid()}, ${user.id}, ${payload?.userId || null}, ${user.username}, ${event}, ${payload ? JSON.stringify(payload) : null}
+        await db.get(SQL`INSERT INTO audit_log (id, userId, aboutUserId, username, event, payload) VALUES (
+            ${ulid()}, ${user.id}, ${payload?.userId || null}, ${user.username}, ${event}, ${compressedPayload}
         )`);
     } catch (error) {
         Sentry.captureException(error);
     }
 };
+
+export const fetchAuditLog = async (username: string, userId: string, aboutUserId: string) => {
+    const db = await connect();
+
+    const entries = await db.all(SQL`
+        SELECT * FROM audit_log
+        WHERE username = ${username} OR userId = ${userId} OR aboutUserId = ${aboutUserId}
+        ORDER BY id DESC
+    `);
+
+    return await Promise.all(entries.map(async (entry) => {
+        const payload = entry.payload ? JSON.parse((await gunzip(entry.payload)).toString()) : null;
+        return { ...entry, payload };
+    }));
+};

server/auditMigration.ts

@@ -0,0 +1,91 @@
+// pnpm run-file server/auditMigration.ts
+
+import zlib from 'node:zlib';
+import { promisify } from 'util';
+
+import SQL from 'sql-template-strings';
+import * as sqlite from 'sqlite';
+import sqlite3 from 'sqlite3';
+
+import dbConnection from './db.ts';
+
+import { rootDir } from '~/server/paths.ts';
+
+const gzip = promisify(zlib.gzip);
+
+const connectAudit = async (): Promise<sqlite.Database> => {
+    const connection = await sqlite.open({
+        filename: `${rootDir}/audit.sqlite`,
+        driver: sqlite3.Database,
+    });
+
+    await connection.exec(`
+        CREATE TABLE IF NOT EXISTS audit_log
+        (
+            id          TEXT NOT NULL PRIMARY KEY,
+            userId      TEXT,
+            username    TEXT,
+            aboutUserId TEXT NULL,
+            event       TEXT NOT NULL,
+            payload     BLOB NULL
+        );
+        CREATE INDEX IF NOT EXISTS "audit_log_userId" ON "audit_log" ("userId");
+        CREATE INDEX IF NOT EXISTS "audit_log_username" ON "audit_log" ("username");
+        CREATE INDEX IF NOT EXISTS "audit_log_aboutUserId" ON "audit_log" ("aboutUserId");
+    `);
+
+    return connection;
+};
+
+async function migrate(): Promise<void> {
+    const oldDb = await dbConnection();
+    const auditDb = await connectAudit();
+
+    const { count } = await oldDb.get(SQL`SELECT COUNT(*) as count FROM audit_log`);
+    let processed = 0;
+
+    const startedAt = new Date();
+
+    while (true) {
+        const entries = await oldDb.all(SQL`
+            SELECT *
+            FROM audit_log
+            ORDER BY id DESC
+            LIMIT 10000
+        `);
+
+        const progress = Math.round(processed / count * 1000) / 1000;
+        const elapsedMinutes = Math.round((new Date().getTime() - startedAt.getTime()) / 1000 / 60 * 100) / 100;
+        console.log(`[Done: ${processed}/${count} (${100 * progress}%)] [Elapsed: ${elapsedMinutes} min] Processing ${entries.length} entries…`);
+
+        if (entries.length === 0) {
+            break;
+        }
+
+        for (const entry of entries) {
+            const compressedPayload = entry.payload
+                ? await gzip(Buffer.from(entry.payload) as Uint8Array)
+                : null;
+
+            try {
+                await auditDb.get(SQL`INSERT INTO audit_log (id, userId, aboutUserId, username, event, payload) VALUES (
+                    ${entry.id}, ${entry.userId}, ${entry.aboutUserId}, ${entry.username}, ${entry.event}, ${compressedPayload}
+                )`);
+            } catch (error) {
+                // likely unique constraint issue, because we can't be atomic across databases. report and ignore.
+                console.error(error);
+            }
+
+            await oldDb.get(SQL`DELETE FROM audit_log WHERE id = ${entry.id}`);
+        }
+
+        processed += entries.length;
+    }
+
+    await oldDb.get(SQL`VACUUM`);
+
+    await oldDb.close();
+    await auditDb.close();
+}
+
+await migrate();

server/express/admin.ts

@@ -12,7 +12,7 @@ import allLocales from '../../locale/locales.ts';
 import type { LocaleDescription } from '../../locale/locales.ts';
 import buildLocaleList from '../../src/buildLocaleList.ts';
 import { buildDict, now, shuffle, handleErrorAsync, filterObjectKeys } from '../../src/helpers.ts';
-import auditLog from '../audit.ts';
+import { auditLog, fetchAuditLog } from '../audit.ts';
 import avatar from '../avatar.ts';
 import { archiveBan, liftBan } from '../ban.ts';
 import type { Database } from '../db.ts';
@@ -714,11 +714,7 @@ router.get('/admin/audit-log/:username/:id', handleErrorAsync(async (req, res) =
         return res.status(401).json({ error: 'Unauthorised' });
     }
 
-    return res.json(await req.db.all(SQL`
-        SELECT * FROM audit_log
-        WHERE username = ${req.params.username} OR userId = ${req.params.id} OR aboutUserId = ${req.params.id}
-        ORDER BY id DESC
-    `));
+    return res.json(await fetchAuditLog(req.params.username, req.params.id, req.params.id));
 }));
 
 router.get('/admin/authenticators/:id', handleErrorAsync(async (req, res) => {

server/express/census.ts

@@ -9,7 +9,7 @@ import type { Aggregate } from '../../locale/config.ts';
 import { groupBy, handleErrorAsync } from '../../src/helpers.ts';
 import { intersection, difference } from '../../src/sets.ts';
 import { buildChart } from '../../src/stats.ts';
-import auditLog from '../audit.ts';
+import { auditLog } from '../audit.ts';
 
 interface CensusRow {
     id: string;

server/express/images.ts

@@ -11,7 +11,7 @@ import SQL from 'sql-template-strings';
 import { ulid } from 'ulid';
 
 import { handleErrorAsync } from '../../src/helpers.ts';
-import auditLog from '../audit.ts';
+import { auditLog } from '../audit.ts';
 import { awsConfig, awsParams } from '../aws.ts';
 import { rootDir } from '../paths.ts';
 

server/express/inclusive.ts

@@ -5,7 +5,7 @@ import { ulid } from 'ulid';
 
 import { handleErrorAsync, sortClearedLinkedText } from '../../src/helpers.ts';
 import type { User } from '../../src/user.ts';
-import auditLog from '../audit.ts';
+import { auditLog } from '../audit.ts';
 import type { Database } from '../db.ts';
 
 interface InclusiveRow {

server/express/mfa.ts

@@ -3,7 +3,7 @@ import speakeasy from 'speakeasy';
 
 import { longtimeCookieSetting } from '../../src/cookieSettings.ts';
 import { handleErrorAsync } from '../../src/helpers.ts';
-import auditLog from '../audit.ts';
+import { auditLog } from '../audit.ts';
 import type { Database } from '../db.ts';
 
 import {

server/express/names.ts

@@ -4,7 +4,7 @@ import SQL from 'sql-template-strings';
 import { ulid } from 'ulid';
 
 import { handleErrorAsync } from '../../src/helpers.ts';
-import auditLog from '../audit.ts';
+import { auditLog } from '../audit.ts';
 import type { Database } from '../db.ts';
 
 interface NameRow {

server/express/nouns.ts

@@ -7,7 +7,7 @@ import { ulid } from 'ulid';
 import type { Translations } from '../../locale/translations.ts';
 import { clearKey, handleErrorAsync } from '../../src/helpers.ts';
 import type { User } from '../../src/user.ts';
-import auditLog from '../audit.ts';
+import { auditLog } from '../audit.ts';
 import type { Database } from '../db.ts';
 import { loadSuml } from '../loader.ts';
 import { registerLocaleFont } from '../localeFont.ts';

server/express/profile.ts

@@ -1,5 +1,5 @@
 import fs from 'fs';
-import zlib from 'zlib';
+import zlib from 'node:zlib';
 
 import { S3, NoSuchKey } from '@aws-sdk/client-s3';
 import * as Sentry from '@sentry/node';
@@ -29,7 +29,7 @@ import type {
 } from '../../src/profile.ts';
 import { socialProviders } from '../../src/socialProviders.ts';
 import { colours, styles } from '../../src/styling.ts';
-import auditLog from '../audit.ts';
+import { auditLog } from '../audit.ts';
 import avatar from '../avatar.ts';
 import { awsConfig, awsParams } from '../aws.ts';
 import crypto from '../crypto.ts';

server/express/sources.ts

@@ -4,7 +4,7 @@ import SQL from 'sql-template-strings';
 import { ulid } from 'ulid';
 
 import { clearKey, handleErrorAsync } from '../../src/helpers.ts';
-import auditLog from '../audit.ts';
+import { auditLog } from '../audit.ts';
 import type { Database } from '../db.ts';
 
 export interface SourceRow {

server/express/subscription.ts

@@ -3,7 +3,7 @@ import SQL from 'sql-template-strings';
 import { ulid } from 'ulid';
 
 import { handleErrorAsync } from '../../src/helpers.ts';
-import auditLog from '../audit.ts';
+import { auditLog } from '../audit.ts';
 
 import { validateEmail } from './user.ts';
 

server/express/terms.ts

@@ -5,7 +5,7 @@ import { ulid } from 'ulid';
 
 import { handleErrorAsync, sortClearedLinkedText, clearKey } from '../../src/helpers.ts';
 import type { User } from '../../src/user.ts';
-import auditLog from '../audit.ts';
+import { auditLog } from '../audit.ts';
 import type { Database } from '../db.ts';
 
 interface TermDb {

server/express/translations.ts

@@ -3,7 +3,7 @@ import SQL from 'sql-template-strings';
 import { ulid } from 'ulid';
 
 import { findAdmins, handleErrorAsync } from '../../src/helpers.ts';
-import auditLog from '../audit.ts';
+import { auditLog } from '../audit.ts';
 
 import { deduplicateEmailPreset } from './user.ts';
 

server/express/user.ts

@@ -12,7 +12,7 @@ import { longtimeCookieSetting } from '../../src/cookieSettings.ts';
 import { buildDict, makeId, now, handleErrorAsync, obfuscateEmail } from '../../src/helpers.ts';
 import type { User } from '../../src/user.ts';
 import { usernameRegex, usernameUnsafeRegex } from '../../src/username.ts';
-import auditLog from '../audit.ts';
+import { auditLog } from '../audit.ts';
 import avatar from '../avatar.ts';
 import { lookupBanArchive } from '../ban.ts';
 import { validateCaptcha } from '../captcha.ts';
@@ -106,7 +106,7 @@ export const saveAuthenticator = async (
             payload,
         });
     }
-    await auditLog({ db, user }, 'auth/saved_authenticator', {
+    await auditLog({ user }, 'auth/saved_authenticator', {
         type,
         payloadId: payload.id || undefined,
         name: payload.name || undefined,
@@ -478,7 +478,7 @@ router.post('/user/init', handleErrorAsync(async (req, res) => {
     if (user) {
         const { block } = await fetchLoginAttempts(req.db, user.id);
         if (block) {
-            await auditLog({ db: req.db, user }, 'auth/login_too_many_attempts');
+            await auditLog({ user }, 'auth/login_too_many_attempts');
             return res.json({ error: 'user.tooManyAttempts' });
         }
     }
@@ -490,12 +490,12 @@ router.post('/user/init', handleErrorAsync(async (req, res) => {
     };
 
     if (!await validateEmail(payload.email)) {
-        await auditLog({ db: req.db, user }, 'auth/email_invalid', { email: payload.email });
+        await auditLog({ user }, 'auth/email_invalid', { email: payload.email });
         return res.json({ error: 'user.account.changeEmail.invalid' });
     }
 
     if (!user && await lookupBanArchive(req.db, 'email', payload)) {
-        await auditLog({ db: req.db, user }, 'auth/blocked_archive_ban');
+        await auditLog({ user }, 'auth/blocked_archive_ban');
         return res.status(401).json({ error: 'Unauthorised' });
     }
 
@@ -511,12 +511,12 @@ router.post('/user/init', handleErrorAsync(async (req, res) => {
 
                 mailer(payload.email, 'confirmCode', { code: payload.code });
 
-                await auditLog({ db: req.db, user }, 'auth/requested_email_code', { email: payload.email });
+                await auditLog({ user }, 'auth/requested_email_code', { email: payload.email });
             },
             async () => {
                 const auth = await findLatestEmailAuthenticator(req.db, payload.email, 'email');
                 codeKey = auth ? auth.id : null;
-                await auditLog({ db: req.db, user }, 'auth/requested_email_code_duplicate', { email: payload.email });
+                await auditLog({ user }, 'auth/requested_email_code_duplicate', { email: payload.email });
             },
         );
     }

server/index.ts

@@ -41,6 +41,7 @@ import translationsRoute from './express/translations.ts';
 import userRoute from './express/user.ts';
 import { config } from './social.ts';
 
+import { closeAuditLogConnection } from '~/server/audit.ts';
 import useAuthentication from '~/server/utils/useAuthentication.ts';
 
 const MemoryStore = memorystore(session);
@@ -132,6 +133,7 @@ router.use(async function (req, res, next) {
         };
         res.on('finish', async () => {
             await req.db.close();
+            await closeAuditLogConnection();
         });
         res.set('Access-Control-Allow-Origin', '*');
         res.set('Access-Control-Allow-Headers', 'authorization,content-type');
@@ -171,6 +173,7 @@ router.use((err: Error, req: Request, res: Response, _next: NextFunction) => {
     console.error(formatError(err, req));
     res.status(500).send('Unexpected server error');
     req.db.close();
+    closeAuditLogConnection();
 });
 
 export default useBase('/api', defineExpressHandler(router));