diff --git a/.gitignore b/.gitignore index 63d95f429..3d923fbce 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ /db.sqlite-wal /db.sqlite-* +/audit.sqlite + /*.sqlite /daemonise.json diff --git a/pages/admin/audit-log/[username]/[id].vue b/pages/admin/audit-log/[username]/[id].vue index 58e79d526..7f6f4a733 100644 --- a/pages/admin/audit-log/[username]/[id].vue +++ b/pages/admin/audit-log/[username]/[id].vue @@ -65,7 +65,7 @@ {{ logEntry.event.split('/')[0] }}/{{ logEntry.event.split('/')[1] }} - 
{{ JSON.stringify(JSON.parse(logEntry.payload), null, 4) }}
+ 
{{ JSON.stringify(logEntry.payload, null, 4) }}
diff --git a/server/audit.ts b/server/audit.ts index 4b51b254b..6b7f85b94 100644 --- a/server/audit.ts +++ b/server/audit.ts @@ -1,26 +1,94 @@ +import zlib from 'node:zlib'; +import { promisify } from 'util'; + import * as Sentry from '@sentry/node'; import SQL from 'sql-template-strings'; +import * as sqlite from 'sqlite'; +import sqlite3 from 'sqlite3'; import { ulid } from 'ulid'; -import type { Database } from './db.ts'; import type { UserRow } from './express/user.ts'; +import { rootDir } from '~/server/paths.ts'; + interface Payload { userId?: string; [key: string]: unknown; } -export default async ( - req: { db: Database; user?: Pick | null; rawUser?: Pick | undefined }, +let connection: sqlite.Database | null = null; +const connect = async (): Promise => { + if (!connection) { + connection = await sqlite.open({ + filename: `${rootDir}/audit.sqlite`, + driver: sqlite3.Database, + }); + + // we don't want to replicate the migration setup just for this, so a little workaround here. + // it's already executed on live db, so only creating the table on dev + if (process.env.NODE_ENV === 'development') { + await connection.exec(` + CREATE TABLE IF NOT EXISTS audit_log + ( + id TEXT NOT NULL PRIMARY KEY, + userId TEXT, + username TEXT, + aboutUserId TEXT NULL, + event TEXT NOT NULL, + payload BLOB NULL + ); + CREATE INDEX IF NOT EXISTS "audit_log_userId" ON "audit_log" ("userId"); + CREATE INDEX IF NOT EXISTS "audit_log_username" ON "audit_log" ("username"); + CREATE INDEX IF NOT EXISTS "audit_log_aboutUserId" ON "audit_log" ("aboutUserId"); + `); + } + } + + return connection; +}; + +export const closeAuditLogConnection = async () => { + if (connection) { + await connection.close(); + connection = null; + } +}; + +const gzip = promisify(zlib.gzip); +const gunzip = promisify(zlib.gunzip); + +export const auditLog = async ( + req: { user?: Pick | null; rawUser?: Pick | undefined }, event: string, payload: Payload | null = null, ): Promise => { + const db = await connect(); + try { + const compressedPayload = payload + ? await gzip(Buffer.from(JSON.stringify(payload)) as Uint8Array) + : null; + const user = req.user || req.rawUser || { id: null, username: null }; - await req.db.get(SQL`INSERT INTO audit_log (id, userId, aboutUserId, username, event, payload) VALUES ( - ${ulid()}, ${user.id}, ${payload?.userId || null}, ${user.username}, ${event}, ${payload ? JSON.stringify(payload) : null} + await db.get(SQL`INSERT INTO audit_log (id, userId, aboutUserId, username, event, payload) VALUES ( + ${ulid()}, ${user.id}, ${payload?.userId || null}, ${user.username}, ${event}, ${compressedPayload} )`); } catch (error) { Sentry.captureException(error); } }; + +export const fetchAuditLog = async (username: string, userId: string, aboutUserId: string) => { + const db = await connect(); + + const entries = await db.all(SQL` + SELECT * FROM audit_log + WHERE username = ${username} OR userId = ${userId} OR aboutUserId = ${aboutUserId} + ORDER BY id DESC + `); + + return await Promise.all(entries.map(async (entry) => { + const payload = entry.payload ? JSON.parse((await gunzip(entry.payload)).toString()) : null; + return { ...entry, payload }; + })); +}; diff --git a/server/auditMigration.ts b/server/auditMigration.ts new file mode 100644 index 000000000..de019b7be --- /dev/null +++ b/server/auditMigration.ts @@ -0,0 +1,91 @@ +// pnpm run-file server/auditMigration.ts + +import zlib from 'node:zlib'; +import { promisify } from 'util'; + +import SQL from 'sql-template-strings'; +import * as sqlite from 'sqlite'; +import sqlite3 from 'sqlite3'; + +import dbConnection from './db.ts'; + +import { rootDir } from '~/server/paths.ts'; + +const gzip = promisify(zlib.gzip); + +const connectAudit = async (): Promise => { + const connection = await sqlite.open({ + filename: `${rootDir}/audit.sqlite`, + driver: sqlite3.Database, + }); + + await connection.exec(` + CREATE TABLE IF NOT EXISTS audit_log + ( + id TEXT NOT NULL PRIMARY KEY, + userId TEXT, + username TEXT, + aboutUserId TEXT NULL, + event TEXT NOT NULL, + payload BLOB NULL + ); + CREATE INDEX IF NOT EXISTS "audit_log_userId" ON "audit_log" ("userId"); + CREATE INDEX IF NOT EXISTS "audit_log_username" ON "audit_log" ("username"); + CREATE INDEX IF NOT EXISTS "audit_log_aboutUserId" ON "audit_log" ("aboutUserId"); + `); + + return connection; +}; + +async function migrate(): Promise { + const oldDb = await dbConnection(); + const auditDb = await connectAudit(); + + const { count } = await oldDb.get(SQL`SELECT COUNT(*) as count FROM audit_log`); + let processed = 0; + + const startedAt = new Date(); + + while (true) { + const entries = await oldDb.all(SQL` + SELECT * + FROM audit_log + ORDER BY id DESC + LIMIT 10000 + `); + + const progress = Math.round(processed / count * 1000) / 1000; + const elapsedMinutes = Math.round((new Date().getTime() - startedAt.getTime()) / 1000 / 60 * 100) / 100; + console.log(`[Done: ${processed}/${count} (${100 * progress}%)] [Elapsed: ${elapsedMinutes} min] Processing ${entries.length} entries…`); + + if (entries.length === 0) { + break; + } + + for (const entry of entries) { + const compressedPayload = entry.payload + ? await gzip(Buffer.from(entry.payload) as Uint8Array) + : null; + + try { + await auditDb.get(SQL`INSERT INTO audit_log (id, userId, aboutUserId, username, event, payload) VALUES ( + ${entry.id}, ${entry.userId}, ${entry.aboutUserId}, ${entry.username}, ${entry.event}, ${compressedPayload} + )`); + } catch (error) { + // likely unique constraint issue, because we can't be atomic across databases. report and ignore. + console.error(error); + } + + await oldDb.get(SQL`DELETE FROM audit_log WHERE id = ${entry.id}`); + } + + processed += entries.length; + } + + await oldDb.get(SQL`VACUUM`); + + await oldDb.close(); + await auditDb.close(); +} + +await migrate(); diff --git a/server/express/admin.ts b/server/express/admin.ts index 8bde25e50..c67a4bf40 100644 --- a/server/express/admin.ts +++ b/server/express/admin.ts @@ -12,7 +12,7 @@ import allLocales from '../../locale/locales.ts'; import type { LocaleDescription } from '../../locale/locales.ts'; import buildLocaleList from '../../src/buildLocaleList.ts'; import { buildDict, now, shuffle, handleErrorAsync, filterObjectKeys } from '../../src/helpers.ts'; -import auditLog from '../audit.ts'; +import { auditLog, fetchAuditLog } from '../audit.ts'; import avatar from '../avatar.ts'; import { archiveBan, liftBan } from '../ban.ts'; import type { Database } from '../db.ts'; @@ -714,11 +714,7 @@ router.get('/admin/audit-log/:username/:id', handleErrorAsync(async (req, res) = return res.status(401).json({ error: 'Unauthorised' }); } - return res.json(await req.db.all(SQL` - SELECT * FROM audit_log - WHERE username = ${req.params.username} OR userId = ${req.params.id} OR aboutUserId = ${req.params.id} - ORDER BY id DESC - `)); + return res.json(await fetchAuditLog(req.params.username, req.params.id, req.params.id)); })); router.get('/admin/authenticators/:id', handleErrorAsync(async (req, res) => { diff --git a/server/express/census.ts b/server/express/census.ts index 6c06ebea5..93575e0f8 100644 --- a/server/express/census.ts +++ b/server/express/census.ts @@ -9,7 +9,7 @@ import type { Aggregate } from '../../locale/config.ts'; import { groupBy, handleErrorAsync } from '../../src/helpers.ts'; import { intersection, difference } from '../../src/sets.ts'; import { buildChart } from '../../src/stats.ts'; -import auditLog from '../audit.ts'; +import { auditLog } from '../audit.ts'; interface CensusRow { id: string; diff --git a/server/express/images.ts b/server/express/images.ts index 809aa602b..d72b50443 100644 --- a/server/express/images.ts +++ b/server/express/images.ts @@ -11,7 +11,7 @@ import SQL from 'sql-template-strings'; import { ulid } from 'ulid'; import { handleErrorAsync } from '../../src/helpers.ts'; -import auditLog from '../audit.ts'; +import { auditLog } from '../audit.ts'; import { awsConfig, awsParams } from '../aws.ts'; import { rootDir } from '../paths.ts'; diff --git a/server/express/inclusive.ts b/server/express/inclusive.ts index c7f34dd31..c8307ac59 100644 --- a/server/express/inclusive.ts +++ b/server/express/inclusive.ts @@ -5,7 +5,7 @@ import { ulid } from 'ulid'; import { handleErrorAsync, sortClearedLinkedText } from '../../src/helpers.ts'; import type { User } from '../../src/user.ts'; -import auditLog from '../audit.ts'; +import { auditLog } from '../audit.ts'; import type { Database } from '../db.ts'; interface InclusiveRow { diff --git a/server/express/mfa.ts b/server/express/mfa.ts index 3e2652d40..e82e0fb07 100644 --- a/server/express/mfa.ts +++ b/server/express/mfa.ts @@ -3,7 +3,7 @@ import speakeasy from 'speakeasy'; import { longtimeCookieSetting } from '../../src/cookieSettings.ts'; import { handleErrorAsync } from '../../src/helpers.ts'; -import auditLog from '../audit.ts'; +import { auditLog } from '../audit.ts'; import type { Database } from '../db.ts'; import { diff --git a/server/express/names.ts b/server/express/names.ts index 3e25eef56..bf15f9a60 100644 --- a/server/express/names.ts +++ b/server/express/names.ts @@ -4,7 +4,7 @@ import SQL from 'sql-template-strings'; import { ulid } from 'ulid'; import { handleErrorAsync } from '../../src/helpers.ts'; -import auditLog from '../audit.ts'; +import { auditLog } from '../audit.ts'; import type { Database } from '../db.ts'; interface NameRow { diff --git a/server/express/nouns.ts b/server/express/nouns.ts index 4b9e36b8b..61be38f18 100644 --- a/server/express/nouns.ts +++ b/server/express/nouns.ts @@ -7,7 +7,7 @@ import { ulid } from 'ulid'; import type { Translations } from '../../locale/translations.ts'; import { clearKey, handleErrorAsync } from '../../src/helpers.ts'; import type { User } from '../../src/user.ts'; -import auditLog from '../audit.ts'; +import { auditLog } from '../audit.ts'; import type { Database } from '../db.ts'; import { loadSuml } from '../loader.ts'; import { registerLocaleFont } from '../localeFont.ts'; diff --git a/server/express/profile.ts b/server/express/profile.ts index f4ebf38da..9bc009046 100644 --- a/server/express/profile.ts +++ b/server/express/profile.ts @@ -1,5 +1,5 @@ import fs from 'fs'; -import zlib from 'zlib'; +import zlib from 'node:zlib'; import { S3, NoSuchKey } from '@aws-sdk/client-s3'; import * as Sentry from '@sentry/node'; @@ -29,7 +29,7 @@ import type { } from '../../src/profile.ts'; import { socialProviders } from '../../src/socialProviders.ts'; import { colours, styles } from '../../src/styling.ts'; -import auditLog from '../audit.ts'; +import { auditLog } from '../audit.ts'; import avatar from '../avatar.ts'; import { awsConfig, awsParams } from '../aws.ts'; import crypto from '../crypto.ts'; diff --git a/server/express/sources.ts b/server/express/sources.ts index 59a50bebd..ff21452e5 100644 --- a/server/express/sources.ts +++ b/server/express/sources.ts @@ -4,7 +4,7 @@ import SQL from 'sql-template-strings'; import { ulid } from 'ulid'; import { clearKey, handleErrorAsync } from '../../src/helpers.ts'; -import auditLog from '../audit.ts'; +import { auditLog } from '../audit.ts'; import type { Database } from '../db.ts'; export interface SourceRow { diff --git a/server/express/subscription.ts b/server/express/subscription.ts index 63fb5037b..cdebb912d 100644 --- a/server/express/subscription.ts +++ b/server/express/subscription.ts @@ -3,7 +3,7 @@ import SQL from 'sql-template-strings'; import { ulid } from 'ulid'; import { handleErrorAsync } from '../../src/helpers.ts'; -import auditLog from '../audit.ts'; +import { auditLog } from '../audit.ts'; import { validateEmail } from './user.ts'; diff --git a/server/express/terms.ts b/server/express/terms.ts index da623deba..6c7fce1ad 100644 --- a/server/express/terms.ts +++ b/server/express/terms.ts @@ -5,7 +5,7 @@ import { ulid } from 'ulid'; import { handleErrorAsync, sortClearedLinkedText, clearKey } from '../../src/helpers.ts'; import type { User } from '../../src/user.ts'; -import auditLog from '../audit.ts'; +import { auditLog } from '../audit.ts'; import type { Database } from '../db.ts'; interface TermDb { diff --git a/server/express/translations.ts b/server/express/translations.ts index 63101d43e..63b1b136f 100644 --- a/server/express/translations.ts +++ b/server/express/translations.ts @@ -3,7 +3,7 @@ import SQL from 'sql-template-strings'; import { ulid } from 'ulid'; import { findAdmins, handleErrorAsync } from '../../src/helpers.ts'; -import auditLog from '../audit.ts'; +import { auditLog } from '../audit.ts'; import { deduplicateEmailPreset } from './user.ts'; diff --git a/server/express/user.ts b/server/express/user.ts index 0fad0b031..e61047af3 100644 --- a/server/express/user.ts +++ b/server/express/user.ts @@ -12,7 +12,7 @@ import { longtimeCookieSetting } from '../../src/cookieSettings.ts'; import { buildDict, makeId, now, handleErrorAsync, obfuscateEmail } from '../../src/helpers.ts'; import type { User } from '../../src/user.ts'; import { usernameRegex, usernameUnsafeRegex } from '../../src/username.ts'; -import auditLog from '../audit.ts'; +import { auditLog } from '../audit.ts'; import avatar from '../avatar.ts'; import { lookupBanArchive } from '../ban.ts'; import { validateCaptcha } from '../captcha.ts'; @@ -106,7 +106,7 @@ export const saveAuthenticator = async ( payload, }); } - await auditLog({ db, user }, 'auth/saved_authenticator', { + await auditLog({ user }, 'auth/saved_authenticator', { type, payloadId: payload.id || undefined, name: payload.name || undefined, @@ -478,7 +478,7 @@ router.post('/user/init', handleErrorAsync(async (req, res) => { if (user) { const { block } = await fetchLoginAttempts(req.db, user.id); if (block) { - await auditLog({ db: req.db, user }, 'auth/login_too_many_attempts'); + await auditLog({ user }, 'auth/login_too_many_attempts'); return res.json({ error: 'user.tooManyAttempts' }); } } @@ -490,12 +490,12 @@ router.post('/user/init', handleErrorAsync(async (req, res) => { }; if (!await validateEmail(payload.email)) { - await auditLog({ db: req.db, user }, 'auth/email_invalid', { email: payload.email }); + await auditLog({ user }, 'auth/email_invalid', { email: payload.email }); return res.json({ error: 'user.account.changeEmail.invalid' }); } if (!user && await lookupBanArchive(req.db, 'email', payload)) { - await auditLog({ db: req.db, user }, 'auth/blocked_archive_ban'); + await auditLog({ user }, 'auth/blocked_archive_ban'); return res.status(401).json({ error: 'Unauthorised' }); } @@ -511,12 +511,12 @@ router.post('/user/init', handleErrorAsync(async (req, res) => { mailer(payload.email, 'confirmCode', { code: payload.code }); - await auditLog({ db: req.db, user }, 'auth/requested_email_code', { email: payload.email }); + await auditLog({ user }, 'auth/requested_email_code', { email: payload.email }); }, async () => { const auth = await findLatestEmailAuthenticator(req.db, payload.email, 'email'); codeKey = auth ? auth.id : null; - await auditLog({ db: req.db, user }, 'auth/requested_email_code_duplicate', { email: payload.email }); + await auditLog({ user }, 'auth/requested_email_code_duplicate', { email: payload.email }); }, ); } diff --git a/server/index.ts b/server/index.ts index 16c28e85d..54e8c0fee 100644 --- a/server/index.ts +++ b/server/index.ts @@ -41,6 +41,7 @@ import translationsRoute from './express/translations.ts'; import userRoute from './express/user.ts'; import { config } from './social.ts'; +import { closeAuditLogConnection } from '~/server/audit.ts'; import useAuthentication from '~/server/utils/useAuthentication.ts'; const MemoryStore = memorystore(session); @@ -132,6 +133,7 @@ router.use(async function (req, res, next) { }; res.on('finish', async () => { await req.db.close(); + await closeAuditLogConnection(); }); res.set('Access-Control-Allow-Origin', '*'); res.set('Access-Control-Allow-Headers', 'authorization,content-type'); @@ -171,6 +173,7 @@ router.use((err: Error, req: Request, res: Response, _next: NextFunction) => { console.error(formatError(err, req)); res.status(500).send('Unexpected server error'); req.db.close(); + closeAuditLogConnection(); }); export default useBase('/api', defineExpressHandler(router));