TheCloud/PronounsPage
1
0
Fork 0
mirror of https://gitlab.com/PronounsPage/PronounsPage.git synced 2025-09-24 05:05:20 -04:00
Code Issues Packages Projects Releases Wiki Activity

auditMigration – batch it up!

Browse Source
This commit is contained in:
Andrea Vos 2025-01-02 21:31:14 +01:00
parent 14d629bc26
commit 25367f5af0
1 changed files with 25 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
server/auditMigration.ts
View File

@ -50,8 +50,8 @@ async function migrate(): Promise<void> {
const entries = await oldDb.all(SQL`
SELECT *
FROM audit_log
ORDER BY id DESC
LIMIT 10000
ORDER BY id ASC
LIMIT 1000
`);
const progress = Math.round(processed / count * 1000) / 1000;
@ -62,22 +62,31 @@ async function migrate(): Promise<void> {
break;
}
for (const entry of entries) {
const compressedPayload = entry.payload
? await gzip(Buffer.from(entry.payload) as Uint8Array)
: null;
const compressedPayloads = await Promise.all(
entries.map((entry) =>
entry.payload
? gzip(Buffer.from(entry.payload) as Uint8Array)
: Promise.resolve(null)
)
);
try {
await auditDb.get(SQL`INSERT INTO audit_log (id, userId, aboutUserId, username, event, payload) VALUES (
${entry.id}, ${entry.userId}, ${entry.aboutUserId}, ${entry.username}, ${entry.event}, ${compressedPayload}
)`);
} catch (error) {
// likely unique constraint issue, because we can't be atomic across databases. report and ignore.
console.error(error);
}
const placeholders = entries.map(() => '(?, ?, ?, ?, ?, ?)').join(', ');
const values = entries.flatMap((entry, index) => [
entry.id,
entry.userId,
entry.aboutUserId,
entry.username,
entry.event,
compressedPayloads[index],
]);
await oldDb.get(SQL`DELETE FROM audit_log WHERE id = ${entry.id}`);
}
await auditDb.run(
`INSERT OR IGNORE INTO audit_log (id, userId, aboutUserId, username, event, payload) VALUES ${placeholders}`,
values
);
const idsToDelete = entries.map((entry) => entry.id).join('","');
await oldDb.run(`DELETE FROM audit_log WHERE id IN ("${idsToDelete}")`);
processed += entries.length;
}