TheCloud/PronounsPage
1
0
Fork 0
mirror of https://gitlab.com/PronounsPage/PronounsPage.git synced 2025-09-27 15:05:38 -04:00
Code Issues Packages Projects Releases Wiki Activity

#254 clean up S3 images - cleanup script - optimise

Browse Source
This commit is contained in:
Andrea Vos 2021-08-24 21:56:01 +00:00
parent 20328fb6d0
commit a7c532c378
1 changed files with 21 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
server/cleanupImages.js
View File

@ -11,49 +11,52 @@ async function cleanup() {
console.log('--- Fetching ids expected to stay ---'); console.log('--- Fetching ids expected to stay ---');
const db = await dbConnection(); const db = await dbConnection();
const avatars = (await db.all(` const avatars = {}
for (let row of await db.all(`
SELECT avatarSource SELECT avatarSource
FROM users FROM users
WHERE avatarSource LIKE 'https://pronouns-page.s3-eu-west-1.amazonaws.com/images/%'` WHERE avatarSource LIKE 'https://pronouns-page.s3-eu-west-1.amazonaws.com/images/%'`
)).map(row => row.avatarSource.match('https://pronouns-page.s3-eu-west-1.amazonaws.com/images/(.*)-thumb.png')[1]); )) {
avatars[row.avatarSource.match('https://pronouns-page.s3-eu-west-1.amazonaws.com/images/(.*)-thumb.png')[1]] = true;
}
const flags = []; const flags = {};
for (let row of await db.all(` for (let row of await db.all(`
SELECT customFlags SELECT customFlags
FROM profiles FROM profiles
WHERE customFlags != '{}' WHERE customFlags != '{}'
`)) { `)) {
for (let key of Object.keys(JSON.parse(row.customFlags))) { for (let key of Object.keys(JSON.parse(row.customFlags))) {
flags.push(key); flags[key] = true;
} }
} }
const sources = []; const sources = {};
for (let row of await db.all(` for (let row of await db.all(`
SELECT images SELECT images
FROM sources FROM sources
WHERE images is not null AND images != '' WHERE images is not null AND images != ''
`)) { `)) {
for (let key of row.images.split(',')) { for (let key of row.images.split(',')) {
sources.push(key); sources[key] = true;
} }
} }
const terms = []; const terms = {};
for (let row of await db.all(` for (let row of await db.all(`
SELECT images SELECT images
FROM terms FROM terms
WHERE images is not null AND images != '' WHERE images is not null AND images != ''
`)) { `)) {
for (let key of row.images.split(',')) { for (let key of row.images.split(',')) {
terms.push(key); terms[key] = true;
} }
} }
console.log('Avatars: ' + avatars.length); console.log('Avatars: ' + Object.keys(avatars).length);
console.log('Flags: ' + flags.length); console.log('Flags: ' + Object.keys(flags).length);
console.log('Sources: ' + sources.length); console.log('Sources: ' + Object.keys(sources).length);
console.log('Terms: ' + terms.length); console.log('Terms: ' + Object.keys(terms).length);
await db.close(); await db.close();
@ -77,7 +80,7 @@ async function cleanup() {
const toRemove = []; const toRemove = [];
const remove = async (object, reason) => { const remove = async (object, reason) => {
console.log(`REMOVING: ${object.Key} (${reason})`); console.log(`REMOVING: ${object.Key} (${reason})`);
toRemove.append({Key: object.Key}); toRemove.push({Key: object.Key});
removed += 1; removed += 1;
removedSize += object.Size; removedSize += object.Size;
} }
@ -93,19 +96,19 @@ async function cleanup() {
const [, id, size] = object.Key.match('images/(.*)-(.*).png'); const [, id, size] = object.Key.match('images/(.*)-(.*).png');
if (avatars.includes(id)) { if (avatars[id]) {
if (size !== 'thumb') { if (size !== 'thumb') {
await remove(object, 'avatar'); await remove(object, 'avatar');
} }
} else if (flags.includes(id)) { } else if (flags[id]) {
if (size !== 'flag') { if (size !== 'flag') {
await remove(object, 'flag'); await remove(object, 'flag');
} }
} else if (sources.includes(id)) { } else if (sources[id]) {
if (size !== 'big' && size !== 'thumb') { if (size !== 'big' && size !== 'thumb') {
await remove(object, 'source'); await remove(object, 'source');
} }
} else if (terms.includes(id)) { } else if (terms[id]) {
if (size !== 'big' && size !== 'thumb') { if (size !== 'big' && size !== 'thumb') {
await remove(object, 'term'); await remove(object, 'term');
} }
@ -122,7 +125,7 @@ async function cleanup() {
} }
}); });
} }
throw 'aaa';
if (objects.Contents.length < chunkSize) { if (objects.Contents.length < chunkSize) {
break; break;
} }