Merge branch 'benpai-s3-migrate' into 'main'

Benpai s3 migrate

See merge request PronounsPage/PronounsPage!635

.env.dist

@@ -23,8 +23,14 @@ DISCORD_SECRET=
 AWS_REGION=eu-west-1
 AWS_KEY=
 AWS_SECRET=
-AWS_S3_BUCKET=
-AWS_CLOUDFRONT_ID=
+
+S3_PROVIDER=
+S3_REGION=
+S3_KEY=
+S3_SECRET=
+S3_BUCKET=
+S3_ENDPOINT=
+S3_PUBLIC_ACCESS=
 
 NARAKEET_API_KEY=
 

migrations/092-r2-migration.sql

@@ -0,0 +1,51 @@
+-- Up
+
+UPDATE users
+SET avatarSource = REPLACE(
+    avatarSource,
+    'https://dclu0bpcdglik.cloudfront.net/',
+    'https://cdn.pronouns.page/'
+)
+WHERE avatarSource LIKE 'https://dclu0bpcdglik.cloudfront.net/%';
+
+UPDATE profiles
+SET card = REPLACE(
+    card,
+    'https://pronouns-page.s3.eu-west-1.amazonaws.com/',
+    'https://cdn.pronouns.page/'
+)
+WHERE card LIKE 'https://pronouns-page.s3.eu-west-1.amazonaws.com/%';
+
+UPDATE profiles
+SET cardDark = REPLACE(
+    cardDark,
+    'https://pronouns-page.s3.eu-west-1.amazonaws.com/',
+    'https://cdn.pronouns.page/'
+)
+WHERE cardDark LIKE 'https://pronouns-page.s3.eu-west-1.amazonaws.com/%';
+
+-- Down
+
+UPDATE users
+SET avatarSource = REPLACE(
+    avatarSource,
+    'https://cdn.pronouns.page/',
+    'https://dclu0bpcdglik.cloudfront.net/'
+)
+WHERE avatarSource LIKE 'https://cdn.pronouns.page/%';
+
+UPDATE profiles
+SET card = REPLACE(
+    card,
+    'https://cdn.pronouns.page/',
+    'https://pronouns-page.s3.eu-west-1.amazonaws.com/'
+)
+WHERE card LIKE 'https://cdn.pronouns.page/%';
+
+UPDATE profiles
+SET cardDark = REPLACE(
+    cardDark,
+    'https://cdn.pronouns.page/',
+    'https://pronouns-page.s3.eu-west-1.amazonaws.com/'
+)
+WHERE cardDark LIKE 'https://cdn.pronouns.page/%';

server/aws.ts

@@ -1,11 +0,0 @@
-export const awsConfig = {
-    region: process.env.AWS_REGION,
-    credentials: {
-        accessKeyId: process.env.AWS_KEY!,
-        secretAccessKey: process.env.AWS_SECRET!,
-    },
-};
-
-export const awsParams = {
-    Bucket: process.env.AWS_S3_BUCKET!,
-};

server/cards.ts

@@ -1,21 +1,18 @@
 import './setup.ts';
 
-import { S3 } from '@aws-sdk/client-s3';
 import * as Sentry from '@sentry/node';
 import Pageres from 'pageres';
 import type { Screenshot } from 'pageres';
 import { ulid } from 'ulid';
 
-import { awsConfig, awsParams } from './aws.ts';
 import dbConnection from './db.ts';
 import type { Database } from './db.ts';
 import isHighLoadTime from './overload.js';
 
+import { s3, s3Config, s3BucketParams } from '~/server/cloudServices.ts';
 import jwt from '~/server/jwt.ts';
 import { getLocaleUrls } from '~/src/domain.ts';
 
-const s3 = new S3(awsConfig);
-
 const urlBases: Record<string, string> = {};
 
 for (const [locale, url] of Object.entries(getLocaleUrls(process.env.NUXT_PUBLIC_DOMAIN_BASE))) {
@@ -109,13 +106,12 @@ const shoot = async (db: Database, mode: 'light' | 'dark'): Promise<void> => {
             Body: buffer,
             ContentType: 'image/png',
             ACL: 'public-read',
-            ...awsParams,
+            ...s3BucketParams,
         });
 
         await db.get(`
             UPDATE profiles
-            SET ${mode === 'dark' ? 'cardDark' : 'card'
-            }='https://${awsParams.Bucket}.s3.${awsConfig.region}.amazonaws.com/${key}'
+            SET ${mode === 'dark' ? 'cardDark' : 'card'}='${s3Config.publicAccess}/${key}'
             WHERE id='${id}'`);
     }
 };

server/cleanupImages.ts

@@ -1,9 +1,8 @@
 import './setup.ts';
 
-import { S3 } from '@aws-sdk/client-s3';
 import type { ListObjectsV2Output, ObjectIdentifier, _Object } from '@aws-sdk/client-s3';
 
-import { awsConfig, awsParams } from './aws.ts';
+import { s3, s3BucketParams, s3Config } from './cloudServices.ts';
 import dbConnection from './db.ts';
 
 import type { Profile } from '~/src/profile.ts';
@@ -89,7 +88,7 @@ async function cleanup(): Promise<void> {
         WHERE card is not null AND card != ''
     `)) {
         cards[row.card!.match(
-            /(https:\/\/pronouns-page.s3.eu-west-1.amazonaws.com\/card\/[^/]+\/.+-([^-]+)\.png)/,
+            `${s3Config.publicAccess}/card/[^/]+/.+-([^-]+)\.png`,
         )![1]] = true;
     }
     for (const row of await db.all(`
@@ -97,8 +96,8 @@ async function cleanup(): Promise<void> {
         FROM profiles
         WHERE cardDark is not null AND cardDark != ''
     `)) {
-        const m = row.cardDark.match(
-            /https:\/\/pronouns-page.s3.eu-west-1.amazonaws.com\/card\/[^/]+\/.+-([^-]+)-dark\.png/,
+        const m = row.cardDark!.match(
+            `${s3Config.publicAccess}/card/[^/]+/.+-([^-]+)-dark\.png`,
         );
         if (!m) {
             console.error(row.cardDark);
@@ -114,7 +113,6 @@ async function cleanup(): Promise<void> {
     validateIds('Cards', cards, 50_000);
 
     console.log('--- Cleaning up S3: Images ---');
-    const s3 = new S3(awsConfig);
     let overall = 0;
     let fresh = 0;
     let removed = 0;
@@ -130,7 +128,7 @@ async function cleanup(): Promise<void> {
             Prefix: 'images/',
             MaxKeys: chunkSize,
             ContinuationToken: continuationToken,
-            ...awsParams,
+            ...s3BucketParams,
         });
         if (!objects.Contents) {
             break;
@@ -183,7 +181,7 @@ async function cleanup(): Promise<void> {
                 Delete: {
                     Objects: toRemove,
                 },
-                ...awsParams,
+                ...s3BucketParams,
             });
         }
 
@@ -201,7 +199,7 @@ async function cleanup(): Promise<void> {
             Prefix: 'card/',
             MaxKeys: chunkSize,
             ContinuationToken: continuationToken,
-            ...awsParams,
+            ...s3BucketParams,
         });
         if (!objects.Contents) {
             break;
@@ -237,7 +235,7 @@ async function cleanup(): Promise<void> {
                 Delete: {
                     Objects: toRemove,
                 },
-                ...awsParams,
+                ...s3BucketParams,
             });
         }
 

server/cloudServices.ts

@@ -0,0 +1,41 @@
+import { S3 } from '@aws-sdk/client-s3';
+import type { S3ClientConfig } from '@aws-sdk/client-s3';
+
+export enum SupportedProviders {
+    AWS = 'aws',
+    CLOUDFLARE = 'cloudflare',
+    LOCAL = 'local',
+}
+
+export interface S3Config extends S3ClientConfig {
+    provider: SupportedProviders;
+    /**
+     * url under which the stored items are publicly reachable
+     */
+    publicAccess?: string;
+}
+
+export const s3Config: S3Config = {
+    provider: process.env.S3_PROVIDER as SupportedProviders || SupportedProviders.AWS,
+    region: process.env.S3_REGION || undefined,
+    credentials: {
+        accessKeyId: process.env.S3_KEY!,
+        secretAccessKey: process.env.S3_SECRET!,
+    },
+    endpoint: process.env.S3_ENDPOINT || undefined,
+    publicAccess: process.env.S3_PUBLIC_ACCESS,
+};
+
+export const s3BucketParams = {
+    Bucket: process.env.S3_BUCKET!,
+};
+
+export const s3 = new S3(s3Config);
+
+export const awsConfig = {
+    region: process.env.AWS_REGION,
+    credentials: {
+        accessKeyId: process.env.AWS_KEY!,
+        secretAccessKey: process.env.AWS_SECRET!,
+    },
+};

server/dotenv.ts

@@ -9,4 +9,4 @@ try {
     console.error(error);
 }
 
-process.env.NUXT_PUBLIC_CLOUDFRONT = `https://${process.env.AWS_CLOUDFRONT_ID}.cloudfront.net`;
+process.env.NUXT_PUBLIC_CLOUDFRONT = process.env.S3_PUBLIC_ACCESS;

server/express/images.ts

@@ -1,7 +1,6 @@
 import fs from 'fs';
 import path from 'path';
 
-import { S3 } from '@aws-sdk/client-s3';
 import { loadImage, createCanvas } from 'canvas';
 import type { Canvas } from 'canvas';
 import { Router } from 'express';
@@ -10,12 +9,12 @@ import sharp from 'sharp';
 import SQL from 'sql-template-strings';
 import { ulid } from 'ulid';
 
-import { handleErrorAsync } from '../../src/helpers.ts';
 import { auditLog } from '../audit.ts';
-import { awsConfig, awsParams } from '../aws.ts';
+import { s3, s3BucketParams } from '../cloudServices.ts';
 import { rootDir } from '../paths.ts';
 
 import { getLocale } from '~/server/data.ts';
+import { handleErrorAsync } from '~/src/helpers.ts';
 
 const sizes = {
     big: [1200, false],
@@ -67,8 +66,6 @@ router.post('/images/upload', handleErrorAsync(async (req, res) => {
         return res.status(400);
     }
 
-    const s3 = new S3(awsConfig);
-
     const requestedSizes = req.query.sizes === undefined || req.query.sizes === 'all'
         ? null
         : (req.query.sizes as string).split(',');
@@ -103,7 +100,7 @@ router.post('/images/upload', handleErrorAsync(async (req, res) => {
                 Body: canvas.toBuffer('image/png'),
                 ContentType: 'image/png',
                 ACL: 'public-read',
-                ...awsParams,
+                ...s3BucketParams,
             });
         }
 

server/express/profile.ts

@@ -2,7 +2,7 @@
 import fs from 'fs';
 import zlib from 'node:zlib';
 
-import { S3, NoSuchKey } from '@aws-sdk/client-s3';
+import { NoSuchKey } from '@aws-sdk/client-s3';
 import * as Sentry from '@sentry/node';
 import { Router } from 'express';
 import type { Request, Response } from 'express';
@@ -37,7 +37,7 @@ import { socialProviders } from '../../src/socialProviders.ts';
 import { colours, styles } from '../../src/styling.ts';
 import { auditLog } from '../audit.ts';
 import avatar from '../avatar.ts';
-import { awsConfig, awsParams } from '../aws.ts';
+import { s3, s3BucketParams } from '../cloudServices.ts';
 import crypto from '../crypto.ts';
 import type { Database } from '../db.ts';
 import { rootDir } from '../paths.ts';
@@ -1177,13 +1177,12 @@ router.get('/profile/export', handleErrorAsync(async (req: Request, res: Respons
         profiles[profile.locale] = exportProfile;
     }
 
-    const s3 = new S3(awsConfig);
     const images: ProfileExportData['images'] = {};
     for (const id of customFlagIds) {
         try {
             const data = await s3.getObject({
                 Key: `images/${id}-flag.png`,
-                ...awsParams,
+                ...s3BucketParams,
             });
             images[id] = {
                 flag: await data.Body!.transformToString('base64'),
@@ -1243,13 +1242,12 @@ router.post('/profile/import', handleErrorAsync(async (req, res) => {
 
     const { profiles, images } = JSON.parse(Buffer.from(payload, 'base64').toString('utf-8')) as ProfileExportData;
 
-    const s3 = new S3(awsConfig);
     for (const [id, sizes] of Object.entries(images)) {
         for (const [size, content] of Object.entries(sizes)) {
             try {
                 await s3.headObject({
                     Key: `images/${id}-${size}.png`,
-                    ...awsParams,
+                    ...s3BucketParams,
                 });
                 continue;
             } catch (error) {
@@ -1263,7 +1261,7 @@ router.post('/profile/import', handleErrorAsync(async (req, res) => {
                 Body: Buffer.from(content, 'base64'),
                 ContentType: 'image/png',
                 ACL: 'public-read',
-                ...awsParams,
+                ...s3BucketParams,
             });
         }
     }

server/express/pronounce.ts

@@ -1,20 +1,21 @@
 import { Polly } from '@aws-sdk/client-polly';
-import { S3, NoSuchKey } from '@aws-sdk/client-s3';
+import { NoSuchKey } from '@aws-sdk/client-s3';
+import type { S3 } from '@aws-sdk/client-s3';
 import type { NodeJsClient } from '@smithy/types';
 import { Router } from 'express';
 import { getH3Event } from 'h3-express';
 import { Base64 } from 'js-base64';
 import sha1 from 'sha1';
 
+import { s3, awsConfig, s3BucketParams } from '../cloudServices.ts';
+
 import type { PronunciationVoiceConfig,
     AwsPollyPronunciationVoiceConfig,
-    NarakeetPronunciationVoiceConfig } from '../../locale/config.ts';
+    NarakeetPronunciationVoiceConfig } from '~/locale/config.ts';
+import { getLocale, loadConfig } from '~/server/data.ts';
 import { convertPronunciationStringToSsml,
     convertPronunciationStringToNarakeetFormat,
-    handleErrorAsync } from '../../src/helpers.ts';
-import { awsConfig, awsParams } from '../aws.ts';
-
-import { getLocale, loadConfig } from '~/server/data.ts';
+    handleErrorAsync } from '~/src/helpers.ts';
 
 const router = Router();
 
@@ -91,14 +92,12 @@ router.get('/pronounce/:voice/:pronunciation', handleErrorAsync(async (req, res)
         return res.status(404).json({ error: 'Not found' });
     }
 
-    const s3 = new S3(awsConfig) as NodeJsClient<S3>;
-
     const provider = providers[(voice.provider || 'aws_polly') as ProviderKey];
     const tokenised = provider.tokenised(text);
     const key = `pronunciation/${config.locale}-${req.params.voice}/${sha1(tokenised)}.mp3`;
 
     try {
-        const s3Response = await s3.getObject({ Key: key, ...awsParams });
+        const s3Response = await (s3 as NodeJsClient<S3>).getObject({ Key: key, ...s3BucketParams });
         res.set('content-type', s3Response.ContentType);
         return s3Response.Body!.pipe(res);
     } catch (error) {
@@ -112,7 +111,7 @@ router.get('/pronounce/:voice/:pronunciation', handleErrorAsync(async (req, res)
             Key: key,
             Body: buffer,
             ContentType: contentType,
-            ...awsParams,
+            ...s3BucketParams,
         });
 
         res.set('content-type', contentType);

server/imageCopy.ts

@@ -1,11 +1,8 @@
-import { S3 } from '@aws-sdk/client-s3';
 import { loadImage, createCanvas } from 'canvas';
 
-import { awsConfig, awsParams } from '~/server/aws.ts';
+import { s3, s3BucketParams } from '~/server/cloudServices.ts';
 import { newDate, sha256 } from '~/src/helpers.ts';
 
-const s3 = new S3(awsConfig);
-
 const convertToPng = async (original: Buffer): Promise<Buffer> => {
     const image = await loadImage(original);
     const canvas = createCanvas(image.width, image.height);
@@ -24,7 +21,7 @@ export default async (prefix: string, url: string, ttlDays: number | null = null
     const key = `${prefix}/${await sha256(url)}.${isSvg ? 'svg' : 'png'}`;
 
     try {
-        const metadata = await s3.headObject({ Key: key, ...awsParams });
+        const metadata = await s3.headObject({ Key: key, ...s3BucketParams });
 
         if (
             ttlDays !== null &&
@@ -49,7 +46,7 @@ export default async (prefix: string, url: string, ttlDays: number | null = null
                     ? 'image/svg+xml'
                     : 'image/png',
                 ACL: 'public-read',
-                ...awsParams,
+                ...s3BucketParams,
             });
 
             return `${process.env.NUXT_PUBLIC_CLOUDFRONT}/${key}`;