Merge branch 'sessions-outside-memory-p' into 'main'

use filesystem-based storage instead of memory storage for sessions

See merge request PronounsPage/PronounsPage!594

README.md

@@ -228,6 +228,22 @@ A non-comprehensive list of things to look out during development and review:
 
 ### Integrations
 
+#### Redis
+
+Redis is used to store sessions and caches in production.
+To run it locally, start the container in `docker/docker-compose.yml':
+
+```bash
+$ cd docker
+$ docker compose up
+```
+
+If you need to inspect redis via `redis-cli` you can use:
+
+```bash
+$ docker compose run --interactive --tty --rm redis redis-cli -h redis
+```
+
 #### AWS
 
 If you want to test with AWS, you can fill in all the `AWS_*` values with your own credentials in `.env.dist`.
@@ -255,7 +271,11 @@ Due to how [caches in Nitro](https://nitro.build/guide/cache) work, they automat
 when their function changes. However, this heuristic does not cover other functions the cached function depends on.
 You can manually clear caches:
 ```bash
-rm -rf .nuxt/cache
+# when using the filesystem
+$ rm -rf .nuxt/cache
+# when using redis
+$ cd docker
+$ docker compose exec redis sh -c "redis-cli KEYS \"cache:*\" | xargs -n 100 redis-cli DEL"
 ```
 
 #### Module did not self-register

components/admin/AdminStorageItem.vue

@@ -54,8 +54,7 @@ const remove = async () => {
                 </span>
             </span>
         </summary>
-
+        <div v-if="isOpen" class="ps-3">
-        <div class="ps-3">
             <template v-if="contentAsyncData.status.value === 'success'">
                 <img
                     v-if="itemKey.endsWith('.png')"

components/admin/AdminStorageTree.vue

@@ -10,6 +10,8 @@ const props = defineProps<{
 
 const chunk = computed(() => props.itemKey.replace(/^.+:/, ''));
 
+const isOpen = ref(false);
+
 const sumSize = (tree: Tree): number => {
     return [...tree.values()].map((node) => {
         if (node instanceof Map) {
@@ -22,7 +24,7 @@ const sumSize = (tree: Tree): number => {
 </script>
 
 <template>
-    <details>
+    <details @toggle="isOpen = $event.newState === 'open'">
         <summary>
             <span class="py-1 d-inline-flex justify-content-between">
                 <code>{{ chunk }}</code>
@@ -31,7 +33,7 @@ const sumSize = (tree: Tree): number => {
                 </span>
             </span>
         </summary>
-        <div class="ps-2 border-start border-secondary">
+        <div v-if="isOpen" class="ps-2 border-start border-secondary">
             <template v-for="[childChunk, childNode] of tree.entries()" :key="childChunk">
                 <AdminStorageTree
                     v-if="childNode instanceof Map"

docker/docker-compose.yml

@@ -0,0 +1,5 @@
+services:
+  redis:
+    image: 'redis:alpine'
+    ports:
+      - '6379:6379'

nuxt.config.ts

@@ -113,7 +113,19 @@ export default defineNuxtConfig({
         esbuild: {
             options: esBuildOptions,
         },
-        // preset: 'node-cluster',
+        preset: 'node-cluster',
+        storage: {
+            calendar: {
+                driver: 'fs-lite',
+                base: 'calendar',
+            },
+        },
+        devStorage: {
+            session: {
+                driver: 'fs',
+                base: './.data/session',
+            },
+        },
     },
     serverHandlers: [
         {

package.json

@@ -3,7 +3,9 @@
     "version": "1.0.0",
     "private": true,
     "type": "module",
-    "sideEffects": ["./server/dotenv.ts"],
+    "sideEffects": [
+        "./server/dotenv.ts"
+    ],
     "scripts": {
         "prepare-dev": "nuxi prepare && pnpm run-file locale/generateSchemas.ts",
         "dev": "nuxi dev",
@@ -107,12 +109,12 @@
         "globals": "^13.24.0",
         "h3-express": "https://github.com/pixunil/h3-express#built",
         "html-validate": "^9.2.2",
+        "ioredis": "^5.6.0",
         "markdown-it": "^14.0.0",
         "markdown-it-mark": "^4.0.0",
         "markdown-it-sub": "^2.0.0",
         "markdown-it-sup": "^2.0.0",
         "marked": "^0.7.0",
-        "memorystore": "^1.6.7",
         "nuxt": "^3.16.0",
         "path-to-regexp": "0.1.12",
         "postcss-rtl": "^2.0.0",

pnpm-lock.yaml

@@ -282,6 +282,9 @@ importers:
       html-validate:
         specifier: ^9.2.2
         version: 9.3.0(vitest@3.0.8(@types/node@20.16.5)(jiti@2.4.2)(jsdom@24.1.3(canvas@3.1.0))(sass@1.32.12)(terser@5.33.0)(yaml@2.7.0))
+      ioredis:
+        specifier: ^5.6.0
+        version: 5.6.0
       markdown-it:
         specifier: ^14.0.0
         version: 14.1.0
@@ -297,9 +300,6 @@ importers:
       marked:
         specifier: ^0.7.0
         version: 0.7.0
-      memorystore:
-        specifier: ^1.6.7
-        version: 1.6.7
       nuxt:
         specifier: ^3.16.0
         version: 3.16.0(@parcel/watcher@2.5.1)(@types/node@20.16.5)(db0@0.3.1(sqlite3@5.1.7))(encoding@0.1.13)(eslint@9.22.0(jiti@2.4.2))(ioredis@5.6.0)(magicast@0.3.5)(optionator@0.9.4)(rollup@4.35.0)(sass@1.32.12)(sqlite3@5.1.7)(terser@5.33.0)(typescript@5.7.2)(vite@6.2.2(@types/node@20.16.5)(jiti@2.4.2)(sass@1.32.12)(terser@5.33.0)(yaml@2.7.0))(vue-tsc@2.2.0(typescript@5.7.2))(yaml@2.7.0)
@@ -5403,9 +5403,6 @@ packages:
   lru-cache@10.4.3:
     resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==}
 
-  lru-cache@4.1.5:
-    resolution: {integrity: sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==}
-
   lru-cache@5.1.1:
     resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==}
 
@@ -5503,10 +5500,6 @@ packages:
     resolution: {integrity: sha512-qX2bG48pTqYRVmDB37rn/6PT7LcR8T7oAX3bf99u1Tt1nzxYfxkgqDwUwolPlXweM0XzBOBFzSx4kfp7KP1s/w==}
     engines: {node: '>=6'}
 
-  memorystore@1.6.7:
-    resolution: {integrity: sha512-OZnmNY/NDrKohPQ+hxp0muBcBKrzKNtHr55DbqSx9hLsYVNnomSAMRAtI7R64t3gf3ID7tHQA7mG4oL3Hu9hdw==}
-    engines: {node: '>=0.10'}
-
   meow@3.7.0:
     resolution: {integrity: sha512-TNdwZs0skRlpPpCUK25StC4VH+tP5GgeY1HQOOGP+lQ2xtdkN2VtT/5tiX9k3IWpkBPV9b3LsAWXn4GGi/PrSA==}
     engines: {node: '>=0.10.0'}
@@ -6451,9 +6444,6 @@ packages:
   proxy-from-env@1.1.0:
     resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==}
 
-  pseudomap@1.0.2:
-    resolution: {integrity: sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==}
-
   psl@1.9.0:
     resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==}
 
@@ -8098,9 +8088,6 @@ packages:
     resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==}
     engines: {node: '>=10'}
 
-  yallist@2.1.2:
-    resolution: {integrity: sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==}
-
   yallist@3.1.1:
     resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==}
 
@@ -14535,11 +14522,6 @@ snapshots:
 
   lru-cache@10.4.3: {}
 
-  lru-cache@4.1.5:
-    dependencies:
-      pseudomap: 1.0.2
-      yallist: 2.1.2
-
   lru-cache@5.1.1:
     dependencies:
       yallist: 3.1.1
@@ -14651,13 +14633,6 @@ snapshots:
       mimic-fn: 2.1.0
       p-is-promise: 2.1.0
 
-  memorystore@1.6.7:
-    dependencies:
-      debug: 4.4.0(supports-color@9.4.0)
-      lru-cache: 4.1.5
-    transitivePeerDependencies:
-      - supports-color
-
   meow@3.7.0:
     dependencies:
       camelcase-keys: 2.1.0
@@ -15779,8 +15754,6 @@ snapshots:
 
   proxy-from-env@1.1.0: {}
 
-  pseudomap@1.0.2: {}
-
   psl@1.9.0: {}
 
   pump@3.0.2:
@@ -17658,8 +17631,6 @@ snapshots:
 
   y18n@5.0.8: {}
 
-  yallist@2.1.2: {}
-
   yallist@3.1.1: {}
 
   yallist@4.0.0: {}

server/index.ts

@@ -6,7 +6,6 @@ import session from 'express-session';
 import grant from 'grant';
 import { useBase } from 'h3';
 import { defineExpressHandler, getH3Event } from 'h3-express';
-import memorystore from 'memorystore';
 import SQL from 'sql-template-strings';
 
 import buildLocaleList from '../src/buildLocaleList.ts';
@@ -43,7 +42,26 @@ import { config } from './social.ts';
 import { closeAuditLogConnection } from '~/server/audit.ts';
 import { getLocale } from '~/server/data.ts';
 
-const MemoryStore = memorystore(session);
+class StorageStore extends session.Store {
+    get(sid: string, callback: (err: unknown, session?: (session.SessionData | null)) => void): void {
+        useStorage('session').getItem<session.SessionData>(sid)
+            .then((session) => callback(null, session))
+            .catch((error) => callback(error));
+    }
+
+    set(sid: string, session: session.SessionData, callback?: (err?: unknown) => void): void {
+        // unwrap session to make it a primitive object (otherwise unstorage will reject serializing the object)
+        useStorage('session').setItem(sid, { ...session }, { ttl: 24 * 60 * 60 })
+            .then(() => callback?.())
+            .catch((error) => callback?.(error));
+    }
+
+    destroy(sid: string, callback?: (err?: unknown) => void): void {
+        useStorage('session').removeItem(sid)
+            .then(() => callback?.())
+            .catch((error) => callback?.(error));
+    }
+}
 
 const router = express.Router();
 
@@ -52,9 +70,7 @@ router.use(session({
     cookie: { ...longtimeCookieSetting, sameSite: undefined }, // somehow, sameSite=lax breaks sign-in with apple 🙄
     resave: false,
     saveUninitialized: false,
-    store: new MemoryStore({
+    store: new StorageStore(),
-        checkPeriod: 86400000, // 24h
-    }),
 }));
 
 export class LazyDatabase implements Database {

server/plugins/storage.ts

@@ -1,9 +1,15 @@
-import { defineNitroPlugin, useStorage } from 'nitropack/runtime';
+import redisDriver from 'unstorage/drivers/redis';
-import fsLiteDriver from 'unstorage/drivers/fs-lite';
 
-export default defineNitroPlugin(async () => {
+export default defineNitroPlugin(() => {
-    useStorage().mount('calendar', fsLiteDriver({ base: 'calendar' }));
+    const runtimeConfig = useRuntimeConfig();
-    // should be resolved when Nitro releases a nev version https://github.com/nitrojs/nitro/issues/3017
+    const storage = useStorage();
-    await useStorage().unmount('data');
+
-    useStorage().mount('data', fsLiteDriver({ base: '.data/kv' }));
+    if (!import.meta.dev) {
+        storage.mount('cache', redisDriver({
+            base: `${runtimeConfig.public.env}:cache`,
+        }));
+        storage.mount('session', redisDriver({
+            base: `${runtimeConfig.public.env}:session`,
+        }));
+    }
 });