From c7910e938eb2bb51f3679425d0798f6e0de077d5 Mon Sep 17 00:00:00 2001 From: Valentyne Stigloher Date: Mon, 17 Mar 2025 13:00:33 +0100 Subject: [PATCH 1/8] (api) use filesystem-based storage instead of memory storage for sessions the implementation is incomplete, e.g. it misses pruning old sessions, this is more meant as proof-of-concept this should reenable the usage of multiple workers --- package.json | 1 - pnpm-lock.yaml | 32 -------------------------------- server/index.ts | 26 +++++++++++++++++++++----- 3 files changed, 21 insertions(+), 38 deletions(-) diff --git a/package.json b/package.json index 2c3c4fabf..f3ef6953d 100644 --- a/package.json +++ b/package.json @@ -112,7 +112,6 @@ "markdown-it-sub": "^2.0.0", "markdown-it-sup": "^2.0.0", "marked": "^0.7.0", - "memorystore": "^1.6.7", "nuxt": "^3.16.0", "path-to-regexp": "0.1.12", "postcss-rtl": "^2.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 19d1453f2..abae874ec 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -297,9 +297,6 @@ importers: marked: specifier: ^0.7.0 version: 0.7.0 - memorystore: - specifier: ^1.6.7 - version: 1.6.7 nuxt: specifier: ^3.16.0 version: 3.16.0(@parcel/watcher@2.5.1)(@types/node@20.16.5)(db0@0.3.1(sqlite3@5.1.7))(encoding@0.1.13)(eslint@9.22.0(jiti@2.4.2))(ioredis@5.6.0)(magicast@0.3.5)(optionator@0.9.4)(rollup@4.35.0)(sass@1.32.12)(sqlite3@5.1.7)(terser@5.33.0)(typescript@5.7.2)(vite@6.2.2(@types/node@20.16.5)(jiti@2.4.2)(sass@1.32.12)(terser@5.33.0)(yaml@2.7.0))(vue-tsc@2.2.0(typescript@5.7.2))(yaml@2.7.0) @@ -5403,9 +5400,6 @@ packages: lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} - lru-cache@4.1.5: - resolution: {integrity: sha512-sWZlbEP2OsHNkXrMl5GYk/jKk70MBng6UU4YI/qGDYbgf6YbP4EvmqISbXCoJiRKs+1bSpFHVgQxvJ17F2li5g==} - lru-cache@5.1.1: resolution: {integrity: sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==} @@ -5503,10 +5497,6 @@ packages: resolution: {integrity: sha512-qX2bG48pTqYRVmDB37rn/6PT7LcR8T7oAX3bf99u1Tt1nzxYfxkgqDwUwolPlXweM0XzBOBFzSx4kfp7KP1s/w==} engines: {node: '>=6'} - memorystore@1.6.7: - resolution: {integrity: sha512-OZnmNY/NDrKohPQ+hxp0muBcBKrzKNtHr55DbqSx9hLsYVNnomSAMRAtI7R64t3gf3ID7tHQA7mG4oL3Hu9hdw==} - engines: {node: '>=0.10'} - meow@3.7.0: resolution: {integrity: sha512-TNdwZs0skRlpPpCUK25StC4VH+tP5GgeY1HQOOGP+lQ2xtdkN2VtT/5tiX9k3IWpkBPV9b3LsAWXn4GGi/PrSA==} engines: {node: '>=0.10.0'} @@ -6451,9 +6441,6 @@ packages: proxy-from-env@1.1.0: resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} - pseudomap@1.0.2: - resolution: {integrity: sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ==} - psl@1.9.0: resolution: {integrity: sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag==} @@ -8098,9 +8085,6 @@ packages: resolution: {integrity: sha512-0pfFzegeDWJHJIAmTLRP2DwHjdF5s7jo9tuztdQxAhINCdvS+3nGINqPd00AphqJR/0LhANUS6/+7SCb98YOfA==} engines: {node: '>=10'} - yallist@2.1.2: - resolution: {integrity: sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A==} - yallist@3.1.1: resolution: {integrity: sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==} @@ -14535,11 +14519,6 @@ snapshots: lru-cache@10.4.3: {} - lru-cache@4.1.5: - dependencies: - pseudomap: 1.0.2 - yallist: 2.1.2 - lru-cache@5.1.1: dependencies: yallist: 3.1.1 @@ -14651,13 +14630,6 @@ snapshots: mimic-fn: 2.1.0 p-is-promise: 2.1.0 - memorystore@1.6.7: - dependencies: - debug: 4.4.0(supports-color@9.4.0) - lru-cache: 4.1.5 - transitivePeerDependencies: - - supports-color - meow@3.7.0: dependencies: camelcase-keys: 2.1.0 @@ -15779,8 +15751,6 @@ snapshots: proxy-from-env@1.1.0: {} - pseudomap@1.0.2: {} - psl@1.9.0: {} pump@3.0.2: @@ -17658,8 +17628,6 @@ snapshots: y18n@5.0.8: {} - yallist@2.1.2: {} - yallist@3.1.1: {} yallist@4.0.0: {} diff --git a/server/index.ts b/server/index.ts index 69edef19e..bf23c0b65 100644 --- a/server/index.ts +++ b/server/index.ts @@ -6,7 +6,6 @@ import session from 'express-session'; import grant from 'grant'; import { useBase } from 'h3'; import { defineExpressHandler, getH3Event } from 'h3-express'; -import memorystore from 'memorystore'; import SQL from 'sql-template-strings'; import buildLocaleList from '../src/buildLocaleList.ts'; @@ -43,7 +42,26 @@ import { config } from './social.ts'; import { closeAuditLogConnection } from '~/server/audit.ts'; import { getLocale } from '~/server/data.ts'; -const MemoryStore = memorystore(session); +class StorageStore extends session.Store { + get(sid: string, callback: (err: unknown, session?: (session.SessionData | null)) => void): void { + useStorage('data').getItem(`session:${sid}`) + .then((session) => callback(null, session)) + .catch((error) => callback(error)); + } + + set(sid: string, session: session.SessionData, callback?: (err?: unknown) => void): void { + // unwrap session to make it a primitive object (otherwise unstorage will reject serializing the object) + useStorage('data').setItem(`session:${sid}`, { ...session }) + .then(() => callback?.()) + .catch((error) => callback?.(error)); + } + + destroy(sid: string, callback?: (err?: unknown) => void): void { + useStorage('data').removeItem(`session:${sid}`) + .then(() => callback?.()) + .catch((error) => callback?.(error)); + } +} const router = express.Router(); @@ -52,9 +70,7 @@ router.use(session({ cookie: { ...longtimeCookieSetting, sameSite: undefined }, // somehow, sameSite=lax breaks sign-in with apple 🙄 resave: false, saveUninitialized: false, - store: new MemoryStore({ - checkPeriod: 86400000, // 24h - }), + store: new StorageStore(), })); export class LazyDatabase implements Database { From 338b26ff6706e8e13ba03d468695e545d70c0c10 Mon Sep 17 00:00:00 2001 From: Valentyne Stigloher Date: Mon, 17 Mar 2025 13:01:53 +0100 Subject: [PATCH 2/8] (fmt) --- package.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index f3ef6953d..400144959 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,9 @@ "version": "1.0.0", "private": true, "type": "module", - "sideEffects": ["./server/dotenv.ts"], + "sideEffects": [ + "./server/dotenv.ts" + ], "scripts": { "prepare-dev": "nuxi prepare && pnpm run-file locale/generateSchemas.ts", "dev": "nuxi dev", From f6ba9ff4ae9394661513328fa3eb5fba3d1d35c2 Mon Sep 17 00:00:00 2001 From: Valentyne Stigloher Date: Mon, 17 Mar 2025 13:02:19 +0100 Subject: [PATCH 3/8] (nuxt) reenable node-cluster preset --- nuxt.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nuxt.config.ts b/nuxt.config.ts index 889d5679e..cec5af8a6 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -113,7 +113,7 @@ export default defineNuxtConfig({ esbuild: { options: esBuildOptions, }, - // preset: 'node-cluster', + preset: 'node-cluster', }, serverHandlers: [ { From 546326fe6ae53883c105f04efaa98a4e22993c5e Mon Sep 17 00:00:00 2001 From: Valentyne Stigloher Date: Wed, 19 Mar 2025 13:38:17 +0100 Subject: [PATCH 4/8] (nuxt) use redis to store sessions --- README.md | 15 +++++++++++++++ docker/docker-compose.yml | 5 +++++ nuxt.config.ts | 12 ++++++++++++ package.json | 1 + pnpm-lock.yaml | 3 +++ server/index.ts | 6 +++--- 6 files changed, 39 insertions(+), 3 deletions(-) create mode 100644 docker/docker-compose.yml diff --git a/README.md b/README.md index c2f029dbb..3e98c3603 100644 --- a/README.md +++ b/README.md @@ -228,6 +228,21 @@ A non-comprehensive list of things to look out during development and review: ### Integrations +#### Redis + +Redis is used to store sessions. To run it locally, start the container in `docker/docker-compose.yml': + +```bash +$ cd docker +$ docker compose up +``` + +If you need to inspect redis via `redis-cli` you can use: + +```bash +$ docker compose run --interactive --tty --rm redis redis-cli -h redis +``` + #### AWS If you want to test with AWS, you can fill in all the `AWS_*` values with your own credentials in `.env.dist`. diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 000000000..ec6894371 --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,5 @@ +services: + redis: + image: 'redis:alpine' + ports: + - '6379:6379' diff --git a/nuxt.config.ts b/nuxt.config.ts index cec5af8a6..1390bee52 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -114,6 +114,18 @@ export default defineNuxtConfig({ options: esBuildOptions, }, preset: 'node-cluster', + storage: { + session: { + driver: 'redis', + base: 'session', + }, + }, + devStorage: { + session: { + driver: 'fs', + base: './.data/session', + }, + }, }, serverHandlers: [ { diff --git a/package.json b/package.json index 400144959..287c2c5cb 100644 --- a/package.json +++ b/package.json @@ -109,6 +109,7 @@ "globals": "^13.24.0", "h3-express": "https://github.com/pixunil/h3-express#built", "html-validate": "^9.2.2", + "ioredis": "^5.6.0", "markdown-it": "^14.0.0", "markdown-it-mark": "^4.0.0", "markdown-it-sub": "^2.0.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index abae874ec..68011874d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -282,6 +282,9 @@ importers: html-validate: specifier: ^9.2.2 version: 9.3.0(vitest@3.0.8(@types/node@20.16.5)(jiti@2.4.2)(jsdom@24.1.3(canvas@3.1.0))(sass@1.32.12)(terser@5.33.0)(yaml@2.7.0)) + ioredis: + specifier: ^5.6.0 + version: 5.6.0 markdown-it: specifier: ^14.0.0 version: 14.1.0 diff --git a/server/index.ts b/server/index.ts index bf23c0b65..3c8eb6d2d 100644 --- a/server/index.ts +++ b/server/index.ts @@ -44,20 +44,20 @@ import { getLocale } from '~/server/data.ts'; class StorageStore extends session.Store { get(sid: string, callback: (err: unknown, session?: (session.SessionData | null)) => void): void { - useStorage('data').getItem(`session:${sid}`) + useStorage('session').getItem(sid) .then((session) => callback(null, session)) .catch((error) => callback(error)); } set(sid: string, session: session.SessionData, callback?: (err?: unknown) => void): void { // unwrap session to make it a primitive object (otherwise unstorage will reject serializing the object) - useStorage('data').setItem(`session:${sid}`, { ...session }) + useStorage('session').setItem(sid, { ...session }, { ttl: 24 * 60 * 60 }) .then(() => callback?.()) .catch((error) => callback?.(error)); } destroy(sid: string, callback?: (err?: unknown) => void): void { - useStorage('data').removeItem(`session:${sid}`) + useStorage('session').removeItem(sid) .then(() => callback?.()) .catch((error) => callback?.(error)); } From b34fe7ea69e4f9ce8a16929061fd07aad9bb981f Mon Sep 17 00:00:00 2001 From: Valentyne Stigloher Date: Wed, 19 Mar 2025 14:27:23 +0100 Subject: [PATCH 5/8] (nuxt) use redis to store cache --- README.md | 9 +++++++-- nuxt.config.ts | 4 ++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 3e98c3603..728a7fbaf 100644 --- a/README.md +++ b/README.md @@ -230,7 +230,8 @@ A non-comprehensive list of things to look out during development and review: #### Redis -Redis is used to store sessions. To run it locally, start the container in `docker/docker-compose.yml': +Redis is used to store sessions and caches in production. +To run it locally, start the container in `docker/docker-compose.yml': ```bash $ cd docker @@ -270,7 +271,11 @@ Due to how [caches in Nitro](https://nitro.build/guide/cache) work, they automat when their function changes. However, this heuristic does not cover other functions the cached function depends on. You can manually clear caches: ```bash -rm -rf .nuxt/cache +# when using the filesystem +$ rm -rf .nuxt/cache +# when using redis +$ cd docker +$ docker compose exec redis sh -c "redis-cli KEYS \"cache:*\" | xargs -n 100 redis-cli DEL" ``` #### Module did not self-register diff --git a/nuxt.config.ts b/nuxt.config.ts index 1390bee52..ff8c013df 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -115,6 +115,10 @@ export default defineNuxtConfig({ }, preset: 'node-cluster', storage: { + cache: { + driver: 'redis', + base: 'cache', + }, session: { driver: 'redis', base: 'session', From 8db436dab10fc18f934d4c2958b3cfb96ca17dfa Mon Sep 17 00:00:00 2001 From: Valentyne Stigloher Date: Wed, 19 Mar 2025 14:29:40 +0100 Subject: [PATCH 6/8] (nuxt) move storage config from nitro plugin into nuxt.config.ts because the workaround should be unnecessary after newest Nitro release --- nuxt.config.ts | 4 ++++ server/plugins/storage.ts | 9 --------- 2 files changed, 4 insertions(+), 9 deletions(-) delete mode 100644 server/plugins/storage.ts diff --git a/nuxt.config.ts b/nuxt.config.ts index ff8c013df..2f1dbf89c 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -123,6 +123,10 @@ export default defineNuxtConfig({ driver: 'redis', base: 'session', }, + calendar: { + driver: 'fs-lite', + base: 'calendar', + }, }, devStorage: { session: { diff --git a/server/plugins/storage.ts b/server/plugins/storage.ts deleted file mode 100644 index 4b0bc6a3a..000000000 --- a/server/plugins/storage.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { defineNitroPlugin, useStorage } from 'nitropack/runtime'; -import fsLiteDriver from 'unstorage/drivers/fs-lite'; - -export default defineNitroPlugin(async () => { - useStorage().mount('calendar', fsLiteDriver({ base: 'calendar' })); - // should be resolved when Nitro releases a nev version https://github.com/nitrojs/nitro/issues/3017 - await useStorage().unmount('data'); - useStorage().mount('data', fsLiteDriver({ base: '.data/kv' })); -}); From 95c2ca0b7ca73432065ba5ec5b79abcd0c8f1889 Mon Sep 17 00:00:00 2001 From: Valentyne Stigloher Date: Thu, 20 Mar 2025 10:14:36 +0100 Subject: [PATCH 7/8] (nuxt) add env runtime config to redis keys to ensure that production and test environments are separated in redis --- nuxt.config.ts | 8 -------- server/plugins/storage.ts | 15 +++++++++++++++ 2 files changed, 15 insertions(+), 8 deletions(-) create mode 100644 server/plugins/storage.ts diff --git a/nuxt.config.ts b/nuxt.config.ts index 2f1dbf89c..91f50c2c6 100644 --- a/nuxt.config.ts +++ b/nuxt.config.ts @@ -115,14 +115,6 @@ export default defineNuxtConfig({ }, preset: 'node-cluster', storage: { - cache: { - driver: 'redis', - base: 'cache', - }, - session: { - driver: 'redis', - base: 'session', - }, calendar: { driver: 'fs-lite', base: 'calendar', diff --git a/server/plugins/storage.ts b/server/plugins/storage.ts new file mode 100644 index 000000000..f687f56b4 --- /dev/null +++ b/server/plugins/storage.ts @@ -0,0 +1,15 @@ +import redisDriver from 'unstorage/drivers/redis'; + +export default defineNitroPlugin(() => { + const runtimeConfig = useRuntimeConfig(); + const storage = useStorage(); + + if (!import.meta.dev) { + storage.mount('cache', redisDriver({ + base: `${runtimeConfig.public.env}:cache`, + })); + storage.mount('session', redisDriver({ + base: `${runtimeConfig.public.env}:session`, + })); + } +}); From 366e1da0d5dff08db4d8103e46befda5b0db134c Mon Sep 17 00:00:00 2001 From: Valentyne Stigloher Date: Thu, 20 Mar 2025 10:19:05 +0100 Subject: [PATCH 8/8] (admin) hide detais contents in storage view until it is opened to shrink DOM size and improve performance --- components/admin/AdminStorageItem.vue | 3 +-- components/admin/AdminStorageTree.vue | 6 ++++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/components/admin/AdminStorageItem.vue b/components/admin/AdminStorageItem.vue index 9c65d2a0c..af5992ab6 100644 --- a/components/admin/AdminStorageItem.vue +++ b/components/admin/AdminStorageItem.vue @@ -54,8 +54,7 @@ const remove = async () => { - -
+