Merge branch 'rewrite-fastify' into 'main'

Merge rewrite code into main branch

See merge request PronounsPage/PronounsPage!372
This commit is contained in:
tecc 2023-09-19 14:14:56 +00:00
commit a4e89e907e
32 changed files with 4116 additions and 0 deletions

5
new/.eslintrc.json Normal file
View File

@ -0,0 +1,5 @@
{
"root": true,
"extends": ["prettier", "plugin:@typescript-eslint/recommended"],
"parser": "@typescript-eslint/parser"
}

16
new/.gitignore vendored Normal file
View File

@ -0,0 +1,16 @@
# This may repeat some stuff in the top-level gitignore.
# Any such case is intentional.
# Node
node_modules/
dist/
*.tsbuildinfo
# Project-specific
.env
/locales
# Editors
.idea/
*.iml
.vscode/

2
new/.npmrc Normal file
View File

@ -0,0 +1,2 @@
shamefully-hoist=true
link-workspace-packages=false

4
new/.prettierignore Normal file
View File

@ -0,0 +1,4 @@
node_modules/
dist/
pnpm-lock.yaml

7
new/.prettierrc Normal file
View File

@ -0,0 +1,7 @@
{
"trailingComma": "es5",
"tabWidth": 4,
"useTabs": false,
"semi": true,
"singleQuote": false
}

50
new/README.md Normal file
View File

@ -0,0 +1,50 @@
# `/new`
This is the working directory for the Pronouns.page rewrite.
## Setup instructions
> Note that this section will assume you are using [pnpm](https://pnpm.io).
1. Install dependencies using `pnpm install`.
2. Run the `setup.mjs` script using `node ./setup.mjs`.
If you are using a package manager *other* than pnpm, you should specify that as an argument (`node ./setup.mjs yarn`).
This is not recommended, however.
3. Build the `common` package (`pnpm build`). This step has to be done every time you change it.
4. The next step depends _entirely_ on what part of the project you're working on:
1. For the backend: Run `pnpm dev`. This will automatically build and restart the server when you change the code.
It's actually quite fast.
## Development environment
### `setup.mjs`
Run `setup.mjs` for a quick setup. It'll symlink some directories and will work as the equivalent of `make switch` et al.
The reason for making this a script is that make was causing problems for Windows users.
This script requires a package manager to run, but does not have any non-builtin dependencies.
If you are using a package manager other than `pnpm`, you should specify it as the first argument to the script.
```
node setup.mjs [pnpm/npm/yarn/bun]
```
### Package managers
pnpm should work without any prior setup.
I haven't tested yarn and npm here, and I don't think I will either.
Just use pnpm for as little friction as possible.
### Style and lints
Prettier and ESLint are used for formatting and linting respectively.
You can run `pnpm format` from the top-level directory of the rewrite to format all the code easily.
### Other notes
Currently, the packages here are using `"type": "module"`. This is mostly experimental and can be changed if it turns out to be annoying/code-breaking.
## Process and collaboration
Most decisions should be discussed in the team Discord.

8
new/backend/.gitignore vendored Normal file
View File

@ -0,0 +1,8 @@
node_modules/
# Keep environment variables out of version control
.env
# Remove the database file
db.sqlite
# ...oh, and Prisma's dist directory for safety
/prisma/dist

30
new/backend/example.env Normal file
View File

@ -0,0 +1,30 @@
# -- Example environment file, provided for convenience.
# -- It is recommended that you read through it.
# -- Specifies the environment. If this variable is not provided, it defaults to "development".
# -- Allowed values:
# "development", "dev": Development environment
# "production", "prod": Production environment
ENVIRONMENT=development
# -- By default, ENVIRONMENT dictates the log level.
# -- However, the log level can be set manually through LOG_LEVEL.
# -- See pino's log levels. They're conventional. This value is also case-insensitive.
# LOG_LEVEL=
# -- Whether or not to allow API calls for unpublished (e.g. not-yet-ready) locales
ALLOW_UNPUBLISHED_LOCALES=true
# -- This base URL only refers to the backend.
# -- This also means we don't need HOME_URL, since the API doesn't have a homepage per se.
HTTP_BASE_URL=http://localhost:4000
# -- These two variables configure the bind address. It defaults to 0.0.0.0:4000.
# HTTP_HOST=0.0.0.0
# HTTP_PORT=4000
# -- This variable determines the database to connect to.
# -- Note that this has to be of the same database type as specified in the Prisma schema.
# -- Right now it's SQLite since we already use it.
DATABASE_URL=file:./db.sqlite
# -- Do change this variable, lest you wish to anger the computer.
SECURITY_SECRET=changeme

3
new/backend/nodemon.json Normal file
View File

@ -0,0 +1,3 @@
{
"watch": ["dist/**", "package.json", "tsconfig.json"]
}

49
new/backend/package.json Normal file
View File

@ -0,0 +1,49 @@
{
"name": "@pronounspage/backend",
"version": "1.0.0",
"description": "",
"private": true,
"type": "module",
"main": "dist/index.js",
"scripts": {
"format": "prettier -w .",
"lint": "eslint src/",
"clean": "rimraf ./dist ./tsconfig.tsbuildinfo",
"build:server": "tsc",
"build": "run-p build:*",
"watch:server": "tsc --watch",
"watch": "run-p watch:*",
"cleanbuild": "run-s clean build",
"start": "node dist/index.js",
"start:fresh": "run-s build start",
"start:watch": "run-p build:watch",
"dev:start": "nodemon dist/index.js",
"dev": "run-p watch dev:start",
"db:generate": "prisma generate"
},
"keywords": [],
"author": "",
"dependencies": {
".prisma/client": "link:prisma/dist",
"@fastify/type-provider-typebox": "^3.5.0",
"@prisma/client": "5.3.1",
"@pronounspage/common": "workspace:*",
"@sinclair/typebox": "^0.31.14",
"csv-parse": "^5.5.0",
"dotenv": "^16.3.1",
"fastify": "^4.21.0",
"pino": "^8.15.0"
},
"devDependencies": {
"@types/node": "^20.5.6",
"nodemon": "^3.0.1",
"npm-run-all": "^4.1.5",
"pino-pretty": "^10.2.0",
"prisma": "^5.3.1",
"rimraf": "^5.0.1",
"typescript": "^5.2.2"
},
"imports": {
"#self/*": "./dist/*.js"
}
}

View File

@ -0,0 +1,334 @@
generator client {
provider = "prisma-client-js"
output = "dist"
}
datasource db {
provider = "sqlite"
url = env("DATABASE_URL")
}
model OldAuditLog {
id String @id
userId String?
username String?
event String
payload String?
@@map("audit_log")
}
model NewAuditLog {
id Int @id @default(autoincrement())
userId String
username String
email String
loggedAt Int
locale String
actionType String
actionData String
@@map("audit_logs_new")
}
model Authenticator {
id String @id
userId String?
type String
payload String
validUntil Int?
users User? @relation(fields: [userId], references: [id], onDelete: Cascade, onUpdate: NoAction)
@@index([userId], map: "authenticators_userId")
@@index([type], map: "authenticators_type")
@@map("authenticators")
}
// Some manual changes here:
// 1. `bannedBy` is a nullable
model BanProposal {
id String @id
userId String
bannedBy String?
bannedTerms String
bannedReason String
bannedByUser User? @relation("ban_proposals_bannedByTousers", fields: [bannedBy], references: [id], onDelete: SetNull, onUpdate: NoAction)
user User @relation("ban_proposals_userIdTousers", fields: [userId], references: [id], onDelete: Cascade, onUpdate: NoAction)
@@index([userId], map: "ban_proposals_userId")
@@map("ban_proposals")
}
model Ban {
type String
value String
@@id([type, value])
@@map("ban")
}
model Inclusive {
id String @id
insteadOf String
say String
because String
locale String
approved Int
base_id String?
author_id String?
categories String?
links String?
deleted Int @default(0)
clarification String?
users User? @relation(fields: [author_id], references: [id], onUpdate: NoAction)
@@index([insteadOf], map: "inclusive_insteadOf")
@@index([locale], map: "inclusive_locale")
@@map("inclusive")
}
/// The underlying table does not contain a valid unique identifier and can therefore currently not be handled by Prisma Client.
model Link {
url String? @id
expiresAt Int?
favicon String?
relMe String?
nodeinfo String?
@@map("links")
@@ignore
}
model Name {
id String @id
name String
locale String
origin String?
meaning String?
usage String?
legally String?
pros String?
cons String?
notablePeople String?
links String?
namedays String?
namedaysComment String?
deleted Int @default(0)
approved Int @default(0)
base_id String?
author_id String?
@@index([locale], map: "names_locale")
@@index([name], map: "names_name")
@@index([locale, name], map: "names_name_locale")
@@map("names")
}
model Noun {
id String @id
masc String
fem String
neutr String
mascPl String
femPl String
neutrPl String
approved Int
base_id String?
locale String @default("pl")
author_id String?
deleted Int @default(0)
sources String?
users User? @relation(fields: [author_id], references: [id], onUpdate: NoAction)
@@index([masc], map: "nouns_masc")
@@index([locale], map: "nouns_locale")
@@map("nouns")
}
model Profile {
id String @id
userId String
locale String
names String
pronouns String
description String
birthday String?
links String
flags String
words String
active Int
teamName String?
footerName String?
footerAreas String?
customFlags String @default("{}")
card String?
credentials String?
credentialsLevel Int?
credentialsName Int?
cardDark String?
opinions String @default("{}")
timezone String?
sensitive String @default("[]")
users User @relation(fields: [userId], references: [id], onDelete: Cascade, onUpdate: NoAction)
user_connections UserConnection[]
@@unique([userId, locale], map: "sqlite_autoindex_profiles_2")
@@index([footerAreas], map: "profiles_footerAreas")
@@index([footerName], map: "profiles_footerName")
@@index([teamName], map: "profiles_teamName")
@@index([locale, userId], map: "profiles_locale_userId")
@@index([userId], map: "profiles_userId")
@@index([locale], map: "profiles_locale")
@@map("profiles")
}
model Report {
id String @id
userId String?
reporterId String?
comment String
isAutomatic Int?
isHandled Int?
snapshot String?
users_reports_reporterIdTousers User? @relation("reports_reporterIdTousers", fields: [reporterId], references: [id], onUpdate: NoAction)
users_reports_userIdTousers User? @relation("reports_userIdTousers", fields: [userId], references: [id], onDelete: Cascade, onUpdate: NoAction)
@@index([userId], map: "reports_userId")
@@index([isHandled], map: "reports_isHandled")
@@index([isAutomatic], map: "reports_isAutomatic")
@@map("reports")
}
/// The underlying table does not contain a valid unique identifier and can therefore currently not be handled by Prisma Client.
model SocialLookup {
userId String
provider String
identifier String
users User @relation(fields: [userId], references: [id], onDelete: Cascade, onUpdate: NoAction)
@@index([provider, identifier], map: "social_lookup_provider_identifier")
@@map("social_lookup")
@@ignore
}
model Source {
id String @id
locale String
pronouns String
type String
author String?
title String
extra String?
year Int?
fragments String
comment String?
link String?
submitter_id String?
approved Int? @default(0)
deleted Int? @default(0)
base_id String?
key String?
images String?
spoiler Int @default(0)
users User? @relation(fields: [submitter_id], references: [id], onUpdate: NoAction)
@@index([locale], map: "sources_locale")
@@map("sources")
}
model Stat {
id String
locale String
users Int
data String
@@id([id, locale])
@@map("stats")
}
/// The underlying table does not contain a valid unique identifier and can therefore currently not be handled by Prisma Client.
model SubscriptionMessage {
id String? @id
subscription_id String
campaign String
@@map("subscription_messages")
@@ignore
}
model Term {
id String @id
term String
original String?
definition String
locale String
approved Int
base_id String?
author_id String?
deleted Int @default(0)
flags String @default("[]")
category String?
images String @default("")
key String?
users User? @relation(fields: [author_id], references: [id], onUpdate: NoAction)
@@index([term], map: "terms_term")
@@index([locale], map: "terms_locale")
@@map("terms")
}
model UserConnection {
id String @id
from_profileId String
to_userId String
relationship String
users User @relation(fields: [to_userId], references: [id], onDelete: Cascade, onUpdate: NoAction)
profiles Profile @relation(fields: [from_profileId], references: [id], onDelete: Cascade, onUpdate: NoAction)
@@index([to_userId], map: "user_connections_to_userId")
@@index([from_profileId], map: "user_connections_from_profileId")
@@map("user_connections")
}
model UserMessage {
id String @id
userId String
adminId String
message String
@@map("user_messages")
}
model User {
id String @id
username String @unique(map: "users_username")
email String @unique(map: "users_email")
roles String
avatarSource String?
bannedReason String?
suspiciousChecked Unsupported("tinyint") @default(dbgenerated("0"))
usernameNorm String? @unique(map: "users_usernameNorm")
bannedTerms String?
bannedBy String?
lastActive Int?
banSnapshot String?
inactiveWarning Int?
adminNotifications Int @default(7)
loginAttempts String?
timesheets String?
socialLookup Int @default(0)
authenticators Authenticator[]
ban_proposals_ban_proposals_bannedByTousers BanProposal[] @relation("ban_proposals_bannedByTousers")
ban_proposals_ban_proposals_userIdTousers BanProposal[] @relation("ban_proposals_userIdTousers")
inclusive Inclusive[]
nouns Noun[]
profiles Profile[]
reports_reports_reporterIdTousers Report[] @relation("reports_reporterIdTousers")
reports_reports_userIdTousers Report[] @relation("reports_userIdTousers")
social_lookup SocialLookup[] @ignore
sources Source[]
terms Term[]
user_connections UserConnection[]
@@map("users")
}

138
new/backend/src/config.ts Normal file
View File

@ -0,0 +1,138 @@
import "dotenv/config";
import {
identity,
parseBool,
parseInteger,
toLowerCase,
} from "@pronounspage/common/util";
import * as path from "node:path";
import * as fs from "node:fs";
import type { log } from "#self/log";
export enum Environment {
DEVELOPMENT = "dev",
PRODUCTION = "production",
}
export interface Config {
environment: Environment;
logLevel?: (typeof log)["level"];
http: {
baseUrl: string;
host: string;
port: number;
};
database: {
url: string;
}
security: {
secret: string;
};
localeDataPath: string;
allowUnpublishedLocales: boolean;
}
function envVarOrDefault<T>(
key: string,
parse: (value: string) => T,
defaultValue: T
): T;
function envVarOrDefault<T>(
key: string,
parse: (value: string) => T,
defaultValue?: undefined
): T | undefined;
function envVarOrDefault<T>(
key: string,
parse: (value: string) => T,
defaultValue: T | undefined = undefined
): T | undefined {
const value = process.env[key];
return value == null ? defaultValue : parse(String(value));
}
function envVarNotNull<T>(key: string, parse: (value: string) => T): T {
const value = envVarOrDefault(key, parse);
if (value === undefined) {
throw new Error(`Environment variable ${key} is missing`);
}
return value;
}
function parseEnvironment(value: string): Environment {
switch (value.toLowerCase()) {
case "dev":
case "development":
return Environment.DEVELOPMENT;
case "prod":
case "production":
return Environment.PRODUCTION;
default:
throw new Error(`"${value}" is not a valid environment`);
}
}
function parsePath(value: string): string {
return path.resolve(process.cwd(), value);
}
export function loadConfigFromEnv(): Config {
return {
environment: envVarOrDefault(
"ENVIRONMENT",
parseEnvironment,
Environment.DEVELOPMENT
),
logLevel: envVarOrDefault("LOG_LEVEL", toLowerCase, undefined),
http: {
baseUrl: envVarNotNull("HTTP_BASE_URL", identity),
host: envVarOrDefault("HTTP_HOST", identity, "0.0.0.0"),
port: envVarOrDefault("HTTP_PORT", parseInteger, 4000),
},
database: {
url: envVarNotNull("DATABASE_URL", identity),
},
security: {
secret: envVarNotNull("SECURITY_SECRET", identity),
},
localeDataPath: envVarOrDefault(
"LOCALE_DATA_PATH",
parsePath,
"../locales"
),
allowUnpublishedLocales: envVarOrDefault(
"ALLOW_UNPUBLISHED_LOCALES",
parseBool,
false
),
};
}
export function validateConfig(config: Config): string | undefined {
if (!fs.existsSync(config.localeDataPath)) {
return `Locale data path is set to ${config.localeDataPath}, but it does not exist`;
}
// I mean these two next ones really aren't necessary, but I find it funny
// ...who said we can't have a little fun here and there?
const secretLower = config.security.secret.toLowerCase();
if (secretLower === "changeme") {
return `You didn't change the secret? The secret is quite literally "${secretLower}"`;
}
if (
secretLower === "changemeonprod!" &&
config.environment === Environment.PRODUCTION
) {
return `Hey now, this is production! You have to change the secret on prod! Look, it says so: ${config.security.secret}`;
}
return undefined;
}
let loadedConfig: Config | undefined = undefined;
export function getConfig() {
if (loadedConfig == undefined) {
loadedConfig = loadConfigFromEnv();
}
return loadedConfig;
}

33
new/backend/src/global.d.ts vendored Normal file
View File

@ -0,0 +1,33 @@
import type {
FastifyInstance,
FastifyPluginAsync,
// FastifyPluginCallback,
FastifyPluginOptions,
FastifyRequest,
FastifyReply,
RawReplyDefaultExpression,
RawRequestDefaultExpression,
RawServerDefault,
} from "fastify";
import type { TypeBoxTypeProvider } from "@fastify/type-provider-typebox";
import type { log } from "#self/log";
type RawServer = RawServerDefault;
type RequestExpr = RawRequestDefaultExpression<RawServer>;
type ReplyExpr = RawReplyDefaultExpression<RawServer>;
type Log = typeof log;
type TypeProvider = TypeBoxTypeProvider;
declare global {
type AppInstance = FastifyInstance<
RawServer,
RequestExpr,
ReplyExpr,
typeof log,
TypeProvider
>;
type AppPluginAsync<
Options extends FastifyPluginOptions = Record<never, never>,
> = FastifyPluginAsync<Options, RawServer, TypeProvider, Log>;
type AppReply = FastifyReply<RawServer, RequestExpr, ReplyExpr>;
}

41
new/backend/src/index.ts Normal file
View File

@ -0,0 +1,41 @@
import { startServer } from "#self/server";
import { getConfig, validateConfig } from "#self/config";
import log from "#self/log";
import { exit, pid, ppid } from "node:process";
import {
getActiveLocaleDescriptions,
loadAllLocales,
loadLocaleDescriptions,
} from "./locales.js";
log.info(`Pronouns.page Backend Server - PID: ${pid}, PPID: ${ppid}`);
try {
const validationError = validateConfig(getConfig());
if (validationError != undefined) {
log.error(`Configuration is invalid: ${validationError}`);
exit(1);
}
} catch (e) {
log.error(`Could not validate configuration: ${e}`);
exit(1);
}
try {
const localeDescriptions = await loadLocaleDescriptions();
const active = getActiveLocaleDescriptions();
await loadAllLocales();
log.info(
`${localeDescriptions.length} locale(s) are registered, of which ${active.length} are active`
);
} catch (e) {
log.error(`Could not load locales: ${e}`);
exit(1);
}
const memUsage = process.memoryUsage();
const memUsagePercent = (memUsage.heapUsed / memUsage.heapTotal) * 100;
log.info(
`Finished preparations - system stats: MEM ${memUsagePercent.toFixed(2)}%`
);
await startServer();

347
new/backend/src/locales.ts Normal file
View File

@ -0,0 +1,347 @@
/*
* This file is for interop with the old locales.js file to reduce duplication.
*/
import { getConfig } from "#self/config";
import * as path from "node:path";
import * as suml from "@pronounspage/common/suml";
import fsp from "node:fs/promises";
import log from "#self/log";
import * as csv from "csv-parse/sync";
import {
capitalizeFirstLetter,
compileTemplate,
isNotBlank,
parseBool,
} from "@pronounspage/common/util";
export interface LocaleDescription {
code: string;
name: string;
url: string;
published: boolean;
/**
* This locale's language family.
*
* @experimental
* This is currently not used for published versions.
*/
symbol: string;
/**
* This locale's language family.
*
* @experimental
* This is currently not used for published versions.
*
* @privateRemarks
* This could technically be an enum but for now it's just a string as there
* doesn't seem to be any apparent benefit to including locales
*/
family: string;
}
let loadedDescriptions: Array<LocaleDescription>;
let indexedDescriptions: Record<string, LocaleDescription>;
let activeLocaleDescriptions: Array<LocaleDescription> | undefined;
export async function loadLocaleDescriptions(): Promise<
Array<LocaleDescription>
> {
if (loadedDescriptions == undefined) {
loadedDescriptions = (
await import(
"file://" +
path.resolve(getConfig().localeDataPath, "locales.js")
)
).default;
indexedDescriptions = Object.fromEntries(
loadedDescriptions.map((v) => [v.code, v]),
);
activeLocaleDescriptions = undefined;
}
return loadedDescriptions;
}
export function getCachedLocaleDescriptions(): Array<LocaleDescription> {
return loadedDescriptions;
}
export function getActiveLocaleDescriptions(): Array<LocaleDescription> {
if (activeLocaleDescriptions == undefined) {
const config = getConfig();
const cached = getCachedLocaleDescriptions();
activeLocaleDescriptions = [];
for (const locale of cached) {
if (config.allowUnpublishedLocales || locale.published) {
activeLocaleDescriptions.push(locale);
}
}
}
return activeLocaleDescriptions;
}
export function isLocaleActive(localeCode: string): boolean {
return getActiveLocaleDescriptions().some((v) => v.code === localeCode);
}
export interface Pronoun {
keys: Array<string>;
description: string;
history: string;
source?: string;
isNormative: boolean;
isPlural: boolean;
isPluralHonorific: boolean;
isPronounceable: boolean;
thirdForm?: string;
smallForm?: string;
forms: Record<string, { written: string; pronounced: string }>;
}
export interface PronounExample {
singular?: string;
plural?: string;
isHonorific: boolean;
}
export class Locale {
private _code: string;
private _dataDirectory: string;
constructor(code: string, dataDirectory: string) {
this._code = code;
this._dataDirectory = dataDirectory;
}
private path(pathParts: Array<string>) {
return path.resolve(this._dataDirectory, ...pathParts);
}
private async readFile(parts: Array<string>): Promise<string>;
private async readFile<T>(
parts: Array<string>,
parser: (value: string) => T,
): Promise<T>;
private async readFile<T>(
pathParts: Array<string>,
parser?: (value: string) => T,
) {
// Some may complain about this
const filePath = this.path(pathParts);
try {
log.trace(`[${this.code}] Loading file ${filePath}`);
const data = await fsp.readFile(filePath, "utf-8");
if (parser) {
return parser(data);
} else {
return data;
}
} catch (e) {
const err = new Error(
`[${this.code}] Could not load file ${filePath}: ${e}`,
);
err.cause = e;
throw err;
}
}
private async importFile(parts: Array<string>): Promise<unknown> {
const filePath = this.path(parts);
try {
log.trace(`[${this.code}] Importing file ${filePath} for locale`);
return await import(`file://${filePath}`);
} catch (e) {
const err = new Error(
`[${this.code}] Could not import file ${filePath}: ${e}`,
);
err.cause = e;
throw err;
}
}
private _config: unknown | null = null;
private _translations: unknown | null = null;
private _pronouns: Array<Pronoun> = [];
private _pronounsByAlias: Record<string, number> = {};
private _examples: Array<PronounExample> = []; // TODO: Type these properly
// private _morphemes: Array<string>;
public async load() {
const tsvParse = (data: string) =>
csv.parse(data, {
columns: true,
cast: false,
relaxColumnCount: true,
relaxQuotes: true,
delimiter: "\t",
});
this._config = await this.readFile(["config.suml"], suml.parse);
this._translations = await this.readFile(
["translations.suml"],
suml.parse,
);
// NOTE(tecc): This currently doesn't work because the morphemes.js files (as well as many others) rely on ESM syntax.
// This is both inconsistent with the locales.js and expectedTranslations.js files,
// and causes Node to throw an error.
// this._morphemes = (await this.importFile("pronouns/morphemes.js")).default;
this._pronouns = [];
this._pronounsByAlias = {};
const pronouns = await this.readFile(
["pronouns/pronouns.tsv"],
tsvParse,
);
for (const pronoun of pronouns) {
const partial: Partial<Pronoun> = { forms: {} };
for (const [key, value] of Object.entries<string>(pronoun)) {
switch (key) {
case "key":
partial.keys = value.split(",");
break;
case "normative":
partial.isNormative = parseBool(value);
break;
case "plural":
partial.isPlural = parseBool(value);
break;
case "pluralHonorific":
partial.isPluralHonorific = parseBool(value);
break;
case "pronounceable":
partial.isPronounceable = parseBool(value);
break;
case "description":
case "history":
case "thirdForm":
case "smallForm":
partial[key] = isNotBlank(value) ? value : undefined;
break;
case "sourcesInfo":
partial.source = value;
break;
default:
const [written, pronounced] = value.split("|");
partial.forms![key] = { written, pronounced };
break;
}
}
const i = this._pronouns.length;
const obj = partial as Pronoun;
this._pronouns.push(obj);
for (const key of obj.keys) {
this._pronounsByAlias[key] = i;
}
}
const examples = await this.readFile(
["pronouns/examples.tsv"],
tsvParse,
);
this._examples = [];
for (const example of examples) {
this._examples.push({
singular: isNotBlank(example.singular)
? example.singular
: undefined,
plural: isNotBlank(example.plural) ? example.plural : undefined,
isHonorific: example.isHonorific,
} satisfies PronounExample);
}
}
public get code() {
return this._code;
}
public get description() {
return indexedDescriptions[this.code];
}
public get config() {
return this._config;
}
public get pronouns() {
return Object.values(this._pronouns);
}
public pronoun(key: string): Pronoun | null {
const index = this._pronounsByAlias[key];
if (index == null) {
return null;
}
return this._pronouns[index];
}
public get examples() {
return this._examples;
}
}
export function examplesFor(
pronoun: Pronoun,
examples: Array<PronounExample>,
): Array<string> {
const finished = [];
const templating: Record<string, string> = {};
for (const [key, value] of Object.entries(pronoun.forms)) {
templating[key] = value.written;
templating[`'${key}`] = capitalizeFirstLetter(value.written);
}
for (const example of examples) {
let template = example.singular;
if (pronoun.isPlural && example.plural) {
template = example.plural;
}
if (template == null) {
continue;
}
finished.push(compileTemplate(template, templating));
}
return finished;
}
const loadedLocales: Record<string, Locale> = {};
/**
* Load a locale.
* This will only succeed if the locale is both
* @param localeCode the locale code
*/
export function getLocale(localeCode: string): Locale | null {
if (!isLocaleActive(localeCode)) {
return null;
}
let locale = loadedLocales[localeCode];
if (locale == null) {
locale = new Locale(
localeCode,
path.resolve(getConfig().localeDataPath, localeCode),
);
loadedLocales[localeCode] = locale;
}
return locale;
}
export async function loadAllLocales() {
const descriptions = await loadLocaleDescriptions();
const promises = [];
for (const description of descriptions) {
const locale = getLocale(description.code);
if (locale) {
promises.push(
locale
.load()
.then(() =>
log.debug(`Successfully loaded locale ${locale.code}`),
),
);
}
}
await Promise.all(promises);
log.info("All active locales have been loaded into memory");
}

35
new/backend/src/log.ts Normal file
View File

@ -0,0 +1,35 @@
import { pino, LoggerOptions } from "pino";
import { getConfig, Environment } from "#self/config";
const loggerConfigurations = {
[Environment.DEVELOPMENT]: {
level: "debug",
transport: {
target: "pino-pretty",
options: {
translateTime: "HH:MM:ss Z",
ignore: "pid,hostname",
},
},
},
[Environment.PRODUCTION]: {
level: "info",
},
} satisfies Record<Environment, LoggerOptions>;
let environment, logLevel;
try {
const config = getConfig();
environment = config.environment;
logLevel = config.logLevel;
} catch (e) {
environment = Environment.DEVELOPMENT;
}
const configuration = loggerConfigurations[environment];
if (logLevel) {
configuration.level = logLevel;
}
export const log = pino(configuration);
export default log;

40
new/backend/src/server.ts Normal file
View File

@ -0,0 +1,40 @@
import fastify from "fastify";
import log from "#self/log";
import { getConfig } from "#self/config";
import v1 from "#self/server/v1";
export async function startServer() {
const config = getConfig();
const app = fastify({
logger: log,
ajv: {
customOptions: {
coerceTypes: "array",
},
},
});
let start: Date;
app.get("/ping", (req, reply) => {
reply.send({
message: `Pong! The server is running!`,
status: {
uptime: Date.now() - start.getTime(),
},
});
});
app.register(v1, { prefix: "/v1" });
try {
await app.listen({
host: config.http.host,
port: config.http.port,
});
start = new Date(Date.now());
log.info(`(Public base URL: ${config.http.baseUrl})`);
} catch (e) {
log.error(e);
return;
}
}

View File

@ -0,0 +1,89 @@
import { Type } from "@sinclair/typebox";
import { examplesFor, getLocale, Pronoun, PronounExample } from "#self/locales";
import pronouns from "#self/server/v1/pronouns";
export type V1AppInstance = AppInstance;
export interface ApiErrorOptions {
code: number;
message: string;
detail?: string;
}
/**
* Creates an error object for the v1 API endpoints.
* This function may seem a bit extraneous but it's a useful utility for consistency.
* @param options
*/
export function error<T extends ApiErrorOptions>(options: T) {
const response: Record<string, unknown> = {
code: options.code,
message: options.message,
};
if (options.detail) {
response.detail = options.detail;
}
return response;
}
export function replyError<T extends ApiErrorOptions>(
reply: AppReply,
options: T,
): AppReply {
const response = error(options);
return reply.status(options.code).send(response);
}
export const ApiError = {
INVALID_LOCALE: {
code: 404,
message: "Invalid locale",
},
UNKNOWN_PRONOUN: {
code: 404,
message: "We aren't aware of any such pronoun. Perhaps there's a typo?\nNote that for custom pronouns, you need to specify all necessary forms: https://en.pronouns.page/faq#custom-pronouns",
},
} satisfies Record<string, ApiErrorOptions>;
export const localeSpecific = Type.Object({
locale: Type.String(),
});
export function transformPronoun(
pronoun: Pronoun,
examples: Array<PronounExample>,
) {
const morphemes: Record<string, string> = {};
const pronunciations: Record<string, string> = {};
for (const [name, form] of Object.entries(pronoun.forms)) {
morphemes[name] = form.written;
pronunciations[name] = form.pronounced;
}
return {
canonicalName: pronoun.keys[0],
description: pronoun.description,
normative: pronoun.isNormative,
morphemes,
pronunciations,
plural: [pronoun.isPlural],
pluralHonorific: [pronoun.isPluralHonorific],
aliases: pronoun.keys.slice(1),
history: pronoun.history ?? "",
pronounceable: pronoun.isPronounceable,
thirdForm: pronoun.thirdForm ?? null,
smallForm: pronoun.smallForm ?? null,
sourcesInfo: pronoun.source ?? null,
examples: examplesFor(pronoun, examples),
};
}
/*
* Version 1 of the API. This would be the "legacy" version.
* Since the output of every endpoint is locale-specific,
* every route is prefixed with the respective locale ID.
*/
export const routes = async function(app: AppInstance) {
app.register(pronouns);
} satisfies AppPluginAsync;
export default routes;

View File

@ -0,0 +1,84 @@
import {
localeSpecific,
ApiError,
replyError,
transformPronoun,
} from "#self/server/v1";
import { getLocale, PronounExample } from "#self/locales";
import { isNotBlank, parseBool } from "@pronounspage/common/util";
import { Type } from "@sinclair/typebox";
export const plugin = async function (app: AppInstance) {
// TODO: "Memoize" this route to reduce memory usage
app.get(
"/:locale/pronouns",
{
schema: {
params: localeSpecific,
},
},
(request, reply) => {
const locale = getLocale(request.params.locale);
if (locale === null) {
return replyError(reply, ApiError.INVALID_LOCALE);
}
const obj: Record<string, unknown> = {};
for (const pronoun of locale.pronouns) {
obj[pronoun.keys[0]] = transformPronoun(
pronoun,
locale.examples
);
}
return obj;
}
);
function parseExample(s: string): PronounExample {
const [singular, plural, isHonorific] = s.split("|");
return {
singular: isNotBlank(singular) ? singular : undefined,
plural: isNotBlank(plural) ? plural : undefined,
isHonorific: isNotBlank(isHonorific) && parseBool(isHonorific),
};
}
// TODO: See previous TODO. Although maybe not with query parameters.
app.get(
"/:locale/pronouns/*",
{
schema: {
params: Type.Intersect([
localeSpecific,
Type.Object({ "*": Type.String() }),
]),
querystring: Type.Object({
"examples[]": Type.Array(Type.String(), { default: [] }),
}),
},
},
(req, reply) => {
const locale = getLocale(req.params.locale);
if (locale == null) {
return replyError(reply, ApiError.INVALID_LOCALE);
}
const key = req.params["*"].split("/").filter(isNotBlank).join("/"); // This is to get rid of any extraneous parts
const found = locale.pronoun(key);
if (found == null) {
return replyError(reply, ApiError.UNKNOWN_PRONOUN);
}
const examples = req.query["examples[]"];
return transformPronoun(
found,
examples.length < 1
? locale.examples
: examples.map(parseExample)
);
}
);
} satisfies AppPluginAsync;
export default plugin;

View File

20
new/backend/tsconfig.json Normal file
View File

@ -0,0 +1,20 @@
{
"extends": "../tsconfig.json",
"compilerOptions": {
"module": "NodeNext",
"moduleResolution": "NodeNext",
"target": "ESNext",
"baseUrl": ".",
"rootDir": "./src",
"outDir": "./dist",
"resolvePackageJsonExports": true,
"paths": {
"#self/*": ["./src/*.js"]
},
"incremental": true,
"strictNullChecks": true,
"strict": true,
"skipLibCheck": true
},
"include": ["./src/**/*"]
}

1
new/common/.npmignore Normal file
View File

@ -0,0 +1 @@
/tsconfig.json

45
new/common/package.json Normal file
View File

@ -0,0 +1,45 @@
{
"name": "@pronounspage/common",
"version": "1.0.0",
"description": "",
"private": true,
"type": "module",
"scripts": {
"lint": "eslint src/**",
"clean": "rimraf dist",
"build:esm": "tsc --module es6 --outDir dist/esm",
"build:cjs": "tsc --module commonjs --outDir dist/cjs",
"build:types": "tsc --emitDeclarationOnly --declaration --outDir dist/types",
"build": "run-p build:esm build:cjs build:types",
"cleanbuild": "run-s clean build"
},
"exports": {
"./*": {
"types": "./dist/types/*.d.ts",
"require": "./dist/cjs/*.js",
"import": "./dist/esm/*.js"
},
".": {
"types": "./dist/types/index.d.ts",
"require": "./dist/cjs/index.d.ts",
"import": "./dist/esm/index.js"
}
},
"typesVersions": {
"*": {
"*": [
"./dist/types/*.d.ts"
]
}
},
"keywords": [],
"author": "",
"devDependencies": {
"npm-run-all": "^4.1.5",
"rimraf": "^5.0.1",
"typescript": "^5.2.2"
},
"dependencies": {
"suml": "^0.2.2"
}
}

1
new/common/src/global.d.ts vendored Normal file
View File

@ -0,0 +1 @@
declare module "suml";

14
new/common/src/suml.ts Normal file
View File

@ -0,0 +1,14 @@
import * as SumlImpl from "suml";
declare module "suml";
console.log(SumlImpl);
const instance = new SumlImpl.default();
export function parse(value: string): unknown {
return instance.parse(value);
}
export function dump(value: unknown): string {
return instance.dump(value);
}

104
new/common/src/util.ts Normal file
View File

@ -0,0 +1,104 @@
export function identity<T>(value: T): T {
return value;
}
// Generally, don't use this function to define commonly used functions.
// Prefer to write a manual implementation.
export function not<Args extends Array<unknown>>(
f: (...args: Args) => boolean
): (...args: Args) => boolean {
return (...args: Args) => !f(...args);
}
export function isNotNull<T>(value: T | null | undefined): value is T {
return value != null;
}
export const isNull = not(isNotNull); // I know I said not to do this, but it's not used yet *however* it irks me to not have completeness.
export function isNotBlank(value: string | null | undefined): boolean {
if (!isNotNull(value)) {
return false;
}
if (value.length < 1) {
return false;
}
return value.trim().length > 1;
}
export const isBlank = not(isNotBlank);
// This is unnecessarily typed but I don't really care, it works doesn't it?
export function toLowerCase<S extends string>(value: S): Lowercase<S> {
return value.toLowerCase() as Lowercase<S>;
}
export function parseInteger(value: string, radix?: number): number {
if (radix != null && (radix < 2 || radix > 36)) {
throw new Error(
`invalid radix ${radix} - must be between 2 and 36 (inclusive)`
);
}
const parsed = parseInt(value, radix);
if (isNaN(parsed)) {
throw new Error("parsed value is NaN");
}
return parsed;
}
export function parseBool(value: unknown): boolean {
switch (typeof value) {
case "boolean":
return value;
case "string":
value = value.trim().toLowerCase();
if (value === "true" || value === "1" || value === "yes") {
return true;
}
if (value === "false" || value === "0" || value === "no") {
return false;
}
throw new Error(`unknown boolean constant: ${value}`);
case "number":
return value != 0;
default:
throw new Error(`Cannot convert value ${value} to a boolean`);
}
}
export function capitalizeFirstLetter(value: string) {
return value.charAt(0).toUpperCase() + value.substring(1);
}
// TODO: Benchmark this function with different implementations.
export function compileTemplate(
template: string,
values: Record<string, string>
): string {
let string = "";
let key = null;
for (let i = 0; i < template.length; i++) {
const c = template.charAt(i);
if (key === null) {
switch (c) {
case "{":
key = "";
continue;
default:
string += c;
break;
}
} else {
switch (c) {
case "}":
string += values[key];
key = null;
continue;
default:
key += c;
break;
}
}
}
/*for (const key in values) {
const value = values[key];
string = string.replace(`{${key}}`, value);
}*/
return string;
}

8
new/common/tsconfig.json Normal file
View File

@ -0,0 +1,8 @@
{
"extends": "../tsconfig.json",
"compilerOptions": {
"rootDir": "./src",
"outDir": "dist"
},
"include": ["./src/**/*"]
}

14
new/package.json Normal file
View File

@ -0,0 +1,14 @@
{
"private": true,
"scripts": {
"format": "prettier -w ."
},
"devDependencies": {
"@typescript-eslint/eslint-plugin": "^6.4.1",
"@typescript-eslint/parser": "^6.4.1",
"eslint": "^8.48.0",
"eslint-config-prettier": "^9.0.0",
"prettier": "^3.0.2",
"typescript": "^5.2.2"
}
}

2453
new/pnpm-lock.yaml generated Normal file

File diff suppressed because it is too large Load Diff

3
new/pnpm-workspace.yaml Normal file
View File

@ -0,0 +1,3 @@
packages:
- "common"
- "backend"

130
new/setup.mjs Normal file
View File

@ -0,0 +1,130 @@
import fsp from "node:fs/promises";
import fs from "node:fs";
import path from "node:path";
import process from "node:process";
import * as child_process from "node:child_process";
console.log("Setting up project");
const argv = process.argv.slice(2);
const packageManager = argv[0] ?? "pnpm";
let currentlySettingUp = "";
let needsToInstallDependencies = false;
const url = new URL(import.meta.url);
let projectDir = url.pathname
.split("/")
.filter((v) => v && v.length > 0)
.join(path.sep)
// This is a cheap but probably check to see if we're *not* on Windows.
if (path.sep !== "\\") {
// You could inline `path.sep` as "\" but why not go that extra mile
// We're already using it so
projectDir = path.sep + projectDir;
}
projectDir = path.dirname(path.resolve(projectDir));
const legacyDir = path.resolve(projectDir, "..");
function message(str) {
let prefix = "";
if (!!currentlySettingUp && currentlySettingUp.trim().length >= 0) {
prefix = `[${currentlySettingUp}] `;
}
console.log(" " + prefix + str);
}
console.log(`
Project directory: ${projectDir}
Legacy directory: ${legacyDir}
`);
async function symlink(to, from) {
if (!fs.existsSync(from)) {
message(`Creating symlink from ${from} to ${to}`);
await fsp.symlink(to, from, "dir");
} else {
message(`File ${from} already exists - not symlinking to ${to}`);
}
}
// Create temporary symlinks
// This step may very well be removed in the future.
{
currentlySettingUp = "locales";
const to = path.resolve(legacyDir, "locale"),
from = path.resolve(projectDir, "locales");
await symlink(to, from);
}
{
currentlySettingUp = "common";
const commonDir = path.resolve(projectDir, "common");
const commonDistDir = path.resolve(commonDir, "dist");
if (!fs.existsSync(commonDistDir)) {
message("Building common package");
child_process.spawnSync(packageManager, ["run", "build"], {
cwd: commonDir,
stdio: "inherit",
});
}
}
{
currentlySettingUp = "backend";
const backendDir = path.resolve(projectDir, "backend");
const envFile = path.resolve(backendDir, ".env"),
exampleEnvFile = path.resolve(backendDir, "example.env");
if (!fs.existsSync(envFile)) {
message(`Missing .env file for backend, creating from example.env`);
await fsp.cp(exampleEnvFile, envFile);
} else {
message(`Environment file already exists, continuing`);
}
const prismaDistDir = path.resolve(backendDir, "prisma", "dist")
if (!fs.existsSync(prismaDistDir)) {
message("Generating Prisma client");
child_process.spawnSync(packageManager, ["run", "db:generate"], {
cwd: backendDir,
stdio: "inherit"
});
} else {
message("Prisma client has already been generated and is not broken");
}
// This is a cheap fix for a bug that occurs when the .prisma/client
// package has already been linked, but the target (prisma/dist) doesn't exist
const prismaClientFile = path.resolve(backendDir, "node_modules", ".prisma", "client");
let shouldFixPrismaClient = true;
if (fs.existsSync(prismaClientFile)) {
shouldFixPrismaClient &&= !(await fsp.stat(prismaClientFile)).isDirectory();
}
if (shouldFixPrismaClient ) {
message("Fixing .prisma/client package")
if (fs.existsSync(prismaClientFile)) {
await fsp.rm(prismaClientFile);
}
needsToInstallDependencies = true;
}
}
{
currentlySettingUp = "finalise";
if (needsToInstallDependencies) {
message("Installing dependencies")
child_process.spawnSync(packageManager, ["install"], {
cwd: projectDir,
stdio: "inherit"
});
}
}
console.log(`\nDone!`);

8
new/tsconfig.json Normal file
View File

@ -0,0 +1,8 @@
{
"compilerOptions": {
"moduleResolution": "node",
"resolveJsonModule": true,
"esModuleInterop": true,
"strict": true
}
}