(helper) helper for merging proposals to translations.suml

server/sumlAst.ts

@@ -0,0 +1,414 @@
+class ParseError extends Error {
+    line: number;
+    context: string;
+
+    constructor(message = '', line = -1, context = '') {
+        let combinedMessage = `Cannot parse line ${line + 1}`;
+        if (context !== undefined) {
+            combinedMessage += ` near \`${context}\``;
+        }
+        if (message) {
+            combinedMessage += `: ${message}`;
+        }
+        super(combinedMessage);
+        this.line = line;
+        this.context = context;
+    }
+}
+
+type FormattingNode = EmptyLineNode | CommentNode;
+
+export class EmptyLineNode {
+    toString(): string {
+        return '\n';
+    }
+}
+
+export class CommentNode {
+    constructor(public comment: string) {}
+
+    toString(indent: number = 0) {
+        return `${' '.repeat(4 * indent)}#${this.comment}\n`;
+    }
+}
+
+export type Node = DictNode | ListNode | InlineNode | VerbatimStringNode | FoldedStringNode;
+
+export class DictNode {
+    constructor(public items: DictEntry[], public formatting: FormattingNode[]) {}
+
+    get separator() {
+        return '\n';
+    }
+
+    toString(indent: number = 0): string {
+        return this.items.map((item) => item.toString(indent)).join('\n');
+    }
+}
+
+export class DictEntry {
+    constructor(public key: string, public value: Node, public formatting: FormattingNode[]) {}
+
+    toString(indent: number = 0): string {
+        return `${this.formatting.map((node) => node.toString(indent)).join('')}${' '.repeat(4 * indent)}${this.key}:${this.value.separator}${this.value.toString(indent + 1)}`;
+    }
+}
+
+export class ListNode {
+    constructor(public items: ListItem[], public formatting: FormattingNode[]) {}
+
+    get separator() {
+        return '\n';
+    }
+
+    toString(indent: number = 0): string {
+        return this.items.map((item) => item.toString(indent)).join('\n');
+    }
+}
+
+export class ListItem {
+    constructor(public value: Node, public formatting: FormattingNode[]) {}
+
+    toString(indent: number = 0): string {
+        return `${this.formatting.map((node) => node.toString(indent)).join('')}${' '.repeat(4 * indent)}-${this.value.separator}${this.value.toString(indent + 1)}`;
+    }
+}
+
+type InlineNode = SingleLineScalarNode | InlineDictNode | InlineListNode;
+
+export class SingleLineScalarNode {
+    constructor(
+        public value: null | boolean | number | string | Date,
+        public formatting: FormattingNode[],
+        public comment: string | undefined,
+    ) {}
+
+    get separator() {
+        return ' ';
+    }
+
+    get representation(): string {
+        if (this.value === null) {
+            return '~';
+        }
+        if (typeof this.value === 'string') {
+            return `'${this.value.replaceAll('\'', '\'\'')}'`;
+        }
+        return this.value.toString();
+    }
+
+    toString(): string {
+        return `${this.representation}${this.comment ? ` # ${this.comment}` : ''}`;
+    }
+}
+
+export class InlineListNode {
+    constructor(public items: Node[]) {}
+
+    get separator() {
+        return ' ';
+    }
+
+    toString(): string {
+        return `[${this.items.map((item) => item.toString()).join(', ')}]`;
+    }
+}
+
+export class InlineDictNode {
+    constructor(public items: Map<string, Node>) {}
+
+    get separator() {
+        return ' ';
+    }
+
+    toString(): string {
+        if (this.items.size === 0) {
+            return '{}';
+        }
+        const formatEntry = ([key, value]: [string, Node]): string => `${key}: ${value.toString()}`;
+        return `{ ${[...this.items.entries()].map(formatEntry).join(', ')} }`;
+    }
+}
+
+export class VerbatimStringNode {
+    constructor(public lines: string[]) {}
+
+    get separator() {
+        return ' ';
+    }
+
+    toString(indent: number = 0): string {
+        const formatLine = (line: string) => {
+            if (!line) {
+                return '';
+            }
+            return `${' '.repeat(4 * indent)}${line}`;
+        };
+        return `|\n${this.lines.map(formatLine).join('\n')}`;
+    }
+}
+
+export class FoldedStringNode {
+    constructor(public lines: string[]) {}
+
+    get separator() {
+        return ' ';
+    }
+
+    toString(indent: number = 0): string {
+        const formatLine = (line: string) => {
+            if (!line) {
+                return '';
+            }
+            return `${' '.repeat(4 * indent)}${line}`;
+        };
+        return `>\n${this.lines.map(formatLine).join('\n')}`;
+    }
+}
+
+export const parse = (value: string): Node => {
+    value = value.replaceAll('\r\n', '\n').replaceAll('\r', '\n');
+    const parser = new Parser();
+    return parser.parseLines(value.split('\n'));
+};
+
+const regexComment = / *#(?<comment>.*?)$/;
+const regexLineComment = new RegExp(`^${regexComment.source}$`);
+const regexPartComment = new RegExp(`(?:${regexComment.source})?$`);
+const regexPartDate = '(?:\\d\\d\\d\\d-\\d\\d-\\d\\d)';
+const regexPartTime = '(?:\\d\\d:\\d\\d:\\d\\d(?:[+-]\\d\\d\\d\\d)?)';
+const regexPartDictKey = '([^:#\' {}]+):';
+const regexPartInlineElement = '((?:[^,\']+)|(?:\'[^\']*\'))\\s*,?';
+
+const regexNull = new RegExp(`^~${regexPartComment.source}$`);
+const regexIntDec = new RegExp(`^([+-]?[0-9]+)${regexPartComment.source}$`);
+const regexIntBin = new RegExp(`^([+-])?0b([0-1]+)${regexPartComment.source}`);
+const regexIntOct = new RegExp(`^([+-])?0o([0-7]+)${regexPartComment.source}`);
+const regexIntHex = new RegExp(`^([+-])?0x([0-9A-Ea-e]+)${regexPartComment.source}`);
+const regexInf = new RegExp(`^([+-])?inf${regexPartComment.source}`);
+const regexFloat = new RegExp(`^([+-]?[0-9]*\\.[0-9]*([Ee][+-][0-9]+)?)${regexPartComment.source}`);
+const regexBool = new RegExp(`^(true|false)?${regexPartComment.source}`);
+const regexDatetime = new RegExp(`^(${regexPartDate}|${regexPartTime}|${regexPartDate} ${regexPartTime}|(?:@\\d+))${regexPartComment.source}`);
+const regexStringInline = new RegExp(`^'((?:[^']|'')*)'${regexPartComment.source}`);
+const regexStringBlock = new RegExp(`^([>|])${regexPartComment.source}`);
+const regexDict = new RegExp(`^${regexPartDictKey}( .*?|)$`);
+const regexListInline = new RegExp(`^\\[(.*)\\]${regexPartComment.source}`);
+const regexDictInline = new RegExp(`^\\{(.*)\\}${regexPartComment.source}`);
+const regexList = new RegExp('^-( .*?|)$');
+
+class Parser {
+    lines: string[] = [];
+    currentLineNumber: number = 0;
+    currentIndent: number = 0;
+    formatting: FormattingNode[] = [];
+
+    private get isCurrentLineValid(): boolean {
+        if (this.currentLineNumber >= this.lines.length) {
+            return false;
+        }
+        const currentLine = this.lines[this.currentLineNumber];
+        return currentLine.startsWith(' '.repeat(4 * this.currentIndent)) || currentLine.trim().length === 0;
+    }
+
+    private get currentLine() {
+        return this.lines[this.currentLineNumber].substring(4 * this.currentIndent);
+    }
+
+    private createError(message: string): ParseError {
+        return new ParseError(message, this.currentLineNumber, this.currentLine);
+    }
+
+    private getAndClearFormatting(): FormattingNode[] {
+        const formatting = this.formatting;
+        this.formatting = [];
+        return formatting;
+    }
+
+    parseLines(lines: string[]): Node {
+        this.lines = lines;
+        this.currentLineNumber = 0;
+        this.currentIndent = 0;
+        this.formatting = [];
+        return this.parse();
+    }
+
+    private parse(): Node {
+        let node: Node | undefined = undefined;
+        let match: RegExpMatchArray | null = null;
+
+        while (this.isCurrentLineValid) {
+            if (this.currentLine.trim().length === 0) {
+                this.formatting.push(new EmptyLineNode());
+                this.currentLineNumber++;
+                continue;
+            }
+            match = this.currentLine.match(regexLineComment);
+            if (match) {
+                this.formatting.push(new CommentNode(match.groups!.comment));
+                this.currentLineNumber++;
+                continue;
+            }
+
+            match = this.currentLine.match(regexDict);
+            if (match) {
+                if (node === undefined) {
+                    node = new DictNode([], this.getAndClearFormatting());
+                }
+                if (!(node instanceof DictNode)) {
+                    throw this.createError('Dict in wrong context');
+                }
+                const formatting = this.getAndClearFormatting();
+                node.items.push(new DictEntry(match[1].trim(), this.parseBlock(match[2].trim()), formatting));
+                continue;
+            }
+
+            match = this.currentLine.match(regexList);
+            if (match) {
+                if (node === undefined) {
+                    node = new ListNode([], this.getAndClearFormatting());
+                } else if (!(node instanceof ListNode)) {
+                    throw this.createError('List in wrong context');
+                }
+                const formatting = this.getAndClearFormatting();
+                node.items.push(new ListItem(this.parseBlock(match[1].trim()), formatting));
+                continue;
+            }
+
+            match = this.currentLine.match(regexStringBlock);
+            if (match) {
+                return this.parseBlock(match[1]);
+            }
+
+            if (node !== undefined) {
+                throw this.createError('Scalar in wrong context');
+            }
+
+            node = this.parseInline(this.currentLine);
+            this.currentLineNumber++;
+        }
+        if (node === undefined) {
+            throw new Error('undefined state');
+        }
+        return node;
+    }
+
+    private parseInline(value: string): InlineNode {
+        const formatting = this.getAndClearFormatting();
+
+        let match;
+        match = value.match(regexNull);
+        if (match) {
+            return new SingleLineScalarNode(null, formatting, match.groups?.comment?.trim());
+        }
+        match = value.match(regexIntDec);
+        if (match) {
+            return new SingleLineScalarNode(parseInt(match[1]), formatting, match.groups?.comment?.trim());
+        }
+        match = value.match(regexIntBin);
+        if (match) {
+            throw new Error('unimplemented int bin');
+        }
+        match = value.match(regexIntOct);
+        if (match) {
+            throw new Error('unimplemented int oct');
+        }
+        match = value.match(regexIntHex);
+        if (match) {
+            throw new Error('unimplemented int hex');
+        }
+        match = value.match(regexInf);
+        if (match) {
+            throw new Error('unimplemented inf');
+        }
+        match = value.match(regexFloat);
+        if (match) {
+            throw new Error('unimplemented float');
+        }
+        match = value.match(regexBool);
+        if (match) {
+            return new SingleLineScalarNode(match[1] === 'true', formatting, match.groups?.comment?.trim());
+        }
+        match = value.match(regexDatetime);
+        if (match) {
+            throw new Error('unimplemented date');
+        }
+        match = value.match(regexStringInline);
+        if (match) {
+            return new SingleLineScalarNode(match[1].replaceAll('\'\'', '\''), formatting, match.groups?.comment?.trim());
+        }
+        match = value.match(regexListInline);
+        if (match) {
+            const result = [];
+            let str = match[1].trim();
+            let itemMatch;
+            while (str.length > 0) {
+                itemMatch = str.match(new RegExp(`^${regexPartInlineElement}`));
+                if (itemMatch) {
+                    result.push(this.parseInline(itemMatch[1]));
+                    str = str.substr(itemMatch[0].length).trim();
+                } else {
+                    throw this.createError('');
+                }
+            }
+            return new InlineListNode(result);
+        }
+        match = value.match(regexDictInline);
+        if (match) {
+            const result = new Map();
+            let str = match[1].trim();
+            let itemMatch;
+            while (str.length > 0) {
+                itemMatch = str.match(new RegExp(`^${regexPartDictKey} *${regexPartInlineElement}`));
+                if (itemMatch) {
+                    result.set(itemMatch[1], this.parseInline(itemMatch[2]));
+                    str = str.substring(itemMatch[0].length).trim();
+                } else {
+                    throw this.createError('');
+                }
+            }
+            return new InlineDictNode(result);
+        }
+
+        throw this.createError('');
+    }
+
+    private parseBlock(value: string): Node {
+        switch (value.trim().substring(0, 1)) {
+            case '':
+            case '#': {
+                this.currentIndent++;
+                this.currentLineNumber++;
+                const block = this.parse();
+                this.currentIndent--;
+                return block;
+            }
+            case '|': {
+                this.currentIndent++;
+                this.currentLineNumber++;
+                const lines = [];
+                while (this.isCurrentLineValid) {
+                    lines.push(this.currentLine);
+                    this.currentLineNumber++;
+                }
+                this.currentIndent--;
+                return new VerbatimStringNode(lines);
+            }
+            case '>': {
+                this.currentIndent++;
+                this.currentLineNumber++;
+                const lines = [];
+                while (this.isCurrentLineValid) {
+                    lines.push(this.currentLine);
+                    this.currentLineNumber++;
+                }
+                this.currentIndent--;
+                return new FoldedStringNode(lines);
+            }
+            default: {
+                const node = this.parseInline(value);
+                this.currentLineNumber++;
+                return node;
+            }
+        }
+    }
+}

server/translationsHelper.ts

@@ -0,0 +1,81 @@
+import fs from 'node:fs/promises';
+
+import { rootDir } from '~/server/paths.ts';
+import { DictNode, parse } from '~/server/sumlAst.ts';
+import type { Node } from '~/server/sumlAst.ts';
+
+const loadDocument = async (name: string) => {
+    return parse(await fs.readFile(`${rootDir}/${name}.suml`, 'utf-8'));
+};
+
+const saveDocument = async (name: string, document: Node) => {
+    await fs.writeFile(`${rootDir}/${name}.suml`, `${document.toString()}\n`);
+};
+
+const mergeTranslationProposals = async (locale: string, proposalsFile: string) => {
+    const baseTranslationsDocument = await loadDocument('locale/_base/translations');
+    const localeTranslationsDocument = await loadDocument(`locale/${locale}/translations`);
+    const proposalsDocument = await loadDocument(proposalsFile);
+
+    if (!(proposalsDocument instanceof DictNode) ||
+        !(baseTranslationsDocument instanceof DictNode) ||
+        !(localeTranslationsDocument instanceof DictNode)) {
+        throw new Error('input nodes must be DictNode');
+    }
+
+    merge(proposalsDocument, baseTranslationsDocument, localeTranslationsDocument);
+    await saveDocument(`locale/${locale}/translations`, localeTranslationsDocument);
+};
+
+const merge = (source: DictNode, base: DictNode, target: DictNode) => {
+    const baseKeys = base.items.map((entry) => entry.key);
+    for (const sourceEntry of source.items) {
+        const baseEntry = base.items.find((baseEntry) => baseEntry.key === sourceEntry.key);
+        const targetEntry = target.items.find((targetEntry) => targetEntry.key === sourceEntry.key);
+        if (targetEntry === undefined) {
+            let insertIndex = 0;
+            if (baseEntry === undefined) {
+                insertIndex = target.items.length;
+            } else {
+                const baseIndex = baseKeys.indexOf(baseEntry.key);
+                const previousKeys = baseKeys.slice(0, baseIndex).toReversed();
+                for (const previousKey of previousKeys) {
+                    const targetIndex = target.items.findIndex((targetEntry) => targetEntry.key === previousKey);
+                    if (targetIndex !== -1) {
+                        insertIndex = targetIndex + 1;
+                        break;
+                    }
+                }
+            }
+            target.items.splice(insertIndex, 0, sourceEntry);
+        } else if (targetEntry.value instanceof DictNode) {
+            if (!(sourceEntry.value instanceof DictNode)) {
+                throw new Error('source entry must be an DictNode when merging');
+            }
+            const childBase = baseEntry?.value instanceof DictNode ? baseEntry.value : new DictNode([], []);
+            merge(sourceEntry.value, childBase, targetEntry.value);
+        } else {
+            targetEntry.value = sourceEntry.value;
+        }
+    }
+};
+
+const main = async () => {
+    if (process.argv.length < 2) {
+        console.log('Missing action.');
+        return;
+    }
+
+    switch (process.argv[2]) {
+        case 'merge':
+            if (process.argv.length < 4) {
+                console.error('Missing locale and proposal file');
+            }
+            await mergeTranslationProposals(process.argv[3], process.argv[4]);
+            break;
+        default:
+            console.error(`Unknown action ${process.argv[2]}`);
+    }
+};
+
+await main();