From 832c0035cf11caa1b077e49c64c4f02e71dda7a0 Mon Sep 17 00:00:00 2001 From: Valentyne Stigloher Date: Wed, 2 Oct 2024 18:43:48 +0200 Subject: [PATCH 1/8] (helper) helper for merging proposals to translations.suml --- server/sumlAst.ts | 414 +++++++++++++++++++++++++++++++++++ server/translationsHelper.ts | 81 +++++++ 2 files changed, 495 insertions(+) create mode 100644 server/sumlAst.ts create mode 100644 server/translationsHelper.ts diff --git a/server/sumlAst.ts b/server/sumlAst.ts new file mode 100644 index 000000000..bc9b171c9 --- /dev/null +++ b/server/sumlAst.ts @@ -0,0 +1,414 @@ +class ParseError extends Error { + line: number; + context: string; + + constructor(message = '', line = -1, context = '') { + let combinedMessage = `Cannot parse line ${line + 1}`; + if (context !== undefined) { + combinedMessage += ` near \`${context}\``; + } + if (message) { + combinedMessage += `: ${message}`; + } + super(combinedMessage); + this.line = line; + this.context = context; + } +} + +type FormattingNode = EmptyLineNode | CommentNode; + +export class EmptyLineNode { + toString(): string { + return '\n'; + } +} + +export class CommentNode { + constructor(public comment: string) {} + + toString(indent: number = 0) { + return `${' '.repeat(4 * indent)}#${this.comment}\n`; + } +} + +export type Node = DictNode | ListNode | InlineNode | VerbatimStringNode | FoldedStringNode; + +export class DictNode { + constructor(public items: DictEntry[], public formatting: FormattingNode[]) {} + + get separator() { + return '\n'; + } + + toString(indent: number = 0): string { + return this.items.map((item) => item.toString(indent)).join('\n'); + } +} + +export class DictEntry { + constructor(public key: string, public value: Node, public formatting: FormattingNode[]) {} + + toString(indent: number = 0): string { + return `${this.formatting.map((node) => node.toString(indent)).join('')}${' '.repeat(4 * indent)}${this.key}:${this.value.separator}${this.value.toString(indent + 1)}`; + } +} + +export class ListNode { + constructor(public items: ListItem[], public formatting: FormattingNode[]) {} + + get separator() { + return '\n'; + } + + toString(indent: number = 0): string { + return this.items.map((item) => item.toString(indent)).join('\n'); + } +} + +export class ListItem { + constructor(public value: Node, public formatting: FormattingNode[]) {} + + toString(indent: number = 0): string { + return `${this.formatting.map((node) => node.toString(indent)).join('')}${' '.repeat(4 * indent)}-${this.value.separator}${this.value.toString(indent + 1)}`; + } +} + +type InlineNode = SingleLineScalarNode | InlineDictNode | InlineListNode; + +export class SingleLineScalarNode { + constructor( + public value: null | boolean | number | string | Date, + public formatting: FormattingNode[], + public comment: string | undefined, + ) {} + + get separator() { + return ' '; + } + + get representation(): string { + if (this.value === null) { + return '~'; + } + if (typeof this.value === 'string') { + return `'${this.value.replaceAll('\'', '\'\'')}'`; + } + return this.value.toString(); + } + + toString(): string { + return `${this.representation}${this.comment ? ` # ${this.comment}` : ''}`; + } +} + +export class InlineListNode { + constructor(public items: Node[]) {} + + get separator() { + return ' '; + } + + toString(): string { + return `[${this.items.map((item) => item.toString()).join(', ')}]`; + } +} + +export class InlineDictNode { + constructor(public items: Map) {} + + get separator() { + return ' '; + } + + toString(): string { + if (this.items.size === 0) { + return '{}'; + } + const formatEntry = ([key, value]: [string, Node]): string => `${key}: ${value.toString()}`; + return `{ ${[...this.items.entries()].map(formatEntry).join(', ')} }`; + } +} + +export class VerbatimStringNode { + constructor(public lines: string[]) {} + + get separator() { + return ' '; + } + + toString(indent: number = 0): string { + const formatLine = (line: string) => { + if (!line) { + return ''; + } + return `${' '.repeat(4 * indent)}${line}`; + }; + return `|\n${this.lines.map(formatLine).join('\n')}`; + } +} + +export class FoldedStringNode { + constructor(public lines: string[]) {} + + get separator() { + return ' '; + } + + toString(indent: number = 0): string { + const formatLine = (line: string) => { + if (!line) { + return ''; + } + return `${' '.repeat(4 * indent)}${line}`; + }; + return `>\n${this.lines.map(formatLine).join('\n')}`; + } +} + +export const parse = (value: string): Node => { + value = value.replaceAll('\r\n', '\n').replaceAll('\r', '\n'); + const parser = new Parser(); + return parser.parseLines(value.split('\n')); +}; + +const regexComment = / *#(?.*?)$/; +const regexLineComment = new RegExp(`^${regexComment.source}$`); +const regexPartComment = new RegExp(`(?:${regexComment.source})?$`); +const regexPartDate = '(?:\\d\\d\\d\\d-\\d\\d-\\d\\d)'; +const regexPartTime = '(?:\\d\\d:\\d\\d:\\d\\d(?:[+-]\\d\\d\\d\\d)?)'; +const regexPartDictKey = '([^:#\' {}]+):'; +const regexPartInlineElement = '((?:[^,\']+)|(?:\'[^\']*\'))\\s*,?'; + +const regexNull = new RegExp(`^~${regexPartComment.source}$`); +const regexIntDec = new RegExp(`^([+-]?[0-9]+)${regexPartComment.source}$`); +const regexIntBin = new RegExp(`^([+-])?0b([0-1]+)${regexPartComment.source}`); +const regexIntOct = new RegExp(`^([+-])?0o([0-7]+)${regexPartComment.source}`); +const regexIntHex = new RegExp(`^([+-])?0x([0-9A-Ea-e]+)${regexPartComment.source}`); +const regexInf = new RegExp(`^([+-])?inf${regexPartComment.source}`); +const regexFloat = new RegExp(`^([+-]?[0-9]*\\.[0-9]*([Ee][+-][0-9]+)?)${regexPartComment.source}`); +const regexBool = new RegExp(`^(true|false)?${regexPartComment.source}`); +const regexDatetime = new RegExp(`^(${regexPartDate}|${regexPartTime}|${regexPartDate} ${regexPartTime}|(?:@\\d+))${regexPartComment.source}`); +const regexStringInline = new RegExp(`^'((?:[^']|'')*)'${regexPartComment.source}`); +const regexStringBlock = new RegExp(`^([>|])${regexPartComment.source}`); +const regexDict = new RegExp(`^${regexPartDictKey}( .*?|)$`); +const regexListInline = new RegExp(`^\\[(.*)\\]${regexPartComment.source}`); +const regexDictInline = new RegExp(`^\\{(.*)\\}${regexPartComment.source}`); +const regexList = new RegExp('^-( .*?|)$'); + +class Parser { + lines: string[] = []; + currentLineNumber: number = 0; + currentIndent: number = 0; + formatting: FormattingNode[] = []; + + private get isCurrentLineValid(): boolean { + if (this.currentLineNumber >= this.lines.length) { + return false; + } + const currentLine = this.lines[this.currentLineNumber]; + return currentLine.startsWith(' '.repeat(4 * this.currentIndent)) || currentLine.trim().length === 0; + } + + private get currentLine() { + return this.lines[this.currentLineNumber].substring(4 * this.currentIndent); + } + + private createError(message: string): ParseError { + return new ParseError(message, this.currentLineNumber, this.currentLine); + } + + private getAndClearFormatting(): FormattingNode[] { + const formatting = this.formatting; + this.formatting = []; + return formatting; + } + + parseLines(lines: string[]): Node { + this.lines = lines; + this.currentLineNumber = 0; + this.currentIndent = 0; + this.formatting = []; + return this.parse(); + } + + private parse(): Node { + let node: Node | undefined = undefined; + let match: RegExpMatchArray | null = null; + + while (this.isCurrentLineValid) { + if (this.currentLine.trim().length === 0) { + this.formatting.push(new EmptyLineNode()); + this.currentLineNumber++; + continue; + } + match = this.currentLine.match(regexLineComment); + if (match) { + this.formatting.push(new CommentNode(match.groups!.comment)); + this.currentLineNumber++; + continue; + } + + match = this.currentLine.match(regexDict); + if (match) { + if (node === undefined) { + node = new DictNode([], this.getAndClearFormatting()); + } + if (!(node instanceof DictNode)) { + throw this.createError('Dict in wrong context'); + } + const formatting = this.getAndClearFormatting(); + node.items.push(new DictEntry(match[1].trim(), this.parseBlock(match[2].trim()), formatting)); + continue; + } + + match = this.currentLine.match(regexList); + if (match) { + if (node === undefined) { + node = new ListNode([], this.getAndClearFormatting()); + } else if (!(node instanceof ListNode)) { + throw this.createError('List in wrong context'); + } + const formatting = this.getAndClearFormatting(); + node.items.push(new ListItem(this.parseBlock(match[1].trim()), formatting)); + continue; + } + + match = this.currentLine.match(regexStringBlock); + if (match) { + return this.parseBlock(match[1]); + } + + if (node !== undefined) { + throw this.createError('Scalar in wrong context'); + } + + node = this.parseInline(this.currentLine); + this.currentLineNumber++; + } + if (node === undefined) { + throw new Error('undefined state'); + } + return node; + } + + private parseInline(value: string): InlineNode { + const formatting = this.getAndClearFormatting(); + + let match; + match = value.match(regexNull); + if (match) { + return new SingleLineScalarNode(null, formatting, match.groups?.comment?.trim()); + } + match = value.match(regexIntDec); + if (match) { + return new SingleLineScalarNode(parseInt(match[1]), formatting, match.groups?.comment?.trim()); + } + match = value.match(regexIntBin); + if (match) { + throw new Error('unimplemented int bin'); + } + match = value.match(regexIntOct); + if (match) { + throw new Error('unimplemented int oct'); + } + match = value.match(regexIntHex); + if (match) { + throw new Error('unimplemented int hex'); + } + match = value.match(regexInf); + if (match) { + throw new Error('unimplemented inf'); + } + match = value.match(regexFloat); + if (match) { + throw new Error('unimplemented float'); + } + match = value.match(regexBool); + if (match) { + return new SingleLineScalarNode(match[1] === 'true', formatting, match.groups?.comment?.trim()); + } + match = value.match(regexDatetime); + if (match) { + throw new Error('unimplemented date'); + } + match = value.match(regexStringInline); + if (match) { + return new SingleLineScalarNode(match[1].replaceAll('\'\'', '\''), formatting, match.groups?.comment?.trim()); + } + match = value.match(regexListInline); + if (match) { + const result = []; + let str = match[1].trim(); + let itemMatch; + while (str.length > 0) { + itemMatch = str.match(new RegExp(`^${regexPartInlineElement}`)); + if (itemMatch) { + result.push(this.parseInline(itemMatch[1])); + str = str.substr(itemMatch[0].length).trim(); + } else { + throw this.createError(''); + } + } + return new InlineListNode(result); + } + match = value.match(regexDictInline); + if (match) { + const result = new Map(); + let str = match[1].trim(); + let itemMatch; + while (str.length > 0) { + itemMatch = str.match(new RegExp(`^${regexPartDictKey} *${regexPartInlineElement}`)); + if (itemMatch) { + result.set(itemMatch[1], this.parseInline(itemMatch[2])); + str = str.substring(itemMatch[0].length).trim(); + } else { + throw this.createError(''); + } + } + return new InlineDictNode(result); + } + + throw this.createError(''); + } + + private parseBlock(value: string): Node { + switch (value.trim().substring(0, 1)) { + case '': + case '#': { + this.currentIndent++; + this.currentLineNumber++; + const block = this.parse(); + this.currentIndent--; + return block; + } + case '|': { + this.currentIndent++; + this.currentLineNumber++; + const lines = []; + while (this.isCurrentLineValid) { + lines.push(this.currentLine); + this.currentLineNumber++; + } + this.currentIndent--; + return new VerbatimStringNode(lines); + } + case '>': { + this.currentIndent++; + this.currentLineNumber++; + const lines = []; + while (this.isCurrentLineValid) { + lines.push(this.currentLine); + this.currentLineNumber++; + } + this.currentIndent--; + return new FoldedStringNode(lines); + } + default: { + const node = this.parseInline(value); + this.currentLineNumber++; + return node; + } + } + } +} diff --git a/server/translationsHelper.ts b/server/translationsHelper.ts new file mode 100644 index 000000000..eb981e4d8 --- /dev/null +++ b/server/translationsHelper.ts @@ -0,0 +1,81 @@ +import fs from 'node:fs/promises'; + +import { rootDir } from '~/server/paths.ts'; +import { DictNode, parse } from '~/server/sumlAst.ts'; +import type { Node } from '~/server/sumlAst.ts'; + +const loadDocument = async (name: string) => { + return parse(await fs.readFile(`${rootDir}/${name}.suml`, 'utf-8')); +}; + +const saveDocument = async (name: string, document: Node) => { + await fs.writeFile(`${rootDir}/${name}.suml`, `${document.toString()}\n`); +}; + +const mergeTranslationProposals = async (locale: string, proposalsFile: string) => { + const baseTranslationsDocument = await loadDocument('locale/_base/translations'); + const localeTranslationsDocument = await loadDocument(`locale/${locale}/translations`); + const proposalsDocument = await loadDocument(proposalsFile); + + if (!(proposalsDocument instanceof DictNode) || + !(baseTranslationsDocument instanceof DictNode) || + !(localeTranslationsDocument instanceof DictNode)) { + throw new Error('input nodes must be DictNode'); + } + + merge(proposalsDocument, baseTranslationsDocument, localeTranslationsDocument); + await saveDocument(`locale/${locale}/translations`, localeTranslationsDocument); +}; + +const merge = (source: DictNode, base: DictNode, target: DictNode) => { + const baseKeys = base.items.map((entry) => entry.key); + for (const sourceEntry of source.items) { + const baseEntry = base.items.find((baseEntry) => baseEntry.key === sourceEntry.key); + const targetEntry = target.items.find((targetEntry) => targetEntry.key === sourceEntry.key); + if (targetEntry === undefined) { + let insertIndex = 0; + if (baseEntry === undefined) { + insertIndex = target.items.length; + } else { + const baseIndex = baseKeys.indexOf(baseEntry.key); + const previousKeys = baseKeys.slice(0, baseIndex).toReversed(); + for (const previousKey of previousKeys) { + const targetIndex = target.items.findIndex((targetEntry) => targetEntry.key === previousKey); + if (targetIndex !== -1) { + insertIndex = targetIndex + 1; + break; + } + } + } + target.items.splice(insertIndex, 0, sourceEntry); + } else if (targetEntry.value instanceof DictNode) { + if (!(sourceEntry.value instanceof DictNode)) { + throw new Error('source entry must be an DictNode when merging'); + } + const childBase = baseEntry?.value instanceof DictNode ? baseEntry.value : new DictNode([], []); + merge(sourceEntry.value, childBase, targetEntry.value); + } else { + targetEntry.value = sourceEntry.value; + } + } +}; + +const main = async () => { + if (process.argv.length < 2) { + console.log('Missing action.'); + return; + } + + switch (process.argv[2]) { + case 'merge': + if (process.argv.length < 4) { + console.error('Missing locale and proposal file'); + } + await mergeTranslationProposals(process.argv[3], process.argv[4]); + break; + default: + console.error(`Unknown action ${process.argv[2]}`); + } +}; + +await main(); From 9f089578febf33b4aee609afb742e054eabe8734 Mon Sep 17 00:00:00 2001 From: Valentyne Stigloher Date: Sun, 27 Apr 2025 16:54:50 +0200 Subject: [PATCH 2/8] (helper) create minimal translations.suml --- server/translationsHelper.ts | 47 ++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/server/translationsHelper.ts b/server/translationsHelper.ts index eb981e4d8..d201425b0 100644 --- a/server/translationsHelper.ts +++ b/server/translationsHelper.ts @@ -1,8 +1,16 @@ import fs from 'node:fs/promises'; +import { deepKeys } from 'dot-prop'; +import Suml from 'suml'; + +import type { Config } from '~/locale/config.ts'; +import type { Translations } from '~/locale/translations.ts'; +import { loadSuml } from '~/server/loader.ts'; import { rootDir } from '~/server/paths.ts'; import { DictNode, parse } from '~/server/sumlAst.ts'; import type { Node } from '~/server/sumlAst.ts'; +import { deepGet, deepListKeys, deepSet } from '~/src/helpers.ts'; +import { listMissingTranslations } from '~/src/missingTranslations.ts'; const loadDocument = async (name: string) => { return parse(await fs.readFile(`${rootDir}/${name}.suml`, 'utf-8')); @@ -60,6 +68,42 @@ const merge = (source: DictNode, base: DictNode, target: DictNode) => { } }; +const createTranslationFiles = async (locale: string) => { + const baseTranslations = await loadSuml('locale/_base/translations.suml'); + const config = await loadSuml('locale/_base/config.suml'); + + const configDocument = await loadDocument('locale/_base/config'); + const translationsDocument = await loadDocument('locale/_base/translations'); + + const requiredTranslationKeys = listMissingTranslations({}, baseTranslations, config); + for (const key of deepListKeys(baseTranslations)) { + if (!requiredTranslationKeys.includes(key)) { + remove(translationsDocument, key.split('.')); + } + } + + await fs.mkdir(`${rootDir}/locale/${locale}`, { recursive: true }); + await saveDocument(`locale/${locale}/config`, configDocument); + await saveDocument(`locale/${locale}/translations`, translationsDocument); +}; + +const remove = (node: Node, path: string[]) => { + if (!(node instanceof DictNode)) { + throw new Error('node must be DictNode'); + } + + if (path.length === 1) { + node.items = node.items.filter((item) => item.key !== path[0]); + return; + } + const childNode = node.items.find((item) => item.key === path[0])?.value; + if (!childNode) { + return; + } + remove(childNode, path.slice(1)); + node.items = node.items.filter((item) => !(item.value instanceof DictNode) || item.value.items.length > 0); +}; + const main = async () => { if (process.argv.length < 2) { console.log('Missing action.'); @@ -73,6 +117,9 @@ const main = async () => { } await mergeTranslationProposals(process.argv[3], process.argv[4]); break; + case 'create': + await createTranslationFiles(process.argv[3]); + break; default: console.error(`Unknown action ${process.argv[2]}`); } From 056d921c4b0d47dc6eba451745aa37d3e1398d7d Mon Sep 17 00:00:00 2001 From: Benjamin Date: Thu, 12 Jun 2025 17:33:39 -0400 Subject: [PATCH 3/8] (lint) --- server/translationsHelper.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/server/translationsHelper.ts b/server/translationsHelper.ts index d201425b0..1b8c70822 100644 --- a/server/translationsHelper.ts +++ b/server/translationsHelper.ts @@ -1,7 +1,9 @@ import fs from 'node:fs/promises'; -import { deepKeys } from 'dot-prop'; -import Suml from 'suml'; +/* Commented out due to being unused (prevents lint error) +import { deepKeys as _deepKeys } from 'dot-prop'; +import _Suml from 'suml'; +*/ import type { Config } from '~/locale/config.ts'; import type { Translations } from '~/locale/translations.ts'; @@ -9,7 +11,9 @@ import { loadSuml } from '~/server/loader.ts'; import { rootDir } from '~/server/paths.ts'; import { DictNode, parse } from '~/server/sumlAst.ts'; import type { Node } from '~/server/sumlAst.ts'; -import { deepGet, deepListKeys, deepSet } from '~/src/helpers.ts'; +import { deepListKeys } from '~/src/helpers.ts'; +// Commented out due to being unused (prevents linting error) +// import { deepSet, deepGet } from '~/src/helpers.ts'; import { listMissingTranslations } from '~/src/missingTranslations.ts'; const loadDocument = async (name: string) => { From 4a41e6966d3458b4c1db34f018d78ad06a58339d Mon Sep 17 00:00:00 2001 From: Benjamin Date: Fri, 13 Jun 2025 15:45:32 -0400 Subject: [PATCH 4/8] CLI Finalizations --- helpers/merge/index.ts | 69 ++++++++++++++++ {server => helpers/merge}/sumlAst.ts | 40 ++++++---- .../merge}/translationsHelper.ts | 46 ++--------- package.json | 4 +- pnpm-lock.yaml | 78 +++++++++++++++++++ 5 files changed, 184 insertions(+), 53 deletions(-) create mode 100644 helpers/merge/index.ts rename {server => helpers/merge}/sumlAst.ts (92%) rename {server => helpers/merge}/translationsHelper.ts (74%) diff --git a/helpers/merge/index.ts b/helpers/merge/index.ts new file mode 100644 index 000000000..b00dde787 --- /dev/null +++ b/helpers/merge/index.ts @@ -0,0 +1,69 @@ +import yargs from 'yargs'; +import { hideBin } from 'yargs/helpers'; + +import { mergeTranslationProposals, createTranslationFiles } from '~/helpers/merge/translationsHelper.ts'; +import Locales from '~/locale/locales.ts'; + +yargs(hideBin(process.argv)) + .scriptName('merger') + .help() + .showHidden() + .version() + .command( + 'merge', + 'Merge in pending translation changes', + (build) => { + build + .options({ + l: { + alias: ['locale'], + describe: 'Locale to be merged into', + demandOption: true, + type: 'string', + nargs: 1, + }, + f: { + alias: ['file'], + describe: 'The file location of the pending translations', + demandOption: true, + normalise: true, + nargs: 1, + }, + }) + .usage('$0 merge -l -f '); + }, + async (args) => { + if (!Locales.map((l) => l.code).includes(String(args.l).toLowerCase() as string)) { + throw new RangeError('Locale Code does not exist, please check your spelling'); + } + await mergeTranslationProposals(String(args.l).toLowerCase(), args.f as string); + }, + ) + .command( + 'create', + 'Create a new language system', + (build) => { + build + .options({ + l: { + alias: ['locale'], + describe: 'Locale to be merged into', + demandOption: true, + type: 'string', + nargs: 1, + }, + }) + .usage('$0 create -l '); + }, + async (args) => { + await createTranslationFiles(String(args.l).toLowerCase() as string); + }, + ) + .example([ + ['$0 merge -l en -f ./to-merge.suml', 'Merge pending translations into the English (en) locale'], + ['$0 create -l he', 'Create a new Hebrew (he) locale'], + ]) + .usage('$0 [options]') + .showHelpOnFail(false, 'Specify --help for available options') + .wrap(null) + .parse(); diff --git a/server/sumlAst.ts b/helpers/merge/sumlAst.ts similarity index 92% rename from server/sumlAst.ts rename to helpers/merge/sumlAst.ts index bc9b171c9..a0f374902 100644 --- a/server/sumlAst.ts +++ b/helpers/merge/sumlAst.ts @@ -25,7 +25,7 @@ export class EmptyLineNode { } export class CommentNode { - constructor(public comment: string) {} + constructor(public comment: string) { } toString(indent: number = 0) { return `${' '.repeat(4 * indent)}#${this.comment}\n`; @@ -34,8 +34,12 @@ export class CommentNode { export type Node = DictNode | ListNode | InlineNode | VerbatimStringNode | FoldedStringNode; +export function initFormatting(formatting: FormattingNode[], indent: number) { + return `${formatting.map((node) => node.toString(indent)).join('')}${' '.repeat(4 * indent)}`; +} + export class DictNode { - constructor(public items: DictEntry[], public formatting: FormattingNode[]) {} + constructor(public items: DictEntry[], public formatting: FormattingNode[]) { } get separator() { return '\n'; @@ -47,15 +51,17 @@ export class DictNode { } export class DictEntry { - constructor(public key: string, public value: Node, public formatting: FormattingNode[]) {} + constructor(public key: string, public value: Node, public formatting: FormattingNode[]) { } toString(indent: number = 0): string { - return `${this.formatting.map((node) => node.toString(indent)).join('')}${' '.repeat(4 * indent)}${this.key}:${this.value.separator}${this.value.toString(indent + 1)}`; + return `${initFormatting(this.formatting, indent) + }${this.key}:${this.value.separator + }${this.value.toString(indent + 1)}`; } } export class ListNode { - constructor(public items: ListItem[], public formatting: FormattingNode[]) {} + constructor(public items: ListItem[], public formatting: FormattingNode[]) { } get separator() { return '\n'; @@ -67,10 +73,10 @@ export class ListNode { } export class ListItem { - constructor(public value: Node, public formatting: FormattingNode[]) {} + constructor(public value: Node, public formatting: FormattingNode[]) { } toString(indent: number = 0): string { - return `${this.formatting.map((node) => node.toString(indent)).join('')}${' '.repeat(4 * indent)}-${this.value.separator}${this.value.toString(indent + 1)}`; + return `${initFormatting(this.formatting, indent)}-${this.value.separator}${this.value.toString(indent + 1)}`; } } @@ -81,7 +87,7 @@ export class SingleLineScalarNode { public value: null | boolean | number | string | Date, public formatting: FormattingNode[], public comment: string | undefined, - ) {} + ) { } get separator() { return ' '; @@ -103,7 +109,7 @@ export class SingleLineScalarNode { } export class InlineListNode { - constructor(public items: Node[]) {} + constructor(public items: Node[]) { } get separator() { return ' '; @@ -115,7 +121,7 @@ export class InlineListNode { } export class InlineDictNode { - constructor(public items: Map) {} + constructor(public items: Map) { } get separator() { return ' '; @@ -131,7 +137,7 @@ export class InlineDictNode { } export class VerbatimStringNode { - constructor(public lines: string[]) {} + constructor(public lines: string[]) { } get separator() { return ' '; @@ -149,7 +155,7 @@ export class VerbatimStringNode { } export class FoldedStringNode { - constructor(public lines: string[]) {} + constructor(public lines: string[]) { } get separator() { return ' '; @@ -188,7 +194,9 @@ const regexIntHex = new RegExp(`^([+-])?0x([0-9A-Ea-e]+)${regexPartComment.sourc const regexInf = new RegExp(`^([+-])?inf${regexPartComment.source}`); const regexFloat = new RegExp(`^([+-]?[0-9]*\\.[0-9]*([Ee][+-][0-9]+)?)${regexPartComment.source}`); const regexBool = new RegExp(`^(true|false)?${regexPartComment.source}`); -const regexDatetime = new RegExp(`^(${regexPartDate}|${regexPartTime}|${regexPartDate} ${regexPartTime}|(?:@\\d+))${regexPartComment.source}`); +const regexDatetime = new RegExp( + `^(${regexPartDate}|${regexPartTime}|${regexPartDate} ${regexPartTime}|(?:@\\d+))${regexPartComment.source}`, +); const regexStringInline = new RegExp(`^'((?:[^']|'')*)'${regexPartComment.source}`); const regexStringBlock = new RegExp(`^([>|])${regexPartComment.source}`); const regexDict = new RegExp(`^${regexPartDictKey}( .*?|)$`); @@ -334,7 +342,11 @@ class Parser { } match = value.match(regexStringInline); if (match) { - return new SingleLineScalarNode(match[1].replaceAll('\'\'', '\''), formatting, match.groups?.comment?.trim()); + return new SingleLineScalarNode( + match[1].replaceAll('\'\'', '\''), + formatting, + match.groups?.comment?.trim(), + ); } match = value.match(regexListInline); if (match) { diff --git a/server/translationsHelper.ts b/helpers/merge/translationsHelper.ts similarity index 74% rename from server/translationsHelper.ts rename to helpers/merge/translationsHelper.ts index 1b8c70822..e47087db5 100644 --- a/server/translationsHelper.ts +++ b/helpers/merge/translationsHelper.ts @@ -1,30 +1,23 @@ import fs from 'node:fs/promises'; -/* Commented out due to being unused (prevents lint error) -import { deepKeys as _deepKeys } from 'dot-prop'; -import _Suml from 'suml'; -*/ - +import { DictNode, parse } from '~/helpers/merge/sumlAst.ts'; +import type { Node } from '~/helpers/merge/sumlAst.ts'; import type { Config } from '~/locale/config.ts'; import type { Translations } from '~/locale/translations.ts'; import { loadSuml } from '~/server/loader.ts'; import { rootDir } from '~/server/paths.ts'; -import { DictNode, parse } from '~/server/sumlAst.ts'; -import type { Node } from '~/server/sumlAst.ts'; import { deepListKeys } from '~/src/helpers.ts'; -// Commented out due to being unused (prevents linting error) -// import { deepSet, deepGet } from '~/src/helpers.ts'; import { listMissingTranslations } from '~/src/missingTranslations.ts'; -const loadDocument = async (name: string) => { +export const loadDocument = async (name: string) => { return parse(await fs.readFile(`${rootDir}/${name}.suml`, 'utf-8')); }; -const saveDocument = async (name: string, document: Node) => { +export const saveDocument = async (name: string, document: Node) => { await fs.writeFile(`${rootDir}/${name}.suml`, `${document.toString()}\n`); }; -const mergeTranslationProposals = async (locale: string, proposalsFile: string) => { +export const mergeTranslationProposals = async (locale: string, proposalsFile: string) => { const baseTranslationsDocument = await loadDocument('locale/_base/translations'); const localeTranslationsDocument = await loadDocument(`locale/${locale}/translations`); const proposalsDocument = await loadDocument(proposalsFile); @@ -39,7 +32,7 @@ const mergeTranslationProposals = async (locale: string, proposalsFile: string) await saveDocument(`locale/${locale}/translations`, localeTranslationsDocument); }; -const merge = (source: DictNode, base: DictNode, target: DictNode) => { +export const merge = (source: DictNode, base: DictNode, target: DictNode) => { const baseKeys = base.items.map((entry) => entry.key); for (const sourceEntry of source.items) { const baseEntry = base.items.find((baseEntry) => baseEntry.key === sourceEntry.key); @@ -72,7 +65,7 @@ const merge = (source: DictNode, base: DictNode, target: DictNode) => { } }; -const createTranslationFiles = async (locale: string) => { +export const createTranslationFiles = async (locale: string) => { const baseTranslations = await loadSuml('locale/_base/translations.suml'); const config = await loadSuml('locale/_base/config.suml'); @@ -91,7 +84,7 @@ const createTranslationFiles = async (locale: string) => { await saveDocument(`locale/${locale}/translations`, translationsDocument); }; -const remove = (node: Node, path: string[]) => { +export const remove = (node: Node, path: string[]) => { if (!(node instanceof DictNode)) { throw new Error('node must be DictNode'); } @@ -107,26 +100,3 @@ const remove = (node: Node, path: string[]) => { remove(childNode, path.slice(1)); node.items = node.items.filter((item) => !(item.value instanceof DictNode) || item.value.items.length > 0); }; - -const main = async () => { - if (process.argv.length < 2) { - console.log('Missing action.'); - return; - } - - switch (process.argv[2]) { - case 'merge': - if (process.argv.length < 4) { - console.error('Missing locale and proposal file'); - } - await mergeTranslationProposals(process.argv[3], process.argv[4]); - break; - case 'create': - await createTranslationFiles(process.argv[3]); - break; - default: - console.error(`Unknown action ${process.argv[2]}`); - } -}; - -await main(); diff --git a/package.json b/package.json index 760e6a9cb..69ff76c80 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "twitter": "^1.7.1", "ulid": "^2.3.0", "uuid": "^8.3.2", + "yargs": "^18.0.0", "zh_cn_zh_tw": "^1.0.7" }, "devDependencies": { @@ -83,6 +84,7 @@ "@types/papaparse": "^5.3.14", "@types/speakeasy": "^2.0.10", "@types/uuid": "8.3.2", + "@types/yargs": "^17.0.33", "@vite-pwa/nuxt": "^0.10.6", "@vitest/coverage-v8": "^3.1.2", "@vue/test-utils": "^2.4.6", @@ -124,7 +126,7 @@ "vue-tsc": "^2.2.10", "vuedraggable": "^4.1.0" }, - "packageManager": "pnpm@10.4.1+sha256.4b702887986995933d4300836b04d6d02a43bc72b52e4f7e93a4ca608b959197", + "packageManager": "pnpm@10.12.1", "pnpm": { "onlyBuiltDependencies": [ "@parcel/watcher", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index fbc78cd7d..7a9c6e6f2 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -128,6 +128,9 @@ importers: uuid: specifier: ^8.3.2 version: 8.3.2 + yargs: + specifier: ^18.0.0 + version: 18.0.0 zh_cn_zh_tw: specifier: ^1.0.7 version: 1.0.7 @@ -204,6 +207,9 @@ importers: '@types/uuid': specifier: 8.3.2 version: 8.3.2 + '@types/yargs': + specifier: ^17.0.33 + version: 17.0.33 '@vite-pwa/nuxt': specifier: ^0.10.6 version: 0.10.6(magicast@0.3.5)(vite@6.3.5(@types/node@22.15.29)(jiti@2.4.2)(sass@1.32.12)(terser@5.33.0)(yaml@2.7.0))(workbox-build@7.3.0)(workbox-window@7.3.0) @@ -2852,6 +2858,12 @@ packages: '@types/web-bluetooth@0.0.21': resolution: {integrity: sha512-oIQLCGWtcFZy2JW77j9k8nHzAOpqMHLQejDA48XXMWH6tjCQHz5RCFz1bzsmROyL6PUm+LLnUiI4BCn221inxA==} + '@types/yargs-parser@21.0.3': + resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==} + + '@types/yargs@17.0.33': + resolution: {integrity: sha512-WpxBCKWPLr4xSsHgz511rFJAM+wS28w2zEO1QDNY5zM/S8ok70NNfztH0xwhqKyaK0OHCbN98LDAZuy1ctxDkA==} + '@types/yauzl@2.10.3': resolution: {integrity: sha512-oJoftv0LSuaDZE3Le4DbKX+KS9G36NzOeSap90UIK0yMA/NhKJhqlSGtNDORNRaIbQfzjXDrQa0ytJ6mNRGz/Q==} @@ -3695,6 +3707,10 @@ packages: resolution: {integrity: sha512-BSeNnyus75C4//NQ9gQt1/csTXyo/8Sb+afLAkzAptFuMsod9HFokGNudZpi/oQV73hnVK+sR+5PVRMd+Dr7YQ==} engines: {node: '>=12'} + cliui@9.0.1: + resolution: {integrity: sha512-k7ndgKhwoQveBL+/1tqGJYNz097I7WOvwbmmU2AR5+magtbjPWQTS1C5vzGkBC8Ym8UWRzfKUzUUqFLypY4Q+w==} + engines: {node: '>=20'} + cluster-key-slot@1.1.2: resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} engines: {node: '>=0.10.0'} @@ -4284,6 +4300,9 @@ packages: elliptic@6.5.7: resolution: {integrity: sha512-ESVCtTwiA+XhY3wyh24QqRGBoP3rEdDUl3EDUUo9tft074fi19IrdpH7hLCMMP3CIj7jb3W96rn8lt/BqIlt5Q==} + emoji-regex@10.4.0: + resolution: {integrity: sha512-EC+0oUMY1Rqm4O6LLrgjtYDvcVYTy7chDnM4Q7030tP4Kwj3u/pR6gP9ygnp2CJMK5Gq+9Q2oqmrFJAz01DXjw==} + emoji-regex@8.0.0: resolution: {integrity: sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==} @@ -4869,6 +4888,10 @@ packages: resolution: {integrity: sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg==} engines: {node: 6.* || 8.* || >= 10.*} + get-east-asian-width@1.3.0: + resolution: {integrity: sha512-vpeMIQKxczTD/0s2CdEWHcb0eeJe6TFjxb+J5xgX7hScxqrGuyjmv4c1D4A/gelKfyox0gJJwIHF+fLjeaM8kQ==} + engines: {node: '>=18'} + get-intrinsic@1.3.0: resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} engines: {node: '>= 0.4'} @@ -7440,6 +7463,10 @@ packages: resolution: {integrity: sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==} engines: {node: '>=12'} + string-width@7.2.0: + resolution: {integrity: sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==} + engines: {node: '>=18'} + string.prototype.matchall@4.0.12: resolution: {integrity: sha512-6CC9uyBL+/48dYizRf7H7VAYCMCNTBeM78x/VTUe9bFEaxBepPJDa1Ow99LqI/1yF7kuy7Q3cQsYMrcjGUcskA==} engines: {node: '>= 0.4'} @@ -8469,6 +8496,10 @@ packages: resolution: {integrity: sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==} engines: {node: '>=12'} + wrap-ansi@9.0.0: + resolution: {integrity: sha512-G8ura3S+3Z2G+mkgNRq8dqaFZAuxfsxpBB8OCTGRTCtp+l/v9nbFNmCUP1BZMts3G1142MsZfn6eeUKrr4PD1Q==} + engines: {node: '>=18'} + wrappy@1.0.2: resolution: {integrity: sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==} @@ -8542,10 +8573,18 @@ packages: resolution: {integrity: sha512-tVpsJW7DdjecAiFpbIB1e3qxIQsE6NoPc5/eTdrbbIC4h0LVsWhnoa3g+m2HclBIujHzsxZ4VJVA+GUuc2/LBw==} engines: {node: '>=12'} + yargs-parser@22.0.0: + resolution: {integrity: sha512-rwu/ClNdSMpkSrUb+d6BRsSkLUq1fmfsY6TOpYzTwvwkg1/NRG85KBy3kq++A8LKQwX6lsu+aWad+2khvuXrqw==} + engines: {node: ^20.19.0 || ^22.12.0 || >=23} + yargs@17.7.2: resolution: {integrity: sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w==} engines: {node: '>=12'} + yargs@18.0.0: + resolution: {integrity: sha512-4UEqdc2RYGHZc7Doyqkrqiln3p9X2DZVxaGbwhn2pi7MrRagKaOcIKe8L3OxYcbhXLgLFUS3zAYuQjKBQgmuNg==} + engines: {node: ^20.19.0 || ^22.12.0 || >=23} + yauzl@2.10.0: resolution: {integrity: sha512-p4a9I6X6nu6IhoGmBqAcbJy1mlC4j27vEPZX9F4L4/vZT3Lyq1VkFHw/V/PUcB9Buo+DG3iHkT0x3Qya58zc3g==} @@ -11909,6 +11948,12 @@ snapshots: '@types/web-bluetooth@0.0.21': {} + '@types/yargs-parser@21.0.3': {} + + '@types/yargs@17.0.33': + dependencies: + '@types/yargs-parser': 21.0.3 + '@types/yauzl@2.10.3': dependencies: '@types/node': 22.15.29 @@ -12931,6 +12976,12 @@ snapshots: strip-ansi: 6.0.1 wrap-ansi: 7.0.0 + cliui@9.0.1: + dependencies: + string-width: 7.2.0 + strip-ansi: 7.1.0 + wrap-ansi: 9.0.0 + cluster-key-slot@1.1.2: {} color-convert@1.9.3: @@ -13488,6 +13539,8 @@ snapshots: minimalistic-crypto-utils: 1.0.1 optional: true + emoji-regex@10.4.0: {} + emoji-regex@8.0.0: {} emoji-regex@9.2.2: {} @@ -14294,6 +14347,8 @@ snapshots: get-caller-file@2.0.5: {} + get-east-asian-width@1.3.0: {} + get-intrinsic@1.3.0: dependencies: call-bind-apply-helpers: 1.0.2 @@ -17281,6 +17336,12 @@ snapshots: emoji-regex: 9.2.2 strip-ansi: 7.1.0 + string-width@7.2.0: + dependencies: + emoji-regex: 10.4.0 + get-east-asian-width: 1.3.0 + strip-ansi: 7.1.0 + string.prototype.matchall@4.0.12: dependencies: call-bind: 1.0.8 @@ -18475,6 +18536,12 @@ snapshots: string-width: 5.1.2 strip-ansi: 7.1.0 + wrap-ansi@9.0.0: + dependencies: + ansi-styles: 6.2.1 + string-width: 7.2.0 + strip-ansi: 7.1.0 + wrappy@1.0.2: {} write-file-atomic@6.0.0: @@ -18514,6 +18581,8 @@ snapshots: yargs-parser@21.1.1: {} + yargs-parser@22.0.0: {} + yargs@17.7.2: dependencies: cliui: 8.0.1 @@ -18524,6 +18593,15 @@ snapshots: y18n: 5.0.8 yargs-parser: 21.1.1 + yargs@18.0.0: + dependencies: + cliui: 9.0.1 + escalade: 3.2.0 + get-caller-file: 2.0.5 + string-width: 7.2.0 + y18n: 5.0.8 + yargs-parser: 22.0.0 + yauzl@2.10.0: dependencies: buffer-crc32: 0.2.13 From a47138e7776b5f73d6ea996e5afe6999c20a5b50 Mon Sep 17 00:00:00 2001 From: Valentyne Stigloher Date: Fri, 20 Jun 2025 14:24:09 +0200 Subject: [PATCH 5/8] (helper) specify .suml extension in arguments --- helpers/merge/translationsHelper.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/helpers/merge/translationsHelper.ts b/helpers/merge/translationsHelper.ts index e47087db5..b081bacea 100644 --- a/helpers/merge/translationsHelper.ts +++ b/helpers/merge/translationsHelper.ts @@ -10,16 +10,16 @@ import { deepListKeys } from '~/src/helpers.ts'; import { listMissingTranslations } from '~/src/missingTranslations.ts'; export const loadDocument = async (name: string) => { - return parse(await fs.readFile(`${rootDir}/${name}.suml`, 'utf-8')); + return parse(await fs.readFile(`${rootDir}/${name}`, 'utf-8')); }; export const saveDocument = async (name: string, document: Node) => { - await fs.writeFile(`${rootDir}/${name}.suml`, `${document.toString()}\n`); + await fs.writeFile(`${rootDir}/${name}`, `${document.toString()}\n`); }; export const mergeTranslationProposals = async (locale: string, proposalsFile: string) => { - const baseTranslationsDocument = await loadDocument('locale/_base/translations'); - const localeTranslationsDocument = await loadDocument(`locale/${locale}/translations`); + const baseTranslationsDocument = await loadDocument('locale/_base/translations.suml'); + const localeTranslationsDocument = await loadDocument(`locale/${locale}/translations.suml`); const proposalsDocument = await loadDocument(proposalsFile); if (!(proposalsDocument instanceof DictNode) || @@ -29,7 +29,7 @@ export const mergeTranslationProposals = async (locale: string, proposalsFile: s } merge(proposalsDocument, baseTranslationsDocument, localeTranslationsDocument); - await saveDocument(`locale/${locale}/translations`, localeTranslationsDocument); + await saveDocument(`locale/${locale}/translations.suml`, localeTranslationsDocument); }; export const merge = (source: DictNode, base: DictNode, target: DictNode) => { @@ -69,8 +69,8 @@ export const createTranslationFiles = async (locale: string) => { const baseTranslations = await loadSuml('locale/_base/translations.suml'); const config = await loadSuml('locale/_base/config.suml'); - const configDocument = await loadDocument('locale/_base/config'); - const translationsDocument = await loadDocument('locale/_base/translations'); + const configDocument = await loadDocument('locale/_base/config.suml'); + const translationsDocument = await loadDocument('locale/_base/translations.suml'); const requiredTranslationKeys = listMissingTranslations({}, baseTranslations, config); for (const key of deepListKeys(baseTranslations)) { @@ -80,8 +80,8 @@ export const createTranslationFiles = async (locale: string) => { } await fs.mkdir(`${rootDir}/locale/${locale}`, { recursive: true }); - await saveDocument(`locale/${locale}/config`, configDocument); - await saveDocument(`locale/${locale}/translations`, translationsDocument); + await saveDocument(`locale/${locale}/config.suml`, configDocument); + await saveDocument(`locale/${locale}/translations.suml`, translationsDocument); }; export const remove = (node: Node, path: string[]) => { From 500dd5c04fd6735fccd16332d1a8c7dacbcb5240 Mon Sep 17 00:00:00 2001 From: Valentyne Stigloher Date: Mon, 23 Jun 2025 15:50:15 +0200 Subject: [PATCH 6/8] (helper) fix comment parsing (it is not entirely consistent, but good enough for the current codebase) --- helpers/merge/sumlAst.ts | 75 ++++++++++++++++++++++++---------------- 1 file changed, 45 insertions(+), 30 deletions(-) diff --git a/helpers/merge/sumlAst.ts b/helpers/merge/sumlAst.ts index a0f374902..65e500fd5 100644 --- a/helpers/merge/sumlAst.ts +++ b/helpers/merge/sumlAst.ts @@ -25,18 +25,18 @@ export class EmptyLineNode { } export class CommentNode { - constructor(public comment: string) { } + constructor(public comment: string, public whitespace: string = '') { } toString(indent: number = 0) { - return `${' '.repeat(4 * indent)}#${this.comment}\n`; + return `${' '.repeat(4 * indent)}${this.whitespace}#${this.comment}\n`; } } export type Node = DictNode | ListNode | InlineNode | VerbatimStringNode | FoldedStringNode; -export function initFormatting(formatting: FormattingNode[], indent: number) { +const initFormatting = (formatting: FormattingNode[], indent: number) => { return `${formatting.map((node) => node.toString(indent)).join('')}${' '.repeat(4 * indent)}`; -} +}; export class DictNode { constructor(public items: DictEntry[], public formatting: FormattingNode[]) { } @@ -46,16 +46,22 @@ export class DictNode { } toString(indent: number = 0): string { - return this.items.map((item) => item.toString(indent)).join('\n'); + return this.formatting.map((node) => node.toString(indent)).join('') + + this.items.map((item) => item.toString(indent)).join('\n'); } } export class DictEntry { - constructor(public key: string, public value: Node, public formatting: FormattingNode[]) { } + constructor( + public key: string, + public value: Node, + public formatting: FormattingNode[], + public comment?: CommentNode, + ) { } toString(indent: number = 0): string { return `${initFormatting(this.formatting, indent) - }${this.key}:${this.value.separator + }${this.key}:${this.comment ? this.comment.toString() : this.value.separator }${this.value.toString(indent + 1)}`; } } @@ -137,7 +143,7 @@ export class InlineDictNode { } export class VerbatimStringNode { - constructor(public lines: string[]) { } + constructor(public lines: string[], public comment?: CommentNode) { } get separator() { return ' '; @@ -150,12 +156,12 @@ export class VerbatimStringNode { } return `${' '.repeat(4 * indent)}${line}`; }; - return `|\n${this.lines.map(formatLine).join('\n')}`; + return `|${this.comment ? this.comment.toString() : '\n'}${this.lines.map(formatLine).join('\n')}`; } } export class FoldedStringNode { - constructor(public lines: string[]) { } + constructor(public lines: string[], public comment?: CommentNode) { } get separator() { return ' '; @@ -168,7 +174,7 @@ export class FoldedStringNode { } return `${' '.repeat(4 * indent)}${line}`; }; - return `>\n${this.lines.map(formatLine).join('\n')}`; + return `>${this.comment ? this.comment.toString() : '\n'}${this.lines.map(formatLine).join('\n')}`; } } @@ -178,9 +184,8 @@ export const parse = (value: string): Node => { return parser.parseLines(value.split('\n')); }; -const regexComment = / *#(?.*?)$/; -const regexLineComment = new RegExp(`^${regexComment.source}$`); -const regexPartComment = new RegExp(`(?:${regexComment.source})?$`); +const regexLineComment = /^(? *)#(?.*?)$/; +const regexPartComment = /(?: *#(?.*?))?$/; const regexPartDate = '(?:\\d\\d\\d\\d-\\d\\d-\\d\\d)'; const regexPartTime = '(?:\\d\\d:\\d\\d:\\d\\d(?:[+-]\\d\\d\\d\\d)?)'; const regexPartDictKey = '([^:#\' {}]+):'; @@ -252,7 +257,7 @@ class Parser { } match = this.currentLine.match(regexLineComment); if (match) { - this.formatting.push(new CommentNode(match.groups!.comment)); + this.formatting.push(new CommentNode(match.groups!.comment, match.groups!.whitespace ?? '')); this.currentLineNumber++; continue; } @@ -266,7 +271,9 @@ class Parser { throw this.createError('Dict in wrong context'); } const formatting = this.getAndClearFormatting(); - node.items.push(new DictEntry(match[1].trim(), this.parseBlock(match[2].trim()), formatting)); + const value = this.parseBlock(match[2].trim()); + const comment = value instanceof DictNode ? this.extractComment(match[0]) : undefined; + node.items.push(new DictEntry(match[1].trim(), value, formatting, comment)); continue; } @@ -385,36 +392,37 @@ class Parser { } private parseBlock(value: string): Node { - switch (value.trim().substring(0, 1)) { + const blockCharacter = value.trim().substring(0, 1); + switch (blockCharacter) { case '': case '#': { this.currentIndent++; this.currentLineNumber++; const block = this.parse(); + for (const formattingNode of this.formatting) { + if (formattingNode instanceof CommentNode) { + formattingNode.whitespace += ' '.repeat(4); + } + } this.currentIndent--; return block; } - case '|': { - this.currentIndent++; - this.currentLineNumber++; - const lines = []; - while (this.isCurrentLineValid) { - lines.push(this.currentLine); - this.currentLineNumber++; - } - this.currentIndent--; - return new VerbatimStringNode(lines); - } + case '|': case '>': { this.currentIndent++; this.currentLineNumber++; - const lines = []; + const lines: string[] = []; while (this.isCurrentLineValid) { lines.push(this.currentLine); this.currentLineNumber++; } this.currentIndent--; - return new FoldedStringNode(lines); + const comment = this.extractComment(value); + if (blockCharacter === '|') { + return new VerbatimStringNode(lines, comment); + } else { + return new FoldedStringNode(lines, comment); + } } default: { const node = this.parseInline(value); @@ -423,4 +431,11 @@ class Parser { } } } + + private extractComment(value: string): CommentNode | undefined { + const match = value.match(regexPartComment); + return match?.groups?.comment + ? new CommentNode(match.groups.comment, ' ') + : undefined; + } } From c61076ad6b74529594291fa628e1767dc891d0a8 Mon Sep 17 00:00:00 2001 From: Valentyne Stigloher Date: Mon, 23 Jun 2025 15:53:08 +0200 Subject: [PATCH 7/8] (helper) ignore last trailing line for string blocks --- helpers/merge/sumlAst.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/helpers/merge/sumlAst.ts b/helpers/merge/sumlAst.ts index 65e500fd5..a74cbfa27 100644 --- a/helpers/merge/sumlAst.ts +++ b/helpers/merge/sumlAst.ts @@ -220,6 +220,9 @@ class Parser { return false; } const currentLine = this.lines[this.currentLineNumber]; + if (this.currentLineNumber === this.lines.length - 1) { + return currentLine.trim().length !== 0; + } return currentLine.startsWith(' '.repeat(4 * this.currentIndent)) || currentLine.trim().length === 0; } From 9c64d52f0d8b8c00128708d352bd982ac9b449ee Mon Sep 17 00:00:00 2001 From: Valentyne Stigloher Date: Mon, 23 Jun 2025 16:01:22 +0200 Subject: [PATCH 8/8] (trans) remove some superflous comments in config.suml and translations.suml --- locale/ar/config.suml | 1 - locale/eo/config.suml | 1 - locale/et/config.suml | 14 +------------- locale/et/translations.suml | 2 -- locale/fo/config.suml | 1 - locale/fo/translations.suml | 3 --- locale/hbs/config.suml | 14 +------------- locale/he/config.suml | 14 +------------- locale/it/config.suml | 1 - locale/it/translations.suml | 6 ------ locale/ko/config.suml | 1 - locale/ko/translations.suml | 1 - locale/nb/translations.suml | 2 -- locale/nn/translations.suml | 2 -- locale/ro/config.suml | 1 - locale/ro/translations.suml | 1 - locale/sv/config.suml | 1 - locale/sv/translations.suml | 2 -- locale/tok/config.suml | 2 -- locale/tok/translations.suml | 8 +------- 20 files changed, 4 insertions(+), 74 deletions(-) diff --git a/locale/ar/config.suml b/locale/ar/config.suml index 3a00cb03c..b2cfe126c 100644 --- a/locale/ar/config.suml +++ b/locale/ar/config.suml @@ -128,7 +128,6 @@ names: people: enabled: false -# optional, but would be nice to have english: enabled: false route: 'english' diff --git a/locale/eo/config.suml b/locale/eo/config.suml index 8e40e7290..9ded2ddd6 100644 --- a/locale/eo/config.suml +++ b/locale/eo/config.suml @@ -167,7 +167,6 @@ names: people: enabled: false -# optional, but would be nice to have english: enabled: true route: 'english' diff --git a/locale/et/config.suml b/locale/et/config.suml index 62cd1c8c8..988af0b46 100644 --- a/locale/et/config.suml +++ b/locale/et/config.suml @@ -110,20 +110,8 @@ names: people: enabled: false -# optional, but would be nice to have english: - enabled: true - route: 'english' - pronounGroups: - - - name: 'Normative forms' - description: - - > - Because of the limitations of grammar, or simply because they just prefer it that way, - many nonbinary people decide to simply use “he” ({/on=„on”}) or “she” ({/ona=„ona”}) - – either the same as their gender assigned at birth or the opposite. - That doesn't make them any less nonbinary! Pronouns ≠ gender. - table: { on: 'Masculine', ona: 'Feminine' } + enabled: false faq: enabled: false diff --git a/locale/et/translations.suml b/locale/et/translations.suml index c4ed9e0ec..e80f373da 100644 --- a/locale/et/translations.suml +++ b/locale/et/translations.suml @@ -651,8 +651,6 @@ profile: language: header: 'Keel' description: '@%username% kaart on saadaval ka järgnevates keeltes' - # if your language has declension and it's hard to fit the username in that sentence, - # just make is 'This card is…' circles: header: 'Minu ringkond' info: > diff --git a/locale/fo/config.suml b/locale/fo/config.suml index 5c093a52a..362c31a14 100644 --- a/locale/fo/config.suml +++ b/locale/fo/config.suml @@ -233,7 +233,6 @@ names: people: enabled: false -# optional, but would be nice to have english: enabled: true route: 'english' diff --git a/locale/fo/translations.suml b/locale/fo/translations.suml index 4824e1e93..77673ab3b 100644 --- a/locale/fo/translations.suml +++ b/locale/fo/translations.suml @@ -681,9 +681,6 @@ links: people: ~ -# this section is a short English-language explanation of how gendered and gender-neutral forms work in your language -# see for example: https://zaimki.pl/english -# it's optional, but would be nice to have english: header: 'English' headerLong: 'An overview in English' diff --git a/locale/hbs/config.suml b/locale/hbs/config.suml index 60872c112..bf2d1887b 100644 --- a/locale/hbs/config.suml +++ b/locale/hbs/config.suml @@ -179,20 +179,8 @@ names: people: enabled: false -# optional, but would be nice to have english: - enabled: true - route: 'english' - pronounGroups: - - - name: 'Normative forms' - description: - - > - Because of the limitations of grammar, or simply because they just prefer it that way, - many nonbinary people decide to simply use “he” ({/on=„on”}) or “she” ({/ona=„ona”}) - – either the same as their gender assigned at birth or the opposite. - That doesn't make them any less nonbinary! Pronouns ≠ gender. - table: { on: 'Masculine', ona: 'Feminine' } + enabled: false faq: enabled: true diff --git a/locale/he/config.suml b/locale/he/config.suml index b4071ec16..dc24c512f 100644 --- a/locale/he/config.suml +++ b/locale/he/config.suml @@ -118,20 +118,8 @@ names: people: enabled: false -# optional, but would be nice to have english: - enabled: true - route: 'english' - pronounGroups: - - - name: 'Normative forms' - description: - - > - Because of the limitations of grammar, or simply because they just prefer it that way, - many nonbinary people decide to simply use “he” ({/on=„on”}) or “she” ({/ona=„ona”}) - – either the same as their gender assigned at birth or the opposite. - That doesn't make them any less nonbinary! Pronouns ≠ gender. - table: { on: 'Masculine', ona: 'Feminine' } + enabled: false faq: enabled: true diff --git a/locale/it/config.suml b/locale/it/config.suml index 48822dd2a..7992834d0 100644 --- a/locale/it/config.suml +++ b/locale/it/config.suml @@ -362,7 +362,6 @@ names: people: enabled: false -# optional, but would be nice to have english: enabled: true route: 'italian' diff --git a/locale/it/translations.suml b/locale/it/translations.suml index 585c3632f..d068b88d1 100644 --- a/locale/it/translations.suml +++ b/locale/it/translations.suml @@ -532,9 +532,6 @@ links: people: ~ -# this section is a short English-language explanation of how gendered and gender-neutral forms work in your language -# see for example: https://zaimki.pl/english -# it's optional, but would be nice to have english: header: 'English' headerLong: 'An overview in English' @@ -912,9 +909,6 @@ profile: language: header: 'Lingua' description: 'La card di @%username% è disponibile anche in altre lingue' - # if your language has declension and it's hard to fit the username in that sentence, - # just make is 'This card is…' - timezone: areas: Antarctica: 'Antartide' diff --git a/locale/ko/config.suml b/locale/ko/config.suml index a71781763..96d56c2b0 100644 --- a/locale/ko/config.suml +++ b/locale/ko/config.suml @@ -112,7 +112,6 @@ names: people: enabled: false -# optional, but would be nice to have english: enabled: true route: 'english' diff --git a/locale/ko/translations.suml b/locale/ko/translations.suml index 340765e42..7b7ff65c2 100644 --- a/locale/ko/translations.suml +++ b/locale/ko/translations.suml @@ -400,7 +400,6 @@ links: people: ~ -# optional, but would be nice to have english: header: 'English' headerLong: 'An overview in English' diff --git a/locale/nb/translations.suml b/locale/nb/translations.suml index bb8fc3e39..382f9a95c 100644 --- a/locale/nb/translations.suml +++ b/locale/nb/translations.suml @@ -1011,8 +1011,6 @@ profile: language: header: 'Språk' description: '@%username%s kort er også tilgjengelig i disse språkene' - # if your language has declension and it's hard to fit the username in that sentence, - # just make is 'This card is…' circles: header: 'Min sirkel' info: > diff --git a/locale/nn/translations.suml b/locale/nn/translations.suml index ab3089494..ccde545d9 100644 --- a/locale/nn/translations.suml +++ b/locale/nn/translations.suml @@ -1013,8 +1013,6 @@ profile: language: header: 'Språk' description: '@%username% sitt kort er ogso tilgjengeleg i desse språka' - # if your language has declension and it's hard to fit the username in that sentence, - # just make is 'This card is…' circles: header: 'Sirkelen min' info: > diff --git a/locale/ro/config.suml b/locale/ro/config.suml index 20111e652..1f054d3c2 100644 --- a/locale/ro/config.suml +++ b/locale/ro/config.suml @@ -187,7 +187,6 @@ names: people: enabled: false -# optional, but would be nice to have english: enabled: true route: 'english' diff --git a/locale/ro/translations.suml b/locale/ro/translations.suml index 98e485922..da3d72b5f 100644 --- a/locale/ro/translations.suml +++ b/locale/ro/translations.suml @@ -551,7 +551,6 @@ links: people: ~ -# optional, but would be nice to have english: # TODO header: 'English' headerLong: 'An overview in English' diff --git a/locale/sv/config.suml b/locale/sv/config.suml index 495aad93a..8712d6307 100644 --- a/locale/sv/config.suml +++ b/locale/sv/config.suml @@ -124,7 +124,6 @@ names: people: enabled: false -# optional, but would be nice to have english: enabled: true route: 'english' diff --git a/locale/sv/translations.suml b/locale/sv/translations.suml index f2f174cf4..dd101eef0 100644 --- a/locale/sv/translations.suml +++ b/locale/sv/translations.suml @@ -396,7 +396,6 @@ links: people: ~ -# optional, but would be nice to have english: header: 'English' headerLong: 'An overview in English' @@ -1116,7 +1115,6 @@ flags: Two_Spirit: 'Two-spirit' Xenogender: 'Xenogender' -# optional, but would be nice to have calendar: # TODO header: 'Kalender' headerLong: 'Queerkalender' diff --git a/locale/tok/config.suml b/locale/tok/config.suml index 39e7ebb18..6d62e44fc 100644 --- a/locale/tok/config.suml +++ b/locale/tok/config.suml @@ -65,10 +65,8 @@ names: people: enabled: false -# optional, but would be nice to have english: enabled: false - route: 'english' faq: enabled: true diff --git a/locale/tok/translations.suml b/locale/tok/translations.suml index 4595d25b6..d5d8e5665 100644 --- a/locale/tok/translations.suml +++ b/locale/tok/translations.suml @@ -62,9 +62,6 @@ links: people: ~ -# this section is a short English-language explanation of how gendered and gender-neutral forms work in your language -# see for example: https://zaimki.pl/english -# it's optional, but would be nice to have english: header: 'English' headerLong: 'An overview in English' @@ -73,8 +70,7 @@ english: intro: - > Toki Pona doesn't have gendered pronouns at all. The pronoun {/li/ona=ona} can refer to anyone and anything. So this version of Pronouns.page makes no mention of pronouns, and is essentially a website for telling people what your name is. - - > - TODO(tbodt): finish writing this + # TODO(tbodt): finish writing this contact: header: 'o toki tawa mi' @@ -324,8 +320,6 @@ profile: language: header: 'toki' description: 'toki ante ni la lipu ni li lon' - # if your language has declension and it's hard to fit the username in that sentence, - # just make is 'This card is…' circles: header: 'kulupu mi' yourMentions: