diff --git a/pontoon/base/migrations/0101_webext_as_mf2.py b/pontoon/base/migrations/0101_webext_as_mf2.py new file mode 100644 index 0000000000..f85c8a99b0 --- /dev/null +++ b/pontoon/base/migrations/0101_webext_as_mf2.py @@ -0,0 +1,116 @@ +from json import dumps +from re import compile + +from moz.l10n.formats.mf2 import mf2_parse_message, mf2_serialize_message +from moz.l10n.formats.webext import webext_serialize_message +from moz.l10n.message import message_from_json +from moz.l10n.model import Expression, PatternMessage, VariableRef + +from django.db import migrations + + +webext_placeholder = compile(r"\$([a-zA-Z0-9_@]+)\$|(\$[1-9])|\$(\$+)") + + +def webext_translation_parse(translation): + db_source = translation.string + declarations = message_from_json(translation.entity.value).declarations + pattern = [] + pos = 0 + for m in webext_placeholder.finditer(db_source): + start = m.start() + if start > pos: + pattern.append(db_source[pos:start]) + if m[1]: + # Named placeholder, with content & optional example in placeholders object + ph_name = m[1].replace("@", "_") + if ph_name[0].isdigit(): + ph_name = f"_{ph_name}" + ph_name = next( + (name for name in declarations if name.lower() == ph_name.lower()), + ph_name, + ) + var = VariableRef(ph_name) + pattern.append(Expression(var, attributes={"source": m[0]})) + elif m[2]: + # Indexed placeholder + var = VariableRef(f"arg{m[2][1]}") + pattern.append(Expression(var, attributes={"source": m[0]})) + else: + # Escaped literal dollar sign + if pattern and isinstance(pattern[-1], str): + pattern[-1] += m[3] + else: + pattern.append(m[3]) + pos = m.end() + if pos < len(db_source): + pattern.append(db_source[pos:]) + return PatternMessage(pattern, declarations) + + +def mf2_entity_changed(entity): + db_source = entity.string + msg = message_from_json(entity.value) + mf2_source = mf2_serialize_message(msg) + if mf2_source == db_source: + return False + entity.string = mf2_source + entity.meta = [m for m in entity.meta if m[0] != "placeholders"] + return True + + +def mf2_translation_changed(translation): + db_source = translation.string + msg = webext_translation_parse(translation) + mf2_source = mf2_serialize_message(msg) + if mf2_source == db_source: + return False + translation.string = mf2_source + return True + + +def webext_as_mf2(apps, schema_editor): + Entity = apps.get_model("base", "Entity") + entities = Entity.objects.filter(resource__format="webext") + ent_fixed = [e for e in entities if mf2_entity_changed(e)] + n = Entity.objects.bulk_update(ent_fixed, ["meta", "string"], batch_size=10_000) + print(f" ({n} entities)", end="", flush=True) + + Translation = apps.get_model("base", "Translation") + translations = Translation.objects.filter( + entity__resource__format="webext" + ).select_related("entity") + trans_fixed = [t for t in translations if mf2_translation_changed(t)] + n = Translation.objects.bulk_update(trans_fixed, ["string"], batch_size=10_000) + print(f" ({n} translations)", end="", flush=True) + + +def webext_string_changed(obj, with_placeholders: bool): + mf2_source = obj.string + msg = mf2_parse_message(mf2_source) + string, placeholders = webext_serialize_message(msg) + if string == mf2_source: + return False + obj.string = string + if with_placeholders: + obj.meta.append(["placeholders", dumps(placeholders)]) + return True + + +def mf2_as_webext(apps, schema_editor): + Entity = apps.get_model("base", "Entity") + entities = Entity.objects.filter(resource__format="webext") + ent_fixed = [e for e in entities if webext_string_changed(e, True)] + n = Entity.objects.bulk_update(ent_fixed, ["meta", "string"], batch_size=10_000) + print(f" ({n} entities)", end="", flush=True) + + Translation = apps.get_model("base", "Translation") + translations = Translation.objects.filter(entity__resource__format="webext") + trans_fixed = [t for t in translations if webext_string_changed(t, False)] + n = Translation.objects.bulk_update(trans_fixed, ["string"], batch_size=10_000) + print(f" ({n} translations)", end="", flush=True) + + +class Migration(migrations.Migration): + dependencies = [("base", "0100_android_as_mf2")] + operations = [migrations.RunPython(webext_as_mf2, reverse_code=mf2_as_webext)] diff --git a/pontoon/base/simple_preview.py b/pontoon/base/simple_preview.py index ad9e15f03b..fdbd61b5ba 100644 --- a/pontoon/base/simple_preview.py +++ b/pontoon/base/simple_preview.py @@ -46,7 +46,7 @@ def get_simple_preview(format: str, string: str): msg = mf2_parse_message(string) return android_simple_preview(msg) - case Resource.Format.GETTEXT: + case Resource.Format.GETTEXT | Resource.Format.WEBEXT: msg = mf2_parse_message(string) msg = as_pattern_message(msg) return serialize_message(None, msg) diff --git a/pontoon/checks/libraries/__init__.py b/pontoon/checks/libraries/__init__.py index b74710db0f..6a1179ae40 100644 --- a/pontoon/checks/libraries/__init__.py +++ b/pontoon/checks/libraries/__init__.py @@ -1,4 +1,5 @@ from moz.l10n.formats.mf2 import mf2_parse_message, mf2_serialize_pattern +from moz.l10n.formats.webext import webext_serialize_message from moz.l10n.model import CatchallKey, Pattern, PatternMessage, SelectMessage from pontoon.base.models import Entity, Resource @@ -107,6 +108,13 @@ def run_checks( (as_gettext(src_msg.pattern), as_gettext(tgt_msg.pattern)) ) + case Resource.Format.WEBEXT: + src_msg = mf2_parse_message(entity.string) + tgt_msg = mf2_parse_message(string) + src_str, _ = webext_serialize_message(src_msg) + tgt_str, _ = webext_serialize_message(tgt_msg) + tt_patterns.append((src_str, tgt_str)) + case _: tt_patterns.append((entity.string, string)) tt_warnings = {} diff --git a/pontoon/checks/libraries/custom.py b/pontoon/checks/libraries/custom.py index ce87e858b7..4544b5d517 100644 --- a/pontoon/checks/libraries/custom.py +++ b/pontoon/checks/libraries/custom.py @@ -1,10 +1,12 @@ from html import escape +from re import fullmatch from typing import Iterable, Iterator from fluent.syntax import FluentParser, ast from fluent.syntax.visitor import Visitor from moz.l10n.formats.android import android_parse_message from moz.l10n.formats.mf2 import mf2_parse_message +from moz.l10n.formats.webext import webext_parse_message, webext_serialize_message from moz.l10n.model import ( Expression, Markup, @@ -161,6 +163,41 @@ def run_custom_checks(entity: Entity, string: str) -> dict[str, list[str]]: if visitor.is_empty: warnings.append("Empty translation") + case Resource.Format.WEBEXT: + try: + msg = mf2_parse_message(string) + except ValueError as e: + msg = None + errors.append(f"Parse error: {e}") + if isinstance(msg, PatternMessage): + try: + orig_msg = mf2_parse_message(entity.string) + _, placeholders = webext_serialize_message(orig_msg) + except ValueError: + placeholders = None + + # The default moz.l10n serialization would escape $ in literal content, + # which we don't want here -- instead looking for typos in placeholders. + webext_src = "" + for part in msg.pattern: + if isinstance(part, str): + webext_src += part + else: + part_source = part.attributes.get("source", None) + if isinstance(part_source, str): + webext_src += part_source + else: + errors.append(f"Unsupported placeholder: {part}") + try: + webext_parse_message(webext_src, placeholders) + except Exception as e: + bad_ph = fullmatch(r"Missing placeholders entry for (\w+)", str(e)) + errors.append( + f"Placeholder ${bad_ph.group(1).upper()}$ not found in reference" + if bad_ph + else f"Parse error: {e}" + ) + checks: dict[str, list[str]] = {} if errors: checks["pErrors"] = errors diff --git a/pontoon/checks/tests/test_custom.py b/pontoon/checks/tests/test_custom.py index 3b41f4b5a3..39ead04cc1 100644 --- a/pontoon/checks/tests/test_custom.py +++ b/pontoon/checks/tests/test_custom.py @@ -16,6 +16,8 @@ def mock_entity( ext = "ftl" case "gettext": ext = "po" + case "webext": + ext = "json" case _: ext = format entity = MagicMock() @@ -258,3 +260,69 @@ def test_android_bad_html(): "pErrors": ["Placeholder not found in reference"], "pndbWarnings": ["Placeholder not found in translation"], } + + +def test_webext_literal_index_placeholder_as_placeholder(): + original = "Source string with a {$arg1 @source=|$1|}" + translation = "Translation with a {$arg1 @source=|$1|}" + entity = mock_entity("webext", string=original) + assert run_custom_checks(entity, translation) == {} + + +def test_webext_literal_index_placeholder_as_literal(): + original = "Source string with a {$arg1 @source=|$1|}" + translation = "Translation with a $1" + entity = mock_entity("webext", string=original) + assert run_custom_checks(entity, translation) == {} + + +def test_webext_literal_named_placeholder_as_placeholder(): + original = ( + ".local $FOO = {$arg1 @source=|$1|}\n" + + "{{Source string with a {$FOO @source=|$FOO$|}}}" + ) + translation = ( + ".local $FOO = {$arg1 @source=|$1|}\n" + + "{{Translation with a {$FOO @source=|$FOO$|}}}" + ) + entity = mock_entity("webext", string=original) + assert run_custom_checks(entity, translation) == {} + + +def test_webext_literal_named_placeholder_as_literal(): + original = ( + ".local $FOO = {$arg1 @source=|$1|}\n" + + "{{Source string with a {$FOO @source=|$FOO$|}}}" + ) + translation = "Translation with a $FOO$" + entity = mock_entity("webext", string=original) + assert run_custom_checks(entity, translation) == {} + + +def test_webext_extra_index_placeholder(): + original = "Source string" + translation = "Translation with a $1" + entity = mock_entity("webext", string=original) + # This should probably also be caught + assert run_custom_checks(entity, translation) == {} + + +def test_webext_extra_named_placeholder_as_literal(): + original = "Source string" + translation = "Translation with a $FOO$" + entity = mock_entity("webext", string=original) + assert run_custom_checks(entity, translation) == { + "pErrors": ["Placeholder $FOO$ not found in reference"] + } + + +def test_webext_extra_named_placeholder_as_placeholder(): + original = "Source string" + translation = ( + ".local $FOO = {$arg1 @source=|$1|}\n" + + "{{Translation with a {$FOO @source=|$FOO$|}}}" + ) + entity = mock_entity("webext", string=original) + assert run_custom_checks(entity, translation) == { + "pErrors": ["Placeholder $FOO$ not found in reference"] + } diff --git a/pontoon/pretranslation/pretranslate.py b/pontoon/pretranslation/pretranslate.py index cbe54e8d26..b3f34b8e67 100644 --- a/pontoon/pretranslation/pretranslate.py +++ b/pontoon/pretranslation/pretranslate.py @@ -58,7 +58,11 @@ def get_pretranslation( set_accesskey(entry, key, prop) pt_res = FluentSerializer().serialize_entry(fluent_astify_entry(entry)) else: - if entity.resource.format in {Resource.Format.ANDROID, Resource.Format.GETTEXT}: + if entity.resource.format in { + Resource.Format.ANDROID, + Resource.Format.GETTEXT, + Resource.Format.WEBEXT, + }: format = Format.mf2 msg = parse_message(format, entity.string) else: @@ -82,7 +86,11 @@ def __init__(self, entity: Entity, locale: Locale, preserve_placeables: bool): match entity.resource.format: case Resource.Format.FLUENT: self.format = Format.fluent - case Resource.Format.ANDROID | Resource.Format.GETTEXT: + case ( + Resource.Format.ANDROID + | Resource.Format.GETTEXT + | Resource.Format.WEBEXT + ): self.format = Format.mf2 case _: self.format = None diff --git a/pontoon/sync/core/translations_to_repo.py b/pontoon/sync/core/translations_to_repo.py index 21614eefe6..c4de76c1a0 100644 --- a/pontoon/sync/core/translations_to_repo.py +++ b/pontoon/sync/core/translations_to_repo.py @@ -4,21 +4,18 @@ from datetime import datetime from os import makedirs, remove from os.path import commonpath, dirname, isfile, join, normpath -from re import compile from moz.l10n.formats import Format from moz.l10n.message import parse_message from moz.l10n.model import ( CatchallKey, Entry, - Expression, Id, Metadata, PatternMessage, Resource, Section, SelectMessage, - VariableRef, ) from moz.l10n.paths import L10nConfigPaths, L10nDiscoverPaths from moz.l10n.resource import parse_resource, serialize_resource @@ -331,9 +328,6 @@ def set_translations( ] -webext_placeholder = compile(r"\$([a-zA-Z0-9_@]+)\$|(\$[1-9])|\$(\$+)") - - def set_translation( translations: list[Translation], format: Format | None, @@ -353,7 +347,7 @@ def set_translation( return False match format: - case Format.android | Format.gettext: + case Format.android | Format.gettext | Format.webext: msg = parse_message(Format.mf2, tx.string) if isinstance(entry.value, SelectMessage): entry.value.variants = ( @@ -373,39 +367,6 @@ def set_translation( elif fuzzy_flag in entry.meta: entry.meta = [m for m in entry.meta if m != fuzzy_flag] - case Format.webext if ( - isinstance(entry.value, PatternMessage) and entry.value.declarations - ): - # With a message value, placeholders in string parts would have their - # $ characters doubled to escape them. - entry.value.pattern = [] - pos = 0 - for m in webext_placeholder.finditer(tx.string): - start = m.start() - if start > pos: - entry.value.pattern.append(tx.string[pos:start]) - if m[1]: - ph_name = m[1].replace("@", "_") - if ph_name[0].isdigit(): - ph_name = f"_{ph_name}" - ph_name = next( - ( - name - for name in entry.value.declarations - if name.lower() == ph_name.lower() - ), - ph_name, - ) - pass - else: - ph_name = m[0] - entry.value.pattern.append( - Expression(VariableRef(ph_name), attributes={"source": m[0]}) - ) - pos = m.end() - if pos < len(tx.string): - entry.value.pattern.append(tx.string[pos:]) - case _: entry.value = tx.string diff --git a/pontoon/sync/formats/__init__.py b/pontoon/sync/formats/__init__.py index f748327399..06c2c3ee28 100644 --- a/pontoon/sync/formats/__init__.py +++ b/pontoon/sync/formats/__init__.py @@ -2,7 +2,6 @@ Parsing resource files. """ -from json import dumps from os.path import splitext from re import Match, compile from typing import Iterator @@ -10,7 +9,6 @@ from fluent.syntax import FluentSerializer from moz.l10n.formats import Format, detect_format, l10n_extensions from moz.l10n.formats.fluent import fluent_astify_entry -from moz.l10n.formats.webext import webext_serialize_message from moz.l10n.message import message_to_json, serialize_message from moz.l10n.model import Entry, Id as L10nId, Message, Resource as MozL10nResource @@ -51,7 +49,7 @@ def _as_string(format: Format | None, entry: Entry[Message]) -> str: case Format.fluent: fluent_entry = fluent_astify_entry(entry, lambda _: "") return _fluent_serializer.serialize_entry(fluent_entry) - case Format.android: + case Format.android | Format.webext: return serialize_message(Format.mf2, entry.value) case Format.properties: string = serialize_message(Format.properties, entry.value) @@ -92,19 +90,6 @@ def as_entity( match format: case Format.gettext: return gettext_as_entity(entry, kwargs) - case Format.webext: - string, placeholders = webext_serialize_message(entry.value) - meta = [[m.key, m.value] for m in entry.meta] - if placeholders: - meta.append(["placeholders", dumps(placeholders)]) - return Entity( - key=list(section_id + entry.id), - value=message_to_json(entry.value), - string=string, - comment=entry.comment, - meta=meta, - **kwargs, - ) case Format.xliff: return xliff_as_entity(section_id, entry, kwargs) case _: diff --git a/pontoon/sync/tests/formats/test_json_extensions.py b/pontoon/sync/tests/formats/test_json_extensions.py index f16f094d04..eeeefd8b02 100644 --- a/pontoon/sync/tests/formats/test_json_extensions.py +++ b/pontoon/sync/tests/formats/test_json_extensions.py @@ -55,7 +55,11 @@ def test_webext(self): assert t2.string == "Translated No Comments or Sources" assert e3.key == ["placeholder"] - assert e3.string == "Hello $YOUR_NAME$" + assert ( + e3.string + == ".local $YOUR_NAME = {$arg1 @source=|$1| @example=Cira}\n" + + "{{Hello {$YOUR_NAME @source=|$YOUR_NAME$|}}}" + ) assert e3.value == { "decl": { "YOUR_NAME": {"$": "arg1", "attr": {"example": "Cira", "source": "$1"}} @@ -63,9 +67,11 @@ def test_webext(self): "msg": ["Hello ", {"$": "YOUR_NAME", "attr": {"source": "$YOUR_NAME$"}}], } assert e3.comment == "Peer greeting" - assert e3.meta == [ - ["placeholders", '{"YOUR_NAME": {"content": "$1", "example": "Cira"}}'], - ] + assert e3.meta == [] assert t3.key == ("placeholder",) - assert t3.string == "Hello $YOUR_NAME$" + assert ( + t3.string + == ".local $YOUR_NAME = {$arg1 @source=|$1| @example=Cira}\n" + + "{{Hello {$YOUR_NAME @source=|$YOUR_NAME$|}}}" + ) diff --git a/pontoon/sync/tests/test_e2e.py b/pontoon/sync/tests/test_e2e.py index bacc115bd9..87c05ca3cc 100644 --- a/pontoon/sync/tests/test_e2e.py +++ b/pontoon/sync/tests/test_e2e.py @@ -659,27 +659,27 @@ def test_webext(): ) entity = EntityFactory.create( - resource=res, key=["number"], string="Entity for $1" + resource=res, key=["number"], string="Entity for {$arg1 @source=|$1|}" ) TranslationFactory.create( entity=entity, locale=locale, - string="Translation for $1", + string="Translation for {$arg1 @source=|$1|}", active=True, approved=True, ) - ph = '{"ORIGIN": {"content": "$1", "example": "developer.mozilla.org"}}' entity = EntityFactory.create( resource=res, key=["name"], - string="Entity for $ORIGIN$", - meta=[["placeholders", ph]], + string=".local $ORIGIN = {$arg1 @source=|$1| @example=developer.mozilla.org}\n" + + "{{Entity for {$ORIGIN @source=|$ORIGIN$|}}}", ) TranslationFactory.create( entity=entity, locale=locale, - string="Translation for $ORIGIN$", + string=".local $ORIGIN = {$arg1 @source=|$1| @example=developer.mozilla.org}\n" + + "{{Translation for {$ORIGIN @source=|$ORIGIN$|}}}", active=True, approved=True, ) diff --git a/translate/src/context/Editor.tsx b/translate/src/context/Editor.tsx index 77b3680150..8afc11cea0 100644 --- a/translate/src/context/Editor.tsx +++ b/translate/src/context/Editor.tsx @@ -28,7 +28,7 @@ import { Locale } from './Locale'; import { MachineryTranslations } from './MachineryTranslations'; import { UnsavedActions } from './UnsavedChanges'; import { getMessageEntryFormat } from '../utils/message/getMessageEntryFormat'; -import { androidPlaceholders } from '~/utils/message/android'; +import { getPlaceholderMap } from '../utils/message/placeholders'; export type EditFieldHandle = { get value(): string; @@ -251,7 +251,8 @@ export function EditorProvider({ children }: { children: React.ReactElement }) { break; } case 'android': - case 'gettext': { + case 'gettext': + case 'webext': { const entry = parseEntry(format, str); if (entry) { next.entry = entry; @@ -332,8 +333,8 @@ export function EditorProvider({ children }: { children: React.ReactElement }) { source = serializeEntry(entry); sourceView = true; } - if (entry.format === 'android') { - placeholders = androidPlaceholders(entry.value); + if (format === 'android' || format === 'webext') { + placeholders = getPlaceholderMap(format, entry.value!); } } else { const entry_ = parseEntry(format, source); @@ -344,9 +345,9 @@ export function EditorProvider({ children }: { children: React.ReactElement }) { entry = createSimpleMessageEntry(format, entity.key, source); sourceView = format === 'fluent'; } - if (format === 'android') { + if (format === 'android' || format === 'webext') { const orig = parseEntry(format, entity.original); - if (orig?.value) placeholders = androidPlaceholders(orig.value); + if (orig?.value) placeholders = getPlaceholderMap(format, orig.value); } } diff --git a/translate/src/modules/entitydetails/components/FluentAttribute.test.jsx b/translate/src/modules/entitydetails/components/FluentAttribute.test.jsx index 2db85a550b..af9d827ede 100644 --- a/translate/src/modules/entitydetails/components/FluentAttribute.test.jsx +++ b/translate/src/modules/entitydetails/components/FluentAttribute.test.jsx @@ -1,6 +1,7 @@ import ftl from '@fluent/dedent'; import { shallow } from 'enzyme'; import React from 'react'; +import { parseEntry } from '~/utils/message'; import { FluentAttribute } from './FluentAttribute'; describe('isSimpleSingleAttributeMessage', () => { @@ -9,9 +10,8 @@ describe('isSimpleSingleAttributeMessage', () => { my-entry = .an-atribute = Hello! `; - const wrapper = shallow( - , - ); + const entry = parseEntry('fluent', original); + const wrapper = shallow(); expect(wrapper.isEmptyRender()).toEqual(false); }); @@ -20,9 +20,8 @@ describe('isSimpleSingleAttributeMessage', () => { my-entry = Something .an-atribute = Hello! `; - const wrapper = shallow( - , - ); + const entry = parseEntry('fluent', original); + const wrapper = shallow(); expect(wrapper.isEmptyRender()).toEqual(true); }); @@ -32,9 +31,8 @@ describe('isSimpleSingleAttributeMessage', () => { .an-atribute = Hello! .two-attrites = World! `; - const wrapper = shallow( - , - ); + const entry = parseEntry('fluent', original); + const wrapper = shallow(); expect(wrapper.isEmptyRender()).toEqual(true); }); }); diff --git a/translate/src/modules/entitydetails/components/FluentAttribute.tsx b/translate/src/modules/entitydetails/components/FluentAttribute.tsx index ec4ddee6e1..a230c7c2aa 100644 --- a/translate/src/modules/entitydetails/components/FluentAttribute.tsx +++ b/translate/src/modules/entitydetails/components/FluentAttribute.tsx @@ -1,27 +1,19 @@ import React from 'react'; import { Localized } from '@fluent/react'; +import { isSelectMessage } from '@mozilla/l10n'; -import type { Entity } from '~/api/entity'; -import { parseEntry } from '~/utils/message'; +import type { MessageEntry } from '~/utils/message'; import { Property } from './Property'; -import { isSelectMessage } from '@mozilla/l10n'; - -type Props = { - readonly entity: Entity; -}; /** * Get attribute of a simple single-attribute Fluent message. */ export function FluentAttribute({ - entity: { format, original }, -}: Props): null | React.ReactElement { - if (format !== 'fluent') { - return null; - } - - const entry = parseEntry(format, original); + entry, +}: { + readonly entry: MessageEntry | null; +}): null | React.ReactElement { if (!entry || entry.value || entry.attributes?.size !== 1) { return null; } diff --git a/translate/src/modules/entitydetails/components/Metadata.test.jsx b/translate/src/modules/entitydetails/components/Metadata.test.jsx index 1f289a0342..b848025ea6 100644 --- a/translate/src/modules/entitydetails/components/Metadata.test.jsx +++ b/translate/src/modules/entitydetails/components/Metadata.test.jsx @@ -81,29 +81,29 @@ describe('', () => { ); }); - it('handles sources as an object with examples', () => { - const phData = { - arg1: { content: '', example: 'example_1' }, - arg2: { content: '', example: 'example_2' }, - }; + it('finds examples for placeholders with source', () => { const wrapper = createMetadata({ ...ENTITY, - meta: [['placeholders', JSON.stringify(phData)]], + format: 'webext', + original: ` + .local $MAXIMUM = {$arg2 @source=|$2| @example=5} + .local $REMAINING = {$arg1 @source=|$1| @example=1} + {{{$REMAINING @source=|$REMAINING$|}/{$MAXIMUM @source=|$MAXIMUM$|} masks available.}}`, }); expect(wrapper.find('div.placeholder .content').text()).toBe( - '$ARG1$: example_1, $ARG2$: example_2', + '$MAXIMUM$: 5, $REMAINING$: 1', ); }); - it('handles sources as an object without examples', () => { - const phData = { - arg1: { content: '' }, - arg2: { content: '' }, - }; + it('only shows examples for placeholders with source and example', () => { const wrapper = createMetadata({ ...ENTITY, - meta: [['placeholders', JSON.stringify(phData)]], + format: 'webext', + original: ` + .local $MAXIMUM = {$arg2 @source=|$2|} + .local $REMAINING = {$arg1 @source=|$1| @example=1} + {{{$REMAINING}/{$MAXIMUM @source=|$MAXIMUM$|} masks available.}}`, }); expect(wrapper.find('div.placeholder')).toHaveLength(0); diff --git a/translate/src/modules/entitydetails/components/Metadata.tsx b/translate/src/modules/entitydetails/components/Metadata.tsx index dbce891451..3642f2af1a 100644 --- a/translate/src/modules/entitydetails/components/Metadata.tsx +++ b/translate/src/modules/entitydetails/components/Metadata.tsx @@ -12,6 +12,8 @@ import { FluentAttribute } from './FluentAttribute'; import { Property } from './Property'; import './Metadata.css'; +import { MessageEntry, parseEntry } from '~/utils/message'; +import { getPlaceholderMap } from '~/utils/message/placeholders'; type Props = { entity: Entity; @@ -151,23 +153,24 @@ function SourceReferences({ meta }: { meta: Entity['meta'] }) { return refs.length > 0 ?
    {refs}
: null; } -function SourcePlaceholders({ meta }: { meta: Entity['meta'] }) { +function SourceExamples({ entry }: { readonly entry: MessageEntry | null }) { + if (!entry?.value || Array.isArray(entry.value)) return null; + const phMap = getPlaceholderMap(entry.format, entry.value); + if (!phMap) return null; + const placeholders = Array.from(phMap.values()); const examples: string[] = []; - for (let [key, value] of meta) { - if (key === 'placeholders') { - try { - for (let [name, { example }] of Object.entries( - JSON.parse(value), - )) { - if (example) { - examples.push(`$${name.toUpperCase()}$: ${example}`); - } - } - } catch {} + for (const [name, exp] of Object.entries(entry.value.decl)) { + const example = exp.attr?.example; + if (typeof example === 'string') { + const ph = placeholders.find((ph) => ph.$ === name); + const source = ph?.attr?.source; + if (typeof source === 'string') { + examples.push(`${source}: ${example}`); + } } } - return ( + return examples.length ? ( {examples.join(', ')} - ); + ) : null; } const EntityContext = ({ @@ -233,6 +236,7 @@ export function Metadata({ teamComments, }: Props): React.ReactElement { const { code } = useContext(Locale); + const entry = parseEntry(entity.format, entity.original); return (
@@ -240,9 +244,9 @@ export function Metadata({ - + - + (format === 'fluent' ? fluentMode : commonMode), + StreamLanguage.define( + format === 'fluent' + ? fluentMode + : format === 'webext' + ? webextMode + : commonMode, + ), syntaxHighlighting(style), decoratorPlugin, keymap.of([ diff --git a/translate/src/modules/translationform/utils/editFieldModes.ts b/translate/src/modules/translationform/utils/editFieldModes.ts index 4a42304858..499a57b82a 100644 --- a/translate/src/modules/translationform/utils/editFieldModes.ts +++ b/translate/src/modules/translationform/utils/editFieldModes.ts @@ -141,3 +141,17 @@ export const commonMode: StreamParser> = { } }, }; + +export const webextMode: StreamParser<[]> = { + name: 'webext', + languageData: { closeBrackets: { brackets: ['<'] } }, + startState: () => [], + token(stream, state) { + if (stream.match(/\$[a-zA-Z0-9_@]+\$|\$[1-9]|\$+/)) { + return 'keyword'; + } else { + stream.eatWhile(/[^$]+/); + return 'string'; + } + }, +}; diff --git a/translate/src/utils/message/android.ts b/translate/src/utils/message/android.ts deleted file mode 100644 index c34c2060f4..0000000000 --- a/translate/src/utils/message/android.ts +++ /dev/null @@ -1,59 +0,0 @@ -import { - Expression, - isExpression, - mf2SerializePattern, - Markup, - Pattern, - Message, -} from '@mozilla/l10n'; - -export function androidPlaceholders( - msg: Message, -): Map | null { - const placeholders = new Map(); - if (Array.isArray(msg)) addPlaceholders(msg, placeholders); - else if (msg.msg) addPlaceholders(msg.msg, placeholders); - else for (const v of msg.alt) addPlaceholders(v.pat, placeholders); - return placeholders.size ? placeholders : null; -} - -function addPlaceholders( - pattern: Pattern, - placeholders: Map, -): void { - for (const part of pattern) { - if (typeof part !== 'string') { - placeholders.set(androidEditPlaceholder(part), part); - } - } -} - -export function androidEditPattern(pattern: Pattern): string { - let str = ''; - for (const part of pattern) { - str += typeof part === 'string' ? part : androidEditPlaceholder(part); - } - return str; -} - -function androidEditPlaceholder(part: Expression | Markup): string { - if (typeof part.attr?.source === 'string') return part.attr.source; - if (isExpression(part)) { - if (part.fn === 'html' && part._) return part._; - if (part.fn === 'entity' && part.$) return `&${part.$};`; - } else if (part.open) { - let str = `<${part.open}`; - if (part.opt) { - for (const [name, val] of Object.entries(part.opt)) { - const opt = typeof val === 'string' ? JSON.stringify(val) : '$' + val.$; - str += ` ${name}=${opt}}`; - } - } - return str + '>'; - } else if (part.close) { - if (!part.opt || Object.keys(part.opt).length === 0) - return ``; - } - // Fallback; this is an error - return mf2SerializePattern([part], false); -} diff --git a/translate/src/utils/message/editMessageEntry.tsx b/translate/src/utils/message/editMessageEntry.tsx index 6534a3b28d..8c7bc457c1 100644 --- a/translate/src/utils/message/editMessageEntry.tsx +++ b/translate/src/utils/message/editMessageEntry.tsx @@ -1,9 +1,9 @@ +import { CatchallKey, isSelectMessage, Message, Pattern } from '@mozilla/l10n'; import type { EditorField } from '~/context/Editor'; import type { MessageEntry } from '.'; import { findPluralSelectors } from './findPluralSelectors'; +import { editablePattern } from './placeholders'; import { serializeEntry } from './serializeEntry'; -import { CatchallKey, isSelectMessage, Message, Pattern } from '@mozilla/l10n'; -import { androidEditPattern } from './android'; const emptyHandleRef = (value: string) => ({ current: { @@ -80,7 +80,9 @@ export function patternAsString( format: MessageEntry['format'], pattern: Pattern, ) { - if (format === 'android') return androidEditPattern(pattern); + if (format === 'android' || format === 'webext') { + return editablePattern(format, pattern); + } switch (pattern.length) { case 0: return ''; diff --git a/translate/src/utils/message/getMessageEntryFormat.tsx b/translate/src/utils/message/getMessageEntryFormat.tsx index 58ae8d3554..9bf2515ea7 100644 --- a/translate/src/utils/message/getMessageEntryFormat.tsx +++ b/translate/src/utils/message/getMessageEntryFormat.tsx @@ -5,6 +5,7 @@ export function getMessageEntryFormat(format: string): MessageEntry['format'] { case 'android': case 'gettext': case 'fluent': + case 'webext': return format; default: return 'plain'; diff --git a/translate/src/utils/message/getPlainMessage.test.js b/translate/src/utils/message/getPlainMessage.test.js index cb86238480..8142623ee5 100644 --- a/translate/src/utils/message/getPlainMessage.test.js +++ b/translate/src/utils/message/getPlainMessage.test.js @@ -176,7 +176,7 @@ describe('getPlainMessage', () => { describe('Unicode MessageFormat', () => { it('works for an MF2 string', () => { - expect(getPlainMessage('{{quoted pattern}}', 'gettext')).toEqual( + expect(getPlainMessage('{{quoted pattern}}', 'webext')).toEqual( 'quoted pattern', ); }); diff --git a/translate/src/utils/message/getPlainMessage.ts b/translate/src/utils/message/getPlainMessage.ts index 624670b38b..f996d921ef 100644 --- a/translate/src/utils/message/getPlainMessage.ts +++ b/translate/src/utils/message/getPlainMessage.ts @@ -1,13 +1,12 @@ import { fluentSerializePattern, - FormatKey, Pattern, serializePattern, type Message, } from '@mozilla/l10n'; import type { MessageEntry } from '.'; import { parseEntry } from './parseEntry'; -import { androidEditPattern } from './android'; +import { editablePattern } from './placeholders'; /** * Return a plain string representation of a given message. @@ -68,7 +67,8 @@ function previewMessage(format: string, message: Message): string { onError: () => {}, }); case 'android': - return androidEditPattern(pattern); + case 'webext': + return editablePattern(format, pattern); default: return serializePattern('plain', pattern); } diff --git a/translate/src/utils/message/index.ts b/translate/src/utils/message/index.ts index 5cf899c66f..46f09dcbb8 100644 --- a/translate/src/utils/message/index.ts +++ b/translate/src/utils/message/index.ts @@ -3,7 +3,7 @@ import type { Message } from '@mozilla/l10n'; export type MessageEntry = | { id: string; - format: 'android' | 'fluent' | 'gettext' | 'plain'; + format: 'android' | 'fluent' | 'gettext' | 'plain' | 'webext'; value: Message; attributes?: never; } diff --git a/translate/src/utils/message/parseEntry.ts b/translate/src/utils/message/parseEntry.ts index 5d00a5dddd..b97b2daf16 100644 --- a/translate/src/utils/message/parseEntry.ts +++ b/translate/src/utils/message/parseEntry.ts @@ -9,7 +9,7 @@ import { import type { MessageEntry } from '.'; /** - * Parse a `'fluent'`, `'android'`, or `'gettext'` message source as a {@link MessageEntry}. + * Parse a `'fluent'`, `'android'`, `'gettext'`, or `'webext'` message source as a {@link MessageEntry}. * * @returns `null` on parse error or unsupported format */ @@ -34,9 +34,13 @@ export function parseEntry( case 'android': case 'gettext': + case 'webext': return { format, id: '', value: mf2ParseMessage(source) }; } - } catch {} + } catch (error) { + const msg = error instanceof Error ? error.message : String(error); + console.warn(`Parse error: ${msg}, with entry source:\n${source}`); + } return null; } diff --git a/translate/src/utils/message/placeholders.ts b/translate/src/utils/message/placeholders.ts new file mode 100644 index 0000000000..b3e5383018 --- /dev/null +++ b/translate/src/utils/message/placeholders.ts @@ -0,0 +1,71 @@ +import { + Expression, + isExpression, + mf2SerializePattern, + Markup, + Pattern, + Message, +} from '@mozilla/l10n'; +import type { MessageEntry } from '.'; + +export function getPlaceholderMap( + format: MessageEntry['format'], + msg: Message, +): Map | null { + const placeholders = new Map(); + if (Array.isArray(msg)) addPlaceholders(format, msg, placeholders); + else if (msg.msg) addPlaceholders(format, msg.msg, placeholders); + else for (const v of msg.alt) addPlaceholders(format, v.pat, placeholders); + return placeholders.size ? placeholders : null; +} + +function addPlaceholders( + format: string, + pattern: Pattern, + placeholders: Map, +): void { + for (const part of pattern) { + if (typeof part !== 'string') { + placeholders.set(editablePlaceholder(format, part), part); + } + } +} + +export function editablePattern( + format: MessageEntry['format'], + pattern: Pattern, +): string { + let str = ''; + for (const part of pattern) { + str += typeof part === 'string' ? part : editablePlaceholder(format, part); + } + return str; +} + +function editablePlaceholder( + format: string, + part: Expression | Markup, +): string { + if (typeof part.attr?.source === 'string') return part.attr.source; + if (format === 'android') { + if (isExpression(part)) { + if (part.fn === 'html' && part._) return part._; + if (part.fn === 'entity' && part.$) return `&${part.$};`; + } else if (part.open) { + let str = `<${part.open}`; + if (part.opt) { + for (const [name, val] of Object.entries(part.opt)) { + const opt = + typeof val === 'string' ? JSON.stringify(val) : '$' + val.$; + str += ` ${name}=${opt}}`; + } + } + return str + '>'; + } else if (part.close) { + if (!part.opt || Object.keys(part.opt).length === 0) + return ``; + } + } + // Fallback; this is an error + return mf2SerializePattern([part], false); +} diff --git a/translate/src/utils/message/serializeEntry.ts b/translate/src/utils/message/serializeEntry.ts index 136613dbbb..93d2cb0d16 100644 --- a/translate/src/utils/message/serializeEntry.ts +++ b/translate/src/utils/message/serializeEntry.ts @@ -28,6 +28,7 @@ export function serializeEntry(entry: MessageEntry | null): string { case 'android': case 'gettext': + case 'webext': return entry.value ? mf2SerializeMessage(entry.value) : ''; default: