diff --git a/pontoon/base/migrations/0100_android_as_mf2.py b/pontoon/base/migrations/0100_android_as_mf2.py index eb63da037c..6b65f9d192 100644 --- a/pontoon/base/migrations/0100_android_as_mf2.py +++ b/pontoon/base/migrations/0100_android_as_mf2.py @@ -3,10 +3,11 @@ from moz.l10n.formats.android import android_parse_message, android_serialize_message from moz.l10n.formats.mf2 import mf2_parse_message, mf2_serialize_message +from moz.l10n.model import PatternMessage from django.db import migrations -from pontoon.base.simple_preview import android_simple_preview +from pontoon.base.simple_preview import preview_placeholder android_nl = compile(r"\s*\n\s*") @@ -32,6 +33,10 @@ def mf2_string_changed(obj): return True +def android_simple_preview(msg: PatternMessage) -> str: + return "".join(preview_placeholder(part) for part in msg.pattern) + + def mf2_tm_changed(tm): changed = False src_prev = tm.source diff --git a/pontoon/base/migrations/0103_xliff_as_mf2.py b/pontoon/base/migrations/0103_xliff_as_mf2.py new file mode 100644 index 0000000000..c033e6034b --- /dev/null +++ b/pontoon/base/migrations/0103_xliff_as_mf2.py @@ -0,0 +1,82 @@ +from html import escape, unescape + +from moz.l10n.formats.mf2 import mf2_parse_message, mf2_serialize_message +from moz.l10n.formats.xliff import xliff_parse_message, xliff_serialize_message +from moz.l10n.message import message_from_json + +from django.db import migrations + + +def mf2_entity_changed(entity): + db_source = entity.string + msg = message_from_json(entity.value) + mf2_source = mf2_serialize_message(msg) + if mf2_source == db_source: + return False + entity.string = mf2_source + return True + + +def mf2_translation_changed(translation, is_xcode: bool): + db_source = translation.string + msg = xliff_parse_message(escape(db_source), is_xcode=is_xcode) + mf2_source = mf2_serialize_message(msg) + if mf2_source == db_source: + return False + translation.string = mf2_source + return True + + +def xliff_as_mf2(apps, schema_editor): + Entity = apps.get_model("base", "Entity") + Translation = apps.get_model("base", "Translation") + + entities = Entity.objects.filter(resource__format__in=["xliff", "xcode"]) + ent_fixed = [e for e in entities if mf2_entity_changed(e)] + n = Entity.objects.bulk_update(ent_fixed, ["string"], batch_size=10_000) + print(f" ({n} entities)", end="", flush=True) + + translations = Translation.objects.filter( + entity__resource__format="xcode" + ).select_related("entity") + trans_fixed = [t for t in translations if mf2_translation_changed(t, True)] + n = Translation.objects.bulk_update(trans_fixed, ["string"], batch_size=10_000) + print(f" ({n} xcode translations)", end="", flush=True) + + translations = Translation.objects.filter( + entity__resource__format="xliff" + ).select_related("entity") + trans_fixed = [t for t in translations if mf2_translation_changed(t, False)] + n = Translation.objects.bulk_update(trans_fixed, ["string"], batch_size=10_000) + print(f" ({n} xliff translations)", end="", flush=True) + + +def xliff_string_changed(obj): + mf2_source = obj.string + msg = mf2_parse_message(mf2_source) + string = unescape(xliff_serialize_message(msg)) + if string == mf2_source: + return False + obj.string = string + return True + + +def mf2_as_xliff(apps, schema_editor): + Entity = apps.get_model("base", "Entity") + entities = Entity.objects.filter(resource__format__in=["xliff", "xcode"]) + ent_fixed = [e for e in entities if xliff_string_changed(e)] + n = Entity.objects.bulk_update(ent_fixed, ["meta", "string"], batch_size=10_000) + print(f" ({n} entities)", end="", flush=True) + + Translation = apps.get_model("base", "Translation") + translations = Translation.objects.filter( + entity__resource__format__in=["xliff", "xcode"] + ) + trans_fixed = [t for t in translations if xliff_string_changed(t)] + n = Translation.objects.bulk_update(trans_fixed, ["string"], batch_size=10_000) + print(f" ({n} translations)", end="", flush=True) + + +class Migration(migrations.Migration): + dependencies = [("base", "0102_add_pretranslators_group")] + operations = [migrations.RunPython(xliff_as_mf2, reverse_code=mf2_as_xliff)] diff --git a/pontoon/base/simple_preview.py b/pontoon/base/simple_preview.py index fdbd61b5ba..7696e37cac 100644 --- a/pontoon/base/simple_preview.py +++ b/pontoon/base/simple_preview.py @@ -1,9 +1,7 @@ from json import dumps -from moz.l10n.formats import Format -from moz.l10n.formats.fluent import fluent_parse_entry +from moz.l10n.formats.fluent import fluent_parse_entry, fluent_serialize_message from moz.l10n.formats.mf2 import mf2_parse_message, mf2_serialize_message -from moz.l10n.message import serialize_message from moz.l10n.model import ( CatchallKey, Expression, @@ -11,14 +9,13 @@ Message, Pattern, PatternMessage, - SelectMessage, VariableRef, ) from pontoon.base.models import Resource -def get_simple_preview(format: str, string: str): +def get_simple_preview(format: str, msg: str | Message | Pattern) -> str: """ Flatten a message entry as a simple string. @@ -27,58 +24,58 @@ def get_simple_preview(format: str, string: str): For Fluent, selects the value if it's not empty, or the first non-empty attribute. """ - try: - match format: - case Resource.Format.FLUENT: - entry = fluent_parse_entry(string, with_linepos=False) - if not entry.value.is_empty(): - msg = entry.value - else: - msg = next( + if format == Resource.Format.FLUENT: + if isinstance(msg, str): + try: + entry = fluent_parse_entry(msg, with_linepos=False) + msg = ( + entry.value + if not entry.value.is_empty() + else next( prop for prop in entry.properties.values() if not prop.is_empty() ) - msg = as_pattern_message(msg) - return serialize_message(Format.fluent, msg) + ) + except Exception: + return msg + pattern = as_simple_pattern(msg) + return fluent_serialize_message(PatternMessage(pattern)) - case Resource.Format.ANDROID: - msg = mf2_parse_message(string) - return android_simple_preview(msg) - - case Resource.Format.GETTEXT | Resource.Format.WEBEXT: - msg = mf2_parse_message(string) - msg = as_pattern_message(msg) - return serialize_message(None, msg) - except Exception: - pass - return string - - -def as_pattern_message(msg: Message) -> PatternMessage: - if isinstance(msg, SelectMessage): - default_pattern = next( - pattern - for keys, pattern in msg.variants.items() - if all(isinstance(key, CatchallKey) for key in keys) - ) - return PatternMessage(default_pattern) - else: + if format in ( + Resource.Format.ANDROID, + Resource.Format.GETTEXT, + Resource.Format.WEBEXT, + Resource.Format.XCODE, + Resource.Format.XLIFF, + ): + if isinstance(msg, str): + try: + msg = mf2_parse_message(msg) + except Exception: + return msg + elif isinstance(msg, str): return msg - -def android_simple_preview(msg: Message | Pattern) -> str: - """ - Matches the JS androidEditPattern() from translate/src/utils/message/android.ts - """ preview = "" - pattern = msg if isinstance(msg, list) else as_pattern_message(msg).pattern - for part in pattern: - preview += android_placeholder_preview(part) + for part in as_simple_pattern(msg): + preview += preview_placeholder(part) return preview -def android_placeholder_preview(part: str | Expression | Markup) -> str: +def as_simple_pattern(msg: Message | Pattern) -> Pattern: + if isinstance(msg, list): + return msg + if isinstance(msg, PatternMessage): + return msg.pattern + return next( + pattern + for keys, pattern in msg.variants.items() + if all(isinstance(key, CatchallKey) for key in keys) + ) + + +def preview_placeholder(part: str | Expression | Markup) -> str: if isinstance(part, str): return part if isinstance(ps := part.attributes.get("source", None), str): @@ -88,12 +85,12 @@ def android_placeholder_preview(part: str | Expression | Markup) -> str: return part.arg elif part.function == "entity" and isinstance(part.arg, VariableRef): return part.arg.name - elif part.kind == "open": + elif part.kind in ("open", "standalone"): res = "<" + part.name for name, val in part.options.items(): valstr = dumps(val) if isinstance(val, str) else "$" + val.name res += f" {name}={valstr}" - res += ">" + res += ">" if part.kind == "open" else " />" return res elif part.kind == "close" and not part.options: return f"" diff --git a/pontoon/checks/libraries/__init__.py b/pontoon/checks/libraries/__init__.py index 6a1179ae40..e2029ba759 100644 --- a/pontoon/checks/libraries/__init__.py +++ b/pontoon/checks/libraries/__init__.py @@ -3,7 +3,7 @@ from moz.l10n.model import CatchallKey, Pattern, PatternMessage, SelectMessage from pontoon.base.models import Entity, Resource -from pontoon.base.simple_preview import android_simple_preview +from pontoon.base.simple_preview import get_simple_preview from . import compare_locales, translate_toolkit from .custom import run_custom_checks @@ -75,19 +75,21 @@ def run_checks( case Resource.Format.ANDROID: src_msg = mf2_parse_message(entity.string) tgt_msg = mf2_parse_message(string) - src0 = android_simple_preview(src_msg) + src0 = get_simple_preview(res_format, src_msg) if isinstance(src_msg, SelectMessage) and isinstance( tgt_msg, SelectMessage ): for keys, pattern in tgt_msg.variants.items(): src = ( - android_simple_preview(src_msg.variants[keys]) + get_simple_preview(res_format, src_msg.variants[keys]) if keys == ("one",) and keys in src_msg.variants else src0 ) - tt_patterns.append((src, android_simple_preview(pattern))) + tt_patterns.append( + (src, get_simple_preview(res_format, pattern)) + ) else: - tt_patterns.append((src0, android_simple_preview(tgt_msg))) + tt_patterns.append((src0, get_simple_preview(res_format, tgt_msg))) case Resource.Format.GETTEXT: src_msg = mf2_parse_message(entity.string) diff --git a/pontoon/checks/libraries/custom.py b/pontoon/checks/libraries/custom.py index 4544b5d517..723dbf16c8 100644 --- a/pontoon/checks/libraries/custom.py +++ b/pontoon/checks/libraries/custom.py @@ -17,10 +17,7 @@ ) from pontoon.base.models import Entity, Resource -from pontoon.base.simple_preview import ( - android_placeholder_preview, - android_simple_preview, -) +from pontoon.base.simple_preview import get_simple_preview, preview_placeholder parser = FluentParser() @@ -81,7 +78,7 @@ def run_custom_checks(entity: Entity, string: str) -> dict[str, list[str]]: for el in pattern if not isinstance(el, str) ) - orig_ps = {android_placeholder_preview(ph) for ph in orig_ph_iter} + orig_ps = {preview_placeholder(ph) for ph in orig_ph_iter} except ValueError: orig_msg = None orig_ps = set() @@ -99,11 +96,11 @@ def run_custom_checks(entity: Entity, string: str) -> dict[str, list[str]]: try: for pattern in patterns: android_msg = android_parse_message( - escape(android_simple_preview(pattern)) + escape(get_simple_preview(Resource.Format.ANDROID, pattern)) ) for el in android_msg.pattern: if not isinstance(el, str): - ps = android_placeholder_preview(el) + ps = preview_placeholder(el) if ps in orig_ps: found_ps.add(ps) else: diff --git a/pontoon/pretranslation/pretranslate.py b/pontoon/pretranslation/pretranslate.py index f658581ab3..4f16689d3b 100644 --- a/pontoon/pretranslation/pretranslate.py +++ b/pontoon/pretranslation/pretranslate.py @@ -62,6 +62,8 @@ def get_pretranslation( Resource.Format.ANDROID, Resource.Format.GETTEXT, Resource.Format.WEBEXT, + Resource.Format.XCODE, + Resource.Format.XLIFF, }: format = Format.mf2 msg = parse_message(format, entity.string) @@ -90,6 +92,8 @@ def __init__(self, entity: Entity, locale: Locale, preserve_placeables: bool): Resource.Format.ANDROID | Resource.Format.GETTEXT | Resource.Format.WEBEXT + | Resource.Format.XCODE + | Resource.Format.XLIFF ): self.format = Format.mf2 case _: diff --git a/pontoon/sync/core/translations_from_repo.py b/pontoon/sync/core/translations_from_repo.py index b8cb207cf7..2abd2b5c46 100644 --- a/pontoon/sync/core/translations_from_repo.py +++ b/pontoon/sync/core/translations_from_repo.py @@ -32,7 +32,7 @@ from pontoon.checks.utils import bulk_run_checks from pontoon.sync.core.checkout import Checkout, Checkouts from pontoon.sync.core.paths import UploadPaths -from pontoon.sync.formats import as_vcs_translations +from pontoon.sync.formats import as_repo_translations log = logging.getLogger(__name__) @@ -162,7 +162,7 @@ def find_db_updates( translated_resources[db_path].add(locale.pk) translations.update( ((db_path, tx.key, locale.pk), (tx.string, tx.fuzzy)) - for tx in as_vcs_translations(l10n_res) + for tx in as_repo_translations(l10n_res) ) except Exception as error: scope = f"[{project.slug}:{db_path}, {locale.code}]" diff --git a/pontoon/sync/core/translations_to_repo.py b/pontoon/sync/core/translations_to_repo.py index c4de76c1a0..ea008012b5 100644 --- a/pontoon/sync/core/translations_to_repo.py +++ b/pontoon/sync/core/translations_to_repo.py @@ -347,7 +347,7 @@ def set_translation( return False match format: - case Format.android | Format.gettext | Format.webext: + case Format.android | Format.gettext | Format.webext | Format.xliff: msg = parse_message(Format.mf2, tx.string) if isinstance(entry.value, SelectMessage): entry.value.variants = ( diff --git a/pontoon/sync/formats/__init__.py b/pontoon/sync/formats/__init__.py index 06c2c3ee28..4b95c9611e 100644 --- a/pontoon/sync/formats/__init__.py +++ b/pontoon/sync/formats/__init__.py @@ -2,6 +2,7 @@ Parsing resource files. """ +from dataclasses import dataclass from os.path import splitext from re import Match, compile from typing import Iterator @@ -10,13 +11,30 @@ from moz.l10n.formats import Format, detect_format, l10n_extensions from moz.l10n.formats.fluent import fluent_astify_entry from moz.l10n.message import message_to_json, serialize_message -from moz.l10n.model import Entry, Id as L10nId, Message, Resource as MozL10nResource +from moz.l10n.model import ( + CatchallKey, + Entry, + Expression, + Id as L10nId, + Message, + PatternMessage, + Resource as MozL10nResource, + SelectMessage, + VariableRef, +) from pontoon.base.models import Entity -from .common import VCSTranslation -from .gettext import gettext_as_entity, gettext_as_translation -from .xliff import xliff_as_entity, xliff_as_translation + +@dataclass +class RepoTranslation: + """ + A single translation of a source string into another language. + """ + + key: tuple[str, ...] + string: str + fuzzy: bool = False def are_compatible_files(file_a, file_b): @@ -49,7 +67,7 @@ def _as_string(format: Format | None, entry: Entry[Message]) -> str: case Format.fluent: fluent_entry = fluent_astify_entry(entry, lambda _: "") return _fluent_serializer.serialize_entry(fluent_entry) - case Format.android | Format.webext: + case Format.android | Format.gettext | Format.webext | Format.xliff: return serialize_message(Format.mf2, entry.value) case Format.properties: string = serialize_message(Format.properties, entry.value) @@ -60,24 +78,24 @@ def _as_string(format: Format | None, entry: Entry[Message]) -> str: return serialize_message(format, entry.value) -def as_vcs_translations(res: MozL10nResource[Message]) -> Iterator[VCSTranslation]: +def as_repo_translations(res: MozL10nResource[Message]) -> Iterator[RepoTranslation]: for section in res.sections: if res.format == Format.android and section.id: continue for entry in section.entries: - if isinstance(entry, Entry): - match res.format: - case Format.gettext: - tx = gettext_as_translation(entry) - case Format.xliff: - tx = xliff_as_translation(section.id, entry) - case _: - tx = VCSTranslation( - key=section.id + entry.id, - string=_as_string(res.format, entry), - ) - if tx is not None: - yield tx + if isinstance(entry, Entry) and not ( + res.format in {Format.gettext, Format.xliff} and entry.value.is_empty() + ): + fuzzy = ( + any(m.key == "flag" and m.value == "fuzzy" for m in entry.meta) + if res.format == Format.gettext + else False + ) + yield RepoTranslation( + key=section.id + entry.id, + string=_as_string(res.format, entry), + fuzzy=fuzzy, + ) def as_entity( @@ -87,25 +105,41 @@ def as_entity( **kwargs, ) -> Entity: """At least `order`, `resource`, and `section` should be set as `kwargs`.""" - match format: - case Format.gettext: - return gettext_as_entity(entry, kwargs) - case Format.xliff: - return xliff_as_entity(section_id, entry, kwargs) - case _: - return Entity( - key=list(section_id + entry.id), - value=message_to_json(entry.value), - properties=( - { - name: message_to_json(value) - for name, value in entry.properties.items() - } - if entry.properties - else None - ), - string=_as_string(format, entry), - comment=entry.comment, - meta=[[m.key, m.value] for m in entry.meta], - **kwargs, + if format == Format.gettext: + source_str = entry.id[0] + plural_str = entry.get_meta("plural") + source_msg = ( + PatternMessage([source_str]) + if plural_str is None + else SelectMessage( + declarations={"n": Expression(VariableRef("n"), "number")}, + selectors=(VariableRef("n"),), + variants={ + ("one",): [source_str], + (CatchallKey("other"),): [plural_str], + }, ) + ) + + return Entity( + key=list(entry.id), + value=message_to_json(source_msg), + string=serialize_message(Format.mf2, source_msg), + comment=entry.comment, + meta=[[m.key, m.value] for m in entry.meta], + **kwargs, + ) + + return Entity( + key=list(section_id + entry.id), + value=message_to_json(entry.value), + properties=( + {name: message_to_json(value) for name, value in entry.properties.items()} + if entry.properties + else None + ), + string=_as_string(format, entry), + comment=entry.comment, + meta=[[m.key, m.value] for m in entry.meta], + **kwargs, + ) diff --git a/pontoon/sync/formats/common.py b/pontoon/sync/formats/common.py deleted file mode 100644 index d6936f3b29..0000000000 --- a/pontoon/sync/formats/common.py +++ /dev/null @@ -1,12 +0,0 @@ -from dataclasses import dataclass - - -@dataclass -class VCSTranslation: - """ - A single translation of a source string into another language. - """ - - key: tuple[str, ...] - string: str - fuzzy: bool = False diff --git a/pontoon/sync/formats/gettext.py b/pontoon/sync/formats/gettext.py deleted file mode 100644 index 09081872f5..0000000000 --- a/pontoon/sync/formats/gettext.py +++ /dev/null @@ -1,48 +0,0 @@ -from typing import Any - -from moz.l10n.formats import Format -from moz.l10n.message import message_to_json, serialize_message -from moz.l10n.model import ( - CatchallKey, - Entry, - Expression, - Message, - PatternMessage, - SelectMessage, - VariableRef, -) - -from pontoon.base.models import Entity - -from .common import VCSTranslation - - -def gettext_as_translation(entry: Entry[Message]): - if entry.value.is_empty(): - return None - string = serialize_message(Format.mf2, entry.value) - fuzzy = any(m.key == "flag" and m.value == "fuzzy" for m in entry.meta) - return VCSTranslation(key=entry.id, string=string, fuzzy=fuzzy) - - -def gettext_as_entity(entry: Entry[Message], kwargs: dict[str, Any]) -> Entity: - source_str = entry.id[0] - plural_str = entry.get_meta("plural") - source_msg = ( - PatternMessage([source_str]) - if plural_str is None - else SelectMessage( - declarations={"n": Expression(VariableRef("n"), "number")}, - selectors=(VariableRef("n"),), - variants={("one",): [source_str], (CatchallKey("other"),): [plural_str]}, - ) - ) - - return Entity( - key=list(entry.id), - value=message_to_json(source_msg), - string=serialize_message(Format.mf2, source_msg), - comment=entry.comment, - meta=[[m.key, m.value] for m in entry.meta], - **kwargs, - ) diff --git a/pontoon/sync/formats/xliff.py b/pontoon/sync/formats/xliff.py deleted file mode 100644 index 0c2f8fb4d2..0000000000 --- a/pontoon/sync/formats/xliff.py +++ /dev/null @@ -1,36 +0,0 @@ -""" -Parser for the xliff translation format. -""" - -from __future__ import annotations - -from html import unescape -from typing import Any - -from moz.l10n.formats import Format -from moz.l10n.message import message_to_json, serialize_message -from moz.l10n.model import Entry, Id as L10nId, Message - -from pontoon.base.models import Entity - -from .common import VCSTranslation - - -def xliff_as_translation(section_id: L10nId, entry: Entry): - # Here, entry.value is from the - string = unescape(serialize_message(Format.xliff, entry.value)) - return VCSTranslation(key=section_id + entry.id, string=string) if string else None - - -def xliff_as_entity( - section_id: L10nId, entry: Entry[Message], kwargs: dict[str, Any] -) -> Entity: - # Here, entry.value is from the - return Entity( - key=list(section_id + entry.id), - value=message_to_json(entry.value), - string=unescape(serialize_message(Format.xliff, entry.value)), - comment=entry.comment, - meta=[[m.key, m.value] for m in entry.meta], - **kwargs, - ) diff --git a/pontoon/sync/tests/formats/test_ftl.py b/pontoon/sync/tests/formats/test_ftl.py index 52fb5682de..a80a433009 100644 --- a/pontoon/sync/tests/formats/test_ftl.py +++ b/pontoon/sync/tests/formats/test_ftl.py @@ -5,7 +5,7 @@ from moz.l10n.formats import Format from moz.l10n.resource import parse_resource -from pontoon.sync.formats import as_entity, as_vcs_translations +from pontoon.sync.formats import as_entity, as_repo_translations class FTLTests(TestCase): @@ -41,7 +41,7 @@ def test_fluent(self): as_entity(Format.fluent, (), entry, date_created=datetime.now()) for entry in res.all_entries() ) - t0, t1, t2, t3, t4, t5, t6 = as_vcs_translations(res) + t0, t1, t2, t3, t4, t5, t6 = as_repo_translations(res) # basic assert e0.comment == "Sample comment" diff --git a/pontoon/sync/tests/formats/test_gettext.py b/pontoon/sync/tests/formats/test_gettext.py index c7aee47fc8..7a670f9a2a 100644 --- a/pontoon/sync/tests/formats/test_gettext.py +++ b/pontoon/sync/tests/formats/test_gettext.py @@ -5,7 +5,7 @@ from moz.l10n.formats import Format from moz.l10n.resource import parse_resource -from pontoon.sync.formats import as_entity, as_vcs_translations +from pontoon.sync.formats import as_entity, as_repo_translations class GettextTests(TestCase): @@ -69,7 +69,7 @@ def test_gettext(self): as_entity(Format.gettext, (), entry, date_created=datetime.now()) for entry in res.all_entries() ) - t0, t1, t2, t3, t4, t6, t7 = as_vcs_translations(res) + t0, t1, t2, t3, t4, t6, t7 = as_repo_translations(res) # basic assert e0.comment == "Sample comment" @@ -237,7 +237,7 @@ def test_context_and_empty_messages(self): as_entity(Format.gettext, (), entry, date_created=datetime.now()) for entry in res.all_entries() ) - assert list(as_vcs_translations(res)) == [] + assert list(as_repo_translations(res)) == [] assert e0.key == ["Source", "Main context"] assert e0.string == "Source" diff --git a/pontoon/sync/tests/formats/test_json_extensions.py b/pontoon/sync/tests/formats/test_json_extensions.py index eeeefd8b02..0e397300c3 100644 --- a/pontoon/sync/tests/formats/test_json_extensions.py +++ b/pontoon/sync/tests/formats/test_json_extensions.py @@ -5,7 +5,7 @@ from moz.l10n.formats import Format from moz.l10n.resource import parse_resource -from pontoon.sync.formats import as_entity, as_vcs_translations +from pontoon.sync.formats import as_entity, as_repo_translations class JsonExtensionsTests(TestCase): @@ -45,7 +45,7 @@ def test_webext(self): as_entity(Format.webext, (), entry, date_created=datetime.now()) for entry in res.all_entries() ) - t0, t1, t2, t3 = as_vcs_translations(res) + t0, t1, t2, t3 = as_repo_translations(res) assert e0.comment == "Sample comment" assert t0.string == "Translated String" diff --git a/pontoon/sync/tests/formats/test_json_keyvalue.py b/pontoon/sync/tests/formats/test_json_keyvalue.py index 24d3ef0c18..68cac77760 100644 --- a/pontoon/sync/tests/formats/test_json_keyvalue.py +++ b/pontoon/sync/tests/formats/test_json_keyvalue.py @@ -5,7 +5,7 @@ from moz.l10n.formats import Format from moz.l10n.resource import parse_resource -from pontoon.sync.formats import as_entity, as_vcs_translations +from pontoon.sync.formats import as_entity, as_repo_translations class JsonKeyValueTests(TestCase): @@ -24,7 +24,7 @@ def test_plain_json(self): as_entity(Format.plain_json, (), entry, date_created=datetime.now()) for entry in res.all_entries() ) - t0, t1 = as_vcs_translations(res) + t0, t1 = as_repo_translations(res) assert e0.key == ["No Comments or Sources"] assert e0.string == "Translated No Comments or Sources" diff --git a/pontoon/sync/tests/formats/test_properties.py b/pontoon/sync/tests/formats/test_properties.py index affdb956e1..a39de48d0e 100644 --- a/pontoon/sync/tests/formats/test_properties.py +++ b/pontoon/sync/tests/formats/test_properties.py @@ -5,7 +5,7 @@ from moz.l10n.formats import Format from moz.l10n.resource import parse_resource -from pontoon.sync.formats import as_entity, as_vcs_translations +from pontoon.sync.formats import as_entity, as_repo_translations class PropertiesTests(TestCase): @@ -28,7 +28,7 @@ def test_properties(self): as_entity(Format.properties, (), entry, date_created=datetime.now()) for entry in res.all_entries() ) - t0, t1, t2, t3 = as_vcs_translations(res) + t0, t1, t2, t3 = as_repo_translations(res) # basic assert e0.comment == "Sample comment" diff --git a/pontoon/sync/tests/formats/test_xliff.py b/pontoon/sync/tests/formats/test_xliff.py index 3b5affbba8..dcec236274 100644 --- a/pontoon/sync/tests/formats/test_xliff.py +++ b/pontoon/sync/tests/formats/test_xliff.py @@ -6,7 +6,7 @@ from moz.l10n.model import Entry from moz.l10n.resource import parse_resource -from pontoon.sync.formats import as_entity, as_vcs_translations +from pontoon.sync.formats import as_entity, as_repo_translations class XLIFFTests(TestCase): @@ -46,7 +46,7 @@ def test_xliff(self): if isinstance(entry, Entry) ) t_res = parse_resource(Format.xliff, src) - t0, t1, t2 = as_vcs_translations(t_res) + t0, t1, t2 = as_repo_translations(t_res) # basic assert e0.comment == "Sample comment" diff --git a/pontoon/sync/tests/formats/test_xml.py b/pontoon/sync/tests/formats/test_xml.py index 435adfd8af..1a3d34ef94 100644 --- a/pontoon/sync/tests/formats/test_xml.py +++ b/pontoon/sync/tests/formats/test_xml.py @@ -5,7 +5,7 @@ from moz.l10n.formats import Format from moz.l10n.resource import parse_resource -from pontoon.sync.formats import as_entity, as_vcs_translations +from pontoon.sync.formats import as_entity, as_repo_translations class AndroidXMLTests(TestCase): @@ -30,7 +30,7 @@ def test_android(self): as_entity(Format.android, (), entry, date_created=datetime.now()) for entry in res.all_entries() ) - t0, t1, t2, t3 = as_vcs_translations(res) + t0, t1, t2, t3 = as_repo_translations(res) # basic assert e0.comment == "Sample comment" @@ -81,7 +81,7 @@ def test_android_quotes(self): """).strip() res = parse_resource(Format.android, src) - (t0,) = as_vcs_translations(res) + (t0,) = as_repo_translations(res) assert t0.string == "'" def test_android_escapes_and_trimming(self): @@ -92,7 +92,7 @@ def test_android_escapes_and_trimming(self): """) res = parse_resource(Format.android, src) - (t0,) = as_vcs_translations(res) + (t0,) = as_repo_translations(res) assert t0.string == " \n" def test_android_xliffg(self): @@ -108,7 +108,7 @@ def test_android_xliffg(self): as_entity(Format.android, (), entry, date_created=datetime.now()) for entry in res.all_entries() ) - (t0,) = as_vcs_translations(res) + (t0,) = as_repo_translations(res) assert ( e0.string diff --git a/translate/src/context/Editor.tsx b/translate/src/context/Editor.tsx index 8afc11cea0..4a55cf8ea5 100644 --- a/translate/src/context/Editor.tsx +++ b/translate/src/context/Editor.tsx @@ -28,7 +28,10 @@ import { Locale } from './Locale'; import { MachineryTranslations } from './MachineryTranslations'; import { UnsavedActions } from './UnsavedChanges'; import { getMessageEntryFormat } from '../utils/message/getMessageEntryFormat'; -import { getPlaceholderMap } from '../utils/message/placeholders'; +import { + getPlaceholderMap, + placeholderFormats, +} from '../utils/message/placeholders'; export type EditFieldHandle = { get value(): string; @@ -252,7 +255,9 @@ export function EditorProvider({ children }: { children: React.ReactElement }) { } case 'android': case 'gettext': - case 'webext': { + case 'webext': + case 'xcode': + case 'xliff': { const entry = parseEntry(format, str); if (entry) { next.entry = entry; @@ -333,8 +338,8 @@ export function EditorProvider({ children }: { children: React.ReactElement }) { source = serializeEntry(entry); sourceView = true; } - if (format === 'android' || format === 'webext') { - placeholders = getPlaceholderMap(format, entry.value!); + if (placeholderFormats.has(format)) { + placeholders = getPlaceholderMap(entry.value!); } } else { const entry_ = parseEntry(format, source); @@ -345,9 +350,9 @@ export function EditorProvider({ children }: { children: React.ReactElement }) { entry = createSimpleMessageEntry(format, entity.key, source); sourceView = format === 'fluent'; } - if (format === 'android' || format === 'webext') { + if (placeholderFormats.has(format)) { const orig = parseEntry(format, entity.original); - if (orig?.value) placeholders = getPlaceholderMap(format, orig.value); + if (orig?.value) placeholders = getPlaceholderMap(orig.value); } } diff --git a/translate/src/modules/entitydetails/components/Metadata.tsx b/translate/src/modules/entitydetails/components/Metadata.tsx index 3642f2af1a..9177f12ca3 100644 --- a/translate/src/modules/entitydetails/components/Metadata.tsx +++ b/translate/src/modules/entitydetails/components/Metadata.tsx @@ -155,7 +155,7 @@ function SourceReferences({ meta }: { meta: Entity['meta'] }) { function SourceExamples({ entry }: { readonly entry: MessageEntry | null }) { if (!entry?.value || Array.isArray(entry.value)) return null; - const phMap = getPlaceholderMap(entry.format, entry.value); + const phMap = getPlaceholderMap(entry.value); if (!phMap) return null; const placeholders = Array.from(phMap.values()); const examples: string[] = []; diff --git a/translate/src/modules/translation/components/Translation.tsx b/translate/src/modules/translation/components/Translation.tsx index a3c2878c5c..4879c49476 100644 --- a/translate/src/modules/translation/components/Translation.tsx +++ b/translate/src/modules/translation/components/Translation.tsx @@ -3,6 +3,7 @@ import React from 'react'; import { getPlainMessage } from '~/utils/message'; import { GenericTranslation } from './GenericTranslation'; +import { placeholderFormats } from '~/utils/message/placeholders'; type Props = { content: string; @@ -23,9 +24,8 @@ export function Translation({ if ( format === 'fluent' || - format === 'android' || format === 'gettext' || - format === 'webext' + placeholderFormats.has(format) ) { content = getPlainMessage(content, format); diffTarget &&= getPlainMessage(diffTarget, format); diff --git a/translate/src/modules/translationform/utils/editFieldExtensions.ts b/translate/src/modules/translationform/utils/editFieldExtensions.ts index 5501702d8d..3d18652956 100644 --- a/translate/src/modules/translationform/utils/editFieldExtensions.ts +++ b/translate/src/modules/translationform/utils/editFieldExtensions.ts @@ -25,7 +25,7 @@ import { EditorActions } from '~/context/Editor'; import { useCopyOriginalIntoEditor } from '~/modules/editor'; import { placeholder } from '~/modules/placeable/placeholder'; import { MessageEntry, parseEntry } from '~/utils/message'; -import { patternAsString } from '~/utils/message/editMessageEntry'; +import { editablePattern } from '~/utils/message/placeholders'; import { decoratorPlugin } from './decoratorPlugin'; import { useHandleCtrlShiftArrow, @@ -188,9 +188,9 @@ function* entryPatterns(format: string, source: string) { } function* msgPatterns(format: MessageEntry['format'], msg: Message) { - if (Array.isArray(msg)) yield patternAsString(format, msg); - else if (msg.msg) yield patternAsString(format, msg.msg); - else for (const v of msg.alt) yield patternAsString(format, v.pat); + if (Array.isArray(msg)) yield editablePattern(msg); + else if (msg.msg) yield editablePattern(msg.msg); + else for (const v of msg.alt) yield editablePattern(v.pat); } function completePlaceholder( diff --git a/translate/src/utils/message/editMessageEntry.tsx b/translate/src/utils/message/editMessageEntry.tsx index 8c7bc457c1..e1ff08ff75 100644 --- a/translate/src/utils/message/editMessageEntry.tsx +++ b/translate/src/utils/message/editMessageEntry.tsx @@ -1,4 +1,4 @@ -import { CatchallKey, isSelectMessage, Message, Pattern } from '@mozilla/l10n'; +import { isSelectMessage, type CatchallKey, type Message } from '@mozilla/l10n'; import type { EditorField } from '~/context/Editor'; import type { MessageEntry } from '.'; import { findPluralSelectors } from './findPluralSelectors'; @@ -69,30 +69,13 @@ function* genPatterns( label: (typeof key === 'string' ? key : key['*']) || 'other', plural: plurals.has(i), })); - yield [keys, labels, patternAsString(format, pat)]; + yield [keys, labels, editablePattern(pat)]; } } else { - yield [[], [], patternAsString(format, Array.isArray(msg) ? msg : msg.msg)]; + yield [[], [], editablePattern(Array.isArray(msg) ? msg : msg.msg)]; } } -export function patternAsString( - format: MessageEntry['format'], - pattern: Pattern, -) { - if (format === 'android' || format === 'webext') { - return editablePattern(format, pattern); - } - switch (pattern.length) { - case 0: - return ''; - case 1: - if (typeof pattern[0] === 'string') return pattern[0]; - throw new Error('Unsupported message element'); - } - throw new Error(`Unsupported message pattern length ${pattern.length}`); -} - function getId(name: string, keys: (string | CatchallKey)[]) { return [ name, diff --git a/translate/src/utils/message/getMessageEntryFormat.tsx b/translate/src/utils/message/getMessageEntryFormat.tsx index 9bf2515ea7..eb16783dd3 100644 --- a/translate/src/utils/message/getMessageEntryFormat.tsx +++ b/translate/src/utils/message/getMessageEntryFormat.tsx @@ -6,6 +6,8 @@ export function getMessageEntryFormat(format: string): MessageEntry['format'] { case 'gettext': case 'fluent': case 'webext': + case 'xcode': + case 'xliff': return format; default: return 'plain'; diff --git a/translate/src/utils/message/getPlainMessage.ts b/translate/src/utils/message/getPlainMessage.ts index f996d921ef..4b3bafd934 100644 --- a/translate/src/utils/message/getPlainMessage.ts +++ b/translate/src/utils/message/getPlainMessage.ts @@ -60,16 +60,10 @@ function previewMessage(format: string, message: Message): string { pattern = catchall.pat; } - switch (format) { - case 'fluent': - return fluentSerializePattern(pattern, { + return format === 'fluent' + ? fluentSerializePattern(pattern, { escapeSyntax: false, onError: () => {}, - }); - case 'android': - case 'webext': - return editablePattern(format, pattern); - default: - return serializePattern('plain', pattern); - } + }) + : editablePattern(pattern); } diff --git a/translate/src/utils/message/index.ts b/translate/src/utils/message/index.ts index 46f09dcbb8..fc3aaf1392 100644 --- a/translate/src/utils/message/index.ts +++ b/translate/src/utils/message/index.ts @@ -3,7 +3,14 @@ import type { Message } from '@mozilla/l10n'; export type MessageEntry = | { id: string; - format: 'android' | 'fluent' | 'gettext' | 'plain' | 'webext'; + format: + | 'android' + | 'fluent' + | 'gettext' + | 'plain' + | 'webext' + | 'xcode' + | 'xliff'; value: Message; attributes?: never; } diff --git a/translate/src/utils/message/parseEntry.ts b/translate/src/utils/message/parseEntry.ts index b97b2daf16..e7531c4ead 100644 --- a/translate/src/utils/message/parseEntry.ts +++ b/translate/src/utils/message/parseEntry.ts @@ -9,7 +9,9 @@ import { import type { MessageEntry } from '.'; /** - * Parse a `'fluent'`, `'android'`, `'gettext'`, or `'webext'` message source as a {@link MessageEntry}. + * Parse a + * `'fluent'`, `'android'`, `'gettext'`, `'webext'`, `'xcode'`, or `'xliff'` + * message source as a {@link MessageEntry}. * * @returns `null` on parse error or unsupported format */ @@ -35,6 +37,8 @@ export function parseEntry( case 'android': case 'gettext': case 'webext': + case 'xcode': + case 'xliff': return { format, id: '', value: mf2ParseMessage(source) }; } } catch (error) { diff --git a/translate/src/utils/message/placeholders.ts b/translate/src/utils/message/placeholders.ts index b3e5383018..ef140eaa66 100644 --- a/translate/src/utils/message/placeholders.ts +++ b/translate/src/utils/message/placeholders.ts @@ -6,65 +6,60 @@ import { Pattern, Message, } from '@mozilla/l10n'; -import type { MessageEntry } from '.'; + +export const placeholderFormats = new Set([ + 'android', + 'webext', + 'xcode', + 'xliff', +]); export function getPlaceholderMap( - format: MessageEntry['format'], msg: Message, ): Map | null { const placeholders = new Map(); - if (Array.isArray(msg)) addPlaceholders(format, msg, placeholders); - else if (msg.msg) addPlaceholders(format, msg.msg, placeholders); - else for (const v of msg.alt) addPlaceholders(format, v.pat, placeholders); + if (Array.isArray(msg)) addPlaceholders(msg, placeholders); + else if (msg.msg) addPlaceholders(msg.msg, placeholders); + else for (const v of msg.alt) addPlaceholders(v.pat, placeholders); return placeholders.size ? placeholders : null; } function addPlaceholders( - format: string, pattern: Pattern, placeholders: Map, ): void { for (const part of pattern) { if (typeof part !== 'string') { - placeholders.set(editablePlaceholder(format, part), part); + placeholders.set(editablePlaceholder(part), part); } } } -export function editablePattern( - format: MessageEntry['format'], - pattern: Pattern, -): string { +export function editablePattern(pattern: Pattern): string { let str = ''; for (const part of pattern) { - str += typeof part === 'string' ? part : editablePlaceholder(format, part); + str += typeof part === 'string' ? part : editablePlaceholder(part); } return str; } -function editablePlaceholder( - format: string, - part: Expression | Markup, -): string { +function editablePlaceholder(part: Expression | Markup): string { if (typeof part.attr?.source === 'string') return part.attr.source; - if (format === 'android') { - if (isExpression(part)) { - if (part.fn === 'html' && part._) return part._; - if (part.fn === 'entity' && part.$) return `&${part.$};`; - } else if (part.open) { - let str = `<${part.open}`; - if (part.opt) { - for (const [name, val] of Object.entries(part.opt)) { - const opt = - typeof val === 'string' ? JSON.stringify(val) : '$' + val.$; - str += ` ${name}=${opt}}`; - } + if (isExpression(part)) { + if (part.fn === 'html' && part._) return part._; // android-only + if (part.fn === 'entity' && part.$) return `&${part.$};`; // android-only + } else if (part.open || part.elem) { + let str = `<${part.open ?? part.elem}`; + if (part.opt) { + for (const [name, val] of Object.entries(part.opt)) { + const opt = typeof val === 'string' ? JSON.stringify(val) : '$' + val.$; + str += ` ${name}=${opt}}`; } - return str + '>'; - } else if (part.close) { - if (!part.opt || Object.keys(part.opt).length === 0) - return ``; } + return str + (part.open ? '>' : ' />'); + } else if (part.close) { + if (!part.opt || Object.keys(part.opt).length === 0) + return ``; } // Fallback; this is an error return mf2SerializePattern([part], false); diff --git a/translate/src/utils/message/serializeEntry.ts b/translate/src/utils/message/serializeEntry.ts index 93d2cb0d16..74e542995a 100644 --- a/translate/src/utils/message/serializeEntry.ts +++ b/translate/src/utils/message/serializeEntry.ts @@ -29,6 +29,8 @@ export function serializeEntry(entry: MessageEntry | null): string { case 'android': case 'gettext': case 'webext': + case 'xcode': + case 'xliff': return entry.value ? mf2SerializeMessage(entry.value) : ''; default: