diff --git a/babel/messages/extract.py b/babel/messages/extract.py index 4676ee613..b22dfce9c 100644 --- a/babel/messages/extract.py +++ b/babel/messages/extract.py @@ -759,7 +759,7 @@ def extract_javascript( contents of javascript template strings. :param lineno: line number offset (for parsing embedded fragments) """ - from babel.messages.jslexer import Token, tokenize, unquote_string + from babel.messages.jslexer import Token, _tokenize, _unquote_string funcname = message_lineno = None messages = [] @@ -770,7 +770,7 @@ def extract_javascript( last_token = None call_stack = -1 dotted = any('.' in kw for kw in keywords) - for token in tokenize( + for token in _tokenize( fileobj.read().decode(encoding), jsx=options.get("jsx", True), template_string=options.get("template_string", True), @@ -786,7 +786,7 @@ def extract_javascript( and token.type == 'template_string' ): message_lineno = token.lineno - messages = [unquote_string(token.value)] + messages = [_unquote_string(token.value)] call_stack = 0 token = Token('operator', ')', token.lineno) @@ -864,7 +864,7 @@ def extract_javascript( call_stack = -1 elif token.type in ('string', 'template_string'): - new_value = unquote_string(token.value) + new_value = _unquote_string(token.value) if concatenate_next: last_argument = (last_argument or '') + new_value concatenate_next = False diff --git a/babel/messages/jslexer.py b/babel/messages/jslexer.py index 6e123449b..fa9b74294 100644 --- a/babel/messages/jslexer.py +++ b/babel/messages/jslexer.py @@ -12,6 +12,7 @@ from __future__ import annotations import re +import warnings from collections.abc import Generator from typing import NamedTuple @@ -64,7 +65,7 @@ class Token(NamedTuple): ] # fmt: skip -def get_rules( +def _get_rules( jsx: bool, dotted: bool, template_string: bool, @@ -88,7 +89,7 @@ def get_rules( return rules -def indicates_division(token: Token) -> bool: +def _indicates_division(token: Token) -> bool: """A helper function that helps the tokenizer to decide if the current token may be followed by a division operator. """ @@ -97,7 +98,7 @@ def indicates_division(token: Token) -> bool: return token.type in ('name', 'number', 'string', 'regexp') -def unquote_string(string: str) -> str: +def _unquote_string(string: str) -> str: """Unquote a string with JavaScript rules. The string has to start with string delimiters (``'``, ``"`` or the back-tick/grave accent (for template strings).) """ @@ -173,7 +174,7 @@ def unquote_string(string: str) -> str: return ''.join(result) -def tokenize( +def _tokenize( source: str, jsx: bool = True, dotted: bool = True, @@ -192,7 +193,7 @@ def tokenize( may_divide = False pos = 0 end = len(source) - rules = get_rules(jsx=jsx, dotted=dotted, template_string=template_string) + rules = _get_rules(jsx=jsx, dotted=dotted, template_string=template_string) while pos < end: # handle regular rules first @@ -203,7 +204,7 @@ def tokenize( # if we don't have a match we don't give up yet, but check for # division operators or regular expression literals, based on # the status of `may_divide` which is determined by the last - # processed non-whitespace token using `indicates_division`. + # processed non-whitespace token using `_indicates_division`. else: if may_divide: match = division_re.match(source, pos) @@ -219,7 +220,56 @@ def tokenize( token_value = match.group() if token_type is not None: token = Token(token_type, token_value, lineno) - may_divide = indicates_division(token) + may_divide = _indicates_division(token) yield token lineno += len(line_re.findall(token_value)) pos = match.end() + + +def _warn_deprecated(name: str) -> None: + warnings.warn( + f"`babel.messages.jslexer.{name}` is an implementation detail of the JavaScript " + f"message extractor; it is deprecated and will be removed in a future version of Babel.", + DeprecationWarning, + stacklevel=3, + ) + + +def get_rules( + jsx: bool, + dotted: bool, + template_string: bool, +) -> list[tuple[str | None, re.Pattern[str]]]: + """Deprecated; this is an implementation detail of the JavaScript message extractor.""" + _warn_deprecated("get_rules") + return _get_rules(jsx=jsx, dotted=dotted, template_string=template_string) + + +def indicates_division(token: Token) -> bool: + """Deprecated; this is an implementation detail of the JavaScript message extractor.""" + _warn_deprecated("indicates_division") + return _indicates_division(token) + + +def tokenize( + source: str, + jsx: bool = True, + dotted: bool = True, + template_string: bool = True, + lineno: int = 1, +) -> Generator[Token, None, None]: + """Deprecated; this is an implementation detail of the JavaScript message extractor.""" + _warn_deprecated("tokenize") + return _tokenize( + source, + jsx=jsx, + dotted=dotted, + template_string=template_string, + lineno=lineno, + ) + + +def unquote_string(string: str) -> str: + """Deprecated; this is an implementation detail of the JavaScript message extractor.""" + _warn_deprecated("unquote_string") + return _unquote_string(string) diff --git a/tests/messages/test_js_unicode_codepoints.py b/tests/messages/test_js_unicode_codepoints.py index 5fe3b7255..4b9180e9f 100644 --- a/tests/messages/test_js_unicode_codepoints.py +++ b/tests/messages/test_js_unicode_codepoints.py @@ -3,7 +3,7 @@ import pytest from babel.messages.extract import extract -from babel.messages.jslexer import unquote_string +from babel.messages.jslexer import _unquote_string @pytest.mark.parametrize('quote', ["'", '"', '`']) @@ -16,7 +16,7 @@ (r'\u{000000000041}', 'A'), ]) def test_unicode_codepoint_escapes(quote, body, expected): - assert unquote_string(quote + body + quote) == expected + assert _unquote_string(quote + body + quote) == expected @pytest.mark.parametrize('quote', ["'", '"', '`']) @@ -37,4 +37,4 @@ def test_unicode_codepoint_extraction_uses_runtime_message(quote): (r'\u{41', 'u{41'), ]) def test_codepoint_escapes_preserve_existing_escape_behavior(body, expected): - assert unquote_string('"' + body + '"') == expected + assert _unquote_string('"' + body + '"') == expected diff --git a/tests/messages/test_jslexer.py b/tests/messages/test_jslexer.py index 788e29a41..76bca9695 100644 --- a/tests/messages/test_jslexer.py +++ b/tests/messages/test_jslexer.py @@ -1,21 +1,23 @@ +import pytest + from babel.messages import jslexer def test_unquote(): - assert jslexer.unquote_string('""') == '' - assert jslexer.unquote_string(r'"h\u00ebllo"') == "hëllo" - assert jslexer.unquote_string(r'"h\xebllo"') == "hëllo" - assert jslexer.unquote_string(r'"\xebb"') == "ëb" - assert jslexer.unquote_string(r'"a\vb"') == "a\vb" - assert jslexer.unquote_string(r'"\\v"') == r"\v" + assert jslexer._unquote_string('""') == '' + assert jslexer._unquote_string(r'"h\u00ebllo"') == "hëllo" + assert jslexer._unquote_string(r'"h\xebllo"') == "hëllo" + assert jslexer._unquote_string(r'"\xebb"') == "ëb" + assert jslexer._unquote_string(r'"a\vb"') == "a\vb" + assert jslexer._unquote_string(r'"\\v"') == r"\v" def test_dollar_in_identifier(): - assert list(jslexer.tokenize('dollar$dollar')) == [('name', 'dollar$dollar', 1)] + assert list(jslexer._tokenize('dollar$dollar')) == [('name', 'dollar$dollar', 1)] def test_dotted_name(): - assert list(jslexer.tokenize("foo.bar(quux)", dotted=True)) == [ + assert list(jslexer._tokenize("foo.bar(quux)", dotted=True)) == [ ('name', 'foo.bar', 1), ('operator', '(', 1), ('name', 'quux', 1), @@ -24,20 +26,20 @@ def test_dotted_name(): def test_dotted_name_end(): - assert list(jslexer.tokenize("foo.bar", dotted=True)) == [ + assert list(jslexer._tokenize("foo.bar", dotted=True)) == [ ('name', 'foo.bar', 1), ] def test_template_string(): - assert list(jslexer.tokenize("gettext `foo\"bar\"p`", template_string=True)) == [ + assert list(jslexer._tokenize("gettext `foo\"bar\"p`", template_string=True)) == [ ('name', 'gettext', 1), ('template_string', '`foo"bar"p`', 1), ] def test_jsx(): - assert list(jslexer.tokenize(""" + assert list(jslexer._tokenize(""" @@ -125,3 +127,16 @@ def test_jsx(): ('jsx_tag', '', 8), ] + + +def test_deprecated_functions(): + with pytest.warns(DeprecationWarning, match="jslexer.tokenize"): + tokens = list(jslexer.tokenize("foo / 2")) + assert tokens == list(jslexer._tokenize("foo / 2")) + with pytest.warns(DeprecationWarning, match="jslexer.get_rules"): + rules = jslexer.get_rules(jsx=False, dotted=False, template_string=False) + assert rules == jslexer._get_rules(jsx=False, dotted=False, template_string=False) + with pytest.warns(DeprecationWarning, match="jslexer.indicates_division"): + assert jslexer.indicates_division(tokens[0]) + with pytest.warns(DeprecationWarning, match="jslexer.unquote_string"): + assert jslexer.unquote_string('"a\\tb"') == "a\tb"