diff --git a/babel/messages/extract.py b/babel/messages/extract.py
index 4676ee613..b22dfce9c 100644
--- a/babel/messages/extract.py
+++ b/babel/messages/extract.py
@@ -759,7 +759,7 @@ def extract_javascript(
contents of javascript template strings.
:param lineno: line number offset (for parsing embedded fragments)
"""
- from babel.messages.jslexer import Token, tokenize, unquote_string
+ from babel.messages.jslexer import Token, _tokenize, _unquote_string
funcname = message_lineno = None
messages = []
@@ -770,7 +770,7 @@ def extract_javascript(
last_token = None
call_stack = -1
dotted = any('.' in kw for kw in keywords)
- for token in tokenize(
+ for token in _tokenize(
fileobj.read().decode(encoding),
jsx=options.get("jsx", True),
template_string=options.get("template_string", True),
@@ -786,7 +786,7 @@ def extract_javascript(
and token.type == 'template_string'
):
message_lineno = token.lineno
- messages = [unquote_string(token.value)]
+ messages = [_unquote_string(token.value)]
call_stack = 0
token = Token('operator', ')', token.lineno)
@@ -864,7 +864,7 @@ def extract_javascript(
call_stack = -1
elif token.type in ('string', 'template_string'):
- new_value = unquote_string(token.value)
+ new_value = _unquote_string(token.value)
if concatenate_next:
last_argument = (last_argument or '') + new_value
concatenate_next = False
diff --git a/babel/messages/jslexer.py b/babel/messages/jslexer.py
index 6e123449b..fa9b74294 100644
--- a/babel/messages/jslexer.py
+++ b/babel/messages/jslexer.py
@@ -12,6 +12,7 @@
from __future__ import annotations
import re
+import warnings
from collections.abc import Generator
from typing import NamedTuple
@@ -64,7 +65,7 @@ class Token(NamedTuple):
] # fmt: skip
-def get_rules(
+def _get_rules(
jsx: bool,
dotted: bool,
template_string: bool,
@@ -88,7 +89,7 @@ def get_rules(
return rules
-def indicates_division(token: Token) -> bool:
+def _indicates_division(token: Token) -> bool:
"""A helper function that helps the tokenizer to decide if the current
token may be followed by a division operator.
"""
@@ -97,7 +98,7 @@ def indicates_division(token: Token) -> bool:
return token.type in ('name', 'number', 'string', 'regexp')
-def unquote_string(string: str) -> str:
+def _unquote_string(string: str) -> str:
"""Unquote a string with JavaScript rules. The string has to start with
string delimiters (``'``, ``"`` or the back-tick/grave accent (for template strings).)
"""
@@ -173,7 +174,7 @@ def unquote_string(string: str) -> str:
return ''.join(result)
-def tokenize(
+def _tokenize(
source: str,
jsx: bool = True,
dotted: bool = True,
@@ -192,7 +193,7 @@ def tokenize(
may_divide = False
pos = 0
end = len(source)
- rules = get_rules(jsx=jsx, dotted=dotted, template_string=template_string)
+ rules = _get_rules(jsx=jsx, dotted=dotted, template_string=template_string)
while pos < end:
# handle regular rules first
@@ -203,7 +204,7 @@ def tokenize(
# if we don't have a match we don't give up yet, but check for
# division operators or regular expression literals, based on
# the status of `may_divide` which is determined by the last
- # processed non-whitespace token using `indicates_division`.
+ # processed non-whitespace token using `_indicates_division`.
else:
if may_divide:
match = division_re.match(source, pos)
@@ -219,7 +220,56 @@ def tokenize(
token_value = match.group()
if token_type is not None:
token = Token(token_type, token_value, lineno)
- may_divide = indicates_division(token)
+ may_divide = _indicates_division(token)
yield token
lineno += len(line_re.findall(token_value))
pos = match.end()
+
+
+def _warn_deprecated(name: str) -> None:
+ warnings.warn(
+ f"`babel.messages.jslexer.{name}` is an implementation detail of the JavaScript "
+ f"message extractor; it is deprecated and will be removed in a future version of Babel.",
+ DeprecationWarning,
+ stacklevel=3,
+ )
+
+
+def get_rules(
+ jsx: bool,
+ dotted: bool,
+ template_string: bool,
+) -> list[tuple[str | None, re.Pattern[str]]]:
+ """Deprecated; this is an implementation detail of the JavaScript message extractor."""
+ _warn_deprecated("get_rules")
+ return _get_rules(jsx=jsx, dotted=dotted, template_string=template_string)
+
+
+def indicates_division(token: Token) -> bool:
+ """Deprecated; this is an implementation detail of the JavaScript message extractor."""
+ _warn_deprecated("indicates_division")
+ return _indicates_division(token)
+
+
+def tokenize(
+ source: str,
+ jsx: bool = True,
+ dotted: bool = True,
+ template_string: bool = True,
+ lineno: int = 1,
+) -> Generator[Token, None, None]:
+ """Deprecated; this is an implementation detail of the JavaScript message extractor."""
+ _warn_deprecated("tokenize")
+ return _tokenize(
+ source,
+ jsx=jsx,
+ dotted=dotted,
+ template_string=template_string,
+ lineno=lineno,
+ )
+
+
+def unquote_string(string: str) -> str:
+ """Deprecated; this is an implementation detail of the JavaScript message extractor."""
+ _warn_deprecated("unquote_string")
+ return _unquote_string(string)
diff --git a/tests/messages/test_js_unicode_codepoints.py b/tests/messages/test_js_unicode_codepoints.py
index 5fe3b7255..4b9180e9f 100644
--- a/tests/messages/test_js_unicode_codepoints.py
+++ b/tests/messages/test_js_unicode_codepoints.py
@@ -3,7 +3,7 @@
import pytest
from babel.messages.extract import extract
-from babel.messages.jslexer import unquote_string
+from babel.messages.jslexer import _unquote_string
@pytest.mark.parametrize('quote', ["'", '"', '`'])
@@ -16,7 +16,7 @@
(r'\u{000000000041}', 'A'),
])
def test_unicode_codepoint_escapes(quote, body, expected):
- assert unquote_string(quote + body + quote) == expected
+ assert _unquote_string(quote + body + quote) == expected
@pytest.mark.parametrize('quote', ["'", '"', '`'])
@@ -37,4 +37,4 @@ def test_unicode_codepoint_extraction_uses_runtime_message(quote):
(r'\u{41', 'u{41'),
])
def test_codepoint_escapes_preserve_existing_escape_behavior(body, expected):
- assert unquote_string('"' + body + '"') == expected
+ assert _unquote_string('"' + body + '"') == expected
diff --git a/tests/messages/test_jslexer.py b/tests/messages/test_jslexer.py
index 788e29a41..76bca9695 100644
--- a/tests/messages/test_jslexer.py
+++ b/tests/messages/test_jslexer.py
@@ -1,21 +1,23 @@
+import pytest
+
from babel.messages import jslexer
def test_unquote():
- assert jslexer.unquote_string('""') == ''
- assert jslexer.unquote_string(r'"h\u00ebllo"') == "hëllo"
- assert jslexer.unquote_string(r'"h\xebllo"') == "hëllo"
- assert jslexer.unquote_string(r'"\xebb"') == "ëb"
- assert jslexer.unquote_string(r'"a\vb"') == "a\vb"
- assert jslexer.unquote_string(r'"\\v"') == r"\v"
+ assert jslexer._unquote_string('""') == ''
+ assert jslexer._unquote_string(r'"h\u00ebllo"') == "hëllo"
+ assert jslexer._unquote_string(r'"h\xebllo"') == "hëllo"
+ assert jslexer._unquote_string(r'"\xebb"') == "ëb"
+ assert jslexer._unquote_string(r'"a\vb"') == "a\vb"
+ assert jslexer._unquote_string(r'"\\v"') == r"\v"
def test_dollar_in_identifier():
- assert list(jslexer.tokenize('dollar$dollar')) == [('name', 'dollar$dollar', 1)]
+ assert list(jslexer._tokenize('dollar$dollar')) == [('name', 'dollar$dollar', 1)]
def test_dotted_name():
- assert list(jslexer.tokenize("foo.bar(quux)", dotted=True)) == [
+ assert list(jslexer._tokenize("foo.bar(quux)", dotted=True)) == [
('name', 'foo.bar', 1),
('operator', '(', 1),
('name', 'quux', 1),
@@ -24,20 +26,20 @@ def test_dotted_name():
def test_dotted_name_end():
- assert list(jslexer.tokenize("foo.bar", dotted=True)) == [
+ assert list(jslexer._tokenize("foo.bar", dotted=True)) == [
('name', 'foo.bar', 1),
]
def test_template_string():
- assert list(jslexer.tokenize("gettext `foo\"bar\"p`", template_string=True)) == [
+ assert list(jslexer._tokenize("gettext `foo\"bar\"p`", template_string=True)) == [
('name', 'gettext', 1),
('template_string', '`foo"bar"p`', 1),
]
def test_jsx():
- assert list(jslexer.tokenize("""
+ assert list(jslexer._tokenize("""
@@ -125,3 +127,16 @@ def test_jsx():
('jsx_tag', '', 8),
]
+
+
+def test_deprecated_functions():
+ with pytest.warns(DeprecationWarning, match="jslexer.tokenize"):
+ tokens = list(jslexer.tokenize("foo / 2"))
+ assert tokens == list(jslexer._tokenize("foo / 2"))
+ with pytest.warns(DeprecationWarning, match="jslexer.get_rules"):
+ rules = jslexer.get_rules(jsx=False, dotted=False, template_string=False)
+ assert rules == jslexer._get_rules(jsx=False, dotted=False, template_string=False)
+ with pytest.warns(DeprecationWarning, match="jslexer.indicates_division"):
+ assert jslexer.indicates_division(tokens[0])
+ with pytest.warns(DeprecationWarning, match="jslexer.unquote_string"):
+ assert jslexer.unquote_string('"a\\tb"') == "a\tb"