From 01bf8ec6d73e46d57773b655d0d999a0392df33f Mon Sep 17 00:00:00 2001 From: Cheryl Sabella Date: Mon, 17 Feb 2020 13:27:28 -0500 Subject: [PATCH 1/4] bpo-39666: Refactor common code between hyperparser and editor --- Lib/idlelib/editor.py | 47 ++++++++++++---------- Lib/idlelib/hyperparser.py | 40 +++--------------- Lib/idlelib/idle_test/test_autocomplete.py | 4 ++ Lib/idlelib/idle_test/test_hyperparser.py | 2 + Lib/idlelib/idle_test/test_parenmatch.py | 4 +- 5 files changed, 40 insertions(+), 57 deletions(-) diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index a178eaf93c013ab..27b25e21b8b89c9 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -1396,28 +1396,7 @@ def newline_and_indent_event(self, event): # Adjust indentation for continuations and block open/close. # First need to find the last statement. - lno = index2line(text.index('insert')) - y = pyparse.Parser(self.indentwidth, self.tabwidth) - if not self.prompt_last_line: - for context in self.num_context_lines: - startat = max(lno - context, 1) - startatindex = repr(startat) + ".0" - rawtext = text.get(startatindex, "insert") - y.set_code(rawtext) - bod = y.find_good_parse_start( - self._build_char_in_string_func(startatindex)) - if bod is not None or startat == 1: - break - y.set_lo(bod or 0) - else: - r = text.tag_prevrange("console", "insert") - if r: - startatindex = r[1] - else: - startatindex = "1.0" - rawtext = text.get(startatindex, "insert") - y.set_code(rawtext) - y.set_lo(0) + y = self.parser() c = y.get_continuation_type() if c != pyparse.C_NONE: @@ -1474,6 +1453,30 @@ def inner(offset, _startindex=startindex, return _icis(_startindex + "+%dc" % offset) return inner + def parser(self, index="insert", lineend=""): + "Create a PyParse instance and find the last statement." + text = self.text + p = pyparse.Parser(self.indentwidth, self.tabwidth) + lineno = self.getlineno(index) + stopatindex = index if not lineend else f"{lineno}.end" + + if self.prompt_last_line: + r = text.tag_prevrange("console", index) + startatindex = r[1] if r else "1.0" + p.set_code(text.get(startatindex, stopatindex) + lineend) + return p + + for context in self.num_context_lines: + startat = max(lineno - context, 1) + startatindex = f"{startat}.0" + p.set_code(text.get(startatindex, stopatindex) + lineend) + bod = p.find_good_parse_start( + self._build_char_in_string_func(startatindex)) + if bod is not None or startat == 1: + break + p.set_lo(bod or 0) + return p + # XXX this isn't bound to anything -- see tabwidth comments ## def change_tabwidth_event(self, event): ## new = self._asktabwidth() diff --git a/Lib/idlelib/hyperparser.py b/Lib/idlelib/hyperparser.py index 77baca782b3fdc0..e1f6afbaaebfa5c 100644 --- a/Lib/idlelib/hyperparser.py +++ b/Lib/idlelib/hyperparser.py @@ -29,46 +29,18 @@ def __init__(self, editwin, index): self.editwin = editwin self.text = text = editwin.text - parser = pyparse.Parser(editwin.indentwidth, editwin.tabwidth) - - def index2line(index): - return int(float(index)) - lno = index2line(text.index(index)) - - if not editwin.prompt_last_line: - for context in editwin.num_context_lines: - startat = max(lno - context, 1) - startatindex = repr(startat) + ".0" - stopatindex = "%d.end" % lno - # We add the newline because PyParse requires a newline - # at end. We add a space so that index won't be at end - # of line, so that its status will be the same as the - # char before it, if should. - parser.set_code(text.get(startatindex, stopatindex)+' \n') - bod = parser.find_good_parse_start( - editwin._build_char_in_string_func(startatindex)) - if bod is not None or startat == 1: - break - parser.set_lo(bod or 0) - else: - r = text.tag_prevrange("console", index) - if r: - startatindex = r[1] - else: - startatindex = "1.0" - stopatindex = "%d.end" % lno - # We add the newline because PyParse requires it. We add a - # space so that index won't be at end of line, so that its - # status will be the same as the char before it, if should. - parser.set_code(text.get(startatindex, stopatindex)+' \n') - parser.set_lo(0) + # We add the newline because PyParse requires a newline + # at end. We add a space so that index won't be at end + # of line, so that its status will be the same as the + # char before it. + parser = editwin.parser(index=index, lineend=" \n") # We want what the parser has, minus the last newline and space. self.rawtext = parser.code[:-2] # Parser.code apparently preserves the statement we are in, so # that stopatindex can be used to synchronize the string with # the text box indices. - self.stopatindex = stopatindex + self.stopatindex = f"{editwin.getlineno(index)}.end" self.bracketing = parser.get_last_stmt_bracketing() # find which pairs of bracketing are openers. These always # correspond to a character of rawtext. diff --git a/Lib/idlelib/idle_test/test_autocomplete.py b/Lib/idlelib/idle_test/test_autocomplete.py index 1841495fcf1a0c6..e598356eed8af7b 100644 --- a/Lib/idlelib/idle_test/test_autocomplete.py +++ b/Lib/idlelib/idle_test/test_autocomplete.py @@ -9,6 +9,7 @@ import idlelib.autocomplete as ac import idlelib.autocomplete_w as acw +from idlelib.editor import EditorWindow from idlelib.idle_test.mock_idle import Func from idlelib.idle_test.mock_tk import Event @@ -21,6 +22,9 @@ def __init__(self, root, text): self.tabwidth = 8 self.prompt_last_line = '>>>' # Currently not used by autocomplete. + getlineno = EditorWindow.getlineno + parser = EditorWindow.parser + class AutoCompleteTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_hyperparser.py b/Lib/idlelib/idle_test/test_hyperparser.py index 343843c4166e97d..c4bdecc4142f7fd 100644 --- a/Lib/idlelib/idle_test/test_hyperparser.py +++ b/Lib/idlelib/idle_test/test_hyperparser.py @@ -16,6 +16,8 @@ def __init__(self, text): _build_char_in_string_func = EditorWindow._build_char_in_string_func is_char_in_string = EditorWindow.is_char_in_string + getlineno = EditorWindow.getlineno + parser = EditorWindow.parser class HyperParserTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_parenmatch.py b/Lib/idlelib/idle_test/test_parenmatch.py index 4a41d8433d54836..341c791154a02db 100644 --- a/Lib/idlelib/idle_test/test_parenmatch.py +++ b/Lib/idlelib/idle_test/test_parenmatch.py @@ -10,6 +10,7 @@ import unittest from unittest.mock import Mock from tkinter import Tk, Text +from idlelib.editor import EditorWindow class DummyEditwin: @@ -18,7 +19,8 @@ def __init__(self, text): self.indentwidth = 8 self.tabwidth = 8 self.prompt_last_line = '>>>' # Currently not used by parenmatch. - + getlineno = EditorWindow.getlineno + parser = EditorWindow.parser class ParenMatchTest(unittest.TestCase): From d58c2b9be03ac8188da830834096f186d0be72b3 Mon Sep 17 00:00:00 2001 From: Cheryl Sabella Date: Mon, 17 Feb 2020 18:42:08 -0500 Subject: [PATCH 2/4] Add tests for parser method. * Move num_context_lines into editor.parser method since it's not used anywhere else. --- Lib/idlelib/editor.py | 16 ++++----- Lib/idlelib/idle_test/test_editor.py | 40 +++++++++++++++++++++++ Lib/idlelib/idle_test/test_hyperparser.py | 1 - 3 files changed, 48 insertions(+), 9 deletions(-) diff --git a/Lib/idlelib/editor.py b/Lib/idlelib/editor.py index 27b25e21b8b89c9..90b6c2dede6553b 100644 --- a/Lib/idlelib/editor.py +++ b/Lib/idlelib/editor.py @@ -248,13 +248,6 @@ def __init__(self, flist=None, filename=None, key=None, root=None): idleConf.blink_off_time = self.text['insertofftime'] self.update_cursor_blink() - # When searching backwards for a reliable place to begin parsing, - # first start num_context_lines[0] lines back, then - # num_context_lines[1] lines back if that didn't work, and so on. - # The last value should be huge (larger than the # of lines in a - # conceivable file). - # Making the initial values larger slows things down more often. - self.num_context_lines = 50, 500, 5000000 self.per = per = self.Percolator(text) self.undo = undo = self.UndoDelegator() per.insertfilter(undo) @@ -1466,7 +1459,14 @@ def parser(self, index="insert", lineend=""): p.set_code(text.get(startatindex, stopatindex) + lineend) return p - for context in self.num_context_lines: + # When searching backwards for a reliable place to begin parsing, + # first start num_context_lines[0] lines back, then + # num_context_lines[1] lines back if that didn't work, and so on. + # The last value should be huge (larger than the # of lines in a + # conceivable file). + # Making the initial values larger slows things down more often. + num_context_lines = 50, 500, 5000000 + for context in num_context_lines: startat = max(lineno - context, 1) startatindex = f"{startat}.0" p.set_code(text.get(startatindex, stopatindex) + lineend) diff --git a/Lib/idlelib/idle_test/test_editor.py b/Lib/idlelib/idle_test/test_editor.py index 443dcf021679fcb..c603b29aec341e6 100644 --- a/Lib/idlelib/idle_test/test_editor.py +++ b/Lib/idlelib/idle_test/test_editor.py @@ -120,6 +120,10 @@ def tearDownClass(cls): cls.root.destroy() del cls.root + @classmethod + def tearDown(self): + self.window.text.delete("1.0", "end-1c") + def test_indent_and_newline_event(self): eq = self.assertEqual w = self.window @@ -189,6 +193,42 @@ def test_indent_and_newline_event(self): nl(None) eq(get('1.0', 'end'), '>>> \na =\n') + def test_parser(self): + eq = self.assertEqual + w = self.window + text = w.text + p = w.parser + + w.prompt_last_line = '' + eq(p().code, "") + + # Two lines with block opener. + insert(" def f1(self, a, b):\n return a + b\n") + eq(p().code, " return a + b\n") + + # No newline in code. + insert(" def f1(") + eq(p(lineend=" \n").code, " def f1( \n") + + # Two lines without block opener. + code = ("a = 1\n" + "b = 2\n") + insert(code) + eq(p().code, code) + + # Emulate shell. + w.prompt_last_line = '>>> ' + code = (">>> def f1(self, a, b):\n") + insert(code) + text.tag_add("console", "1.0", "1.4") + # The prompt is removed with the parser. + eq(p().code, code[4:]) + + code = (">>> def f1(self, a, b):\n return a + b\n") + insert(code) + text.tag_add("console", "1.0", "1.4") + eq(p().code, code[4:]) + class RMenuTest(unittest.TestCase): diff --git a/Lib/idlelib/idle_test/test_hyperparser.py b/Lib/idlelib/idle_test/test_hyperparser.py index c4bdecc4142f7fd..8d13854be67687b 100644 --- a/Lib/idlelib/idle_test/test_hyperparser.py +++ b/Lib/idlelib/idle_test/test_hyperparser.py @@ -12,7 +12,6 @@ def __init__(self, text): self.indentwidth = 8 self.tabwidth = 8 self.prompt_last_line = '>>>' - self.num_context_lines = 50, 500, 1000 _build_char_in_string_func = EditorWindow._build_char_in_string_func is_char_in_string = EditorWindow.is_char_in_string From d7c813a98e207ccd0cc892e453bf766a3f4766fc Mon Sep 17 00:00:00 2001 From: Cheryl Sabella Date: Tue, 18 Feb 2020 07:28:09 -0500 Subject: [PATCH 3/4] Separate editor and shell tests --- Lib/idlelib/idle_test/test_editor.py | 41 ++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/Lib/idlelib/idle_test/test_editor.py b/Lib/idlelib/idle_test/test_editor.py index c603b29aec341e6..616fdfef60d01d8 100644 --- a/Lib/idlelib/idle_test/test_editor.py +++ b/Lib/idlelib/idle_test/test_editor.py @@ -124,12 +124,13 @@ def tearDownClass(cls): def tearDown(self): self.window.text.delete("1.0", "end-1c") - def test_indent_and_newline_event(self): + def test_indent_and_newline_event_editor(self): eq = self.assertEqual w = self.window text = w.text get = text.get nl = w.newline_and_indent_event + w.prompt_last_line = '' TestInfo = namedtuple('Tests', ['label', 'text', 'expected', 'mark']) @@ -171,7 +172,6 @@ def test_indent_and_newline_event(self): '2.end'), ) - w.prompt_last_line = '' for test in tests: with self.subTest(label=test.label): insert(text, test.text) @@ -186,20 +186,26 @@ def test_indent_and_newline_event(self): # Deletes selected text before adding new line. eq(get('1.0', 'end'), ' def f1(self, a,\n \n return a + b\n') - # Preserves the whitespace in shell prompt. + def test_indent_and_newline_event_shell(self): + eq = self.assertEqual + w = self.window + text = w.text + get = text.get + nl = w.newline_and_indent_event w.prompt_last_line = '>>> ' insert(text, '>>> \t\ta =') text.mark_set('insert', '1.5') + text.tag_add("console", "1.0", "1.4") nl(None) eq(get('1.0', 'end'), '>>> \na =\n') - def test_parser(self): + def test_parser_editor(self): eq = self.assertEqual w = self.window text = w.text p = w.parser - w.prompt_last_line = '' + eq(p().code, "") # Two lines with block opener. @@ -216,19 +222,42 @@ def test_parser(self): insert(code) eq(p().code, code) - # Emulate shell. + # No newline in code. + insert(" def f1(") + eq(p(lineend=" \n").code, " def f1( \n") + + def test_parser_shell(self): + eq = self.assertEqual + w = self.window + text = w.text + p = w.parser w.prompt_last_line = '>>> ' + + # One line with block opener. code = (">>> def f1(self, a, b):\n") insert(code) text.tag_add("console", "1.0", "1.4") # The prompt is removed with the parser. eq(p().code, code[4:]) + # Two lines with block opener parses everything but prompt. code = (">>> def f1(self, a, b):\n return a + b\n") insert(code) text.tag_add("console", "1.0", "1.4") eq(p().code, code[4:]) + # Two lines without block opener. + code = (">>> a = 1\n" + ">>> b = 2\n") + insert(code) + text.tag_add("console", "2.0", "2.4") + eq(p().code, "b = 2\n") + + # No newline in code. + insert(">>> def f1(") + text.tag_add("console", "1.0", "1.4") + eq(p(lineend=" \n").code, " def f1( \n") + class RMenuTest(unittest.TestCase): From 6a9eebd7c888615dd3980cf140ffb92b45ac046f Mon Sep 17 00:00:00 2001 From: Cheryl Sabella Date: Tue, 9 Jun 2020 19:24:23 -0400 Subject: [PATCH 4/4] Merge master --- Lib/idlelib/idle_test/test_editor.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/Lib/idlelib/idle_test/test_editor.py b/Lib/idlelib/idle_test/test_editor.py index 616fdfef60d01d8..19475b878f819a0 100644 --- a/Lib/idlelib/idle_test/test_editor.py +++ b/Lib/idlelib/idle_test/test_editor.py @@ -209,21 +209,21 @@ def test_parser_editor(self): eq(p().code, "") # Two lines with block opener. - insert(" def f1(self, a, b):\n return a + b\n") + insert(text, " def f1(self, a, b):\n return a + b\n") eq(p().code, " return a + b\n") # No newline in code. - insert(" def f1(") + insert(text, " def f1(") eq(p(lineend=" \n").code, " def f1( \n") # Two lines without block opener. code = ("a = 1\n" "b = 2\n") - insert(code) + insert(text, code) eq(p().code, code) # No newline in code. - insert(" def f1(") + insert(text, " def f1(") eq(p(lineend=" \n").code, " def f1( \n") def test_parser_shell(self): @@ -235,26 +235,26 @@ def test_parser_shell(self): # One line with block opener. code = (">>> def f1(self, a, b):\n") - insert(code) + insert(text, code) text.tag_add("console", "1.0", "1.4") # The prompt is removed with the parser. eq(p().code, code[4:]) # Two lines with block opener parses everything but prompt. code = (">>> def f1(self, a, b):\n return a + b\n") - insert(code) + insert(text, code) text.tag_add("console", "1.0", "1.4") eq(p().code, code[4:]) # Two lines without block opener. code = (">>> a = 1\n" ">>> b = 2\n") - insert(code) + insert(text, code) text.tag_add("console", "2.0", "2.4") eq(p().code, "b = 2\n") # No newline in code. - insert(">>> def f1(") + insert(text, ">>> def f1(") text.tag_add("console", "1.0", "1.4") eq(p(lineend=" \n").code, " def f1( \n")