diff --git a/sqlparse/engine/grouping.py b/sqlparse/engine/grouping.py
index a63f4da2..0b7b5485 100644
--- a/sqlparse/engine/grouping.py
+++ b/sqlparse/engine/grouping.py
@@ -7,7 +7,7 @@
 
 from sqlparse import sql
 from sqlparse import tokens as T
-from sqlparse.utils import recurse, imt
+from sqlparse.utils import recurse
 
 T_NUMERICAL = (T.Number, T.Number.Integer, T.Number.Float)
 T_STRING = (T.String, T.String.Single, T.String.Symbol)
@@ -17,36 +17,28 @@
 def _group_matching(tlist, cls):
     """Groups Tokens that have beginning and end."""
     opens = []
-    tidx_offset = 0
-    for idx, token in enumerate(list(tlist)):
-        tidx = idx - tidx_offset
+    n = len(tlist.tokens)
+    for idx in range(n):
+        token = tlist.tokens[idx]
+        if token is None:
+            continue
 
         if token.is_whitespace:
-            # ~50% of tokens will be whitespace. Will checking early
-            # for them avoid 3 comparisons, but then add 1 more comparison
-            # for the other ~50% of tokens...
             continue
 
         if token.is_group and not isinstance(token, cls):
-            # Check inside previously grouped (i.e. parenthesis) if group
-            # of different type is inside (i.e., case). though ideally  should
-            # should check for all open/close tokens at once to avoid recursion
             _group_matching(token, cls)
             continue
 
         if token.match(*cls.M_OPEN):
-            opens.append(tidx)
+            opens.append(idx)
 
         elif token.match(*cls.M_CLOSE):
             try:
                 open_idx = opens.pop()
             except IndexError:
-                # this indicates invalid sql and unbalanced tokens.
-                # instead of break, continue in case other "valid" groups exist
                 continue
-            close_idx = tidx
-            tlist.group_tokens(cls, open_idx, close_idx)
-            tidx_offset += close_idx - open_idx
+            tlist.group_tokens(cls, open_idx, idx)
 
 
 def group_brackets(tlist):
@@ -114,7 +106,7 @@ def group_typed_literal(tlist):
     # https://www.postgresql.org/docs/9.1/datatype-datetime.html
     # https://www.postgresql.org/docs/9.1/functions-datetime.html
     def match(token):
-        return imt(token, m=sql.TypedLiteral.M_OPEN)
+        return any(token.match(*pattern) for pattern in sql.TypedLiteral.M_OPEN)
 
     def match_to_extend(token):
         return isinstance(token, sql.TypedLiteral)
@@ -147,9 +139,8 @@ def match(token):
         return False
 
     def valid_prev(token):
-        sqlcls = sql.SquareBrackets, sql.Identifier
-        ttypes = T.Name, T.String.Symbol
-        return imt(token, i=sqlcls, t=ttypes)
+        return (isinstance(token, (sql.SquareBrackets, sql.Identifier))
+                or token.ttype in (T.Name, T.String.Symbol))
 
     def valid_next(token):
         # issue261, allow invalid next token
@@ -157,10 +148,11 @@ def valid_next(token):
 
     def post(tlist, pidx, tidx, nidx):
         # next_ validation is being performed here. issue261
-        sqlcls = sql.SquareBrackets, sql.Function
-        ttypes = T.Name, T.String.Symbol, T.Wildcard, T.String.Single
         next_ = tlist[nidx] if nidx is not None else None
-        valid_next = imt(next_, i=sqlcls, t=ttypes)
+        valid_next = (next_ is not None
+                      and (isinstance(next_, (sql.SquareBrackets, sql.Function))
+                           or next_.ttype in (T.Name, T.String.Symbol,
+                                              T.Wildcard, T.String.Single)))
 
         return (pidx, nidx) if valid_next else (pidx, tidx)
 
@@ -175,8 +167,7 @@ def valid_prev(token):
         return token.normalized == 'NULL' or not token.is_keyword
 
     def valid_next(token):
-        ttypes = T.DML, T.DDL, T.CTE
-        return not imt(token, t=ttypes) and token is not None
+        return token is not None and token.ttype not in (T.DML, T.DDL, T.CTE)
 
     def post(tlist, pidx, tidx, nidx):
         return pidx, nidx
@@ -210,12 +201,13 @@ def match(token):
         return token.ttype == T.Operator.Comparison
 
     def valid(token):
-        if imt(token, t=ttypes, i=sqlcls):
+        if token is None:
+            return False
+        if isinstance(token, sqlcls) or token.ttype in ttypes:
             return True
-        elif token and token.is_keyword and token.normalized == 'NULL':
+        if token.is_keyword and token.normalized == 'NULL':
             return True
-        else:
-            return False
+        return False
 
     def post(tlist, pidx, tidx, nidx):
         return pidx, nidx
@@ -240,7 +232,9 @@ def group_over(tlist):
     tidx, token = tlist.token_next_by(m=sql.Over.M_OPEN)
     while token:
         nidx, next_ = tlist.token_next(tidx)
-        if imt(next_, i=sql.Parenthesis, t=T.Name):
+        if (next_ is not None
+                and (isinstance(next_, sql.Parenthesis)
+                     or next_.ttype in T.Name)):
             tlist.group_tokens(sql.Over, tidx, nidx)
         tidx, token = tlist.token_next_by(m=sql.Over.M_OPEN, idx=tidx)
 
@@ -253,7 +247,7 @@ def match(token):
         return isinstance(token, sql.SquareBrackets)
 
     def valid_prev(token):
-        return imt(token, i=sqlcls, t=ttypes)
+        return isinstance(token, sqlcls) or token.ttype in ttypes
 
     def valid_next(token):
         return True
@@ -271,13 +265,16 @@ def group_operator(tlist):
               sql.Identifier, sql.Operation, sql.TypedLiteral)
 
     def match(token):
-        return imt(token, t=(T.Operator, T.Wildcard))
+        return token.ttype in (T.Operator, T.Wildcard)
 
     def valid(token):
-        return imt(token, i=sqlcls, t=ttypes) \
-            or (token and token.match(
-                T.Keyword,
-                ('CURRENT_DATE', 'CURRENT_TIME', 'CURRENT_TIMESTAMP')))
+        if token is None:
+            return False
+        return (isinstance(token, sqlcls)
+                or token.ttype in ttypes
+                or token.match(
+                    T.Keyword,
+                    ('CURRENT_DATE', 'CURRENT_TIME', 'CURRENT_TIMESTAMP')))
 
     def post(tlist, pidx, tidx, nidx):
         tlist[tidx].ttype = T.Operator
@@ -299,7 +296,11 @@ def match(token):
         return token.match(T.Punctuation, ',')
 
     def valid(token):
-        return imt(token, i=sqlcls, m=m_role, t=ttypes)
+        if token is None:
+            return False
+        return (isinstance(token, sqlcls)
+                or token.match(*m_role)
+                or token.ttype in ttypes)
 
     def post(tlist, pidx, tidx, nidx):
         return pidx, nidx
@@ -314,7 +315,7 @@ def group_comments(tlist):
     tidx, token = tlist.token_next_by(t=T.Comment)
     while token:
         eidx, end = tlist.token_not_matching(
-            lambda tk: imt(tk, t=T.Comment) or tk.is_newline, idx=tidx)
+            lambda tk: tk.ttype in T.Comment or tk.is_newline, idx=tidx)
         if end is not None:
             eidx, end = tlist.token_prev(eidx, skip_ws=False)
             tlist.group_tokens(sql.Comment, tidx, eidx)
@@ -358,11 +359,13 @@ def group_functions(tlist):
     has_table = False
     has_as = False
     for tmp_token in tlist.tokens:
-        if tmp_token.value.upper() == 'CREATE':
+        val = str(tmp_token)
+        val_upper = val.upper()
+        if val_upper == 'CREATE':
             has_create = True
-        if tmp_token.value.upper() == 'TABLE':
+        if val_upper == 'TABLE':
             has_table = True
-        if tmp_token.value == 'AS':
+        if val == 'AS':
             has_as = True
     if has_create and has_table and not has_as:
         return
@@ -386,7 +389,9 @@ def group_order(tlist):
     tidx, token = tlist.token_next_by(t=T.Keyword.Order)
     while token:
         pidx, prev_ = tlist.token_prev(tidx)
-        if imt(prev_, i=sql.Identifier, t=T.Number):
+        if (prev_ is not None
+                and (isinstance(prev_, sql.Identifier)
+                     or prev_.ttype in T.Number)):
             tlist.group_tokens(sql.Identifier, pidx, tidx)
             tidx = pidx
         tidx, token = tlist.token_next_by(t=T.Keyword.Order, idx=tidx)
@@ -415,6 +420,14 @@ def group_values(tlist):
         tlist.group_tokens(sql.Values, start_idx, end_idx, extend=True)
 
 
+def _compact_all(tlist):
+    """Recursively compact None sentinels from all token lists."""
+    tlist._compact_tokens()
+    for token in tlist.tokens:
+        if token.is_group:
+            _compact_all(token)
+
+
 def group(stmt):
     for func in [
         group_comments,
@@ -448,6 +461,7 @@ def group(stmt):
         group_values,
     ]:
         func(stmt)
+        _compact_all(stmt)
     return stmt
 
 
@@ -460,11 +474,9 @@ def _group(tlist, cls, match,
            ):
     """Groups together tokens that are joined by a middle token. i.e. x < y"""
 
-    tidx_offset = 0
     pidx, prev_ = None, None
     for idx, token in enumerate(list(tlist)):
-        tidx = idx - tidx_offset
-        if tidx < 0:  # tidx shouldn't get negative
+        if token is None:
             continue
 
         if token.is_whitespace:
@@ -473,14 +485,17 @@ def _group(tlist, cls, match,
         if recurse and token.is_group and not isinstance(token, cls):
             _group(token, cls, match, valid_prev, valid_next, post, extend)
 
+        # Token may have been consumed by a prior group_tokens in this pass
+        if tlist.tokens[idx] is None:
+            continue
+
         if match(token):
-            nidx, next_ = tlist.token_next(tidx)
+            nidx, next_ = tlist.token_next(idx)
             if prev_ and valid_prev(prev_) and valid_next(next_):
-                from_idx, to_idx = post(tlist, pidx, tidx, nidx)
+                from_idx, to_idx = post(tlist, pidx, idx, nidx)
                 grp = tlist.group_tokens(cls, from_idx, to_idx, extend=extend)
 
-                tidx_offset += to_idx - from_idx
                 pidx, prev_ = from_idx, grp
                 continue
 
-        pidx, prev_ = tidx, token
+        pidx, prev_ = idx, token
diff --git a/sqlparse/filters/others.py b/sqlparse/filters/others.py
index 40a808df..2d8f552f 100644
--- a/sqlparse/filters/others.py
+++ b/sqlparse/filters/others.py
@@ -74,7 +74,8 @@ def _get_insert_token(token):
             tidx, token = get_next_comment(idx=tidx)
 
     def process(self, stmt):
-        [self.process(sgroup) for sgroup in stmt.get_sublists()]
+        [self.process(sgroup) for sgroup in stmt.get_sublists()
+         if not isinstance(sgroup, sql.Comment)]
         StripCommentsFilter._process(stmt)
         return stmt
 
diff --git a/sqlparse/filters/reindent.py b/sqlparse/filters/reindent.py
index 1ba36060..3256cb5b 100644
--- a/sqlparse/filters/reindent.py
+++ b/sqlparse/filters/reindent.py
@@ -94,35 +94,61 @@ def _next_token(self, tlist, idx=-1):
         return tidx, token
 
     def _split_kwds(self, tlist):
+        # Pass 1: scan unmodified list for all keyword positions
+        inserts = {}   # idx -> token to insert before
+        deletes = set()
+
         tidx, token = self._next_token(tlist)
         while token:
             pidx, prev_ = tlist.token_prev(tidx, skip_ws=False)
             uprev = str(prev_)
 
             if prev_ and prev_.is_whitespace:
-                del tlist.tokens[pidx]
-                tidx -= 1
+                deletes.add(pidx)
 
             if not (uprev.endswith('\n') or uprev.endswith('\r')):
-                tlist.insert_before(tidx, self.nl())
-                tidx += 1
+                inserts[tidx] = self.nl()
 
             tidx, token = self._next_token(tlist, tidx)
 
+        # Pass 2: rebuild token list in O(n)
+        if inserts or deletes:
+            new_tokens = []
+            for i, tok in enumerate(tlist.tokens):
+                if i in inserts:
+                    nl_tok = inserts[i]
+                    nl_tok.parent = tlist
+                    new_tokens.append(nl_tok)
+                if i not in deletes:
+                    new_tokens.append(tok)
+            tlist.tokens = new_tokens
+
     def _split_statements(self, tlist):
         ttypes = T.Keyword.DML, T.Keyword.DDL
+        inserts = {}
+        deletes = set()
+
         tidx, token = tlist.token_next_by(t=ttypes)
         while token:
             pidx, prev_ = tlist.token_prev(tidx, skip_ws=False)
             if prev_ and prev_.is_whitespace:
-                del tlist.tokens[pidx]
-                tidx -= 1
+                deletes.add(pidx)
             # only break if it's not the first token
-            if prev_:
-                tlist.insert_before(tidx, self.nl())
-                tidx += 1
+            if prev_ is not None:
+                inserts[tidx] = self.nl()
             tidx, token = tlist.token_next_by(t=ttypes, idx=tidx)
 
+        if inserts or deletes:
+            new_tokens = []
+            for i, tok in enumerate(tlist.tokens):
+                if i in inserts:
+                    nl_tok = inserts[i]
+                    nl_tok.parent = tlist
+                    new_tokens.append(nl_tok)
+                if i not in deletes:
+                    new_tokens.append(tok)
+            tlist.tokens = new_tokens
+
     def _process(self, tlist):
         func_name = f'_process_{type(tlist).__name__}'
         func = getattr(self, func_name.lower(), self._process_default)
@@ -172,7 +198,7 @@ def _process_identifierlist(self, tlist):
                 shift = 0
                 for token in identifiers:
                     # Add 1 for the "," separator
-                    position += len(token.value) + 1
+                    position += len(str(token)) + 1
                     if position > (self.wrap_after - self.offset):
                         adjust = 0
                         tidx = token_to_idx[id(token)] + shift
@@ -215,12 +241,12 @@ def _process_identifierlist(self, tlist):
                             adj_i, sql.Token(T.Whitespace, ' '))
                         ws_shift += 1
 
-            end_at = self.offset + sum(len(i.value) + 1 for i in identifiers)
+            end_at = self.offset + sum(len(str(i)) + 1 for i in identifiers)
             adjusted_offset = 0
             if (self.wrap_after > 0
                     and end_at > (self.wrap_after - self.offset)
                     and self._last_func):
-                adjusted_offset = -len(self._last_func.value) - 1
+                adjusted_offset = -len(str(self._last_func)) - 1
 
             # Rebuild index mapping after whitespace insertions
             token_to_idx = {id(t): i
@@ -234,7 +260,7 @@ def _process_identifierlist(self, tlist):
                 position = 0
                 for token in identifiers:
                     # Add 1 for the "," separator
-                    position += len(token.value) + 1
+                    position += len(str(token)) + 1
                     if (self.wrap_after > 0
                             and position > (self.wrap_after - self.offset)):
                         tidx = token_to_idx[id(token)] + shift
diff --git a/sqlparse/lexer.py b/sqlparse/lexer.py
index 07779b43..62415cdb 100644
--- a/sqlparse/lexer.py
+++ b/sqlparse/lexer.py
@@ -17,7 +17,6 @@
 from io import TextIOBase
 
 from sqlparse import tokens, keywords
-from sqlparse.utils import consume
 
 
 class Lexer:
@@ -81,6 +80,29 @@ def clear(self):
     def set_SQL_REGEX(self, SQL_REGEX):
         """Set the list of regex that will parse the SQL."""
         FLAGS = re.IGNORECASE | re.UNICODE
+
+        # Build combined alternation regex for single-pass tokenization.
+        # The dollar-quoted string pattern uses a backreference (\1) which
+        # breaks in a combined alternation, so it's handled separately.
+        self._dollar_quote_re = None
+        self._dollar_quote_action = None
+
+        combined_parts = []
+        actions = []
+        group_idx = 0
+        for rx, action in SQL_REGEX:
+            if r'\1' in rx or r'\2' in rx:
+                self._dollar_quote_re = re.compile(rx, FLAGS)
+                self._dollar_quote_action = action
+                continue
+            combined_parts.append(f'(?P<g{group_idx}>{rx})')
+            actions.append(action)
+            group_idx += 1
+
+        self._combined_re = re.compile('|'.join(combined_parts), FLAGS)
+        self._actions = actions
+
+        # Keep legacy list for external code that accesses _SQL_REGEX
         self._SQL_REGEX = [
             (re.compile(rx, FLAGS).match, tt) for rx, tt in SQL_REGEX
         ]
@@ -135,22 +157,41 @@ def get_tokens(self, text, encoding=None):
                     type(text))
             )
 
-        iterable = enumerate(text)
-        for pos, char in iterable:
-            for rexmatch, action in self._SQL_REGEX:
-                m = rexmatch(text, pos)
-
-                if not m:
+        combined_re = self._combined_re
+        actions = self._actions
+        dollar_re = self._dollar_quote_re
+        dollar_action = self._dollar_quote_action
+        text_len = len(text)
+        _PROCESS_AS_KEYWORD = keywords.PROCESS_AS_KEYWORD
+        _TokenType = tokens._TokenType
+        is_keyword = self.is_keyword
+        pos = 0
+
+        while pos < text_len:
+            # Dollar-quoted strings: backreference pattern handled separately
+            if dollar_re is not None and text[pos] == '$':
+                m = dollar_re.match(text, pos)
+                if m:
+                    yield dollar_action, m.group(), pos
+                    pos = m.end()
                     continue
-                elif isinstance(action, tokens._TokenType):
-                    yield action, m.group(), pos
-                elif action is keywords.PROCESS_AS_KEYWORD:
-                    yield (*self.is_keyword(m.group()), pos)
 
-                consume(iterable, m.end() - pos - 1)
-                break
-            else:
-                yield tokens.Error, char, pos
+            m = combined_re.match(text, pos)
+            if m is None:
+                yield tokens.Error, text[pos], pos
+                pos += 1
+                continue
+
+            idx = int(m.lastgroup[1:])  # 'g12' -> 12
+            action = actions[idx]
+            value = m.group()
+
+            if isinstance(action, _TokenType):
+                yield action, value, pos
+            elif action is _PROCESS_AS_KEYWORD:
+                yield (*is_keyword(value), pos)
+
+            pos = m.end()
 
 
 def tokenize(sql, encoding=None):
diff --git a/sqlparse/sql.py b/sqlparse/sql.py
index 19cb06f5..3d1f7ab6 100644
--- a/sqlparse/sql.py
+++ b/sqlparse/sql.py
@@ -170,8 +170,9 @@ class TokenList(Token):
 
     def __init__(self, tokens=None):
         self.tokens = tokens or []
-        [setattr(token, "parent", self) for token in self.tokens]
-        super().__init__(None, str(self))
+        for token in self.tokens:
+            token.parent = self
+        super().__init__(None, '')
         self.is_group = True
         if (
             len(self.tokens) > 0
@@ -183,9 +184,17 @@ def __init__(self, tokens=None):
                 self.tokens[-1].position - self.tokens[0].position
             ) + self.tokens[-1].length
 
-    def __str__(self):
+    @property
+    def value(self):
         return "".join(token.value for token in self.flatten())
 
+    @value.setter
+    def value(self, val):
+        pass  # value is computed from children
+
+    def __str__(self):
+        return self.value
+
     # weird bug
     # def __len__(self):
     #     return len(self.tokens)
@@ -231,6 +240,8 @@ def flatten(self):
         This method is recursively called for all child tokens.
         """
         for token in self.tokens:
+            if token is None:
+                continue
             if token.is_group:
                 yield from token.flatten()
             else:
@@ -238,7 +249,7 @@ def flatten(self):
 
     def get_sublists(self):
         for token in self.tokens:
-            if token.is_group:
+            if token is not None and token.is_group:
                 yield token
 
     @property
@@ -262,6 +273,8 @@ def _token_matching(self, funcs, start=0, end=None, reverse=False):
             indexes = range(start, end)
         for idx in indexes:
             token = self.tokens[idx]
+            if token is None:
+                continue
             for func in funcs:
                 if func(token):
                     return idx, token
@@ -277,11 +290,11 @@ def token_first(self, skip_ws=True, skip_cm=False):
         ignored too.
         """
 
-        # this on is inconsistent, using Comment instead of T.Comment...
         def matcher(tk):
             return not (
                 (skip_ws and tk.is_whitespace)
-                or (skip_cm and imt(tk, t=T.Comment, i=Comment))
+                or (skip_cm and (isinstance(tk, Comment)
+                                 or tk.ttype in T.Comment))
             )
 
         return self._token_matching(matcher)[1]
@@ -322,7 +335,8 @@ def token_next(self, idx, skip_ws=True, skip_cm=False, _reverse=False):
         def matcher(tk):
             return not (
                 (skip_ws and tk.is_whitespace)
-                or (skip_cm and imt(tk, t=T.Comment, i=Comment))
+                or (skip_cm and (isinstance(tk, Comment)
+                                 or tk.ttype in T.Comment))
             )
 
         return self._token_matching(matcher, idx, reverse=_reverse)
@@ -336,24 +350,29 @@ def group_tokens(self, grp_cls, start, end,
                      include_end=True, extend=False):
         """Replace tokens by an instance of *grp_cls*."""
         start_idx = start
-        start = self.tokens[start_idx]
+        start_token = self.tokens[start_idx]
 
         end_idx = end + include_end
 
-        # will be needed later for new group_clauses
-        # while skip_ws and tokens and tokens[-1].is_whitespace:
-        #     tokens = tokens[:-1]
-
-        if extend and isinstance(start, grp_cls):
-            subtokens = self.tokens[start_idx + 1:end_idx]
-
-            grp = start
+        if extend and isinstance(start_token, grp_cls):
+            # Scan backwards: non-None tokens are at end of range
+            subtokens = []
+            for i in range(end_idx - 1, start_idx, -1):
+                t = self.tokens[i]
+                if t is None:
+                    break
+                subtokens.append(t)
+                self.tokens[i] = None
+            subtokens.reverse()
+            grp = start_token
             grp.tokens.extend(subtokens)
-            del self.tokens[start_idx + 1:end_idx]
         else:
-            subtokens = self.tokens[start_idx:end_idx]
+            subtokens = [t for t in self.tokens[start_idx:end_idx]
+                         if t is not None]
             grp = grp_cls(subtokens)
-            self.tokens[start_idx:end_idx] = [grp]
+            self.tokens[start_idx] = grp
+            for i in range(start_idx + 1, end_idx):
+                self.tokens[i] = None
             grp.parent = self
 
         for token in subtokens:
@@ -361,6 +380,10 @@ def group_tokens(self, grp_cls, start, end,
 
         return grp
 
+    def _compact_tokens(self):
+        """Remove None sentinels left by deferred group_tokens."""
+        self.tokens = [t for t in self.tokens if t is not None]
+
     def insert_before(self, where, token):
         """Inserts *token* before *where*."""
         if not isinstance(where, int):
@@ -407,7 +430,7 @@ def get_parent_name(self):
         """
         dot_idx, _ = self.token_next_by(m=(T.Punctuation, "."))
         _, prev_ = self.token_prev(dot_idx)
-        return remove_quotes(prev_.value) if prev_ is not None else None
+        return remove_quotes(str(prev_)) if prev_ is not None else None
 
     def _get_first_name(self, idx=None, reverse=False,
                         keywords=False, real_name=False):
@@ -494,7 +517,7 @@ def get_typecast(self):
         """Returns the typecast or ``None`` of this object as a string."""
         midx, marker = self.token_next_by(m=(T.Punctuation, "::"))
         nidx, next_ = self.token_next(midx, skip_ws=False)
-        return next_.value if next_ else None
+        return str(next_) if next_ else None
 
     def get_ordering(self):
         """Returns the ordering or ``None`` as uppercase string."""