From 875b7d4933d2622bc6fb6c48d274dc91b6f5f6c3 Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 00:32:57 +0100 Subject: [PATCH 1/3] Look up parents in a map when computing metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tree-sitter doesn't store parent pointers: `tree_sitter::Node::parent` walks down from the root of the tree, so it costs O(depth). Some metrics walk up through all the ancestors of a node (e.g. the C++ LOC metric for every declaration), which then costs O(depth²) per node. On very deep trees, like the ones of some generated files in mozilla-central, computing the metrics of a single file took hours. `metrics` now computes the parent of each node once, with a single cursor pass over the tree, and the nodes it visits look their parent up in that map. Nodes created elsewhere keep using tree-sitter's lookup. Since `Node` now has a private field, it can no longer be constructed directly outside of the crate; `From` is provided instead. Fixes #1258. --- src/node.rs | 155 +++++++++++++++++++++++++++++++++++++++++++++----- src/spaces.rs | 8 ++- 2 files changed, 147 insertions(+), 16 deletions(-) diff --git a/src/node.rs b/src/node.rs index b5f1ebae6..364c27da4 100644 --- a/src/node.rs +++ b/src/node.rs @@ -1,3 +1,6 @@ +use std::collections::HashMap; +use std::fmt; + use tree_sitter::Node as OtherNode; use tree_sitter::Tree as OtherTree; use tree_sitter::{Parser, TreeCursor}; @@ -19,7 +22,50 @@ impl Tree { } pub(crate) fn get_root(&self) -> Node<'_> { - Node(self.0.root_node()) + Node(self.0.root_node(), None) + } +} + +/// The parent of each node of a tree. +/// +/// tree-sitter doesn't store parent pointers: `tree_sitter::Node::parent` +/// walks down from the root of the tree, so it costs O(depth). Walking up +/// through all the ancestors of a node then costs O(depth²), which takes +/// hours on very deep trees (e.g. generated files). Nodes created from a +/// root with [`Node::with_parents`] look their parent up here instead. +pub(crate) struct Parents<'a>(HashMap>); + +impl<'a> Parents<'a> { + /// Computes the parent of each node of the tree rooted at `root`. + pub(crate) fn new(root: &Node<'a>) -> Self { + let mut parents = HashMap::with_capacity(root.0.descendant_count()); + let mut ancestors: Vec> = Vec::new(); + let mut cursor = root.0.walk(); + 'walk: loop { + let node = cursor.node(); + if let Some(&parent) = ancestors.last() { + parents.insert(node.id(), parent); + } + if cursor.goto_first_child() { + ancestors.push(node); + continue; + } + while !cursor.goto_next_sibling() { + if !cursor.goto_parent() { + break 'walk; + } + ancestors.pop(); + } + } + Self(parents) + } +} + +impl fmt::Debug for Parents<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Parents") + .field("len", &self.0.len()) + .finish_non_exhaustive() } } @@ -27,10 +73,36 @@ impl Tree { /// /// The inner `tree_sitter::Node` is exposed for advanced use cases /// where direct access to the underlying tree-sitter API is needed. -#[derive(Clone, Copy, Debug)] -pub struct Node<'a>(pub OtherNode<'a>); +#[derive(Clone, Copy)] +pub struct Node<'a>(pub OtherNode<'a>, Option<&'a Parents<'a>>); + +impl fmt::Debug for Node<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("Node").field(&self.0).finish() + } +} + +impl<'a> From> for Node<'a> { + fn from(node: OtherNode<'a>) -> Self { + Self(node, None) + } +} impl<'a> Node<'a> { + /// Returns this node, using `parents` to look up the parent of this + /// node and of all the nodes reached from it. + pub(crate) fn with_parents<'b>(&self, parents: &'b Parents<'a>) -> Node<'b> + where + 'a: 'b, + { + Node(self.0, Some(parents)) + } + + /// Wraps another node of the same tree. + fn wrap(&self, node: OtherNode<'a>) -> Node<'a> { + Node(node, self.1) + } + /// Checks if a node represents a syntax error or contains any syntax errors /// anywhere within it. pub fn has_error(&self) -> bool { @@ -80,24 +152,27 @@ impl<'a> Node<'a> { } pub(crate) fn parent(&self) -> Option> { - self.0.parent().map(Node) + match self.1 { + Some(parents) => parents.0.get(&self.0.id()).map(|&parent| self.wrap(parent)), + None => self.0.parent().map(|parent| self.wrap(parent)), + } } #[inline(always)] pub(crate) fn has_sibling(&self, id: u16) -> bool { - self.0.parent().is_some_and(|parent| { + self.parent().is_some_and(|parent| { self.0 - .children(&mut parent.walk()) + .children(&mut parent.0.walk()) .any(|child| child.kind_id() == id) }) } pub(crate) fn previous_sibling(&self) -> Option> { - self.0.prev_sibling().map(Node) + self.0.prev_sibling().map(|node| self.wrap(node)) } pub(crate) fn next_sibling(&self) -> Option> { - self.0.next_sibling().map(Node) + self.0.next_sibling().map(|node| self.wrap(node)) } #[inline(always)] @@ -111,12 +186,12 @@ impl<'a> Node<'a> { self.0.child_count() } - pub(crate) fn child_by_field_name(&self, name: &str) -> Option> { - self.0.child_by_field_name(name).map(Node) + pub(crate) fn child_by_field_name(&self, name: &str) -> Option> { + self.0.child_by_field_name(name).map(|node| self.wrap(node)) } pub(crate) fn child(&self, pos: usize) -> Option> { - self.0.child(pos as u32).map(Node) + self.0.child(pos as u32).map(|node| self.wrap(node)) } pub(crate) fn children(&self) -> impl ExactSizeIterator> + use<'a> { @@ -130,7 +205,7 @@ impl<'a> Node<'a> { } pub(crate) fn cursor(&self) -> Cursor<'a> { - Cursor(self.0.walk()) + Cursor(self.0.walk(), self.1) } #[allow(dead_code)] @@ -183,11 +258,12 @@ impl<'a> Node<'a> { /// An `AST` cursor. #[derive(Clone)] -pub struct Cursor<'a>(TreeCursor<'a>); +pub struct Cursor<'a>(TreeCursor<'a>, Option<&'a Parents<'a>>); impl<'a> Cursor<'a> { pub(crate) fn reset(&mut self, node: &Node<'a>) { self.0.reset(node.0); + self.1 = node.1; } pub(crate) fn goto_next_sibling(&mut self) -> bool { @@ -199,7 +275,7 @@ impl<'a> Cursor<'a> { } pub(crate) fn node(&self) -> Node<'a> { - Node(self.0.node()) + Node(self.0.node(), self.1) } } @@ -266,3 +342,54 @@ impl<'a> Search<'a> for Node<'a> { } } } + +#[cfg(test)] +mod tests { + use std::path::Path; + + use super::*; + use crate::{CppParser, ParserTrait, metrics}; + + #[test] + fn parents_match_tree_sitter() { + let code = "namespace ns { + int f(int a) { + if (a > 0) { + return [](int b) { return b * 2; }(a); + } + for (int i = 0; i < a; i++) { a += i; } + return a; + } + struct S { int x; void g(); }; + }"; + let parser = CppParser::new(code.as_bytes().to_vec(), Path::new("foo.cpp"), None); + let root = parser.get_root(); + let parents = Parents::new(&root); + + let mut stack = vec![root.with_parents(&parents)]; + let mut count = 0; + while let Some(node) = stack.pop() { + assert_eq!( + node.parent().map(|parent| parent.id()), + node.0.parent().map(|parent| parent.id()), + "Wrong parent for {node:?}" + ); + stack.extend(node.children()); + count += 1; + } + assert_eq!(count, root.0.descendant_count()); + } + + // Every declaration below is nested in all the namespaces before it, and + // the LOC metric walks up through all its ancestors. With O(depth) parent + // lookups, this test takes about ten times longer (a deeper tree makes the + // difference much larger, see #1258). + #[test] + fn metrics_on_deep_tree() { + let depth = 500; + let code = "namespace a { int x; ".repeat(depth) + &"}".repeat(depth); + let parser = CppParser::new(code.into_bytes(), Path::new("foo.cpp"), None); + let space = metrics(&parser, Path::new("foo.cpp")).unwrap(); + assert_eq!(space.metrics.loc.lloc(), depth as f64); + } +} diff --git a/src/spaces.rs b/src/spaces.rs index 3e053c415..46e862a0e 100644 --- a/src/spaces.rs +++ b/src/spaces.rs @@ -5,7 +5,7 @@ use std::fmt; use std::path::{Path, PathBuf}; use crate::checker::Checker; -use crate::node::Node; +use crate::node::{Node, Parents}; use crate::abc::{self, Abc}; use crate::cognitive::{self, Cognitive}; @@ -285,7 +285,11 @@ struct State<'a> { /// ``` pub fn metrics<'a, T: ParserTrait>(parser: &'a T, path: &'a Path) -> Option { let code = parser.get_code(); - let node = parser.get_root(); + let root = parser.get_root(); + // Some metrics walk up through the ancestors of a node, so look parents up + // in a map instead of using the O(depth) `tree_sitter::Node::parent`. + let parents = Parents::new(&root); + let node = root.with_parents(&parents); let mut cursor = node.cursor(); let mut stack = Vec::new(); let mut children = Vec::new(); From 21f1c20dc61768c0a8dac9c73608eb1025fa610a Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Fri, 20 Nov 2026 01:55:00 +0100 Subject: [PATCH 2/3] Revert "Exclude two slow files from the grammar check tasks" This reverts commit ac94f0c. With parents looked up in a map, computing the metrics of security/manager/ssl/nsSTSPreloadList.inc and third_party/wasm2c/src/prebuilt/lexer-keywords.cc takes seconds instead of hours, so they no longer need to be excluded. --- check-grammar-crate.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/check-grammar-crate.py b/check-grammar-crate.py index 22ddfc57c..c96d48257 100755 --- a/check-grammar-crate.py +++ b/check-grammar-crate.py @@ -64,13 +64,6 @@ ], } -# Files excluded from the metrics computation, because computing their metrics -# takes hours (https://github.com/mozilla/rust-code-analysis/issues/1258). -EXCLUDED_FILES = [ - "*/security/manager/ssl/nsSTSPreloadList.inc", - "*/third_party/wasm2c/src/prebuilt/lexer-keywords.cc", -] - # Run a subprocess. def run_subprocess(cmd: str, *args: T.Union[str, pathlib.Path]) -> None: subprocess.run([cmd, *args]) @@ -98,8 +91,6 @@ def run_rca( "--progress=log", "-I", *include_grammars, - "-X", - *EXCLUDED_FILES, "-p", repo_dir, "-o", From 182ae1fd6b8667b99f0e39c482f064d476f0b4bb Mon Sep 17 00:00:00 2001 From: Marco Castelluccio Date: Sat, 21 Nov 2026 01:52:51 +0100 Subject: [PATCH 3/3] Fall back to tree-sitter for parents missing from the map The parent map only contains the nodes below the root it was computed from. A ParserTrait implementation can return a subtree from get_root(), and then the ancestors above it were no longer found: e.g. with the arrow function of `const f = () => 1` as root, the JavaScript checker didn't see the enclosing variable declarator and counted a closure instead of a function. On a miss, look the parent up with tree-sitter instead. For the root of the whole tree, that returns immediately. --- src/node.rs | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/src/node.rs b/src/node.rs index 364c27da4..f362375a6 100644 --- a/src/node.rs +++ b/src/node.rs @@ -152,10 +152,13 @@ impl<'a> Node<'a> { } pub(crate) fn parent(&self) -> Option> { - match self.1 { - Some(parents) => parents.0.get(&self.0.id()).map(|&parent| self.wrap(parent)), - None => self.0.parent().map(|parent| self.wrap(parent)), - } + // The map only contains the nodes below the root it was computed + // from, which may be a subtree: fall back to tree-sitter for the + // others. For the root of the whole tree, this returns immediately. + self.1 + .and_then(|parents| parents.0.get(&self.0.id()).copied()) + .or_else(|| self.0.parent()) + .map(|parent| self.wrap(parent)) } #[inline(always)] @@ -348,7 +351,10 @@ mod tests { use std::path::Path; use super::*; - use crate::{CppParser, ParserTrait, metrics}; + use crate::{ + Checker, CppParser, Javascript, JavascriptCode, JavascriptParser, ParserTrait, Search, + metrics, + }; #[test] fn parents_match_tree_sitter() { @@ -380,6 +386,27 @@ mod tests { assert_eq!(count, root.0.descendant_count()); } + // A parser can return a subtree as its root: the ancestors above it must + // still be found, e.g. to tell a function from a closure. + #[test] + fn parents_above_subtree_root() { + let code = "const f = () => 1;"; + let parser = JavascriptParser::new(code.as_bytes().to_vec(), Path::new("foo.js"), None); + let arrow = parser + .get_root() + .first_occurrence(|id| id == Javascript::ArrowFunction) + .unwrap(); + let parents = Parents::new(&arrow); + let arrow = arrow.with_parents(&parents); + + assert_eq!( + arrow.parent().map(|parent| parent.kind_id()), + Some(Javascript::VariableDeclarator as u16) + ); + assert!(JavascriptCode::is_func(&arrow)); + assert!(!JavascriptCode::is_closure(&arrow)); + } + // Every declaration below is nested in all the namespaces before it, and // the LOC metric walks up through all its ancestors. With O(depth) parent // lookups, this test takes about ten times longer (a deeper tree makes the