Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 0 additions & 9 deletions check-grammar-crate.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,13 +64,6 @@
],
}

# Files excluded from the metrics computation, because computing their metrics
# takes hours (https://github.com/mozilla/rust-code-analysis/issues/1258).
EXCLUDED_FILES = [
"*/security/manager/ssl/nsSTSPreloadList.inc",
"*/third_party/wasm2c/src/prebuilt/lexer-keywords.cc",
]

# Run a subprocess.
def run_subprocess(cmd: str, *args: T.Union[str, pathlib.Path]) -> None:
subprocess.run([cmd, *args])
Expand Down Expand Up @@ -98,8 +91,6 @@ def run_rca(
"--progress=log",
"-I",
*include_grammars,
"-X",
*EXCLUDED_FILES,
"-p",
repo_dir,
"-o",
Expand Down
182 changes: 168 additions & 14 deletions src/node.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
use std::collections::HashMap;
use std::fmt;

use tree_sitter::Node as OtherNode;
use tree_sitter::Tree as OtherTree;
use tree_sitter::{Parser, TreeCursor};
Expand All @@ -19,18 +22,87 @@ impl Tree {
}

pub(crate) fn get_root(&self) -> Node<'_> {
Node(self.0.root_node())
Node(self.0.root_node(), None)
}
}

/// The parent of each node of a tree.
///
/// tree-sitter doesn't store parent pointers: `tree_sitter::Node::parent`
/// walks down from the root of the tree, so it costs O(depth). Walking up
/// through all the ancestors of a node then costs O(depth²), which takes
/// hours on very deep trees (e.g. generated files). Nodes created from a
/// root with [`Node::with_parents`] look their parent up here instead.
pub(crate) struct Parents<'a>(HashMap<usize, OtherNode<'a>>);

impl<'a> Parents<'a> {
/// Computes the parent of each node of the tree rooted at `root`.
pub(crate) fn new(root: &Node<'a>) -> Self {
let mut parents = HashMap::with_capacity(root.0.descendant_count());
let mut ancestors: Vec<OtherNode<'a>> = Vec::new();
let mut cursor = root.0.walk();
'walk: loop {
let node = cursor.node();
if let Some(&parent) = ancestors.last() {
parents.insert(node.id(), parent);
}
if cursor.goto_first_child() {
ancestors.push(node);
continue;
}
while !cursor.goto_next_sibling() {
if !cursor.goto_parent() {
break 'walk;
}
ancestors.pop();
}
}
Self(parents)
}
}

impl fmt::Debug for Parents<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Parents")
.field("len", &self.0.len())
.finish_non_exhaustive()
}
}

/// An `AST` node.
///
/// The inner `tree_sitter::Node` is exposed for advanced use cases
/// where direct access to the underlying tree-sitter API is needed.
#[derive(Clone, Copy, Debug)]
pub struct Node<'a>(pub OtherNode<'a>);
#[derive(Clone, Copy)]
pub struct Node<'a>(pub OtherNode<'a>, Option<&'a Parents<'a>>);

impl fmt::Debug for Node<'_> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_tuple("Node").field(&self.0).finish()
}
}

impl<'a> From<OtherNode<'a>> for Node<'a> {
fn from(node: OtherNode<'a>) -> Self {
Self(node, None)
}
}

impl<'a> Node<'a> {
/// Returns this node, using `parents` to look up the parent of this
/// node and of all the nodes reached from it.
pub(crate) fn with_parents<'b>(&self, parents: &'b Parents<'a>) -> Node<'b>
where
'a: 'b,
{
Node(self.0, Some(parents))
}

/// Wraps another node of the same tree.
fn wrap(&self, node: OtherNode<'a>) -> Node<'a> {
Node(node, self.1)
}

/// Checks if a node represents a syntax error or contains any syntax errors
/// anywhere within it.
pub fn has_error(&self) -> bool {
Expand Down Expand Up @@ -80,24 +152,30 @@ impl<'a> Node<'a> {
}

pub(crate) fn parent(&self) -> Option<Node<'a>> {
self.0.parent().map(Node)
// The map only contains the nodes below the root it was computed
// from, which may be a subtree: fall back to tree-sitter for the
// others. For the root of the whole tree, this returns immediately.
self.1
.and_then(|parents| parents.0.get(&self.0.id()).copied())
.or_else(|| self.0.parent())
.map(|parent| self.wrap(parent))
}

#[inline(always)]
pub(crate) fn has_sibling(&self, id: u16) -> bool {
self.0.parent().is_some_and(|parent| {
self.parent().is_some_and(|parent| {
self.0
.children(&mut parent.walk())
.children(&mut parent.0.walk())
.any(|child| child.kind_id() == id)
})
}

pub(crate) fn previous_sibling(&self) -> Option<Node<'a>> {
self.0.prev_sibling().map(Node)
self.0.prev_sibling().map(|node| self.wrap(node))
}

pub(crate) fn next_sibling(&self) -> Option<Node<'a>> {
self.0.next_sibling().map(Node)
self.0.next_sibling().map(|node| self.wrap(node))
}

#[inline(always)]
Expand All @@ -111,12 +189,12 @@ impl<'a> Node<'a> {
self.0.child_count()
}

pub(crate) fn child_by_field_name(&self, name: &str) -> Option<Node<'_>> {
self.0.child_by_field_name(name).map(Node)
pub(crate) fn child_by_field_name(&self, name: &str) -> Option<Node<'a>> {
self.0.child_by_field_name(name).map(|node| self.wrap(node))
}

pub(crate) fn child(&self, pos: usize) -> Option<Node<'a>> {
self.0.child(pos as u32).map(Node)
self.0.child(pos as u32).map(|node| self.wrap(node))
}

pub(crate) fn children(&self) -> impl ExactSizeIterator<Item = Node<'a>> + use<'a> {
Expand All @@ -130,7 +208,7 @@ impl<'a> Node<'a> {
}

pub(crate) fn cursor(&self) -> Cursor<'a> {
Cursor(self.0.walk())
Cursor(self.0.walk(), self.1)
}

#[allow(dead_code)]
Expand Down Expand Up @@ -183,11 +261,12 @@ impl<'a> Node<'a> {

/// An `AST` cursor.
#[derive(Clone)]
pub struct Cursor<'a>(TreeCursor<'a>);
pub struct Cursor<'a>(TreeCursor<'a>, Option<&'a Parents<'a>>);

impl<'a> Cursor<'a> {
pub(crate) fn reset(&mut self, node: &Node<'a>) {
self.0.reset(node.0);
self.1 = node.1;
}

pub(crate) fn goto_next_sibling(&mut self) -> bool {
Expand All @@ -199,7 +278,7 @@ impl<'a> Cursor<'a> {
}

pub(crate) fn node(&self) -> Node<'a> {
Node(self.0.node())
Node(self.0.node(), self.1)
}
}

Expand Down Expand Up @@ -266,3 +345,78 @@ impl<'a> Search<'a> for Node<'a> {
}
}
}

#[cfg(test)]
mod tests {
use std::path::Path;

use super::*;
use crate::{
Checker, CppParser, Javascript, JavascriptCode, JavascriptParser, ParserTrait, Search,
metrics,
};

#[test]
fn parents_match_tree_sitter() {
let code = "namespace ns {
int f(int a) {
if (a > 0) {
return [](int b) { return b * 2; }(a);
}
for (int i = 0; i < a; i++) { a += i; }
return a;
}
struct S { int x; void g(); };
}";
let parser = CppParser::new(code.as_bytes().to_vec(), Path::new("foo.cpp"), None);
let root = parser.get_root();
let parents = Parents::new(&root);

let mut stack = vec![root.with_parents(&parents)];
let mut count = 0;
while let Some(node) = stack.pop() {
assert_eq!(
node.parent().map(|parent| parent.id()),
node.0.parent().map(|parent| parent.id()),
"Wrong parent for {node:?}"
);
stack.extend(node.children());
count += 1;
}
assert_eq!(count, root.0.descendant_count());
}

// A parser can return a subtree as its root: the ancestors above it must
// still be found, e.g. to tell a function from a closure.
#[test]
fn parents_above_subtree_root() {
let code = "const f = () => 1;";
let parser = JavascriptParser::new(code.as_bytes().to_vec(), Path::new("foo.js"), None);
let arrow = parser
.get_root()
.first_occurrence(|id| id == Javascript::ArrowFunction)
.unwrap();
let parents = Parents::new(&arrow);
let arrow = arrow.with_parents(&parents);

assert_eq!(
arrow.parent().map(|parent| parent.kind_id()),
Some(Javascript::VariableDeclarator as u16)
);
assert!(JavascriptCode::is_func(&arrow));
assert!(!JavascriptCode::is_closure(&arrow));
}

// Every declaration below is nested in all the namespaces before it, and
// the LOC metric walks up through all its ancestors. With O(depth) parent
// lookups, this test takes about ten times longer (a deeper tree makes the
// difference much larger, see #1258).
#[test]
fn metrics_on_deep_tree() {
let depth = 500;
let code = "namespace a { int x; ".repeat(depth) + &"}".repeat(depth);
let parser = CppParser::new(code.into_bytes(), Path::new("foo.cpp"), None);
let space = metrics(&parser, Path::new("foo.cpp")).unwrap();
assert_eq!(space.metrics.loc.lloc(), depth as f64);
}
}
8 changes: 6 additions & 2 deletions src/spaces.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ use std::fmt;
use std::path::{Path, PathBuf};

use crate::checker::Checker;
use crate::node::Node;
use crate::node::{Node, Parents};

use crate::abc::{self, Abc};
use crate::cognitive::{self, Cognitive};
Expand Down Expand Up @@ -285,7 +285,11 @@ struct State<'a> {
/// ```
pub fn metrics<'a, T: ParserTrait>(parser: &'a T, path: &'a Path) -> Option<FuncSpace> {
let code = parser.get_code();
let node = parser.get_root();
let root = parser.get_root();
// Some metrics walk up through the ancestors of a node, so look parents up
// in a map instead of using the O(depth) `tree_sitter::Node::parent`.
let parents = Parents::new(&root);
let node = root.with_parents(&parents);
let mut cursor = node.cursor();
let mut stack = Vec::new();
let mut children = Vec::new();
Expand Down