diff --git a/src/Analysis/Engine/Impl/Indexing/ISymbolIndex.cs b/src/Analysis/Engine/Impl/Indexing/ISymbolIndex.cs new file mode 100644 index 000000000..fad8f742c --- /dev/null +++ b/src/Analysis/Engine/Impl/Indexing/ISymbolIndex.cs @@ -0,0 +1,26 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; + +namespace Microsoft.PythonTools.Analysis.Indexing { + internal interface ISymbolIndex { + IEnumerable HierarchicalDocumentSymbols(Uri uri); + + IEnumerable WorkspaceSymbols(string query); + } +} diff --git a/src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs b/src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs new file mode 100644 index 000000000..cfc2f8dd4 --- /dev/null +++ b/src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs @@ -0,0 +1,39 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; + +namespace Microsoft.PythonTools.Analysis.Indexing { + internal static class SymbolExtensions { + public static IEnumerable Flatten(this IEnumerable docSyms, Uri uri, string parent = null, int? depthLimit = null) { + foreach (var sym in docSyms) { + yield return new FlatSymbol(sym.Name, sym.Kind, uri, sym.SelectionRange, parent); + + if (depthLimit != null) { + if (depthLimit < 1) { + yield break; + } + depthLimit--; + } + + foreach (var si in sym.Children.Flatten(uri, sym.Name, depthLimit)) { + yield return si; + } + } + } + } +} diff --git a/src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs b/src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs new file mode 100644 index 000000000..4f1fc24cf --- /dev/null +++ b/src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs @@ -0,0 +1,64 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using Microsoft.PythonTools.Analysis.Infrastructure; +using Microsoft.PythonTools.Parsing.Ast; + +namespace Microsoft.PythonTools.Analysis.Indexing { + internal class SymbolIndex : ISymbolIndex { + private readonly ConcurrentDictionary> _index = new ConcurrentDictionary>(); + + public SymbolIndex() { } + + public IEnumerable HierarchicalDocumentSymbols(Uri uri) + => _index.TryGetValue(uri, out var list) ? list : Enumerable.Empty(); + + public IEnumerable WorkspaceSymbols(string query) { + foreach (var kvp in _index) { + foreach (var found in WorkspaceSymbolsQuery(query, kvp.Key, kvp.Value)) { + yield return found; + } + } + } + + private IEnumerable WorkspaceSymbolsQuery(string query, Uri uri, IEnumerable symbols) { + // Some semblance of a BFS. + var queue = new Queue<(HierarchicalSymbol, string)>(symbols.Select(s => (s, (string)null))); + + while (queue.Count > 0) { + var (sym, parent) = queue.Dequeue(); + + if (sym.Name.ContainsOrdinal(query, ignoreCase: true)) { + yield return new FlatSymbol(sym.Name, sym.Kind, uri, sym.SelectionRange, parent); + } + + foreach (var child in sym.Children.MaybeEnumerate()) { + queue.Enqueue((child, sym.Name)); + } + } + } + + public void UpdateParseTree(Uri uri, PythonAst ast) { + var walker = new SymbolIndexWalker(ast); + ast.Walk(walker); + _index[uri] = walker.Symbols; + } + } +} diff --git a/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs b/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs new file mode 100644 index 000000000..a879ec46b --- /dev/null +++ b/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs @@ -0,0 +1,340 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using Microsoft.PythonTools.Analysis.Infrastructure; +using Microsoft.PythonTools.Parsing.Ast; + +namespace Microsoft.PythonTools.Analysis.Indexing { + internal class SymbolIndexWalker : PythonWalker { + private static readonly Regex DoubleUnderscore = new Regex(@"^__.*__$", RegexOptions.Compiled); + private static readonly Regex ConstantLike = new Regex(@"^[\p{Lu}\p{N}_]+$", RegexOptions.Compiled); + + private readonly PythonAst _ast; + private readonly SymbolStack _stack = new SymbolStack(); + + public SymbolIndexWalker(PythonAst ast) { + _ast = ast; + } + + public IReadOnlyList Symbols => _stack.Root; + + public override bool Walk(ClassDefinition node) { + _stack.Enter(SymbolKind.Class); + node.Body?.Walk(this); + var children = _stack.Exit(); + + _stack.AddSymbol(new HierarchicalSymbol( + node.Name, + SymbolKind.Class, + node.GetSpan(_ast), + node.NameExpression.GetSpan(_ast), + children, + FunctionKind.Class + )); + + return false; + } + + public override bool Walk(FunctionDefinition node) { + _stack.Enter(SymbolKind.Function); + foreach (var p in node.Parameters) { + AddVarSymbol(p.NameExpression); + } + node.Body?.Walk(this); + var children = _stack.Exit(); + + var span = node.GetSpan(_ast); + + var ds = new HierarchicalSymbol( + node.Name, + SymbolKind.Function, + span, + node.IsLambda ? span : node.NameExpression.GetSpan(_ast), + children, + FunctionKind.Function + ); + + if (_stack.Parent == SymbolKind.Class) { + switch (ds.Name) { + case "__init__": + ds.Kind = SymbolKind.Constructor; + break; + case var name when DoubleUnderscore.IsMatch(name): + ds.Kind = SymbolKind.Operator; + break; + default: + ds.Kind = SymbolKind.Method; + + if (node.Decorators != null) { + foreach (var dec in node.Decorators.Decorators) { + var maybeKind = DecoratorExpressionToKind(dec); + if (maybeKind.HasValue) { + ds.Kind = maybeKind.Value.kind; + ds._functionKind = maybeKind.Value.functionKind; + break; + } + } + } + + break; + } + } + + _stack.AddSymbol(ds); + + return false; + } + + public override bool Walk(ImportStatement node) { + foreach (var (nameNode, nameString) in node.Names.Zip(node.AsNames, (name, asName) => asName != null ? (asName, asName.Name) : ((Node)name, name.MakeString()))) { + var span = nameNode.GetSpan(_ast); + _stack.AddSymbol(new HierarchicalSymbol(nameString, SymbolKind.Module, span)); + } + + return false; + } + + public override bool Walk(FromImportStatement node) { + if (node.IsFromFuture) { + return false; + } + + foreach (var name in node.Names.Zip(node.AsNames, (name, asName) => asName ?? name)) { + var span = name.GetSpan(_ast); + _stack.AddSymbol(new HierarchicalSymbol(name.Name, SymbolKind.Module, span)); + } + + return false; + } + + public override bool Walk(AssignmentStatement node) { + node.Right?.Walk(this); + foreach (var ne in node.Left.OfType()) { + AddVarSymbol(ne); + } + return false; + } + + public override bool Walk(AugmentedAssignStatement node) { + node.Right?.Walk(this); + AddVarSymbol(node.Left as NameExpression); + return false; + } + + public override bool Walk(IfStatement node) { + WalkAndDeclareAll(node.Tests); + WalkAndDeclare(node.ElseStatement); + + return false; + } + + public override bool Walk(TryStatement node) { + WalkAndDeclare(node.Body); + WalkAndDeclareAll(node.Handlers); + WalkAndDeclare(node.Else); + WalkAndDeclare(node.Finally); + + return false; + } + + public override bool Walk(ForStatement node) { + _stack.EnterDeclared(); + AddVarSymbolRecursive(node.Left); + node.List?.Walk(this); + node.Body?.Walk(this); + _stack.ExitDeclaredAndMerge(); + + _stack.EnterDeclared(); + node.Else?.Walk(this); + _stack.ExitDeclaredAndMerge(); + + return false; + } + + public override bool Walk(ComprehensionFor node) { + AddVarSymbolRecursive(node.Left); + return base.Walk(node); + } + + public override bool Walk(ListComprehension node) { + _stack.Enter(SymbolKind.None); + return base.Walk(node); + } + + public override void PostWalk(ListComprehension node) => ExitComprehension(node); + + public override bool Walk(DictionaryComprehension node) { + _stack.Enter(SymbolKind.None); + return base.Walk(node); + } + + public override void PostWalk(DictionaryComprehension node) => ExitComprehension(node); + + public override bool Walk(SetComprehension node) { + _stack.Enter(SymbolKind.None); + return base.Walk(node); + } + + public override void PostWalk(SetComprehension node) => ExitComprehension(node); + + public override bool Walk(GeneratorExpression node) { + _stack.Enter(SymbolKind.None); + return base.Walk(node); + } + + public override void PostWalk(GeneratorExpression node) => ExitComprehension(node); + + private void ExitComprehension(Comprehension node) { + var children = _stack.Exit(); + var span = node.GetSpan(_ast); + + _stack.AddSymbol(new HierarchicalSymbol( + $"<{node.NodeName}>", + SymbolKind.None, + span, + children: children + )); + } + + + private void AddVarSymbol(NameExpression node) { + if (node == null) { + return; + } + + var kind = SymbolKind.Variable; + + switch (node.Name) { + case "*": + case "_": + return; + case var s when (_stack.Parent == null || _stack.Parent == SymbolKind.Class) && ConstantLike.IsMatch(s): + kind = SymbolKind.Constant; + break; + } + + var span = node.GetSpan(_ast); + + _stack.AddSymbol(new HierarchicalSymbol(node.Name, kind, span)); + } + + private void AddVarSymbolRecursive(Expression node) { + if (node == null) { + return; + } + + switch (node) { + case NameExpression ne: + AddVarSymbol(ne); + return; + + case SequenceExpression se: + foreach (var item in se.Items.MaybeEnumerate()) { + AddVarSymbolRecursive(item); + } + return; + } + } + + private (SymbolKind kind, string functionKind)? DecoratorExpressionToKind(Expression exp) { + switch (exp) { + case NameExpression ne when NameIsProperty(ne.Name): + case MemberExpression me when NameIsProperty(me.Name): + return (SymbolKind.Property, FunctionKind.Property); + + case NameExpression ne when NameIsStaticMethod(ne.Name): + case MemberExpression me when NameIsStaticMethod(me.Name): + return (SymbolKind.Method, FunctionKind.StaticMethod); + + case NameExpression ne when NameIsClassMethod(ne.Name): + case MemberExpression me when NameIsClassMethod(me.Name): + return (SymbolKind.Method, FunctionKind.ClassMethod); + } + + return null; + } + + private bool NameIsProperty(string name) => + name == "property" + || name == "abstractproperty" + || name == "classproperty" + || name == "abstractclassproperty"; + + private bool NameIsStaticMethod(string name) => + name == "staticmethod" + || name == "abstractstaticmethod"; + + private bool NameIsClassMethod(string name) => + name == "classmethod" + || name == "abstractclassmethod"; + + private void WalkAndDeclare(Node node) { + if (node == null) { + return; + } + + _stack.EnterDeclared(); + node.Walk(this); + _stack.ExitDeclaredAndMerge(); + } + + private void WalkAndDeclareAll(IEnumerable nodes) { + foreach (var node in nodes.MaybeEnumerate()) { + WalkAndDeclare(node); + } + } + + private class SymbolStack { + private readonly Stack<(SymbolKind? kind, List symbols)> _symbols; + private readonly Stack> _declared = new Stack>(new[] { new HashSet() }); + + public List Root { get; } = new List(); + + public SymbolKind? Parent => _symbols.Peek().kind; + + public SymbolStack() { + _symbols = new Stack<(SymbolKind?, List)>(new (SymbolKind?, List)[] { (null, Root) }); + } + + public void Enter(SymbolKind parent) { + _symbols.Push((parent, new List())); + EnterDeclared(); + } + + public List Exit() { + ExitDeclared(); + return _symbols.Pop().symbols; + } + public void EnterDeclared() => _declared.Push(new HashSet()); + + public void ExitDeclared() => _declared.Pop(); + + public void ExitDeclaredAndMerge() => _declared.Peek().UnionWith(_declared.Pop()); + + public void AddSymbol(HierarchicalSymbol sym) { + if (sym.Kind == SymbolKind.Variable && _declared.Peek().Contains(sym.Name)) { + return; + } + + _symbols.Peek().symbols.Add(sym); + _declared.Peek().Add(sym.Name); + } + } + } +} diff --git a/src/Analysis/Engine/Impl/Indexing/Symbols.cs b/src/Analysis/Engine/Impl/Indexing/Symbols.cs new file mode 100644 index 000000000..654766846 --- /dev/null +++ b/src/Analysis/Engine/Impl/Indexing/Symbols.cs @@ -0,0 +1,97 @@ +using System; +using System.Collections.Generic; + +namespace Microsoft.PythonTools.Analysis.Indexing { + // From LSP. + internal enum SymbolKind { + None = 0, + File = 1, + Module = 2, + Namespace = 3, + Package = 4, + Class = 5, + Method = 6, + Property = 7, + Field = 8, + Constructor = 9, + Enum = 10, + Interface = 11, + Function = 12, + Variable = 13, + Constant = 14, + String = 15, + Number = 16, + Boolean = 17, + Array = 18, + Object = 19, + Key = 20, + Null = 21, + EnumMember = 22, + Struct = 23, + Event = 24, + Operator = 25, + TypeParameter = 26 + } + + internal class FunctionKind { + public const string None = ""; + public const string Function = "function"; + public const string Property = "property"; + public const string StaticMethod = "staticmethod"; + public const string ClassMethod = "classmethod"; + public const string Class = "class"; + } + + // Analagous to LSP's DocumentSymbol. + internal class HierarchicalSymbol { + public string Name; + public string Detail; + public SymbolKind Kind; + public bool? Deprecated; + public SourceSpan Range; + public SourceSpan SelectionRange; + public IList Children; + + public string _functionKind; + + public HierarchicalSymbol( + string name, + SymbolKind kind, + SourceSpan range, + SourceSpan? selectionRange = null, + IList children = null, + string functionKind = FunctionKind.None + ) { + Name = name; + Kind = kind; + Range = range; + SelectionRange = selectionRange ?? range; + Children = children; + _functionKind = functionKind; + } + } + + // Analagous to LSP's SymbolInformation. + internal class FlatSymbol { + public string Name; + public SymbolKind Kind; + public bool? Deprecated; + public Uri DocumentUri; + public SourceSpan Range; + public string ContainerName; + + public FlatSymbol( + string name, + SymbolKind kind, + Uri documentUri, + SourceSpan range, + string containerName = null + ) { + Name = name; + Kind = kind; + DocumentUri = documentUri; + Range = range; + ContainerName = containerName; + } + } +} diff --git a/src/Analysis/Engine/Impl/Infrastructure/Extensions/StringExtensions.cs b/src/Analysis/Engine/Impl/Infrastructure/Extensions/StringExtensions.cs index bd3ca29a9..fb6b871a1 100644 --- a/src/Analysis/Engine/Impl/Infrastructure/Extensions/StringExtensions.cs +++ b/src/Analysis/Engine/Impl/Infrastructure/Extensions/StringExtensions.cs @@ -201,6 +201,9 @@ public static bool EqualsOrdinal(this string s, string other, bool ignoreCase) public static bool EqualsOrdinal(this string s, int index, string other, int otherIndex, int length, bool ignoreCase = false) => string.Compare(s, index, other, otherIndex, length, ignoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal) == 0; + public static bool ContainsOrdinal(this string s, string value, bool ignoreCase = false) + => s.IndexOfOrdinal(value, ignoreCase: ignoreCase) != -1; + public static string[] Split(this string s, char separator, int startIndex, int length) { var count = 0; var endIndex = startIndex + length; diff --git a/src/Analysis/Engine/Test/SymbolIndexTests.cs b/src/Analysis/Engine/Test/SymbolIndexTests.cs new file mode 100644 index 000000000..995a3cde2 --- /dev/null +++ b/src/Analysis/Engine/Test/SymbolIndexTests.cs @@ -0,0 +1,730 @@ +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.IO; +using FluentAssertions; +using Microsoft.Python.Tests.Utilities.FluentAssertions; +using Microsoft.PythonTools; +using Microsoft.PythonTools.Analysis.FluentAssertions; +using Microsoft.PythonTools.Analysis.Indexing; +using Microsoft.PythonTools.Parsing; +using Microsoft.PythonTools.Parsing.Ast; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace AnalysisTests { + [TestClass] + public class SymbolIndexTests { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() { + TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + } + + [TestCleanup] + public void TestCleanup() { + TestEnvironmentImpl.TestCleanup(); + } + + [TestMethod, Priority(0)] + public void WalkerAssignments() { + var code = @"x = 1 +y = x +z = y"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 1, 2, 2)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(3, 1, 3, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerMultipleAssignments() { + var code = @"x = y = z = 1"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 5, 1, 6)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(1, 9, 1, 10)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerUnderscore() { + var code = @"_ = 1"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public void WalkerIfStatement() { + var code = @"if foo(): + x = 1 +elif bar(): + x = 2 +else: + y = 3 +"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(4, 5, 4, 6)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(6, 5, 6, 6)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerTryExceptFinally() { + var code = @"try: + x = 1 +except Exception: + x = 2 +finally: + y = 3 +"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(4, 5, 4, 6)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(6, 5, 6, 6)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerReassign() { + var code = @"x = 1 +x = 2"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerAugmentedAssign() { + var code = @"x += 1"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerTopLevelConstant() { + var code = @"FOO_BAR_3 = 1234"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("FOO_BAR_3", SymbolKind.Constant, new SourceSpan(1, 1, 1, 10)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerFunction() { + var code = @"def func(x, y): + z = x + y + return z"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 3, 13), new SourceSpan(1, 5, 1, 9), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 10, 1, 11)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerFunctionStarredArgs() { + var code = @"def func(*args, **kwargs): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 31), new SourceSpan(1, 5, 1, 9), new[] { + new HierarchicalSymbol("args", SymbolKind.Variable, new SourceSpan(1, 11, 1, 15)), + new HierarchicalSymbol("kwargs", SymbolKind.Variable, new SourceSpan(1, 19, 1, 25)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerFunctionUnderscoreArg() { + var code = @"def func(_): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 17), new SourceSpan(1, 5, 1, 9), new List(), FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerImports() { + var code = @"import sys +import numpy as np +from os.path import join as osjoin +from os.path import ( join as osjoin2, exists as osexists, expanduser ) +"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("sys", SymbolKind.Module, new SourceSpan(1, 8, 1, 11)), + new HierarchicalSymbol("np", SymbolKind.Module, new SourceSpan(2, 17, 2, 19)), + new HierarchicalSymbol("osjoin", SymbolKind.Module, new SourceSpan(3, 29, 3, 35)), + new HierarchicalSymbol("osjoin2", SymbolKind.Module, new SourceSpan(4, 31, 4, 38)), + new HierarchicalSymbol("osexists", SymbolKind.Module, new SourceSpan(4, 50, 4, 58)), + new HierarchicalSymbol("expanduser", SymbolKind.Module, new SourceSpan(4, 60, 4, 70)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerImportFromFuture() { + var code = @"from __future__ import print_function"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public void WalkerClass() { + var code = @"class Foo(object): + ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 8), new SourceSpan(1, 7, 1, 10), new List(), FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassConstant() { + var code = @"class Foo(object): + CONSTANT = 1234"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 20), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("CONSTANT", SymbolKind.Constant, new SourceSpan(2, 5, 2, 13)), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerConstructor() { + var code = @"class Foo(object): + def __init__(self, x): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 31), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("__init__", SymbolKind.Constructor, new SourceSpan(2, 5, 2, 31), new SourceSpan(2, 9, 2, 17), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 18, 2, 22)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 24, 2, 25)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerMethod() { + var code = @"class Foo(object): + def foo(self, x): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 26), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("foo", SymbolKind.Method, new SourceSpan(2, 5, 2, 26), new SourceSpan(2, 9, 2, 12), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 13, 2, 17)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 19, 2, 20)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerDoubleUnderscoreMethod() { + var code = @"class Foo(object): + def __lt__(self, x): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 29), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("__lt__", SymbolKind.Operator, new SourceSpan(2, 5, 2, 29), new SourceSpan(2, 9, 2, 15), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 16, 2, 20)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 22, 2, 23)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerProperties() { + var code = @"class Foo(object): + @property + def func1(self): ... + + @abstractproperty + def func2(self): ... + + @classproperty + def func3(self): ... + + @abstractclassproperty + def func4(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 12, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func2", SymbolKind.Property, new SourceSpan(5, 5, 6, 25), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(6, 15, 6, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func3", SymbolKind.Property, new SourceSpan(8, 5, 9, 25), new SourceSpan(9, 9, 9, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(9, 15, 9, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func4", SymbolKind.Property, new SourceSpan(11, 5, 12, 25), new SourceSpan(12, 9, 12, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(12, 15, 12, 19)), + }, FunctionKind.Property), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerAbcProperties() { + var code = @"class Foo(object): + @abc.abstractproperty + def func1(self): ... + + @abc.abstractclassproperty + def func2(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 6, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func2", SymbolKind.Property, new SourceSpan(5, 5, 6, 25), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(6, 15, 6, 19)), + }, FunctionKind.Property), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerStaticMethods() { + var code = @"class Foo(object): + @staticmethod + def func1(arg): ... + + @abstractstaticmethod + def func2(arg): ... + + @abc.abstractstaticmethod + def func3(arg): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 9, 24), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 24), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("arg", SymbolKind.Variable, new SourceSpan(3, 15, 3, 18)), + }, FunctionKind.StaticMethod), + new HierarchicalSymbol("func2", SymbolKind.Method, new SourceSpan(5, 5, 6, 24), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("arg", SymbolKind.Variable, new SourceSpan(6, 15, 6, 18)), + }, FunctionKind.StaticMethod), + new HierarchicalSymbol("func3", SymbolKind.Method, new SourceSpan(8, 5, 9, 24), new SourceSpan(9, 9, 9, 14), new[] { + new HierarchicalSymbol("arg", SymbolKind.Variable, new SourceSpan(9, 15, 9, 18)), + }, FunctionKind.StaticMethod), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassMethods() { + var code = @"class Foo(object): + @classmethod + def func1(cls): ... + + @abstractclassmethod + def func2(cls): ... + + @abc.abstractclassmethod + def func3(cls): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 9, 24), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 24), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("cls", SymbolKind.Variable, new SourceSpan(3, 15, 3, 18)), + }, FunctionKind.ClassMethod), + new HierarchicalSymbol("func2", SymbolKind.Method, new SourceSpan(5, 5, 6, 24), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("cls", SymbolKind.Variable, new SourceSpan(6, 15, 6, 18)), + }, FunctionKind.ClassMethod), + new HierarchicalSymbol("func3", SymbolKind.Method, new SourceSpan(8, 5, 9, 24), new SourceSpan(9, 9, 9, 14), new[] { + new HierarchicalSymbol("cls", SymbolKind.Variable, new SourceSpan(9, 15, 9, 18)), + }, FunctionKind.ClassMethod), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerTopLevelFunctionDecorator() { + var code = @"@something +def func1(x, y): ... + +@something_else() +def func2(x, y): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func1", SymbolKind.Function, new SourceSpan(1, 1, 2, 21), new SourceSpan(2, 5, 2, 10), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 11, 2, 12)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 14, 2, 15)), + }, FunctionKind.Function), + new HierarchicalSymbol("func2", SymbolKind.Function, new SourceSpan(4, 1, 5, 21), new SourceSpan(5, 5, 5, 10), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(5, 11, 5, 12)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(5, 14, 5, 15)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassFunctionDecorator() { + var code = @"class Foo(object): + @something + def func1(self): ... + + @something_else() + def func2(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 6, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), + }, FunctionKind.Function), + new HierarchicalSymbol("func2", SymbolKind.Method, new SourceSpan(5, 5, 6, 25), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(6, 15, 6, 19)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassFunctionMultiDecorator() { + var code = @"class Foo(object): + @property + @something + def func1(self): ... + + @something + @property + def func2(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 8, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 4, 25), new SourceSpan(4, 9, 4, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(4, 15, 4, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func2", SymbolKind.Property, new SourceSpan(6, 5, 8, 25), new SourceSpan(8, 9, 8, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(8, 15, 8, 19)), + }, FunctionKind.Property), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerLambda() { + var code = @"f = lambda x, y: x + y"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.Function, new SourceSpan(1, 5, 1, 23), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 12, 1, 13)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 15, 1, 16)), + }, functionKind: FunctionKind.Function), + new HierarchicalSymbol("f", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerForLoop() { + var code = @"z = False +for [x, y, (p, q)] in [[1, 2, [3, 4]]]: + z += x +else: + z = None"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 6, 2, 7)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 9, 2, 10)), + new HierarchicalSymbol("p", SymbolKind.Variable, new SourceSpan(2, 13, 2, 14)), + new HierarchicalSymbol("q", SymbolKind.Variable, new SourceSpan(2, 16, 2, 17)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(3, 5, 3, 6)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(5, 5, 5, 6)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerListComprehension() { + var code = @"flat_list = [item for sublist in l for item in sublist]"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 13, 1, 56), children: new[] { + new HierarchicalSymbol("sublist", SymbolKind.Variable, new SourceSpan(1, 23, 1, 30)), + new HierarchicalSymbol("item", SymbolKind.Variable, new SourceSpan(1, 40, 1, 44)), + }), + new HierarchicalSymbol("flat_list", SymbolKind.Variable, new SourceSpan(1, 1, 1, 10)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerDictionaryComprehension() { + var code = @"d = { x: y for x, y in zip(range(10), range(10)) }"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 51), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 19, 1, 20)), + }), + new HierarchicalSymbol("d", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerSetComprehension() { + var code = @"s = { x for x in range(10) }"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 29), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + }), + new HierarchicalSymbol("s", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerGenerator() { + var code = @"g = (x + y for x, y in zip(range(10), range(10)))"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 50), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 19, 1, 20)), + }), + new HierarchicalSymbol("g", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerNestedListComprehension() { + var code = @"l = [ + x for x in [ + y for y in range(10) + ] +]"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 5, 2), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 11, 2, 12)), + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(2, 16, 4, 6), children: new[] { + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(3, 15, 3, 16)), + }), + }), + new HierarchicalSymbol("l", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerIncompleteFunction() { + var code = @"def func(x, y):"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 16), new SourceSpan(1, 5, 1, 9), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 10, 1, 11)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerIncompleteClass() { + var code = @"class Foo(object):"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 1, 19), new SourceSpan(1, 7, 1, 10), new List(), FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerIncompleteAssign() { + var code = @"x ="; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerAugmentedAssignLambda() { + var code = @"x += lambda x, y: x + y"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.Function, new SourceSpan(1, 6, 1, 24), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), + }, functionKind: FunctionKind.Function), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void IndexHierarchicalDocument() { + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + var ast = GetParse("x = 1"); + index.UpdateParseTree(uri, ast); + + var symbols = index.HierarchicalDocumentSymbols(uri); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void IndexHierarchicalDocumentUpdate() { + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + var ast = GetParse("x = 1"); + index.UpdateParseTree(uri, ast); + + var symbols = index.HierarchicalDocumentSymbols(uri); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + + ast = GetParse("y = 1"); + index.UpdateParseTree(uri, ast); + + symbols = index.HierarchicalDocumentSymbols(uri); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void IndexHierarchicalDocumentNotFound() { + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + + var symbols = index.HierarchicalDocumentSymbols(uri); + symbols.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public void IndexWorkspaceSymbolsFlatten() { + var code = @"class Foo(object): + def foo(self, x): ..."; + + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + var ast = GetParse(code); + index.UpdateParseTree(uri, ast); + + var symbols = index.WorkspaceSymbols(""); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new FlatSymbol("Foo", SymbolKind.Class, uri, new SourceSpan(1, 7, 1, 10)), + new FlatSymbol("foo", SymbolKind.Method, uri, new SourceSpan(2, 9, 2, 12), "Foo"), + new FlatSymbol("self", SymbolKind.Variable, uri, new SourceSpan(2, 13, 2, 17), "foo"), + new FlatSymbol("x", SymbolKind.Variable, uri, new SourceSpan(2, 19, 2, 20), "foo"), + }); + } + + [TestMethod, Priority(0)] + public void IndexWorkspaceSymbolsFiltered() { + var code = @"class Foo(object): + def foo(self, x): ..."; + + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + var ast = GetParse(code); + index.UpdateParseTree(uri, ast); + + var symbols = index.WorkspaceSymbols("x"); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new FlatSymbol("x", SymbolKind.Variable, uri, new SourceSpan(2, 19, 2, 20), "foo"), + }); + } + + [TestMethod, Priority(0)] + public void IndexWorkspaceSymbolsNotFound() { + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + + var symbols = index.WorkspaceSymbols(""); + symbols.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public void IndexWorkspaceSymbolsCaseInsensitive() { + var code = @"class Foo(object): + def foo(self, x): ..."; + + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + var ast = GetParse(code); + index.UpdateParseTree(uri, ast); + + var symbols = index.WorkspaceSymbols("foo"); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new FlatSymbol("Foo", SymbolKind.Class, uri, new SourceSpan(1, 7, 1, 10)), + new FlatSymbol("foo", SymbolKind.Method, uri, new SourceSpan(2, 9, 2, 12), "Foo"), + }); + } + + private PythonAst GetParse(string code, PythonLanguageVersion version = PythonLanguageVersion.V37) + => Parser.CreateParser(new StringReader(code), version).ParseFile(); + + private IReadOnlyList WalkSymbols(string code, PythonLanguageVersion version = PythonLanguageVersion.V37) { + var ast = GetParse(code); + var walker = new SymbolIndexWalker(ast); + ast.Walk(walker); + return walker.Symbols; + } + } +} diff --git a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs index 98bb72841..aef505e54 100644 --- a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs +++ b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs @@ -16,11 +16,13 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.PythonTools; using Microsoft.PythonTools.Analysis; +using Microsoft.PythonTools.Analysis.Indexing; using Microsoft.PythonTools.Analysis.Infrastructure; using Microsoft.PythonTools.Analysis.Values; using Microsoft.PythonTools.Interpreter; @@ -30,42 +32,50 @@ public sealed partial class Server { private static int _symbolHierarchyDepthLimit = 10; private static int _symbolHierarchyMaxSymbols = 1000; - public override async Task WorkspaceSymbols(WorkspaceSymbolParams @params, CancellationToken cancellationToken) { - await WaitForCompleteAnalysisAsync(cancellationToken); - - var members = Enumerable.Empty(); - var opts = GetMemberOptions.ExcludeBuiltins | GetMemberOptions.DeclaredOnly; - - foreach (var entry in ProjectFiles) { - members = members.Concat( - await GetModuleVariablesAsync(entry as ProjectEntry, opts, @params.query, 50, cancellationToken) - ); - } + public override Task WorkspaceSymbols(WorkspaceSymbolParams @params, CancellationToken cancellationToken) { + var symbols = SymbolIndex.WorkspaceSymbols(@params.query); + + var result = symbols.Take(_symbolHierarchyMaxSymbols).Select(s => { + cancellationToken.ThrowIfCancellationRequested(); + return new SymbolInformation { + name = s.Name, + kind = (SymbolKind)s.Kind, + location = new Location { + range = s.Range, + uri = s.DocumentUri, + }, + containerName = s.ContainerName, + }; + }).ToArray(); - members = members.GroupBy(mr => mr.Name).Select(g => g.First()); - return members.Select(ToSymbolInformation).ToArray(); + return Task.FromResult(result); } - public override async Task DocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { - await WaitForCompleteAnalysisAsync(cancellationToken); - var opts = GetMemberOptions.ExcludeBuiltins | GetMemberOptions.DeclaredOnly; - var entry = ProjectFiles.GetEntry(@params.textDocument.uri); - - var members = await GetModuleVariablesAsync(entry as ProjectEntry, opts, string.Empty, 50, cancellationToken); - return members - .GroupBy(mr => mr.Name) - .Select(g => g.First()) - .Select(ToSymbolInformation) - .ToArray(); - } + public override Task DocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { + var uri = @params.textDocument.uri; + var symbols = SymbolIndex.HierarchicalDocumentSymbols(uri); + + var result = symbols.Flatten(uri, depthLimit: _symbolHierarchyDepthLimit).Take(_symbolHierarchyMaxSymbols).Select(s => { + cancellationToken.ThrowIfCancellationRequested(); + return new SymbolInformation { + name = s.Name, + kind = (SymbolKind)s.Kind, + location = new Location { + range = s.Range, + uri = s.DocumentUri, + }, + containerName = s.ContainerName, + }; + }).ToArray(); - public override async Task HierarchicalDocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { - await WaitForCompleteAnalysisAsync(cancellationToken); - var opts = GetMemberOptions.ExcludeBuiltins | GetMemberOptions.DeclaredOnly; - var entry = ProjectFiles.GetEntry(@params.textDocument.uri); + return Task.FromResult(result); + } - var members = await GetModuleVariablesAsync(entry as ProjectEntry, opts, string.Empty, 50, cancellationToken); - return ToDocumentSymbols(members); + public override Task HierarchicalDocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { + var uri = @params.textDocument.uri; + var symbols = SymbolIndex.HierarchicalDocumentSymbols(uri); + var result = ToDocumentSymbols(symbols, cancellationToken); + return Task.FromResult(result); } private static async Task> GetModuleVariablesAsync(ProjectEntry entry, GetMemberOptions opts, string prefix, int timeout, CancellationToken token) { @@ -78,7 +88,7 @@ private static IEnumerable GetModuleVariables(ProjectEntry entry, var all = breadthFirst.SelectMany(c => analysis.GetAllAvailableMembersFromScope(c, opts)); var result = all .Where(m => { - if (m.Values.Any(v => v.DeclaringModule == entry || + if (m.Values.Any(v => v.DeclaringModule == entry || v.Locations .MaybeEnumerate() .ExcludeDefault() @@ -91,6 +101,28 @@ private static IEnumerable GetModuleVariables(ProjectEntry entry, return result; } + private DocumentSymbol[] ToDocumentSymbols(IEnumerable hSymbols, CancellationToken cancellationToken = default(CancellationToken)) { + return hSymbols.MaybeEnumerate().Select(hSym => new DocumentSymbol { + name = hSym.Name, + detail = hSym.Detail, + kind = (SymbolKind)hSym.Kind, + deprecated = hSym.Deprecated ?? false, + range = hSym.Range, + selectionRange = hSym.SelectionRange, + children = ToDocumentSymbols(hSym.Children), + _functionKind = hSym._functionKind, + }).Select(v => { + cancellationToken.ThrowIfCancellationRequested(); + + if (v.selectionRange.start < v.range.start || v.selectionRange.end > v.range.end) { + Debug.Fail($"selectionRange {v.selectionRange} not encompassed by range {v.range}"); + v.selectionRange = v.range; + } + + return v; + }).ToArray(); + } + private SymbolInformation ToSymbolInformation(IMemberResult m) { var res = new SymbolInformation { name = m.Name, diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index 8a6a1e6c0..625d19b3a 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -27,6 +27,7 @@ using Microsoft.Python.LanguageServer.Extensions; using Microsoft.PythonTools.Analysis; using Microsoft.PythonTools.Analysis.Documentation; +using Microsoft.PythonTools.Analysis.Indexing; using Microsoft.PythonTools.Analysis.Infrastructure; using Microsoft.PythonTools.Intellisense; using Microsoft.PythonTools.Interpreter; @@ -97,6 +98,9 @@ public Server() { ParseQueue = new ParseQueue(); _editorFiles = new EditorFiles(this); + SymbolIndex = new SymbolIndex(); + OnParseComplete += UpdateSymbolIndex; + _disposableBag .Add(() => { foreach (var ext in _extensions.Values) { @@ -106,6 +110,7 @@ public Server() { (ext as IDisposable)?.Dispose(); } }) + .Add(() => OnParseComplete -= UpdateSymbolIndex) .Add(ProjectFiles) .Add(() => Analyzer?.Dispose()) .Add(AnalysisQueue) @@ -120,6 +125,7 @@ private void Analysis_UnhandledException(object sender, UnhandledExceptionEventA internal AnalysisQueue AnalysisQueue { get; } internal ParseQueue ParseQueue { get; } + internal SymbolIndex SymbolIndex { get; } internal PythonAnalyzer Analyzer { get; private set; } internal ServerSettings Settings { get; private set; } = new ServerSettings(); @@ -709,5 +715,13 @@ private async Task WaitForCompleteAnalysisWorker(CancellationToken cancellationT } TraceMessage($"Analysis complete."); } + + private void UpdateSymbolIndex(object sender, ParseCompleteEventArgs args) { + ProjectFiles.GetEntry(args.uri, args.version, out var entry, out var tree); + var ast = entry.GetCurrentParse()?.Tree; + if (ast != null) { + SymbolIndex.UpdateParseTree(args.uri, ast); + } + } } } diff --git a/src/UnitTests/Core/Impl/FluentAssertions/CollectionAssertionsExtensions.cs b/src/UnitTests/Core/Impl/FluentAssertions/CollectionAssertionsExtensions.cs index 453af3a7a..c84edf772 100644 --- a/src/UnitTests/Core/Impl/FluentAssertions/CollectionAssertionsExtensions.cs +++ b/src/UnitTests/Core/Impl/FluentAssertions/CollectionAssertionsExtensions.cs @@ -37,5 +37,9 @@ public static AndConstraint OnlyContain( public static AndConstraint OnlyContain( this StringCollectionAssertions assertions, IReadOnlyCollection expected, string because = "", params object[] reasonArgs) => assertions.HaveCount(expected.Count, because, reasonArgs).And.Contain(expected, because, reasonArgs); + + public static AndConstraint> BeEquivalentToWithStrictOrdering( + this GenericCollectionAssertions assertions, IEnumerable expected, string because = "", params object[] reasonArgs) + => assertions.BeEquivalentTo(expected, options => options.WithStrictOrdering(), because, reasonArgs); } }