From e89df57e8127194a9fc4b91d41eec9fec2c3a7f3 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Fri, 4 Jan 2019 15:56:53 -0800 Subject: [PATCH 01/10] first working indexer prototype --- .../Engine/Impl/Indexing/ISymbolIndex.cs | 28 ++ .../Engine/Impl/Indexing/IndexerWalker.cs | 245 ++++++++++++++++++ .../Engine/Impl/Indexing/SymbolExtensions.cs | 39 +++ .../Engine/Impl/Indexing/SymbolIndex.cs | 80 ++++++ src/Analysis/Engine/Impl/Indexing/Symbols.cs | 57 ++++ .../Implementation/Server.WorkspaceSymbols.cs | 115 ++++++-- .../Impl/Implementation/Server.cs | 14 + 7 files changed, 551 insertions(+), 27 deletions(-) create mode 100644 src/Analysis/Engine/Impl/Indexing/ISymbolIndex.cs create mode 100644 src/Analysis/Engine/Impl/Indexing/IndexerWalker.cs create mode 100644 src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs create mode 100644 src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs create mode 100644 src/Analysis/Engine/Impl/Indexing/Symbols.cs diff --git a/src/Analysis/Engine/Impl/Indexing/ISymbolIndex.cs b/src/Analysis/Engine/Impl/Indexing/ISymbolIndex.cs new file mode 100644 index 000000000..37c06254c --- /dev/null +++ b/src/Analysis/Engine/Impl/Indexing/ISymbolIndex.cs @@ -0,0 +1,28 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.PythonTools.Analysis.Indexing { + internal interface ISymbolIndex { + Task> HierarchicalDocumentSymbolsAsync(Uri uri, CancellationToken token = default); + + Task> WorkspaceSymbolsAsync(string query, CancellationToken token = default); + } +} diff --git a/src/Analysis/Engine/Impl/Indexing/IndexerWalker.cs b/src/Analysis/Engine/Impl/Indexing/IndexerWalker.cs new file mode 100644 index 000000000..0ab7e6d3c --- /dev/null +++ b/src/Analysis/Engine/Impl/Indexing/IndexerWalker.cs @@ -0,0 +1,245 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using Microsoft.PythonTools.Parsing.Ast; + +namespace Microsoft.PythonTools.Analysis.Indexing { + internal class IndexerWalker : PythonWalker { + private readonly PythonAst _ast; + + private readonly Stack _scope = new Stack(); + private HierarchicalSymbol Parent => _scope.Count > 0 ? _scope.Peek() : null; + + private bool _skipNext; + private bool SkipNext { + get { + var v = _skipNext; + _skipNext = false; + return v; + } + set { _skipNext = value; } + } + + public List Symbols = new List(); + + private void AddHierarchicalSymbol(HierarchicalSymbol sym) { + if (SkipNext) { + return; + } + + var parent = Parent; + + if (parent == null) { + Symbols.Add(sym); + } else { + parent.Children = parent.Children ?? new List(); + parent.Children.Add(sym); + } + } + + public IndexerWalker(PythonAst ast) { + _ast = ast; + } + + public override bool Walk(NameExpression node) { + var kind = SymbolKind.Variable; + + switch (node.Name) { + case "*": + case "_": + return base.Walk(node); + case var s when (Parent == null || Parent.Kind == SymbolKind.Class) && Regex.IsMatch(s, @"^[\p{Lu}\p{N}_]+$"): + kind = SymbolKind.Constant; + break; + } + + var span = node.GetSpan(_ast); + + AddHierarchicalSymbol(new HierarchicalSymbol { + Name = node.Name, + Kind = kind, + Range = span, + SelectionRange = span, + }); + + return base.Walk(node); + } + + public override bool Walk(ClassDefinition node) { + var ds = new HierarchicalSymbol { + Name = node.Name, + Kind = SymbolKind.Class, + Range = node.GetSpan(_ast), + SelectionRange = node.NameExpression.GetSpan(_ast), + }; + AddHierarchicalSymbol(ds); + _scope.Push(ds); + SkipNext = true; + return base.Walk(node); + } + + public override void PostWalk(ClassDefinition node) { + _scope.Pop(); + base.PostWalk(node); + } + + public override bool Walk(FunctionDefinition node) { + if (node.IsLambda) { + return base.Walk(node); + } + + var ds = new HierarchicalSymbol { + Name = node.Name, + Kind = SymbolKind.Function, + Range = node.GetSpan(_ast), + SelectionRange = node.NameExpression.GetSpan(_ast), + }; + + if (Parent?.Kind == SymbolKind.Class) { + switch (ds.Name) { + case "__init__": + ds.Kind = SymbolKind.Constructor; + break; + case var name when Regex.IsMatch(name, @"^__.*__$"): + ds.Kind = SymbolKind.Operator; + break; + default: + ds.Kind = SymbolKind.Method; + + if (node.Decorators != null) { + foreach (var dec in node.Decorators.Decorators) { + var maybeKind = DecoratorExpressionToKind(dec); + if (maybeKind.HasValue) { + ds.Kind = maybeKind.Value; + break; + } + } + } + + break; + } + } + + AddHierarchicalSymbol(ds); + _scope.Push(ds); + SkipNext = true; + return base.Walk(node); + } + + public override void PostWalk(FunctionDefinition node) { + if (node.IsLambda) { + base.PostWalk(node); + return; + } + + _scope.Pop(); + base.PostWalk(node); + } + + public override bool Walk(Parameter node) { + var span = node.GetSpan(_ast); + AddHierarchicalSymbol(new HierarchicalSymbol { + Name = node.Name, + Kind = SymbolKind.Variable, + Range = span, + SelectionRange = span, + }); + + return base.Walk(node); + } + + public override bool Walk(ImportStatement node) { + foreach (var imp in node.Names.Zip(node.AsNames, (name, asName) => (name, asName))) { + var span = imp.name.GetSpan(_ast); + AddHierarchicalSymbol(new HierarchicalSymbol { + Name = imp.name.MakeString(), + Kind = SymbolKind.Package, + Range = span, + SelectionRange = span, + }); + + if (imp.asName != null) { + span = imp.asName.GetSpan(_ast); + AddHierarchicalSymbol(new HierarchicalSymbol { + Name = imp.asName.Name, + Kind = SymbolKind.Package, + Range = span, + SelectionRange = span, + }); + } + } + + return base.Walk(node); + } + + public override bool Walk(FromImportStatement node) { + var root = node.Root; + var rootSpan = root.GetSpan(_ast); + AddHierarchicalSymbol(new HierarchicalSymbol { + Name = root.MakeString(), + Kind = SymbolKind.Package, + Range = rootSpan, + SelectionRange = rootSpan, + }); + + foreach (var imp in node.Names.Zip(node.AsNames, (name, asName) => (name, asName))) { + var span = imp.name.GetSpan(_ast); + AddHierarchicalSymbol(new HierarchicalSymbol { + Name = imp.name.Name, + Kind = SymbolKind.Package, + Range = span, + SelectionRange = span, + }); + + if (imp.asName != null) { + span = imp.asName.GetSpan(_ast); + AddHierarchicalSymbol(new HierarchicalSymbol { + Name = imp.asName.Name, + Kind = SymbolKind.Package, + Range = span, + SelectionRange = span, + }); + } + } + + return base.Walk(node); + } + + public override bool Walk(DecoratorStatement node) { + return base.Walk(node); + } + + private SymbolKind? DecoratorExpressionToKind(Expression exp) { + switch (exp) { + case NameExpression ne when + ne.Name == "property" || ne.Name == "abstractproperty" + || ne.Name == "classproperty" || ne.Name == "abstractclassproperty": + return SymbolKind.Property; + + case CallExpression ce when ce.Target is NameExpression ne: + return DecoratorExpressionToKind(ne); + + case CallExpression ce when ce.Target is MemberExpression me: + return DecoratorExpressionToKind(me.Target); + } + + return null; + } + } +} diff --git a/src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs b/src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs new file mode 100644 index 000000000..2c1651043 --- /dev/null +++ b/src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs @@ -0,0 +1,39 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.PythonTools.Analysis.Indexing { + internal static class SymbolExtensions { + public static IEnumerable Flatten(this IEnumerable docSyms, Uri uri, string parent = null) { + foreach (var sym in docSyms ?? Enumerable.Empty()) { + yield return new FlatSymbol { + Name = sym.Name, + Kind = sym.Kind, + DocumentUri = uri, + Range = sym.SelectionRange, + ContainerName = parent, + }; + + foreach (var si in sym.Children.Flatten(uri, sym.Name)) { + yield return si; + } + } + } + } +} diff --git a/src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs b/src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs new file mode 100644 index 000000000..6b6e6deee --- /dev/null +++ b/src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs @@ -0,0 +1,80 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.PythonTools.Analysis.Infrastructure; +using Microsoft.PythonTools.Parsing.Ast; + +namespace Microsoft.PythonTools.Analysis.Indexing { + internal class SymbolIndex : ISymbolIndex { + private readonly ConcurrentDictionary> _index = new ConcurrentDictionary>(); + + public SymbolIndex() { } + + public async Task> HierarchicalDocumentSymbolsAsync(Uri uri, CancellationToken token = default) { + return _index.TryGetValue(uri, out var list) ? list : null; + } + + public Task> WorkspaceSymbolsAsync(string query, CancellationToken token = default) { + return Task.FromResult(WorkspaceSymbols(query)); + } + + private IEnumerable WorkspaceSymbols(string query, CancellationToken token = default) { + foreach (var kvp in _index) { + foreach (var found in WorkspaceSymbolsQuery(query, kvp.Key, kvp.Value, token: token)) { + yield return found; + } + } + } + + private IEnumerable WorkspaceSymbolsQuery(string query, Uri uri, IEnumerable symbols, CancellationToken token = default) { + // Some semblance of a BFS. + var ll = new LinkedList<(HierarchicalSymbol, string)>(symbols.Select(s => (s, (string)null))); + + while (ll.First != null) { + token.ThrowIfCancellationRequested(); + + var (sym, parent) = ll.First.Value; + ll.RemoveFirst(); + + if (sym.Name.Contains(query)) { + yield return new FlatSymbol { + Name = sym.Name, + Kind = sym.Kind, + DocumentUri = uri, + Range = sym.SelectionRange, + ContainerName = parent, + }; + } + + foreach (var child in sym.Children.MaybeEnumerate()) { + ll.AddLast((child, sym.Name)); + } + } + } + + public void UpdateParseTree(Uri uri, PythonAst ast) { + var walker = new IndexerWalker(ast); + ast.Walk(walker); + _index[uri] = walker.Symbols; + } + } +} diff --git a/src/Analysis/Engine/Impl/Indexing/Symbols.cs b/src/Analysis/Engine/Impl/Indexing/Symbols.cs new file mode 100644 index 000000000..106436e52 --- /dev/null +++ b/src/Analysis/Engine/Impl/Indexing/Symbols.cs @@ -0,0 +1,57 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.PythonTools.Analysis.Indexing { + // From LSP. + internal enum SymbolKind { + None = 0, + File = 1, + Module = 2, + Namespace = 3, + Package = 4, + Class = 5, + Method = 6, + Property = 7, + Field = 8, + Constructor = 9, + Enum = 10, + Interface = 11, + Function = 12, + Variable = 13, + Constant = 14, + String = 15, + Number = 16, + Boolean = 17, + Array = 18, + Object = 19, + Key = 20, + Null = 21, + EnumMember = 22, + Struct = 23, + Event = 24, + Operator = 25, + TypeParameter = 26 + } + + // Analagous to LSP's DocumentSymbol. + internal class HierarchicalSymbol { + public string Name; + public string Detail; + public SymbolKind Kind; + public bool? Deprecated; + public SourceSpan Range; + public SourceSpan SelectionRange; + public List Children; + } + + // Analagous to LSP's SymbolInformation. + internal class FlatSymbol { + public string Name; + public SymbolKind Kind; + public bool? Deprecated; + public Uri DocumentUri; + public SourceSpan Range; + public string ContainerName; + } +} diff --git a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs index 98bb72841..26f9f1e0a 100644 --- a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs +++ b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs @@ -16,14 +16,17 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading; using System.Threading.Tasks; using Microsoft.PythonTools; using Microsoft.PythonTools.Analysis; +using Microsoft.PythonTools.Analysis.Indexing; using Microsoft.PythonTools.Analysis.Infrastructure; using Microsoft.PythonTools.Analysis.Values; using Microsoft.PythonTools.Interpreter; +using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.Python.LanguageServer.Implementation { public sealed partial class Server { @@ -31,41 +34,77 @@ public sealed partial class Server { private static int _symbolHierarchyMaxSymbols = 1000; public override async Task WorkspaceSymbols(WorkspaceSymbolParams @params, CancellationToken cancellationToken) { - await WaitForCompleteAnalysisAsync(cancellationToken); + var symbols = await SymbolIndex.WorkspaceSymbolsAsync(@params.query, cancellationToken); + return symbols.Take(_symbolHierarchyMaxSymbols).Select(s => new SymbolInformation { + name = s.Name, + kind = (SymbolKind)s.Kind, + location = new Location { + range = s.Range, + uri = s.DocumentUri, + }, + containerName = s.ContainerName, + }).ToArray(); - var members = Enumerable.Empty(); - var opts = GetMemberOptions.ExcludeBuiltins | GetMemberOptions.DeclaredOnly; - - foreach (var entry in ProjectFiles) { - members = members.Concat( - await GetModuleVariablesAsync(entry as ProjectEntry, opts, @params.query, 50, cancellationToken) - ); - } - - members = members.GroupBy(mr => mr.Name).Select(g => g.First()); - return members.Select(ToSymbolInformation).ToArray(); + // await WaitForCompleteAnalysisAsync(cancellationToken); + // + // var members = Enumerable.Empty(); + // var opts = GetMemberOptions.ExcludeBuiltins | GetMemberOptions.DeclaredOnly; + // + // foreach (var entry in ProjectFiles) { + // members = members.Concat( + // await GetModuleVariablesAsync(entry as ProjectEntry, opts, @params.query, 50, cancellationToken) + // ); + // } + // + // members = members.GroupBy(mr => mr.Name).Select(g => g.First()); + // return members.Select(ToSymbolInformation).ToArray(); } public override async Task DocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { - await WaitForCompleteAnalysisAsync(cancellationToken); - var opts = GetMemberOptions.ExcludeBuiltins | GetMemberOptions.DeclaredOnly; - var entry = ProjectFiles.GetEntry(@params.textDocument.uri); - - var members = await GetModuleVariablesAsync(entry as ProjectEntry, opts, string.Empty, 50, cancellationToken); - return members - .GroupBy(mr => mr.Name) - .Select(g => g.First()) - .Select(ToSymbolInformation) - .ToArray(); + // var uri = @params.textDocument.uri; + // var symbols = await SymbolIndex.HierarchicalDocumentSymbolsAsync(uri, cancellationToken); + // var flat = symbols.Flatten(uri); + + throw new Exception("this function is not called"); + + // var uri = @params.textDocument.uri; + // var symbols = await SymbolIndex.WorkspaceSymbolsAsync(string.Empty, cancellationToken); + // return Array.Empty(); + + // var ast = GetCurrentAst(uri); + // var walker = new IndexerWalker(ast); + // ast.Walk(walker); + // return IndexerWalker.DocSymsToSymInfos(uri, walker.Symbols).ToArray(); + // + // await WaitForCompleteAnalysisAsync(cancellationToken); + // var opts = GetMemberOptions.ExcludeBuiltins | GetMemberOptions.DeclaredOnly; + // var entry = ProjectFiles.GetEntry(@params.textDocument.uri); + // + // var members = await GetModuleVariablesAsync(entry as ProjectEntry, opts, string.Empty, 50, cancellationToken); + // return members + // .GroupBy(mr => mr.Name) + // .Select(g => g.First()) + // .Select(ToSymbolInformation) + // .ToArray(); } public override async Task HierarchicalDocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { - await WaitForCompleteAnalysisAsync(cancellationToken); - var opts = GetMemberOptions.ExcludeBuiltins | GetMemberOptions.DeclaredOnly; - var entry = ProjectFiles.GetEntry(@params.textDocument.uri); + var uri = @params.textDocument.uri; + var symbols = await SymbolIndex.HierarchicalDocumentSymbolsAsync(uri, cancellationToken); + return ToDocumentSymbols(symbols); + + // var uri = @params.textDocument.uri; + // var ast = GetCurrentAst(uri); + // var walker = new IndexerWalker(ast); + // ast.Walk(walker); + // return walker.Symbols.ToArray(); - var members = await GetModuleVariablesAsync(entry as ProjectEntry, opts, string.Empty, 50, cancellationToken); - return ToDocumentSymbols(members); + // await WaitForCompleteAnalysisAsync(cancellationToken); + // var opts = GetMemberOptions.ExcludeBuiltins | GetMemberOptions.DeclaredOnly; + // var entry = ProjectFiles.GetEntry(@params.textDocument.uri); + // + // var members = await GetModuleVariablesAsync(entry as ProjectEntry, opts, string.Empty, 50, cancellationToken); + // return ToDocumentSymbols(members); } private static async Task> GetModuleVariablesAsync(ProjectEntry entry, GetMemberOptions opts, string prefix, int timeout, CancellationToken token) { @@ -91,6 +130,28 @@ private static IEnumerable GetModuleVariables(ProjectEntry entry, return result; } + private DocumentSymbol[] ToDocumentSymbols(IEnumerable hSymbols) { + return hSymbols.MaybeEnumerate().Select(hSym => new DocumentSymbol { + name = hSym.Name, + detail = hSym.Detail, + kind = (SymbolKind)hSym.Kind, + deprecated = hSym.Deprecated ?? false, + range = hSym.Range, + selectionRange = hSym.SelectionRange, + children = ToDocumentSymbols(hSym.Children), + }).Select(v => { + if (v.selectionRange.start < v.range.start) { + Debug.Fail("out of range"); + } + + if (v.selectionRange.end > v.range.end) { + Debug.Fail("out of range"); + } + + return v; + }).ToArray(); + } + private SymbolInformation ToSymbolInformation(IMemberResult m) { var res = new SymbolInformation { name = m.Name, diff --git a/src/LanguageServer/Impl/Implementation/Server.cs b/src/LanguageServer/Impl/Implementation/Server.cs index 8a6a1e6c0..625d19b3a 100644 --- a/src/LanguageServer/Impl/Implementation/Server.cs +++ b/src/LanguageServer/Impl/Implementation/Server.cs @@ -27,6 +27,7 @@ using Microsoft.Python.LanguageServer.Extensions; using Microsoft.PythonTools.Analysis; using Microsoft.PythonTools.Analysis.Documentation; +using Microsoft.PythonTools.Analysis.Indexing; using Microsoft.PythonTools.Analysis.Infrastructure; using Microsoft.PythonTools.Intellisense; using Microsoft.PythonTools.Interpreter; @@ -97,6 +98,9 @@ public Server() { ParseQueue = new ParseQueue(); _editorFiles = new EditorFiles(this); + SymbolIndex = new SymbolIndex(); + OnParseComplete += UpdateSymbolIndex; + _disposableBag .Add(() => { foreach (var ext in _extensions.Values) { @@ -106,6 +110,7 @@ public Server() { (ext as IDisposable)?.Dispose(); } }) + .Add(() => OnParseComplete -= UpdateSymbolIndex) .Add(ProjectFiles) .Add(() => Analyzer?.Dispose()) .Add(AnalysisQueue) @@ -120,6 +125,7 @@ private void Analysis_UnhandledException(object sender, UnhandledExceptionEventA internal AnalysisQueue AnalysisQueue { get; } internal ParseQueue ParseQueue { get; } + internal SymbolIndex SymbolIndex { get; } internal PythonAnalyzer Analyzer { get; private set; } internal ServerSettings Settings { get; private set; } = new ServerSettings(); @@ -709,5 +715,13 @@ private async Task WaitForCompleteAnalysisWorker(CancellationToken cancellationT } TraceMessage($"Analysis complete."); } + + private void UpdateSymbolIndex(object sender, ParseCompleteEventArgs args) { + ProjectFiles.GetEntry(args.uri, args.version, out var entry, out var tree); + var ast = entry.GetCurrentParse()?.Tree; + if (ast != null) { + SymbolIndex.UpdateParseTree(args.uri, ast); + } + } } } From b5497d4149f8d0f8175264430e16b6ed5379a570 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Mon, 14 Jan 2019 14:02:39 -0800 Subject: [PATCH 02/10] rework indexer, add VS-specific fields, full walker testing --- .../Engine/Impl/Indexing/IndexerWalker.cs | 245 --------- .../Engine/Impl/Indexing/SymbolExtensions.cs | 11 +- .../Engine/Impl/Indexing/SymbolIndex.cs | 4 +- .../Engine/Impl/Indexing/SymbolIndexWalker.cs | 267 +++++++++ src/Analysis/Engine/Impl/Indexing/Symbols.cs | 30 +- src/Analysis/Engine/Test/SymbolIndexTests.cs | 509 ++++++++++++++++++ .../Implementation/Server.WorkspaceSymbols.cs | 77 +-- 7 files changed, 831 insertions(+), 312 deletions(-) delete mode 100644 src/Analysis/Engine/Impl/Indexing/IndexerWalker.cs create mode 100644 src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs create mode 100644 src/Analysis/Engine/Test/SymbolIndexTests.cs diff --git a/src/Analysis/Engine/Impl/Indexing/IndexerWalker.cs b/src/Analysis/Engine/Impl/Indexing/IndexerWalker.cs deleted file mode 100644 index 0ab7e6d3c..000000000 --- a/src/Analysis/Engine/Impl/Indexing/IndexerWalker.cs +++ /dev/null @@ -1,245 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System.Collections.Generic; -using System.Linq; -using System.Text.RegularExpressions; -using Microsoft.PythonTools.Parsing.Ast; - -namespace Microsoft.PythonTools.Analysis.Indexing { - internal class IndexerWalker : PythonWalker { - private readonly PythonAst _ast; - - private readonly Stack _scope = new Stack(); - private HierarchicalSymbol Parent => _scope.Count > 0 ? _scope.Peek() : null; - - private bool _skipNext; - private bool SkipNext { - get { - var v = _skipNext; - _skipNext = false; - return v; - } - set { _skipNext = value; } - } - - public List Symbols = new List(); - - private void AddHierarchicalSymbol(HierarchicalSymbol sym) { - if (SkipNext) { - return; - } - - var parent = Parent; - - if (parent == null) { - Symbols.Add(sym); - } else { - parent.Children = parent.Children ?? new List(); - parent.Children.Add(sym); - } - } - - public IndexerWalker(PythonAst ast) { - _ast = ast; - } - - public override bool Walk(NameExpression node) { - var kind = SymbolKind.Variable; - - switch (node.Name) { - case "*": - case "_": - return base.Walk(node); - case var s when (Parent == null || Parent.Kind == SymbolKind.Class) && Regex.IsMatch(s, @"^[\p{Lu}\p{N}_]+$"): - kind = SymbolKind.Constant; - break; - } - - var span = node.GetSpan(_ast); - - AddHierarchicalSymbol(new HierarchicalSymbol { - Name = node.Name, - Kind = kind, - Range = span, - SelectionRange = span, - }); - - return base.Walk(node); - } - - public override bool Walk(ClassDefinition node) { - var ds = new HierarchicalSymbol { - Name = node.Name, - Kind = SymbolKind.Class, - Range = node.GetSpan(_ast), - SelectionRange = node.NameExpression.GetSpan(_ast), - }; - AddHierarchicalSymbol(ds); - _scope.Push(ds); - SkipNext = true; - return base.Walk(node); - } - - public override void PostWalk(ClassDefinition node) { - _scope.Pop(); - base.PostWalk(node); - } - - public override bool Walk(FunctionDefinition node) { - if (node.IsLambda) { - return base.Walk(node); - } - - var ds = new HierarchicalSymbol { - Name = node.Name, - Kind = SymbolKind.Function, - Range = node.GetSpan(_ast), - SelectionRange = node.NameExpression.GetSpan(_ast), - }; - - if (Parent?.Kind == SymbolKind.Class) { - switch (ds.Name) { - case "__init__": - ds.Kind = SymbolKind.Constructor; - break; - case var name when Regex.IsMatch(name, @"^__.*__$"): - ds.Kind = SymbolKind.Operator; - break; - default: - ds.Kind = SymbolKind.Method; - - if (node.Decorators != null) { - foreach (var dec in node.Decorators.Decorators) { - var maybeKind = DecoratorExpressionToKind(dec); - if (maybeKind.HasValue) { - ds.Kind = maybeKind.Value; - break; - } - } - } - - break; - } - } - - AddHierarchicalSymbol(ds); - _scope.Push(ds); - SkipNext = true; - return base.Walk(node); - } - - public override void PostWalk(FunctionDefinition node) { - if (node.IsLambda) { - base.PostWalk(node); - return; - } - - _scope.Pop(); - base.PostWalk(node); - } - - public override bool Walk(Parameter node) { - var span = node.GetSpan(_ast); - AddHierarchicalSymbol(new HierarchicalSymbol { - Name = node.Name, - Kind = SymbolKind.Variable, - Range = span, - SelectionRange = span, - }); - - return base.Walk(node); - } - - public override bool Walk(ImportStatement node) { - foreach (var imp in node.Names.Zip(node.AsNames, (name, asName) => (name, asName))) { - var span = imp.name.GetSpan(_ast); - AddHierarchicalSymbol(new HierarchicalSymbol { - Name = imp.name.MakeString(), - Kind = SymbolKind.Package, - Range = span, - SelectionRange = span, - }); - - if (imp.asName != null) { - span = imp.asName.GetSpan(_ast); - AddHierarchicalSymbol(new HierarchicalSymbol { - Name = imp.asName.Name, - Kind = SymbolKind.Package, - Range = span, - SelectionRange = span, - }); - } - } - - return base.Walk(node); - } - - public override bool Walk(FromImportStatement node) { - var root = node.Root; - var rootSpan = root.GetSpan(_ast); - AddHierarchicalSymbol(new HierarchicalSymbol { - Name = root.MakeString(), - Kind = SymbolKind.Package, - Range = rootSpan, - SelectionRange = rootSpan, - }); - - foreach (var imp in node.Names.Zip(node.AsNames, (name, asName) => (name, asName))) { - var span = imp.name.GetSpan(_ast); - AddHierarchicalSymbol(new HierarchicalSymbol { - Name = imp.name.Name, - Kind = SymbolKind.Package, - Range = span, - SelectionRange = span, - }); - - if (imp.asName != null) { - span = imp.asName.GetSpan(_ast); - AddHierarchicalSymbol(new HierarchicalSymbol { - Name = imp.asName.Name, - Kind = SymbolKind.Package, - Range = span, - SelectionRange = span, - }); - } - } - - return base.Walk(node); - } - - public override bool Walk(DecoratorStatement node) { - return base.Walk(node); - } - - private SymbolKind? DecoratorExpressionToKind(Expression exp) { - switch (exp) { - case NameExpression ne when - ne.Name == "property" || ne.Name == "abstractproperty" - || ne.Name == "classproperty" || ne.Name == "abstractclassproperty": - return SymbolKind.Property; - - case CallExpression ce when ce.Target is NameExpression ne: - return DecoratorExpressionToKind(ne); - - case CallExpression ce when ce.Target is MemberExpression me: - return DecoratorExpressionToKind(me.Target); - } - - return null; - } - } -} diff --git a/src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs b/src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs index 2c1651043..e37214e11 100644 --- a/src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs +++ b/src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs @@ -20,7 +20,7 @@ namespace Microsoft.PythonTools.Analysis.Indexing { internal static class SymbolExtensions { - public static IEnumerable Flatten(this IEnumerable docSyms, Uri uri, string parent = null) { + public static IEnumerable Flatten(this IEnumerable docSyms, Uri uri, string parent = null, int? depthLimit = null) { foreach (var sym in docSyms ?? Enumerable.Empty()) { yield return new FlatSymbol { Name = sym.Name, @@ -30,7 +30,14 @@ public static IEnumerable Flatten(this IEnumerable> _index = new ConcurrentDictionary>(); + private readonly ConcurrentDictionary> _index = new ConcurrentDictionary>(); public SymbolIndex() { } @@ -72,7 +72,7 @@ private IEnumerable WorkspaceSymbolsQuery(string query, Uri uri, IEn } public void UpdateParseTree(Uri uri, PythonAst ast) { - var walker = new IndexerWalker(ast); + var walker = new SymbolIndexWalker(ast); ast.Walk(walker); _index[uri] = walker.Symbols; } diff --git a/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs b/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs new file mode 100644 index 000000000..6b0397ee1 --- /dev/null +++ b/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs @@ -0,0 +1,267 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using Microsoft.PythonTools.Analysis.Infrastructure; +using Microsoft.PythonTools.Parsing.Ast; + +namespace Microsoft.PythonTools.Analysis.Indexing { + internal class SymbolIndexWalker : PythonWalker { + private static readonly Regex DoubleUnderscore = new Regex(@"^__.*__$", RegexOptions.Compiled); + private static readonly Regex ConstantLike = new Regex(@"^[\p{Lu}\p{N}_]+$", RegexOptions.Compiled); + + private readonly PythonAst _ast; + private readonly SymbolStack _stack = new SymbolStack(); + + public SymbolIndexWalker(PythonAst ast) { + _ast = ast; + } + + public IReadOnlyList Symbols => _stack.Root; + + public override bool Walk(ClassDefinition node) { + _stack.Enter(SymbolKind.Class); + node.Body?.Walk(this); + var children = _stack.Exit(); + + _stack.AddSymbol(new HierarchicalSymbol( + node.Name, + SymbolKind.Class, + node.GetSpan(_ast), + node.NameExpression.GetSpan(_ast), + children, + FunctionKind.Class + )); + + return false; + } + + public override bool Walk(FunctionDefinition node) { + _stack.Enter(SymbolKind.Function); + foreach (var p in node.Parameters) { + AddVarSymbol(p.NameExpression); + } + node.Body?.Walk(this); + var children = _stack.Exit(); + + var span = node.GetSpan(_ast); + + var ds = new HierarchicalSymbol( + node.Name, + SymbolKind.Function, + span, + node.IsLambda ? span : node.NameExpression.GetSpan(_ast), + children, + FunctionKind.Function + ); + + if (_stack.Parent == SymbolKind.Class) { + switch (ds.Name) { + case "__init__": + ds.Kind = SymbolKind.Constructor; + break; + case var name when DoubleUnderscore.IsMatch(name): + ds.Kind = SymbolKind.Operator; + break; + default: + ds.Kind = SymbolKind.Method; + + if (node.Decorators != null) { + foreach (var dec in node.Decorators.Decorators) { + var maybeKind = DecoratorExpressionToKind(dec); + if (maybeKind.HasValue) { + ds.Kind = maybeKind.Value.Item1; + ds._functionKind = maybeKind.Value.Item2; + break; + } + } + } + + break; + } + } + + _stack.AddSymbol(ds); + + return false; + } + + public override bool Walk(ImportStatement node) { + foreach (var imp in node.Names.Zip(node.AsNames, (name, asName) => (name, asName))) { + if (imp.asName == null) { + var span = imp.name.GetSpan(_ast); + _stack.AddSymbol(new HierarchicalSymbol(imp.name.MakeString(), SymbolKind.Module, span)); + } else { + var span = imp.asName.GetSpan(_ast); + _stack.AddSymbol(new HierarchicalSymbol(imp.asName.Name, SymbolKind.Module, span)); + } + } + + return false; + } + + public override bool Walk(FromImportStatement node) { + foreach (var imp in node.Names.Zip(node.AsNames, (name, asName) => (name, asName))) { + if (imp.asName == null) { + var span = imp.name.GetSpan(_ast); + _stack.AddSymbol(new HierarchicalSymbol(imp.name.Name, SymbolKind.Module, span)); + } else { + var span = imp.asName.GetSpan(_ast); + _stack.AddSymbol(new HierarchicalSymbol(imp.asName.Name, SymbolKind.Module, span)); + } + } + + return false; + } + + public override bool Walk(AssignmentStatement node) { + foreach (var ne in node.Left.OfType()) { + AddVarSymbol(ne); + } + node.Right?.Walk(this); + return false; + } + + public override bool Walk(AugmentedAssignStatement node) { + AddVarSymbol(node.Left as NameExpression); + return false; + } + + public override bool Walk(IfStatement node) { + WalkAndDeclareAll(node.Tests); + WalkAndDeclare(node.ElseStatement); + + return false; + } + + public override bool Walk(TryStatement node) { + WalkAndDeclare(node.Body); + WalkAndDeclareAll(node.Handlers); + WalkAndDeclare(node.Else); + WalkAndDeclare(node.Finally); + + return false; + } + + private void AddVarSymbol(NameExpression node) { + if (node == null) { + return; + } + + var kind = SymbolKind.Variable; + + switch (node.Name) { + case "*": + case "_": + return; + case var s when (_stack.Parent == null || _stack.Parent == SymbolKind.Class) && ConstantLike.IsMatch(s): + kind = SymbolKind.Constant; + break; + } + + var span = node.GetSpan(_ast); + + _stack.AddSymbol(new HierarchicalSymbol(node.Name, kind, span)); + } + + private (SymbolKind, string)? DecoratorExpressionToKind(Expression exp) { + switch (exp) { + case NameExpression ne when NameIsProperty(ne.Name): + case MemberExpression me when NameIsProperty(me.Name): + return (SymbolKind.Property, FunctionKind.Property); + + case NameExpression ne when NameIsStaticMethod(ne.Name): + case MemberExpression me when NameIsStaticMethod(me.Name): + return (SymbolKind.Method, FunctionKind.StaticMethod); + + case NameExpression ne when NameIsClassMethod(ne.Name): + case MemberExpression me when NameIsClassMethod(me.Name): + return (SymbolKind.Method, FunctionKind.ClassMethod); + } + + return null; + } + + private bool NameIsProperty(string name) => + name == "property" + || name == "abstractproperty" + || name == "classproperty" + || name == "abstractclassproperty"; + + private bool NameIsStaticMethod(string name) => + name == "staticmethod" + || name == "abstractstaticmethod"; + + private bool NameIsClassMethod(string name) => + name == "classmethod" + || name == "abstractclassmethod"; + + private void WalkAndDeclare(Node node) { + if (node == null) { + return; + } + + _stack.EnterDeclared(); + node.Walk(this); + _stack.ExitDeclaredAndMerge(); + } + + private void WalkAndDeclareAll(IEnumerable nodes) { + foreach (var node in nodes.MaybeEnumerate()) { + WalkAndDeclare(node); + } + } + + private class SymbolStack { + private readonly Stack<(SymbolKind?, List)> _symbols; + private readonly Stack> _declared = new Stack>(new[] { new HashSet() }); + + public List Root { get; } = new List(); + + public SymbolKind? Parent => _symbols.Peek().Item1; + + public SymbolStack() { + _symbols = new Stack<(SymbolKind?, List)>(new (SymbolKind?, List)[] { (null, Root) }); + } + + public void Enter(SymbolKind parent) { + _symbols.Push((parent, new List())); + EnterDeclared(); + } + + public List Exit() { + ExitDeclared(); + return _symbols.Pop().Item2; + } + public void EnterDeclared() => _declared.Push(new HashSet()); + + public void ExitDeclared() => _declared.Pop(); + + public void ExitDeclaredAndMerge() => _declared.Peek().UnionWith(_declared.Pop()); + + public void AddSymbol(HierarchicalSymbol sym) { + if (sym.Kind == SymbolKind.Variable && _declared.Peek().Contains(sym.Name)) { + return; + } + + _symbols.Peek().Item2.Add(sym); + _declared.Peek().Add(sym.Name); + } + } + } +} diff --git a/src/Analysis/Engine/Impl/Indexing/Symbols.cs b/src/Analysis/Engine/Impl/Indexing/Symbols.cs index 106436e52..6417712dc 100644 --- a/src/Analysis/Engine/Impl/Indexing/Symbols.cs +++ b/src/Analysis/Engine/Impl/Indexing/Symbols.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Text; namespace Microsoft.PythonTools.Analysis.Indexing { // From LSP. @@ -34,6 +33,15 @@ internal enum SymbolKind { TypeParameter = 26 } + internal class FunctionKind { + public const string None = ""; + public const string Function = "function"; + public const string Property = "property"; + public const string StaticMethod = "staticmethod"; + public const string ClassMethod = "classmethod"; + public const string Class = "class"; + } + // Analagous to LSP's DocumentSymbol. internal class HierarchicalSymbol { public string Name; @@ -42,7 +50,25 @@ internal class HierarchicalSymbol { public bool? Deprecated; public SourceSpan Range; public SourceSpan SelectionRange; - public List Children; + public IList Children; + + public string _functionKind; + + public HierarchicalSymbol( + string name, + SymbolKind kind, + SourceSpan range, + SourceSpan? selectionRange = null, + IList children = null, + string functionKind = FunctionKind.None + ) { + Name = name; + Kind = kind; + Range = range; + SelectionRange = selectionRange ?? range; + Children = children; + _functionKind = functionKind; + } } // Analagous to LSP's SymbolInformation. diff --git a/src/Analysis/Engine/Test/SymbolIndexTests.cs b/src/Analysis/Engine/Test/SymbolIndexTests.cs new file mode 100644 index 000000000..bf17feb5b --- /dev/null +++ b/src/Analysis/Engine/Test/SymbolIndexTests.cs @@ -0,0 +1,509 @@ +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABLITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using FluentAssertions; +using Microsoft.PythonTools; +using Microsoft.PythonTools.Analysis.FluentAssertions; +using Microsoft.PythonTools.Analysis.Indexing; +using Microsoft.PythonTools.Parsing; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using TestUtilities; + +namespace AnalysisTests { + [TestClass] + public class SymbolIndexTests { + public TestContext TestContext { get; set; } + + [TestInitialize] + public void TestInitialize() { + TestEnvironmentImpl.TestInitialize($"{TestContext.FullyQualifiedTestClassName}.{TestContext.TestName}"); + } + + [TestCleanup] + public void TestCleanup() { + TestEnvironmentImpl.TestCleanup(); + } + + [TestMethod, Priority(0)] + public void WalkerAssignments() { + var code = @"x = 1 +y = x +z = y"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 1, 2, 2)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(3, 1, 3, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerMultipleAssignments() { + var code = @"x = y = z = 1"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 5, 1, 6)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(1, 9, 1, 10)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerUnderscore() { + var code = @"_ = 1"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public void WalkerIfStatement() { + var code = @"if foo(): + x = 1 +elif bar(): + x = 2 +else: + y = 3 +"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(4, 5, 4, 6)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(6, 5, 6, 6)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerTryExceptFinally() { + var code = @"try: + x = 1 +except Exception: + x = 2 +finally: + y = 3 +"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(4, 5, 4, 6)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(6, 5, 6, 6)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerReassign() { + var code = @"x = 1 +x = 2"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerAugmentedAssign() { + var code = @"x += 1"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerTopLevelConstant() { + var code = @"FOO_BAR_3 = 1234"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("FOO_BAR_3", SymbolKind.Constant, new SourceSpan(1, 1, 1, 10)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerFunction() { + var code = @"def func(x, y): + z = x + y + return z"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 3, 13), new SourceSpan(1, 5, 1, 9), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 10, 1, 11)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerFunctionStarredArgs() { + var code = @"def func(*args, **kwargs): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 31), new SourceSpan(1, 5, 1, 9), new[] { + new HierarchicalSymbol("args", SymbolKind.Variable, new SourceSpan(1, 11, 1, 15)), + new HierarchicalSymbol("kwargs", SymbolKind.Variable, new SourceSpan(1, 19, 1, 25)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerFunctionUnderscoreArg() { + var code = @"def func(_): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 17), new SourceSpan(1, 5, 1, 9), new List(), FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerImports() { + var code = @"import sys +import numpy as np +from os.path import join as osjoin +from os.path import ( join as osjoin2, exists as osexists, expanduser ) +"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("sys", SymbolKind.Module, new SourceSpan(1, 8, 1, 11)), + new HierarchicalSymbol("np", SymbolKind.Module, new SourceSpan(2, 17, 2, 19)), + new HierarchicalSymbol("osjoin", SymbolKind.Module, new SourceSpan(3, 29, 3, 35)), + new HierarchicalSymbol("osjoin2", SymbolKind.Module, new SourceSpan(4, 31, 4, 38)), + new HierarchicalSymbol("osexists", SymbolKind.Module, new SourceSpan(4, 50, 4, 58)), + new HierarchicalSymbol("expanduser", SymbolKind.Module, new SourceSpan(4, 60, 4, 70)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClass() { + var code = @"class Foo(object): + ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 8), new SourceSpan(1, 7, 1, 10), new List(), FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassConstant() { + var code = @"class Foo(object): + CONSTANT = 1234"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 20), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("CONSTANT", SymbolKind.Constant, new SourceSpan(2, 5, 2, 13)), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerConstructor() { + var code = @"class Foo(object): + def __init__(self, x): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 31), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("__init__", SymbolKind.Constructor, new SourceSpan(2, 5, 2, 31), new SourceSpan(2, 9, 2, 17), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 18, 2, 22)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 24, 2, 25)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerMethod() { + var code = @"class Foo(object): + def foo(self, x): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 26), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("foo", SymbolKind.Method, new SourceSpan(2, 5, 2, 26), new SourceSpan(2, 9, 2, 12), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 13, 2, 17)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 19, 2, 20)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerDoubleUnderscoreMethod() { + var code = @"class Foo(object): + def __lt__(self, x): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 29), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("__lt__", SymbolKind.Operator, new SourceSpan(2, 5, 2, 29), new SourceSpan(2, 9, 2, 15), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 16, 2, 20)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 22, 2, 23)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerProperties() { + var code = @"class Foo(object): + @property + def func1(self): ... + + @abstractproperty + def func2(self): ... + + @classproperty + def func3(self): ... + + @abstractclassproperty + def func4(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 12, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func2", SymbolKind.Property, new SourceSpan(5, 5, 6, 25), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(6, 15, 6, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func3", SymbolKind.Property, new SourceSpan(8, 5, 9, 25), new SourceSpan(9, 9, 9, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(9, 15, 9, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func4", SymbolKind.Property, new SourceSpan(11, 5, 12, 25), new SourceSpan(12, 9, 12, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(12, 15, 12, 19)), + }, FunctionKind.Property), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerAbcProperties() { + var code = @"class Foo(object): + @abc.abstractproperty + def func1(self): ... + + @abc.abstractclassproperty + def func2(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 6, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func2", SymbolKind.Property, new SourceSpan(5, 5, 6, 25), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(6, 15, 6, 19)), + }, FunctionKind.Property), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerStaticMethods() { + var code = @"class Foo(object): + @staticmethod + def func1(arg): ... + + @abstractstaticmethod + def func2(arg): ... + + @abc.abstractstaticmethod + def func3(arg): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 9, 24), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 24), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("arg", SymbolKind.Variable, new SourceSpan(3, 15, 3, 18)), + }, FunctionKind.StaticMethod), + new HierarchicalSymbol("func2", SymbolKind.Method, new SourceSpan(5, 5, 6, 24), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("arg", SymbolKind.Variable, new SourceSpan(6, 15, 6, 18)), + }, FunctionKind.StaticMethod), + new HierarchicalSymbol("func3", SymbolKind.Method, new SourceSpan(8, 5, 9, 24), new SourceSpan(9, 9, 9, 14), new[] { + new HierarchicalSymbol("arg", SymbolKind.Variable, new SourceSpan(9, 15, 9, 18)), + }, FunctionKind.StaticMethod), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassMethods() { + var code = @"class Foo(object): + @classmethod + def func1(cls): ... + + @abstractclassmethod + def func2(cls): ... + + @abc.abstractclassmethod + def func3(cls): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 9, 24), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 24), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("cls", SymbolKind.Variable, new SourceSpan(3, 15, 3, 18)), + }, FunctionKind.ClassMethod), + new HierarchicalSymbol("func2", SymbolKind.Method, new SourceSpan(5, 5, 6, 24), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("cls", SymbolKind.Variable, new SourceSpan(6, 15, 6, 18)), + }, FunctionKind.ClassMethod), + new HierarchicalSymbol("func3", SymbolKind.Method, new SourceSpan(8, 5, 9, 24), new SourceSpan(9, 9, 9, 14), new[] { + new HierarchicalSymbol("cls", SymbolKind.Variable, new SourceSpan(9, 15, 9, 18)), + }, FunctionKind.ClassMethod), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerTopLevelFunctionDecorator() { + var code = @"@something +def func1(x, y): ... + +@something_else() +def func2(x, y): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("func1", SymbolKind.Function, new SourceSpan(1, 1, 2, 21), new SourceSpan(2, 5, 2, 10), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 11, 2, 12)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 14, 2, 15)), + }, FunctionKind.Function), + new HierarchicalSymbol("func2", SymbolKind.Function, new SourceSpan(4, 1, 5, 21), new SourceSpan(5, 5, 5, 10), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(5, 11, 5, 12)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(5, 14, 5, 15)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassFunctionDecorator() { + var code = @"class Foo(object): + @something + def func1(self): ... + + @something_else() + def func2(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 6, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), + }, FunctionKind.Function), + new HierarchicalSymbol("func2", SymbolKind.Method, new SourceSpan(5, 5, 6, 25), new SourceSpan(6, 9, 6, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(6, 15, 6, 19)), + }, FunctionKind.Function), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerClassFunctionMultiDecorator() { + var code = @"class Foo(object): + @property + @something + def func1(self): ... + + @something + @property + def func2(self): ..."; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 8, 25), new SourceSpan(1, 7, 1, 10), new[] { + new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 4, 25), new SourceSpan(4, 9, 4, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(4, 15, 4, 19)), + }, FunctionKind.Property), + new HierarchicalSymbol("func2", SymbolKind.Property, new SourceSpan(6, 5, 8, 25), new SourceSpan(8, 9, 8, 14), new[] { + new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(8, 15, 8, 19)), + }, FunctionKind.Property), + }, FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerLambda() { + var code = @"f = lambda x, y: x + y"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("f", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + new HierarchicalSymbol("", SymbolKind.Function, new SourceSpan(1, 5, 1, 23), new SourceSpan(1, 5, 1, 23), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 12, 1, 13)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 15, 1, 16)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerIncompleteFunction() { + var code = @"def func(x, y):"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 16), new SourceSpan(1, 5, 1, 9), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 10, 1, 11)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + }, FunctionKind.Function), + }); + } + + [TestMethod, Priority(0)] + public void WalkerIncompleteClass() { + var code = @"class Foo(object):"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 1, 19), new SourceSpan(1, 7, 1, 10), new List(), FunctionKind.Class), + }); + } + + [TestMethod, Priority(0)] + public void WalkerIncompleteAssycn() { + var code = @"x ="; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + private IReadOnlyList WalkSymbols(string code, PythonLanguageVersion version = PythonLanguageVersion.V37) { + using (var reader = new StringReader(code)) { + var parser = Parser.CreateParser(reader, PythonLanguageVersion.V37); + var ast = parser.ParseFile(); + var walker = new SymbolIndexWalker(ast); + ast.Walk(walker); + return walker.Symbols; + } + } + } +} diff --git a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs index 26f9f1e0a..b3a4f6208 100644 --- a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs +++ b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs @@ -26,7 +26,6 @@ using Microsoft.PythonTools.Analysis.Infrastructure; using Microsoft.PythonTools.Analysis.Values; using Microsoft.PythonTools.Interpreter; -using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.Python.LanguageServer.Implementation { public sealed partial class Server { @@ -44,67 +43,26 @@ public override async Task WorkspaceSymbols(WorkspaceSymbol }, containerName = s.ContainerName, }).ToArray(); - - // await WaitForCompleteAnalysisAsync(cancellationToken); - // - // var members = Enumerable.Empty(); - // var opts = GetMemberOptions.ExcludeBuiltins | GetMemberOptions.DeclaredOnly; - // - // foreach (var entry in ProjectFiles) { - // members = members.Concat( - // await GetModuleVariablesAsync(entry as ProjectEntry, opts, @params.query, 50, cancellationToken) - // ); - // } - // - // members = members.GroupBy(mr => mr.Name).Select(g => g.First()); - // return members.Select(ToSymbolInformation).ToArray(); } public override async Task DocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { - // var uri = @params.textDocument.uri; - // var symbols = await SymbolIndex.HierarchicalDocumentSymbolsAsync(uri, cancellationToken); - // var flat = symbols.Flatten(uri); - - throw new Exception("this function is not called"); - - // var uri = @params.textDocument.uri; - // var symbols = await SymbolIndex.WorkspaceSymbolsAsync(string.Empty, cancellationToken); - // return Array.Empty(); - - // var ast = GetCurrentAst(uri); - // var walker = new IndexerWalker(ast); - // ast.Walk(walker); - // return IndexerWalker.DocSymsToSymInfos(uri, walker.Symbols).ToArray(); - // - // await WaitForCompleteAnalysisAsync(cancellationToken); - // var opts = GetMemberOptions.ExcludeBuiltins | GetMemberOptions.DeclaredOnly; - // var entry = ProjectFiles.GetEntry(@params.textDocument.uri); - // - // var members = await GetModuleVariablesAsync(entry as ProjectEntry, opts, string.Empty, 50, cancellationToken); - // return members - // .GroupBy(mr => mr.Name) - // .Select(g => g.First()) - // .Select(ToSymbolInformation) - // .ToArray(); + var uri = @params.textDocument.uri; + var symbols = await SymbolIndex.HierarchicalDocumentSymbolsAsync(uri, cancellationToken); + return symbols.Flatten(uri, depthLimit: _symbolHierarchyDepthLimit).Take(_symbolHierarchyMaxSymbols).Select(s => new SymbolInformation { + name = s.Name, + kind = (SymbolKind)s.Kind, + location = new Location { + range = s.Range, + uri = s.DocumentUri, + }, + containerName = s.ContainerName, + }).ToArray(); } public override async Task HierarchicalDocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { var uri = @params.textDocument.uri; var symbols = await SymbolIndex.HierarchicalDocumentSymbolsAsync(uri, cancellationToken); return ToDocumentSymbols(symbols); - - // var uri = @params.textDocument.uri; - // var ast = GetCurrentAst(uri); - // var walker = new IndexerWalker(ast); - // ast.Walk(walker); - // return walker.Symbols.ToArray(); - - // await WaitForCompleteAnalysisAsync(cancellationToken); - // var opts = GetMemberOptions.ExcludeBuiltins | GetMemberOptions.DeclaredOnly; - // var entry = ProjectFiles.GetEntry(@params.textDocument.uri); - // - // var members = await GetModuleVariablesAsync(entry as ProjectEntry, opts, string.Empty, 50, cancellationToken); - // return ToDocumentSymbols(members); } private static async Task> GetModuleVariablesAsync(ProjectEntry entry, GetMemberOptions opts, string prefix, int timeout, CancellationToken token) { @@ -117,7 +75,7 @@ private static IEnumerable GetModuleVariables(ProjectEntry entry, var all = breadthFirst.SelectMany(c => analysis.GetAllAvailableMembersFromScope(c, opts)); var result = all .Where(m => { - if (m.Values.Any(v => v.DeclaringModule == entry || + if (m.Values.Any(v => v.DeclaringModule == entry || v.Locations .MaybeEnumerate() .ExcludeDefault() @@ -139,15 +97,12 @@ private DocumentSymbol[] ToDocumentSymbols(IEnumerable hSymb range = hSym.Range, selectionRange = hSym.SelectionRange, children = ToDocumentSymbols(hSym.Children), + _functionKind = hSym._functionKind, }).Select(v => { - if (v.selectionRange.start < v.range.start) { - Debug.Fail("out of range"); - } - - if (v.selectionRange.end > v.range.end) { - Debug.Fail("out of range"); + if (v.selectionRange.start < v.range.start || v.selectionRange.end > v.range.end) { + Debug.Fail($"selectionRange {v.selectionRange} not encompassed by range {v.range}"); + v.selectionRange = v.range; } - return v; }).ToArray(); } From eba0c1f837db4f1863d99372b388fe6ca0127ee3 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Mon, 14 Jan 2019 15:20:55 -0800 Subject: [PATCH 03/10] basic indexer testing --- .../Engine/Impl/Indexing/SymbolExtensions.cs | 8 +------- src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs | 10 ++-------- src/Analysis/Engine/Impl/Indexing/Symbols.cs | 14 ++++++++++++++ 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs b/src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs index e37214e11..8caa4c744 100644 --- a/src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs +++ b/src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs @@ -22,13 +22,7 @@ namespace Microsoft.PythonTools.Analysis.Indexing { internal static class SymbolExtensions { public static IEnumerable Flatten(this IEnumerable docSyms, Uri uri, string parent = null, int? depthLimit = null) { foreach (var sym in docSyms ?? Enumerable.Empty()) { - yield return new FlatSymbol { - Name = sym.Name, - Kind = sym.Kind, - DocumentUri = uri, - Range = sym.SelectionRange, - ContainerName = parent, - }; + yield return new FlatSymbol(sym.Name, sym.Kind, uri, sym.SelectionRange, parent); if (depthLimit != null) { if (depthLimit < 1) { diff --git a/src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs b/src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs index cb503b673..4ea31472f 100644 --- a/src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs +++ b/src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs @@ -30,7 +30,7 @@ internal class SymbolIndex : ISymbolIndex { public SymbolIndex() { } public async Task> HierarchicalDocumentSymbolsAsync(Uri uri, CancellationToken token = default) { - return _index.TryGetValue(uri, out var list) ? list : null; + return _index.TryGetValue(uri, out var list) ? list : Enumerable.Empty(); } public Task> WorkspaceSymbolsAsync(string query, CancellationToken token = default) { @@ -56,13 +56,7 @@ private IEnumerable WorkspaceSymbolsQuery(string query, Uri uri, IEn ll.RemoveFirst(); if (sym.Name.Contains(query)) { - yield return new FlatSymbol { - Name = sym.Name, - Kind = sym.Kind, - DocumentUri = uri, - Range = sym.SelectionRange, - ContainerName = parent, - }; + yield return new FlatSymbol(sym.Name, sym.Kind, uri, sym.SelectionRange, parent); } foreach (var child in sym.Children.MaybeEnumerate()) { diff --git a/src/Analysis/Engine/Impl/Indexing/Symbols.cs b/src/Analysis/Engine/Impl/Indexing/Symbols.cs index 6417712dc..654766846 100644 --- a/src/Analysis/Engine/Impl/Indexing/Symbols.cs +++ b/src/Analysis/Engine/Impl/Indexing/Symbols.cs @@ -79,5 +79,19 @@ internal class FlatSymbol { public Uri DocumentUri; public SourceSpan Range; public string ContainerName; + + public FlatSymbol( + string name, + SymbolKind kind, + Uri documentUri, + SourceSpan range, + string containerName = null + ) { + Name = name; + Kind = kind; + DocumentUri = documentUri; + Range = range; + ContainerName = containerName; + } } } From 9b3b79e9f21f539c22eec561746f3bdbb8e06b26 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Mon, 14 Jan 2019 16:46:41 -0800 Subject: [PATCH 04/10] drop async, case insensitive match --- .../Engine/Impl/Indexing/ISymbolIndex.cs | 6 +- .../Engine/Impl/Indexing/SymbolIndex.cs | 20 +-- .../Extensions/StringExtensions.cs | 3 + src/Analysis/Engine/Test/SymbolIndexTests.cs | 121 ++++++++++++++++-- .../Implementation/Server.WorkspaceSymbols.cs | 64 +++++---- 5 files changed, 163 insertions(+), 51 deletions(-) diff --git a/src/Analysis/Engine/Impl/Indexing/ISymbolIndex.cs b/src/Analysis/Engine/Impl/Indexing/ISymbolIndex.cs index 37c06254c..fad8f742c 100644 --- a/src/Analysis/Engine/Impl/Indexing/ISymbolIndex.cs +++ b/src/Analysis/Engine/Impl/Indexing/ISymbolIndex.cs @@ -16,13 +16,11 @@ using System; using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; namespace Microsoft.PythonTools.Analysis.Indexing { internal interface ISymbolIndex { - Task> HierarchicalDocumentSymbolsAsync(Uri uri, CancellationToken token = default); + IEnumerable HierarchicalDocumentSymbols(Uri uri); - Task> WorkspaceSymbolsAsync(string query, CancellationToken token = default); + IEnumerable WorkspaceSymbols(string query); } } diff --git a/src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs b/src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs index 4ea31472f..d59742d24 100644 --- a/src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs +++ b/src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs @@ -18,8 +18,6 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; -using System.Threading; -using System.Threading.Tasks; using Microsoft.PythonTools.Analysis.Infrastructure; using Microsoft.PythonTools.Parsing.Ast; @@ -29,33 +27,27 @@ internal class SymbolIndex : ISymbolIndex { public SymbolIndex() { } - public async Task> HierarchicalDocumentSymbolsAsync(Uri uri, CancellationToken token = default) { + public IEnumerable HierarchicalDocumentSymbols(Uri uri) { return _index.TryGetValue(uri, out var list) ? list : Enumerable.Empty(); } - public Task> WorkspaceSymbolsAsync(string query, CancellationToken token = default) { - return Task.FromResult(WorkspaceSymbols(query)); - } - - private IEnumerable WorkspaceSymbols(string query, CancellationToken token = default) { + public IEnumerable WorkspaceSymbols(string query) { foreach (var kvp in _index) { - foreach (var found in WorkspaceSymbolsQuery(query, kvp.Key, kvp.Value, token: token)) { + foreach (var found in WorkspaceSymbolsQuery(query, kvp.Key, kvp.Value)) { yield return found; } } } - private IEnumerable WorkspaceSymbolsQuery(string query, Uri uri, IEnumerable symbols, CancellationToken token = default) { + private IEnumerable WorkspaceSymbolsQuery(string query, Uri uri, IEnumerable symbols) { // Some semblance of a BFS. var ll = new LinkedList<(HierarchicalSymbol, string)>(symbols.Select(s => (s, (string)null))); - - while (ll.First != null) { - token.ThrowIfCancellationRequested(); + while (ll.First != null) { var (sym, parent) = ll.First.Value; ll.RemoveFirst(); - if (sym.Name.Contains(query)) { + if (sym.Name.ContainsOrdinal(query, ignoreCase: true)) { yield return new FlatSymbol(sym.Name, sym.Kind, uri, sym.SelectionRange, parent); } diff --git a/src/Analysis/Engine/Impl/Infrastructure/Extensions/StringExtensions.cs b/src/Analysis/Engine/Impl/Infrastructure/Extensions/StringExtensions.cs index bd3ca29a9..fb6b871a1 100644 --- a/src/Analysis/Engine/Impl/Infrastructure/Extensions/StringExtensions.cs +++ b/src/Analysis/Engine/Impl/Infrastructure/Extensions/StringExtensions.cs @@ -201,6 +201,9 @@ public static bool EqualsOrdinal(this string s, string other, bool ignoreCase) public static bool EqualsOrdinal(this string s, int index, string other, int otherIndex, int length, bool ignoreCase = false) => string.Compare(s, index, other, otherIndex, length, ignoreCase ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal) == 0; + public static bool ContainsOrdinal(this string s, string value, bool ignoreCase = false) + => s.IndexOfOrdinal(value, ignoreCase: ignoreCase) != -1; + public static string[] Split(this string s, char separator, int startIndex, int length) { var count = 0; var endIndex = startIndex + length; diff --git a/src/Analysis/Engine/Test/SymbolIndexTests.cs b/src/Analysis/Engine/Test/SymbolIndexTests.cs index bf17feb5b..39a94c99b 100644 --- a/src/Analysis/Engine/Test/SymbolIndexTests.cs +++ b/src/Analysis/Engine/Test/SymbolIndexTests.cs @@ -12,15 +12,14 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; using System.IO; -using System.Linq; using FluentAssertions; using Microsoft.PythonTools; using Microsoft.PythonTools.Analysis.FluentAssertions; using Microsoft.PythonTools.Analysis.Indexing; using Microsoft.PythonTools.Parsing; +using Microsoft.PythonTools.Parsing.Ast; using Microsoft.VisualStudio.TestTools.UnitTesting; using TestUtilities; @@ -496,14 +495,118 @@ public void WalkerIncompleteAssycn() { }); } + [TestMethod, Priority(0)] + public void IndexHierarchicalDocument() { + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + var ast = GetParse("x = 1"); + index.UpdateParseTree(uri, ast); + + var symbols = index.HierarchicalDocumentSymbols(uri); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void IndexHierarchicalDocumentUpdate() { + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + var ast = GetParse("x = 1"); + index.UpdateParseTree(uri, ast); + + var symbols = index.HierarchicalDocumentSymbols(uri); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + + ast = GetParse("y = 1"); + index.UpdateParseTree(uri, ast); + + symbols = index.HierarchicalDocumentSymbols(uri); + symbols.Should().BeEquivalentTo(new[] { + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void IndexHierarchicalDocumentNotFound() { + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + + var symbols = index.HierarchicalDocumentSymbols(uri); + symbols.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public void IndexWorkspaceSymbolsFlatten() { + var code = @"class Foo(object): + def foo(self, x): ..."; + + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + var ast = GetParse(code); + index.UpdateParseTree(uri, ast); + + var symbols = index.WorkspaceSymbols(""); + symbols.Should().BeEquivalentTo(new[] { + new FlatSymbol("Foo", SymbolKind.Class, uri, new SourceSpan(1, 7, 1, 10)), + new FlatSymbol("foo", SymbolKind.Method, uri, new SourceSpan(2, 9, 2, 12), "Foo"), + new FlatSymbol("self", SymbolKind.Variable, uri, new SourceSpan(2, 13, 2, 17), "foo"), + new FlatSymbol("x", SymbolKind.Variable, uri, new SourceSpan(2, 19, 2, 20), "foo"), + }); + } + + [TestMethod, Priority(0)] + public void IndexWorkspaceSymbolsFiltered() { + var code = @"class Foo(object): + def foo(self, x): ..."; + + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + var ast = GetParse(code); + index.UpdateParseTree(uri, ast); + + var symbols = index.WorkspaceSymbols("x"); + symbols.Should().BeEquivalentTo(new[] { + new FlatSymbol("x", SymbolKind.Variable, uri, new SourceSpan(2, 19, 2, 20), "foo"), + }); + } + + [TestMethod, Priority(0)] + public void IndexWorkspaceSymbolsNotFound() { + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + + var symbols = index.WorkspaceSymbols(""); + symbols.Should().BeEmpty(); + } + + [TestMethod, Priority(0)] + public void IndexWorkspaceSymbolsCaseInsensitive() { + var code = @"class Foo(object): + def foo(self, x): ..."; + + var index = new SymbolIndex(); + var uri = TestData.GetDefaultModuleUri(); + var ast = GetParse(code); + index.UpdateParseTree(uri, ast); + + var symbols = index.WorkspaceSymbols("foo"); + symbols.Should().BeEquivalentTo(new[] { + new FlatSymbol("Foo", SymbolKind.Class, uri, new SourceSpan(1, 7, 1, 10)), + new FlatSymbol("foo", SymbolKind.Method, uri, new SourceSpan(2, 9, 2, 12), "Foo"), + }); + } + + private PythonAst GetParse(string code, PythonLanguageVersion version = PythonLanguageVersion.V37) + => Parser.CreateParser(new StringReader(code), version).ParseFile(); + private IReadOnlyList WalkSymbols(string code, PythonLanguageVersion version = PythonLanguageVersion.V37) { - using (var reader = new StringReader(code)) { - var parser = Parser.CreateParser(reader, PythonLanguageVersion.V37); - var ast = parser.ParseFile(); - var walker = new SymbolIndexWalker(ast); - ast.Walk(walker); - return walker.Symbols; - } + var ast = GetParse(code); + var walker = new SymbolIndexWalker(ast); + ast.Walk(walker); + return walker.Symbols; } } } diff --git a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs index b3a4f6208..aef505e54 100644 --- a/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs +++ b/src/LanguageServer/Impl/Implementation/Server.WorkspaceSymbols.cs @@ -32,37 +32,50 @@ public sealed partial class Server { private static int _symbolHierarchyDepthLimit = 10; private static int _symbolHierarchyMaxSymbols = 1000; - public override async Task WorkspaceSymbols(WorkspaceSymbolParams @params, CancellationToken cancellationToken) { - var symbols = await SymbolIndex.WorkspaceSymbolsAsync(@params.query, cancellationToken); - return symbols.Take(_symbolHierarchyMaxSymbols).Select(s => new SymbolInformation { - name = s.Name, - kind = (SymbolKind)s.Kind, - location = new Location { - range = s.Range, - uri = s.DocumentUri, - }, - containerName = s.ContainerName, + public override Task WorkspaceSymbols(WorkspaceSymbolParams @params, CancellationToken cancellationToken) { + var symbols = SymbolIndex.WorkspaceSymbols(@params.query); + + var result = symbols.Take(_symbolHierarchyMaxSymbols).Select(s => { + cancellationToken.ThrowIfCancellationRequested(); + return new SymbolInformation { + name = s.Name, + kind = (SymbolKind)s.Kind, + location = new Location { + range = s.Range, + uri = s.DocumentUri, + }, + containerName = s.ContainerName, + }; }).ToArray(); + + return Task.FromResult(result); } - public override async Task DocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { + public override Task DocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { var uri = @params.textDocument.uri; - var symbols = await SymbolIndex.HierarchicalDocumentSymbolsAsync(uri, cancellationToken); - return symbols.Flatten(uri, depthLimit: _symbolHierarchyDepthLimit).Take(_symbolHierarchyMaxSymbols).Select(s => new SymbolInformation { - name = s.Name, - kind = (SymbolKind)s.Kind, - location = new Location { - range = s.Range, - uri = s.DocumentUri, - }, - containerName = s.ContainerName, + var symbols = SymbolIndex.HierarchicalDocumentSymbols(uri); + + var result = symbols.Flatten(uri, depthLimit: _symbolHierarchyDepthLimit).Take(_symbolHierarchyMaxSymbols).Select(s => { + cancellationToken.ThrowIfCancellationRequested(); + return new SymbolInformation { + name = s.Name, + kind = (SymbolKind)s.Kind, + location = new Location { + range = s.Range, + uri = s.DocumentUri, + }, + containerName = s.ContainerName, + }; }).ToArray(); + + return Task.FromResult(result); } - public override async Task HierarchicalDocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { + public override Task HierarchicalDocumentSymbol(DocumentSymbolParams @params, CancellationToken cancellationToken) { var uri = @params.textDocument.uri; - var symbols = await SymbolIndex.HierarchicalDocumentSymbolsAsync(uri, cancellationToken); - return ToDocumentSymbols(symbols); + var symbols = SymbolIndex.HierarchicalDocumentSymbols(uri); + var result = ToDocumentSymbols(symbols, cancellationToken); + return Task.FromResult(result); } private static async Task> GetModuleVariablesAsync(ProjectEntry entry, GetMemberOptions opts, string prefix, int timeout, CancellationToken token) { @@ -88,7 +101,7 @@ private static IEnumerable GetModuleVariables(ProjectEntry entry, return result; } - private DocumentSymbol[] ToDocumentSymbols(IEnumerable hSymbols) { + private DocumentSymbol[] ToDocumentSymbols(IEnumerable hSymbols, CancellationToken cancellationToken = default(CancellationToken)) { return hSymbols.MaybeEnumerate().Select(hSym => new DocumentSymbol { name = hSym.Name, detail = hSym.Detail, @@ -99,10 +112,13 @@ private DocumentSymbol[] ToDocumentSymbols(IEnumerable hSymb children = ToDocumentSymbols(hSym.Children), _functionKind = hSym._functionKind, }).Select(v => { + cancellationToken.ThrowIfCancellationRequested(); + if (v.selectionRange.start < v.range.start || v.selectionRange.end > v.range.end) { Debug.Fail($"selectionRange {v.selectionRange} not encompassed by range {v.range}"); v.selectionRange = v.range; } + return v; }).ToArray(); } From 2a8fe555989a3f13c07fa5cff21c479eaac1aa6b Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Tue, 15 Jan 2019 09:58:52 -0800 Subject: [PATCH 05/10] remove null check in Flatten --- src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs b/src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs index 8caa4c744..cfc2f8dd4 100644 --- a/src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs +++ b/src/Analysis/Engine/Impl/Indexing/SymbolExtensions.cs @@ -16,12 +16,11 @@ using System; using System.Collections.Generic; -using System.Linq; namespace Microsoft.PythonTools.Analysis.Indexing { internal static class SymbolExtensions { public static IEnumerable Flatten(this IEnumerable docSyms, Uri uri, string parent = null, int? depthLimit = null) { - foreach (var sym in docSyms ?? Enumerable.Empty()) { + foreach (var sym in docSyms) { yield return new FlatSymbol(sym.Name, sym.Kind, uri, sym.SelectionRange, parent); if (depthLimit != null) { From 2e50be33804154d38384de30761f79386f729946 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Tue, 15 Jan 2019 12:57:53 -0800 Subject: [PATCH 06/10] check symbol ordering strictly, process RHS of assignments before lefthand names, use queue instead of linked list --- .../Engine/Impl/Indexing/SymbolIndex.cs | 14 ++- .../Engine/Impl/Indexing/SymbolIndexWalker.cs | 26 +++--- src/Analysis/Engine/Test/SymbolIndexTests.cs | 85 +++++++++++-------- .../CollectionAssertionsExtensions.cs | 4 + 4 files changed, 71 insertions(+), 58 deletions(-) diff --git a/src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs b/src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs index d59742d24..4f1fc24cf 100644 --- a/src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs +++ b/src/Analysis/Engine/Impl/Indexing/SymbolIndex.cs @@ -27,9 +27,8 @@ internal class SymbolIndex : ISymbolIndex { public SymbolIndex() { } - public IEnumerable HierarchicalDocumentSymbols(Uri uri) { - return _index.TryGetValue(uri, out var list) ? list : Enumerable.Empty(); - } + public IEnumerable HierarchicalDocumentSymbols(Uri uri) + => _index.TryGetValue(uri, out var list) ? list : Enumerable.Empty(); public IEnumerable WorkspaceSymbols(string query) { foreach (var kvp in _index) { @@ -41,18 +40,17 @@ public IEnumerable WorkspaceSymbols(string query) { private IEnumerable WorkspaceSymbolsQuery(string query, Uri uri, IEnumerable symbols) { // Some semblance of a BFS. - var ll = new LinkedList<(HierarchicalSymbol, string)>(symbols.Select(s => (s, (string)null))); + var queue = new Queue<(HierarchicalSymbol, string)>(symbols.Select(s => (s, (string)null))); - while (ll.First != null) { - var (sym, parent) = ll.First.Value; - ll.RemoveFirst(); + while (queue.Count > 0) { + var (sym, parent) = queue.Dequeue(); if (sym.Name.ContainsOrdinal(query, ignoreCase: true)) { yield return new FlatSymbol(sym.Name, sym.Kind, uri, sym.SelectionRange, parent); } foreach (var child in sym.Children.MaybeEnumerate()) { - ll.AddLast((child, sym.Name)); + queue.Enqueue((child, sym.Name)); } } } diff --git a/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs b/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs index 6b0397ee1..a4abeb29d 100644 --- a/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs +++ b/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs @@ -102,13 +102,13 @@ public override bool Walk(FunctionDefinition node) { } public override bool Walk(ImportStatement node) { - foreach (var imp in node.Names.Zip(node.AsNames, (name, asName) => (name, asName))) { - if (imp.asName == null) { - var span = imp.name.GetSpan(_ast); - _stack.AddSymbol(new HierarchicalSymbol(imp.name.MakeString(), SymbolKind.Module, span)); + foreach (var (name, asName) in node.Names.Zip(node.AsNames, (a, b) => (a, b))) { + if (asName == null) { + var span = name.GetSpan(_ast); + _stack.AddSymbol(new HierarchicalSymbol(name.MakeString(), SymbolKind.Module, span)); } else { - var span = imp.asName.GetSpan(_ast); - _stack.AddSymbol(new HierarchicalSymbol(imp.asName.Name, SymbolKind.Module, span)); + var span = asName.GetSpan(_ast); + _stack.AddSymbol(new HierarchicalSymbol(asName.Name, SymbolKind.Module, span)); } } @@ -116,28 +116,24 @@ public override bool Walk(ImportStatement node) { } public override bool Walk(FromImportStatement node) { - foreach (var imp in node.Names.Zip(node.AsNames, (name, asName) => (name, asName))) { - if (imp.asName == null) { - var span = imp.name.GetSpan(_ast); - _stack.AddSymbol(new HierarchicalSymbol(imp.name.Name, SymbolKind.Module, span)); - } else { - var span = imp.asName.GetSpan(_ast); - _stack.AddSymbol(new HierarchicalSymbol(imp.asName.Name, SymbolKind.Module, span)); - } + foreach (var name in node.Names.Zip(node.AsNames, (name, asName) => asName ?? name)) { + var span = name.GetSpan(_ast); + _stack.AddSymbol(new HierarchicalSymbol(name.Name, SymbolKind.Module, span)); } return false; } public override bool Walk(AssignmentStatement node) { + node.Right?.Walk(this); foreach (var ne in node.Left.OfType()) { AddVarSymbol(ne); } - node.Right?.Walk(this); return false; } public override bool Walk(AugmentedAssignStatement node) { + node.Right?.Walk(this); AddVarSymbol(node.Left as NameExpression); return false; } diff --git a/src/Analysis/Engine/Test/SymbolIndexTests.cs b/src/Analysis/Engine/Test/SymbolIndexTests.cs index 39a94c99b..6f7a3493c 100644 --- a/src/Analysis/Engine/Test/SymbolIndexTests.cs +++ b/src/Analysis/Engine/Test/SymbolIndexTests.cs @@ -15,6 +15,7 @@ using System.Collections.Generic; using System.IO; using FluentAssertions; +using Microsoft.Python.Tests.Utilities.FluentAssertions; using Microsoft.PythonTools; using Microsoft.PythonTools.Analysis.FluentAssertions; using Microsoft.PythonTools.Analysis.Indexing; @@ -45,7 +46,7 @@ public void WalkerAssignments() { z = y"; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 1, 2, 2)), new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(3, 1, 3, 2)), @@ -57,7 +58,7 @@ public void WalkerMultipleAssignments() { var code = @"x = y = z = 1"; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 5, 1, 6)), new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(1, 9, 1, 10)), @@ -83,7 +84,7 @@ elif bar(): "; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(4, 5, 4, 6)), new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(6, 5, 6, 6)), @@ -101,7 +102,7 @@ public void WalkerTryExceptFinally() { "; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 5, 2, 6)), new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(4, 5, 4, 6)), new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(6, 5, 6, 6)), @@ -114,7 +115,7 @@ public void WalkerReassign() { x = 2"; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), }); } @@ -124,7 +125,7 @@ public void WalkerAugmentedAssign() { var code = @"x += 1"; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), }); } @@ -134,7 +135,7 @@ public void WalkerTopLevelConstant() { var code = @"FOO_BAR_3 = 1234"; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("FOO_BAR_3", SymbolKind.Constant, new SourceSpan(1, 1, 1, 10)), }); } @@ -146,7 +147,7 @@ public void WalkerFunction() { return z"; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 3, 13), new SourceSpan(1, 5, 1, 9), new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 10, 1, 11)), new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), @@ -160,7 +161,7 @@ public void WalkerFunctionStarredArgs() { var code = @"def func(*args, **kwargs): ..."; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 31), new SourceSpan(1, 5, 1, 9), new[] { new HierarchicalSymbol("args", SymbolKind.Variable, new SourceSpan(1, 11, 1, 15)), new HierarchicalSymbol("kwargs", SymbolKind.Variable, new SourceSpan(1, 19, 1, 25)), @@ -173,7 +174,7 @@ public void WalkerFunctionUnderscoreArg() { var code = @"def func(_): ..."; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 17), new SourceSpan(1, 5, 1, 9), new List(), FunctionKind.Function), }); } @@ -187,7 +188,7 @@ from os.path import ( join as osjoin2, exists as osexists, expanduser ) "; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("sys", SymbolKind.Module, new SourceSpan(1, 8, 1, 11)), new HierarchicalSymbol("np", SymbolKind.Module, new SourceSpan(2, 17, 2, 19)), new HierarchicalSymbol("osjoin", SymbolKind.Module, new SourceSpan(3, 29, 3, 35)), @@ -203,7 +204,7 @@ public void WalkerClass() { ..."; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 8), new SourceSpan(1, 7, 1, 10), new List(), FunctionKind.Class), }); } @@ -214,7 +215,7 @@ public void WalkerClassConstant() { CONSTANT = 1234"; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 20), new SourceSpan(1, 7, 1, 10), new[] { new HierarchicalSymbol("CONSTANT", SymbolKind.Constant, new SourceSpan(2, 5, 2, 13)), }, FunctionKind.Class), @@ -227,7 +228,7 @@ public void WalkerConstructor() { def __init__(self, x): ..."; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 31), new SourceSpan(1, 7, 1, 10), new[] { new HierarchicalSymbol("__init__", SymbolKind.Constructor, new SourceSpan(2, 5, 2, 31), new SourceSpan(2, 9, 2, 17), new[] { new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 18, 2, 22)), @@ -243,7 +244,7 @@ public void WalkerMethod() { def foo(self, x): ..."; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 26), new SourceSpan(1, 7, 1, 10), new[] { new HierarchicalSymbol("foo", SymbolKind.Method, new SourceSpan(2, 5, 2, 26), new SourceSpan(2, 9, 2, 12), new[] { new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 13, 2, 17)), @@ -259,7 +260,7 @@ public void WalkerDoubleUnderscoreMethod() { def __lt__(self, x): ..."; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 2, 29), new SourceSpan(1, 7, 1, 10), new[] { new HierarchicalSymbol("__lt__", SymbolKind.Operator, new SourceSpan(2, 5, 2, 29), new SourceSpan(2, 9, 2, 15), new[] { new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(2, 16, 2, 20)), @@ -285,7 +286,7 @@ def func3(self): ... def func4(self): ..."; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 12, 25), new SourceSpan(1, 7, 1, 10), new[] { new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), @@ -313,7 +314,7 @@ def func1(self): ... def func2(self): ..."; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 6, 25), new SourceSpan(1, 7, 1, 10), new[] { new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), @@ -338,7 +339,7 @@ def func2(arg): ... def func3(arg): ..."; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 9, 24), new SourceSpan(1, 7, 1, 10), new[] { new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 24), new SourceSpan(3, 9, 3, 14), new[] { new HierarchicalSymbol("arg", SymbolKind.Variable, new SourceSpan(3, 15, 3, 18)), @@ -366,7 +367,7 @@ def func2(cls): ... def func3(cls): ..."; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 9, 24), new SourceSpan(1, 7, 1, 10), new[] { new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 24), new SourceSpan(3, 9, 3, 14), new[] { new HierarchicalSymbol("cls", SymbolKind.Variable, new SourceSpan(3, 15, 3, 18)), @@ -390,7 +391,7 @@ def func1(x, y): ... def func2(x, y): ..."; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("func1", SymbolKind.Function, new SourceSpan(1, 1, 2, 21), new SourceSpan(2, 5, 2, 10), new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 11, 2, 12)), new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 14, 2, 15)), @@ -412,7 +413,7 @@ def func1(self): ... def func2(self): ..."; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 6, 25), new SourceSpan(1, 7, 1, 10), new[] { new HierarchicalSymbol("func1", SymbolKind.Method, new SourceSpan(2, 5, 3, 25), new SourceSpan(3, 9, 3, 14), new[] { new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(3, 15, 3, 19)), @@ -436,7 +437,7 @@ def func1(self): ... def func2(self): ..."; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 8, 25), new SourceSpan(1, 7, 1, 10), new[] { new HierarchicalSymbol("func1", SymbolKind.Property, new SourceSpan(2, 5, 4, 25), new SourceSpan(4, 9, 4, 14), new[] { new HierarchicalSymbol("self", SymbolKind.Variable, new SourceSpan(4, 15, 4, 19)), @@ -453,12 +454,12 @@ public void WalkerLambda() { var code = @"f = lambda x, y: x + y"; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { - new HierarchicalSymbol("f", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("", SymbolKind.Function, new SourceSpan(1, 5, 1, 23), new SourceSpan(1, 5, 1, 23), new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 12, 1, 13)), new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 15, 1, 16)), }, FunctionKind.Function), + new HierarchicalSymbol("f", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), }); } @@ -467,7 +468,7 @@ public void WalkerIncompleteFunction() { var code = @"def func(x, y):"; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("func", SymbolKind.Function, new SourceSpan(1, 1, 1, 16), new SourceSpan(1, 5, 1, 9), new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 10, 1, 11)), new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), @@ -480,17 +481,31 @@ public void WalkerIncompleteClass() { var code = @"class Foo(object):"; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("Foo", SymbolKind.Class, new SourceSpan(1, 1, 1, 19), new SourceSpan(1, 7, 1, 10), new List(), FunctionKind.Class), }); } [TestMethod, Priority(0)] - public void WalkerIncompleteAssycn() { + public void WalkerIncompleteAssign() { var code = @"x ="; var symbols = WalkSymbols(code); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerAugmentedAssignLambda() { + var code = @"x += lambda x, y: x + y"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.Function, new SourceSpan(1, 6, 1, 24), new SourceSpan(1, 6, 1, 24), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), + }, FunctionKind.Function), new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), }); } @@ -503,7 +518,7 @@ public void IndexHierarchicalDocument() { index.UpdateParseTree(uri, ast); var symbols = index.HierarchicalDocumentSymbols(uri); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), }); } @@ -516,7 +531,7 @@ public void IndexHierarchicalDocumentUpdate() { index.UpdateParseTree(uri, ast); var symbols = index.HierarchicalDocumentSymbols(uri); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), }); @@ -524,7 +539,7 @@ public void IndexHierarchicalDocumentUpdate() { index.UpdateParseTree(uri, ast); symbols = index.HierarchicalDocumentSymbols(uri); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), }); } @@ -549,7 +564,7 @@ public void IndexWorkspaceSymbolsFlatten() { index.UpdateParseTree(uri, ast); var symbols = index.WorkspaceSymbols(""); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new FlatSymbol("Foo", SymbolKind.Class, uri, new SourceSpan(1, 7, 1, 10)), new FlatSymbol("foo", SymbolKind.Method, uri, new SourceSpan(2, 9, 2, 12), "Foo"), new FlatSymbol("self", SymbolKind.Variable, uri, new SourceSpan(2, 13, 2, 17), "foo"), @@ -568,7 +583,7 @@ public void IndexWorkspaceSymbolsFiltered() { index.UpdateParseTree(uri, ast); var symbols = index.WorkspaceSymbols("x"); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new FlatSymbol("x", SymbolKind.Variable, uri, new SourceSpan(2, 19, 2, 20), "foo"), }); } @@ -593,7 +608,7 @@ public void IndexWorkspaceSymbolsCaseInsensitive() { index.UpdateParseTree(uri, ast); var symbols = index.WorkspaceSymbols("foo"); - symbols.Should().BeEquivalentTo(new[] { + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { new FlatSymbol("Foo", SymbolKind.Class, uri, new SourceSpan(1, 7, 1, 10)), new FlatSymbol("foo", SymbolKind.Method, uri, new SourceSpan(2, 9, 2, 12), "Foo"), }); diff --git a/src/UnitTests/Core/Impl/FluentAssertions/CollectionAssertionsExtensions.cs b/src/UnitTests/Core/Impl/FluentAssertions/CollectionAssertionsExtensions.cs index 453af3a7a..c84edf772 100644 --- a/src/UnitTests/Core/Impl/FluentAssertions/CollectionAssertionsExtensions.cs +++ b/src/UnitTests/Core/Impl/FluentAssertions/CollectionAssertionsExtensions.cs @@ -37,5 +37,9 @@ public static AndConstraint OnlyContain( public static AndConstraint OnlyContain( this StringCollectionAssertions assertions, IReadOnlyCollection expected, string because = "", params object[] reasonArgs) => assertions.HaveCount(expected.Count, because, reasonArgs).And.Contain(expected, because, reasonArgs); + + public static AndConstraint> BeEquivalentToWithStrictOrdering( + this GenericCollectionAssertions assertions, IEnumerable expected, string because = "", params object[] reasonArgs) + => assertions.BeEquivalentTo(expected, options => options.WithStrictOrdering(), because, reasonArgs); } } From 4e26002794a104f3cc0980d830c7215db55041f3 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Tue, 15 Jan 2019 13:19:18 -0800 Subject: [PATCH 07/10] inline zip, name tuple elements --- .../Engine/Impl/Indexing/SymbolIndexWalker.cs | 25 ++++++++----------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs b/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs index a4abeb29d..005af72cb 100644 --- a/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs +++ b/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs @@ -85,8 +85,8 @@ public override bool Walk(FunctionDefinition node) { foreach (var dec in node.Decorators.Decorators) { var maybeKind = DecoratorExpressionToKind(dec); if (maybeKind.HasValue) { - ds.Kind = maybeKind.Value.Item1; - ds._functionKind = maybeKind.Value.Item2; + ds.Kind = maybeKind.Value.kind; + ds._functionKind = maybeKind.Value.functionKind; break; } } @@ -102,14 +102,9 @@ public override bool Walk(FunctionDefinition node) { } public override bool Walk(ImportStatement node) { - foreach (var (name, asName) in node.Names.Zip(node.AsNames, (a, b) => (a, b))) { - if (asName == null) { - var span = name.GetSpan(_ast); - _stack.AddSymbol(new HierarchicalSymbol(name.MakeString(), SymbolKind.Module, span)); - } else { - var span = asName.GetSpan(_ast); - _stack.AddSymbol(new HierarchicalSymbol(asName.Name, SymbolKind.Module, span)); - } + foreach (var (nameNode, nameString) in node.Names.Zip(node.AsNames, (name, asName) => asName != null ? (asName, asName.Name) : ((Node)name, name.MakeString()))) { + var span = nameNode.GetSpan(_ast); + _stack.AddSymbol(new HierarchicalSymbol(nameString, SymbolKind.Module, span)); } return false; @@ -175,7 +170,7 @@ private void AddVarSymbol(NameExpression node) { _stack.AddSymbol(new HierarchicalSymbol(node.Name, kind, span)); } - private (SymbolKind, string)? DecoratorExpressionToKind(Expression exp) { + private (SymbolKind kind, string functionKind)? DecoratorExpressionToKind(Expression exp) { switch (exp) { case NameExpression ne when NameIsProperty(ne.Name): case MemberExpression me when NameIsProperty(me.Name): @@ -224,12 +219,12 @@ private void WalkAndDeclareAll(IEnumerable nodes) { } private class SymbolStack { - private readonly Stack<(SymbolKind?, List)> _symbols; + private readonly Stack<(SymbolKind? kind, List symbols)> _symbols; private readonly Stack> _declared = new Stack>(new[] { new HashSet() }); public List Root { get; } = new List(); - public SymbolKind? Parent => _symbols.Peek().Item1; + public SymbolKind? Parent => _symbols.Peek().kind; public SymbolStack() { _symbols = new Stack<(SymbolKind?, List)>(new (SymbolKind?, List)[] { (null, Root) }); @@ -242,7 +237,7 @@ public void Enter(SymbolKind parent) { public List Exit() { ExitDeclared(); - return _symbols.Pop().Item2; + return _symbols.Pop().symbols; } public void EnterDeclared() => _declared.Push(new HashSet()); @@ -255,7 +250,7 @@ public void AddSymbol(HierarchicalSymbol sym) { return; } - _symbols.Peek().Item2.Add(sym); + _symbols.Peek().symbols.Add(sym); _declared.Peek().Add(sym.Name); } } From f8e1e5b3bc916f5182870819190040120c61f3d2 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Tue, 15 Jan 2019 16:38:45 -0800 Subject: [PATCH 08/10] support comprehensions --- .../Engine/Impl/Indexing/SymbolIndexWalker.cs | 78 +++++++++++++++++++ src/Analysis/Engine/Test/SymbolIndexTests.cs | 75 ++++++++++++++++++ 2 files changed, 153 insertions(+) diff --git a/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs b/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs index 005af72cb..d25db6bf8 100644 --- a/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs +++ b/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs @@ -149,6 +149,66 @@ public override bool Walk(TryStatement node) { return false; } + public override bool Walk(ForStatement node) { + _stack.EnterDeclared(); + AddVarSymbolRecursive(node.Left); + node.List?.Walk(this); + node.Body?.Walk(this); + _stack.ExitDeclaredAndMerge(); + + _stack.EnterDeclared(); + node.Else?.Walk(this); + _stack.ExitDeclaredAndMerge(); + + return false; + } + + public override bool Walk(ComprehensionFor node) { + AddVarSymbolRecursive(node.Left); + return false; + } + + public override bool Walk(ListComprehension node) { + _stack.Enter(SymbolKind.None); + return base.Walk(node); + } + + public override void PostWalk(ListComprehension node) => ExitComprehension(node); + + public override bool Walk(DictionaryComprehension node) { + _stack.Enter(SymbolKind.None); + return base.Walk(node); + } + + public override void PostWalk(DictionaryComprehension node) => ExitComprehension(node); + + public override bool Walk(SetComprehension node) { + _stack.Enter(SymbolKind.None); + return base.Walk(node); + } + + public override void PostWalk(SetComprehension node) => ExitComprehension(node); + + public override bool Walk(GeneratorExpression node) { + _stack.Enter(SymbolKind.None); + return base.Walk(node); + } + + public override void PostWalk(GeneratorExpression node) => ExitComprehension(node); + + private void ExitComprehension(Comprehension node) { + var children = _stack.Exit(); + var span = node.GetSpan(_ast); + + _stack.AddSymbol(new HierarchicalSymbol( + $"<{node.NodeName}>", + SymbolKind.None, + span, + children: children + )); + } + + private void AddVarSymbol(NameExpression node) { if (node == null) { return; @@ -170,6 +230,24 @@ private void AddVarSymbol(NameExpression node) { _stack.AddSymbol(new HierarchicalSymbol(node.Name, kind, span)); } + private void AddVarSymbolRecursive(Expression node) { + if (node == null) { + return; + } + + switch (node) { + case NameExpression ne: + AddVarSymbol(ne); + return; + + case SequenceExpression se: + foreach (var item in se.Items.MaybeEnumerate()) { + AddVarSymbolRecursive(item); + } + return; + } + } + private (SymbolKind kind, string functionKind)? DecoratorExpressionToKind(Expression exp) { switch (exp) { case NameExpression ne when NameIsProperty(ne.Name): diff --git a/src/Analysis/Engine/Test/SymbolIndexTests.cs b/src/Analysis/Engine/Test/SymbolIndexTests.cs index 6f7a3493c..ad09829ea 100644 --- a/src/Analysis/Engine/Test/SymbolIndexTests.cs +++ b/src/Analysis/Engine/Test/SymbolIndexTests.cs @@ -463,6 +463,81 @@ public void WalkerLambda() { }); } + [TestMethod, Priority(0)] + public void WalkerForLoop() { + var code = @"z = False +for [x, y, (p, q)] in [[1, 2, [3, 4]]]: + z += x +else: + z = None"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 6, 2, 7)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(2, 9, 2, 10)), + new HierarchicalSymbol("p", SymbolKind.Variable, new SourceSpan(2, 13, 2, 14)), + new HierarchicalSymbol("q", SymbolKind.Variable, new SourceSpan(2, 16, 2, 17)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(3, 5, 3, 6)), + new HierarchicalSymbol("z", SymbolKind.Variable, new SourceSpan(5, 5, 5, 6)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerListComprehension() { + var code = @"flat_list = [item for sublist in l for item in sublist]"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 13, 1, 56), new SourceSpan(1, 13, 1, 56), new[] { + new HierarchicalSymbol("sublist", SymbolKind.Variable, new SourceSpan(1, 23, 1, 30)), + new HierarchicalSymbol("item", SymbolKind.Variable, new SourceSpan(1, 40, 1, 44)), + }), + new HierarchicalSymbol("flat_list", SymbolKind.Variable, new SourceSpan(1, 1, 1, 10)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerDictionaryComprehension() { + var code = @"d = { x: y for x, y in zip(range(10), range(10)) }"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 51), new SourceSpan(1, 5, 1, 51), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 19, 1, 20)), + }), + new HierarchicalSymbol("d", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerSetComprehension() { + var code = @"s = { x for x in range(10) }"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 29), new SourceSpan(1, 5, 1, 29), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), + }), + new HierarchicalSymbol("s", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + + [TestMethod, Priority(0)] + public void WalkerGenerator() { + var code = @"g = (x + y for x, y in zip(range(10), range(10)))"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 50), new SourceSpan(1, 5, 1, 50), new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 19, 1, 20)), + }), + new HierarchicalSymbol("g", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + [TestMethod, Priority(0)] public void WalkerIncompleteFunction() { var code = @"def func(x, y):"; From ba622fbb635697889465fbedd8f0480b3d8d8700 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Wed, 16 Jan 2019 09:53:25 -0800 Subject: [PATCH 09/10] handle nested comprehensions --- .../Engine/Impl/Indexing/SymbolIndexWalker.cs | 2 +- src/Analysis/Engine/Test/SymbolIndexTests.cs | 36 ++++++++++++++----- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs b/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs index d25db6bf8..0349d3851 100644 --- a/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs +++ b/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs @@ -165,7 +165,7 @@ public override bool Walk(ForStatement node) { public override bool Walk(ComprehensionFor node) { AddVarSymbolRecursive(node.Left); - return false; + return base.Walk(node); } public override bool Walk(ListComprehension node) { diff --git a/src/Analysis/Engine/Test/SymbolIndexTests.cs b/src/Analysis/Engine/Test/SymbolIndexTests.cs index ad09829ea..ae6efae04 100644 --- a/src/Analysis/Engine/Test/SymbolIndexTests.cs +++ b/src/Analysis/Engine/Test/SymbolIndexTests.cs @@ -455,10 +455,10 @@ public void WalkerLambda() { var symbols = WalkSymbols(code); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("", SymbolKind.Function, new SourceSpan(1, 5, 1, 23), new SourceSpan(1, 5, 1, 23), new[] { + new HierarchicalSymbol("", SymbolKind.Function, new SourceSpan(1, 5, 1, 23), children: new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 12, 1, 13)), new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 15, 1, 16)), - }, FunctionKind.Function), + }, functionKind: FunctionKind.Function), new HierarchicalSymbol("f", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), }); } @@ -489,7 +489,7 @@ public void WalkerListComprehension() { var symbols = WalkSymbols(code); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 13, 1, 56), new SourceSpan(1, 13, 1, 56), new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 13, 1, 56), children: new[] { new HierarchicalSymbol("sublist", SymbolKind.Variable, new SourceSpan(1, 23, 1, 30)), new HierarchicalSymbol("item", SymbolKind.Variable, new SourceSpan(1, 40, 1, 44)), }), @@ -503,7 +503,7 @@ public void WalkerDictionaryComprehension() { var symbols = WalkSymbols(code); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 51), new SourceSpan(1, 5, 1, 51), new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 51), children: new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 19, 1, 20)), }), @@ -517,7 +517,7 @@ public void WalkerSetComprehension() { var symbols = WalkSymbols(code); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 29), new SourceSpan(1, 5, 1, 29), new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 29), children: new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), }), new HierarchicalSymbol("s", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), @@ -530,7 +530,7 @@ public void WalkerGenerator() { var symbols = WalkSymbols(code); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 50), new SourceSpan(1, 5, 1, 50), new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 1, 50), children: new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 19, 1, 20)), }), @@ -538,6 +538,26 @@ public void WalkerGenerator() { }); } + [TestMethod, Priority(0)] + public void WalkerNestedListComprehension() { + var code = @"l = [ + x for x in [ + y for y in range(10) + ] +]"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEquivalentToWithStrictOrdering(new[] { + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(1, 5, 5, 2), children: new[] { + new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(2, 11, 2, 12)), + new HierarchicalSymbol("", SymbolKind.None, new SourceSpan(2, 16, 4, 6), children: new[] { + new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(3, 15, 3, 16)), + }), + }), + new HierarchicalSymbol("l", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), + }); + } + [TestMethod, Priority(0)] public void WalkerIncompleteFunction() { var code = @"def func(x, y):"; @@ -577,10 +597,10 @@ public void WalkerAugmentedAssignLambda() { var symbols = WalkSymbols(code); symbols.Should().BeEquivalentToWithStrictOrdering(new[] { - new HierarchicalSymbol("", SymbolKind.Function, new SourceSpan(1, 6, 1, 24), new SourceSpan(1, 6, 1, 24), new[] { + new HierarchicalSymbol("", SymbolKind.Function, new SourceSpan(1, 6, 1, 24), children: new[] { new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 13, 1, 14)), new HierarchicalSymbol("y", SymbolKind.Variable, new SourceSpan(1, 16, 1, 17)), - }, FunctionKind.Function), + }, functionKind: FunctionKind.Function), new HierarchicalSymbol("x", SymbolKind.Variable, new SourceSpan(1, 1, 1, 2)), }); } From c46ea7e0d60169ab92dc16c0d406a8b64b16afe6 Mon Sep 17 00:00:00 2001 From: Jake Bailey <5341706+jakebailey@users.noreply.github.com> Date: Wed, 16 Jan 2019 13:54:13 -0800 Subject: [PATCH 10/10] ignore __future__ imports --- src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs | 4 ++++ src/Analysis/Engine/Test/SymbolIndexTests.cs | 8 ++++++++ 2 files changed, 12 insertions(+) diff --git a/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs b/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs index 0349d3851..a879ec46b 100644 --- a/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs +++ b/src/Analysis/Engine/Impl/Indexing/SymbolIndexWalker.cs @@ -111,6 +111,10 @@ public override bool Walk(ImportStatement node) { } public override bool Walk(FromImportStatement node) { + if (node.IsFromFuture) { + return false; + } + foreach (var name in node.Names.Zip(node.AsNames, (name, asName) => asName ?? name)) { var span = name.GetSpan(_ast); _stack.AddSymbol(new HierarchicalSymbol(name.Name, SymbolKind.Module, span)); diff --git a/src/Analysis/Engine/Test/SymbolIndexTests.cs b/src/Analysis/Engine/Test/SymbolIndexTests.cs index ae6efae04..995a3cde2 100644 --- a/src/Analysis/Engine/Test/SymbolIndexTests.cs +++ b/src/Analysis/Engine/Test/SymbolIndexTests.cs @@ -198,6 +198,14 @@ from os.path import ( join as osjoin2, exists as osexists, expanduser ) }); } + [TestMethod, Priority(0)] + public void WalkerImportFromFuture() { + var code = @"from __future__ import print_function"; + + var symbols = WalkSymbols(code); + symbols.Should().BeEmpty(); + } + [TestMethod, Priority(0)] public void WalkerClass() { var code = @"class Foo(object):