From 8bca56b1fa4ab8b282ae6491de25525c90aa9efe Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Fri, 1 Mar 2019 09:51:05 -0800 Subject: [PATCH 1/4] Fix #668 (partial) --- .../Analyzer/Handlers/ConditionalHandler.cs | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs b/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs index cb41801b9..b104acb94 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Handlers/ConditionalHandler.cs @@ -66,24 +66,6 @@ public bool HandleIf(IfStatement node) { someRecognized = true; } } - - // Handle basic check such as - // if isinstance(value, type): - // return value - // by assigning type to the value unless clause is raising exception. - var ce = node.Tests.FirstOrDefault()?.Test as CallExpression; - if (ce?.Target is NameExpression ne && ne.Name == @"isinstance" && ce.Args.Count == 2) { - var nex = ce.Args[0].Expression as NameExpression; - var name = nex?.Name; - var typeName = (ce.Args[1].Expression as NameExpression)?.Name; - if (name != null && typeName != null) { - var typeId = typeName.GetTypeId(); - if (typeId != BuiltinTypeId.Unknown) { - var t = new PythonType(typeName, Module, string.Empty, LocationInfo.Empty, typeId); - Eval.DeclareVariable(name, t, VariableSource.Declaration, nex); - } - } - } return !someRecognized; } From 7ffc9db4f0807314d4d36a8540380b988c26ee2e Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 4 Mar 2019 16:43:30 -0800 Subject: [PATCH 2/4] Tests --- src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs | 10 ++++++++-- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 12 ++++-------- src/Analysis/Ast/Test/BasicTests.cs | 14 ++++++++------ 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index 205e64d01..a30c5a97b 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -227,8 +227,14 @@ private async Task AnalyzeAffectedEntriesAsync(IDependencyChainWalker !k.IsTypeshed).Count == 0) { + + var count = walker.MissingKeys.Where(k => !k.IsTypeshed).Count; + _log?.Log(TraceEventType.Verbose, $"Walker count is {count}, missing keys:"); + foreach (var key in walker.MissingKeys) { + _log?.Log(TraceEventType.Verbose, $" Name: {key.Name}, Path: {key.FilePath}"); + } + + if (count == 0) { Interlocked.Exchange(ref _runningTasks, 0); _analysisCompleteEvent.Set(); } diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index 7be31275e..ce6949fc5 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -111,6 +111,7 @@ internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer s ContentState = State.Analyzed; } InitializeContent(creationOptions.Content); + Log?.Log(TraceEventType.Verbose, $"Created module: {Name} {FilePath}"); } #region IPythonType @@ -385,21 +386,16 @@ private void Parse(CancellationToken cancellationToken) { _diagnosticsService?.Replace(Uri, _parseErrors); } - ContentState = State.Parsed; - } - - NewAst?.Invoke(this, EventArgs.Empty); - - if (ContentState < State.Analyzing) { ContentState = State.Analyzing; + Log?.Log(TraceEventType.Verbose, $"Enqueue: {Name} {FilePath}"); var analyzer = Services.GetService(); analyzer.EnqueueDocumentForAnalysis(this, ast, version, _allProcessingCts.Token); - } - lock (AnalysisLock) { _parsingTask = null; } + + NewAst?.Invoke(this, EventArgs.Empty); } private class CollectingErrorSink : ErrorSink { diff --git a/src/Analysis/Ast/Test/BasicTests.cs b/src/Analysis/Ast/Test/BasicTests.cs index 47d47b8cd..c8b64e2f2 100644 --- a/src/Analysis/Ast/Test/BasicTests.cs +++ b/src/Analysis/Ast/Test/BasicTests.cs @@ -75,12 +75,14 @@ public async Task ImportTest() { import sys x = sys.path "; - var analysis = await GetAnalysisAsync(code); - analysis.GlobalScope.Variables.Count.Should().Be(2); - - analysis.Should() - .HaveVariable("sys").OfType(BuiltinTypeId.Module) - .And.HaveVariable("x").OfType(BuiltinTypeId.List); + for (var i = 0; i < 20; i++) { + var analysis = await GetAnalysisAsync(code); + analysis.GlobalScope.Variables.Count.Should().Be(2); + + analysis.Should() + .HaveVariable("sys").OfType(BuiltinTypeId.Module) + .And.HaveVariable("x").OfType(BuiltinTypeId.List); + } } [TestMethod, Priority(0)] From 6303c45f234ba1c40058520c8700f6815c303035 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Mon, 4 Mar 2019 18:11:48 -0800 Subject: [PATCH 3/4] Revert "Tests" This reverts commit 7ffc9db4f0807314d4d36a8540380b988c26ee2e. --- src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs | 10 ++-------- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 12 ++++++++---- src/Analysis/Ast/Test/BasicTests.cs | 14 ++++++-------- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs index a30c5a97b..205e64d01 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzer.cs @@ -227,14 +227,8 @@ private async Task AnalyzeAffectedEntriesAsync(IDependencyChainWalker !k.IsTypeshed).Count; - _log?.Log(TraceEventType.Verbose, $"Walker count is {count}, missing keys:"); - foreach (var key in walker.MissingKeys) { - _log?.Log(TraceEventType.Verbose, $" Name: {key.Name}, Path: {key.FilePath}"); - } - - if (count == 0) { + + if (walker.MissingKeys.Where(k => !k.IsTypeshed).Count == 0) { Interlocked.Exchange(ref _runningTasks, 0); _analysisCompleteEvent.Set(); } diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index ce6949fc5..7be31275e 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -111,7 +111,6 @@ internal PythonModule(ModuleCreationOptions creationOptions, IServiceContainer s ContentState = State.Analyzed; } InitializeContent(creationOptions.Content); - Log?.Log(TraceEventType.Verbose, $"Created module: {Name} {FilePath}"); } #region IPythonType @@ -386,16 +385,21 @@ private void Parse(CancellationToken cancellationToken) { _diagnosticsService?.Replace(Uri, _parseErrors); } + ContentState = State.Parsed; + } + + NewAst?.Invoke(this, EventArgs.Empty); + + if (ContentState < State.Analyzing) { ContentState = State.Analyzing; - Log?.Log(TraceEventType.Verbose, $"Enqueue: {Name} {FilePath}"); var analyzer = Services.GetService(); analyzer.EnqueueDocumentForAnalysis(this, ast, version, _allProcessingCts.Token); + } + lock (AnalysisLock) { _parsingTask = null; } - - NewAst?.Invoke(this, EventArgs.Empty); } private class CollectingErrorSink : ErrorSink { diff --git a/src/Analysis/Ast/Test/BasicTests.cs b/src/Analysis/Ast/Test/BasicTests.cs index c8b64e2f2..47d47b8cd 100644 --- a/src/Analysis/Ast/Test/BasicTests.cs +++ b/src/Analysis/Ast/Test/BasicTests.cs @@ -75,14 +75,12 @@ public async Task ImportTest() { import sys x = sys.path "; - for (var i = 0; i < 20; i++) { - var analysis = await GetAnalysisAsync(code); - analysis.GlobalScope.Variables.Count.Should().Be(2); - - analysis.Should() - .HaveVariable("sys").OfType(BuiltinTypeId.Module) - .And.HaveVariable("x").OfType(BuiltinTypeId.List); - } + var analysis = await GetAnalysisAsync(code); + analysis.GlobalScope.Variables.Count.Should().Be(2); + + analysis.Should() + .HaveVariable("sys").OfType(BuiltinTypeId.Module) + .And.HaveVariable("x").OfType(BuiltinTypeId.List); } [TestMethod, Priority(0)] From 954852f319596d5caad359d3e6ff6ad1328e9649 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 12 Mar 2019 18:42:02 -0700 Subject: [PATCH 4/4] Cache document content --- src/Analysis/Ast/Impl/Documents/DocumentBuffer.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Analysis/Ast/Impl/Documents/DocumentBuffer.cs b/src/Analysis/Ast/Impl/Documents/DocumentBuffer.cs index f8fa67c76..7e042c1a3 100644 --- a/src/Analysis/Ast/Impl/Documents/DocumentBuffer.cs +++ b/src/Analysis/Ast/Impl/Documents/DocumentBuffer.cs @@ -17,17 +17,15 @@ using System.Collections.Generic; using System.Linq; using System.Text; -using System.Threading; -using Microsoft.Python.Core.Diagnostics; using Microsoft.Python.Parsing; namespace Microsoft.Python.Analysis.Documents { internal sealed class DocumentBuffer { private readonly StringBuilder _sb = new StringBuilder(); - private readonly int _ownerThreadId = Thread.CurrentThread.ManagedThreadId; + private string _cachedText; public int Version { get; private set; } - public string Text => _sb.ToString(); + public string Text => _cachedText ?? (_cachedText = _sb.ToString()); public void Reset(int version, string content) { Version = version; @@ -35,9 +33,11 @@ public void Reset(int version, string content) { if (!string.IsNullOrEmpty(content)) { _sb.Append(content); } + _cachedText = null; } public void Update(IEnumerable changes) { + var lastStart = int.MaxValue; var lineLoc = SplitLines(_sb).ToArray(); @@ -57,6 +57,7 @@ public void Update(IEnumerable changes) { } } Version++; + _cachedText = null; } private static IEnumerable SplitLines(StringBuilder text) {