From c6a9c223be9265f72b042dfcf41aad7006e9f9ba Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Mon, 30 Sep 2019 12:03:05 -0700 Subject: [PATCH 1/7] Remove stale reference --- .../Impl/Microsoft.Python.LanguageServer.csproj | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj index ca455737f..a33882e0d 100644 --- a/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj +++ b/src/LanguageServer/Impl/Microsoft.Python.LanguageServer.csproj @@ -33,14 +33,6 @@ - - - $(AnalysisReference)/Microsoft.Python.Analysis.Engine.dll - - - PreserveNewest - - From 6109ac761f1c2398c5ca6b4f38826cfb28051454 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 12 Nov 2019 13:55:07 -0800 Subject: [PATCH 2/7] Don't suppress LHS diagnostics on augmented assign --- .../UndefinedVariables/UndefinedVariablesWalker.cs | 2 -- src/Analysis/Ast/Test/AssignmentTests.cs | 9 +++++++++ src/Analysis/Ast/Test/LintUndefinedVarsTests.cs | 12 ++++++++++++ 3 files changed, 21 insertions(+), 2 deletions(-) diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs index 7365bf08d..42ac73bf0 100644 --- a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs @@ -49,9 +49,7 @@ public override bool Walk(SuiteStatement node) { HandleNonLocal(nls); break; case AugmentedAssignStatement augs: - _suppressDiagnostics = true; augs.Left?.Walk(new ExpressionWalker(this)); - _suppressDiagnostics = false; augs.Right?.Walk(new ExpressionWalker(this)); break; case AssignmentStatement asst: diff --git a/src/Analysis/Ast/Test/AssignmentTests.cs b/src/Analysis/Ast/Test/AssignmentTests.cs index cf3a54344..298454515 100644 --- a/src/Analysis/Ast/Test/AssignmentTests.cs +++ b/src/Analysis/Ast/Test/AssignmentTests.cs @@ -219,6 +219,15 @@ def __init__(self): .Which.Should().HaveMembers("abc", "y", "__class__"); } + [TestMethod, Priority(0)] + public async Task AugmentedAssignToUndefined() { + const string code = @" +x += 1 +"; + var analysis = await GetAnalysisAsync(code); + analysis.Should().NotHaveVariable("x"); + } + [TestMethod, Priority(0)] public async Task BaseInstanceVariable() { const string code = @" diff --git a/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs b/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs index cdee6b70f..dec8c849a 100644 --- a/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs +++ b/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs @@ -821,6 +821,18 @@ class Subclass(MyClass): d.Should().BeEmpty(); } + [TestMethod, Priority(0)] + public async Task AugmentedAssignToUndefined() { + const string code = @" +x += 1 +"; + var d = await LintAsync(code); + d.Should().HaveCount(1); + d[0].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); + d[0].SourceSpan.Should().Be(2, 1, 2, 2); + } + + private async Task> LintAsync(string code, InterpreterConfiguration configuration = null) { var analysis = await GetAnalysisAsync(code, configuration ?? PythonVersions.LatestAvailable3X); var a = Services.GetService(); From bcfc3b7246947674cbc18d7a76d2b90c105acf89 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 12 Nov 2019 15:30:27 -0800 Subject: [PATCH 3/7] Revert "Don't suppress LHS diagnostics on augmented assign" This reverts commit 6109ac761f1c2398c5ca6b4f38826cfb28051454. --- .../UndefinedVariables/UndefinedVariablesWalker.cs | 2 ++ src/Analysis/Ast/Test/AssignmentTests.cs | 9 --------- src/Analysis/Ast/Test/LintUndefinedVarsTests.cs | 12 ------------ 3 files changed, 2 insertions(+), 21 deletions(-) diff --git a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs index 42ac73bf0..7365bf08d 100644 --- a/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs +++ b/src/Analysis/Ast/Impl/Linting/UndefinedVariables/UndefinedVariablesWalker.cs @@ -49,7 +49,9 @@ public override bool Walk(SuiteStatement node) { HandleNonLocal(nls); break; case AugmentedAssignStatement augs: + _suppressDiagnostics = true; augs.Left?.Walk(new ExpressionWalker(this)); + _suppressDiagnostics = false; augs.Right?.Walk(new ExpressionWalker(this)); break; case AssignmentStatement asst: diff --git a/src/Analysis/Ast/Test/AssignmentTests.cs b/src/Analysis/Ast/Test/AssignmentTests.cs index 298454515..cf3a54344 100644 --- a/src/Analysis/Ast/Test/AssignmentTests.cs +++ b/src/Analysis/Ast/Test/AssignmentTests.cs @@ -219,15 +219,6 @@ def __init__(self): .Which.Should().HaveMembers("abc", "y", "__class__"); } - [TestMethod, Priority(0)] - public async Task AugmentedAssignToUndefined() { - const string code = @" -x += 1 -"; - var analysis = await GetAnalysisAsync(code); - analysis.Should().NotHaveVariable("x"); - } - [TestMethod, Priority(0)] public async Task BaseInstanceVariable() { const string code = @" diff --git a/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs b/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs index dec8c849a..cdee6b70f 100644 --- a/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs +++ b/src/Analysis/Ast/Test/LintUndefinedVarsTests.cs @@ -821,18 +821,6 @@ class Subclass(MyClass): d.Should().BeEmpty(); } - [TestMethod, Priority(0)] - public async Task AugmentedAssignToUndefined() { - const string code = @" -x += 1 -"; - var d = await LintAsync(code); - d.Should().HaveCount(1); - d[0].ErrorCode.Should().Be(ErrorCodes.UndefinedVariable); - d[0].SourceSpan.Should().Be(2, 1, 2, 2); - } - - private async Task> LintAsync(string code, InterpreterConfiguration configuration = null) { var analysis = await GetAnalysisAsync(code, configuration ?? PythonVersions.LatestAvailable3X); var a = Services.GetService(); From 1f716e8ee2a14d558676317768231b4d71f55e34 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Thu, 14 Nov 2019 18:40:30 -0800 Subject: [PATCH 4/7] Sync builtins creation --- .../Impl/Analyzer/Definitions/IAnalyzable.cs | 6 ++++ .../Impl/Analyzer/PythonAnalyzerSession.cs | 10 +++--- .../Ast/Impl/Analyzer/PythonInterpreter.cs | 1 - .../Ast/Impl/Modules/BuiltinsPythonModule.cs | 19 ++++++++++- src/Analysis/Ast/Impl/Modules/PythonModule.cs | 33 ++++++++++++------- .../Resolution/MainModuleResolution.cs | 19 +++++------ 6 files changed, 59 insertions(+), 29 deletions(-) diff --git a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs index 2b9182661..b13891861 100644 --- a/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs +++ b/src/Analysis/Ast/Impl/Analyzer/Definitions/IAnalyzable.cs @@ -14,6 +14,7 @@ // permissions and limitations under the License. using Microsoft.Python.Analysis.Dependencies; +using Microsoft.Python.Parsing.Ast; namespace Microsoft.Python.Analysis.Analyzer { /// @@ -30,6 +31,11 @@ internal interface IAnalyzable { /// void NotifyAnalysisBegins(); + /// + /// Performs standard analysis pass. Does not include any restoration from databases. + /// + ModuleWalker Analyze(PythonAst ast); + /// /// Notifies document that its analysis is now complete. /// diff --git a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs index caeb42dca..d52d21d29 100644 --- a/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs +++ b/src/Analysis/Ast/Impl/Analyzer/PythonAnalyzerSession.cs @@ -527,11 +527,11 @@ private IDocumentAnalysis RestoreOrAnalyzeModule(IDependencyChainSingleNode _hiddenNames.Contains(name) ? null : base.GetMember(name); public IMember GetAnyMember(string name) => base.GetMember(name); public override IEnumerable GetMemberNames() => base.GetMemberNames().Except(_hiddenNames).ToArray(); + #endregion + + public void Initialize() => ParseAndLogExceptions(CancellationToken.None); protected override string[] GetScrapeArguments(IPythonInterpreter interpreter) => !InstallPath.TryGetFile("scrape_module.py", out var sb) ? null : new[] { "-W", "ignore", "-B", "-E", sb }; + protected override void Parse() { } + + protected override void Analyze(PythonAst ast, int version) { + NotifyAnalysisBegins(); + var walker = Analyze(ast); + var analysis = new DocumentAnalysis(this, version, walker.GlobalScope, walker.Eval, walker.StarImportMemberNames); + NotifyAnalysisComplete(analysis); + } + protected override void OnAnalysisComplete() { SpecializeTypes(); SpecializeFunctions(); diff --git a/src/Analysis/Ast/Impl/Modules/PythonModule.cs b/src/Analysis/Ast/Impl/Modules/PythonModule.cs index d1463ef62..ba9b118bb 100644 --- a/src/Analysis/Ast/Impl/Modules/PythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/PythonModule.cs @@ -22,10 +22,11 @@ using System.Threading; using System.Threading.Tasks; using Microsoft.Python.Analysis.Analyzer; +using Microsoft.Python.Analysis.Analyzer.Evaluation; +using Microsoft.Python.Analysis.Analyzer.Handlers; using Microsoft.Python.Analysis.Dependencies; using Microsoft.Python.Analysis.Diagnostics; using Microsoft.Python.Analysis.Documents; -using Microsoft.Python.Analysis.Specializations.Typing; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; using Microsoft.Python.Core; @@ -305,7 +306,7 @@ public void Invalidate() { Services.GetService().InvalidateAnalysis(this); } - private void Parse() { + protected virtual void Parse() { _parseCts?.Cancel(); _parseCts = new CancellationTokenSource(); @@ -316,7 +317,7 @@ private void Parse() { _parsingTask = Task.Run(() => ParseAndLogExceptions(_linkedParseCts.Token), _linkedParseCts.Token); } - private void ParseAndLogExceptions(CancellationToken cancellationToken) { + protected void ParseAndLogExceptions(CancellationToken cancellationToken) { try { Parse(cancellationToken); } catch (Exception ex) when (!(ex is OperationCanceledException)) { @@ -325,6 +326,15 @@ private void ParseAndLogExceptions(CancellationToken cancellationToken) { } } + protected virtual void Analyze(PythonAst ast, int version) { + if (ContentState < State.Analyzing) { + ContentState = State.Analyzing; + + var analyzer = Services.GetService(); + analyzer.EnqueueDocumentForAnalysis(this, ast, version); + } + } + private void Parse(CancellationToken cancellationToken) { CollectingErrorSink sink = null; int version; @@ -345,7 +355,6 @@ private void Parse(CancellationToken cancellationToken) { } var ast = parser.ParseFile(Uri); - // Log?.Log(TraceEventType.Verbose, $"Parse complete: {Name} ({ModuleType})"); lock (_syncObj) { @@ -370,13 +379,7 @@ private void Parse(CancellationToken cancellationToken) { } NewAst?.Invoke(this, EventArgs.Empty); - - if (ContentState < State.Analyzing) { - ContentState = State.Analyzing; - - var analyzer = Services.GetService(); - analyzer.EnqueueDocumentForAnalysis(this, ast, version); - } + Analyze(ast, version); lock (_syncObj) { _parsingTask = null; @@ -423,6 +426,14 @@ public void NotifyAnalysisBegins() { } } + public ModuleWalker Analyze(PythonAst ast) { + var eval = new ExpressionEval(Services, this, ast); + var walker = new ModuleWalker(eval, SimpleImportedVariableHandler.Instance); + ast.Walk(walker); + walker.Complete(); + return walker; + } + public void NotifyAnalysisComplete(IDocumentAnalysis analysis) { lock (_syncObj) { if (analysis.Version < Analysis.Version) { diff --git a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs index f138c820b..12aa1e184 100644 --- a/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs +++ b/src/Analysis/Ast/Impl/Modules/Resolution/MainModuleResolution.cs @@ -39,6 +39,7 @@ namespace Microsoft.Python.Analysis.Modules.Resolution { internal sealed class MainModuleResolution : ModuleResolutionBase, IModuleManagement { private readonly ConcurrentDictionary _specialized = new ConcurrentDictionary(); private readonly IUIService _ui; + private BuiltinsPythonModule _builtins; private IModuleDatabaseService _dbService; private IRunningDocumentTable _rdt; @@ -53,7 +54,7 @@ public MainModuleResolution(string root, IServiceContainer services, ImmutableAr public string BuiltinModuleName => BuiltinTypeId.Unknown.GetModuleName(Interpreter.LanguageVersion); public ImmutableArray LibraryPaths { get; private set; } = ImmutableArray.Empty; - public IBuiltinsPythonModule BuiltinsModule { get; private set; } + public IBuiltinsPythonModule BuiltinsModule => _builtins; public IEnumerable GetImportedModules(CancellationToken cancellationToken) { foreach (var module in _specialized.Values) { @@ -185,12 +186,8 @@ public IPythonModule GetSpecializedModule(string fullName, bool allowCreation = public bool IsSpecializedModule(string fullName, string modulePath = null) => _specialized.ContainsKey(fullName); - private async Task AddBuiltinTypesToPathResolverAsync(CancellationToken cancellationToken = default) { - var analyzer = Services.GetService(); - await analyzer.GetAnalysisAsync(BuiltinsModule, Timeout.Infinite, cancellationToken); - + private void AddBuiltinTypesToPathResolver() { Check.InvalidOperation(!(BuiltinsModule.Analysis is EmptyAnalysis), "Builtins analysis did not complete correctly."); - // Add built-in module names var builtinModuleNamesMember = BuiltinsModule.GetAnyMember("__builtin_module_names__"); var value = builtinModuleNamesMember is IVariable variable ? variable.Value : builtinModuleNamesMember; @@ -222,16 +219,16 @@ public async Task ReloadAsync(CancellationToken cancellationToken = default) { ReloadModulePaths(addedRoots, cancellationToken); if (!builtinsIsCreated) { - var builtinsModule = CreateBuiltinsModule(Services, Interpreter, StubCache); - BuiltinsModule = builtinsModule; - builtinsRef = new ModuleRef(builtinsModule); + _builtins = CreateBuiltinsModule(Services, Interpreter, StubCache); + builtinsRef = new ModuleRef(_builtins); + _builtins.Initialize(); } Modules[BuiltinModuleName] = builtinsRef; - await AddBuiltinTypesToPathResolverAsync(cancellationToken); + AddBuiltinTypesToPathResolver(); } - private static IBuiltinsPythonModule CreateBuiltinsModule(IServiceContainer services, IPythonInterpreter interpreter, IStubCache stubCache) { + private static BuiltinsPythonModule CreateBuiltinsModule(IServiceContainer services, IPythonInterpreter interpreter, IStubCache stubCache) { var moduleName = BuiltinTypeId.Unknown.GetModuleName(interpreter.LanguageVersion); var modulePath = stubCache.GetCacheFilePath(interpreter.Configuration.InterpreterPath); return new BuiltinsPythonModule(moduleName, modulePath, services); From e3045074a9d817a08b1a080faae7a890a6229990 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Thu, 14 Nov 2019 21:22:24 -0800 Subject: [PATCH 5/7] Sync builtins --- src/LanguageServer/Impl/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageServer/Impl/Program.cs b/src/LanguageServer/Impl/Program.cs index 25528cdb5..85abd38b6 100644 --- a/src/LanguageServer/Impl/Program.cs +++ b/src/LanguageServer/Impl/Program.cs @@ -13,7 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -// #define WAIT_FOR_DEBUGGER +#define WAIT_FOR_DEBUGGER using System; using System.Diagnostics; From 6e1a119198191015dd3d628ef54d2f4a039c331a Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Fri, 15 Nov 2019 11:07:50 -0800 Subject: [PATCH 6/7] Usings --- src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs index a49273ebf..c8caf615c 100644 --- a/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs +++ b/src/Analysis/Ast/Impl/Modules/BuiltinsPythonModule.cs @@ -19,9 +19,6 @@ using System.Linq; using System.Threading; using Microsoft.Python.Analysis.Analyzer; -using Microsoft.Python.Analysis.Analyzer.Evaluation; -using Microsoft.Python.Analysis.Analyzer.Handlers; -using Microsoft.Python.Analysis.Documents; using Microsoft.Python.Analysis.Specializations; using Microsoft.Python.Analysis.Types; using Microsoft.Python.Analysis.Values; From 6093070396316208e112a9918002ed168a6a1bac Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Fri, 15 Nov 2019 11:08:42 -0800 Subject: [PATCH 7/7] Debug code --- src/LanguageServer/Impl/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageServer/Impl/Program.cs b/src/LanguageServer/Impl/Program.cs index 85abd38b6..25528cdb5 100644 --- a/src/LanguageServer/Impl/Program.cs +++ b/src/LanguageServer/Impl/Program.cs @@ -13,7 +13,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -#define WAIT_FOR_DEBUGGER +// #define WAIT_FOR_DEBUGGER using System; using System.Diagnostics;