From 51da19c06af2dbbe7007881056dfef538ce8dd78 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 23 Oct 2018 19:40:03 -0700 Subject: [PATCH 01/19] Walk --- .../Impl/Interpreter/Ast/AstAnalysisWalker.cs | 11 +++++-- .../Impl/Interpreter/Ast/NameLookupContext.cs | 5 ++- src/Analysis/Engine/Test/AstAnalysisTests.cs | 33 +++++++++++++++++++ src/LanguageServer/Impl/Program.cs | 2 +- .../TestData/AstAnalysis/ForwardRefFunc1.py | 10 ++++++ .../TestData/AstAnalysis/ForwardRefFunc2.py | 16 +++++++++ .../TestData/AstAnalysis/ForwardRefProp1.py | 11 +++++++ 7 files changed, 83 insertions(+), 5 deletions(-) create mode 100644 src/UnitTests/TestData/AstAnalysis/ForwardRefFunc1.py create mode 100644 src/UnitTests/TestData/AstAnalysis/ForwardRefFunc2.py create mode 100644 src/UnitTests/TestData/AstAnalysis/ForwardRefProp1.py diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs index 596a93512..713317eed 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs @@ -27,7 +27,7 @@ namespace Microsoft.PythonTools.Interpreter.Ast { class AstAnalysisWalker : PythonWalker { private readonly IPythonModule _module; private readonly Dictionary _members; - private readonly List _postWalkers = new List(); + private readonly List _functionWalkers = new List(); private readonly AnalysisLogWriter _log; private readonly Dictionary _typingScope; @@ -57,6 +57,7 @@ public AstAnalysisWalker( filePath, documentUri, includeLocationInfo, + _functionWalkers, log: warnAboutUndefinedValues ? _log : null ); _typingScope = new Dictionary(); @@ -85,7 +86,11 @@ public override void PostWalk(PythonAst node) { } public void Complete() { - foreach (var walker in _postWalkers) { + // Do not use foreach since walker list is dynamically + // modified and walkers are removed as they are done. + while (_functionWalkers.Count > 0) { + var walker = _functionWalkers[0]; + _functionWalkers.RemoveAt(0); walker.Walk(); } @@ -387,7 +392,7 @@ private IPythonFunctionOverload CreateFunctionOverload(NameLookupContext funcSco var overload = new AstPythonFunctionOverload(parameters, funcScope.GetLocOfName(node, node.NameExpression)); var funcWalk = new AstAnalysisFunctionWalker(funcScope, node, overload); - _postWalkers.Add(funcWalk); + _functionWalkers.Add(funcWalk); return overload; } diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/NameLookupContext.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/NameLookupContext.cs index 59a9372cc..b54ea9ee0 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/NameLookupContext.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/NameLookupContext.cs @@ -29,7 +29,7 @@ sealed class NameLookupContext { private readonly Stack> _scopes = new Stack>(); private readonly Lazy _builtinModule; private readonly AnalysisLogWriter _log; - + private readonly List _functionWalkers; internal readonly IPythonType _unknownType; public NameLookupContext( @@ -40,6 +40,7 @@ public NameLookupContext( string filePath, Uri documentUri, bool includeLocationInfo, + List functionWalkers, IPythonModule builtinModule = null, AnalysisLogWriter log = null ) { @@ -50,6 +51,7 @@ public NameLookupContext( FilePath = filePath; DocumentUri = documentUri; IncludeLocationInfo = includeLocationInfo; + _functionWalkers = functionWalkers; DefaultLookupOptions = LookupOptions.Normal; @@ -80,6 +82,7 @@ public NameLookupContext Clone(bool copyScopeContents = false) { FilePath, DocumentUri, IncludeLocationInfo, + _functionWalkers, _builtinModule.IsValueCreated ? _builtinModule.Value : null, _log ); diff --git a/src/Analysis/Engine/Test/AstAnalysisTests.cs b/src/Analysis/Engine/Test/AstAnalysisTests.cs index 81a3b268c..9eb54af6d 100644 --- a/src/Analysis/Engine/Test/AstAnalysisTests.cs +++ b/src/Analysis/Engine/Test/AstAnalysisTests.cs @@ -212,6 +212,39 @@ public async Task AstMultiValues() { } } + [TestMethod, Priority(0)] + public async Task AstForwardRefProperty1() { + using (var server = await CreateServerAsync()) { + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(@" +from ForwardRefProp1 import * +x = B().getA().methodA() +"); + analysis.Should().HaveVariable("x").OfTypes(BuiltinTypeId.Str); + } + } + + [TestMethod, Priority(0)] + public async Task AstForwardRefFunction1() { + using (var server = await CreateServerAsync()) { + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(@" +from ForwardRefFunc1 import * +x = B().getA().methodA() +"); + analysis.Should().HaveVariable("x").OfTypes(BuiltinTypeId.Str); + } + } + + [TestMethod, Priority(0)] + public async Task AstForwardRefFunction2() { + using (var server = await CreateServerAsync()) { + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(@" +from ForwardRefFunc2 import * +x = B().getA().methodA() +"); + analysis.Should().HaveVariable("x").OfTypes(BuiltinTypeId.Str); + } + } + [TestMethod, Priority(0)] public void AstImports() { var mod = Parse("Imports.py", PythonLanguageVersion.V35); diff --git a/src/LanguageServer/Impl/Program.cs b/src/LanguageServer/Impl/Program.cs index 27ab814f4..2ff35f851 100644 --- a/src/LanguageServer/Impl/Program.cs +++ b/src/LanguageServer/Impl/Program.cs @@ -14,7 +14,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -// #define WAIT_FOR_DEBUGGER +#define WAIT_FOR_DEBUGGER using System; using Microsoft.Python.LanguageServer.Services; diff --git a/src/UnitTests/TestData/AstAnalysis/ForwardRefFunc1.py b/src/UnitTests/TestData/AstAnalysis/ForwardRefFunc1.py new file mode 100644 index 000000000..8a3c55b08 --- /dev/null +++ b/src/UnitTests/TestData/AstAnalysis/ForwardRefFunc1.py @@ -0,0 +1,10 @@ +class A(object): + def methodA(self): + return 's' + +class B(object): + def getA(self): + return self.funcA() + + def funcA(self): + return A() diff --git a/src/UnitTests/TestData/AstAnalysis/ForwardRefFunc2.py b/src/UnitTests/TestData/AstAnalysis/ForwardRefFunc2.py new file mode 100644 index 000000000..a149c8c15 --- /dev/null +++ b/src/UnitTests/TestData/AstAnalysis/ForwardRefFunc2.py @@ -0,0 +1,16 @@ +class A(object): + def methodA(self): + return 's' + +class B(object): + def getA(self): + return self.func1() + + def func1(self): + return self.func2() + + def func2(self): + return self.func3() + + def func3(self): + return A() diff --git a/src/UnitTests/TestData/AstAnalysis/ForwardRefProp1.py b/src/UnitTests/TestData/AstAnalysis/ForwardRefProp1.py new file mode 100644 index 000000000..47e2051c9 --- /dev/null +++ b/src/UnitTests/TestData/AstAnalysis/ForwardRefProp1.py @@ -0,0 +1,11 @@ +class A(object): + def methodA(self): + return 's' + +class B(object): + def getA(self): + return self.propA + + @property + def propA(self): + return A() From 381c930e849fe288cff3ac61c2a8f6c0dc0aadca Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 23 Oct 2018 22:23:04 -0700 Subject: [PATCH 02/19] Tests --- .../Ast/AstAnalysisFunctionWalker.cs | 18 +- .../Ast/AstAnalysisFunctionWalkerSet.cs | 50 +++++ .../Impl/Interpreter/Ast/AstAnalysisWalker.cs | 10 +- .../Impl/Interpreter/Ast/AstPythonFunction.cs | 23 +- .../Impl/Interpreter/Ast/AstPythonProperty.cs | 21 +- .../Impl/Interpreter/Ast/NameLookupContext.cs | 196 +++++++++++------- .../Definitions/IBuiltinProperty.cs | 23 +- .../Definitions/IPythonFunction.cs | 41 ++-- 8 files changed, 214 insertions(+), 168 deletions(-) create mode 100644 src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs index 01724bbd2..e80c0fc57 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs @@ -22,7 +22,6 @@ namespace Microsoft.PythonTools.Interpreter.Ast { class AstAnalysisFunctionWalker : PythonWalker { - private readonly FunctionDefinition _target; private readonly NameLookupContext _scope; private readonly AstPythonFunctionOverload _overload; private AstPythonType _selfType; @@ -33,12 +32,11 @@ public AstAnalysisFunctionWalker( AstPythonFunctionOverload overload ) { _scope = scope ?? throw new ArgumentNullException(nameof(scope)); - _target = targetFunction ?? throw new ArgumentNullException(nameof(targetFunction)); + Target = targetFunction ?? throw new ArgumentNullException(nameof(targetFunction)); _overload = overload ?? throw new ArgumentNullException(nameof(overload)); } - public IList ReturnTypes => _overload.ReturnTypes; - public IPythonFunctionOverload Overload => _overload; + public FunctionDefinition Target { get; } private void GetMethodType(FunctionDefinition node, out bool classmethod, out bool staticmethod) { classmethod = false; @@ -51,7 +49,7 @@ private void GetMethodType(FunctionDefinition node, out bool classmethod, out bo var classmethodObj = _scope.Interpreter.GetBuiltinType(BuiltinTypeId.ClassMethod); var staticmethodObj = _scope.Interpreter.GetBuiltinType(BuiltinTypeId.StaticMethod); - foreach (var d in (_target.Decorators?.Decorators).MaybeEnumerate().ExcludeDefault()) { + foreach (var d in (Target.Decorators?.Decorators).MaybeEnumerate().ExcludeDefault()) { var m = _scope.GetValueFromExpression(d); if (m == classmethodObj) { classmethod = true; @@ -65,21 +63,21 @@ public void Walk() { var self = GetSelf(); _selfType = (self as AstPythonConstant)?.Type as AstPythonType; - _overload.ReturnTypes.AddRange(_scope.GetTypesFromAnnotation(_target.ReturnAnnotation).ExcludeDefault()); + _overload.ReturnTypes.AddRange(_scope.GetTypesFromAnnotation(Target.ReturnAnnotation).ExcludeDefault()); _scope.PushScope(); if (self != null) { - var p0 = _target.Parameters.FirstOrDefault(); + var p0 = Target.Parameters.FirstOrDefault(); if (p0 != null && !string.IsNullOrEmpty(p0.Name)) { _scope.SetInScope(p0.Name, self); } } - _target.Walk(this); + Target.Walk(this); _scope.PopScope(); } public override bool Walk(FunctionDefinition node) { - if (node != _target) { + if (node != Target) { // Do not walk nested functions (yet) return false; } @@ -171,7 +169,7 @@ public override bool Walk(ReturnStatement node) { private IMember GetSelf() { bool classmethod, staticmethod; - GetMethodType(_target, out classmethod, out staticmethod); + GetMethodType(Target, out classmethod, out staticmethod); var self = _scope.LookupNameInScopes("__class__", NameLookupContext.LookupOptions.Local); if (!staticmethod && !classmethod) { var cls = self as IPythonType; diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs new file mode 100644 index 000000000..ddc1b48de --- /dev/null +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs @@ -0,0 +1,50 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; +using System.Linq; +using Microsoft.PythonTools.Parsing.Ast; + +namespace Microsoft.PythonTools.Interpreter.Ast { + class AstAnalysisFunctionWalkerSet { + private readonly Dictionary _functionWalkers + = new Dictionary(); + + public void Add(AstAnalysisFunctionWalker walker) + => _functionWalkers[walker.Target] = walker; + + public void ProcessSet() { + // Do not use foreach since walker list is dynamically + // modified and walkers are removed as they are done. + while (_functionWalkers.Count > 0) { + var walker = _functionWalkers.First().Value; + ProcessWalker(walker); + } + } + + public void ProcessFunction(FunctionDefinition fn) { + if (_functionWalkers.TryGetValue(fn, out var w)) { + ProcessWalker(w); + } + } + + private void ProcessWalker(AstAnalysisFunctionWalker walker) { + // Remove walker before processing as to prevent reentrancy. + _functionWalkers.Remove(walker.Target); + walker.Walk(); + } + } +} diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs index 713317eed..5068b91bf 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs @@ -27,9 +27,9 @@ namespace Microsoft.PythonTools.Interpreter.Ast { class AstAnalysisWalker : PythonWalker { private readonly IPythonModule _module; private readonly Dictionary _members; - private readonly List _functionWalkers = new List(); private readonly AnalysisLogWriter _log; private readonly Dictionary _typingScope; + private readonly AstAnalysisFunctionWalkerSet _functionWalkers = new AstAnalysisFunctionWalkerSet(); private IPythonInterpreter _interpreter => Scope.Interpreter; private PythonAst _ast => Scope.Ast; @@ -86,13 +86,7 @@ public override void PostWalk(PythonAst node) { } public void Complete() { - // Do not use foreach since walker list is dynamically - // modified and walkers are removed as they are done. - while (_functionWalkers.Count > 0) { - var walker = _functionWalkers[0]; - _functionWalkers.RemoveAt(0); - walker.Walk(); - } + _functionWalkers.ProcessSet(); if (_module.Name != "typing" && Scope.FilePath.EndsWithOrdinal(".pyi", ignoreCase: true)) { // Do not expose members directly imported from typing diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunction.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunction.cs index 9458d6a15..f9c210a39 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunction.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunction.cs @@ -22,7 +22,7 @@ using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.PythonTools.Interpreter.Ast { - class AstPythonFunction : IPythonFunction, ILocatedMember, IHasQualifiedName { + class AstPythonFunction : IPythonFunction2, ILocatedMember, IHasQualifiedName { private readonly List _overloads; private readonly string _doc; @@ -35,13 +35,14 @@ ILocationInfo loc ) { DeclaringModule = declModule ?? throw new ArgumentNullException(nameof(declModule)); DeclaringType = declType; + FunctionDefinition = def; - Name = def.Name; + Name = FunctionDefinition.Name; if (Name == "__init__") { _doc = declType?.Documentation; } - foreach (var dec in (def.Decorators?.Decorators).MaybeEnumerate().ExcludeDefault().OfType()) { + foreach (var dec in (FunctionDefinition.Decorators?.Decorators).MaybeEnumerate().ExcludeDefault().OfType()) { if (dec.Name == "classmethod") { IsClassMethod = true; } else if (dec.Name == "staticmethod") { @@ -54,21 +55,9 @@ ILocationInfo loc Locations = loc != null ? new[] { loc } : Array.Empty(); } - internal AstPythonFunction(IPythonFunction original) { - DeclaringModule = original.DeclaringModule; - DeclaringType = original.DeclaringType; - Name = original.Name; - // Copy the null if _doc isn't set in the original; otherwise calculate the docs - _doc = (original is AstPythonFunction apf) ? apf._doc : original.Documentation; - IsClassMethod = original.IsClassMethod; - IsStatic = original.IsStatic; - _overloads = original.Overloads.ToList(); - Locations = (original as ILocatedMember)?.Locations ?? Array.Empty(); - } + public FunctionDefinition FunctionDefinition { get; } - internal void AddOverload(IPythonFunctionOverload overload) { - _overloads.Add(overload); - } + internal void AddOverload(IPythonFunctionOverload overload) => _overloads.Add(overload); public IPythonModule DeclaringModule {get;} public IPythonType DeclaringType {get;} diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonProperty.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonProperty.cs index 60c4fac7b..69f79bb1d 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonProperty.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonProperty.cs @@ -5,34 +5,31 @@ using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.PythonTools.Interpreter.Ast { - class AstPythonProperty : IBuiltinProperty, ILocatedMember { + class AstPythonProperty : IBuiltinProperty2, ILocatedMember { private IPythonFunctionOverload _getter; public AstPythonProperty( PythonAst ast, - FunctionDefinition getter, + FunctionDefinition definition, ILocationInfo location ) { - Documentation = getter.Documentation; IsReadOnly = true; Locations = new[] { location }; + FunctionDefinition = definition; } - public void AddOverload(IPythonFunctionOverload overload) { - if (_getter == null) { - _getter = overload; - } - } + public FunctionDefinition FunctionDefinition { get; } - public void MakeSettable() { - IsReadOnly = false; - } + public void AddOverload(IPythonFunctionOverload overload) + => _getter = _getter ?? overload; + + public void MakeSettable() => IsReadOnly = false; public IPythonType Type => _getter?.ReturnType.FirstOrDefault(); public bool IsStatic => false; - public string Documentation { get; } + public string Documentation => FunctionDefinition.Documentation; public string Description => Type == null ? "property of unknown type" : "property of type {0}".FormatUI(Type.Name); diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/NameLookupContext.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/NameLookupContext.cs index b54ea9ee0..a0960e8f9 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/NameLookupContext.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/NameLookupContext.cs @@ -9,7 +9,7 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. @@ -27,9 +27,10 @@ namespace Microsoft.PythonTools.Interpreter.Ast { sealed class NameLookupContext { private readonly Stack> _scopes = new Stack>(); + private readonly AstAnalysisFunctionWalkerSet _functionWalkers; private readonly Lazy _builtinModule; private readonly AnalysisLogWriter _log; - private readonly List _functionWalkers; + internal readonly IPythonType _unknownType; public NameLookupContext( @@ -40,7 +41,7 @@ public NameLookupContext( string filePath, Uri documentUri, bool includeLocationInfo, - List functionWalkers, + AstAnalysisFunctionWalkerSet functionWalkers, IPythonModule builtinModule = null, AnalysisLogWriter log = null ) { @@ -51,8 +52,8 @@ public NameLookupContext( FilePath = filePath; DocumentUri = documentUri; IncludeLocationInfo = includeLocationInfo; - _functionWalkers = functionWalkers; + _functionWalkers = functionWalkers; DefaultLookupOptions = LookupOptions.Normal; _unknownType = Interpreter.GetBuiltinType(BuiltinTypeId.Unknown) ?? @@ -85,10 +86,10 @@ public NameLookupContext Clone(bool copyScopeContents = false) { _functionWalkers, _builtinModule.IsValueCreated ? _builtinModule.Value : null, _log - ); - - ctxt.DefaultLookupOptions = DefaultLookupOptions; - ctxt.SuppressBuiltinLookup = SuppressBuiltinLookup; + ) { + DefaultLookupOptions = DefaultLookupOptions, + SuppressBuiltinLookup = SuppressBuiltinLookup + }; foreach (var scope in _scopes.Reverse()) { if (copyScopeContents) { @@ -115,11 +116,9 @@ public Dictionary PushScope(Dictionary scope = return scope; } - public Dictionary PopScope() { - return _scopes.Pop(); - } + public Dictionary PopScope() => _scopes.Pop(); - internal LocationInfo GetLoc(Node node) { + public LocationInfo GetLoc(Node node) { if (!IncludeLocationInfo) { return null; } @@ -132,7 +131,7 @@ internal LocationInfo GetLoc(Node node) { return new LocationInfo(FilePath, DocumentUri, start.Line, start.Column, end.Line, end.Column); } - internal LocationInfo GetLocOfName(Node node, NameExpression header) { + public LocationInfo GetLocOfName(Node node, NameExpression header) { var loc = GetLoc(node); if (loc == null || header == null) { return null; @@ -150,26 +149,6 @@ internal LocationInfo GetLocOfName(Node node, NameExpression header) { return loc; } - private string GetNameFromExpressionWorker(Expression expr) { - if (expr is NameExpression ne) { - return ne.Name; - } - - if (expr is MemberExpression me) { - return "{0}.{1}".FormatInvariant(GetNameFromExpressionWorker(me.Target), me.Name); - } - - throw new FormatException(); - } - - public string GetNameFromExpression(Expression expr) { - try { - return GetNameFromExpressionWorker(expr); - } catch (FormatException) { - return null; - } - } - public IEnumerable GetTypesFromAnnotation(Expression expr) { if (expr == null) { return Enumerable.Empty(); @@ -197,28 +176,42 @@ public IMember GetValueFromExpression(Expression expr, LookupOptions options) { return null; } - var m = GetValueFromName(expr as NameExpression, options) ?? - GetValueFromMember(expr as MemberExpression, options) ?? - GetValueFromCallable(expr as CallExpression, options) ?? - GetValueFromUnaryOp(expr as UnaryExpression, options) ?? - GetValueFromBinaryOp(expr, options) ?? - GetValueFromIndex(expr as IndexExpression, options) ?? - GetValueFromConditional(expr as ConditionalExpression, options) ?? - GetConstantFromLiteral(expr, options); - if (m != null) { - return m; + IMember m; + switch (expr) { + case NameExpression nex: + m = GetValueFromName(nex, options); + break; + case MemberExpression mex: + m = GetValueFromMember(mex, options); + break; + case CallExpression cex: + m = GetValueFromCallable(cex, options); + break; + case UnaryExpression uex: + m = GetValueFromUnaryOp(uex, options); + break; + case IndexExpression iex: + m = GetValueFromIndex(iex, options); + break; + case ConditionalExpression coex: + m = GetValueFromConditional(coex, options); + break; + default: + m = GetValueFromBinaryOp(expr, options) ?? GetConstantFromLiteral(expr, options); + break; } - - _log?.Log(TraceLevel.Verbose, "UnknownExpression", expr.ToCodeString(Ast).Trim()); - return null; + if (m == null) { + _log?.Log(TraceLevel.Verbose, "UnknownExpression", expr.ToCodeString(Ast).Trim()); + } + return m; } private IMember GetValueFromName(NameExpression expr, LookupOptions options) { - if (expr == null || string.IsNullOrEmpty(expr.Name)) { + if (string.IsNullOrEmpty(expr?.Name)) { return null; } - IMember existing = LookupNameInScopes(expr.Name, options); + var existing = LookupNameInScopes(expr.Name, options); if (existing != null) { return existing; } @@ -231,28 +224,40 @@ private IMember GetValueFromName(NameExpression expr, LookupOptions options) { } private IMember GetValueFromMember(MemberExpression expr, LookupOptions options) { - if (expr == null || expr.Target == null || string.IsNullOrEmpty(expr.Name)) { + if (expr?.Target == null || string.IsNullOrEmpty(expr?.Name)) { return null; } var e = GetValueFromExpression(expr.Target); - if (e is IMemberContainer mc) { - var m = mc.GetMember(Context, expr.Name); - if (m != null) { - return m; - } + IMember value = null; + switch (e) { + case IMemberContainer mc: + value = mc.GetMember(Context, expr.Name); + break; + case IPythonMultipleMembers mm: + value = mm.Members.OfType() + .Select(x => x.GetMember(Context, expr.Name)) + .ExcludeDefault() + .FirstOrDefault(); + break; + default: + value = GetValueFromPropertyOrFunction(e, expr); + break; + } + if(value is IBuiltinProperty2 p) { + value = GetPropertyReturnType(p, expr); + } + if (value == null) { _log?.Log(TraceLevel.Verbose, "UnknownMember", expr.ToCodeString(Ast).Trim()); - } else { - _log?.Log(TraceLevel.Verbose, "UnknownMemberContainer", expr.Target.ToCodeString(Ast).Trim()); + return new AstPythonConstant(_unknownType, GetLoc(expr)); } - return new AstPythonConstant(_unknownType, GetLoc(expr)); + return value; } private IMember GetValueFromUnaryOp(UnaryExpression expr, LookupOptions options) { if (expr == null || expr.Expression == null) { return null; } - // Assume that the type after the op is the same as before return GetValueFromExpression(expr.Expression); } @@ -349,27 +354,61 @@ private IMember GetValueFromCallable(CallExpression expr, LookupOptions options) } var m = GetValueFromExpression(expr.Target); - if (m is IPythonType type) { - if (type.TypeId == BuiltinTypeId.Type && type == Interpreter.GetBuiltinType(BuiltinTypeId.Type) && expr.Args.Count >= 1) { - var aType = GetTypeFromValue(GetValueFromExpression(expr.Args[0].Expression)); - if (aType != null) { - return aType; + IMember value = null; + switch (m) { + case IPythonType type: + if (type.TypeId == BuiltinTypeId.Type && type == Interpreter.GetBuiltinType(BuiltinTypeId.Type) && expr.Args.Count >= 1) { + value = GetTypeFromValue(GetValueFromExpression(expr.Args[0].Expression, options)); + } else { + value = new AstPythonConstant(type, GetLoc(expr)); } - } - return new AstPythonConstant(type, GetLoc(expr)); + break; + default: + value = GetValueFromPropertyOrFunction(m, expr); + break; + } + if (value == null) { + _log?.Log(TraceLevel.Verbose, "UnknownCallable", expr.Target.ToCodeString(Ast).Trim()); + return new AstPythonConstant(_unknownType, GetLoc(expr)); } + return value; + } - if (m is IPythonFunction fn) { - // TODO: Select correct overload and handle multiple return types - if (fn.Overloads.Count > 0 && fn.Overloads[0].ReturnType.Count > 0) { - return new AstPythonConstant(fn.Overloads[0].ReturnType[0]); + private IMember GetValueFromPropertyOrFunction(IMember fn, Expression expr) { + switch (fn) { + case IPythonBoundFunction bf: + return GetValueFromFunction(bf.Function as IPythonFunction2, expr); + case IPythonFunction2 f: + return GetValueFromFunction(f, expr); + case IBuiltinProperty2 p: + return GetPropertyReturnType(p, expr); + } + return null; + } + + private IMember GetValueFromFunction(IPythonFunction2 fn, Expression expr) { + var returnType = _unknownType; + if (fn != null) { + returnType = GetFunctionReturnType(fn.Overloads.FirstOrDefault()); + if (IsUnknown(returnType)) { + // Function may not have been walked yet. Do it now. + _functionWalkers.ProcessFunction(fn.FunctionDefinition); + returnType = GetFunctionReturnType(fn.Overloads.FirstOrDefault()); } - _log?.Log(TraceLevel.Verbose, "NoReturn", expr.Target.ToCodeString(Ast).Trim()); - return new AstPythonConstant(Interpreter.GetBuiltinType(BuiltinTypeId.NoneType), GetLoc(expr)); } + return new AstPythonConstant(returnType, GetLoc(expr)); + } - _log?.Log(TraceLevel.Verbose, "UnknownCallable", expr.Target.ToCodeString(Ast).Trim()); - return new AstPythonConstant(_unknownType, GetLoc(expr)); + private IPythonType GetFunctionReturnType(IPythonFunctionOverload o) + => o != null && o.ReturnType.Count > 0 ? o.ReturnType[0] : _unknownType; + + private IMember GetPropertyReturnType(IBuiltinProperty2 p, Expression expr) { + var returnType = p.Type; + if (IsUnknown(p.Type)) { + // Function may not have been walked yet. Do it now. + _functionWalkers.ProcessFunction(p.FunctionDefinition); + } + return new AstPythonConstant(p.Type ?? _unknownType, GetLoc(expr)); } public IPythonConstant GetConstantFromLiteral(Expression expr, LookupOptions options) { @@ -528,17 +567,16 @@ public IMember GetInScope(string name, Dictionary scope = null) return null; } - IMember obj; - if ((scope ?? _scopes.Peek()).TryGetValue(name, out obj)) { + if ((scope ?? _scopes.Peek()).TryGetValue(name, out var obj)) { return obj; } return null; } - private static bool IsUnknown(IMember value) { - return (value as IPythonType)?.TypeId == BuiltinTypeId.Unknown || - (value as IPythonConstant)?.Type?.TypeId == BuiltinTypeId.Unknown; - } + private static bool IsUnknown(IMember value) => + value == null || + (value as IPythonType)?.TypeId == BuiltinTypeId.Unknown || + (value as IPythonConstant)?.Type?.TypeId == BuiltinTypeId.Unknown; public void SetInScope(string name, IMember value, bool mergeWithExisting = true, Dictionary scope = null) { Debug.Assert(_scopes.Count > 0); diff --git a/src/Analysis/Engine/Impl/Interpreter/Definitions/IBuiltinProperty.cs b/src/Analysis/Engine/Impl/Interpreter/Definitions/IBuiltinProperty.cs index 0e5098879..e9cc0ab1f 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Definitions/IBuiltinProperty.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Definitions/IBuiltinProperty.cs @@ -9,11 +9,12 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.PythonTools.Interpreter { /// @@ -23,29 +24,25 @@ public interface IBuiltinProperty : IMember { /// /// The type of the value the property gets/sets. /// - IPythonType Type { - get; - } + IPythonType Type { get; } /// /// True if the property is static (declared on the class) not the instance. /// - bool IsStatic { - get; - } + bool IsStatic { get; } /// /// Documentation for the property. /// - string Documentation { - get; - } + string Documentation { get; } /// /// A user readable description of the property. /// - string Description { - get; - } + string Description { get; } + } + + public interface IBuiltinProperty2: IBuiltinProperty { + FunctionDefinition FunctionDefinition { get; } } } diff --git a/src/Analysis/Engine/Impl/Interpreter/Definitions/IPythonFunction.cs b/src/Analysis/Engine/Impl/Interpreter/Definitions/IPythonFunction.cs index fadf86826..69036e271 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Definitions/IPythonFunction.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Definitions/IPythonFunction.cs @@ -15,46 +15,29 @@ // permissions and limitations under the License. using System.Collections.Generic; +using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.PythonTools.Interpreter { /// /// Represents an object which is a function. Provides documentation for signature help. /// public interface IPythonFunction : IMember { - string Name { - get; - } + string Name { get; } + string Documentation { get; } + bool IsBuiltin { get; } - string Documentation { - get; - } - - bool IsBuiltin { - get; - } - /// /// False if binds instance when in a class, true if always static. /// - bool IsStatic { - get; - } - - bool IsClassMethod { - get; - } - - IReadOnlyList Overloads { - get; - } - - IPythonType DeclaringType { - get; - } + bool IsStatic { get; } + bool IsClassMethod { get; } + IReadOnlyList Overloads { get; } + IPythonType DeclaringType { get; } + IPythonModule DeclaringModule { get; } + } - IPythonModule DeclaringModule { - get; - } + public interface IPythonFunction2 : IPythonFunction { + FunctionDefinition FunctionDefinition { get; } } /// From 993f8bd60be0253719dad406bb269b3d506f40e5 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 23 Oct 2018 22:23:32 -0700 Subject: [PATCH 03/19] Undo --- src/LanguageServer/Impl/Program.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/LanguageServer/Impl/Program.cs b/src/LanguageServer/Impl/Program.cs index 2ff35f851..27ab814f4 100644 --- a/src/LanguageServer/Impl/Program.cs +++ b/src/LanguageServer/Impl/Program.cs @@ -14,7 +14,7 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -#define WAIT_FOR_DEBUGGER +// #define WAIT_FOR_DEBUGGER using System; using Microsoft.Python.LanguageServer.Services; From 99ee98a2cf7edcb27e439d7d9b470cac678d306e Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Tue, 23 Oct 2018 22:27:50 -0700 Subject: [PATCH 04/19] Comments --- .../Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs | 2 +- .../Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs | 6 ++++++ .../Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs | 3 +-- .../Impl/Interpreter/Ast/AstPythonFunctionOverload.cs | 2 +- src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonModule.cs | 2 +- src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs | 2 +- src/Analysis/Engine/Test/AstAnalysisTests.cs | 2 +- src/LanguageServer/Impl/Program.cs | 2 +- 8 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs index e80c0fc57..a7b274194 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs @@ -9,7 +9,7 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs index ddc1b48de..d4e9bfb55 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs @@ -19,6 +19,12 @@ using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.PythonTools.Interpreter.Ast { + /// + /// Represents set of function body walkers. Functions are walked after + /// all classes are collected. If function or property return type is unknown, + /// it can be walked, and so on recursively, until return type is determined + /// or there is nothing left to walk. + /// class AstAnalysisFunctionWalkerSet { private readonly Dictionary _functionWalkers = new Dictionary(); diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs index 1d3285fd4..0976d4c08 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs @@ -9,12 +9,11 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using Microsoft.PythonTools.Analysis; using Microsoft.PythonTools.Parsing.Ast; diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunctionOverload.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunctionOverload.cs index 56ac7c1e2..a0fe051a8 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunctionOverload.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunctionOverload.cs @@ -9,7 +9,7 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonModule.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonModule.cs index 3c4d73e39..85a3c3624 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonModule.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonModule.cs @@ -9,7 +9,7 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs index d2f9cbd51..269f7b090 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs @@ -9,7 +9,7 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. diff --git a/src/Analysis/Engine/Test/AstAnalysisTests.cs b/src/Analysis/Engine/Test/AstAnalysisTests.cs index 9eb54af6d..48713cec2 100644 --- a/src/Analysis/Engine/Test/AstAnalysisTests.cs +++ b/src/Analysis/Engine/Test/AstAnalysisTests.cs @@ -9,7 +9,7 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. diff --git a/src/LanguageServer/Impl/Program.cs b/src/LanguageServer/Impl/Program.cs index 27ab814f4..2021df3c6 100644 --- a/src/LanguageServer/Impl/Program.cs +++ b/src/LanguageServer/Impl/Program.cs @@ -9,7 +9,7 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. From d5774f31cfe8ce650a9075c16d88c0fbc93d90bb Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Fri, 26 Oct 2018 09:30:41 -0700 Subject: [PATCH 05/19] PR feedback --- .../Impl/Interpreter/Ast/AstPythonConstant.cs | 8 +-- .../Impl/Interpreter/Ast/NameLookupContext.cs | 54 ++++++++----------- 2 files changed, 23 insertions(+), 39 deletions(-) diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonConstant.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonConstant.cs index 504b6abbb..42e8009ee 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonConstant.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonConstant.cs @@ -29,7 +29,6 @@ public AstPythonConstant(IPythonType type, params ILocationInfo[] locations) { } public IEnumerable Locations { get; } - public PythonMemberType MemberType => PythonMemberType.Constant; public IPythonType Type { get; } @@ -42,19 +41,16 @@ public IMember GetMember(IModuleContext context, string name) { } m = Type?.GetMember(context, name); - if (m is IPythonFunction f && !f.IsStatic) { m = new AstPythonBoundMethod(f, Type); lock (_cachedMembers) { _cachedMembers[name] = m; } } - return m; } - public IEnumerable GetMemberNames(IModuleContext moduleContext) { - return Type?.GetMemberNames(moduleContext); - } + public IEnumerable GetMemberNames(IModuleContext moduleContext) + => Type?.GetMemberNames(moduleContext); } } diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/NameLookupContext.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/NameLookupContext.cs index a0960e8f9..6b1d206d1 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/NameLookupContext.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/NameLookupContext.cs @@ -244,10 +244,10 @@ private IMember GetValueFromMember(MemberExpression expr, LookupOptions options) value = GetValueFromPropertyOrFunction(e, expr); break; } - if(value is IBuiltinProperty2 p) { + + if (value is IBuiltinProperty2 p) { value = GetPropertyReturnType(p, expr); - } - if (value == null) { + } else if (value == null) { _log?.Log(TraceLevel.Verbose, "UnknownMember", expr.ToCodeString(Ast).Trim()); return new AstPythonConstant(_unknownType, GetLoc(expr)); } @@ -308,7 +308,7 @@ private IMember GetValueFromIndex(IndexExpression expr, LookupOptions options) { } var type = GetTypeFromValue(GetValueFromExpression(expr.Target)); - if (type != null && type != _unknownType) { + if (!IsUnknown(type)) { if (AstTypingModule.IsTypingType(type)) { return type; } @@ -334,7 +334,7 @@ private IMember GetValueFromIndex(IndexExpression expr, LookupOptions options) { } else { _log?.Log(TraceLevel.Verbose, "UnknownIndex", expr.ToCodeString(Ast, CodeFormattingOptions.Traditional).Trim()); } - return new AstPythonConstant(_unknownType, GetLoc(expr)); + return type; } private IMember GetValueFromConditional(ConditionalExpression expr, LookupOptions options) { @@ -356,28 +356,27 @@ private IMember GetValueFromCallable(CallExpression expr, LookupOptions options) var m = GetValueFromExpression(expr.Target); IMember value = null; switch (m) { + case IPythonType type when type == Interpreter.GetBuiltinType(BuiltinTypeId.Type) && expr.Args.Count >= 1: + value = GetTypeFromValue(GetValueFromExpression(expr.Args[0].Expression, options)); + break; case IPythonType type: - if (type.TypeId == BuiltinTypeId.Type && type == Interpreter.GetBuiltinType(BuiltinTypeId.Type) && expr.Args.Count >= 1) { - value = GetTypeFromValue(GetValueFromExpression(expr.Args[0].Expression, options)); - } else { - value = new AstPythonConstant(type, GetLoc(expr)); - } + value = new AstPythonConstant(type, GetLoc(expr)); break; default: value = GetValueFromPropertyOrFunction(m, expr); break; } + if (value == null) { _log?.Log(TraceLevel.Verbose, "UnknownCallable", expr.Target.ToCodeString(Ast).Trim()); - return new AstPythonConstant(_unknownType, GetLoc(expr)); } return value; } private IMember GetValueFromPropertyOrFunction(IMember fn, Expression expr) { switch (fn) { - case IPythonBoundFunction bf: - return GetValueFromFunction(bf.Function as IPythonFunction2, expr); + case IPythonBoundFunction bf when bf.Function is IPythonFunction2 f: + return GetValueFromFunction(f, expr); case IPythonFunction2 f: return GetValueFromFunction(f, expr); case IBuiltinProperty2 p: @@ -387,28 +386,24 @@ private IMember GetValueFromPropertyOrFunction(IMember fn, Expression expr) { } private IMember GetValueFromFunction(IPythonFunction2 fn, Expression expr) { - var returnType = _unknownType; - if (fn != null) { + var returnType = GetFunctionReturnType(fn.Overloads.FirstOrDefault()); + if (IsUnknown(returnType)) { + // Function may not have been walked yet. Do it now. + _functionWalkers.ProcessFunction(fn.FunctionDefinition); returnType = GetFunctionReturnType(fn.Overloads.FirstOrDefault()); - if (IsUnknown(returnType)) { - // Function may not have been walked yet. Do it now. - _functionWalkers.ProcessFunction(fn.FunctionDefinition); - returnType = GetFunctionReturnType(fn.Overloads.FirstOrDefault()); - } } - return new AstPythonConstant(returnType, GetLoc(expr)); + return !IsUnknown(returnType) ? new AstPythonConstant(returnType, GetLoc(expr)) : null; } private IPythonType GetFunctionReturnType(IPythonFunctionOverload o) => o != null && o.ReturnType.Count > 0 ? o.ReturnType[0] : _unknownType; private IMember GetPropertyReturnType(IBuiltinProperty2 p, Expression expr) { - var returnType = p.Type; if (IsUnknown(p.Type)) { // Function may not have been walked yet. Do it now. _functionWalkers.ProcessFunction(p.FunctionDefinition); } - return new AstPythonConstant(p.Type ?? _unknownType, GetLoc(expr)); + return !IsUnknown(p.Type) ? new AstPythonConstant(p.Type, GetLoc(expr)) : null; } public IPythonConstant GetConstantFromLiteral(Expression expr, LookupOptions options) { @@ -420,12 +415,8 @@ public IPythonConstant GetConstantFromLiteral(Expression expr, LookupOptions opt } } - var type = GetTypeFromLiteral(expr); - if (type != null) { - return new AstPythonConstant(type, GetLoc(expr)); - } - - return null; + var type = GetTypeFromLiteral(expr); + return type != null ? new AstPythonConstant(type, GetLoc(expr)) : null; } public IEnumerable GetTypesFromValue(IMember value) { @@ -614,8 +605,6 @@ public IMember LookupNameInScopes(string name) { } public IMember LookupNameInScopes(string name, LookupOptions options) { - IMember value; - var scopes = _scopes.ToList(); if (scopes.Count == 1) { if (!options.HasFlag(LookupOptions.Local) && !options.HasFlag(LookupOptions.Global)) { @@ -637,7 +626,7 @@ public IMember LookupNameInScopes(string name, LookupOptions options) { if (scopes != null) { foreach (var scope in scopes) { - if (scope.TryGetValue(name, out value) && value != null) { + if (scope.TryGetValue(name, out var value) && value != null) { if (value is ILazyMember lm) { value = lm.Get(); scope[name] = value; @@ -650,7 +639,6 @@ public IMember LookupNameInScopes(string name, LookupOptions options) { if (!SuppressBuiltinLookup && options.HasFlag(LookupOptions.Builtins)) { return _builtinModule.Value.GetMember(Context, name); } - return null; } } From 80aabe1f6db41e32cc1814d3849e3a5d1a7610f6 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Fri, 26 Oct 2018 23:11:56 -0700 Subject: [PATCH 06/19] Fix more --- .../Ast/AstAnalysisFunctionWalker.cs | 33 ++- .../Ast/AstAnalysisFunctionWalkerSet.cs | 16 +- .../Impl/Interpreter/Ast/AstAnalysisWalker.cs | 189 +++++++++--------- .../Ast/AstBuiltinsPythonModule.cs | 8 +- .../Impl/Interpreter/Ast/AstPythonModule.cs | 107 +--------- .../Interpreter/Ast/AstScrapedPythonModule.cs | 7 +- .../Ast/AstTypeAnnotationConverter.cs | 20 +- .../Impl/Interpreter/Ast/NameLookupContext.cs | 34 ++-- .../Interpreter/Ast/PythonModuleLoader.cs | 99 +++++++++ .../Impl/Parsing/Ast/FunctionDefinition.cs | 138 ++++--------- .../Impl/Parsing/Ast/PythonNameBinder.cs | 10 +- src/Analysis/Engine/Impl/PythonAnalyzer.cs | 75 +++---- src/Analysis/Engine/Test/AstAnalysisTests.cs | 13 +- .../AstAnalysis/ForwardRefGlobalFunc.py | 8 + 14 files changed, 375 insertions(+), 382 deletions(-) create mode 100644 src/Analysis/Engine/Impl/Interpreter/Ast/PythonModuleLoader.cs create mode 100644 src/UnitTests/TestData/AstAnalysis/ForwardRefGlobalFunc.py diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs index a7b274194..9eecd8018 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs @@ -16,11 +16,13 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using Microsoft.PythonTools.Analysis.Infrastructure; using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.PythonTools.Interpreter.Ast { + [DebuggerDisplay("{Target.Name}")] class AstAnalysisFunctionWalker : PythonWalker { private readonly NameLookupContext _scope; private readonly AstPythonFunctionOverload _overload; @@ -64,14 +66,24 @@ public void Walk() { _selfType = (self as AstPythonConstant)?.Type as AstPythonType; _overload.ReturnTypes.AddRange(_scope.GetTypesFromAnnotation(Target.ReturnAnnotation).ExcludeDefault()); - _scope.PushScope(); + + // Declare self, if any + var skip = 0; if (self != null) { var p0 = Target.Parameters.FirstOrDefault(); if (p0 != null && !string.IsNullOrEmpty(p0.Name)) { _scope.SetInScope(p0.Name, self); + skip++; } } + + // Declare parameters in scope + foreach(var p in Target.Parameters.Skip(skip).Where(p => !string.IsNullOrEmpty(p.Name))) { + var value = _scope.GetValueFromExpression(p.DefaultValue); + _scope.SetInScope(p.Name, value ?? _scope.UnknownType); + } + Target.Walk(this); _scope.PopScope(); } @@ -161,19 +173,28 @@ public override bool Walk(IfStatement node) { } public override bool Walk(ReturnStatement node) { - foreach (var type in _scope.GetTypesFromValue(_scope.GetValueFromExpression(node.Expression)).ExcludeDefault()) { + var types = _scope.GetTypesFromValue(_scope.GetValueFromExpression(node.Expression)).ExcludeDefault(); + foreach (var type in types) { _overload.ReturnTypes.Add(type); } + + // Clean up: if there are None or Unknown types along with real ones, remove them. + var realTypes = _overload.ReturnTypes + .Where(t => t.TypeId != BuiltinTypeId.Unknown && t.TypeId != BuiltinTypeId.NoneType) + .ToList(); + + if (realTypes.Count > 0) { + _overload.ReturnTypes.Clear(); + _overload.ReturnTypes.AddRange(realTypes); + } return true; // We want to evaluate all code so all private variables in __new__ get defined } private IMember GetSelf() { - bool classmethod, staticmethod; - GetMethodType(Target, out classmethod, out staticmethod); + GetMethodType(Target, out var classmethod, out var staticmethod); var self = _scope.LookupNameInScopes("__class__", NameLookupContext.LookupOptions.Local); if (!staticmethod && !classmethod) { - var cls = self as IPythonType; - if (cls == null) { + if (!(self is IPythonType cls)) { self = null; } else { self = new AstPythonConstant(cls, ((cls as ILocatedMember)?.Locations).MaybeEnumerate().ToArray()); diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs index d4e9bfb55..9d026442b 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs @@ -16,6 +16,7 @@ using System.Collections.Generic; using System.Linq; +using Microsoft.PythonTools.Analysis.Infrastructure; using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.PythonTools.Interpreter.Ast { @@ -33,8 +34,18 @@ public void Add(AstAnalysisFunctionWalker walker) => _functionWalkers[walker.Target] = walker; public void ProcessSet() { - // Do not use foreach since walker list is dynamically - // modified and walkers are removed as they are done. + // Do not use foreach since walker list is dynamically modified and walkers are removed + // after processing. Handle __init__ and __new__ first so class variables are initialized. + var constructors = _functionWalkers + .Where(kvp => kvp.Key.Name == "__init__" || kvp.Key.Name == "__new__") + .Select(c => c.Value) + .ExcludeDefault() + .ToArray(); + + foreach (var ctor in constructors) { + ProcessWalker(ctor); + } + while (_functionWalkers.Count > 0) { var walker = _functionWalkers.First().Value; ProcessWalker(walker); @@ -50,6 +61,7 @@ public void ProcessFunction(FunctionDefinition fn) { private void ProcessWalker(AstAnalysisFunctionWalker walker) { // Remove walker before processing as to prevent reentrancy. _functionWalkers.Remove(walker.Target); + var z = walker.Target.Name == "day"; walker.Walk(); } } diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs index 5068b91bf..6d9798b65 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs @@ -9,7 +9,7 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. @@ -28,12 +28,11 @@ class AstAnalysisWalker : PythonWalker { private readonly IPythonModule _module; private readonly Dictionary _members; private readonly AnalysisLogWriter _log; - private readonly Dictionary _typingScope; + private readonly Dictionary _typingScope = new Dictionary(); private readonly AstAnalysisFunctionWalkerSet _functionWalkers = new AstAnalysisFunctionWalkerSet(); - - private IPythonInterpreter _interpreter => Scope.Interpreter; - private PythonAst _ast => Scope.Ast; - private IPythonType _unknownType => Scope._unknownType; + private readonly NameLookupContext _scope; + private readonly PythonAst _ast; + private readonly IPythonInterpreter _interpreter; public AstAnalysisWalker( IPythonInterpreter interpreter, @@ -44,12 +43,13 @@ public AstAnalysisWalker( Dictionary members, bool includeLocationInfo, bool warnAboutUndefinedValues, + bool suppressBuiltinLookup, AnalysisLogWriter log = null ) { _log = log ?? (interpreter as AstPythonInterpreter)?.Log; _module = module ?? throw new ArgumentNullException(nameof(module)); _members = members ?? throw new ArgumentNullException(nameof(members)); - Scope = new NameLookupContext( + _scope = new NameLookupContext( interpreter ?? throw new ArgumentNullException(nameof(interpreter)), interpreter.CreateModuleContext(), ast ?? throw new ArgumentNullException(nameof(ast)), @@ -60,42 +60,43 @@ public AstAnalysisWalker( _functionWalkers, log: warnAboutUndefinedValues ? _log : null ); - _typingScope = new Dictionary(); - Scope.PushScope(_typingScope); + _ast = ast; + _interpreter = interpreter; + _scope.SuppressBuiltinLookup = suppressBuiltinLookup; + _scope.PushScope(_typingScope); WarnAboutUndefinedValues = warnAboutUndefinedValues; } public bool CreateBuiltinTypes { get; set; } public bool WarnAboutUndefinedValues { get; } - public NameLookupContext Scope { get; } public override bool Walk(PythonAst node) { if (_ast != node) { throw new InvalidOperationException("walking wrong AST"); } - FirstPassCollectClasses(); - Scope.PushScope(_members); + CollectTopLevelDefinitions(); + _scope.PushScope(_members); return base.Walk(node); } public override void PostWalk(PythonAst node) { - Scope.PopScope(); + _scope.PopScope(); base.PostWalk(node); } public void Complete() { _functionWalkers.ProcessSet(); - if (_module.Name != "typing" && Scope.FilePath.EndsWithOrdinal(".pyi", ignoreCase: true)) { + if (_module.Name != "typing" && _scope.FilePath.EndsWithOrdinal(".pyi", ignoreCase: true)) { // Do not expose members directly imported from typing _typingScope.Clear(); } } internal LocationInfo GetLoc(ClassDefinition node) { - if (!Scope.IncludeLocationInfo) { + if (!_scope.IncludeLocationInfo) { return null; } if (node == null || node.StartIndex >= node.EndIndex) { @@ -104,34 +105,33 @@ internal LocationInfo GetLoc(ClassDefinition node) { var start = node.NameExpression?.GetStart(_ast) ?? node.GetStart(_ast); var end = node.GetEnd(_ast); - return new LocationInfo(Scope.FilePath, Scope.DocumentUri, start.Line, start.Column, end.Line, end.Column); + return new LocationInfo(_scope.FilePath, _scope.DocumentUri, start.Line, start.Column, end.Line, end.Column); } - internal LocationInfo GetLoc(Node node) => Scope.GetLoc(node); - private IPythonType CurrentClass => Scope.GetInScope("__class__") as IPythonType; + private LocationInfo GetLoc(Node node) => _scope.GetLoc(node); private IMember Clone(IMember member) => member is IPythonMultipleMembers mm ? AstPythonMultipleMembers.Create(mm.Members) : member; public override bool Walk(AssignmentStatement node) { - var value = Scope.GetValueFromExpression(node.Right); + var value = _scope.GetValueFromExpression(node.Right); if ((value == null || value.MemberType == PythonMemberType.Unknown) && WarnAboutUndefinedValues) { _log?.Log(TraceLevel.Warning, "UndefinedValue", node.Right.ToCodeString(_ast).Trim()); } if ((value as IPythonConstant)?.Type?.TypeId == BuiltinTypeId.Ellipsis) { - value = _unknownType; + value = _scope.UnknownType; } foreach (var expr in node.Left.OfType()) { AssignFromAnnotation(expr); - if (value != _unknownType && expr.Expression is NameExpression ne) { - Scope.SetInScope(ne.Name, Clone(value)); + if (value != _scope.UnknownType && expr.Expression is NameExpression ne) { + _scope.SetInScope(ne.Name, Clone(value)); } } foreach (var ne in node.Left.OfType()) { - Scope.SetInScope(ne.Name, Clone(value)); + _scope.SetInScope(ne.Name, Clone(value)); } return base.Walk(node); @@ -148,13 +148,13 @@ private void AssignFromAnnotation(ExpressionWithAnnotation expr) { } if (expr.Expression is NameExpression ne) { - bool any = false; - foreach (var annType in Scope.GetTypesFromAnnotation(expr.Annotation)) { - Scope.SetInScope(ne.Name, new AstPythonConstant(annType, GetLoc(expr.Expression))); + var any = false; + foreach (var annType in _scope.GetTypesFromAnnotation(expr.Annotation)) { + _scope.SetInScope(ne.Name, new AstPythonConstant(annType, GetLoc(expr.Expression))); any = true; } if (!any) { - Scope.SetInScope(ne.Name, _unknownType); + _scope.SetInScope(ne.Name, _scope.UnknownType); } } } @@ -165,18 +165,18 @@ public override bool Walk(ImportStatement node) { } try { - for (int i = 0; i < node.Names.Count; ++i) { + for (var i = 0; i < node.Names.Count; ++i) { var n = node.AsNames?[i] ?? node.Names[i].Names[0]; if (n != null) { if (n.Name == "typing") { - Scope.SetInScope(n.Name, new AstTypingModule(), scope: _typingScope); + _scope.SetInScope(n.Name, new AstTypingModule(), scope: _typingScope); } else if (n.Name == _module.Name) { - Scope.SetInScope(n.Name, _module); + _scope.SetInScope(n.Name, _module); } else { - Scope.SetInScope(n.Name, new AstNestedPythonModule( + _scope.SetInScope(n.Name, new AstNestedPythonModule( _interpreter, n.Name, - ModuleResolver.ResolvePotentialModuleNames(_module.Name, Scope.FilePath, node.Names[i].MakeString(), true).ToArray() + ModuleResolver.ResolvePotentialModuleNames(_module.Name, _scope.FilePath, node.Names[i].MakeString(), true).ToArray() )); } } @@ -212,7 +212,7 @@ public override bool Walk(FromImportStatement node) { IPythonModule mod = null; Dictionary scope = null; - bool isTyping = modName == "typing"; + var isTyping = modName == "typing"; var warnAboutUnknownValues = WarnAboutUndefinedValues; if (isTyping) { mod = new AstTypingModule(); @@ -225,44 +225,44 @@ public override bool Walk(FromImportStatement node) { mod = mod ?? new AstNestedPythonModule( _interpreter, modName, - ModuleResolver.ResolvePotentialModuleNames(_module.Name, Scope.FilePath, modName, true).ToArray() + ModuleResolver.ResolvePotentialModuleNames(_module.Name, _scope.FilePath, modName, true).ToArray() ); foreach (var name in GetImportNames(node.Names, node.AsNames)) { if (name.Key == "*") { - mod.Imported(Scope.Context); + mod.Imported(_scope.Context); // Ensure child modules have been loaded mod.GetChildrenModules(); - foreach (var member in mod.GetMemberNames(Scope.Context)) { - var mem = mod.GetMember(Scope.Context, member); + foreach (var member in mod.GetMemberNames(_scope.Context)) { + var mem = mod.GetMember(_scope.Context, member); if (mem == null) { if (WarnAboutUndefinedValues) { _log?.Log(TraceLevel.Warning, "UndefinedImport", modName, member); } - mem = new AstPythonConstant(_unknownType, ((mod as ILocatedMember)?.Locations).MaybeEnumerate().ToArray()); + mem = new AstPythonConstant(_scope.UnknownType, ((mod as ILocatedMember)?.Locations).MaybeEnumerate().ToArray()); } else if (mem.MemberType == PythonMemberType.Unknown && warnAboutUnknownValues) { _log?.Log(TraceLevel.Warning, "UnknownImport", modName, member); } - Scope.SetInScope(member, mem, scope: scope); - (mem as IPythonModule)?.Imported(Scope.Context); + _scope.SetInScope(member, mem, scope: scope); + (mem as IPythonModule)?.Imported(_scope.Context); } } else { IMember mem; if (mod is AstNestedPythonModule m && !m.IsLoaded) { - mem = new AstNestedPythonModuleMember(name.Key, m, Scope.Context, GetLoc(name.Value)); + mem = new AstNestedPythonModuleMember(name.Key, m, _scope.Context, GetLoc(name.Value)); } else { - mem = mod.GetMember(Scope.Context, name.Key); + mem = mod.GetMember(_scope.Context, name.Key); if (mem == null) { if (WarnAboutUndefinedValues) { _log?.Log(TraceLevel.Warning, "UndefinedImport", modName, name); } - mem = new AstPythonConstant(_unknownType, GetLoc(name.Value)); + mem = new AstPythonConstant(_scope.UnknownType, GetLoc(name.Value)); } else if (mem.MemberType == PythonMemberType.Unknown && warnAboutUnknownValues) { _log?.Log(TraceLevel.Warning, "UnknownImport", modName, name); } - (mem as IPythonModule)?.Imported(Scope.Context); + (mem as IPythonModule)?.Imported(_scope.Context); } - Scope.SetInScope(name.Value.Name, mem, scope: scope); + _scope.SetInScope(name.Value.Name, mem, scope: scope); } } @@ -270,7 +270,7 @@ public override bool Walk(FromImportStatement node) { } public override bool Walk(IfStatement node) { - bool allValidComparisons = true; + var allValidComparisons = true; foreach (var test in node.TestsInternal) { if (test.Test is BinaryExpression cmp && cmp.Left is MemberExpression me && (me.Target as NameExpression)?.Name == "sys" && me.Name == "version_info" && @@ -286,7 +286,7 @@ public override bool Walk(IfStatement node) { return true; } - bool shouldWalk = false; + var shouldWalk = false; switch (cmp.Operator) { case PythonOperator.LessThan: shouldWalk = _ast.LanguageVersion.ToVersion() < v; @@ -320,7 +320,7 @@ public override bool Walk(FunctionDefinition node) { var dec = (node.Decorators?.Decorators).MaybeEnumerate().ExcludeDefault(); foreach (var d in dec) { - var obj = Scope.GetValueFromExpression(d); + var obj = _scope.GetValueFromExpression(d); if (obj == _interpreter.GetBuiltinType(BuiltinTypeId.Property)) { AddProperty(node); return false; @@ -334,7 +334,7 @@ public override bool Walk(FunctionDefinition node) { } foreach (var setter in dec.OfType().Where(n => n.Name == "setter")) { if (setter.Target is NameExpression src) { - var existingProp = Scope.LookupNameInScopes(src.Name, NameLookupContext.LookupOptions.Local) as AstPythonProperty; + var existingProp = _scope.LookupNameInScopes(src.Name, NameLookupContext.LookupOptions.Local) as AstPythonProperty; if (existingProp != null) { // Setter for an existing property, so don't create a function existingProp.MakeSettable(); @@ -343,50 +343,33 @@ public override bool Walk(FunctionDefinition node) { } } - var existing = Scope.LookupNameInScopes(node.Name, NameLookupContext.LookupOptions.Local) as AstPythonFunction; - if (existing == null) { - existing = new AstPythonFunction(_ast, _module, CurrentClass, node, GetLoc(node)); - Scope.SetInScope(node.Name, existing); - } - - var funcScope = Scope.Clone(); - if (CreateBuiltinTypes) { - funcScope.SuppressBuiltinLookup = true; - } - existing.AddOverload(CreateFunctionOverload(funcScope, node)); - + ProcessFunctionDefinition(node); // Do not recurse into functions return false; } private void AddProperty(FunctionDefinition node) { - var existing = Scope.LookupNameInScopes(node.Name, NameLookupContext.LookupOptions.Local) as AstPythonProperty; - + var existing = _scope.LookupNameInScopes(node.Name, NameLookupContext.LookupOptions.Local) as AstPythonProperty; if (existing == null) { existing = new AstPythonProperty(_ast, node, GetLoc(node)); - Scope.SetInScope(node.Name, existing); + _scope.SetInScope(node.Name, existing); } // Treat the rest of the property as a function. "AddOverload" takes the return type // and sets it as the property type. - var funcScope = Scope.Clone(); - if (CreateBuiltinTypes) { - funcScope.SuppressBuiltinLookup = true; - } + var funcScope = _scope.Clone(); + funcScope.SuppressBuiltinLookup = CreateBuiltinTypes; + existing.AddOverload(CreateFunctionOverload(funcScope, node)); } private IPythonFunctionOverload CreateFunctionOverload(NameLookupContext funcScope, FunctionDefinition node) { - var parameters = new List(); - foreach (var p in node.Parameters) { - var annType = Scope.GetTypesFromAnnotation(p.Annotation); - parameters.Add(new AstPythonParameterInfo(_ast, p, annType)); - } + var parameters = node.Parameters + .Select(p => new AstPythonParameterInfo(_ast, p, _scope.GetTypesFromAnnotation(p.Annotation))) + .ToArray(); var overload = new AstPythonFunctionOverload(parameters, funcScope.GetLocOfName(node, node.NameExpression)); - - var funcWalk = new AstAnalysisFunctionWalker(funcScope, node, overload); - _functionWalkers.Add(funcWalk); + _functionWalkers.Add(new AstAnalysisFunctionWalker(funcScope, node, overload)); return overload; } @@ -401,23 +384,23 @@ private AstPythonType CreateType(ClassDefinition node) { if (node == null) { throw new ArgumentNullException(nameof(node)); } - if (CreateBuiltinTypes) { - return new AstPythonBuiltinType(_ast, _module, node, GetDoc(node.Body as SuiteStatement), GetLoc(node)); - } - - return new AstPythonType(_ast, _module, node, GetDoc(node.Body as SuiteStatement), GetLoc(node)); + return CreateBuiltinTypes + ? new AstPythonBuiltinType(_ast, _module, node, GetDoc(node.Body as SuiteStatement), GetLoc(node)) + : new AstPythonType(_ast, _module, node, GetDoc(node.Body as SuiteStatement), GetLoc(node)); } + private void CollectTopLevelDefinitions() { + var s = (_ast.Body as SuiteStatement).Statements.ToArray(); + + foreach (var node in s.OfType()) { + ProcessFunctionDefinition(node); + } - public void FirstPassCollectClasses() { - foreach (var node in (_ast.Body as SuiteStatement).Statements.OfType()) { + foreach (var node in s.OfType()) { _members[node.Name] = CreateType(node); } - foreach (var node in (_ast.Body as SuiteStatement).Statements.OfType()) { - var rhs = node.Right as NameExpression; - if (rhs == null) { - continue; - } + foreach (var node in s.OfType().Where(n => n.Right is NameExpression)) { + var rhs = (NameExpression)node.Right; if (_members.TryGetValue(rhs.Name, out var member)) { foreach (var lhs in node.Left.OfType()) { _members[lhs.Name] = member; @@ -427,17 +410,17 @@ public void FirstPassCollectClasses() { } public override bool Walk(ClassDefinition node) { - var member = Scope.GetInScope(node.Name); - AstPythonType t = member as AstPythonType ?? + var member = _scope.GetInScope(node.Name); + var t = member as AstPythonType ?? (member as IPythonMultipleMembers)?.Members.OfType().FirstOrDefault(pt => pt.StartIndex == node.StartIndex); if (t == null) { t = CreateType(node); - Scope.SetInScope(node.Name, t); + _scope.SetInScope(node.Name, t); } var bases = node.Bases.Where(a => string.IsNullOrEmpty(a.Name)) // We cheat slightly and treat base classes as annotations. - .SelectMany(a => Scope.GetTypesFromAnnotation(a.Expression)) + .SelectMany(a => _scope.GetTypesFromAnnotation(a.Expression)) .ToArray(); try { @@ -446,19 +429,33 @@ public override bool Walk(ClassDefinition node) { // Bases were set while we were working } - Scope.PushScope(); - Scope.SetInScope("__class__", t); + _scope.PushScope(); + _scope.SetInScope("__class__", t); return true; } public override void PostWalk(ClassDefinition node) { - var cls = CurrentClass as AstPythonType; - var m = Scope.PopScope(); + var cls = _scope.GetInScope("__class__") as AstPythonType; // Store before popping the scope + var m = _scope.PopScope(); if (cls != null && m != null) { cls.AddMembers(m, true); } base.PostWalk(node); } + + public void ProcessFunctionDefinition(FunctionDefinition node) { + var existing = _scope.LookupNameInScopes(node.Name, NameLookupContext.LookupOptions.Local) as AstPythonFunction; + if (existing == null) { + var cls = _scope.GetInScope("__class__") as IPythonType; + existing = new AstPythonFunction(_ast, _module, cls, node, GetLoc(node)); + _scope.SetInScope(node.Name, existing); + } + + var funcScope = _scope.Clone(); + funcScope.SuppressBuiltinLookup = CreateBuiltinTypes; + + existing.AddOverload(CreateFunctionOverload(funcScope, node)); + } } } diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstBuiltinsPythonModule.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstBuiltinsPythonModule.cs index d1472f981..2e0f5d5d0 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstBuiltinsPythonModule.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstBuiltinsPythonModule.cs @@ -89,9 +89,13 @@ protected override PythonWalker PrepareWalker(IPythonInterpreter interpreter, Py const bool includeLocations = false; #endif - var walker = new AstAnalysisWalker(interpreter, ast, this, filePath, null, _members, includeLocations, true); + var walker = new AstAnalysisWalker( + interpreter, ast, this, filePath, null, _members, + includeLocations, + warnAboutUndefinedValues: true, + suppressBuiltinLookup: true + ); walker.CreateBuiltinTypes = true; - walker.Scope.SuppressBuiltinLookup = true; return walker; } diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonModule.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonModule.cs index 85a3c3624..db99531bd 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonModule.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonModule.cs @@ -22,105 +22,20 @@ using System.Threading; using Microsoft.PythonTools.Analysis; using Microsoft.PythonTools.Analysis.Infrastructure; -using Microsoft.PythonTools.Parsing; using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.PythonTools.Interpreter.Ast { - public static class PythonModuleLoader { - public static IPythonModule FromFile( - IPythonInterpreter interpreter, - string sourceFile, - PythonLanguageVersion langVersion - ) { - return FromFile(interpreter, sourceFile, langVersion, null); - } - - public static IPythonModule FromFile( - IPythonInterpreter interpreter, - string sourceFile, - PythonLanguageVersion langVersion, - string moduleFullName - ) { - Stream stream = null; - try { - if(Directory.Exists(sourceFile)) { - stream = new MemoryStream(); // Module without __init__.py, create empty stream - } else { - stream = new FileStream(sourceFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); - } - return FromStream(interpreter, stream, sourceFile, langVersion, moduleFullName); - } finally { - stream?.Dispose(); - } - } - - public static IPythonModule FromStream( - IPythonInterpreter interpreter, - Stream sourceFile, - string fileName, - PythonLanguageVersion langVersion - ) { - return FromStream(interpreter, sourceFile, fileName, langVersion, null); - } - - public static IPythonModule FromStream( - IPythonInterpreter interpreter, - Stream sourceFile, - string fileName, - PythonLanguageVersion langVersion, - string moduleFullName - ) { - PythonAst ast; - var sink = KeepParseErrors ? new CollectingErrorSink() : ErrorSink.Null; - var parser = Parser.CreateParser(sourceFile, langVersion, new ParserOptions { - StubFile = fileName.EndsWithOrdinal(".pyi", ignoreCase: true), - ErrorSink = sink - }); - ast = parser.ParseFile(); - - return new AstPythonModule( - moduleFullName ?? ModulePath.FromFullPath(fileName, isPackage: IsPackageCheck).FullName, - interpreter, - ast, - fileName, - (sink as CollectingErrorSink)?.Errors.Select(e => "{0} ({1}): {2}".FormatUI(fileName ?? "(builtins)", e.Span, e.Message)) - ); - } - - public static IPythonModule FromTypeStub( - IPythonInterpreter interpreter, - string stubFile, - PythonLanguageVersion langVersion, - string moduleFullName - ) { - return new AstCachedPythonModule(moduleFullName, stubFile); - } - - // Avoid hitting the filesystem, but exclude non-importable - // paths. Ideally, we'd stop at the first path that's a known - // search path, except we don't know search paths here. - private static bool IsPackageCheck(string path) { - return ModulePath.IsImportable(PathUtils.GetFileName(path.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar))); - } - - internal static bool KeepParseErrors = false; - } - sealed class AstPythonModule : IPythonModule, IProjectEntry, ILocatedMember { private readonly IPythonInterpreter _interpreter; - private readonly Dictionary _properties; - private readonly List _childModules; - private readonly Dictionary _members; + private readonly List _childModules = new List(); + private readonly Dictionary _members = new Dictionary(); private bool _foundChildModules; private string _documentation = string.Empty; internal AstPythonModule() { Name = string.Empty; FilePath = string.Empty; - _properties = new Dictionary(); - _childModules = new List(); _foundChildModules = true; - _members = new Dictionary(); } internal AstPythonModule(string moduleName, IPythonInterpreter interpreter, PythonAst ast, string filePath, IEnumerable parseErrors) { @@ -131,16 +46,16 @@ internal AstPythonModule(string moduleName, IPythonInterpreter interpreter, Pyth Locations = new[] { new LocationInfo(filePath, DocumentUri, 1, 1) }; _interpreter = interpreter; - _properties = new Dictionary(); - _childModules = new List(); - _members = new Dictionary(); - // Do not allow children of named modules - if (!ModulePath.IsInitPyFile(FilePath)) { - _foundChildModules = true; - } + _foundChildModules = !ModulePath.IsInitPyFile(FilePath); + + var walker = new AstAnalysisWalker( + interpreter, ast, this, filePath, DocumentUri, _members, + includeLocationInfo: true, + warnAboutUndefinedValues: true, + suppressBuiltinLookup: false + ); - var walker = new AstAnalysisWalker(interpreter, ast, this, filePath, DocumentUri, _members, true, true); ast.Walk(walker); walker.Complete(); @@ -178,7 +93,7 @@ public string Documentation { public string FilePath { get; } public Uri DocumentUri { get; } public PythonMemberType MemberType => PythonMemberType.Module; - public Dictionary Properties => _properties; + public Dictionary Properties { get; } = new Dictionary(); public IEnumerable Locations { get; } public int AnalysisVersion => 1; diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstScrapedPythonModule.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstScrapedPythonModule.cs index 0fc2b462d..ab6210430 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstScrapedPythonModule.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstScrapedPythonModule.cs @@ -127,7 +127,12 @@ protected virtual PythonWalker PrepareWalker(IPythonInterpreter interpreter, Pyt const Uri uri = null; const bool includeLocations = false; #endif - return new AstAnalysisWalker(interpreter, ast, this, filePath, uri, _members, includeLocations, true); + return new AstAnalysisWalker( + interpreter, ast, this, filePath, uri, _members, + includeLocations, + warnAboutUndefinedValues: true, + suppressBuiltinLookup: false + ); } protected virtual void PostWalk(PythonWalker walker) { diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs index cf4a80797..1d9df306e 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs @@ -50,7 +50,7 @@ public override IPythonType Finalize(IPythonType type) { return null; } - if (type == _scope._unknownType) { + if (type == _scope.UnknownType) { return null; } @@ -86,21 +86,17 @@ public override IPythonType LookupName(string name) { return m as IPythonType; } - public override IPythonType GetTypeMember(IPythonType baseType, string member) { - return AsIPythonType(baseType.GetMember(_scope.Context, member)); - } + public override IPythonType GetTypeMember(IPythonType baseType, string member) + => AsIPythonType(baseType.GetMember(_scope.Context, member)); public override IPythonType MakeNameType(string name) => new NameType(name); public override string GetName(IPythonType type) => (type as NameType)?.Name; - public override IPythonType MakeUnion(IReadOnlyList types) { - return new UnionType(types); - } + public override IPythonType MakeUnion(IReadOnlyList types) => new UnionType(types); - public override IReadOnlyList GetUnionTypes(IPythonType unionType) { - return (unionType as UnionType)?.Types ?? - (unionType as IPythonMultipleMembers)?.Members.OfType().ToArray(); - } + public override IReadOnlyList GetUnionTypes(IPythonType unionType) => + (unionType as UnionType)?.Types ?? + (unionType as IPythonMultipleMembers)?.Members.OfType().ToArray(); public override IPythonType MakeGeneric(IPythonType baseType, IReadOnlyList args) { if (args == null || args.Count == 0 || baseType == null) { @@ -127,7 +123,7 @@ public override IPythonType MakeGeneric(IPythonType baseType, IReadOnlyList _builtinModule; private readonly AnalysisLogWriter _log; - internal readonly IPythonType _unknownType; + internal IPythonType UnknownType { get; } public NameLookupContext( IPythonInterpreter interpreter, @@ -56,7 +56,7 @@ public NameLookupContext( _functionWalkers = functionWalkers; DefaultLookupOptions = LookupOptions.Normal; - _unknownType = Interpreter.GetBuiltinType(BuiltinTypeId.Unknown) ?? + UnknownType = Interpreter.GetBuiltinType(BuiltinTypeId.Unknown) ?? new FallbackBuiltinPythonType(new FallbackBuiltinModule(Ast.LanguageVersion), BuiltinTypeId.Unknown); _builtinModule = builtinModule == null ? new Lazy(ImportBuiltinModule) : new Lazy(() => builtinModule); @@ -220,7 +220,7 @@ private IMember GetValueFromName(NameExpression expr, LookupOptions options) { return Module; } _log?.Log(TraceLevel.Verbose, "UnknownName", expr.Name); - return new AstPythonConstant(_unknownType, GetLoc(expr)); + return new AstPythonConstant(UnknownType, GetLoc(expr)); } private IMember GetValueFromMember(MemberExpression expr, LookupOptions options) { @@ -249,7 +249,7 @@ private IMember GetValueFromMember(MemberExpression expr, LookupOptions options) value = GetPropertyReturnType(p, expr); } else if (value == null) { _log?.Log(TraceLevel.Verbose, "UnknownMember", expr.ToCodeString(Ast).Trim()); - return new AstPythonConstant(_unknownType, GetLoc(expr)); + return new AstPythonConstant(UnknownType, GetLoc(expr)); } return value; } @@ -334,7 +334,7 @@ private IMember GetValueFromIndex(IndexExpression expr, LookupOptions options) { } else { _log?.Log(TraceLevel.Verbose, "UnknownIndex", expr.ToCodeString(Ast, CodeFormattingOptions.Traditional).Trim()); } - return type; + return null; } private IMember GetValueFromConditional(ConditionalExpression expr, LookupOptions options) { @@ -342,10 +342,12 @@ private IMember GetValueFromConditional(ConditionalExpression expr, LookupOption return null; } - return AstPythonMultipleMembers.Combine( - GetValueFromExpression(expr.TrueExpression), - GetValueFromExpression(expr.FalseExpression) - ); + var trueValue = GetValueFromExpression(expr.TrueExpression); + var falseValue = GetValueFromExpression(expr.FalseExpression); + + return trueValue != null || falseValue != null + ? AstPythonMultipleMembers.Combine(trueValue, falseValue) + : null; } private IMember GetValueFromCallable(CallExpression expr, LookupOptions options) { @@ -381,6 +383,8 @@ private IMember GetValueFromPropertyOrFunction(IMember fn, Expression expr) { return GetValueFromFunction(f, expr); case IBuiltinProperty2 p: return GetPropertyReturnType(p, expr); + case IPythonConstant c when c.Type?.MemberType == PythonMemberType.Class: + return c.Type; // typically cls() } return null; } @@ -396,7 +400,7 @@ private IMember GetValueFromFunction(IPythonFunction2 fn, Expression expr) { } private IPythonType GetFunctionReturnType(IPythonFunctionOverload o) - => o != null && o.ReturnType.Count > 0 ? o.ReturnType[0] : _unknownType; + => o != null && o.ReturnType.Count > 0 ? o.ReturnType[0] : UnknownType; private IMember GetPropertyReturnType(IBuiltinProperty2 p, Expression expr) { if (IsUnknown(p.Type)) { @@ -415,8 +419,8 @@ public IPythonConstant GetConstantFromLiteral(Expression expr, LookupOptions opt } } - var type = GetTypeFromLiteral(expr); - return type != null ? new AstPythonConstant(type, GetLoc(expr)) : null; + var type = GetTypeFromLiteral(expr); + return type != null ? new AstPythonConstant(type, GetLoc(expr)) : null; } public IEnumerable GetTypesFromValue(IMember value) { @@ -579,7 +583,7 @@ public void SetInScope(string name, IMember value, bool mergeWithExisting = true s.Remove(name); return; } - if (mergeWithExisting && s.TryGetValue(name, out IMember existing) && existing != null) { + if (mergeWithExisting && s.TryGetValue(name, out var existing) && existing != null) { if (IsUnknown(existing)) { s[name] = value; } else if (!IsUnknown(value)) { @@ -600,9 +604,7 @@ public enum LookupOptions { Normal = Local | Nonlocal | Global | Builtins } - public IMember LookupNameInScopes(string name) { - return LookupNameInScopes(name, DefaultLookupOptions); - } + public IMember LookupNameInScopes(string name) => LookupNameInScopes(name, DefaultLookupOptions); public IMember LookupNameInScopes(string name, LookupOptions options) { var scopes = _scopes.ToList(); diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/PythonModuleLoader.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/PythonModuleLoader.cs new file mode 100644 index 000000000..1ddc468e1 --- /dev/null +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/PythonModuleLoader.cs @@ -0,0 +1,99 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.IO; +using System.Linq; +using Microsoft.PythonTools.Analysis; +using Microsoft.PythonTools.Analysis.Infrastructure; +using Microsoft.PythonTools.Parsing; +using Microsoft.PythonTools.Parsing.Ast; + +namespace Microsoft.PythonTools.Interpreter.Ast { + public static class PythonModuleLoader { + public static IPythonModule FromFile( + IPythonInterpreter interpreter, + string sourceFile, + PythonLanguageVersion langVersion + ) => FromFile(interpreter, sourceFile, langVersion, null); + + public static IPythonModule FromFile( + IPythonInterpreter interpreter, + string sourceFile, + PythonLanguageVersion langVersion, + string moduleFullName + ) { + Stream stream = null; + try { + if (Directory.Exists(sourceFile)) { + stream = new MemoryStream(); // Module without __init__.py, create empty stream + } else { + stream = new FileStream(sourceFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite); + } + return FromStream(interpreter, stream, sourceFile, langVersion, moduleFullName); + } finally { + stream?.Dispose(); + } + } + + public static IPythonModule FromStream( + IPythonInterpreter interpreter, + Stream sourceFile, + string fileName, + PythonLanguageVersion langVersion + ) { + return FromStream(interpreter, sourceFile, fileName, langVersion, null); + } + + public static IPythonModule FromStream( + IPythonInterpreter interpreter, + Stream sourceFile, + string fileName, + PythonLanguageVersion langVersion, + string moduleFullName + ) { + PythonAst ast; + var sink = KeepParseErrors ? new CollectingErrorSink() : ErrorSink.Null; + var parser = Parser.CreateParser(sourceFile, langVersion, new ParserOptions { + StubFile = fileName.EndsWithOrdinal(".pyi", ignoreCase: true), + ErrorSink = sink + }); + ast = parser.ParseFile(); + + return new AstPythonModule( + moduleFullName ?? ModulePath.FromFullPath(fileName, isPackage: IsPackageCheck).FullName, + interpreter, + ast, + fileName, + (sink as CollectingErrorSink)?.Errors.Select(e => "{0} ({1}): {2}".FormatUI(fileName ?? "(builtins)", e.Span, e.Message)) + ); + } + + public static IPythonModule FromTypeStub( + IPythonInterpreter interpreter, + string stubFile, + PythonLanguageVersion langVersion, + string moduleFullName + ) => new AstCachedPythonModule(moduleFullName, stubFile); + + // Avoid hitting the filesystem, but exclude non-importable + // paths. Ideally, we'd stop at the first path that's a known + // search path, except we don't know search paths here. + private static bool IsPackageCheck(string path) + => ModulePath.IsImportable(PathUtils.GetFileName(path.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar))); + + internal static bool KeepParseErrors = false; + } +} diff --git a/src/Analysis/Engine/Impl/Parsing/Ast/FunctionDefinition.cs b/src/Analysis/Engine/Impl/Parsing/Ast/FunctionDefinition.cs index d0c32ee05..c1aa96710 100644 --- a/src/Analysis/Engine/Impl/Parsing/Ast/FunctionDefinition.cs +++ b/src/Analysis/Engine/Impl/Parsing/Ast/FunctionDefinition.cs @@ -15,53 +15,38 @@ // permissions and limitations under the License. using System; -using System.Collections.Generic; +using System.Diagnostics; using System.Text; using Microsoft.PythonTools.Analysis.Infrastructure; namespace Microsoft.PythonTools.Parsing.Ast { - + [DebuggerDisplay("{Name}")] public class FunctionDefinition : ScopeStatement, IMaybeAsyncStatement { - protected Statement _body; - private readonly NameExpression/*!*/ _name; + internal static readonly object WhitespaceAfterAsync = new object(); + private readonly Parameter[] _parameters; - private Expression _returnAnnotation; - private DecoratorStatement _decorators; - private bool _generator; // The function is a generator - private bool _coroutine; - private bool _isLambda; - - private PythonVariable _variable; // The variable corresponding to the function name or null for lambdas - internal PythonVariable _nameVariable; // the variable that refers to the global __name__ - internal bool _hasReturn; - private int _headerIndex; - private int _defIndex; private int? _keywordEndIndex; - internal static readonly object WhitespaceAfterAsync = new object(); + protected Statement _body; public FunctionDefinition(NameExpression name, Parameter[] parameters) - : this(name, parameters, (Statement)null) { + : this(name, parameters, (Statement)null) { } - + public FunctionDefinition(NameExpression name, Parameter[] parameters, Statement body, DecoratorStatement decorators = null) { if (name == null) { - _name = new NameExpression(""); - _isLambda = true; + NameExpression = new NameExpression(""); + IsLambda = true; } else { - _name = name; + NameExpression = name; } _parameters = parameters; _body = body; - _decorators = decorators; + Decorators = decorators; } - public bool IsLambda { - get { - return _isLambda; - } - } + public bool IsLambda { get; } public Parameter[] Parameters => _parameters ?? Array.Empty(); @@ -71,41 +56,21 @@ public bool IsLambda { public override int KeywordEndIndex => _keywordEndIndex ?? (DefIndex + (IsCoroutine ? 9 : 3)); public override int KeywordLength => KeywordEndIndex - StartIndex; - public Expression ReturnAnnotation { - get { return _returnAnnotation; } - set { _returnAnnotation = value; } - } + public Expression ReturnAnnotation { get; set; } - public override Statement Body { - get { return _body; } - } + public override Statement Body => _body; - internal void SetBody(Statement body) { - _body = body; - } + internal void SetBody(Statement body) => _body = body; - public int HeaderIndex { - get { return _headerIndex; } - set { _headerIndex = value; } - } + public int HeaderIndex { get; set; } - public int DefIndex { - get { return _defIndex; } - set { _defIndex = value; } - } + public int DefIndex { get; set; } - public override string/*!*/ Name { - get { return _name.Name ?? ""; } - } + public override string/*!*/ Name => NameExpression.Name ?? ""; - public NameExpression NameExpression { - get { return _name; } - } + public NameExpression NameExpression { get; } - public DecoratorStatement Decorators { - get { return _decorators; } - internal set { _decorators = value; } - } + public DecoratorStatement Decorators { get; internal set; } internal LambdaExpression LambdaExpression { get; set; } @@ -114,29 +79,20 @@ public DecoratorStatement Decorators { /// expression and instead of returning a value when called they return a generator /// object which implements the iterator protocol. /// - public bool IsGenerator { - get { return _generator; } - set { _generator = value; } - } + public bool IsGenerator { get; set; } /// /// True if the function is a coroutine. Coroutines are defined using /// 'async def'. /// - public bool IsCoroutine { - get { return _coroutine; } - set { _coroutine = value; } - } + public bool IsCoroutine { get; set; } bool IMaybeAsyncStatement.IsAsync => IsCoroutine; /// /// Gets the variable that this function is assigned to. /// - public PythonVariable Variable { - get { return _variable; } - set { _variable = value; } - } + public PythonVariable Variable { get; set; } /// /// Gets the variable reference for the specific assignment to the variable for this function definition. @@ -146,7 +102,7 @@ public PythonReference GetVariableReference(PythonAst ast) { } internal override bool ExposesLocalVariable(PythonVariable variable) { - return NeedsLocalsDictionary; + return NeedsLocalsDictionary; } internal override bool TryBindOuter(ScopeStatement from, string name, bool allowGlobals, out PythonVariable variable) { @@ -163,7 +119,7 @@ internal override bool TryBindOuter(ScopeStatement from, string name, bool allow } AddCellVariable(variable); - } else if(allowGlobals) { + } else if (allowGlobals) { from.AddReferencedGlobal(name); } return true; @@ -197,7 +153,7 @@ internal override void Bind(PythonNameBinder binder) { base.Bind(binder); Verify(binder); } - + private void Verify(PythonNameBinder binder) { if (ContainsImportStar && IsClosure) { binder.ReportSyntaxError( @@ -235,27 +191,19 @@ public int GetIndexOfDef(PythonAst ast) { public override void Walk(PythonWalker walker) { if (walker.Walk(this)) { - _name?.Walk(walker); - if (_parameters != null) { - foreach (Parameter p in _parameters) { - p.Walk(walker); - } - } - if (_decorators != null) { - _decorators.Walk(walker); - } - if (_body != null) { - _body.Walk(walker); - } - if (_returnAnnotation != null) { - _returnAnnotation.Walk(walker); + NameExpression?.Walk(walker); + foreach (var p in _parameters.MaybeEnumerate()) { + p.Walk(walker); } + Decorators?.Walk(walker); + _body?.Walk(walker); + ReturnAnnotation?.Walk(walker); } walker.PostWalk(this); } public SourceLocation Header { - get { return GlobalParent.IndexToLocation(_headerIndex); } + get { return GlobalParent.IndexToLocation(HeaderIndex); } } public override string GetLeadingWhiteSpace(PythonAst ast) { @@ -290,10 +238,10 @@ internal override void AppendCodeStringStmt(StringBuilder res, PythonAst ast, Co res.Append(name); if (!this.IsIncompleteNode(ast)) { format.Append( - res, - format.SpaceBeforeFunctionDeclarationParen, - " ", - "", + res, + format.SpaceBeforeFunctionDeclarationParen, + " ", + "", this.GetThirdWhiteSpaceDefaultNull(ast) ); @@ -317,13 +265,13 @@ internal override void AppendCodeStringStmt(StringBuilder res, PythonAst ast, Co format.Append( res, - Parameters.Length != 0 ? + Parameters.Length != 0 ? format.SpaceWithinFunctionDeclarationParens : format.SpaceWithinEmptyParameterList, " ", "", this.GetFourthWhiteSpaceDefaultNull(ast) - ); + ); if (!this.IsMissingCloseGrouping(ast)) { res.Append(')'); @@ -336,11 +284,11 @@ internal override void AppendCodeStringStmt(StringBuilder res, PythonAst ast, Co " ", "", this.GetFifthWhiteSpace(ast) - ); + ); res.Append("->"); - _returnAnnotation.AppendCodeString( - res, - ast, + ReturnAnnotation.AppendCodeString( + res, + ast, format, format.SpaceAroundAnnotationArrow != null ? format.SpaceAroundAnnotationArrow.Value ? " " : "" : diff --git a/src/Analysis/Engine/Impl/Parsing/Ast/PythonNameBinder.cs b/src/Analysis/Engine/Impl/Parsing/Ast/PythonNameBinder.cs index 76503b40c..21ee0e84b 100644 --- a/src/Analysis/Engine/Impl/Parsing/Ast/PythonNameBinder.cs +++ b/src/Analysis/Engine/Impl/Parsing/Ast/PythonNameBinder.cs @@ -346,14 +346,6 @@ public override bool Walk(WhileStatement node) { return false; } - public override bool Walk(ReturnStatement node) { - FunctionDefinition funcDef = _currentScope as FunctionDefinition; - if (funcDef != null) { - funcDef._hasReturn = true; - } - return base.Walk(node); - } - // WithStatement public override bool Walk(WithStatement node) { _currentScope.ContainsExceptionHandling = true; @@ -393,7 +385,7 @@ public override bool Walk(FromImportStatement node) { // FunctionDefinition public override bool Walk(FunctionDefinition node) { - node._nameVariable = _globalScope.EnsureGlobalVariable("__name__"); + _globalScope.EnsureGlobalVariable("__name__"); // Name is defined in the enclosing context if (!node.IsLambda) { diff --git a/src/Analysis/Engine/Impl/PythonAnalyzer.cs b/src/Analysis/Engine/Impl/PythonAnalyzer.cs index d03fc9ab8..9911dca5e 100644 --- a/src/Analysis/Engine/Impl/PythonAnalyzer.cs +++ b/src/Analysis/Engine/Impl/PythonAnalyzer.cs @@ -221,8 +221,7 @@ public IPythonProjectEntry AddModule(string moduleName, string filePath, Uri doc /// /// New in 2.1 public void AddModuleAlias(string moduleName, string moduleAlias) { - ModuleReference modRef; - if (Modules.TryImport(moduleName, out modRef)) { + if (Modules.TryImport(moduleName, out var modRef)) { Modules[moduleAlias] = modRef; } } @@ -279,9 +278,8 @@ public void RemoveModule(IProjectEntry entry, Action onImpo /// '__init__'. /// public IEnumerable GetEntriesThatImportModule(string moduleName, bool includeUnresolved) { - ModuleReference modRef; var entries = new List(); - if (Modules.TryImport(moduleName, out modRef) && modRef.HasReferences) { + if (Modules.TryImport(moduleName, out var modRef) && modRef.HasReferences) { entries.AddRange(modRef.References.Select(m => m.ProjectEntry).OfType()); } @@ -365,8 +363,7 @@ public IMemberResult[] GetModules() { } if (moduleRef.IsValid) { - List l; - if (!d.TryGetValue(modName, out l)) { + if (!d.TryGetValue(modName, out var l)) { d[modName] = l = new List(); } if (moduleRef.HasModule) { @@ -381,7 +378,7 @@ public IMemberResult[] GetModules() { private static IMemberResult[] ModuleDictToMemberResult(Dictionary> d) { var result = new IMemberResult[d.Count]; - int pos = 0; + var pos = 0; foreach (var kvp in d) { var lazyEnumerator = new LazyModuleEnumerator(kvp.Value); result[pos++] = new MemberResult( @@ -437,7 +434,7 @@ public IEnumerable FindNameInAllModules(string name) { if (Interpreter is ICanFindModuleMembers finder) { foreach (var modName in finder.GetModulesNamed(name)) { - int dot = modName.LastIndexOf('.'); + var dot = modName.LastIndexOf('.'); if (dot < 0) { yield return new ExportedMemberInfo(null, modName); } else { @@ -500,7 +497,7 @@ public IEnumerable FindNameInAllModules(string name) { } private static bool GetPackageNameIfMatch(string name, string fullName, out string packageName) { - int lastDot = fullName.LastIndexOf('.'); + var lastDot = fullName.LastIndexOf('.'); if (lastDot < 0) { packageName = null; return false; @@ -529,10 +526,8 @@ private static bool GetPackageNameIfMatch(string name, string fullName, out stri /// /// public IMemberResult[] GetModuleMembers(IModuleContext moduleContext, string[] names, bool includeMembers = false) { - ModuleReference moduleRef; - if (Modules.TryImport(names[0], out moduleRef)) { - var module = moduleRef.Module as IModule; - if (module != null) { + if (Modules.TryImport(names[0], out var moduleRef)) { + if (moduleRef.Module is IModule module) { return GetModuleMembers(moduleContext, names, includeMembers, module); } @@ -542,7 +537,7 @@ public IMemberResult[] GetModuleMembers(IModuleContext moduleContext, string[] n } internal static IMemberResult[] GetModuleMembers(IModuleContext moduleContext, string[] names, bool includeMembers, IModule module) { - for (int i = 1; i < names.Length && module != null; i++) { + for (var i = 1; i < names.Length && module != null; i++) { module = module.GetChildPackage(moduleContext, names[i]); } @@ -564,7 +559,7 @@ internal static IMemberResult[] GetModuleMembers(IModuleContext moduleContext, s results.Add(child.Value); } foreach (var keyValue in module.GetAllMembers(moduleContext)) { - bool anyModules = false; + var anyModules = false; foreach (var ns in keyValue.Value.OfType()) { if (ns.Members.OfType().Any(mod => !(mod is MultipleMemberInfo))) { anyModules = true; @@ -584,10 +579,8 @@ internal static IMemberResult[] GetModuleMembers(IModuleContext moduleContext, s return new IMemberResult[0]; } - private static IMemberResult[] MemberDictToMemberResult(Dictionary> results) { - return results.Select(r => new MemberResult(r.Key, r.Value.SelectMany()) as IMemberResult).ToArray(); - } - + private static IMemberResult[] MemberDictToMemberResult(Dictionary> results) + => results.Select(r => new MemberResult(r.Key, r.Value.SelectMany()) as IMemberResult).ToArray(); /// /// Gets the list of directories which should be analyzed. @@ -602,7 +595,7 @@ private static IMemberResult[] MemberDictToMemberResult(Dictionary TypeStubDirectories => _typeStubPaths.AsLockedEnumerable().ToArray(); public AnalysisLimits Limits { - get { return _limits; } + get => _limits; set { value = value ?? AnalysisLimits.GetDefaultLimits(); var limits = _limits; @@ -707,8 +700,7 @@ internal IKnownClasses ClassInfos { /// Function to create the value. /// The cached value or null. internal AnalysisValue GetCached(object key, Func maker) { - AnalysisValue result; - if (!_itemCache.TryGetValue(key, out result)) { + if (!_itemCache.TryGetValue(key, out var result)) { // Set the key to prevent recursion _itemCache[key] = null; _itemCache[key] = result = maker(); @@ -718,15 +710,12 @@ internal AnalysisValue GetCached(object key, Func maker) { internal BuiltinModule BuiltinModule => _builtinModule; - internal BuiltinInstanceInfo GetInstance(IPythonType type) { - return GetBuiltinType(type).Instance; - } + internal BuiltinInstanceInfo GetInstance(IPythonType type) => GetBuiltinType(type).Instance; - internal BuiltinClassInfo GetBuiltinType(IPythonType type) { - return (BuiltinClassInfo)GetCached(type, + internal BuiltinClassInfo GetBuiltinType(IPythonType type) => + (BuiltinClassInfo)GetCached(type, () => MakeBuiltinType(type) ) ?? ClassInfos[BuiltinTypeId.Object]; - } private BuiltinClassInfo MakeBuiltinType(IPythonType type) { switch (type.TypeId) { @@ -739,8 +728,7 @@ private BuiltinClassInfo MakeBuiltinType(IPythonType type) { } internal IAnalysisSet GetAnalysisSetFromObjects(object objects) { - var typeList = objects as IEnumerable; - if (typeList == null) { + if (!(objects is IEnumerable typeList)) { return AnalysisSet.Empty; } return AnalysisSet.UnionAll(typeList.Select(GetAnalysisValueFromObjects)); @@ -814,8 +802,7 @@ internal IDictionary GetAllMembers(IMemberContainer contai var children = (container as IModule)?.GetChildrenPackages(moduleContext); if (children?.Any() ?? false) { foreach (var child in children) { - IAnalysisSet existing; - if (result.TryGetValue(child.Key, out existing)) { + if (result.TryGetValue(child.Key, out var existing)) { result[child.Key] = existing.Add(child.Value); } else { result[child.Key] = child.Value; @@ -831,8 +818,7 @@ internal IDictionary GetAllMembers(IMemberContainer contai internal ConcurrentDictionary ModulesByFilename { get; } public bool TryGetProjectEntryByPath(string path, out IProjectEntry projEntry) { - ModuleInfo modInfo; - if (ModulesByFilename.TryGetValue(path, out modInfo)) { + if (ModulesByFilename.TryGetValue(path, out var modInfo)) { projEntry = modInfo.ProjectEntry; return true; } @@ -842,7 +828,7 @@ public bool TryGetProjectEntryByPath(string path, out IProjectEntry projEntry) { } internal IAnalysisSet GetConstant(object value) { - object key = value ?? _nullKey; + var key = value ?? _nullKey; return GetCached(key, () => { var constant = value as IPythonConstant; var constantType = constant?.Type; @@ -1012,14 +998,13 @@ internal AggregateProjectEntry GetAggregate(params IProjectEntry[] aggregating) return GetAggregateWorker(aggregating); } - private static void SortAggregates(IProjectEntry[] aggregating) { - Array.Sort(aggregating, (x, y) => x.GetHashCode() - y.GetHashCode()); - } + private static void SortAggregates(IProjectEntry[] aggregating) + => Array.Sort(aggregating, (x, y) => x.GetHashCode() - y.GetHashCode()); internal AggregateProjectEntry GetAggregate(HashSet from, IProjectEntry with) { Debug.Assert(!from.Contains(with)); - IProjectEntry[] all = new IProjectEntry[from.Count + 1]; + var all = new IProjectEntry[from.Count + 1]; from.CopyTo(all); all[from.Count] = with; @@ -1036,13 +1021,11 @@ internal void ClearAggregate(AggregateProjectEntry entry) { } private AggregateProjectEntry GetAggregateWorker(IProjectEntry[] all) { - AggregateProjectEntry agg; - if (!_aggregates.TryGetValue(all, out agg)) { + if (!_aggregates.TryGetValue(all, out var agg)) { _aggregates[all] = agg = new AggregateProjectEntry(new HashSet(all)); foreach (var proj in all) { - IAggregateableProjectEntry aggretable = proj as IAggregateableProjectEntry; - if (aggretable != null) { + if (proj is IAggregateableProjectEntry aggretable) { aggretable.AggregatedInto(agg); } } @@ -1058,7 +1041,7 @@ public bool Equals(IProjectEntry[] x, IProjectEntry[] y) { if (x.Length != y.Length) { return false; } - for (int i = 0; i < x.Length; i++) { + for (var i = 0; i < x.Length; i++) { if (x[i] != y[i]) { return false; } @@ -1067,8 +1050,8 @@ public bool Equals(IProjectEntry[] x, IProjectEntry[] y) { } public int GetHashCode(IProjectEntry[] obj) { - int res = 0; - for (int i = 0; i < obj.Length; i++) { + var res = 0; + for (var i = 0; i < obj.Length; i++) { res ^= obj[i].GetHashCode(); } return res; diff --git a/src/Analysis/Engine/Test/AstAnalysisTests.cs b/src/Analysis/Engine/Test/AstAnalysisTests.cs index 48713cec2..e8e1721c5 100644 --- a/src/Analysis/Engine/Test/AstAnalysisTests.cs +++ b/src/Analysis/Engine/Test/AstAnalysisTests.cs @@ -223,6 +223,17 @@ from ForwardRefProp1 import * } } + [TestMethod, Priority(0)] + public async Task AstForwardRefGlobalFunction() { + using (var server = await CreateServerAsync()) { + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(@" +from ForwardRefGlobalFunc import * +x = func1() +"); + analysis.Should().HaveVariable("x").OfTypes(BuiltinTypeId.Str); + } + } + [TestMethod, Priority(0)] public async Task AstForwardRefFunction1() { using (var server = await CreateServerAsync()) { @@ -810,7 +821,7 @@ private async Task FullStdLibTest(InterpreterConfiguration configuration, params foreach (var r in modules .Where(m => !skip.Contains(m.ModuleName)) .GroupBy(m => { - int i = m.FullName.IndexOf('.'); + var i = m.FullName.IndexOf('.'); return i <= 0 ? m.FullName : m.FullName.Remove(i); }) .AsParallel() diff --git a/src/UnitTests/TestData/AstAnalysis/ForwardRefGlobalFunc.py b/src/UnitTests/TestData/AstAnalysis/ForwardRefGlobalFunc.py new file mode 100644 index 000000000..0eef85ba8 --- /dev/null +++ b/src/UnitTests/TestData/AstAnalysis/ForwardRefGlobalFunc.py @@ -0,0 +1,8 @@ +def func1(): + return func2() + +def func2(): + return func3() + +def func3(): + return 's' From 15a78e158eff7e83e204415be26bfd37a07f8234 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Sat, 27 Oct 2018 10:37:35 -0700 Subject: [PATCH 07/19] Update test --- src/Analysis/Engine/Test/AstAnalysisTests.cs | 11 +++++++-- .../FluentAssertions/VariableDefAssertions.cs | 24 ++++++++++++++++++- .../VariableDefAssertionsExtensions.cs | 12 ++++++++++ 3 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/Analysis/Engine/Test/AstAnalysisTests.cs b/src/Analysis/Engine/Test/AstAnalysisTests.cs index e8e1721c5..fb64f8443 100644 --- a/src/Analysis/Engine/Test/AstAnalysisTests.cs +++ b/src/Analysis/Engine/Test/AstAnalysisTests.cs @@ -978,13 +978,20 @@ public async Task TypeShedJsonMakeScanner() { scanner = _json.make_scanner()"; var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); - analysis.Should().HaveVariable("scanner").WithValue() - .Which.Should().HaveSingleOverload() + var v0 = analysis.Should().HaveVariable("scanner").WithValueAt(0); + var v1 = analysis.Should().HaveVariable("scanner").WithValueAt(1); + + v0.Which.Should().HaveSingleOverload() .Which.Should().HaveName("__call__") .And.HaveParameters("string", "index") .And.HaveParameterAt(0).WithName("string").WithType("str").WithNoDefaultValue() .And.HaveParameterAt(1).WithName("index").WithType("int").WithNoDefaultValue() .And.HaveSingleReturnType("tuple[None, int]"); + + v1.Which.Should().HaveSingleOverload() + .Which.Should().HaveName("__call__") + .And.HaveParameters("*args", "**kwargs") + .And.HaveSingleReturnType("type"); } } diff --git a/src/Analysis/Engine/Test/FluentAssertions/VariableDefAssertions.cs b/src/Analysis/Engine/Test/FluentAssertions/VariableDefAssertions.cs index 9965e9d8c..a7fbeb637 100644 --- a/src/Analysis/Engine/Test/FluentAssertions/VariableDefAssertions.cs +++ b/src/Analysis/Engine/Test/FluentAssertions/VariableDefAssertions.cs @@ -211,7 +211,19 @@ public AndWhichConstraint> var testInfo = new AnalysisValueTestInfo((TValue)value, GetScopeDescription(), _scope); return new AndWhichConstraint>(this, testInfo); } - + + public AndWhichConstraint> HaveValueAt(int index, string because = "", params object[] reasonArgs) + where TValue : IAnalysisValue { + var value = FlattenAnalysisValues(Subject.Types).ToArray()[index]; + + Execute.Assertion.ForCondition(value is TValue) + .BecauseOf(because, reasonArgs) + .FailWith($"Expected {_moduleName}.{_name} to have value of type {typeof(TValue)}{{reason}}, but its value has type {value.GetType()}."); + + var testInfo = new AnalysisValueTestInfo((TValue)value, GetScopeDescription(), _scope); + return new AndWhichConstraint>(this, testInfo); + } + private IAnalysisValue AssertSingle(string because, object[] reasonArgs, IAnalysisValue[] values) { Execute.Assertion.ForCondition(values.Length == 1) .BecauseOf(because, reasonArgs) @@ -222,6 +234,16 @@ private IAnalysisValue AssertSingle(string because, object[] reasonArgs, IAnalys return values[0]; } + private IAnalysisValue[] AssertCount(int count, string because, object[] reasonArgs, IAnalysisValue[] values) { + Execute.Assertion.ForCondition(values.Length != count) + .BecauseOf(because, reasonArgs) + .FailWith(values.Length == count + ? $"Expected variable '{_moduleName}.{_name}' to have {count} values{{reason}}, but found none." + : $"Expected variable '{_moduleName}.{_name}' to have {count} values{{reason}}, but found {values.Length}: {GetQuotedNames(values)}"); + + return values; + } + private string GetScopeDescription() { switch (_scope) { case IFunctionScope functionScope: diff --git a/src/Analysis/Engine/Test/FluentAssertions/VariableDefAssertionsExtensions.cs b/src/Analysis/Engine/Test/FluentAssertions/VariableDefAssertionsExtensions.cs index fc5504561..881a0bf3a 100644 --- a/src/Analysis/Engine/Test/FluentAssertions/VariableDefAssertionsExtensions.cs +++ b/src/Analysis/Engine/Test/FluentAssertions/VariableDefAssertionsExtensions.cs @@ -122,11 +122,23 @@ public static AndWhichConstraint> return new AndWhichConstraint>(andWhichConstraint.And, testInfo); } + public static AndWhichConstraint> + WithValueAt(this AndWhichConstraint andWhichConstraint, int index, string because = "", params object[] reasonArgs) + where TValue : IAnalysisValue { + var testInfo = andWhichConstraint.Which.Should().HaveValueAt(index, because, reasonArgs).Which; + return new AndWhichConstraint>(andWhichConstraint.And, testInfo); + } + public static AndWhichConstraint> WithValue(this AndWhichConstraint constraint, string because = "", params object[] reasonArgs) where TValue : IAnalysisValue => constraint.WithValue(because, reasonArgs); + public static AndWhichConstraint> + WithValueAt(this AndWhichConstraint constraint, int index, string because = "", params object[] reasonArgs) + where TValue : IAnalysisValue + => constraint.WithValueAt(index, because, reasonArgs); + public static AndWhichConstraint> WithValue(this AndWhichConstraint constraint, string because = "", params object[] reasonArgs) where TValue : IAnalysisValue From 134f76ff2ae32794a7deeede31dbe92abd15d053 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Wed, 31 Oct 2018 20:52:27 -0700 Subject: [PATCH 08/19] Basic named tuple annotation processing --- .../Engine/Impl/Values/TypingModuleInfo.cs | 7 +- .../Engine/Impl/Values/TypingTypeInfo.cs | 101 +++++++++++------- src/Analysis/Engine/Test/AnalysisTest.cs | 42 ++++++++ src/Analysis/Engine/Test/AstAnalysisTests.cs | 62 ++++++++++- 4 files changed, 166 insertions(+), 46 deletions(-) diff --git a/src/Analysis/Engine/Impl/Values/TypingModuleInfo.cs b/src/Analysis/Engine/Impl/Values/TypingModuleInfo.cs index 2b17a39c0..bf36814ec 100644 --- a/src/Analysis/Engine/Impl/Values/TypingModuleInfo.cs +++ b/src/Analysis/Engine/Impl/Values/TypingModuleInfo.cs @@ -17,7 +17,6 @@ using System; using System.Collections.Generic; using System.Linq; -using Microsoft.PythonTools.Analysis.Infrastructure; using Microsoft.PythonTools.Interpreter; using Microsoft.PythonTools.Parsing.Ast; @@ -34,9 +33,7 @@ private TypingModuleInfo(BuiltinModule inner) public static BuiltinModule Wrap(BuiltinModule inner) => new TypingModuleInfo(inner); - private IAnalysisSet GetBuiltin(BuiltinTypeId typeId) { - return ProjectState.ClassInfos[typeId]; - } + private IAnalysisSet GetBuiltin(BuiltinTypeId typeId) => ProjectState.ClassInfos[typeId]; private IAnalysisSet GetFunction(Node node, AnalysisUnit unit, string name, CallDelegate callable) { lock (_callables) { @@ -119,6 +116,7 @@ public override IAnalysisSet GetMember(Node node, AnalysisUnit unit, string name case "NamedTuple": case "Generator": case "ClassVar": + case "Type": res = new TypingTypeInfo(name, _inner.GetMember(node, unit, name)?.FirstOrDefault()); break; @@ -142,7 +140,6 @@ public override IAnalysisSet GetMember(Node node, AnalysisUnit unit, string name case "Counter": res = Import("collections", "Counter", node, unit); break; case "Deque": res = Import("collections", "deque", node, unit); break; case "DefaultDict": res = Import("collections", "defaultdict", node, unit); break; - case "Type": res = GetBuiltin(BuiltinTypeId.Type); break; case "ByteString": res = GetBuiltin(BuiltinTypeId.Bytes); break; case "AnyStr": res = GetBuiltin(BuiltinTypeId.Unicode).Union(GetBuiltin(BuiltinTypeId.Bytes), canMutate: false); break; case "Text": res = GetBuiltin(BuiltinTypeId.Str); break; diff --git a/src/Analysis/Engine/Impl/Values/TypingTypeInfo.cs b/src/Analysis/Engine/Impl/Values/TypingTypeInfo.cs index b61f95a0f..54fe201ad 100644 --- a/src/Analysis/Engine/Impl/Values/TypingTypeInfo.cs +++ b/src/Analysis/Engine/Impl/Values/TypingTypeInfo.cs @@ -23,30 +23,36 @@ using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.PythonTools.Analysis.Values { + /// + /// Represents better view on an existing type, typically generic. + /// class TypingTypeInfo : AnalysisValue, IHasRichDescription { private readonly AnalysisValue _innerValue; - private readonly string _baseName; - private readonly IReadOnlyList _args; - - public TypingTypeInfo(string baseName, AnalysisValue innerValue) : this(baseName, innerValue, Array.Empty()) { } - - private TypingTypeInfo(string baseName, AnalysisValue innerValue, IReadOnlyList args) { - _baseName = baseName; + private readonly string _typeName; + private readonly IReadOnlyList _typeArgs; + + public TypingTypeInfo(string typeName, AnalysisValue innerValue) + : this(typeName, innerValue, Array.Empty()) { } + + /// + /// + /// + /// Name of the generic type such as List + /// Analysis value from the real typing module + /// Actual type arguments to the generic type. + private TypingTypeInfo(string typeName, AnalysisValue innerValue, IReadOnlyList typeArgs) { + _typeName = typeName; _innerValue = innerValue; - _args = args; + _typeArgs = typeArgs ?? Array.Empty(); } - public TypingTypeInfo MakeGeneric(IReadOnlyList args) { - if (_args != null) { - return new TypingTypeInfo(_baseName, _innerValue, args); - } - return this; - } + public TypingTypeInfo MakeGeneric(IReadOnlyList args) + => args.Count > 0 ? new TypingTypeInfo(_typeName, _innerValue, args) : this; public override IAnalysisSet GetInstanceType() => this; public override IAnalysisSet Call(Node node, AnalysisUnit unit, IAnalysisSet[] args, NameExpression[] keywordArgNames) { - if ((_args == null || !_args.Any()) && node is CallExpression ce) { + if ((_typeArgs == null || !_typeArgs.Any()) && node is CallExpression ce) { return unit.InterpreterScope.GetOrMakeNodeValue(node, NodeValueKind.TypeAnnotation, n => { // Use annotation converter and reparse the arguments var newArgs = new List(); @@ -59,12 +65,16 @@ public override IAnalysisSet Call(Node node, AnalysisUnit unit, IAnalysisSet[] a foreach (var type in ce.Args.MaybeEnumerate().Where(e => e?.Expression != null).Select(e => new TypeAnnotation(unit.State.LanguageVersion, e.Expression))) { newArgs.Add(type.GetValue(eval) ?? AnalysisSet.Empty); } - return new TypingTypeInfo(_baseName, _innerValue, newArgs); + return new TypingTypeInfo(_typeName, _innerValue, newArgs); }); } return this; } + /// + /// Creates type wrapper for a generic, such as List[str] + /// + /// public override IAnalysisSet GetIndex(Node node, AnalysisUnit unit, IAnalysisSet index) { if (node is IndexExpression ie) { return unit.InterpreterScope.GetOrMakeNodeValue(node, NodeValueKind.TypeAnnotation, n => { @@ -85,7 +95,7 @@ public override IAnalysisSet GetIndex(Node node, AnalysisUnit unit, IAnalysisSet foreach (var type in exprs.Select(e => new TypeAnnotation(unit.State.LanguageVersion, e))) { newArgs.Add(type.GetValue(eval) ?? AnalysisSet.Empty); } - return new TypingTypeInfo(_baseName, _innerValue, newArgs); + return new TypingTypeInfo(_typeName, _innerValue, newArgs); }); } return this; @@ -95,8 +105,8 @@ public IAnalysisSet Finalize(ExpressionEvaluator eval, Node node, AnalysisUnit u if (Push()) { try { var finalizer = new TypingTypeInfoFinalizer(eval, node, unit); - return finalizer.Finalize(_baseName, _args) - ?? finalizer.Finalize(_baseName) + return finalizer.Finalize(_typeName, _typeArgs) + ?? finalizer.Finalize(_typeName) ?? AnalysisSet.Empty; } finally { Pop(); @@ -106,15 +116,15 @@ public IAnalysisSet Finalize(ExpressionEvaluator eval, Node node, AnalysisUnit u } public override string ToString() { - if (_args != null) { - return $""; + if (_typeArgs != null) { + return $""; } - return $""; + return $""; } public IReadOnlyList ToTypeList() { - if (_baseName == " List") { - return _args; + if (_typeName == " List") { + return _typeArgs; } return null; } @@ -127,19 +137,19 @@ public static IReadOnlyList ToTypeList(IAnalysisSet set) { } public IEnumerable> GetRichDescription() { - if (_baseName != " List") { + if (_typeName != " List") { if (_innerValue == null) { - yield return new KeyValuePair(WellKnownRichDescriptionKinds.Name, _baseName); + yield return new KeyValuePair(WellKnownRichDescriptionKinds.Name, _typeName); } else { foreach (var kv in _innerValue.GetRichDescriptions()) { yield return kv; } } } - if (_args != null && _args.Any()) { + if (_typeArgs != null && _typeArgs.Any()) { yield return new KeyValuePair(WellKnownRichDescriptionKinds.Misc, "["); bool addComma = false; - foreach (var arg in _args) { + foreach (var arg in _typeArgs) { if (addComma) { yield return new KeyValuePair(WellKnownRichDescriptionKinds.Comma, ", "); } @@ -157,32 +167,32 @@ public IEnumerable> GetRichDescription() { public override bool Equals(object obj) { if (obj is TypingTypeInfo other) { - if (_baseName != other._baseName) { + if (_typeName != other._typeName) { return false; } - if ((_args == null) != (other._args == null)) { + if ((_typeArgs == null) != (other._typeArgs == null)) { return false; } - if (_args == null || other._args == null) { + if (_typeArgs == null || other._typeArgs == null) { return true; } - return _args.Zip(other._args, (x, y) => ObjectComparer.Instance.Equals(x, y)).All(b => b); + return _typeArgs.Zip(other._typeArgs, (x, y) => ObjectComparer.Instance.Equals(x, y)).All(b => b); } return false; } public override int GetHashCode() { - if (_args != null) { - return _args.Aggregate(_baseName.GetHashCode(), (h, s) => h + 37 * ObjectComparer.Instance.GetHashCode(s)); + if (_typeArgs != null) { + return _typeArgs.Aggregate(_typeName.GetHashCode(), (h, s) => h + 37 * ObjectComparer.Instance.GetHashCode(s)); } - return _baseName.GetHashCode(); + return _typeName.GetHashCode(); } internal override bool UnionEquals(AnalysisValue av, int strength) { if (strength == 0) { return Equals(av); } else { - return _baseName == (av as TypingTypeInfo)?._baseName; + return _typeName == (av as TypingTypeInfo)?._typeName; } } @@ -190,15 +200,15 @@ internal override int UnionHashCode(int strength) { if (strength == 0) { return GetHashCode(); } else { - return _baseName.GetHashCode(); + return _typeName.GetHashCode(); } } internal override AnalysisValue UnionMergeTypes(AnalysisValue av, int strength) { - if (strength == 0 || (_args != null && _args.Count == 0)) { + if (strength == 0 || (_typeArgs != null && _typeArgs.Count == 0)) { return this; } else { - return new TypingTypeInfo(_baseName, _innerValue, Array.Empty()); + return new TypingTypeInfo(_typeName, _innerValue, Array.Empty()); } } } @@ -307,6 +317,10 @@ private bool GetSequenceTypes(string name, IReadOnlyList args, out } + /// + /// Given type name and type arguments provides value + /// based on the actual type to use in analysis. + /// public IAnalysisSet Finalize(string name, IReadOnlyList args) { if (string.IsNullOrEmpty(name) || args == null || args.Count == 0) { return null; @@ -433,6 +447,11 @@ public IAnalysisSet Finalize(string name, IReadOnlyList args) { case "NamedTuple": return CreateNamedTuple(_node, _unit, args.ElementAtOrDefault(0), args.ElementAtOrDefault(1)); + + case "Type": + // Return class rather than instance + return args.Count > 0 ? Finalize(args[0]) : AnalysisSet.Empty; + case " List": return AnalysisSet.UnionAll(args.Select(ToInstance)); } @@ -494,6 +513,9 @@ private IAnalysisSet CreateNamedTuple(Node node, AnalysisUnit unit, IAnalysisSet return res; } + /// + /// Turns type name into actual type to use in analysis. + /// public IAnalysisSet Finalize(string name) { if (string.IsNullOrEmpty(name)) { return null; @@ -526,6 +548,7 @@ public IAnalysisSet Finalize(string name) { case "FrozenSet": return ClassInfo[BuiltinTypeId.FrozenSet]; case "NamedTuple": return ClassInfo[BuiltinTypeId.Tuple]; case "Generator": return ClassInfo[BuiltinTypeId.Generator]; + case "Type": return ClassInfo[BuiltinTypeId.Type]; case "NoReturn": return AnalysisSet.Empty; case " List": return null; } diff --git a/src/Analysis/Engine/Test/AnalysisTest.cs b/src/Analysis/Engine/Test/AnalysisTest.cs index 6e3f89bc9..1aadec087 100644 --- a/src/Analysis/Engine/Test/AnalysisTest.cs +++ b/src/Analysis/Engine/Test/AnalysisTest.cs @@ -3093,6 +3093,48 @@ def Cmeth(self, avar, bvar): } } + [TestMethod, Priority(0)] + public async Task TypingListOfTuples() { + using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) { + var code = @" +from typing import List + +def ls() -> List[tuple]: + pass + +x = ls()[0] +"; + + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); + + analysis + .Should().HaveVariable("ls").Which + .Should().HaveShortDescriptions("module.ls() -> list[tuple]"); + + analysis + .Should().HaveVariable("x").Which + .Should().HaveType(BuiltinTypeId.Tuple); + } + } + + [TestMethod, Priority(0)] + public async Task CollectionsNamedTuple() { + using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) { + var code = @" +from collections import namedtuple +nt = namedtuple('Point', ['x', 'y']) +pt = nt(1, 2) +pt. +"; + + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); + + analysis + .Should().HaveVariable("pt").Which + .Should().HaveType(BuiltinTypeId.Tuple); + } + } + [TestMethod, Priority(0)] public async Task FunctionOverloads() { var code = @" diff --git a/src/Analysis/Engine/Test/AstAnalysisTests.cs b/src/Analysis/Engine/Test/AstAnalysisTests.cs index fb64f8443..356612a4f 100644 --- a/src/Analysis/Engine/Test/AstAnalysisTests.cs +++ b/src/Analysis/Engine/Test/AstAnalysisTests.cs @@ -345,6 +345,30 @@ def overdrawn(self): } } + + [TestMethod, Priority(0)] + public async Task AstNamedTuple() { + using (var server = await CreateServerAsync()) { + var code = @" +from typing import List + +def ls() -> List[tuple]: + pass + +x = ls()[0] +"; + + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); + + analysis + .Should().HaveVariable("ls").Which + .Should().HaveShortDescriptions("module.ls() -> list[tuple]"); + + analysis + .Should().HaveVariable("x").Which + .Should().HaveType(BuiltinTypeId.Tuple); + } + } [TestMethod, Priority(0)] public async Task AstSearchPathsThroughFactory() { using (var factory = CreateInterpreterFactory()) @@ -896,7 +920,8 @@ public async Task AstTypeAnnotationConversion() { } #endregion - #region Type Shed tests + #region Typeshed tests + [TestMethod, Priority(0)] public async Task TypeShedElementTree() { using (var server = await CreateServerAsync()) { @@ -1034,6 +1059,40 @@ public async Task TypeShedSysInfo() { } } + [TestMethod, Priority(0)] + public async Task TypeShedNamedTuple() { + using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) { + server.Analyzer.SetTypeStubPaths(new[] { TypeShedPath }); + var code = "from collections import namedtuple\n"; + + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); + analysis + .Should().HaveFunction("namedtuple").Which + .Should().HaveResolvedReturnTypes(BuiltinTypeId.Tuple); + } + } + + [TestMethod, Priority(0)] + public async Task NamedTupleAnnotation() { + using (var server = await CreateServerAsync()) { + server.Analyzer.SetTypeStubPaths(new[] { TypeShedPath }); + server.Analyzer.Limits = new AnalysisLimits { UseTypeStubPackages = true, UseTypeStubPackagesExclusively = true }; + + var code = @" +from typing import Union, Iterable, Type + +def namedtuple(typename: str, field_names: Union[str, Iterable[str]]) -> Type[tuple]: ... + +nt = namedtuple('Point', ['x', 'y']) +pt = nt(1, 2) +"; + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); + + analysis.Should().HaveVariable("pt").OfTypes(BuiltinTypeId.Tuple); + } + } + + [TestMethod, Priority(0)] public void TypeStubConditionalDefine() { var seen = new HashSet(); @@ -1082,6 +1141,5 @@ public void TypeStubConditionalDefine() { } } #endregion - } } From 649ebded440e0f984b47d68c655d726b48244887 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Wed, 31 Oct 2018 21:49:34 -0700 Subject: [PATCH 09/19] Test fix --- src/Analysis/Engine/Test/AnalysisTest.cs | 3 +- src/Analysis/Engine/Test/AstAnalysisTests.cs | 37 +++++--------------- src/Analysis/Engine/Test/ServerBasedTest.cs | 2 +- 3 files changed, 11 insertions(+), 31 deletions(-) diff --git a/src/Analysis/Engine/Test/AnalysisTest.cs b/src/Analysis/Engine/Test/AnalysisTest.cs index 1aadec087..778194613 100644 --- a/src/Analysis/Engine/Test/AnalysisTest.cs +++ b/src/Analysis/Engine/Test/AnalysisTest.cs @@ -3124,9 +3124,10 @@ public async Task CollectionsNamedTuple() { from collections import namedtuple nt = namedtuple('Point', ['x', 'y']) pt = nt(1, 2) -pt. "; + server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); + server.Analyzer.Limits = new AnalysisLimits { UseTypeStubPackages = true, UseTypeStubPackagesExclusively = false }; var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); analysis diff --git a/src/Analysis/Engine/Test/AstAnalysisTests.cs b/src/Analysis/Engine/Test/AstAnalysisTests.cs index 356612a4f..bf2c5ca86 100644 --- a/src/Analysis/Engine/Test/AstAnalysisTests.cs +++ b/src/Analysis/Engine/Test/AstAnalysisTests.cs @@ -20,7 +20,6 @@ using System.Globalization; using System.IO; using System.Linq; -using System.Runtime.InteropServices; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -41,7 +40,7 @@ namespace AnalysisTests { [TestClass] - public class AstAnalysisTests { + public class AstAnalysisTests: ServerBasedTest { public TestContext TestContext { get; set; } [TestInitialize] @@ -90,26 +89,6 @@ private static Task CreateServerAsync(InterpreterConfiguration configura private static AstPythonInterpreterFactory CreateInterpreterFactory() => CreateInterpreterFactory(PythonVersions.LatestAvailable); - private static readonly Lazy _typeShedPath = new Lazy(FindTypeShedForTest); - private static string TypeShedPath => _typeShedPath.Value; - private static string FindTypeShedForTest() { - var candidate = Path.Combine(PathUtils.GetParent(typeof(AstPythonInterpreterFactory).Assembly.Location), "Typeshed"); - if (Directory.Exists(candidate) && Directory.Exists(Path.Combine(candidate, "stdlib", "2and3"))) { - return candidate; - } - - var root = TestData.GetPath(); - - for (string previousRoot = null; root != previousRoot; previousRoot = root, root = PathUtils.GetParent(root)) { - candidate = Path.Combine(root, "typeshed"); - if (Directory.Exists(Path.Combine(candidate, "stdlib", "2and3"))) { - return candidate; - } - } - - return null; - } - #region Test cases @@ -963,7 +942,7 @@ public async Task TypeShedChildModules() { } using (var server = await CreateServerAsync()) { - server.Analyzer.SetTypeStubPaths(new[] { TypeShedPath }); + server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(@"import urllib"); var secondMembers = server.Analyzer.GetModuleMembers(analysis.InterpreterContext, new[] { "urllib" }) @@ -977,7 +956,7 @@ public async Task TypeShedChildModules() { [TestMethod, Priority(0)] public async Task TypeShedSysExcInfo() { using (var server = await CreateServerAsync()) { - server.Analyzer.SetTypeStubPaths(new[] { TypeShedPath }); + server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); var code = @"import sys e1, e2, e3 = sys.exc_info()"; @@ -997,7 +976,7 @@ public async Task TypeShedSysExcInfo() { [TestMethod, Priority(0)] public async Task TypeShedJsonMakeScanner() { using (var server = await CreateServerAsync()) { - server.Analyzer.SetTypeStubPaths(new[] { TypeShedPath }); + server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); var code = @"import _json scanner = _json.make_scanner()"; @@ -1023,7 +1002,7 @@ public async Task TypeShedJsonMakeScanner() { [TestMethod, Priority(0)] public async Task TypeShedSysInfo() { using (var server = await CreateServerAsync()) { - server.Analyzer.SetTypeStubPaths(new[] { TypeShedPath }); + server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); server.Analyzer.Limits = new AnalysisLimits { UseTypeStubPackages = true, UseTypeStubPackagesExclusively = true }; var code = @"import sys @@ -1062,7 +1041,7 @@ public async Task TypeShedSysInfo() { [TestMethod, Priority(0)] public async Task TypeShedNamedTuple() { using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) { - server.Analyzer.SetTypeStubPaths(new[] { TypeShedPath }); + server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); var code = "from collections import namedtuple\n"; var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); @@ -1075,8 +1054,8 @@ public async Task TypeShedNamedTuple() { [TestMethod, Priority(0)] public async Task NamedTupleAnnotation() { using (var server = await CreateServerAsync()) { - server.Analyzer.SetTypeStubPaths(new[] { TypeShedPath }); - server.Analyzer.Limits = new AnalysisLimits { UseTypeStubPackages = true, UseTypeStubPackagesExclusively = true }; + server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); + server.Analyzer.Limits = new AnalysisLimits { UseTypeStubPackages = true, UseTypeStubPackagesExclusively = false }; var code = @" from typing import Union, Iterable, Type diff --git a/src/Analysis/Engine/Test/ServerBasedTest.cs b/src/Analysis/Engine/Test/ServerBasedTest.cs index 04a3df767..292aacca1 100644 --- a/src/Analysis/Engine/Test/ServerBasedTest.cs +++ b/src/Analysis/Engine/Test/ServerBasedTest.cs @@ -32,7 +32,7 @@ namespace AnalysisTests { public class ServerBasedTest { private Server _server; - protected async Task CreateServerAsync(InterpreterConfiguration configuration = null, Uri rootUri = null) { + protected virtual async Task CreateServerAsync(InterpreterConfiguration configuration = null, Uri rootUri = null) { configuration = configuration ?? PythonVersions.LatestAvailable2X ?? PythonVersions.LatestAvailable3X; configuration.AssertInstalled(); From 1921f797aa7384f91297b6ae89b98dafdeab6b7b Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 1 Nov 2018 13:16:16 -0700 Subject: [PATCH 10/19] Introduce ability to return class definition --- .../Engine/Impl/AnalysisValueSetExtensions.cs | 28 ++---- .../Impl/Analyzer/FunctionAnalysisUnit.cs | 3 +- .../Ast/AstAnalysisFunctionWalker.cs | 1 + .../Impl/Interpreter/Ast/AstAnalysisWalker.cs | 4 +- .../Interpreter/Ast/AstPythonBuiltinType.cs | 15 +-- .../Impl/Interpreter/Ast/AstPythonType.cs | 50 +++++----- .../Ast/AstTypeAnnotationConverter.cs | 9 +- .../Interpreter/Definitions/IPythonType.cs | 39 ++++---- src/Analysis/Engine/Test/AnalysisTest.cs | 17 ++++ src/Analysis/Engine/Test/AstAnalysisTests.cs | 94 +++++++------------ .../ModuleAnalysisAssertions.cs | 6 ++ .../Test/FluentAssertions/ScopeAssertions.cs | 7 ++ .../TestData/AstAnalysis/ReturnAnnotation.py | 3 + 13 files changed, 142 insertions(+), 134 deletions(-) create mode 100644 src/UnitTests/TestData/AstAnalysis/ReturnAnnotation.py diff --git a/src/Analysis/Engine/Impl/AnalysisValueSetExtensions.cs b/src/Analysis/Engine/Impl/AnalysisValueSetExtensions.cs index f6177fdee..3ea088976 100644 --- a/src/Analysis/Engine/Impl/AnalysisValueSetExtensions.cs +++ b/src/Analysis/Engine/Impl/AnalysisValueSetExtensions.cs @@ -78,10 +78,8 @@ public static IAnalysisSet Call(this IAnalysisSet self, Node node, AnalysisUnit foreach (var ns in self) { var call = ns.Call(node, unit, args, keywordArgNames); Debug.Assert(call != null); - res = res.Union(call); } - return res; } @@ -95,7 +93,6 @@ public static IAnalysisSet GetIterator(this IAnalysisSet self, Node node, Analys foreach (var ns in self) { res = res.Union(ns.GetIterator(node, unit)); } - return res; } @@ -109,7 +106,6 @@ public static IAnalysisSet GetAsyncIterator(this IAnalysisSet self, Node node, A foreach (var ns in self) { res = res.Union(ns.GetAsyncIterator(node, unit)); } - return res; } @@ -122,7 +118,6 @@ public static IAnalysisSet GetIndex(this IAnalysisSet self, Node node, AnalysisU foreach (var ns in self) { res = res.Union(ns.GetIndex(node, unit, index)); } - return res; } @@ -140,8 +135,7 @@ public static void SetIndex(this IAnalysisSet self, Node node, AnalysisUnit unit /// Performs a delete index operation propagating the index types into /// the provided object. /// - public static void DeleteIndex(this IAnalysisSet self, Node node, AnalysisUnit unit, IAnalysisSet index) { - } + public static void DeleteIndex(this IAnalysisSet self, Node node, AnalysisUnit unit, IAnalysisSet index) { } /// /// Performs an augmented assignment propagating the value into the @@ -163,7 +157,6 @@ public static IAnalysisSet GetEnumeratorTypes(this IAnalysisSet self, Node node, foreach (var ns in self) { res = res.Union(ns.GetEnumeratorTypes(node, unit)); } - return res; } /// @@ -176,7 +169,6 @@ public static IAnalysisSet GetAsyncEnumeratorTypes(this IAnalysisSet self, Node foreach (var ns in self) { res = res.Union(ns.GetAsyncEnumeratorTypes(node, unit)); } - return res; } @@ -188,7 +180,6 @@ public static IAnalysisSet GetDescriptor(this IAnalysisSet self, Node node, Anal foreach (var ns in self) { res = res.Union(ns.GetDescriptor(node, instance, context, unit)); } - return res; } @@ -200,7 +191,6 @@ public static IAnalysisSet BinaryOperation(this IAnalysisSet self, Node node, An foreach (var ns in self) { res = res.Union(ns.BinaryOperation(node, unit, operation, rhs)); } - return res; } @@ -212,7 +202,6 @@ public static IAnalysisSet UnaryOperation(this IAnalysisSet self, Node node, Ana foreach (var ns in self) { res = res.Union(ns.UnaryOperation(node, unit, operation)); } - return res; } @@ -228,9 +217,8 @@ internal static AnalysisValue GetUnionType(this IAnalysisSet types) { /// /// Gets instance representations of all members of the set. /// - public static IAnalysisSet GetInstanceType(this IAnalysisSet types) { - return AnalysisSet.Create(types.SelectMany(ns => ns.GetInstanceType())); - } + public static IAnalysisSet GetInstanceType(this IAnalysisSet types) + => AnalysisSet.Create(types.SelectMany(ns => (ns.PythonType as IPythonType2)?.IsClass == true ? ns : ns.GetInstanceType())); public static bool IsUnknown(this IAnalysisSet res) { return res == null || @@ -241,17 +229,15 @@ public static bool IsUnknown(this IAnalysisSet res) { /// /// Returns true if the set contains no or only the object or unknown types /// - public static bool IsObjectOrUnknown(this IAnalysisSet res) { - return res.IsUnknown() || res.All(v => v.TypeId == BuiltinTypeId.Object); - } + public static bool IsObjectOrUnknown(this IAnalysisSet res) + => res.IsUnknown() || res.All(v => v.TypeId == BuiltinTypeId.Object); /// /// Returns true if the set contains no types, only the object or unknown /// types, or None. /// - public static bool IsObjectOrUnknownOrNone(this IAnalysisSet res) { - return res.IsObjectOrUnknown() || res.All(v => v.TypeId == BuiltinTypeId.NoneType); - } + public static bool IsObjectOrUnknownOrNone(this IAnalysisSet res) + => res.IsObjectOrUnknown() || res.All(v => v.TypeId == BuiltinTypeId.NoneType); /// /// Returns a sequence of all recognized string values in the set. diff --git a/src/Analysis/Engine/Impl/Analyzer/FunctionAnalysisUnit.cs b/src/Analysis/Engine/Impl/Analyzer/FunctionAnalysisUnit.cs index 8774ae066..fc8c375ca 100644 --- a/src/Analysis/Engine/Impl/Analyzer/FunctionAnalysisUnit.cs +++ b/src/Analysis/Engine/Impl/Analyzer/FunctionAnalysisUnit.cs @@ -9,13 +9,12 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using System.Threading; using Microsoft.PythonTools.Analysis.Infrastructure; diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs index 9eecd8018..989262c4f 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs @@ -65,6 +65,7 @@ public void Walk() { var self = GetSelf(); _selfType = (self as AstPythonConstant)?.Type as AstPythonType; + var index = Target.Name == "namedtuple"; _overload.ReturnTypes.AddRange(_scope.GetTypesFromAnnotation(Target.ReturnAnnotation).ExcludeDefault()); _scope.PushScope(); diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs index 4fe84a063..c081f2723 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs @@ -385,8 +385,8 @@ private AstPythonType CreateType(ClassDefinition node) { throw new ArgumentNullException(nameof(node)); } return CreateBuiltinTypes - ? new AstPythonBuiltinType(_ast, _module, node, GetDoc(node.Body as SuiteStatement), GetLoc(node)) - : new AstPythonType(_ast, _module, node, GetDoc(node.Body as SuiteStatement), GetLoc(node)); + ? new AstPythonBuiltinType(node.Name, _ast, _module, node.StartIndex, GetDoc(node.Body as SuiteStatement), GetLoc(node)) + : new AstPythonType(node.Name, _ast, _module, node.StartIndex, GetDoc(node.Body as SuiteStatement), GetLoc(node)); } private void CollectTopLevelDefinitions() { var s = (_ast.Body as SuiteStatement).Statements.ToArray(); diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs index 0976d4c08..0c2ea2539 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs @@ -27,13 +27,16 @@ public AstPythonBuiltinType(string name, BuiltinTypeId typeId) } public AstPythonBuiltinType( + string name, PythonAst ast, IPythonModule declModule, - ClassDefinition def, + int startIndex, string doc, - LocationInfo loc - ) : base(ast, declModule, def, doc, loc) { - _typeId = BuiltinTypeId.Unknown; + LocationInfo loc, + BuiltinTypeId typeId = BuiltinTypeId.Unknown, + bool isClass = false + ) : base(name, ast, declModule, startIndex, doc, loc, isClass) { + _typeId = typeId; } public bool TrySetTypeId(BuiltinTypeId typeId) { @@ -49,8 +52,8 @@ public bool TrySetTypeId(BuiltinTypeId typeId) { public bool IsHidden { get { - lock (_members) { - return _members.ContainsKey("__hidden__"); + lock (Members) { + return Members.ContainsKey("__hidden__"); } } } diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs index b816dc7fc..7795de97e 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs @@ -23,31 +23,34 @@ using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.PythonTools.Interpreter.Ast { - class AstPythonType : IPythonType, IMemberContainer, ILocatedMember, IHasQualifiedName { + class AstPythonType : IPythonType2, IMemberContainer, ILocatedMember, IHasQualifiedName { private static readonly IPythonModule NoDeclModule = new AstPythonModule(); + private Dictionary _members; private readonly string _name; private IReadOnlyList _mro; private AsyncLocal _isProcessing = new AsyncLocal(); - protected readonly Dictionary _members; + protected Dictionary Members => + _members ?? (_members = new Dictionary()); public AstPythonType(string name): this(name, new Dictionary(), Array.Empty()) { } public AstPythonType( + string name, PythonAst ast, IPythonModule declModule, - ClassDefinition def, + int startIndex, string doc, - ILocationInfo loc + ILocationInfo loc, + bool isClass = false ) { - _members = new Dictionary(); - - _name = def?.Name ?? throw new ArgumentNullException(nameof(def)); + _name = name ?? throw new ArgumentNullException(nameof(name)); Documentation = doc; DeclaringModule = declModule ?? throw new ArgumentNullException(nameof(declModule)); Locations = loc != null ? new[] { loc } : Array.Empty(); - StartIndex = def?.StartIndex ?? 0; + StartIndex = startIndex; + IsClass = isClass; } private AstPythonType(string name, Dictionary members, IEnumerable locations) { @@ -59,30 +62,30 @@ private AstPythonType(string name, Dictionary members, IEnumera } internal void AddMembers(IEnumerable> members, bool overwrite) { - lock (_members) { + lock (Members) { foreach (var kv in members) { if (!overwrite) { - if (_members.TryGetValue(kv.Key, out var existing)) { + if (Members.TryGetValue(kv.Key, out var existing)) { continue; } } - _members[kv.Key] = kv.Value; + Members[kv.Key] = kv.Value; } } } internal void SetBases(IPythonInterpreter interpreter, IEnumerable bases) { - lock (_members) { + lock (Members) { if (Bases != null) { return; // Already set } Bases = bases.MaybeEnumerate().ToArray(); if (Bases.Count > 0) { - _members["__base__"] = Bases[0]; + Members["__base__"] = Bases[0]; } - _members["__bases__"] = new AstPythonSequence( + Members["__bases__"] = new AstPythonSequence( interpreter?.GetBuiltinType(BuiltinTypeId.Tuple), DeclaringModule, Bases, @@ -93,7 +96,7 @@ internal void SetBases(IPythonInterpreter interpreter, IEnumerable public IReadOnlyList Mro { get { - lock (_members) { + lock (Members) { if (_mro != null) { return _mro; } @@ -164,9 +167,9 @@ internal static IReadOnlyList CalculateMro(IPythonType cls, HashSet public string Name { get { - lock (_members) { + lock (Members) { IMember nameMember; - if (_members.TryGetValue("__name__", out nameMember) && nameMember is AstPythonStringLiteral lit) { + if (Members.TryGetValue("__name__", out nameMember) && nameMember is AstPythonStringLiteral lit) { return lit.Value; } } @@ -179,6 +182,7 @@ public string Name { public virtual bool IsBuiltin => false; public PythonMemberType MemberType => PythonMemberType.Class; public virtual BuiltinTypeId TypeId => BuiltinTypeId.Type; + public virtual bool IsClass { get; } /// /// The start index of this class. Used to disambiguate multiple @@ -193,15 +197,15 @@ public string Name { public IMember GetMember(IModuleContext context, string name) { IMember member; - lock (_members) { - if (_members.TryGetValue(name, out member)) { + lock (Members) { + if (Members.TryGetValue(name, out member)) { return member; } - // Special case names that we want to add to our own _members dict + // Special case names that we want to add to our own Members dict switch (name) { case "__mro__": - member = _members[name] = new AstPythonSequence( + member = Members[name] = new AstPythonSequence( (context as IPythonInterpreter)?.GetBuiltinType(BuiltinTypeId.Tuple), DeclaringModule, Mro, @@ -232,8 +236,8 @@ public IMember GetMember(IModuleContext context, string name) { public IEnumerable GetMemberNames(IModuleContext moduleContext) { var names = new HashSet(); - lock (_members) { - names.UnionWith(_members.Keys); + lock (Members) { + names.UnionWith(Members.Keys); } foreach (var m in Mro.Skip(1)) { diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs index 1d9df306e..96662b361 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs @@ -94,7 +94,7 @@ public override IPythonType GetTypeMember(IPythonType baseType, string member) public override IPythonType MakeUnion(IReadOnlyList types) => new UnionType(types); - public override IReadOnlyList GetUnionTypes(IPythonType unionType) => + public override IReadOnlyList GetUnionTypes(IPythonType unionType) => (unionType as UnionType)?.Types ?? (unionType as IPythonMultipleMembers)?.Members.OfType().ToArray(); @@ -129,7 +129,7 @@ public override IPythonType MakeGeneric(IPythonType baseType, IReadOnlyList 0 ? MakeGenericClassType(args[0]) : _scope.Interpreter.GetBuiltinType(BuiltinTypeId.Type); case "Any": return baseType; // TODO: Other types @@ -192,6 +192,11 @@ private IPythonType MakeLookupType(BuiltinTypeId typeId, IReadOnlyList typeArg.IsBuiltin + ? new AstPythonBuiltinType(typeArg.Name, _scope.Ast, _scope.Module, 0, typeArg.Documentation, null, typeArg.TypeId, isClass: true) + : new AstPythonType(typeArg.Name, _scope.Ast, _scope.Module, 0, typeArg.Documentation, null, isClass: true); + private class ModuleType : IPythonType { public ModuleType(IPythonModule module) { DeclaringModule = module; diff --git a/src/Analysis/Engine/Impl/Interpreter/Definitions/IPythonType.cs b/src/Analysis/Engine/Impl/Interpreter/Definitions/IPythonType.cs index 059100de3..a3f64a0e7 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Definitions/IPythonType.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Definitions/IPythonType.cs @@ -9,7 +9,7 @@ // THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS // OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY // IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. +// MERCHANTABILITY OR NON-INFRINGEMENT. // // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. @@ -21,28 +21,29 @@ public interface IPythonType : IMemberContainer, IMember { IPythonFunction GetConstructors(); // PythonType.Get__name__(this); - string Name { - get; - } + string Name { get; } - string Documentation { - get; - } + /// + /// Human-readable documentation that may be displayed in the editor hover tooltip. + /// + string Documentation { get; } - BuiltinTypeId TypeId { - get; - } + BuiltinTypeId TypeId { get; } - IPythonModule DeclaringModule { - get; - } + IPythonModule DeclaringModule { get; } - IReadOnlyList Mro { - get; - } + IReadOnlyList Mro { get; } - bool IsBuiltin { - get; - } + bool IsBuiltin { get; } + } + + public interface IPythonType2 : IPythonType { + /// + /// Indicates that type is a class. Used in cases when function has to return + /// a class rather than the class instance. Example: function annotated as '-> Type[T]' + /// can be called as a T constructor so func() constructs class instance rather than invoking + /// // call on an existing instance. See also collections/namedtuple typing in the Typeshed. + /// + bool IsClass { get; } } } diff --git a/src/Analysis/Engine/Test/AnalysisTest.cs b/src/Analysis/Engine/Test/AnalysisTest.cs index 778194613..abbf8ebf6 100644 --- a/src/Analysis/Engine/Test/AnalysisTest.cs +++ b/src/Analysis/Engine/Test/AnalysisTest.cs @@ -3117,6 +3117,23 @@ def ls() -> List[tuple]: } } + [TestMethod, Priority(0)] + public async Task NamedTupleReturnAnnotation() { + using (var server = await CreateServerAsync()) { + server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); + server.Analyzer.Limits = new AnalysisLimits { UseTypeStubPackages = true, UseTypeStubPackagesExclusively = false }; + + var code = @" +from typing import Union, Iterable, Type +def namedtuple(typename: str, field_names: Union[str, Iterable[str]]) -> Type[tuple]: ... +nt = namedtuple('Point', ['x', 'y']) +pt = nt(1, 2) +"; + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); + analysis.Should().HaveVariable("pt").OfTypes(BuiltinTypeId.Tuple); + } + } + [TestMethod, Priority(0)] public async Task CollectionsNamedTuple() { using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) { diff --git a/src/Analysis/Engine/Test/AstAnalysisTests.cs b/src/Analysis/Engine/Test/AstAnalysisTests.cs index bf2c5ca86..b474ba54a 100644 --- a/src/Analysis/Engine/Test/AstAnalysisTests.cs +++ b/src/Analysis/Engine/Test/AstAnalysisTests.cs @@ -324,30 +324,6 @@ def overdrawn(self): } } - - [TestMethod, Priority(0)] - public async Task AstNamedTuple() { - using (var server = await CreateServerAsync()) { - var code = @" -from typing import List - -def ls() -> List[tuple]: - pass - -x = ls()[0] -"; - - var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); - - analysis - .Should().HaveVariable("ls").Which - .Should().HaveShortDescriptions("module.ls() -> list[tuple]"); - - analysis - .Should().HaveVariable("x").Which - .Should().HaveType(BuiltinTypeId.Tuple); - } - } [TestMethod, Priority(0)] public async Task AstSearchPathsThroughFactory() { using (var factory = CreateInterpreterFactory()) @@ -482,23 +458,6 @@ public async Task AstTypeStubPaths_ExclusiveStubs() { } } - private async Task GetStubBasedAnalysis( - Server server, - string code, - AnalysisLimits limits, - IEnumerable searchPaths, - IEnumerable stubPaths) { - - if (limits != null) { - server.Analyzer.Limits = limits; - } - server.Analyzer.SetSearchPaths(searchPaths); - server.Analyzer.SetTypeStubPaths(stubPaths); - - var uri = await server.OpenDefaultDocumentAndGetUriAsync(code); - return await server.GetAnalysisAsync(uri); - } - [TestMethod, Priority(0)] public void AstMro() { var O = new AstPythonType("O"); @@ -1039,38 +998,55 @@ public async Task TypeShedSysInfo() { } [TestMethod, Priority(0)] - public async Task TypeShedNamedTuple() { - using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) { - server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); - var code = "from collections import namedtuple\n"; + public async Task AstReturnAnnotationList() { + using (var server = await CreateServerAsync()) { + var code = @" +from typing import List + +def ls() -> List[tuple]: + pass + +x = ls()[0] +"; var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); + analysis - .Should().HaveFunction("namedtuple").Which - .Should().HaveResolvedReturnTypes(BuiltinTypeId.Tuple); + .Should().HaveVariable("ls").Which + .Should().HaveShortDescriptions("module.ls() -> list[tuple]"); + + analysis + .Should().HaveVariable("x").Which + .Should().HaveType(BuiltinTypeId.Tuple); } } [TestMethod, Priority(0)] - public async Task NamedTupleAnnotation() { + public async Task AstNamedTupleReturnAnnotation() { using (var server = await CreateServerAsync()) { - server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); - server.Analyzer.Limits = new AnalysisLimits { UseTypeStubPackages = true, UseTypeStubPackagesExclusively = false }; - - var code = @" -from typing import Union, Iterable, Type - -def namedtuple(typename: str, field_names: Union[str, Iterable[str]]) -> Type[tuple]: ... - + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(@" +from ReturnAnnotation import * nt = namedtuple('Point', ['x', 'y']) pt = nt(1, 2) -"; - var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); - +"); analysis.Should().HaveVariable("pt").OfTypes(BuiltinTypeId.Tuple); } } + [TestMethod, Priority(0)] + public async Task TypeShedNamedTuple() { + using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) { + server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); + server.Analyzer.Limits = new AnalysisLimits { UseTypeStubPackages = true, UseTypeStubPackagesExclusively = true }; + var code = "from collections import namedtuple\n"; + + var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); + analysis + .Should().HaveBuiltInFunctionInfo("namedtuple").Which + .Should().HaveOverloadAt(1).Which + .Should().HaveSingleReturnType("tuple"); + } + } [TestMethod, Priority(0)] public void TypeStubConditionalDefine() { diff --git a/src/Analysis/Engine/Test/FluentAssertions/ModuleAnalysisAssertions.cs b/src/Analysis/Engine/Test/FluentAssertions/ModuleAnalysisAssertions.cs index 808f94c58..08b5ecfaf 100644 --- a/src/Analysis/Engine/Test/FluentAssertions/ModuleAnalysisAssertions.cs +++ b/src/Analysis/Engine/Test/FluentAssertions/ModuleAnalysisAssertions.cs @@ -80,6 +80,12 @@ public AndWhichConstraint>(this, constraint.Which); } + public AndWhichConstraint> HaveBuiltInFunctionInfo(string name, string because = "", params object[] reasonArgs) { + NotBeNull(because, reasonArgs); + var constraint = _scopeAssertions.HaveBuiltInFunctionInfo(name, because, reasonArgs); + return new AndWhichConstraint>(this, constraint.Which); + } + public AndWhichConstraint HaveFunction(string name, string because = "", params object[] reasonArgs) { NotBeNull(because, reasonArgs); var constraint = _scopeAssertions.HaveFunction(name, because, reasonArgs); diff --git a/src/Analysis/Engine/Test/FluentAssertions/ScopeAssertions.cs b/src/Analysis/Engine/Test/FluentAssertions/ScopeAssertions.cs index 857a2c339..aecdf81bf 100644 --- a/src/Analysis/Engine/Test/FluentAssertions/ScopeAssertions.cs +++ b/src/Analysis/Engine/Test/FluentAssertions/ScopeAssertions.cs @@ -95,6 +95,13 @@ public AndWhichConstraint return new AndWhichConstraint>((TScopeAssertions)this, assertion.Which); } + public AndWhichConstraint> HaveBuiltInFunctionInfo(string name, string because = "", params object[] reasonArgs) { + var assertion = HaveVariable(name, because, reasonArgs) + .Which.Should().HaveValue(); + + return new AndWhichConstraint>((TScopeAssertions)this, assertion.Which); + } + public AndWhichConstraint HaveFunction(string name, string because = "", params object[] reasonArgs) { var assertion = HaveVariable(name, because, reasonArgs) .Which.Should().HaveValue() diff --git a/src/UnitTests/TestData/AstAnalysis/ReturnAnnotation.py b/src/UnitTests/TestData/AstAnalysis/ReturnAnnotation.py new file mode 100644 index 000000000..d4f4d89a4 --- /dev/null +++ b/src/UnitTests/TestData/AstAnalysis/ReturnAnnotation.py @@ -0,0 +1,3 @@ +from typing import Union, Iterable, Type + +def namedtuple(typename: str, field_names: Union[str, Iterable[str]]) -> Type[tuple]: ... From 3f928976c011dac594e2ca453ca486c82b5e23ea Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Thu, 1 Nov 2018 16:53:07 -0700 Subject: [PATCH 11/19] Named tuple, part II --- .../Impl/Interpreter/Ast/AstAnalysisWalker.cs | 5 +++- .../Ast/AstPythonFunctionOverload.cs | 4 ++- .../Engine/Impl/Values/BuiltinFunctionInfo.cs | 28 +++++++++++-------- .../Engine/Impl/Values/BuiltinMethodInfo.cs | 10 +++++-- src/Analysis/Engine/Impl/Values/Utils.cs | 6 ---- src/Analysis/Engine/Test/AnalysisTest.cs | 3 -- src/Analysis/Engine/Test/AstAnalysisTests.cs | 2 +- 7 files changed, 33 insertions(+), 25 deletions(-) diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs index c081f2723..9055f6551 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs @@ -368,7 +368,10 @@ private IPythonFunctionOverload CreateFunctionOverload(NameLookupContext funcSco .Select(p => new AstPythonParameterInfo(_ast, p, _scope.GetTypesFromAnnotation(p.Annotation))) .ToArray(); - var overload = new AstPythonFunctionOverload(parameters, funcScope.GetLocOfName(node, node.NameExpression)); + var overload = new AstPythonFunctionOverload( + parameters, + funcScope.GetLocOfName(node, node.NameExpression), + node.ReturnAnnotation?.ToCodeString(_ast)); _functionWalkers.Add(new AstAnalysisFunctionWalker(funcScope, node, overload)); return overload; diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunctionOverload.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunctionOverload.cs index a0fe051a8..485c69cc4 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunctionOverload.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonFunctionOverload.cs @@ -25,10 +25,12 @@ class AstPythonFunctionOverload : IPythonFunctionOverload, ILocatedMember { public AstPythonFunctionOverload( IEnumerable parameters, - ILocationInfo loc + ILocationInfo loc, + string returnDocumentation = null ) { _parameters = parameters?.ToArray() ?? throw new ArgumentNullException(nameof(parameters)); Locations = loc != null ? new[] { loc } : Array.Empty(); + ReturnDocumentation = returnDocumentation; } internal void SetDocumentation(string doc) { diff --git a/src/Analysis/Engine/Impl/Values/BuiltinFunctionInfo.cs b/src/Analysis/Engine/Impl/Values/BuiltinFunctionInfo.cs index ba429f34b..ced1cec45 100644 --- a/src/Analysis/Engine/Impl/Values/BuiltinFunctionInfo.cs +++ b/src/Analysis/Engine/Impl/Values/BuiltinFunctionInfo.cs @@ -16,7 +16,6 @@ using System; using System.Collections.Generic; -using System.Collections.ObjectModel; using System.Linq; using Microsoft.PythonTools.Analysis.Infrastructure; using Microsoft.PythonTools.Interpreter; @@ -25,15 +24,13 @@ namespace Microsoft.PythonTools.Analysis.Values { internal class BuiltinFunctionInfo : BuiltinNamespace, IHasRichDescription, IHasQualifiedName { private string _doc; - private ReadOnlyCollection _overloads; - private readonly Lazy _returnTypes; + private OverloadResult[] _overloads; private BuiltinMethodInfo _method; public BuiltinFunctionInfo(IPythonFunction function, PythonAnalyzer projectState) : base(projectState.Types[BuiltinTypeId.BuiltinFunction], projectState) { Function = function; - _returnTypes = new Lazy(() => Utils.GetReturnTypes(function, projectState).GetInstanceType()); } public override IPythonType PythonType => _type; @@ -43,8 +40,10 @@ public override bool IsOfType(IAnalysisSet klass) { klass.Contains(ProjectState.ClassInfos[BuiltinTypeId.BuiltinFunction]); } - public override IAnalysisSet Call(Node node, AnalysisUnit unit, IAnalysisSet[] args, NameExpression[] keywordArgNames) - => _returnTypes.Value; + public override IAnalysisSet Call(Node node, AnalysisUnit unit, IAnalysisSet[] args, NameExpression[] keywordArgNames) + => AnalysisSet.UnionAll(Overloads + .Where(o => o.ReturnType != null) + .Select(o => ProjectState.GetAnalysisSetFromObjects(o.ReturnType))); public override IAnalysisSet GetDescriptor(Node node, AnalysisValue instance, AnalysisValue context, AnalysisUnit unit) { if (Function.IsClassMethod) { @@ -124,12 +123,19 @@ private static IEnumerable> GetParameterString(IPyt public override IEnumerable Overloads { get { if (_overloads == null) { - var overloads = Function.Overloads; - var result = new OverloadResult[overloads.Count]; - for (int i = 0; i < result.Length; i++) { - result[i] = new BuiltinFunctionOverloadResult(ProjectState, Function.Name, overloads[i], 0, () => Description); + var overloads = Function.Overloads.ToArray(); + // If some of the overloads have return type annotations, drop others. + // This helps when function has multiple overloads with some definitions + // comings from the library code and some from the Typeshed. Library code + // has documentation but often lacks return types. + overloads = overloads.Any(o => !string.IsNullOrEmpty(o.ReturnDocumentation)) + ? overloads.Where(o => !string.IsNullOrEmpty(o.ReturnDocumentation)).ToArray() + : overloads; + + _overloads = new OverloadResult[overloads.Length]; + for (var i = 0; i < _overloads.Length; i++) { + _overloads[i] = new BuiltinFunctionOverloadResult(ProjectState, Function.Name, overloads[i], 0, () => Description); } - _overloads = new ReadOnlyCollection(result); } return _overloads; } diff --git a/src/Analysis/Engine/Impl/Values/BuiltinMethodInfo.cs b/src/Analysis/Engine/Impl/Values/BuiltinMethodInfo.cs index 058bf578c..f8978533e 100644 --- a/src/Analysis/Engine/Impl/Values/BuiltinMethodInfo.cs +++ b/src/Analysis/Engine/Impl/Values/BuiltinMethodInfo.cs @@ -34,14 +34,14 @@ public BuiltinMethodInfo(IPythonMethodDescriptor method, PythonAnalyzer projectS var function = method.Function; _memberType = method.MemberType; _function = function; - _returnTypes = Utils.GetReturnTypes(function, projectState); + _returnTypes = GetReturnTypes(function, projectState); } public BuiltinMethodInfo(IPythonFunction function, PythonMemberType memType, PythonAnalyzer projectState) : base(projectState.Types[BuiltinTypeId.BuiltinMethodDescriptor], projectState) { _memberType = memType; _function = function; - _returnTypes = Utils.GetReturnTypes(function, projectState); + _returnTypes = GetReturnTypes(function, projectState); _fromFunction = true; } @@ -105,5 +105,11 @@ public override string Documentation { public override int GetHashCode() => new { hc1 = base.GetHashCode(), hc2 = _function.GetHashCode() }.GetHashCode(); public override bool Equals(object obj) => base.Equals(obj) && obj is BuiltinMethodInfo bmi && _function.Equals(bmi._function); + + private IAnalysisSet GetReturnTypes(IPythonFunction func, PythonAnalyzer projectState) { + return AnalysisSet.UnionAll(func.Overloads + .Where(fn => fn.ReturnType != null) + .Select(fn => projectState.GetAnalysisSetFromObjects(fn.ReturnType))); + } } } diff --git a/src/Analysis/Engine/Impl/Values/Utils.cs b/src/Analysis/Engine/Impl/Values/Utils.cs index 8549186ba..8e64156d6 100644 --- a/src/Analysis/Engine/Impl/Values/Utils.cs +++ b/src/Analysis/Engine/Impl/Values/Utils.cs @@ -46,12 +46,6 @@ internal static string StripDocumentation(string doc) { return result.ToString(); } - internal static IAnalysisSet GetReturnTypes(IPythonFunction func, PythonAnalyzer projectState) { - return AnalysisSet.UnionAll(func.Overloads - .Where(fn => fn.ReturnType != null) - .Select(fn => projectState.GetAnalysisSetFromObjects(fn.ReturnType))); - } - internal static T First(IEnumerable sequence) where T : class { if (sequence == null) { return null; diff --git a/src/Analysis/Engine/Test/AnalysisTest.cs b/src/Analysis/Engine/Test/AnalysisTest.cs index abbf8ebf6..a2258c58f 100644 --- a/src/Analysis/Engine/Test/AnalysisTest.cs +++ b/src/Analysis/Engine/Test/AnalysisTest.cs @@ -3120,9 +3120,6 @@ def ls() -> List[tuple]: [TestMethod, Priority(0)] public async Task NamedTupleReturnAnnotation() { using (var server = await CreateServerAsync()) { - server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); - server.Analyzer.Limits = new AnalysisLimits { UseTypeStubPackages = true, UseTypeStubPackagesExclusively = false }; - var code = @" from typing import Union, Iterable, Type def namedtuple(typename: str, field_names: Union[str, Iterable[str]]) -> Type[tuple]: ... diff --git a/src/Analysis/Engine/Test/AstAnalysisTests.cs b/src/Analysis/Engine/Test/AstAnalysisTests.cs index b474ba54a..7fa1180ab 100644 --- a/src/Analysis/Engine/Test/AstAnalysisTests.cs +++ b/src/Analysis/Engine/Test/AstAnalysisTests.cs @@ -1043,7 +1043,7 @@ public async Task TypeShedNamedTuple() { var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); analysis .Should().HaveBuiltInFunctionInfo("namedtuple").Which - .Should().HaveOverloadAt(1).Which + .Should().HaveSingleOverload().Which .Should().HaveSingleReturnType("tuple"); } } From 8e61522af5a8d947d04508170e577a658812f746 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Fri, 2 Nov 2018 07:31:52 -0700 Subject: [PATCH 12/19] Properly transfer tuple elements --- .../Impl/Analyzer/ExpressionEvaluator.cs | 2 +- .../Interpreter/Ast/AstPythonBuiltinType.cs | 2 +- .../Engine/Impl/Values/BuiltinClassInfo.cs | 18 +++++- .../Engine/Impl/Values/BuiltinFunctionInfo.cs | 57 +++++++------------ .../Definitions/IBuiltinSequenceClassInfo.cs | 23 ++++++++ .../Impl/Values/SequenceBuiltinClassInfo.cs | 4 +- src/Analysis/Engine/Test/AnalysisTest.cs | 14 ----- src/Analysis/Engine/Test/AstAnalysisTests.cs | 1 + 8 files changed, 67 insertions(+), 54 deletions(-) create mode 100644 src/Analysis/Engine/Impl/Values/Definitions/IBuiltinSequenceClassInfo.cs diff --git a/src/Analysis/Engine/Impl/Analyzer/ExpressionEvaluator.cs b/src/Analysis/Engine/Impl/Analyzer/ExpressionEvaluator.cs index 1b0f5ed8e..a86f4573d 100644 --- a/src/Analysis/Engine/Impl/Analyzer/ExpressionEvaluator.cs +++ b/src/Analysis/Engine/Impl/Analyzer/ExpressionEvaluator.cs @@ -530,7 +530,7 @@ internal void AssignTo(Node assignStmt, Expression left, IAnalysisSet values) { // list/tuple var l = (SequenceExpression)left; var valuesArr = values.ToArray(); - for (int i = 0; i < l.Items.Count; i++) { + for (var i = 0; i < l.Items.Count; i++) { if (valuesArr.Length > 0) { foreach (var value in valuesArr) { AssignTo(assignStmt, l.Items[i], value.GetIndex(assignStmt, _unit, ProjectState.GetConstant(i))); diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs index 0c2ea2539..62edf7341 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs @@ -36,7 +36,7 @@ public AstPythonBuiltinType( BuiltinTypeId typeId = BuiltinTypeId.Unknown, bool isClass = false ) : base(name, ast, declModule, startIndex, doc, loc, isClass) { - _typeId = typeId; + _typeId = typeId == BuiltinTypeId.Unknown && isClass ? BuiltinTypeId.Type : typeId; } public bool TrySetTypeId(BuiltinTypeId typeId) { diff --git a/src/Analysis/Engine/Impl/Values/BuiltinClassInfo.cs b/src/Analysis/Engine/Impl/Values/BuiltinClassInfo.cs index 8ca42bd60..c0e27ca8d 100644 --- a/src/Analysis/Engine/Impl/Values/BuiltinClassInfo.cs +++ b/src/Analysis/Engine/Impl/Values/BuiltinClassInfo.cs @@ -19,6 +19,7 @@ using System.Diagnostics; using System.Linq; using Microsoft.PythonTools.Analysis.Analyzer; +using Microsoft.PythonTools.Analysis.Infrastructure; using Microsoft.PythonTools.Interpreter; using Microsoft.PythonTools.Parsing.Ast; @@ -158,7 +159,22 @@ public override void SetMember(Node node, AnalysisUnit unit, string name, IAnaly } public override IAnalysisSet GetIndex(Node node, AnalysisUnit unit, IAnalysisSet index) { - // TODO: Needs to actually do indexing on type + try { + var indices = index.Select(x => x.GetConstantValue()).ExcludeDefault().OfType().ToArray(); + if (this is IPythonSequenceType seqt) { + var a = seqt.IndexTypes.ToArray(); + var values= indices.Where(i => i >= 0 && i < a.Length).Select(i => unit.State.GetAnalysisValueFromObjects(a[i])); + return AnalysisSet.UnionAll(values); + } + if (this is IBuiltinSequenceClassInfo bseqt) { + var a = bseqt.IndexTypes.ToArray(); + var values = indices.Where(i => i >= 0 && i < a.Length).Select(i => a[i]); + return AnalysisSet.UnionAll(values); + } + } catch { + return AnalysisSet.Empty; + } + var clrType = _type as IAdvancedPythonType; if (clrType == null || !clrType.IsGenericTypeDefinition) { return AnalysisSet.Empty; diff --git a/src/Analysis/Engine/Impl/Values/BuiltinFunctionInfo.cs b/src/Analysis/Engine/Impl/Values/BuiltinFunctionInfo.cs index ced1cec45..6689aab97 100644 --- a/src/Analysis/Engine/Impl/Values/BuiltinFunctionInfo.cs +++ b/src/Analysis/Engine/Impl/Values/BuiltinFunctionInfo.cs @@ -14,7 +14,6 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. -using System; using System.Collections.Generic; using System.Linq; using Microsoft.PythonTools.Analysis.Infrastructure; @@ -24,24 +23,21 @@ namespace Microsoft.PythonTools.Analysis.Values { internal class BuiltinFunctionInfo : BuiltinNamespace, IHasRichDescription, IHasQualifiedName { private string _doc; - private OverloadResult[] _overloads; + private IPythonFunctionOverload[] _overloads; private BuiltinMethodInfo _method; public BuiltinFunctionInfo(IPythonFunction function, PythonAnalyzer projectState) : base(projectState.Types[BuiltinTypeId.BuiltinFunction], projectState) { - Function = function; } public override IPythonType PythonType => _type; - public override bool IsOfType(IAnalysisSet klass) { - return klass.Contains(ProjectState.ClassInfos[BuiltinTypeId.Function]) || - klass.Contains(ProjectState.ClassInfos[BuiltinTypeId.BuiltinFunction]); - } + public override bool IsOfType(IAnalysisSet klass) + => klass.Contains(ProjectState.ClassInfos[BuiltinTypeId.Function]) || klass.Contains(ProjectState.ClassInfos[BuiltinTypeId.BuiltinFunction]); public override IAnalysisSet Call(Node node, AnalysisUnit unit, IAnalysisSet[] args, NameExpression[] keywordArgNames) - => AnalysisSet.UnionAll(Overloads + => AnalysisSet.UnionAll(GetFunctionOverloads() .Where(o => o.ReturnType != null) .Select(o => ProjectState.GetAnalysisSetFromObjects(o.ReturnType))); @@ -120,33 +116,10 @@ private static IEnumerable> GetParameterString(IPyt } } - public override IEnumerable Overloads { - get { - if (_overloads == null) { - var overloads = Function.Overloads.ToArray(); - // If some of the overloads have return type annotations, drop others. - // This helps when function has multiple overloads with some definitions - // comings from the library code and some from the Typeshed. Library code - // has documentation but often lacks return types. - overloads = overloads.Any(o => !string.IsNullOrEmpty(o.ReturnDocumentation)) - ? overloads.Where(o => !string.IsNullOrEmpty(o.ReturnDocumentation)).ToArray() - : overloads; - - _overloads = new OverloadResult[overloads.Length]; - for (var i = 0; i < _overloads.Length; i++) { - _overloads[i] = new BuiltinFunctionOverloadResult(ProjectState, Function.Name, overloads[i], 0, () => Description); - } - } - return _overloads; - } - } + public override IEnumerable Overloads + => GetFunctionOverloads().Select(o => new BuiltinFunctionOverloadResult(ProjectState, Function.Name, o, 0, () => Description)); - public override string Documentation { - get { - _doc = _doc ?? Utils.StripDocumentation(Function.Documentation); - return _doc; - } - } + public override string Documentation => _doc = _doc ?? (_doc = Utils.StripDocumentation(Function.Documentation)); public override PythonMemberType MemberType => Function.MemberType; @@ -181,12 +154,26 @@ internal override AnalysisValue UnionMergeTypes(AnalysisValue ns, int strength) } public override bool Equals(object obj) { - if(obj is BuiltinFunctionInfo other && !other.Overloads.SetEquals(Overloads)) { + if (obj is BuiltinFunctionInfo other && !other.Overloads.SetEquals(Overloads)) { return false; } return base.Equals(obj); } public override int GetHashCode() => base.GetHashCode() ^ Overloads.GetHashCode(); + + private IPythonFunctionOverload[] GetFunctionOverloads() { + if (_overloads == null) { + var overloads = Function.Overloads.ToArray(); + // If some of the overloads have return type annotations, drop others. + // This helps when function has multiple overloads with some definitions + // comings from the library code and some from the Typeshed. Library code + // has documentation but often lacks return types. + _overloads = overloads.Any(o => !string.IsNullOrEmpty(o.ReturnDocumentation)) + ? overloads.Where(o => !string.IsNullOrEmpty(o.ReturnDocumentation)).ToArray() + : overloads; + } + return _overloads; + } } } diff --git a/src/Analysis/Engine/Impl/Values/Definitions/IBuiltinSequenceClassInfo.cs b/src/Analysis/Engine/Impl/Values/Definitions/IBuiltinSequenceClassInfo.cs new file mode 100644 index 000000000..5cc8883ff --- /dev/null +++ b/src/Analysis/Engine/Impl/Values/Definitions/IBuiltinSequenceClassInfo.cs @@ -0,0 +1,23 @@ +// Python Tools for Visual Studio +// Copyright(c) Microsoft Corporation +// All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the License); you may not use +// this file except in compliance with the License. You may obtain a copy of the +// License at http://www.apache.org/licenses/LICENSE-2.0 +// +// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY +// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, +// MERCHANTABILITY OR NON-INFRINGEMENT. +// +// See the Apache Version 2.0 License for specific language governing +// permissions and limitations under the License. + +using System.Collections.Generic; + +namespace Microsoft.PythonTools.Analysis.Values { + public interface IBuiltinSequenceClassInfo: IBuiltinClassInfo { + IReadOnlyList IndexTypes { get; } + } +} diff --git a/src/Analysis/Engine/Impl/Values/SequenceBuiltinClassInfo.cs b/src/Analysis/Engine/Impl/Values/SequenceBuiltinClassInfo.cs index 12a9491e2..678d16eb3 100644 --- a/src/Analysis/Engine/Impl/Values/SequenceBuiltinClassInfo.cs +++ b/src/Analysis/Engine/Impl/Values/SequenceBuiltinClassInfo.cs @@ -25,7 +25,7 @@ namespace Microsoft.PythonTools.Analysis.Values { /// /// Specialized ClassInfo for sequence types. /// - abstract class SequenceBuiltinClassInfo : BuiltinClassInfo { + abstract class SequenceBuiltinClassInfo : BuiltinClassInfo, IBuiltinSequenceClassInfo { protected readonly IAnalysisSet[] _indexTypes; public SequenceBuiltinClassInfo(IPythonType classObj, PythonAnalyzer projectState) @@ -38,7 +38,7 @@ public SequenceBuiltinClassInfo(IPythonType classObj, PythonAnalyzer projectStat } } - internal IReadOnlyList IndexTypes => _indexTypes; + public IReadOnlyList IndexTypes => _indexTypes; public override IAnalysisSet Call(Node node, AnalysisUnit unit, IAnalysisSet[] args, NameExpression[] keywordArgNames) { if (args.Length == 1) { diff --git a/src/Analysis/Engine/Test/AnalysisTest.cs b/src/Analysis/Engine/Test/AnalysisTest.cs index a2258c58f..778194613 100644 --- a/src/Analysis/Engine/Test/AnalysisTest.cs +++ b/src/Analysis/Engine/Test/AnalysisTest.cs @@ -3117,20 +3117,6 @@ def ls() -> List[tuple]: } } - [TestMethod, Priority(0)] - public async Task NamedTupleReturnAnnotation() { - using (var server = await CreateServerAsync()) { - var code = @" -from typing import Union, Iterable, Type -def namedtuple(typename: str, field_names: Union[str, Iterable[str]]) -> Type[tuple]: ... -nt = namedtuple('Point', ['x', 'y']) -pt = nt(1, 2) -"; - var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); - analysis.Should().HaveVariable("pt").OfTypes(BuiltinTypeId.Tuple); - } - } - [TestMethod, Priority(0)] public async Task CollectionsNamedTuple() { using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) { diff --git a/src/Analysis/Engine/Test/AstAnalysisTests.cs b/src/Analysis/Engine/Test/AstAnalysisTests.cs index 7fa1180ab..fc25fc621 100644 --- a/src/Analysis/Engine/Test/AstAnalysisTests.cs +++ b/src/Analysis/Engine/Test/AstAnalysisTests.cs @@ -921,6 +921,7 @@ public async Task TypeShedSysExcInfo() { e1, e2, e3 = sys.exc_info()"; var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code); + // sys.exc_info() -> (exception_type, exception_value, traceback) analysis.Should().HaveVariable("e1").OfTypes(BuiltinTypeId.Type) .And.HaveVariable("e2").OfTypes("BaseException") .And.HaveVariable("e3").OfTypes(BuiltinTypeId.NoneType) From ac07b7c6f20f837d31758677413608e78b570685 Mon Sep 17 00:00:00 2001 From: Mikhail Arkhipov Date: Fri, 2 Nov 2018 07:34:43 -0700 Subject: [PATCH 13/19] Fix AF message --- src/Analysis/Engine/Test/AstAnalysisTests.cs | 5 ++--- .../Engine/Test/FluentAssertions/AnalysisValueAssertions.cs | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Analysis/Engine/Test/AstAnalysisTests.cs b/src/Analysis/Engine/Test/AstAnalysisTests.cs index fc25fc621..b3125ff14 100644 --- a/src/Analysis/Engine/Test/AstAnalysisTests.cs +++ b/src/Analysis/Engine/Test/AstAnalysisTests.cs @@ -927,9 +927,8 @@ public async Task TypeShedSysExcInfo() { .And.HaveVariable("e3").OfTypes(BuiltinTypeId.NoneType) .And.HaveVariable("sys").WithValue() .Which.Should().HaveMember("exc_info") - .Which.Should().HaveOverloadCount(2) - .And.HaveOverloadAt(0).WithSingleReturnType("tuple") - .And.HaveOverloadAt(1).WithSingleReturnType("tuple[type, BaseException, None]"); + .Which.Should().HaveSingleOverload() + .WithSingleReturnType("tuple[type BaseException, BaseException, None]"); } } diff --git a/src/Analysis/Engine/Test/FluentAssertions/AnalysisValueAssertions.cs b/src/Analysis/Engine/Test/FluentAssertions/AnalysisValueAssertions.cs index d30d59e43..7239bf33b 100644 --- a/src/Analysis/Engine/Test/FluentAssertions/AnalysisValueAssertions.cs +++ b/src/Analysis/Engine/Test/FluentAssertions/AnalysisValueAssertions.cs @@ -177,7 +177,7 @@ public AndConstraint HaveOverloadCount(int count, string because = var overloads = Subject.Overloads.ToArray(); Execute.Assertion.ForCondition(overloads.Length == count) .BecauseOf(because, reasonArgs) - .FailWith($"Expected {GetName()} to have single overload{{reason}}, but it {GetOverloadsString(overloads.Length)}."); + .FailWith($"Expected {GetName()} to have {count} overloads{{reason}}, but it {GetOverloadsString(overloads.Length)}."); return new AndConstraint((TAssertions)this); } From 74a60f11a732c885a83938b6a683ba4dd5d9c4f5 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Fri, 2 Nov 2018 09:59:57 -0700 Subject: [PATCH 14/19] Remove debug code --- .../Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs | 1 - .../Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs | 1 - .../Engine/Test/FluentAssertions/AnalysisValueAssertions.cs | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs index 989262c4f..9eecd8018 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalker.cs @@ -65,7 +65,6 @@ public void Walk() { var self = GetSelf(); _selfType = (self as AstPythonConstant)?.Type as AstPythonType; - var index = Target.Name == "namedtuple"; _overload.ReturnTypes.AddRange(_scope.GetTypesFromAnnotation(Target.ReturnAnnotation).ExcludeDefault()); _scope.PushScope(); diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs index 9d026442b..cb6300d20 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisFunctionWalkerSet.cs @@ -61,7 +61,6 @@ public void ProcessFunction(FunctionDefinition fn) { private void ProcessWalker(AstAnalysisFunctionWalker walker) { // Remove walker before processing as to prevent reentrancy. _functionWalkers.Remove(walker.Target); - var z = walker.Target.Name == "day"; walker.Walk(); } } diff --git a/src/Analysis/Engine/Test/FluentAssertions/AnalysisValueAssertions.cs b/src/Analysis/Engine/Test/FluentAssertions/AnalysisValueAssertions.cs index 7239bf33b..7f89e32f2 100644 --- a/src/Analysis/Engine/Test/FluentAssertions/AnalysisValueAssertions.cs +++ b/src/Analysis/Engine/Test/FluentAssertions/AnalysisValueAssertions.cs @@ -177,7 +177,7 @@ public AndConstraint HaveOverloadCount(int count, string because = var overloads = Subject.Overloads.ToArray(); Execute.Assertion.ForCondition(overloads.Length == count) .BecauseOf(because, reasonArgs) - .FailWith($"Expected {GetName()} to have {count} overloads{{reason}}, but it {GetOverloadsString(overloads.Length)}."); + .FailWith($"Expected {GetName()} to have {GetOverloadsString(count)}{{reason}}, but it {GetOverloadsString(overloads.Length)}."); return new AndConstraint((TAssertions)this); } From 37467e26f1d7d14c9fbe24457ab66bd99df52039 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Fri, 2 Nov 2018 11:01:17 -0700 Subject: [PATCH 15/19] Fix slicing --- .../Engine/Impl/Values/BuiltinFunctionInfo.cs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/Analysis/Engine/Impl/Values/BuiltinFunctionInfo.cs b/src/Analysis/Engine/Impl/Values/BuiltinFunctionInfo.cs index 6689aab97..80c7bcffc 100644 --- a/src/Analysis/Engine/Impl/Values/BuiltinFunctionInfo.cs +++ b/src/Analysis/Engine/Impl/Values/BuiltinFunctionInfo.cs @@ -36,10 +36,16 @@ public BuiltinFunctionInfo(IPythonFunction function, PythonAnalyzer projectState public override bool IsOfType(IAnalysisSet klass) => klass.Contains(ProjectState.ClassInfos[BuiltinTypeId.Function]) || klass.Contains(ProjectState.ClassInfos[BuiltinTypeId.BuiltinFunction]); - public override IAnalysisSet Call(Node node, AnalysisUnit unit, IAnalysisSet[] args, NameExpression[] keywordArgNames) - => AnalysisSet.UnionAll(GetFunctionOverloads() - .Where(o => o.ReturnType != null) - .Select(o => ProjectState.GetAnalysisSetFromObjects(o.ReturnType))); + public override IAnalysisSet Call(Node node, AnalysisUnit unit, IAnalysisSet[] args, NameExpression[] keywordArgNames) { + var returnTypes = GetFunctionOverloads().Where(o => o.ReturnType != null).SelectMany(o => o.ReturnType); + var types = returnTypes.Select(t => { + var av = ProjectState.GetAnalysisValueFromObjects(t); + return t is IPythonType2 pt2 && pt2.IsClass + ? AnalysisSet.Create(av) + : ProjectState.GetAnalysisValueFromObjects(t).GetInstanceType(); + }); + return AnalysisSet.UnionAll(types); + } public override IAnalysisSet GetDescriptor(Node node, AnalysisValue instance, AnalysisValue context, AnalysisUnit unit) { if (Function.IsClassMethod) { From c869f85b8aa8d18b7ce205f4c61bd0105c5ad489 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Fri, 2 Nov 2018 14:40:58 -0700 Subject: [PATCH 16/19] Add completion test Use builtin types from interpreter --- .../AnalysisOnlyInterpreterFactory.cs | 110 ------------------ .../Impl/Interpreter/Ast/AstAnalysisWalker.cs | 4 +- .../Interpreter/Ast/AstPythonBuiltinType.cs | 24 ++-- .../Impl/Interpreter/Ast/AstPythonType.cs | 28 +++-- .../Ast/AstTypeAnnotationConverter.cs | 20 ++-- .../Engine/Impl/Values/BuiltinClassInfo.cs | 19 +-- .../Engine/Impl/Values/BuiltinFunctionInfo.cs | 2 +- src/Analysis/Engine/Test/CompletionTests.cs | 22 +++- 8 files changed, 73 insertions(+), 156 deletions(-) delete mode 100644 src/Analysis/Engine/Impl/Interpreter/AnalysisOnlyInterpreterFactory.cs diff --git a/src/Analysis/Engine/Impl/Interpreter/AnalysisOnlyInterpreterFactory.cs b/src/Analysis/Engine/Impl/Interpreter/AnalysisOnlyInterpreterFactory.cs deleted file mode 100644 index 4fcef7977..000000000 --- a/src/Analysis/Engine/Impl/Interpreter/AnalysisOnlyInterpreterFactory.cs +++ /dev/null @@ -1,110 +0,0 @@ -// Python Tools for Visual Studio -// Copyright(c) Microsoft Corporation -// All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the License); you may not use -// this file except in compliance with the License. You may obtain a copy of the -// License at http://www.apache.org/licenses/LICENSE-2.0 -// -// THIS CODE IS PROVIDED ON AN *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY -// IMPLIED WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, -// MERCHANTABLITY OR NON-INFRINGEMENT. -// -// See the Apache Version 2.0 License for specific language governing -// permissions and limitations under the License. - -using System; -using System.Collections.Generic; -using System.Threading; -using System.Threading.Tasks; -using Microsoft.PythonTools.Analysis; -using Microsoft.PythonTools.Analysis.Infrastructure; - -namespace Microsoft.PythonTools.Interpreter { - sealed class AnalysisOnlyInterpreterFactory : IPythonInterpreterFactory, ICustomInterpreterSerialization { - private readonly static InterpreterFactoryCreationOptions DefaultCreationOptions = new InterpreterFactoryCreationOptions { - WatchFileSystem = false - }; - - public InterpreterConfiguration Configuration { get; } - - - public AnalysisOnlyInterpreterFactory(InterpreterConfiguration config) { - Configuration = config; - } - - public AnalysisOnlyInterpreterFactory(Version version, string description = null) - : this(GetConfiguration(version, description ?? "Analysis Interpreter {0}".FormatUI(version))) { - } - - - internal AnalysisOnlyInterpreterFactory(Dictionary properties) - : this(InterpreterConfiguration.FromDictionary(properties)) { - } - - private static InterpreterConfiguration GetConfiguration(Version version, string description) { - return new InterpreterConfiguration( - "AnalysisOnly|{0}|{1}".FormatInvariant(version, description), - description, - architecture: InterpreterArchitecture.Unknown, - version: version - ); - } - - public bool GetSerializationInfo(out string assembly, out string typeName, out Dictionary properties) { - assembly = GetType().Assembly.Location; - typeName = GetType().FullName; - properties = new Dictionary { - { nameof(Version), Configuration.Version.ToString() } - }; - return true; - } - - public IPythonInterpreter CreateInterpreter() { - return new AnalysisOnlyInterpreter(this); - } - - public void NotifyImportNamesChanged() { } - } - - sealed class AnalysisOnlyInterpreter : IPythonInterpreter2 { - private IBuiltinPythonModule _builtins; - - public AnalysisOnlyInterpreter(IPythonInterpreterFactory factory) { - Factory = factory; - } - - public IPythonInterpreterFactory Factory { get; } - - public event EventHandler ModuleNamesChanged { add { } remove { } } - - public IModuleContext CreateModuleContext() => null; - - public void Dispose() { } - - public IPythonType GetBuiltinType(BuiltinTypeId id) { - return _builtins?.GetAnyMember(id.GetTypeName(Factory.Configuration.Version)) as IPythonType; - } - - public IList GetModuleNames() { - if (_builtins != null) { - return new[] { _builtins.Name }; - } - return Array.Empty(); - } - - public Task ImportModuleAsync(string name, CancellationToken token) => Task.FromResult(ImportModule(name)); - - public IPythonModule ImportModule(string name) { - if (_builtins != null && _builtins.Name == name) { - return _builtins; - } - return null; - } - - public void Initialize(PythonAnalyzer state) { - _builtins = state.BuiltinModule.InterpreterModule as IBuiltinPythonModule; - } - } -} diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs index 9055f6551..b612b98a1 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstAnalysisWalker.cs @@ -388,8 +388,8 @@ private AstPythonType CreateType(ClassDefinition node) { throw new ArgumentNullException(nameof(node)); } return CreateBuiltinTypes - ? new AstPythonBuiltinType(node.Name, _ast, _module, node.StartIndex, GetDoc(node.Body as SuiteStatement), GetLoc(node)) - : new AstPythonType(node.Name, _ast, _module, node.StartIndex, GetDoc(node.Body as SuiteStatement), GetLoc(node)); + ? new AstPythonBuiltinType(node.Name, _module, node.StartIndex, GetDoc(node.Body as SuiteStatement), GetLoc(node)) + : new AstPythonType(node.Name, _module, node.StartIndex, GetDoc(node.Body as SuiteStatement), GetLoc(node)); } private void CollectTopLevelDefinitions() { var s = (_ast.Body as SuiteStatement).Statements.ToArray(); diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs index 62edf7341..4fb6e0f17 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonBuiltinType.cs @@ -14,8 +14,8 @@ // See the Apache Version 2.0 License for specific language governing // permissions and limitations under the License. +using System.Linq; using Microsoft.PythonTools.Analysis; -using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.PythonTools.Interpreter.Ast { class AstPythonBuiltinType : AstPythonType { @@ -28,14 +28,13 @@ public AstPythonBuiltinType(string name, BuiltinTypeId typeId) public AstPythonBuiltinType( string name, - PythonAst ast, IPythonModule declModule, int startIndex, string doc, LocationInfo loc, BuiltinTypeId typeId = BuiltinTypeId.Unknown, bool isClass = false - ) : base(name, ast, declModule, startIndex, doc, loc, isClass) { + ) : base(name, declModule, startIndex, doc, loc, isClass) { _typeId = typeId == BuiltinTypeId.Unknown && isClass ? BuiltinTypeId.Type : typeId; } @@ -50,12 +49,19 @@ public bool TrySetTypeId(BuiltinTypeId typeId) { public override bool IsBuiltin => true; public override BuiltinTypeId TypeId => _typeId; - public bool IsHidden { - get { - lock (Members) { - return Members.ContainsKey("__hidden__"); - } - } + public bool IsHidden => ContainsMember("__hidden__"); + + /// + /// Clones builtin type as class. Typically used in scenarios where method + /// returns an object that acts like a class constructor, such as namedtuple. + /// + /// + public AstPythonBuiltinType AsClass() { + var clone = new AstPythonBuiltinType(Name, DeclaringModule, StartIndex, Documentation, + Locations.OfType().FirstOrDefault(), + TypeId == BuiltinTypeId.Unknown ? BuiltinTypeId.Type : TypeId, true); + clone.AddMembers(Members, true); + return clone; } } } diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs index 7795de97e..a9393da1b 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs @@ -20,14 +20,15 @@ using System.Threading; using Microsoft.PythonTools.Analysis; using Microsoft.PythonTools.Analysis.Infrastructure; -using Microsoft.PythonTools.Parsing.Ast; namespace Microsoft.PythonTools.Interpreter.Ast { class AstPythonType : IPythonType2, IMemberContainer, ILocatedMember, IHasQualifiedName { private static readonly IPythonModule NoDeclModule = new AstPythonModule(); - private Dictionary _members; private readonly string _name; + private readonly object _lock = new object(); + + private Dictionary _members; private IReadOnlyList _mro; private AsyncLocal _isProcessing = new AsyncLocal(); @@ -38,7 +39,6 @@ class AstPythonType : IPythonType2, IMemberContainer, ILocatedMember, IHasQualif public AstPythonType( string name, - PythonAst ast, IPythonModule declModule, int startIndex, string doc, @@ -62,7 +62,7 @@ private AstPythonType(string name, Dictionary members, IEnumera } internal void AddMembers(IEnumerable> members, bool overwrite) { - lock (Members) { + lock (_lock) { foreach (var kv in members) { if (!overwrite) { if (Members.TryGetValue(kv.Key, out var existing)) { @@ -75,7 +75,7 @@ internal void AddMembers(IEnumerable> members, boo } internal void SetBases(IPythonInterpreter interpreter, IEnumerable bases) { - lock (Members) { + lock (_lock) { if (Bases != null) { return; // Already set } @@ -96,7 +96,7 @@ internal void SetBases(IPythonInterpreter interpreter, IEnumerable public IReadOnlyList Mro { get { - lock (Members) { + lock (_lock) { if (_mro != null) { return _mro; } @@ -167,9 +167,8 @@ internal static IReadOnlyList CalculateMro(IPythonType cls, HashSet public string Name { get { - lock (Members) { - IMember nameMember; - if (Members.TryGetValue("__name__", out nameMember) && nameMember is AstPythonStringLiteral lit) { + lock (_lock) { + if (Members.TryGetValue("__name__", out var nameMember) && nameMember is AstPythonStringLiteral lit) { return lit.Value; } } @@ -197,7 +196,7 @@ public string Name { public IMember GetMember(IModuleContext context, string name) { IMember member; - lock (Members) { + lock (_lock) { if (Members.TryGetValue(name, out member)) { return member; } @@ -236,15 +235,20 @@ public IMember GetMember(IModuleContext context, string name) { public IEnumerable GetMemberNames(IModuleContext moduleContext) { var names = new HashSet(); - lock (Members) { + lock (_lock) { names.UnionWith(Members.Keys); } foreach (var m in Mro.Skip(1)) { names.UnionWith(m.GetMemberNames(moduleContext)); } - return names; } + + protected bool ContainsMember(string name) { + lock (_members) { + return Members.ContainsKey(name); + } + } } } diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs index 96662b361..87a8544c0 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs @@ -94,9 +94,12 @@ public override IPythonType GetTypeMember(IPythonType baseType, string member) public override IPythonType MakeUnion(IReadOnlyList types) => new UnionType(types); - public override IReadOnlyList GetUnionTypes(IPythonType unionType) => - (unionType as UnionType)?.Types ?? - (unionType as IPythonMultipleMembers)?.Members.OfType().ToArray(); + public override IReadOnlyList GetUnionTypes(IPythonType type) => + type is UnionType unionType + ? unionType.Types + : type is IPythonMultipleMembers multipleMembers + ? multipleMembers.Members.OfType().ToArray() + : null; public override IPythonType MakeGeneric(IPythonType baseType, IReadOnlyList args) { if (args == null || args.Count == 0 || baseType == null) { @@ -192,10 +195,13 @@ private IPythonType MakeLookupType(BuiltinTypeId typeId, IReadOnlyList typeArg.IsBuiltin - ? new AstPythonBuiltinType(typeArg.Name, _scope.Ast, _scope.Module, 0, typeArg.Documentation, null, typeArg.TypeId, isClass: true) - : new AstPythonType(typeArg.Name, _scope.Ast, _scope.Module, 0, typeArg.Documentation, null, isClass: true); + private IPythonType MakeGenericClassType(IPythonType typeArg) { + if (typeArg.IsBuiltin) { + var builtinType = _scope.Interpreter.GetBuiltinType(typeArg.TypeId) as AstPythonBuiltinType; + return builtinType.AsClass(); + } + return new AstPythonType(typeArg.Name, _scope.Module, 0, typeArg.Documentation, null, isClass: true); + } private class ModuleType : IPythonType { public ModuleType(IPythonModule module) { diff --git a/src/Analysis/Engine/Impl/Values/BuiltinClassInfo.cs b/src/Analysis/Engine/Impl/Values/BuiltinClassInfo.cs index c0e27ca8d..23ac7ffca 100644 --- a/src/Analysis/Engine/Impl/Values/BuiltinClassInfo.cs +++ b/src/Analysis/Engine/Impl/Values/BuiltinClassInfo.cs @@ -159,20 +159,11 @@ public override void SetMember(Node node, AnalysisUnit unit, string name, IAnaly } public override IAnalysisSet GetIndex(Node node, AnalysisUnit unit, IAnalysisSet index) { - try { - var indices = index.Select(x => x.GetConstantValue()).ExcludeDefault().OfType().ToArray(); - if (this is IPythonSequenceType seqt) { - var a = seqt.IndexTypes.ToArray(); - var values= indices.Where(i => i >= 0 && i < a.Length).Select(i => unit.State.GetAnalysisValueFromObjects(a[i])); - return AnalysisSet.UnionAll(values); - } - if (this is IBuiltinSequenceClassInfo bseqt) { - var a = bseqt.IndexTypes.ToArray(); - var values = indices.Where(i => i >= 0 && i < a.Length).Select(i => a[i]); - return AnalysisSet.UnionAll(values); - } - } catch { - return AnalysisSet.Empty; + var indices = index.Select(x => x.GetConstantValue()).OfType().ToArray(); + if (this is IBuiltinSequenceClassInfo bseqt) { + var a = bseqt.IndexTypes.ToArray(); + var values = indices.Where(i => i >= 0 && i < a.Length).Select(i => a[i]); + return AnalysisSet.UnionAll(values); } var clrType = _type as IAdvancedPythonType; diff --git a/src/Analysis/Engine/Impl/Values/BuiltinFunctionInfo.cs b/src/Analysis/Engine/Impl/Values/BuiltinFunctionInfo.cs index 80c7bcffc..407ca58a5 100644 --- a/src/Analysis/Engine/Impl/Values/BuiltinFunctionInfo.cs +++ b/src/Analysis/Engine/Impl/Values/BuiltinFunctionInfo.cs @@ -125,7 +125,7 @@ private static IEnumerable> GetParameterString(IPyt public override IEnumerable Overloads => GetFunctionOverloads().Select(o => new BuiltinFunctionOverloadResult(ProjectState, Function.Name, o, 0, () => Description)); - public override string Documentation => _doc = _doc ?? (_doc = Utils.StripDocumentation(Function.Documentation)); + public override string Documentation => _doc ?? (_doc = Utils.StripDocumentation(Function.Documentation)); public override PythonMemberType MemberType => Function.MemberType; diff --git a/src/Analysis/Engine/Test/CompletionTests.cs b/src/Analysis/Engine/Test/CompletionTests.cs index b841af40e..e6455319a 100644 --- a/src/Analysis/Engine/Test/CompletionTests.cs +++ b/src/Analysis/Engine/Test/CompletionTests.cs @@ -961,6 +961,26 @@ await AssertCompletion(server, analysis.DocumentUri, } } + [TestMethod, Priority(0)] + public async Task CollectionsNamedTuple() { + using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) { + var code = @" +from collections import namedtuple +nt = namedtuple('Point', ['x', 'y']) +pt = nt(1, 2) +pt. +"; + server.Analyzer.SetTypeStubPaths(new[] { GetTypeshedPath() }); + server.Analyzer.Limits = new AnalysisLimits { UseTypeStubPackages = true, UseTypeStubPackagesExclusively = false }; + + var uri = await server.OpenDefaultDocumentAndGetUriAsync(code); + await server.WaitForCompleteAnalysisAsync(CancellationToken.None); + var completion = await server.SendCompletion(uri, 4, 3); + + completion.Should().HaveLabels("count", "index"); + } + } + [TestMethod, Priority(0)] public async Task Hook() { using (var s = await CreateServerAsync()) { @@ -1119,4 +1139,4 @@ await s.DidOpenTextDocument(new DidOpenTextDocumentParams { return uri; } } -} \ No newline at end of file +} From 1abce37c48d9a309701cfd8f76349af8327b09ec Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Fri, 2 Nov 2018 14:46:01 -0700 Subject: [PATCH 17/19] Move GetIndex up --- src/Analysis/Engine/Impl/Values/BuiltinClassInfo.cs | 7 ------- .../Engine/Impl/Values/SequenceBuiltinClassInfo.cs | 7 +++++++ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Analysis/Engine/Impl/Values/BuiltinClassInfo.cs b/src/Analysis/Engine/Impl/Values/BuiltinClassInfo.cs index 23ac7ffca..c9f837950 100644 --- a/src/Analysis/Engine/Impl/Values/BuiltinClassInfo.cs +++ b/src/Analysis/Engine/Impl/Values/BuiltinClassInfo.cs @@ -159,13 +159,6 @@ public override void SetMember(Node node, AnalysisUnit unit, string name, IAnaly } public override IAnalysisSet GetIndex(Node node, AnalysisUnit unit, IAnalysisSet index) { - var indices = index.Select(x => x.GetConstantValue()).OfType().ToArray(); - if (this is IBuiltinSequenceClassInfo bseqt) { - var a = bseqt.IndexTypes.ToArray(); - var values = indices.Where(i => i >= 0 && i < a.Length).Select(i => a[i]); - return AnalysisSet.UnionAll(values); - } - var clrType = _type as IAdvancedPythonType; if (clrType == null || !clrType.IsGenericTypeDefinition) { return AnalysisSet.Empty; diff --git a/src/Analysis/Engine/Impl/Values/SequenceBuiltinClassInfo.cs b/src/Analysis/Engine/Impl/Values/SequenceBuiltinClassInfo.cs index 678d16eb3..a9a777598 100644 --- a/src/Analysis/Engine/Impl/Values/SequenceBuiltinClassInfo.cs +++ b/src/Analysis/Engine/Impl/Values/SequenceBuiltinClassInfo.cs @@ -76,6 +76,13 @@ public override IAnalysisSet Call(Node node, AnalysisUnit unit, IAnalysisSet[] a return base.Call(node, unit, args, keywordArgNames); } + public override IAnalysisSet GetIndex(Node node, AnalysisUnit unit, IAnalysisSet index) { + var indices = index.Select(x => x.GetConstantValue()).OfType().ToArray(); + var a = IndexTypes.ToArray(); + var values = indices.Where(i => i >= 0 && i < a.Length).Select(i => a[i]); + return AnalysisSet.UnionAll(values); + } + public override IEnumerable> GetRichDescription() { yield return new KeyValuePair(WellKnownRichDescriptionKinds.Type, _type.Name); if (_indexTypes == null || _indexTypes.Length == 0) { From f33f680e0d3598080feab289564bcd833601d575 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Fri, 2 Nov 2018 14:58:52 -0700 Subject: [PATCH 18/19] Return Type when converting to class unknown builtin --- .../Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs | 4 +++- src/Analysis/Engine/Test/AstAnalysisTests.cs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs index 87a8544c0..e57f9d699 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstTypeAnnotationConverter.cs @@ -198,7 +198,9 @@ private IPythonType MakeLookupType(BuiltinTypeId typeId, IReadOnlyList() .Which.Should().HaveMember("exc_info") .Which.Should().HaveSingleOverload() - .WithSingleReturnType("tuple[type BaseException, BaseException, None]"); + .WithSingleReturnType("tuple[type, BaseException, None]"); } } From aeff018f94dc28e2cc599727569dc94ac0d1fef6 Mon Sep 17 00:00:00 2001 From: MikhailArkhipov Date: Fri, 2 Nov 2018 15:00:45 -0700 Subject: [PATCH 19/19] Use proper lock object --- src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs index a9393da1b..1e20fc680 100644 --- a/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs +++ b/src/Analysis/Engine/Impl/Interpreter/Ast/AstPythonType.cs @@ -246,7 +246,7 @@ public IEnumerable GetMemberNames(IModuleContext moduleContext) { } protected bool ContainsMember(string name) { - lock (_members) { + lock (_lock) { return Members.ContainsKey(name); } }