Fix handling of generics and type declarations

src/Analysis/Engine/Impl/Values/Protocols.cs

@@ -731,4 +731,113 @@ public InstanceProtocol(ProtocolInfo self, IReadOnlyList<IAnalysisSet> arguments
         internal override int UnionHashCode(int strength) => GetHashCode();
         protected override Protocol UnionMergeTypes(Protocol p) => this;
     }
+
+    /// <summary>
+    /// List protocol that delegates work to its inner class: List[T] has list methods.
+    /// </summary>
+    class ListProtocol : IterableProtocol {
+        protected readonly AnalysisValue _actualType;
+
+        public ListProtocol(ProtocolInfo self, AnalysisValue actualType, IAnalysisSet valueTypes) : base(self, valueTypes) {
+            _actualType = actualType;
+        }
+
+        public override IDictionary<string, IAnalysisSet> GetAllMembers(IModuleContext moduleContext, GetMemberOptions options = GetMemberOptions.None)
+            => _actualType.GetAllMembers(moduleContext, options);
+    }
+
+    /// <summary>
+    /// Dict protocol that delegates work to its inner class: Dict[T1, T2] has dict methods.
+    /// </summary>
+    class DictProtocol : MappingProtocol {
+        protected readonly AnalysisValue _actualType;
+
+        public DictProtocol(ProtocolInfo self, AnalysisValue actualType, IAnalysisSet keyTypes, IAnalysisSet valueTypes, IAnalysisSet items) 
+            : base(self, keyTypes, valueTypes, items) {
+            _actualType = actualType;
+        }
+
+        public override IDictionary<string, IAnalysisSet> GetAllMembers(IModuleContext moduleContext, GetMemberOptions options = GetMemberOptions.None)
+            => _actualType.GetAllMembers(moduleContext, options);
+    }
+
+    /// <summary>
+    /// Generic protocol that delegates work to its inner class.
+    /// Typically used in NewType and TypeVar scenarios delegating 
+    /// calls to the actual class serving as new type.
+    /// </summary>
+    class TypeDelegateProtocol : Protocol {
+        protected readonly AnalysisValue _actualType;
+
+        /// <summary>
+        /// Creates protocol that delegates to real type, such as TypeVar or NewType
+        /// to the actual underlying type.
+        /// </summary>
+        /// <param name="self">Protocol info</param>
+        /// <param name="baseType">Base type to delegate to, such as 'list'.</param>
+        public TypeDelegateProtocol(ProtocolInfo self, AnalysisValue actualType) : base(self) {
+            _actualType = actualType;
+        }
+
+        protected override bool Equals(Protocol other) => _actualType.Equals((other as TypeDelegateProtocol)._actualType);
+
+        public override string Name => _actualType.Name;
+        public override string Documentation => _actualType.Documentation;
+        public override string Description => _actualType.Description;
+        public override string ShortDescription => _actualType.ShortDescription;
+        public override BuiltinTypeId TypeId => _actualType.TypeId;
+        public override PythonMemberType MemberType => _actualType.MemberType;
+        public override IEnumerable<KeyValuePair<string, string>> GetRichDescription()
+            => _actualType is IHasRichDescription rd
+                ? rd.GetRichDescription()
+                : Enumerable.Empty<KeyValuePair<string, string>>();
+
+        public override IPythonProjectEntry DeclaringModule => _actualType.DeclaringModule;
+        public override int DeclaringVersion => _actualType.DeclaringVersion;
+        public override IPythonType PythonType => _actualType.PythonType;
+        public override bool IsOfType(IAnalysisSet klass) => _actualType.IsOfType(klass);
+        public override bool IsAlive => _actualType.IsAlive;
+
+        public override IDictionary<string, IAnalysisSet> GetAllMembers(IModuleContext moduleContext, GetMemberOptions options = GetMemberOptions.None)
+            => _actualType.GetAllMembers(moduleContext, options);
+        public override IAnalysisSet GetMember(Node node, AnalysisUnit unit, string name) => _actualType.GetMember(node, unit, name);
+        public override IAnalysisSet GetTypeMember(Node node, AnalysisUnit unit, string name) => _actualType.GetTypeMember(node, unit, name);
+        public override IAnalysisSet GetInstanceType() => _actualType.GetInstanceType();
+
+        public override AnalysisUnit AnalysisUnit => _actualType.AnalysisUnit;
+        internal override void AddReference(Node node, AnalysisUnit analysisUnit) => _actualType.AddReference(node, analysisUnit);
+        public override void AugmentAssign(AugmentedAssignStatement node, AnalysisUnit unit, IAnalysisSet value)
+            => _actualType.AugmentAssign(node, unit, value);
+        public override IAnalysisSet Await(Node node, AnalysisUnit unit) => _actualType.Await(node, unit);
+        public override IAnalysisSet BinaryOperation(Node node, AnalysisUnit unit, PythonOperator operation, IAnalysisSet rhs)
+            => _actualType.BinaryOperation(node, unit, operation, rhs);
+        public override IAnalysisSet Call(Node node, AnalysisUnit unit, IAnalysisSet[] args, NameExpression[] keywordArgNames)
+            => _actualType.Call(node, unit, args, keywordArgNames);
+        public override void DeleteMember(Node node, AnalysisUnit unit, string name) => _actualType.DeleteMember(node, unit, name);
+        public override IAnalysisSet GetAsyncEnumeratorTypes(Node node, AnalysisUnit unit) => _actualType.GetAsyncEnumeratorTypes(node, unit);
+        public override IAnalysisSet GetAsyncIterator(Node node, AnalysisUnit unit) => _actualType.GetAsyncIterator(node, unit);
+        public override object GetConstantValue() => _actualType.GetConstantValue();
+        public override IAnalysisSet GetDescriptor(Node node, AnalysisValue instance, AnalysisValue context, AnalysisUnit unit)
+            => _actualType.GetDescriptor(node, instance, context, unit);
+
+        public override IAnalysisSet GetDescriptor(PythonAnalyzer projectState, AnalysisValue instance, AnalysisValue context)
+            => _actualType.GetDescriptor(projectState, instance, context);
+        public override IAnalysisSet GetEnumeratorTypes(Node node, AnalysisUnit unit) => _actualType.GetEnumeratorTypes(node, unit);
+        public override IAnalysisSet GetIndex(Node node, AnalysisUnit unit, IAnalysisSet index) => _actualType.GetIndex(node, unit, index);
+        public override IEnumerable<KeyValuePair<IAnalysisSet, IAnalysisSet>> GetItems() => _actualType.GetItems();
+        public override IAnalysisSet GetIterator(Node node, AnalysisUnit unit) => _actualType.GetIterator(node, unit);
+        public override IAnalysisSet GetReturnForYieldFrom(Node node, AnalysisUnit unit) => _actualType.GetReturnForYieldFrom(node, unit);
+        public override IEnumerable<ILocationInfo> Locations => _actualType.Locations;
+        public override IMro Mro => _actualType.Mro;
+        public override IEnumerable<OverloadResult> Overloads => _actualType.Overloads;
+        public override int? GetLength() => _actualType.GetLength();
+        internal override IAnalysisSet Resolve(AnalysisUnit unit, ResolutionContext context) => _actualType.Resolve(unit, context);
+        internal override IEnumerable<ILocationInfo> References => _actualType.References;
+        public override IAnalysisSet ReverseBinaryOperation(Node node, AnalysisUnit unit, PythonOperator operation, IAnalysisSet rhs)
+            => _actualType.ReverseBinaryOperation(node, unit, operation, rhs);
+
+        public override void SetIndex(Node node, AnalysisUnit unit, IAnalysisSet index, IAnalysisSet value) => _actualType.SetIndex(node, unit, index, value);
+        public override IAnalysisSet UnaryOperation(Node node, AnalysisUnit unit, PythonOperator operation) => _actualType.UnaryOperation(node, unit, operation);
+        public override void SetMember(Node node, AnalysisUnit unit, string name, IAnalysisSet value) => _actualType.SetMember(node, unit, name, value);
+    }
 }

src/Analysis/Engine/Impl/Values/TypingModuleInfo.cs

@@ -54,18 +54,25 @@ private IAnalysisSet GetFunction(Node node, AnalysisUnit unit, string name, Call
         private IAnalysisSet NewType_Call(Node node, AnalysisUnit unit, IAnalysisSet[] args, NameExpression[] keywordArgNames) {
             return unit.InterpreterScope.GetOrMakeNodeValue(node, Analyzer.NodeValueKind.TypeAnnotation, n => {
                 var name = PythonAnalyzer.GetArg(args, keywordArgNames, null, 0).GetConstantValueAsString().FirstOrDefault(x => !string.IsNullOrEmpty(x));
-                var baseType = PythonAnalyzer.GetArg(args, keywordArgNames, null, 1) ?? unit.State.ClassInfos[BuiltinTypeId.Object].Instance;
+                var baseTypeSet = PythonAnalyzer.GetArg(args, keywordArgNames, null, 1) ?? unit.State.ClassInfos[BuiltinTypeId.Object].Instance;
                 if (string.IsNullOrEmpty(name)) {
-                    return baseType;
+                    return baseTypeSet;
                 }
                 var instPi = new ProtocolInfo(unit.ProjectEntry, unit.State);
-                var np = new NameProtocol(instPi, name, memberType: PythonMemberType.Instance);
-                var cls = new NamespaceProtocol(instPi, "__class__");
+                var np = new NameProtocol(instPi, name, memberType: PythonMemberType.Instance, typeId: BuiltinTypeId.Type);
+                var cls = new NamespaceProtocol(instPi, "__class__"); // Declares class type
                 instPi.AddProtocol(np);
                 instPi.AddProtocol(cls);
 
+                // Add base delegate so we can see actual type members
+                var baseType = baseTypeSet.FirstOrDefault();
+                if (baseType != null) {
+                    var bt = new TypeDelegateProtocol(instPi, baseType);
+                    instPi.AddProtocol(bt);
+                }
+
                 var pi = new ProtocolInfo(unit.ProjectEntry, unit.State);
-                pi.AddProtocol(new NameProtocol(pi, name));
+                pi.AddProtocol(new NameProtocol(pi, name, memberType: PythonMemberType.Instance, typeId: BuiltinTypeId.Type));
                 pi.AddProtocol(new InstanceProtocol(pi, Array.Empty<IAnalysisSet>(), instPi));
                 pi.AddReference(n, unit);
 

src/Analysis/Engine/Impl/Values/TypingTypeInfo.cs

@@ -355,10 +355,13 @@ public IAnalysisSet Finalize(string name, IReadOnlyList<IAnalysisSet> args) {
                     if (GetSequenceTypes(name, args, out realType, out keyTypes, out valueTypes)) {
                         var p = new ProtocolInfo(Entry, State);
                         p.AddReference(_node, _unit);
+
                         np = realType == null ? new NameProtocol(p, name) : new NameProtocol(p, realType);
                         np.ExtendDescription(valueTypes.GetRichDescriptions(unionPrefix: "[", unionSuffix: "]", alwaysUsePrefixSuffix: true));
                         p.AddProtocol(np);
-                        p.AddProtocol(new IterableProtocol(p, valueTypes));
+
+                        var actualType = State.GetAnalysisValueFromObjects(realType);
+                        p.AddProtocol(new ListProtocol(p, actualType, valueTypes));
                         if (keyTypes != null) {
                             p.AddProtocol(new GetItemProtocol(p, keyTypes, valueTypes));
                         }
@@ -373,14 +376,17 @@ public IAnalysisSet Finalize(string name, IReadOnlyList<IAnalysisSet> args) {
                     if (GetSequenceTypes(name, args, out realType, out keyTypes, out valueTypes)) {
                         var p = new ProtocolInfo(Entry, State);
                         p.AddReference(_node, _unit);
+
                         np = realType == null ? new NameProtocol(p, name) : new NameProtocol(p, realType);
                         np.ExtendDescription(new KeyValuePair<string, string>(WellKnownRichDescriptionKinds.Misc, "["));
                         np.ExtendDescription(keyTypes.GetRichDescriptions(unionPrefix: "[", unionSuffix: "]", defaultIfEmpty: "Any"));
                         np.ExtendDescription(new KeyValuePair<string, string>(WellKnownRichDescriptionKinds.Comma, ", "));
                         np.ExtendDescription(valueTypes.GetRichDescriptions(unionPrefix: "[", unionSuffix: "]", defaultIfEmpty: "Any"));
                         np.ExtendDescription(new KeyValuePair<string, string>(WellKnownRichDescriptionKinds.Misc, "]"));
                         p.AddProtocol(np);
-                        p.AddProtocol(new MappingProtocol(p, keyTypes, valueTypes, MakeTuple(keyTypes, valueTypes)));
+
+                        var actualType = State.GetAnalysisValueFromObjects(realType);
+                        p.AddProtocol(new DictProtocol(p, actualType, keyTypes, valueTypes, MakeTuple(keyTypes, valueTypes)));
                         return p;
                     }
                     break;

src/Analysis/Engine/Test/AnalysisTest.cs

@@ -4225,7 +4225,84 @@ public async Task PackageRelativeImportAliasedMember() {
                 analysisPackage.Should().HaveVariable("y").OfTypes(BuiltinTypeId.Module, BuiltinTypeId.Function);
             }
         }
-
+
+        [TestMethod, Priority(0)]
+        public async Task GenericListBase() {
+            var code = @"
+from typing import List
+
+def func(a: List[str]):
+    pass
+";
+
+            using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) {
+                var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code);
+                analysis
+                    .Should().HaveFunction("func").Which
+                    .Should().HaveVariable("a")
+                    .OfTypes(BuiltinTypeId.List, BuiltinTypeId.Unknown); // list and 'function argument'
+            }
+        }
+
+        [TestMethod, Priority(0)]
+        public async Task GenericDictBase() {
+            var code = @"
+from typing import Dict
+
+def func(a: Dict[int, str]):
+    pass
+";
+
+            using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) {
+                var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code);
+                analysis
+                    .Should().HaveFunction("func").Which
+                    .Should().HaveVariable("a")
+                    .OfTypes(BuiltinTypeId.Dict, BuiltinTypeId.Unknown); // dict and 'function argument'
+            }
+        }
+
+        [TestMethod, Priority(0)]
+        public async Task NewType() {
+            var code = @"
+from typing import NewType
+
+Foo = NewType('Foo', dict)
+foo: Foo = Foo({ })
+";
+
+            using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) {
+                var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code);
+                analysis
+                    .Should().HaveVariable("Foo")
+                    .OfType(BuiltinTypeId.Type).WithDescription("Foo")
+                    .And.HaveVariable("foo")
+                    .OfType(BuiltinTypeId.Type).WithDescription("Foo");
+            }
+        }
+
+        [TestMethod, Priority(0)]
+        public async Task TypeVar() {
+            var code = @"
+from typing import Sequence, TypeVar
+
+T = TypeVar('T') # Declare type variable
+
+def first(l: Sequence[T]) -> T: # Generic function
+    return l[0]
+
+arr = [1, 2, 3]
+x = first(arr)  # should be int
+";
+
+            using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) {
+                var analysis = await server.OpenDefaultDocumentAndGetAnalysisAsync(code);
+                analysis
+                    .Should().HaveVariable("x")
+                    .OfTypes(BuiltinTypeId.Int, BuiltinTypeId.Type); // int and T
+            }
+        }
+
         [TestMethod, Priority(0)]
         public async Task Defaults() {
             var text = @"

src/Analysis/Engine/Test/CompletionTests.cs

@@ -1054,7 +1054,78 @@ public async Task MarkupKindValid() {
             }
         }
 
-        private static async Task AssertCompletion(Server s, Uri uri, IReadOnlyCollection<string> contains, IReadOnlyCollection<string> excludes, Position? position = null, CompletionContext? context = null, Func<CompletionItem, string> cmpKey = null, string expr = null, Range? applicableSpan = null, InsertTextFormat? allFormat = InsertTextFormat.PlainText) {
+        [TestMethod, Priority(0)]
+        public async Task NewType() {
+            var code = @"
+from typing import NewType
+
+Foo = NewType('Foo', dict)
+foo: Foo = Foo({ })
+foo.
+";
+            using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) {
+                var uri = await server.OpenDefaultDocumentAndGetUriAsync(code);
+                await server.GetAnalysisAsync(uri);
+
+                var completions = await server.SendCompletion(uri, 5, 4);
+                completions.Should().HaveLabels("clear", "copy", "items", "keys", "update", "values");
+            }
+        }
+
+        [TestMethod, Priority(0)]
+        public async Task GenericListBase() {
+            var code = @"
+from typing import List
+
+def func(a: List[str]):
+    a.
+    a[0].
+    pass
+";
+            using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) {
+                var uri = await server.OpenDefaultDocumentAndGetUriAsync(code);
+                await server.GetAnalysisAsync(uri);
+
+                var completions = await server.SendCompletion(uri, 4, 6);
+                completions.Should().HaveLabels("clear", "copy", "count", "index", "remove", "reverse");
+
+                completions = await server.SendCompletion(uri, 5, 9);
+                completions.Should().HaveLabels("capitalize");
+            }
+        }
+
+        [TestMethod, Priority(0)]
+        public async Task GenericDictBase() {
+            var code = @"
+from typing import Dict
+
+def func(a: Dict[int, str]):
+    a.
+    a[0].
+    pass
+";
+            using (var server = await CreateServerAsync(PythonVersions.LatestAvailable3X)) {
+                var uri = await server.OpenDefaultDocumentAndGetUriAsync(code);
+                await server.GetAnalysisAsync(uri);
+
+                var completions = await server.SendCompletion(uri, 4, 6);
+                completions.Should().HaveLabels("keys", "values");
+
+                completions = await server.SendCompletion(uri, 5, 9);
+                completions.Should().HaveLabels("capitalize");
+            }
+        }
+        private static async Task AssertCompletion(
+            Server s, 
+            Uri uri, 
+            IReadOnlyCollection<string> contains, 
+            IReadOnlyCollection<string> excludes, 
+            Position? position = null, 
+            CompletionContext? context = null, 
+            Func<CompletionItem, string> cmpKey = null, 
+            string expr = null, 
+            Range? applicableSpan = null,
+            InsertTextFormat? allFormat = InsertTextFormat.PlainText) {
             await s.WaitForCompleteAnalysisAsync(CancellationToken.None);
             var res = await s.Completion(new CompletionParams {
                 textDocument = new TextDocumentIdentifier { uri = uri },