From 261fe5c7c45752897c82b85d55b4b828df07ebf7 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 19 Oct 2026 18:43:55 +0330 Subject: [PATCH 01/40] feat(proto): add MTPROTO backend type and Masque and Mtproto proxy messages Add BackendType MTPROTO = 2, the Masque message (matching the released Python bridge 0.9.3) and the Mtproto user message, exposed on Proxy as masque = 7 and mtproto = 8. Regenerate the Go code. --- common/service.pb.go | 699 +++++++++++++++++++++++--------------- common/service.proto | 17 + common/service_grpc.pb.go | 4 +- 3 files changed, 449 insertions(+), 271 deletions(-) diff --git a/common/service.pb.go b/common/service.pb.go index d079f8ec..4d99b9b3 100644 --- a/common/service.pb.go +++ b/common/service.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.36.11 -// protoc v7.34.1 +// protoc v7.36.1 // source: common/service.proto package common @@ -26,6 +26,7 @@ type BackendType int32 const ( BackendType_XRAY BackendType = 0 BackendType_WIREGUARD BackendType = 1 + BackendType_MTPROTO BackendType = 2 ) // Enum value maps for BackendType. @@ -33,10 +34,12 @@ var ( BackendType_name = map[int32]string{ 0: "XRAY", 1: "WIREGUARD", + 2: "MTPROTO", } BackendType_value = map[string]int32{ "XRAY": 0, "WIREGUARD": 1, + "MTPROTO": 2, } ) @@ -1305,6 +1308,118 @@ func (x *Hysteria) GetAuth() string { return "" } +type Masque struct { + state protoimpl.MessageState `protogen:"open.v1"` + Pass string `protobuf:"bytes,1,opt,name=pass,proto3" json:"pass,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Masque) Reset() { + *x = Masque{} + mi := &file_common_service_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Masque) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Masque) ProtoMessage() {} + +func (x *Masque) ProtoReflect() protoreflect.Message { + mi := &file_common_service_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Masque.ProtoReflect.Descriptor instead. +func (*Masque) Descriptor() ([]byte, []int) { + return file_common_service_proto_rawDescGZIP(), []int{20} +} + +func (x *Masque) GetPass() string { + if x != nil { + return x.Pass + } + return "" +} + +type Mtproto struct { + state protoimpl.MessageState `protogen:"open.v1"` + Secret string `protobuf:"bytes,1,opt,name=secret,proto3" json:"secret,omitempty"` + UserAdTag string `protobuf:"bytes,2,opt,name=user_ad_tag,json=userAdTag,proto3" json:"user_ad_tag,omitempty"` + MaxTcpConns uint32 `protobuf:"varint,3,opt,name=max_tcp_conns,json=maxTcpConns,proto3" json:"max_tcp_conns,omitempty"` + MaxUniqueIps uint32 `protobuf:"varint,4,opt,name=max_unique_ips,json=maxUniqueIps,proto3" json:"max_unique_ips,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Mtproto) Reset() { + *x = Mtproto{} + mi := &file_common_service_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Mtproto) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Mtproto) ProtoMessage() {} + +func (x *Mtproto) ProtoReflect() protoreflect.Message { + mi := &file_common_service_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Mtproto.ProtoReflect.Descriptor instead. +func (*Mtproto) Descriptor() ([]byte, []int) { + return file_common_service_proto_rawDescGZIP(), []int{21} +} + +func (x *Mtproto) GetSecret() string { + if x != nil { + return x.Secret + } + return "" +} + +func (x *Mtproto) GetUserAdTag() string { + if x != nil { + return x.UserAdTag + } + return "" +} + +func (x *Mtproto) GetMaxTcpConns() uint32 { + if x != nil { + return x.MaxTcpConns + } + return 0 +} + +func (x *Mtproto) GetMaxUniqueIps() uint32 { + if x != nil { + return x.MaxUniqueIps + } + return 0 +} + type Proxy struct { state protoimpl.MessageState `protogen:"open.v1"` Vmess *Vmess `protobuf:"bytes,1,opt,name=vmess,proto3" json:"vmess,omitempty"` @@ -1313,13 +1428,15 @@ type Proxy struct { Shadowsocks *Shadowsocks `protobuf:"bytes,4,opt,name=shadowsocks,proto3" json:"shadowsocks,omitempty"` Wireguard *Wireguard `protobuf:"bytes,5,opt,name=wireguard,proto3" json:"wireguard,omitempty"` Hysteria *Hysteria `protobuf:"bytes,6,opt,name=hysteria,proto3" json:"hysteria,omitempty"` + Masque *Masque `protobuf:"bytes,7,opt,name=masque,proto3" json:"masque,omitempty"` + Mtproto *Mtproto `protobuf:"bytes,8,opt,name=mtproto,proto3" json:"mtproto,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *Proxy) Reset() { *x = Proxy{} - mi := &file_common_service_proto_msgTypes[20] + mi := &file_common_service_proto_msgTypes[22] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1331,7 +1448,7 @@ func (x *Proxy) String() string { func (*Proxy) ProtoMessage() {} func (x *Proxy) ProtoReflect() protoreflect.Message { - mi := &file_common_service_proto_msgTypes[20] + mi := &file_common_service_proto_msgTypes[22] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1344,7 +1461,7 @@ func (x *Proxy) ProtoReflect() protoreflect.Message { // Deprecated: Use Proxy.ProtoReflect.Descriptor instead. func (*Proxy) Descriptor() ([]byte, []int) { - return file_common_service_proto_rawDescGZIP(), []int{20} + return file_common_service_proto_rawDescGZIP(), []int{22} } func (x *Proxy) GetVmess() *Vmess { @@ -1389,6 +1506,20 @@ func (x *Proxy) GetHysteria() *Hysteria { return nil } +func (x *Proxy) GetMasque() *Masque { + if x != nil { + return x.Masque + } + return nil +} + +func (x *Proxy) GetMtproto() *Mtproto { + if x != nil { + return x.Mtproto + } + return nil +} + type User struct { state protoimpl.MessageState `protogen:"open.v1"` Email string `protobuf:"bytes,1,opt,name=email,proto3" json:"email,omitempty"` @@ -1400,7 +1531,7 @@ type User struct { func (x *User) Reset() { *x = User{} - mi := &file_common_service_proto_msgTypes[21] + mi := &file_common_service_proto_msgTypes[23] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1412,7 +1543,7 @@ func (x *User) String() string { func (*User) ProtoMessage() {} func (x *User) ProtoReflect() protoreflect.Message { - mi := &file_common_service_proto_msgTypes[21] + mi := &file_common_service_proto_msgTypes[23] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1425,7 +1556,7 @@ func (x *User) ProtoReflect() protoreflect.Message { // Deprecated: Use User.ProtoReflect.Descriptor instead. func (*User) Descriptor() ([]byte, []int) { - return file_common_service_proto_rawDescGZIP(), []int{21} + return file_common_service_proto_rawDescGZIP(), []int{23} } func (x *User) GetEmail() string { @@ -1458,7 +1589,7 @@ type Users struct { func (x *Users) Reset() { *x = Users{} - mi := &file_common_service_proto_msgTypes[22] + mi := &file_common_service_proto_msgTypes[24] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1470,7 +1601,7 @@ func (x *Users) String() string { func (*Users) ProtoMessage() {} func (x *Users) ProtoReflect() protoreflect.Message { - mi := &file_common_service_proto_msgTypes[22] + mi := &file_common_service_proto_msgTypes[24] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1483,7 +1614,7 @@ func (x *Users) ProtoReflect() protoreflect.Message { // Deprecated: Use Users.ProtoReflect.Descriptor instead. func (*Users) Descriptor() ([]byte, []int) { - return file_common_service_proto_rawDescGZIP(), []int{22} + return file_common_service_proto_rawDescGZIP(), []int{24} } func (x *Users) GetUsers() []*User { @@ -1504,7 +1635,7 @@ type UsersChunk struct { func (x *UsersChunk) Reset() { *x = UsersChunk{} - mi := &file_common_service_proto_msgTypes[23] + mi := &file_common_service_proto_msgTypes[25] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1516,7 +1647,7 @@ func (x *UsersChunk) String() string { func (*UsersChunk) ProtoMessage() {} func (x *UsersChunk) ProtoReflect() protoreflect.Message { - mi := &file_common_service_proto_msgTypes[23] + mi := &file_common_service_proto_msgTypes[25] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1529,7 +1660,7 @@ func (x *UsersChunk) ProtoReflect() protoreflect.Message { // Deprecated: Use UsersChunk.ProtoReflect.Descriptor instead. func (*UsersChunk) Descriptor() ([]byte, []int) { - return file_common_service_proto_rawDescGZIP(), []int{23} + return file_common_service_proto_rawDescGZIP(), []int{25} } func (x *UsersChunk) GetUsers() []*User { @@ -1564,7 +1695,7 @@ type RoutingRule struct { func (x *RoutingRule) Reset() { *x = RoutingRule{} - mi := &file_common_service_proto_msgTypes[24] + mi := &file_common_service_proto_msgTypes[26] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1576,7 +1707,7 @@ func (x *RoutingRule) String() string { func (*RoutingRule) ProtoMessage() {} func (x *RoutingRule) ProtoReflect() protoreflect.Message { - mi := &file_common_service_proto_msgTypes[24] + mi := &file_common_service_proto_msgTypes[26] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1589,7 +1720,7 @@ func (x *RoutingRule) ProtoReflect() protoreflect.Message { // Deprecated: Use RoutingRule.ProtoReflect.Descriptor instead. func (*RoutingRule) Descriptor() ([]byte, []int) { - return file_common_service_proto_rawDescGZIP(), []int{24} + return file_common_service_proto_rawDescGZIP(), []int{26} } func (x *RoutingRule) GetOutboundTag() string { @@ -1615,7 +1746,7 @@ type RoutingRulesResponse struct { func (x *RoutingRulesResponse) Reset() { *x = RoutingRulesResponse{} - mi := &file_common_service_proto_msgTypes[25] + mi := &file_common_service_proto_msgTypes[27] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1627,7 +1758,7 @@ func (x *RoutingRulesResponse) String() string { func (*RoutingRulesResponse) ProtoMessage() {} func (x *RoutingRulesResponse) ProtoReflect() protoreflect.Message { - mi := &file_common_service_proto_msgTypes[25] + mi := &file_common_service_proto_msgTypes[27] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1640,7 +1771,7 @@ func (x *RoutingRulesResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use RoutingRulesResponse.ProtoReflect.Descriptor instead. func (*RoutingRulesResponse) Descriptor() ([]byte, []int) { - return file_common_service_proto_rawDescGZIP(), []int{25} + return file_common_service_proto_rawDescGZIP(), []int{27} } func (x *RoutingRulesResponse) GetRules() []*RoutingRule { @@ -1659,7 +1790,7 @@ type BalancerInfoRequest struct { func (x *BalancerInfoRequest) Reset() { *x = BalancerInfoRequest{} - mi := &file_common_service_proto_msgTypes[26] + mi := &file_common_service_proto_msgTypes[28] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1671,7 +1802,7 @@ func (x *BalancerInfoRequest) String() string { func (*BalancerInfoRequest) ProtoMessage() {} func (x *BalancerInfoRequest) ProtoReflect() protoreflect.Message { - mi := &file_common_service_proto_msgTypes[26] + mi := &file_common_service_proto_msgTypes[28] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1684,7 +1815,7 @@ func (x *BalancerInfoRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use BalancerInfoRequest.ProtoReflect.Descriptor instead. func (*BalancerInfoRequest) Descriptor() ([]byte, []int) { - return file_common_service_proto_rawDescGZIP(), []int{26} + return file_common_service_proto_rawDescGZIP(), []int{28} } func (x *BalancerInfoRequest) GetTag() string { @@ -1704,7 +1835,7 @@ type BalancerInfoResponse struct { func (x *BalancerInfoResponse) Reset() { *x = BalancerInfoResponse{} - mi := &file_common_service_proto_msgTypes[27] + mi := &file_common_service_proto_msgTypes[29] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1716,7 +1847,7 @@ func (x *BalancerInfoResponse) String() string { func (*BalancerInfoResponse) ProtoMessage() {} func (x *BalancerInfoResponse) ProtoReflect() protoreflect.Message { - mi := &file_common_service_proto_msgTypes[27] + mi := &file_common_service_proto_msgTypes[29] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1729,7 +1860,7 @@ func (x *BalancerInfoResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use BalancerInfoResponse.ProtoReflect.Descriptor instead. func (*BalancerInfoResponse) Descriptor() ([]byte, []int) { - return file_common_service_proto_rawDescGZIP(), []int{27} + return file_common_service_proto_rawDescGZIP(), []int{29} } func (x *BalancerInfoResponse) GetOverrideTarget() string { @@ -1764,7 +1895,7 @@ type TestRouteRequest struct { func (x *TestRouteRequest) Reset() { *x = TestRouteRequest{} - mi := &file_common_service_proto_msgTypes[28] + mi := &file_common_service_proto_msgTypes[30] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1776,7 +1907,7 @@ func (x *TestRouteRequest) String() string { func (*TestRouteRequest) ProtoMessage() {} func (x *TestRouteRequest) ProtoReflect() protoreflect.Message { - mi := &file_common_service_proto_msgTypes[28] + mi := &file_common_service_proto_msgTypes[30] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1789,7 +1920,7 @@ func (x *TestRouteRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use TestRouteRequest.ProtoReflect.Descriptor instead. func (*TestRouteRequest) Descriptor() ([]byte, []int) { - return file_common_service_proto_rawDescGZIP(), []int{28} + return file_common_service_proto_rawDescGZIP(), []int{30} } func (x *TestRouteRequest) GetInboundTag() string { @@ -1875,7 +2006,7 @@ type RouteResult struct { func (x *RouteResult) Reset() { *x = RouteResult{} - mi := &file_common_service_proto_msgTypes[29] + mi := &file_common_service_proto_msgTypes[31] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1887,7 +2018,7 @@ func (x *RouteResult) String() string { func (*RouteResult) ProtoMessage() {} func (x *RouteResult) ProtoReflect() protoreflect.Message { - mi := &file_common_service_proto_msgTypes[29] + mi := &file_common_service_proto_msgTypes[31] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1900,7 +2031,7 @@ func (x *RouteResult) ProtoReflect() protoreflect.Message { // Deprecated: Use RouteResult.ProtoReflect.Descriptor instead. func (*RouteResult) Descriptor() ([]byte, []int) { - return file_common_service_proto_rawDescGZIP(), []int{29} + return file_common_service_proto_rawDescGZIP(), []int{31} } func (x *RouteResult) GetOutboundTag() string { @@ -1951,7 +2082,7 @@ type AddRoutingRuleRequest struct { func (x *AddRoutingRuleRequest) Reset() { *x = AddRoutingRuleRequest{} - mi := &file_common_service_proto_msgTypes[30] + mi := &file_common_service_proto_msgTypes[32] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1963,7 +2094,7 @@ func (x *AddRoutingRuleRequest) String() string { func (*AddRoutingRuleRequest) ProtoMessage() {} func (x *AddRoutingRuleRequest) ProtoReflect() protoreflect.Message { - mi := &file_common_service_proto_msgTypes[30] + mi := &file_common_service_proto_msgTypes[32] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1976,7 +2107,7 @@ func (x *AddRoutingRuleRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use AddRoutingRuleRequest.ProtoReflect.Descriptor instead. func (*AddRoutingRuleRequest) Descriptor() ([]byte, []int) { - return file_common_service_proto_rawDescGZIP(), []int{30} + return file_common_service_proto_rawDescGZIP(), []int{32} } func (x *AddRoutingRuleRequest) GetRule() string { @@ -2002,7 +2133,7 @@ type RemoveRoutingRuleRequest struct { func (x *RemoveRoutingRuleRequest) Reset() { *x = RemoveRoutingRuleRequest{} - mi := &file_common_service_proto_msgTypes[31] + mi := &file_common_service_proto_msgTypes[33] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2014,7 +2145,7 @@ func (x *RemoveRoutingRuleRequest) String() string { func (*RemoveRoutingRuleRequest) ProtoMessage() {} func (x *RemoveRoutingRuleRequest) ProtoReflect() protoreflect.Message { - mi := &file_common_service_proto_msgTypes[31] + mi := &file_common_service_proto_msgTypes[33] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2027,7 +2158,7 @@ func (x *RemoveRoutingRuleRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use RemoveRoutingRuleRequest.ProtoReflect.Descriptor instead. func (*RemoveRoutingRuleRequest) Descriptor() ([]byte, []int) { - return file_common_service_proto_rawDescGZIP(), []int{31} + return file_common_service_proto_rawDescGZIP(), []int{33} } func (x *RemoveRoutingRuleRequest) GetRuleTag() string { @@ -2047,7 +2178,7 @@ type OverrideBalancerTargetRequest struct { func (x *OverrideBalancerTargetRequest) Reset() { *x = OverrideBalancerTargetRequest{} - mi := &file_common_service_proto_msgTypes[32] + mi := &file_common_service_proto_msgTypes[34] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -2059,7 +2190,7 @@ func (x *OverrideBalancerTargetRequest) String() string { func (*OverrideBalancerTargetRequest) ProtoMessage() {} func (x *OverrideBalancerTargetRequest) ProtoReflect() protoreflect.Message { - mi := &file_common_service_proto_msgTypes[32] + mi := &file_common_service_proto_msgTypes[34] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -2072,7 +2203,7 @@ func (x *OverrideBalancerTargetRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use OverrideBalancerTargetRequest.ProtoReflect.Descriptor instead. func (*OverrideBalancerTargetRequest) Descriptor() ([]byte, []int) { - return file_common_service_proto_rawDescGZIP(), []int{32} + return file_common_service_proto_rawDescGZIP(), []int{34} } func (x *OverrideBalancerTargetRequest) GetBalancerTag() string { @@ -2092,168 +2223,194 @@ func (x *OverrideBalancerTargetRequest) GetTarget() string { var File_common_service_proto protoreflect.FileDescriptor const file_common_service_proto_rawDesc = "" + - "\x0a\x14common/service.proto\x12\x07service\"" + - "\x07\x0a\x05Empty\"r\x0a\x10BaseInfoResponse\x12\x18\x0a\x07" + - "started\x18\x01 \x01(\x08R\x07started\x12!\x0a\x0ccore_v" + - "ersion\x18\x02 \x01(\x09R\x0bcoreVersion\x12!\x0a\x0cnod" + - "e_version\x18\x03 \x01(\x09R\x0bnodeVersion\"\xba\x01\x0a" + - "\x07Backend\x12(\x0a\x04type\x18\x01 \x01(\x0e2\x14.service" + - ".BackendTypeR\x04type\x12\x16\x0a\x06config\x18\x02 \x01" + - "(\x09R\x06config\x12#\x0a\x05users\x18\x03 \x03(\x0b2\x0d.serv" + - "ice.UserR\x05users\x12\x1d\x0a\x0akeep_alive\x18\x04 " + - "\x01(\x04R\x09keepAlive\x12)\x0a\x10exclude_inboun" + - "ds\x18\x05 \x03(\x09R\x0fexcludeInbounds\"\x1d\x0a\x03Log" + - "\x12\x16\x0a\x06detail\x18\x01 \x01(\x09R\x06detail\"X\x0a\x04Stat" + - "\x12\x12\x0a\x04name\x18\x01 \x01(\x09R\x04name\x12\x12\x0a\x04type\x18\x02 \x01" + - "(\x09R\x04type\x12\x12\x0a\x04link\x18\x03 \x01(\x09R\x04link\x12\x14\x0a\x05" + - "value\x18\x04 \x01(\x03R\x05value\"3\x0a\x0cStatRespon" + - "se\x12#\x0a\x05stats\x18\x01 \x03(\x0b2\x0d.service.Stat" + - "R\x05stats\"^\x0a\x0bStatRequest\x12\x12\x0a\x04name\x18\x01" + - " \x01(\x09R\x04name\x12\x14\x0a\x05reset\x18\x02 \x01(\x08R\x05reset" + - "\x12%\x0a\x04type\x18\x03 \x01(\x0e2\x11.service.StatTyp" + - "eR\x04type\">\x0a\x12OnlineStatResponse\x12\x12\x0a" + - "\x04name\x18\x01 \x01(\x09R\x04name\x12\x14\x0a\x05value\x18\x02 \x01(\x03" + - "R\x05value\"\xa6\x01\x0a\x19StatsOnlineIpListRes" + - "ponse\x12\x12\x0a\x04name\x18\x01 \x01(\x09R\x04name\x12=\x0a\x03ips" + - "\x18\x02 \x03(\x0b2+.service.StatsOnlineIpLi" + - "stResponse.IpsEntryR\x03ips\x1a6\x0a\x08IpsE" + - "ntry\x12\x10\x0a\x03key\x18\x01 \x01(\x09R\x03key\x12\x14\x0a\x05value\x18" + - "\x02 \x01(\x03R\x05value:\x028\x01\"\xbf\x01\x0a\x07Latency\x12\x12\x0a\x04" + - "name\x18\x01 \x01(\x09R\x04name\x12\x14\x0a\x05alive\x18\x02 \x01(\x08R" + - "\x05alive\x12\x14\x0a\x05delay\x18\x03 \x01(\x03R\x05delay\x12\x12\x0a\x04" + - "link\x18\x04 \x01(\x09R\x04link\x12$\x0a\x0elast_seen_ti" + - "me\x18\x05 \x01(\x03R\x0clastSeenTime\x12\"\x0a\x0dlast_t" + - "ry_time\x18\x06 \x01(\x03R\x0blastTryTime\x12\x16\x0a\x06so" + - "urce\x18\x07 \x01(\x09R\x06source\"$\x0a\x0eLatencyReq" + - "uest\x12\x12\x0a\x04name\x18\x01 \x01(\x09R\x04name\"A\x0a\x0fLate" + - "ncyResponse\x12.\x0a\x09latencies\x18\x01 \x03(\x0b2\x10" + - ".service.LatencyR\x09latencies\"\xac\x02\x0a\x14" + - "BackendStatsResponse\x12#\x0a\x0dnum_goro" + - "utine\x18\x01 \x01(\x0dR\x0cnumGoroutine\x12\x15\x0a\x06num" + - "_gc\x18\x02 \x01(\x0dR\x05numGc\x12\x14\x0a\x05alloc\x18\x03 \x01(\x04R" + - "\x05alloc\x12\x1f\x0a\x0btotal_alloc\x18\x04 \x01(\x04R\x0atot" + - "alAlloc\x12\x10\x0a\x03sys\x18\x05 \x01(\x04R\x03sys\x12\x18\x0a\x07mal" + - "locs\x18\x06 \x01(\x04R\x07mallocs\x12\x14\x0a\x05frees\x18\x07 \x01" + - "(\x04R\x05frees\x12!\x0a\x0clive_objects\x18\x08 \x01(\x04R" + - "\x0bliveObjects\x12$\x0a\x0epause_total_ns\x18\x09" + - " \x01(\x04R\x0cpauseTotalNs\x12\x16\x0a\x06uptime\x18\x0a \x01" + - "(\x0dR\x06uptime\"\x93\x02\x0a\x13SystemStatsRespon" + - "se\x12\x1b\x0a\x09mem_total\x18\x01 \x01(\x04R\x08memTotal\x12" + - "\x19\x0a\x08mem_used\x18\x02 \x01(\x04R\x07memUsed\x12\x1b\x0a\x09cp" + - "u_cores\x18\x03 \x01(\x04R\x08cpuCores\x12\x1b\x0a\x09cpu_u" + - "sage\x18\x04 \x01(\x01R\x08cpuUsage\x128\x0a\x18incoming" + - "_bandwidth_speed\x18\x05 \x01(\x04R\x16incoming" + - "BandwidthSpeed\x128\x0a\x18outgoing_bandw" + - "idth_speed\x18\x06 \x01(\x04R\x16outgoingBandwi" + - "dthSpeed\x12\x16\x0a\x06uptime\x18\x07 \x01(\x04R\x06uptime" + - "\"\x17\x0a\x05Vmess\x12\x0e\x0a\x02id\x18\x01 \x01(\x09R\x02id\"+\x0a\x05Vle" + - "ss\x12\x0e\x0a\x02id\x18\x01 \x01(\x09R\x02id\x12\x12\x0a\x04flow\x18\x02 \x01(\x09" + - "R\x04flow\"$\x0a\x06Trojan\x12\x1a\x0a\x08password\x18\x01 \x01" + - "(\x09R\x08password\"A\x0a\x0bShadowsocks\x12\x1a\x0a\x08p" + - "assword\x18\x01 \x01(\x09R\x08password\x12\x16\x0a\x06metho" + - "d\x18\x02 \x01(\x09R\x06method\"k\x0a\x09Wireguard\x12\x1d\x0a\x0a" + - "public_key\x18\x01 \x01(\x09R\x09publicKey\x12\x19\x0a\x08p" + - "eer_ips\x18\x02 \x03(\x09R\x07peerIps\x12$\x0a\x0epre_sh" + - "ared_key\x18\x03 \x01(\x09R\x0cpreSharedKey\"\x1e\x0a\x08" + - "Hysteria\x12\x12\x0a\x04auth\x18\x01 \x01(\x09R\x04auth\"\x95\x02\x0a" + - "\x05Proxy\x12$\x0a\x05vmess\x18\x01 \x01(\x0b2\x0e.service." + - "VmessR\x05vmess\x12$\x0a\x05vless\x18\x02 \x01(\x0b2\x0e.se" + - "rvice.VlessR\x05vless\x12'\x0a\x06trojan\x18\x03 \x01" + - "(\x0b2\x0f.service.TrojanR\x06trojan\x126\x0a\x0bs" + - "hadowsocks\x18\x04 \x01(\x0b2\x14.service.Shado" + - "wsocksR\x0bshadowsocks\x120\x0a\x09wireguard" + - "\x18\x05 \x01(\x0b2\x12.service.WireguardR\x09wire" + - "guard\x12-\x0a\x08hysteria\x18\x06 \x01(\x0b2\x11.servic" + - "e.HysteriaR\x08hysteria\"b\x0a\x04User\x12\x14\x0a\x05" + - "email\x18\x01 \x01(\x09R\x05email\x12(\x0a\x07proxies\x18\x02 " + - "\x01(\x0b2\x0e.service.ProxyR\x07proxies\x12\x1a\x0a\x08" + - "inbounds\x18\x03 \x03(\x09R\x08inbounds\",\x0a\x05User" + - "s\x12#\x0a\x05users\x18\x01 \x03(\x0b2\x0d.service.UserR" + - "\x05users\"[\x0a\x0aUsersChunk\x12#\x0a\x05users\x18\x01 " + - "\x03(\x0b2\x0d.service.UserR\x05users\x12\x14\x0a\x05ind" + - "ex\x18\x02 \x01(\x04R\x05index\x12\x12\x0a\x04last\x18\x03 \x01(\x08R\x04l" + - "ast\"K\x0a\x0bRoutingRule\x12!\x0a\x0coutbound_t" + - "ag\x18\x01 \x01(\x09R\x0boutboundTag\x12\x19\x0a\x08rule_ta" + - "g\x18\x02 \x01(\x09R\x07ruleTag\"B\x0a\x14RoutingRules" + - "Response\x12*\x0a\x05rules\x18\x01 \x03(\x0b2\x14.servic" + - "e.RoutingRuleR\x05rules\"'\x0a\x13Balancer" + - "InfoRequest\x12\x10\x0a\x03tag\x18\x01 \x01(\x09R\x03tag\"j\x0a" + - "\x14BalancerInfoResponse\x12'\x0a\x0foverrid" + - "e_target\x18\x01 \x01(\x09R\x0eoverrideTarget\x12)" + - "\x0a\x10principle_target\x18\x02 \x03(\x09R\x0fprinci" + - "pleTarget\"\xba\x03\x0a\x10TestRouteRequest\x12\x1f" + - "\x0a\x0binbound_tag\x18\x01 \x01(\x09R\x0ainboundTag\x12" + - "\x18\x0a\x07network\x18\x02 \x01(\x09R\x07network\x12\x1b\x0a\x09tar" + - "get_ip\x18\x03 \x01(\x09R\x08targetIp\x12#\x0a\x0dtarget" + - "_domain\x18\x04 \x01(\x09R\x0ctargetDomain\x12\x1f\x0a\x0bt" + - "arget_port\x18\x05 \x01(\x0dR\x0atargetPort\x12\x1a\x0a\x08" + - "protocol\x18\x06 \x01(\x09R\x08protocol\x12\x12\x0a\x04user" + - "\x18\x07 \x01(\x09R\x04user\x12I\x0a\x0aattributes\x18\x08 \x03(\x0b" + - "2).service.TestRouteRequest.Attr" + - "ibutesEntryR\x0aattributes\x12'\x0a\x0ffield" + - "_selectors\x18\x09 \x03(\x09R\x0efieldSelectors" + - "\x12%\x0a\x0epublish_result\x18\x0a \x01(\x08R\x0dpublis" + - "hResult\x1a=\x0a\x0fAttributesEntry\x12\x10\x0a\x03ke" + - "y\x18\x01 \x01(\x09R\x03key\x12\x14\x0a\x05value\x18\x02 \x01(\x09R\x05val" + - "ue:\x028\x01\"\xc0\x01\x0a\x0bRouteResult\x12!\x0a\x0coutbou" + - "nd_tag\x18\x01 \x01(\x09R\x0boutboundTag\x12.\x0a\x13out" + - "bound_group_tags\x18\x02 \x03(\x09R\x11outbound" + - "GroupTags\x12\x1f\x0a\x0binbound_tag\x18\x03 \x01(\x09R\x0a" + - "inboundTag\x12\x18\x0a\x07network\x18\x04 \x01(\x09R\x07net" + - "work\x12#\x0a\x0dtarget_domain\x18\x05 \x01(\x09R\x0ctar" + - "getDomain\"N\x0a\x15AddRoutingRuleReque" + - "st\x12\x12\x0a\x04rule\x18\x01 \x01(\x09R\x04rule\x12!\x0a\x0cshould" + - "_reset\x18\x02 \x01(\x08R\x0bshouldReset\"5\x0a\x18Rem" + - "oveRoutingRuleRequest\x12\x19\x0a\x08rule_ta" + - "g\x18\x01 \x01(\x09R\x07ruleTag\"Z\x0a\x1dOverrideBala" + - "ncerTargetRequest\x12!\x0a\x0cbalancer_ta" + - "g\x18\x01 \x01(\x09R\x0bbalancerTag\x12\x16\x0a\x06target\x18\x02" + - " \x01(\x09R\x06target*&\x0a\x0bBackendType\x12\x08\x0a\x04X" + - "RAY\x10\x00\x12\x0d\x0a\x09WIREGUARD\x10\x01*_\x0a\x08StatType" + - "\x12\x0d\x0a\x09Outbounds\x10\x00\x12\x0c\x0a\x08Outbound\x10\x01\x12\x0c\x0a" + - "\x08Inbounds\x10\x02\x12\x0b\x0a\x07Inbound\x10\x03\x12\x0d\x0a\x09User" + - "sStat\x10\x04\x12\x0c\x0a\x08UserStat\x10\x052\xdc\x09\x0a\x0bNodeSe" + - "rvice\x126\x0a\x05Start\x12\x10.service.Backend" + - "\x1a\x19.service.BaseInfoResponse\"\x00\x12(\x0a" + - "\x04Stop\x12\x0e.service.Empty\x1a\x0e.service." + - "Empty\"\x00\x12:\x0a\x0bGetBaseInfo\x12\x0e.service" + - ".Empty\x1a\x19.service.BaseInfoRespons" + - "e\"\x00\x12+\x0a\x07GetLogs\x12\x0e.service.Empty\x1a\x0c" + - ".service.Log\"\x000\x01\x12@\x0a\x0eGetSystemSta" + - "ts\x12\x0e.service.Empty\x1a\x1c.service.Sys" + - "temStatsResponse\"\x00\x12B\x0a\x0fGetBackend" + - "Stats\x12\x0e.service.Empty\x1a\x1d.service." + - "BackendStatsResponse\"\x00\x129\x0a\x08GetSta" + - "ts\x12\x14.service.StatRequest\x1a\x15.servi" + - "ce.StatResponse\"\x00\x12J\x0a\x13GetOutbound" + - "sLatency\x12\x17.service.LatencyReques" + - "t\x1a\x18.service.LatencyResponse\"\x00\x12I\x0a" + - "\x12GetUserOnlineStats\x12\x14.service.St" + - "atRequest\x1a\x1b.service.OnlineStatRe" + - "sponse\"\x00\x12V\x0a\x18GetUserOnlineIpListS" + - "tats\x12\x14.service.StatRequest\x1a\".ser" + - "vice.StatsOnlineIpListResponse\"\x00" + - "\x12-\x0a\x08SyncUser\x12\x0d.service.User\x1a\x0e.se" + - "rvice.Empty\"\x00(\x01\x12-\x0a\x09SyncUsers\x12\x0e.s" + - "ervice.Users\x1a\x0e.service.Empty\"\x00\x12;" + - "\x0a\x10SyncUsersChunked\x12\x13.service.Use" + - "rsChunk\x1a\x0e.service.Empty\"\x00(\x01\x12C\x0a\x10L" + - "istRoutingRules\x12\x0e.service.Empty\x1a" + - "\x1d.service.RoutingRulesResponse\"\x00" + - "\x12P\x0a\x0fGetBalancerInfo\x12\x1c.service.Ba" + - "lancerInfoRequest\x1a\x1d.service.Bala" + - "ncerInfoResponse\"\x00\x12>\x0a\x09TestRoute\x12" + - "\x19.service.TestRouteRequest\x1a\x14.ser" + - "vice.RouteResult\"\x00\x12B\x0a\x0eAddRouting" + - "Rule\x12\x1e.service.AddRoutingRuleReq" + - "uest\x1a\x0e.service.Empty\"\x00\x12H\x0a\x11Remove" + - "RoutingRule\x12!.service.RemoveRout" + - "ingRuleRequest\x1a\x0e.service.Empty\"\x00" + - "\x12R\x0a\x16OverrideBalancerTarget\x12&.ser" + - "vice.OverrideBalancerTargetReque" + - "st\x1a\x0e.service.Empty\"\x00B#Z!github.c" + - "om/pasarguard/node/commonb\x06proto" + - "3" + "\n" + + "\x14common/service.proto\x12\aservice\"\a\n" + + "\x05Empty\"r\n" + + "\x10BaseInfoResponse\x12\x18\n" + + "\astarted\x18\x01 \x01(\bR\astarted\x12!\n" + + "\fcore_version\x18\x02 \x01(\tR\vcoreVersion\x12!\n" + + "\fnode_version\x18\x03 \x01(\tR\vnodeVersion\"\xba\x01\n" + + "\aBackend\x12(\n" + + "\x04type\x18\x01 \x01(\x0e2\x14.service.BackendTypeR\x04type\x12\x16\n" + + "\x06config\x18\x02 \x01(\tR\x06config\x12#\n" + + "\x05users\x18\x03 \x03(\v2\r.service.UserR\x05users\x12\x1d\n" + + "\n" + + "keep_alive\x18\x04 \x01(\x04R\tkeepAlive\x12)\n" + + "\x10exclude_inbounds\x18\x05 \x03(\tR\x0fexcludeInbounds\"\x1d\n" + + "\x03Log\x12\x16\n" + + "\x06detail\x18\x01 \x01(\tR\x06detail\"X\n" + + "\x04Stat\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x12\n" + + "\x04type\x18\x02 \x01(\tR\x04type\x12\x12\n" + + "\x04link\x18\x03 \x01(\tR\x04link\x12\x14\n" + + "\x05value\x18\x04 \x01(\x03R\x05value\"3\n" + + "\fStatResponse\x12#\n" + + "\x05stats\x18\x01 \x03(\v2\r.service.StatR\x05stats\"^\n" + + "\vStatRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n" + + "\x05reset\x18\x02 \x01(\bR\x05reset\x12%\n" + + "\x04type\x18\x03 \x01(\x0e2\x11.service.StatTypeR\x04type\">\n" + + "\x12OnlineStatResponse\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n" + + "\x05value\x18\x02 \x01(\x03R\x05value\"\xa6\x01\n" + + "\x19StatsOnlineIpListResponse\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12=\n" + + "\x03ips\x18\x02 \x03(\v2+.service.StatsOnlineIpListResponse.IpsEntryR\x03ips\x1a6\n" + + "\bIpsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\x03R\x05value:\x028\x01\"\xbf\x01\n" + + "\aLatency\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n" + + "\x05alive\x18\x02 \x01(\bR\x05alive\x12\x14\n" + + "\x05delay\x18\x03 \x01(\x03R\x05delay\x12\x12\n" + + "\x04link\x18\x04 \x01(\tR\x04link\x12$\n" + + "\x0elast_seen_time\x18\x05 \x01(\x03R\flastSeenTime\x12\"\n" + + "\rlast_try_time\x18\x06 \x01(\x03R\vlastTryTime\x12\x16\n" + + "\x06source\x18\a \x01(\tR\x06source\"$\n" + + "\x0eLatencyRequest\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\"A\n" + + "\x0fLatencyResponse\x12.\n" + + "\tlatencies\x18\x01 \x03(\v2\x10.service.LatencyR\tlatencies\"\xac\x02\n" + + "\x14BackendStatsResponse\x12#\n" + + "\rnum_goroutine\x18\x01 \x01(\rR\fnumGoroutine\x12\x15\n" + + "\x06num_gc\x18\x02 \x01(\rR\x05numGc\x12\x14\n" + + "\x05alloc\x18\x03 \x01(\x04R\x05alloc\x12\x1f\n" + + "\vtotal_alloc\x18\x04 \x01(\x04R\n" + + "totalAlloc\x12\x10\n" + + "\x03sys\x18\x05 \x01(\x04R\x03sys\x12\x18\n" + + "\amallocs\x18\x06 \x01(\x04R\amallocs\x12\x14\n" + + "\x05frees\x18\a \x01(\x04R\x05frees\x12!\n" + + "\flive_objects\x18\b \x01(\x04R\vliveObjects\x12$\n" + + "\x0epause_total_ns\x18\t \x01(\x04R\fpauseTotalNs\x12\x16\n" + + "\x06uptime\x18\n" + + " \x01(\rR\x06uptime\"\x93\x02\n" + + "\x13SystemStatsResponse\x12\x1b\n" + + "\tmem_total\x18\x01 \x01(\x04R\bmemTotal\x12\x19\n" + + "\bmem_used\x18\x02 \x01(\x04R\amemUsed\x12\x1b\n" + + "\tcpu_cores\x18\x03 \x01(\x04R\bcpuCores\x12\x1b\n" + + "\tcpu_usage\x18\x04 \x01(\x01R\bcpuUsage\x128\n" + + "\x18incoming_bandwidth_speed\x18\x05 \x01(\x04R\x16incomingBandwidthSpeed\x128\n" + + "\x18outgoing_bandwidth_speed\x18\x06 \x01(\x04R\x16outgoingBandwidthSpeed\x12\x16\n" + + "\x06uptime\x18\a \x01(\x04R\x06uptime\"\x17\n" + + "\x05Vmess\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\"+\n" + + "\x05Vless\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12\x12\n" + + "\x04flow\x18\x02 \x01(\tR\x04flow\"$\n" + + "\x06Trojan\x12\x1a\n" + + "\bpassword\x18\x01 \x01(\tR\bpassword\"A\n" + + "\vShadowsocks\x12\x1a\n" + + "\bpassword\x18\x01 \x01(\tR\bpassword\x12\x16\n" + + "\x06method\x18\x02 \x01(\tR\x06method\"k\n" + + "\tWireguard\x12\x1d\n" + + "\n" + + "public_key\x18\x01 \x01(\tR\tpublicKey\x12\x19\n" + + "\bpeer_ips\x18\x02 \x03(\tR\apeerIps\x12$\n" + + "\x0epre_shared_key\x18\x03 \x01(\tR\fpreSharedKey\"\x1e\n" + + "\bHysteria\x12\x12\n" + + "\x04auth\x18\x01 \x01(\tR\x04auth\"\x1c\n" + + "\x06Masque\x12\x12\n" + + "\x04pass\x18\x01 \x01(\tR\x04pass\"\x8b\x01\n" + + "\aMtproto\x12\x16\n" + + "\x06secret\x18\x01 \x01(\tR\x06secret\x12\x1e\n" + + "\vuser_ad_tag\x18\x02 \x01(\tR\tuserAdTag\x12\"\n" + + "\rmax_tcp_conns\x18\x03 \x01(\rR\vmaxTcpConns\x12$\n" + + "\x0emax_unique_ips\x18\x04 \x01(\rR\fmaxUniqueIps\"\xea\x02\n" + + "\x05Proxy\x12$\n" + + "\x05vmess\x18\x01 \x01(\v2\x0e.service.VmessR\x05vmess\x12$\n" + + "\x05vless\x18\x02 \x01(\v2\x0e.service.VlessR\x05vless\x12'\n" + + "\x06trojan\x18\x03 \x01(\v2\x0f.service.TrojanR\x06trojan\x126\n" + + "\vshadowsocks\x18\x04 \x01(\v2\x14.service.ShadowsocksR\vshadowsocks\x120\n" + + "\twireguard\x18\x05 \x01(\v2\x12.service.WireguardR\twireguard\x12-\n" + + "\bhysteria\x18\x06 \x01(\v2\x11.service.HysteriaR\bhysteria\x12'\n" + + "\x06masque\x18\a \x01(\v2\x0f.service.MasqueR\x06masque\x12*\n" + + "\amtproto\x18\b \x01(\v2\x10.service.MtprotoR\amtproto\"b\n" + + "\x04User\x12\x14\n" + + "\x05email\x18\x01 \x01(\tR\x05email\x12(\n" + + "\aproxies\x18\x02 \x01(\v2\x0e.service.ProxyR\aproxies\x12\x1a\n" + + "\binbounds\x18\x03 \x03(\tR\binbounds\",\n" + + "\x05Users\x12#\n" + + "\x05users\x18\x01 \x03(\v2\r.service.UserR\x05users\"[\n" + + "\n" + + "UsersChunk\x12#\n" + + "\x05users\x18\x01 \x03(\v2\r.service.UserR\x05users\x12\x14\n" + + "\x05index\x18\x02 \x01(\x04R\x05index\x12\x12\n" + + "\x04last\x18\x03 \x01(\bR\x04last\"K\n" + + "\vRoutingRule\x12!\n" + + "\foutbound_tag\x18\x01 \x01(\tR\voutboundTag\x12\x19\n" + + "\brule_tag\x18\x02 \x01(\tR\aruleTag\"B\n" + + "\x14RoutingRulesResponse\x12*\n" + + "\x05rules\x18\x01 \x03(\v2\x14.service.RoutingRuleR\x05rules\"'\n" + + "\x13BalancerInfoRequest\x12\x10\n" + + "\x03tag\x18\x01 \x01(\tR\x03tag\"j\n" + + "\x14BalancerInfoResponse\x12'\n" + + "\x0foverride_target\x18\x01 \x01(\tR\x0eoverrideTarget\x12)\n" + + "\x10principle_target\x18\x02 \x03(\tR\x0fprincipleTarget\"\xba\x03\n" + + "\x10TestRouteRequest\x12\x1f\n" + + "\vinbound_tag\x18\x01 \x01(\tR\n" + + "inboundTag\x12\x18\n" + + "\anetwork\x18\x02 \x01(\tR\anetwork\x12\x1b\n" + + "\ttarget_ip\x18\x03 \x01(\tR\btargetIp\x12#\n" + + "\rtarget_domain\x18\x04 \x01(\tR\ftargetDomain\x12\x1f\n" + + "\vtarget_port\x18\x05 \x01(\rR\n" + + "targetPort\x12\x1a\n" + + "\bprotocol\x18\x06 \x01(\tR\bprotocol\x12\x12\n" + + "\x04user\x18\a \x01(\tR\x04user\x12I\n" + + "\n" + + "attributes\x18\b \x03(\v2).service.TestRouteRequest.AttributesEntryR\n" + + "attributes\x12'\n" + + "\x0ffield_selectors\x18\t \x03(\tR\x0efieldSelectors\x12%\n" + + "\x0epublish_result\x18\n" + + " \x01(\bR\rpublishResult\x1a=\n" + + "\x0fAttributesEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xc0\x01\n" + + "\vRouteResult\x12!\n" + + "\foutbound_tag\x18\x01 \x01(\tR\voutboundTag\x12.\n" + + "\x13outbound_group_tags\x18\x02 \x03(\tR\x11outboundGroupTags\x12\x1f\n" + + "\vinbound_tag\x18\x03 \x01(\tR\n" + + "inboundTag\x12\x18\n" + + "\anetwork\x18\x04 \x01(\tR\anetwork\x12#\n" + + "\rtarget_domain\x18\x05 \x01(\tR\ftargetDomain\"N\n" + + "\x15AddRoutingRuleRequest\x12\x12\n" + + "\x04rule\x18\x01 \x01(\tR\x04rule\x12!\n" + + "\fshould_reset\x18\x02 \x01(\bR\vshouldReset\"5\n" + + "\x18RemoveRoutingRuleRequest\x12\x19\n" + + "\brule_tag\x18\x01 \x01(\tR\aruleTag\"Z\n" + + "\x1dOverrideBalancerTargetRequest\x12!\n" + + "\fbalancer_tag\x18\x01 \x01(\tR\vbalancerTag\x12\x16\n" + + "\x06target\x18\x02 \x01(\tR\x06target*3\n" + + "\vBackendType\x12\b\n" + + "\x04XRAY\x10\x00\x12\r\n" + + "\tWIREGUARD\x10\x01\x12\v\n" + + "\aMTPROTO\x10\x02*_\n" + + "\bStatType\x12\r\n" + + "\tOutbounds\x10\x00\x12\f\n" + + "\bOutbound\x10\x01\x12\f\n" + + "\bInbounds\x10\x02\x12\v\n" + + "\aInbound\x10\x03\x12\r\n" + + "\tUsersStat\x10\x04\x12\f\n" + + "\bUserStat\x10\x052\xdc\t\n" + + "\vNodeService\x126\n" + + "\x05Start\x12\x10.service.Backend\x1a\x19.service.BaseInfoResponse\"\x00\x12(\n" + + "\x04Stop\x12\x0e.service.Empty\x1a\x0e.service.Empty\"\x00\x12:\n" + + "\vGetBaseInfo\x12\x0e.service.Empty\x1a\x19.service.BaseInfoResponse\"\x00\x12+\n" + + "\aGetLogs\x12\x0e.service.Empty\x1a\f.service.Log\"\x000\x01\x12@\n" + + "\x0eGetSystemStats\x12\x0e.service.Empty\x1a\x1c.service.SystemStatsResponse\"\x00\x12B\n" + + "\x0fGetBackendStats\x12\x0e.service.Empty\x1a\x1d.service.BackendStatsResponse\"\x00\x129\n" + + "\bGetStats\x12\x14.service.StatRequest\x1a\x15.service.StatResponse\"\x00\x12J\n" + + "\x13GetOutboundsLatency\x12\x17.service.LatencyRequest\x1a\x18.service.LatencyResponse\"\x00\x12I\n" + + "\x12GetUserOnlineStats\x12\x14.service.StatRequest\x1a\x1b.service.OnlineStatResponse\"\x00\x12V\n" + + "\x18GetUserOnlineIpListStats\x12\x14.service.StatRequest\x1a\".service.StatsOnlineIpListResponse\"\x00\x12-\n" + + "\bSyncUser\x12\r.service.User\x1a\x0e.service.Empty\"\x00(\x01\x12-\n" + + "\tSyncUsers\x12\x0e.service.Users\x1a\x0e.service.Empty\"\x00\x12;\n" + + "\x10SyncUsersChunked\x12\x13.service.UsersChunk\x1a\x0e.service.Empty\"\x00(\x01\x12C\n" + + "\x10ListRoutingRules\x12\x0e.service.Empty\x1a\x1d.service.RoutingRulesResponse\"\x00\x12P\n" + + "\x0fGetBalancerInfo\x12\x1c.service.BalancerInfoRequest\x1a\x1d.service.BalancerInfoResponse\"\x00\x12>\n" + + "\tTestRoute\x12\x19.service.TestRouteRequest\x1a\x14.service.RouteResult\"\x00\x12B\n" + + "\x0eAddRoutingRule\x12\x1e.service.AddRoutingRuleRequest\x1a\x0e.service.Empty\"\x00\x12H\n" + + "\x11RemoveRoutingRule\x12!.service.RemoveRoutingRuleRequest\x1a\x0e.service.Empty\"\x00\x12R\n" + + "\x16OverrideBalancerTarget\x12&.service.OverrideBalancerTargetRequest\x1a\x0e.service.Empty\"\x00B#Z!github.com/pasarguard/node/commonb\x06proto3" var ( file_common_service_proto_rawDescOnce sync.Once @@ -2268,7 +2425,7 @@ func file_common_service_proto_rawDescGZIP() []byte { } var file_common_service_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_common_service_proto_msgTypes = make([]protoimpl.MessageInfo, 35) +var file_common_service_proto_msgTypes = make([]protoimpl.MessageInfo, 37) var file_common_service_proto_goTypes = []any{ (BackendType)(0), // 0: service.BackendType (StatType)(0), // 1: service.StatType @@ -2292,28 +2449,30 @@ var file_common_service_proto_goTypes = []any{ (*Shadowsocks)(nil), // 19: service.Shadowsocks (*Wireguard)(nil), // 20: service.Wireguard (*Hysteria)(nil), // 21: service.Hysteria - (*Proxy)(nil), // 22: service.Proxy - (*User)(nil), // 23: service.User - (*Users)(nil), // 24: service.Users - (*UsersChunk)(nil), // 25: service.UsersChunk - (*RoutingRule)(nil), // 26: service.RoutingRule - (*RoutingRulesResponse)(nil), // 27: service.RoutingRulesResponse - (*BalancerInfoRequest)(nil), // 28: service.BalancerInfoRequest - (*BalancerInfoResponse)(nil), // 29: service.BalancerInfoResponse - (*TestRouteRequest)(nil), // 30: service.TestRouteRequest - (*RouteResult)(nil), // 31: service.RouteResult - (*AddRoutingRuleRequest)(nil), // 32: service.AddRoutingRuleRequest - (*RemoveRoutingRuleRequest)(nil), // 33: service.RemoveRoutingRuleRequest - (*OverrideBalancerTargetRequest)(nil), // 34: service.OverrideBalancerTargetRequest - nil, // 35: service.StatsOnlineIpListResponse.IpsEntry - nil, // 36: service.TestRouteRequest.AttributesEntry + (*Masque)(nil), // 22: service.Masque + (*Mtproto)(nil), // 23: service.Mtproto + (*Proxy)(nil), // 24: service.Proxy + (*User)(nil), // 25: service.User + (*Users)(nil), // 26: service.Users + (*UsersChunk)(nil), // 27: service.UsersChunk + (*RoutingRule)(nil), // 28: service.RoutingRule + (*RoutingRulesResponse)(nil), // 29: service.RoutingRulesResponse + (*BalancerInfoRequest)(nil), // 30: service.BalancerInfoRequest + (*BalancerInfoResponse)(nil), // 31: service.BalancerInfoResponse + (*TestRouteRequest)(nil), // 32: service.TestRouteRequest + (*RouteResult)(nil), // 33: service.RouteResult + (*AddRoutingRuleRequest)(nil), // 34: service.AddRoutingRuleRequest + (*RemoveRoutingRuleRequest)(nil), // 35: service.RemoveRoutingRuleRequest + (*OverrideBalancerTargetRequest)(nil), // 36: service.OverrideBalancerTargetRequest + nil, // 37: service.StatsOnlineIpListResponse.IpsEntry + nil, // 38: service.TestRouteRequest.AttributesEntry } var file_common_service_proto_depIdxs = []int32{ 0, // 0: service.Backend.type:type_name -> service.BackendType - 23, // 1: service.Backend.users:type_name -> service.User + 25, // 1: service.Backend.users:type_name -> service.User 6, // 2: service.StatResponse.stats:type_name -> service.Stat 1, // 3: service.StatRequest.type:type_name -> service.StatType - 35, // 4: service.StatsOnlineIpListResponse.ips:type_name -> service.StatsOnlineIpListResponse.IpsEntry + 37, // 4: service.StatsOnlineIpListResponse.ips:type_name -> service.StatsOnlineIpListResponse.IpsEntry 11, // 5: service.LatencyResponse.latencies:type_name -> service.Latency 16, // 6: service.Proxy.vmess:type_name -> service.Vmess 17, // 7: service.Proxy.vless:type_name -> service.Vless @@ -2321,54 +2480,56 @@ var file_common_service_proto_depIdxs = []int32{ 19, // 9: service.Proxy.shadowsocks:type_name -> service.Shadowsocks 20, // 10: service.Proxy.wireguard:type_name -> service.Wireguard 21, // 11: service.Proxy.hysteria:type_name -> service.Hysteria - 22, // 12: service.User.proxies:type_name -> service.Proxy - 23, // 13: service.Users.users:type_name -> service.User - 23, // 14: service.UsersChunk.users:type_name -> service.User - 26, // 15: service.RoutingRulesResponse.rules:type_name -> service.RoutingRule - 36, // 16: service.TestRouteRequest.attributes:type_name -> service.TestRouteRequest.AttributesEntry - 4, // 17: service.NodeService.Start:input_type -> service.Backend - 2, // 18: service.NodeService.Stop:input_type -> service.Empty - 2, // 19: service.NodeService.GetBaseInfo:input_type -> service.Empty - 2, // 20: service.NodeService.GetLogs:input_type -> service.Empty - 2, // 21: service.NodeService.GetSystemStats:input_type -> service.Empty - 2, // 22: service.NodeService.GetBackendStats:input_type -> service.Empty - 8, // 23: service.NodeService.GetStats:input_type -> service.StatRequest - 12, // 24: service.NodeService.GetOutboundsLatency:input_type -> service.LatencyRequest - 8, // 25: service.NodeService.GetUserOnlineStats:input_type -> service.StatRequest - 8, // 26: service.NodeService.GetUserOnlineIpListStats:input_type -> service.StatRequest - 23, // 27: service.NodeService.SyncUser:input_type -> service.User - 24, // 28: service.NodeService.SyncUsers:input_type -> service.Users - 25, // 29: service.NodeService.SyncUsersChunked:input_type -> service.UsersChunk - 2, // 30: service.NodeService.ListRoutingRules:input_type -> service.Empty - 28, // 31: service.NodeService.GetBalancerInfo:input_type -> service.BalancerInfoRequest - 30, // 32: service.NodeService.TestRoute:input_type -> service.TestRouteRequest - 32, // 33: service.NodeService.AddRoutingRule:input_type -> service.AddRoutingRuleRequest - 33, // 34: service.NodeService.RemoveRoutingRule:input_type -> service.RemoveRoutingRuleRequest - 34, // 35: service.NodeService.OverrideBalancerTarget:input_type -> service.OverrideBalancerTargetRequest - 3, // 36: service.NodeService.Start:output_type -> service.BaseInfoResponse - 2, // 37: service.NodeService.Stop:output_type -> service.Empty - 3, // 38: service.NodeService.GetBaseInfo:output_type -> service.BaseInfoResponse - 5, // 39: service.NodeService.GetLogs:output_type -> service.Log - 15, // 40: service.NodeService.GetSystemStats:output_type -> service.SystemStatsResponse - 14, // 41: service.NodeService.GetBackendStats:output_type -> service.BackendStatsResponse - 7, // 42: service.NodeService.GetStats:output_type -> service.StatResponse - 13, // 43: service.NodeService.GetOutboundsLatency:output_type -> service.LatencyResponse - 9, // 44: service.NodeService.GetUserOnlineStats:output_type -> service.OnlineStatResponse - 10, // 45: service.NodeService.GetUserOnlineIpListStats:output_type -> service.StatsOnlineIpListResponse - 2, // 46: service.NodeService.SyncUser:output_type -> service.Empty - 2, // 47: service.NodeService.SyncUsers:output_type -> service.Empty - 2, // 48: service.NodeService.SyncUsersChunked:output_type -> service.Empty - 27, // 49: service.NodeService.ListRoutingRules:output_type -> service.RoutingRulesResponse - 29, // 50: service.NodeService.GetBalancerInfo:output_type -> service.BalancerInfoResponse - 31, // 51: service.NodeService.TestRoute:output_type -> service.RouteResult - 2, // 52: service.NodeService.AddRoutingRule:output_type -> service.Empty - 2, // 53: service.NodeService.RemoveRoutingRule:output_type -> service.Empty - 2, // 54: service.NodeService.OverrideBalancerTarget:output_type -> service.Empty - 36, // [36:55] is the sub-list for method output_type - 17, // [17:36] is the sub-list for method input_type - 17, // [17:17] is the sub-list for extension type_name - 17, // [17:17] is the sub-list for extension extendee - 0, // [0:17] is the sub-list for field type_name + 22, // 12: service.Proxy.masque:type_name -> service.Masque + 23, // 13: service.Proxy.mtproto:type_name -> service.Mtproto + 24, // 14: service.User.proxies:type_name -> service.Proxy + 25, // 15: service.Users.users:type_name -> service.User + 25, // 16: service.UsersChunk.users:type_name -> service.User + 28, // 17: service.RoutingRulesResponse.rules:type_name -> service.RoutingRule + 38, // 18: service.TestRouteRequest.attributes:type_name -> service.TestRouteRequest.AttributesEntry + 4, // 19: service.NodeService.Start:input_type -> service.Backend + 2, // 20: service.NodeService.Stop:input_type -> service.Empty + 2, // 21: service.NodeService.GetBaseInfo:input_type -> service.Empty + 2, // 22: service.NodeService.GetLogs:input_type -> service.Empty + 2, // 23: service.NodeService.GetSystemStats:input_type -> service.Empty + 2, // 24: service.NodeService.GetBackendStats:input_type -> service.Empty + 8, // 25: service.NodeService.GetStats:input_type -> service.StatRequest + 12, // 26: service.NodeService.GetOutboundsLatency:input_type -> service.LatencyRequest + 8, // 27: service.NodeService.GetUserOnlineStats:input_type -> service.StatRequest + 8, // 28: service.NodeService.GetUserOnlineIpListStats:input_type -> service.StatRequest + 25, // 29: service.NodeService.SyncUser:input_type -> service.User + 26, // 30: service.NodeService.SyncUsers:input_type -> service.Users + 27, // 31: service.NodeService.SyncUsersChunked:input_type -> service.UsersChunk + 2, // 32: service.NodeService.ListRoutingRules:input_type -> service.Empty + 30, // 33: service.NodeService.GetBalancerInfo:input_type -> service.BalancerInfoRequest + 32, // 34: service.NodeService.TestRoute:input_type -> service.TestRouteRequest + 34, // 35: service.NodeService.AddRoutingRule:input_type -> service.AddRoutingRuleRequest + 35, // 36: service.NodeService.RemoveRoutingRule:input_type -> service.RemoveRoutingRuleRequest + 36, // 37: service.NodeService.OverrideBalancerTarget:input_type -> service.OverrideBalancerTargetRequest + 3, // 38: service.NodeService.Start:output_type -> service.BaseInfoResponse + 2, // 39: service.NodeService.Stop:output_type -> service.Empty + 3, // 40: service.NodeService.GetBaseInfo:output_type -> service.BaseInfoResponse + 5, // 41: service.NodeService.GetLogs:output_type -> service.Log + 15, // 42: service.NodeService.GetSystemStats:output_type -> service.SystemStatsResponse + 14, // 43: service.NodeService.GetBackendStats:output_type -> service.BackendStatsResponse + 7, // 44: service.NodeService.GetStats:output_type -> service.StatResponse + 13, // 45: service.NodeService.GetOutboundsLatency:output_type -> service.LatencyResponse + 9, // 46: service.NodeService.GetUserOnlineStats:output_type -> service.OnlineStatResponse + 10, // 47: service.NodeService.GetUserOnlineIpListStats:output_type -> service.StatsOnlineIpListResponse + 2, // 48: service.NodeService.SyncUser:output_type -> service.Empty + 2, // 49: service.NodeService.SyncUsers:output_type -> service.Empty + 2, // 50: service.NodeService.SyncUsersChunked:output_type -> service.Empty + 29, // 51: service.NodeService.ListRoutingRules:output_type -> service.RoutingRulesResponse + 31, // 52: service.NodeService.GetBalancerInfo:output_type -> service.BalancerInfoResponse + 33, // 53: service.NodeService.TestRoute:output_type -> service.RouteResult + 2, // 54: service.NodeService.AddRoutingRule:output_type -> service.Empty + 2, // 55: service.NodeService.RemoveRoutingRule:output_type -> service.Empty + 2, // 56: service.NodeService.OverrideBalancerTarget:output_type -> service.Empty + 38, // [38:57] is the sub-list for method output_type + 19, // [19:38] is the sub-list for method input_type + 19, // [19:19] is the sub-list for extension type_name + 19, // [19:19] is the sub-list for extension extendee + 0, // [0:19] is the sub-list for field type_name } func init() { file_common_service_proto_init() } @@ -2382,7 +2543,7 @@ func file_common_service_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_common_service_proto_rawDesc), len(file_common_service_proto_rawDesc)), NumEnums: 2, - NumMessages: 35, + NumMessages: 37, NumExtensions: 0, NumServices: 1, }, diff --git a/common/service.proto b/common/service.proto index f44f0f5d..4ff0fbf7 100644 --- a/common/service.proto +++ b/common/service.proto @@ -16,6 +16,7 @@ message BaseInfoResponse { enum BackendType { XRAY = 0; WIREGUARD = 1; + MTPROTO = 2; } message Backend { @@ -138,6 +139,20 @@ message Hysteria { string auth = 1; } +message Masque { + string pass = 1; +} + +message Mtproto { + string secret = 1; + string user_ad_tag = 2; + uint32 max_tcp_conns = 3; + uint32 max_unique_ips = 4; + // Note: data quota and expiry are intentionally omitted. The panel owns + // quota/expiry enforcement (via usage reporting and user sync), the same as + // for the xray and wireguard backends; telemt must not enforce them. +} + message Proxy { Vmess vmess = 1; Vless vless = 2; @@ -145,6 +160,8 @@ message Proxy { Shadowsocks shadowsocks = 4; Wireguard wireguard = 5; Hysteria hysteria = 6; + Masque masque = 7; + Mtproto mtproto = 8; } message User { diff --git a/common/service_grpc.pb.go b/common/service_grpc.pb.go index 86ea1c57..503f2f63 100644 --- a/common/service_grpc.pb.go +++ b/common/service_grpc.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: -// - protoc-gen-go-grpc v1.6.1 -// - protoc v7.34.1 +// - protoc-gen-go-grpc v1.6.2 +// - protoc v7.36.1 // source: common/service.proto package common From 4c147e0a09decdc29654a826e153f21330a5d04f Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 19 Oct 2026 18:49:32 +0330 Subject: [PATCH 02/40] feat(mtproto): add telemt MTProto backend Port node PR #61 (feat/mtproto-telemt, head 4eb26fb) onto dev: the backend/mtproto package that drives telemt as a child process, the MTPROTO case in Controller.StartBackend, TELEMT_EXECUTABLE_PATH (default /usr/local/bin/telemt) in config and .env.example, and the install_telemt Makefile target (TELEMT_VERSION still 3.4.25). Ported PR commits: 8542f27, 90c50a1, 419732c, 9f9eac2, a87168d, 77c62fd, 630df12, 0eae051, ef43229, cd183a4, a66a322, fa79e3b, 098d828, 2c4fce8, 4eb26fb. The proto part of 8542f27 landed in the previous commit. Dockerfile.mtproto (9f9eac2, 4eb26fb) is not ported. Adapt the backend to dev's Backend interface: the process manager now keeps its output in a logstream.Buffer, the capture goroutines Publish into it, MTProto.SubscribeLogs returns a Subscribe channel, and Shutdown closes the buffer. This replaces Logs(), CloseLogs and the log WaitGroup, so a late log line can no longer be sent on a closed channel. Add a test that runs a stand-in process and checks that its stdout and stderr reach a subscriber and that shutdown closes the subscription cleanly. Co-authored-by: x0sina --- .env.example | 3 + Makefile | 51 ++- backend/mtproto/client.go | 275 ++++++++++++ backend/mtproto/config.go | 305 +++++++++++++ backend/mtproto/integration_test.go | 142 ++++++ backend/mtproto/metrics.go | 208 +++++++++ backend/mtproto/mtproto.go | 657 ++++++++++++++++++++++++++++ backend/mtproto/mtproto_test.go | 303 +++++++++++++ backend/mtproto/process.go | 229 ++++++++++ backend/mtproto/process_test.go | 70 +++ backend/mtproto/process_unix.go | 35 ++ backend/mtproto/process_windows.go | 28 ++ backend/mtproto/toml.go | 234 ++++++++++ backend/mtproto/tracker.go | 204 +++++++++ config/config.go | 2 + controller/controller.go | 11 + 16 files changed, 2756 insertions(+), 1 deletion(-) create mode 100644 backend/mtproto/client.go create mode 100644 backend/mtproto/config.go create mode 100644 backend/mtproto/integration_test.go create mode 100644 backend/mtproto/metrics.go create mode 100644 backend/mtproto/mtproto.go create mode 100644 backend/mtproto/mtproto_test.go create mode 100644 backend/mtproto/process.go create mode 100644 backend/mtproto/process_test.go create mode 100644 backend/mtproto/process_unix.go create mode 100644 backend/mtproto/process_windows.go create mode 100644 backend/mtproto/toml.go create mode 100644 backend/mtproto/tracker.go diff --git a/.env.example b/.env.example index 949f47d2..5539cced 100644 --- a/.env.example +++ b/.env.example @@ -9,6 +9,9 @@ NODE_HOST = "0.0.0.0" # pinned by GEOFILES_TAG in the Makefile (not the geo files from the Xray release zip). # These are defaults: `pg-node geofiles` installs its own files and points XRAY_ASSETS_PATH at them. +# MTProto (telemt) backend binary +# TELEMT_EXECUTABLE_PATH = /usr/local/bin/telemt + SSL_CERT_FILE = /var/lib/pg-node/certs/ssl_cert.pem SSL_KEY_FILE = /var/lib/pg-node/certs/ssl_key.pem diff --git a/Makefile b/Makefile index f1411091..99aa1b9d 100644 --- a/Makefile +++ b/Makefile @@ -46,6 +46,19 @@ GEOFILES_DIR ?= /usr/local/share/xray GEOFILES_XRAY ?= /usr/local/bin/xray GEOFILES_CHECK_CONFIG ?= $(CURDIR)/testdata/geofiles-check.json +# telemt (MTProto backend): pinned release; static musl tarball + sha256 verify +TELEMT_REPOSITORY ?= telemt/telemt +TELEMT_VERSION ?= 3.4.25 +TELEMT_OS ?= +TELEMT_ARCH ?= +TELEMT_ARCH_MAP_amd64 = x86_64 +TELEMT_ARCH_MAP_arm64 = aarch64 +TELEMT_GOOS := $(if $(GOOS),$(GOOS),$(shell go env GOOS)) +TELEMT_GOARCH := $(if $(GOARCH),$(GOARCH),$(shell go env GOARCH)) +TELEMT_OS_EFFECTIVE := $(if $(TELEMT_OS),$(TELEMT_OS),$(TELEMT_GOOS)) +TELEMT_ARCH_EFFECTIVE := $(if $(TELEMT_ARCH),$(TELEMT_ARCH),$(TELEMT_ARCH_MAP_$(TELEMT_GOARCH))) +TELEMT_GOARCH_EFFECTIVE := $(TELEMT_GOARCH) + ifeq ($(GOOS),windows) OUTPUT = $(NAME).exe ADDITION = go build -o w$(NAME).exe -trimpath -ldflags "-H windowsgui $(LDFLAGS)" -v $(MAIN) @@ -58,7 +71,7 @@ ADDITION = GOMIPS=softfloat go build -o $(NAME)_softfloat -trimpath -ldflags "$( endif .PHONY: clean build test test-race-wireguard test-integration test-integration-full test-integration-wireguard \ - install_geofiles check_geofiles update_geofiles_pin + install_geofiles check_geofiles update_geofiles_pin install_telemt build: CGO_ENABLED=0 go build -o $(OUTPUT) $(PARAMS) $(MAIN) @@ -264,6 +277,42 @@ else @exit 1 endif +install_telemt: update_os +ifeq ($(UNAME_S),Linux) + @if [ "$(TELEMT_OS_EFFECTIVE)" != "linux" ]; then \ + echo "Unsupported TELEMT target OS: $(TELEMT_OS_EFFECTIVE)"; \ + exit 1; \ + fi + @if [ -z "$(TELEMT_ARCH_EFFECTIVE)" ]; then \ + echo "Unsupported TELEMT target architecture: $(TELEMT_GOARCH_EFFECTIVE)"; \ + exit 1; \ + fi + @set -eu; \ + telemt_asset="telemt-$(TELEMT_ARCH_EFFECTIVE)-linux-musl.tar.gz"; \ + telemt_release="$(TELEMT_VERSION)"; \ + telemt_release="$${telemt_release#refs/tags/}"; \ + if [ -z "$${telemt_release}" ] || [ "$${telemt_release}" = "latest" ]; then \ + telemt_base_url="https://github.com/$(TELEMT_REPOSITORY)/releases/latest/download"; \ + else \ + telemt_base_url="https://github.com/$(TELEMT_REPOSITORY)/releases/download/$${telemt_release}"; \ + fi; \ + tmp_dir="$$(mktemp -d)"; \ + trap 'rm -rf "$$tmp_dir"' EXIT; \ + curl -fL --retry 5 --retry-delay 3 --connect-timeout 10 --max-time 120 \ + -o "$$tmp_dir/$$telemt_asset" "$$telemt_base_url/$$telemt_asset"; \ + curl -fL --retry 5 --retry-delay 3 --connect-timeout 10 --max-time 120 \ + -o "$$tmp_dir/$$telemt_asset.sha256" "$$telemt_base_url/$$telemt_asset.sha256"; \ + ( cd "$$tmp_dir" && sha256sum -c "$$telemt_asset.sha256" ); \ + tar -xzf "$$tmp_dir/$$telemt_asset" -C "$$tmp_dir"; \ + test -f "$$tmp_dir/telemt"; \ + if [ "$$(id -u)" -eq 0 ]; then install_cmd="install"; else install_cmd="sudo install"; fi; \ + $$install_cmd -m 0755 "$$tmp_dir/telemt" /usr/local/bin/telemt + @telemt --version || true +else + @echo "Unsupported operating system: $(UNAME_S)" + @exit 1 +endif + test-integration: $(MAKE) test-integration-full diff --git a/backend/mtproto/client.go b/backend/mtproto/client.go new file mode 100644 index 00000000..1500be41 --- /dev/null +++ b/backend/mtproto/client.go @@ -0,0 +1,275 @@ +package mtproto + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" +) + +type apiClient struct { + baseURL string + authHeader string + httpClient *http.Client +} + +type apiSuccessResponse[T any] struct { + OK bool `json:"ok"` + Data T `json:"data"` + Revision string `json:"revision"` + Error *apiErrorBody `json:"error,omitempty"` +} + +type apiErrorResponse struct { + OK bool `json:"ok"` + Error *apiErrorBody `json:"error"` + Request uint64 `json:"request_id"` + Revision string `json:"revision"` +} + +type apiErrorBody struct { + Code string `json:"code"` + Message string `json:"message"` +} + +type telemtUserInfo struct { + Username string `json:"username"` + InRuntime bool `json:"in_runtime"` + CurrentConnections uint64 `json:"current_connections"` + ActiveUniqueIPsList []string `json:"active_unique_ips_list"` + RecentUniqueIPsList []string `json:"recent_unique_ips_list"` + ActiveUniqueIPs int `json:"active_unique_ips"` + RecentUniqueIPs int `json:"recent_unique_ips"` + UserAdTag string `json:"user_ad_tag"` + MaxTCPConns int `json:"max_tcp_conns"` + MaxUniqueIPs int `json:"max_unique_ips"` + TotalOctets uint64 `json:"total_octets"` +} + +type telemtHealth struct { + Status string `json:"status"` + ReadOnly bool `json:"read_only"` +} + +type telemtCreateUserRequest struct { + Username string `json:"username"` + Secret string `json:"secret,omitempty"` + UserAdTag string `json:"user_ad_tag,omitempty"` + MaxTCPConns *int `json:"max_tcp_conns,omitempty"` + MaxUniqueIPs *int `json:"max_unique_ips,omitempty"` +} + +// telemtPatchUserRequest uses JSON Merge Patch: omitted fields are unchanged, +// JSON null removes the per-user override, and a value sets it. omitempty is +// therefore not used on the clearable fields. +type telemtPatchUserRequest struct { + Secret string `json:"secret,omitempty"` + UserAdTag *string `json:"user_ad_tag"` + MaxTCPConns *int `json:"max_tcp_conns"` + MaxUniqueIPs *int `json:"max_unique_ips"` +} + +type apiStatusError struct { + StatusCode int + Code string + Message string +} + +func (e *apiStatusError) Error() string { + message := strings.TrimSpace(e.Message) + if message != "" { + return message + } + if e.Code != "" { + return e.Code + } + return fmt.Sprintf("telemt API request failed with status %d", e.StatusCode) +} + +func (e *apiStatusError) NotFound() bool { + return e.StatusCode == http.StatusNotFound || e.Code == "not_found" +} + +func newAPIClient(baseURL, authHeader string) *apiClient { + return &apiClient{ + baseURL: strings.TrimRight(baseURL, "/"), + authHeader: authHeader, + httpClient: &http.Client{ + Timeout: 10 * time.Second, + }, + } +} + +func (c *apiClient) Health(ctx context.Context) error { + _, err := doAPIRequest[telemtHealth](ctx, c, http.MethodGet, "/v1/health", nil) + return err +} + +func (c *apiClient) GetUser(ctx context.Context, username string) (*telemtUserInfo, error) { + escapedUsername := url.PathEscape(username) + user, err := doAPIRequest[telemtUserInfo](ctx, c, http.MethodGet, "/v1/users/"+escapedUsername, nil) + if err != nil { + return nil, err + } + return user, nil +} + +func (c *apiClient) CreateUser(ctx context.Context, user *runtimeUser) error { + request, err := buildCreateUserRequest(user) + if err != nil { + return err + } + _, err = doAPIRequest[telemtUserInfo](ctx, c, http.MethodPost, "/v1/users", request) + return err +} + +func (c *apiClient) PatchUser(ctx context.Context, username string, user *runtimeUser) error { + request, err := buildPatchUserRequest(user) + if err != nil { + return err + } + _, err = doAPIRequest[telemtUserInfo](ctx, c, http.MethodPatch, "/v1/users/"+url.PathEscape(username), request) + return err +} + +func (c *apiClient) DeleteUser(ctx context.Context, username string) error { + _, err := doAPIRequest[map[string]any](ctx, c, http.MethodDelete, "/v1/users/"+url.PathEscape(username), nil) + return err +} + +func buildCreateUserRequest(user *runtimeUser) (*telemtCreateUserRequest, error) { + maxTCPConns, err := optionalPositiveInt(user.MaxTCPConns, "max_tcp_conns") + if err != nil { + return nil, err + } + maxUniqueIPs, err := optionalPositiveInt(user.MaxUniqueIPs, "max_unique_ips") + if err != nil { + return nil, err + } + + request := &telemtCreateUserRequest{ + Username: user.Username, + Secret: user.Secret, + UserAdTag: user.UserAdTag, + MaxTCPConns: maxTCPConns, + MaxUniqueIPs: maxUniqueIPs, + } + return request, nil +} + +func buildPatchUserRequest(user *runtimeUser) (*telemtPatchUserRequest, error) { + if user == nil { + return nil, errors.New("mtproto user is nil") + } + + maxTCPConns, err := optionalPositiveInt(user.MaxTCPConns, "max_tcp_conns") + if err != nil { + return nil, err + } + maxUniqueIPs, err := optionalPositiveInt(user.MaxUniqueIPs, "max_unique_ips") + if err != nil { + return nil, err + } + + request := &telemtPatchUserRequest{ + Secret: user.Secret, + MaxTCPConns: maxTCPConns, + MaxUniqueIPs: maxUniqueIPs, + } + if user.UserAdTag != "" { + tag := user.UserAdTag + request.UserAdTag = &tag + } + return request, nil +} + +func optionalPositiveInt(value uint32, field string) (*int, error) { + if value == 0 { + return nil, nil + } + maxInt := int(^uint(0) >> 1) + if uint64(value) > uint64(maxInt) { + return nil, fmt.Errorf("mtproto field %s exceeds native int range", field) + } + converted := int(value) + return &converted, nil +} + +func doAPIRequest[T any](ctx context.Context, client *apiClient, method, path string, body any) (*T, error) { + if client == nil { + return nil, errors.New("mtproto api client is nil") + } + + var requestBody io.Reader + if body != nil { + encoded, err := json.Marshal(body) + if err != nil { + return nil, fmt.Errorf("failed to encode mtproto api request: %w", err) + } + requestBody = bytes.NewReader(encoded) + } + + request, err := http.NewRequestWithContext(ctx, method, client.baseURL+path, requestBody) + if err != nil { + return nil, err + } + + if body != nil { + request.Header.Set("Content-Type", "application/json") + } + if client.authHeader != "" { + request.Header.Set("Authorization", client.authHeader) + } + + response, err := client.httpClient.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + + payload, err := io.ReadAll(response.Body) + if err != nil { + return nil, err + } + + if response.StatusCode < http.StatusOK || response.StatusCode >= http.StatusMultipleChoices { + var failure apiErrorResponse + if json.Unmarshal(payload, &failure) == nil && failure.Error != nil { + return nil, &apiStatusError{ + StatusCode: response.StatusCode, + Code: failure.Error.Code, + Message: failure.Error.Message, + } + } + return nil, &apiStatusError{ + StatusCode: response.StatusCode, + Message: strings.TrimSpace(string(payload)), + } + } + + var success apiSuccessResponse[T] + if err := json.Unmarshal(payload, &success); err != nil { + return nil, fmt.Errorf("failed to decode mtproto api response: %w", err) + } + if !success.OK { + if success.Error != nil { + return nil, &apiStatusError{ + StatusCode: response.StatusCode, + Code: success.Error.Code, + Message: success.Error.Message, + } + } + return nil, &apiStatusError{ + StatusCode: response.StatusCode, + Message: "telemt api returned ok=false", + } + } + + return &success.Data, nil +} diff --git a/backend/mtproto/config.go b/backend/mtproto/config.go new file mode 100644 index 00000000..0176c774 --- /dev/null +++ b/backend/mtproto/config.go @@ -0,0 +1,305 @@ +package mtproto + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "regexp" + "slices" + "strings" + + "github.com/pasarguard/node/common" +) + +const ( + defaultInboundTag = "mtproto" + loopbackV4CIDR = "127.0.0.1/32" + loopbackV6CIDR = "::1/128" +) + +var ( + validUsernamePattern = regexp.MustCompile(`^[A-Za-z0-9_.-]{1,64}$`) + restrictedAccessKeys = []string{ + "users", + "user_ad_tags", + "user_max_tcp_conns", + "user_expirations", + "user_data_quota", + "user_max_unique_ips", + "user_enabled", + } +) + +type Config struct { + base map[string]any + InboundTag string +} + +type runtimeUser struct { + Username string + Secret string + UserAdTag string + MaxTCPConns uint32 + MaxUniqueIPs uint32 +} + +func NewConfig(raw string) (*Config, error) { + if strings.TrimSpace(raw) == "" { + return nil, errors.New("mtproto config string must not be empty") + } + + decoder := json.NewDecoder(strings.NewReader(raw)) + decoder.UseNumber() + + var root map[string]any + if err := decoder.Decode(&root); err != nil { + return nil, fmt.Errorf("failed to decode mtproto config JSON: %w", err) + } + + var extra json.RawMessage + switch err := decoder.Decode(&extra); { + case errors.Is(err, io.EOF): + // exactly one JSON document, as required + case err != nil: + return nil, fmt.Errorf("failed to decode trailing mtproto config JSON: %w", err) + default: + return nil, errors.New("mtproto config must contain exactly one JSON object") + } + + cfg := &Config{ + base: cloneMap(root), + InboundTag: defaultInboundTag, + } + + if inboundTagRaw, exists := cfg.base["inbound_tag"]; exists { + inboundTag, ok := inboundTagRaw.(string) + if !ok { + return nil, errors.New("mtproto config field inbound_tag must be a string") + } + inboundTag = strings.TrimSpace(inboundTag) + if inboundTag == "" { + return nil, errors.New("mtproto config field inbound_tag must not be empty") + } + cfg.InboundTag = inboundTag + delete(cfg.base, "inbound_tag") + } + + if err := validateConfigRoot(cfg.base); err != nil { + return nil, err + } + + return cfg, nil +} + +func validateConfigRoot(root map[string]any) error { + if root == nil { + return errors.New("mtproto config must be a JSON object") + } + + if accessRaw, exists := root["access"]; exists { + access, ok := accessRaw.(map[string]any) + if !ok { + return errors.New("mtproto config field access must be an object") + } + for _, key := range restrictedAccessKeys { + if _, forbidden := access[key]; forbidden { + return fmt.Errorf("mtproto config must not define access.%s; node derives per-user access data", key) + } + } + } + + if serverRaw, exists := root["server"]; exists { + if _, ok := serverRaw.(map[string]any); !ok { + return errors.New("mtproto config field server must be an object") + } + } + + return nil +} + +func (c *Config) Render(users map[string]*runtimeUser, sentinel *runtimeUser, apiPort, metricsPort int, apiAuthHeader string) (string, error) { + if c == nil { + return "", errors.New("mtproto config is nil") + } + + root := cloneMap(c.base) + server := ensureObject(root, "server") + api := ensureObject(server, "api") + access := ensureObject(root, "access") + + // metrics_port enables the Prometheus endpoint; metrics_listen pins it to + // loopback. telemt treats metrics_port as the enable flag, so both are set. + server["metrics_port"] = metricsPort + server["metrics_listen"] = fmt.Sprintf("127.0.0.1:%d", metricsPort) + server["metrics_whitelist"] = []any{loopbackV4CIDR, loopbackV6CIDR} + + api["enabled"] = true + api["listen"] = fmt.Sprintf("127.0.0.1:%d", apiPort) + api["auth_header"] = apiAuthHeader + api["whitelist"] = []any{loopbackV4CIDR, loopbackV6CIDR} + + if err := applyUsersToAccess(access, users, sentinel); err != nil { + return "", err + } + + return renderTOML(root) +} + +func applyUsersToAccess(access map[string]any, users map[string]*runtimeUser, sentinel *runtimeUser) error { + if len(users) == 0 && sentinel == nil { + return errors.New("mtproto config requires at least one user or sentinel") + } + + userSecrets := make(map[string]any, len(users)+1) + userAdTags := make(map[string]any) + userMaxTCPConns := make(map[string]any) + userMaxUniqueIPs := make(map[string]any) + userEnabled := make(map[string]any) + + usernames := make([]string, 0, len(users)) + for username := range users { + usernames = append(usernames, username) + } + slices.Sort(usernames) + + for _, username := range usernames { + user := users[username] + userSecrets[username] = user.Secret + if user.UserAdTag != "" { + userAdTags[username] = user.UserAdTag + } + if user.MaxTCPConns > 0 { + userMaxTCPConns[username] = user.MaxTCPConns + } + if user.MaxUniqueIPs > 0 { + userMaxUniqueIPs[username] = user.MaxUniqueIPs + } + } + + if sentinel != nil { + // Keep [access.users] non-empty so telemt will start, but disable the + // sentinel so it cannot authenticate as a real MTProto client. + userSecrets[sentinel.Username] = sentinel.Secret + userEnabled[sentinel.Username] = false + } + + access["users"] = userSecrets + writeOptionalMap(access, "user_ad_tags", userAdTags) + writeOptionalMap(access, "user_max_tcp_conns", userMaxTCPConns) + writeOptionalMap(access, "user_max_unique_ips", userMaxUniqueIPs) + writeOptionalMap(access, "user_enabled", userEnabled) + + return nil +} + +func writeOptionalMap(target map[string]any, key string, values map[string]any) { + if len(values) == 0 { + delete(target, key) + return + } + target[key] = values +} + +func ActiveUsersFromCommon(users []*common.User, inboundTag string) (map[string]*runtimeUser, error) { + active := make(map[string]*runtimeUser) + for _, user := range users { + runtimeUser, include, err := activeUserFromCommon(user, inboundTag) + if err != nil { + return nil, err + } + if include { + active[runtimeUser.Username] = runtimeUser + } + } + return active, nil +} + +func activeUserFromCommon(user *common.User, inboundTag string) (*runtimeUser, bool, error) { + if user == nil { + return nil, false, errors.New("mtproto user must not be nil") + } + + if !slices.Contains(user.GetInbounds(), inboundTag) { + return nil, false, nil + } + + username := strings.TrimSpace(user.GetEmail()) + if !validUsernamePattern.MatchString(username) { + return nil, false, fmt.Errorf("invalid mtproto username %q; expected [A-Za-z0-9_.-] and length 1..64", username) + } + + proxy := user.GetProxies().GetMtproto() + if proxy == nil { + return nil, false, fmt.Errorf("mtproto user %q is missing proxies.mtproto", username) + } + + secret := strings.TrimSpace(proxy.GetSecret()) + if !isHex32(secret) { + return nil, false, fmt.Errorf("mtproto user %q has an invalid secret; expected exactly 32 hex characters", username) + } + + userAdTag := strings.TrimSpace(proxy.GetUserAdTag()) + if userAdTag != "" && !isHex32(userAdTag) { + return nil, false, fmt.Errorf("mtproto user %q has an invalid user_ad_tag; expected exactly 32 hex characters", username) + } + + return &runtimeUser{ + Username: username, + Secret: strings.ToLower(secret), + UserAdTag: strings.ToLower(userAdTag), + MaxTCPConns: proxy.GetMaxTcpConns(), + MaxUniqueIPs: proxy.GetMaxUniqueIps(), + }, true, nil +} + +func ensureObject(target map[string]any, key string) map[string]any { + if existing, ok := target[key]; ok { + if existingMap, ok := existing.(map[string]any); ok { + return existingMap + } + } + + child := make(map[string]any) + target[key] = child + return child +} + +func cloneMap(input map[string]any) map[string]any { + if input == nil { + return map[string]any{} + } + + cloned := make(map[string]any, len(input)) + for key, value := range input { + cloned[key] = cloneValue(value) + } + return cloned +} + +func cloneValue(value any) any { + switch typed := value.(type) { + case map[string]any: + return cloneMap(typed) + case []any: + cloned := make([]any, len(typed)) + for i, item := range typed { + cloned[i] = cloneValue(item) + } + return cloned + default: + return typed + } +} + +func isHex32(value string) bool { + if len(value) != 32 { + return false + } + for _, r := range value { + if !((r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')) { + return false + } + } + return true +} diff --git a/backend/mtproto/integration_test.go b/backend/mtproto/integration_test.go new file mode 100644 index 00000000..c7281b87 --- /dev/null +++ b/backend/mtproto/integration_test.go @@ -0,0 +1,142 @@ +package mtproto + +import ( + "context" + "errors" + "fmt" + "os" + "testing" + "time" + + "github.com/pasarguard/node/common" + "github.com/pasarguard/node/config" + "github.com/pasarguard/node/pkg/netutil" +) + +// Gated integration test: runs only when MTPROTO_TELEMT_BIN points at a real +// telemt binary. Exercises the full lifecycle (start, health, users, stats, +// sentinel/empty-users) against the actual proxy. +// +// MTPROTO_TELEMT_BIN=/path/to/telemt go test ./backend/mtproto/ -run Integration -v +func telemtBinOrSkip(t *testing.T) string { + t.Helper() + bin := os.Getenv("MTPROTO_TELEMT_BIN") + if bin == "" { + t.Skip("set MTPROTO_TELEMT_BIN to a telemt binary to run telemt integration tests") + } + return bin +} + +func integrationConfig(t *testing.T, bin string) *config.Config { + return &config.Config{ + TelemtExecutablePath: bin, + GeneratedConfigPath: t.TempDir(), + LogBufferSize: 1000, + } +} + +func integrationMTConfig(t *testing.T) *Config { + port := netutil.FindFreePort() + raw := fmt.Sprintf(`{ + "inbound_tag": "mtproto", + "general": {"use_middle_proxy": false, "modes": {"classic": true, "secure": true, "tls": false}}, + "server": {"port": %d, "listeners": [{"ip": "127.0.0.1"}]} + }`, port) + cfg, err := NewConfig(raw) + if err != nil { + t.Fatalf("NewConfig: %v", err) + } + return cfg +} + +func TestIntegration_LifecycleAndUsers(t *testing.T) { + bin := telemtBinOrSkip(t) + + ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second) + defer cancel() + + users := []*common.User{mtUser("alice", "mtproto", testSecret)} + mt, err := New(ctx, integrationConfig(t, bin), integrationMTConfig(t), users) + if err != nil { + t.Fatalf("New: %v", err) + } + defer mt.Shutdown() + + if !mt.Started() { + t.Fatal("backend not started") + } + if v := mt.Version(); v == "" { + t.Error("empty telemt version") + } else { + t.Logf("telemt version: %s", v) + } + + // alice was rendered into the config; she must be present in telemt's runtime. + if _, err := mt.apiClient.GetUser(ctx, "alice"); err != nil { + t.Fatalf("alice not in telemt runtime: %v", err) + } + + // add bob at runtime via the API path + if err := mt.SyncUser(ctx, mtUser("bob", "mtproto", "ffeeddccbbaa00112233445566778899")); err != nil { + t.Fatalf("SyncUser(bob): %v", err) + } + if _, err := mt.apiClient.GetUser(ctx, "bob"); err != nil { + t.Fatalf("bob not added: %v", err) + } + + // remove alice (untag her) + aliceGone := &common.User{Email: "alice", Inbounds: []string{}, Proxies: &common.Proxy{Mtproto: &common.Mtproto{Secret: testSecret}}} + if err := mt.SyncUser(ctx, aliceGone); err != nil { + t.Fatalf("SyncUser(remove alice): %v", err) + } + if _, err := mt.apiClient.GetUser(ctx, "alice"); err == nil { + t.Fatal("alice should have been removed") + } else { + var apiErr *apiStatusError + if !errors.As(err, &apiErr) || !apiErr.NotFound() { + t.Fatalf("expected alice lookup to return not found, got %v", err) + } + } + + // Traffic + online-count stats come from telemt's Prometheus endpoint, which + // only binds once the data plane is up (needs Telegram connectivity). A miss + // must still succeed with empty/zero results so the panel poll does not fail. + if stats, err := mt.GetStats(ctx, &common.StatRequest{Type: common.StatType_UsersStat}); err != nil { + t.Errorf("GetStats: %v", err) + } else if stats == nil { + t.Error("GetStats returned nil") + } + if online, err := mt.GetUserOnlineStats(ctx, "bob"); err != nil { + t.Errorf("GetUserOnlineStats: %v", err) + } else if online == nil || online.GetName() != "bob" { + t.Errorf("GetUserOnlineStats = %+v", online) + } + + // These do not depend on metrics: IP list comes from the control API, sys + // stats from the OS process; both must work against a live telemt. + if _, err := mt.GetUserOnlineIpListStats(ctx, "bob"); err != nil { + t.Errorf("GetUserOnlineIpListStats: %v", err) + } + if _, err := mt.GetSysStats(ctx); err != nil { + t.Errorf("GetSysStats: %v", err) + } +} + +func TestIntegration_EmptyUsersSentinel(t *testing.T) { + bin := telemtBinOrSkip(t) + + ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second) + defer cancel() + + // Zero real users: telemt rejects an empty [access.users], so the sentinel + // user must keep it alive. + mt, err := New(ctx, integrationConfig(t, bin), integrationMTConfig(t), nil) + if err != nil { + t.Fatalf("New with zero users (sentinel path): %v", err) + } + defer mt.Shutdown() + + if !mt.Started() { + t.Fatal("backend not started with sentinel user") + } +} diff --git a/backend/mtproto/metrics.go b/backend/mtproto/metrics.go new file mode 100644 index 00000000..f7f049f6 --- /dev/null +++ b/backend/mtproto/metrics.go @@ -0,0 +1,208 @@ +package mtproto + +import ( + "bufio" + "context" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" +) + +const ( + telemtMetricFromClient = "telemt_user_octets_from_client" + telemtMetricToClient = "telemt_user_octets_to_client" + telemtMetricConnectionsCurrnt = "telemt_user_connections_current" + telemtMetricUniqueIPsCurrent = "telemt_user_unique_ips_current" +) + +// userMetrics holds the per-user counters/gauges scraped from telemt's +// Prometheus endpoint. Octets are monotonic counters (delta-tracked by +// trafficTracker); connections/unique-ips are point-in-time gauges. +type userMetrics struct { + Uplink int64 + Downlink int64 + ConnectionsCurrent int64 + UniqueIPsCurrent int64 +} + +var perUserMetricNames = map[string]struct{}{ + telemtMetricFromClient: {}, + telemtMetricToClient: {}, + telemtMetricConnectionsCurrnt: {}, + telemtMetricUniqueIPsCurrent: {}, +} + +type metricsScraper struct { + url string + httpClient *http.Client +} + +func newMetricsScraper(url string) *metricsScraper { + return &metricsScraper{ + url: url, + httpClient: &http.Client{ + Timeout: 10 * time.Second, + }, + } +} + +func (s *metricsScraper) Scrape(ctx context.Context) (map[string]userMetrics, error) { + if s == nil { + return nil, fmt.Errorf("mtproto metrics scraper is nil") + } + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url, nil) + if err != nil { + return nil, err + } + + response, err := s.httpClient.Do(request) + if err != nil { + return nil, err + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + payload, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) + return nil, fmt.Errorf("mtproto metrics scrape failed with status %d: %s", response.StatusCode, strings.TrimSpace(string(payload))) + } + + return parseMetrics(response.Body) +} + +func parseMetrics(reader io.Reader) (map[string]userMetrics, error) { + counters := make(map[string]userMetrics) + scanner := bufio.NewScanner(reader) + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + + metricName, labels, value, ok, err := parseMetricLine(line) + if err != nil { + return nil, err + } + if !ok { + continue + } + + username, exists := labels["user"] + if !exists || username == "" { + continue + } + + entry := counters[username] + switch metricName { + case telemtMetricFromClient: + entry.Uplink = value + case telemtMetricToClient: + entry.Downlink = value + case telemtMetricConnectionsCurrnt: + entry.ConnectionsCurrent = value + case telemtMetricUniqueIPsCurrent: + entry.UniqueIPsCurrent = value + default: + continue + } + counters[username] = entry + } + + if err := scanner.Err(); err != nil { + return nil, err + } + + return counters, nil +} + +func parseMetricLine(line string) (string, map[string]string, int64, bool, error) { + splitIndex := strings.IndexAny(line, " \t") + if splitIndex <= 0 { + return "", nil, 0, false, nil + } + + metricWithLabels := strings.TrimSpace(line[:splitIndex]) + valueText := strings.TrimSpace(line[splitIndex+1:]) + if valueText == "" { + return "", nil, 0, false, nil + } + + metricName := metricWithLabels + labels := make(map[string]string) + + if braceIndex := strings.IndexByte(metricWithLabels, '{'); braceIndex >= 0 { + if !strings.HasSuffix(metricWithLabels, "}") { + return "", nil, 0, false, fmt.Errorf("invalid prometheus metric line %q", line) + } + + metricName = metricWithLabels[:braceIndex] + parsedLabels, err := parsePrometheusLabels(metricWithLabels[braceIndex+1 : len(metricWithLabels)-1]) + if err != nil { + return "", nil, 0, false, err + } + labels = parsedLabels + } + + if _, tracked := perUserMetricNames[metricName]; !tracked { + return "", nil, 0, false, nil + } + + metricValue, err := strconv.ParseFloat(strings.Fields(valueText)[0], 64) + if err != nil { + return "", nil, 0, false, fmt.Errorf("invalid prometheus metric value %q: %w", valueText, err) + } + + return metricName, labels, int64(metricValue), true, nil +} + +func parsePrometheusLabels(raw string) (map[string]string, error) { + result := make(map[string]string) + if strings.TrimSpace(raw) == "" { + return result, nil + } + + parts := splitLabelPairs(raw) + for _, part := range parts { + key, value, found := strings.Cut(part, "=") + if !found { + return nil, fmt.Errorf("invalid prometheus label pair %q", part) + } + key = strings.TrimSpace(key) + value = strings.TrimSpace(value) + decoded, err := strconv.Unquote(value) + if err != nil { + return nil, fmt.Errorf("invalid prometheus label value %q: %w", value, err) + } + result[key] = decoded + } + + return result, nil +} + +func splitLabelPairs(raw string) []string { + parts := make([]string, 0) + start := 0 + inQuotes := false + escapeNext := false + + for index, r := range raw { + switch { + case escapeNext: + escapeNext = false + case r == '\\': + escapeNext = true + case r == '"': + inQuotes = !inQuotes + case r == ',' && !inQuotes: + parts = append(parts, strings.TrimSpace(raw[start:index])) + start = index + 1 + } + } + + parts = append(parts, strings.TrimSpace(raw[start:])) + return parts +} diff --git a/backend/mtproto/mtproto.go b/backend/mtproto/mtproto.go new file mode 100644 index 00000000..9295c2ab --- /dev/null +++ b/backend/mtproto/mtproto.go @@ -0,0 +1,657 @@ +package mtproto + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + gopsutilprocess "github.com/shirou/gopsutil/v4/process" + + "github.com/pasarguard/node/common" + "github.com/pasarguard/node/config" + "github.com/pasarguard/node/pkg/netutil" +) + +const ( + telemtStartupTimeout = 30 * time.Second +) + +type MTProto struct { + cfg *config.Config + config *Config + process *processManager + apiClient *apiClient + metricsScraper *metricsScraper + tracker *trafficTracker + sentinel *runtimeUser + + apiPort int + metricsPort int + apiAuthHeader string + + mu sync.RWMutex + desired map[string]*runtimeUser + shutdownOnce sync.Once +} + +func New(ctx context.Context, cfg *config.Config, mtConfig *Config, users []*common.User) (*MTProto, error) { + if cfg == nil { + return nil, errors.New("config is nil") + } + if mtConfig == nil { + return nil, errors.New("mtproto config is nil") + } + + apiPort := netutil.FindFreePort() + metricsPort := netutil.FindFreePort() + for metricsPort == apiPort { + metricsPort = netutil.FindFreePort() + } + + authHeader, err := generateBearerToken() + if err != nil { + return nil, err + } + + generatedRoot, err := filepath.Abs(cfg.GeneratedConfigPath) + if err != nil { + return nil, err + } + telemtDir := filepath.Join(generatedRoot, "telemt") + process, err := newProcessManager( + cfg.TelemtExecutablePath, + filepath.Join(telemtDir, "config.toml"), + filepath.Join(telemtDir, "telemt.pid"), + cfg.LogBufferSize, + ) + if err != nil { + return nil, err + } + + activeUsers, err := ActiveUsersFromCommon(users, mtConfig.InboundTag) + if err != nil { + return nil, err + } + + sentinel, err := newSentinelUser() + if err != nil { + return nil, err + } + + mt := &MTProto{ + cfg: cfg, + config: mtConfig, + process: process, + apiClient: newAPIClient(fmt.Sprintf("http://127.0.0.1:%d", apiPort), authHeader), + metricsScraper: newMetricsScraper(fmt.Sprintf("http://127.0.0.1:%d/metrics", metricsPort)), + tracker: newTrafficTracker(), + sentinel: sentinel, + apiPort: apiPort, + metricsPort: metricsPort, + apiAuthHeader: authHeader, + desired: cloneRuntimeUserMap(activeUsers), + } + + if err := mt.writeRenderedConfigLocked(mt.desired); err != nil { + return nil, err + } + if err := mt.process.Start(); err != nil { + return nil, err + } + if err := mt.waitForHealth(ctx); err != nil { + mt.Shutdown() + return nil, err + } + + return mt, nil +} + +func (m *MTProto) Started() bool { + return m.process.Started() +} + +func (m *MTProto) Version() string { + return m.process.Version() +} + +func (m *MTProto) SubscribeLogs(ctx context.Context) <-chan string { + return m.process.SubscribeLogs(ctx) +} + +func (m *MTProto) Restart() error { + m.mu.Lock() + defer m.mu.Unlock() + + if err := m.writeRenderedConfigLocked(m.desired); err != nil { + return err + } + if err := m.process.Restart(); err != nil { + return err + } + + return m.waitForHealth(context.Background()) +} + +func (m *MTProto) Shutdown() { + m.shutdownOnce.Do(func() { + m.process.Shutdown() + m.process.logs.Close() + }) +} + +func (m *MTProto) SyncUser(ctx context.Context, user *common.User) error { + return m.updateUsersPartial(ctx, []*common.User{user}) +} + +func (m *MTProto) SyncUsers(ctx context.Context, users []*common.User) error { + m.mu.Lock() + defer m.mu.Unlock() + + desired, err := ActiveUsersFromCommon(users, m.config.InboundTag) + if err != nil { + return err + } + + return m.applyFullDesiredLocked(ctx, desired) +} + +func (m *MTProto) UpdateUsers(ctx context.Context, users []*common.User) error { + return m.updateUsersPartial(ctx, users) +} + +func (m *MTProto) UpdateUsersAndRestart(ctx context.Context, users []*common.User) error { + m.mu.Lock() + defer m.mu.Unlock() + + nextDesired, err := m.buildNextDesiredLocked(users) + if err != nil { + return err + } + + return m.applyFullDesiredLocked(ctx, nextDesired) +} + +func (m *MTProto) GetSysStats(_ context.Context) (*common.BackendStatsResponse, error) { + if !m.process.Started() { + return nil, errors.New("mtproto backend is not started") + } + + pid := m.process.PID() + if pid == 0 { + return nil, errors.New("telemt process pid is unavailable") + } + + process, err := gopsutilprocess.NewProcess(pid) + if err != nil { + return nil, err + } + + result := &common.BackendStatsResponse{} + if memoryInfo, err := process.MemoryInfo(); err == nil && memoryInfo != nil { + result.Alloc = memoryInfo.RSS + result.Sys = memoryInfo.VMS + } + + if createTimeMillis, err := process.CreateTime(); err == nil && createTimeMillis > 0 { + uptimeSeconds := uint64(0) + nowMillis := time.Now().UnixMilli() + if nowMillis > createTimeMillis { + uptimeSeconds = uint64((nowMillis - createTimeMillis) / 1000) + } + if uptimeSeconds > uint64(^uint32(0)) { + uptimeSeconds = uint64(^uint32(0)) + } + result.Uptime = uint32(uptimeSeconds) + } + + return result, nil +} + +func (m *MTProto) GetStats(ctx context.Context, request *common.StatRequest) (*common.StatResponse, error) { + if !m.process.Started() { + return nil, errors.New("mtproto backend is not started") + } + + // Prometheus only binds after telemt's data plane is up (Telegram egress). + // Treat a scrape miss as empty stats rather than failing the panel poll. + if err := m.refreshTracker(ctx); err != nil { + return &common.StatResponse{Stats: []*common.Stat{}}, nil + } + + link := m.config.InboundTag + switch request.GetType() { + case common.StatType_UsersStat: + return m.tracker.UsersStats(link, request.GetReset_()), nil + case common.StatType_UserStat: + return m.tracker.UserStats(request.GetName(), link, request.GetReset_()), nil + case common.StatType_Outbounds, common.StatType_Outbound: + if request.GetType() == common.StatType_Outbound && request.GetName() != "" && request.GetName() != link { + return &common.StatResponse{Stats: []*common.Stat{}}, nil + } + return m.tracker.AggregateStats(link, "mtproto", request.GetReset_()), nil + case common.StatType_Inbounds, common.StatType_Inbound: + if request.GetType() == common.StatType_Inbound && request.GetName() != "" && request.GetName() != link { + return &common.StatResponse{Stats: []*common.Stat{}}, nil + } + return m.tracker.AggregateStats(link, link, request.GetReset_()), nil + default: + return nil, errors.New("unsupported mtproto stat type") + } +} + +// GetOutboundsLatency is not applicable to an MTProto proxy (no outbounds); +// it returns an empty result to satisfy the backend.Backend interface. +func (m *MTProto) GetOutboundsLatency(_ context.Context, _ *common.LatencyRequest) (*common.LatencyResponse, error) { + return &common.LatencyResponse{Latencies: []*common.Latency{}}, nil +} + +func (m *MTProto) GetUserOnlineStats(ctx context.Context, email string) (*common.OnlineStatResponse, error) { + if !m.process.Started() { + return nil, errors.New("mtproto backend is not started") + } + + if !m.isDesiredUser(email) { + return &common.OnlineStatResponse{Name: email, Value: 0}, nil + } + + // Online count comes from telemt's per-user gauge (telemt_user_connections_current) + // via the metrics scrape, rather than a per-user API call. + snapshot, err := m.metricsScraper.Scrape(ctx) + if err != nil { + return &common.OnlineStatResponse{Name: email, Value: 0}, nil + } + + return &common.OnlineStatResponse{ + Name: email, + Value: snapshot[email].ConnectionsCurrent, + }, nil +} + +func (m *MTProto) GetUserOnlineIpListStats(ctx context.Context, email string) (*common.StatsOnlineIpListResponse, error) { + if !m.process.Started() { + return nil, errors.New("mtproto backend is not started") + } + + response := &common.StatsOnlineIpListResponse{ + Name: email, + Ips: make(map[string]int64), + } + + if !m.isDesiredUser(email) { + return response, nil + } + + // The active-IP *list* is only available from the API (metrics expose the count only). + userInfo, err := m.apiClient.GetUser(ctx, email) + if err != nil { + var apiErr *apiStatusError + if errors.As(err, &apiErr) && apiErr.NotFound() { + return response, nil + } + return nil, err + } + + now := time.Now().Unix() + for _, ip := range userInfo.ActiveUniqueIPsList { + if ip == "" { + continue + } + response.Ips[ip] = now + } + + return response, nil +} + +func (m *MTProto) updateUsersPartial(ctx context.Context, users []*common.User) error { + m.mu.Lock() + defer m.mu.Unlock() + + nextDesired, err := m.buildNextDesiredLocked(users) + if err != nil { + return err + } + + if len(m.desired) == 0 || len(nextDesired) == 0 { + return m.applyFullDesiredLocked(ctx, nextDesired) + } + + if err := m.applyPartialDesiredLocked(ctx, users, nextDesired); err != nil { + return m.applyFullDesiredLocked(ctx, nextDesired) + } + + m.desired = cloneRuntimeUserMap(nextDesired) + return nil +} + +func (m *MTProto) buildNextDesiredLocked(users []*common.User) (map[string]*runtimeUser, error) { + nextDesired := cloneRuntimeUserMap(m.desired) + for _, user := range users { + runtimeUser, include, err := activeUserFromCommon(user, m.config.InboundTag) + if err != nil { + return nil, err + } + + username := strings.TrimSpace(user.GetEmail()) + if include { + nextDesired[runtimeUser.Username] = runtimeUser + continue + } + delete(nextDesired, username) + } + return nextDesired, nil +} + +func (m *MTProto) applyPartialDesiredLocked(ctx context.Context, users []*common.User, nextDesired map[string]*runtimeUser) error { + for _, user := range users { + runtimeUser, include, err := activeUserFromCommon(user, m.config.InboundTag) + if err != nil { + return err + } + + if include { + if err := m.upsertUser(ctx, runtimeUser); err != nil { + return err + } + continue + } + + username := strings.TrimSpace(user.GetEmail()) + if username == "" { + continue + } + if err := m.deleteUser(ctx, username); err != nil { + return err + } + } + + _ = nextDesired + return nil +} + +func (m *MTProto) upsertUser(ctx context.Context, user *runtimeUser) error { + _, err := m.apiClient.GetUser(ctx, user.Username) + if err == nil { + return m.apiClient.PatchUser(ctx, user.Username, user) + } + + var apiErr *apiStatusError + if errors.As(err, &apiErr) && apiErr.NotFound() { + return m.apiClient.CreateUser(ctx, user) + } + + return err +} + +func (m *MTProto) deleteUser(ctx context.Context, username string) error { + err := m.apiClient.DeleteUser(ctx, username) + if err == nil { + return nil + } + + var apiErr *apiStatusError + if errors.As(err, &apiErr) && apiErr.NotFound() { + return nil + } + return err +} + +func (m *MTProto) applyFullDesiredLocked(ctx context.Context, desired map[string]*runtimeUser) error { + previousDesired := cloneRuntimeUserMap(m.desired) + + if err := m.writeRenderedConfigLocked(desired); err != nil { + return err + } + + if !m.process.Started() { + if err := m.process.Start(); err != nil { + return err + } + if err := m.waitForHealth(ctx); err != nil { + m.process.Shutdown() + return err + } + if err := m.waitForRuntimeUsers(ctx, previousDesired, desired); err != nil { + return err + } + m.desired = cloneRuntimeUserMap(desired) + return nil + } + + if err := m.process.Reload(); err != nil { + if restartErr := m.process.Restart(); restartErr != nil { + return fmt.Errorf("telemt reload failed: %w (restart failed: %v)", err, restartErr) + } + if healthErr := m.waitForHealth(ctx); healthErr != nil { + return healthErr + } + if runtimeErr := m.waitForRuntimeUsers(ctx, previousDesired, desired); runtimeErr != nil { + return runtimeErr + } + m.desired = cloneRuntimeUserMap(desired) + return nil + } + + if err := m.waitForHealth(ctx); err != nil { + if restartErr := m.process.Restart(); restartErr != nil { + return fmt.Errorf("telemt reload health check failed: %w (restart failed: %v)", err, restartErr) + } + if healthErr := m.waitForHealth(ctx); healthErr != nil { + return healthErr + } + } + if err := m.waitForRuntimeUsers(ctx, previousDesired, desired); err != nil { + return err + } + + m.desired = cloneRuntimeUserMap(desired) + return nil +} + +func (m *MTProto) writeRenderedConfigLocked(desired map[string]*runtimeUser) error { + var sentinel *runtimeUser + if len(desired) == 0 { + sentinel = m.sentinel + } + + rendered, err := m.config.Render(desired, sentinel, m.apiPort, m.metricsPort, m.apiAuthHeader) + if err != nil { + return err + } + + return writeFileAtomic(m.process.configPath, []byte(rendered)) +} + +func (m *MTProto) waitForHealth(ctx context.Context) error { + waitContext := ctx + if _, hasDeadline := ctx.Deadline(); !hasDeadline { + var cancel context.CancelFunc + waitContext, cancel = context.WithTimeout(ctx, telemtStartupTimeout) + defer cancel() + } + + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + + waitCh := m.process.WaitChan() + + for { + if err := m.apiClient.Health(waitContext); err == nil { + return nil + } + + select { + case <-waitContext.Done(): + return waitContext.Err() + case err := <-waitCh: + if err == nil { + return errors.New("telemt process exited before becoming healthy") + } + return fmt.Errorf("telemt process exited before becoming healthy: %w", err) + case <-ticker.C: + } + } +} + +func (m *MTProto) waitForRuntimeUsers(ctx context.Context, previousDesired, desired map[string]*runtimeUser) error { + waitContext := ctx + if _, hasDeadline := ctx.Deadline(); !hasDeadline { + var cancel context.CancelFunc + waitContext, cancel = context.WithTimeout(ctx, telemtStartupTimeout) + defer cancel() + } + + expectedPresent := make(map[string]struct{}, len(desired)) + for username := range desired { + expectedPresent[username] = struct{}{} + } + if len(expectedPresent) == 0 { + expectedPresent[m.sentinel.Username] = struct{}{} + } + + expectedAbsent := make(map[string]struct{}) + for username := range previousDesired { + if _, keep := desired[username]; !keep { + expectedAbsent[username] = struct{}{} + } + } + // Going from zero desired users to real users drops the sentinel, but it is + // not in previousDesired, so require it to be gone explicitly. + if len(previousDesired) == 0 && len(desired) > 0 { + expectedAbsent[m.sentinel.Username] = struct{}{} + } + + ticker := time.NewTicker(200 * time.Millisecond) + defer ticker.Stop() + + for { + converged := true + + for username := range expectedPresent { + if _, err := m.apiClient.GetUser(waitContext, username); err != nil { + converged = false + break + } + } + + if converged { + for username := range expectedAbsent { + _, err := m.apiClient.GetUser(waitContext, username) + if err == nil { + converged = false + break + } + + var apiErr *apiStatusError + if !errors.As(err, &apiErr) || !apiErr.NotFound() { + converged = false + break + } + } + } + + if converged { + return nil + } + + select { + case <-waitContext.Done(): + return waitContext.Err() + case <-ticker.C: + } + } +} + +func (m *MTProto) refreshTracker(ctx context.Context) error { + m.mu.RLock() + activeUsers := make(map[string]struct{}, len(m.desired)) + for username := range m.desired { + activeUsers[username] = struct{}{} + } + scraper := m.metricsScraper + tracker := m.tracker + m.mu.RUnlock() + + snapshot, err := scraper.Scrape(ctx) + if err != nil { + return fmt.Errorf("failed to scrape telemt metrics: %w", err) + } + + tracker.Sync(snapshot, activeUsers) + return nil +} + +func (m *MTProto) isDesiredUser(username string) bool { + m.mu.RLock() + defer m.mu.RUnlock() + _, exists := m.desired[username] + return exists +} + +func newSentinelUser() (*runtimeUser, error) { + secret := make([]byte, 16) + usernameSuffix := make([]byte, 4) + if _, err := rand.Read(secret); err != nil { + return nil, fmt.Errorf("failed to generate mtproto sentinel secret: %w", err) + } + if _, err := rand.Read(usernameSuffix); err != nil { + return nil, fmt.Errorf("failed to generate mtproto sentinel username: %w", err) + } + + return &runtimeUser{ + Username: "pg-node-internal-" + hex.EncodeToString(usernameSuffix), + Secret: hex.EncodeToString(secret), + }, nil +} + +func generateBearerToken() (string, error) { + token := make([]byte, 24) + if _, err := rand.Read(token); err != nil { + return "", err + } + return "Bearer " + hex.EncodeToString(token), nil +} + +func cloneRuntimeUserMap(input map[string]*runtimeUser) map[string]*runtimeUser { + cloned := make(map[string]*runtimeUser, len(input)) + for username, user := range input { + if user == nil { + continue + } + copyUser := *user + cloned[username] = ©User + } + return cloned +} + +// writeFileAtomic writes payload to path via a temp file and rename. The +// rendered config holds the API bearer token and per-user secrets, so the +// directory and file are created with owner-only permissions. +func writeFileAtomic(path string, payload []byte) error { + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + + tempFile, err := os.CreateTemp(dir, filepath.Base(path)+".*.tmp") + if err != nil { + return err + } + tempPath := tempFile.Name() + defer func() { _ = os.Remove(tempPath) }() + + if _, err := tempFile.Write(payload); err != nil { + _ = tempFile.Close() + return err + } + if err := tempFile.Close(); err != nil { + return err + } + return os.Rename(tempPath, path) +} diff --git a/backend/mtproto/mtproto_test.go b/backend/mtproto/mtproto_test.go new file mode 100644 index 00000000..0a4999f8 --- /dev/null +++ b/backend/mtproto/mtproto_test.go @@ -0,0 +1,303 @@ +package mtproto + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "strings" + "testing" + + "github.com/pasarguard/node/common" +) + +const testSecret = "00112233445566778899aabbccddeeff" + +func mtUser(email, tag, secret string) *common.User { + return &common.User{ + Email: email, + Inbounds: []string{tag}, + Proxies: &common.Proxy{Mtproto: &common.Mtproto{Secret: secret}}, + } +} + +func TestNewConfig_InboundTagAndValidation(t *testing.T) { + // default inbound tag + cfg, err := NewConfig(`{"server":{"port":2222}}`) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.InboundTag != defaultInboundTag { + t.Errorf("default inbound tag = %q, want %q", cfg.InboundTag, defaultInboundTag) + } + if _, ok := cfg.base["inbound_tag"]; ok { + t.Errorf("inbound_tag must be stripped from base config") + } + + // custom inbound tag + cfg, err = NewConfig(`{"inbound_tag":"mtp-eu","server":{"port":2222}}`) + if err != nil { + t.Fatalf("NewConfig returned error: %v", err) + } + if cfg.InboundTag != "mtp-eu" { + t.Errorf("custom inbound tag = %q, want %q", cfg.InboundTag, "mtp-eu") + } + + // panel must not set managed access keys + if _, err := NewConfig(`{"access":{"users":{"a":"x"}}}`); err == nil { + t.Errorf("expected error when panel sets access.users") + } + if _, err := NewConfig(`{"access":{"user_enabled":{"a":false}}}`); err == nil { + t.Errorf("expected error when panel sets access.user_enabled") + } + + // empty / invalid JSON + if _, err := NewConfig(" "); err == nil { + t.Errorf("expected error for empty config") + } + if _, err := NewConfig(`{`); err == nil { + t.Errorf("expected error for invalid JSON") + } + if _, err := NewConfig(`{} {}`); err == nil { + t.Errorf("expected error for trailing JSON object") + } +} + +func TestActiveUserFromCommon_Selection(t *testing.T) { + // not tagged -> excluded, no error + if _, include, err := activeUserFromCommon(mtUser("bob", "other", testSecret), "mtproto"); err != nil || include { + t.Errorf("untagged user: include=%v err=%v, want include=false err=nil", include, err) + } + + // tagged + valid secret -> included, lowercased + u, include, err := activeUserFromCommon(mtUser("alice", "mtproto", strings.ToUpper(testSecret)), "mtproto") + if err != nil || !include { + t.Fatalf("valid user not included: include=%v err=%v", include, err) + } + if u.Username != "alice" || u.Secret != testSecret { + t.Errorf("mapped user = %+v, want username=alice secret lowercased", u) + } + + // tagged but missing mtproto proxy -> error + bad := &common.User{Email: "c", Inbounds: []string{"mtproto"}, Proxies: &common.Proxy{}} + if _, _, err := activeUserFromCommon(bad, "mtproto"); err == nil { + t.Errorf("expected error for missing proxies.mtproto") + } + + // invalid secret length -> error + if _, _, err := activeUserFromCommon(mtUser("d", "mtproto", "abc"), "mtproto"); err == nil { + t.Errorf("expected error for short secret") + } + + // invalid username -> error + if _, _, err := activeUserFromCommon(mtUser("has space", "mtproto", testSecret), "mtproto"); err == nil { + t.Errorf("expected error for invalid username") + } +} + +func TestConfigRender_InjectsManagedSections(t *testing.T) { + cfg, err := NewConfig(`{"server":{"port":2222},"censorship":{"tls_domain":"example.com"}}`) + if err != nil { + t.Fatal(err) + } + users := map[string]*runtimeUser{"alice": {Username: "alice", Secret: testSecret}} + out, err := cfg.Render(users, nil, 12345, 23456, "Bearer testtoken") + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "enabled = true", + `listen = "127.0.0.1:12345"`, + `auth_header = "Bearer testtoken"`, + `metrics_port = 23456`, + `metrics_listen = "127.0.0.1:23456"`, + "[access.users]", + `alice = "` + testSecret + `"`, + `tls_domain = "example.com"`, + } { + if !strings.Contains(out, want) { + t.Errorf("rendered config missing %q\n---\n%s", want, out) + } + } + if strings.Contains(out, "user_enabled") { + t.Errorf("real users must not disable anyone via user_enabled") + } + + // empty users + sentinel keeps telemt happy (telemt rejects empty [access.users]) + sentinel := &runtimeUser{Username: "pg-node-internal-x", Secret: testSecret} + out, err = cfg.Render(map[string]*runtimeUser{}, sentinel, 1, 2, "") + if err != nil { + t.Fatalf("render with sentinel failed: %v", err) + } + if !strings.Contains(out, "pg-node-internal-x") { + t.Errorf("sentinel user missing from rendered config") + } + if !strings.Contains(out, "[access.user_enabled]") || !strings.Contains(out, "pg-node-internal-x = false") { + t.Errorf("sentinel must be rendered disabled, got:\n%s", out) + } + + // empty users + no sentinel -> error (would crash telemt) + if _, err := cfg.Render(map[string]*runtimeUser{}, nil, 1, 2, ""); err == nil { + t.Errorf("expected error rendering with zero users and no sentinel") + } +} + +func TestTracker_DeltaResetAndCounterRestart(t *testing.T) { + tr := newTrafficTracker() + active := map[string]struct{}{"alice": {}} + + // telemt is always started fresh by the node, so counters begin at 0: + // the first sample establishes the baseline and yields no delta. + tr.Sync(map[string]userMetrics{"alice": {Uplink: 0, Downlink: 0}}, active) + if got := tr.UsersStats("mtproto", false); len(got.Stats) != 0 { + t.Fatalf("first sample should yield zero delta, got %+v", got.Stats) + } + + // growth -> unreported pending delta (read without reset) + tr.Sync(map[string]userMetrics{"alice": {Uplink: 50, Downlink: 60}}, active) + assertDelta(t, tr.UsersStats("mtproto", false), 50, 60) + + // counter restart (telemt restarted -> counters drop): unreported 50/60 must be + // PRESERVED, not lost, and never negative. + tr.Sync(map[string]userMetrics{"alice": {Uplink: 5, Downlink: 3}}, active) + assertDelta(t, tr.UsersStats("mtproto", false), 50, 60) + + // growth after restart accumulates on top of the preserved pending total + tr.Sync(map[string]userMetrics{"alice": {Uplink: 25, Downlink: 13}}, active) + assertDelta(t, tr.UsersStats("mtproto", true), 70, 70) // 50 preserved + (25-5); 60 + (13-3) + + // after reset with no new growth -> empty + if got := tr.UsersStats("mtproto", false); len(got.Stats) != 0 { + t.Fatalf("after reset with no growth, want empty, got %+v", got.Stats) + } + + // normal post-reset growth + tr.Sync(map[string]userMetrics{"alice": {Uplink: 35, Downlink: 20}}, active) + assertDelta(t, tr.UsersStats("mtproto", false), 10, 7) +} + +func assertDelta(t *testing.T, resp *common.StatResponse, wantUp, wantDown int64) { + t.Helper() + var up, down int64 + for _, s := range resp.GetStats() { + switch s.GetType() { + case "uplink": + up = s.GetValue() + case "downlink": + down = s.GetValue() + } + } + if up != wantUp || down != wantDown { + t.Errorf("delta = up:%d down:%d, want up:%d down:%d", up, down, wantUp, wantDown) + } +} + +func TestParseMetrics(t *testing.T) { + body := `# HELP telemt_user_octets_from_client Per-user bytes received +# TYPE telemt_user_octets_from_client counter +telemt_user_octets_from_client{user="alice"} 1024 +telemt_user_octets_to_client{user="alice"} 2048 +telemt_user_connections_current{user="alice"} 3 +telemt_user_unique_ips_current{user="alice"} 2 +telemt_stats_user_entries 1 +telemt_user_octets_from_client{user="bob"} 7 +` + got, err := parseMetrics(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + alice := got["alice"] + if alice.Uplink != 1024 || alice.Downlink != 2048 || alice.ConnectionsCurrent != 3 || alice.UniqueIPsCurrent != 2 { + t.Errorf("alice metrics = %+v", alice) + } + if got["bob"].Uplink != 7 { + t.Errorf("bob uplink = %d, want 7", got["bob"].Uplink) + } +} + +func TestGetOutboundsLatency_Empty(t *testing.T) { + m := &MTProto{} + resp, err := m.GetOutboundsLatency(context.Background(), &common.LatencyRequest{}) + if err != nil { + t.Fatal(err) + } + if resp == nil || len(resp.GetLatencies()) != 0 { + t.Errorf("expected empty latency response, got %+v", resp) + } +} + +func TestBuildPatchUserRequest_ClearsOptionalFields(t *testing.T) { + payload, err := buildPatchUserRequest(&runtimeUser{ + Username: "alice", + Secret: testSecret, + }) + if err != nil { + t.Fatal(err) + } + encoded, err := json.Marshal(payload) + if err != nil { + t.Fatal(err) + } + got := string(encoded) + for _, want := range []string{ + `"secret":"` + testSecret + `"`, + `"user_ad_tag":null`, + `"max_tcp_conns":null`, + `"max_unique_ips":null`, + } { + if !strings.Contains(got, want) { + t.Errorf("patch JSON missing %s: %s", want, got) + } + } + + tag := "aabbccddeeff00112233445566778899" + setPayload, err := buildPatchUserRequest(&runtimeUser{ + Username: "alice", + Secret: testSecret, + UserAdTag: tag, + MaxTCPConns: 4, + MaxUniqueIPs: 2, + }) + if err != nil { + t.Fatal(err) + } + if setPayload.UserAdTag == nil || *setPayload.UserAdTag != tag { + t.Errorf("user_ad_tag = %v, want %s", setPayload.UserAdTag, tag) + } + if setPayload.MaxTCPConns == nil || *setPayload.MaxTCPConns != 4 { + t.Errorf("max_tcp_conns = %v, want 4", setPayload.MaxTCPConns) + } + if setPayload.MaxUniqueIPs == nil || *setPayload.MaxUniqueIPs != 2 { + t.Errorf("max_unique_ips = %v, want 2", setPayload.MaxUniqueIPs) + } +} + +func TestGetStats_MetricsUnavailable(t *testing.T) { + m := metricsUnavailableBackend() + + stats, err := m.GetStats(context.Background(), &common.StatRequest{Type: common.StatType_UsersStat}) + if err != nil { + t.Fatalf("GetStats should be best-effort, got %v", err) + } + if stats == nil || len(stats.GetStats()) != 0 { + t.Errorf("expected empty stats, got %+v", stats) + } + + online, err := m.GetUserOnlineStats(context.Background(), "alice") + if err != nil { + t.Fatalf("GetUserOnlineStats should be best-effort, got %v", err) + } + if online == nil || online.GetName() != "alice" || online.GetValue() != 0 { + t.Errorf("expected zero online count, got %+v", online) + } +} + +func metricsUnavailableBackend() *MTProto { + return &MTProto{ + config: &Config{InboundTag: "mtproto"}, + process: &processManager{cmd: &exec.Cmd{Process: &os.Process{Pid: 1}}}, + metricsScraper: newMetricsScraper("http://127.0.0.1:1/metrics"), + tracker: newTrafficTracker(), + desired: map[string]*runtimeUser{"alice": {Username: "alice", Secret: testSecret}}, + } +} diff --git a/backend/mtproto/process.go b/backend/mtproto/process.go new file mode 100644 index 00000000..89a20e6f --- /dev/null +++ b/backend/mtproto/process.go @@ -0,0 +1,229 @@ +package mtproto + +import ( + "bufio" + "context" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/pasarguard/node/pkg/logstream" +) + +// telemtCommandTimeout bounds the short-lived telemt invocations (--version, +// reload) so a wedged binary cannot block startup or a live reload forever. +const telemtCommandTimeout = 30 * time.Second + +type commandFactory func(executable string, args ...string) *exec.Cmd + +type processManager struct { + executablePath string + configPath string + pidFilePath string + logs *logstream.Buffer + version string + commandFactory commandFactory + + mu sync.RWMutex + cmd *exec.Cmd + waitCh chan error +} + +func newProcessManager(executablePath, configPath, pidFilePath string, logBufferSize int) (*processManager, error) { + if strings.TrimSpace(executablePath) == "" { + return nil, fmt.Errorf("telemt executable path must not be empty") + } + + absExecutablePath, err := filepath.Abs(executablePath) + if err != nil { + return nil, err + } + absConfigPath, err := filepath.Abs(configPath) + if err != nil { + return nil, err + } + absPIDFilePath, err := filepath.Abs(pidFilePath) + if err != nil { + return nil, err + } + + manager := &processManager{ + executablePath: absExecutablePath, + configPath: absConfigPath, + pidFilePath: absPIDFilePath, + logs: logstream.NewBuffer(logBufferSize), + commandFactory: func(executable string, args ...string) *exec.Cmd { + return exec.Command(executable, args...) + }, + } + + manager.version, err = manager.detectVersion() + if err != nil { + return nil, err + } + + return manager, nil +} + +func (p *processManager) detectVersion() (string, error) { + ctx, cancel := context.WithTimeout(context.Background(), telemtCommandTimeout) + defer cancel() + + cmd := exec.CommandContext(ctx, p.executablePath, "--version") + output, err := cmd.CombinedOutput() + if err != nil { + if ctx.Err() == context.DeadlineExceeded { + return "", fmt.Errorf("telemt --version timed out after %s: %w", telemtCommandTimeout, ctx.Err()) + } + return "", fmt.Errorf("failed to detect telemt version: %w", err) + } + + line := strings.TrimSpace(string(output)) + fields := strings.Fields(line) + if len(fields) >= 2 && strings.EqualFold(fields[0], "telemt") { + return fields[1], nil + } + if line == "" { + return "", fmt.Errorf("failed to detect telemt version: empty output") + } + return line, nil +} + +func (p *processManager) Version() string { + p.mu.RLock() + defer p.mu.RUnlock() + return p.version +} + +func (p *processManager) SubscribeLogs(ctx context.Context) <-chan string { + return p.logs.Subscribe(ctx) +} + +func (p *processManager) Start() error { + p.mu.Lock() + defer p.mu.Unlock() + + if p.startedLocked() { + return fmt.Errorf("telemt process is already running") + } + + if err := os.MkdirAll(filepath.Dir(p.configPath), 0o755); err != nil { + return fmt.Errorf("failed to create telemt config directory: %w", err) + } + if err := os.MkdirAll(filepath.Dir(p.pidFilePath), 0o755); err != nil { + return fmt.Errorf("failed to create telemt pid directory: %w", err) + } + + cmd := p.commandFactory(p.executablePath, "run", "--pid-file", p.pidFilePath, p.configPath) + // Pin telemt's working directory to the generated-config dir so its + // relative-path artifacts (e.g. the tls_front cache) stay contained there + // instead of littering the node's working directory. + cmd.Dir = filepath.Dir(p.configPath) + setProcessAttributes(cmd) + + stdout, err := cmd.StdoutPipe() + if err != nil { + return err + } + stderr, err := cmd.StderrPipe() + if err != nil { + return err + } + + if err := cmd.Start(); err != nil { + return err + } + + waitCh := make(chan error, 1) + go func() { + waitCh <- cmd.Wait() + close(waitCh) + }() + + go p.captureLogs(stdout) + go p.captureLogs(stderr) + + p.cmd = cmd + p.waitCh = waitCh + return nil +} + +func (p *processManager) WaitChan() <-chan error { + p.mu.RLock() + defer p.mu.RUnlock() + return p.waitCh +} + +func (p *processManager) PID() int32 { + p.mu.RLock() + defer p.mu.RUnlock() + if p.cmd == nil || p.cmd.Process == nil { + return 0 + } + return int32(p.cmd.Process.Pid) +} + +func (p *processManager) Started() bool { + p.mu.RLock() + defer p.mu.RUnlock() + return p.startedLocked() +} + +func (p *processManager) startedLocked() bool { + return p.cmd != nil && p.cmd.Process != nil && p.cmd.ProcessState == nil +} + +func (p *processManager) Reload() error { + ctx, cancel := context.WithTimeout(context.Background(), telemtCommandTimeout) + defer cancel() + + cmd := exec.CommandContext(ctx, p.executablePath, "reload", "--pid-file", p.pidFilePath) + output, err := cmd.CombinedOutput() + if err != nil { + if ctx.Err() == context.DeadlineExceeded { + return fmt.Errorf("telemt reload timed out after %s: %w", telemtCommandTimeout, ctx.Err()) + } + return fmt.Errorf("failed to reload telemt: %w (%s)", err, strings.TrimSpace(string(output))) + } + return nil +} + +func (p *processManager) Shutdown() { + p.mu.Lock() + cmd := p.cmd + waitCh := p.waitCh + p.cmd = nil + p.waitCh = nil + p.mu.Unlock() + + if cmd == nil { + return + } + + _ = terminateProcess(cmd) + if waitCh != nil { + select { + case <-waitCh: + case <-time.After(5 * time.Second): + _ = killProcess(cmd) + <-waitCh + } + } +} + +func (p *processManager) Restart() error { + p.Shutdown() + return p.Start() +} + +func (p *processManager) captureLogs(pipe io.Reader) { + scanner := bufio.NewScanner(pipe) + for scanner.Scan() { + p.logs.Publish(scanner.Text()) + } +} diff --git a/backend/mtproto/process_test.go b/backend/mtproto/process_test.go new file mode 100644 index 00000000..81378721 --- /dev/null +++ b/backend/mtproto/process_test.go @@ -0,0 +1,70 @@ +//go:build !windows + +package mtproto + +import ( + "context" + "os/exec" + "path/filepath" + "testing" + "time" + + "github.com/pasarguard/node/pkg/logstream" +) + +// TestProcessManager_LogsReachSubscribersAndCloseCleanly runs a stand-in for +// telemt that writes to stdout and stderr, then shuts it down and closes the +// log buffer while a subscriber is attached. Neither step may panic, and the +// subscriber channel must be closed afterwards. +func TestProcessManager_LogsReachSubscribersAndCloseCleanly(t *testing.T) { + sh, err := exec.LookPath("sh") + if err != nil { + t.Skip("sh not available") + } + + dir := t.TempDir() + p := &processManager{ + executablePath: sh, + configPath: filepath.Join(dir, "config.toml"), + pidFilePath: filepath.Join(dir, "telemt.pid"), + logs: logstream.NewBuffer(16), + commandFactory: func(executable string, _ ...string) *exec.Cmd { + return exec.Command(executable, "-c", "echo out-line; echo err-line >&2; exec sleep 30") + }, + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + logs := p.SubscribeLogs(ctx) + + if err := p.Start(); err != nil { + t.Fatalf("Start: %v", err) + } + + seen := map[string]bool{} + deadline := time.After(5 * time.Second) + for !seen["out-line"] || !seen["err-line"] { + select { + case line := <-logs: + seen[line] = true + case <-deadline: + t.Fatalf("timed out waiting for stdout and stderr lines, got %v", seen) + } + } + + p.Shutdown() + p.logs.Close() + + select { + case _, ok := <-logs: + if ok { + t.Fatal("expected the subscriber channel to be closed after Close") + } + case <-time.After(5 * time.Second): + t.Fatal("subscriber channel was not closed after Close") + } + + if _, ok := <-p.SubscribeLogs(context.Background()); ok { + t.Fatal("expected a closed channel when subscribing after Close") + } +} diff --git a/backend/mtproto/process_unix.go b/backend/mtproto/process_unix.go new file mode 100644 index 00000000..83e0e395 --- /dev/null +++ b/backend/mtproto/process_unix.go @@ -0,0 +1,35 @@ +//go:build !windows + +package mtproto + +import ( + "os/exec" + "syscall" +) + +func setProcessAttributes(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setpgid: true, + Pgid: 0, + } +} + +func terminateProcess(cmd *exec.Cmd) error { + if cmd == nil || cmd.Process == nil { + return nil + } + + if err := syscall.Kill(-cmd.Process.Pid, syscall.SIGTERM); err == nil { + return nil + } + return cmd.Process.Signal(syscall.SIGTERM) +} + +func killProcess(cmd *exec.Cmd) error { + if cmd == nil || cmd.Process == nil { + return nil + } + + _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) + return cmd.Process.Kill() +} diff --git a/backend/mtproto/process_windows.go b/backend/mtproto/process_windows.go new file mode 100644 index 00000000..ab7d3ac8 --- /dev/null +++ b/backend/mtproto/process_windows.go @@ -0,0 +1,28 @@ +//go:build windows + +package mtproto + +import ( + "os/exec" + "syscall" +) + +func setProcessAttributes(cmd *exec.Cmd) { + cmd.SysProcAttr = &syscall.SysProcAttr{ + CreationFlags: syscall.CREATE_NEW_PROCESS_GROUP, + } +} + +func terminateProcess(cmd *exec.Cmd) error { + if cmd == nil || cmd.Process == nil { + return nil + } + return cmd.Process.Kill() +} + +func killProcess(cmd *exec.Cmd) error { + if cmd == nil || cmd.Process == nil { + return nil + } + return cmd.Process.Kill() +} diff --git a/backend/mtproto/toml.go b/backend/mtproto/toml.go new file mode 100644 index 00000000..072ddbe1 --- /dev/null +++ b/backend/mtproto/toml.go @@ -0,0 +1,234 @@ +package mtproto + +import ( + "encoding/json" + "fmt" + "reflect" + "slices" + "strconv" + "strings" +) + +type tomlLiteral string + +func renderTOML(root map[string]any) (string, error) { + var builder strings.Builder + if err := writeTOMLSection(&builder, nil, root, tomlHeaderNone); err != nil { + return "", err + } + return builder.String(), nil +} + +type tomlHeaderKind uint8 + +const ( + tomlHeaderNone tomlHeaderKind = iota + tomlHeaderTable + tomlHeaderArrayTable +) + +func writeTOMLSection(builder *strings.Builder, path []string, section map[string]any, headerKind tomlHeaderKind) error { + scalarKeys, tableKeys, arrayTableKeys, err := classifyTOMLSection(section) + if err != nil { + return err + } + + if headerKind != tomlHeaderNone { + writeSectionSpacing(builder) + headerText := joinPath(path) + switch headerKind { + case tomlHeaderTable: + builder.WriteString("[") + builder.WriteString(headerText) + builder.WriteString("]\n") + case tomlHeaderArrayTable: + builder.WriteString("[[") + builder.WriteString(headerText) + builder.WriteString("]]\n") + } + } + + for _, key := range scalarKeys { + inline, err := renderInlineValue(section[key]) + if err != nil { + return fmt.Errorf("failed to render TOML value for %s: %w", joinPath(appendPath(path, key)), err) + } + builder.WriteString(renderKey(key)) + builder.WriteString(" = ") + builder.WriteString(inline) + builder.WriteByte('\n') + } + + for _, key := range tableKeys { + child, _ := section[key].(map[string]any) + if err := writeTOMLSection(builder, appendPath(path, key), child, tomlHeaderTable); err != nil { + return err + } + } + + for _, key := range arrayTableKeys { + items, _ := section[key].([]any) + for index, rawItem := range items { + child, ok := rawItem.(map[string]any) + if !ok { + return fmt.Errorf("failed to render TOML array table %s[%d]: expected object element", joinPath(appendPath(path, key)), index) + } + if err := writeTOMLSection(builder, appendPath(path, key), child, tomlHeaderArrayTable); err != nil { + return err + } + } + } + + return nil +} + +func classifyTOMLSection(section map[string]any) ([]string, []string, []string, error) { + keys := make([]string, 0, len(section)) + for key := range section { + keys = append(keys, key) + } + slices.Sort(keys) + + scalarKeys := make([]string, 0, len(keys)) + tableKeys := make([]string, 0, len(keys)) + arrayTableKeys := make([]string, 0) + + for _, key := range keys { + switch value := section[key].(type) { + case map[string]any: + tableKeys = append(tableKeys, key) + case []any: + if isArrayOfObjects(value) { + arrayTableKeys = append(arrayTableKeys, key) + continue + } + scalarKeys = append(scalarKeys, key) + default: + scalarKeys = append(scalarKeys, key) + } + } + + return scalarKeys, tableKeys, arrayTableKeys, nil +} + +func isArrayOfObjects(values []any) bool { + if len(values) == 0 { + return false + } + for _, value := range values { + if _, ok := value.(map[string]any); !ok { + return false + } + } + return true +} + +func renderInlineValue(value any) (string, error) { + switch typed := value.(type) { + case nil: + return "", fmt.Errorf("nil values are not valid in TOML") + case tomlLiteral: + return string(typed), nil + case string: + return strconv.Quote(typed), nil + case bool: + return strconv.FormatBool(typed), nil + case json.Number: + return typed.String(), nil + case []any: + rendered := make([]string, 0, len(typed)) + for _, item := range typed { + inline, err := renderInlineValue(item) + if err != nil { + return "", err + } + rendered = append(rendered, inline) + } + return "[" + strings.Join(rendered, ", ") + "]", nil + case map[string]any: + keys := make([]string, 0, len(typed)) + for key := range typed { + keys = append(keys, key) + } + slices.Sort(keys) + + parts := make([]string, 0, len(keys)) + for _, key := range keys { + inline, err := renderInlineValue(typed[key]) + if err != nil { + return "", err + } + parts = append(parts, fmt.Sprintf("%s = %s", renderKey(key), inline)) + } + return "{ " + strings.Join(parts, ", ") + " }", nil + default: + return renderPrimitiveValue(typed) + } +} + +func renderPrimitiveValue(value any) (string, error) { + typedValue := reflect.ValueOf(value) + switch typedValue.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return strconv.FormatInt(typedValue.Int(), 10), nil + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return strconv.FormatUint(typedValue.Uint(), 10), nil + case reflect.Float32: + return strconv.FormatFloat(typedValue.Float(), 'f', -1, 32), nil + case reflect.Float64: + return strconv.FormatFloat(typedValue.Float(), 'f', -1, 64), nil + default: + return "", fmt.Errorf("unsupported TOML value type %T", value) + } +} + +func renderKey(key string) string { + if isBareKey(key) { + return key + } + return strconv.Quote(key) +} + +func isBareKey(key string) bool { + if key == "" { + return false + } + for _, r := range key { + if !(r >= 'a' && r <= 'z') && + !(r >= 'A' && r <= 'Z') && + !(r >= '0' && r <= '9') && + r != '_' && + r != '-' { + return false + } + } + return true +} + +func joinPath(path []string) string { + if len(path) == 0 { + return "" + } + parts := make([]string, 0, len(path)) + for _, segment := range path { + parts = append(parts, renderKey(segment)) + } + return strings.Join(parts, ".") +} + +func appendPath(path []string, key string) []string { + result := make([]string, 0, len(path)+1) + result = append(result, path...) + result = append(result, key) + return result +} + +func writeSectionSpacing(builder *strings.Builder) { + if builder.Len() == 0 { + return + } + if !strings.HasSuffix(builder.String(), "\n") { + builder.WriteByte('\n') + } + builder.WriteByte('\n') +} diff --git a/backend/mtproto/tracker.go b/backend/mtproto/tracker.go new file mode 100644 index 00000000..249d7303 --- /dev/null +++ b/backend/mtproto/tracker.go @@ -0,0 +1,204 @@ +package mtproto + +import ( + "sync" + + "github.com/pasarguard/node/common" +) + +type trafficTracker struct { + mu sync.RWMutex + entries map[string]*trafficEntry +} + +type trafficEntry struct { + CurrentUplink int64 + CurrentDownlink int64 + BaseUplink int64 + BaseDownlink int64 + LastDeltaUplink int64 + LastDeltaDown int64 + Deleted bool +} + +func newTrafficTracker() *trafficTracker { + return &trafficTracker{ + entries: make(map[string]*trafficEntry), + } +} + +func (t *trafficTracker) Sync(snapshot map[string]userMetrics, activeUsers map[string]struct{}) { + t.mu.Lock() + defer t.mu.Unlock() + + for username := range activeUsers { + counters := snapshot[username] + t.applySampleLocked(username, counters) + } + + for username, entry := range t.entries { + if _, ok := activeUsers[username]; ok { + entry.Deleted = false + continue + } + if entry.Deleted { + continue + } + entry.LastDeltaUplink = entry.CurrentUplink - entry.BaseUplink + entry.LastDeltaDown = entry.CurrentDownlink - entry.BaseDownlink + entry.Deleted = true + } +} + +func (t *trafficTracker) applySampleLocked(username string, counters userMetrics) { + entry, exists := t.entries[username] + if !exists { + t.entries[username] = &trafficEntry{ + CurrentUplink: counters.Uplink, + CurrentDownlink: counters.Downlink, + } + return + } + + readded := false + if entry.Deleted { + pendingUplink := entry.LastDeltaUplink + pendingDownlink := entry.LastDeltaDown + entry.BaseUplink = counters.Uplink - pendingUplink + entry.BaseDownlink = counters.Downlink - pendingDownlink + entry.LastDeltaUplink = 0 + entry.LastDeltaDown = 0 + entry.Deleted = false + readded = true + } + + if !readded && (counters.Uplink < entry.CurrentUplink || counters.Downlink < entry.CurrentDownlink) { + pendingUplink := entry.CurrentUplink - entry.BaseUplink + if pendingUplink < 0 { + pendingUplink = 0 + } + pendingDownlink := entry.CurrentDownlink - entry.BaseDownlink + if pendingDownlink < 0 { + pendingDownlink = 0 + } + entry.BaseUplink = counters.Uplink - pendingUplink + entry.BaseDownlink = counters.Downlink - pendingDownlink + } + + entry.CurrentUplink = counters.Uplink + entry.CurrentDownlink = counters.Downlink +} + +func (t *trafficTracker) UserStats(username, link string, reset bool) *common.StatResponse { + if reset { + t.mu.Lock() + defer t.mu.Unlock() + } else { + t.mu.RLock() + defer t.mu.RUnlock() + } + + response := &common.StatResponse{Stats: []*common.Stat{}} + entry, exists := t.entries[username] + if !exists { + return response + } + + response.Stats = append(response.Stats, buildTrafficStats(username, link, entry.deltaUplink(), entry.deltaDownlink())...) + if reset { + t.resetEntryLocked(username, entry) + } + return response +} + +func (t *trafficTracker) UsersStats(link string, reset bool) *common.StatResponse { + if reset { + t.mu.Lock() + defer t.mu.Unlock() + } else { + t.mu.RLock() + defer t.mu.RUnlock() + } + + response := &common.StatResponse{Stats: make([]*common.Stat, 0, len(t.entries)*2)} + for username, entry := range t.entries { + response.Stats = append(response.Stats, buildTrafficStats(username, link, entry.deltaUplink(), entry.deltaDownlink())...) + if reset { + t.resetEntryLocked(username, entry) + } + } + return response +} + +func (t *trafficTracker) AggregateStats(name, link string, reset bool) *common.StatResponse { + if reset { + t.mu.Lock() + defer t.mu.Unlock() + } else { + t.mu.RLock() + defer t.mu.RUnlock() + } + + var totalUplink int64 + var totalDownlink int64 + for username, entry := range t.entries { + totalUplink += entry.deltaUplink() + totalDownlink += entry.deltaDownlink() + if reset { + t.resetEntryLocked(username, entry) + } + } + + return &common.StatResponse{ + Stats: buildTrafficStats(name, link, totalUplink, totalDownlink), + } +} + +func (t *trafficTracker) resetEntryLocked(username string, entry *trafficEntry) { + entry.BaseUplink = entry.CurrentUplink + entry.BaseDownlink = entry.CurrentDownlink + if entry.Deleted { + delete(t.entries, username) + } +} + +func (e *trafficEntry) deltaUplink() int64 { + delta := e.CurrentUplink - e.BaseUplink + if delta < 0 { + return 0 + } + return delta +} + +func (e *trafficEntry) deltaDownlink() int64 { + delta := e.CurrentDownlink - e.BaseDownlink + if delta < 0 { + return 0 + } + return delta +} + +func buildTrafficStats(name, link string, uplink, downlink int64) []*common.Stat { + if uplink == 0 && downlink == 0 { + return nil + } + + stats := make([]*common.Stat, 0, 2) + if uplink > 0 { + stats = append(stats, &common.Stat{ + Name: name, + Type: "uplink", + Link: link, + Value: uplink, + }) + } + if downlink > 0 { + stats = append(stats, &common.Stat{ + Name: name, + Type: "downlink", + Link: link, + Value: downlink, + }) + } + return stats +} diff --git a/config/config.go b/config/config.go index b69f0dc3..6be786ac 100644 --- a/config/config.go +++ b/config/config.go @@ -17,6 +17,7 @@ type Config struct { ServicePort int NodeHost string XrayExecutablePath string + TelemtExecutablePath string XrayAssetsPath string SslCertFile string SslKeyFile string @@ -47,6 +48,7 @@ func Load() (*Config, error) { cfg := &Config{ ServicePort: GetEnvAsInt("SERVICE_PORT", 62050), XrayExecutablePath: GetEnv("XRAY_EXECUTABLE_PATH", "/usr/local/bin/xray"), + TelemtExecutablePath: GetEnv("TELEMT_EXECUTABLE_PATH", "/usr/local/bin/telemt"), XrayAssetsPath: GetEnv("XRAY_ASSETS_PATH", "/usr/local/share/xray"), SslCertFile: GetEnv("SSL_CERT_FILE", "/var/lib/pg-node/certs/ssl_cert.pem"), SslKeyFile: GetEnv("SSL_KEY_FILE", "/var/lib/pg-node/certs/ssl_key.pem"), diff --git a/controller/controller.go b/controller/controller.go index 9b6aac94..530fa4eb 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -12,6 +12,7 @@ import ( "google.golang.org/protobuf/proto" "github.com/pasarguard/node/backend" + "github.com/pasarguard/node/backend/mtproto" "github.com/pasarguard/node/backend/wireguard" "github.com/pasarguard/node/backend/xray" "github.com/pasarguard/node/common" @@ -311,6 +312,16 @@ func (c *Controller) StartBackend(ctx context.Context, backend *common.Backend) return err } c.backend = newBackend + case common.BackendType_MTPROTO: + config, err := mtproto.NewConfig(backend.GetConfig()) + if err != nil { + return err + } + newBackend, err := mtproto.New(ctx, c.cfg, config, backend.GetUsers()) + if err != nil { + return err + } + c.backend = newBackend default: return errors.New("invalid backend type") } From ca7c724af2321e6bb46fb32ee474e94ade8b326f Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 19 Oct 2026 19:08:24 +0330 Subject: [PATCH 03/40] fix(mtproto): harden rendered telemt config and skip invalid users Config: - Reject panel JSON that sets node-managed or host-reaching telemt keys: include, web, server.api, server.admin_api, server.metrics_port, server.metrics_listen, server.metrics_whitelist, access.user_rate_limits and access.user_source_deny (next to the existing restricted per-user access maps). - Validate object sections and ports (1-65535, integers) and add Config.ListenPorts (server.port, default 443, plus listener ports). - Render forces general.disable_colors, general.telemetry.user_enabled and server.api.read_only=false, and fails when a listen port equals the node's telemt API or metrics port. TOML: - Render strings and keys as TOML basic strings (\uXXXX for control characters) instead of strconv.Quote, whose \a, \v and \x escapes are not TOML. - Drop null object fields; a null array element stays an error. - Always quote a key named include: telemt expands any `include = "..."` line as a file include before parsing. Users: - ActiveUsersFromCommon logs and skips invalid users and returns their emails, so one bad user no longer fails Start or a full sync. Partial updates treat an invalid user as not included. SyncUser still returns the validation error for its single user. Tests use github.com/pelletier/go-toml v1.9.5 (already in go.sum) as an independent TOML decoder; it is now a direct requirement. --- backend/mtproto/config.go | 206 +++++++++++++++-- backend/mtproto/config_test.go | 379 ++++++++++++++++++++++++++++++++ backend/mtproto/mtproto.go | 50 ++--- backend/mtproto/mtproto_test.go | 121 ---------- backend/mtproto/toml.go | 51 ++++- backend/mtproto/toml_test.go | 153 +++++++++++++ go.mod | 1 + 7 files changed, 797 insertions(+), 164 deletions(-) create mode 100644 backend/mtproto/config_test.go create mode 100644 backend/mtproto/toml_test.go diff --git a/backend/mtproto/config.go b/backend/mtproto/config.go index 0176c774..e84eb365 100644 --- a/backend/mtproto/config.go +++ b/backend/mtproto/config.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "log" "regexp" "slices" "strings" @@ -16,6 +17,8 @@ const ( defaultInboundTag = "mtproto" loopbackV4CIDR = "127.0.0.1/32" loopbackV6CIDR = "::1/128" + // telemtDefaultPort is the server.port telemt uses when the config omits it. + telemtDefaultPort = 443 ) var ( @@ -29,8 +32,33 @@ var ( "user_max_unique_ips", "user_enabled", } + // rejectedConfigKeys are telemt config paths the panel JSON must not set: + // the node manages them, or they would reach past the node. + rejectedConfigKeys = []rejectedConfigKey{ + {path: []string{"include"}, reason: "telemt would read other files on the node host"}, + {path: []string{"web"}, reason: "the web transport can serve files from the node host"}, + {path: []string{"server", "api"}, reason: "the node manages the telemt control API"}, + {path: []string{"server", "admin_api"}, reason: "it is an alias of server.api, which the node manages"}, + {path: []string{"server", "metrics_port"}, reason: "the node manages the metrics endpoint"}, + {path: []string{"server", "metrics_listen"}, reason: "the node manages the metrics endpoint"}, + {path: []string{"server", "metrics_whitelist"}, reason: "the node manages the metrics endpoint"}, + {path: []string{"access", "user_rate_limits"}, reason: "per-user maps are keyed by node-managed usernames"}, + {path: []string{"access", "user_source_deny"}, reason: "per-user maps are keyed by node-managed usernames"}, + } + // objectConfigPaths must hold a JSON object when present. + objectConfigPaths = [][]string{ + {"general"}, + {"general", "telemetry"}, + {"server"}, + {"access"}, + } ) +type rejectedConfigKey struct { + path []string + reason string +} + type Config struct { base map[string]any InboundTag string @@ -92,42 +120,148 @@ func NewConfig(raw string) (*Config, error) { return cfg, nil } +// validateConfigRoot checks the panel JSON. A JSON null counts as not set, +// matching the renderer, which drops null fields. func validateConfigRoot(root map[string]any) error { if root == nil { return errors.New("mtproto config must be a JSON object") } - if accessRaw, exists := root["access"]; exists { - access, ok := accessRaw.(map[string]any) - if !ok { - return errors.New("mtproto config field access must be an object") + for _, path := range objectConfigPaths { + if _, err := objectAt(root, path...); err != nil { + return err } - for _, key := range restrictedAccessKeys { - if _, forbidden := access[key]; forbidden { - return fmt.Errorf("mtproto config must not define access.%s; node derives per-user access data", key) - } + } + + access, _ := objectAt(root, "access") + for _, key := range restrictedAccessKeys { + if isSet(access, key) { + return fmt.Errorf("mtproto config must not define access.%s; node derives per-user access data", key) } } - if serverRaw, exists := root["server"]; exists { - if _, ok := serverRaw.(map[string]any); !ok { - return errors.New("mtproto config field server must be an object") + for _, rejected := range rejectedConfigKeys { + last := len(rejected.path) - 1 + parent, err := objectAt(root, rejected.path[:last]...) + if err != nil { + return err + } + if isSet(parent, rejected.path[last]) { + return fmt.Errorf("mtproto config must not define %s: %s", strings.Join(rejected.path, "."), rejected.reason) } } + if _, err := listenPorts(root); err != nil { + return err + } + return nil } +// ListenPorts returns the client-facing TCP ports of the config, sorted and +// without duplicates: server.port (telemt's default 443 when unset) and every +// server.listeners[].port. +func (c *Config) ListenPorts() []int { + if c == nil { + return nil + } + ports, _ := listenPorts(c.base) + return ports +} + +func listenPorts(root map[string]any) ([]int, error) { + server, err := objectAt(root, "server") + if err != nil { + return nil, err + } + + serverPort := telemtDefaultPort + if isSet(server, "port") { + if serverPort, err = portValue("server.port", server["port"]); err != nil { + return nil, err + } + } + ports := []int{serverPort} + + if isSet(server, "listeners") { + listeners, ok := server["listeners"].([]any) + if !ok { + return nil, errors.New("mtproto config field server.listeners must be an array of objects") + } + for index, rawListener := range listeners { + listener, ok := rawListener.(map[string]any) + if !ok { + return nil, fmt.Errorf("mtproto config field server.listeners[%d] must be an object", index) + } + // A listener without a port uses server.port, already listed. + if !isSet(listener, "port") { + continue + } + port, err := portValue(fmt.Sprintf("server.listeners[%d].port", index), listener["port"]) + if err != nil { + return nil, err + } + ports = append(ports, port) + } + } + + slices.Sort(ports) + return slices.Compact(ports), nil +} + +func portValue(field string, value any) (int, error) { + var port int64 + switch typed := value.(type) { + case json.Number: + parsed, err := typed.Int64() + if err != nil { + return 0, fmt.Errorf("mtproto config field %s must be an integer port, got %s", field, typed) + } + port = parsed + case int: + port = int64(typed) + case int64: + port = typed + default: + return 0, fmt.Errorf("mtproto config field %s must be an integer port", field) + } + if port < 1 || port > 65535 { + return 0, fmt.Errorf("mtproto config field %s must be between 1 and 65535, got %d", field, port) + } + return int(port), nil +} + func (c *Config) Render(users map[string]*runtimeUser, sentinel *runtimeUser, apiPort, metricsPort int, apiAuthHeader string) (string, error) { if c == nil { return "", errors.New("mtproto config is nil") } + ports, err := listenPorts(c.base) + if err != nil { + return "", err + } + for _, port := range ports { + switch port { + case apiPort: + return "", fmt.Errorf("mtproto config listen port %d conflicts with the node's telemt API port", port) + case metricsPort: + return "", fmt.Errorf("mtproto config listen port %d conflicts with the node's telemt metrics port", port) + } + } + root := cloneMap(c.base) + general := ensureObject(root, "general") + telemetry := ensureObject(general, "telemetry") server := ensureObject(root, "server") api := ensureObject(server, "api") access := ensureObject(root, "access") + // telemt's output is streamed to the panel log viewer, where ANSI colours + // are noise. Per-user counters feed traffic accounting; with user + // telemetry off telemt reports zero for every user. + general["disable_colors"] = true + telemetry["user_enabled"] = true + // metrics_port enables the Prometheus endpoint; metrics_listen pins it to // loopback. telemt treats metrics_port as the enable flag, so both are set. server["metrics_port"] = metricsPort @@ -135,6 +269,7 @@ func (c *Config) Render(users map[string]*runtimeUser, sentinel *runtimeUser, ap server["metrics_whitelist"] = []any{loopbackV4CIDR, loopbackV6CIDR} api["enabled"] = true + api["read_only"] = false api["listen"] = fmt.Sprintf("127.0.0.1:%d", apiPort) api["auth_header"] = apiAuthHeader api["whitelist"] = []any{loopbackV4CIDR, loopbackV6CIDR} @@ -201,18 +336,36 @@ func writeOptionalMap(target map[string]any, key string, values map[string]any) target[key] = values } -func ActiveUsersFromCommon(users []*common.User, inboundTag string) (map[string]*runtimeUser, error) { +// ActiveUsersFromCommon returns the users that carry inboundTag, keyed by +// username. A tagged user that fails validation is logged and left out, so one +// bad record cannot fail a whole sync; the second value lists their emails. +func ActiveUsersFromCommon(users []*common.User, inboundTag string) (map[string]*runtimeUser, []string) { active := make(map[string]*runtimeUser) + var skipped []string + seenSkipped := make(map[string]struct{}) for _, user := range users { + if user == nil { + continue + } runtimeUser, include, err := activeUserFromCommon(user, inboundTag) if err != nil { - return nil, err + email := strings.TrimSpace(user.GetEmail()) + if _, seen := seenSkipped[email]; !seen { + seenSkipped[email] = struct{}{} + skipped = append(skipped, email) + logSkippedUser(email, err) + } + continue } if include { active[runtimeUser.Username] = runtimeUser } } - return active, nil + return active, skipped +} + +func logSkippedUser(email string, err error) { + log.Printf("mtproto: skipping user %s: %v", email, err) } func activeUserFromCommon(user *common.User, inboundTag string) (*runtimeUser, bool, error) { @@ -253,6 +406,31 @@ func activeUserFromCommon(user *common.User, inboundTag string) (*runtimeUser, b }, true, nil } +// objectAt returns the object at path, or nil when any part of the path is +// absent or null. A part that holds something other than an object is an error. +func objectAt(root map[string]any, path ...string) (map[string]any, error) { + current := root + for index, key := range path { + if current == nil { + return nil, nil + } + raw := current[key] + if raw == nil { + return nil, nil + } + child, ok := raw.(map[string]any) + if !ok { + return nil, fmt.Errorf("mtproto config field %s must be an object", strings.Join(path[:index+1], ".")) + } + current = child + } + return current, nil +} + +func isSet(object map[string]any, key string) bool { + return object != nil && object[key] != nil +} + func ensureObject(target map[string]any, key string) map[string]any { if existing, ok := target[key]; ok { if existingMap, ok := existing.(map[string]any); ok { diff --git a/backend/mtproto/config_test.go b/backend/mtproto/config_test.go new file mode 100644 index 00000000..ab149895 --- /dev/null +++ b/backend/mtproto/config_test.go @@ -0,0 +1,379 @@ +package mtproto + +import ( + "context" + "slices" + "strconv" + "strings" + "testing" + + "github.com/pasarguard/node/common" +) + +func TestNewConfig_InboundTagAndValidation(t *testing.T) { + // default inbound tag + cfg, err := NewConfig(`{"server":{"port":2222}}`) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if cfg.InboundTag != defaultInboundTag { + t.Errorf("default inbound tag = %q, want %q", cfg.InboundTag, defaultInboundTag) + } + if _, ok := cfg.base["inbound_tag"]; ok { + t.Errorf("inbound_tag must be stripped from base config") + } + + // custom inbound tag + cfg, err = NewConfig(`{"inbound_tag":"mtp-eu","server":{"port":2222}}`) + if err != nil { + t.Fatalf("NewConfig returned error: %v", err) + } + if cfg.InboundTag != "mtp-eu" { + t.Errorf("custom inbound tag = %q, want %q", cfg.InboundTag, "mtp-eu") + } + + // panel must not set managed access keys + if _, err := NewConfig(`{"access":{"users":{"a":"x"}}}`); err == nil { + t.Errorf("expected error when panel sets access.users") + } + if _, err := NewConfig(`{"access":{"user_enabled":{"a":false}}}`); err == nil { + t.Errorf("expected error when panel sets access.user_enabled") + } + + // empty / invalid JSON + if _, err := NewConfig(" "); err == nil { + t.Errorf("expected error for empty config") + } + if _, err := NewConfig(`{`); err == nil { + t.Errorf("expected error for invalid JSON") + } + if _, err := NewConfig(`{} {}`); err == nil { + t.Errorf("expected error for trailing JSON object") + } +} + +func TestNewConfigRejectsNodeManagedKeys(t *testing.T) { + cases := []struct { + raw string + key string + }{ + {`{"include": "x"}`, "include"}, + {`{"server": {"api": {"read_only": true}}}`, "server.api"}, + {`{"server": {"admin_api": {}}}`, "server.admin_api"}, + {`{"server": {"metrics_port": 1}}`, "server.metrics_port"}, + {`{"server": {"metrics_listen": "0.0.0.0:9090"}}`, "server.metrics_listen"}, + {`{"server": {"metrics_whitelist": ["0.0.0.0/0"]}}`, "server.metrics_whitelist"}, + {`{"web": {}}`, "web"}, + {`{"access": {"user_rate_limits": {}}}`, "access.user_rate_limits"}, + {`{"access": {"user_source_deny": {}}}`, "access.user_source_deny"}, + {`{"access": {"users": {}}}`, "access.users"}, + {`{"access": {"user_ad_tags": {}}}`, "access.user_ad_tags"}, + {`{"access": {"user_max_tcp_conns": {}}}`, "access.user_max_tcp_conns"}, + {`{"access": {"user_expirations": {}}}`, "access.user_expirations"}, + {`{"access": {"user_data_quota": {}}}`, "access.user_data_quota"}, + {`{"access": {"user_max_unique_ips": {}}}`, "access.user_max_unique_ips"}, + {`{"access": {"user_enabled": {}}}`, "access.user_enabled"}, + } + for _, tc := range cases { + t.Run(tc.key, func(t *testing.T) { + _, err := NewConfig(tc.raw) + if err == nil { + t.Fatalf("NewConfig(%s) succeeded, want an error naming %s", tc.raw, tc.key) + } + if !strings.Contains(err.Error(), "must not define "+tc.key) { + t.Errorf("error %q does not name %s", err, tc.key) + } + }) + } + + // Keys that only share a prefix with a rejected one stay allowed. + if _, err := NewConfig(`{"server": {"port": 443, "listen_backlog": 128}, "access": {"replay_check_len": 65536}}`); err != nil { + t.Errorf("unrelated keys must be accepted, got %v", err) + } +} + +func TestNewConfigValidatesShapes(t *testing.T) { + cases := map[string]string{ + "general not object": `{"general": 1}`, + "telemetry not object": `{"general": {"telemetry": true}}`, + "server not object": `{"server": []}`, + "access not object": `{"access": "x"}`, + "port zero": `{"server": {"port": 0}}`, + "port too large": `{"server": {"port": 65536}}`, + "port string": `{"server": {"port": "443"}}`, + "port fraction": `{"server": {"port": 443.5}}`, + "listeners not array": `{"server": {"listeners": {"ip": "::"}}}`, + "listener not object": `{"server": {"listeners": [1]}}`, + "listener port invalid": `{"server": {"listeners": [{"ip": "::", "port": -1}]}}`, + } + for name, raw := range cases { + t.Run(name, func(t *testing.T) { + if _, err := NewConfig(raw); err == nil { + t.Errorf("NewConfig(%s) succeeded, want an error", raw) + } + }) + } + + // JSON null means "not set" for objects and ports. + if _, err := NewConfig(`{"general": null, "server": {"port": null, "api": null}, "access": null}`); err != nil { + t.Errorf("null sections must be treated as absent, got %v", err) + } +} + +func TestConfigListenPorts(t *testing.T) { + cases := []struct { + name string + raw string + want []int + }{ + {"telemt default port", `{}`, []int{443}}, + {"server port", `{"server": {"port": 2222}}`, []int{2222}}, + { + "listeners with and without port", + `{"server": {"port": 8443, "listeners": [{"ip": "0.0.0.0"}, {"ip": "::", "port": 9443}, {"ip": "::1", "port": 8443}]}}`, + []int{8443, 9443}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg, err := NewConfig(tc.raw) + if err != nil { + t.Fatal(err) + } + if got := cfg.ListenPorts(); !slices.Equal(got, tc.want) { + t.Errorf("ListenPorts() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestRenderForcesNodeKeys(t *testing.T) { + const apiPort, metricsPort = 12345, 23456 + users := map[string]*runtimeUser{"alice": {Username: "alice", Secret: testSecret}} + + fromPanel, err := NewConfig(`{"general": {"disable_colors": false, "telemetry": {"user_enabled": false, "core_enabled": false}}, "server": {"port": 2222}}`) + if err != nil { + t.Fatal(err) + } + minimal, err := NewConfig(`{"server": {"port": 2222}}`) + if err != nil { + t.Fatal(err) + } + // NewConfig rejects server.api; Render must still force it if a base + // carries it some other way. + direct := &Config{InboundTag: defaultInboundTag, base: map[string]any{ + "server": map[string]any{"api": map[string]any{"read_only": true, "enabled": false}}, + }} + + for name, cfg := range map[string]*Config{"panel overrides": fromPanel, "minimal": minimal, "direct base": direct} { + t.Run(name, func(t *testing.T) { + out, err := cfg.Render(users, nil, apiPort, metricsPort, "Bearer testtoken") + if err != nil { + t.Fatal(err) + } + tree := decodeTOML(t, out) + expect := map[string]any{ + "general.disable_colors": true, + "general.telemetry.user_enabled": true, + "server.api.enabled": true, + "server.api.read_only": false, + "server.api.listen": "127.0.0.1:" + strconv.Itoa(apiPort), + "server.api.auth_header": "Bearer testtoken", + "server.metrics_port": int64(metricsPort), + "server.metrics_listen": "127.0.0.1:" + strconv.Itoa(metricsPort), + "access.users.alice": testSecret, + } + for path, want := range expect { + if got := tree.GetPath(strings.Split(path, ".")); got != want { + t.Errorf("%s = %#v, want %#v\n---\n%s", path, got, want, out) + } + } + }) + } + + // Render must not mutate the stored base. + if telemetry := fromPanel.base["general"].(map[string]any)["telemetry"].(map[string]any); telemetry["user_enabled"] != false { + t.Errorf("Render mutated the base config: telemetry = %v", telemetry) + } +} + +func TestRenderRejectsPortConflict(t *testing.T) { + const apiPort, metricsPort = 12345, 23456 + users := map[string]*runtimeUser{"alice": {Username: "alice", Secret: testSecret}} + + cases := map[string]string{ + "server port is api port": `{"server": {"port": 12345}}`, + "server port is metrics port": `{"server": {"port": 23456}}`, + "listener port is api port": `{"server": {"port": 443, "listeners": [{"ip": "::", "port": 12345}]}}`, + "listener port is metrics port": `{"server": {"port": 443, "listeners": [{"ip": "0.0.0.0", "port": 23456}]}}`, + "listener inherits server port": `{"server": {"port": 23456, "listeners": [{"ip": "0.0.0.0"}]}}`, + } + for name, raw := range cases { + t.Run(name, func(t *testing.T) { + cfg, err := NewConfig(raw) + if err != nil { + t.Fatal(err) + } + _, err = cfg.Render(users, nil, apiPort, metricsPort, "Bearer x") + if err == nil { + t.Fatalf("Render succeeded for %s, want a port conflict error", raw) + } + if !strings.Contains(err.Error(), "conflicts") { + t.Errorf("error %q does not describe a port conflict", err) + } + }) + } + + cfg, err := NewConfig(`{"server": {"port": 443, "listeners": [{"ip": "::", "port": 8443}]}}`) + if err != nil { + t.Fatal(err) + } + if _, err := cfg.Render(users, nil, apiPort, metricsPort, "Bearer x"); err != nil { + t.Errorf("Render without a conflict failed: %v", err) + } +} + +func TestConfigRender_InjectsManagedSections(t *testing.T) { + cfg, err := NewConfig(`{"server":{"port":2222},"censorship":{"tls_domain":"example.com"}}`) + if err != nil { + t.Fatal(err) + } + users := map[string]*runtimeUser{"alice": {Username: "alice", Secret: testSecret}} + out, err := cfg.Render(users, nil, 12345, 23456, "Bearer testtoken") + if err != nil { + t.Fatal(err) + } + for _, want := range []string{ + "enabled = true", + `listen = "127.0.0.1:12345"`, + `auth_header = "Bearer testtoken"`, + `metrics_port = 23456`, + `metrics_listen = "127.0.0.1:23456"`, + "[access.users]", + `alice = "` + testSecret + `"`, + `tls_domain = "example.com"`, + } { + if !strings.Contains(out, want) { + t.Errorf("rendered config missing %q\n---\n%s", want, out) + } + } + if strings.Contains(out, "user_enabled = {") || strings.Contains(out, "[access.user_enabled]") { + t.Errorf("real users must not disable anyone via access.user_enabled") + } + + // empty users + sentinel keeps telemt happy (telemt rejects empty [access.users]) + sentinel := &runtimeUser{Username: "pg-node-internal-x", Secret: testSecret} + out, err = cfg.Render(map[string]*runtimeUser{}, sentinel, 1, 2, "") + if err != nil { + t.Fatalf("render with sentinel failed: %v", err) + } + if !strings.Contains(out, "pg-node-internal-x") { + t.Errorf("sentinel user missing from rendered config") + } + if !strings.Contains(out, "[access.user_enabled]") || !strings.Contains(out, "pg-node-internal-x = false") { + t.Errorf("sentinel must be rendered disabled, got:\n%s", out) + } + + // empty users + no sentinel -> error (would crash telemt) + if _, err := cfg.Render(map[string]*runtimeUser{}, nil, 1, 2, ""); err == nil { + t.Errorf("expected error rendering with zero users and no sentinel") + } +} + +func TestActiveUserFromCommon_Selection(t *testing.T) { + // not tagged -> excluded, no error + if _, include, err := activeUserFromCommon(mtUser("bob", "other", testSecret), "mtproto"); err != nil || include { + t.Errorf("untagged user: include=%v err=%v, want include=false err=nil", include, err) + } + + // tagged + valid secret -> included, lowercased + u, include, err := activeUserFromCommon(mtUser("alice", "mtproto", strings.ToUpper(testSecret)), "mtproto") + if err != nil || !include { + t.Fatalf("valid user not included: include=%v err=%v", include, err) + } + if u.Username != "alice" || u.Secret != testSecret { + t.Errorf("mapped user = %+v, want username=alice secret lowercased", u) + } + + // tagged but missing mtproto proxy -> error + bad := &common.User{Email: "c", Inbounds: []string{"mtproto"}, Proxies: &common.Proxy{}} + if _, _, err := activeUserFromCommon(bad, "mtproto"); err == nil { + t.Errorf("expected error for missing proxies.mtproto") + } + + // invalid secret length -> error + if _, _, err := activeUserFromCommon(mtUser("d", "mtproto", "abc"), "mtproto"); err == nil { + t.Errorf("expected error for short secret") + } + + // invalid username -> error + if _, _, err := activeUserFromCommon(mtUser("has space", "mtproto", testSecret), "mtproto"); err == nil { + t.Errorf("expected error for invalid username") + } +} + +func TestActiveUsersSkipsInvalidUsers(t *testing.T) { + users := []*common.User{ + mtUser("1", "mtproto", testSecret), + {Email: "2", Inbounds: []string{"mtproto"}, Proxies: &common.Proxy{}}, + mtUser("3", "mtproto", testSecret[:31]), + // Not tagged: neither returned nor skipped, even though it is invalid. + {Email: "4", Inbounds: []string{"vless"}, Proxies: &common.Proxy{}}, + nil, + } + + active, skipped := ActiveUsersFromCommon(users, "mtproto") + if len(active) != 1 || active["1"] == nil { + t.Errorf("active = %v, want only user 1", active) + } + if !slices.Equal(skipped, []string{"2", "3"}) { + t.Errorf("skipped = %v, want [2 3]", skipped) + } + + // An email repeated in one call is reported once. + _, skipped = ActiveUsersFromCommon([]*common.User{ + mtUser("5", "mtproto", "bad"), + mtUser("5", "mtproto", "bad"), + }, "mtproto") + if !slices.Equal(skipped, []string{"5"}) { + t.Errorf("skipped = %v, want [5]", skipped) + } +} + +func TestBuildNextDesiredSkipsInvalidUsers(t *testing.T) { + m := &MTProto{ + config: &Config{InboundTag: "mtproto"}, + desired: map[string]*runtimeUser{ + "1": {Username: "1", Secret: testSecret}, + "2": {Username: "2", Secret: testSecret}, + }, + } + newSecret := "ffeeddccbbaa99887766554433221100" + + next := m.buildNextDesiredLocked([]*common.User{ + mtUser("2", "mtproto", "not-hex"), // invalid: treated as not included + mtUser("3", "mtproto", newSecret), // valid: added + nil, + }) + + if next["1"] == nil { + t.Errorf("untouched user 1 was dropped: %v", next) + } + if _, ok := next["2"]; ok { + t.Errorf("invalid user 2 must be removed from the desired set: %v", next) + } + if next["3"] == nil || next["3"].Secret != newSecret { + t.Errorf("valid user 3 was not applied: %v", next) + } + if _, ok := m.desired["3"]; ok { + t.Errorf("buildNextDesiredLocked must not mutate m.desired") + } +} + +func TestSyncUserReturnsValidationError(t *testing.T) { + m := &MTProto{config: &Config{InboundTag: "mtproto"}} + err := m.SyncUser(context.Background(), mtUser("1", "mtproto", testSecret[:31])) + if err == nil || !strings.Contains(err.Error(), "invalid secret") { + t.Errorf("SyncUser error = %v, want the secret validation error", err) + } +} diff --git a/backend/mtproto/mtproto.go b/backend/mtproto/mtproto.go index 9295c2ab..c05fab3f 100644 --- a/backend/mtproto/mtproto.go +++ b/backend/mtproto/mtproto.go @@ -75,10 +75,7 @@ func New(ctx context.Context, cfg *config.Config, mtConfig *Config, users []*com return nil, err } - activeUsers, err := ActiveUsersFromCommon(users, mtConfig.InboundTag) - if err != nil { - return nil, err - } + activeUsers, _ := ActiveUsersFromCommon(users, mtConfig.InboundTag) sentinel, err := newSentinelUser() if err != nil { @@ -147,6 +144,11 @@ func (m *MTProto) Shutdown() { } func (m *MTProto) SyncUser(ctx context.Context, user *common.User) error { + // The batch paths skip a user that fails validation; a single-user sync + // reports it to the caller instead. + if _, _, err := activeUserFromCommon(user, m.config.InboundTag); err != nil { + return err + } return m.updateUsersPartial(ctx, []*common.User{user}) } @@ -154,10 +156,7 @@ func (m *MTProto) SyncUsers(ctx context.Context, users []*common.User) error { m.mu.Lock() defer m.mu.Unlock() - desired, err := ActiveUsersFromCommon(users, m.config.InboundTag) - if err != nil { - return err - } + desired, _ := ActiveUsersFromCommon(users, m.config.InboundTag) return m.applyFullDesiredLocked(ctx, desired) } @@ -170,10 +169,7 @@ func (m *MTProto) UpdateUsersAndRestart(ctx context.Context, users []*common.Use m.mu.Lock() defer m.mu.Unlock() - nextDesired, err := m.buildNextDesiredLocked(users) - if err != nil { - return err - } + nextDesired := m.buildNextDesiredLocked(users) return m.applyFullDesiredLocked(ctx, nextDesired) } @@ -313,10 +309,7 @@ func (m *MTProto) updateUsersPartial(ctx context.Context, users []*common.User) m.mu.Lock() defer m.mu.Unlock() - nextDesired, err := m.buildNextDesiredLocked(users) - if err != nil { - return err - } + nextDesired := m.buildNextDesiredLocked(users) if len(m.desired) == 0 || len(nextDesired) == 0 { return m.applyFullDesiredLocked(ctx, nextDesired) @@ -330,32 +323,39 @@ func (m *MTProto) updateUsersPartial(ctx context.Context, users []*common.User) return nil } -func (m *MTProto) buildNextDesiredLocked(users []*common.User) (map[string]*runtimeUser, error) { +// buildNextDesiredLocked applies users on top of m.desired. A user that fails +// validation is logged and treated as not included, like an untagged one. +func (m *MTProto) buildNextDesiredLocked(users []*common.User) map[string]*runtimeUser { nextDesired := cloneRuntimeUserMap(m.desired) for _, user := range users { - runtimeUser, include, err := activeUserFromCommon(user, m.config.InboundTag) - if err != nil { - return nil, err + if user == nil { + continue } username := strings.TrimSpace(user.GetEmail()) + runtimeUser, include, err := activeUserFromCommon(user, m.config.InboundTag) + if err != nil { + logSkippedUser(username, err) + } if include { nextDesired[runtimeUser.Username] = runtimeUser continue } delete(nextDesired, username) } - return nextDesired, nil + return nextDesired } func (m *MTProto) applyPartialDesiredLocked(ctx context.Context, users []*common.User, nextDesired map[string]*runtimeUser) error { for _, user := range users { - runtimeUser, include, err := activeUserFromCommon(user, m.config.InboundTag) - if err != nil { - return err + if user == nil { + continue } - if include { + // An invalid user (already logged by buildNextDesiredLocked) is + // removed like an untagged one. + runtimeUser, include, err := activeUserFromCommon(user, m.config.InboundTag) + if err == nil && include { if err := m.upsertUser(ctx, runtimeUser); err != nil { return err } diff --git a/backend/mtproto/mtproto_test.go b/backend/mtproto/mtproto_test.go index 0a4999f8..549862a1 100644 --- a/backend/mtproto/mtproto_test.go +++ b/backend/mtproto/mtproto_test.go @@ -21,127 +21,6 @@ func mtUser(email, tag, secret string) *common.User { } } -func TestNewConfig_InboundTagAndValidation(t *testing.T) { - // default inbound tag - cfg, err := NewConfig(`{"server":{"port":2222}}`) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if cfg.InboundTag != defaultInboundTag { - t.Errorf("default inbound tag = %q, want %q", cfg.InboundTag, defaultInboundTag) - } - if _, ok := cfg.base["inbound_tag"]; ok { - t.Errorf("inbound_tag must be stripped from base config") - } - - // custom inbound tag - cfg, err = NewConfig(`{"inbound_tag":"mtp-eu","server":{"port":2222}}`) - if err != nil { - t.Fatalf("NewConfig returned error: %v", err) - } - if cfg.InboundTag != "mtp-eu" { - t.Errorf("custom inbound tag = %q, want %q", cfg.InboundTag, "mtp-eu") - } - - // panel must not set managed access keys - if _, err := NewConfig(`{"access":{"users":{"a":"x"}}}`); err == nil { - t.Errorf("expected error when panel sets access.users") - } - if _, err := NewConfig(`{"access":{"user_enabled":{"a":false}}}`); err == nil { - t.Errorf("expected error when panel sets access.user_enabled") - } - - // empty / invalid JSON - if _, err := NewConfig(" "); err == nil { - t.Errorf("expected error for empty config") - } - if _, err := NewConfig(`{`); err == nil { - t.Errorf("expected error for invalid JSON") - } - if _, err := NewConfig(`{} {}`); err == nil { - t.Errorf("expected error for trailing JSON object") - } -} - -func TestActiveUserFromCommon_Selection(t *testing.T) { - // not tagged -> excluded, no error - if _, include, err := activeUserFromCommon(mtUser("bob", "other", testSecret), "mtproto"); err != nil || include { - t.Errorf("untagged user: include=%v err=%v, want include=false err=nil", include, err) - } - - // tagged + valid secret -> included, lowercased - u, include, err := activeUserFromCommon(mtUser("alice", "mtproto", strings.ToUpper(testSecret)), "mtproto") - if err != nil || !include { - t.Fatalf("valid user not included: include=%v err=%v", include, err) - } - if u.Username != "alice" || u.Secret != testSecret { - t.Errorf("mapped user = %+v, want username=alice secret lowercased", u) - } - - // tagged but missing mtproto proxy -> error - bad := &common.User{Email: "c", Inbounds: []string{"mtproto"}, Proxies: &common.Proxy{}} - if _, _, err := activeUserFromCommon(bad, "mtproto"); err == nil { - t.Errorf("expected error for missing proxies.mtproto") - } - - // invalid secret length -> error - if _, _, err := activeUserFromCommon(mtUser("d", "mtproto", "abc"), "mtproto"); err == nil { - t.Errorf("expected error for short secret") - } - - // invalid username -> error - if _, _, err := activeUserFromCommon(mtUser("has space", "mtproto", testSecret), "mtproto"); err == nil { - t.Errorf("expected error for invalid username") - } -} - -func TestConfigRender_InjectsManagedSections(t *testing.T) { - cfg, err := NewConfig(`{"server":{"port":2222},"censorship":{"tls_domain":"example.com"}}`) - if err != nil { - t.Fatal(err) - } - users := map[string]*runtimeUser{"alice": {Username: "alice", Secret: testSecret}} - out, err := cfg.Render(users, nil, 12345, 23456, "Bearer testtoken") - if err != nil { - t.Fatal(err) - } - for _, want := range []string{ - "enabled = true", - `listen = "127.0.0.1:12345"`, - `auth_header = "Bearer testtoken"`, - `metrics_port = 23456`, - `metrics_listen = "127.0.0.1:23456"`, - "[access.users]", - `alice = "` + testSecret + `"`, - `tls_domain = "example.com"`, - } { - if !strings.Contains(out, want) { - t.Errorf("rendered config missing %q\n---\n%s", want, out) - } - } - if strings.Contains(out, "user_enabled") { - t.Errorf("real users must not disable anyone via user_enabled") - } - - // empty users + sentinel keeps telemt happy (telemt rejects empty [access.users]) - sentinel := &runtimeUser{Username: "pg-node-internal-x", Secret: testSecret} - out, err = cfg.Render(map[string]*runtimeUser{}, sentinel, 1, 2, "") - if err != nil { - t.Fatalf("render with sentinel failed: %v", err) - } - if !strings.Contains(out, "pg-node-internal-x") { - t.Errorf("sentinel user missing from rendered config") - } - if !strings.Contains(out, "[access.user_enabled]") || !strings.Contains(out, "pg-node-internal-x = false") { - t.Errorf("sentinel must be rendered disabled, got:\n%s", out) - } - - // empty users + no sentinel -> error (would crash telemt) - if _, err := cfg.Render(map[string]*runtimeUser{}, nil, 1, 2, ""); err == nil { - t.Errorf("expected error rendering with zero users and no sentinel") - } -} - func TestTracker_DeltaResetAndCounterRestart(t *testing.T) { tr := newTrafficTracker() active := map[string]struct{}{"alice": {}} diff --git a/backend/mtproto/toml.go b/backend/mtproto/toml.go index 072ddbe1..2d9fc3cc 100644 --- a/backend/mtproto/toml.go +++ b/backend/mtproto/toml.go @@ -95,6 +95,9 @@ func classifyTOMLSection(section map[string]any) ([]string, []string, []string, for _, key := range keys { switch value := section[key].(type) { + case nil: + // JSON null means "not set"; TOML has no null. + continue case map[string]any: tableKeys = append(tableKeys, key) case []any: @@ -126,11 +129,11 @@ func isArrayOfObjects(values []any) bool { func renderInlineValue(value any) (string, error) { switch typed := value.(type) { case nil: - return "", fmt.Errorf("nil values are not valid in TOML") + return "", fmt.Errorf("null array elements are not valid in TOML") case tomlLiteral: return string(typed), nil case string: - return strconv.Quote(typed), nil + return quoteTOMLString(typed), nil case bool: return strconv.FormatBool(typed), nil case json.Number: @@ -154,6 +157,9 @@ func renderInlineValue(value any) (string, error) { parts := make([]string, 0, len(keys)) for _, key := range keys { + if typed[key] == nil { + continue + } inline, err := renderInlineValue(typed[key]) if err != nil { return "", err @@ -183,10 +189,47 @@ func renderPrimitiveValue(value any) (string, error) { } func renderKey(key string) string { - if isBareKey(key) { + // telemt expands any line of the form `include = ""` as a file + // include before it parses TOML, in every table. A quoted key is not + // matched, so a key named include is always quoted. + if isBareKey(key) && key != "include" { return key } - return strconv.Quote(key) + return quoteTOMLString(key) +} + +// quoteTOMLString renders value as a TOML basic string. strconv.Quote does not +// fit: it emits Go escapes such as \a, \v and \x01 that TOML rejects. +func quoteTOMLString(value string) string { + var builder strings.Builder + builder.Grow(len(value) + 2) + builder.WriteByte('"') + for _, r := range value { + switch r { + case '"': + builder.WriteString(`\"`) + case '\\': + builder.WriteString(`\\`) + case '\b': + builder.WriteString(`\b`) + case '\t': + builder.WriteString(`\t`) + case '\n': + builder.WriteString(`\n`) + case '\f': + builder.WriteString(`\f`) + case '\r': + builder.WriteString(`\r`) + default: + if r < 0x20 || r == 0x7f { + fmt.Fprintf(&builder, `\u%04X`, r) + continue + } + builder.WriteRune(r) + } + } + builder.WriteByte('"') + return builder.String() } func isBareKey(key string) bool { diff --git a/backend/mtproto/toml_test.go b/backend/mtproto/toml_test.go new file mode 100644 index 00000000..40e27434 --- /dev/null +++ b/backend/mtproto/toml_test.go @@ -0,0 +1,153 @@ +package mtproto + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/pelletier/go-toml" +) + +// decodeTOML parses rendered output with an independent TOML decoder, so the +// tests check real TOML validity rather than the renderer's own idea of it. +func decodeTOML(t *testing.T, rendered string) *toml.Tree { + t.Helper() + tree, err := toml.Load(rendered) + if err != nil { + t.Fatalf("rendered TOML does not decode: %v\n---\n%s", err, rendered) + } + return tree +} + +func decodeJSONObject(t *testing.T, raw string) map[string]any { + t.Helper() + decoder := json.NewDecoder(strings.NewReader(raw)) + decoder.UseNumber() + var root map[string]any + if err := decoder.Decode(&root); err != nil { + t.Fatal(err) + } + return root +} + +func TestTOMLEscapesControlCharacters(t *testing.T) { + value := "bell\a vtab\v soh\x01 quote\" backslash\\ newline\n tab\t del\x7f esc\x1b unicodeé" + key := "key\x01with\\backslash\ttab" + + out, err := renderTOML(map[string]any{ + "general": map[string]any{"value": value, key: "k"}, + }) + if err != nil { + t.Fatal(err) + } + + // go-toml v1.9.5 cannot decode \" inside a quoted key (TOML 1.0 allows + // it), so this key is checked on the rendered text only. + quoted, err := renderTOML(map[string]any{"quoted": map[string]any{`say "hi"`: "q"}}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(quoted, `"say \"hi\"" = "q"`) { + t.Errorf("key with a quote is not rendered as an escaped basic string\n---\n%s", quoted) + } + + for _, want := range []string{`\u0007`, `\u000B`, `\u0001`, `\"`, `\\`, `\n`, `\u007F`, `\u001B`} { + if !strings.Contains(out, want) { + t.Errorf("rendered TOML missing escape %s\n---\n%s", want, out) + } + } + for _, invalid := range []string{`\a`, `\v`, `\x`} { + if strings.Contains(out, invalid) { + t.Errorf("rendered TOML contains non-TOML escape %s\n---\n%s", invalid, out) + } + } + for _, line := range strings.Split(out, "\n") { + for _, r := range line { + if (r < 0x20 && r != '\t') || r == 0x7f { + t.Errorf("rendered line contains raw control character %U: %q", r, line) + } + } + } + + tree := decodeTOML(t, out) + if got := tree.GetPath([]string{"general", "value"}); got != value { + t.Errorf("value round trip = %q, want %q", got, value) + } + if got := tree.GetPath([]string{"general", key}); got != "k" { + t.Errorf("escaped key round trip = %v, want %q", got, "k") + } +} + +func TestTOMLDropsNullValues(t *testing.T) { + out, err := renderTOML(decodeJSONObject(t, `{"general": {"ad_tag": null, "x": 1}, "server": {"listeners": [{"ip": "::", "announce": null}]}}`)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(out, "ad_tag") || strings.Contains(out, "announce") { + t.Errorf("null values must be dropped\n---\n%s", out) + } + tree := decodeTOML(t, out) + if got := tree.GetPath([]string{"general", "x"}); got != int64(1) { + t.Errorf("general.x = %#v, want 1", got) + } + + // Inline tables (objects inside a mixed array) drop nulls too. + out, err = renderTOML(decodeJSONObject(t, `{"general": {"mixed": [1, {"a": null, "b": 2}]}}`)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(out, "a =") { + t.Errorf("null inside an inline table must be dropped\n---\n%s", out) + } + + // A null array element has no TOML meaning and is an error. + if out, err := renderTOML(decodeJSONObject(t, `{"general": {"list": [1, null]}}`)); err == nil { + t.Errorf("null inside an array must be an error, got\n%s", out) + } +} + +func TestTOMLArrayOfTables(t *testing.T) { + out, err := renderTOML(decodeJSONObject(t, `{"server": {"port": 443, "listeners": [{"ip": "0.0.0.0"}, {"ip": "::"}]}}`)) + if err != nil { + t.Fatal(err) + } + if got := strings.Count(out, "[[server.listeners]]"); got != 2 { + t.Errorf("[[server.listeners]] appears %d times, want 2\n---\n%s", got, out) + } + + tree := decodeTOML(t, out) + listeners, ok := tree.GetPath([]string{"server", "listeners"}).([]*toml.Tree) + if !ok || len(listeners) != 2 { + t.Fatalf("server.listeners = %#v, want two tables", tree.GetPath([]string{"server", "listeners"})) + } + if listeners[0].Get("ip") != "0.0.0.0" || listeners[1].Get("ip") != "::" { + t.Errorf("listener ips = %v, %v", listeners[0].Get("ip"), listeners[1].Get("ip")) + } + if got := tree.GetPath([]string{"server", "port"}); got != int64(443) { + t.Errorf("server.port = %#v, want 443", got) + } +} + +// telemt expands any line of the form `include = ""` as a textual +// include before it parses TOML (src/config/load/includes.rs), whatever table +// it sits in. A key named include must therefore never render bare. +func TestTOMLQuotesIncludeKeys(t *testing.T) { + out, err := renderTOML(map[string]any{ + "general": map[string]any{"include": "/etc/shadow"}, + "access": map[string]any{"users": map[string]any{"include": testSecret}}, + }) + if err != nil { + t.Fatal(err) + } + for _, line := range strings.Split(out, "\n") { + rest, found := strings.CutPrefix(strings.TrimSpace(line), "include") + if found && strings.HasPrefix(strings.TrimSpace(rest), "=") { + t.Errorf("line would be expanded by telemt as an include: %q", line) + } + } + + tree := decodeTOML(t, out) + if got := tree.GetPath([]string{"general", "include"}); got != "/etc/shadow" { + t.Errorf("general.include = %v, want the quoted key to round trip", got) + } +} diff --git a/go.mod b/go.mod index 3bd60c52..7fd05fd2 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/google/uuid v1.6.0 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/joho/godotenv v1.5.1 + github.com/pelletier/go-toml v1.9.5 github.com/shirou/gopsutil/v4 v4.26.9 github.com/vishvananda/netlink v1.3.1 github.com/xtls/xray-core v1.260327.1-0.20260930074004-b26a91de4f32 From f9bca96f1fb4cab6b28020ae7cadb935536a9be4 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 19 Oct 2026 19:23:07 +0330 Subject: [PATCH 04/40] fix(mtproto): force stderr logging and reject web listeners - Render forces logging.destination = "stderr": the node reads telemt's stderr for its log stream and startup errors, and a panel config with destination "file" or "syslog" would leave both empty. logging must be an object when present. - NewConfig rejects a server.listeners[].transport other than "mtproxy". telemt WEB listeners need [[web.vhosts]], which the node rejects, so they would only make telemt exit at startup. --- backend/mtproto/config.go | 29 +++++++++++++++++++++++--- backend/mtproto/config_test.go | 38 +++++++++++++++++++++++++++++++++- 2 files changed, 63 insertions(+), 4 deletions(-) diff --git a/backend/mtproto/config.go b/backend/mtproto/config.go index e84eb365..f2c6943f 100644 --- a/backend/mtproto/config.go +++ b/backend/mtproto/config.go @@ -51,6 +51,7 @@ var ( {"general", "telemetry"}, {"server"}, {"access"}, + {"logging"}, } ) @@ -155,6 +156,25 @@ func validateConfigRoot(root map[string]any) error { return err } + return validateListenerTransports(root) +} + +// validateListenerTransports allows only MTProxy listeners. telemt WEB +// listeners need [[web.vhosts]], and the node rejects the web section, so a +// web listener would only make telemt exit at startup. Call it after +// listenPorts, which checks the listener shapes. +func validateListenerTransports(root map[string]any) error { + server, _ := objectAt(root, "server") + listeners, _ := server["listeners"].([]any) + for index, rawListener := range listeners { + listener, _ := rawListener.(map[string]any) + if !isSet(listener, "transport") { + continue + } + if transport, ok := listener["transport"].(string); !ok || transport != "mtproxy" { + return fmt.Errorf("mtproto config field server.listeners[%d].transport must be \"mtproxy\"; the node supports only MTProxy listeners", index) + } + } return nil } @@ -255,10 +275,13 @@ func (c *Config) Render(users map[string]*runtimeUser, sentinel *runtimeUser, ap server := ensureObject(root, "server") api := ensureObject(server, "api") access := ensureObject(root, "access") + logging := ensureObject(root, "logging") - // telemt's output is streamed to the panel log viewer, where ANSI colours - // are noise. Per-user counters feed traffic accounting; with user - // telemetry off telemt reports zero for every user. + // The node reads telemt's stderr for its log stream and startup errors, + // and streams it to the panel log viewer, where ANSI colours are noise. + // Per-user counters feed traffic accounting; with user telemetry off + // telemt reports zero for every user. + logging["destination"] = "stderr" general["disable_colors"] = true telemetry["user_enabled"] = true diff --git a/backend/mtproto/config_test.go b/backend/mtproto/config_test.go index ab149895..1722369e 100644 --- a/backend/mtproto/config_test.go +++ b/backend/mtproto/config_test.go @@ -98,6 +98,7 @@ func TestNewConfigValidatesShapes(t *testing.T) { "telemetry not object": `{"general": {"telemetry": true}}`, "server not object": `{"server": []}`, "access not object": `{"access": "x"}`, + "logging not object": `{"logging": "file"}`, "port zero": `{"server": {"port": 0}}`, "port too large": `{"server": {"port": 65536}}`, "port string": `{"server": {"port": "443"}}`, @@ -120,6 +121,36 @@ func TestNewConfigValidatesShapes(t *testing.T) { } } +// telemt WEB listeners need [[web.vhosts]], which the node rejects, so a web +// listener would only crash telemt at startup. +func TestNewConfigRejectsNonMTProxyListeners(t *testing.T) { + cases := map[string]string{ + "web": `{"server": {"listeners": [{"ip": "::", "transport": "web"}]}}`, + "upper case": `{"server": {"listeners": [{"ip": "::"}, {"ip": "0.0.0.0", "transport": "WEB"}]}}`, + "not string": `{"server": {"listeners": [{"ip": "::", "transport": 1}]}}`, + } + for name, raw := range cases { + t.Run(name, func(t *testing.T) { + _, err := NewConfig(raw) + if err == nil { + t.Fatalf("NewConfig(%s) succeeded, want an error", raw) + } + if !strings.Contains(err.Error(), "server.listeners[") || !strings.Contains(err.Error(), "].transport") { + t.Errorf("error %q does not name the listener transport key", err) + } + }) + } + + for _, raw := range []string{ + `{"server": {"listeners": [{"ip": "::", "transport": "mtproxy"}]}}`, + `{"server": {"listeners": [{"ip": "::", "transport": null}, {"ip": "0.0.0.0"}]}}`, + } { + if _, err := NewConfig(raw); err != nil { + t.Errorf("NewConfig(%s) failed: %v", raw, err) + } + } +} + func TestConfigListenPorts(t *testing.T) { cases := []struct { name string @@ -159,13 +190,17 @@ func TestRenderForcesNodeKeys(t *testing.T) { if err != nil { t.Fatal(err) } + logToFile, err := NewConfig(`{"logging": {"destination": "file", "path": "/tmp/x"}, "server": {"port": 2222}}`) + if err != nil { + t.Fatal(err) + } // NewConfig rejects server.api; Render must still force it if a base // carries it some other way. direct := &Config{InboundTag: defaultInboundTag, base: map[string]any{ "server": map[string]any{"api": map[string]any{"read_only": true, "enabled": false}}, }} - for name, cfg := range map[string]*Config{"panel overrides": fromPanel, "minimal": minimal, "direct base": direct} { + for name, cfg := range map[string]*Config{"panel overrides": fromPanel, "minimal": minimal, "log to file": logToFile, "direct base": direct} { t.Run(name, func(t *testing.T) { out, err := cfg.Render(users, nil, apiPort, metricsPort, "Bearer testtoken") if err != nil { @@ -175,6 +210,7 @@ func TestRenderForcesNodeKeys(t *testing.T) { expect := map[string]any{ "general.disable_colors": true, "general.telemetry.user_enabled": true, + "logging.destination": "stderr", "server.api.enabled": true, "server.api.read_only": false, "server.api.listen": "127.0.0.1:" + strconv.Itoa(apiPort), From a2c71b834d6cab0b5b258662b7e94e025c17b405 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 19 Oct 2026 19:56:46 +0330 Subject: [PATCH 05/40] fix(mtproto): supervise telemt 3.5.14 safely across start, crash and shutdown Process manager (process.go): - Refuse telemt older than 3.5.14 (parsed from `telemt X.Y.Z`). - Run `telemt run ` with no --pid-file, so a stale pid file can no longer block startup; reload sends SIGHUP to the child (Windows returns an error and the caller restarts). - Started() checks a waitDone channel and never reads ProcessState (data race under -race). Stop marks the run as stopping; any other exit is logged and published as "telemt exited unexpectedly: ", with an OOM hint on SIGKILL. - Read stdout and stderr to EOF before Wait so a crash's last lines are kept; overlong lines are cut instead of stalling the reader. - Keep the last 50 lines since Start for startup errors (StartupTail). - Kill an orphaned telemt whose command line runs exactly this config path before starting a new one. - Add Exited(), Generation(), Close(); Start after Close fails. Lifecycle (lifecycle.go): - New picks the API and metrics ports again while they clash with each other or with the config's listen ports, creates /telemt with mode 0700 and resolves symlinks (telemt 3.5.8+ rejects a symlinked config path). - Wait up to 90 s for /v1/health (telemt binds its API only after the Telegram DC check), then fail with telemt's recent output instead of a bare exit status. /v1/health/ready gets a 30 s budget and only logs a warning. - A supervisor restarts telemt after an unexpected exit with backoff from 1 s doubling to 60 s, re-rendering the desired users. - syncMu serializes every mutating call, the supervisor and Shutdown. Shutdown sets closed, stops telemt, closes the log buffer and deletes config.toml; later syncs return errClosed instead of starting a new telemt and crashing the node on a closed log channel. Also bump TELEMT_VERSION to 3.5.14 and give the integration tests a 150 s context. Unit tests drive a fake telemt (the test binary re-run as a helper process) for slow APIs, crashes, orphans and startup failures. --- Makefile | 5 +- backend/mtproto/client.go | 21 ++ backend/mtproto/fake_telemt_test.go | 303 +++++++++++++++++++++ backend/mtproto/integration_test.go | 8 +- backend/mtproto/lifecycle.go | 344 +++++++++++++++++++++++ backend/mtproto/lifecycle_test.go | 304 +++++++++++++++++++++ backend/mtproto/logtail.go | 59 ++++ backend/mtproto/mtproto.go | 233 +++++----------- backend/mtproto/mtproto_test.go | 4 +- backend/mtproto/process.go | 409 ++++++++++++++++++++-------- backend/mtproto/process_test.go | 314 +++++++++++++++++++-- backend/mtproto/process_unix.go | 18 ++ backend/mtproto/process_windows.go | 16 ++ 13 files changed, 1732 insertions(+), 306 deletions(-) create mode 100644 backend/mtproto/fake_telemt_test.go create mode 100644 backend/mtproto/lifecycle.go create mode 100644 backend/mtproto/lifecycle_test.go create mode 100644 backend/mtproto/logtail.go diff --git a/Makefile b/Makefile index 99aa1b9d..3266505f 100644 --- a/Makefile +++ b/Makefile @@ -46,9 +46,10 @@ GEOFILES_DIR ?= /usr/local/share/xray GEOFILES_XRAY ?= /usr/local/bin/xray GEOFILES_CHECK_CONFIG ?= $(CURDIR)/testdata/geofiles-check.json -# telemt (MTProto backend): pinned release; static musl tarball + sha256 verify +# telemt (MTProto backend): pinned release; static musl tarball + sha256 verify. +# 3.5.14 is also the floor: the node refuses to start an older telemt. TELEMT_REPOSITORY ?= telemt/telemt -TELEMT_VERSION ?= 3.4.25 +TELEMT_VERSION ?= 3.5.14 TELEMT_OS ?= TELEMT_ARCH ?= TELEMT_ARCH_MAP_amd64 = x86_64 diff --git a/backend/mtproto/client.go b/backend/mtproto/client.go index 1500be41..328ae6f7 100644 --- a/backend/mtproto/client.go +++ b/backend/mtproto/client.go @@ -57,6 +57,12 @@ type telemtHealth struct { ReadOnly bool `json:"read_only"` } +type telemtHealthReady struct { + Ready bool `json:"ready"` + Status string `json:"status"` + Reason string `json:"reason"` +} + type telemtCreateUserRequest struct { Username string `json:"username"` Secret string `json:"secret,omitempty"` @@ -111,6 +117,21 @@ func (c *apiClient) Health(ctx context.Context) error { return err } +// Ready returns nil when /v1/health/ready answers 200: telemt admits clients +// and has a healthy Telegram upstream. On 503 the error carries telemt's +// reason, such as no_healthy_upstreams. +func (c *apiClient) Ready(ctx context.Context) error { + _, err := doAPIRequest[telemtHealthReady](ctx, c, http.MethodGet, "/v1/health/ready", nil) + var apiErr *apiStatusError + if errors.As(err, &apiErr) && apiErr.StatusCode == http.StatusServiceUnavailable { + var body apiSuccessResponse[telemtHealthReady] + if json.Unmarshal([]byte(apiErr.Message), &body) == nil && body.Data.Reason != "" { + return fmt.Errorf("telemt is not ready: %s", body.Data.Reason) + } + } + return err +} + func (c *apiClient) GetUser(ctx context.Context, username string) (*telemtUserInfo, error) { escapedUsername := url.PathEscape(username) user, err := doAPIRequest[telemtUserInfo](ctx, c, http.MethodGet, "/v1/users/"+escapedUsername, nil) diff --git a/backend/mtproto/fake_telemt_test.go b/backend/mtproto/fake_telemt_test.go new file mode 100644 index 00000000..23c99881 --- /dev/null +++ b/backend/mtproto/fake_telemt_test.go @@ -0,0 +1,303 @@ +//go:build !windows + +package mtproto + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "net/http" + "os" + "os/signal" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "github.com/pasarguard/node/config" + "github.com/pasarguard/node/pkg/netutil" +) + +// fakeTelemtEnv names the file with the fakeTelemt options; the wrapper +// script sets it for the helper process. +const fakeTelemtEnv = "MTPROTO_FAKE_TELEMT_MODE" + +// fakeTelemt describes how the stand-in telemt behaves. The test binary +// re-executes itself as TestHelperFakeTelemt (through a wrapper script that +// writeFakeTelemt puts in a temp dir), so the fake runs as a real child process +// the node starts, signals and reaps. +type fakeTelemt struct { + Version string `json:"version"` // --version output, default 3.5.14 + // RecordFile receives the fake's argv, cwd and pid once its signal + // handlers are installed, so a test can wait for it to be ready. + RecordFile string `json:"record_file"` + Stdout []string `json:"stdout"` + Stderr []string `json:"stderr"` + // ExitAfter makes the fake exit with ExitCode after this delay. With + // ExitOnceFile set it exits only on the run that creates that file. + ExitAfter time.Duration `json:"exit_after"` + ExitCode int `json:"exit_code"` + ExitOnceFile string `json:"exit_once_file"` + // HealthDelay delays binding the API, like telemt waiting for Telegram DCs. + HealthDelay time.Duration `json:"health_delay"` + NoAPI bool `json:"no_api"` + ReadyStatus int `json:"ready_status"` // default 200 + // SignalFile gets one line per SIGHUP received. + SignalFile string `json:"signal_file"` + // Lifetime bounds the fake so a broken test never leaves it running. + Lifetime time.Duration `json:"lifetime"` +} + +type fakeTelemtRecord struct { + Args []string `json:"args"` + Cwd string `json:"cwd"` + PID int `json:"pid"` +} + +// TestHelperFakeTelemt is not a real test: it is the fake telemt process. +func TestHelperFakeTelemt(t *testing.T) { + optionsPath := os.Getenv(fakeTelemtEnv) + if optionsPath == "" { + t.Skip("helper process for the fake telemt; runs only when re-executed by a test") + } + os.Exit(runFakeTelemt(optionsPath, flag.Args())) +} + +func runFakeTelemt(optionsPath string, args []string) int { + raw, err := os.ReadFile(optionsPath) + if err != nil { + fmt.Fprintln(os.Stderr, "fake telemt: bad options:", err) + return 2 + } + var f fakeTelemt + if err := json.Unmarshal(raw, &f); err != nil { + fmt.Fprintln(os.Stderr, "fake telemt: bad options:", err) + return 2 + } + + if len(args) > 0 && args[0] == "--version" { + version := f.Version + if version == "" { + version = "3.5.14" + } + fmt.Println("telemt " + version) + return 0 + } + + hup := make(chan os.Signal, 1) + signal.Notify(hup, syscall.SIGHUP) + + if f.RecordFile != "" { + cwd, _ := os.Getwd() + payload, _ := json.Marshal(fakeTelemtRecord{Args: args, Cwd: cwd, PID: os.Getpid()}) + tmp := f.RecordFile + ".tmp" + if err := os.WriteFile(tmp, payload, 0o600); err == nil { + _ = os.Rename(tmp, f.RecordFile) + } + } + + for _, line := range f.Stdout { + fmt.Fprintln(os.Stdout, line) + } + for _, line := range f.Stderr { + fmt.Fprintln(os.Stderr, line) + } + + var exit <-chan time.Time + if f.ExitAfter > 0 { + firstRun := true + if f.ExitOnceFile != "" { + if _, err := os.Stat(f.ExitOnceFile); err == nil { + firstRun = false + } else { + _ = os.WriteFile(f.ExitOnceFile, nil, 0o600) + } + } + if firstRun { + exit = time.After(f.ExitAfter) + } + } + + if !f.NoAPI && len(args) > 0 { + go serveFakeTelemtAPI(f, args[len(args)-1]) + } + + lifetime := f.Lifetime + if lifetime <= 0 { + lifetime = time.Minute + } + deadline := time.After(lifetime) + + for { + select { + case <-hup: + if f.SignalFile != "" { + file, err := os.OpenFile(f.SignalFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) + if err == nil { + _, _ = file.WriteString("SIGHUP\n") + _ = file.Close() + } + } + case <-exit: + fmt.Fprintln(os.Stderr, "fake telemt exiting") + return f.ExitCode + case <-deadline: + return 0 + } + } +} + +func serveFakeTelemtAPI(f fakeTelemt, configPath string) { + if f.HealthDelay > 0 { + time.Sleep(f.HealthDelay) + } + + listen, err := fakeTelemtAPIListen(configPath) + if err != nil { + fmt.Fprintln(os.Stderr, "fake telemt:", err) + os.Exit(1) + } + + readyStatus := f.ReadyStatus + if readyStatus == 0 { + readyStatus = http.StatusOK + } + + mux := http.NewServeMux() + mux.HandleFunc("/v1/health", func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"ok":true,"data":{"status":"ok","read_only":false},"revision":"r"}`)) + }) + mux.HandleFunc("/v1/health/ready", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(readyStatus) + if readyStatus == http.StatusOK { + _, _ = w.Write([]byte(`{"ok":true,"data":{"ready":true,"status":"ready"},"revision":"r"}`)) + return + } + _, _ = w.Write([]byte(`{"ok":true,"data":{"ready":false,"status":"not_ready","reason":"no_healthy_upstreams"},"revision":"r"}`)) + }) + + if err := http.ListenAndServe(listen, mux); err != nil { + fmt.Fprintln(os.Stderr, "fake telemt: api:", err) + os.Exit(1) + } +} + +// fakeTelemtAPIListen reads server.api.listen from the rendered config. +func fakeTelemtAPIListen(configPath string) (string, error) { + data, err := os.ReadFile(configPath) + if err != nil { + return "", err + } + section := "" + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "[") { + section = strings.Trim(line, "[]") + continue + } + key, value, ok := strings.Cut(line, "=") + if ok && section == "server.api" && strings.TrimSpace(key) == "listen" { + return strconv.Unquote(strings.TrimSpace(value)) + } + } + return "", errors.New("server.api.listen not found in " + configPath) +} + +// writeFakeTelemt writes a wrapper script that runs the fake telemt with +// options f and returns its path. +func writeFakeTelemt(t *testing.T, f fakeTelemt) string { + t.Helper() + + payload, err := json.Marshal(f) + if err != nil { + t.Fatal(err) + } + testBinary, err := filepath.Abs(os.Args[0]) + if err != nil { + t.Fatal(err) + } + + dir := t.TempDir() + optionsPath := filepath.Join(dir, "options.json") + if err := os.WriteFile(optionsPath, payload, 0o600); err != nil { + t.Fatal(err) + } + + script := fmt.Sprintf("#!/bin/sh\n%s=%s\nexport %s\nexec %s -test.run='^TestHelperFakeTelemt$' -- \"$@\"\n", + fakeTelemtEnv, shellQuote(optionsPath), fakeTelemtEnv, shellQuote(testBinary)) + + path := filepath.Join(dir, "telemt") + if err := os.WriteFile(path, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + return path +} + +func shellQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", `'\''`) + "'" +} + +// readFakeTelemtRecord waits for the fake to write its record file. +func readFakeTelemtRecord(t *testing.T, path string) fakeTelemtRecord { + t.Helper() + + var record fakeTelemtRecord + waitFor(t, 10*time.Second, "the fake telemt record file", func() bool { + data, err := os.ReadFile(path) + if err != nil { + return false + } + return json.Unmarshal(data, &record) == nil + }) + return record +} + +// waitFor polls cond until it holds or the timeout passes. +func waitFor(t *testing.T, timeout time.Duration, what string, cond func() bool) { + t.Helper() + + deadline := time.NewTimer(timeout) + defer deadline.Stop() + ticker := time.NewTicker(20 * time.Millisecond) + defer ticker.Stop() + + for !cond() { + select { + case <-deadline.C: + t.Fatalf("timed out after %s waiting for %s", timeout, what) + case <-ticker.C: + } + } +} + +func testNodeConfig(t *testing.T, executable string) *config.Config { + t.Helper() + return &config.Config{ + TelemtExecutablePath: executable, + GeneratedConfigPath: t.TempDir(), + LogBufferSize: 500, + } +} + +func testMTConfig(t *testing.T) *Config { + t.Helper() + raw := fmt.Sprintf(`{"inbound_tag": "mtproto", "server": {"port": %d, "listeners": [{"ip": "127.0.0.1"}]}}`, netutil.FindFreePort()) + cfg, err := NewConfig(raw) + if err != nil { + t.Fatalf("NewConfig: %v", err) + } + return cfg +} + +// lowerStartupTimeout sets telemtStartupTimeout for one test. Register it +// before the backend's Shutdown cleanup, so the restore runs after the +// supervisor goroutine has exited. +func lowerStartupTimeout(t *testing.T, timeout time.Duration) { + t.Helper() + previous := telemtStartupTimeout + telemtStartupTimeout = timeout + t.Cleanup(func() { telemtStartupTimeout = previous }) +} diff --git a/backend/mtproto/integration_test.go b/backend/mtproto/integration_test.go index c7281b87..3f609af8 100644 --- a/backend/mtproto/integration_test.go +++ b/backend/mtproto/integration_test.go @@ -52,7 +52,9 @@ func integrationMTConfig(t *testing.T) *Config { func TestIntegration_LifecycleAndUsers(t *testing.T) { bin := telemtBinOrSkip(t) - ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second) + // telemt binds its API only after the Telegram DC check (up to about 60 s + // with slow DCs), and New allows it telemtStartupTimeout. + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Second) defer cancel() users := []*common.User{mtUser("alice", "mtproto", testSecret)} @@ -125,7 +127,9 @@ func TestIntegration_LifecycleAndUsers(t *testing.T) { func TestIntegration_EmptyUsersSentinel(t *testing.T) { bin := telemtBinOrSkip(t) - ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second) + // telemt binds its API only after the Telegram DC check (up to about 60 s + // with slow DCs), and New allows it telemtStartupTimeout. + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Second) defer cancel() // Zero real users: telemt rejects an empty [access.users], so the sentinel diff --git a/backend/mtproto/lifecycle.go b/backend/mtproto/lifecycle.go new file mode 100644 index 00000000..7eab018c --- /dev/null +++ b/backend/mtproto/lifecycle.go @@ -0,0 +1,344 @@ +package mtproto + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "slices" + "strings" + "time" + + "github.com/pasarguard/node/common" + "github.com/pasarguard/node/config" + "github.com/pasarguard/node/pkg/netutil" +) + +// telemtStartupTimeout bounds the wait for telemt's API after a start. telemt +// binds its API and metrics only after the STUN probe and the Telegram DC +// connectivity check, which took 19-58 s in tests with slow or unreachable +// DCs. It is a variable so tests can lower it. +var telemtStartupTimeout = 90 * time.Second + +const ( + // telemtReadyTimeout bounds the wait for /v1/health/ready once the API is + // up. Not being ready (no reachable Telegram upstream yet) is only logged: + // Telegram egress problems must not fail a start. + telemtReadyTimeout = 30 * time.Second + // telemtStartupTailSize is how many lines of telemt output are kept for a + // startup error. + telemtStartupTailSize = 50 + // telemtCrashTailSize is how many of those lines are logged after a crash. + telemtCrashTailSize = 10 + + supervisorInitialBackoff = time.Second + supervisorMaxBackoff = 60 * time.Second + + internalPortAttempts = 32 + healthPollInterval = 250 * time.Millisecond + readyPollInterval = 500 * time.Millisecond +) + +var errClosed = errors.New("mtproto backend is shut down") + +func New(ctx context.Context, cfg *config.Config, mtConfig *Config, users []*common.User) (*MTProto, error) { + if cfg == nil { + return nil, errors.New("config is nil") + } + if mtConfig == nil { + return nil, errors.New("mtproto config is nil") + } + + apiPort, metricsPort, err := pickInternalPorts(mtConfig.ListenPorts(), netutil.FindFreePort) + if err != nil { + return nil, err + } + + authHeader, err := generateBearerToken() + if err != nil { + return nil, err + } + sentinel, err := newSentinelUser() + if err != nil { + return nil, err + } + + telemtDir, err := prepareTelemtDir(cfg.GeneratedConfigPath) + if err != nil { + return nil, err + } + process, err := newProcessManager(cfg.TelemtExecutablePath, telemtDir, cfg.LogBufferSize, telemtStartupTailSize) + if err != nil { + return nil, err + } + + activeUsers, _ := ActiveUsersFromCommon(users, mtConfig.InboundTag) + + lifetime, cancel := context.WithCancel(context.Background()) + mt := &MTProto{ + cfg: cfg, + config: mtConfig, + process: process, + configPath: filepath.Join(telemtDir, "config.toml"), + apiClient: newAPIClient(fmt.Sprintf("http://127.0.0.1:%d", apiPort), authHeader), + metricsScraper: newMetricsScraper(fmt.Sprintf("http://127.0.0.1:%d/metrics", metricsPort)), + tracker: newTrafficTracker(), + sentinel: sentinel, + apiPort: apiPort, + metricsPort: metricsPort, + apiAuthHeader: authHeader, + desired: cloneRuntimeUserMap(activeUsers), + lifetime: lifetime, + cancel: cancel, + } + + mt.syncMu.Lock() + err = mt.writeRenderedConfigLocked(mt.desired) + if err == nil { + err = mt.startLocked(ctx) + } + mt.syncMu.Unlock() + if err != nil { + mt.Shutdown() + return nil, err + } + + mt.supervisorDone = make(chan struct{}) + go mt.supervise() + return mt, nil +} + +// Shutdown stops the supervisor and telemt, closes the log stream and removes +// the rendered config, which holds user secrets and the API token. Every +// mutating call afterwards returns errClosed. +func (m *MTProto) Shutdown() { + m.cancel() + + m.syncMu.Lock() + if !m.closed { + m.closed = true + m.process.Close() + if err := os.Remove(m.configPath); err != nil && !errors.Is(err, os.ErrNotExist) { + log.Printf("warning: failed to remove the telemt config: %v", err) + } + } + m.syncMu.Unlock() + + if m.supervisorDone != nil { + <-m.supervisorDone + } +} + +// prepareTelemtDir creates /telemt with owner-only permissions and +// returns it with symlinks resolved: telemt 3.5.8+ opens the config path one +// component at a time with O_NOFOLLOW, so a symlinked directory fails. +func prepareTelemtDir(generatedConfigPath string) (string, error) { + root, err := filepath.Abs(generatedConfigPath) + if err != nil { + return "", err + } + dir := filepath.Join(root, "telemt") + if err := os.MkdirAll(dir, 0o700); err != nil { + return "", fmt.Errorf("failed to create the telemt config directory: %w", err) + } + // MkdirAll keeps the mode of an existing directory. + if err := os.Chmod(dir, 0o700); err != nil { + return "", fmt.Errorf("failed to restrict the telemt config directory: %w", err) + } + resolved, err := filepath.EvalSymlinks(dir) + if err != nil { + return "", fmt.Errorf("failed to resolve the telemt config directory: %w", err) + } + return resolved, nil +} + +// pickInternalPorts picks free ports for telemt's loopback API and metrics +// endpoints, distinct from each other and from the config's client ports. +func pickInternalPorts(listenPorts []int, findFreePort func() int) (apiPort, metricsPort int, err error) { + pick := func(name string, taken ...int) (int, error) { + for range internalPortAttempts { + port := findFreePort() + if port >= 1 && port <= 65535 && !slices.Contains(listenPorts, port) && !slices.Contains(taken, port) { + return port, nil + } + } + return 0, fmt.Errorf("failed to find a free telemt %s port outside the mtproto listen ports %v after %d attempts", + name, listenPorts, internalPortAttempts) + } + + if apiPort, err = pick("API"); err != nil { + return 0, 0, err + } + if metricsPort, err = pick("metrics", apiPort); err != nil { + return 0, 0, err + } + return apiPort, metricsPort, nil +} + +// startLocked starts telemt on the rendered config and waits for its API. +// A failure stops telemt and carries its last output. Requires syncMu. +func (m *MTProto) startLocked(ctx context.Context) error { + if err := m.process.Start(m.configPath); err != nil { + return err + } + if err := m.waitForHealth(ctx); err != nil { + m.process.Stop() + return m.startupError(err) + } + if err := m.waitForReady(ctx, telemtReadyTimeout); err != nil && m.lifetime.Err() == nil { + log.Printf("warning: telemt is running but not ready to serve clients yet: %v", err) + } + return nil +} + +func (m *MTProto) startupError(err error) error { + tail := m.process.StartupTail(telemtStartupTailSize) + if len(tail) == 0 { + return fmt.Errorf("failed to start telemt: %w", err) + } + return fmt.Errorf("failed to start telemt: %w; recent telemt output:\n%s", err, strings.Join(tail, "\n")) +} + +// waitForHealth polls /v1/health until it answers, telemt exits, the backend +// shuts down, or telemtStartupTimeout passes. +func (m *MTProto) waitForHealth(ctx context.Context) error { + timeout := telemtStartupTimeout + waitCtx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + exited := m.process.Exited() + ticker := time.NewTicker(healthPollInterval) + defer ticker.Stop() + + for { + if err := m.apiClient.Health(waitCtx); err == nil { + return nil + } + + select { + case <-exited: + return errors.New("telemt exited before its API became healthy") + case <-m.lifetime.Done(): + return errClosed + case <-waitCtx.Done(): + if err := ctx.Err(); err != nil { + return fmt.Errorf("stopped waiting for the telemt API: %w", err) + } + return fmt.Errorf("telemt API did not become healthy within %s", timeout) + case <-ticker.C: + } + } +} + +// waitForReady polls /v1/health/ready for up to budget and returns the last +// reason telemt gave for not being ready. +func (m *MTProto) waitForReady(ctx context.Context, budget time.Duration) error { + waitCtx, cancel := context.WithTimeout(ctx, budget) + defer cancel() + + ticker := time.NewTicker(readyPollInterval) + defer ticker.Stop() + + var lastErr error + for { + err := m.apiClient.Ready(waitCtx) + if err == nil { + return nil + } + // Keep a real reason over the error of a request the deadline cut off. + if lastErr == nil || waitCtx.Err() == nil { + lastErr = err + } + + select { + case <-m.lifetime.Done(): + return errClosed + case <-waitCtx.Done(): + return fmt.Errorf("not ready after %s: %w", budget, lastErr) + case <-ticker.C: + } + } +} + +// supervise restarts telemt whenever it is found not running, until +// Shutdown. The delay starts at supervisorInitialBackoff and doubles up to +// supervisorMaxBackoff; it resets once a telemt has stayed up that long. +func (m *MTProto) supervise() { + defer close(m.supervisorDone) + + backoff := supervisorInitialBackoff + for { + if exited := m.process.Exited(); exited != nil { + watchedSince := time.Now() + select { + case <-exited: + case <-m.lifetime.Done(): + return + } + if time.Since(watchedSince) >= supervisorMaxBackoff { + backoff = supervisorInitialBackoff + } + } + + closed, running := m.supervisorState() + if closed { + return + } + if running { + // A sync or Restart already started a new telemt. + continue + } + + if tail := m.process.StartupTail(telemtCrashTailSize); len(tail) > 0 { + log.Printf("last %d telemt log lines:\n%s", len(tail), strings.Join(tail, "\n")) + } + log.Printf("telemt is not running, restarting it in %s", backoff) + select { + case <-time.After(backoff): + case <-m.lifetime.Done(): + return + } + + restarted, err := m.restartIfStopped() + switch { + case errors.Is(err, errClosed): + return + case err != nil: + log.Printf("failed to restart telemt: %v", err) + case !restarted: + continue + default: + log.Println("telemt restarted") + } + backoff = min(backoff*2, supervisorMaxBackoff) + } +} + +func (m *MTProto) supervisorState() (closed, running bool) { + m.syncMu.Lock() + defer m.syncMu.Unlock() + return m.closed, m.process.Started() +} + +// restartIfStopped starts telemt again unless it already runs, re-rendering +// the config from the current desired users. +func (m *MTProto) restartIfStopped() (bool, error) { + m.syncMu.Lock() + defer m.syncMu.Unlock() + + if m.closed { + return false, errClosed + } + if m.process.Started() { + return false, nil + } + if err := m.writeRenderedConfigLocked(m.desired); err != nil { + return false, err + } + if err := m.startLocked(m.lifetime); err != nil { + return false, err + } + return true, nil +} diff --git a/backend/mtproto/lifecycle_test.go b/backend/mtproto/lifecycle_test.go new file mode 100644 index 00000000..5517f56c --- /dev/null +++ b/backend/mtproto/lifecycle_test.go @@ -0,0 +1,304 @@ +//go:build !windows + +package mtproto + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "syscall" + "testing" + "time" + + "github.com/pasarguard/node/common" + "github.com/pasarguard/node/config" +) + +// newTestBackend starts the backend against the fake telemt and shuts it +// down when the test ends. +func newTestBackend(t *testing.T, cfg *config.Config, users []*common.User) *MTProto { + t.Helper() + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + mt, err := New(ctx, cfg, testMTConfig(t), users) + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(mt.Shutdown) + return mt +} + +func TestStartFailureIncludesLogTail(t *testing.T) { + exe := writeFakeTelemt(t, fakeTelemt{ + NoAPI: true, + Stderr: []string{"[telemt] Error: Config error: bad listener"}, + ExitAfter: 10 * time.Millisecond, + ExitCode: 1, + }) + + mt, err := New(context.Background(), testNodeConfig(t, exe), testMTConfig(t), nil) + if err == nil { + mt.Shutdown() + t.Fatal("New succeeded although telemt exited") + } + if !strings.Contains(err.Error(), "bad listener") { + t.Errorf("error %q does not include telemt's output", err) + } + if strings.TrimSpace(err.Error()) == "exit status 1" { + t.Error("the startup error is a bare exit status") + } +} + +// TestStartTimeoutIncludesLogTail covers telemt that never binds its API +// (Telegram unreachable): New gives up after telemtStartupTimeout, reports +// the output so far, and leaves no telemt running. +func TestStartTimeoutIncludesLogTail(t *testing.T) { + // Long enough for the fake (a race-instrumented test binary) to start and + // print its line before the deadline. + lowerStartupTimeout(t, 5*time.Second) + record := filepath.Join(t.TempDir(), "record.json") + exe := writeFakeTelemt(t, fakeTelemt{ + NoAPI: true, + RecordFile: record, + Stderr: []string{"checking Telegram DC connectivity"}, + }) + + mt, err := New(context.Background(), testNodeConfig(t, exe), testMTConfig(t), nil) + if err == nil { + mt.Shutdown() + t.Fatal("New succeeded although the telemt API never came up") + } + for _, want := range []string{"did not become healthy within 5s", "checking Telegram DC connectivity"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error %q does not contain %q", err, want) + } + } + + pid := readFakeTelemtRecord(t, record).PID + if err := syscall.Kill(pid, 0); !errors.Is(err, syscall.ESRCH) { + t.Errorf("telemt %d is still running after the failed start (kill 0: %v)", pid, err) + } +} + +func TestStartWaitsForSlowAPI(t *testing.T) { + // The API binds 2 s after the fake starts; the timeout leaves room for a + // slow start of the fake itself under -race. + lowerStartupTimeout(t, 10*time.Second) + exe := writeFakeTelemt(t, fakeTelemt{HealthDelay: 2 * time.Second}) + + mt := newTestBackend(t, testNodeConfig(t, exe), nil) + if !mt.Started() { + t.Fatal("backend not started") + } + if got := mt.Version(); got != "3.5.14" { + t.Errorf("Version() = %q, want 3.5.14", got) + } +} + +func TestGeneratedDirSymlinkResolved(t *testing.T) { + realDir := t.TempDir() + link := filepath.Join(t.TempDir(), "generated") + if err := os.Symlink(realDir, link); err != nil { + t.Fatal(err) + } + record := filepath.Join(t.TempDir(), "record.json") + cfg := testNodeConfig(t, writeFakeTelemt(t, fakeTelemt{RecordFile: record})) + cfg.GeneratedConfigPath = link + + newTestBackend(t, cfg, nil) + + args := readFakeTelemtRecord(t, record).Args + if len(args) != 2 { + t.Fatalf("telemt argv = %q", args) + } + resolvedRoot, err := filepath.EvalSymlinks(realDir) + if err != nil { + t.Fatal(err) + } + if want := filepath.Join(resolvedRoot, "telemt", "config.toml"); args[1] != want { + t.Errorf("config path = %q, want %q", args[1], want) + } + if resolved, err := filepath.EvalSymlinks(args[1]); err != nil || resolved != args[1] { + t.Errorf("config path %q still has a symlink component (resolves to %q, %v)", args[1], resolved, err) + } +} + +func TestTelemtDirIsPrivate(t *testing.T) { + cfg := testNodeConfig(t, writeFakeTelemt(t, fakeTelemt{})) + telemtDir := filepath.Join(cfg.GeneratedConfigPath, "telemt") + // A directory left by an older node build may be world readable. + if err := os.MkdirAll(telemtDir, 0o755); err != nil { + t.Fatal(err) + } + + newTestBackend(t, cfg, nil) + + info, err := os.Stat(telemtDir) + if err != nil { + t.Fatal(err) + } + if mode := info.Mode().Perm(); mode != 0o700 { + t.Errorf("telemt dir mode = %o, want 700", mode) + } +} + +func TestShutdownRemovesRenderedConfig(t *testing.T) { + cfg := testNodeConfig(t, writeFakeTelemt(t, fakeTelemt{})) + mt := newTestBackend(t, cfg, []*common.User{mtUser("alice", "mtproto", testSecret)}) + + configPath := mt.configPath + if _, err := os.Stat(configPath); err != nil { + t.Fatalf("rendered config missing while running: %v", err) + } + + mt.Shutdown() + + if _, err := os.Stat(configPath); !errors.Is(err, os.ErrNotExist) { + t.Errorf("config.toml still exists after Shutdown (stat: %v)", err) + } + if mt.Started() { + t.Error("Started() is true after Shutdown") + } + mt.Shutdown() // a second Shutdown is a no-op +} + +// TestSyncAfterShutdownReturnsErrClosed is the regression for a sync that +// reached a shut-down backend, started a new telemt and crashed the node +// with "send on closed channel". +func TestSyncAfterShutdownReturnsErrClosed(t *testing.T) { + cfg := testNodeConfig(t, writeFakeTelemt(t, fakeTelemt{})) + mt := newTestBackend(t, cfg, nil) + mt.Shutdown() + + ctx := context.Background() + users := []*common.User{mtUser("alice", "mtproto", testSecret)} + calls := map[string]func() error{ + "SyncUsers": func() error { return mt.SyncUsers(ctx, users) }, + "UpdateUsers": func() error { return mt.UpdateUsers(ctx, users) }, + "SyncUser": func() error { return mt.SyncUser(ctx, users[0]) }, + "UpdateUsersAndRestart": func() error { return mt.UpdateUsersAndRestart(ctx, users) }, + "Restart": mt.Restart, + } + for name, call := range calls { + if err := call(); !errors.Is(err, errClosed) { + t.Errorf("%s after Shutdown = %v, want errClosed", name, err) + } + } + + if mt.Started() || mt.process.Generation() != 1 { + t.Errorf("a telemt was started after Shutdown (started %v, generation %d)", mt.Started(), mt.process.Generation()) + } + if _, err := os.Stat(mt.configPath); !errors.Is(err, os.ErrNotExist) { + t.Errorf("a call after Shutdown wrote the config again (stat: %v)", err) + } +} + +func TestSupervisorRestartsCrashedTelemt(t *testing.T) { + exe := writeFakeTelemt(t, fakeTelemt{ + ExitAfter: time.Second, + ExitCode: 1, + ExitOnceFile: filepath.Join(t.TempDir(), "crashed-once"), + }) + mt := newTestBackend(t, testNodeConfig(t, exe), nil) + if mt.process.Generation() != 1 { + t.Fatalf("Generation() = %d after New, want 1", mt.process.Generation()) + } + + waitFor(t, 10*time.Second, "the supervisor to restart telemt", func() bool { + return mt.Started() && mt.process.Generation() == 2 + }) + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + logs := mt.SubscribeLogs(ctx) + for { + select { + case line := <-logs: + if strings.HasPrefix(line, "telemt exited unexpectedly: exit status 1") { + return + } + case <-ctx.Done(): + t.Fatal(`no "telemt exited unexpectedly" line was published`) + } + } +} + +func TestPickInternalPorts(t *testing.T) { + sequence := func(ports ...int) func() int { + return func() int { + port := ports[0] + if len(ports) > 1 { + ports = ports[1:] + } + return port + } + } + + apiPort, metricsPort, err := pickInternalPorts([]int{443, 8443}, sequence(443, 65536, 20000, 20000, 8443, 30000)) + if err != nil { + t.Fatalf("pickInternalPorts: %v", err) + } + if apiPort != 20000 || metricsPort != 30000 { + t.Errorf("ports = %d, %d; want 20000, 30000", apiPort, metricsPort) + } + + if _, _, err := pickInternalPorts([]int{443}, sequence(443)); err == nil || !strings.Contains(err.Error(), "API port") { + t.Errorf("an endless clash with the listen port must fail naming the API port, got %v", err) + } + if _, _, err := pickInternalPorts([]int{443}, sequence(20000)); err == nil || !strings.Contains(err.Error(), "metrics port") { + t.Errorf("an endless clash with the API port must fail naming the metrics port, got %v", err) + } +} + +func TestAPIClientReady(t *testing.T) { + status := http.StatusServiceUnavailable + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/v1/health/ready" { + http.NotFound(w, r) + return + } + w.WriteHeader(status) + if status == http.StatusOK { + _, _ = w.Write([]byte(`{"ok":true,"data":{"ready":true,"status":"ready"},"revision":"r"}`)) + return + } + _, _ = w.Write([]byte(`{"ok":true,"data":{"ready":false,"status":"not_ready","reason":"admission_closed"},"revision":"r"}`)) + })) + defer server.Close() + + client := newAPIClient(server.URL, "Bearer test") + err := client.Ready(context.Background()) + if err == nil || !strings.Contains(err.Error(), "admission_closed") { + t.Errorf("Ready on 503 = %v, want an error naming the reason", err) + } + + status = http.StatusOK + if err := client.Ready(context.Background()); err != nil { + t.Errorf("Ready on 200 = %v, want nil", err) + } +} + +// TestWaitForReadyGivesUp checks the readiness wait is bounded and reports +// the reason; New only logs this, so Start never fails on Telegram egress. +func TestWaitForReadyGivesUp(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + _, _ = w.Write([]byte(`{"ok":true,"data":{"ready":false,"reason":"no_healthy_upstreams"},"revision":"r"}`)) + })) + defer server.Close() + + lifetime, cancel := context.WithCancel(context.Background()) + defer cancel() + mt := &MTProto{apiClient: newAPIClient(server.URL, ""), lifetime: lifetime} + + err := mt.waitForReady(context.Background(), 300*time.Millisecond) + if err == nil || !strings.Contains(err.Error(), "no_healthy_upstreams") { + t.Errorf("waitForReady = %v, want an error naming the reason", err) + } +} diff --git a/backend/mtproto/logtail.go b/backend/mtproto/logtail.go new file mode 100644 index 00000000..fad483bb --- /dev/null +++ b/backend/mtproto/logtail.go @@ -0,0 +1,59 @@ +package mtproto + +import "sync" + +// lineRing keeps the last lines telemt wrote, so startup and crash errors can +// show them. +type lineRing struct { + mu sync.Mutex + lines []string + next int + full bool +} + +func newLineRing(size int) *lineRing { + if size <= 0 { + size = 1 + } + return &lineRing{lines: make([]string, size)} +} + +func (r *lineRing) reset() { + r.mu.Lock() + defer r.mu.Unlock() + r.next = 0 + r.full = false + clear(r.lines) +} + +func (r *lineRing) add(line string) { + r.mu.Lock() + defer r.mu.Unlock() + r.lines[r.next] = line + r.next = (r.next + 1) % len(r.lines) + if r.next == 0 { + r.full = true + } +} + +// tail returns the last n lines, oldest first; n <= 0 returns all of them. +func (r *lineRing) tail(n int) []string { + r.mu.Lock() + defer r.mu.Unlock() + + count := r.next + oldest := 0 + if r.full { + count = len(r.lines) + oldest = r.next + } + if n <= 0 || n > count { + n = count + } + + out := make([]string, 0, n) + for i := count - n; i < count; i++ { + out = append(out, r.lines[(oldest+i)%len(r.lines)]) + } + return out +} diff --git a/backend/mtproto/mtproto.go b/backend/mtproto/mtproto.go index c05fab3f..28224012 100644 --- a/backend/mtproto/mtproto.go +++ b/backend/mtproto/mtproto.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "errors" "fmt" + "log" "os" "path/filepath" "strings" @@ -16,17 +17,13 @@ import ( "github.com/pasarguard/node/common" "github.com/pasarguard/node/config" - "github.com/pasarguard/node/pkg/netutil" -) - -const ( - telemtStartupTimeout = 30 * time.Second ) type MTProto struct { cfg *config.Config config *Config process *processManager + configPath string apiClient *apiClient metricsScraper *metricsScraper tracker *trafficTracker @@ -36,78 +33,19 @@ type MTProto struct { metricsPort int apiAuthHeader string - mu sync.RWMutex - desired map[string]*runtimeUser - shutdownOnce sync.Once -} + // syncMu serializes every operation that changes users or the telemt + // process: syncs, restarts, the supervisor and Shutdown. + syncMu sync.Mutex + closed bool // guarded by syncMu -func New(ctx context.Context, cfg *config.Config, mtConfig *Config, users []*common.User) (*MTProto, error) { - if cfg == nil { - return nil, errors.New("config is nil") - } - if mtConfig == nil { - return nil, errors.New("mtproto config is nil") - } - - apiPort := netutil.FindFreePort() - metricsPort := netutil.FindFreePort() - for metricsPort == apiPort { - metricsPort = netutil.FindFreePort() - } - - authHeader, err := generateBearerToken() - if err != nil { - return nil, err - } - - generatedRoot, err := filepath.Abs(cfg.GeneratedConfigPath) - if err != nil { - return nil, err - } - telemtDir := filepath.Join(generatedRoot, "telemt") - process, err := newProcessManager( - cfg.TelemtExecutablePath, - filepath.Join(telemtDir, "config.toml"), - filepath.Join(telemtDir, "telemt.pid"), - cfg.LogBufferSize, - ) - if err != nil { - return nil, err - } - - activeUsers, _ := ActiveUsersFromCommon(users, mtConfig.InboundTag) - - sentinel, err := newSentinelUser() - if err != nil { - return nil, err - } - - mt := &MTProto{ - cfg: cfg, - config: mtConfig, - process: process, - apiClient: newAPIClient(fmt.Sprintf("http://127.0.0.1:%d", apiPort), authHeader), - metricsScraper: newMetricsScraper(fmt.Sprintf("http://127.0.0.1:%d/metrics", metricsPort)), - tracker: newTrafficTracker(), - sentinel: sentinel, - apiPort: apiPort, - metricsPort: metricsPort, - apiAuthHeader: authHeader, - desired: cloneRuntimeUserMap(activeUsers), - } + // mu guards desired for readers; writers also hold syncMu. + mu sync.RWMutex + desired map[string]*runtimeUser - if err := mt.writeRenderedConfigLocked(mt.desired); err != nil { - return nil, err - } - if err := mt.process.Start(); err != nil { - return nil, err - } - if err := mt.waitForHealth(ctx); err != nil { - mt.Shutdown() - return nil, err - } - - return mt, nil + // lifetime is cancelled by Shutdown so waits for telemt stop early. + lifetime context.Context + cancel context.CancelFunc + supervisorDone chan struct{} } func (m *MTProto) Started() bool { @@ -123,24 +61,17 @@ func (m *MTProto) SubscribeLogs(ctx context.Context) <-chan string { } func (m *MTProto) Restart() error { - m.mu.Lock() - defer m.mu.Unlock() + m.syncMu.Lock() + defer m.syncMu.Unlock() - if err := m.writeRenderedConfigLocked(m.desired); err != nil { - return err + if m.closed { + return errClosed } - if err := m.process.Restart(); err != nil { + if err := m.writeRenderedConfigLocked(m.desired); err != nil { return err } - - return m.waitForHealth(context.Background()) -} - -func (m *MTProto) Shutdown() { - m.shutdownOnce.Do(func() { - m.process.Shutdown() - m.process.logs.Close() - }) + m.process.Stop() + return m.startLocked(m.lifetime) } func (m *MTProto) SyncUser(ctx context.Context, user *common.User) error { @@ -153,9 +84,12 @@ func (m *MTProto) SyncUser(ctx context.Context, user *common.User) error { } func (m *MTProto) SyncUsers(ctx context.Context, users []*common.User) error { - m.mu.Lock() - defer m.mu.Unlock() + m.syncMu.Lock() + defer m.syncMu.Unlock() + if m.closed { + return errClosed + } desired, _ := ActiveUsersFromCommon(users, m.config.InboundTag) return m.applyFullDesiredLocked(ctx, desired) @@ -166,9 +100,12 @@ func (m *MTProto) UpdateUsers(ctx context.Context, users []*common.User) error { } func (m *MTProto) UpdateUsersAndRestart(ctx context.Context, users []*common.User) error { - m.mu.Lock() - defer m.mu.Unlock() + m.syncMu.Lock() + defer m.syncMu.Unlock() + if m.closed { + return errClosed + } nextDesired := m.buildNextDesiredLocked(users) return m.applyFullDesiredLocked(ctx, nextDesired) @@ -184,7 +121,7 @@ func (m *MTProto) GetSysStats(_ context.Context) (*common.BackendStatsResponse, return nil, errors.New("telemt process pid is unavailable") } - process, err := gopsutilprocess.NewProcess(pid) + process, err := gopsutilprocess.NewProcess(int32(pid)) if err != nil { return nil, err } @@ -215,8 +152,9 @@ func (m *MTProto) GetStats(ctx context.Context, request *common.StatRequest) (*c return nil, errors.New("mtproto backend is not started") } - // Prometheus only binds after telemt's data plane is up (Telegram egress). - // Treat a scrape miss as empty stats rather than failing the panel poll. + // telemt binds metrics together with its API, after the Telegram DC + // check. Treat a scrape miss (telemt restarting, say) as empty stats + // rather than failing the panel poll. if err := m.refreshTracker(ctx); err != nil { return &common.StatResponse{Stats: []*common.Stat{}}, nil } @@ -306,9 +244,12 @@ func (m *MTProto) GetUserOnlineIpListStats(ctx context.Context, email string) (* } func (m *MTProto) updateUsersPartial(ctx context.Context, users []*common.User) error { - m.mu.Lock() - defer m.mu.Unlock() + m.syncMu.Lock() + defer m.syncMu.Unlock() + if m.closed { + return errClosed + } nextDesired := m.buildNextDesiredLocked(users) if len(m.desired) == 0 || len(nextDesired) == 0 { @@ -319,7 +260,7 @@ func (m *MTProto) updateUsersPartial(ctx context.Context, users []*common.User) return m.applyFullDesiredLocked(ctx, nextDesired) } - m.desired = cloneRuntimeUserMap(nextDesired) + m.setDesiredLocked(nextDesired) return nil } @@ -410,50 +351,47 @@ func (m *MTProto) applyFullDesiredLocked(ctx context.Context, desired map[string } if !m.process.Started() { - if err := m.process.Start(); err != nil { + if err := m.startLocked(ctx); err != nil { return err } - if err := m.waitForHealth(ctx); err != nil { - m.process.Shutdown() - return err - } - if err := m.waitForRuntimeUsers(ctx, previousDesired, desired); err != nil { - return err - } - m.desired = cloneRuntimeUserMap(desired) - return nil + } else if err := m.reloadLocked(ctx); err != nil { + return err } - if err := m.process.Reload(); err != nil { - if restartErr := m.process.Restart(); restartErr != nil { - return fmt.Errorf("telemt reload failed: %w (restart failed: %v)", err, restartErr) - } - if healthErr := m.waitForHealth(ctx); healthErr != nil { - return healthErr - } - if runtimeErr := m.waitForRuntimeUsers(ctx, previousDesired, desired); runtimeErr != nil { - return runtimeErr - } - m.desired = cloneRuntimeUserMap(desired) - return nil + if err := m.waitForRuntimeUsers(ctx, previousDesired, desired); err != nil { + return err } - if err := m.waitForHealth(ctx); err != nil { - if restartErr := m.process.Restart(); restartErr != nil { - return fmt.Errorf("telemt reload health check failed: %w (restart failed: %v)", err, restartErr) - } - if healthErr := m.waitForHealth(ctx); healthErr != nil { - return healthErr - } + m.setDesiredLocked(desired) + return nil +} + +// reloadLocked makes the running telemt re-read the rendered config and +// restarts it when the reload fails. Requires syncMu. +func (m *MTProto) reloadLocked(ctx context.Context) error { + err := m.process.Reload() + if err == nil { + err = m.waitForHealth(ctx) } - if err := m.waitForRuntimeUsers(ctx, previousDesired, desired); err != nil { + if err == nil || errors.Is(err, errClosed) { return err } - m.desired = cloneRuntimeUserMap(desired) + log.Printf("telemt reload failed, restarting it: %v", err) + m.process.Stop() + if restartErr := m.startLocked(ctx); restartErr != nil { + return fmt.Errorf("telemt reload failed: %w (restart failed: %v)", err, restartErr) + } return nil } +// setDesiredLocked replaces the desired users. Requires syncMu. +func (m *MTProto) setDesiredLocked(desired map[string]*runtimeUser) { + m.mu.Lock() + defer m.mu.Unlock() + m.desired = cloneRuntimeUserMap(desired) +} + func (m *MTProto) writeRenderedConfigLocked(desired map[string]*runtimeUser) error { var sentinel *runtimeUser if len(desired) == 0 { @@ -465,38 +403,7 @@ func (m *MTProto) writeRenderedConfigLocked(desired map[string]*runtimeUser) err return err } - return writeFileAtomic(m.process.configPath, []byte(rendered)) -} - -func (m *MTProto) waitForHealth(ctx context.Context) error { - waitContext := ctx - if _, hasDeadline := ctx.Deadline(); !hasDeadline { - var cancel context.CancelFunc - waitContext, cancel = context.WithTimeout(ctx, telemtStartupTimeout) - defer cancel() - } - - ticker := time.NewTicker(500 * time.Millisecond) - defer ticker.Stop() - - waitCh := m.process.WaitChan() - - for { - if err := m.apiClient.Health(waitContext); err == nil { - return nil - } - - select { - case <-waitContext.Done(): - return waitContext.Err() - case err := <-waitCh: - if err == nil { - return errors.New("telemt process exited before becoming healthy") - } - return fmt.Errorf("telemt process exited before becoming healthy: %w", err) - case <-ticker.C: - } - } + return writeFileAtomic(m.configPath, []byte(rendered)) } func (m *MTProto) waitForRuntimeUsers(ctx context.Context, previousDesired, desired map[string]*runtimeUser) error { @@ -563,6 +470,8 @@ func (m *MTProto) waitForRuntimeUsers(ctx context.Context, previousDesired, desi select { case <-waitContext.Done(): return waitContext.Err() + case <-m.lifetime.Done(): + return errClosed case <-ticker.C: } } diff --git a/backend/mtproto/mtproto_test.go b/backend/mtproto/mtproto_test.go index 549862a1..0d653897 100644 --- a/backend/mtproto/mtproto_test.go +++ b/backend/mtproto/mtproto_test.go @@ -3,8 +3,6 @@ package mtproto import ( "context" "encoding/json" - "os" - "os/exec" "strings" "testing" @@ -174,7 +172,7 @@ func TestGetStats_MetricsUnavailable(t *testing.T) { func metricsUnavailableBackend() *MTProto { return &MTProto{ config: &Config{InboundTag: "mtproto"}, - process: &processManager{cmd: &exec.Cmd{Process: &os.Process{Pid: 1}}}, + process: &processManager{run: &telemtRun{pid: 1, waitDone: make(chan struct{})}}, metricsScraper: newMetricsScraper("http://127.0.0.1:1/metrics"), tracker: newTrafficTracker(), desired: map[string]*runtimeUser{"alice": {Username: "alice", Secret: testSecret}}, diff --git a/backend/mtproto/process.go b/backend/mtproto/process.go index 89a20e6f..491628e3 100644 --- a/backend/mtproto/process.go +++ b/backend/mtproto/process.go @@ -3,100 +3,143 @@ package mtproto import ( "bufio" "context" + "errors" "fmt" "io" + "log" "os" "os/exec" "path/filepath" + "regexp" + "slices" + "strconv" "strings" "sync" + "sync/atomic" "time" + gopsutilprocess "github.com/shirou/gopsutil/v4/process" + "github.com/pasarguard/node/pkg/logstream" ) -// telemtCommandTimeout bounds the short-lived telemt invocations (--version, -// reload) so a wedged binary cannot block startup or a live reload forever. -const telemtCommandTimeout = 30 * time.Second +const ( + // telemtCommandTimeout bounds telemt --version so a wedged binary cannot + // block startup forever. + telemtCommandTimeout = 30 * time.Second + // telemtStopTimeout is how long Stop waits after SIGTERM, and again after + // SIGKILL, for telemt to exit. telemt's graceful shutdown allows 5 s for + // the middle end and 5 s for the control plane. + telemtStopTimeout = 15 * time.Second + // orphanExitTimeout bounds the wait for a killed orphan to go away. + orphanExitTimeout = 5 * time.Second + // maxTelemtLogLine caps one line of telemt output; the rest of a longer + // line is skipped. + maxTelemtLogLine = 64 * 1024 +) + +// minTelemtVersion is the oldest telemt the node runs: the release the +// backend is written and tested against. Older releases differ in ways the +// node cannot detect at runtime, such as the per-user octet metric names +// (renamed in 3.5.4), which would make traffic read as zero. +var minTelemtVersion = [3]int{3, 5, 14} -type commandFactory func(executable string, args ...string) *exec.Cmd +var telemtVersionPattern = regexp.MustCompile(`(?i)\btelemt\s+v?(\d+)\.(\d+)\.(\d+)(\S*)`) + +var errProcessManagerClosed = errors.New("telemt process manager is closed") + +// telemtRun is one started telemt process. +type telemtRun struct { + cmd *exec.Cmd + pid int + waitDone chan struct{} // closed once the process is reaped and its output read + stopping atomic.Bool // set by Stop, so the exit is not reported as a crash +} type processManager struct { executablePath string - configPath string - pidFilePath string - logs *logstream.Buffer + workDir string version string - commandFactory commandFactory + logs *logstream.Buffer + startupTail *lineRing - mu sync.RWMutex - cmd *exec.Cmd - waitCh chan error + mu sync.Mutex + run *telemtRun + generation uint64 + closed bool } -func newProcessManager(executablePath, configPath, pidFilePath string, logBufferSize int) (*processManager, error) { +// newProcessManager checks the telemt binary and returns a manager that runs +// it with workDir as its working directory. telemt older than 3.5.14 is +// refused. +func newProcessManager(executablePath, workDir string, logBufferSize, startupTailSize int) (*processManager, error) { if strings.TrimSpace(executablePath) == "" { - return nil, fmt.Errorf("telemt executable path must not be empty") + return nil, errors.New("telemt executable path must not be empty") } absExecutablePath, err := filepath.Abs(executablePath) if err != nil { return nil, err } - absConfigPath, err := filepath.Abs(configPath) + absWorkDir, err := filepath.Abs(workDir) if err != nil { return nil, err } - absPIDFilePath, err := filepath.Abs(pidFilePath) + + version, err := detectTelemtVersion(absExecutablePath) if err != nil { return nil, err } - manager := &processManager{ + return &processManager{ executablePath: absExecutablePath, - configPath: absConfigPath, - pidFilePath: absPIDFilePath, + workDir: absWorkDir, + version: version, logs: logstream.NewBuffer(logBufferSize), - commandFactory: func(executable string, args ...string) *exec.Cmd { - return exec.Command(executable, args...) - }, - } - - manager.version, err = manager.detectVersion() - if err != nil { - return nil, err - } - - return manager, nil + startupTail: newLineRing(startupTailSize), + }, nil } -func (p *processManager) detectVersion() (string, error) { +func detectTelemtVersion(executablePath string) (string, error) { ctx, cancel := context.WithTimeout(context.Background(), telemtCommandTimeout) defer cancel() - cmd := exec.CommandContext(ctx, p.executablePath, "--version") - output, err := cmd.CombinedOutput() + output, err := exec.CommandContext(ctx, executablePath, "--version").CombinedOutput() if err != nil { if ctx.Err() == context.DeadlineExceeded { return "", fmt.Errorf("telemt --version timed out after %s: %w", telemtCommandTimeout, ctx.Err()) } return "", fmt.Errorf("failed to detect telemt version: %w", err) } + return parseTelemtVersion(string(output)) +} + +// parseTelemtVersion reads `telemt X.Y.Z` and refuses versions below +// minTelemtVersion. A suffix such as -rc.1 is kept but not compared. +func parseTelemtVersion(output string) (string, error) { + match := telemtVersionPattern.FindStringSubmatch(output) + if match == nil { + return "", fmt.Errorf("failed to parse telemt version from %q", strings.TrimSpace(output)) + } - line := strings.TrimSpace(string(output)) - fields := strings.Fields(line) - if len(fields) >= 2 && strings.EqualFold(fields[0], "telemt") { - return fields[1], nil + var found [3]int + for i := range found { + value, err := strconv.Atoi(match[i+1]) + if err != nil { + return "", fmt.Errorf("failed to parse telemt version from %q: %w", strings.TrimSpace(output), err) + } + found[i] = value } - if line == "" { - return "", fmt.Errorf("failed to detect telemt version: empty output") + + version := fmt.Sprintf("%d.%d.%d%s", found[0], found[1], found[2], match[4]) + if slices.Compare(found[:], minTelemtVersion[:]) < 0 { + return "", fmt.Errorf("telemt %s is not supported: the node requires telemt %d.%d.%d or newer", + version, minTelemtVersion[0], minTelemtVersion[1], minTelemtVersion[2]) } - return line, nil + return version, nil } func (p *processManager) Version() string { - p.mu.RLock() - defer p.mu.RUnlock() return p.version } @@ -104,26 +147,28 @@ func (p *processManager) SubscribeLogs(ctx context.Context) <-chan string { return p.logs.Subscribe(ctx) } -func (p *processManager) Start() error { +// Start runs `telemt run `. Any telemt still running for the same +// config path (left over from a killed node) is killed first. +func (p *processManager) Start(configPath string) error { p.mu.Lock() defer p.mu.Unlock() + if p.closed { + return errProcessManagerClosed + } if p.startedLocked() { - return fmt.Errorf("telemt process is already running") + return errors.New("telemt process is already running") } - if err := os.MkdirAll(filepath.Dir(p.configPath), 0o755); err != nil { - return fmt.Errorf("failed to create telemt config directory: %w", err) - } - if err := os.MkdirAll(filepath.Dir(p.pidFilePath), 0o755); err != nil { - return fmt.Errorf("failed to create telemt pid directory: %w", err) - } + cleanupOrphanedTelemt(configPath) - cmd := p.commandFactory(p.executablePath, "run", "--pid-file", p.pidFilePath, p.configPath) - // Pin telemt's working directory to the generated-config dir so its - // relative-path artifacts (e.g. the tls_front cache) stay contained there - // instead of littering the node's working directory. - cmd.Dir = filepath.Dir(p.configPath) + // No --pid-file: a pid file that survives a hard kill can name a live + // process later and make telemt refuse to start. The node reloads by + // signalling its own child instead. + cmd := exec.Command(p.executablePath, "run", configPath) + // telemt writes relative-path artifacts (the tls_front cache, for one) + // into its working directory; keep them in the generated config dir. + cmd.Dir = p.workDir setProcessAttributes(cmd) stdout, err := cmd.StdoutPipe() @@ -135,95 +180,245 @@ func (p *processManager) Start() error { return err } + p.startupTail.reset() if err := cmd.Start(); err != nil { - return err + return fmt.Errorf("failed to start telemt: %w", err) } - waitCh := make(chan error, 1) - go func() { - waitCh <- cmd.Wait() - close(waitCh) - }() - - go p.captureLogs(stdout) - go p.captureLogs(stderr) + run := &telemtRun{ + cmd: cmd, + pid: cmd.Process.Pid, + waitDone: make(chan struct{}), + } + p.run = run + p.generation++ - p.cmd = cmd - p.waitCh = waitCh + go p.wait(run, stdout, stderr) return nil } -func (p *processManager) WaitChan() <-chan error { - p.mu.RLock() - defer p.mu.RUnlock() - return p.waitCh +// wait reads telemt's output to EOF, then reaps the process. os/exec closes +// the pipes in Wait, so calling it while the readers run can lose the last +// lines of a crash. +func (p *processManager) wait(run *telemtRun, stdout, stderr io.Reader) { + var readers sync.WaitGroup + readers.Add(2) + go func() { + defer readers.Done() + p.captureLogs(stdout) + }() + go func() { + defer readers.Done() + p.captureLogs(stderr) + }() + readers.Wait() + + err := run.cmd.Wait() + if !run.stopping.Load() { + message := "telemt exited unexpectedly" + if err != nil { + message = fmt.Sprintf("%s: %v", message, err) + if strings.Contains(strings.ToLower(err.Error()), "signal: killed") { + message += " (process was killed externally; check container/system OOM and memory limits)" + } + } else { + message += " with exit status 0" + } + log.Println(message) + p.publish(message) + } + close(run.waitDone) } -func (p *processManager) PID() int32 { - p.mu.RLock() - defer p.mu.RUnlock() - if p.cmd == nil || p.cmd.Process == nil { - return 0 +func (p *processManager) captureLogs(pipe io.Reader) { + reader := bufio.NewReaderSize(pipe, maxTelemtLogLine) + for { + line, err := reader.ReadSlice('\n') + if errors.Is(err, bufio.ErrBufferFull) { + p.publish(string(line) + " [truncated]") + for errors.Is(err, bufio.ErrBufferFull) { + _, err = reader.ReadSlice('\n') + } + if err != nil { + return + } + continue + } + if text := strings.TrimRight(string(line), "\r\n"); text != "" || err == nil { + p.publish(text) + } + if err != nil { + return + } } - return int32(p.cmd.Process.Pid) +} + +func (p *processManager) publish(line string) { + p.startupTail.add(line) + p.logs.Publish(line) } func (p *processManager) Started() bool { - p.mu.RLock() - defer p.mu.RUnlock() + p.mu.Lock() + defer p.mu.Unlock() return p.startedLocked() } +// startedLocked requires p.mu. It checks waitDone, never cmd.ProcessState, +// which Wait writes without the lock. func (p *processManager) startedLocked() bool { - return p.cmd != nil && p.cmd.Process != nil && p.cmd.ProcessState == nil + if p.run == nil { + return false + } + select { + case <-p.run.waitDone: + return false + default: + return true + } +} + +// PID returns the running telemt's pid, or 0. +func (p *processManager) PID() int { + p.mu.Lock() + defer p.mu.Unlock() + if !p.startedLocked() { + return 0 + } + return p.run.pid +} + +// Exited returns a channel closed when the current telemt exits, or nil when +// none was started (or it was stopped). +func (p *processManager) Exited() <-chan struct{} { + p.mu.Lock() + defer p.mu.Unlock() + if p.run == nil { + return nil + } + return p.run.waitDone +} + +// Generation counts successful Starts. +func (p *processManager) Generation() uint64 { + p.mu.Lock() + defer p.mu.Unlock() + return p.generation +} + +// StartupTail returns the last n lines (all kept lines when n <= 0) telemt +// wrote since the latest Start, including the exit message after a crash. +func (p *processManager) StartupTail(n int) []string { + return p.startupTail.tail(n) } +// Reload asks the running telemt to re-read its config (SIGHUP). func (p *processManager) Reload() error { - ctx, cancel := context.WithTimeout(context.Background(), telemtCommandTimeout) - defer cancel() + p.mu.Lock() + defer p.mu.Unlock() + if !p.startedLocked() { + return errors.New("telemt process is not running") + } + if err := reloadProcess(p.run.cmd); err != nil { + return fmt.Errorf("failed to signal telemt to reload: %w", err) + } + return nil +} - cmd := exec.CommandContext(ctx, p.executablePath, "reload", "--pid-file", p.pidFilePath) - output, err := cmd.CombinedOutput() - if err != nil { - if ctx.Err() == context.DeadlineExceeded { - return fmt.Errorf("telemt reload timed out after %s: %w", telemtCommandTimeout, ctx.Err()) +// Stop terminates telemt and waits for it to exit, killing it after +// telemtStopTimeout. +func (p *processManager) Stop() { + p.mu.Lock() + run := p.run + p.mu.Unlock() + if run == nil { + return + } + + run.stopping.Store(true) + select { + case <-run.waitDone: + default: + _ = terminateProcess(run.cmd) + select { + case <-run.waitDone: + case <-time.After(telemtStopTimeout): + log.Printf("telemt process %d did not exit within %s, killing it", run.pid, telemtStopTimeout) + _ = killProcess(run.cmd) + select { + case <-run.waitDone: + case <-time.After(telemtStopTimeout): + log.Printf("warning: telemt process %d is still running after SIGKILL", run.pid) + } } - return fmt.Errorf("failed to reload telemt: %w (%s)", err, strings.TrimSpace(string(output))) } - return nil + + p.mu.Lock() + if p.run == run { + p.run = nil + } + p.mu.Unlock() } -func (p *processManager) Shutdown() { +// Close stops telemt and closes the log buffer. Start fails afterwards. +func (p *processManager) Close() { p.mu.Lock() - cmd := p.cmd - waitCh := p.waitCh - p.cmd = nil - p.waitCh = nil + p.closed = true p.mu.Unlock() - if cmd == nil { + p.Stop() + p.logs.Close() +} + +// cleanupOrphanedTelemt kills telemt processes still running configPath, for +// example after the node was SIGKILLed. Only a command line that runs exactly +// this config matches, so nothing else is ever killed. +func cleanupOrphanedTelemt(configPath string) { + processes, err := gopsutilprocess.Processes() + if err != nil { + log.Printf("warning: failed to list processes for telemt orphan cleanup: %v", err) return } - _ = terminateProcess(cmd) - if waitCh != nil { - select { - case <-waitCh: - case <-time.After(5 * time.Second): - _ = killProcess(cmd) - <-waitCh + self := int32(os.Getpid()) + for _, proc := range processes { + if proc.Pid == self { + continue + } + args, err := proc.CmdlineSlice() + if err != nil || !runsTelemtConfig(args, configPath) { + continue } + + log.Printf("killing orphaned telemt process %d running %s", proc.Pid, configPath) + if err := killOrphan(int(proc.Pid)); err != nil { + log.Printf("warning: failed to kill orphaned telemt process %d: %v", proc.Pid, err) + continue + } + waitProcessGone(proc, orphanExitTimeout) } } -func (p *processManager) Restart() error { - p.Shutdown() - return p.Start() +// runsTelemtConfig reports whether args look like `... run `. +func runsTelemtConfig(args []string, configPath string) bool { + if len(args) < 3 || args[len(args)-1] != configPath { + return false + } + return slices.Contains(args[1:len(args)-1], "run") } -func (p *processManager) captureLogs(pipe io.Reader) { - scanner := bufio.NewScanner(pipe) - for scanner.Scan() { - p.logs.Publish(scanner.Text()) +func waitProcessGone(proc *gopsutilprocess.Process, timeout time.Duration) { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + running, err := proc.IsRunning() + if err != nil || !running { + return + } + // A zombie has released its ports; only its parent can reap it. + if status, err := proc.Status(); err == nil && slices.Contains(status, gopsutilprocess.Zombie) { + return + } + time.Sleep(50 * time.Millisecond) } + log.Printf("warning: orphaned telemt process %d did not exit within %s", proc.Pid, timeout) } diff --git a/backend/mtproto/process_test.go b/backend/mtproto/process_test.go index 81378721..d24f0f92 100644 --- a/backend/mtproto/process_test.go +++ b/backend/mtproto/process_test.go @@ -4,45 +4,158 @@ package mtproto import ( "context" + "fmt" + "os" "os/exec" "path/filepath" + "slices" + "strings" + "sync" "testing" "time" - - "github.com/pasarguard/node/pkg/logstream" ) -// TestProcessManager_LogsReachSubscribersAndCloseCleanly runs a stand-in for -// telemt that writes to stdout and stderr, then shuts it down and closes the -// log buffer while a subscriber is attached. Neither step may panic, and the -// subscriber channel must be closed afterwards. -func TestProcessManager_LogsReachSubscribersAndCloseCleanly(t *testing.T) { - sh, err := exec.LookPath("sh") +// newTestProcessManager builds a process manager for the fake telemt and +// closes it when the test ends, so a failing test never leaves a child running. +func newTestProcessManager(t *testing.T, f fakeTelemt) *processManager { + t.Helper() + + manager, err := newProcessManager(writeFakeTelemt(t, f), t.TempDir(), 500, 50) if err != nil { - t.Skip("sh not available") + t.Fatalf("newProcessManager: %v", err) } + t.Cleanup(manager.Close) + return manager +} - dir := t.TempDir() - p := &processManager{ - executablePath: sh, - configPath: filepath.Join(dir, "config.toml"), - pidFilePath: filepath.Join(dir, "telemt.pid"), - logs: logstream.NewBuffer(16), - commandFactory: func(executable string, _ ...string) *exec.Cmd { - return exec.Command(executable, "-c", "echo out-line; echo err-line >&2; exec sleep 30") - }, +func TestParseTelemtVersion(t *testing.T) { + for output, want := range map[string]string{ + "telemt 3.5.14\n": "3.5.14", + "telemt 3.5.15": "3.5.15", + "telemt 3.6.0\n": "3.6.0", + "telemt 4.0.0": "4.0.0", + "telemt 3.5.14-rc.1\n": "3.5.14-rc.1", + } { + got, err := parseTelemtVersion(output) + if err != nil || got != want { + t.Errorf("parseTelemtVersion(%q) = %q, %v; want %q", output, got, err, want) + } } + for _, output := range []string{"telemt 3.5.13", "telemt 3.4.25", "telemt 2.9.99", "", "garbage", "telemt x.y.z"} { + if _, err := parseTelemtVersion(output); err == nil { + t.Errorf("parseTelemtVersion(%q) succeeded, want an error", output) + } + } +} + +func TestProcessRefusesOldTelemt(t *testing.T) { + _, err := newProcessManager(writeFakeTelemt(t, fakeTelemt{Version: "3.4.25"}), t.TempDir(), 16, 50) + if err == nil { + t.Fatal("newProcessManager accepted telemt 3.4.25") + } + if !strings.Contains(err.Error(), "3.5.14") || !strings.Contains(err.Error(), "3.4.25") { + t.Errorf("error %q should name the found version and the 3.5.14 floor", err) + } +} + +func TestProcessAcceptsSupportedTelemt(t *testing.T) { + manager := newTestProcessManager(t, fakeTelemt{}) + if got := manager.Version(); got != "3.5.14" { + t.Errorf("Version() = %q, want 3.5.14", got) + } + if manager.Started() || manager.PID() != 0 || manager.Exited() != nil || manager.Generation() != 0 { + t.Error("a new process manager must report not started, pid 0, nil Exited and generation 0") + } +} + +func TestProcessRunArgsHaveNoPidFile(t *testing.T) { + record := filepath.Join(t.TempDir(), "record.json") + manager := newTestProcessManager(t, fakeTelemt{RecordFile: record, NoAPI: true}) + + configPath := filepath.Join(t.TempDir(), "config.toml") + if err := manager.Start(configPath); err != nil { + t.Fatalf("Start: %v", err) + } + + got := readFakeTelemtRecord(t, record) + if want := []string{"run", configPath}; !slices.Equal(got.Args, want) { + t.Errorf("telemt argv = %q, want %q", got.Args, want) + } + wantDir, err := filepath.EvalSymlinks(manager.workDir) + if err != nil { + t.Fatal(err) + } + if gotDir, _ := filepath.EvalSymlinks(got.Cwd); gotDir != wantDir { + t.Errorf("telemt cwd = %q, want the work dir %q", got.Cwd, wantDir) + } + if manager.PID() != got.PID { + t.Errorf("PID() = %d, want %d", manager.PID(), got.PID) + } + if manager.Generation() != 1 { + t.Errorf("Generation() = %d, want 1", manager.Generation()) + } + if err := manager.Start(configPath); err == nil { + t.Error("a second Start while telemt runs must fail") + } +} + +// TestProcessStartedNoRace must run under -race: Started reads process state +// while the wait goroutine reaps the exiting child. +func TestProcessStartedNoRace(t *testing.T) { + manager := newTestProcessManager(t, fakeTelemt{NoAPI: true, ExitAfter: 50 * time.Millisecond}) + + if err := manager.Start(filepath.Join(t.TempDir(), "config.toml")); err != nil { + t.Fatalf("Start: %v", err) + } + exited := manager.Exited() + if exited == nil { + t.Fatal("Exited() is nil after Start") + } + + var wg sync.WaitGroup + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + for { + _ = manager.Started() + _ = manager.PID() + _ = manager.Generation() + select { + case <-exited: + return + default: + } + } + }() + } + + select { + case <-exited: + case <-time.After(10 * time.Second): + t.Fatal("the fake telemt did not exit") + } + wg.Wait() + + if manager.Started() { + t.Error("Started() is true after the process exited") + } +} + +func TestProcessLogsReachSubscribersAndCloseCleanly(t *testing.T) { + manager := newTestProcessManager(t, fakeTelemt{NoAPI: true, Stdout: []string{"out-line"}, Stderr: []string{"err-line"}}) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() - logs := p.SubscribeLogs(ctx) + logs := manager.SubscribeLogs(ctx) - if err := p.Start(); err != nil { + if err := manager.Start(filepath.Join(t.TempDir(), "config.toml")); err != nil { t.Fatalf("Start: %v", err) } seen := map[string]bool{} - deadline := time.After(5 * time.Second) + deadline := time.After(10 * time.Second) for !seen["out-line"] || !seen["err-line"] { select { case line := <-logs: @@ -52,19 +165,160 @@ func TestProcessManager_LogsReachSubscribersAndCloseCleanly(t *testing.T) { } } - p.Shutdown() - p.logs.Close() + manager.Stop() + if manager.Started() || manager.Exited() != nil { + t.Error("after Stop the manager must report not started and a nil Exited") + } + for _, line := range manager.StartupTail(0) { + if strings.Contains(line, "exited unexpectedly") { + t.Errorf("Stop was reported as an unexpected exit: %q", line) + } + } - select { - case _, ok := <-logs: - if ok { - t.Fatal("expected the subscriber channel to be closed after Close") + manager.Close() + closed := time.After(5 * time.Second) + for open := true; open; { + select { + case _, open = <-logs: + case <-closed: + t.Fatal("subscriber channel was not closed after Close") } - case <-time.After(5 * time.Second): - t.Fatal("subscriber channel was not closed after Close") } - if _, ok := <-p.SubscribeLogs(context.Background()); ok { + if _, ok := <-manager.SubscribeLogs(context.Background()); ok { t.Fatal("expected a closed channel when subscribing after Close") } + if err := manager.Start(filepath.Join(t.TempDir(), "config.toml")); err == nil { + t.Fatal("Start after Close must fail") + } +} + +// TestProcessLastLinesSurviveExit checks that output written right before +// telemt dies reaches the startup tail: the pipes are read to EOF before Wait +// closes them. +func TestProcessLastLinesSurviveExit(t *testing.T) { + stderr := make([]string, 0, 301) + for i := range 300 { + stderr = append(stderr, fmt.Sprintf("fake line %d", i)) + } + stderr = append(stderr, "Config error: last words") + manager := newTestProcessManager(t, fakeTelemt{NoAPI: true, Stderr: stderr, ExitAfter: time.Millisecond, ExitCode: 3}) + + if err := manager.Start(filepath.Join(t.TempDir(), "config.toml")); err != nil { + t.Fatalf("Start: %v", err) + } + select { + case <-manager.Exited(): + case <-time.After(10 * time.Second): + t.Fatal("the fake telemt did not exit") + } + + tail := manager.StartupTail(3) + want := []string{"Config error: last words", "fake telemt exiting", "telemt exited unexpectedly: exit status 3"} + if !slices.Equal(tail, want) { + t.Errorf("StartupTail(3) = %q, want %q", tail, want) + } + if got := len(manager.StartupTail(0)); got != 50 { + t.Errorf("startup tail holds %d lines, want the ring size 50", got) + } +} + +// TestProcessLongLineKeepsReading checks that an overlong line is cut and +// the lines after it still arrive. +func TestProcessLongLineKeepsReading(t *testing.T) { + long := strings.Repeat("y", 3*maxTelemtLogLine) + manager := newTestProcessManager(t, fakeTelemt{NoAPI: true, Stderr: []string{"before", long, "after"}, ExitAfter: time.Millisecond}) + + if err := manager.Start(filepath.Join(t.TempDir(), "config.toml")); err != nil { + t.Fatalf("Start: %v", err) + } + select { + case <-manager.Exited(): + case <-time.After(10 * time.Second): + t.Fatal("the fake telemt did not exit") + } + + tail := manager.StartupTail(0) + if len(tail) < 3 || tail[0] != "before" || tail[2] != "after" { + t.Fatalf("StartupTail = %.80q, want before, the cut line, after", tail) + } + if want := strings.Repeat("y", maxTelemtLogLine) + " [truncated]"; tail[1] != want { + t.Errorf("the long line was kept as %d bytes, want %d", len(tail[1]), len(want)) + } +} + +func TestProcessReloadSendsSIGHUP(t *testing.T) { + dir := t.TempDir() + record := filepath.Join(dir, "record.json") + signals := filepath.Join(dir, "signals") + manager := newTestProcessManager(t, fakeTelemt{NoAPI: true, RecordFile: record, SignalFile: signals}) + + if err := manager.Reload(); err == nil { + t.Error("Reload without a running telemt must fail") + } + if err := manager.Start(filepath.Join(dir, "config.toml")); err != nil { + t.Fatalf("Start: %v", err) + } + readFakeTelemtRecord(t, record) + + if err := manager.Reload(); err != nil { + t.Fatalf("Reload: %v", err) + } + waitFor(t, 10*time.Second, "the fake telemt to receive SIGHUP", func() bool { + data, _ := os.ReadFile(signals) + return string(data) == "SIGHUP\n" + }) + if !manager.Started() { + t.Error("telemt must keep running after a reload") + } +} + +// TestOrphanCleanupKillsStaleTelemt starts a telemt for the same config path +// outside the manager (left over from a killed node); Start must kill it, and +// must leave a process for a different config path alone. +func TestOrphanCleanupKillsStaleTelemt(t *testing.T) { + configPath := filepath.Join(t.TempDir(), "config.toml") + + startOutside := func(args ...string) <-chan struct{} { + t.Helper() + record := filepath.Join(t.TempDir(), "record.json") + cmd := exec.Command(writeFakeTelemt(t, fakeTelemt{NoAPI: true, RecordFile: record}), args...) + if err := cmd.Start(); err != nil { + t.Fatalf("starting the stray telemt: %v", err) + } + done := make(chan struct{}) + go func() { + _ = cmd.Wait() + close(done) + }() + t.Cleanup(func() { + _ = cmd.Process.Kill() + <-done + }) + readFakeTelemtRecord(t, record) + return done + } + + orphanDone := startOutside("run", configPath) + bystanderDone := startOutside("run", configPath+".old") + + manager := newTestProcessManager(t, fakeTelemt{NoAPI: true}) + if err := manager.Start(configPath); err != nil { + t.Fatalf("Start: %v", err) + } + + select { + case <-orphanDone: + case <-time.After(10 * time.Second): + t.Fatal("the orphaned telemt for the same config path is still running") + } + + select { + case <-bystanderDone: + t.Fatal("a process for another config path was killed") + default: + } + if !manager.Started() { + t.Error("the manager's own telemt is not running") + } } diff --git a/backend/mtproto/process_unix.go b/backend/mtproto/process_unix.go index 83e0e395..3d44d95b 100644 --- a/backend/mtproto/process_unix.go +++ b/backend/mtproto/process_unix.go @@ -3,6 +3,7 @@ package mtproto import ( + "errors" "os/exec" "syscall" ) @@ -33,3 +34,20 @@ func killProcess(cmd *exec.Cmd) error { _ = syscall.Kill(-cmd.Process.Pid, syscall.SIGKILL) return cmd.Process.Kill() } + +// reloadProcess sends SIGHUP, which telemt handles as a config reload. +func reloadProcess(cmd *exec.Cmd) error { + return cmd.Process.Signal(syscall.SIGHUP) +} + +// killOrphan SIGKILLs a stray telemt and, when it leads its own process group +// (the node starts telemt with Setpgid), the rest of that group. +func killOrphan(pid int) error { + if pgid, err := syscall.Getpgid(pid); err == nil && pgid == pid { + _ = syscall.Kill(-pid, syscall.SIGKILL) + } + if err := syscall.Kill(pid, syscall.SIGKILL); err != nil && !errors.Is(err, syscall.ESRCH) { + return err + } + return nil +} diff --git a/backend/mtproto/process_windows.go b/backend/mtproto/process_windows.go index ab7d3ac8..c5d8d892 100644 --- a/backend/mtproto/process_windows.go +++ b/backend/mtproto/process_windows.go @@ -3,6 +3,8 @@ package mtproto import ( + "errors" + "os" "os/exec" "syscall" ) @@ -26,3 +28,17 @@ func killProcess(cmd *exec.Cmd) error { } return cmd.Process.Kill() } + +// reloadProcess fails on Windows, which has no SIGHUP; the caller falls back +// to a restart. +func reloadProcess(_ *exec.Cmd) error { + return errors.New("reloading telemt by signal is not supported on Windows") +} + +func killOrphan(pid int) error { + process, err := os.FindProcess(pid) + if err != nil { + return err + } + return process.Kill() +} From abd6cd9d95d479d2600b556d0fe278952eba3426 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 19 Oct 2026 20:20:14 +0330 Subject: [PATCH 06/40] fix(mtproto): detach telemt startup from request deadlines - New starts telemt on context.WithoutCancel(ctx) bounded by telemtStartupTimeout, so a telemt that binds its API after the panel's 10 s Start deadline still comes up and the panel's retry attaches to it. Once spawned, startLocked waits for health on a detached ctx too; only the startup timeout and Shutdown (errClosed) end that wait. - The /v1/health/ready probe moved to a background watcher bound to the backend lifetime and to that telemt's exit: it warns after 30 s and logs when telemt becomes ready later. Start and every restart return right after health instead of holding syncMu and the panel deadline. - A sync whose ctx is already done (it queued on syncMu past its deadline) returns ctx.Err() before touching the config; reloadLocked never restarts telemt when the caller gave up, and startLocked spawns nothing for a done ctx or after Shutdown. - The supervisor and readiness watchers share one WaitGroup that Shutdown waits on. --- backend/mtproto/lifecycle.go | 83 ++++++++++++++++++++++------ backend/mtproto/lifecycle_test.go | 90 +++++++++++++++++++++++++++++++ backend/mtproto/mtproto.go | 31 +++++++++-- 3 files changed, 184 insertions(+), 20 deletions(-) diff --git a/backend/mtproto/lifecycle.go b/backend/mtproto/lifecycle.go index 7eab018c..7a4e1039 100644 --- a/backend/mtproto/lifecycle.go +++ b/backend/mtproto/lifecycle.go @@ -23,10 +23,14 @@ import ( var telemtStartupTimeout = 90 * time.Second const ( - // telemtReadyTimeout bounds the wait for /v1/health/ready once the API is - // up. Not being ready (no reachable Telegram upstream yet) is only logged: - // Telegram egress problems must not fail a start. + // telemtReadyTimeout is how long the background readiness watcher waits + // for /v1/health/ready before it warns. Not being ready (no reachable + // Telegram upstream yet) is only logged: Telegram egress problems must + // not fail or delay a start. telemtReadyTimeout = 30 * time.Second + // readyRecheckInterval is how often the watcher checks again after the + // warning, to log when telemt becomes ready. + readyRecheckInterval = 10 * time.Second // telemtStartupTailSize is how many lines of telemt output are kept for a // startup error. telemtStartupTailSize = 50 @@ -94,10 +98,16 @@ func New(ctx context.Context, cfg *config.Config, mtConfig *Config, users []*com cancel: cancel, } + // The panel's Start call has a short deadline (10 s by default) while + // telemt may need far longer to bind its API. Let a slow start finish + // anyway: the panel's retry then attaches to the running backend. + startCtx, cancelStart := context.WithTimeout(context.WithoutCancel(ctx), telemtStartupTimeout) + defer cancelStart() + mt.syncMu.Lock() err = mt.writeRenderedConfigLocked(mt.desired) if err == nil { - err = mt.startLocked(ctx) + err = mt.startLocked(startCtx) } mt.syncMu.Unlock() if err != nil { @@ -105,8 +115,7 @@ func New(ctx context.Context, cfg *config.Config, mtConfig *Config, users []*com return nil, err } - mt.supervisorDone = make(chan struct{}) - go mt.supervise() + mt.background.Go(mt.supervise) return mt, nil } @@ -126,9 +135,9 @@ func (m *MTProto) Shutdown() { } m.syncMu.Unlock() - if m.supervisorDone != nil { - <-m.supervisorDone - } + // The supervisor and readiness watchers end on the lifetime cancel. They + // are only added under syncMu before closed is set, so no Add races this. + m.background.Wait() } // prepareTelemtDir creates /telemt with owner-only permissions and @@ -178,21 +187,65 @@ func pickInternalPorts(listenPorts []int, findFreePort func() int) (apiPort, met } // startLocked starts telemt on the rendered config and waits for its API. -// A failure stops telemt and carries its last output. Requires syncMu. +// A failure stops telemt and carries its last output. Readiness is watched in +// the background. Requires syncMu. func (m *MTProto) startLocked(ctx context.Context) error { + if m.lifetime.Err() != nil { + return errClosed + } + if err := ctx.Err(); err != nil { + return err + } if err := m.process.Start(m.configPath); err != nil { return err } - if err := m.waitForHealth(ctx); err != nil { + // Once telemt runs, only telemtStartupTimeout and Shutdown bound its + // startup: a caller that gives up must not kill a telemt still binding. + if err := m.waitForHealth(context.WithoutCancel(ctx)); err != nil { m.process.Stop() return m.startupError(err) } - if err := m.waitForReady(ctx, telemtReadyTimeout); err != nil && m.lifetime.Err() == nil { - log.Printf("warning: telemt is running but not ready to serve clients yet: %v", err) - } + + exited := m.process.Exited() + m.background.Go(func() { m.watchReadiness(exited) }) return nil } +// watchReadiness logs whether a started telemt gets ready to serve clients: +// a warning after telemtReadyTimeout, then a line once it is ready. It stops +// when exited (that telemt's Exited channel) closes or on Shutdown. +func (m *MTProto) watchReadiness(exited <-chan struct{}) { + ctx, cancel := context.WithCancel(m.lifetime) + defer cancel() + go func() { + select { + case <-exited: + cancel() + case <-ctx.Done(): + } + }() + + err := m.waitForReady(ctx, telemtReadyTimeout) + if err == nil || ctx.Err() != nil { + return + } + log.Printf("warning: telemt is running but not ready to serve clients yet: %v", err) + + ticker := time.NewTicker(readyRecheckInterval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + if m.apiClient.Ready(ctx) == nil { + log.Println("telemt is now ready to serve clients") + return + } + } +} + func (m *MTProto) startupError(err error) error { tail := m.process.StartupTail(telemtStartupTailSize) if len(tail) == 0 { @@ -266,8 +319,6 @@ func (m *MTProto) waitForReady(ctx context.Context, budget time.Duration) error // Shutdown. The delay starts at supervisorInitialBackoff and doubles up to // supervisorMaxBackoff; it resets once a telemt has stayed up that long. func (m *MTProto) supervise() { - defer close(m.supervisorDone) - backoff := supervisorInitialBackoff for { if exited := m.process.Exited(); exited != nil { diff --git a/backend/mtproto/lifecycle_test.go b/backend/mtproto/lifecycle_test.go index 5517f56c..49729df8 100644 --- a/backend/mtproto/lifecycle_test.go +++ b/backend/mtproto/lifecycle_test.go @@ -302,3 +302,93 @@ func TestWaitForReadyGivesUp(t *testing.T) { t.Errorf("waitForReady = %v, want an error naming the reason", err) } } + +// TestStartOutlivesRequestDeadline models the panel's 10 s Start call against +// a telemt that binds its API later: the start must complete anyway, so the +// panel's retry attaches to a running backend. +func TestStartOutlivesRequestDeadline(t *testing.T) { + exe := writeFakeTelemt(t, fakeTelemt{HealthDelay: 1500 * time.Millisecond}) + + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + mt, err := New(ctx, testNodeConfig(t, exe), testMTConfig(t), nil) + if err != nil { + t.Fatalf("New failed once the request deadline passed: %v", err) + } + t.Cleanup(mt.Shutdown) + + if ctx.Err() == nil { + t.Fatal("the request deadline had not passed yet; the test proves nothing") + } + if !mt.Started() { + t.Error("backend not started") + } +} + +// TestStartDoesNotWaitForReadiness: telemt is healthy but not ready (no +// Telegram upstream); New must return right after health, not after the +// readiness budget. +func TestStartDoesNotWaitForReadiness(t *testing.T) { + exe := writeFakeTelemt(t, fakeTelemt{ReadyStatus: http.StatusServiceUnavailable}) + + started := time.Now() + mt := newTestBackend(t, testNodeConfig(t, exe), nil) + if elapsed := time.Since(started); elapsed >= 10*time.Second { + t.Errorf("New took %s with /v1/health/ready answering 503; readiness must not block start", elapsed) + } + if !mt.Started() { + t.Error("backend not started") + } +} + +// TestCancelledSyncKeepsTelemtRunning: a sync whose request already expired +// (it queued on syncMu) must not restart a healthy telemt. +func TestCancelledSyncKeepsTelemtRunning(t *testing.T) { + mt := newTestBackend(t, testNodeConfig(t, writeFakeTelemt(t, fakeTelemt{})), nil) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + users := []*common.User{mtUser("alice", "mtproto", testSecret)} + for name, call := range map[string]func() error{ + "SyncUsers": func() error { return mt.SyncUsers(ctx, users) }, + "UpdateUsers": func() error { return mt.UpdateUsers(ctx, users) }, + "SyncUser": func() error { return mt.SyncUser(ctx, users[0]) }, + "UpdateUsersAndRestart": func() error { return mt.UpdateUsersAndRestart(ctx, users) }, + } { + if err := call(); !errors.Is(err, context.Canceled) { + t.Errorf("%s with a cancelled ctx = %v, want context.Canceled", name, err) + } + } + + if !mt.Started() || mt.process.Generation() != 1 { + t.Errorf("telemt was restarted (started %v, generation %d)", mt.Started(), mt.process.Generation()) + } +} + +// TestCancelledContextNeverRestartsTelemt covers the paths below the entry +// check: a reload whose caller is gone keeps telemt, and a start with a done +// context spawns nothing. +func TestCancelledContextNeverRestartsTelemt(t *testing.T) { + mt := newTestBackend(t, testNodeConfig(t, writeFakeTelemt(t, fakeTelemt{})), nil) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + mt.syncMu.Lock() + defer mt.syncMu.Unlock() + + if err := mt.reloadLocked(ctx); !errors.Is(err, context.Canceled) { + t.Errorf("reloadLocked with a cancelled ctx = %v, want context.Canceled", err) + } + if !mt.process.Started() || mt.process.Generation() != 1 { + t.Fatalf("reloadLocked restarted telemt (started %v, generation %d)", mt.process.Started(), mt.process.Generation()) + } + + mt.process.Stop() + if err := mt.startLocked(ctx); !errors.Is(err, context.Canceled) { + t.Errorf("startLocked with a cancelled ctx = %v, want context.Canceled", err) + } + if mt.process.Generation() != 1 { + t.Errorf("startLocked spawned telemt for a cancelled ctx (generation %d)", mt.process.Generation()) + } +} diff --git a/backend/mtproto/mtproto.go b/backend/mtproto/mtproto.go index 28224012..778e8b54 100644 --- a/backend/mtproto/mtproto.go +++ b/backend/mtproto/mtproto.go @@ -43,9 +43,10 @@ type MTProto struct { desired map[string]*runtimeUser // lifetime is cancelled by Shutdown so waits for telemt stop early. - lifetime context.Context - cancel context.CancelFunc - supervisorDone chan struct{} + lifetime context.Context + cancel context.CancelFunc + // background tracks the supervisor and readiness watchers. + background sync.WaitGroup } func (m *MTProto) Started() bool { @@ -90,6 +91,10 @@ func (m *MTProto) SyncUsers(ctx context.Context, users []*common.User) error { if m.closed { return errClosed } + // The call may have queued on syncMu past its deadline. + if err := ctx.Err(); err != nil { + return err + } desired, _ := ActiveUsersFromCommon(users, m.config.InboundTag) return m.applyFullDesiredLocked(ctx, desired) @@ -106,6 +111,9 @@ func (m *MTProto) UpdateUsersAndRestart(ctx context.Context, users []*common.Use if m.closed { return errClosed } + if err := ctx.Err(); err != nil { + return err + } nextDesired := m.buildNextDesiredLocked(users) return m.applyFullDesiredLocked(ctx, nextDesired) @@ -250,6 +258,9 @@ func (m *MTProto) updateUsersPartial(ctx context.Context, users []*common.User) if m.closed { return errClosed } + if err := ctx.Err(); err != nil { + return err + } nextDesired := m.buildNextDesiredLocked(users) if len(m.desired) == 0 || len(nextDesired) == 0 { @@ -344,6 +355,11 @@ func (m *MTProto) deleteUser(ctx context.Context, username string) error { } func (m *MTProto) applyFullDesiredLocked(ctx context.Context, desired map[string]*runtimeUser) error { + // A partial sync that failed on an expired ctx falls back here; do not + // rewrite the config for a caller that is gone. + if err := ctx.Err(); err != nil { + return err + } previousDesired := cloneRuntimeUserMap(m.desired) if err := m.writeRenderedConfigLocked(desired); err != nil { @@ -367,8 +383,12 @@ func (m *MTProto) applyFullDesiredLocked(ctx context.Context, desired map[string } // reloadLocked makes the running telemt re-read the rendered config and -// restarts it when the reload fails. Requires syncMu. +// restarts it when the reload fails. A caller that gave up (ctx done) is not +// a reload failure, so it never restarts a healthy telemt. Requires syncMu. func (m *MTProto) reloadLocked(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } err := m.process.Reload() if err == nil { err = m.waitForHealth(ctx) @@ -376,6 +396,9 @@ func (m *MTProto) reloadLocked(ctx context.Context) error { if err == nil || errors.Is(err, errClosed) { return err } + if ctxErr := ctx.Err(); ctxErr != nil { + return fmt.Errorf("telemt reload not confirmed: %w", ctxErr) + } log.Printf("telemt reload failed, restarting it: %v", err) m.process.Stop() From c0ec733ef0ffd85acfb95b7b562815fad8db1592 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 19 Oct 2026 20:44:32 +0330 Subject: [PATCH 07/40] fix(mtproto): apply user changes without per-user reads and confirm reloads User changes are now decided from the confirmed desired state, never with a GET per user. Unchanged users are skipped; up to 32 changes go through telemt's user API (PATCH falls back to POST on 404, POST to PATCH on 409 user_exists, DELETE of a missing user is done). Larger batches, changes to or from zero users, a failed partial apply and every full sync render the config once and reload it through POST /v1/system/reload (instant, rollback), polling the reload status for the outcome and retrying while another reload is in progress. desired changes only after telemt confirmed the new users. A rolled_back, failed or refused reload re-renders the previous users to disk and returns telemt's error. A reload whose caller gave up leaves telemt running and forces the next sync to apply all users. Only a reload request that cannot reach telemt restarts it. The online IP list uses one GET /v1/stats/users/active-ips call. The SIGHUP reload and waitForRuntimeUsers are removed, and DELETE responses (a JSON string) decode again instead of failing into the full-render fallback. --- backend/mtproto/client.go | 155 +++++- backend/mtproto/fake_telemt_test.go | 43 +- backend/mtproto/integration_test.go | 46 +- backend/mtproto/lifecycle.go | 1 + backend/mtproto/lifecycle_test.go | 20 + backend/mtproto/mtproto.go | 328 +------------ backend/mtproto/process.go | 17 +- backend/mtproto/process_test.go | 27 -- backend/mtproto/process_unix.go | 5 - backend/mtproto/process_windows.go | 7 - backend/mtproto/sync.go | 332 +++++++++++++ backend/mtproto/sync_test.go | 713 ++++++++++++++++++++++++++++ 12 files changed, 1287 insertions(+), 407 deletions(-) create mode 100644 backend/mtproto/sync.go create mode 100644 backend/mtproto/sync_test.go diff --git a/backend/mtproto/client.go b/backend/mtproto/client.go index 328ae6f7..c430d383 100644 --- a/backend/mtproto/client.go +++ b/backend/mtproto/client.go @@ -9,10 +9,22 @@ import ( "io" "net/http" "net/url" + "strconv" "strings" "time" ) +const ( + // reloadRetryInterval is how often a reload is requested again while + // telemt runs another one. + reloadRetryInterval = 200 * time.Millisecond + // reloadPollInterval is how often an accepted reload is checked. + reloadPollInterval = 100 * time.Millisecond +) + +// errAPIUnreachable marks a reload request that got no answer from telemt. +var errAPIUnreachable = errors.New("telemt API unreachable") + type apiClient struct { baseURL string authHeader string @@ -52,6 +64,24 @@ type telemtUserInfo struct { TotalOctets uint64 `json:"total_octets"` } +type telemtUserActiveIPs struct { + Username string `json:"username"` + ActiveIPs []string `json:"active_ips"` +} + +type telemtReloadRequest struct { + Mode string `json:"mode"` + FailurePolicy string `json:"failure_policy"` +} + +// telemtReloadStatus is both the 202 answer to a reload request and the +// status of that reload. +type telemtReloadStatus struct { + ReloadID uint64 `json:"reload_id"` + State string `json:"state"` + Error string `json:"error"` +} + type telemtHealth struct { Status string `json:"status"` ReadOnly bool `json:"read_only"` @@ -102,6 +132,35 @@ func (e *apiStatusError) NotFound() bool { return e.StatusCode == http.StatusNotFound || e.Code == "not_found" } +func (e *apiStatusError) UserExists() bool { + return e.StatusCode == http.StatusConflict && e.Code == "user_exists" +} + +// ReloadInProgress reports telemt's answer to a reload request while another +// reload runs: 409 reload_in_progress, "Reload N is already in progress". +func (e *apiStatusError) ReloadInProgress() bool { + return e.StatusCode == http.StatusConflict && + (strings.Contains(e.Code, "in_progress") || strings.Contains(strings.ToLower(e.Message), "already in progress")) +} + +// reloadRejectedError reports a reload telemt refused, rolled back or failed. +// telemt keeps running the config it had before. +type reloadRejectedError struct { + State string // rolled_back or failed; empty when the request was refused + Err error +} + +func (e *reloadRejectedError) Error() string { + if e.State == "" { + return fmt.Sprintf("telemt refused the config reload: %v", e.Err) + } + return fmt.Sprintf("telemt config reload %s: %v", e.State, e.Err) +} + +func (e *reloadRejectedError) Unwrap() error { + return e.Err +} + func newAPIClient(baseURL, authHeader string) *apiClient { return &apiClient{ baseURL: strings.TrimRight(baseURL, "/"), @@ -132,13 +191,85 @@ func (c *apiClient) Ready(ctx context.Context) error { return err } -func (c *apiClient) GetUser(ctx context.Context, username string) (*telemtUserInfo, error) { - escapedUsername := url.PathEscape(username) - user, err := doAPIRequest[telemtUserInfo](ctx, c, http.MethodGet, "/v1/users/"+escapedUsername, nil) +// ListUsers returns the users of telemt's config file; InRuntime tells +// whether the running config has each one too. telemt loads the whole config +// for any user read, so one list beats a read per user. +func (c *apiClient) ListUsers(ctx context.Context) ([]telemtUserInfo, error) { + users, err := doAPIRequest[[]telemtUserInfo](ctx, c, http.MethodGet, "/v1/users", nil) if err != nil { return nil, err } - return user, nil + return *users, nil +} + +// ActiveIPs returns the active client IPs per username, for the users that +// have any. +func (c *apiClient) ActiveIPs(ctx context.Context) (map[string][]string, error) { + entries, err := doAPIRequest[[]telemtUserActiveIPs](ctx, c, http.MethodGet, "/v1/stats/users/active-ips", nil) + if err != nil { + return nil, err + } + activeIPs := make(map[string][]string, len(*entries)) + for _, entry := range *entries { + activeIPs[entry.Username] = entry.ActiveIPs + } + return activeIPs, nil +} + +// Reload makes telemt load its config file into a new runtime generation and +// waits for the outcome. It returns nil once the reload succeeded, a +// *reloadRejectedError when telemt refused it or ended it rolled_back or +// failed, an error wrapping errAPIUnreachable when the request got no answer, +// and ctx's error when ctx ends first. While telemt runs another reload it +// answers 409, and the request is repeated until ctx ends. +func (c *apiClient) Reload(ctx context.Context) error { + request := telemtReloadRequest{Mode: "instant", FailurePolicy: "rollback"} + + var accepted *telemtReloadStatus + for { + var err error + accepted, err = doAPIRequest[telemtReloadStatus](ctx, c, http.MethodPost, "/v1/system/reload", request) + if err == nil { + break + } + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + var apiErr *apiStatusError + if !errors.As(err, &apiErr) { + return fmt.Errorf("%w: %w", errAPIUnreachable, err) + } + if !apiErr.ReloadInProgress() { + return &reloadRejectedError{Err: apiErr} + } + if err := sleepContext(ctx, reloadRetryInterval); err != nil { + return err + } + } + + statusPath := "/v1/system/reload/" + strconv.FormatUint(accepted.ReloadID, 10) + for { + status, err := doAPIRequest[telemtReloadStatus](ctx, c, http.MethodGet, statusPath, nil) + if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return fmt.Errorf("failed to read the state of telemt reload %d: %w", accepted.ReloadID, err) + } + switch status.State { + case "succeeded": + return nil + case "rolled_back", "failed": + reason := status.Error + if reason == "" { + reason = "telemt gave no reason" + } + return &reloadRejectedError{State: status.State, Err: errors.New(reason)} + } + if err := sleepContext(ctx, reloadPollInterval); err != nil { + return err + } + } } func (c *apiClient) CreateUser(ctx context.Context, user *runtimeUser) error { @@ -159,8 +290,10 @@ func (c *apiClient) PatchUser(ctx context.Context, username string, user *runtim return err } +// DeleteUser removes the user. telemt answers with the deleted username as a +// JSON string. func (c *apiClient) DeleteUser(ctx context.Context, username string) error { - _, err := doAPIRequest[map[string]any](ctx, c, http.MethodDelete, "/v1/users/"+url.PathEscape(username), nil) + _, err := doAPIRequest[string](ctx, c, http.MethodDelete, "/v1/users/"+url.PathEscape(username), nil) return err } @@ -294,3 +427,15 @@ func doAPIRequest[T any](ctx context.Context, client *apiClient, method, path st return &success.Data, nil } + +// sleepContext waits for d or until ctx ends, and returns ctx's error then. +func sleepContext(ctx context.Context, d time.Duration) error { + timer := time.NewTimer(d) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} diff --git a/backend/mtproto/fake_telemt_test.go b/backend/mtproto/fake_telemt_test.go index 23c99881..e12a65d9 100644 --- a/backend/mtproto/fake_telemt_test.go +++ b/backend/mtproto/fake_telemt_test.go @@ -9,11 +9,9 @@ import ( "fmt" "net/http" "os" - "os/signal" "path/filepath" "strconv" "strings" - "syscall" "testing" "time" @@ -31,8 +29,8 @@ const fakeTelemtEnv = "MTPROTO_FAKE_TELEMT_MODE" // the node starts, signals and reaps. type fakeTelemt struct { Version string `json:"version"` // --version output, default 3.5.14 - // RecordFile receives the fake's argv, cwd and pid once its signal - // handlers are installed, so a test can wait for it to be ready. + // RecordFile receives the fake's argv, cwd and pid once it runs, so a + // test can wait for it. RecordFile string `json:"record_file"` Stdout []string `json:"stdout"` Stderr []string `json:"stderr"` @@ -45,8 +43,9 @@ type fakeTelemt struct { HealthDelay time.Duration `json:"health_delay"` NoAPI bool `json:"no_api"` ReadyStatus int `json:"ready_status"` // default 200 - // SignalFile gets one line per SIGHUP received. - SignalFile string `json:"signal_file"` + // DropReload closes reload requests without an answer, like an API that + // cannot be reached. + DropReload bool `json:"drop_reload"` // Lifetime bounds the fake so a broken test never leaves it running. Lifetime time.Duration `json:"lifetime"` } @@ -87,9 +86,6 @@ func runFakeTelemt(optionsPath string, args []string) int { return 0 } - hup := make(chan os.Signal, 1) - signal.Notify(hup, syscall.SIGHUP) - if f.RecordFile != "" { cwd, _ := os.Getwd() payload, _ := json.Marshal(fakeTelemtRecord{Args: args, Cwd: cwd, PID: os.Getpid()}) @@ -131,22 +127,12 @@ func runFakeTelemt(optionsPath string, args []string) int { } deadline := time.After(lifetime) - for { - select { - case <-hup: - if f.SignalFile != "" { - file, err := os.OpenFile(f.SignalFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) - if err == nil { - _, _ = file.WriteString("SIGHUP\n") - _ = file.Close() - } - } - case <-exit: - fmt.Fprintln(os.Stderr, "fake telemt exiting") - return f.ExitCode - case <-deadline: - return 0 - } + select { + case <-exit: + fmt.Fprintln(os.Stderr, "fake telemt exiting") + return f.ExitCode + case <-deadline: + return 0 } } @@ -178,6 +164,13 @@ func serveFakeTelemtAPI(f fakeTelemt, configPath string) { } _, _ = w.Write([]byte(`{"ok":true,"data":{"ready":false,"status":"not_ready","reason":"no_healthy_upstreams"},"revision":"r"}`)) }) + if f.DropReload { + mux.HandleFunc("/v1/system/reload", func(w http.ResponseWriter, _ *http.Request) { + if conn, _, err := http.NewResponseController(w).Hijack(); err == nil { + _ = conn.Close() + } + }) + } if err := http.ListenAndServe(listen, mux); err != nil { fmt.Fprintln(os.Stderr, "fake telemt: api:", err) diff --git a/backend/mtproto/integration_test.go b/backend/mtproto/integration_test.go index 3f609af8..5f873c8c 100644 --- a/backend/mtproto/integration_test.go +++ b/backend/mtproto/integration_test.go @@ -2,7 +2,6 @@ package mtproto import ( "context" - "errors" "fmt" "os" "testing" @@ -49,6 +48,34 @@ func integrationMTConfig(t *testing.T) *Config { return cfg } +// waitTelemtUser waits up to 5 s until telemt's running config has username +// (present) or lacks it. GET /v1/users lists the users in the config file; +// in_runtime tells whether the running config has them too. +func waitTelemtUser(ctx context.Context, t *testing.T, mt *MTProto, username string, present bool) { + t.Helper() + + deadline := time.Now().Add(5 * time.Second) + for { + users, err := mt.apiClient.ListUsers(ctx) + if err != nil { + t.Fatalf("ListUsers: %v", err) + } + inRuntime := false + for _, user := range users { + if user.Username == username && user.InRuntime { + inRuntime = true + } + } + if inRuntime == present { + return + } + if time.Now().After(deadline) { + t.Fatalf("telemt runtime has %s = %v, want %v", username, inRuntime, present) + } + time.Sleep(100 * time.Millisecond) + } +} + func TestIntegration_LifecycleAndUsers(t *testing.T) { bin := telemtBinOrSkip(t) @@ -74,31 +101,20 @@ func TestIntegration_LifecycleAndUsers(t *testing.T) { } // alice was rendered into the config; she must be present in telemt's runtime. - if _, err := mt.apiClient.GetUser(ctx, "alice"); err != nil { - t.Fatalf("alice not in telemt runtime: %v", err) - } + waitTelemtUser(ctx, t, mt, "alice", true) // add bob at runtime via the API path if err := mt.SyncUser(ctx, mtUser("bob", "mtproto", "ffeeddccbbaa00112233445566778899")); err != nil { t.Fatalf("SyncUser(bob): %v", err) } - if _, err := mt.apiClient.GetUser(ctx, "bob"); err != nil { - t.Fatalf("bob not added: %v", err) - } + waitTelemtUser(ctx, t, mt, "bob", true) // remove alice (untag her) aliceGone := &common.User{Email: "alice", Inbounds: []string{}, Proxies: &common.Proxy{Mtproto: &common.Mtproto{Secret: testSecret}}} if err := mt.SyncUser(ctx, aliceGone); err != nil { t.Fatalf("SyncUser(remove alice): %v", err) } - if _, err := mt.apiClient.GetUser(ctx, "alice"); err == nil { - t.Fatal("alice should have been removed") - } else { - var apiErr *apiStatusError - if !errors.As(err, &apiErr) || !apiErr.NotFound() { - t.Fatalf("expected alice lookup to return not found, got %v", err) - } - } + waitTelemtUser(ctx, t, mt, "alice", false) // Traffic + online-count stats come from telemt's Prometheus endpoint, which // only binds once the data plane is up (needs Telegram connectivity). A miss diff --git a/backend/mtproto/lifecycle.go b/backend/mtproto/lifecycle.go index 7a4e1039..7d3d094c 100644 --- a/backend/mtproto/lifecycle.go +++ b/backend/mtproto/lifecycle.go @@ -391,5 +391,6 @@ func (m *MTProto) restartIfStopped() (bool, error) { if err := m.startLocked(m.lifetime); err != nil { return false, err } + m.needsFullApply = false return true, nil } diff --git a/backend/mtproto/lifecycle_test.go b/backend/mtproto/lifecycle_test.go index 49729df8..744daa6b 100644 --- a/backend/mtproto/lifecycle_test.go +++ b/backend/mtproto/lifecycle_test.go @@ -9,6 +9,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "slices" "strings" "syscall" "testing" @@ -365,6 +366,25 @@ func TestCancelledSyncKeepsTelemtRunning(t *testing.T) { } } +// TestReloadUnreachableRestartsTelemt: a reload request that cannot reach +// telemt's API restarts telemt on the new config, and that start confirms it. +func TestReloadUnreachableRestartsTelemt(t *testing.T) { + exe := writeFakeTelemt(t, fakeTelemt{DropReload: true}) + mt := newTestBackend(t, testNodeConfig(t, exe), []*common.User{mtUser("alice", "mtproto", testSecret)}) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := mt.SyncUsers(ctx, []*common.User{mtUser("bob", "mtproto", testSecret)}); err != nil { + t.Fatalf("SyncUsers: %v", err) + } + if !mt.Started() || mt.process.Generation() != 2 { + t.Errorf("telemt was not restarted (started %v, generation %d)", mt.Started(), mt.process.Generation()) + } + if got := desiredNames(mt); !slices.Equal(got, []string{"bob"}) { + t.Errorf("desired = %q, want bob", got) + } +} + // TestCancelledContextNeverRestartsTelemt covers the paths below the entry // check: a reload whose caller is gone keeps telemt, and a start with a done // context spawns nothing. diff --git a/backend/mtproto/mtproto.go b/backend/mtproto/mtproto.go index 778e8b54..9fe41ba7 100644 --- a/backend/mtproto/mtproto.go +++ b/backend/mtproto/mtproto.go @@ -6,10 +6,8 @@ import ( "encoding/hex" "errors" "fmt" - "log" "os" "path/filepath" - "strings" "sync" "time" @@ -37,9 +35,14 @@ type MTProto struct { // process: syncs, restarts, the supervisor and Shutdown. syncMu sync.Mutex closed bool // guarded by syncMu - - // mu guards desired for readers; writers also hold syncMu. - mu sync.RWMutex + // needsFullApply is set when telemt may run users other than desired: a + // reload whose outcome is unknown, or user API calls that failed midway. + // The next sync then renders and reloads every user. Guarded by syncMu. + needsFullApply bool + + // stateMu guards desired for readers; writers also hold syncMu. desired + // holds only users telemt confirmed it runs. + stateMu sync.RWMutex desired map[string]*runtimeUser // lifetime is cancelled by Shutdown so waits for telemt stop early. @@ -72,51 +75,11 @@ func (m *MTProto) Restart() error { return err } m.process.Stop() - return m.startLocked(m.lifetime) -} - -func (m *MTProto) SyncUser(ctx context.Context, user *common.User) error { - // The batch paths skip a user that fails validation; a single-user sync - // reports it to the caller instead. - if _, _, err := activeUserFromCommon(user, m.config.InboundTag); err != nil { - return err - } - return m.updateUsersPartial(ctx, []*common.User{user}) -} - -func (m *MTProto) SyncUsers(ctx context.Context, users []*common.User) error { - m.syncMu.Lock() - defer m.syncMu.Unlock() - - if m.closed { - return errClosed - } - // The call may have queued on syncMu past its deadline. - if err := ctx.Err(); err != nil { - return err - } - desired, _ := ActiveUsersFromCommon(users, m.config.InboundTag) - - return m.applyFullDesiredLocked(ctx, desired) -} - -func (m *MTProto) UpdateUsers(ctx context.Context, users []*common.User) error { - return m.updateUsersPartial(ctx, users) -} - -func (m *MTProto) UpdateUsersAndRestart(ctx context.Context, users []*common.User) error { - m.syncMu.Lock() - defer m.syncMu.Unlock() - - if m.closed { - return errClosed - } - if err := ctx.Err(); err != nil { + if err := m.startLocked(m.lifetime); err != nil { return err } - nextDesired := m.buildNextDesiredLocked(users) - - return m.applyFullDesiredLocked(ctx, nextDesired) + m.needsFullApply = false + return nil } func (m *MTProto) GetSysStats(_ context.Context) (*common.BackendStatsResponse, error) { @@ -230,18 +193,16 @@ func (m *MTProto) GetUserOnlineIpListStats(ctx context.Context, email string) (* return response, nil } - // The active-IP *list* is only available from the API (metrics expose the count only). - userInfo, err := m.apiClient.GetUser(ctx, email) + // The active-IP list is only available from the API (metrics expose the + // count only). A read of one user loads telemt's whole config, so this + // takes the list of every user in one call. + activeIPs, err := m.apiClient.ActiveIPs(ctx) if err != nil { - var apiErr *apiStatusError - if errors.As(err, &apiErr) && apiErr.NotFound() { - return response, nil - } return nil, err } now := time.Now().Unix() - for _, ip := range userInfo.ActiveUniqueIPsList { + for _, ip := range activeIPs[email] { if ip == "" { continue } @@ -251,264 +212,15 @@ func (m *MTProto) GetUserOnlineIpListStats(ctx context.Context, email string) (* return response, nil } -func (m *MTProto) updateUsersPartial(ctx context.Context, users []*common.User) error { - m.syncMu.Lock() - defer m.syncMu.Unlock() - - if m.closed { - return errClosed - } - if err := ctx.Err(); err != nil { - return err - } - nextDesired := m.buildNextDesiredLocked(users) - - if len(m.desired) == 0 || len(nextDesired) == 0 { - return m.applyFullDesiredLocked(ctx, nextDesired) - } - - if err := m.applyPartialDesiredLocked(ctx, users, nextDesired); err != nil { - return m.applyFullDesiredLocked(ctx, nextDesired) - } - - m.setDesiredLocked(nextDesired) - return nil -} - -// buildNextDesiredLocked applies users on top of m.desired. A user that fails -// validation is logged and treated as not included, like an untagged one. -func (m *MTProto) buildNextDesiredLocked(users []*common.User) map[string]*runtimeUser { - nextDesired := cloneRuntimeUserMap(m.desired) - for _, user := range users { - if user == nil { - continue - } - - username := strings.TrimSpace(user.GetEmail()) - runtimeUser, include, err := activeUserFromCommon(user, m.config.InboundTag) - if err != nil { - logSkippedUser(username, err) - } - if include { - nextDesired[runtimeUser.Username] = runtimeUser - continue - } - delete(nextDesired, username) - } - return nextDesired -} - -func (m *MTProto) applyPartialDesiredLocked(ctx context.Context, users []*common.User, nextDesired map[string]*runtimeUser) error { - for _, user := range users { - if user == nil { - continue - } - - // An invalid user (already logged by buildNextDesiredLocked) is - // removed like an untagged one. - runtimeUser, include, err := activeUserFromCommon(user, m.config.InboundTag) - if err == nil && include { - if err := m.upsertUser(ctx, runtimeUser); err != nil { - return err - } - continue - } - - username := strings.TrimSpace(user.GetEmail()) - if username == "" { - continue - } - if err := m.deleteUser(ctx, username); err != nil { - return err - } - } - - _ = nextDesired - return nil -} - -func (m *MTProto) upsertUser(ctx context.Context, user *runtimeUser) error { - _, err := m.apiClient.GetUser(ctx, user.Username) - if err == nil { - return m.apiClient.PatchUser(ctx, user.Username, user) - } - - var apiErr *apiStatusError - if errors.As(err, &apiErr) && apiErr.NotFound() { - return m.apiClient.CreateUser(ctx, user) - } - - return err -} - -func (m *MTProto) deleteUser(ctx context.Context, username string) error { - err := m.apiClient.DeleteUser(ctx, username) - if err == nil { - return nil - } - - var apiErr *apiStatusError - if errors.As(err, &apiErr) && apiErr.NotFound() { - return nil - } - return err -} - -func (m *MTProto) applyFullDesiredLocked(ctx context.Context, desired map[string]*runtimeUser) error { - // A partial sync that failed on an expired ctx falls back here; do not - // rewrite the config for a caller that is gone. - if err := ctx.Err(); err != nil { - return err - } - previousDesired := cloneRuntimeUserMap(m.desired) - - if err := m.writeRenderedConfigLocked(desired); err != nil { - return err - } - - if !m.process.Started() { - if err := m.startLocked(ctx); err != nil { - return err - } - } else if err := m.reloadLocked(ctx); err != nil { - return err - } - - if err := m.waitForRuntimeUsers(ctx, previousDesired, desired); err != nil { - return err - } - - m.setDesiredLocked(desired) - return nil -} - -// reloadLocked makes the running telemt re-read the rendered config and -// restarts it when the reload fails. A caller that gave up (ctx done) is not -// a reload failure, so it never restarts a healthy telemt. Requires syncMu. -func (m *MTProto) reloadLocked(ctx context.Context) error { - if err := ctx.Err(); err != nil { - return err - } - err := m.process.Reload() - if err == nil { - err = m.waitForHealth(ctx) - } - if err == nil || errors.Is(err, errClosed) { - return err - } - if ctxErr := ctx.Err(); ctxErr != nil { - return fmt.Errorf("telemt reload not confirmed: %w", ctxErr) - } - - log.Printf("telemt reload failed, restarting it: %v", err) - m.process.Stop() - if restartErr := m.startLocked(ctx); restartErr != nil { - return fmt.Errorf("telemt reload failed: %w (restart failed: %v)", err, restartErr) - } - return nil -} - -// setDesiredLocked replaces the desired users. Requires syncMu. -func (m *MTProto) setDesiredLocked(desired map[string]*runtimeUser) { - m.mu.Lock() - defer m.mu.Unlock() - m.desired = cloneRuntimeUserMap(desired) -} - -func (m *MTProto) writeRenderedConfigLocked(desired map[string]*runtimeUser) error { - var sentinel *runtimeUser - if len(desired) == 0 { - sentinel = m.sentinel - } - - rendered, err := m.config.Render(desired, sentinel, m.apiPort, m.metricsPort, m.apiAuthHeader) - if err != nil { - return err - } - - return writeFileAtomic(m.configPath, []byte(rendered)) -} - -func (m *MTProto) waitForRuntimeUsers(ctx context.Context, previousDesired, desired map[string]*runtimeUser) error { - waitContext := ctx - if _, hasDeadline := ctx.Deadline(); !hasDeadline { - var cancel context.CancelFunc - waitContext, cancel = context.WithTimeout(ctx, telemtStartupTimeout) - defer cancel() - } - - expectedPresent := make(map[string]struct{}, len(desired)) - for username := range desired { - expectedPresent[username] = struct{}{} - } - if len(expectedPresent) == 0 { - expectedPresent[m.sentinel.Username] = struct{}{} - } - - expectedAbsent := make(map[string]struct{}) - for username := range previousDesired { - if _, keep := desired[username]; !keep { - expectedAbsent[username] = struct{}{} - } - } - // Going from zero desired users to real users drops the sentinel, but it is - // not in previousDesired, so require it to be gone explicitly. - if len(previousDesired) == 0 && len(desired) > 0 { - expectedAbsent[m.sentinel.Username] = struct{}{} - } - - ticker := time.NewTicker(200 * time.Millisecond) - defer ticker.Stop() - - for { - converged := true - - for username := range expectedPresent { - if _, err := m.apiClient.GetUser(waitContext, username); err != nil { - converged = false - break - } - } - - if converged { - for username := range expectedAbsent { - _, err := m.apiClient.GetUser(waitContext, username) - if err == nil { - converged = false - break - } - - var apiErr *apiStatusError - if !errors.As(err, &apiErr) || !apiErr.NotFound() { - converged = false - break - } - } - } - - if converged { - return nil - } - - select { - case <-waitContext.Done(): - return waitContext.Err() - case <-m.lifetime.Done(): - return errClosed - case <-ticker.C: - } - } -} - func (m *MTProto) refreshTracker(ctx context.Context) error { - m.mu.RLock() + m.stateMu.RLock() activeUsers := make(map[string]struct{}, len(m.desired)) for username := range m.desired { activeUsers[username] = struct{}{} } scraper := m.metricsScraper tracker := m.tracker - m.mu.RUnlock() + m.stateMu.RUnlock() snapshot, err := scraper.Scrape(ctx) if err != nil { @@ -520,8 +232,8 @@ func (m *MTProto) refreshTracker(ctx context.Context) error { } func (m *MTProto) isDesiredUser(username string) bool { - m.mu.RLock() - defer m.mu.RUnlock() + m.stateMu.RLock() + defer m.stateMu.RUnlock() _, exists := m.desired[username] return exists } diff --git a/backend/mtproto/process.go b/backend/mtproto/process.go index 491628e3..35efdb5f 100644 --- a/backend/mtproto/process.go +++ b/backend/mtproto/process.go @@ -163,8 +163,8 @@ func (p *processManager) Start(configPath string) error { cleanupOrphanedTelemt(configPath) // No --pid-file: a pid file that survives a hard kill can name a live - // process later and make telemt refuse to start. The node reloads by - // signalling its own child instead. + // process later and make telemt refuse to start. The node reloads + // through telemt's API, which needs no pid. cmd := exec.Command(p.executablePath, "run", configPath) // telemt writes relative-path artifacts (the tls_front cache, for one) // into its working directory; keep them in the generated config dir. @@ -312,19 +312,6 @@ func (p *processManager) StartupTail(n int) []string { return p.startupTail.tail(n) } -// Reload asks the running telemt to re-read its config (SIGHUP). -func (p *processManager) Reload() error { - p.mu.Lock() - defer p.mu.Unlock() - if !p.startedLocked() { - return errors.New("telemt process is not running") - } - if err := reloadProcess(p.run.cmd); err != nil { - return fmt.Errorf("failed to signal telemt to reload: %w", err) - } - return nil -} - // Stop terminates telemt and waits for it to exit, killing it after // telemtStopTimeout. func (p *processManager) Stop() { diff --git a/backend/mtproto/process_test.go b/backend/mtproto/process_test.go index d24f0f92..9d2bf4ee 100644 --- a/backend/mtproto/process_test.go +++ b/backend/mtproto/process_test.go @@ -5,7 +5,6 @@ package mtproto import ( "context" "fmt" - "os" "os/exec" "path/filepath" "slices" @@ -247,32 +246,6 @@ func TestProcessLongLineKeepsReading(t *testing.T) { } } -func TestProcessReloadSendsSIGHUP(t *testing.T) { - dir := t.TempDir() - record := filepath.Join(dir, "record.json") - signals := filepath.Join(dir, "signals") - manager := newTestProcessManager(t, fakeTelemt{NoAPI: true, RecordFile: record, SignalFile: signals}) - - if err := manager.Reload(); err == nil { - t.Error("Reload without a running telemt must fail") - } - if err := manager.Start(filepath.Join(dir, "config.toml")); err != nil { - t.Fatalf("Start: %v", err) - } - readFakeTelemtRecord(t, record) - - if err := manager.Reload(); err != nil { - t.Fatalf("Reload: %v", err) - } - waitFor(t, 10*time.Second, "the fake telemt to receive SIGHUP", func() bool { - data, _ := os.ReadFile(signals) - return string(data) == "SIGHUP\n" - }) - if !manager.Started() { - t.Error("telemt must keep running after a reload") - } -} - // TestOrphanCleanupKillsStaleTelemt starts a telemt for the same config path // outside the manager (left over from a killed node); Start must kill it, and // must leave a process for a different config path alone. diff --git a/backend/mtproto/process_unix.go b/backend/mtproto/process_unix.go index 3d44d95b..71e74fe6 100644 --- a/backend/mtproto/process_unix.go +++ b/backend/mtproto/process_unix.go @@ -35,11 +35,6 @@ func killProcess(cmd *exec.Cmd) error { return cmd.Process.Kill() } -// reloadProcess sends SIGHUP, which telemt handles as a config reload. -func reloadProcess(cmd *exec.Cmd) error { - return cmd.Process.Signal(syscall.SIGHUP) -} - // killOrphan SIGKILLs a stray telemt and, when it leads its own process group // (the node starts telemt with Setpgid), the rest of that group. func killOrphan(pid int) error { diff --git a/backend/mtproto/process_windows.go b/backend/mtproto/process_windows.go index c5d8d892..8be0d514 100644 --- a/backend/mtproto/process_windows.go +++ b/backend/mtproto/process_windows.go @@ -3,7 +3,6 @@ package mtproto import ( - "errors" "os" "os/exec" "syscall" @@ -29,12 +28,6 @@ func killProcess(cmd *exec.Cmd) error { return cmd.Process.Kill() } -// reloadProcess fails on Windows, which has no SIGHUP; the caller falls back -// to a restart. -func reloadProcess(_ *exec.Cmd) error { - return errors.New("reloading telemt by signal is not supported on Windows") -} - func killOrphan(pid int) error { process, err := os.FindProcess(pid) if err != nil { diff --git a/backend/mtproto/sync.go b/backend/mtproto/sync.go new file mode 100644 index 00000000..6882e96b --- /dev/null +++ b/backend/mtproto/sync.go @@ -0,0 +1,332 @@ +package mtproto + +import ( + "context" + "errors" + "fmt" + "log" + "maps" + "strings" + + "github.com/pasarguard/node/common" +) + +// partialApplyMaxUsers is the most user changes a batch sends one by one +// through telemt's user API. telemt loads, validates and rewrites its whole +// config for each such call, so a larger batch renders the config once and +// reloads it instead. +const partialApplyMaxUsers = 32 + +// userChange is one user to create, patch or delete in telemt. +type userChange struct { + username string + user *runtimeUser // the new state; nil deletes the user + known bool // the user is in m.desired +} + +func (m *MTProto) SyncUser(ctx context.Context, user *common.User) error { + // The batch paths skip a user that fails validation; a single-user sync + // reports it to the caller instead. + if _, _, err := activeUserFromCommon(user, m.config.InboundTag); err != nil { + return err + } + return m.UpdateUsers(ctx, []*common.User{user}) +} + +// SyncUsers replaces all users through one config render and reload. +func (m *MTProto) SyncUsers(ctx context.Context, users []*common.User) error { + m.syncMu.Lock() + defer m.syncMu.Unlock() + + if err := m.syncAllowedLocked(ctx); err != nil { + return err + } + desired, _ := ActiveUsersFromCommon(users, m.config.InboundTag) + + // A reload cuts every telemt session; skip one that changes nothing. + if m.confirmedLocked() && maps.EqualFunc(m.desired, desired, runtimeUserEqual) { + return nil + } + return m.applyFullLocked(ctx, desired) +} + +// UpdateUsers applies users on top of the current ones. Up to +// partialApplyMaxUsers changes go through telemt's user API; more, a change +// to or from zero users, or an unconfirmed telemt state go through a config +// render and reload. Users that did not change are not sent at all. +func (m *MTProto) UpdateUsers(ctx context.Context, users []*common.User) error { + m.syncMu.Lock() + defer m.syncMu.Unlock() + + if err := m.syncAllowedLocked(ctx); err != nil { + return err + } + next := m.buildNextDesiredLocked(users) + changes := m.userChangesLocked(users, next) + + if !m.confirmedLocked() || len(changes) > partialApplyMaxUsers { + return m.applyFullLocked(ctx, next) + } + if len(changes) == 0 { + return nil + } + // telemt refuses to delete its last user, and zero users means the + // sentinel, which only a render adds or drops. + if len(m.desired) == 0 || len(next) == 0 { + return m.applyFullLocked(ctx, next) + } + + if err := m.applyUserChangesLocked(ctx, changes); err != nil { + // Some changes may have reached telemt: its users are unknown until + // a full apply succeeds. + m.needsFullApply = true + log.Printf("mtproto: applying %d user changes through the telemt API failed, rendering all users instead: %v", len(changes), err) + return m.applyFullLocked(ctx, next) + } + m.confirmLocked(next) + return nil +} + +// UpdateUsersAndRestart applies users on top of the current ones through a +// config render and reload. +func (m *MTProto) UpdateUsersAndRestart(ctx context.Context, users []*common.User) error { + m.syncMu.Lock() + defer m.syncMu.Unlock() + + if err := m.syncAllowedLocked(ctx); err != nil { + return err + } + return m.applyFullLocked(ctx, m.buildNextDesiredLocked(users)) +} + +// syncAllowedLocked refuses a sync after Shutdown, or one whose caller gave +// up while it queued on syncMu. Requires syncMu. +func (m *MTProto) syncAllowedLocked(ctx context.Context) error { + if m.closed { + return errClosed + } + return ctx.Err() +} + +// confirmedLocked reports whether telemt runs and runs exactly m.desired. +// Requires syncMu. +func (m *MTProto) confirmedLocked() bool { + return !m.needsFullApply && m.process.Started() +} + +// buildNextDesiredLocked applies users on top of m.desired. A user that fails +// validation is logged and treated as not included, like an untagged one. +// Requires syncMu. +func (m *MTProto) buildNextDesiredLocked(users []*common.User) map[string]*runtimeUser { + nextDesired := cloneRuntimeUserMap(m.desired) + for _, user := range users { + if user == nil { + continue + } + + username := strings.TrimSpace(user.GetEmail()) + runtimeUser, include, err := activeUserFromCommon(user, m.config.InboundTag) + if err != nil { + logSkippedUser(username, err) + } + if include { + nextDesired[runtimeUser.Username] = runtimeUser + continue + } + delete(nextDesired, username) + } + return nextDesired +} + +// userChangesLocked lists, in batch order, the users of the batch whose +// state in next differs from m.desired. Requires syncMu. +func (m *MTProto) userChangesLocked(users []*common.User, next map[string]*runtimeUser) []userChange { + var changes []userChange + seen := make(map[string]struct{}, len(users)) + for _, user := range users { + if user == nil { + continue + } + username := strings.TrimSpace(user.GetEmail()) + if _, done := seen[username]; done { + continue + } + seen[username] = struct{}{} + + current, known := m.desired[username] + wanted := next[username] + if !runtimeUserEqual(current, wanted) { + changes = append(changes, userChange{username: username, user: wanted, known: known}) + } + } + return changes +} + +// applyUserChangesLocked sends each change to telemt's user API. m.desired +// picks create or patch; when telemt disagrees, its answer corrects the +// choice: a patch of a missing user creates it, a create of an existing user +// patches it, and deleting a missing user is done. Requires syncMu. +func (m *MTProto) applyUserChangesLocked(ctx context.Context, changes []userChange) error { + for _, change := range changes { + var err error + switch { + case change.user == nil: + err = m.deleteUser(ctx, change.username) + case change.known: + err = m.patchOrCreateUser(ctx, change.user) + default: + err = m.createOrPatchUser(ctx, change.user) + } + if err != nil { + return fmt.Errorf("user %s: %w", change.username, err) + } + } + return nil +} + +func (m *MTProto) patchOrCreateUser(ctx context.Context, user *runtimeUser) error { + err := m.apiClient.PatchUser(ctx, user.Username, user) + var apiErr *apiStatusError + if errors.As(err, &apiErr) && apiErr.NotFound() { + return m.apiClient.CreateUser(ctx, user) + } + return err +} + +func (m *MTProto) createOrPatchUser(ctx context.Context, user *runtimeUser) error { + err := m.apiClient.CreateUser(ctx, user) + var apiErr *apiStatusError + if errors.As(err, &apiErr) && apiErr.UserExists() { + return m.apiClient.PatchUser(ctx, user.Username, user) + } + return err +} + +func (m *MTProto) deleteUser(ctx context.Context, username string) error { + err := m.apiClient.DeleteUser(ctx, username) + var apiErr *apiStatusError + if errors.As(err, &apiErr) && apiErr.NotFound() { + return nil + } + return err +} + +// applyFullLocked renders desired and makes telemt run it: a reload of the +// running telemt, or a start when none runs. m.desired takes desired only +// once telemt confirmed it. Requires syncMu. +func (m *MTProto) applyFullLocked(ctx context.Context, desired map[string]*runtimeUser) error { + // A partial apply that failed on an expired ctx falls back here; do not + // rewrite the config for a caller that is gone. + if err := ctx.Err(); err != nil { + return err + } + if err := m.writeRenderedConfigLocked(desired); err != nil { + return err + } + + if !m.process.Started() { + if err := m.startLocked(ctx); err != nil { + return err + } + m.confirmLocked(desired) + return nil + } + + err := m.reloadLocked(ctx) + var rejected *reloadRejectedError + switch { + case err == nil: + m.confirmLocked(desired) + return nil + case errors.As(err, &rejected): + // telemt still runs the users it had. Put them back on disk too: + // telemt watches the config file and would load the rejected one. + if restoreErr := m.writeRenderedConfigLocked(m.desired); restoreErr != nil { + return fmt.Errorf("%w (restoring the previous telemt config failed: %v)", err, restoreErr) + } + return err + default: + // The outcome is unknown, and telemt may still load the new config. + m.needsFullApply = true + return err + } +} + +// reloadLocked makes the running telemt load the rendered config through its +// reload API and waits for the outcome. Only a request that cannot reach +// telemt restarts it, on the rendered config; a caller that gave up or a +// rejected config never does. Requires syncMu. +func (m *MTProto) reloadLocked(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + waitCtx, cancel := m.reloadContext(ctx) + defer cancel() + + err := m.apiClient.Reload(waitCtx) + switch { + case err == nil: + return nil + case m.lifetime.Err() != nil: + return errClosed + case waitCtx.Err() != nil: + return fmt.Errorf("telemt reload not confirmed: %w", err) + case !errors.Is(err, errAPIUnreachable): + return err + } + + log.Printf("telemt API unreachable for a reload, restarting telemt: %v", err) + m.process.Stop() + // telemt is down now: start it again even if the caller gives up. + if restartErr := m.startLocked(context.WithoutCancel(ctx)); restartErr != nil { + return fmt.Errorf("telemt reload failed: %w (restart failed: %v)", err, restartErr) + } + return nil +} + +// reloadContext bounds the wait for a reload by ctx, by Shutdown, and by +// telemtStartupTimeout when ctx has no deadline: syncMu is held throughout. +func (m *MTProto) reloadContext(ctx context.Context) (context.Context, context.CancelFunc) { + var waitCtx context.Context + var cancel context.CancelFunc + if _, hasDeadline := ctx.Deadline(); hasDeadline { + waitCtx, cancel = context.WithCancel(ctx) + } else { + waitCtx, cancel = context.WithTimeout(ctx, telemtStartupTimeout) + } + stop := context.AfterFunc(m.lifetime, cancel) + return waitCtx, func() { + stop() + cancel() + } +} + +// confirmLocked records desired as the users telemt runs. Requires syncMu. +func (m *MTProto) confirmLocked(desired map[string]*runtimeUser) { + m.needsFullApply = false + m.stateMu.Lock() + defer m.stateMu.Unlock() + m.desired = cloneRuntimeUserMap(desired) +} + +func (m *MTProto) writeRenderedConfigLocked(desired map[string]*runtimeUser) error { + var sentinel *runtimeUser + if len(desired) == 0 { + sentinel = m.sentinel + } + + rendered, err := m.config.Render(desired, sentinel, m.apiPort, m.metricsPort, m.apiAuthHeader) + if err != nil { + return err + } + + return writeFileAtomic(m.configPath, []byte(rendered)) +} + +// runtimeUserEqual reports whether a and b give telemt the same user. +func runtimeUserEqual(a, b *runtimeUser) bool { + if a == nil || b == nil { + return a == b + } + return *a == *b +} diff --git a/backend/mtproto/sync_test.go b/backend/mtproto/sync_test.go new file mode 100644 index 00000000..727f408b --- /dev/null +++ b/backend/mtproto/sync_test.go @@ -0,0 +1,713 @@ +package mtproto + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/pasarguard/node/common" +) + +const otherSecret = "ffeeddccbbaa00112233445566778899" + +// fakeTelemtAPI stands in for telemt's control API. It keeps the runtime +// user set, records every call, and runs reloads through telemt's states: +// accepted, then preparing for reloadPending polls, then a terminal state. +type fakeTelemtAPI struct { + t *testing.T + server *httptest.Server + configPath string + + mu sync.Mutex + users map[string]string // runtime users: username to secret + calls []string // "METHOD /path" per request + // reloadBodies holds the body of every reload request. + reloadBodies []string + + // reloadInProgress answers this many reload requests with telemt's 409. + reloadInProgress int + // reloadRejectStatus, when set, answers reload requests with this status + // and a bad_request error, like telemt for a config it cannot load. + reloadRejectStatus int + reloadPending int // status polls that report preparing first + reloadHang bool // the reload never finishes + reloadState string + reloadError string + nextReloadID int + pendingPolls map[int]int + // failPatch answers PATCH requests with a 500. + failPatch bool + activeIPs map[string][]string +} + +func newFakeTelemtAPI(t *testing.T) *fakeTelemtAPI { + t.Helper() + f := &fakeTelemtAPI{ + t: t, + configPath: filepath.Join(t.TempDir(), "config.toml"), + users: make(map[string]string), + pendingPolls: make(map[int]int), + activeIPs: make(map[string][]string), + } + f.server = httptest.NewServer(http.HandlerFunc(f.serve)) + t.Cleanup(f.server.Close) + return f +} + +// takeCalls returns the calls recorded so far and forgets them. +func (f *fakeTelemtAPI) takeCalls() []string { + f.mu.Lock() + defer f.mu.Unlock() + calls := f.calls + f.calls = nil + return calls +} + +func (f *fakeTelemtAPI) runtimeUsers() map[string]string { + f.mu.Lock() + defer f.mu.Unlock() + users := make(map[string]string, len(f.users)) + for name, secret := range f.users { + users[name] = secret + } + return users +} + +func (f *fakeTelemtAPI) set(apply func(f *fakeTelemtAPI)) { + f.mu.Lock() + defer f.mu.Unlock() + apply(f) +} + +// loadUsersLocked reads [access.users] from the rendered config, as a +// telemt (re)start or a successful reload does. +func (f *fakeTelemtAPI) loadUsersLocked() { + data, err := os.ReadFile(f.configPath) + if err != nil { + f.t.Errorf("fake telemt: reading the config: %v", err) + return + } + users := make(map[string]string) + section := "" + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "[") { + section = strings.Trim(line, "[]") + continue + } + key, value, ok := strings.Cut(line, "=") + if !ok || section != "access.users" { + continue + } + secret, err := strconv.Unquote(strings.TrimSpace(value)) + if err != nil { + f.t.Errorf("fake telemt: bad user line %q", line) + continue + } + users[strings.Trim(strings.TrimSpace(key), `"`)] = secret + } + f.users = users +} + +func (f *fakeTelemtAPI) serve(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + + body, _ := io.ReadAll(r.Body) + path := r.URL.Path + f.calls = append(f.calls, r.Method+" "+path) + + switch { + case r.Method == http.MethodGet && path == "/v1/stats/users/active-ips": + type entry struct { + Username string `json:"username"` + ActiveIPs []string `json:"active_ips"` + } + data := []entry{} + for name, ips := range f.activeIPs { + data = append(data, entry{Username: name, ActiveIPs: ips}) + } + writeFakeSuccess(w, http.StatusOK, data) + + case r.Method == http.MethodGet && path == "/v1/users": + data := []map[string]any{} + for name := range f.users { + data = append(data, map[string]any{"username": name, "in_runtime": true}) + } + writeFakeSuccess(w, http.StatusOK, data) + + case r.Method == http.MethodPost && path == "/v1/users": + var request telemtCreateUserRequest + if err := json.Unmarshal(body, &request); err != nil { + writeFakeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + if _, exists := f.users[request.Username]; exists { + writeFakeError(w, http.StatusConflict, "user_exists", "User already exists") + return + } + f.users[request.Username] = request.Secret + writeFakeSuccess(w, http.StatusCreated, map[string]any{"user": map[string]any{"username": request.Username}}) + + case strings.HasPrefix(path, "/v1/users/"): + name := strings.TrimPrefix(path, "/v1/users/") + if _, exists := f.users[name]; !exists { + writeFakeError(w, http.StatusNotFound, "not_found", "User not found") + return + } + switch r.Method { + case http.MethodPatch: + if f.failPatch { + writeFakeError(w, http.StatusInternalServerError, "internal_error", "patch failed") + return + } + var request telemtPatchUserRequest + if err := json.Unmarshal(body, &request); err != nil { + writeFakeError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + if request.Secret != "" { + f.users[name] = request.Secret + } + writeFakeSuccess(w, http.StatusOK, map[string]any{"username": name}) + case http.MethodDelete: + if len(f.users) == 1 { + writeFakeError(w, http.StatusConflict, "last_user_forbidden", "Cannot delete the last user") + return + } + delete(f.users, name) + writeFakeSuccess(w, http.StatusOK, name) + default: + // A GET of one user loads the whole config in telemt; the node + // must not need it. + writeFakeError(w, http.StatusMethodNotAllowed, "method_not_allowed", "unexpected "+r.Method) + } + + case r.Method == http.MethodPost && path == "/v1/system/reload": + f.reloadBodies = append(f.reloadBodies, string(body)) + if f.reloadInProgress > 0 { + f.reloadInProgress-- + writeFakeError(w, http.StatusConflict, "reload_in_progress", "Reload 7 is already in progress") + return + } + if f.reloadRejectStatus != 0 { + writeFakeError(w, f.reloadRejectStatus, "bad_request", "invalid runtime config: bad listener") + return + } + f.nextReloadID++ + f.pendingPolls[f.nextReloadID] = f.reloadPending + writeFakeSuccess(w, http.StatusAccepted, map[string]any{ + "reload_id": f.nextReloadID, "target_generation": f.nextReloadID + 1, + "config_revision": "r", "state": "accepted", "mode": "instant", "failure_policy": "rollback", + }) + + case r.Method == http.MethodGet && strings.HasPrefix(path, "/v1/system/reload/"): + id, err := strconv.Atoi(strings.TrimPrefix(path, "/v1/system/reload/")) + if _, known := f.pendingPolls[id]; err != nil || !known { + writeFakeError(w, http.StatusNotFound, "reload_not_found", "Reload was not found") + return + } + status := map[string]any{"reload_id": id, "mode": "instant", "failure_policy": "rollback"} + switch { + case f.reloadHang || f.pendingPolls[id] > 0: + f.pendingPolls[id]-- + status["state"] = "preparing" + case f.reloadState == "" || f.reloadState == "succeeded": + f.loadUsersLocked() + status["state"] = "succeeded" + default: + status["state"] = f.reloadState + status["error"] = f.reloadError + } + writeFakeSuccess(w, http.StatusOK, status) + + default: + writeFakeError(w, http.StatusNotFound, "not_found", "Route not found") + } +} + +func writeFakeSuccess(w http.ResponseWriter, status int, data any) { + payload, _ := json.Marshal(map[string]any{"ok": true, "data": data, "revision": "r"}) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write(payload) +} + +func writeFakeError(w http.ResponseWriter, status int, code, message string) { + payload, _ := json.Marshal(map[string]any{ + "ok": false, + "error": map[string]string{"code": code, "message": message}, + "request_id": 1, + }) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write(payload) +} + +// newSyncTestBackend returns a backend whose telemt "runs" on api: the +// process is a stand-in that counts as started, and the fake's runtime users +// are loaded from the initial render, as a telemt start would. +func newSyncTestBackend(t *testing.T, api *fakeTelemtAPI, users []*common.User) *MTProto { + t.Helper() + + mtConfig, err := NewConfig(`{"inbound_tag": "mtproto", "server": {"port": 8443}}`) + if err != nil { + t.Fatal(err) + } + sentinel, err := newSentinelUser() + if err != nil { + t.Fatal(err) + } + desired, _ := ActiveUsersFromCommon(users, mtConfig.InboundTag) + + lifetime, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + m := &MTProto{ + config: mtConfig, + process: &processManager{run: &telemtRun{pid: 1, waitDone: make(chan struct{})}}, + configPath: api.configPath, + apiClient: newAPIClient(api.server.URL, "Bearer test"), + metricsScraper: newMetricsScraper("http://127.0.0.1:1/metrics"), + tracker: newTrafficTracker(), + sentinel: sentinel, + apiPort: 20001, + metricsPort: 20002, + apiAuthHeader: "Bearer test", + desired: cloneRuntimeUserMap(desired), + lifetime: lifetime, + cancel: cancel, + } + if err := m.writeRenderedConfigLocked(m.desired); err != nil { + t.Fatal(err) + } + api.set(func(f *fakeTelemtAPI) { f.loadUsersLocked() }) + return m +} + +func desiredNames(m *MTProto) []string { + m.stateMu.RLock() + defer m.stateMu.RUnlock() + names := make([]string, 0, len(m.desired)) + for name := range m.desired { + names = append(names, name) + } + slices.Sort(names) + return names +} + +func desiredSecret(m *MTProto, name string) string { + m.stateMu.RLock() + defer m.stateMu.RUnlock() + if user := m.desired[name]; user != nil { + return user.Secret + } + return "" +} + +func assertCalls(t *testing.T, got []string, want ...string) { + t.Helper() + if !slices.Equal(got, want) { + t.Errorf("telemt API calls = %q, want %q", got, want) + } +} + +func untagged(email string) *common.User { + return &common.User{Email: email, Proxies: &common.Proxy{Mtproto: &common.Mtproto{Secret: testSecret}}} +} + +func numberedSecret(i int) string { + return fmt.Sprintf("%032x", i+1) +} + +func TestPartialSkipsUnchangedUsers(t *testing.T) { + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{ + mtUser("u1", "mtproto", testSecret), + mtUser("u2", "mtproto", testSecret), + }) + + err := m.UpdateUsers(context.Background(), []*common.User{ + mtUser("u1", "mtproto", testSecret), + mtUser("u2", "mtproto", otherSecret), + }) + if err != nil { + t.Fatalf("UpdateUsers: %v", err) + } + assertCalls(t, api.takeCalls(), "PATCH /v1/users/u2") + if got := desiredSecret(m, "u2"); got != otherSecret { + t.Errorf("desired u2 secret = %q, want %q", got, otherSecret) + } + if got := api.runtimeUsers()["u2"]; got != otherSecret { + t.Errorf("telemt u2 secret = %q, want %q", got, otherSecret) + } + + // A batch with nothing new reaches telemt not at all. + if err := m.UpdateUsers(context.Background(), []*common.User{mtUser("u1", "mtproto", testSecret)}); err != nil { + t.Fatalf("UpdateUsers (unchanged): %v", err) + } + assertCalls(t, api.takeCalls()) +} + +func TestPartialCreatesAndDeletesWithoutGet(t *testing.T) { + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{ + mtUser("u1", "mtproto", testSecret), + mtUser("u2", "mtproto", testSecret), + }) + ctx := context.Background() + + if err := m.UpdateUsers(ctx, []*common.User{mtUser("u3", "mtproto", testSecret), untagged("u1")}); err != nil { + t.Fatalf("UpdateUsers: %v", err) + } + assertCalls(t, api.takeCalls(), "POST /v1/users", "DELETE /v1/users/u1") + if got := desiredNames(m); !slices.Equal(got, []string{"u2", "u3"}) { + t.Errorf("desired = %q, want u2, u3", got) + } + + t.Run("patch of a user telemt lacks creates it", func(t *testing.T) { + api.set(func(f *fakeTelemtAPI) { delete(f.users, "u2") }) + if err := m.SyncUser(ctx, mtUser("u2", "mtproto", otherSecret)); err != nil { + t.Fatalf("SyncUser: %v", err) + } + assertCalls(t, api.takeCalls(), "PATCH /v1/users/u2", "POST /v1/users") + if got := api.runtimeUsers()["u2"]; got != otherSecret { + t.Errorf("telemt u2 secret = %q, want %q", got, otherSecret) + } + }) + + t.Run("create of a user telemt has patches it", func(t *testing.T) { + api.set(func(f *fakeTelemtAPI) { f.users["u4"] = testSecret }) + if err := m.SyncUser(ctx, mtUser("u4", "mtproto", otherSecret)); err != nil { + t.Fatalf("SyncUser: %v", err) + } + assertCalls(t, api.takeCalls(), "POST /v1/users", "PATCH /v1/users/u4") + if got := api.runtimeUsers()["u4"]; got != otherSecret { + t.Errorf("telemt u4 secret = %q, want %q", got, otherSecret) + } + }) + + t.Run("delete of a user telemt lacks succeeds", func(t *testing.T) { + api.set(func(f *fakeTelemtAPI) { delete(f.users, "u3") }) + if err := m.SyncUser(ctx, untagged("u3")); err != nil { + t.Fatalf("SyncUser: %v", err) + } + assertCalls(t, api.takeCalls(), "DELETE /v1/users/u3") + if got := desiredNames(m); !slices.Equal(got, []string{"u2", "u4"}) { + t.Errorf("desired = %q, want u2, u4", got) + } + }) +} + +func TestLargeBatchUsesFullRenderAndReload(t *testing.T) { + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{mtUser("u0", "mtproto", testSecret)}) + before, err := os.Stat(api.configPath) + if err != nil { + t.Fatal(err) + } + + users := make([]*common.User, 0, partialApplyMaxUsers+1) + for i := 1; i <= partialApplyMaxUsers+1; i++ { + users = append(users, mtUser(fmt.Sprintf("u%d", i), "mtproto", numberedSecret(i))) + } + if err := m.UpdateUsers(context.Background(), users); err != nil { + t.Fatalf("UpdateUsers: %v", err) + } + + assertCalls(t, api.takeCalls(), "POST /v1/system/reload", "GET /v1/system/reload/1") + after, err := os.Stat(api.configPath) + if err != nil { + t.Fatal(err) + } + if os.SameFile(before, after) { + t.Error("the config was not rendered again before the reload") + } + runtime := api.runtimeUsers() + if len(runtime) != partialApplyMaxUsers+2 || runtime["u33"] != numberedSecret(33) || runtime["u0"] != testSecret { + t.Errorf("telemt users after the reload = %v, want u0 to u33", runtime) + } + if got := len(desiredNames(m)); got != partialApplyMaxUsers+2 { + t.Errorf("desired has %d users, want %d", got, partialApplyMaxUsers+2) + } +} + +func TestFullSyncReportsRejectedReload(t *testing.T) { + cases := map[string]func(f *fakeTelemtAPI) string{ + "rolled back": func(f *fakeTelemtAPI) string { + f.reloadState, f.reloadError = "rolled_back", "bad config" + return "bad config" + }, + "failed": func(f *fakeTelemtAPI) string { + f.reloadState, f.reloadError = "failed", "listener bind failed" + return "listener bind failed" + }, + "refused": func(f *fakeTelemtAPI) string { + f.reloadRejectStatus = http.StatusBadRequest + return "bad listener" + }, + } + for name, setup := range cases { + t.Run(name, func(t *testing.T) { + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{mtUser("u1", "mtproto", testSecret)}) + var want string + api.set(func(f *fakeTelemtAPI) { want = setup(f) }) + + err := m.SyncUsers(context.Background(), []*common.User{mtUser("u2", "mtproto", otherSecret)}) + if err == nil || !strings.Contains(err.Error(), want) { + t.Fatalf("SyncUsers = %v, want an error containing %q", err, want) + } + if got := desiredNames(m); !slices.Equal(got, []string{"u1"}) { + t.Errorf("desired = %q after a rejected reload, want u1", got) + } + if !m.process.Started() { + t.Error("a rejected reload stopped telemt") + } + // telemt watches the file: it must hold the confirmed users again. + data, err := os.ReadFile(api.configPath) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), "u1 = ") || strings.Contains(string(data), "u2 = ") { + t.Errorf("config after a rejected reload is not the previous user set:\n%s", data) + } + }) + } +} + +func TestFullSyncRetriesReloadInProgress(t *testing.T) { + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{mtUser("u1", "mtproto", testSecret)}) + api.set(func(f *fakeTelemtAPI) { + f.reloadInProgress = 1 + f.reloadPending = 2 + }) + + if err := m.SyncUsers(context.Background(), []*common.User{mtUser("u2", "mtproto", testSecret)}); err != nil { + t.Fatalf("SyncUsers: %v", err) + } + assertCalls(t, api.takeCalls(), + "POST /v1/system/reload", "POST /v1/system/reload", + "GET /v1/system/reload/1", "GET /v1/system/reload/1", "GET /v1/system/reload/1") + api.set(func(f *fakeTelemtAPI) { + for _, body := range f.reloadBodies { + if body != `{"mode":"instant","failure_policy":"rollback"}` { + t.Errorf("reload body = %s", body) + } + } + }) + if got := desiredNames(m); !slices.Equal(got, []string{"u2"}) { + t.Errorf("desired = %q, want u2", got) + } + if runtime := api.runtimeUsers(); len(runtime) != 1 || runtime["u2"] == "" { + t.Errorf("telemt users = %v, want u2", runtime) + } +} + +// TestFullSyncWithoutChangesSkipsReload: a reload cuts every telemt session, +// so a full sync of the confirmed users sends nothing; once the state is +// unconfirmed, the same sync reloads. +func TestFullSyncWithoutChangesSkipsReload(t *testing.T) { + api := newFakeTelemtAPI(t) + users := []*common.User{mtUser("u1", "mtproto", testSecret), mtUser("u2", "mtproto", testSecret)} + m := newSyncTestBackend(t, api, users) + + if err := m.SyncUsers(context.Background(), users); err != nil { + t.Fatalf("SyncUsers: %v", err) + } + assertCalls(t, api.takeCalls()) + + m.syncMu.Lock() + m.needsFullApply = true + m.syncMu.Unlock() + if err := m.SyncUsers(context.Background(), users); err != nil { + t.Fatalf("SyncUsers (unconfirmed): %v", err) + } + assertCalls(t, api.takeCalls(), "POST /v1/system/reload", "GET /v1/system/reload/1") +} + +func TestZeroUsersUsesSentinelPath(t *testing.T) { + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{mtUser("u1", "mtproto", testSecret)}) + ctx := context.Background() + + if err := m.UpdateUsers(ctx, []*common.User{untagged("u1")}); err != nil { + t.Fatalf("UpdateUsers to zero users: %v", err) + } + assertCalls(t, api.takeCalls(), "POST /v1/system/reload", "GET /v1/system/reload/1") + if got := desiredNames(m); len(got) != 0 { + t.Errorf("desired = %q, want none", got) + } + if runtime := api.runtimeUsers(); len(runtime) != 1 || runtime[m.sentinel.Username] == "" { + t.Errorf("telemt users = %v, want only the sentinel", runtime) + } + + if err := m.SyncUser(ctx, mtUser("u2", "mtproto", testSecret)); err != nil { + t.Fatalf("SyncUser from zero users: %v", err) + } + assertCalls(t, api.takeCalls(), "POST /v1/system/reload", "GET /v1/system/reload/2") + if runtime := api.runtimeUsers(); len(runtime) != 1 || runtime["u2"] == "" { + t.Errorf("telemt users = %v, want only u2", runtime) + } +} + +func TestInvalidUserDoesNotFailBatch(t *testing.T) { + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{mtUser("u1", "mtproto", testSecret)}) + + broken := &common.User{Email: "u2", Inbounds: []string{"mtproto"}} + if err := m.SyncUsers(context.Background(), []*common.User{mtUser("u3", "mtproto", testSecret), broken}); err != nil { + t.Fatalf("SyncUsers: %v", err) + } + if got := desiredNames(m); !slices.Equal(got, []string{"u3"}) { + t.Errorf("desired = %q, want u3", got) + } + + if err := m.UpdateUsers(context.Background(), []*common.User{mtUser("u4", "mtproto", testSecret), broken}); err != nil { + t.Fatalf("UpdateUsers: %v", err) + } + if got := desiredNames(m); !slices.Equal(got, []string{"u3", "u4"}) { + t.Errorf("desired = %q, want u3, u4", got) + } +} + +func TestOnlineIPListUsesBulkEndpoint(t *testing.T) { + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{ + mtUser("u1", "mtproto", testSecret), + mtUser("u2", "mtproto", testSecret), + }) + api.set(func(f *fakeTelemtAPI) { + f.activeIPs = map[string][]string{"u1": {"203.0.113.7", "2001:db8::1"}, "u2": {"198.51.100.1"}} + }) + + response, err := m.GetUserOnlineIpListStats(context.Background(), "u1") + if err != nil { + t.Fatalf("GetUserOnlineIpListStats: %v", err) + } + assertCalls(t, api.takeCalls(), "GET /v1/stats/users/active-ips") + if len(response.GetIps()) != 2 || response.GetIps()["203.0.113.7"] == 0 || response.GetIps()["2001:db8::1"] == 0 { + t.Errorf("Ips = %v, want the two addresses of u1", response.GetIps()) + } +} + +// TestUnconfirmedReloadForcesFullApply: the caller gives up while telemt is +// still reloading. telemt keeps running, desired keeps the confirmed users, +// and the next sync renders and reloads everything instead of a diff. +func TestUnconfirmedReloadForcesFullApply(t *testing.T) { + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{ + mtUser("u1", "mtproto", testSecret), + mtUser("u2", "mtproto", testSecret), + }) + api.set(func(f *fakeTelemtAPI) { f.reloadHang = true }) + + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + err := m.SyncUsers(ctx, []*common.User{mtUser("u1", "mtproto", testSecret), mtUser("u3", "mtproto", testSecret)}) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("SyncUsers = %v, want a deadline error", err) + } + if got := desiredNames(m); !slices.Equal(got, []string{"u1", "u2"}) { + t.Errorf("desired = %q after an unconfirmed reload, want u1, u2", got) + } + if !m.process.Started() { + t.Error("an unconfirmed reload stopped telemt") + } + api.takeCalls() + + api.set(func(f *fakeTelemtAPI) { f.reloadHang = false }) + if err := m.UpdateUsers(context.Background(), []*common.User{mtUser("u2", "mtproto", otherSecret)}); err != nil { + t.Fatalf("UpdateUsers: %v", err) + } + assertCalls(t, api.takeCalls(), "POST /v1/system/reload", "GET /v1/system/reload/2") + if got := desiredNames(m); !slices.Equal(got, []string{"u1", "u2"}) { + t.Errorf("desired = %q, want u1, u2", got) + } + + // Confirmed again: the next small change is a diff. + if err := m.UpdateUsers(context.Background(), []*common.User{mtUser("u1", "mtproto", otherSecret)}); err != nil { + t.Fatalf("UpdateUsers: %v", err) + } + assertCalls(t, api.takeCalls(), "PATCH /v1/users/u1") +} + +func TestPartialFailureFallsBackToFullApply(t *testing.T) { + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{ + mtUser("u1", "mtproto", testSecret), + mtUser("u2", "mtproto", testSecret), + }) + api.set(func(f *fakeTelemtAPI) { f.failPatch = true }) + + if err := m.UpdateUsers(context.Background(), []*common.User{mtUser("u2", "mtproto", otherSecret)}); err != nil { + t.Fatalf("UpdateUsers: %v", err) + } + assertCalls(t, api.takeCalls(), "PATCH /v1/users/u2", "POST /v1/system/reload", "GET /v1/system/reload/1") + if got := api.runtimeUsers()["u2"]; got != otherSecret { + t.Errorf("telemt u2 secret = %q, want %q", got, otherSecret) + } + if got := desiredSecret(m, "u2"); got != otherSecret { + t.Errorf("desired u2 secret = %q, want %q", got, otherSecret) + } +} + +func TestRuntimeUserEqual(t *testing.T) { + base := runtimeUser{Username: "u1", Secret: testSecret, UserAdTag: otherSecret, MaxTCPConns: 2, MaxUniqueIPs: 3} + same := base + if !runtimeUserEqual(&base, &same) { + t.Error("identical users compare unequal") + } + for name, change := range map[string]func(u *runtimeUser){ + "secret": func(u *runtimeUser) { u.Secret = otherSecret }, + "user_ad_tag": func(u *runtimeUser) { u.UserAdTag = "" }, + "max_tcp_conns": func(u *runtimeUser) { u.MaxTCPConns = 0 }, + "max_unique_ips": func(u *runtimeUser) { u.MaxUniqueIPs = 9 }, + } { + changed := base + change(&changed) + if runtimeUserEqual(&base, &changed) { + t.Errorf("users differing in %s compare equal", name) + } + } + if runtimeUserEqual(&base, nil) || !runtimeUserEqual(nil, nil) { + t.Error("nil handling is wrong") + } +} + +func TestAPIClientListUsersAndActiveIPs(t *testing.T) { + api := newFakeTelemtAPI(t) + api.set(func(f *fakeTelemtAPI) { + f.users = map[string]string{"u1": testSecret} + f.activeIPs = map[string][]string{"u1": {"203.0.113.7"}} + }) + client := newAPIClient(api.server.URL, "Bearer test") + + users, err := client.ListUsers(context.Background()) + if err != nil { + t.Fatalf("ListUsers: %v", err) + } + if len(users) != 1 || users[0].Username != "u1" || !users[0].InRuntime { + t.Errorf("ListUsers = %+v", users) + } + + ips, err := client.ActiveIPs(context.Background()) + if err != nil { + t.Fatalf("ActiveIPs: %v", err) + } + if !slices.Equal(ips["u1"], []string{"203.0.113.7"}) { + t.Errorf("ActiveIPs = %v", ips) + } +} From 47cd1618b853ace104fa56b311e17f02f42c329a Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 19 Oct 2026 20:59:07 +0330 Subject: [PATCH 08/40] fix(mtproto): confirm hot reloads with a marker user instead of the reload API telemt's API reload swaps the runtime generation and, in instant mode, stops every session of the old one. A hot reload (SIGHUP, or telemt's config file watcher) updates the users live and cancels only changed or removed ones, so full applies go back to it. Every render now carries one disabled marker user with a fresh random name (pgm_<16 hex>). After writing the config the node sends SIGHUP (on Windows it relies on telemt's file watcher) and polls GET /v1/users until the marker is in telemt's running config, for up to 30 s, detached from the caller and ended by Shutdown. A config telemt did not load by then is rejected: the previous users are rendered again with a new marker and the error carries telemt's last output. A caller that gives up leaves telemt running and forces the next sync to apply all users; only an API that gets no answer restarts telemt. The marker replaces the zero-user sentinel and keeps telemt's user list non-empty, so the last real user is deleted, and the first one created, through the user API. The reload API client code is removed. --- backend/mtproto/client.go | 118 -------- backend/mtproto/config.go | 20 +- backend/mtproto/config_test.go | 20 +- backend/mtproto/fake_telemt_test.go | 102 ++++++- backend/mtproto/integration_test.go | 10 +- backend/mtproto/lifecycle.go | 6 - backend/mtproto/lifecycle_test.go | 33 +- backend/mtproto/mtproto.go | 35 ++- backend/mtproto/process.go | 19 +- backend/mtproto/process_test.go | 27 ++ backend/mtproto/process_unix.go | 8 + backend/mtproto/process_windows.go | 6 + backend/mtproto/sync.go | 156 +++++++--- backend/mtproto/sync_test.go | 446 +++++++++++++++++----------- 14 files changed, 599 insertions(+), 407 deletions(-) diff --git a/backend/mtproto/client.go b/backend/mtproto/client.go index c430d383..5d8baeb4 100644 --- a/backend/mtproto/client.go +++ b/backend/mtproto/client.go @@ -9,22 +9,10 @@ import ( "io" "net/http" "net/url" - "strconv" "strings" "time" ) -const ( - // reloadRetryInterval is how often a reload is requested again while - // telemt runs another one. - reloadRetryInterval = 200 * time.Millisecond - // reloadPollInterval is how often an accepted reload is checked. - reloadPollInterval = 100 * time.Millisecond -) - -// errAPIUnreachable marks a reload request that got no answer from telemt. -var errAPIUnreachable = errors.New("telemt API unreachable") - type apiClient struct { baseURL string authHeader string @@ -69,19 +57,6 @@ type telemtUserActiveIPs struct { ActiveIPs []string `json:"active_ips"` } -type telemtReloadRequest struct { - Mode string `json:"mode"` - FailurePolicy string `json:"failure_policy"` -} - -// telemtReloadStatus is both the 202 answer to a reload request and the -// status of that reload. -type telemtReloadStatus struct { - ReloadID uint64 `json:"reload_id"` - State string `json:"state"` - Error string `json:"error"` -} - type telemtHealth struct { Status string `json:"status"` ReadOnly bool `json:"read_only"` @@ -136,31 +111,6 @@ func (e *apiStatusError) UserExists() bool { return e.StatusCode == http.StatusConflict && e.Code == "user_exists" } -// ReloadInProgress reports telemt's answer to a reload request while another -// reload runs: 409 reload_in_progress, "Reload N is already in progress". -func (e *apiStatusError) ReloadInProgress() bool { - return e.StatusCode == http.StatusConflict && - (strings.Contains(e.Code, "in_progress") || strings.Contains(strings.ToLower(e.Message), "already in progress")) -} - -// reloadRejectedError reports a reload telemt refused, rolled back or failed. -// telemt keeps running the config it had before. -type reloadRejectedError struct { - State string // rolled_back or failed; empty when the request was refused - Err error -} - -func (e *reloadRejectedError) Error() string { - if e.State == "" { - return fmt.Sprintf("telemt refused the config reload: %v", e.Err) - } - return fmt.Sprintf("telemt config reload %s: %v", e.State, e.Err) -} - -func (e *reloadRejectedError) Unwrap() error { - return e.Err -} - func newAPIClient(baseURL, authHeader string) *apiClient { return &apiClient{ baseURL: strings.TrimRight(baseURL, "/"), @@ -216,62 +166,6 @@ func (c *apiClient) ActiveIPs(ctx context.Context) (map[string][]string, error) return activeIPs, nil } -// Reload makes telemt load its config file into a new runtime generation and -// waits for the outcome. It returns nil once the reload succeeded, a -// *reloadRejectedError when telemt refused it or ended it rolled_back or -// failed, an error wrapping errAPIUnreachable when the request got no answer, -// and ctx's error when ctx ends first. While telemt runs another reload it -// answers 409, and the request is repeated until ctx ends. -func (c *apiClient) Reload(ctx context.Context) error { - request := telemtReloadRequest{Mode: "instant", FailurePolicy: "rollback"} - - var accepted *telemtReloadStatus - for { - var err error - accepted, err = doAPIRequest[telemtReloadStatus](ctx, c, http.MethodPost, "/v1/system/reload", request) - if err == nil { - break - } - if ctxErr := ctx.Err(); ctxErr != nil { - return ctxErr - } - var apiErr *apiStatusError - if !errors.As(err, &apiErr) { - return fmt.Errorf("%w: %w", errAPIUnreachable, err) - } - if !apiErr.ReloadInProgress() { - return &reloadRejectedError{Err: apiErr} - } - if err := sleepContext(ctx, reloadRetryInterval); err != nil { - return err - } - } - - statusPath := "/v1/system/reload/" + strconv.FormatUint(accepted.ReloadID, 10) - for { - status, err := doAPIRequest[telemtReloadStatus](ctx, c, http.MethodGet, statusPath, nil) - if err != nil { - if ctxErr := ctx.Err(); ctxErr != nil { - return ctxErr - } - return fmt.Errorf("failed to read the state of telemt reload %d: %w", accepted.ReloadID, err) - } - switch status.State { - case "succeeded": - return nil - case "rolled_back", "failed": - reason := status.Error - if reason == "" { - reason = "telemt gave no reason" - } - return &reloadRejectedError{State: status.State, Err: errors.New(reason)} - } - if err := sleepContext(ctx, reloadPollInterval); err != nil { - return err - } - } -} - func (c *apiClient) CreateUser(ctx context.Context, user *runtimeUser) error { request, err := buildCreateUserRequest(user) if err != nil { @@ -427,15 +321,3 @@ func doAPIRequest[T any](ctx context.Context, client *apiClient, method, path st return &success.Data, nil } - -// sleepContext waits for d or until ctx ends, and returns ctx's error then. -func sleepContext(ctx context.Context, d time.Duration) error { - timer := time.NewTimer(d) - defer timer.Stop() - select { - case <-ctx.Done(): - return ctx.Err() - case <-timer.C: - return nil - } -} diff --git a/backend/mtproto/config.go b/backend/mtproto/config.go index f2c6943f..d7af06f5 100644 --- a/backend/mtproto/config.go +++ b/backend/mtproto/config.go @@ -251,7 +251,7 @@ func portValue(field string, value any) (int, error) { return int(port), nil } -func (c *Config) Render(users map[string]*runtimeUser, sentinel *runtimeUser, apiPort, metricsPort int, apiAuthHeader string) (string, error) { +func (c *Config) Render(users map[string]*runtimeUser, marker *runtimeUser, apiPort, metricsPort int, apiAuthHeader string) (string, error) { if c == nil { return "", errors.New("mtproto config is nil") } @@ -297,16 +297,16 @@ func (c *Config) Render(users map[string]*runtimeUser, sentinel *runtimeUser, ap api["auth_header"] = apiAuthHeader api["whitelist"] = []any{loopbackV4CIDR, loopbackV6CIDR} - if err := applyUsersToAccess(access, users, sentinel); err != nil { + if err := applyUsersToAccess(access, users, marker); err != nil { return "", err } return renderTOML(root) } -func applyUsersToAccess(access map[string]any, users map[string]*runtimeUser, sentinel *runtimeUser) error { - if len(users) == 0 && sentinel == nil { - return errors.New("mtproto config requires at least one user or sentinel") +func applyUsersToAccess(access map[string]any, users map[string]*runtimeUser, marker *runtimeUser) error { + if len(users) == 0 && marker == nil { + return errors.New("mtproto config requires at least one user or a marker user") } userSecrets := make(map[string]any, len(users)+1) @@ -335,11 +335,11 @@ func applyUsersToAccess(access map[string]any, users map[string]*runtimeUser, se } } - if sentinel != nil { - // Keep [access.users] non-empty so telemt will start, but disable the - // sentinel so it cannot authenticate as a real MTProto client. - userSecrets[sentinel.Username] = sentinel.Secret - userEnabled[sentinel.Username] = false + if marker != nil { + // The marker keeps [access.users] non-empty so telemt will start; it + // is disabled so it cannot authenticate as a real MTProto client. + userSecrets[marker.Username] = marker.Secret + userEnabled[marker.Username] = false } access["users"] = userSecrets diff --git a/backend/mtproto/config_test.go b/backend/mtproto/config_test.go index 1722369e..cca109f2 100644 --- a/backend/mtproto/config_test.go +++ b/backend/mtproto/config_test.go @@ -297,22 +297,22 @@ func TestConfigRender_InjectsManagedSections(t *testing.T) { t.Errorf("real users must not disable anyone via access.user_enabled") } - // empty users + sentinel keeps telemt happy (telemt rejects empty [access.users]) - sentinel := &runtimeUser{Username: "pg-node-internal-x", Secret: testSecret} - out, err = cfg.Render(map[string]*runtimeUser{}, sentinel, 1, 2, "") + // empty users + marker keeps telemt happy (telemt rejects empty [access.users]) + marker := &runtimeUser{Username: "pgm_0123456789abcdef", Secret: testSecret} + out, err = cfg.Render(map[string]*runtimeUser{}, marker, 1, 2, "") if err != nil { - t.Fatalf("render with sentinel failed: %v", err) + t.Fatalf("render with marker failed: %v", err) } - if !strings.Contains(out, "pg-node-internal-x") { - t.Errorf("sentinel user missing from rendered config") + if !strings.Contains(out, "pgm_0123456789abcdef") { + t.Errorf("marker user missing from rendered config") } - if !strings.Contains(out, "[access.user_enabled]") || !strings.Contains(out, "pg-node-internal-x = false") { - t.Errorf("sentinel must be rendered disabled, got:\n%s", out) + if !strings.Contains(out, "[access.user_enabled]") || !strings.Contains(out, "pgm_0123456789abcdef = false") { + t.Errorf("marker must be rendered disabled, got:\n%s", out) } - // empty users + no sentinel -> error (would crash telemt) + // empty users + no marker -> error (would crash telemt) if _, err := cfg.Render(map[string]*runtimeUser{}, nil, 1, 2, ""); err == nil { - t.Errorf("expected error rendering with zero users and no sentinel") + t.Errorf("expected error rendering with zero users and no marker") } } diff --git a/backend/mtproto/fake_telemt_test.go b/backend/mtproto/fake_telemt_test.go index e12a65d9..f9a7d670 100644 --- a/backend/mtproto/fake_telemt_test.go +++ b/backend/mtproto/fake_telemt_test.go @@ -9,9 +9,12 @@ import ( "fmt" "net/http" "os" + "os/signal" "path/filepath" "strconv" "strings" + "sync" + "syscall" "testing" "time" @@ -43,9 +46,14 @@ type fakeTelemt struct { HealthDelay time.Duration `json:"health_delay"` NoAPI bool `json:"no_api"` ReadyStatus int `json:"ready_status"` // default 200 - // DropReload closes reload requests without an answer, like an API that - // cannot be reached. - DropReload bool `json:"drop_reload"` + // ServeUsers serves GET /v1/users like telemt: the users of the config + // file, in_runtime for those loaded at start or on the latest SIGHUP. + ServeUsers bool `json:"serve_users"` + // DropUsers closes GET /v1/users requests without an answer, like an API + // that cannot be reached. + DropUsers bool `json:"drop_users"` + // SignalFile gets one line per SIGHUP received. + SignalFile string `json:"signal_file"` // Lifetime bounds the fake so a broken test never leaves it running. Lifetime time.Duration `json:"lifetime"` } @@ -86,6 +94,16 @@ func runFakeTelemt(optionsPath string, args []string) int { return 0 } + hup := make(chan os.Signal, 1) + signal.Notify(hup, syscall.SIGHUP) + + var configPath string + if len(args) > 0 { + configPath = args[len(args)-1] + } + runtime := &fakeTelemtRuntime{} + runtime.load(configPath) + if f.RecordFile != "" { cwd, _ := os.Getwd() payload, _ := json.Marshal(fakeTelemtRecord{Args: args, Cwd: cwd, PID: os.Getpid()}) @@ -117,8 +135,8 @@ func runFakeTelemt(optionsPath string, args []string) int { } } - if !f.NoAPI && len(args) > 0 { - go serveFakeTelemtAPI(f, args[len(args)-1]) + if !f.NoAPI && configPath != "" { + go serveFakeTelemtAPI(f, configPath, runtime) } lifetime := f.Lifetime @@ -127,16 +145,52 @@ func runFakeTelemt(optionsPath string, args []string) int { } deadline := time.After(lifetime) - select { - case <-exit: - fmt.Fprintln(os.Stderr, "fake telemt exiting") - return f.ExitCode - case <-deadline: - return 0 + for { + select { + case <-hup: + runtime.load(configPath) + if f.SignalFile != "" { + file, err := os.OpenFile(f.SignalFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) + if err == nil { + _, _ = file.WriteString("SIGHUP\n") + _ = file.Close() + } + } + case <-exit: + fmt.Fprintln(os.Stderr, "fake telemt exiting") + return f.ExitCode + case <-deadline: + return 0 + } + } +} + +// fakeTelemtRuntime holds the users of the fake's running config. +type fakeTelemtRuntime struct { + mu sync.Mutex + users map[string]string +} + +// load reads the users of the config file into the running config, as a +// telemt start or SIGHUP reload does. +func (r *fakeTelemtRuntime) load(configPath string) { + data, err := os.ReadFile(configPath) + if err != nil { + return } + r.mu.Lock() + defer r.mu.Unlock() + r.users = parseConfigUsers(string(data)) +} + +func (r *fakeTelemtRuntime) has(username string) bool { + r.mu.Lock() + defer r.mu.Unlock() + _, ok := r.users[username] + return ok } -func serveFakeTelemtAPI(f fakeTelemt, configPath string) { +func serveFakeTelemtAPI(f fakeTelemt, configPath string, runtime *fakeTelemtRuntime) { if f.HealthDelay > 0 { time.Sleep(f.HealthDelay) } @@ -164,13 +218,29 @@ func serveFakeTelemtAPI(f fakeTelemt, configPath string) { } _, _ = w.Write([]byte(`{"ok":true,"data":{"ready":false,"status":"not_ready","reason":"no_healthy_upstreams"},"revision":"r"}`)) }) - if f.DropReload { - mux.HandleFunc("/v1/system/reload", func(w http.ResponseWriter, _ *http.Request) { + mux.HandleFunc("/v1/users", func(w http.ResponseWriter, _ *http.Request) { + if f.DropUsers { if conn, _, err := http.NewResponseController(w).Hijack(); err == nil { _ = conn.Close() } - }) - } + return + } + if !f.ServeUsers { + http.NotFound(w, nil) + return + } + data, err := os.ReadFile(configPath) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + return + } + users := []map[string]any{} + for name := range parseConfigUsers(string(data)) { + users = append(users, map[string]any{"username": name, "in_runtime": runtime.has(name)}) + } + payload, _ := json.Marshal(map[string]any{"ok": true, "data": users, "revision": "r"}) + _, _ = w.Write(payload) + }) if err := http.ListenAndServe(listen, mux); err != nil { fmt.Fprintln(os.Stderr, "fake telemt: api:", err) diff --git a/backend/mtproto/integration_test.go b/backend/mtproto/integration_test.go index 5f873c8c..76e6fd30 100644 --- a/backend/mtproto/integration_test.go +++ b/backend/mtproto/integration_test.go @@ -14,7 +14,7 @@ import ( // Gated integration test: runs only when MTPROTO_TELEMT_BIN points at a real // telemt binary. Exercises the full lifecycle (start, health, users, stats, -// sentinel/empty-users) against the actual proxy. +// marker user/empty-users) against the actual proxy. // // MTPROTO_TELEMT_BIN=/path/to/telemt go test ./backend/mtproto/ -run Integration -v func telemtBinOrSkip(t *testing.T) string { @@ -140,7 +140,7 @@ func TestIntegration_LifecycleAndUsers(t *testing.T) { } } -func TestIntegration_EmptyUsersSentinel(t *testing.T) { +func TestIntegration_EmptyUsersMarker(t *testing.T) { bin := telemtBinOrSkip(t) // telemt binds its API only after the Telegram DC check (up to about 60 s @@ -148,15 +148,15 @@ func TestIntegration_EmptyUsersSentinel(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 150*time.Second) defer cancel() - // Zero real users: telemt rejects an empty [access.users], so the sentinel + // Zero real users: telemt rejects an empty [access.users], so the marker // user must keep it alive. mt, err := New(ctx, integrationConfig(t, bin), integrationMTConfig(t), nil) if err != nil { - t.Fatalf("New with zero users (sentinel path): %v", err) + t.Fatalf("New with zero users (marker only): %v", err) } defer mt.Shutdown() if !mt.Started() { - t.Fatal("backend not started with sentinel user") + t.Fatal("backend not started with the marker user only") } } diff --git a/backend/mtproto/lifecycle.go b/backend/mtproto/lifecycle.go index 7d3d094c..1a3410fc 100644 --- a/backend/mtproto/lifecycle.go +++ b/backend/mtproto/lifecycle.go @@ -64,11 +64,6 @@ func New(ctx context.Context, cfg *config.Config, mtConfig *Config, users []*com if err != nil { return nil, err } - sentinel, err := newSentinelUser() - if err != nil { - return nil, err - } - telemtDir, err := prepareTelemtDir(cfg.GeneratedConfigPath) if err != nil { return nil, err @@ -89,7 +84,6 @@ func New(ctx context.Context, cfg *config.Config, mtConfig *Config, users []*com apiClient: newAPIClient(fmt.Sprintf("http://127.0.0.1:%d", apiPort), authHeader), metricsScraper: newMetricsScraper(fmt.Sprintf("http://127.0.0.1:%d/metrics", metricsPort)), tracker: newTrafficTracker(), - sentinel: sentinel, apiPort: apiPort, metricsPort: metricsPort, apiAuthHeader: authHeader, diff --git a/backend/mtproto/lifecycle_test.go b/backend/mtproto/lifecycle_test.go index 744daa6b..df62e63e 100644 --- a/backend/mtproto/lifecycle_test.go +++ b/backend/mtproto/lifecycle_test.go @@ -366,10 +366,35 @@ func TestCancelledSyncKeepsTelemtRunning(t *testing.T) { } } -// TestReloadUnreachableRestartsTelemt: a reload request that cannot reach -// telemt's API restarts telemt on the new config, and that start confirms it. -func TestReloadUnreachableRestartsTelemt(t *testing.T) { - exe := writeFakeTelemt(t, fakeTelemt{DropReload: true}) +// TestFullSyncSignalsTelemt: a full apply makes telemt reload with SIGHUP +// and returns once the render's marker user is in telemt's running config. +// This fake has no config watcher, so only the signal can confirm it. +func TestFullSyncSignalsTelemt(t *testing.T) { + signals := filepath.Join(t.TempDir(), "signals") + exe := writeFakeTelemt(t, fakeTelemt{ServeUsers: true, SignalFile: signals}) + mt := newTestBackend(t, testNodeConfig(t, exe), []*common.User{mtUser("alice", "mtproto", testSecret)}) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := mt.SyncUsers(ctx, []*common.User{mtUser("bob", "mtproto", testSecret)}); err != nil { + t.Fatalf("SyncUsers: %v", err) + } + if data, _ := os.ReadFile(signals); !strings.Contains(string(data), "SIGHUP") { + t.Errorf("telemt got no SIGHUP (signal log %q)", data) + } + if !mt.Started() || mt.process.Generation() != 1 { + t.Errorf("telemt was restarted (started %v, generation %d)", mt.Started(), mt.process.Generation()) + } + if got := desiredNames(mt); !slices.Equal(got, []string{"bob"}) { + t.Errorf("desired = %q, want bob", got) + } +} + +// TestUsersAPIUnreachableRestartsTelemt: when telemt's API gets no answer +// while the node waits for a reload, the node restarts telemt on the new +// config, and that start confirms it. +func TestUsersAPIUnreachableRestartsTelemt(t *testing.T) { + exe := writeFakeTelemt(t, fakeTelemt{DropUsers: true}) mt := newTestBackend(t, testNodeConfig(t, exe), []*common.User{mtUser("alice", "mtproto", testSecret)}) ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) diff --git a/backend/mtproto/mtproto.go b/backend/mtproto/mtproto.go index 9fe41ba7..8ea03a58 100644 --- a/backend/mtproto/mtproto.go +++ b/backend/mtproto/mtproto.go @@ -25,7 +25,6 @@ type MTProto struct { apiClient *apiClient metricsScraper *metricsScraper tracker *trafficTracker - sentinel *runtimeUser apiPort int metricsPort int @@ -40,10 +39,12 @@ type MTProto struct { // The next sync then renders and reloads every user. Guarded by syncMu. needsFullApply bool - // stateMu guards desired for readers; writers also hold syncMu. desired - // holds only users telemt confirmed it runs. + // stateMu guards desired and marker for readers; writers also hold + // syncMu. desired holds only users telemt confirmed it runs. marker is + // the disabled user of the latest render, never a real user. stateMu sync.RWMutex desired map[string]*runtimeUser + marker *runtimeUser // lifetime is cancelled by Shutdown so waits for telemt stop early. lifetime context.Context @@ -238,18 +239,36 @@ func (m *MTProto) isDesiredUser(username string) bool { return exists } -func newSentinelUser() (*runtimeUser, error) { +// markerUsernamePrefix starts every marker username. Panel usernames are +// numeric user ids, so no real user has it. +const markerUsernamePrefix = "pgm_" + +// markerUsername returns the marker user of the latest render. Stats must +// never report it; older markers share markerUsernamePrefix. +func (m *MTProto) markerUsername() string { + m.stateMu.RLock() + defer m.stateMu.RUnlock() + if m.marker == nil { + return "" + } + return m.marker.Username +} + +// newMarkerUser returns a marker user with a fresh random name and secret. +// Every render carries one, disabled: once telemt's running config has it, +// telemt runs that render. It also keeps telemt's user list non-empty. +func newMarkerUser() (*runtimeUser, error) { secret := make([]byte, 16) - usernameSuffix := make([]byte, 4) + usernameSuffix := make([]byte, 8) if _, err := rand.Read(secret); err != nil { - return nil, fmt.Errorf("failed to generate mtproto sentinel secret: %w", err) + return nil, fmt.Errorf("failed to generate the mtproto marker secret: %w", err) } if _, err := rand.Read(usernameSuffix); err != nil { - return nil, fmt.Errorf("failed to generate mtproto sentinel username: %w", err) + return nil, fmt.Errorf("failed to generate the mtproto marker username: %w", err) } return &runtimeUser{ - Username: "pg-node-internal-" + hex.EncodeToString(usernameSuffix), + Username: markerUsernamePrefix + hex.EncodeToString(usernameSuffix), Secret: hex.EncodeToString(secret), }, nil } diff --git a/backend/mtproto/process.go b/backend/mtproto/process.go index 35efdb5f..22dd9806 100644 --- a/backend/mtproto/process.go +++ b/backend/mtproto/process.go @@ -163,8 +163,8 @@ func (p *processManager) Start(configPath string) error { cleanupOrphanedTelemt(configPath) // No --pid-file: a pid file that survives a hard kill can name a live - // process later and make telemt refuse to start. The node reloads - // through telemt's API, which needs no pid. + // process later and make telemt refuse to start. The node reloads by + // signalling its own child instead. cmd := exec.Command(p.executablePath, "run", configPath) // telemt writes relative-path artifacts (the tls_front cache, for one) // into its working directory; keep them in the generated config dir. @@ -312,6 +312,21 @@ func (p *processManager) StartupTail(n int) []string { return p.startupTail.tail(n) } +// Reload asks the running telemt to re-read its config: SIGHUP on Unix. On +// Windows it does nothing, and telemt's config file watcher picks the change +// up. +func (p *processManager) Reload() error { + p.mu.Lock() + defer p.mu.Unlock() + if !p.startedLocked() { + return errors.New("telemt process is not running") + } + if err := reloadProcess(p.run.cmd); err != nil { + return fmt.Errorf("failed to signal telemt to reload: %w", err) + } + return nil +} + // Stop terminates telemt and waits for it to exit, killing it after // telemtStopTimeout. func (p *processManager) Stop() { diff --git a/backend/mtproto/process_test.go b/backend/mtproto/process_test.go index 9d2bf4ee..d24f0f92 100644 --- a/backend/mtproto/process_test.go +++ b/backend/mtproto/process_test.go @@ -5,6 +5,7 @@ package mtproto import ( "context" "fmt" + "os" "os/exec" "path/filepath" "slices" @@ -246,6 +247,32 @@ func TestProcessLongLineKeepsReading(t *testing.T) { } } +func TestProcessReloadSendsSIGHUP(t *testing.T) { + dir := t.TempDir() + record := filepath.Join(dir, "record.json") + signals := filepath.Join(dir, "signals") + manager := newTestProcessManager(t, fakeTelemt{NoAPI: true, RecordFile: record, SignalFile: signals}) + + if err := manager.Reload(); err == nil { + t.Error("Reload without a running telemt must fail") + } + if err := manager.Start(filepath.Join(dir, "config.toml")); err != nil { + t.Fatalf("Start: %v", err) + } + readFakeTelemtRecord(t, record) + + if err := manager.Reload(); err != nil { + t.Fatalf("Reload: %v", err) + } + waitFor(t, 10*time.Second, "the fake telemt to receive SIGHUP", func() bool { + data, _ := os.ReadFile(signals) + return string(data) == "SIGHUP\n" + }) + if !manager.Started() { + t.Error("telemt must keep running after a reload") + } +} + // TestOrphanCleanupKillsStaleTelemt starts a telemt for the same config path // outside the manager (left over from a killed node); Start must kill it, and // must leave a process for a different config path alone. diff --git a/backend/mtproto/process_unix.go b/backend/mtproto/process_unix.go index 71e74fe6..0efbc7e7 100644 --- a/backend/mtproto/process_unix.go +++ b/backend/mtproto/process_unix.go @@ -35,6 +35,14 @@ func killProcess(cmd *exec.Cmd) error { return cmd.Process.Kill() } +// reloadProcess sends SIGHUP, which telemt handles as a config reload. +func reloadProcess(cmd *exec.Cmd) error { + if cmd == nil || cmd.Process == nil { + return nil + } + return cmd.Process.Signal(syscall.SIGHUP) +} + // killOrphan SIGKILLs a stray telemt and, when it leads its own process group // (the node starts telemt with Setpgid), the rest of that group. func killOrphan(pid int) error { diff --git a/backend/mtproto/process_windows.go b/backend/mtproto/process_windows.go index 8be0d514..7852ef45 100644 --- a/backend/mtproto/process_windows.go +++ b/backend/mtproto/process_windows.go @@ -28,6 +28,12 @@ func killProcess(cmd *exec.Cmd) error { return cmd.Process.Kill() } +// reloadProcess does nothing: Windows has no SIGHUP, and telemt's config +// file watcher (it polls every 3 s) loads the new config. +func reloadProcess(_ *exec.Cmd) error { + return nil +} + func killOrphan(pid int) error { process, err := os.FindProcess(pid) if err != nil { diff --git a/backend/mtproto/sync.go b/backend/mtproto/sync.go index 6882e96b..a7a1ec54 100644 --- a/backend/mtproto/sync.go +++ b/backend/mtproto/sync.go @@ -7,6 +7,7 @@ import ( "log" "maps" "strings" + "time" "github.com/pasarguard/node/common" ) @@ -17,6 +18,19 @@ import ( // reloads it instead. const partialApplyMaxUsers = 32 +// markerPollInterval is how often the node checks whether telemt loaded a +// rendered config. +const markerPollInterval = 200 * time.Millisecond + +// markerConfirmTimeout bounds the wait for telemt to load a rendered config; +// a config it did not load by then counts as rejected. It is a variable so +// tests can lower it. +var markerConfirmTimeout = 30 * time.Second + +// errReloadRejected marks a rendered config telemt did not load. telemt +// keeps running the config it had. +var errReloadRejected = errors.New("telemt did not load the new config") + // userChange is one user to create, patch or delete in telemt. type userChange struct { username string @@ -43,7 +57,7 @@ func (m *MTProto) SyncUsers(ctx context.Context, users []*common.User) error { } desired, _ := ActiveUsersFromCommon(users, m.config.InboundTag) - // A reload cuts every telemt session; skip one that changes nothing. + // Nothing to apply: skip the render and the wait for telemt. if m.confirmedLocked() && maps.EqualFunc(m.desired, desired, runtimeUserEqual) { return nil } @@ -51,9 +65,9 @@ func (m *MTProto) SyncUsers(ctx context.Context, users []*common.User) error { } // UpdateUsers applies users on top of the current ones. Up to -// partialApplyMaxUsers changes go through telemt's user API; more, a change -// to or from zero users, or an unconfirmed telemt state go through a config -// render and reload. Users that did not change are not sent at all. +// partialApplyMaxUsers changes go through telemt's user API; more, or an +// unconfirmed telemt state, go through a config render and reload. Users +// that did not change are not sent at all. func (m *MTProto) UpdateUsers(ctx context.Context, users []*common.User) error { m.syncMu.Lock() defer m.syncMu.Unlock() @@ -70,11 +84,6 @@ func (m *MTProto) UpdateUsers(ctx context.Context, users []*common.User) error { if len(changes) == 0 { return nil } - // telemt refuses to delete its last user, and zero users means the - // sentinel, which only a render adds or drops. - if len(m.desired) == 0 || len(next) == 0 { - return m.applyFullLocked(ctx, next) - } if err := m.applyUserChangesLocked(ctx, changes); err != nil { // Some changes may have reached telemt: its users are unknown until @@ -211,8 +220,8 @@ func (m *MTProto) deleteUser(ctx context.Context, username string) error { return err } -// applyFullLocked renders desired and makes telemt run it: a reload of the -// running telemt, or a start when none runs. m.desired takes desired only +// applyFullLocked renders desired and makes telemt run it: a hot reload of +// the running telemt, or a start when none runs. m.desired takes desired only // once telemt confirmed it. Requires syncMu. func (m *MTProto) applyFullLocked(ctx context.Context, desired map[string]*runtimeUser) error { // A partial apply that failed on an expired ctx falls back here; do not @@ -233,17 +242,19 @@ func (m *MTProto) applyFullLocked(ctx context.Context, desired map[string]*runti } err := m.reloadLocked(ctx) - var rejected *reloadRejectedError switch { case err == nil: m.confirmLocked(desired) return nil - case errors.As(err, &rejected): + case errors.Is(err, errReloadRejected): // telemt still runs the users it had. Put them back on disk too: - // telemt watches the config file and would load the rejected one. + // telemt watches the config file and would retry the rejected one. if restoreErr := m.writeRenderedConfigLocked(m.desired); restoreErr != nil { return fmt.Errorf("%w (restoring the previous telemt config failed: %v)", err, restoreErr) } + if signalErr := m.process.Reload(); signalErr != nil { + log.Printf("warning: %v", signalErr) + } return err default: // The outcome is unknown, and telemt may still load the new config. @@ -252,30 +263,29 @@ func (m *MTProto) applyFullLocked(ctx context.Context, desired map[string]*runti } } -// reloadLocked makes the running telemt load the rendered config through its -// reload API and waits for the outcome. Only a request that cannot reach -// telemt restarts it, on the rendered config; a caller that gave up or a -// rejected config never does. Requires syncMu. +// reloadLocked makes the running telemt hot-reload the rendered config: +// SIGHUP (telemt's config watcher also sees the file change), then a wait +// until the render's marker user is in telemt's running config. A hot reload +// replaces only the users that changed; an API reload would cut every +// session. Only an API that cannot be reached, or a telemt that cannot be +// signalled, restarts telemt, on the rendered config; a caller that gave up +// or a rejected config never does. Requires syncMu. func (m *MTProto) reloadLocked(ctx context.Context) error { if err := ctx.Err(); err != nil { return err } - waitCtx, cancel := m.reloadContext(ctx) - defer cancel() - - err := m.apiClient.Reload(waitCtx) + err := m.process.Reload() + if err == nil { + err = m.waitForMarker(ctx, m.marker.Username) + } switch { - case err == nil: - return nil - case m.lifetime.Err() != nil: - return errClosed - case waitCtx.Err() != nil: - return fmt.Errorf("telemt reload not confirmed: %w", err) - case !errors.Is(err, errAPIUnreachable): + case err == nil || errors.Is(err, errClosed) || errors.Is(err, errReloadRejected): return err + case ctx.Err() != nil: + return fmt.Errorf("telemt reload not confirmed: %w", ctx.Err()) } - log.Printf("telemt API unreachable for a reload, restarting telemt: %v", err) + log.Printf("telemt did not answer a reload, restarting it: %v", err) m.process.Stop() // telemt is down now: start it again even if the caller gives up. if restartErr := m.startLocked(context.WithoutCancel(ctx)); restartErr != nil { @@ -284,21 +294,62 @@ func (m *MTProto) reloadLocked(ctx context.Context) error { return nil } -// reloadContext bounds the wait for a reload by ctx, by Shutdown, and by -// telemtStartupTimeout when ctx has no deadline: syncMu is held throughout. -func (m *MTProto) reloadContext(ctx context.Context) (context.Context, context.CancelFunc) { - var waitCtx context.Context - var cancel context.CancelFunc - if _, hasDeadline := ctx.Deadline(); hasDeadline { - waitCtx, cancel = context.WithCancel(ctx) - } else { - waitCtx, cancel = context.WithTimeout(ctx, telemtStartupTimeout) - } +// waitForMarker polls GET /v1/users until telemt's running config has the +// marker user, for up to markerConfirmTimeout. It returns ctx's error once +// the caller gives up, errClosed on Shutdown, an errReloadRejected error with +// telemt's last output when the time is up, and the request error when the +// API cannot be reached. +func (m *MTProto) waitForMarker(ctx context.Context, marker string) error { + // Requests do not end with the caller's ctx, so an aborted request is + // never mistaken for an unreachable API. + waitCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), markerConfirmTimeout) + defer cancel() stop := context.AfterFunc(m.lifetime, cancel) - return waitCtx, func() { - stop() - cancel() + defer stop() + + ticker := time.NewTicker(markerPollInterval) + defer ticker.Stop() + + for { + users, err := m.apiClient.ListUsers(waitCtx) + if err == nil && userInRuntime(users, marker) { + return nil + } + var apiErr *apiStatusError + if err != nil && !errors.As(err, &apiErr) && waitCtx.Err() == nil { + return fmt.Errorf("telemt API unreachable: %w", err) + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-waitCtx.Done(): + if m.lifetime.Err() != nil { + return errClosed + } + return m.reloadRejectedError() + case <-ticker.C: + } + } +} + +// reloadRejectedError reports a config telemt did not load in time, with +// its last output, which carries the reason. +func (m *MTProto) reloadRejectedError() error { + tail := m.process.StartupTail(telemtCrashTailSize) + if len(tail) == 0 { + return fmt.Errorf("%w within %s", errReloadRejected, markerConfirmTimeout) + } + return fmt.Errorf("%w within %s; recent telemt output:\n%s", errReloadRejected, markerConfirmTimeout, strings.Join(tail, "\n")) +} + +func userInRuntime(users []telemtUserInfo, username string) bool { + for _, user := range users { + if user.Username == username { + return user.InRuntime + } } + return false } // confirmLocked records desired as the users telemt runs. Requires syncMu. @@ -309,18 +360,25 @@ func (m *MTProto) confirmLocked(desired map[string]*runtimeUser) { m.desired = cloneRuntimeUserMap(desired) } +// writeRenderedConfigLocked writes desired plus a new marker user to the +// config file and records that marker. Requires syncMu. func (m *MTProto) writeRenderedConfigLocked(desired map[string]*runtimeUser) error { - var sentinel *runtimeUser - if len(desired) == 0 { - sentinel = m.sentinel + marker, err := newMarkerUser() + if err != nil { + return err } - - rendered, err := m.config.Render(desired, sentinel, m.apiPort, m.metricsPort, m.apiAuthHeader) + rendered, err := m.config.Render(desired, marker, m.apiPort, m.metricsPort, m.apiAuthHeader) if err != nil { return err } + if err := writeFileAtomic(m.configPath, []byte(rendered)); err != nil { + return err + } - return writeFileAtomic(m.configPath, []byte(rendered)) + m.stateMu.Lock() + defer m.stateMu.Unlock() + m.marker = marker + return nil } // runtimeUserEqual reports whether a and b give telemt the same user. diff --git a/backend/mtproto/sync_test.go b/backend/mtproto/sync_test.go index 727f408b..4f36ffd0 100644 --- a/backend/mtproto/sync_test.go +++ b/backend/mtproto/sync_test.go @@ -6,10 +6,12 @@ import ( "errors" "fmt" "io" + "maps" "net/http" "net/http/httptest" "os" "path/filepath" + "regexp" "slices" "strconv" "strings" @@ -22,44 +24,43 @@ import ( const otherSecret = "ffeeddccbbaa00112233445566778899" -// fakeTelemtAPI stands in for telemt's control API. It keeps the runtime -// user set, records every call, and runs reloads through telemt's states: -// accepted, then preparing for reloadPending polls, then a terminal state. +var markerUsernamePattern = regexp.MustCompile(`^pgm_[0-9a-f]{16}$`) + +// fakeTelemtAPI stands in for telemt's control API. It keeps the users of +// the config file (disk) and of the running config (runtime), records every +// call, and plays telemt's config watcher: a changed config file reaches the +// running config when GET /v1/users is polled, after reloadDelayPolls polls, +// unless rejectReload is set. type fakeTelemtAPI struct { t *testing.T server *httptest.Server configPath string - mu sync.Mutex - users map[string]string // runtime users: username to secret - calls []string // "METHOD /path" per request - // reloadBodies holds the body of every reload request. - reloadBodies []string - - // reloadInProgress answers this many reload requests with telemt's 409. - reloadInProgress int - // reloadRejectStatus, when set, answers reload requests with this status - // and a bad_request error, like telemt for a config it cannot load. - reloadRejectStatus int - reloadPending int // status polls that report preparing first - reloadHang bool // the reload never finishes - reloadState string - reloadError string - nextReloadID int - pendingPolls map[int]int - // failPatch answers PATCH requests with a 500. - failPatch bool + mu sync.Mutex + disk map[string]string // username to secret + runtime map[string]string + calls []string // "METHOD /path" per request + + rejectReload bool + reloadDelayPolls int + appliedConfig string // the config file content runtime was loaded from + pendingConfig string // a changed config file waiting to be applied + pendingPolls int + // rejectedConfig is the last changed config file rejectReload kept out. + rejectedConfig string + + failPatch bool // PATCH answers 500 activeIPs map[string][]string } func newFakeTelemtAPI(t *testing.T) *fakeTelemtAPI { t.Helper() f := &fakeTelemtAPI{ - t: t, - configPath: filepath.Join(t.TempDir(), "config.toml"), - users: make(map[string]string), - pendingPolls: make(map[int]int), - activeIPs: make(map[string][]string), + t: t, + configPath: filepath.Join(t.TempDir(), "config.toml"), + disk: make(map[string]string), + runtime: make(map[string]string), + activeIPs: make(map[string][]string), } f.server = httptest.NewServer(http.HandlerFunc(f.serve)) t.Cleanup(f.server.Close) @@ -75,33 +76,82 @@ func (f *fakeTelemtAPI) takeCalls() []string { return calls } -func (f *fakeTelemtAPI) runtimeUsers() map[string]string { +// realRuntimeUsers returns the running users without marker users. +func (f *fakeTelemtAPI) realRuntimeUsers() map[string]string { f.mu.Lock() defer f.mu.Unlock() - users := make(map[string]string, len(f.users)) - for name, secret := range f.users { - users[name] = secret + users := make(map[string]string) + for name, secret := range f.runtime { + if !strings.HasPrefix(name, markerUsernamePrefix) { + users[name] = secret + } } return users } +// runtimeMarkers lists the marker users in the running config. +func (f *fakeTelemtAPI) runtimeMarkers() []string { + f.mu.Lock() + defer f.mu.Unlock() + var markers []string + for name := range f.runtime { + if strings.HasPrefix(name, markerUsernamePrefix) { + markers = append(markers, name) + } + } + return markers +} + func (f *fakeTelemtAPI) set(apply func(f *fakeTelemtAPI)) { f.mu.Lock() defer f.mu.Unlock() apply(f) } -// loadUsersLocked reads [access.users] from the rendered config, as a -// telemt (re)start or a successful reload does. -func (f *fakeTelemtAPI) loadUsersLocked() { +// startLocked loads the config file into disk and runtime, as a telemt start. +func (f *fakeTelemtAPI) startLocked() { data, err := os.ReadFile(f.configPath) if err != nil { f.t.Errorf("fake telemt: reading the config: %v", err) return } + f.disk = parseConfigUsers(string(data)) + f.runtime = maps.Clone(f.disk) + f.appliedConfig = string(data) +} + +// watchConfigLocked plays telemt's config watcher. +func (f *fakeTelemtAPI) watchConfigLocked() { + data, err := os.ReadFile(f.configPath) + if err != nil { + return // no config file: the test set the users directly + } + content := string(data) + f.disk = parseConfigUsers(content) + if content == f.appliedConfig { + return + } + if f.rejectReload { + f.rejectedConfig = content + return + } + if content != f.pendingConfig { + f.pendingConfig = content + f.pendingPolls = f.reloadDelayPolls + } + if f.pendingPolls > 0 { + f.pendingPolls-- + return + } + f.runtime = maps.Clone(f.disk) + f.appliedConfig = content +} + +// parseConfigUsers reads [access.users] from a rendered config. +func parseConfigUsers(config string) map[string]string { users := make(map[string]string) section := "" - for _, line := range strings.Split(string(data), "\n") { + for _, line := range strings.Split(config, "\n") { line = strings.TrimSpace(line) if strings.HasPrefix(line, "[") { section = strings.Trim(line, "[]") @@ -113,12 +163,22 @@ func (f *fakeTelemtAPI) loadUsersLocked() { } secret, err := strconv.Unquote(strings.TrimSpace(value)) if err != nil { - f.t.Errorf("fake telemt: bad user line %q", line) continue } users[strings.Trim(strings.TrimSpace(key), `"`)] = secret } - f.users = users + return users +} + +// configMarkers lists the marker users of a rendered config. +func configMarkers(config string) []string { + var markers []string + for name := range parseConfigUsers(config) { + if strings.HasPrefix(name, markerUsernamePrefix) { + markers = append(markers, name) + } + } + return markers } func (f *fakeTelemtAPI) serve(w http.ResponseWriter, r *http.Request) { @@ -142,9 +202,13 @@ func (f *fakeTelemtAPI) serve(w http.ResponseWriter, r *http.Request) { writeFakeSuccess(w, http.StatusOK, data) case r.Method == http.MethodGet && path == "/v1/users": + // telemt lists the users of the config file and computes in_runtime + // from the running config. + f.watchConfigLocked() data := []map[string]any{} - for name := range f.users { - data = append(data, map[string]any{"username": name, "in_runtime": true}) + for name := range f.disk { + _, inRuntime := f.runtime[name] + data = append(data, map[string]any{"username": name, "in_runtime": inRuntime}) } writeFakeSuccess(w, http.StatusOK, data) @@ -154,16 +218,17 @@ func (f *fakeTelemtAPI) serve(w http.ResponseWriter, r *http.Request) { writeFakeError(w, http.StatusBadRequest, "bad_request", err.Error()) return } - if _, exists := f.users[request.Username]; exists { + if _, exists := f.runtime[request.Username]; exists { writeFakeError(w, http.StatusConflict, "user_exists", "User already exists") return } - f.users[request.Username] = request.Secret + f.runtime[request.Username] = request.Secret + f.disk[request.Username] = request.Secret writeFakeSuccess(w, http.StatusCreated, map[string]any{"user": map[string]any{"username": request.Username}}) case strings.HasPrefix(path, "/v1/users/"): name := strings.TrimPrefix(path, "/v1/users/") - if _, exists := f.users[name]; !exists { + if _, exists := f.runtime[name]; !exists { writeFakeError(w, http.StatusNotFound, "not_found", "User not found") return } @@ -179,15 +244,17 @@ func (f *fakeTelemtAPI) serve(w http.ResponseWriter, r *http.Request) { return } if request.Secret != "" { - f.users[name] = request.Secret + f.runtime[name] = request.Secret + f.disk[name] = request.Secret } writeFakeSuccess(w, http.StatusOK, map[string]any{"username": name}) case http.MethodDelete: - if len(f.users) == 1 { + if len(f.runtime) == 1 { writeFakeError(w, http.StatusConflict, "last_user_forbidden", "Cannot delete the last user") return } - delete(f.users, name) + delete(f.runtime, name) + delete(f.disk, name) writeFakeSuccess(w, http.StatusOK, name) default: // A GET of one user loads the whole config in telemt; the node @@ -195,44 +262,6 @@ func (f *fakeTelemtAPI) serve(w http.ResponseWriter, r *http.Request) { writeFakeError(w, http.StatusMethodNotAllowed, "method_not_allowed", "unexpected "+r.Method) } - case r.Method == http.MethodPost && path == "/v1/system/reload": - f.reloadBodies = append(f.reloadBodies, string(body)) - if f.reloadInProgress > 0 { - f.reloadInProgress-- - writeFakeError(w, http.StatusConflict, "reload_in_progress", "Reload 7 is already in progress") - return - } - if f.reloadRejectStatus != 0 { - writeFakeError(w, f.reloadRejectStatus, "bad_request", "invalid runtime config: bad listener") - return - } - f.nextReloadID++ - f.pendingPolls[f.nextReloadID] = f.reloadPending - writeFakeSuccess(w, http.StatusAccepted, map[string]any{ - "reload_id": f.nextReloadID, "target_generation": f.nextReloadID + 1, - "config_revision": "r", "state": "accepted", "mode": "instant", "failure_policy": "rollback", - }) - - case r.Method == http.MethodGet && strings.HasPrefix(path, "/v1/system/reload/"): - id, err := strconv.Atoi(strings.TrimPrefix(path, "/v1/system/reload/")) - if _, known := f.pendingPolls[id]; err != nil || !known { - writeFakeError(w, http.StatusNotFound, "reload_not_found", "Reload was not found") - return - } - status := map[string]any{"reload_id": id, "mode": "instant", "failure_policy": "rollback"} - switch { - case f.reloadHang || f.pendingPolls[id] > 0: - f.pendingPolls[id]-- - status["state"] = "preparing" - case f.reloadState == "" || f.reloadState == "succeeded": - f.loadUsersLocked() - status["state"] = "succeeded" - default: - status["state"] = f.reloadState - status["error"] = f.reloadError - } - writeFakeSuccess(w, http.StatusOK, status) - default: writeFakeError(w, http.StatusNotFound, "not_found", "Route not found") } @@ -257,8 +286,9 @@ func writeFakeError(w http.ResponseWriter, status int, code, message string) { } // newSyncTestBackend returns a backend whose telemt "runs" on api: the -// process is a stand-in that counts as started, and the fake's runtime users -// are loaded from the initial render, as a telemt start would. +// process is a stand-in that counts as started (signalling it does nothing, +// like telemt on Windows, where its config watcher picks the file up), and +// the fake is started on the initial render. func newSyncTestBackend(t *testing.T, api *fakeTelemtAPI, users []*common.User) *MTProto { t.Helper() @@ -266,22 +296,20 @@ func newSyncTestBackend(t *testing.T, api *fakeTelemtAPI, users []*common.User) if err != nil { t.Fatal(err) } - sentinel, err := newSentinelUser() - if err != nil { - t.Fatal(err) - } desired, _ := ActiveUsersFromCommon(users, mtConfig.InboundTag) lifetime, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) m := &MTProto{ - config: mtConfig, - process: &processManager{run: &telemtRun{pid: 1, waitDone: make(chan struct{})}}, + config: mtConfig, + process: &processManager{ + run: &telemtRun{pid: 1, waitDone: make(chan struct{})}, + startupTail: newLineRing(telemtStartupTailSize), + }, configPath: api.configPath, apiClient: newAPIClient(api.server.URL, "Bearer test"), metricsScraper: newMetricsScraper("http://127.0.0.1:1/metrics"), tracker: newTrafficTracker(), - sentinel: sentinel, apiPort: 20001, metricsPort: 20002, apiAuthHeader: "Bearer test", @@ -292,10 +320,18 @@ func newSyncTestBackend(t *testing.T, api *fakeTelemtAPI, users []*common.User) if err := m.writeRenderedConfigLocked(m.desired); err != nil { t.Fatal(err) } - api.set(func(f *fakeTelemtAPI) { f.loadUsersLocked() }) + api.set(func(f *fakeTelemtAPI) { f.startLocked() }) return m } +// lowerMarkerConfirmTimeout sets markerConfirmTimeout for one test. +func lowerMarkerConfirmTimeout(t *testing.T, timeout time.Duration) { + t.Helper() + previous := markerConfirmTimeout + markerConfirmTimeout = timeout + t.Cleanup(func() { markerConfirmTimeout = previous }) +} + func desiredNames(m *MTProto) []string { m.stateMu.RLock() defer m.stateMu.RUnlock() @@ -316,6 +352,15 @@ func desiredSecret(m *MTProto, name string) string { return "" } +func readConfig(t *testing.T, path string) string { + t.Helper() + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + return string(data) +} + func assertCalls(t *testing.T, got []string, want ...string) { t.Helper() if !slices.Equal(got, want) { @@ -349,7 +394,7 @@ func TestPartialSkipsUnchangedUsers(t *testing.T) { if got := desiredSecret(m, "u2"); got != otherSecret { t.Errorf("desired u2 secret = %q, want %q", got, otherSecret) } - if got := api.runtimeUsers()["u2"]; got != otherSecret { + if got := api.realRuntimeUsers()["u2"]; got != otherSecret { t.Errorf("telemt u2 secret = %q, want %q", got, otherSecret) } @@ -377,29 +422,29 @@ func TestPartialCreatesAndDeletesWithoutGet(t *testing.T) { } t.Run("patch of a user telemt lacks creates it", func(t *testing.T) { - api.set(func(f *fakeTelemtAPI) { delete(f.users, "u2") }) + api.set(func(f *fakeTelemtAPI) { delete(f.runtime, "u2") }) if err := m.SyncUser(ctx, mtUser("u2", "mtproto", otherSecret)); err != nil { t.Fatalf("SyncUser: %v", err) } assertCalls(t, api.takeCalls(), "PATCH /v1/users/u2", "POST /v1/users") - if got := api.runtimeUsers()["u2"]; got != otherSecret { + if got := api.realRuntimeUsers()["u2"]; got != otherSecret { t.Errorf("telemt u2 secret = %q, want %q", got, otherSecret) } }) t.Run("create of a user telemt has patches it", func(t *testing.T) { - api.set(func(f *fakeTelemtAPI) { f.users["u4"] = testSecret }) + api.set(func(f *fakeTelemtAPI) { f.runtime["u4"] = testSecret }) if err := m.SyncUser(ctx, mtUser("u4", "mtproto", otherSecret)); err != nil { t.Fatalf("SyncUser: %v", err) } assertCalls(t, api.takeCalls(), "POST /v1/users", "PATCH /v1/users/u4") - if got := api.runtimeUsers()["u4"]; got != otherSecret { + if got := api.realRuntimeUsers()["u4"]; got != otherSecret { t.Errorf("telemt u4 secret = %q, want %q", got, otherSecret) } }) t.Run("delete of a user telemt lacks succeeds", func(t *testing.T) { - api.set(func(f *fakeTelemtAPI) { delete(f.users, "u3") }) + api.set(func(f *fakeTelemtAPI) { delete(f.runtime, "u3") }) if err := m.SyncUser(ctx, untagged("u3")); err != nil { t.Fatalf("SyncUser: %v", err) } @@ -426,7 +471,8 @@ func TestLargeBatchUsesFullRenderAndReload(t *testing.T) { t.Fatalf("UpdateUsers: %v", err) } - assertCalls(t, api.takeCalls(), "POST /v1/system/reload", "GET /v1/system/reload/1") + // No per-user call: one render, then one poll that finds the marker. + assertCalls(t, api.takeCalls(), "GET /v1/users") after, err := os.Stat(api.configPath) if err != nil { t.Fatal(err) @@ -434,7 +480,7 @@ func TestLargeBatchUsesFullRenderAndReload(t *testing.T) { if os.SameFile(before, after) { t.Error("the config was not rendered again before the reload") } - runtime := api.runtimeUsers() + runtime := api.realRuntimeUsers() if len(runtime) != partialApplyMaxUsers+2 || runtime["u33"] != numberedSecret(33) || runtime["u0"] != testSecret { t.Errorf("telemt users after the reload = %v, want u0 to u33", runtime) } @@ -443,82 +489,101 @@ func TestLargeBatchUsesFullRenderAndReload(t *testing.T) { } } -func TestFullSyncReportsRejectedReload(t *testing.T) { - cases := map[string]func(f *fakeTelemtAPI) string{ - "rolled back": func(f *fakeTelemtAPI) string { - f.reloadState, f.reloadError = "rolled_back", "bad config" - return "bad config" - }, - "failed": func(f *fakeTelemtAPI) string { - f.reloadState, f.reloadError = "failed", "listener bind failed" - return "listener bind failed" - }, - "refused": func(f *fakeTelemtAPI) string { - f.reloadRejectStatus = http.StatusBadRequest - return "bad listener" - }, - } - for name, setup := range cases { - t.Run(name, func(t *testing.T) { - api := newFakeTelemtAPI(t) - m := newSyncTestBackend(t, api, []*common.User{mtUser("u1", "mtproto", testSecret)}) - var want string - api.set(func(f *fakeTelemtAPI) { want = setup(f) }) +// TestMarkerUserPerRender: every render carries exactly one disabled marker +// user with a fresh name, the backend reports the current one, and the +// marker never becomes a desired user. +func TestMarkerUserPerRender(t *testing.T) { + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{mtUser("u1", "mtproto", testSecret)}) - err := m.SyncUsers(context.Background(), []*common.User{mtUser("u2", "mtproto", otherSecret)}) - if err == nil || !strings.Contains(err.Error(), want) { - t.Fatalf("SyncUsers = %v, want an error containing %q", err, want) - } - if got := desiredNames(m); !slices.Equal(got, []string{"u1"}) { - t.Errorf("desired = %q after a rejected reload, want u1", got) - } - if !m.process.Started() { - t.Error("a rejected reload stopped telemt") - } - // telemt watches the file: it must hold the confirmed users again. - data, err := os.ReadFile(api.configPath) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(string(data), "u1 = ") || strings.Contains(string(data), "u2 = ") { - t.Errorf("config after a rejected reload is not the previous user set:\n%s", data) - } - }) + seen := map[string]bool{} + for _, users := range [][]*common.User{ + {mtUser("u2", "mtproto", testSecret)}, + {mtUser("u3", "mtproto", testSecret)}, + } { + if err := m.SyncUsers(context.Background(), users); err != nil { + t.Fatalf("SyncUsers: %v", err) + } + config := readConfig(t, api.configPath) + markers := configMarkers(config) + if len(markers) != 1 || !markerUsernamePattern.MatchString(markers[0]) { + t.Fatalf("config markers = %q, want one pgm_<16 hex>", markers) + } + marker := markers[0] + if seen[marker] { + t.Errorf("marker %s was reused", marker) + } + seen[marker] = true + if got := m.markerUsername(); got != marker { + t.Errorf("markerUsername() = %q, want %q", got, marker) + } + if !strings.Contains(config, "[access.user_enabled]\n"+marker+" = false") { + t.Errorf("marker %s is not rendered disabled:\n%s", marker, config) + } + if slices.Contains(desiredNames(m), marker) { + t.Errorf("marker %s is in desired", marker) + } + if got := api.runtimeMarkers(); !slices.Equal(got, []string{marker}) { + t.Errorf("telemt markers = %q, want %q", got, marker) + } } } -func TestFullSyncRetriesReloadInProgress(t *testing.T) { +// TestHotReloadWaitsForMarker: telemt applies a config some polls after the +// write (its watcher or an earlier reload is still busy); the sync waits for +// the marker instead of failing. +func TestHotReloadWaitsForMarker(t *testing.T) { api := newFakeTelemtAPI(t) m := newSyncTestBackend(t, api, []*common.User{mtUser("u1", "mtproto", testSecret)}) - api.set(func(f *fakeTelemtAPI) { - f.reloadInProgress = 1 - f.reloadPending = 2 - }) + api.set(func(f *fakeTelemtAPI) { f.reloadDelayPolls = 3 }) if err := m.SyncUsers(context.Background(), []*common.User{mtUser("u2", "mtproto", testSecret)}); err != nil { t.Fatalf("SyncUsers: %v", err) } - assertCalls(t, api.takeCalls(), - "POST /v1/system/reload", "POST /v1/system/reload", - "GET /v1/system/reload/1", "GET /v1/system/reload/1", "GET /v1/system/reload/1") - api.set(func(f *fakeTelemtAPI) { - for _, body := range f.reloadBodies { - if body != `{"mode":"instant","failure_policy":"rollback"}` { - t.Errorf("reload body = %s", body) - } - } - }) + assertCalls(t, api.takeCalls(), "GET /v1/users", "GET /v1/users", "GET /v1/users", "GET /v1/users") if got := desiredNames(m); !slices.Equal(got, []string{"u2"}) { t.Errorf("desired = %q, want u2", got) } - if runtime := api.runtimeUsers(); len(runtime) != 1 || runtime["u2"] == "" { + if runtime := api.realRuntimeUsers(); len(runtime) != 1 || runtime["u2"] == "" { t.Errorf("telemt users = %v, want u2", runtime) } } -// TestFullSyncWithoutChangesSkipsReload: a reload cuts every telemt session, -// so a full sync of the confirmed users sends nothing; once the state is -// unconfirmed, the same sync reloads. +// TestRejectedHotReloadRestoresPreviousUsers: telemt never loads the new +// config (it rejected it), so the marker never reaches its running config. +// The sync fails with telemt's last output, desired is unchanged, and the +// previous users are rendered again with a new marker. +func TestRejectedHotReloadRestoresPreviousUsers(t *testing.T) { + lowerMarkerConfirmTimeout(t, 600*time.Millisecond) + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{mtUser("u1", "mtproto", testSecret)}) + m.process.startupTail.add("config reload rejected: bad listener") + api.set(func(f *fakeTelemtAPI) { f.rejectReload = true }) + + err := m.SyncUsers(context.Background(), []*common.User{mtUser("u2", "mtproto", otherSecret)}) + if err == nil || !strings.Contains(err.Error(), "bad listener") { + t.Fatalf("SyncUsers = %v, want an error with telemt's last output", err) + } + if got := desiredNames(m); !slices.Equal(got, []string{"u1"}) { + t.Errorf("desired = %q after a rejected reload, want u1", got) + } + if !m.process.Started() { + t.Error("a rejected reload stopped telemt") + } + + config := readConfig(t, api.configPath) + users := parseConfigUsers(config) + if users["u1"] != testSecret || users["u2"] != "" { + t.Errorf("config after a rejected reload is not the previous user set:\n%s", config) + } + var rejected string + api.set(func(f *fakeTelemtAPI) { rejected = f.rejectedConfig }) + rejectedMarkers, restoredMarkers := configMarkers(rejected), configMarkers(config) + if len(rejectedMarkers) != 1 || len(restoredMarkers) != 1 || rejectedMarkers[0] == restoredMarkers[0] { + t.Errorf("markers: rejected %q, restored %q; want one each and a new one for the restore", rejectedMarkers, restoredMarkers) + } +} + func TestFullSyncWithoutChangesSkipsReload(t *testing.T) { api := newFakeTelemtAPI(t) users := []*common.User{mtUser("u1", "mtproto", testSecret), mtUser("u2", "mtproto", testSecret)} @@ -535,10 +600,13 @@ func TestFullSyncWithoutChangesSkipsReload(t *testing.T) { if err := m.SyncUsers(context.Background(), users); err != nil { t.Fatalf("SyncUsers (unconfirmed): %v", err) } - assertCalls(t, api.takeCalls(), "POST /v1/system/reload", "GET /v1/system/reload/1") + assertCalls(t, api.takeCalls(), "GET /v1/users") } -func TestZeroUsersUsesSentinelPath(t *testing.T) { +// TestLastRealUserGoesThroughAPI: the marker keeps telemt's user list +// non-empty, so the last real user is deleted, and the first one created, +// through the user API. +func TestLastRealUserGoesThroughAPI(t *testing.T) { api := newFakeTelemtAPI(t) m := newSyncTestBackend(t, api, []*common.User{mtUser("u1", "mtproto", testSecret)}) ctx := context.Background() @@ -546,20 +614,32 @@ func TestZeroUsersUsesSentinelPath(t *testing.T) { if err := m.UpdateUsers(ctx, []*common.User{untagged("u1")}); err != nil { t.Fatalf("UpdateUsers to zero users: %v", err) } - assertCalls(t, api.takeCalls(), "POST /v1/system/reload", "GET /v1/system/reload/1") + assertCalls(t, api.takeCalls(), "DELETE /v1/users/u1") if got := desiredNames(m); len(got) != 0 { t.Errorf("desired = %q, want none", got) } - if runtime := api.runtimeUsers(); len(runtime) != 1 || runtime[m.sentinel.Username] == "" { - t.Errorf("telemt users = %v, want only the sentinel", runtime) + if runtime := api.realRuntimeUsers(); len(runtime) != 0 { + t.Errorf("telemt users = %v, want only the marker", runtime) } if err := m.SyncUser(ctx, mtUser("u2", "mtproto", testSecret)); err != nil { t.Fatalf("SyncUser from zero users: %v", err) } - assertCalls(t, api.takeCalls(), "POST /v1/system/reload", "GET /v1/system/reload/2") - if runtime := api.runtimeUsers(); len(runtime) != 1 || runtime["u2"] == "" { - t.Errorf("telemt users = %v, want only u2", runtime) + assertCalls(t, api.takeCalls(), "POST /v1/users") + if runtime := api.realRuntimeUsers(); len(runtime) != 1 || runtime["u2"] == "" { + t.Errorf("telemt users = %v, want u2", runtime) + } + + // A full sync to zero users renders the marker alone. + if err := m.SyncUsers(ctx, nil); err != nil { + t.Fatalf("SyncUsers to zero users: %v", err) + } + assertCalls(t, api.takeCalls(), "GET /v1/users") + if runtime := api.realRuntimeUsers(); len(runtime) != 0 { + t.Errorf("telemt users = %v, want only the marker", runtime) + } + if markers := api.runtimeMarkers(); len(markers) != 1 { + t.Errorf("telemt markers = %q, want one", markers) } } @@ -603,16 +683,16 @@ func TestOnlineIPListUsesBulkEndpoint(t *testing.T) { } } -// TestUnconfirmedReloadForcesFullApply: the caller gives up while telemt is -// still reloading. telemt keeps running, desired keeps the confirmed users, -// and the next sync renders and reloads everything instead of a diff. +// TestUnconfirmedReloadForcesFullApply: the caller gives up before telemt +// loaded the config. telemt keeps running, desired keeps the confirmed +// users, and the next sync renders and reloads everything instead of a diff. func TestUnconfirmedReloadForcesFullApply(t *testing.T) { api := newFakeTelemtAPI(t) m := newSyncTestBackend(t, api, []*common.User{ mtUser("u1", "mtproto", testSecret), mtUser("u2", "mtproto", testSecret), }) - api.set(func(f *fakeTelemtAPI) { f.reloadHang = true }) + api.set(func(f *fakeTelemtAPI) { f.rejectReload = true }) ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) defer cancel() @@ -628,14 +708,17 @@ func TestUnconfirmedReloadForcesFullApply(t *testing.T) { } api.takeCalls() - api.set(func(f *fakeTelemtAPI) { f.reloadHang = false }) + api.set(func(f *fakeTelemtAPI) { f.rejectReload = false }) if err := m.UpdateUsers(context.Background(), []*common.User{mtUser("u2", "mtproto", otherSecret)}); err != nil { t.Fatalf("UpdateUsers: %v", err) } - assertCalls(t, api.takeCalls(), "POST /v1/system/reload", "GET /v1/system/reload/2") + assertCalls(t, api.takeCalls(), "GET /v1/users") if got := desiredNames(m); !slices.Equal(got, []string{"u1", "u2"}) { t.Errorf("desired = %q, want u1, u2", got) } + if runtime := api.realRuntimeUsers(); len(runtime) != 2 || runtime["u2"] != otherSecret { + t.Errorf("telemt users = %v, want u1 and the new u2", runtime) + } // Confirmed again: the next small change is a diff. if err := m.UpdateUsers(context.Background(), []*common.User{mtUser("u1", "mtproto", otherSecret)}); err != nil { @@ -655,8 +738,8 @@ func TestPartialFailureFallsBackToFullApply(t *testing.T) { if err := m.UpdateUsers(context.Background(), []*common.User{mtUser("u2", "mtproto", otherSecret)}); err != nil { t.Fatalf("UpdateUsers: %v", err) } - assertCalls(t, api.takeCalls(), "PATCH /v1/users/u2", "POST /v1/system/reload", "GET /v1/system/reload/1") - if got := api.runtimeUsers()["u2"]; got != otherSecret { + assertCalls(t, api.takeCalls(), "PATCH /v1/users/u2", "GET /v1/users") + if got := api.realRuntimeUsers()["u2"]; got != otherSecret { t.Errorf("telemt u2 secret = %q, want %q", got, otherSecret) } if got := desiredSecret(m, "u2"); got != otherSecret { @@ -690,7 +773,8 @@ func TestRuntimeUserEqual(t *testing.T) { func TestAPIClientListUsersAndActiveIPs(t *testing.T) { api := newFakeTelemtAPI(t) api.set(func(f *fakeTelemtAPI) { - f.users = map[string]string{"u1": testSecret} + f.disk = map[string]string{"u1": testSecret, "u2": testSecret} + f.runtime = map[string]string{"u1": testSecret} f.activeIPs = map[string][]string{"u1": {"203.0.113.7"}} }) client := newAPIClient(api.server.URL, "Bearer test") @@ -699,8 +783,12 @@ func TestAPIClientListUsersAndActiveIPs(t *testing.T) { if err != nil { t.Fatalf("ListUsers: %v", err) } - if len(users) != 1 || users[0].Username != "u1" || !users[0].InRuntime { - t.Errorf("ListUsers = %+v", users) + inRuntime := map[string]bool{} + for _, user := range users { + inRuntime[user.Username] = user.InRuntime + } + if len(users) != 2 || !inRuntime["u1"] || inRuntime["u2"] { + t.Errorf("ListUsers = %+v, want u1 in runtime and u2 only on disk", users) } ips, err := client.ActiveIPs(context.Background()) From 0afe9517e236729b1c9d089d5e43cfc65789110a Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 19 Oct 2026 21:36:28 +0330 Subject: [PATCH 09/40] fix(mtproto): settle hot reloads independently of callers and clear stale overrides telemt stages an admission override for every user API write and clears it only when a published config carries the same value. A full render that changed such a user again before telemt published the API's own write left the user refused while the node reported success. The node now records users written through the API since the last confirmed render or telemt start, and before a render writes again each one whose rendered state differs, so the render's publish clears the override. A write that fails stays unknown and keeps the state unconfirmed. A full apply now runs to telemt's verdict in the background under syncMu, which a context-aware lock lets callers wait for only until their own deadline. A caller that gives up gets its ctx error at once; the apply still confirms, or restores the previous users, and logs the outcome. A rejection goes to telemt's log stream, which the panel shows. A 5xx "failed to load config" from GET /v1/users is an immediate rejection with telemt's message. telemt is restarted only after 3 polls in a row get no answer or when it exited, not on one failed request. Reload code moves to reload.go and the per-user API writes to userapi.go. --- backend/mtproto/lifecycle.go | 3 + backend/mtproto/lifecycle_test.go | 7 +- backend/mtproto/mtproto.go | 12 +- backend/mtproto/process.go | 7 + backend/mtproto/reload.go | 147 ++++++++ backend/mtproto/sync.go | 302 +++++++---------- backend/mtproto/sync_test.go | 536 ++++++++++++++++++++++++------ backend/mtproto/synclock.go | 44 +++ backend/mtproto/userapi.go | 129 +++++++ 9 files changed, 894 insertions(+), 293 deletions(-) create mode 100644 backend/mtproto/reload.go create mode 100644 backend/mtproto/synclock.go create mode 100644 backend/mtproto/userapi.go diff --git a/backend/mtproto/lifecycle.go b/backend/mtproto/lifecycle.go index 1a3410fc..af51a48f 100644 --- a/backend/mtproto/lifecycle.go +++ b/backend/mtproto/lifecycle.go @@ -200,6 +200,9 @@ func (m *MTProto) startLocked(ctx context.Context) error { return m.startupError(err) } + // A new telemt has fresh user admission state: no API write is pending. + m.apiSent = nil + exited := m.process.Exited() m.background.Go(func() { m.watchReadiness(exited) }) return nil diff --git a/backend/mtproto/lifecycle_test.go b/backend/mtproto/lifecycle_test.go index df62e63e..6a1af729 100644 --- a/backend/mtproto/lifecycle_test.go +++ b/backend/mtproto/lifecycle_test.go @@ -390,9 +390,10 @@ func TestFullSyncSignalsTelemt(t *testing.T) { } } -// TestUsersAPIUnreachableRestartsTelemt: when telemt's API gets no answer -// while the node waits for a reload, the node restarts telemt on the new -// config, and that start confirms it. +// TestUsersAPIUnreachableRestartsTelemt: when telemt's API keeps getting no +// answer (markerUnreachablePolls polls in a row) while the node waits for a +// reload, the node restarts telemt on the new config, and that start +// confirms it. func TestUsersAPIUnreachableRestartsTelemt(t *testing.T) { exe := writeFakeTelemt(t, fakeTelemt{DropUsers: true}) mt := newTestBackend(t, testNodeConfig(t, exe), []*common.User{mtUser("alice", "mtproto", testSecret)}) diff --git a/backend/mtproto/mtproto.go b/backend/mtproto/mtproto.go index 8ea03a58..7497a3c3 100644 --- a/backend/mtproto/mtproto.go +++ b/backend/mtproto/mtproto.go @@ -31,13 +31,17 @@ type MTProto struct { apiAuthHeader string // syncMu serializes every operation that changes users or the telemt - // process: syncs, restarts, the supervisor and Shutdown. - syncMu sync.Mutex + // process: syncs, restarts, the supervisor and Shutdown. A full apply + // holds it until telemt's verdict, even after its caller gave up. + syncMu syncLock closed bool // guarded by syncMu // needsFullApply is set when telemt may run users other than desired: a // reload whose outcome is unknown, or user API calls that failed midway. // The next sync then renders and reloads every user. Guarded by syncMu. needsFullApply bool + // apiSent holds the users written through telemt's user API since the + // last confirmed render or telemt start. Guarded by syncMu. + apiSent map[string]apiWrite // stateMu guards desired and marker for readers; writers also hold // syncMu. desired holds only users telemt confirmed it runs. marker is @@ -256,7 +260,9 @@ func (m *MTProto) markerUsername() string { // newMarkerUser returns a marker user with a fresh random name and secret. // Every render carries one, disabled: once telemt's running config has it, -// telemt runs that render. It also keeps telemt's user list non-empty. +// telemt has published that render (its user admission follows from the +// published config shortly after). It also keeps telemt's user list +// non-empty. func newMarkerUser() (*runtimeUser, error) { secret := make([]byte, 16) usernameSuffix := make([]byte, 8) diff --git a/backend/mtproto/process.go b/backend/mtproto/process.go index 22dd9806..80715f58 100644 --- a/backend/mtproto/process.go +++ b/backend/mtproto/process.go @@ -327,6 +327,13 @@ func (p *processManager) Reload() error { return nil } +// Notice logs a node message about telemt and adds it to telemt's log +// stream, which the panel shows. +func (p *processManager) Notice(message string) { + log.Println(message) + p.logs.Publish(message) +} + // Stop terminates telemt and waits for it to exit, killing it after // telemtStopTimeout. func (p *processManager) Stop() { diff --git a/backend/mtproto/reload.go b/backend/mtproto/reload.go new file mode 100644 index 00000000..1097aa23 --- /dev/null +++ b/backend/mtproto/reload.go @@ -0,0 +1,147 @@ +package mtproto + +import ( + "context" + "errors" + "fmt" + "log" + "net/http" + "strings" + "time" +) + +const ( + // markerPollInterval is how often the node checks whether telemt loaded + // a rendered config. + markerPollInterval = 200 * time.Millisecond + // markerUnreachablePolls is how many polls in a row must get no answer + // before telemt counts as unreachable and is restarted, unless it exited. + markerUnreachablePolls = 3 +) + +// markerConfirmTimeout bounds the wait for telemt to load a rendered config; +// a config it did not load by then counts as rejected. It is a variable so +// tests can lower it. +var markerConfirmTimeout = 30 * time.Second + +// errReloadRejected marks a rendered config telemt did not load. telemt +// keeps running the config it had. +var errReloadRejected = errors.New("telemt did not load the new config") + +// reloadLocked makes the running telemt hot-reload the rendered config: +// SIGHUP (telemt's config watcher also sees the file change), then a wait +// until the render's marker user is in telemt's running config. A hot reload +// replaces only the users that changed; an API reload would cut every +// session. Only an API that stays unreachable, or a telemt that exited or +// cannot be signalled, restarts telemt, on the rendered config; a rejected +// config never does. Requires syncMu. +func (m *MTProto) reloadLocked(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + // telemt installs its SIGHUP handler in its config watcher, which starts + // before the API the node waited for at startup, so a healthy telemt + // handles the signal. Should one not, SIGHUP ends it and the restart + // below recovers it. + err := m.process.Reload() + if err == nil { + err = m.waitForMarker(ctx, m.marker.Username) + } + switch { + case err == nil || errors.Is(err, errClosed) || errors.Is(err, errReloadRejected): + return err + case ctx.Err() != nil: + return fmt.Errorf("telemt reload not confirmed: %w", ctx.Err()) + } + + log.Printf("telemt did not answer a reload, restarting it: %v", err) + m.process.Stop() + // telemt is down now: start it again even if ctx ends. + if restartErr := m.startLocked(context.WithoutCancel(ctx)); restartErr != nil { + return fmt.Errorf("telemt reload failed: %w (restart failed: %v)", err, restartErr) + } + return nil +} + +// waitForMarker polls GET /v1/users until telemt's running config has the +// marker user, for up to markerConfirmTimeout. It returns an errReloadRejected +// error when telemt cannot read the rendered file or the time is up, +// errClosed on Shutdown, ctx's error when ctx ends, and the request error +// when the API stays unreachable or telemt exited. +func (m *MTProto) waitForMarker(ctx context.Context, marker string) error { + waitCtx, cancel := context.WithTimeout(ctx, markerConfirmTimeout) + defer cancel() + stop := context.AfterFunc(m.lifetime, cancel) + defer stop() + + exited := m.process.Exited() + ticker := time.NewTicker(markerPollInterval) + defer ticker.Stop() + + unanswered := 0 + for { + users, err := m.apiClient.ListUsers(waitCtx) + var apiErr *apiStatusError + switch { + case err == nil: + unanswered = 0 + if userInRuntime(users, marker) { + return nil + } + case errors.As(err, &apiErr): + unanswered = 0 + // telemt reads the config file for this list: it cannot parse + // the render. + if apiErr.StatusCode >= http.StatusInternalServerError && strings.Contains(apiErr.Message, "failed to load config") { + return fmt.Errorf("%w: %s", errReloadRejected, apiErr.Message) + } + case waitCtx.Err() == nil: + unanswered++ + if unanswered >= markerUnreachablePolls || isClosed(exited) { + return fmt.Errorf("telemt API unreachable: %w", err) + } + } + + select { + case <-waitCtx.Done(): + switch { + case m.lifetime.Err() != nil: + return errClosed + case ctx.Err() != nil: + return ctx.Err() + default: + return m.reloadRejectedError() + } + case <-ticker.C: + } + } +} + +// reloadRejectedError reports a config telemt did not load in time, with +// its last output, which carries the reason. +func (m *MTProto) reloadRejectedError() error { + tail := m.process.StartupTail(telemtCrashTailSize) + if len(tail) == 0 { + return fmt.Errorf("%w within %s", errReloadRejected, markerConfirmTimeout) + } + return fmt.Errorf("%w within %s; recent telemt output:\n%s", errReloadRejected, markerConfirmTimeout, strings.Join(tail, "\n")) +} + +func userInRuntime(users []telemtUserInfo, username string) bool { + for _, user := range users { + if user.Username == username { + return user.InRuntime + } + } + return false +} + +// isClosed reports whether ch is closed; a nil ch never is. +func isClosed(ch <-chan struct{}) bool { + select { + case <-ch: + return true + default: + return false + } +} diff --git a/backend/mtproto/sync.go b/backend/mtproto/sync.go index a7a1ec54..91f09879 100644 --- a/backend/mtproto/sync.go +++ b/backend/mtproto/sync.go @@ -7,7 +7,6 @@ import ( "log" "maps" "strings" - "time" "github.com/pasarguard/node/common" ) @@ -18,26 +17,6 @@ import ( // reloads it instead. const partialApplyMaxUsers = 32 -// markerPollInterval is how often the node checks whether telemt loaded a -// rendered config. -const markerPollInterval = 200 * time.Millisecond - -// markerConfirmTimeout bounds the wait for telemt to load a rendered config; -// a config it did not load by then counts as rejected. It is a variable so -// tests can lower it. -var markerConfirmTimeout = 30 * time.Second - -// errReloadRejected marks a rendered config telemt did not load. telemt -// keeps running the config it had. -var errReloadRejected = errors.New("telemt did not load the new config") - -// userChange is one user to create, patch or delete in telemt. -type userChange struct { - username string - user *runtimeUser // the new state; nil deletes the user - known bool // the user is in m.desired -} - func (m *MTProto) SyncUser(ctx context.Context, user *common.User) error { // The batch paths skip a user that fails validation; a single-user sync // reports it to the caller instead. @@ -49,19 +28,17 @@ func (m *MTProto) SyncUser(ctx context.Context, user *common.User) error { // SyncUsers replaces all users through one config render and reload. func (m *MTProto) SyncUsers(ctx context.Context, users []*common.User) error { - m.syncMu.Lock() - defer m.syncMu.Unlock() - - if err := m.syncAllowedLocked(ctx); err != nil { + if err := m.lockSync(ctx); err != nil { return err } desired, _ := ActiveUsersFromCommon(users, m.config.InboundTag) // Nothing to apply: skip the render and the wait for telemt. if m.confirmedLocked() && maps.EqualFunc(m.desired, desired, runtimeUserEqual) { + m.syncMu.Unlock() return nil } - return m.applyFullLocked(ctx, desired) + return m.applyFullDetachedLocked(ctx, desired) } // UpdateUsers applies users on top of the current ones. Up to @@ -69,56 +46,75 @@ func (m *MTProto) SyncUsers(ctx context.Context, users []*common.User) error { // unconfirmed telemt state, go through a config render and reload. Users // that did not change are not sent at all. func (m *MTProto) UpdateUsers(ctx context.Context, users []*common.User) error { - m.syncMu.Lock() - defer m.syncMu.Unlock() - - if err := m.syncAllowedLocked(ctx); err != nil { + if err := m.lockSync(ctx); err != nil { return err } - next := m.buildNextDesiredLocked(users) - changes := m.userChangesLocked(users, next) - - if !m.confirmedLocked() || len(changes) > partialApplyMaxUsers { - return m.applyFullLocked(ctx, next) - } - if len(changes) == 0 { + next, full := m.updateUsersLocked(ctx, users) + if !full { + m.syncMu.Unlock() return nil } - - if err := m.applyUserChangesLocked(ctx, changes); err != nil { - // Some changes may have reached telemt: its users are unknown until - // a full apply succeeds. - m.needsFullApply = true - log.Printf("mtproto: applying %d user changes through the telemt API failed, rendering all users instead: %v", len(changes), err) - return m.applyFullLocked(ctx, next) - } - m.confirmLocked(next) - return nil + return m.applyFullDetachedLocked(ctx, next) } // UpdateUsersAndRestart applies users on top of the current ones through a // config render and reload. func (m *MTProto) UpdateUsersAndRestart(ctx context.Context, users []*common.User) error { - m.syncMu.Lock() - defer m.syncMu.Unlock() + if err := m.lockSync(ctx); err != nil { + return err + } + return m.applyFullDetachedLocked(ctx, m.buildNextDesiredLocked(users)) +} +// lockSync takes syncMu for a sync call. A caller waiting behind another +// apply gives up at its own deadline, and every waiter on Shutdown. A closed +// backend or a caller already gone is refused. +func (m *MTProto) lockSync(ctx context.Context) error { + if err := m.syncMu.LockContext(ctx, m.lifetime.Done()); err != nil { + return err + } if err := m.syncAllowedLocked(ctx); err != nil { + m.syncMu.Unlock() return err } - return m.applyFullLocked(ctx, m.buildNextDesiredLocked(users)) + return nil } // syncAllowedLocked refuses a sync after Shutdown, or one whose caller gave // up while it queued on syncMu. Requires syncMu. func (m *MTProto) syncAllowedLocked(ctx context.Context) error { - if m.closed { + if m.closed || m.lifetime.Err() != nil { return errClosed } return ctx.Err() } -// confirmedLocked reports whether telemt runs and runs exactly m.desired. -// Requires syncMu. +// updateUsersLocked applies users through telemt's user API when it can and +// reports whether next needs a full apply instead. Requires syncMu. +func (m *MTProto) updateUsersLocked(ctx context.Context, users []*common.User) (next map[string]*runtimeUser, full bool) { + next = m.buildNextDesiredLocked(users) + changes := m.userChangesLocked(users, next) + + if !m.confirmedLocked() || len(changes) > partialApplyMaxUsers { + return next, true + } + if len(changes) == 0 { + return next, false + } + if err := m.applyUserChangesLocked(ctx, changes); err != nil { + // Some changes may have reached telemt: its users are unknown until + // a full apply succeeds. + m.needsFullApply = true + log.Printf("mtproto: applying %d user changes through the telemt API failed, rendering all users instead: %v", len(changes), err) + return next, true + } + m.confirmLocked(next) + return next, false +} + +// confirmedLocked reports whether telemt runs and has published exactly +// m.desired, with no user API write whose outcome is unknown. Requires +// syncMu. func (m *MTProto) confirmedLocked() bool { return !m.needsFullApply && m.process.Started() } @@ -171,69 +167,82 @@ func (m *MTProto) userChangesLocked(users []*common.User, next map[string]*runti return changes } -// applyUserChangesLocked sends each change to telemt's user API. m.desired -// picks create or patch; when telemt disagrees, its answer corrects the -// choice: a patch of a missing user creates it, a create of an existing user -// patches it, and deleting a missing user is done. Requires syncMu. +// applyUserChangesLocked sends each change to telemt's user API, stopping at +// the first failure. Requires syncMu. func (m *MTProto) applyUserChangesLocked(ctx context.Context, changes []userChange) error { for _, change := range changes { - var err error - switch { - case change.user == nil: - err = m.deleteUser(ctx, change.username) - case change.known: - err = m.patchOrCreateUser(ctx, change.user) - default: - err = m.createOrPatchUser(ctx, change.user) - } - if err != nil { + if err := m.sendUserChangeLocked(ctx, change); err != nil { return fmt.Errorf("user %s: %w", change.username, err) } } return nil } -func (m *MTProto) patchOrCreateUser(ctx context.Context, user *runtimeUser) error { - err := m.apiClient.PatchUser(ctx, user.Username, user) - var apiErr *apiStatusError - if errors.As(err, &apiErr) && apiErr.NotFound() { - return m.apiClient.CreateUser(ctx, user) +// applyFullDetachedLocked runs applyFullLocked in the background and waits +// for it or for ctx. It takes over syncMu from the caller and releases it +// once the apply settles, so the render, telemt's verdict and the restore of +// a rejected config all happen even when the caller gives up, which a panel +// deadline (10 to 15 s) does before markerConfirmTimeout. That caller gets +// ctx's error at once, and the outcome is logged instead. +func (m *MTProto) applyFullDetachedLocked(ctx context.Context, desired map[string]*runtimeUser) error { + // Do not start a render for a caller already gone. + if err := ctx.Err(); err != nil { + m.syncMu.Unlock() + return err } - return err -} -func (m *MTProto) createOrPatchUser(ctx context.Context, user *runtimeUser) error { - err := m.apiClient.CreateUser(ctx, user) - var apiErr *apiStatusError - if errors.As(err, &apiErr) && apiErr.UserExists() { - return m.apiClient.PatchUser(ctx, user.Username, user) + applyCtx, cancel := context.WithCancel(context.WithoutCancel(ctx)) + stopOnShutdown := context.AfterFunc(m.lifetime, cancel) + result := make(chan error) + callerGone := make(chan struct{}) + // Shutdown waits for this goroutine: it is added under syncMu before + // closed is set. + m.background.Go(func() { + defer m.syncMu.Unlock() + defer cancel() + defer stopOnShutdown() + + err := m.applyFullLocked(applyCtx, desired) + select { + case result <- err: + case <-callerGone: + logSettledApply(err) + } + }) + + select { + case err := <-result: + return err + case <-ctx.Done(): + close(callerGone) + return fmt.Errorf("the users are still being applied: %w", ctx.Err()) } - return err } -func (m *MTProto) deleteUser(ctx context.Context, username string) error { - err := m.apiClient.DeleteUser(ctx, username) - var apiErr *apiStatusError - if errors.As(err, &apiErr) && apiErr.NotFound() { - return nil +// logSettledApply logs how a full apply ended after its caller stopped +// waiting. A rejection is already in telemt's log stream. +func logSettledApply(err error) { + switch { + case err == nil: + log.Println("mtproto: telemt loaded the new users after the caller stopped waiting") + case errors.Is(err, errClosed), errors.Is(err, errReloadRejected): + default: + log.Printf("mtproto: applying the users failed after the caller stopped waiting: %v", err) } - return err } // applyFullLocked renders desired and makes telemt run it: a hot reload of // the running telemt, or a start when none runs. m.desired takes desired only -// once telemt confirmed it. Requires syncMu. +// once telemt published it. Requires syncMu. func (m *MTProto) applyFullLocked(ctx context.Context, desired map[string]*runtimeUser) error { - // A partial apply that failed on an expired ctx falls back here; do not - // rewrite the config for a caller that is gone. if err := ctx.Err(); err != nil { return err } - if err := m.writeRenderedConfigLocked(desired); err != nil { - return err - } if !m.process.Started() { + if err := m.writeRenderedConfigLocked(desired); err != nil { + return err + } if err := m.startLocked(ctx); err != nil { return err } @@ -241,20 +250,23 @@ func (m *MTProto) applyFullLocked(ctx context.Context, desired map[string]*runti return nil } + m.matchAPIWritesLocked(ctx, desired) + if err := m.writeRenderedConfigLocked(desired); err != nil { + return err + } + err := m.reloadLocked(ctx) switch { case err == nil: m.confirmLocked(desired) + m.forgetSettledAPIWritesLocked() return nil case errors.Is(err, errReloadRejected): - // telemt still runs the users it had. Put them back on disk too: - // telemt watches the config file and would retry the rejected one. - if restoreErr := m.writeRenderedConfigLocked(m.desired); restoreErr != nil { + m.process.Notice("mtproto: " + strings.ReplaceAll(err.Error(), "\n", " | ") + "; rendering the previous users again") + if restoreErr := m.restoreLocked(ctx); restoreErr != nil { + m.needsFullApply = true return fmt.Errorf("%w (restoring the previous telemt config failed: %v)", err, restoreErr) } - if signalErr := m.process.Reload(); signalErr != nil { - log.Printf("warning: %v", signalErr) - } return err default: // The outcome is unknown, and telemt may still load the new config. @@ -263,95 +275,25 @@ func (m *MTProto) applyFullLocked(ctx context.Context, desired map[string]*runti } } -// reloadLocked makes the running telemt hot-reload the rendered config: -// SIGHUP (telemt's config watcher also sees the file change), then a wait -// until the render's marker user is in telemt's running config. A hot reload -// replaces only the users that changed; an API reload would cut every -// session. Only an API that cannot be reached, or a telemt that cannot be -// signalled, restarts telemt, on the rendered config; a caller that gave up -// or a rejected config never does. Requires syncMu. -func (m *MTProto) reloadLocked(ctx context.Context) error { - if err := ctx.Err(); err != nil { - return err - } - err := m.process.Reload() - if err == nil { - err = m.waitForMarker(ctx, m.marker.Username) +// restoreLocked renders m.desired again after telemt did not load a render: +// telemt keeps running the previous users, and it watches the config file, +// so the rejected file must not stay there. The restore is not confirmed. If +// telemt was only slow and loads the rejected render after all, the restore +// render follows it. Requires syncMu. +func (m *MTProto) restoreLocked(ctx context.Context) error { + m.matchAPIWritesLocked(ctx, m.desired) + if m.hasUnknownAPIWritesLocked() { + m.needsFullApply = true } - switch { - case err == nil || errors.Is(err, errClosed) || errors.Is(err, errReloadRejected): + if err := m.writeRenderedConfigLocked(m.desired); err != nil { return err - case ctx.Err() != nil: - return fmt.Errorf("telemt reload not confirmed: %w", ctx.Err()) } - - log.Printf("telemt did not answer a reload, restarting it: %v", err) - m.process.Stop() - // telemt is down now: start it again even if the caller gives up. - if restartErr := m.startLocked(context.WithoutCancel(ctx)); restartErr != nil { - return fmt.Errorf("telemt reload failed: %w (restart failed: %v)", err, restartErr) + if err := m.process.Reload(); err != nil { + log.Printf("warning: %v", err) } return nil } -// waitForMarker polls GET /v1/users until telemt's running config has the -// marker user, for up to markerConfirmTimeout. It returns ctx's error once -// the caller gives up, errClosed on Shutdown, an errReloadRejected error with -// telemt's last output when the time is up, and the request error when the -// API cannot be reached. -func (m *MTProto) waitForMarker(ctx context.Context, marker string) error { - // Requests do not end with the caller's ctx, so an aborted request is - // never mistaken for an unreachable API. - waitCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), markerConfirmTimeout) - defer cancel() - stop := context.AfterFunc(m.lifetime, cancel) - defer stop() - - ticker := time.NewTicker(markerPollInterval) - defer ticker.Stop() - - for { - users, err := m.apiClient.ListUsers(waitCtx) - if err == nil && userInRuntime(users, marker) { - return nil - } - var apiErr *apiStatusError - if err != nil && !errors.As(err, &apiErr) && waitCtx.Err() == nil { - return fmt.Errorf("telemt API unreachable: %w", err) - } - - select { - case <-ctx.Done(): - return ctx.Err() - case <-waitCtx.Done(): - if m.lifetime.Err() != nil { - return errClosed - } - return m.reloadRejectedError() - case <-ticker.C: - } - } -} - -// reloadRejectedError reports a config telemt did not load in time, with -// its last output, which carries the reason. -func (m *MTProto) reloadRejectedError() error { - tail := m.process.StartupTail(telemtCrashTailSize) - if len(tail) == 0 { - return fmt.Errorf("%w within %s", errReloadRejected, markerConfirmTimeout) - } - return fmt.Errorf("%w within %s; recent telemt output:\n%s", errReloadRejected, markerConfirmTimeout, strings.Join(tail, "\n")) -} - -func userInRuntime(users []telemtUserInfo, username string) bool { - for _, user := range users { - if user.Username == username { - return user.InRuntime - } - } - return false -} - // confirmLocked records desired as the users telemt runs. Requires syncMu. func (m *MTProto) confirmLocked(desired map[string]*runtimeUser) { m.needsFullApply = false diff --git a/backend/mtproto/sync_test.go b/backend/mtproto/sync_test.go index 4f36ffd0..b990fbc0 100644 --- a/backend/mtproto/sync_test.go +++ b/backend/mtproto/sync_test.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "maps" "net/http" "net/http/httptest" "os" @@ -20,37 +19,56 @@ import ( "time" "github.com/pasarguard/node/common" + "github.com/pasarguard/node/pkg/logstream" ) const otherSecret = "ffeeddccbbaa00112233445566778899" +const thirdSecret = "0123456789abcdef0123456789abcdef" var markerUsernamePattern = regexp.MustCompile(`^pgm_[0-9a-f]{16}$`) -// fakeTelemtAPI stands in for telemt's control API. It keeps the users of -// the config file (disk) and of the running config (runtime), records every -// call, and plays telemt's config watcher: a changed config file reaches the -// running config when GET /v1/users is polled, after reloadDelayPolls polls, -// unless rejectReload is set. +// fakeOverride is an admission override telemt stages for a user API write. +type fakeOverride struct { + deleted bool + secret string +} + +// fakeTelemtAPI stands in for telemt's control API and the parts of telemt +// behind it that the node depends on: +// - user API writes change the config file, as telemt saves them, and +// stage an admission override that only a published config with the same +// value clears; +// - the config watcher publishes a changed config file when GET /v1/users +// is polled (or connects asks), after reloadDelayPolls polls, unless +// rejectReload is set; +// - GET /v1/users lists the users of the config file, with in_runtime from +// the published config. type fakeTelemtAPI struct { t *testing.T server *httptest.Server configPath string - mu sync.Mutex - disk map[string]string // username to secret - runtime map[string]string - calls []string // "METHOD /path" per request + mu sync.Mutex + runtime map[string]string // published users: username to secret + overrides map[string]fakeOverride + calls []string // "METHOD /path" per request rejectReload bool reloadDelayPolls int - appliedConfig string // the config file content runtime was loaded from - pendingConfig string // a changed config file waiting to be applied + appliedConfig string // the config file content runtime was published from + pendingConfig string // a changed config file waiting to be published pendingPolls int // rejectedConfig is the last changed config file rejectReload kept out. rejectedConfig string - failPatch bool // PATCH answers 500 - activeIPs map[string][]string + // loadError makes reads of the config file fail like a file telemt + // cannot parse: 500 "failed to load config: ". + loadError string + // dropUsersRequests closes this many GET /v1/users requests without an + // answer. + dropUsersRequests int + failPatch bool // PATCH answers 500 + activeIPs map[string][]string } func newFakeTelemtAPI(t *testing.T) *fakeTelemtAPI { @@ -58,8 +76,8 @@ func newFakeTelemtAPI(t *testing.T) *fakeTelemtAPI { f := &fakeTelemtAPI{ t: t, configPath: filepath.Join(t.TempDir(), "config.toml"), - disk: make(map[string]string), runtime: make(map[string]string), + overrides: make(map[string]fakeOverride), activeIPs: make(map[string][]string), } f.server = httptest.NewServer(http.HandlerFunc(f.serve)) @@ -76,10 +94,12 @@ func (f *fakeTelemtAPI) takeCalls() []string { return calls } -// realRuntimeUsers returns the running users without marker users. +// realRuntimeUsers returns the published users without marker users, after +// the watcher picked up the config file. func (f *fakeTelemtAPI) realRuntimeUsers() map[string]string { f.mu.Lock() defer f.mu.Unlock() + f.watchConfigLocked() users := make(map[string]string) for name, secret := range f.runtime { if !strings.HasPrefix(name, markerUsernamePrefix) { @@ -89,10 +109,11 @@ func (f *fakeTelemtAPI) realRuntimeUsers() map[string]string { return users } -// runtimeMarkers lists the marker users in the running config. +// runtimeMarkers lists the marker users in the published config. func (f *fakeTelemtAPI) runtimeMarkers() []string { f.mu.Lock() defer f.mu.Unlock() + f.watchConfigLocked() var markers []string for name := range f.runtime { if strings.HasPrefix(name, markerUsernamePrefix) { @@ -102,32 +123,62 @@ func (f *fakeTelemtAPI) runtimeMarkers() []string { return markers } +// connects reports whether a client with secret gets past telemt's +// handshake as username: the published config and the effective admission +// state (a staged override wins) must both hold that secret. The watcher +// picks up the config file first, as time passing would. +func (f *fakeTelemtAPI) connects(username, secret string) bool { + f.mu.Lock() + defer f.mu.Unlock() + f.watchConfigLocked() + configured, ok := f.runtime[username] + if !ok || configured != secret { + return false + } + if override, staged := f.overrides[username]; staged { + return !override.deleted && override.secret == secret + } + return true +} + func (f *fakeTelemtAPI) set(apply func(f *fakeTelemtAPI)) { f.mu.Lock() defer f.mu.Unlock() apply(f) } -// startLocked loads the config file into disk and runtime, as a telemt start. +// startLocked publishes the config file, as a telemt start: fresh admission +// state, no overrides. func (f *fakeTelemtAPI) startLocked() { data, err := os.ReadFile(f.configPath) if err != nil { f.t.Errorf("fake telemt: reading the config: %v", err) return } - f.disk = parseConfigUsers(string(data)) - f.runtime = maps.Clone(f.disk) - f.appliedConfig = string(data) + f.publishLocked(string(data)) + f.overrides = make(map[string]fakeOverride) +} + +// publishLocked makes content the running config and clears the overrides +// it matches, like telemt's admission reconcile. +func (f *fakeTelemtAPI) publishLocked(content string) { + f.runtime = parseConfigUsers(content) + f.appliedConfig = content + for name, override := range f.overrides { + configured, present := f.runtime[name] + if (override.deleted && !present) || (!override.deleted && present && configured == override.secret) { + delete(f.overrides, name) + } + } } // watchConfigLocked plays telemt's config watcher. func (f *fakeTelemtAPI) watchConfigLocked() { data, err := os.ReadFile(f.configPath) - if err != nil { - return // no config file: the test set the users directly + if err != nil || f.loadError != "" { + return } content := string(data) - f.disk = parseConfigUsers(content) if content == f.appliedConfig { return } @@ -143,8 +194,18 @@ func (f *fakeTelemtAPI) watchConfigLocked() { f.pendingPolls-- return } - f.runtime = maps.Clone(f.disk) - f.appliedConfig = content + f.publishLocked(content) +} + +// editDiskLocked changes the users of the config file, as telemt's user API +// saves a change. +func (f *fakeTelemtAPI) editDiskLocked(edit func(users map[string]string)) { + data, _ := os.ReadFile(f.configPath) + users := parseConfigUsers(string(data)) + edit(users) + if err := os.WriteFile(f.configPath, []byte(rewriteConfigUsers(string(data), users)), 0o600); err != nil { + f.t.Errorf("fake telemt: writing the config: %v", err) + } } // parseConfigUsers reads [access.users] from a rendered config. @@ -170,6 +231,42 @@ func parseConfigUsers(config string) map[string]string { return users } +// rewriteConfigUsers replaces the [access.users] entries of config. +func rewriteConfigUsers(config string, users map[string]string) string { + names := make([]string, 0, len(users)) + for name := range users { + names = append(names, name) + } + slices.Sort(names) + entries := make([]string, 0, len(names)) + for _, name := range names { + entries = append(entries, name+" = "+strconv.Quote(users[name])) + } + + var out []string + section, wrote := "", false + for _, line := range strings.Split(config, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "[") { + section = strings.Trim(trimmed, "[]") + out = append(out, line) + if section == "access.users" { + out = append(out, entries...) + wrote = true + } + continue + } + if section == "access.users" && strings.Contains(trimmed, "=") { + continue + } + out = append(out, line) + } + if !wrote { + out = append(append(out, "[access.users]"), entries...) + } + return strings.Join(out, "\n") + "\n" +} + // configMarkers lists the marker users of a rendered config. func configMarkers(config string) []string { var markers []string @@ -189,8 +286,7 @@ func (f *fakeTelemtAPI) serve(w http.ResponseWriter, r *http.Request) { path := r.URL.Path f.calls = append(f.calls, r.Method+" "+path) - switch { - case r.Method == http.MethodGet && path == "/v1/stats/users/active-ips": + if r.Method == http.MethodGet && path == "/v1/stats/users/active-ips" { type entry struct { Username string `json:"username"` ActiveIPs []string `json:"active_ips"` @@ -200,17 +296,37 @@ func (f *fakeTelemtAPI) serve(w http.ResponseWriter, r *http.Request) { data = append(data, entry{Username: name, ActiveIPs: ips}) } writeFakeSuccess(w, http.StatusOK, data) + return + } + if r.Method == http.MethodGet && path == "/v1/users" && f.dropUsersRequests > 0 { + f.dropUsersRequests-- + if conn, _, err := http.NewResponseController(w).Hijack(); err == nil { + _ = conn.Close() + } + return + } + if !strings.HasPrefix(path, "/v1/users") { + writeFakeError(w, http.StatusNotFound, "not_found", "Route not found") + return + } + // Every users route loads the config file first. + data, err := os.ReadFile(f.configPath) + if f.loadError != "" || err != nil { + writeFakeError(w, http.StatusInternalServerError, "internal_error", "failed to load config: "+f.loadError) + return + } + disk := parseConfigUsers(string(data)) + + switch { case r.Method == http.MethodGet && path == "/v1/users": - // telemt lists the users of the config file and computes in_runtime - // from the running config. f.watchConfigLocked() - data := []map[string]any{} - for name := range f.disk { + users := []map[string]any{} + for name := range disk { _, inRuntime := f.runtime[name] - data = append(data, map[string]any{"username": name, "in_runtime": inRuntime}) + users = append(users, map[string]any{"username": name, "in_runtime": inRuntime}) } - writeFakeSuccess(w, http.StatusOK, data) + writeFakeSuccess(w, http.StatusOK, users) case r.Method == http.MethodPost && path == "/v1/users": var request telemtCreateUserRequest @@ -218,17 +334,17 @@ func (f *fakeTelemtAPI) serve(w http.ResponseWriter, r *http.Request) { writeFakeError(w, http.StatusBadRequest, "bad_request", err.Error()) return } - if _, exists := f.runtime[request.Username]; exists { + if _, exists := disk[request.Username]; exists { writeFakeError(w, http.StatusConflict, "user_exists", "User already exists") return } - f.runtime[request.Username] = request.Secret - f.disk[request.Username] = request.Secret + f.editDiskLocked(func(users map[string]string) { users[request.Username] = request.Secret }) + f.overrides[request.Username] = fakeOverride{secret: request.Secret} writeFakeSuccess(w, http.StatusCreated, map[string]any{"user": map[string]any{"username": request.Username}}) case strings.HasPrefix(path, "/v1/users/"): name := strings.TrimPrefix(path, "/v1/users/") - if _, exists := f.runtime[name]; !exists { + if _, exists := disk[name]; !exists { writeFakeError(w, http.StatusNotFound, "not_found", "User not found") return } @@ -243,18 +359,20 @@ func (f *fakeTelemtAPI) serve(w http.ResponseWriter, r *http.Request) { writeFakeError(w, http.StatusBadRequest, "bad_request", err.Error()) return } + secret := disk[name] if request.Secret != "" { - f.runtime[name] = request.Secret - f.disk[name] = request.Secret + secret = request.Secret } + f.editDiskLocked(func(users map[string]string) { users[name] = secret }) + f.overrides[name] = fakeOverride{secret: secret} writeFakeSuccess(w, http.StatusOK, map[string]any{"username": name}) case http.MethodDelete: - if len(f.runtime) == 1 { + if len(disk) == 1 { writeFakeError(w, http.StatusConflict, "last_user_forbidden", "Cannot delete the last user") return } - delete(f.runtime, name) - delete(f.disk, name) + f.editDiskLocked(func(users map[string]string) { delete(users, name) }) + f.overrides[name] = fakeOverride{deleted: true} writeFakeSuccess(w, http.StatusOK, name) default: // A GET of one user loads the whole config in telemt; the node @@ -299,11 +417,11 @@ func newSyncTestBackend(t *testing.T, api *fakeTelemtAPI, users []*common.User) desired, _ := ActiveUsersFromCommon(users, mtConfig.InboundTag) lifetime, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) m := &MTProto{ config: mtConfig, process: &processManager{ run: &telemtRun{pid: 1, waitDone: make(chan struct{})}, + logs: logstream.NewBuffer(100), startupTail: newLineRing(telemtStartupTailSize), }, configPath: api.configPath, @@ -317,6 +435,11 @@ func newSyncTestBackend(t *testing.T, api *fakeTelemtAPI, users []*common.User) lifetime: lifetime, cancel: cancel, } + // End a detached apply before the fake's server closes. + t.Cleanup(func() { + cancel() + m.background.Wait() + }) if err := m.writeRenderedConfigLocked(m.desired); err != nil { t.Fatal(err) } @@ -324,7 +447,14 @@ func newSyncTestBackend(t *testing.T, api *fakeTelemtAPI, users []*common.User) return m } -// lowerMarkerConfirmTimeout sets markerConfirmTimeout for one test. +// waitSettled waits until no apply holds syncMu. +func waitSettled(m *MTProto) { + m.syncMu.Lock() + m.syncMu.Unlock() +} + +// lowerMarkerConfirmTimeout sets markerConfirmTimeout for one test. Register +// it before newSyncTestBackend, so a detached apply ends before the restore. func lowerMarkerConfirmTimeout(t *testing.T, timeout time.Duration) { t.Helper() previous := markerConfirmTimeout @@ -332,6 +462,22 @@ func lowerMarkerConfirmTimeout(t *testing.T, timeout time.Duration) { t.Cleanup(func() { markerConfirmTimeout = previous }) } +// logLines returns what the backend published to telemt's log stream. +func logLines(m *MTProto) []string { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + stream := m.SubscribeLogs(ctx) + var lines []string + for { + select { + case line := <-stream: + lines = append(lines, line) + default: + return lines + } + } +} + func desiredNames(m *MTProto) []string { m.stateMu.RLock() defer m.stateMu.RUnlock() @@ -394,8 +540,8 @@ func TestPartialSkipsUnchangedUsers(t *testing.T) { if got := desiredSecret(m, "u2"); got != otherSecret { t.Errorf("desired u2 secret = %q, want %q", got, otherSecret) } - if got := api.realRuntimeUsers()["u2"]; got != otherSecret { - t.Errorf("telemt u2 secret = %q, want %q", got, otherSecret) + if !api.connects("u2", otherSecret) { + t.Error("u2 cannot connect with the new secret") } // A batch with nothing new reaches telemt not at all. @@ -422,29 +568,29 @@ func TestPartialCreatesAndDeletesWithoutGet(t *testing.T) { } t.Run("patch of a user telemt lacks creates it", func(t *testing.T) { - api.set(func(f *fakeTelemtAPI) { delete(f.runtime, "u2") }) + api.set(func(f *fakeTelemtAPI) { f.editDiskLocked(func(users map[string]string) { delete(users, "u2") }) }) if err := m.SyncUser(ctx, mtUser("u2", "mtproto", otherSecret)); err != nil { t.Fatalf("SyncUser: %v", err) } assertCalls(t, api.takeCalls(), "PATCH /v1/users/u2", "POST /v1/users") - if got := api.realRuntimeUsers()["u2"]; got != otherSecret { - t.Errorf("telemt u2 secret = %q, want %q", got, otherSecret) + if !api.connects("u2", otherSecret) { + t.Error("u2 cannot connect with the new secret") } }) t.Run("create of a user telemt has patches it", func(t *testing.T) { - api.set(func(f *fakeTelemtAPI) { f.runtime["u4"] = testSecret }) + api.set(func(f *fakeTelemtAPI) { f.editDiskLocked(func(users map[string]string) { users["u4"] = testSecret }) }) if err := m.SyncUser(ctx, mtUser("u4", "mtproto", otherSecret)); err != nil { t.Fatalf("SyncUser: %v", err) } assertCalls(t, api.takeCalls(), "POST /v1/users", "PATCH /v1/users/u4") - if got := api.realRuntimeUsers()["u4"]; got != otherSecret { - t.Errorf("telemt u4 secret = %q, want %q", got, otherSecret) + if !api.connects("u4", otherSecret) { + t.Error("u4 cannot connect with the new secret") } }) t.Run("delete of a user telemt lacks succeeds", func(t *testing.T) { - api.set(func(f *fakeTelemtAPI) { delete(f.runtime, "u3") }) + api.set(func(f *fakeTelemtAPI) { f.editDiskLocked(func(users map[string]string) { delete(users, "u3") }) }) if err := m.SyncUser(ctx, untagged("u3")); err != nil { t.Fatalf("SyncUser: %v", err) } @@ -551,8 +697,9 @@ func TestHotReloadWaitsForMarker(t *testing.T) { // TestRejectedHotReloadRestoresPreviousUsers: telemt never loads the new // config (it rejected it), so the marker never reaches its running config. -// The sync fails with telemt's last output, desired is unchanged, and the -// previous users are rendered again with a new marker. +// The sync fails with telemt's last output, desired is unchanged, the +// previous users are rendered again with a new marker, and the rejection +// reaches the log stream the panel shows. func TestRejectedHotReloadRestoresPreviousUsers(t *testing.T) { lowerMarkerConfirmTimeout(t, 600*time.Millisecond) api := newFakeTelemtAPI(t) @@ -582,6 +729,203 @@ func TestRejectedHotReloadRestoresPreviousUsers(t *testing.T) { if len(rejectedMarkers) != 1 || len(restoredMarkers) != 1 || rejectedMarkers[0] == restoredMarkers[0] { t.Errorf("markers: rejected %q, restored %q; want one each and a new one for the restore", rejectedMarkers, restoredMarkers) } + if lines := logLines(m); !slices.ContainsFunc(lines, func(line string) bool { + return strings.Contains(line, "did not load the new config") + }) { + t.Errorf("the rejection is not in the log stream: %q", lines) + } +} + +// TestLoadFailureRejectsAtOnce: telemt cannot load the rendered file, so GET +// /v1/users answers 500 "failed to load config"; that is a rejection with +// telemt's message, without waiting out markerConfirmTimeout. +func TestLoadFailureRejectsAtOnce(t *testing.T) { + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{mtUser("u1", "mtproto", testSecret)}) + api.set(func(f *fakeTelemtAPI) { f.loadError = "TOML parse error at line 3" }) + + started := time.Now() + err := m.SyncUsers(context.Background(), []*common.User{mtUser("u2", "mtproto", testSecret)}) + if !errors.Is(err, errReloadRejected) || !strings.Contains(err.Error(), "TOML parse error at line 3") { + t.Fatalf("SyncUsers = %v, want a rejection with telemt's message", err) + } + if elapsed := time.Since(started); elapsed > 5*time.Second { + t.Errorf("the rejection took %s", elapsed) + } + if got := desiredNames(m); !slices.Equal(got, []string{"u1"}) { + t.Errorf("desired = %q, want u1", got) + } + if users := parseConfigUsers(readConfig(t, api.configPath)); users["u1"] == "" || users["u2"] != "" { + t.Errorf("the previous users were not rendered again: %v", users) + } +} + +// TestCallerDeadlineDoesNotStopApply: a panel call's deadline (here 1 s) +// ends before telemt's verdict. The caller gets its ctx error at once; the +// apply goes on, finds the render rejected, renders the previous users +// again and logs telemt's reason for the panel. +func TestCallerDeadlineDoesNotStopApply(t *testing.T) { + lowerMarkerConfirmTimeout(t, 2*time.Second) + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{mtUser("u1", "mtproto", testSecret)}) + m.process.startupTail.add("config reload rejected: bad listener") + api.set(func(f *fakeTelemtAPI) { f.rejectReload = true }) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + started := time.Now() + err := m.SyncUsers(ctx, []*common.User{mtUser("u2", "mtproto", otherSecret)}) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("SyncUsers = %v, want the caller's deadline", err) + } + if elapsed := time.Since(started); elapsed > 1500*time.Millisecond { + t.Errorf("the caller waited %s after its 1 s deadline", elapsed) + } + + waitSettled(m) + if users := parseConfigUsers(readConfig(t, api.configPath)); users["u1"] != testSecret || users["u2"] != "" { + t.Errorf("the previous users were not rendered again: %v", users) + } + if got := desiredNames(m); !slices.Equal(got, []string{"u1"}) { + t.Errorf("desired = %q, want u1", got) + } + if lines := logLines(m); !slices.ContainsFunc(lines, func(line string) bool { + return strings.Contains(line, "did not load the new config") && strings.Contains(line, "bad listener") + }) { + t.Errorf("the rejection and telemt's reason are not in the log stream: %q", lines) + } + + // telemt still runs the confirmed users, so the next change is a diff. + api.set(func(f *fakeTelemtAPI) { f.rejectReload = false }) + api.takeCalls() + if err := m.UpdateUsers(context.Background(), []*common.User{mtUser("u1", "mtproto", otherSecret)}); err != nil { + t.Fatalf("UpdateUsers: %v", err) + } + assertCalls(t, api.takeCalls(), "PATCH /v1/users/u1") +} + +// TestQueuedCallerGivesUpAtDeadline: a call waiting for syncMu behind a +// long apply returns its ctx error at its own deadline, not when the apply +// ends. +func TestQueuedCallerGivesUpAtDeadline(t *testing.T) { + lowerMarkerConfirmTimeout(t, 3*time.Second) + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{mtUser("u1", "mtproto", testSecret)}) + api.set(func(f *fakeTelemtAPI) { f.rejectReload = true }) + + initialMarker := m.markerUsername() + firstDone := make(chan error, 1) + go func() { + firstDone <- m.SyncUsers(context.Background(), []*common.User{mtUser("u2", "mtproto", testSecret)}) + }() + // The first apply holds syncMu once it rendered its marker. + deadline := time.Now().Add(5 * time.Second) + for m.markerUsername() == initialMarker { + if time.Now().After(deadline) { + t.Fatal("the first apply never rendered") + } + time.Sleep(10 * time.Millisecond) + } + + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + started := time.Now() + err := m.UpdateUsers(ctx, []*common.User{mtUser("u3", "mtproto", testSecret)}) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("queued UpdateUsers = %v, want its deadline", err) + } + if elapsed := time.Since(started); elapsed > 1500*time.Millisecond { + t.Errorf("the queued caller waited %s with a 500 ms deadline", elapsed) + } + if err := <-firstDone; !errors.Is(err, errReloadRejected) { + t.Errorf("first SyncUsers = %v, want the rejection", err) + } +} + +// TestFullRenderClearsStaleAPIOverrides: telemt stages an admission +// override for every user API write and clears it only when a published +// config carries the same value. A full render that changes such a user +// again, before telemt published the API's own write, must first make the +// override agree, or the user is locked out. +func TestFullRenderClearsStaleAPIOverrides(t *testing.T) { + cases := map[string]struct { + apiChange *common.User + render []*common.User + user string + secret string + calls []string + }{ + "patched, then changed by a render": { + apiChange: mtUser("u2", "mtproto", otherSecret), + render: []*common.User{mtUser("u1", "mtproto", testSecret), mtUser("u2", "mtproto", thirdSecret)}, + user: "u2", + secret: thirdSecret, + calls: []string{"PATCH /v1/users/u2", "PATCH /v1/users/u2", "GET /v1/users"}, + }, + "deleted, then added back by a render": { + apiChange: untagged("u2"), + render: []*common.User{mtUser("u1", "mtproto", testSecret), mtUser("u2", "mtproto", thirdSecret)}, + user: "u2", + secret: thirdSecret, + calls: []string{"DELETE /v1/users/u2", "POST /v1/users", "GET /v1/users"}, + }, + "created, then removed by a render": { + apiChange: mtUser("u3", "mtproto", otherSecret), + render: []*common.User{mtUser("u1", "mtproto", testSecret), mtUser("u2", "mtproto", testSecret)}, + user: "u3", + calls: []string{"POST /v1/users", "DELETE /v1/users/u3", "GET /v1/users"}, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{ + mtUser("u1", "mtproto", testSecret), + mtUser("u2", "mtproto", testSecret), + }) + + // The API write and, before telemt's watcher read the file, the + // full render. + if err := m.SyncUser(context.Background(), tc.apiChange); err != nil { + t.Fatalf("SyncUser: %v", err) + } + if err := m.SyncUsers(context.Background(), tc.render); err != nil { + t.Fatalf("SyncUsers: %v", err) + } + assertCalls(t, api.takeCalls(), tc.calls...) + + if tc.secret != "" && !api.connects(tc.user, tc.secret) { + var override fakeOverride + api.set(func(f *fakeTelemtAPI) { override = f.overrides[tc.user] }) + t.Errorf("%s cannot connect with %s after the render (staged override %+v)", tc.user, tc.secret, override) + } + api.set(func(f *fakeTelemtAPI) { + if len(f.overrides) != 0 { + t.Errorf("overrides left after the render: %+v", f.overrides) + } + }) + }) + } +} + +// TestSingleAPIErrorKeepsPolling: polls without an answer, fewer than +// markerUnreachablePolls in a row from a telemt that still runs, are not an +// unreachable telemt; the wait goes on and telemt is not restarted. +func TestSingleAPIErrorKeepsPolling(t *testing.T) { + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{mtUser("u1", "mtproto", testSecret)}) + api.set(func(f *fakeTelemtAPI) { f.dropUsersRequests = markerUnreachablePolls - 1 }) + + if err := m.SyncUsers(context.Background(), []*common.User{mtUser("u2", "mtproto", testSecret)}); err != nil { + t.Fatalf("SyncUsers: %v", err) + } + assertCalls(t, api.takeCalls(), "GET /v1/users", "GET /v1/users", "GET /v1/users") + if !m.process.Started() { + t.Error("telemt was stopped after failed requests") + } + if got := desiredNames(m); !slices.Equal(got, []string{"u2"}) { + t.Errorf("desired = %q, want u2", got) + } } func TestFullSyncWithoutChangesSkipsReload(t *testing.T) { @@ -626,15 +970,16 @@ func TestLastRealUserGoesThroughAPI(t *testing.T) { t.Fatalf("SyncUser from zero users: %v", err) } assertCalls(t, api.takeCalls(), "POST /v1/users") - if runtime := api.realRuntimeUsers(); len(runtime) != 1 || runtime["u2"] == "" { - t.Errorf("telemt users = %v, want u2", runtime) + if !api.connects("u2", testSecret) { + t.Error("u2 cannot connect") } - // A full sync to zero users renders the marker alone. + // A full sync to zero users renders the marker alone; u2, last written + // through the API, is deleted through it again first. if err := m.SyncUsers(ctx, nil); err != nil { t.Fatalf("SyncUsers to zero users: %v", err) } - assertCalls(t, api.takeCalls(), "GET /v1/users") + assertCalls(t, api.takeCalls(), "DELETE /v1/users/u2", "GET /v1/users") if runtime := api.realRuntimeUsers(); len(runtime) != 0 { t.Errorf("telemt users = %v, want only the marker", runtime) } @@ -683,68 +1028,40 @@ func TestOnlineIPListUsesBulkEndpoint(t *testing.T) { } } -// TestUnconfirmedReloadForcesFullApply: the caller gives up before telemt -// loaded the config. telemt keeps running, desired keeps the confirmed -// users, and the next sync renders and reloads everything instead of a diff. -func TestUnconfirmedReloadForcesFullApply(t *testing.T) { +// TestPartialFailureFallsBackToFullApply: a failed PATCH may or may not have +// reached telemt, so the full apply writes the user through the API again +// before the render. When that fails too, the render still lands, but the +// state stays unconfirmed so the next sync tries again. +func TestPartialFailureFallsBackToFullApply(t *testing.T) { api := newFakeTelemtAPI(t) m := newSyncTestBackend(t, api, []*common.User{ mtUser("u1", "mtproto", testSecret), mtUser("u2", "mtproto", testSecret), }) - api.set(func(f *fakeTelemtAPI) { f.rejectReload = true }) - - ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) - defer cancel() - err := m.SyncUsers(ctx, []*common.User{mtUser("u1", "mtproto", testSecret), mtUser("u3", "mtproto", testSecret)}) - if !errors.Is(err, context.DeadlineExceeded) { - t.Fatalf("SyncUsers = %v, want a deadline error", err) - } - if got := desiredNames(m); !slices.Equal(got, []string{"u1", "u2"}) { - t.Errorf("desired = %q after an unconfirmed reload, want u1, u2", got) - } - if !m.process.Started() { - t.Error("an unconfirmed reload stopped telemt") - } - api.takeCalls() + api.set(func(f *fakeTelemtAPI) { f.failPatch = true }) - api.set(func(f *fakeTelemtAPI) { f.rejectReload = false }) if err := m.UpdateUsers(context.Background(), []*common.User{mtUser("u2", "mtproto", otherSecret)}); err != nil { t.Fatalf("UpdateUsers: %v", err) } - assertCalls(t, api.takeCalls(), "GET /v1/users") - if got := desiredNames(m); !slices.Equal(got, []string{"u1", "u2"}) { - t.Errorf("desired = %q, want u1, u2", got) + assertCalls(t, api.takeCalls(), "PATCH /v1/users/u2", "PATCH /v1/users/u2", "GET /v1/users") + if !api.connects("u2", otherSecret) { + t.Error("u2 cannot connect with the new secret") } - if runtime := api.realRuntimeUsers(); len(runtime) != 2 || runtime["u2"] != otherSecret { - t.Errorf("telemt users = %v, want u1 and the new u2", runtime) + if got := desiredSecret(m, "u2"); got != otherSecret { + t.Errorf("desired u2 secret = %q, want %q", got, otherSecret) } - - // Confirmed again: the next small change is a diff. - if err := m.UpdateUsers(context.Background(), []*common.User{mtUser("u1", "mtproto", otherSecret)}); err != nil { - t.Fatalf("UpdateUsers: %v", err) + m.syncMu.Lock() + needsFullApply := m.needsFullApply + m.syncMu.Unlock() + if !needsFullApply { + t.Error("a user whose API write failed twice must keep the state unconfirmed") } - assertCalls(t, api.takeCalls(), "PATCH /v1/users/u1") -} -func TestPartialFailureFallsBackToFullApply(t *testing.T) { - api := newFakeTelemtAPI(t) - m := newSyncTestBackend(t, api, []*common.User{ - mtUser("u1", "mtproto", testSecret), - mtUser("u2", "mtproto", testSecret), - }) - api.set(func(f *fakeTelemtAPI) { f.failPatch = true }) - - if err := m.UpdateUsers(context.Background(), []*common.User{mtUser("u2", "mtproto", otherSecret)}); err != nil { + api.set(func(f *fakeTelemtAPI) { f.failPatch = false }) + if err := m.UpdateUsers(context.Background(), []*common.User{mtUser("u1", "mtproto", testSecret)}); err != nil { t.Fatalf("UpdateUsers: %v", err) } assertCalls(t, api.takeCalls(), "PATCH /v1/users/u2", "GET /v1/users") - if got := api.realRuntimeUsers()["u2"]; got != otherSecret { - t.Errorf("telemt u2 secret = %q, want %q", got, otherSecret) - } - if got := desiredSecret(m, "u2"); got != otherSecret { - t.Errorf("desired u2 secret = %q, want %q", got, otherSecret) - } } func TestRuntimeUserEqual(t *testing.T) { @@ -773,7 +1090,12 @@ func TestRuntimeUserEqual(t *testing.T) { func TestAPIClientListUsersAndActiveIPs(t *testing.T) { api := newFakeTelemtAPI(t) api.set(func(f *fakeTelemtAPI) { - f.disk = map[string]string{"u1": testSecret, "u2": testSecret} + f.editDiskLocked(func(users map[string]string) { + users["u1"] = testSecret + users["u2"] = testSecret + }) + data, _ := os.ReadFile(f.configPath) + f.appliedConfig = string(data) f.runtime = map[string]string{"u1": testSecret} f.activeIPs = map[string][]string{"u1": {"203.0.113.7"}} }) diff --git a/backend/mtproto/synclock.go b/backend/mtproto/synclock.go new file mode 100644 index 00000000..6c05de24 --- /dev/null +++ b/backend/mtproto/synclock.go @@ -0,0 +1,44 @@ +package mtproto + +import ( + "context" + "sync" +) + +// syncLock is a mutex whose waiters can give up when their context ends, +// and whose holder may hand it to another goroutine to release. The zero +// value is unlocked. +type syncLock struct { + once sync.Once + ch chan struct{} +} + +func (l *syncLock) sem() chan struct{} { + l.once.Do(func() { l.ch = make(chan struct{}, 1) }) + return l.ch +} + +func (l *syncLock) Lock() { + l.sem() <- struct{}{} +} + +func (l *syncLock) Unlock() { + select { + case <-l.sem(): + default: + panic("mtproto: unlock of an unlocked syncLock") + } +} + +// LockContext takes the lock. It gives up with ctx's error when ctx ends, +// and with errClosed when closed is closed, whichever comes first. +func (l *syncLock) LockContext(ctx context.Context, closed <-chan struct{}) error { + select { + case l.sem() <- struct{}{}: + return nil + case <-ctx.Done(): + return ctx.Err() + case <-closed: + return errClosed + } +} diff --git a/backend/mtproto/userapi.go b/backend/mtproto/userapi.go new file mode 100644 index 00000000..39e2c1e9 --- /dev/null +++ b/backend/mtproto/userapi.go @@ -0,0 +1,129 @@ +package mtproto + +import ( + "context" + "errors" + "log" + "maps" + "slices" +) + +// userChange is one user to create, patch or delete in telemt. +type userChange struct { + username string + user *runtimeUser // the new state; nil deletes the user + known bool // telemt is expected to have the user +} + +// apiWrite is the last write of one user through telemt's user API since +// the last confirmed render: the state it set (nil for a delete), or unknown +// when the call failed and may or may not have reached telemt. +// +// telemt stages each such write as an admission override, which only a +// published config carrying the same value clears; until then the user can +// connect only if the published config agrees with the override. +type apiWrite struct { + user *runtimeUser + unknown bool +} + +// sendUserChangeLocked writes one change through telemt's user API and +// records it in m.apiSent. Create or patch follows change.known, and +// telemt's answer corrects that choice: a patch of a missing user creates +// it, a create of an existing user patches it, and deleting a missing user +// is done. Requires syncMu. +func (m *MTProto) sendUserChangeLocked(ctx context.Context, change userChange) error { + var err error + switch { + case change.user == nil: + err = m.deleteUser(ctx, change.username) + case change.known: + err = m.patchOrCreateUser(ctx, change.user) + default: + err = m.createOrPatchUser(ctx, change.user) + } + + if m.apiSent == nil { + m.apiSent = make(map[string]apiWrite) + } + if err != nil { + m.apiSent[change.username] = apiWrite{unknown: true} + return err + } + var sent *runtimeUser + if change.user != nil { + copied := *change.user + sent = &copied + } + m.apiSent[change.username] = apiWrite{user: sent} + return nil +} + +// matchAPIWritesLocked makes telemt's admission overrides agree with users +// before users are rendered. A render that changed a user written through +// the API, before telemt published that write, would leave the override +// different from the published config, and the user refused. So each user +// written through the API since the last confirmed render whose rendered +// state differs from that write is written again; the render's publish then +// clears the override. A write that fails is logged and stays unknown. +// Requires syncMu. +func (m *MTProto) matchAPIWritesLocked(ctx context.Context, users map[string]*runtimeUser) { + for _, username := range slices.Sorted(maps.Keys(m.apiSent)) { + write := m.apiSent[username] + wanted := users[username] + if !write.unknown && runtimeUserEqual(write.user, wanted) { + continue + } + change := userChange{username: username, user: wanted, known: write.unknown || write.user != nil} + if err := m.sendUserChangeLocked(ctx, change); err != nil { + log.Printf("warning: mtproto: writing user %s through the telemt API before a render failed; telemt may refuse the user until the next full apply: %v", username, err) + } + } +} + +// forgetSettledAPIWritesLocked runs after telemt published a render that +// matchAPIWritesLocked prepared: that publish cleared every override the +// node knows of. Writes whose outcome is unknown stay, and keep the state +// unconfirmed so the next sync renders again. Requires syncMu. +func (m *MTProto) forgetSettledAPIWritesLocked() { + maps.DeleteFunc(m.apiSent, func(_ string, write apiWrite) bool { return !write.unknown }) + if m.hasUnknownAPIWritesLocked() { + m.needsFullApply = true + } +} + +func (m *MTProto) hasUnknownAPIWritesLocked() bool { + for _, write := range m.apiSent { + if write.unknown { + return true + } + } + return false +} + +func (m *MTProto) patchOrCreateUser(ctx context.Context, user *runtimeUser) error { + err := m.apiClient.PatchUser(ctx, user.Username, user) + var apiErr *apiStatusError + if errors.As(err, &apiErr) && apiErr.NotFound() { + return m.apiClient.CreateUser(ctx, user) + } + return err +} + +func (m *MTProto) createOrPatchUser(ctx context.Context, user *runtimeUser) error { + err := m.apiClient.CreateUser(ctx, user) + var apiErr *apiStatusError + if errors.As(err, &apiErr) && apiErr.UserExists() { + return m.apiClient.PatchUser(ctx, user.Username, user) + } + return err +} + +func (m *MTProto) deleteUser(ctx context.Context, username string) error { + err := m.apiClient.DeleteUser(ctx, username) + var apiErr *apiStatusError + if errors.As(err, &apiErr) && apiErr.NotFound() { + return nil + } + return err +} From 71b16bbba26b296c65c554fb60bc43f42b0afd33 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 19 Oct 2026 22:07:53 +0330 Subject: [PATCH 10/40] fix(mtproto): account traffic per stat family without double counting - Give users, outbound and inbound stats their own baseline per user, so the panel's 10 s user poll and 30 s outbound poll no longer consume each other's traffic. - Track every user telemt reports, desired or not. A user missing from a snapshot keeps its counters, so a removed and re-added user is not billed again and its last interval is still reported. Absent users are dropped after 25 h (telemt forgets idle stats after 24 h), keeping the outbound and inbound traffic still pending. - Count from zero after a telemt restart (process generation) or a counter decrease, carrying the pending delta of every family. - Serialize refreshes with refreshMu and share a snapshot younger than 1 s, so concurrent polls never apply an older scrape after a newer one. A scrape that overlaps a telemt restart is discarded. - Read telemt 3.5.14's metric names, filter by name before parsing labels, and switch to GET /v1/users totals (reported as downlink) while telemt_telemetry_user_series_suppressed is set; a source switch reseeds without double counting. - Report distinct client IPs as the online count, drop marker users from every result, and log failed refreshes at most once a minute. - Move the stats endpoints into stats.go. --- backend/mtproto/metrics.go | 188 ++++++++---- backend/mtproto/metrics_test.go | 116 +++++++ backend/mtproto/mtproto.go | 168 +--------- backend/mtproto/mtproto_test.go | 103 ------- backend/mtproto/stats.go | 274 +++++++++++++++++ backend/mtproto/stats_test.go | 523 ++++++++++++++++++++++++++++++++ backend/mtproto/tracker.go | 310 +++++++++++-------- backend/mtproto/tracker_test.go | 213 +++++++++++++ 8 files changed, 1438 insertions(+), 457 deletions(-) create mode 100644 backend/mtproto/metrics_test.go create mode 100644 backend/mtproto/stats.go create mode 100644 backend/mtproto/stats_test.go create mode 100644 backend/mtproto/tracker_test.go diff --git a/backend/mtproto/metrics.go b/backend/mtproto/metrics.go index f7f049f6..2ec3d226 100644 --- a/backend/mtproto/metrics.go +++ b/backend/mtproto/metrics.go @@ -3,24 +3,36 @@ package mtproto import ( "bufio" "context" + "errors" "fmt" "io" + "math" "net/http" "strconv" "strings" "time" ) +// telemt 3.5.14 metric names (src/metrics/render/users.rs). The octet +// counters were renamed with a _total suffix in 3.5.4. const ( - telemtMetricFromClient = "telemt_user_octets_from_client" - telemtMetricToClient = "telemt_user_octets_to_client" - telemtMetricConnectionsCurrnt = "telemt_user_connections_current" - telemtMetricUniqueIPsCurrent = "telemt_user_unique_ips_current" + // metricOctetsFromClient counts bytes received from the client: uplink. + metricOctetsFromClient = "telemt_user_octets_from_client_total" + // metricOctetsToClient counts bytes sent to the client: downlink. + metricOctetsToClient = "telemt_user_octets_to_client_total" + metricConnectionsCurrent = "telemt_user_connections_current" + metricUniqueIPsCurrent = "telemt_user_unique_ips_current" + // metricUserSeriesSuppressed is above 0 when telemt left users out of + // the per-user series: it exports them for at most 4096 users. + metricUserSeriesSuppressed = "telemt_telemetry_user_series_suppressed" ) -// userMetrics holds the per-user counters/gauges scraped from telemt's -// Prometheus endpoint. Octets are monotonic counters (delta-tracked by -// trafficTracker); connections/unique-ips are point-in-time gauges. +// maxMetricsLine bounds one line of telemt's metrics output. +const maxMetricsLine = 1 << 20 + +// userMetrics holds one user's telemt counters and gauges. Octets are +// cumulative counters (the tracker turns them into deltas); connections and +// unique IPs are point-in-time gauges. type userMetrics struct { Uplink int64 Downlink int64 @@ -28,13 +40,6 @@ type userMetrics struct { UniqueIPsCurrent int64 } -var perUserMetricNames = map[string]struct{}{ - telemtMetricFromClient: {}, - telemtMetricToClient: {}, - telemtMetricConnectionsCurrnt: {}, - telemtMetricUniqueIPsCurrent: {}, -} - type metricsScraper struct { url string httpClient *http.Client @@ -49,33 +54,41 @@ func newMetricsScraper(url string) *metricsScraper { } } -func (s *metricsScraper) Scrape(ctx context.Context) (map[string]userMetrics, error) { +// Scrape reads telemt's metrics endpoint; see parseMetrics. +func (s *metricsScraper) Scrape(ctx context.Context) (map[string]userMetrics, int64, error) { if s == nil { - return nil, fmt.Errorf("mtproto metrics scraper is nil") + return nil, 0, errors.New("mtproto metrics scraper is nil") } request, err := http.NewRequestWithContext(ctx, http.MethodGet, s.url, nil) if err != nil { - return nil, err + return nil, 0, err } response, err := s.httpClient.Do(request) if err != nil { - return nil, err + return nil, 0, err } defer response.Body.Close() if response.StatusCode != http.StatusOK { payload, _ := io.ReadAll(io.LimitReader(response.Body, 4096)) - return nil, fmt.Errorf("mtproto metrics scrape failed with status %d: %s", response.StatusCode, strings.TrimSpace(string(payload))) + return nil, 0, fmt.Errorf("mtproto metrics scrape failed with status %d: %s", response.StatusCode, strings.TrimSpace(string(payload))) } return parseMetrics(response.Body) } -func parseMetrics(reader io.Reader) (map[string]userMetrics, error) { - counters := make(map[string]userMetrics) +// parseMetrics reads telemt's Prometheus text output and returns the +// per-user metrics by username and the value of the series suppression +// gauge. Other metrics are skipped before their labels are read, so an odd +// label elsewhere cannot fail the scrape. A per-user line it cannot read +// fails the whole parse: skipping it would read as a counter reset. +func parseMetrics(reader io.Reader) (map[string]userMetrics, int64, error) { + users := make(map[string]userMetrics) + var suppressed int64 scanner := bufio.NewScanner(reader) + scanner.Buffer(make([]byte, 0, 64*1024), maxMetricsLine) for scanner.Scan() { line := strings.TrimSpace(scanner.Text()) @@ -83,80 +96,117 @@ func parseMetrics(reader io.Reader) (map[string]userMetrics, error) { continue } - metricName, labels, value, ok, err := parseMetricLine(line) - if err != nil { - return nil, err - } - if !ok { + name, rest := splitMetricName(line) + switch name { + case metricOctetsFromClient, metricOctetsToClient, metricConnectionsCurrent, metricUniqueIPsCurrent: + case metricUserSeriesSuppressed: + _, value, err := parseSample(rest) + if err != nil { + return nil, 0, fmt.Errorf("invalid prometheus line %q: %w", line, err) + } + suppressed = value + continue + default: continue } - username, exists := labels["user"] - if !exists || username == "" { + labels, value, err := parseSample(rest) + if err != nil { + return nil, 0, fmt.Errorf("invalid prometheus line %q: %w", line, err) + } + username := labels["user"] + if username == "" { continue } - entry := counters[username] - switch metricName { - case telemtMetricFromClient: + entry := users[username] + switch name { + case metricOctetsFromClient: entry.Uplink = value - case telemtMetricToClient: + case metricOctetsToClient: entry.Downlink = value - case telemtMetricConnectionsCurrnt: + case metricConnectionsCurrent: entry.ConnectionsCurrent = value - case telemtMetricUniqueIPsCurrent: + case metricUniqueIPsCurrent: entry.UniqueIPsCurrent = value - default: - continue } - counters[username] = entry + users[username] = entry } if err := scanner.Err(); err != nil { - return nil, err + return nil, 0, err } - - return counters, nil + return users, suppressed, nil } -func parseMetricLine(line string) (string, map[string]string, int64, bool, error) { - splitIndex := strings.IndexAny(line, " \t") - if splitIndex <= 0 { - return "", nil, 0, false, nil - } - - metricWithLabels := strings.TrimSpace(line[:splitIndex]) - valueText := strings.TrimSpace(line[splitIndex+1:]) - if valueText == "" { - return "", nil, 0, false, nil +// splitMetricName splits a sample line into the metric name and the rest: +// labels, if any, then the value. +func splitMetricName(line string) (name, rest string) { + end := strings.IndexAny(line, "{ \t") + if end < 0 { + return line, "" } + return line[:end], line[end:] +} - metricName := metricWithLabels - labels := make(map[string]string) - - if braceIndex := strings.IndexByte(metricWithLabels, '{'); braceIndex >= 0 { - if !strings.HasSuffix(metricWithLabels, "}") { - return "", nil, 0, false, fmt.Errorf("invalid prometheus metric line %q", line) +// parseSample reads `{labels} value [timestamp]` or `value [timestamp]`. +func parseSample(rest string) (map[string]string, int64, error) { + labels := map[string]string{} + if strings.HasPrefix(rest, "{") { + end := labelsEnd(rest) + if end < 0 { + return nil, 0, errors.New("unterminated label set") } - - metricName = metricWithLabels[:braceIndex] - parsedLabels, err := parsePrometheusLabels(metricWithLabels[braceIndex+1 : len(metricWithLabels)-1]) + parsed, err := parsePrometheusLabels(rest[1:end]) if err != nil { - return "", nil, 0, false, err + return nil, 0, err } - labels = parsedLabels + labels = parsed + rest = rest[end+1:] } - if _, tracked := perUserMetricNames[metricName]; !tracked { - return "", nil, 0, false, nil + fields := strings.Fields(rest) + if len(fields) == 0 { + return nil, 0, errors.New("missing value") } - - metricValue, err := strconv.ParseFloat(strings.Fields(valueText)[0], 64) + value, err := parseMetricValue(fields[0]) if err != nil { - return "", nil, 0, false, fmt.Errorf("invalid prometheus metric value %q: %w", valueText, err) + return nil, 0, err + } + return labels, value, nil +} + +// labelsEnd returns the index of the brace closing the label set that s +// starts with, or -1. +func labelsEnd(s string) int { + inQuotes := false + escapeNext := false + for index := 1; index < len(s); index++ { + switch c := s[index]; { + case escapeNext: + escapeNext = false + case c == '\\': + escapeNext = true + case c == '"': + inQuotes = !inQuotes + case c == '}' && !inQuotes: + return index + } } + return -1 +} - return metricName, labels, int64(metricValue), true, nil +// parseMetricValue reads a sample value as an integer. telemt writes +// counters and gauges as integers; a float form is accepted too. +func parseMetricValue(text string) (int64, error) { + if value, err := strconv.ParseInt(text, 10, 64); err == nil { + return value, nil + } + value, err := strconv.ParseFloat(text, 64) + if err != nil || math.IsNaN(value) || math.Abs(value) >= math.MaxInt64 { + return 0, fmt.Errorf("invalid prometheus metric value %q", text) + } + return int64(value), nil } func parsePrometheusLabels(raw string) (map[string]string, error) { @@ -167,6 +217,10 @@ func parsePrometheusLabels(raw string) (map[string]string, error) { parts := splitLabelPairs(raw) for _, part := range parts { + if part == "" { + // A trailing comma is valid. + continue + } key, value, found := strings.Cut(part, "=") if !found { return nil, fmt.Errorf("invalid prometheus label pair %q", part) diff --git a/backend/mtproto/metrics_test.go b/backend/mtproto/metrics_test.go new file mode 100644 index 00000000..a7afae13 --- /dev/null +++ b/backend/mtproto/metrics_test.go @@ -0,0 +1,116 @@ +package mtproto + +import ( + "strings" + "testing" +) + +// telemt3514Metrics is the per-user part of telemt 3.5.14's /metrics output +// (src/metrics/render/users.rs), plus the pre-3.5.4 octet names, which must +// be ignored. +const telemt3514Metrics = `# HELP telemt_user_connections_total Per-user total connections +# TYPE telemt_user_connections_total counter +# HELP telemt_user_connections_current Per-user active connections +# TYPE telemt_user_connections_current gauge +# HELP telemt_user_octets_from_client_total Per-user total bytes received +# TYPE telemt_user_octets_from_client_total counter +# HELP telemt_user_octets_to_client_total Per-user total bytes sent +# TYPE telemt_user_octets_to_client_total counter +telemt_ip_reservation_rollback_total{reason="tcp_limit"} 0 +telemt_ip_tracker_users{scope="active"} 2 +telemt_user_connections_total{user="alice"} 9 +telemt_user_connections_current{user="alice"} 3 +telemt_user_octets_from_client_total{user="alice"} 1024 +telemt_user_octets_to_client_total{user="alice"} 2048 +telemt_user_msgs_from_client_total{user="alice"} 11 +telemt_user_msgs_to_client_total{user="alice"} 12 +telemt_user_connections_total{user="bob"} 1 +telemt_user_connections_current{user="bob"} 0 +telemt_user_octets_from_client_total{user="bob"} 7 +telemt_user_octets_to_client_total{user="bob"} 0 +telemt_user_octets_from_client{user="alice"} 999999 +telemt_user_octets_to_client{user="alice"} 999999 +# HELP telemt_user_unique_ips_current Per-user current number of unique active IPs +# TYPE telemt_user_unique_ips_current gauge +telemt_user_unique_ips_current{user="alice"} 2 +telemt_user_unique_ips_recent_window{user="alice"} 4 +telemt_user_unique_ips_limit{user="alice"} 0 +telemt_user_unique_ips_utilization{user="alice"} 0.000000 +telemt_user_unique_ips_current{user="bob"} 0 +telemt_user_unique_ips_current{user="carol"} 1 +# HELP telemt_telemetry_user_series_suppressed User-labeled metric series suppression flag +# TYPE telemt_telemetry_user_series_suppressed gauge +telemt_telemetry_user_series_suppressed 0 +telemt_telemetry_user_series_users{family="stats",status="emitted"} 2 +telemt_telemetry_user_series_users{family="stats",status="suppressed"} 0 +` + +func TestParseMetricsNewNames(t *testing.T) { + users, suppressed, err := parseMetrics(strings.NewReader(telemt3514Metrics)) + if err != nil { + t.Fatal(err) + } + if suppressed != 0 { + t.Errorf("suppressed = %d, want 0", suppressed) + } + want := map[string]userMetrics{ + "alice": {Uplink: 1024, Downlink: 2048, ConnectionsCurrent: 3, UniqueIPsCurrent: 2}, + "bob": {Uplink: 7}, + "carol": {UniqueIPsCurrent: 1}, + } + if len(users) != len(want) { + t.Errorf("users = %+v, want %+v", users, want) + } + for name, metrics := range want { + if users[name] != metrics { + t.Errorf("%s = %+v, want %+v", name, users[name], metrics) + } + } + + _, suppressed, err = parseMetrics(strings.NewReader("telemt_telemetry_user_series_suppressed 1\n")) + if err != nil || suppressed != 1 { + t.Errorf("suppressed = %d, %v; want 1", suppressed, err) + } +} + +func TestParseMetricsIgnoresBadLabelsOnOtherMetrics(t *testing.T) { + body := `telemt_tls_front_profile{domain="bad\q.example"} 1 +telemt_other{broken} 1 +telemt_unterminated{a="b" 1 +telemt_user_octets_from_client_total{user="alice"} 10 +telemt_user_octets_to_client_total{user="alice"} 20 +` + users, _, err := parseMetrics(strings.NewReader(body)) + if err != nil { + t.Fatalf("a bad label on an unrelated metric failed the scrape: %v", err) + } + if got := users["alice"]; got != (userMetrics{Uplink: 10, Downlink: 20}) { + t.Errorf("alice = %+v, want uplink 10, downlink 20", got) + } +} + +// TestParseMetricsRejectsBadUserSeries: a per-user series the node cannot +// read fails the whole scrape. Skipping it would read as a counter reset. +func TestParseMetricsRejectsBadUserSeries(t *testing.T) { + for _, line := range []string{ + `telemt_user_octets_to_client_total{user="alice"} many`, + `telemt_user_octets_to_client_total{user="alice} 1`, + `telemt_user_octets_to_client_total{user="alice"}`, + `telemt_telemetry_user_series_suppressed yes`, + } { + if _, _, err := parseMetrics(strings.NewReader(line + "\n")); err == nil { + t.Errorf("parseMetrics(%q) succeeded, want an error", line) + } + } +} + +func TestParseMetricsLabelValueWithSpaces(t *testing.T) { + body := `telemt_user_octets_from_client_total{user="alice",note="a b"} 5 1700000000000` + "\n" + users, _, err := parseMetrics(strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + if users["alice"].Uplink != 5 { + t.Errorf("alice uplink = %d, want 5", users["alice"].Uplink) + } +} diff --git a/backend/mtproto/mtproto.go b/backend/mtproto/mtproto.go index 7497a3c3..96afd969 100644 --- a/backend/mtproto/mtproto.go +++ b/backend/mtproto/mtproto.go @@ -4,16 +4,12 @@ import ( "context" "crypto/rand" "encoding/hex" - "errors" "fmt" "os" "path/filepath" "sync" "time" - gopsutilprocess "github.com/shirou/gopsutil/v4/process" - - "github.com/pasarguard/node/common" "github.com/pasarguard/node/config" ) @@ -50,6 +46,14 @@ type MTProto struct { desired map[string]*runtimeUser marker *runtimeUser + // refreshMu serializes stats refreshes, so snapshots reach tracker in + // the order telemt produced them; it guards refresh. Stats readers never + // take syncMu. + refreshMu sync.Mutex + refresh refreshState + // statsClock is the time source of refreshes; nil means time.Now. + statsClock func() time.Time + // lifetime is cancelled by Shutdown so waits for telemt stop early. lifetime context.Context cancel context.CancelFunc @@ -87,162 +91,6 @@ func (m *MTProto) Restart() error { return nil } -func (m *MTProto) GetSysStats(_ context.Context) (*common.BackendStatsResponse, error) { - if !m.process.Started() { - return nil, errors.New("mtproto backend is not started") - } - - pid := m.process.PID() - if pid == 0 { - return nil, errors.New("telemt process pid is unavailable") - } - - process, err := gopsutilprocess.NewProcess(int32(pid)) - if err != nil { - return nil, err - } - - result := &common.BackendStatsResponse{} - if memoryInfo, err := process.MemoryInfo(); err == nil && memoryInfo != nil { - result.Alloc = memoryInfo.RSS - result.Sys = memoryInfo.VMS - } - - if createTimeMillis, err := process.CreateTime(); err == nil && createTimeMillis > 0 { - uptimeSeconds := uint64(0) - nowMillis := time.Now().UnixMilli() - if nowMillis > createTimeMillis { - uptimeSeconds = uint64((nowMillis - createTimeMillis) / 1000) - } - if uptimeSeconds > uint64(^uint32(0)) { - uptimeSeconds = uint64(^uint32(0)) - } - result.Uptime = uint32(uptimeSeconds) - } - - return result, nil -} - -func (m *MTProto) GetStats(ctx context.Context, request *common.StatRequest) (*common.StatResponse, error) { - if !m.process.Started() { - return nil, errors.New("mtproto backend is not started") - } - - // telemt binds metrics together with its API, after the Telegram DC - // check. Treat a scrape miss (telemt restarting, say) as empty stats - // rather than failing the panel poll. - if err := m.refreshTracker(ctx); err != nil { - return &common.StatResponse{Stats: []*common.Stat{}}, nil - } - - link := m.config.InboundTag - switch request.GetType() { - case common.StatType_UsersStat: - return m.tracker.UsersStats(link, request.GetReset_()), nil - case common.StatType_UserStat: - return m.tracker.UserStats(request.GetName(), link, request.GetReset_()), nil - case common.StatType_Outbounds, common.StatType_Outbound: - if request.GetType() == common.StatType_Outbound && request.GetName() != "" && request.GetName() != link { - return &common.StatResponse{Stats: []*common.Stat{}}, nil - } - return m.tracker.AggregateStats(link, "mtproto", request.GetReset_()), nil - case common.StatType_Inbounds, common.StatType_Inbound: - if request.GetType() == common.StatType_Inbound && request.GetName() != "" && request.GetName() != link { - return &common.StatResponse{Stats: []*common.Stat{}}, nil - } - return m.tracker.AggregateStats(link, link, request.GetReset_()), nil - default: - return nil, errors.New("unsupported mtproto stat type") - } -} - -// GetOutboundsLatency is not applicable to an MTProto proxy (no outbounds); -// it returns an empty result to satisfy the backend.Backend interface. -func (m *MTProto) GetOutboundsLatency(_ context.Context, _ *common.LatencyRequest) (*common.LatencyResponse, error) { - return &common.LatencyResponse{Latencies: []*common.Latency{}}, nil -} - -func (m *MTProto) GetUserOnlineStats(ctx context.Context, email string) (*common.OnlineStatResponse, error) { - if !m.process.Started() { - return nil, errors.New("mtproto backend is not started") - } - - if !m.isDesiredUser(email) { - return &common.OnlineStatResponse{Name: email, Value: 0}, nil - } - - // Online count comes from telemt's per-user gauge (telemt_user_connections_current) - // via the metrics scrape, rather than a per-user API call. - snapshot, err := m.metricsScraper.Scrape(ctx) - if err != nil { - return &common.OnlineStatResponse{Name: email, Value: 0}, nil - } - - return &common.OnlineStatResponse{ - Name: email, - Value: snapshot[email].ConnectionsCurrent, - }, nil -} - -func (m *MTProto) GetUserOnlineIpListStats(ctx context.Context, email string) (*common.StatsOnlineIpListResponse, error) { - if !m.process.Started() { - return nil, errors.New("mtproto backend is not started") - } - - response := &common.StatsOnlineIpListResponse{ - Name: email, - Ips: make(map[string]int64), - } - - if !m.isDesiredUser(email) { - return response, nil - } - - // The active-IP list is only available from the API (metrics expose the - // count only). A read of one user loads telemt's whole config, so this - // takes the list of every user in one call. - activeIPs, err := m.apiClient.ActiveIPs(ctx) - if err != nil { - return nil, err - } - - now := time.Now().Unix() - for _, ip := range activeIPs[email] { - if ip == "" { - continue - } - response.Ips[ip] = now - } - - return response, nil -} - -func (m *MTProto) refreshTracker(ctx context.Context) error { - m.stateMu.RLock() - activeUsers := make(map[string]struct{}, len(m.desired)) - for username := range m.desired { - activeUsers[username] = struct{}{} - } - scraper := m.metricsScraper - tracker := m.tracker - m.stateMu.RUnlock() - - snapshot, err := scraper.Scrape(ctx) - if err != nil { - return fmt.Errorf("failed to scrape telemt metrics: %w", err) - } - - tracker.Sync(snapshot, activeUsers) - return nil -} - -func (m *MTProto) isDesiredUser(username string) bool { - m.stateMu.RLock() - defer m.stateMu.RUnlock() - _, exists := m.desired[username] - return exists -} - // markerUsernamePrefix starts every marker username. Panel usernames are // numeric user ids, so no real user has it. const markerUsernamePrefix = "pgm_" diff --git a/backend/mtproto/mtproto_test.go b/backend/mtproto/mtproto_test.go index 0d653897..495d0aa5 100644 --- a/backend/mtproto/mtproto_test.go +++ b/backend/mtproto/mtproto_test.go @@ -19,79 +19,6 @@ func mtUser(email, tag, secret string) *common.User { } } -func TestTracker_DeltaResetAndCounterRestart(t *testing.T) { - tr := newTrafficTracker() - active := map[string]struct{}{"alice": {}} - - // telemt is always started fresh by the node, so counters begin at 0: - // the first sample establishes the baseline and yields no delta. - tr.Sync(map[string]userMetrics{"alice": {Uplink: 0, Downlink: 0}}, active) - if got := tr.UsersStats("mtproto", false); len(got.Stats) != 0 { - t.Fatalf("first sample should yield zero delta, got %+v", got.Stats) - } - - // growth -> unreported pending delta (read without reset) - tr.Sync(map[string]userMetrics{"alice": {Uplink: 50, Downlink: 60}}, active) - assertDelta(t, tr.UsersStats("mtproto", false), 50, 60) - - // counter restart (telemt restarted -> counters drop): unreported 50/60 must be - // PRESERVED, not lost, and never negative. - tr.Sync(map[string]userMetrics{"alice": {Uplink: 5, Downlink: 3}}, active) - assertDelta(t, tr.UsersStats("mtproto", false), 50, 60) - - // growth after restart accumulates on top of the preserved pending total - tr.Sync(map[string]userMetrics{"alice": {Uplink: 25, Downlink: 13}}, active) - assertDelta(t, tr.UsersStats("mtproto", true), 70, 70) // 50 preserved + (25-5); 60 + (13-3) - - // after reset with no new growth -> empty - if got := tr.UsersStats("mtproto", false); len(got.Stats) != 0 { - t.Fatalf("after reset with no growth, want empty, got %+v", got.Stats) - } - - // normal post-reset growth - tr.Sync(map[string]userMetrics{"alice": {Uplink: 35, Downlink: 20}}, active) - assertDelta(t, tr.UsersStats("mtproto", false), 10, 7) -} - -func assertDelta(t *testing.T, resp *common.StatResponse, wantUp, wantDown int64) { - t.Helper() - var up, down int64 - for _, s := range resp.GetStats() { - switch s.GetType() { - case "uplink": - up = s.GetValue() - case "downlink": - down = s.GetValue() - } - } - if up != wantUp || down != wantDown { - t.Errorf("delta = up:%d down:%d, want up:%d down:%d", up, down, wantUp, wantDown) - } -} - -func TestParseMetrics(t *testing.T) { - body := `# HELP telemt_user_octets_from_client Per-user bytes received -# TYPE telemt_user_octets_from_client counter -telemt_user_octets_from_client{user="alice"} 1024 -telemt_user_octets_to_client{user="alice"} 2048 -telemt_user_connections_current{user="alice"} 3 -telemt_user_unique_ips_current{user="alice"} 2 -telemt_stats_user_entries 1 -telemt_user_octets_from_client{user="bob"} 7 -` - got, err := parseMetrics(strings.NewReader(body)) - if err != nil { - t.Fatal(err) - } - alice := got["alice"] - if alice.Uplink != 1024 || alice.Downlink != 2048 || alice.ConnectionsCurrent != 3 || alice.UniqueIPsCurrent != 2 { - t.Errorf("alice metrics = %+v", alice) - } - if got["bob"].Uplink != 7 { - t.Errorf("bob uplink = %d, want 7", got["bob"].Uplink) - } -} - func TestGetOutboundsLatency_Empty(t *testing.T) { m := &MTProto{} resp, err := m.GetOutboundsLatency(context.Background(), &common.LatencyRequest{}) @@ -148,33 +75,3 @@ func TestBuildPatchUserRequest_ClearsOptionalFields(t *testing.T) { t.Errorf("max_unique_ips = %v, want 2", setPayload.MaxUniqueIPs) } } - -func TestGetStats_MetricsUnavailable(t *testing.T) { - m := metricsUnavailableBackend() - - stats, err := m.GetStats(context.Background(), &common.StatRequest{Type: common.StatType_UsersStat}) - if err != nil { - t.Fatalf("GetStats should be best-effort, got %v", err) - } - if stats == nil || len(stats.GetStats()) != 0 { - t.Errorf("expected empty stats, got %+v", stats) - } - - online, err := m.GetUserOnlineStats(context.Background(), "alice") - if err != nil { - t.Fatalf("GetUserOnlineStats should be best-effort, got %v", err) - } - if online == nil || online.GetName() != "alice" || online.GetValue() != 0 { - t.Errorf("expected zero online count, got %+v", online) - } -} - -func metricsUnavailableBackend() *MTProto { - return &MTProto{ - config: &Config{InboundTag: "mtproto"}, - process: &processManager{run: &telemtRun{pid: 1, waitDone: make(chan struct{})}}, - metricsScraper: newMetricsScraper("http://127.0.0.1:1/metrics"), - tracker: newTrafficTracker(), - desired: map[string]*runtimeUser{"alice": {Username: "alice", Secret: testSecret}}, - } -} diff --git a/backend/mtproto/stats.go b/backend/mtproto/stats.go new file mode 100644 index 00000000..a7c186da --- /dev/null +++ b/backend/mtproto/stats.go @@ -0,0 +1,274 @@ +package mtproto + +import ( + "context" + "errors" + "fmt" + "log" + "maps" + "math" + "strings" + "time" + + gopsutilprocess "github.com/shirou/gopsutil/v4/process" + + "github.com/pasarguard/node/common" +) + +const ( + // statsSnapshotMaxAge is how long one refresh serves later readers. The + // panel's user, node and online polls often arrive together, and in + // API-totals mode each refresh makes telemt load its whole config. + statsSnapshotMaxAge = time.Second + // refreshErrorLogInterval is the least time between two logged refresh + // failures. + refreshErrorLogInterval = time.Minute + // refreshFailedMessage starts the log line of a failed refresh. + refreshFailedMessage = "warning: mtproto: failed to read telemt traffic counters" +) + +// refreshState is the outcome of the latest stats refresh. Guarded by +// MTProto.refreshMu. +type refreshState struct { + at time.Time + snapshot trafficSnapshot + err error + // loggedAt is when a failure was last logged, and unlogged how many + // failures came after it. + loggedAt time.Time + unlogged int +} + +func (m *MTProto) GetStats(ctx context.Context, request *common.StatRequest) (*common.StatResponse, error) { + if !m.process.Started() { + return nil, errors.New("mtproto backend is not started") + } + + // A failed read (telemt restarting, say) answers empty rather than + // failing the panel poll. The tracker keeps the unreported traffic for + // the next read that works. + if _, err := m.refreshStats(ctx); err != nil { + return &common.StatResponse{Stats: []*common.Stat{}}, nil + } + + link := m.config.InboundTag + switch request.GetType() { + case common.StatType_UsersStat: + return m.tracker.UsersStats(link, request.GetReset_()), nil + case common.StatType_UserStat: + return m.tracker.UserStats(request.GetName(), link, request.GetReset_()), nil + case common.StatType_Outbounds, common.StatType_Outbound: + if request.GetType() == common.StatType_Outbound && request.GetName() != "" && request.GetName() != link { + return &common.StatResponse{Stats: []*common.Stat{}}, nil + } + return m.tracker.AggregateStats(familyOutbound, link, "mtproto", request.GetReset_()), nil + case common.StatType_Inbounds, common.StatType_Inbound: + if request.GetType() == common.StatType_Inbound && request.GetName() != "" && request.GetName() != link { + return &common.StatResponse{Stats: []*common.Stat{}}, nil + } + return m.tracker.AggregateStats(familyInbound, link, link, request.GetReset_()), nil + default: + return nil, errors.New("unsupported mtproto stat type") + } +} + +// GetUserOnlineStats returns the user's distinct client IPs, as xray does: +// one Telegram client opens several connections. +func (m *MTProto) GetUserOnlineStats(ctx context.Context, email string) (*common.OnlineStatResponse, error) { + if !m.process.Started() { + return nil, errors.New("mtproto backend is not started") + } + + response := &common.OnlineStatResponse{Name: email} + if !m.isDesiredUser(email) { + return response, nil + } + snapshot, err := m.refreshStats(ctx) + if err != nil { + return response, nil + } + response.Value = snapshot.Users[email].UniqueIPsCurrent + return response, nil +} + +func (m *MTProto) GetUserOnlineIpListStats(ctx context.Context, email string) (*common.StatsOnlineIpListResponse, error) { + if !m.process.Started() { + return nil, errors.New("mtproto backend is not started") + } + + response := &common.StatsOnlineIpListResponse{ + Name: email, + Ips: make(map[string]int64), + } + + if !m.isDesiredUser(email) { + return response, nil + } + + // The active-IP list is only available from the API (metrics expose the + // count only). A read of one user loads telemt's whole config, so this + // takes the list of every user in one call. + activeIPs, err := m.apiClient.ActiveIPs(ctx) + if err != nil { + return nil, err + } + + now := time.Now().Unix() + for _, ip := range activeIPs[email] { + if ip == "" { + continue + } + response.Ips[ip] = now + } + + return response, nil +} + +// GetOutboundsLatency is not applicable to an MTProto proxy (no outbounds); +// it returns an empty result to satisfy the backend.Backend interface. +func (m *MTProto) GetOutboundsLatency(_ context.Context, _ *common.LatencyRequest) (*common.LatencyResponse, error) { + return &common.LatencyResponse{Latencies: []*common.Latency{}}, nil +} + +func (m *MTProto) GetSysStats(_ context.Context) (*common.BackendStatsResponse, error) { + if !m.process.Started() { + return nil, errors.New("mtproto backend is not started") + } + + pid := m.process.PID() + if pid == 0 { + return nil, errors.New("telemt process pid is unavailable") + } + + process, err := gopsutilprocess.NewProcess(int32(pid)) + if err != nil { + return nil, err + } + + result := &common.BackendStatsResponse{} + if memoryInfo, err := process.MemoryInfo(); err == nil && memoryInfo != nil { + result.Alloc = memoryInfo.RSS + result.Sys = memoryInfo.VMS + } + + if createTimeMillis, err := process.CreateTime(); err == nil && createTimeMillis > 0 { + uptimeSeconds := uint64(0) + nowMillis := time.Now().UnixMilli() + if nowMillis > createTimeMillis { + uptimeSeconds = uint64((nowMillis - createTimeMillis) / 1000) + } + if uptimeSeconds > uint64(^uint32(0)) { + uptimeSeconds = uint64(^uint32(0)) + } + result.Uptime = uint32(uptimeSeconds) + } + + return result, nil +} + +// refreshStats reads telemt's per-user counters and applies them to the +// tracker. Refreshes are serialized, so snapshots reach the tracker in the +// order telemt produced them, and callers within statsSnapshotMaxAge of the +// last refresh share its outcome. It never takes syncMu. +func (m *MTProto) refreshStats(ctx context.Context) (trafficSnapshot, error) { + m.refreshMu.Lock() + defer m.refreshMu.Unlock() + + if !m.refresh.at.IsZero() && m.statsNow().Sub(m.refresh.at) < statsSnapshotMaxAge { + return m.refresh.snapshot, m.refresh.err + } + + snapshot, err := m.readTraffic(ctx) + // Stamped after the read, so callers queued behind a slow one share it. + now := m.statsNow() + if err != nil { + if ctx.Err() != nil { + // The caller gave up; the next one tries again. + return trafficSnapshot{}, err + } + m.logRefreshErrorLocked(now, err) + m.refresh.at, m.refresh.snapshot, m.refresh.err = now, trafficSnapshot{}, err + return trafficSnapshot{}, err + } + + m.tracker.Apply(snapshot) + m.refresh.at, m.refresh.snapshot, m.refresh.err = now, snapshot, nil + return snapshot, nil +} + +// readTraffic takes one snapshot of telemt's per-user counters, without +// marker users. It reads the metrics endpoint, and the user list only while +// telemt suppresses per-user series: telemt loads and parses its whole +// config file for that list. +func (m *MTProto) readTraffic(ctx context.Context) (trafficSnapshot, error) { + generation := m.process.Generation() + + users, suppressed, err := m.metricsScraper.Scrape(ctx) + if err != nil { + return trafficSnapshot{}, fmt.Errorf("failed to scrape telemt metrics: %w", err) + } + source := sourceMetrics + if suppressed > 0 { + listed, err := m.apiClient.ListUsers(ctx) + if err != nil { + return trafficSnapshot{}, fmt.Errorf("telemt suppresses per-user metrics and listing its users failed: %w", err) + } + users = usersFromAPI(listed) + source = sourceAPITotals + } + + // A telemt started during the read may have answered instead of the + // one it began with; its counters cannot be told apart. + if m.process.Generation() != generation { + return trafficSnapshot{}, errors.New("telemt restarted while its counters were read") + } + + maps.DeleteFunc(users, func(username string, _ userMetrics) bool { + return strings.HasPrefix(username, markerUsernamePrefix) + }) + return trafficSnapshot{Users: users, Source: source, Generation: generation}, nil +} + +// usersFromAPI maps telemt's user list to per-user metrics: total_octets +// (both directions summed) as downlink. +func usersFromAPI(listed []telemtUserInfo) map[string]userMetrics { + users := make(map[string]userMetrics, len(listed)) + for _, user := range listed { + users[user.Username] = userMetrics{ + Downlink: int64(min(user.TotalOctets, math.MaxInt64)), + ConnectionsCurrent: int64(min(user.CurrentConnections, math.MaxInt64)), + UniqueIPsCurrent: int64(user.ActiveUniqueIPs), + } + } + return users +} + +// logRefreshErrorLocked logs a failed refresh at most once per +// refreshErrorLogInterval. Requires refreshMu. +func (m *MTProto) logRefreshErrorLocked(now time.Time, err error) { + if !m.refresh.loggedAt.IsZero() && now.Sub(m.refresh.loggedAt) < refreshErrorLogInterval { + m.refresh.unlogged++ + return + } + suffix := "" + if m.refresh.unlogged > 0 { + suffix = fmt.Sprintf(" (%d more failures since the last report)", m.refresh.unlogged) + } + log.Printf("%s; usage is kept and reported once a read works%s: %v", refreshFailedMessage, suffix, err) + m.refresh.loggedAt = now + m.refresh.unlogged = 0 +} + +func (m *MTProto) statsNow() time.Time { + if m.statsClock != nil { + return m.statsClock() + } + return time.Now() +} + +func (m *MTProto) isDesiredUser(username string) bool { + m.stateMu.RLock() + defer m.stateMu.RUnlock() + _, exists := m.desired[username] + return exists +} diff --git a/backend/mtproto/stats_test.go b/backend/mtproto/stats_test.go new file mode 100644 index 00000000..fb25a864 --- /dev/null +++ b/backend/mtproto/stats_test.go @@ -0,0 +1,523 @@ +package mtproto + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "log" + "maps" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/pasarguard/node/common" +) + +const testMarker = "pgm_0011223344556677" + +// fakeStatsServer serves telemt's /metrics and GET /v1/users from fields a +// test sets, and counts the requests. +type fakeStatsServer struct { + server *httptest.Server + + mu sync.Mutex + suppressed int + metrics map[string]userMetrics + metricsStatus int // 0 answers 200 + // onMetrics runs for each /metrics request before it is answered. + onMetrics func(f *fakeStatsServer) + apiUsers []telemtUserInfo + usersStatus int // 0 answers 200 + calls map[string]int +} + +func newFakeStatsServer(t *testing.T) *fakeStatsServer { + t.Helper() + f := &fakeStatsServer{metrics: map[string]userMetrics{}, calls: map[string]int{}} + f.server = httptest.NewServer(http.HandlerFunc(f.serve)) + t.Cleanup(f.server.Close) + return f +} + +func (f *fakeStatsServer) set(apply func(f *fakeStatsServer)) { + f.mu.Lock() + defer f.mu.Unlock() + apply(f) +} + +func (f *fakeStatsServer) callCount(path string) int { + f.mu.Lock() + defer f.mu.Unlock() + return f.calls[path] +} + +func (f *fakeStatsServer) serve(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls[r.URL.Path]++ + + switch { + case r.URL.Path == "/metrics": + if f.onMetrics != nil { + f.onMetrics(f) + } + if f.metricsStatus != 0 { + http.Error(w, "metrics unavailable", f.metricsStatus) + return + } + _, _ = fmt.Fprint(w, renderFakeMetrics(f.suppressed, f.metrics)) + case r.Method == http.MethodGet && r.URL.Path == "/v1/users": + if f.usersStatus != 0 { + writeFakeError(w, f.usersStatus, "internal_error", "failed to load config") + return + } + writeFakeSuccess(w, http.StatusOK, f.apiUsers) + default: + writeFakeError(w, http.StatusNotFound, "not_found", "Route not found") + } +} + +// renderFakeMetrics writes users the way telemt 3.5.14 does. +func renderFakeMetrics(suppressed int, users map[string]userMetrics) string { + var b strings.Builder + for _, name := range slices.Sorted(maps.Keys(users)) { + u := users[name] + fmt.Fprintf(&b, "telemt_user_connections_total{user=%q} 1\n", name) + fmt.Fprintf(&b, "telemt_user_connections_current{user=%q} %d\n", name, u.ConnectionsCurrent) + fmt.Fprintf(&b, "telemt_user_octets_from_client_total{user=%q} %d\n", name, u.Uplink) + fmt.Fprintf(&b, "telemt_user_octets_to_client_total{user=%q} %d\n", name, u.Downlink) + fmt.Fprintf(&b, "telemt_user_unique_ips_current{user=%q} %d\n", name, u.UniqueIPsCurrent) + } + fmt.Fprintf(&b, "telemt_telemetry_user_series_suppressed %d\n", suppressed) + return b.String() +} + +// testClock is a clock tests move by hand; with step set, every reading +// also moves it on by step. +type testClock struct { + mu sync.Mutex + now time.Time + step time.Duration +} + +func newTestClock(step time.Duration) *testClock { + return &testClock{now: time.Unix(1_800_000_000, 0), step: step} +} + +func (c *testClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + c.now = c.now.Add(c.step) + return c.now +} + +func (c *testClock) Advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.now = c.now.Add(d) +} + +// newStatsTestBackend returns a started backend reading stats from server. +// Its clock moves 2 s per reading, so every call refreshes, unless a test +// sets the step to zero. +func newStatsTestBackend(t *testing.T, server *fakeStatsServer, users ...string) (*MTProto, *testClock) { + t.Helper() + desired := make(map[string]*runtimeUser, len(users)) + for _, name := range users { + desired[name] = &runtimeUser{Username: name, Secret: testSecret} + } + clock := newTestClock(2 * statsSnapshotMaxAge) + m := &MTProto{ + config: &Config{InboundTag: "mtproto"}, + process: &processManager{run: &telemtRun{pid: 1, waitDone: make(chan struct{})}, generation: 1}, + apiClient: newAPIClient(server.server.URL, "Bearer test"), + metricsScraper: newMetricsScraper(server.server.URL + "/metrics"), + tracker: newTrafficTracker(), + desired: desired, + statsClock: clock.Now, + } + return m, clock +} + +func getStats(t *testing.T, m *MTProto, statType common.StatType, reset bool) *common.StatResponse { + t.Helper() + response, err := m.GetStats(context.Background(), &common.StatRequest{Type: statType, Reset_: reset}) + if err != nil { + t.Fatalf("GetStats(%v): %v", statType, err) + } + return response +} + +func TestGetStats_MetricsUnavailable(t *testing.T) { + m := metricsUnavailableBackend() + + stats, err := m.GetStats(context.Background(), &common.StatRequest{Type: common.StatType_UsersStat}) + if err != nil { + t.Fatalf("GetStats should be best-effort, got %v", err) + } + if stats == nil || len(stats.GetStats()) != 0 { + t.Errorf("expected empty stats, got %+v", stats) + } + + online, err := m.GetUserOnlineStats(context.Background(), "alice") + if err != nil { + t.Fatalf("GetUserOnlineStats should be best-effort, got %v", err) + } + if online == nil || online.GetName() != "alice" || online.GetValue() != 0 { + t.Errorf("expected zero online count, got %+v", online) + } +} + +func metricsUnavailableBackend() *MTProto { + return &MTProto{ + config: &Config{InboundTag: "mtproto"}, + process: &processManager{run: &telemtRun{pid: 1, waitDone: make(chan struct{})}}, + metricsScraper: newMetricsScraper("http://127.0.0.1:1/metrics"), + tracker: newTrafficTracker(), + desired: map[string]*runtimeUser{"alice": {Username: "alice", Secret: testSecret}}, + } +} + +// TestReaddedUserNotBilledTwice: telemt keeps a removed user's counters, so +// the tracker follows every user telemt reports, desired or not. +func TestReaddedUserNotBilledTwice(t *testing.T) { + server := newFakeStatsServer(t) + m, _ := newStatsTestBackend(t, server, "u1") + + server.set(func(f *fakeStatsServer) { f.metrics["u1"] = traffic(1000, 0) }) + assertTraffic(t, getStats(t, m, common.StatType_UsersStat, true), map[string]counters{"u1": {1000, 0}}) + + m.stateMu.Lock() + m.desired = map[string]*runtimeUser{} + m.stateMu.Unlock() + assertTraffic(t, getStats(t, m, common.StatType_UsersStat, true), map[string]counters{}) + + m.stateMu.Lock() + m.desired = map[string]*runtimeUser{"u1": {Username: "u1", Secret: testSecret}} + m.stateMu.Unlock() + server.set(func(f *fakeStatsServer) { f.metrics["u1"] = traffic(1200, 0) }) + assertTraffic(t, getStats(t, m, common.StatType_UsersStat, true), map[string]counters{"u1": {200, 0}}) +} + +// TestRemovedUserTrafficIsBilled: traffic telemt counts for a user after it +// left the desired set (its last interval, or a session still open) is +// reported. +func TestRemovedUserTrafficIsBilled(t *testing.T) { + server := newFakeStatsServer(t) + m, _ := newStatsTestBackend(t, server) + + server.set(func(f *fakeStatsServer) { f.metrics["u1"] = traffic(40, 60) }) + assertTraffic(t, getStats(t, m, common.StatType_UsersStat, true), map[string]counters{"u1": {40, 60}}) + assertTraffic(t, getStats(t, m, common.StatType_Outbounds, true), map[string]counters{"mtproto": {40, 60}}) +} + +func TestConcurrentRefreshDoesNotDoubleCount(t *testing.T) { + server := newFakeStatsServer(t) + m, _ := newStatsTestBackend(t, server, "u1") + + // Every scrape sees the counters grown by 10/20. + server.set(func(f *fakeStatsServer) { + f.onMetrics = func(f *fakeStatsServer) { + u := f.metrics["u1"] + f.metrics["u1"] = traffic(u.Uplink+10, u.Downlink+20) + } + }) + + const workers, calls = 4, 25 + var mu sync.Mutex + var reported counters + var wg sync.WaitGroup + for range workers { + wg.Go(func() { + for range calls { + response, err := m.GetStats(context.Background(), &common.StatRequest{Type: common.StatType_UsersStat, Reset_: true}) + if err != nil { + t.Errorf("GetStats: %v", err) + return + } + got := trafficByName(t, response)["u1"] + mu.Lock() + reported.Up += got.Up + reported.Down += got.Down + mu.Unlock() + } + }) + } + wg.Wait() + + scrapes := server.callCount("/metrics") + if scrapes != workers*calls { + t.Errorf("%d scrapes, want %d (the clock makes every refresh stale)", scrapes, workers*calls) + } + var final userMetrics + server.set(func(f *fakeStatsServer) { final = f.metrics["u1"] }) + if reported != (counters{final.Uplink, final.Downlink}) { + t.Errorf("reported %v in total, want the final counters %d/%d", reported, final.Uplink, final.Downlink) + } +} + +func TestRefreshReusesRecentSnapshot(t *testing.T) { + server := newFakeStatsServer(t) + m, clock := newStatsTestBackend(t, server, "u1") + clock.step = 0 + server.set(func(f *fakeStatsServer) { f.metrics["u1"] = traffic(5, 5) }) + + getStats(t, m, common.StatType_UsersStat, false) + getStats(t, m, common.StatType_Outbounds, false) + if _, err := m.GetUserOnlineStats(context.Background(), "u1"); err != nil { + t.Fatal(err) + } + clock.Advance(statsSnapshotMaxAge - time.Millisecond) + getStats(t, m, common.StatType_UsersStat, false) + if got := server.callCount("/metrics"); got != 1 { + t.Errorf("%d scrapes within %s, want 1", got, statsSnapshotMaxAge) + } + + clock.Advance(time.Millisecond) + getStats(t, m, common.StatType_UsersStat, false) + if got := server.callCount("/metrics"); got != 2 { + t.Errorf("%d scrapes after %s, want 2", got, statsSnapshotMaxAge) + } +} + +// TestRestartDuringScrapeDiscardsSnapshot: a scrape that overlaps a telemt +// restart may hold either process's counters, so it is dropped. +func TestRestartDuringScrapeDiscardsSnapshot(t *testing.T) { + server := newFakeStatsServer(t) + m, _ := newStatsTestBackend(t, server, "u1") + + server.set(func(f *fakeStatsServer) { + f.metrics["u1"] = traffic(70, 80) + f.onMetrics = func(f *fakeStatsServer) { + m.process.mu.Lock() + m.process.generation++ + m.process.mu.Unlock() + f.onMetrics = nil + } + }) + assertTraffic(t, getStats(t, m, common.StatType_UsersStat, true), map[string]counters{}) + assertTraffic(t, getStats(t, m, common.StatType_UsersStat, true), map[string]counters{"u1": {70, 80}}) +} + +func TestSuppressedSeriesSwitchToAPITotals(t *testing.T) { + server := newFakeStatsServer(t) + m, _ := newStatsTestBackend(t, server, "u1") + var users, outbound counters + poll := func(statType common.StatType, total *counters, want map[string]counters) { + t.Helper() + response := getStats(t, m, statType, true) + assertTraffic(t, response, want) + for _, got := range trafficByName(t, response) { + total.Up += got.Up + total.Down += got.Down + } + } + + server.set(func(f *fakeStatsServer) { + f.metrics["u1"] = traffic(100, 200) + f.metrics[testMarker] = traffic(9, 9) + }) + poll(common.StatType_UsersStat, &users, map[string]counters{"u1": {100, 200}}) + server.set(func(f *fakeStatsServer) { f.metrics["u1"] = traffic(150, 250) }) + poll(common.StatType_Outbounds, &outbound, map[string]counters{"mtproto": {150, 250}}) + if got := server.callCount("/v1/users"); got != 0 { + t.Errorf("%d user list calls while per-user metrics are complete, want 0", got) + } + + // More users than telemt exports series for: u1's series are gone (or + // stale), and its total comes from the user list. + server.set(func(f *fakeStatsServer) { + f.suppressed = 3 + f.metrics = map[string]userMetrics{"u9": traffic(1, 1)} + f.apiUsers = []telemtUserInfo{ + {Username: "u1", TotalOctets: 400}, + {Username: testMarker, TotalOctets: 999}, + } + }) + poll(common.StatType_UsersStat, &users, map[string]counters{"u1": {50, 50}}) + + server.set(func(f *fakeStatsServer) { f.apiUsers[0].TotalOctets = 500 }) + poll(common.StatType_UsersStat, &users, map[string]counters{"u1": {0, 100}}) + poll(common.StatType_Outbounds, &outbound, map[string]counters{"mtproto": {0, 100}}) + usersCalls := server.callCount("/v1/users") + if usersCalls != 3 { + t.Errorf("%d user list calls in API-totals mode, want 3", usersCalls) + } + + server.set(func(f *fakeStatsServer) { + f.suppressed = 0 + f.metrics = map[string]userMetrics{"u1": traffic(200, 300)} + }) + poll(common.StatType_UsersStat, &users, map[string]counters{}) + server.set(func(f *fakeStatsServer) { f.metrics["u1"] = traffic(210, 320) }) + poll(common.StatType_UsersStat, &users, map[string]counters{"u1": {10, 20}}) + poll(common.StatType_Outbounds, &outbound, map[string]counters{"mtproto": {10, 20}}) + if got := server.callCount("/v1/users"); got != usersCalls { + t.Errorf("%d user list calls after per-user metrics came back, want %d", got, usersCalls) + } + + // Every byte telemt counted (210+320) is billed once per family. + if users.Up+users.Down != 530 || outbound.Up+outbound.Down != 530 { + t.Errorf("billed users %v, outbound %v; want 530 bytes each", users, outbound) + } +} + +// TestFailedRefreshKeepsPendingTraffic: a user list that fails (telemt +// answers 5xx while a bad config is on disk) keeps the previous counters. +func TestFailedRefreshKeepsPendingTraffic(t *testing.T) { + server := newFakeStatsServer(t) + m, _ := newStatsTestBackend(t, server, "u1") + + server.set(func(f *fakeStatsServer) { + f.suppressed = 1 + f.apiUsers = []telemtUserInfo{{Username: "u1", TotalOctets: 100}} + }) + assertTraffic(t, getStats(t, m, common.StatType_UsersStat, true), map[string]counters{"u1": {0, 100}}) + + server.set(func(f *fakeStatsServer) { + f.usersStatus = http.StatusInternalServerError + f.apiUsers[0].TotalOctets = 150 + }) + assertTraffic(t, getStats(t, m, common.StatType_UsersStat, true), map[string]counters{}) + + server.set(func(f *fakeStatsServer) { + f.usersStatus = 0 + f.apiUsers[0].TotalOctets = 170 + }) + assertTraffic(t, getStats(t, m, common.StatType_UsersStat, true), map[string]counters{"u1": {0, 70}}) +} + +func TestMarkerUsersNeverReported(t *testing.T) { + server := newFakeStatsServer(t) + m, _ := newStatsTestBackend(t, server, "u1") + server.set(func(f *fakeStatsServer) { + f.metrics["u1"] = userMetrics{Uplink: 1, Downlink: 2, UniqueIPsCurrent: 1} + f.metrics[testMarker] = userMetrics{Uplink: 5, Downlink: 6, UniqueIPsCurrent: 1} + f.metrics["pgm_ffeeddccbbaa9988"] = userMetrics{Uplink: 7, Downlink: 8, UniqueIPsCurrent: 1} + }) + + assertTraffic(t, getStats(t, m, common.StatType_UsersStat, false), map[string]counters{"u1": {1, 2}}) + assertTraffic(t, getStats(t, m, common.StatType_UserStat, false), map[string]counters{}) + response, err := m.GetStats(context.Background(), &common.StatRequest{Type: common.StatType_UserStat, Name: testMarker}) + if err != nil { + t.Fatal(err) + } + assertTraffic(t, response, map[string]counters{}) + assertTraffic(t, getStats(t, m, common.StatType_Outbounds, false), map[string]counters{"mtproto": {1, 2}}) + assertTraffic(t, getStats(t, m, common.StatType_Inbounds, false), map[string]counters{"mtproto": {1, 2}}) + + online, err := m.GetUserOnlineStats(context.Background(), testMarker) + if err != nil || online.GetValue() != 0 { + t.Errorf("marker online = %+v, %v; want 0", online, err) + } +} + +func TestOnlineStatIsUniqueIPs(t *testing.T) { + server := newFakeStatsServer(t) + m, _ := newStatsTestBackend(t, server, "u1") + server.set(func(f *fakeStatsServer) { + f.metrics["u1"] = userMetrics{ConnectionsCurrent: 5, UniqueIPsCurrent: 2} + }) + + online, err := m.GetUserOnlineStats(context.Background(), "u1") + if err != nil { + t.Fatal(err) + } + if online.GetName() != "u1" || online.GetValue() != 2 { + t.Errorf("online = %+v, want u1 with 2 unique IPs", online) + } + + // With per-user series suppressed, the count comes from the user list. + server.set(func(f *fakeStatsServer) { + f.suppressed = 1 + f.apiUsers = []telemtUserInfo{{Username: "u1", CurrentConnections: 7, ActiveUniqueIPs: 3}} + }) + online, err = m.GetUserOnlineStats(context.Background(), "u1") + if err != nil { + t.Fatal(err) + } + if online.GetValue() != 3 { + t.Errorf("online = %d with suppressed series, want 3", online.GetValue()) + } +} + +// lockedBuffer is a bytes.Buffer safe for the log package and a test to +// share. +type lockedBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +// captureLog sends the standard logger's output to a buffer for one test. +func captureLog(t *testing.T) *lockedBuffer { + t.Helper() + buffer := &lockedBuffer{} + previous := log.Writer() + log.SetOutput(buffer) + t.Cleanup(func() { log.SetOutput(previous) }) + return buffer +} + +func TestScrapeErrorIsLogged(t *testing.T) { + logs := captureLog(t) + server := newFakeStatsServer(t) + m, clock := newStatsTestBackend(t, server, "u1") + clock.step = 0 + server.set(func(f *fakeStatsServer) { f.metricsStatus = http.StatusInternalServerError }) + + failures := func() int { return strings.Count(logs.String(), refreshFailedMessage) } + for i := range 3 { + if i > 0 { + clock.Advance(2 * statsSnapshotMaxAge) + } + response := getStats(t, m, common.StatType_UsersStat, true) + assertTraffic(t, response, map[string]counters{}) + if response.GetStats() == nil { + t.Error("GetStats returned nil stats, want an empty list") + } + } + if got := server.callCount("/metrics"); got != 3 { + t.Errorf("%d scrapes, want 3", got) + } + if got := failures(); got != 1 { + t.Errorf("logged %d failures in 4 s, want 1:\n%s", got, logs.String()) + } + + clock.Advance(refreshErrorLogInterval) + getStats(t, m, common.StatType_UsersStat, true) + if got := failures(); got != 2 { + t.Errorf("logged %d failures after a minute, want 2:\n%s", got, logs.String()) + } + if !strings.Contains(logs.String(), "500") { + t.Errorf("the log does not carry the cause:\n%s", logs.String()) + } +} + +func TestAPIUsersDecodeTotals(t *testing.T) { + payload := `{"ok":true,"data":[{"username":"u1","total_octets":12345,"current_connections":4,"active_unique_ips":2}]}` + var decoded apiSuccessResponse[[]telemtUserInfo] + if err := json.Unmarshal([]byte(payload), &decoded); err != nil { + t.Fatal(err) + } + got := usersFromAPI(decoded.Data) + if want := (userMetrics{Downlink: 12345, ConnectionsCurrent: 4, UniqueIPsCurrent: 2}); got["u1"] != want { + t.Errorf("u1 = %+v, want %+v", got["u1"], want) + } +} diff --git a/backend/mtproto/tracker.go b/backend/mtproto/tracker.go index 249d7303..ae232885 100644 --- a/backend/mtproto/tracker.go +++ b/backend/mtproto/tracker.go @@ -1,204 +1,260 @@ package mtproto import ( + "log" "sync" + "time" "github.com/pasarguard/node/common" ) -type trafficTracker struct { - mu sync.RWMutex - entries map[string]*trafficEntry +// statFamily is a group of stats the panel reads and resets on its own +// schedule (users every 10 s, outbounds every 30 s). Like xray's separate +// counters, each family keeps its own baseline per user, so resetting one +// family never hides traffic from another. +type statFamily uint8 + +const ( + familyUsers statFamily = iota + familyOutbound + familyInbound + familyCount +) + +// counterSource is where a snapshot's per-user counters come from. +type counterSource uint8 + +const ( + // sourceMetrics is telemt's per-user Prometheus counters, one per + // direction. + sourceMetrics counterSource = iota + // sourceAPITotals is total_octets from GET /v1/users, used while telemt + // suppresses per-user series (more than 4096 users with stats). It sums + // both directions, so it is reported as downlink with uplink 0. + sourceAPITotals +) + +// absentEntryTTL is how long the tracker keeps a user that no snapshot +// reports. telemt keeps a user's counters after the user leaves its config +// until they have been idle for 24 h, so a user re-added sooner comes back +// with its old total, which must not be billed again. +const absentEntryTTL = 25 * time.Hour + +// trafficSnapshot is one read of telemt's per-user counters. Generation is +// the telemt process (processManager.Generation) the counters came from. +type trafficSnapshot struct { + Users map[string]userMetrics + Source counterSource + Generation uint64 +} + +// counters is a byte count per direction. +type counters struct { + Up int64 + Down int64 } +// trafficEntry follows one user's telemt counters. Current is the latest +// sample, and Base[f] the point family f last reported up to, so the +// family's unreported traffic is Current - Base[f]. type trafficEntry struct { - CurrentUplink int64 - CurrentDownlink int64 - BaseUplink int64 - BaseDownlink int64 - LastDeltaUplink int64 - LastDeltaDown int64 - Deleted bool + Current counters + Base [familyCount]counters + Source counterSource + Generation uint64 + AbsentSince time.Time // zero while snapshots report the user +} + +// trafficTracker turns telemt's cumulative per-user counters into the +// deltas each stat family reports. It follows every user telemt reports, +// desired or not: telemt keeps counting a removed user until it forgets the +// user's stats. +type trafficTracker struct { + mu sync.RWMutex + entries map[string]*trafficEntry + // retired holds the unreported outbound and inbound traffic of dropped + // entries. + retired [familyCount]counters + source counterSource + applied bool + now func() time.Time } func newTrafficTracker() *trafficTracker { return &trafficTracker{ entries: make(map[string]*trafficEntry), + now: time.Now, } } -func (t *trafficTracker) Sync(snapshot map[string]userMetrics, activeUsers map[string]struct{}) { +// Apply takes a snapshot. A user missing from it is not sampled: its +// counters are unknown, not zero. Snapshots must be applied in the order +// they were read. +func (t *trafficTracker) Apply(s trafficSnapshot) { t.mu.Lock() defer t.mu.Unlock() - for username := range activeUsers { - counters := snapshot[username] - t.applySampleLocked(username, counters) + if t.applied && s.Source != t.source { + logSourceChange(s.Source) } + t.applied = true + t.source = s.Source + for username, metrics := range s.Users { + sample := counters{Up: metrics.Uplink, Down: metrics.Downlink} + entry, exists := t.entries[username] + if !exists { + // telemt counts a user from zero, and keeps counting while it + // reports the user, so the whole counter is new traffic. + t.entries[username] = &trafficEntry{Current: sample, Source: s.Source, Generation: s.Generation} + continue + } + + switch { + case entry.Generation != s.Generation: + // A new telemt process counts from zero, whichever source. + entry.carryPending(counters{}) + case entry.Source != s.Source: + // The new source counts the same traffic differently: keep only + // what was pending. + entry.carryPending(sample) + case sample.Up < entry.Current.Up || sample.Down < entry.Current.Down: + // telemt dropped the user's stats after 24 h idle and counts + // from zero again. + entry.carryPending(counters{}) + } + entry.Current = sample + entry.Source = s.Source + entry.Generation = s.Generation + entry.AbsentSince = time.Time{} + } + + now := t.now() for username, entry := range t.entries { - if _, ok := activeUsers[username]; ok { - entry.Deleted = false + if _, present := s.Users[username]; present { continue } - if entry.Deleted { + if entry.AbsentSince.IsZero() { + entry.AbsentSince = now + } + if now.Sub(entry.AbsentSince) < absentEntryTTL || entry.delta(familyUsers) != (counters{}) { continue } - entry.LastDeltaUplink = entry.CurrentUplink - entry.BaseUplink - entry.LastDeltaDown = entry.CurrentDownlink - entry.BaseDownlink - entry.Deleted = true + // Users are reported one by one, so a user is dropped only once + // reported; the aggregates keep its pending traffic. + for _, family := range []statFamily{familyOutbound, familyInbound} { + t.retired[family] = t.retired[family].plus(entry.delta(family)) + } + delete(t.entries, username) } } -func (t *trafficTracker) applySampleLocked(username string, counters userMetrics) { - entry, exists := t.entries[username] - if !exists { - t.entries[username] = &trafficEntry{ - CurrentUplink: counters.Uplink, - CurrentDownlink: counters.Downlink, - } +func logSourceChange(source counterSource) { + if source == sourceAPITotals { + log.Println("mtproto: telemt suppresses per-user metrics (more than 4096 users with stats); " + + "counting traffic from its user API totals, reported as downlink") return } + log.Println("mtproto: telemt exports per-user metrics again; counting traffic from them") +} - readded := false - if entry.Deleted { - pendingUplink := entry.LastDeltaUplink - pendingDownlink := entry.LastDeltaDown - entry.BaseUplink = counters.Uplink - pendingUplink - entry.BaseDownlink = counters.Downlink - pendingDownlink - entry.LastDeltaUplink = 0 - entry.LastDeltaDown = 0 - entry.Deleted = false - readded = true +// carryPending rebases every family onto a counter that now stands at start, +// keeping each family's unreported traffic: Base = start - pending. +func (e *trafficEntry) carryPending(start counters) { + for family := range familyCount { + pending := e.delta(family) + e.Base[family] = counters{Up: start.Up - pending.Up, Down: start.Down - pending.Down} } +} - if !readded && (counters.Uplink < entry.CurrentUplink || counters.Downlink < entry.CurrentDownlink) { - pendingUplink := entry.CurrentUplink - entry.BaseUplink - if pendingUplink < 0 { - pendingUplink = 0 - } - pendingDownlink := entry.CurrentDownlink - entry.BaseDownlink - if pendingDownlink < 0 { - pendingDownlink = 0 - } - entry.BaseUplink = counters.Uplink - pendingUplink - entry.BaseDownlink = counters.Downlink - pendingDownlink +// delta is family f's unreported traffic, never negative. +func (e *trafficEntry) delta(f statFamily) counters { + return counters{ + Up: max(0, e.Current.Up-e.Base[f].Up), + Down: max(0, e.Current.Down-e.Base[f].Down), } +} - entry.CurrentUplink = counters.Uplink - entry.CurrentDownlink = counters.Downlink +func (c counters) plus(other counters) counters { + return counters{Up: c.Up + other.Up, Down: c.Down + other.Down} } -func (t *trafficTracker) UserStats(username, link string, reset bool) *common.StatResponse { +// lock takes the write lock when reset changes baselines, else the read lock, +// and returns the matching unlock. +func (t *trafficTracker) lock(reset bool) func() { if reset { t.mu.Lock() - defer t.mu.Unlock() - } else { - t.mu.RLock() - defer t.mu.RUnlock() + return t.mu.Unlock } + t.mu.RLock() + return t.mu.RUnlock +} + +// UserStats reports one user's traffic. A reset covers only that user. +func (t *trafficTracker) UserStats(username, link string, reset bool) *common.StatResponse { + defer t.lock(reset)() response := &common.StatResponse{Stats: []*common.Stat{}} entry, exists := t.entries[username] if !exists { return response } - - response.Stats = append(response.Stats, buildTrafficStats(username, link, entry.deltaUplink(), entry.deltaDownlink())...) + response.Stats = append(response.Stats, buildTrafficStats(username, link, entry.delta(familyUsers))...) if reset { - t.resetEntryLocked(username, entry) + entry.Base[familyUsers] = entry.Current } return response } +// UsersStats reports every user with traffic. A reset covers the users it +// reported. func (t *trafficTracker) UsersStats(link string, reset bool) *common.StatResponse { - if reset { - t.mu.Lock() - defer t.mu.Unlock() - } else { - t.mu.RLock() - defer t.mu.RUnlock() - } + defer t.lock(reset)() response := &common.StatResponse{Stats: make([]*common.Stat, 0, len(t.entries)*2)} for username, entry := range t.entries { - response.Stats = append(response.Stats, buildTrafficStats(username, link, entry.deltaUplink(), entry.deltaDownlink())...) + delta := entry.delta(familyUsers) + if delta == (counters{}) { + continue + } + response.Stats = append(response.Stats, buildTrafficStats(username, link, delta)...) if reset { - t.resetEntryLocked(username, entry) + entry.Base[familyUsers] = entry.Current } } return response } -func (t *trafficTracker) AggregateStats(name, link string, reset bool) *common.StatResponse { - if reset { - t.mu.Lock() - defer t.mu.Unlock() - } else { - t.mu.RLock() - defer t.mu.RUnlock() +// AggregateStats reports the traffic of all users as one stat for family f, +// which must be familyOutbound or familyInbound. +func (t *trafficTracker) AggregateStats(f statFamily, name, link string, reset bool) *common.StatResponse { + response := &common.StatResponse{Stats: []*common.Stat{}} + if f != familyOutbound && f != familyInbound { + return response } + defer t.lock(reset)() - var totalUplink int64 - var totalDownlink int64 - for username, entry := range t.entries { - totalUplink += entry.deltaUplink() - totalDownlink += entry.deltaDownlink() + total := t.retired[f] + for _, entry := range t.entries { + total = total.plus(entry.delta(f)) if reset { - t.resetEntryLocked(username, entry) + entry.Base[f] = entry.Current } } - - return &common.StatResponse{ - Stats: buildTrafficStats(name, link, totalUplink, totalDownlink), - } -} - -func (t *trafficTracker) resetEntryLocked(username string, entry *trafficEntry) { - entry.BaseUplink = entry.CurrentUplink - entry.BaseDownlink = entry.CurrentDownlink - if entry.Deleted { - delete(t.entries, username) - } -} - -func (e *trafficEntry) deltaUplink() int64 { - delta := e.CurrentUplink - e.BaseUplink - if delta < 0 { - return 0 - } - return delta -} - -func (e *trafficEntry) deltaDownlink() int64 { - delta := e.CurrentDownlink - e.BaseDownlink - if delta < 0 { - return 0 + if reset { + t.retired[f] = counters{} } - return delta + response.Stats = append(response.Stats, buildTrafficStats(name, link, total)...) + return response } -func buildTrafficStats(name, link string, uplink, downlink int64) []*common.Stat { - if uplink == 0 && downlink == 0 { - return nil - } - +func buildTrafficStats(name, link string, traffic counters) []*common.Stat { stats := make([]*common.Stat, 0, 2) - if uplink > 0 { - stats = append(stats, &common.Stat{ - Name: name, - Type: "uplink", - Link: link, - Value: uplink, - }) - } - if downlink > 0 { - stats = append(stats, &common.Stat{ - Name: name, - Type: "downlink", - Link: link, - Value: downlink, - }) + if traffic.Up > 0 { + stats = append(stats, &common.Stat{Name: name, Type: "uplink", Link: link, Value: traffic.Up}) + } + if traffic.Down > 0 { + stats = append(stats, &common.Stat{Name: name, Type: "downlink", Link: link, Value: traffic.Down}) } return stats } diff --git a/backend/mtproto/tracker_test.go b/backend/mtproto/tracker_test.go new file mode 100644 index 00000000..0e8052ae --- /dev/null +++ b/backend/mtproto/tracker_test.go @@ -0,0 +1,213 @@ +package mtproto + +import ( + "maps" + "testing" + "time" + + "github.com/pasarguard/node/common" +) + +func traffic(up, down int64) userMetrics { + return userMetrics{Uplink: up, Downlink: down} +} + +func metricsSnapshot(generation uint64, users map[string]userMetrics) trafficSnapshot { + return trafficSnapshot{Users: users, Source: sourceMetrics, Generation: generation} +} + +func apiSnapshot(generation uint64, users map[string]userMetrics) trafficSnapshot { + return trafficSnapshot{Users: users, Source: sourceAPITotals, Generation: generation} +} + +// trafficByName sums a response's uplink and downlink values per stat name. +func trafficByName(t *testing.T, resp *common.StatResponse) map[string]counters { + t.Helper() + got := make(map[string]counters) + for _, stat := range resp.GetStats() { + total := got[stat.GetName()] + switch stat.GetType() { + case "uplink": + total.Up += stat.GetValue() + case "downlink": + total.Down += stat.GetValue() + default: + t.Errorf("unexpected stat type %q", stat.GetType()) + } + got[stat.GetName()] = total + } + return got +} + +// assertTraffic checks that resp reports exactly want: every name with its +// uplink and downlink, and no other name. +func assertTraffic(t *testing.T, resp *common.StatResponse, want map[string]counters) { + t.Helper() + if got := trafficByName(t, resp); !maps.Equal(got, want) { + t.Errorf("stats = %v, want %v", got, want) + } +} + +func TestFamiliesHaveIndependentBaselines(t *testing.T) { + tr := newTrafficTracker() + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(100, 200)})) + + assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {100, 200}}) + assertTraffic(t, tr.AggregateStats(familyOutbound, "mtproto", "mtproto", true), map[string]counters{"mtproto": {100, 200}}) + assertTraffic(t, tr.AggregateStats(familyInbound, "mtproto", "mtproto", true), map[string]counters{"mtproto": {100, 200}}) + + // The other order: an outbound poll first must not hide user traffic. + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(150, 260)})) + assertTraffic(t, tr.AggregateStats(familyOutbound, "mtproto", "mtproto", true), map[string]counters{"mtproto": {50, 60}}) + assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {50, 60}}) + assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{}) + assertTraffic(t, tr.AggregateStats(familyOutbound, "mtproto", "mtproto", true), map[string]counters{}) + assertTraffic(t, tr.AggregateStats(familyInbound, "mtproto", "mtproto", true), map[string]counters{"mtproto": {50, 60}}) +} + +func TestUserStatResetsOnlyThatUser(t *testing.T) { + tr := newTrafficTracker() + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(10, 20), "u2": traffic(30, 40)})) + + assertTraffic(t, tr.UserStats("u1", "mtproto", true), map[string]counters{"u1": {10, 20}}) + assertTraffic(t, tr.UserStats("u1", "mtproto", true), map[string]counters{}) + assertTraffic(t, tr.UsersStats("mtproto", false), map[string]counters{"u2": {30, 40}}) + assertTraffic(t, tr.AggregateStats(familyOutbound, "mtproto", "mtproto", false), map[string]counters{"mtproto": {40, 60}}) +} + +func TestAbsentUserIsNotACounterReset(t *testing.T) { + tr := newTrafficTracker() + var reported counters + report := func() { + t.Helper() + got := trafficByName(t, tr.UsersStats("mtproto", true)) + reported.Up += got["u1"].Up + reported.Down += got["u1"].Down + } + + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(1000, 100), "u2": traffic(1, 1)})) + report() + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u2": traffic(2, 2)})) + report() + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(1500, 150), "u2": traffic(3, 3)})) + report() + + if reported != (counters{1500, 150}) { + t.Errorf("u1 reported %v in total, want {1500 150}", reported) + } + assertTraffic(t, tr.AggregateStats(familyOutbound, "mtproto", "mtproto", true), map[string]counters{"mtproto": {1503, 153}}) +} + +func TestGenerationChangeKeepsPendingDelta(t *testing.T) { + tr := newTrafficTracker() + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(500, 50)})) + // telemt restarted: its counters start from zero again. + tr.Apply(metricsSnapshot(2, map[string]userMetrics{"u1": traffic(100, 10)})) + + assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {600, 60}}) + assertTraffic(t, tr.AggregateStats(familyOutbound, "mtproto", "mtproto", true), map[string]counters{"mtproto": {600, 60}}) + + tr.Apply(metricsSnapshot(2, map[string]userMetrics{"u1": traffic(130, 15)})) + assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {30, 5}}) +} + +// TestCounterDecreaseCarriesPending: a counter that goes down within one +// telemt process means telemt dropped the user's stats entry (24 h idle) and +// counts from zero again. Moved from the old tracker test, with the traffic +// after the decrease now counted too. +func TestCounterDecreaseCarriesPending(t *testing.T) { + tr := newTrafficTracker() + + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"alice": traffic(0, 0)})) + assertTraffic(t, tr.UsersStats("mtproto", false), map[string]counters{}) + + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"alice": traffic(50, 60)})) + assertTraffic(t, tr.UsersStats("mtproto", false), map[string]counters{"alice": {50, 60}}) + + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"alice": traffic(5, 3)})) + assertTraffic(t, tr.UsersStats("mtproto", false), map[string]counters{"alice": {55, 63}}) + + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"alice": traffic(25, 13)})) + assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"alice": {75, 73}}) + assertTraffic(t, tr.UsersStats("mtproto", false), map[string]counters{}) + + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"alice": traffic(35, 20)})) + assertTraffic(t, tr.UsersStats("mtproto", false), map[string]counters{"alice": {10, 7}}) +} + +// TestSourceChangeReseeds: API totals sum both directions, so switching +// sources must neither bill a counter twice nor lose what was pending. +func TestSourceChangeReseeds(t *testing.T) { + tr := newTrafficTracker() + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(100, 200)})) + assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {100, 200}}) + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(150, 250)})) + + tr.Apply(apiSnapshot(1, map[string]userMetrics{"u1": traffic(0, 400)})) + assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {50, 50}}) + + tr.Apply(apiSnapshot(1, map[string]userMetrics{"u1": traffic(0, 500)})) + assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {0, 100}}) + + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(200, 300)})) + assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{}) + + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(210, 320)})) + assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {10, 20}}) + assertTraffic(t, tr.AggregateStats(familyOutbound, "mtproto", "mtproto", true), map[string]counters{"mtproto": {160, 370}}) +} + +// TestAbsentEntryIsKeptUntilTelemtForgetsIt: telemt keeps a removed user's +// counters until they sit idle for 24 h, and its user list omits users not in +// the config. A user re-added within that time comes back with its old total. +func TestAbsentEntryIsKeptUntilTelemtForgetsIt(t *testing.T) { + now := time.Unix(1_800_000_000, 0) + tr := newTrafficTracker() + tr.now = func() time.Time { return now } + + tr.Apply(apiSnapshot(1, map[string]userMetrics{"u1": traffic(0, 1000), "u2": traffic(0, 5)})) + assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {0, 1000}, "u2": {0, 5}}) + + tr.Apply(apiSnapshot(1, map[string]userMetrics{"u2": traffic(0, 5)})) + now = now.Add(time.Hour) + tr.Apply(apiSnapshot(1, map[string]userMetrics{"u1": traffic(0, 1200), "u2": traffic(0, 5)})) + assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {0, 200}}) + + // Removed for longer than telemt keeps idle counters: the entry goes, + // and its unreported outbound traffic stays in the aggregate. + tr.Apply(apiSnapshot(1, map[string]userMetrics{"u2": traffic(0, 5)})) + now = now.Add(absentEntryTTL) + tr.Apply(apiSnapshot(1, map[string]userMetrics{"u2": traffic(0, 5)})) + tr.mu.RLock() + _, kept := tr.entries["u1"] + tr.mu.RUnlock() + if kept { + t.Fatal("u1 is still tracked after being absent for absentEntryTTL") + } + + tr.Apply(apiSnapshot(1, map[string]userMetrics{"u1": traffic(0, 300), "u2": traffic(0, 5)})) + assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {0, 300}}) + assertTraffic(t, tr.AggregateStats(familyOutbound, "mtproto", "mtproto", true), map[string]counters{"mtproto": {0, 1505}}) +} + +// TestAbsentEntryWithPendingTrafficIsKept: an entry is never dropped while +// its user traffic is unreported. +func TestAbsentEntryWithPendingTrafficIsKept(t *testing.T) { + now := time.Unix(1_800_000_000, 0) + tr := newTrafficTracker() + tr.now = func() time.Time { return now } + + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(10, 20)})) + tr.Apply(metricsSnapshot(1, map[string]userMetrics{})) + now = now.Add(2 * absentEntryTTL) + tr.Apply(metricsSnapshot(1, map[string]userMetrics{})) + + assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {10, 20}}) + tr.Apply(metricsSnapshot(1, map[string]userMetrics{})) + tr.mu.RLock() + remaining := len(tr.entries) + tr.mu.RUnlock() + if remaining != 0 { + t.Errorf("%d entries remain after the pending traffic was reported", remaining) + } +} From 809914912897bd49f35366e1143d8997df3073d6 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 19 Oct 2026 22:26:02 +0330 Subject: [PATCH 11/40] fix(mtproto): carry exact traffic across a stats source switch The reseed on a switch between per-user metrics and API totals dropped the bytes telemt counted between the last sample of one source and the first of the other, in every family. Both sources count the same telemt stats entry (total_octets is octets_from_client + octets_to_client), so within one telemt process the increment is known exactly: credit it as downlink. A total below the last sample means telemt dropped the entry in between, which counts from zero like a restart. The suppression trigger also covers more than 4096 configured users; say so in the comments and the switch log line. --- backend/mtproto/metrics.go | 3 +- backend/mtproto/stats_test.go | 24 ++++++----- backend/mtproto/tracker.go | 26 +++++++++--- backend/mtproto/tracker_test.go | 72 +++++++++++++++++++++++++++------ 4 files changed, 95 insertions(+), 30 deletions(-) diff --git a/backend/mtproto/metrics.go b/backend/mtproto/metrics.go index 2ec3d226..bef1e876 100644 --- a/backend/mtproto/metrics.go +++ b/backend/mtproto/metrics.go @@ -23,7 +23,8 @@ const ( metricConnectionsCurrent = "telemt_user_connections_current" metricUniqueIPsCurrent = "telemt_user_unique_ips_current" // metricUserSeriesSuppressed is above 0 when telemt left users out of - // the per-user series: it exports them for at most 4096 users. + // the per-user series: it exports them for at most 4096 users with + // stats, and unique-IP series for at most 4096 configured users. metricUserSeriesSuppressed = "telemt_telemetry_user_series_suppressed" ) diff --git a/backend/mtproto/stats_test.go b/backend/mtproto/stats_test.go index fb25a864..cbe30f5e 100644 --- a/backend/mtproto/stats_test.go +++ b/backend/mtproto/stats_test.go @@ -330,40 +330,42 @@ func TestSuppressedSeriesSwitchToAPITotals(t *testing.T) { } // More users than telemt exports series for: u1's series are gone (or - // stale), and its total comes from the user list. + // stale), and its total comes from the user list. u1 moved 50 more + // bytes since the last scrape; they count as downlink. server.set(func(f *fakeStatsServer) { f.suppressed = 3 f.metrics = map[string]userMetrics{"u9": traffic(1, 1)} f.apiUsers = []telemtUserInfo{ - {Username: "u1", TotalOctets: 400}, + {Username: "u1", TotalOctets: 450}, {Username: testMarker, TotalOctets: 999}, } }) - poll(common.StatType_UsersStat, &users, map[string]counters{"u1": {50, 50}}) + poll(common.StatType_UsersStat, &users, map[string]counters{"u1": {50, 100}}) server.set(func(f *fakeStatsServer) { f.apiUsers[0].TotalOctets = 500 }) - poll(common.StatType_UsersStat, &users, map[string]counters{"u1": {0, 100}}) + poll(common.StatType_UsersStat, &users, map[string]counters{"u1": {0, 50}}) poll(common.StatType_Outbounds, &outbound, map[string]counters{"mtproto": {0, 100}}) usersCalls := server.callCount("/v1/users") if usersCalls != 3 { t.Errorf("%d user list calls in API-totals mode, want 3", usersCalls) } + // Back to per-user metrics, 30 bytes past the last total of 500. server.set(func(f *fakeStatsServer) { f.suppressed = 0 - f.metrics = map[string]userMetrics{"u1": traffic(200, 300)} + f.metrics = map[string]userMetrics{"u1": traffic(210, 320)} }) - poll(common.StatType_UsersStat, &users, map[string]counters{}) - server.set(func(f *fakeStatsServer) { f.metrics["u1"] = traffic(210, 320) }) + poll(common.StatType_UsersStat, &users, map[string]counters{"u1": {0, 30}}) + server.set(func(f *fakeStatsServer) { f.metrics["u1"] = traffic(220, 340) }) poll(common.StatType_UsersStat, &users, map[string]counters{"u1": {10, 20}}) - poll(common.StatType_Outbounds, &outbound, map[string]counters{"mtproto": {10, 20}}) + poll(common.StatType_Outbounds, &outbound, map[string]counters{"mtproto": {10, 50}}) if got := server.callCount("/v1/users"); got != usersCalls { t.Errorf("%d user list calls after per-user metrics came back, want %d", got, usersCalls) } - // Every byte telemt counted (210+320) is billed once per family. - if users.Up+users.Down != 530 || outbound.Up+outbound.Down != 530 { - t.Errorf("billed users %v, outbound %v; want 530 bytes each", users, outbound) + // Every byte telemt counted (220+340) is billed once per family. + if users != (counters{160, 400}) || outbound != (counters{160, 400}) { + t.Errorf("billed users %v, outbound %v; want {160 400} each", users, outbound) } } diff --git a/backend/mtproto/tracker.go b/backend/mtproto/tracker.go index ae232885..526d7c41 100644 --- a/backend/mtproto/tracker.go +++ b/backend/mtproto/tracker.go @@ -29,8 +29,9 @@ const ( // direction. sourceMetrics counterSource = iota // sourceAPITotals is total_octets from GET /v1/users, used while telemt - // suppresses per-user series (more than 4096 users with stats). It sums - // both directions, so it is reported as downlink with uplink 0. + // suppresses per-user series (more than 4096 users with stats or + // configured users). It sums both directions, so it is reported as + // downlink with uplink 0. sourceAPITotals ) @@ -115,9 +116,7 @@ func (t *trafficTracker) Apply(s trafficSnapshot) { // A new telemt process counts from zero, whichever source. entry.carryPending(counters{}) case entry.Source != s.Source: - // The new source counts the same traffic differently: keep only - // what was pending. - entry.carryPending(sample) + entry.carryAcrossSource(sample) case sample.Up < entry.Current.Up || sample.Down < entry.Current.Down: // telemt dropped the user's stats after 24 h idle and counts // from zero again. @@ -151,13 +150,28 @@ func (t *trafficTracker) Apply(s trafficSnapshot) { func logSourceChange(source counterSource) { if source == sourceAPITotals { - log.Println("mtproto: telemt suppresses per-user metrics (more than 4096 users with stats); " + + log.Println("mtproto: telemt suppresses per-user metrics (more than 4096 users with stats or configured users); " + "counting traffic from its user API totals, reported as downlink") return } log.Println("mtproto: telemt exports per-user metrics again; counting traffic from them") } +// carryAcrossSource rebases every family onto sample from the other source. +// Both sources count the same telemt stats entry (total_octets is +// octets_from_client + octets_to_client), so the bytes counted since the +// last sample are known as a total: they are credited as downlink. +func (e *trafficEntry) carryAcrossSource(sample counters) { + increment := sample.Up + sample.Down - (e.Current.Up + e.Current.Down) + if increment < 0 { + // telemt dropped the stats entry in between and counts from zero + // again. + e.carryPending(counters{}) + return + } + e.carryPending(counters{Up: sample.Up, Down: sample.Down - increment}) +} + // carryPending rebases every family onto a counter that now stands at start, // keeping each family's unreported traffic: Base = start - pending. func (e *trafficEntry) carryPending(start counters) { diff --git a/backend/mtproto/tracker_test.go b/backend/mtproto/tracker_test.go index 0e8052ae..bd90b1b3 100644 --- a/backend/mtproto/tracker_test.go +++ b/backend/mtproto/tracker_test.go @@ -135,26 +135,74 @@ func TestCounterDecreaseCarriesPending(t *testing.T) { assertTraffic(t, tr.UsersStats("mtproto", false), map[string]counters{"alice": {10, 7}}) } -// TestSourceChangeReseeds: API totals sum both directions, so switching -// sources must neither bill a counter twice nor lose what was pending. -func TestSourceChangeReseeds(t *testing.T) { +// TestSourceSwitchCarriesBoundaryTraffic: both sources count the same telemt +// stats entry (total_octets is from + to), so the bytes counted between the +// last sample of one source and the first of the other are billed once, as +// downlink, in every family. +func TestSourceSwitchCarriesBoundaryTraffic(t *testing.T) { tr := newTrafficTracker() + var users, outbound counters + poll := func(resp *common.StatResponse, total *counters, want map[string]counters) { + t.Helper() + assertTraffic(t, resp, want) + for _, got := range trafficByName(t, resp) { + *total = total.plus(got) + } + } + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(100, 200)})) - assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {100, 200}}) + poll(tr.UsersStats("mtproto", true), &users, map[string]counters{"u1": {100, 200}}) tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(150, 250)})) + poll(tr.AggregateStats(familyOutbound, "mtproto", "mtproto", true), &outbound, map[string]counters{"mtproto": {150, 250}}) - tr.Apply(apiSnapshot(1, map[string]userMetrics{"u1": traffic(0, 400)})) - assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {50, 50}}) + // Metrics to API totals, 50 bytes past 150+250. + tr.Apply(apiSnapshot(1, map[string]userMetrics{"u1": traffic(0, 450)})) + poll(tr.UsersStats("mtproto", true), &users, map[string]counters{"u1": {50, 100}}) tr.Apply(apiSnapshot(1, map[string]userMetrics{"u1": traffic(0, 500)})) - assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {0, 100}}) - - tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(200, 300)})) - assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{}) + poll(tr.UsersStats("mtproto", true), &users, map[string]counters{"u1": {0, 50}}) + poll(tr.AggregateStats(familyOutbound, "mtproto", "mtproto", true), &outbound, map[string]counters{"mtproto": {0, 100}}) + // API totals to metrics, 30 bytes past 500. tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(210, 320)})) - assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {10, 20}}) - assertTraffic(t, tr.AggregateStats(familyOutbound, "mtproto", "mtproto", true), map[string]counters{"mtproto": {160, 370}}) + poll(tr.UsersStats("mtproto", true), &users, map[string]counters{"u1": {0, 30}}) + + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(220, 340)})) + poll(tr.UsersStats("mtproto", true), &users, map[string]counters{"u1": {10, 20}}) + poll(tr.AggregateStats(familyOutbound, "mtproto", "mtproto", true), &outbound, map[string]counters{"mtproto": {10, 50}}) + + if users != (counters{160, 400}) || outbound != (counters{160, 400}) { + t.Errorf("billed users %v, outbound %v; want {160 400} (220+340 bytes) each", users, outbound) + } +} + +// TestSourceSwitchAfterEvictionCountsFromZero: a total below the last +// sample means telemt dropped the stats entry in between and counts from +// zero again. +func TestSourceSwitchAfterEvictionCountsFromZero(t *testing.T) { + tr := newTrafficTracker() + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(100, 200)})) + assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {100, 200}}) + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(120, 230)})) + + tr.Apply(apiSnapshot(1, map[string]userMetrics{"u1": traffic(0, 40)})) + assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {20, 70}}) + assertTraffic(t, tr.AggregateStats(familyOutbound, "mtproto", "mtproto", true), map[string]counters{"mtproto": {120, 270}}) + + // The other direction: metrics below the last API total. + tr.Apply(apiSnapshot(1, map[string]userMetrics{"u1": traffic(0, 90)})) + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(5, 15)})) + assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {5, 65}}) + assertTraffic(t, tr.AggregateStats(familyOutbound, "mtproto", "mtproto", true), map[string]counters{"mtproto": {5, 65}}) +} + +// TestGenerationChangeBeatsSourceChange: a new telemt counts from zero, so +// the whole first sample is new traffic, whatever the source. +func TestGenerationChangeBeatsSourceChange(t *testing.T) { + tr := newTrafficTracker() + tr.Apply(metricsSnapshot(1, map[string]userMetrics{"u1": traffic(100, 200)})) + tr.Apply(apiSnapshot(2, map[string]userMetrics{"u1": traffic(0, 400)})) + assertTraffic(t, tr.UsersStats("mtproto", true), map[string]counters{"u1": {100, 600}}) } // TestAbsentEntryIsKeptUntilTelemtForgetsIt: telemt keeps a removed user's From 5d9b826f4c59a64ae67bc58da5e5f9a9ea9c8fc7 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 19 Oct 2026 23:24:39 +0330 Subject: [PATCH 12/40] build(mtproto): ship pinned telemt 3.5.14 in the node image - Dockerfile: the builder runs make install_telemt after install_xray, and the final stage carries /usr/local/bin/telemt plus the telemt license files in /usr/share/licenses/telemt/. - Makefile: install_telemt checks the tarball against our own SHA256 per arch (x86_64, aarch64) instead of the release's .sha256 file, refuses "latest", and needs TELEMT_SHA256 for any version other than the pinned one. TELEMT_INSTALL_DIR selects the target dir; sudo is used only when that dir is not writable. --version runs only for a host-arch binary. - CI: install telemt and point MTPROTO_TELEMT_BIN at it, so the gated MTProto integration tests run. - third_party/telemt: telemt 3.5.14 LICENSE and the canonical English license text, copied verbatim, with a note on the unmodified binary. - README: MTProto backend, Xray and MTProto on one host, third-party note. --- .github/workflows/run-tests.yml | 5 + Dockerfile | 4 + Makefile | 67 +++++++--- README.md | 38 ++++++ third_party/telemt/LICENSE | 169 ++++++++++++++++++++++++ third_party/telemt/README.md | 3 + third_party/telemt/TELEMT-LICENSE.en.md | 120 +++++++++++++++++ 7 files changed, 389 insertions(+), 17 deletions(-) create mode 100644 third_party/telemt/LICENSE create mode 100644 third_party/telemt/README.md create mode 100644 third_party/telemt/TELEMT-LICENSE.en.md diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index f41cb193..ef68d0be 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -31,6 +31,9 @@ jobs: - name: Install xray-core run: make install_xray + - name: Install telemt + run: make install_telemt + - name: install wg run: make install_wg @@ -41,6 +44,8 @@ jobs: make generate_client_cert - name: Run regular test suite + env: + MTPROTO_TELEMT_BIN: /usr/local/bin/telemt run: TEST_INTEGRATION=true go test ./... -v -p 1 - name: Check Xray process status races diff --git a/Dockerfile b/Dockerfile index 2ee626f9..977102bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,6 +16,8 @@ COPY . . RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} make NAME=main build # install_xray also installs the pinned geo files (Makefile install_geofiles) and fails the build unless xray loads them RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} XRAY_TAG=${XRAY_TAG} make install_xray +# MTProto backend: the telemt version and SHA256 per arch are pinned in the Makefile (amd64 and arm64 only) +RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} make install_telemt FROM alpine:latest @@ -27,5 +29,7 @@ WORKDIR /app COPY --from=builder /src/main /app/main COPY --from=builder /usr/local/bin/xray /usr/local/bin/xray COPY --from=builder /usr/local/share/xray /usr/local/share/xray +COPY --from=builder /usr/local/bin/telemt /usr/local/bin/telemt +COPY third_party/telemt/ /usr/share/licenses/telemt/ ENTRYPOINT ["./main"] diff --git a/Makefile b/Makefile index 3266505f..823b91f8 100644 --- a/Makefile +++ b/Makefile @@ -46,10 +46,19 @@ GEOFILES_DIR ?= /usr/local/share/xray GEOFILES_XRAY ?= /usr/local/bin/xray GEOFILES_CHECK_CONFIG ?= $(CURDIR)/testdata/geofiles-check.json -# telemt (MTProto backend): pinned release; static musl tarball + sha256 verify. +# telemt (MTProto backend): pinned release, static musl tarball. # 3.5.14 is also the floor: the node refuses to start an older telemt. -TELEMT_REPOSITORY ?= telemt/telemt -TELEMT_VERSION ?= 3.5.14 +# The tarball is checked against our own SHA256 per arch below, never the .sha256 file of the release. +# Any other version also needs its checksum, and "latest" is always refused: +# make install_telemt TELEMT_VERSION=X.Y.Z TELEMT_SHA256=-linux-musl.tar.gz> +# On a bump, update TELEMT_PINNED_VERSION, both checksums and third_party/telemt/ together. +TELEMT_REPOSITORY ?= telemt/telemt +TELEMT_PINNED_VERSION := 3.5.14 +TELEMT_VERSION ?= $(TELEMT_PINNED_VERSION) +TELEMT_SHA256 ?= +TELEMT_SHA256_PINNED_x86_64 := 71c1454f79cf755edea0985500629daba553002f9863be754e8e38a418822773 +TELEMT_SHA256_PINNED_aarch64 := 21795430c4165cf964405459714d08eea319edaa5c9027beac4de373c356d84d +TELEMT_INSTALL_DIR ?= /usr/local/bin TELEMT_OS ?= TELEMT_ARCH ?= TELEMT_ARCH_MAP_amd64 = x86_64 @@ -285,30 +294,54 @@ ifeq ($(UNAME_S),Linux) exit 1; \ fi @if [ -z "$(TELEMT_ARCH_EFFECTIVE)" ]; then \ - echo "Unsupported TELEMT target architecture: $(TELEMT_GOARCH_EFFECTIVE)"; \ + echo "Unsupported TELEMT target architecture: $(TELEMT_GOARCH_EFFECTIVE) (telemt is installed for amd64 and arm64 only)"; \ exit 1; \ fi @set -eu; \ telemt_asset="telemt-$(TELEMT_ARCH_EFFECTIVE)-linux-musl.tar.gz"; \ telemt_release="$(TELEMT_VERSION)"; \ - telemt_release="$${telemt_release#refs/tags/}"; \ - if [ -z "$${telemt_release}" ] || [ "$${telemt_release}" = "latest" ]; then \ - telemt_base_url="https://github.com/$(TELEMT_REPOSITORY)/releases/latest/download"; \ - else \ - telemt_base_url="https://github.com/$(TELEMT_REPOSITORY)/releases/download/$${telemt_release}"; \ + telemt_sha256="$(TELEMT_SHA256)"; \ + case "$$telemt_release" in \ + ""|latest|*[!0-9A-Za-z._-]*) \ + echo "TELEMT_VERSION must name a telemt release such as $(TELEMT_PINNED_VERSION), got '$$telemt_release'" >&2; \ + exit 1;; \ + esac; \ + if [ -z "$$telemt_sha256" ]; then \ + if [ "$$telemt_release" != "$(TELEMT_PINNED_VERSION)" ]; then \ + echo "telemt $$telemt_release is not the pinned $(TELEMT_PINNED_VERSION): also pass TELEMT_SHA256=" >&2; \ + exit 1; \ + fi; \ + telemt_sha256="$(TELEMT_SHA256_PINNED_$(TELEMT_ARCH_EFFECTIVE))"; \ + if [ -z "$$telemt_sha256" ]; then \ + echo "no pinned SHA256 for $$telemt_asset: pass TELEMT_SHA256" >&2; \ + exit 1; \ + fi; \ fi; \ + telemt_sha256="$$(printf '%s' "$$telemt_sha256" | tr 'A-F' 'a-f')"; \ + case "$$telemt_sha256" in \ + *[!0-9a-f]*) echo "TELEMT_SHA256 must be 64 hex characters" >&2; exit 1;; \ + esac; \ + if [ "$${#telemt_sha256}" -ne 64 ]; then \ + echo "TELEMT_SHA256 must be 64 hex characters" >&2; \ + exit 1; \ + fi; \ + echo "Installing telemt $$telemt_release ($$telemt_asset)"; \ tmp_dir="$$(mktemp -d)"; \ trap 'rm -rf "$$tmp_dir"' EXIT; \ - curl -fL --retry 5 --retry-delay 3 --connect-timeout 10 --max-time 120 \ - -o "$$tmp_dir/$$telemt_asset" "$$telemt_base_url/$$telemt_asset"; \ - curl -fL --retry 5 --retry-delay 3 --connect-timeout 10 --max-time 120 \ - -o "$$tmp_dir/$$telemt_asset.sha256" "$$telemt_base_url/$$telemt_asset.sha256"; \ + curl -fsSL --retry 5 --retry-delay 3 --connect-timeout 10 --max-time 600 \ + -o "$$tmp_dir/$$telemt_asset" \ + "https://github.com/$(TELEMT_REPOSITORY)/releases/download/$$telemt_release/$$telemt_asset"; \ + printf '%s %s\n' "$$telemt_sha256" "$$telemt_asset" > "$$tmp_dir/$$telemt_asset.sha256"; \ ( cd "$$tmp_dir" && sha256sum -c "$$telemt_asset.sha256" ); \ - tar -xzf "$$tmp_dir/$$telemt_asset" -C "$$tmp_dir"; \ + tar -xzf "$$tmp_dir/$$telemt_asset" -C "$$tmp_dir" telemt; \ test -f "$$tmp_dir/telemt"; \ - if [ "$$(id -u)" -eq 0 ]; then install_cmd="install"; else install_cmd="sudo install"; fi; \ - $$install_cmd -m 0755 "$$tmp_dir/telemt" /usr/local/bin/telemt - @telemt --version || true + if [ -w "$(TELEMT_INSTALL_DIR)" ]; then install_cmd="install"; else install_cmd="sudo install"; fi; \ + $$install_cmd -m 0755 "$$tmp_dir/telemt" "$(TELEMT_INSTALL_DIR)/telemt" + @if [ "$(TELEMT_ARCH_EFFECTIVE)" = "$(UNAME_M)" ]; then \ + "$(TELEMT_INSTALL_DIR)/telemt" --version; \ + else \ + echo "Skipping telemt --version: $(TELEMT_ARCH_EFFECTIVE) binary on a $(UNAME_M) host"; \ + fi else @echo "Unsupported operating system: $(UNAME_S)" @exit 1 diff --git a/README.md b/README.md index b93efc7a..43a62fbf 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,44 @@ The easiest way to install PasarGuard Node is using our automated installation s sudo bash -c "$(curl -sL https://github.com/PasarGuard/scripts/raw/main/pg-node.sh)" @ install ``` +# MTProto backend +Besides Xray and WireGuard, the node can serve Telegram MTProto proxy users. The panel picks the backend per core, and a node runs one backend at a time. + +The MTProto backend runs [telemt](https://github.com/telemt/telemt) as a separate process: +- The node image ships telemt 3.5.14 at `/usr/local/bin/telemt` (amd64 and arm64). The node refuses telemt older than 3.5.14. +- Set `TELEMT_EXECUTABLE_PATH` to use another binary. +- Outside Docker, `make install_telemt` installs the pinned release after checking it against the SHA256 pinned in the `Makefile`. +- telemt needs outbound access to Telegram. On start it checks the Telegram data centers, which can take up to a minute. + +## Xray and MTProto on one host +One node runs one backend, so serving Xray (or WireGuard) and MTProto from one server takes two node instances, each added to the panel as its own node: +- Each instance needs its own `SERVICE_PORT` and its own data directory (certificates and generated configs). With the installer, `--name` creates a separate instance, for example `... @ install --name pg-node-mtproto`. +- Both instances use the host network, so only one of them can listen on public port 443. Give the other one's inbound another port, for example 8443. + +With Docker Compose (trimmed, see `docker-compose.yml` for the full service): +```yaml +services: + node: # Xray or WireGuard + image: pasarguard/node:latest + network_mode: host + environment: + SERVICE_PORT: 62050 + volumes: + - /var/lib/pg-node:/var/lib/pg-node + node-mtproto: # MTProto + image: pasarguard/node:latest + network_mode: host + environment: + SERVICE_PORT: 62051 + volumes: + - /var/lib/pg-node-mtproto:/var/lib/pg-node +``` + +## Third-party software +- telemt 3.5.14, TELEMT License 3.3, Copyright (c) 2026 Telemt. Source: https://github.com/telemt/telemt/tree/3.5.14 +- The image contains the unmodified upstream release binary. The license text is in [`third_party/telemt/`](third_party/telemt/) and in the image at `/usr/share/licenses/telemt/`. +- The telemt license asks operators of a public service to credit telemt (for example in the service docs), without implying endorsement by the Telemt maintainers. + # Donation You can help PasarGuard team with your donations, [Click Here](https://donate.pasarguard.org/) diff --git a/third_party/telemt/LICENSE b/third_party/telemt/LICENSE new file mode 100644 index 00000000..6adee358 --- /dev/null +++ b/third_party/telemt/LICENSE @@ -0,0 +1,169 @@ +######## TELEMT LICENSE 3.3 ######### +##### Copyright (c) 2026 Telemt ##### + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this Software and associated documentation files (the "Software"), +to use, reproduce, modify, prepare derivative works of, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to permit +persons to whom the Software is furnished to do so, provided that all +copyright notices, license terms, and conditions set forth in this License +are preserved and complied with. + +### Official Translations + +The canonical version of this License is the English version. +Official translations are provided for informational purposes only +and for convenience, and do not have legal force. In case of any +discrepancy, the English version of this License shall prevail + +/----------------------------------------------------------\ +| Language | Location | +|-------------|--------------------------------------------| +| English | docs/LICENSE/TELEMT-LICENSE.en.md | +| German | docs/LICENSE/TELEMT-LICENSE.de.md | +| Russian | docs/LICENSE/TELEMT-LICENSE.ru.md | +\----------------------------------------------------------/ + +### License Versioning Policy + +This License is version 3 of the TELEMT Public License. +Each version of the Software is licensed under the License that +accompanies its corresponding source code distribution. + +Future versions of the Software may be distributed under a different +version of the TELEMT Public License or under a different license, +as determined by the Telemt maintainers. + +Any such change of license applies only to the versions of the +Software distributed with the new license and SHALL NOT retroactively +affect any previously released versions of the Software. + +Recipients of the Software are granted rights only under the License +provided with the version of the Software they received. + +Redistributions of the Software, including Modified Versions, MUST +preserve the copyright notices, license text, and conditions of this +License for all portions of the Software derived from Telemt. + +Additional terms or licenses may be applied to modifications or +additional code added by a redistributor, provided that such terms +do not restrict or alter the rights granted under this License for +the original Telemt Software. + +Nothing in this section limits the rights granted under this License +for versions of the Software already released. + +### Definitions + +For the purposes of this License: +- "Software" means the Telemt software, including source code, documentation, +and any associated files distributed under this License. +- "Contributor" means any person or entity that submits code, patches, +documentation, or other contributions to the Software that are accepted +into the Software by the maintainers. +- "Contribution" means any work of authorship intentionally submitted +to the Software for inclusion in the Software. +- "Modified Version" means any version of the Software that has been +changed, adapted, extended, or otherwise modified from the original +Software. +- "Maintainers" means the individuals or entities responsible for +the official Telemt project and its releases. + +#### 1 Attribution + +Redistributions of the Software, in source or binary form, MUST RETAIN the +above copyright notice, this license text, and any existing attribution +notices. + +#### 2 Modification Notice + +If you modify the Software, you MUST clearly state that the Software has been +modified and include a brief description of the changes made. + +Modified versions MUST NOT be presented as the original Telemt. + +#### 3 Trademark and Branding + +This license DOES NOT grant permission to use the name "Telemt", +the Telemt logo, or any Telemt trademarks or branding. + +Redistributed or modified versions of the Software MAY NOT use the Telemt +name in a way that suggests endorsement or official origin without explicit +permission from the Telemt maintainers. + +Use of the name "Telemt" to describe a modified version of the Software +is permitted only if the modified version is clearly identified as a +modified or unofficial version. + +Any distribution that could reasonably confuse users into believing that +the software is an official Telemt release is prohibited. + +#### 4 Binary Distribution Transparency + +If you distribute compiled binaries of the Software, +you are ENCOURAGED to provide access to the corresponding +source code and build instructions where reasonably possible. + +This helps preserve transparency and allows recipients to verify the +integrity and reproducibility of distributed builds. + +#### 5 Patent Grant and Defensive Termination Clause + +Each contributor grants you a perpetual, worldwide, non-exclusive, +no-charge, royalty-free, irrevocable patent license to make, have made, +use, offer to sell, sell, import, and otherwise transfer the Software. + +This patent license applies only to those patent claims necessarily +infringed by the contributor’s contribution alone or by combination of +their contribution with the Software. + +If you initiate or participate in any patent litigation, including +cross-claims or counterclaims, alleging that the Software or any +contribution incorporated within the Software constitutes patent +infringement, then **all rights granted to you under this license shall +terminate immediately** as of the date such litigation is filed. + +Additionally, if you initiate legal action alleging that the +Software itself infringes your patent or other intellectual +property rights, then all rights granted to you under this +license SHALL TERMINATE automatically. + +#### 6 Contributions + +Unless you explicitly state otherwise, any Contribution intentionally +submitted for inclusion in the Software shall be licensed under the terms +of this License. + +By submitting a Contribution, you grant the Telemt maintainers and all +recipients of the Software the rights described in this License with +respect to that Contribution. + +#### 7 Network Use Attribution + +If the Software is used to provide a publicly accessible network service, +the operator of such service SHOULD provide attribution to Telemt in at least +one of the following locations: + +- service documentation +- service description +- an "About" or similar informational page +- other user-visible materials reasonably associated with the service + +Such attribution MUST NOT imply endorsement by the Telemt project or its +maintainers. + +#### 8 Disclaimer of Warranty and Severability Clause + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE +USE OR OTHER DEALINGS IN THE SOFTWARE + +IF ANY PROVISION OF THIS LICENSE IS HELD TO BE INVALID OR UNENFORCEABLE, +SUCH PROVISION SHALL BE INTERPRETED TO REFLECT THE ORIGINAL INTENT +OF THE PARTIES AS CLOSELY AS POSSIBLE, AND THE REMAINING PROVISIONS +SHALL REMAIN IN FULL FORCE AND EFFECT diff --git a/third_party/telemt/README.md b/third_party/telemt/README.md new file mode 100644 index 00000000..5ed816f8 --- /dev/null +++ b/third_party/telemt/README.md @@ -0,0 +1,3 @@ +# telemt + +[telemt](https://github.com/telemt/telemt) is an MTProto proxy for Telegram, written in Rust by the Telemt maintainers; the node's MTProto backend runs it as a separate process. The node image ships the unmodified upstream release binary of telemt 3.5.14 (`telemt-x86_64-linux-musl.tar.gz` or `telemt-aarch64-linux-musl.tar.gz`, checked against the SHA256 values pinned in the node `Makefile`), and its source code and build instructions are at https://github.com/telemt/telemt/tree/3.5.14. telemt is licensed under the TELEMT License 3.3, Copyright (c) 2026 Telemt: `LICENSE` is the root license file of the 3.5.14 tag and `TELEMT-LICENSE.en.md` is the English text that file names as canonical (`docs/LICENSE/TELEMT-LICENSE.en.md`), both copied verbatim. The PasarGuard node is not a Telemt release and is not endorsed by the Telemt maintainers. diff --git a/third_party/telemt/TELEMT-LICENSE.en.md b/third_party/telemt/TELEMT-LICENSE.en.md new file mode 100644 index 00000000..74e6a44d --- /dev/null +++ b/third_party/telemt/TELEMT-LICENSE.en.md @@ -0,0 +1,120 @@ +# TELEMT License 3.3 + +***Copyright (c) 2026 Telemt*** + +Permission is hereby granted, free of charge, to any person obtaining a copy of this Software and associated documentation files (the "Software"), to use, reproduce, modify, prepare derivative works of, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, provided that all copyright notices, license terms, and conditions set forth in this License are preserved and complied with. + +### Official Translations + +The canonical version of this License is the English version. +Official translations are provided for informational purposes only and for convenience, and do not have legal force. In case of any discrepancy, the English version of this License shall prevail. + +| Language | Location | +|-------------|----------| +| English | [docs/LICENSE/TELEMT-LICENSE.en.md](https://github.com/telemt/telemt/tree/main/docs/LICENSE/TELEMT-LICENSE.en.md)| +| German | [docs/LICENSE/TELEMT-LICENSE.de.md](https://github.com/telemt/telemt/tree/main/docs/LICENSE/TELEMT-LICENSE.de.md)| +| Russian | [docs/LICENSE/TELEMT-LICENSE.ru.md](https://github.com/telemt/telemt/tree/main/docs/LICENSE/TELEMT-LICENSE.ru.md)| + +### License Versioning Policy + +This License is version 3.3 of the TELEMT License. +Each version of the Software is licensed under the License that accompanies its corresponding source code distribution. + +Future versions of the Software may be distributed under a different version of the TELEMT Public License or under a different license, as determined by the Telemt maintainers. + +Any such change of license applies only to the versions of the Software distributed with the new license and SHALL NOT retroactively affect any previously released versions of the Software. + +Recipients of the Software are granted rights only under the License provided with the version of the Software they received. + +Redistributions of the Software, including Modified Versions, MUST preserve the copyright notices, license text, and conditions of this License for all portions of the Software derived from Telemt. + +Additional terms or licenses may be applied to modifications or additional code added by a redistributor, provided that such terms do not restrict or alter the rights granted under this License for the original Telemt Software. + +Nothing in this section limits the rights granted under this License for versions of the Software already released. + +### Definitions + +For the purposes of this License: + +**"Software"** means the Telemt software, including source code, documentation, and any associated files distributed under this License. + +**"Contributor"** means any person or entity that submits code, patches, documentation, or other contributions to the Software that are accepted into the Software by the maintainers. + +**"Contribution"** means any work of authorship intentionally submitted to the Software for inclusion in the Software. + +**"Modified Version"** means any version of the Software that has been changed, adapted, extended, or otherwise modified from the original Software. + +**"Maintainers"** means the individuals or entities responsible for the official Telemt project and its releases. + +### 1 Attribution + +Redistributions of the Software, in source or binary form, MUST RETAIN: + +- the above copyright notice; +- this license text; +- any existing attribution notices. + +### 2 Modification Notice + +If you modify the Software, you MUST clearly state that the Software has been modified and include a brief description of the changes made. + +Modified versions MUST NOT be presented as the original Telemt. + +### 3 Trademark and Branding + +This license DOES NOT grant permission to use the name "Telemt", the Telemt logo, or any Telemt trademarks or branding. + +Redistributed or modified versions of the Software MAY NOT use the Telemt name in a way that suggests endorsement or official origin without explicit permission from the Telemt maintainers. + +Use of the name "Telemt" to describe a modified version of the Software is permitted only if the modified version is clearly identified as a modified or unofficial version. + +Any distribution that could reasonably confuse users into believing that the software is an official Telemt release is prohibited. + +### 4 Binary Distribution Transparency + +If you distribute compiled binaries of the Software, you are ENCOURAGED to provide access to the corresponding source code and build instructions where reasonably possible. + +This helps preserve transparency and allows recipients to verify the integrity and reproducibility of distributed builds. + +### 5 Patent Grant and Defensive Termination Clause + +Each contributor grants you a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable patent license to: + +- make, +- have made, +- use, +- offer to sell, +- sell, +- import, +- and otherwise transfer the Software. + +This patent license applies only to those patent claims necessarily infringed by the contributor’s contribution alone or by combination of their contribution with the Software. + +If you initiate or participate in any patent litigation, including cross-claims or counterclaims, alleging that the Software or any contribution incorporated within the Software constitutes patent infringement, then **all rights granted to you under this license shall terminate immediately** as of the date such litigation is filed. + +Additionally, if you initiate legal action alleging that the Software itself infringes your patent or other intellectual property rights, then all rights granted to you under this license SHALL TERMINATE automatically. + +### 6 Contributions + +Unless you explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Software shall be licensed under the terms of this License. + +By submitting a Contribution, you grant the Telemt maintainers and all recipients of the Software the rights described in this License with respect to that Contribution. + +### 7 Network Use Attribution + +If the Software is used to provide a publicly accessible network service, the operator of such service SHOULD provide attribution to Telemt in at least one of the following locations: + +- service documentation; +- service description; +- an "About" or similar informational page; +- other user-visible materials reasonably associated with the service. + +Such attribution MUST NOT imply endorsement by the Telemt project or its maintainers. + +### 8 Disclaimer of Warranty and Severability Clause + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +IF ANY PROVISION OF THIS LICENSE IS HELD TO BE INVALID OR UNENFORCEABLE, SUCH PROVISION SHALL BE INTERPRETED TO REFLECT THE ORIGINAL INTENT OF THE PARTIES AS CLOSELY AS POSSIBLE, AND THE REMAINING PROVISIONS SHALL REMAIN IN FULL FORCE AND EFFECT. \ No newline at end of file From af0fc339d22a80880a0c26ffc30749e769caed02 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 19 Oct 2026 23:53:26 +0330 Subject: [PATCH 13/40] fix(mtproto): read telemt 3.5.14's answer to a user delete telemt 3.5.14 answers DELETE /v1/users/{name} with an object, {"username": ..., "in_runtime": ...} (status 202 while the running config still has the user), not with the username as a JSON string. The node failed to decode every successful delete, logged the batch as failed and fell back to a full render and hot reload for each user removal. The fake telemt API now answers deletes like telemt 3.5.14, and a client test pins the answer. Found by the live tests against telemt 3.5.14. --- backend/mtproto/client.go | 11 ++++++++--- backend/mtproto/sync_test.go | 31 ++++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/backend/mtproto/client.go b/backend/mtproto/client.go index 5d8baeb4..802aaf7b 100644 --- a/backend/mtproto/client.go +++ b/backend/mtproto/client.go @@ -57,6 +57,11 @@ type telemtUserActiveIPs struct { ActiveIPs []string `json:"active_ips"` } +type telemtDeleteUserResponse struct { + Username string `json:"username"` + InRuntime bool `json:"in_runtime"` +} + type telemtHealth struct { Status string `json:"status"` ReadOnly bool `json:"read_only"` @@ -184,10 +189,10 @@ func (c *apiClient) PatchUser(ctx context.Context, username string, user *runtim return err } -// DeleteUser removes the user. telemt answers with the deleted username as a -// JSON string. +// DeleteUser removes the user. telemt answers with the username and whether +// its running config still has the user (status 202 then, else 200). func (c *apiClient) DeleteUser(ctx context.Context, username string) error { - _, err := doAPIRequest[string](ctx, c, http.MethodDelete, "/v1/users/"+url.PathEscape(username), nil) + _, err := doAPIRequest[telemtDeleteUserResponse](ctx, c, http.MethodDelete, "/v1/users/"+url.PathEscape(username), nil) return err } diff --git a/backend/mtproto/sync_test.go b/backend/mtproto/sync_test.go index b990fbc0..ddbd080b 100644 --- a/backend/mtproto/sync_test.go +++ b/backend/mtproto/sync_test.go @@ -373,7 +373,14 @@ func (f *fakeTelemtAPI) serve(w http.ResponseWriter, r *http.Request) { } f.editDiskLocked(func(users map[string]string) { delete(users, name) }) f.overrides[name] = fakeOverride{deleted: true} - writeFakeSuccess(w, http.StatusOK, name) + // telemt 3.5.14 answers 202 while the running config still has + // the user, 200 otherwise. + _, inRuntime := f.runtime[name] + status := http.StatusOK + if inRuntime { + status = http.StatusAccepted + } + writeFakeSuccess(w, status, map[string]any{"username": name, "in_runtime": inRuntime}) default: // A GET of one user loads the whole config in telemt; the node // must not need it. @@ -1121,3 +1128,25 @@ func TestAPIClientListUsersAndActiveIPs(t *testing.T) { t.Errorf("ActiveIPs = %v", ips) } } + +// telemt 3.5.14 answers DELETE /v1/users/{name} with an object, 202 while +// the running config still has the user. Reading it as anything else made +// every API delete look failed and fall back to a full render. +func TestAPIClientDeleteUserReadsTelemtAnswer(t *testing.T) { + api := newFakeTelemtAPI(t) + api.set(func(f *fakeTelemtAPI) { + f.editDiskLocked(func(users map[string]string) { + users["u1"] = testSecret + users["u2"] = testSecret + }) + f.runtime = map[string]string{"u1": testSecret, "u2": testSecret} + }) + client := newAPIClient(api.server.URL, "Bearer test") + + if err := client.DeleteUser(context.Background(), "u1"); err != nil { + t.Fatalf("DeleteUser: %v", err) + } + if _, onDisk := parseConfigUsers(readConfig(t, api.configPath))["u1"]; onDisk { + t.Error("u1 is still in the config file") + } +} From 1f79fb5c065fc44f1d2d1f4001f032bcdd035783 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 19 Oct 2026 23:55:09 +0330 Subject: [PATCH 14/40] fix(mtproto): reject a render telemt cannot read without waiting 30 s telemt reads the config file and its includes before it parses them, and answers GET /v1/users with 500 "failed to read config graph: ..." when that read fails, for example on a rendered file above its 8 MiB limit per config file. The node only took "failed to load config" (a parse error) as a rejection, so a read failure kept the sync waiting the whole markerConfirmTimeout under the sync lock before the restore. Both answers now reject the render at once with telemt's message. Found by the live tests against telemt 3.5.14. --- backend/mtproto/reload.go | 18 ++++++++++-- backend/mtproto/sync_test.go | 56 ++++++++++++++++++++++-------------- 2 files changed, 50 insertions(+), 24 deletions(-) diff --git a/backend/mtproto/reload.go b/backend/mtproto/reload.go index 1097aa23..fc21215f 100644 --- a/backend/mtproto/reload.go +++ b/backend/mtproto/reload.go @@ -90,9 +90,9 @@ func (m *MTProto) waitForMarker(ctx context.Context, marker string) error { } case errors.As(err, &apiErr): unanswered = 0 - // telemt reads the config file for this list: it cannot parse - // the render. - if apiErr.StatusCode >= http.StatusInternalServerError && strings.Contains(apiErr.Message, "failed to load config") { + // telemt reads the config file for this list: it cannot read or + // parse the render. + if cannotLoadConfig(apiErr) { return fmt.Errorf("%w: %s", errReloadRejected, apiErr.Message) } case waitCtx.Err() == nil: @@ -127,6 +127,18 @@ func (m *MTProto) reloadRejectedError() error { return fmt.Errorf("%w within %s; recent telemt output:\n%s", errReloadRejected, markerConfirmTimeout, strings.Join(tail, "\n")) } +// cannotLoadConfig reports whether telemt answered that it cannot use its +// config file: it reads the file and its includes first ("failed to read +// config graph", which also enforces telemt's 8 MiB limit per file), then +// parses them ("failed to load config"). +func cannotLoadConfig(apiErr *apiStatusError) bool { + if apiErr.StatusCode < http.StatusInternalServerError { + return false + } + return strings.Contains(apiErr.Message, "failed to read config graph") || + strings.Contains(apiErr.Message, "failed to load config") +} + func userInRuntime(users []telemtUserInfo, username string) bool { for _, user := range users { if user.Username == username { diff --git a/backend/mtproto/sync_test.go b/backend/mtproto/sync_test.go index ddbd080b..90e58ee9 100644 --- a/backend/mtproto/sync_test.go +++ b/backend/mtproto/sync_test.go @@ -61,8 +61,8 @@ type fakeTelemtAPI struct { // rejectedConfig is the last changed config file rejectReload kept out. rejectedConfig string - // loadError makes reads of the config file fail like a file telemt - // cannot parse: 500 "failed to load config: ". + // loadError makes every users route fail like telemt when it cannot + // read or parse the config file: 500 with this message. loadError string // dropUsersRequests closes this many GET /v1/users requests without an // answer. @@ -313,7 +313,11 @@ func (f *fakeTelemtAPI) serve(w http.ResponseWriter, r *http.Request) { // Every users route loads the config file first. data, err := os.ReadFile(f.configPath) if f.loadError != "" || err != nil { - writeFakeError(w, http.StatusInternalServerError, "internal_error", "failed to load config: "+f.loadError) + message := f.loadError + if message == "" { + message = "failed to load config: " + err.Error() + } + writeFakeError(w, http.StatusInternalServerError, "internal_error", message) return } disk := parseConfigUsers(string(data)) @@ -744,26 +748,36 @@ func TestRejectedHotReloadRestoresPreviousUsers(t *testing.T) { } // TestLoadFailureRejectsAtOnce: telemt cannot load the rendered file, so GET -// /v1/users answers 500 "failed to load config"; that is a rejection with -// telemt's message, without waiting out markerConfirmTimeout. +// /v1/users answers 500 when telemt cannot read or parse the rendered file; +// that is a rejection with telemt's message, without waiting out +// markerConfirmTimeout. func TestLoadFailureRejectsAtOnce(t *testing.T) { - api := newFakeTelemtAPI(t) - m := newSyncTestBackend(t, api, []*common.User{mtUser("u1", "mtproto", testSecret)}) - api.set(func(f *fakeTelemtAPI) { f.loadError = "TOML parse error at line 3" }) + for name, message := range map[string]string{ + "parse": "failed to load config: TOML parse error at line 3", + // telemt 3.5.14 checks its 8 MiB source limit while it reads the + // config graph, before it parses anything. + "read": "failed to read config graph: Config error: file exceeds configured size limit", + } { + t.Run(name, func(t *testing.T) { + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{mtUser("u1", "mtproto", testSecret)}) + api.set(func(f *fakeTelemtAPI) { f.loadError = message }) - started := time.Now() - err := m.SyncUsers(context.Background(), []*common.User{mtUser("u2", "mtproto", testSecret)}) - if !errors.Is(err, errReloadRejected) || !strings.Contains(err.Error(), "TOML parse error at line 3") { - t.Fatalf("SyncUsers = %v, want a rejection with telemt's message", err) - } - if elapsed := time.Since(started); elapsed > 5*time.Second { - t.Errorf("the rejection took %s", elapsed) - } - if got := desiredNames(m); !slices.Equal(got, []string{"u1"}) { - t.Errorf("desired = %q, want u1", got) - } - if users := parseConfigUsers(readConfig(t, api.configPath)); users["u1"] == "" || users["u2"] != "" { - t.Errorf("the previous users were not rendered again: %v", users) + started := time.Now() + err := m.SyncUsers(context.Background(), []*common.User{mtUser("u2", "mtproto", testSecret)}) + if !errors.Is(err, errReloadRejected) || !strings.Contains(err.Error(), message) { + t.Fatalf("SyncUsers = %v, want a rejection with telemt's message", err) + } + if elapsed := time.Since(started); elapsed > 5*time.Second { + t.Errorf("the rejection took %s", elapsed) + } + if got := desiredNames(m); !slices.Equal(got, []string{"u1"}) { + t.Errorf("desired = %q, want u1", got) + } + if users := parseConfigUsers(readConfig(t, api.configPath)); users["u1"] == "" || users["u2"] != "" { + t.Errorf("the previous users were not rendered again: %v", users) + } + }) } } From c67617e4c9ff1b4d47ff69108e92954f360c1e95 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Tue, 20 Oct 2026 00:02:31 +0330 Subject: [PATCH 15/40] fix(mtproto): do not SIGHUP telemt before it handles the signal telemt installs its SIGHUP handler in its config watcher, which waits for the runtime to activate after the API is up, reads the whole config and sets up its file watches first. A SIGHUP in that window ends telemt with the default action. The node assumed the opposite order, so a sync right after a start (or a restore after a quick rejection) could kill telemt: the reload then found the API gone, restarted telemt and dropped every session. For telemtSignalGrace (5 s) after telemt's API first answers, the node now sends no SIGHUP. The watcher loads most renders of that time by itself, but misses one written while it sets up its watches, so a reload whose marker telemt has not loaded when the grace ends gets its SIGHUP then. The fake telemt can now play the watcher, including that blind spot, and a late SIGHUP handler. Found by the live tests against telemt 3.5.14, where nine telemts started at once made the window wide. --- backend/mtproto/fake_telemt_test.go | 65 +++++++++++++++++++++++++++-- backend/mtproto/lifecycle.go | 1 + backend/mtproto/lifecycle_test.go | 57 +++++++++++++++++++++++++ backend/mtproto/mtproto.go | 3 ++ backend/mtproto/reload.go | 65 ++++++++++++++++++++--------- backend/mtproto/sync.go | 2 +- 6 files changed, 169 insertions(+), 24 deletions(-) diff --git a/backend/mtproto/fake_telemt_test.go b/backend/mtproto/fake_telemt_test.go index f9a7d670..82440009 100644 --- a/backend/mtproto/fake_telemt_test.go +++ b/backend/mtproto/fake_telemt_test.go @@ -54,6 +54,17 @@ type fakeTelemt struct { DropUsers bool `json:"drop_users"` // SignalFile gets one line per SIGHUP received. SignalFile string `json:"signal_file"` + // SignalDelay makes the fake handle SIGHUP only this long after it binds + // its API, like telemt, whose config watcher installs the handler after + // the API is up; a SIGHUP before that ends the fake. + SignalDelay time.Duration `json:"signal_delay"` + // WatchConfig plays telemt's config watcher. Once the API is up it reads + // the config file, spends WatchSetup setting up its watches, then loads + // the file if that first read differed from the running config, and + // every later change. Like telemt's, it misses a change made while it + // sets up: the watches start from the file as it is then. + WatchConfig bool `json:"watch_config"` + WatchSetup time.Duration `json:"watch_setup"` // Lifetime bounds the fake so a broken test never leaves it running. Lifetime time.Duration `json:"lifetime"` } @@ -95,7 +106,14 @@ func runFakeTelemt(optionsPath string, args []string) int { } hup := make(chan os.Signal, 1) - signal.Notify(hup, syscall.SIGHUP) + if f.SignalDelay > 0 { + go func() { + time.Sleep(f.HealthDelay + f.SignalDelay) + signal.Notify(hup, syscall.SIGHUP) + }() + } else { + signal.Notify(hup, syscall.SIGHUP) + } var configPath string if len(args) > 0 { @@ -138,6 +156,9 @@ func runFakeTelemt(optionsPath string, args []string) int { if !f.NoAPI && configPath != "" { go serveFakeTelemtAPI(f, configPath, runtime) } + if f.WatchConfig && configPath != "" { + go watchFakeTelemtConfig(f, configPath, runtime) + } lifetime := f.Lifetime if lifetime <= 0 { @@ -167,8 +188,9 @@ func runFakeTelemt(optionsPath string, args []string) int { // fakeTelemtRuntime holds the users of the fake's running config. type fakeTelemtRuntime struct { - mu sync.Mutex - users map[string]string + mu sync.Mutex + users map[string]string + content string // the config file content users were loaded from } // load reads the users of the config file into the running config, as a @@ -181,6 +203,33 @@ func (r *fakeTelemtRuntime) load(configPath string) { r.mu.Lock() defer r.mu.Unlock() r.users = parseConfigUsers(string(data)) + r.content = string(data) +} + +func (r *fakeTelemtRuntime) loaded() string { + r.mu.Lock() + defer r.mu.Unlock() + return r.content +} + +// watchFakeTelemtConfig plays telemt's config watcher; see +// fakeTelemt.WatchConfig. +func watchFakeTelemtConfig(f fakeTelemt, configPath string, runtime *fakeTelemtRuntime) { + time.Sleep(f.HealthDelay) + first, _ := os.ReadFile(configPath) + time.Sleep(f.WatchSetup) + watched, _ := os.ReadFile(configPath) + if string(first) != runtime.loaded() { + runtime.load(configPath) + } + for { + time.Sleep(50 * time.Millisecond) + current, err := os.ReadFile(configPath) + if err == nil && string(current) != string(watched) { + watched = current + runtime.load(configPath) + } + } } func (r *fakeTelemtRuntime) has(username string) bool { @@ -364,3 +413,13 @@ func lowerStartupTimeout(t *testing.T, timeout time.Duration) { telemtStartupTimeout = timeout t.Cleanup(func() { telemtStartupTimeout = previous }) } + +// lowerSignalGrace sets telemtSignalGrace for one test, so a fake telemt +// without a config watcher gets the SIGHUP of a sync right after its start. +// Register it before the backend's Shutdown cleanup. +func lowerSignalGrace(t *testing.T, grace time.Duration) { + t.Helper() + previous := telemtSignalGrace + telemtSignalGrace = grace + t.Cleanup(func() { telemtSignalGrace = previous }) +} diff --git a/backend/mtproto/lifecycle.go b/backend/mtproto/lifecycle.go index af51a48f..915a7529 100644 --- a/backend/mtproto/lifecycle.go +++ b/backend/mtproto/lifecycle.go @@ -199,6 +199,7 @@ func (m *MTProto) startLocked(ctx context.Context) error { m.process.Stop() return m.startupError(err) } + m.healthyAt = time.Now() // A new telemt has fresh user admission state: no API write is pending. m.apiSent = nil diff --git a/backend/mtproto/lifecycle_test.go b/backend/mtproto/lifecycle_test.go index 6a1af729..7b5f78e9 100644 --- a/backend/mtproto/lifecycle_test.go +++ b/backend/mtproto/lifecycle_test.go @@ -370,6 +370,7 @@ func TestCancelledSyncKeepsTelemtRunning(t *testing.T) { // and returns once the render's marker user is in telemt's running config. // This fake has no config watcher, so only the signal can confirm it. func TestFullSyncSignalsTelemt(t *testing.T) { + lowerSignalGrace(t, 0) signals := filepath.Join(t.TempDir(), "signals") exe := writeFakeTelemt(t, fakeTelemt{ServeUsers: true, SignalFile: signals}) mt := newTestBackend(t, testNodeConfig(t, exe), []*common.User{mtUser("alice", "mtproto", testSecret)}) @@ -394,6 +395,62 @@ func TestFullSyncSignalsTelemt(t *testing.T) { // answer (markerUnreachablePolls polls in a row) while the node waits for a // reload, the node restarts telemt on the new config, and that start // confirms it. +// telemt installs its SIGHUP handler with its config watcher, after its API +// is up, and a SIGHUP before that ends it. A sync right after the start must +// leave the reload to the watcher instead of killing telemt. +func TestSyncRightAfterStartDoesNotSignalTelemt(t *testing.T) { + signals := filepath.Join(t.TempDir(), "signals") + exe := writeFakeTelemt(t, fakeTelemt{ServeUsers: true, WatchConfig: true, SignalDelay: time.Second, SignalFile: signals}) + mt := newTestBackend(t, testNodeConfig(t, exe), []*common.User{mtUser("alice", "mtproto", testSecret)}) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := mt.SyncUsers(ctx, []*common.User{mtUser("bob", "mtproto", testSecret)}); err != nil { + t.Fatalf("SyncUsers: %v", err) + } + if !mt.Started() || mt.process.Generation() != 1 { + t.Errorf("telemt was restarted (started %v, generation %d)", mt.Started(), mt.process.Generation()) + } + if data, _ := os.ReadFile(signals); len(data) != 0 { + t.Errorf("telemt got a SIGHUP right after its start (signal log %q)", data) + } + if got := desiredNames(mt); !slices.Equal(got, []string{"bob"}) { + t.Errorf("desired = %q, want bob", got) + } +} + +// telemt's config watcher misses a render written while it sets itself up +// right after the start, and only SIGHUP makes telemt load it then. The +// node sends it once telemtSignalGrace has passed. +func TestRenderMissedByWatcherGetsSIGHUPAfterGrace(t *testing.T) { + lowerSignalGrace(t, 1500*time.Millisecond) + lowerMarkerConfirmTimeout(t, 10*time.Second) + signals := filepath.Join(t.TempDir(), "signals") + exe := writeFakeTelemt(t, fakeTelemt{ + ServeUsers: true, + WatchConfig: true, + WatchSetup: 600 * time.Millisecond, + SignalDelay: 700 * time.Millisecond, + SignalFile: signals, + }) + mt := newTestBackend(t, testNodeConfig(t, exe), []*common.User{mtUser("alice", "mtproto", testSecret)}) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := mt.SyncUsers(ctx, []*common.User{mtUser("bob", "mtproto", testSecret)}); err != nil { + t.Fatalf("SyncUsers: %v", err) + } + if !mt.Started() || mt.process.Generation() != 1 { + t.Errorf("telemt was restarted (started %v, generation %d)", mt.Started(), mt.process.Generation()) + } + if data, _ := os.ReadFile(signals); strings.Count(string(data), "SIGHUP") != 1 { + t.Errorf("signal log %q, want one SIGHUP", data) + } + if got := desiredNames(mt); !slices.Equal(got, []string{"bob"}) { + t.Errorf("desired = %q, want bob", got) + } +} + func TestUsersAPIUnreachableRestartsTelemt(t *testing.T) { exe := writeFakeTelemt(t, fakeTelemt{DropUsers: true}) mt := newTestBackend(t, testNodeConfig(t, exe), []*common.User{mtUser("alice", "mtproto", testSecret)}) diff --git a/backend/mtproto/mtproto.go b/backend/mtproto/mtproto.go index 96afd969..d57eec37 100644 --- a/backend/mtproto/mtproto.go +++ b/backend/mtproto/mtproto.go @@ -38,6 +38,9 @@ type MTProto struct { // apiSent holds the users written through telemt's user API since the // last confirmed render or telemt start. Guarded by syncMu. apiSent map[string]apiWrite + // healthyAt is when the running telemt's API first answered. Guarded by + // syncMu. + healthyAt time.Time // stateMu guards desired and marker for readers; writers also hold // syncMu. desired holds only users telemt confirmed it runs. marker is diff --git a/backend/mtproto/reload.go b/backend/mtproto/reload.go index fc21215f..a0c2840c 100644 --- a/backend/mtproto/reload.go +++ b/backend/mtproto/reload.go @@ -19,6 +19,14 @@ const ( markerUnreachablePolls = 3 ) +// telemtSignalGrace is how long after telemt's API first answers the node +// does not send SIGHUP. telemt's config watcher installs the SIGHUP handler +// only after the API is up and the watcher has read the config and set up +// its watches, and a SIGHUP before that ends telemt. The watcher loads most +// renders of that time on its own, but misses one written while it sets up. +// It is a variable so tests can lower it. +var telemtSignalGrace = 5 * time.Second + // markerConfirmTimeout bounds the wait for telemt to load a rendered config; // a config it did not load by then counts as rejected. It is a variable so // tests can lower it. @@ -28,25 +36,21 @@ var markerConfirmTimeout = 30 * time.Second // keeps running the config it had. var errReloadRejected = errors.New("telemt did not load the new config") -// reloadLocked makes the running telemt hot-reload the rendered config: -// SIGHUP (telemt's config watcher also sees the file change), then a wait -// until the render's marker user is in telemt's running config. A hot reload -// replaces only the users that changed; an API reload would cut every -// session. Only an API that stays unreachable, or a telemt that exited or -// cannot be signalled, restarts telemt, on the rendered config; a rejected -// config never does. Requires syncMu. +// reloadLocked makes the running telemt hot-reload the rendered config and +// waits until the render's marker user is in telemt's running config. +// telemt's config watcher sees the file change, and SIGHUP makes sure once +// telemtSignalGrace allows it (see waitForMarker). A hot reload replaces +// only the users that changed; an API reload would cut every session. Only +// an API that stays unreachable, or a telemt that exited or cannot be +// signalled, restarts telemt, on the rendered config; a rejected config +// never does. Requires syncMu. func (m *MTProto) reloadLocked(ctx context.Context) error { if err := ctx.Err(); err != nil { return err } - // telemt installs its SIGHUP handler in its config watcher, which starts - // before the API the node waited for at startup, so a healthy telemt - // handles the signal. Should one not, SIGHUP ends it and the restart - // below recovers it. - err := m.process.Reload() - if err == nil { - err = m.waitForMarker(ctx, m.marker.Username) - } + // Should telemt not handle the signal after all, SIGHUP ends it and the + // restart below recovers it. + err := m.waitForMarker(ctx, m.marker.Username, m.healthyAt.Add(telemtSignalGrace)) switch { case err == nil || errors.Is(err, errClosed) || errors.Is(err, errReloadRejected): return err @@ -63,12 +67,25 @@ func (m *MTProto) reloadLocked(ctx context.Context) error { return nil } +// signalReloadLocked sends telemt SIGHUP, unless its API first answered +// less than telemtSignalGrace ago; telemt's config watcher sees the change +// then. Requires syncMu. +func (m *MTProto) signalReloadLocked() error { + if time.Since(m.healthyAt) < telemtSignalGrace { + return nil + } + return m.process.Reload() +} + // waitForMarker polls GET /v1/users until telemt's running config has the -// marker user, for up to markerConfirmTimeout. It returns an errReloadRejected -// error when telemt cannot read the rendered file or the time is up, -// errClosed on Shutdown, ctx's error when ctx ends, and the request error -// when the API stays unreachable or telemt exited. -func (m *MTProto) waitForMarker(ctx context.Context, marker string) error { +// marker user, for up to markerConfirmTimeout, and sends telemt SIGHUP once +// signalAt has passed: at once in steady state, or when telemtSignalGrace +// ends if the watcher has not loaded the render by then. It returns an +// errReloadRejected error when telemt cannot read the rendered file or the +// time is up, errClosed on Shutdown, ctx's error when ctx ends, and the +// signal or request error when telemt cannot be signalled, the API stays +// unreachable or telemt exited. +func (m *MTProto) waitForMarker(ctx context.Context, marker string, signalAt time.Time) error { waitCtx, cancel := context.WithTimeout(ctx, markerConfirmTimeout) defer cancel() stop := context.AfterFunc(m.lifetime, cancel) @@ -78,8 +95,16 @@ func (m *MTProto) waitForMarker(ctx context.Context, marker string) error { ticker := time.NewTicker(markerPollInterval) defer ticker.Stop() + signalled := false unanswered := 0 for { + if !signalled && !time.Now().Before(signalAt) { + signalled = true + if err := m.process.Reload(); err != nil { + return err + } + } + users, err := m.apiClient.ListUsers(waitCtx) var apiErr *apiStatusError switch { diff --git a/backend/mtproto/sync.go b/backend/mtproto/sync.go index 91f09879..bd063cf2 100644 --- a/backend/mtproto/sync.go +++ b/backend/mtproto/sync.go @@ -288,7 +288,7 @@ func (m *MTProto) restoreLocked(ctx context.Context) error { if err := m.writeRenderedConfigLocked(m.desired); err != nil { return err } - if err := m.process.Reload(); err != nil { + if err := m.signalReloadLocked(); err != nil { log.Printf("warning: %v", err) } return nil From f69d55926510c77629878780825ceb4a133ca77c Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Tue, 20 Oct 2026 00:10:28 +0330 Subject: [PATCH 16/40] test(mtproto): cover telemt 3.5.14 lifecycle, reload and removal end to end The integration tests (run when MTPROTO_TELEMT_BIN names a telemt binary) now check what telemt does, not only what its config file says. A fake MTProto client performs the obfuscated2 handshake with a user's secret, and telemt's per-user connection gauge tells whether it admitted the client. telemt's dc_overrides point the client's DC at a local listener, so sessions stay open without Telegram. Covered live: start, version 3.5.14, health and stats, then Shutdown removing config.toml and leaving no telemt process; user create, secret change and delete through the user API; full syncs down to the marker alone and back on the same process, with the watcher and with SIGHUP; unchanged users keeping their sessions across a hot reload while removed users lose theirs; API deletes cutting sessions, including the last real user; a start without users; a secret written through the API and changed by a render before telemt published it; a symlinked generated path; and a render above telemt's 8 MiB limit, rejected at once with telemt's reason while the previous users and their sessions stay. The tests run in parallel, each with its own telemt. --- backend/mtproto/integration_test.go | 649 +++++++++++++++++++++---- backend/mtproto/mtproto_client_test.go | 143 ++++++ 2 files changed, 703 insertions(+), 89 deletions(-) create mode 100644 backend/mtproto/mtproto_client_test.go diff --git a/backend/mtproto/integration_test.go b/backend/mtproto/integration_test.go index 76e6fd30..26ff20ed 100644 --- a/backend/mtproto/integration_test.go +++ b/backend/mtproto/integration_test.go @@ -2,21 +2,68 @@ package mtproto import ( "context" + "errors" "fmt" + "net" "os" + "path/filepath" + "slices" + "strings" "testing" "time" + gopsutilprocess "github.com/shirou/gopsutil/v4/process" + "github.com/pasarguard/node/common" "github.com/pasarguard/node/config" "github.com/pasarguard/node/pkg/netutil" ) -// Gated integration test: runs only when MTPROTO_TELEMT_BIN points at a real -// telemt binary. Exercises the full lifecycle (start, health, users, stats, -// marker user/empty-users) against the actual proxy. +// Integration tests against a real telemt binary. They run only when +// MTPROTO_TELEMT_BIN points at one: +// +// MTPROTO_TELEMT_BIN=/path/to/telemt go test ./backend/mtproto/ -run Integration -v -timeout 30m // -// MTPROTO_TELEMT_BIN=/path/to/telemt go test ./backend/mtproto/ -run Integration -v +// telemt binds its API only after a STUN probe and a Telegram DC check, which +// take up to about a minute on a host with slow or unreachable DCs, so each +// test runs telemt in parallel with the others and allows it +// integrationTimeout. Clients are fake (dialMTProto), and their DC is a local +// listener (startFakeDC), so no test needs Telegram. + +// integrationTimeout bounds one integration test, startup included. +const integrationTimeout = 150 * time.Second + +const ( + // admitWait is how long a handshake may take to show up in telemt's + // per-user connection gauge; telemt counts it within milliseconds. + admitWait = 5 * time.Second + // refuseWait is how long a refused handshake must stay uncounted. + refuseWait = 3 * time.Second + // liveWait bounds waits for telemt to act on a change: a socket it + // closes, a user it publishes. + liveWait = 10 * time.Second +) + +// Secrets of integration users, distinct so that a handshake names one user. +const ( + secretAlice = testSecret + secretBob = otherSecret + secretCarol = thirdSecret + secretDave = "11111111111111111111111111111111" + secretErin = "22222222222222222222222222222222" + secretOld = "33333333333333333333333333333333" + secretNew = "44444444444444444444444444444444" + secretFinal = "55555555555555555555555555555555" +) + +// liveTelemt is a backend running the real telemt. +type liveTelemt struct { + t *testing.T + ctx context.Context + mt *MTProto + addr string // the MTProxy listener +} + func telemtBinOrSkip(t *testing.T) string { t.Helper() bin := os.Getenv("MTPROTO_TELEMT_BIN") @@ -26,137 +73,561 @@ func telemtBinOrSkip(t *testing.T) string { return bin } -func integrationConfig(t *testing.T, bin string) *config.Config { - return &config.Config{ - TelemtExecutablePath: bin, - GeneratedConfigPath: t.TempDir(), - LogBufferSize: 1000, +// startLiveTelemt starts a backend on the real telemt with users, rendering +// into generatedPath (a new temp dir when empty). The test runs in parallel +// with the other integration tests. +func startLiveTelemt(t *testing.T, generatedPath string, users ...*common.User) *liveTelemt { + t.Helper() + bin := telemtBinOrSkip(t) + t.Parallel() + + ctx, cancel := context.WithTimeout(context.Background(), integrationTimeout) + t.Cleanup(cancel) + if generatedPath == "" { + generatedPath = t.TempDir() } -} -func integrationMTConfig(t *testing.T) *Config { port := netutil.FindFreePort() - raw := fmt.Sprintf(`{ + mtConfig, err := NewConfig(fmt.Sprintf(`{ "inbound_tag": "mtproto", "general": {"use_middle_proxy": false, "modes": {"classic": true, "secure": true, "tls": false}}, - "server": {"port": %d, "listeners": [{"ip": "127.0.0.1"}]} - }`, port) - cfg, err := NewConfig(raw) + "server": {"port": %d, "listeners": [{"ip": "127.0.0.1"}]}, + "dc_overrides": {"%d": %q} + }`, port, mtprotoTestDC, startFakeDC(t))) if err != nil { t.Fatalf("NewConfig: %v", err) } - return cfg + nodeConfig := &config.Config{ + TelemtExecutablePath: bin, + GeneratedConfigPath: generatedPath, + LogBufferSize: 1000, + } + + started := time.Now() + mt, err := New(ctx, nodeConfig, mtConfig, users) + if err != nil { + t.Fatalf("New: %v", err) + } + t.Cleanup(mt.Shutdown) + t.Logf("telemt %s ready after %s", mt.Version(), time.Since(started).Round(time.Millisecond)) + + return &liveTelemt{t: t, ctx: ctx, mt: mt, addr: fmt.Sprintf("127.0.0.1:%d", port)} } -// waitTelemtUser waits up to 5 s until telemt's running config has username -// (present) or lacks it. GET /v1/users lists the users in the config file; -// in_runtime tells whether the running config has them too. -func waitTelemtUser(ctx context.Context, t *testing.T, mt *MTProto, username string, present bool) { - t.Helper() +// users returns GET /v1/users by username: the users of the config file, +// with InRuntime telling whether telemt's running config has them too. +func (l *liveTelemt) users() map[string]telemtUserInfo { + l.t.Helper() + listed, err := l.mt.apiClient.ListUsers(l.ctx) + if err != nil { + l.t.Fatalf("ListUsers: %v", err) + } + users := make(map[string]telemtUserInfo, len(listed)) + for _, user := range listed { + users[user.Username] = user + } + return users +} + +// waitListed waits until the config file and the running config both have +// username, or the config file lacks it. A user missing from the file may +// still run, so absence must also be checked with expectRefused. +func (l *liveTelemt) waitListed(username string, listed bool) { + l.t.Helper() + waitFor(l.t, liveWait, fmt.Sprintf("telemt to list %s = %v", username, listed), func() bool { + user, found := l.users()[username] + if listed { + return found && user.InRuntime + } + return !found + }) +} + +// connections reads telemt_user_connections_current for username. +func (l *liveTelemt) connections(username string) int64 { + l.t.Helper() + users, _, err := l.mt.metricsScraper.Scrape(l.ctx) + if err != nil { + l.t.Fatalf("scrape telemt metrics: %v", err) + } + return users[username].ConnectionsCurrent +} + +func (l *liveTelemt) waitConnections(username string, want int64) { + l.t.Helper() + waitFor(l.t, liveWait, fmt.Sprintf("%d telemt connections of %s", want, username), func() bool { + return l.connections(username) == want + }) +} - deadline := time.Now().Add(5 * time.Second) +func (l *liveTelemt) dial(secret string) net.Conn { + l.t.Helper() + conn, err := dialMTProto(l.addr, secret) + if err != nil { + l.t.Fatalf("dial telemt: %v", err) + } + l.t.Cleanup(func() { _ = conn.Close() }) + return conn +} + +// expectAdmitted opens a client session of username with secret and waits +// until telemt counts it, redialling while a change telemt confirmed is +// still being published to its handshake path. It returns the open session. +func (l *liveTelemt) expectAdmitted(username, secret string) net.Conn { + l.t.Helper() + deadline := time.Now().Add(liveWait) for { - users, err := mt.apiClient.ListUsers(ctx) - if err != nil { - t.Fatalf("ListUsers: %v", err) + before := l.connections(username) + conn := l.dial(secret) + admitted := waitUntil(admitWait, func() bool { return l.connections(username) > before }) + if admitted && !connClosed(conn, 100*time.Millisecond) { + return conn } - inRuntime := false - for _, user := range users { - if user.Username == username && user.InRuntime { - inRuntime = true - } + _ = conn.Close() + if time.Now().After(deadline) { + l.t.Fatalf("telemt did not admit %s with secret %s", username, secret) } - if inRuntime == present { + } +} + +// expectRefused checks that telemt stops admitting clients of username with +// secret within liveWait: a change telemt confirmed may take a moment to +// reach its handshake path. telemt keeps a refused socket open (it relays +// it to the mask host), so only its connection gauge tells. +func (l *liveTelemt) expectRefused(username, secret string) { + l.t.Helper() + deadline := time.Now().Add(liveWait) + for { + before := l.connections(username) + conn := l.dial(secret) + admitted := waitUntil(refuseWait, func() bool { return l.connections(username) > before }) + _ = conn.Close() + if !admitted { return } if time.Now().After(deadline) { - t.Fatalf("telemt runtime has %s = %v, want %v", username, inRuntime, present) + l.t.Fatalf("telemt still admits %s with secret %s", username, secret) + } + l.waitConnections(username, before) + } +} + +// expectCut waits until telemt closed session, a client of username, and no +// longer counts any connection of username. +func (l *liveTelemt) expectCut(session net.Conn, username string) { + l.t.Helper() + waitFor(l.t, liveWait, fmt.Sprintf("telemt to close the session of %s", username), func() bool { + return connClosed(session, 100*time.Millisecond) + }) + l.waitConnections(username, 0) +} + +// expectOpen checks that session is still open and that telemt counts +// connections sessions of username. +func (l *liveTelemt) expectOpen(session net.Conn, username string, connections int64) { + l.t.Helper() + if connClosed(session, time.Second) { + l.t.Fatalf("telemt closed the session of %s", username) + } + if got := l.connections(username); got != connections { + l.t.Fatalf("telemt counts %d connections of %s, want %d", got, username, connections) + } +} + +// expectNewMarker checks that the latest render carries a marker other than +// previous and that telemt runs it, and returns it. +func (l *liveTelemt) expectNewMarker(previous string) string { + l.t.Helper() + marker := l.mt.markerUsername() + if !markerUsernamePattern.MatchString(marker) || marker == previous { + l.t.Fatalf("marker after a render = %q, previous %q", marker, previous) + } + l.waitListed(marker, true) + return marker +} + +func (l *liveTelemt) expectPID(want int) { + l.t.Helper() + if got := l.mt.process.PID(); got != want { + l.t.Fatalf("telemt pid = %d, want %d (telemt was restarted)", got, want) + } +} + +// passSignalGrace waits until telemtSignalGrace has passed since telemt's +// API first answered, so the next reload sends SIGHUP instead of leaving +// the change to telemt's config watcher. +func (l *liveTelemt) passSignalGrace() { + l.t.Helper() + l.mt.syncMu.Lock() + healthyAt := l.mt.healthyAt + l.mt.syncMu.Unlock() + time.Sleep(time.Until(healthyAt.Add(telemtSignalGrace))) +} + +// sighups counts the SIGHUP reloads telemt logged. +func (l *liveTelemt) sighups() int { + count := 0 + for _, line := range logLines(l.mt) { + if strings.Contains(line, "SIGHUP received") { + count++ + } + } + return count +} + +// waitUntil polls cond every 100 ms for up to timeout and reports whether +// it held. +func waitUntil(timeout time.Duration, cond func() bool) bool { + deadline := time.Now().Add(timeout) + for { + if cond() { + return true + } + if time.Now().After(deadline) { + return false } time.Sleep(100 * time.Millisecond) } } -func TestIntegration_LifecycleAndUsers(t *testing.T) { - bin := telemtBinOrSkip(t) +func TestIntegrationLifecycle(t *testing.T) { + l := startLiveTelemt(t, "", mtUser("alice", "mtproto", secretAlice)) + mt := l.mt - // telemt binds its API only after the Telegram DC check (up to about 60 s - // with slow DCs), and New allows it telemtStartupTimeout. - ctx, cancel := context.WithTimeout(context.Background(), 150*time.Second) - defer cancel() + if got := mt.Version(); got != "3.5.14" { + t.Errorf("Version() = %q, want 3.5.14", got) + } + if !mt.Started() { + t.Fatal("Started() = false") + } + if err := mt.apiClient.Health(l.ctx); err != nil { + t.Fatalf("telemt health: %v", err) + } + pid := mt.process.PID() + if pid == 0 { + t.Fatal("no telemt pid") + } - users := []*common.User{mtUser("alice", "mtproto", testSecret)} - mt, err := New(ctx, integrationConfig(t, bin), integrationMTConfig(t), users) + l.waitListed("alice", true) + l.expectAdmitted("alice", secretAlice) + + // Stats read the live telemt: one client address, the client's uplink + // bytes, then nothing new after a reset. + if online, err := mt.GetUserOnlineStats(l.ctx, "alice"); err != nil || online.GetValue() != 1 { + t.Errorf("GetUserOnlineStats(alice) = %v, %v; want 1 unique IP", online, err) + } + ips, err := mt.GetUserOnlineIpListStats(l.ctx, "alice") if err != nil { - t.Fatalf("New: %v", err) + t.Fatalf("GetUserOnlineIpListStats: %v", err) + } + if _, found := ips.GetIps()["127.0.0.1"]; !found || len(ips.GetIps()) != 1 { + t.Errorf("GetUserOnlineIpListStats(alice) = %v, want only 127.0.0.1", ips.GetIps()) + } + var uplink int64 + waitFor(t, liveWait, "alice's uplink in the user stats", func() bool { + stats, err := mt.GetStats(l.ctx, &common.StatRequest{Type: common.StatType_UsersStat, Reset_: true}) + if err != nil { + t.Fatalf("GetStats: %v", err) + } + for _, stat := range stats.GetStats() { + if stat.GetName() == "alice" && stat.GetType() == "uplink" && stat.GetLink() == "mtproto" { + uplink += stat.GetValue() + } + } + return uplink > 0 + }) + time.Sleep(statsSnapshotMaxAge) + if stats, err := mt.GetStats(l.ctx, &common.StatRequest{Type: common.StatType_UsersStat, Reset_: true}); err != nil || len(stats.GetStats()) != 0 { + t.Errorf("user stats after a reset = %v, %v; want none", stats.GetStats(), err) + } + if sys, err := mt.GetSysStats(l.ctx); err != nil || sys.GetAlloc() == 0 { + t.Errorf("GetSysStats = %v, %v; want telemt's memory", sys, err) } - defer mt.Shutdown() - if !mt.Started() { - t.Fatal("backend not started") + configPath := mt.configPath + mt.Shutdown() + if _, err := os.Stat(configPath); !errors.Is(err, os.ErrNotExist) { + t.Errorf("config.toml after Shutdown: %v, want it removed", err) + } + if exists, err := gopsutilprocess.PidExists(int32(pid)); err != nil || exists { + t.Errorf("telemt pid %d exists after Shutdown: %v, %v", pid, exists, err) } - if v := mt.Version(); v == "" { - t.Error("empty telemt version") - } else { - t.Logf("telemt version: %s", v) + processes, err := gopsutilprocess.Processes() + if err != nil { + t.Fatalf("list processes: %v", err) } + for _, proc := range processes { + if args, err := proc.CmdlineSlice(); err == nil && runsTelemtConfig(args, configPath) { + t.Errorf("telemt process %d still runs %s after Shutdown", proc.Pid, configPath) + } + } +} - // alice was rendered into the config; she must be present in telemt's runtime. - waitTelemtUser(ctx, t, mt, "alice", true) +// User create, secret change and delete through UpdateUsers go through +// telemt's user API (the marker never changes) and reach telemt's handshake +// path, not only its config file. +func TestIntegrationUpdateUsersThroughAPI(t *testing.T) { + l := startLiveTelemt(t, "", mtUser("alice", "mtproto", secretAlice)) + marker := l.mt.markerUsername() - // add bob at runtime via the API path - if err := mt.SyncUser(ctx, mtUser("bob", "mtproto", "ffeeddccbbaa00112233445566778899")); err != nil { - t.Fatalf("SyncUser(bob): %v", err) + if err := l.mt.UpdateUsers(l.ctx, []*common.User{mtUser("bob", "mtproto", secretOld)}); err != nil { + t.Fatalf("UpdateUsers(create bob): %v", err) } - waitTelemtUser(ctx, t, mt, "bob", true) + l.waitListed("bob", true) + session := l.expectAdmitted("bob", secretOld) + _ = session.Close() + l.waitConnections("bob", 0) + + if err := l.mt.UpdateUsers(l.ctx, []*common.User{mtUser("bob", "mtproto", secretNew)}); err != nil { + t.Fatalf("UpdateUsers(patch bob): %v", err) + } + session = l.expectAdmitted("bob", secretNew) + _ = session.Close() + l.waitConnections("bob", 0) + l.expectRefused("bob", secretOld) + + if err := l.mt.UpdateUsers(l.ctx, []*common.User{untagged("bob")}); err != nil { + t.Fatalf("UpdateUsers(delete bob): %v", err) + } + l.waitListed("bob", false) + l.expectRefused("bob", secretNew) + + if got := l.mt.markerUsername(); got != marker { + t.Errorf("marker changed from %s to %s: a user API batch rendered the config", marker, got) + } + l.expectAdmitted("alice", secretAlice) +} - // remove alice (untag her) - aliceGone := &common.User{Email: "alice", Inbounds: []string{}, Proxies: &common.Proxy{Mtproto: &common.Mtproto{Secret: testSecret}}} - if err := mt.SyncUser(ctx, aliceGone); err != nil { - t.Fatalf("SyncUser(remove alice): %v", err) +// Full syncs hot-reload the same telemt process, down to zero real users +// (marker only) and back, and telemt confirms each render's marker. The +// first syncs come right after the start, when the node leaves the reload +// to telemt's config watcher; the last one comes after telemtSignalGrace +// and sends SIGHUP. +func TestIntegrationFullSyncKeepsProcess(t *testing.T) { + l := startLiveTelemt(t, "", mtUser("alice", "mtproto", secretAlice)) + pid := l.mt.process.PID() + marker := l.mt.markerUsername() + + err := l.mt.SyncUsers(l.ctx, []*common.User{ + mtUser("carol", "mtproto", secretCarol), + mtUser("dave", "mtproto", secretDave), + }) + if err != nil { + t.Fatalf("SyncUsers(carol, dave): %v", err) } - waitTelemtUser(ctx, t, mt, "alice", false) + marker = l.expectNewMarker(marker) + l.expectPID(pid) + l.waitListed("carol", true) + l.waitListed("dave", true) + l.waitListed("alice", false) + l.expectAdmitted("carol", secretCarol) + l.expectRefused("alice", secretAlice) - // Traffic + online-count stats come from telemt's Prometheus endpoint, which - // only binds once the data plane is up (needs Telegram connectivity). A miss - // must still succeed with empty/zero results so the panel poll does not fail. - if stats, err := mt.GetStats(ctx, &common.StatRequest{Type: common.StatType_UsersStat}); err != nil { - t.Errorf("GetStats: %v", err) - } else if stats == nil { - t.Error("GetStats returned nil") + if err := l.mt.SyncUsers(l.ctx, nil); err != nil { + t.Fatalf("SyncUsers(no users): %v", err) } - if online, err := mt.GetUserOnlineStats(ctx, "bob"); err != nil { - t.Errorf("GetUserOnlineStats: %v", err) - } else if online == nil || online.GetName() != "bob" { - t.Errorf("GetUserOnlineStats = %+v", online) + marker = l.expectNewMarker(marker) + l.expectPID(pid) + if users := l.users(); len(users) != 1 { + t.Errorf("telemt users after syncing none = %v, want only the marker %s", users, marker) } + l.expectRefused("dave", secretDave) - // These do not depend on metrics: IP list comes from the control API, sys - // stats from the OS process; both must work against a live telemt. - if _, err := mt.GetUserOnlineIpListStats(ctx, "bob"); err != nil { - t.Errorf("GetUserOnlineIpListStats: %v", err) + l.passSignalGrace() + sighups := l.sighups() + if err := l.mt.SyncUsers(l.ctx, []*common.User{mtUser("erin", "mtproto", secretErin)}); err != nil { + t.Fatalf("SyncUsers(erin): %v", err) } - if _, err := mt.GetSysStats(ctx); err != nil { - t.Errorf("GetSysStats: %v", err) + l.expectNewMarker(marker) + if l.sighups() <= sighups { + t.Error("telemt logged no SIGHUP for a sync after telemtSignalGrace") } + l.expectPID(pid) + l.expectAdmitted("erin", secretErin) } -func TestIntegration_EmptyUsersMarker(t *testing.T) { - bin := telemtBinOrSkip(t) +// A hot reload keeps the sessions of users it does not change and cuts the +// sessions of users it removes, right after the start (telemt's config +// watcher reloads) and after telemtSignalGrace (SIGHUP reloads). +func TestIntegrationHotReloadKeepsUnchangedSessions(t *testing.T) { + alice := mtUser("alice", "mtproto", secretAlice) + carol := mtUser("carol", "mtproto", secretCarol) + l := startLiveTelemt(t, "", alice, mtUser("bob", "mtproto", secretBob)) + pid := l.mt.process.PID() + + aliceSession := l.expectAdmitted("alice", secretAlice) + bobSession := l.expectAdmitted("bob", secretBob) + if err := l.mt.SyncUsers(l.ctx, []*common.User{alice, carol}); err != nil { + t.Fatalf("SyncUsers without bob: %v", err) + } + l.expectCut(bobSession, "bob") + l.expectOpen(aliceSession, "alice", 1) + l.expectPID(pid) + + carolSession := l.expectAdmitted("carol", secretCarol) + l.passSignalGrace() + sighups := l.sighups() + if err := l.mt.SyncUsers(l.ctx, []*common.User{alice, mtUser("dave", "mtproto", secretDave)}); err != nil { + t.Fatalf("SyncUsers without carol: %v", err) + } + if l.sighups() <= sighups { + t.Error("telemt logged no SIGHUP for a sync after telemtSignalGrace") + } + l.expectCut(carolSession, "carol") + l.expectOpen(aliceSession, "alice", 1) + l.expectPID(pid) + l.expectAdmitted("dave", secretDave) +} - // telemt binds its API only after the Telegram DC check (up to about 60 s - // with slow DCs), and New allows it telemtStartupTimeout. - ctx, cancel := context.WithTimeout(context.Background(), 150*time.Second) - defer cancel() +// A user deleted through the API loses its open session, and deleting the +// last real user leaves telemt running with only the marker. +func TestIntegrationAPIDeleteCutsSessions(t *testing.T) { + l := startLiveTelemt(t, "", mtUser("alice", "mtproto", secretAlice), mtUser("bob", "mtproto", secretBob)) + pid := l.mt.process.PID() + marker := l.mt.markerUsername() - // Zero real users: telemt rejects an empty [access.users], so the marker - // user must keep it alive. - mt, err := New(ctx, integrationConfig(t, bin), integrationMTConfig(t), nil) + bobSession := l.expectAdmitted("bob", secretBob) + if err := l.mt.UpdateUsers(l.ctx, []*common.User{untagged("bob")}); err != nil { + t.Fatalf("UpdateUsers(delete bob): %v", err) + } + l.expectCut(bobSession, "bob") + + aliceSession := l.expectAdmitted("alice", secretAlice) + if err := l.mt.UpdateUsers(l.ctx, []*common.User{untagged("alice")}); err != nil { + t.Fatalf("UpdateUsers(delete the last real user): %v", err) + } + l.expectCut(aliceSession, "alice") + users := l.users() + if _, found := users[marker]; !found || len(users) != 1 || l.mt.markerUsername() != marker { + t.Errorf("telemt users after deleting the last real user = %v, want only the marker %s", users, marker) + } + l.expectPID(pid) + if err := l.mt.apiClient.Health(l.ctx); err != nil { + t.Fatalf("telemt health with only the marker: %v", err) + } + l.expectRefused("alice", secretAlice) + + if err := l.mt.UpdateUsers(l.ctx, []*common.User{mtUser("carol", "mtproto", secretCarol)}); err != nil { + t.Fatalf("UpdateUsers(create carol): %v", err) + } + l.expectAdmitted("carol", secretCarol) +} + +// Starting with no real users runs telemt on the marker alone, and users +// added later through the API connect. +func TestIntegrationStartWithoutUsers(t *testing.T) { + l := startLiveTelemt(t, "") + + marker := l.mt.markerUsername() + if users := l.users(); len(users) != 1 || !users[marker].InRuntime { + t.Fatalf("telemt users = %v, want only the marker %s", users, marker) + } + if err := l.mt.UpdateUsers(l.ctx, []*common.User{mtUser("alice", "mtproto", secretAlice)}); err != nil { + t.Fatalf("UpdateUsers(alice): %v", err) + } + l.expectAdmitted("alice", secretAlice) +} + +// A secret written through the API and changed again by a full render before +// telemt published it must not lock the user out: telemt admits the user +// with the rendered secret only. +func TestIntegrationRenderAfterAPIWriteAdmitsRenderedSecret(t *testing.T) { + l := startLiveTelemt(t, "", mtUser("alice", "mtproto", secretAlice), mtUser("bob", "mtproto", secretOld)) + marker := l.mt.markerUsername() + + if err := l.mt.SyncUser(l.ctx, mtUser("bob", "mtproto", secretNew)); err != nil { + t.Fatalf("SyncUser(bob, new secret): %v", err) + } + batch := []*common.User{mtUser("bob", "mtproto", secretFinal)} + for i := range partialApplyMaxUsers { + batch = append(batch, mtUser(fmt.Sprintf("u%d", i), "mtproto", numberedSecret(i))) + } + if err := l.mt.UpdateUsers(l.ctx, batch); err != nil { + t.Fatalf("UpdateUsers(%d changes): %v", len(batch), err) + } + l.expectNewMarker(marker) + + l.expectAdmitted("bob", secretFinal) + l.expectRefused("bob", secretNew) + l.expectRefused("bob", secretOld) + l.expectAdmitted("u0", numberedSecret(0)) +} + +// GeneratedConfigPath may be a symlink: telemt refuses a config path with a +// symlinked component, so the node renders into the resolved directory. +func TestIntegrationSymlinkedGeneratedPath(t *testing.T) { + base := t.TempDir() + target := filepath.Join(base, "real") + if err := os.Mkdir(target, 0o700); err != nil { + t.Fatal(err) + } + link := filepath.Join(base, "link") + if err := os.Symlink(target, link); err != nil { + t.Fatal(err) + } + + l := startLiveTelemt(t, link, mtUser("alice", "mtproto", secretAlice)) + resolved, err := filepath.EvalSymlinks(target) if err != nil { - t.Fatalf("New with zero users (marker only): %v", err) + t.Fatal(err) } - defer mt.Shutdown() + if want := filepath.Join(resolved, "telemt", "config.toml"); l.mt.configPath != want { + t.Errorf("config path = %s, want %s", l.mt.configPath, want) + } + l.expectAdmitted("alice", secretAlice) - if !mt.Started() { - t.Fatal("backend not started with the marker user only") + marker := l.mt.markerUsername() + if err := l.mt.SyncUsers(l.ctx, []*common.User{mtUser("bob", "mtproto", secretBob)}); err != nil { + t.Fatalf("SyncUsers(bob): %v", err) + } + l.expectNewMarker(marker) + l.expectAdmitted("bob", secretBob) +} + +// rejectedRenderUsers is enough users to push the rendered config past +// telemt's 8 MiB limit for one config source; each renders as a line of +// about 45 bytes. The node itself accepts every one of them. +const rejectedRenderUsers = 220_000 + +// A render telemt refuses is reported with telemt's reason, and the previous +// users are rendered again: telemt keeps running them, sessions included. +func TestIntegrationRejectedRenderRestoresUsers(t *testing.T) { + alice := mtUser("alice", "mtproto", secretAlice) + l := startLiveTelemt(t, "", alice) + pid := l.mt.process.PID() + marker := l.mt.markerUsername() + aliceSession := l.expectAdmitted("alice", secretAlice) + + users := []*common.User{alice} + for i := range rejectedRenderUsers { + users = append(users, mtUser(fmt.Sprint(1_000_000+i), "mtproto", fmt.Sprintf("%032x", 1_000_000+i))) + } + started := time.Now() + err := l.mt.SyncUsers(l.ctx, users) + const reason = "exceeds configured size limit" + if !errors.Is(err, errReloadRejected) || !strings.Contains(err.Error(), reason) { + t.Fatalf("SyncUsers(%d users) = %v, want a rejection with telemt's reason %q", len(users), err, reason) + } + // telemt's answer rejects the render; the node must not wait out + // markerConfirmTimeout. + if elapsed := time.Since(started); elapsed >= markerConfirmTimeout/2 { + t.Errorf("the rejection took %s", elapsed) + } + if !slices.ContainsFunc(logLines(l.mt), func(line string) bool { + return strings.Contains(line, reason) && strings.Contains(line, "rendering the previous users again") + }) { + t.Errorf("telemt's log stream has no rejection notice with its reason") + } + + if got := desiredNames(l.mt); !slices.Equal(got, []string{"alice"}) { + t.Errorf("desired users after the rejection = %v, want [alice]", got) + } + l.expectNewMarker(marker) + l.expectPID(pid) + if info, err := os.Stat(l.mt.configPath); err != nil || info.Size() > 1<<20 { + t.Errorf("config file after the restore: %v, %v; want the small previous render", info, err) } + l.expectOpen(aliceSession, "alice", 1) + l.expectAdmitted("alice", secretAlice) + l.expectRefused("1000000", fmt.Sprintf("%032x", 1_000_000)) } diff --git a/backend/mtproto/mtproto_client_test.go b/backend/mtproto/mtproto_client_test.go new file mode 100644 index 00000000..811ab807 --- /dev/null +++ b/backend/mtproto/mtproto_client_test.go @@ -0,0 +1,143 @@ +package mtproto + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "errors" + "io" + "net" + "os" + "sync" + "testing" + "time" +) + +// mtprotoTestDC is the Telegram DC the fake client asks for. Integration +// configs point it at startFakeDC through telemt's dc_overrides. +const mtprotoTestDC = 2 + +// obfuscated2 initial packets must not look like another protocol, or the +// server would not read them as MTProto. +var reservedInitPrefixes = [][]byte{ + []byte("HEAD"), []byte("POST"), []byte("GET "), []byte("OPTI"), + {0x16, 0x03, 0x01, 0x02}, {0xdd, 0xdd, 0xdd, 0xdd}, {0xee, 0xee, 0xee, 0xee}, +} + +// startFakeDC listens on loopback as a Telegram DC that accepts telemt's +// relay connections and reads them until they close, so a client session +// stays open until telemt itself ends it. A test host cannot reach the real +// DCs, and telemt closes a session at once when its DC refuses. +func startFakeDC(t *testing.T) string { + t.Helper() + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("fake DC listen: %v", err) + } + + var ( + mu sync.Mutex + conns []net.Conn + ) + go func() { + for { + conn, err := listener.Accept() + if err != nil { + return + } + mu.Lock() + conns = append(conns, conn) + mu.Unlock() + go func() { + _, _ = io.Copy(io.Discard, conn) + _ = conn.Close() + }() + } + }() + t.Cleanup(func() { + _ = listener.Close() + mu.Lock() + defer mu.Unlock() + for _, conn := range conns { + _ = conn.Close() + } + }) + return listener.Addr().String() +} + +// dialMTProto connects to addr as an MTProto client of the user whose secret +// is secretHex: the obfuscated2 handshake keyed with the secret, the padded +// intermediate transport (`dd`), then one framed packet. telemt counts an +// admitted connection in telemt_user_connections_current right away, even +// though it cannot reach Telegram; a handshake with a wrong secret is never +// counted for any user. +func dialMTProto(addr, secretHex string) (net.Conn, error) { + secret, err := hex.DecodeString(secretHex) + if err != nil { + return nil, err + } + + init := make([]byte, 64) + for { + if _, err := rand.Read(init); err != nil { + return nil, err + } + if validInitPacket(init) { + break + } + } + copy(init[56:60], []byte{0xdd, 0xdd, 0xdd, 0xdd}) + binary.LittleEndian.PutUint16(init[60:62], mtprotoTestDC) + + key := sha256.Sum256(append(append([]byte{}, init[8:40]...), secret...)) + block, err := aes.NewCipher(key[:]) + if err != nil { + return nil, err + } + stream := cipher.NewCTR(block, init[40:56]) + encrypted := make([]byte, len(init)) + stream.XORKeyStream(encrypted, init) + + frame := make([]byte, 4+200) + binary.LittleEndian.PutUint32(frame, 200) + stream.XORKeyStream(frame, frame) + + conn, err := net.DialTimeout("tcp", addr, 5*time.Second) + if err != nil { + return nil, err + } + handshake := append(append([]byte{}, init[:56]...), encrypted[56:]...) + if _, err := conn.Write(append(handshake, frame...)); err != nil { + conn.Close() + return nil, err + } + return conn, nil +} + +func validInitPacket(init []byte) bool { + if init[0] == 0xef || bytes.Equal(init[4:8], []byte{0, 0, 0, 0}) { + return false + } + for _, prefix := range reservedInitPrefixes { + if bytes.Equal(init[:4], prefix) { + return false + } + } + return true +} + +// connClosed reports whether the server closed conn, waiting up to wait for +// it. telemt never sends a fake client anything, so a read that times out +// means the connection is still open. +func connClosed(conn net.Conn, wait time.Duration) bool { + if err := conn.SetReadDeadline(time.Now().Add(wait)); err != nil { + return true + } + buf := make([]byte, 1) + _, err := conn.Read(buf) + return err != nil && !errors.Is(err, os.ErrDeadlineExceeded) +} From 77cf00009790922e2707cc8a859848a74d5c556e Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Tue, 20 Oct 2026 00:12:45 +0330 Subject: [PATCH 17/40] refactor(mtproto): keep mtproto.go to the backend struct and drop dead code mtproto.go now holds the MTProto struct and its one-line accessors. The helpers it collected move next to their callers: Restart and the API token to lifecycle.go, the marker user to reload.go, the atomic file write and user map clone to sync.go. markerUsername, read only by tests, moves to the tests, and the unused config field goes. Also removes the never-built tomlLiteral type, the always-nil error of classifyTOMLSection, and decoded telemt API fields the node never reads. No behavior change. --- backend/mtproto/client.go | 29 ++++----- backend/mtproto/lifecycle.go | 32 +++++++++- backend/mtproto/mtproto.go | 113 ++--------------------------------- backend/mtproto/reload.go | 27 +++++++++ backend/mtproto/sync.go | 40 +++++++++++++ backend/mtproto/sync_test.go | 10 ++++ backend/mtproto/toml.go | 20 +++---- 7 files changed, 130 insertions(+), 141 deletions(-) diff --git a/backend/mtproto/client.go b/backend/mtproto/client.go index 802aaf7b..c8033eaa 100644 --- a/backend/mtproto/client.go +++ b/backend/mtproto/client.go @@ -20,17 +20,13 @@ type apiClient struct { } type apiSuccessResponse[T any] struct { - OK bool `json:"ok"` - Data T `json:"data"` - Revision string `json:"revision"` - Error *apiErrorBody `json:"error,omitempty"` + OK bool `json:"ok"` + Data T `json:"data"` + Error *apiErrorBody `json:"error,omitempty"` } type apiErrorResponse struct { - OK bool `json:"ok"` - Error *apiErrorBody `json:"error"` - Request uint64 `json:"request_id"` - Revision string `json:"revision"` + Error *apiErrorBody `json:"error"` } type apiErrorBody struct { @@ -38,18 +34,13 @@ type apiErrorBody struct { Message string `json:"message"` } +// telemtUserInfo is the part of telemt's user record the node reads. type telemtUserInfo struct { - Username string `json:"username"` - InRuntime bool `json:"in_runtime"` - CurrentConnections uint64 `json:"current_connections"` - ActiveUniqueIPsList []string `json:"active_unique_ips_list"` - RecentUniqueIPsList []string `json:"recent_unique_ips_list"` - ActiveUniqueIPs int `json:"active_unique_ips"` - RecentUniqueIPs int `json:"recent_unique_ips"` - UserAdTag string `json:"user_ad_tag"` - MaxTCPConns int `json:"max_tcp_conns"` - MaxUniqueIPs int `json:"max_unique_ips"` - TotalOctets uint64 `json:"total_octets"` + Username string `json:"username"` + InRuntime bool `json:"in_runtime"` + CurrentConnections uint64 `json:"current_connections"` + ActiveUniqueIPs int `json:"active_unique_ips"` + TotalOctets uint64 `json:"total_octets"` } type telemtUserActiveIPs struct { diff --git a/backend/mtproto/lifecycle.go b/backend/mtproto/lifecycle.go index 915a7529..b6bc1129 100644 --- a/backend/mtproto/lifecycle.go +++ b/backend/mtproto/lifecycle.go @@ -2,6 +2,8 @@ package mtproto import ( "context" + "crypto/rand" + "encoding/hex" "errors" "fmt" "log" @@ -77,7 +79,6 @@ func New(ctx context.Context, cfg *config.Config, mtConfig *Config, users []*com lifetime, cancel := context.WithCancel(context.Background()) mt := &MTProto{ - cfg: cfg, config: mtConfig, process: process, configPath: filepath.Join(telemtDir, "config.toml"), @@ -180,6 +181,16 @@ func pickInternalPorts(listenPorts []int, findFreePort func() int) (apiPort, met return apiPort, metricsPort, nil } +// generateBearerToken returns a random Authorization header value for +// telemt's API. +func generateBearerToken() (string, error) { + token := make([]byte, 24) + if _, err := rand.Read(token); err != nil { + return "", err + } + return "Bearer " + hex.EncodeToString(token), nil +} + // startLocked starts telemt on the rendered config and waits for its API. // A failure stops telemt and carries its last output. Readiness is watched in // the background. Requires syncMu. @@ -371,6 +382,25 @@ func (m *MTProto) supervisorState() (closed, running bool) { return m.closed, m.process.Started() } +// Restart renders the desired users again and restarts telemt on them. +func (m *MTProto) Restart() error { + m.syncMu.Lock() + defer m.syncMu.Unlock() + + if m.closed { + return errClosed + } + if err := m.writeRenderedConfigLocked(m.desired); err != nil { + return err + } + m.process.Stop() + if err := m.startLocked(m.lifetime); err != nil { + return err + } + m.needsFullApply = false + return nil +} + // restartIfStopped starts telemt again unless it already runs, re-rendering // the config from the current desired users. func (m *MTProto) restartIfStopped() (bool, error) { diff --git a/backend/mtproto/mtproto.go b/backend/mtproto/mtproto.go index d57eec37..06d4e6e8 100644 --- a/backend/mtproto/mtproto.go +++ b/backend/mtproto/mtproto.go @@ -2,19 +2,16 @@ package mtproto import ( "context" - "crypto/rand" - "encoding/hex" - "fmt" - "os" - "path/filepath" "sync" "time" - - "github.com/pasarguard/node/config" ) +// MTProto runs telemt as the node's MTProto proxy backend: it renders the +// panel config plus the users into telemt's config file, supervises the +// telemt process (lifecycle.go), applies user changes through telemt's user +// API or a hot reload (sync.go, reload.go) and reads traffic from telemt's +// metrics (stats.go). type MTProto struct { - cfg *config.Config config *Config process *processManager configPath string @@ -75,103 +72,3 @@ func (m *MTProto) Version() string { func (m *MTProto) SubscribeLogs(ctx context.Context) <-chan string { return m.process.SubscribeLogs(ctx) } - -func (m *MTProto) Restart() error { - m.syncMu.Lock() - defer m.syncMu.Unlock() - - if m.closed { - return errClosed - } - if err := m.writeRenderedConfigLocked(m.desired); err != nil { - return err - } - m.process.Stop() - if err := m.startLocked(m.lifetime); err != nil { - return err - } - m.needsFullApply = false - return nil -} - -// markerUsernamePrefix starts every marker username. Panel usernames are -// numeric user ids, so no real user has it. -const markerUsernamePrefix = "pgm_" - -// markerUsername returns the marker user of the latest render. Stats must -// never report it; older markers share markerUsernamePrefix. -func (m *MTProto) markerUsername() string { - m.stateMu.RLock() - defer m.stateMu.RUnlock() - if m.marker == nil { - return "" - } - return m.marker.Username -} - -// newMarkerUser returns a marker user with a fresh random name and secret. -// Every render carries one, disabled: once telemt's running config has it, -// telemt has published that render (its user admission follows from the -// published config shortly after). It also keeps telemt's user list -// non-empty. -func newMarkerUser() (*runtimeUser, error) { - secret := make([]byte, 16) - usernameSuffix := make([]byte, 8) - if _, err := rand.Read(secret); err != nil { - return nil, fmt.Errorf("failed to generate the mtproto marker secret: %w", err) - } - if _, err := rand.Read(usernameSuffix); err != nil { - return nil, fmt.Errorf("failed to generate the mtproto marker username: %w", err) - } - - return &runtimeUser{ - Username: markerUsernamePrefix + hex.EncodeToString(usernameSuffix), - Secret: hex.EncodeToString(secret), - }, nil -} - -func generateBearerToken() (string, error) { - token := make([]byte, 24) - if _, err := rand.Read(token); err != nil { - return "", err - } - return "Bearer " + hex.EncodeToString(token), nil -} - -func cloneRuntimeUserMap(input map[string]*runtimeUser) map[string]*runtimeUser { - cloned := make(map[string]*runtimeUser, len(input)) - for username, user := range input { - if user == nil { - continue - } - copyUser := *user - cloned[username] = ©User - } - return cloned -} - -// writeFileAtomic writes payload to path via a temp file and rename. The -// rendered config holds the API bearer token and per-user secrets, so the -// directory and file are created with owner-only permissions. -func writeFileAtomic(path string, payload []byte) error { - dir := filepath.Dir(path) - if err := os.MkdirAll(dir, 0o700); err != nil { - return err - } - - tempFile, err := os.CreateTemp(dir, filepath.Base(path)+".*.tmp") - if err != nil { - return err - } - tempPath := tempFile.Name() - defer func() { _ = os.Remove(tempPath) }() - - if _, err := tempFile.Write(payload); err != nil { - _ = tempFile.Close() - return err - } - if err := tempFile.Close(); err != nil { - return err - } - return os.Rename(tempPath, path) -} diff --git a/backend/mtproto/reload.go b/backend/mtproto/reload.go index a0c2840c..82ff713a 100644 --- a/backend/mtproto/reload.go +++ b/backend/mtproto/reload.go @@ -2,6 +2,8 @@ package mtproto import ( "context" + "crypto/rand" + "encoding/hex" "errors" "fmt" "log" @@ -36,6 +38,31 @@ var markerConfirmTimeout = 30 * time.Second // keeps running the config it had. var errReloadRejected = errors.New("telemt did not load the new config") +// markerUsernamePrefix starts every marker username. Panel usernames are +// numeric user ids, so no real user has it. +const markerUsernamePrefix = "pgm_" + +// newMarkerUser returns a marker user with a fresh random name and secret. +// Every render carries one, disabled: once telemt's running config has it, +// telemt has published that render (its user admission follows from the +// published config shortly after). It also keeps telemt's user list +// non-empty. +func newMarkerUser() (*runtimeUser, error) { + secret := make([]byte, 16) + usernameSuffix := make([]byte, 8) + if _, err := rand.Read(secret); err != nil { + return nil, fmt.Errorf("failed to generate the mtproto marker secret: %w", err) + } + if _, err := rand.Read(usernameSuffix); err != nil { + return nil, fmt.Errorf("failed to generate the mtproto marker username: %w", err) + } + + return &runtimeUser{ + Username: markerUsernamePrefix + hex.EncodeToString(usernameSuffix), + Secret: hex.EncodeToString(secret), + }, nil +} + // reloadLocked makes the running telemt hot-reload the rendered config and // waits until the render's marker user is in telemt's running config. // telemt's config watcher sees the file change, and SIGHUP makes sure once diff --git a/backend/mtproto/sync.go b/backend/mtproto/sync.go index bd063cf2..29256662 100644 --- a/backend/mtproto/sync.go +++ b/backend/mtproto/sync.go @@ -6,6 +6,8 @@ import ( "fmt" "log" "maps" + "os" + "path/filepath" "strings" "github.com/pasarguard/node/common" @@ -323,6 +325,44 @@ func (m *MTProto) writeRenderedConfigLocked(desired map[string]*runtimeUser) err return nil } +// writeFileAtomic writes payload to path via a temp file and rename. The +// rendered config holds the API bearer token and per-user secrets, so the +// directory and file are created with owner-only permissions. +func writeFileAtomic(path string, payload []byte) error { + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0o700); err != nil { + return err + } + + tempFile, err := os.CreateTemp(dir, filepath.Base(path)+".*.tmp") + if err != nil { + return err + } + tempPath := tempFile.Name() + defer func() { _ = os.Remove(tempPath) }() + + if _, err := tempFile.Write(payload); err != nil { + _ = tempFile.Close() + return err + } + if err := tempFile.Close(); err != nil { + return err + } + return os.Rename(tempPath, path) +} + +func cloneRuntimeUserMap(input map[string]*runtimeUser) map[string]*runtimeUser { + cloned := make(map[string]*runtimeUser, len(input)) + for username, user := range input { + if user == nil { + continue + } + copyUser := *user + cloned[username] = ©User + } + return cloned +} + // runtimeUserEqual reports whether a and b give telemt the same user. func runtimeUserEqual(a, b *runtimeUser) bool { if a == nil || b == nil { diff --git a/backend/mtproto/sync_test.go b/backend/mtproto/sync_test.go index 90e58ee9..f14e6d4f 100644 --- a/backend/mtproto/sync_test.go +++ b/backend/mtproto/sync_test.go @@ -489,6 +489,16 @@ func logLines(m *MTProto) []string { } } +// markerUsername returns the marker user of the latest render. +func (m *MTProto) markerUsername() string { + m.stateMu.RLock() + defer m.stateMu.RUnlock() + if m.marker == nil { + return "" + } + return m.marker.Username +} + func desiredNames(m *MTProto) []string { m.stateMu.RLock() defer m.stateMu.RUnlock() diff --git a/backend/mtproto/toml.go b/backend/mtproto/toml.go index 2d9fc3cc..fd2ba9f3 100644 --- a/backend/mtproto/toml.go +++ b/backend/mtproto/toml.go @@ -9,8 +9,6 @@ import ( "strings" ) -type tomlLiteral string - func renderTOML(root map[string]any) (string, error) { var builder strings.Builder if err := writeTOMLSection(&builder, nil, root, tomlHeaderNone); err != nil { @@ -28,10 +26,7 @@ const ( ) func writeTOMLSection(builder *strings.Builder, path []string, section map[string]any, headerKind tomlHeaderKind) error { - scalarKeys, tableKeys, arrayTableKeys, err := classifyTOMLSection(section) - if err != nil { - return err - } + scalarKeys, tableKeys, arrayTableKeys := classifyTOMLSection(section) if headerKind != tomlHeaderNone { writeSectionSpacing(builder) @@ -82,16 +77,17 @@ func writeTOMLSection(builder *strings.Builder, path []string, section map[strin return nil } -func classifyTOMLSection(section map[string]any) ([]string, []string, []string, error) { +// classifyTOMLSection sorts the keys of section into scalars (inline +// values), tables and arrays of tables, each in key order. +func classifyTOMLSection(section map[string]any) (scalarKeys, tableKeys, arrayTableKeys []string) { keys := make([]string, 0, len(section)) for key := range section { keys = append(keys, key) } slices.Sort(keys) - scalarKeys := make([]string, 0, len(keys)) - tableKeys := make([]string, 0, len(keys)) - arrayTableKeys := make([]string, 0) + scalarKeys = make([]string, 0, len(keys)) + tableKeys = make([]string, 0, len(keys)) for _, key := range keys { switch value := section[key].(type) { @@ -111,7 +107,7 @@ func classifyTOMLSection(section map[string]any) ([]string, []string, []string, } } - return scalarKeys, tableKeys, arrayTableKeys, nil + return scalarKeys, tableKeys, arrayTableKeys } func isArrayOfObjects(values []any) bool { @@ -130,8 +126,6 @@ func renderInlineValue(value any) (string, error) { switch typed := value.(type) { case nil: return "", fmt.Errorf("null array elements are not valid in TOML") - case tomlLiteral: - return string(typed), nil case string: return quoteTOMLString(typed), nil case bool: From 03f8529d67e69b5315352ef82818518625a09ade Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Tue, 20 Oct 2026 01:12:22 +0330 Subject: [PATCH 18/40] chore(mtproto): fix the Windows test build and document telemt limits - Build integration_test.go only off Windows: it uses waitFor from the Unix-only fake telemt helpers, so GOOS=windows go vet failed. - Move the TestUsersAPIUnreachableRestartsTelemt doc comment back above its test. - README: Start can take up to 90 s and the panel's retry attaches; above 4096 users telemt stops per-user series and the node bills API totals as downlink; telemt refuses configs above 8 MiB (about 150k to 190k users). --- README.md | 3 +++ backend/mtproto/integration_test.go | 2 ++ backend/mtproto/lifecycle_test.go | 8 ++++---- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 43a62fbf..b295ef02 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,9 @@ The MTProto backend runs [telemt](https://github.com/telemt/telemt) as a separat - Set `TELEMT_EXECUTABLE_PATH` to use another binary. - Outside Docker, `make install_telemt` installs the pinned release after checking it against the SHA256 pinned in the `Makefile`. - telemt needs outbound access to Telegram. On start it checks the Telegram data centers, which can take up to a minute. +- Start can take up to 90 s while telemt probes Telegram, and the panel's retry attaches to the backend that is starting or running. +- Above 4096 users with stats or configured users, telemt stops its per-user series and the node bills from telemt's API totals, reported as downlink (no uplink and downlink split). +- telemt refuses a config file above 8 MiB, which is roughly 150k to 190k users on one node. ## Xray and MTProto on one host One node runs one backend, so serving Xray (or WireGuard) and MTProto from one server takes two node instances, each added to the panel as its own node: diff --git a/backend/mtproto/integration_test.go b/backend/mtproto/integration_test.go index 26ff20ed..9d40ac85 100644 --- a/backend/mtproto/integration_test.go +++ b/backend/mtproto/integration_test.go @@ -1,3 +1,5 @@ +//go:build !windows + package mtproto import ( diff --git a/backend/mtproto/lifecycle_test.go b/backend/mtproto/lifecycle_test.go index 7b5f78e9..2d1e0468 100644 --- a/backend/mtproto/lifecycle_test.go +++ b/backend/mtproto/lifecycle_test.go @@ -391,10 +391,6 @@ func TestFullSyncSignalsTelemt(t *testing.T) { } } -// TestUsersAPIUnreachableRestartsTelemt: when telemt's API keeps getting no -// answer (markerUnreachablePolls polls in a row) while the node waits for a -// reload, the node restarts telemt on the new config, and that start -// confirms it. // telemt installs its SIGHUP handler with its config watcher, after its API // is up, and a SIGHUP before that ends it. A sync right after the start must // leave the reload to the watcher instead of killing telemt. @@ -451,6 +447,10 @@ func TestRenderMissedByWatcherGetsSIGHUPAfterGrace(t *testing.T) { } } +// TestUsersAPIUnreachableRestartsTelemt: when telemt's API keeps getting no +// answer (markerUnreachablePolls polls in a row) while the node waits for a +// reload, the node restarts telemt on the new config, and that start +// confirms it. func TestUsersAPIUnreachableRestartsTelemt(t *testing.T) { exe := writeFakeTelemt(t, fakeTelemt{DropUsers: true}) mt := newTestBackend(t, testNodeConfig(t, exe), []*common.User{mtUser("alice", "mtproto", testSecret)}) From d036914cde01110f34b9bda8c7081e61662f9090 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Tue, 20 Oct 2026 01:26:12 +0330 Subject: [PATCH 19/40] fix(mtproto): send the first SIGHUP once telemt handles it telemt's config watcher installs its SIGHUP handler at the end of its own setup, which runs concurrently with the API coming up and can finish after the API first answers; a SIGHUP before that ends telemt, and once installed the handler stays for the process lifetime. On live telemt 3.5.14 the handler appeared about 3 ms after the API first answered, but a slow STUN probe, TLS front or large config can stretch that past any fixed grace. On Linux the node now sends SIGHUP only once bit 0 of the SigCgt mask in /proc//status is set, checked on each marker poll and before the restore render's signal. The 5 s grace after the first API answer stays as the fallback on other platforms or when /proc cannot be read. The fake telemt now models the mask too: with a signal delay it resets SIGHUP to its default action until signal.Notify, since the Go runtime otherwise catches SIGHUP from the start. The grace test forces the fallback, a new Linux test leaves only the mask to let the signal through, and the live tests wait for the handler instead of the clock. --- backend/mtproto/fake_telemt_linux_test.go | 25 ++++++ backend/mtproto/fake_telemt_other_test.go | 8 ++ backend/mtproto/fake_telemt_test.go | 41 +++++++--- backend/mtproto/integration_test.go | 35 +++++---- backend/mtproto/lifecycle_test.go | 16 ++-- backend/mtproto/reload.go | 59 +++++++++----- backend/mtproto/sighup_linux.go | 43 +++++++++++ backend/mtproto/sighup_linux_test.go | 94 +++++++++++++++++++++++ backend/mtproto/sighup_other.go | 11 +++ 9 files changed, 278 insertions(+), 54 deletions(-) create mode 100644 backend/mtproto/fake_telemt_linux_test.go create mode 100644 backend/mtproto/fake_telemt_other_test.go create mode 100644 backend/mtproto/sighup_linux.go create mode 100644 backend/mtproto/sighup_linux_test.go create mode 100644 backend/mtproto/sighup_other.go diff --git a/backend/mtproto/fake_telemt_linux_test.go b/backend/mtproto/fake_telemt_linux_test.go new file mode 100644 index 00000000..0ee36e76 --- /dev/null +++ b/backend/mtproto/fake_telemt_linux_test.go @@ -0,0 +1,25 @@ +//go:build linux + +package mtproto + +import ( + "fmt" + "os" + "os/signal" + "syscall" + "unsafe" +) + +// defaultSIGHUP gives SIGHUP its default action, which ends the process, so +// the fake's SigCgt mask lacks SIGHUP until signal.Notify, like telemt's +// before its config watcher installs the handler; the Go runtime catches +// SIGHUP from the start otherwise. signal.Ignore makes the runtime let go of +// SIGHUP first, so a later signal.Notify installs its handler again. +func defaultSIGHUP() { + signal.Ignore(syscall.SIGHUP) + var action [4]uint64 // SIG_DFL, no flags, empty mask + _, _, errno := syscall.RawSyscall6(syscall.SYS_RT_SIGACTION, uintptr(syscall.SIGHUP), uintptr(unsafe.Pointer(&action)), 0, 8, 0, 0) + if errno != 0 { + fmt.Fprintln(os.Stderr, "fake telemt: failed to reset SIGHUP:", errno) + } +} diff --git a/backend/mtproto/fake_telemt_other_test.go b/backend/mtproto/fake_telemt_other_test.go new file mode 100644 index 00000000..07ccebe7 --- /dev/null +++ b/backend/mtproto/fake_telemt_other_test.go @@ -0,0 +1,8 @@ +//go:build !linux && !windows + +package mtproto + +// defaultSIGHUP does nothing here: the Go runtime already ends the fake on a +// SIGHUP before signal.Notify, and outside Linux the node cannot read signal +// handlers, so it waits telemtSignalGrace instead. +func defaultSIGHUP() {} diff --git a/backend/mtproto/fake_telemt_test.go b/backend/mtproto/fake_telemt_test.go index 82440009..1dbb9c08 100644 --- a/backend/mtproto/fake_telemt_test.go +++ b/backend/mtproto/fake_telemt_test.go @@ -55,8 +55,9 @@ type fakeTelemt struct { // SignalFile gets one line per SIGHUP received. SignalFile string `json:"signal_file"` // SignalDelay makes the fake handle SIGHUP only this long after it binds - // its API, like telemt, whose config watcher installs the handler after - // the API is up; a SIGHUP before that ends the fake. + // its API, like telemt, whose config watcher installs the handler at the + // end of its setup; a SIGHUP before that ends the fake, and on Linux its + // SigCgt mask shows the handler only from then on. SignalDelay time.Duration `json:"signal_delay"` // WatchConfig plays telemt's config watcher. Once the API is up it reads // the config file, spends WatchSetup setting up its watches, then loads @@ -107,6 +108,7 @@ func runFakeTelemt(optionsPath string, args []string) int { hup := make(chan os.Signal, 1) if f.SignalDelay > 0 { + defaultSIGHUP() go func() { time.Sleep(f.HealthDelay + f.SignalDelay) signal.Notify(hup, syscall.SIGHUP) @@ -170,13 +172,7 @@ func runFakeTelemt(optionsPath string, args []string) int { select { case <-hup: runtime.load(configPath) - if f.SignalFile != "" { - file, err := os.OpenFile(f.SignalFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) - if err == nil { - _, _ = file.WriteString("SIGHUP\n") - _ = file.Close() - } - } + appendFakeLine(f.SignalFile, "SIGHUP") case <-exit: fmt.Fprintln(os.Stderr, "fake telemt exiting") return f.ExitCode @@ -186,6 +182,18 @@ func runFakeTelemt(optionsPath string, args []string) int { } } +// appendFakeLine appends line to path, unless path is empty. +func appendFakeLine(path, line string) { + if path == "" { + return + } + file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) + if err == nil { + _, _ = file.WriteString(line + "\n") + _ = file.Close() + } +} + // fakeTelemtRuntime holds the users of the fake's running config. type fakeTelemtRuntime struct { mu sync.Mutex @@ -415,11 +423,22 @@ func lowerStartupTimeout(t *testing.T, timeout time.Duration) { } // lowerSignalGrace sets telemtSignalGrace for one test, so a fake telemt -// without a config watcher gets the SIGHUP of a sync right after its start. -// Register it before the backend's Shutdown cleanup. +// without a config watcher gets the SIGHUP of a sync right after its start +// where the node cannot read signal handlers. Register it before the +// backend's Shutdown cleanup. func lowerSignalGrace(t *testing.T, grace time.Duration) { t.Helper() previous := telemtSignalGrace telemtSignalGrace = grace t.Cleanup(func() { telemtSignalGrace = previous }) } + +// withoutSIGHUPProbe makes the node unable to read telemt's signal +// handlers for one test, so it falls back to telemtSignalGrace on every +// platform. Register it before the backend's Shutdown cleanup. +func withoutSIGHUPProbe(t *testing.T) { + t.Helper() + previous := readSIGHUPCaught + readSIGHUPCaught = func(int) (bool, error) { return false, errors.ErrUnsupported } + t.Cleanup(func() { readSIGHUPCaught = previous }) +} diff --git a/backend/mtproto/integration_test.go b/backend/mtproto/integration_test.go index 9d40ac85..fad5d270 100644 --- a/backend/mtproto/integration_test.go +++ b/backend/mtproto/integration_test.go @@ -255,15 +255,17 @@ func (l *liveTelemt) expectPID(want int) { } } -// passSignalGrace waits until telemtSignalGrace has passed since telemt's -// API first answered, so the next reload sends SIGHUP instead of leaving -// the change to telemt's config watcher. -func (l *liveTelemt) passSignalGrace() { +// waitHandlesSIGHUP waits until the node may send telemt SIGHUP (telemt +// installed its handler, or telemtSignalGrace passed where the node cannot +// tell), so the next reload sends SIGHUP instead of leaving the change to +// telemt's config watcher. +func (l *liveTelemt) waitHandlesSIGHUP() { l.t.Helper() - l.mt.syncMu.Lock() - healthyAt := l.mt.healthyAt - l.mt.syncMu.Unlock() - time.Sleep(time.Until(healthyAt.Add(telemtSignalGrace))) + waitFor(l.t, telemtSignalGrace+liveWait, "telemt to handle SIGHUP", func() bool { + l.mt.syncMu.Lock() + defer l.mt.syncMu.Unlock() + return l.mt.telemtHandlesSIGHUPLocked() + }) } // sighups counts the SIGHUP reloads telemt logged. @@ -402,9 +404,9 @@ func TestIntegrationUpdateUsersThroughAPI(t *testing.T) { // Full syncs hot-reload the same telemt process, down to zero real users // (marker only) and back, and telemt confirms each render's marker. The -// first syncs come right after the start, when the node leaves the reload -// to telemt's config watcher; the last one comes after telemtSignalGrace -// and sends SIGHUP. +// first syncs come right after the start, when telemt may not handle SIGHUP +// yet and its config watcher reloads; the last one comes once telemt +// handles SIGHUP and sends it. func TestIntegrationFullSyncKeepsProcess(t *testing.T) { l := startLiveTelemt(t, "", mtUser("alice", "mtproto", secretAlice)) pid := l.mt.process.PID() @@ -435,14 +437,14 @@ func TestIntegrationFullSyncKeepsProcess(t *testing.T) { } l.expectRefused("dave", secretDave) - l.passSignalGrace() + l.waitHandlesSIGHUP() sighups := l.sighups() if err := l.mt.SyncUsers(l.ctx, []*common.User{mtUser("erin", "mtproto", secretErin)}); err != nil { t.Fatalf("SyncUsers(erin): %v", err) } l.expectNewMarker(marker) if l.sighups() <= sighups { - t.Error("telemt logged no SIGHUP for a sync after telemtSignalGrace") + t.Error("telemt logged no SIGHUP for a sync once it handled SIGHUP") } l.expectPID(pid) l.expectAdmitted("erin", secretErin) @@ -450,7 +452,8 @@ func TestIntegrationFullSyncKeepsProcess(t *testing.T) { // A hot reload keeps the sessions of users it does not change and cuts the // sessions of users it removes, right after the start (telemt's config -// watcher reloads) and after telemtSignalGrace (SIGHUP reloads). +// watcher reloads, or SIGHUP if telemt handles it already) and once telemt +// handles SIGHUP (SIGHUP reloads). func TestIntegrationHotReloadKeepsUnchangedSessions(t *testing.T) { alice := mtUser("alice", "mtproto", secretAlice) carol := mtUser("carol", "mtproto", secretCarol) @@ -467,13 +470,13 @@ func TestIntegrationHotReloadKeepsUnchangedSessions(t *testing.T) { l.expectPID(pid) carolSession := l.expectAdmitted("carol", secretCarol) - l.passSignalGrace() + l.waitHandlesSIGHUP() sighups := l.sighups() if err := l.mt.SyncUsers(l.ctx, []*common.User{alice, mtUser("dave", "mtproto", secretDave)}); err != nil { t.Fatalf("SyncUsers without carol: %v", err) } if l.sighups() <= sighups { - t.Error("telemt logged no SIGHUP for a sync after telemtSignalGrace") + t.Error("telemt logged no SIGHUP for a sync once it handled SIGHUP") } l.expectCut(carolSession, "carol") l.expectOpen(aliceSession, "alice", 1) diff --git a/backend/mtproto/lifecycle_test.go b/backend/mtproto/lifecycle_test.go index 2d1e0468..cdb22fbf 100644 --- a/backend/mtproto/lifecycle_test.go +++ b/backend/mtproto/lifecycle_test.go @@ -391,9 +391,10 @@ func TestFullSyncSignalsTelemt(t *testing.T) { } } -// telemt installs its SIGHUP handler with its config watcher, after its API -// is up, and a SIGHUP before that ends it. A sync right after the start must -// leave the reload to the watcher instead of killing telemt. +// telemt installs its SIGHUP handler at the end of its config watcher's +// setup, which can finish after its API first answers, and a SIGHUP before +// that ends it. A sync right after the start must leave the reload to the +// watcher instead of killing telemt. func TestSyncRightAfterStartDoesNotSignalTelemt(t *testing.T) { signals := filepath.Join(t.TempDir(), "signals") exe := writeFakeTelemt(t, fakeTelemt{ServeUsers: true, WatchConfig: true, SignalDelay: time.Second, SignalFile: signals}) @@ -416,16 +417,19 @@ func TestSyncRightAfterStartDoesNotSignalTelemt(t *testing.T) { } // telemt's config watcher misses a render written while it sets itself up -// right after the start, and only SIGHUP makes telemt load it then. The -// node sends it once telemtSignalGrace has passed. +// right after the start, and only SIGHUP makes telemt load it then. Where +// the node cannot read telemt's signal handlers, it sends SIGHUP once +// telemtSignalGrace has passed. The watcher's setup outlasts the grace, so +// the render is missed whenever the sync lands. func TestRenderMissedByWatcherGetsSIGHUPAfterGrace(t *testing.T) { + withoutSIGHUPProbe(t) lowerSignalGrace(t, 1500*time.Millisecond) lowerMarkerConfirmTimeout(t, 10*time.Second) signals := filepath.Join(t.TempDir(), "signals") exe := writeFakeTelemt(t, fakeTelemt{ ServeUsers: true, WatchConfig: true, - WatchSetup: 600 * time.Millisecond, + WatchSetup: 2 * time.Second, SignalDelay: 700 * time.Millisecond, SignalFile: signals, }) diff --git a/backend/mtproto/reload.go b/backend/mtproto/reload.go index 82ff713a..a6422b91 100644 --- a/backend/mtproto/reload.go +++ b/backend/mtproto/reload.go @@ -22,13 +22,14 @@ const ( ) // telemtSignalGrace is how long after telemt's API first answers the node -// does not send SIGHUP. telemt's config watcher installs the SIGHUP handler -// only after the API is up and the watcher has read the config and set up -// its watches, and a SIGHUP before that ends telemt. The watcher loads most -// renders of that time on its own, but misses one written while it sets up. -// It is a variable so tests can lower it. +// does not send SIGHUP where it cannot see telemt's signal handlers (see +// telemtHandlesSIGHUPLocked). It is a variable so tests can lower it. var telemtSignalGrace = 5 * time.Second +// readSIGHUPCaught is sighupCaught; tests replace it to make the node fall +// back to telemtSignalGrace. +var readSIGHUPCaught = sighupCaught + // markerConfirmTimeout bounds the wait for telemt to load a rendered config; // a config it did not load by then counts as rejected. It is a variable so // tests can lower it. @@ -66,7 +67,7 @@ func newMarkerUser() (*runtimeUser, error) { // reloadLocked makes the running telemt hot-reload the rendered config and // waits until the render's marker user is in telemt's running config. // telemt's config watcher sees the file change, and SIGHUP makes sure once -// telemtSignalGrace allows it (see waitForMarker). A hot reload replaces +// telemt handles it (see waitForMarkerLocked). A hot reload replaces // only the users that changed; an API reload would cut every session. Only // an API that stays unreachable, or a telemt that exited or cannot be // signalled, restarts telemt, on the rendered config; a rejected config @@ -77,7 +78,7 @@ func (m *MTProto) reloadLocked(ctx context.Context) error { } // Should telemt not handle the signal after all, SIGHUP ends it and the // restart below recovers it. - err := m.waitForMarker(ctx, m.marker.Username, m.healthyAt.Add(telemtSignalGrace)) + err := m.waitForMarkerLocked(ctx, m.marker.Username) switch { case err == nil || errors.Is(err, errClosed) || errors.Is(err, errReloadRejected): return err @@ -94,25 +95,41 @@ func (m *MTProto) reloadLocked(ctx context.Context) error { return nil } -// signalReloadLocked sends telemt SIGHUP, unless its API first answered -// less than telemtSignalGrace ago; telemt's config watcher sees the change -// then. Requires syncMu. +// signalReloadLocked sends telemt SIGHUP if it handles the signal yet (see +// telemtHandlesSIGHUPLocked); before that, telemt's config watcher sees the +// change. Requires syncMu. func (m *MTProto) signalReloadLocked() error { - if time.Since(m.healthyAt) < telemtSignalGrace { + if !m.telemtHandlesSIGHUPLocked() { return nil } return m.process.Reload() } -// waitForMarker polls GET /v1/users until telemt's running config has the -// marker user, for up to markerConfirmTimeout, and sends telemt SIGHUP once -// signalAt has passed: at once in steady state, or when telemtSignalGrace -// ends if the watcher has not loaded the render by then. It returns an -// errReloadRejected error when telemt cannot read the rendered file or the -// time is up, errClosed on Shutdown, ctx's error when ctx ends, and the -// signal or request error when telemt cannot be signalled, the API stays -// unreachable or telemt exited. -func (m *MTProto) waitForMarker(ctx context.Context, marker string, signalAt time.Time) error { +// telemtHandlesSIGHUPLocked reports whether the running telemt can take +// SIGHUP. telemt's config watcher installs its SIGHUP handler at the end of +// its own setup, which runs concurrently with the API coming up and can +// finish after the API first answers; a SIGHUP before that ends telemt. +// Once installed, the handler stays for the process lifetime. The watcher +// loads most renders of that window on its own, but misses one written +// while it sets up. On Linux the node reads the handler from telemt's +// SigCgt mask (sighupCaught); elsewhere, or when that read fails, it waits +// telemtSignalGrace after the API first answered. Requires syncMu. +func (m *MTProto) telemtHandlesSIGHUPLocked() bool { + if caught, err := readSIGHUPCaught(m.process.PID()); err == nil { + return caught + } + return time.Since(m.healthyAt) >= telemtSignalGrace +} + +// waitForMarkerLocked polls GET /v1/users until telemt's running config has +// the marker user, for up to markerConfirmTimeout, and sends telemt SIGHUP +// once telemt handles it: at once in steady state, or once the config +// watcher installed the handler if it has not loaded the render by then. +// It returns an errReloadRejected error when telemt cannot read the +// rendered file or the time is up, errClosed on Shutdown, ctx's error when +// ctx ends, and the signal or request error when telemt cannot be +// signalled, the API stays unreachable or telemt exited. Requires syncMu. +func (m *MTProto) waitForMarkerLocked(ctx context.Context, marker string) error { waitCtx, cancel := context.WithTimeout(ctx, markerConfirmTimeout) defer cancel() stop := context.AfterFunc(m.lifetime, cancel) @@ -125,7 +142,7 @@ func (m *MTProto) waitForMarker(ctx context.Context, marker string, signalAt tim signalled := false unanswered := 0 for { - if !signalled && !time.Now().Before(signalAt) { + if !signalled && m.telemtHandlesSIGHUPLocked() { signalled = true if err := m.process.Reload(); err != nil { return err diff --git a/backend/mtproto/sighup_linux.go b/backend/mtproto/sighup_linux.go new file mode 100644 index 00000000..d326ac6e --- /dev/null +++ b/backend/mtproto/sighup_linux.go @@ -0,0 +1,43 @@ +//go:build linux + +package mtproto + +import ( + "errors" + "fmt" + "os" + "strconv" + "strings" +) + +// sighupCaught reports whether process pid has a SIGHUP handler installed, +// from the SigCgt mask in /proc//status. +func sighupCaught(pid int) (bool, error) { + if pid <= 0 { + return false, errors.New("telemt process is not running") + } + status, err := os.ReadFile("/proc/" + strconv.Itoa(pid) + "/status") + if err != nil { + return false, err + } + return parseSIGHUPCaught(string(status)) +} + +// parseSIGHUPCaught reads the SIGHUP bit of the SigCgt hex mask in the text +// of /proc//status: signal n is bit n-1, so SIGHUP (1) is bit 0. +func parseSIGHUPCaught(status string) (bool, error) { + for _, line := range strings.Split(status, "\n") { + mask, ok := strings.CutPrefix(line, "SigCgt:") + if !ok { + continue + } + mask = strings.TrimSpace(mask) + if mask == "" || strings.Trim(mask, "0123456789abcdefABCDEF") != "" { + return false, fmt.Errorf("unexpected SigCgt mask %q", mask) + } + // The last hex digit holds signals 1 to 4, whatever the mask width. + digit, _ := strconv.ParseUint(mask[len(mask)-1:], 16, 8) + return digit&1 != 0, nil + } + return false, errors.New("no SigCgt line in the process status") +} diff --git a/backend/mtproto/sighup_linux_test.go b/backend/mtproto/sighup_linux_test.go new file mode 100644 index 00000000..f2daba9f --- /dev/null +++ b/backend/mtproto/sighup_linux_test.go @@ -0,0 +1,94 @@ +//go:build linux + +package mtproto + +import ( + "context" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + "github.com/pasarguard/node/common" +) + +func TestParseSIGHUPCaught(t *testing.T) { + status := func(sigCgt string) string { + return "Name:\ttelemt\nState:\tS (sleeping)\nPid:\t4242\n" + + "SigQ:\t0/63414\nSigPnd:\t0000000000000000\nShdPnd:\t0000000000000000\n" + + "SigBlk:\t0000000000000000\nSigIgn:\t0000000000001000\n" + + "SigCgt:\t" + sigCgt + "\nCapInh:\t0000000000000000\n" + } + for name, test := range map[string]struct { + status string + caught bool + fails bool + }{ + // telemt 3.5.14 right after exec: SIGBUS and SIGSEGV only. + "without SIGHUP": {status: status("0000000000000440")}, + "with SIGHUP": {status: status("0000000000004e47"), caught: true}, + "every other signal": {status: status("fffffffffffffffe")}, + "upper-case hex": {status: status("FFFFFFFD7FC1FEFF"), caught: true}, + "128-bit mask": {status: status("00000000000000000000000000000441"), caught: true}, + "no SigCgt line": {status: "Name:\ttelemt\nSigIgn:\t0000000000000000\n", fails: true}, + "empty mask": {status: status(""), fails: true}, + "not a hex mask": {status: status("00000000000004g1"), fails: true}, + "SigCgt in the value": {status: "Name:\tSigCgt:\t1\n", fails: true}, + } { + t.Run(name, func(t *testing.T) { + caught, err := parseSIGHUPCaught(test.status) + if test.fails { + if err == nil { + t.Fatalf("parseSIGHUPCaught = %v, want an error", caught) + } + return + } + if err != nil || caught != test.caught { + t.Fatalf("parseSIGHUPCaught = %v, %v; want %v", caught, err, test.caught) + } + }) + } +} + +func TestSIGHUPCaughtWithoutProcess(t *testing.T) { + if _, err := sighupCaught(0); err == nil { + t.Error("sighupCaught(0) succeeded") + } +} + +// TestRenderMissedByWatcherGetsSIGHUPOnceHandled: on Linux the node sends +// the SIGHUP for a render telemt's config watcher missed once telemt's +// SigCgt mask shows the handler, not on a clock. telemtSignalGrace is out +// of reach here, so only the mask can let the signal through; one sent +// before the handler would end the fake (a restart), and none would leave +// the render unloaded. +func TestRenderMissedByWatcherGetsSIGHUPOnceHandled(t *testing.T) { + lowerSignalGrace(t, time.Hour) + lowerMarkerConfirmTimeout(t, 10*time.Second) + signals := filepath.Join(t.TempDir(), "signals") + exe := writeFakeTelemt(t, fakeTelemt{ + ServeUsers: true, + WatchConfig: true, + WatchSetup: 2 * time.Second, + SignalDelay: 700 * time.Millisecond, + SignalFile: signals, + }) + mt := newTestBackend(t, testNodeConfig(t, exe), []*common.User{mtUser("alice", "mtproto", testSecret)}) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if err := mt.SyncUsers(ctx, []*common.User{mtUser("bob", "mtproto", testSecret)}); err != nil { + t.Fatalf("SyncUsers: %v", err) + } + if !mt.Started() || mt.process.Generation() != 1 { + t.Errorf("telemt was restarted (started %v, generation %d)", mt.Started(), mt.process.Generation()) + } + if data, _ := os.ReadFile(signals); strings.Count(string(data), "SIGHUP") != 1 { + t.Errorf("signal log %q, want one SIGHUP", data) + } + if got := desiredNames(mt); !slices.Equal(got, []string{"bob"}) { + t.Errorf("desired = %q, want bob", got) + } +} diff --git a/backend/mtproto/sighup_other.go b/backend/mtproto/sighup_other.go new file mode 100644 index 00000000..1e447f12 --- /dev/null +++ b/backend/mtproto/sighup_other.go @@ -0,0 +1,11 @@ +//go:build !linux + +package mtproto + +import "errors" + +// sighupCaught cannot read signal handlers outside Linux, so the node falls +// back to telemtSignalGrace. +func sighupCaught(int) (bool, error) { + return false, errors.ErrUnsupported +} From 759f4807194f1c45c7170615f8006bab3ab373fa Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Tue, 20 Oct 2026 01:30:00 +0330 Subject: [PATCH 20/40] fix(mtproto): restart telemt whose API hangs during a reload A telemt whose API accepts connections but never answers was diagnosed as a rejected reload: each users poll ran into the 10 s client timeout, so the 30 s marker budget ran out before three unanswered polls, the node restored the previous render and reported a rejection, and nothing restarted telemt. Each marker poll now gets its own timeout, a quarter of the budget (7.5 s), so three hung polls in a row reach the existing restart branch at about 22.5 s. A shorter timeout would misread a large node: telemt reads its whole config for each poll, and listing 150k users took about 2.5 s on live telemt 3.5.14. --- backend/mtproto/reload.go | 26 ++++++++++++++++++----- backend/mtproto/sync_test.go | 40 ++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 5 deletions(-) diff --git a/backend/mtproto/reload.go b/backend/mtproto/reload.go index a6422b91..6810f0fe 100644 --- a/backend/mtproto/reload.go +++ b/backend/mtproto/reload.go @@ -35,6 +35,16 @@ var readSIGHUPCaught = sighupCaught // tests can lower it. var markerConfirmTimeout = 30 * time.Second +// markerPollTimeout bounds one poll of the marker wait, so an API that +// accepts connections but never answers counts as unanswered, and +// markerUnreachablePolls such polls in a row end inside markerConfirmTimeout +// (and restart telemt). It is a quarter of that budget, 7.5 s, because +// telemt reads its whole config for each poll: listing 150k users took +// about 2.5 s on a fast host. +func markerPollTimeout() time.Duration { + return markerConfirmTimeout / (markerUnreachablePolls + 1) +} + // errReloadRejected marks a rendered config telemt did not load. telemt // keeps running the config it had. var errReloadRejected = errors.New("telemt did not load the new config") @@ -125,10 +135,11 @@ func (m *MTProto) telemtHandlesSIGHUPLocked() bool { // the marker user, for up to markerConfirmTimeout, and sends telemt SIGHUP // once telemt handles it: at once in steady state, or once the config // watcher installed the handler if it has not loaded the render by then. -// It returns an errReloadRejected error when telemt cannot read the -// rendered file or the time is up, errClosed on Shutdown, ctx's error when -// ctx ends, and the signal or request error when telemt cannot be -// signalled, the API stays unreachable or telemt exited. Requires syncMu. +// Each poll gets markerPollTimeout. It returns an errReloadRejected error +// when telemt cannot read the rendered file or the time is up, errClosed on +// Shutdown, ctx's error when ctx ends, and the signal or request error when +// telemt cannot be signalled, the API stays unreachable or telemt exited. +// Requires syncMu. func (m *MTProto) waitForMarkerLocked(ctx context.Context, marker string) error { waitCtx, cancel := context.WithTimeout(ctx, markerConfirmTimeout) defer cancel() @@ -139,6 +150,7 @@ func (m *MTProto) waitForMarkerLocked(ctx context.Context, marker string) error ticker := time.NewTicker(markerPollInterval) defer ticker.Stop() + pollTimeout := markerPollTimeout() signalled := false unanswered := 0 for { @@ -149,7 +161,9 @@ func (m *MTProto) waitForMarkerLocked(ctx context.Context, marker string) error } } - users, err := m.apiClient.ListUsers(waitCtx) + pollCtx, cancelPoll := context.WithTimeout(waitCtx, pollTimeout) + users, err := m.apiClient.ListUsers(pollCtx) + cancelPoll() var apiErr *apiStatusError switch { case err == nil: @@ -165,6 +179,8 @@ func (m *MTProto) waitForMarkerLocked(ctx context.Context, marker string) error return fmt.Errorf("%w: %s", errReloadRejected, apiErr.Message) } case waitCtx.Err() == nil: + // No answer: a refused or reset connection, or a poll that ran + // out of pollTimeout. unanswered++ if unanswered >= markerUnreachablePolls || isClosed(exited) { return fmt.Errorf("telemt API unreachable: %w", err) diff --git a/backend/mtproto/sync_test.go b/backend/mtproto/sync_test.go index f14e6d4f..2ad30f3e 100644 --- a/backend/mtproto/sync_test.go +++ b/backend/mtproto/sync_test.go @@ -15,6 +15,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "testing" "time" @@ -959,6 +960,45 @@ func TestSingleAPIErrorKeepsPolling(t *testing.T) { } } +// TestHungUsersAPIIsUnreachable: an API that accepts connections but never +// answers gets markerPollTimeout per poll, so markerUnreachablePolls polls +// in a row end the marker wait inside markerConfirmTimeout as an +// unreachable telemt, which reloadLocked restarts, not as a rejected config. +func TestHungUsersAPIIsUnreachable(t *testing.T) { + lowerMarkerConfirmTimeout(t, 4*time.Second) + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{mtUser("u1", "mtproto", testSecret)}) + + var polls atomic.Int32 + release := make(chan struct{}) + hung := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + polls.Add(1) + select { + case <-r.Context().Done(): + case <-release: + } + })) + t.Cleanup(hung.Close) + t.Cleanup(func() { close(release) }) + m.apiClient = newAPIClient(hung.URL, "Bearer test") + + m.syncMu.Lock() + started := time.Now() + err := m.waitForMarkerLocked(context.Background(), m.markerUsername()) + elapsed := time.Since(started) + m.syncMu.Unlock() + + if err == nil || errors.Is(err, errReloadRejected) || !strings.Contains(err.Error(), "telemt API unreachable") { + t.Fatalf("waitForMarkerLocked = %v, want telemt API unreachable", err) + } + if elapsed >= markerConfirmTimeout { + t.Errorf("the wait took %s, not inside markerConfirmTimeout (%s)", elapsed, markerConfirmTimeout) + } + if got := polls.Load(); got != markerUnreachablePolls { + t.Errorf("%d polls reached the API, want %d", got, markerUnreachablePolls) + } +} + func TestFullSyncWithoutChangesSkipsReload(t *testing.T) { api := newFakeTelemtAPI(t) users := []*common.User{mtUser("u1", "mtproto", testSecret), mtUser("u2", "mtproto", testSecret)} From b48561da8c154d952cfd7d395a6bc633701bf82f Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Tue, 20 Oct 2026 01:35:50 +0330 Subject: [PATCH 21/40] test(mtproto): cover Shutdown during a full apply in the marker wait A full apply blocked waiting for telemt to load its render must not hold Shutdown for the 30 s marker budget. The test starts a sync against a fake telemt that never loads the render, waits until the marker wait polls GET /v1/users (the fake now logs those requests), and calls Shutdown: Shutdown returns promptly, the apply ends with errClosed, needsFullApply is set on the closed backend, desired keeps the confirmed users and telemt is not restarted. --- backend/mtproto/fake_telemt_test.go | 3 ++ backend/mtproto/lifecycle_test.go | 50 +++++++++++++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/backend/mtproto/fake_telemt_test.go b/backend/mtproto/fake_telemt_test.go index 1dbb9c08..208f82cc 100644 --- a/backend/mtproto/fake_telemt_test.go +++ b/backend/mtproto/fake_telemt_test.go @@ -52,6 +52,8 @@ type fakeTelemt struct { // DropUsers closes GET /v1/users requests without an answer, like an API // that cannot be reached. DropUsers bool `json:"drop_users"` + // UsersRequestFile gets one line per GET /v1/users request. + UsersRequestFile string `json:"users_request_file"` // SignalFile gets one line per SIGHUP received. SignalFile string `json:"signal_file"` // SignalDelay makes the fake handle SIGHUP only this long after it binds @@ -276,6 +278,7 @@ func serveFakeTelemtAPI(f fakeTelemt, configPath string, runtime *fakeTelemtRunt _, _ = w.Write([]byte(`{"ok":true,"data":{"ready":false,"status":"not_ready","reason":"no_healthy_upstreams"},"revision":"r"}`)) }) mux.HandleFunc("/v1/users", func(w http.ResponseWriter, _ *http.Request) { + appendFakeLine(f.UsersRequestFile, "GET /v1/users") if f.DropUsers { if conn, _, err := http.NewResponseController(w).Hijack(); err == nil { _ = conn.Close() diff --git a/backend/mtproto/lifecycle_test.go b/backend/mtproto/lifecycle_test.go index cdb22fbf..015c25df 100644 --- a/backend/mtproto/lifecycle_test.go +++ b/backend/mtproto/lifecycle_test.go @@ -472,6 +472,56 @@ func TestUsersAPIUnreachableRestartsTelemt(t *testing.T) { } } +// TestShutdownEndsApplyInMarkerWait: Shutdown while a full apply waits for +// telemt to load its render returns at once instead of after +// markerConfirmTimeout. The apply ends with errClosed and leaves +// needsFullApply set, desired keeps the confirmed users, and telemt is not +// restarted. telemt here never loads the render: it has no config watcher +// and never handles SIGHUP, so the node never sends it. +func TestShutdownEndsApplyInMarkerWait(t *testing.T) { + lowerSignalGrace(t, time.Hour) + polls := filepath.Join(t.TempDir(), "polls") + exe := writeFakeTelemt(t, fakeTelemt{ServeUsers: true, SignalDelay: time.Hour, UsersRequestFile: polls}) + mt := newTestBackend(t, testNodeConfig(t, exe), []*common.User{mtUser("alice", "mtproto", testSecret)}) + + applied := make(chan error, 1) + go func() { + applied <- mt.SyncUsers(context.Background(), []*common.User{mtUser("bob", "mtproto", testSecret)}) + }() + // Only the marker wait lists users here. + waitFor(t, 10*time.Second, "the marker wait to poll telemt", func() bool { + data, _ := os.ReadFile(polls) + return len(data) > 0 + }) + + started := time.Now() + mt.Shutdown() + if elapsed := time.Since(started); elapsed > 5*time.Second { + t.Errorf("Shutdown took %s during an apply", elapsed) + } + select { + case err := <-applied: + if !errors.Is(err, errClosed) { + t.Errorf("SyncUsers = %v, want errClosed", err) + } + case <-time.After(5 * time.Second): + t.Fatal("the apply did not end on Shutdown") + } + + mt.syncMu.Lock() + closed, needsFullApply := mt.closed, mt.needsFullApply + mt.syncMu.Unlock() + if !closed || !needsFullApply { + t.Errorf("closed %v, needsFullApply %v after Shutdown cut an apply; want both", closed, needsFullApply) + } + if got := desiredNames(mt); !slices.Equal(got, []string{"alice"}) { + t.Errorf("desired = %q, want the confirmed alice", got) + } + if generation := mt.process.Generation(); generation != 1 { + t.Errorf("telemt was restarted during Shutdown (generation %d)", generation) + } +} + // TestCancelledContextNeverRestartsTelemt covers the paths below the entry // check: a reload whose caller is gone keeps telemt, and a start with a done // context spawns nothing. From 730ac6a10346ba09d9139cd9b741651685cce73d Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Thu, 22 Oct 2026 21:51:31 +0330 Subject: [PATCH 22/40] test(controller): cover a slow MTProto Start outliving its caller Dev now runs every core start under a context that keeps the request's values but not its cancellation (#116), and a same-config Start is a heartbeat only when the running backend passes GetSysStats (#117). The panel relies on both for MTProto: telemt may need up to 90 s to bind its API, the panel gives up on Start after 10 s, and its retry must attach to the telemt that came up instead of restarting it. Run the controller against a fake telemt (the test binary re-executed through a wrapper script, so the node starts and reaps a real child) that binds its API after a delay: a Start whose caller already went away ends with telemt running, telemt keeps running after the start context ends, and a retry that waits behind the start attaches to the same backend. --- controller/mtproto_start_unix_test.go | 257 ++++++++++++++++++++++++++ 1 file changed, 257 insertions(+) create mode 100644 controller/mtproto_start_unix_test.go diff --git a/controller/mtproto_start_unix_test.go b/controller/mtproto_start_unix_test.go new file mode 100644 index 00000000..d48216ba --- /dev/null +++ b/controller/mtproto_start_unix_test.go @@ -0,0 +1,257 @@ +//go:build !windows + +package controller + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/google/uuid" + "google.golang.org/protobuf/proto" + + "github.com/pasarguard/node/backend" + "github.com/pasarguard/node/common" + "github.com/pasarguard/node/config" + "github.com/pasarguard/node/pkg/netutil" +) + +// The fake telemt is this test binary run as TestHelperFakeTelemt through a +// wrapper script, so the node starts, signals and reaps a real child process. +const ( + // fakeTelemtHealthDelayEnv holds how long the fake waits before it binds + // its API, like telemt waiting for Telegram DCs; "never" means it never does. + fakeTelemtHealthDelayEnv = "CONTROLLER_FAKE_TELEMT_HEALTH_DELAY" + // fakeTelemtPIDFileEnv names the file the fake writes its pid to. + fakeTelemtPIDFileEnv = "CONTROLLER_FAKE_TELEMT_PID_FILE" +) + +// TestHelperFakeTelemt is not a real test: it is the fake telemt process. +func TestHelperFakeTelemt(t *testing.T) { + delay, ok := os.LookupEnv(fakeTelemtHealthDelayEnv) + if !ok { + t.Skip("helper process for the fake telemt; runs only when re-executed by a test") + } + os.Exit(runFakeTelemt(delay, os.Getenv(fakeTelemtPIDFileEnv), os.Args)) +} + +func runFakeTelemt(delay, pidFile string, args []string) int { + // The wrapper passes telemt's own arguments after "--". + for i, arg := range args { + if arg == "--" { + args = args[i+1:] + break + } + } + if len(args) > 0 && args[0] == "--version" { + fmt.Println("telemt 3.5.14") + return 0 + } + if len(args) != 2 || args[0] != "run" { + fmt.Fprintln(os.Stderr, "fake telemt: unexpected arguments", args) + return 2 + } + if pidFile != "" { + if err := os.WriteFile(pidFile, []byte(strconv.Itoa(os.Getpid())), 0o600); err != nil { + fmt.Fprintln(os.Stderr, "fake telemt:", err) + return 2 + } + } + // Never outlive a broken test by much. + go func() { + time.Sleep(2 * time.Minute) + os.Exit(3) + }() + + if delay == "never" { + select {} + } + wait, err := time.ParseDuration(delay) + if err != nil { + fmt.Fprintln(os.Stderr, "fake telemt: bad health delay:", err) + return 2 + } + time.Sleep(wait) + + listen, err := fakeTelemtAPIListen(args[1]) + if err != nil { + fmt.Fprintln(os.Stderr, "fake telemt:", err) + return 2 + } + mux := http.NewServeMux() + mux.HandleFunc("/v1/health", func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"ok":true,"data":{"status":"ok","read_only":false},"revision":"r"}`)) + }) + mux.HandleFunc("/v1/health/ready", func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(`{"ok":true,"data":{"ready":true,"status":"ready"},"revision":"r"}`)) + }) + if err := http.ListenAndServe(listen, mux); err != nil { + fmt.Fprintln(os.Stderr, "fake telemt: api:", err) + return 1 + } + return 0 +} + +// fakeTelemtAPIListen reads server.api.listen from the rendered config. +func fakeTelemtAPIListen(configPath string) (string, error) { + data, err := os.ReadFile(configPath) + if err != nil { + return "", err + } + section := "" + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "[") { + section = strings.Trim(line, "[]") + continue + } + key, value, ok := strings.Cut(line, "=") + if ok && section == "server.api" && strings.TrimSpace(key) == "listen" { + return strconv.Unquote(strings.TrimSpace(value)) + } + } + return "", errors.New("server.api.listen not found in " + configPath) +} + +// newFakeTelemtController returns a controller whose telemt is the fake. It +// binds its API after healthDelay ("never" for a start that never ends on +// its own) and writes its pid to the returned file. +func newFakeTelemtController(t *testing.T, healthDelay string) (*Controller, string) { + t.Helper() + testBinary, err := filepath.Abs(os.Args[0]) + if err != nil { + t.Fatal(err) + } + dir := t.TempDir() + pidFile := filepath.Join(dir, "telemt.pid") + script := fmt.Sprintf("#!/bin/sh\nexport %s=%s %s=%s\nexec %s -test.run='^TestHelperFakeTelemt$' -- \"$@\"\n", + fakeTelemtHealthDelayEnv, shellQuote(healthDelay), fakeTelemtPIDFileEnv, shellQuote(pidFile), shellQuote(testBinary)) + executable := filepath.Join(dir, "telemt") + if err := os.WriteFile(executable, []byte(script), 0o700); err != nil { + t.Fatal(err) + } + + cfg := config.NewTestConfig(t.TempDir(), uuid.New()) + cfg.TelemtExecutablePath = executable + c := New(cfg) + t.Cleanup(c.Disconnect) + return c, pidFile +} + +func shellQuote(value string) string { + return "'" + strings.ReplaceAll(value, "'", `'\''`) + "'" +} + +func mtprotoStartRequest() *common.Backend { + return &common.Backend{ + Type: common.BackendType_MTPROTO, + Config: fmt.Sprintf(`{"inbound_tag": "mtproto", "server": {"port": %d, "listeners": [{"ip": "127.0.0.1"}]}}`, + netutil.FindFreePort()), + KeepAlive: 60, + Users: []*common.User{{ + Email: "1", + Inbounds: []string{"mtproto"}, + Proxies: &common.Proxy{Mtproto: &common.Mtproto{Secret: "0123456789abcdef0123456789abcdef"}}, + }}, + } +} + +// readFakeTelemtPID waits for the fake to write its pid. +func readFakeTelemtPID(t *testing.T, pidFile string) int { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for { + if data, err := os.ReadFile(pidFile); err == nil { + if pid, err := strconv.Atoi(string(data)); err == nil { + return pid + } + } + if time.Now().After(deadline) { + t.Fatal("the fake telemt did not start within 10s") + } + time.Sleep(10 * time.Millisecond) + } +} + +// A slow MTProto Start (telemt waits for Telegram DCs before it binds its +// API) must not die with the panel's request, like an xray start. +func TestMTProtoStartOutlivesItsCaller(t *testing.T) { + c, _ := newFakeTelemtController(t, "1s") + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + c.LockControl() + err := c.StartOrAttach(ctx, mtprotoStartRequest()) + back := c.Backend() + c.UnlockControl() + if err != nil { + t.Fatalf("an MTProto Start whose caller went away failed: %v", err) + } + if back == nil || !back.Started() { + t.Fatal("no running telemt after a Start whose caller went away") + } + // The start context ends when StartOrAttach returns: telemt must not + // follow it. + time.Sleep(500 * time.Millisecond) + if !back.Started() { + t.Fatal("telemt stopped once StartOrAttach returned") + } +} + +// The panel retries a timed-out MTProto Start with the same config. The retry +// waits for the start in progress and attaches to it: a same-config Start is +// a heartbeat once the running backend passes its health check. +func TestMTProtoRetryAttachesToAStartInProgress(t *testing.T) { + c, _ := newFakeTelemtController(t, "1s") + first := mtprotoStartRequest() + retry := proto.Clone(first).(*common.Backend) + + type result struct { + err error + back backend.Backend + } + start := func(ctx context.Context, data *common.Backend, locked chan<- struct{}, out chan<- result) { + c.LockControl() + if locked != nil { + close(locked) + } + err := c.StartOrAttach(ctx, data) + back := c.Backend() + c.UnlockControl() + out <- result{err, back} + } + + gaveUp, cancel := context.WithCancel(context.Background()) + cancel() + locked := make(chan struct{}) + firstDone := make(chan result, 1) + go start(gaveUp, first, locked, firstDone) + <-locked + waitForStartInProgress(t, c) + + retryCtx, cancelRetry := context.WithTimeout(context.Background(), time.Minute) + defer cancelRetry() + retryDone := make(chan result, 1) + go start(retryCtx, retry, nil, retryDone) + + r1, r2 := <-firstDone, <-retryDone + if r1.err != nil { + t.Fatalf("first Start: %v", r1.err) + } + if r2.err != nil { + t.Fatalf("retried Start: %v", r2.err) + } + if r2.back != r1.back { + t.Fatal("the retried Start restarted telemt instead of attaching to it") + } + if !r2.back.Started() { + t.Fatal("no running telemt after the retried Start") + } +} From 7e69e36a55acba0b31d23fea51d7da511fbae4be Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Thu, 22 Oct 2026 21:52:30 +0330 Subject: [PATCH 23/40] fix(mtproto): let Disconnect abort a telemt start in progress New detached the telemt start from its context (WithoutCancel plus its own 90 s bound) so a slow start would outlive the panel's 10 s Start request. Dev's controller now does that for every backend (#116): it runs StartOrAttach under a context without the request's cancellation and cancels it only from Disconnect, which process shutdown calls without LockControl, so it does not wait for a start in progress. The second detachment in New and startLocked ignored that abort. A Disconnect during a telemt start that never binds its API (Telegram unreachable) blocked on the controller mutex for the whole 90 s telemtStartupTimeout, and the telemt being started kept running until then. New and startLocked now follow their context; waitForHealth still bounds the wait by telemtStartupTimeout and reports Shutdown as errClosed. Every other start already passes a context the caller cannot cancel: Restart and the supervisor the backend lifetime, a full apply its detached apply context, the restart after a hung reload a WithoutCancel context. The panel sees no change: the Start outlives its request through the controller, as the controller tests show. --- backend/mtproto/lifecycle.go | 29 +++++++++------- backend/mtproto/lifecycle_test.go | 48 +++++++++++++++++++-------- controller/mtproto_start_unix_test.go | 43 ++++++++++++++++++++++++ 3 files changed, 94 insertions(+), 26 deletions(-) diff --git a/backend/mtproto/lifecycle.go b/backend/mtproto/lifecycle.go index b6bc1129..90c4b136 100644 --- a/backend/mtproto/lifecycle.go +++ b/backend/mtproto/lifecycle.go @@ -93,16 +93,16 @@ func New(ctx context.Context, cfg *config.Config, mtConfig *Config, users []*com cancel: cancel, } - // The panel's Start call has a short deadline (10 s by default) while - // telemt may need far longer to bind its API. Let a slow start finish - // anyway: the panel's retry then attaches to the running backend. - startCtx, cancelStart := context.WithTimeout(context.WithoutCancel(ctx), telemtStartupTimeout) - defer cancelStart() - + // telemt may need far longer than the panel's Start call (10 s by + // default) to bind its API. The controller runs a start under a context + // that outlives that call and is cancelled only to abort the start + // (Disconnect), so a slow start finishes and the panel's retry attaches. + // waitForHealth bounds it by telemtStartupTimeout; cancelling ctx stops + // telemt at once. mt.syncMu.Lock() err = mt.writeRenderedConfigLocked(mt.desired) if err == nil { - err = mt.startLocked(startCtx) + err = mt.startLocked(ctx) } mt.syncMu.Unlock() if err != nil { @@ -193,7 +193,10 @@ func generateBearerToken() (string, error) { // startLocked starts telemt on the rendered config and waits for its API. // A failure stops telemt and carries its last output. Readiness is watched in -// the background. Requires syncMu. +// the background. Cancelling ctx aborts the start and stops telemt, so a +// caller that may give up while telemt still binds passes a context detached +// from its own (the controller's start context, the detached apply context). +// Requires syncMu. func (m *MTProto) startLocked(ctx context.Context) error { if m.lifetime.Err() != nil { return errClosed @@ -204,9 +207,7 @@ func (m *MTProto) startLocked(ctx context.Context) error { if err := m.process.Start(m.configPath); err != nil { return err } - // Once telemt runs, only telemtStartupTimeout and Shutdown bound its - // startup: a caller that gives up must not kill a telemt still binding. - if err := m.waitForHealth(context.WithoutCancel(ctx)); err != nil { + if err := m.waitForHealth(ctx); err != nil { m.process.Stop() return m.startupError(err) } @@ -264,7 +265,7 @@ func (m *MTProto) startupError(err error) error { } // waitForHealth polls /v1/health until it answers, telemt exits, the backend -// shuts down, or telemtStartupTimeout passes. +// shuts down, ctx ends, or telemtStartupTimeout passes. func (m *MTProto) waitForHealth(ctx context.Context) error { timeout := telemtStartupTimeout waitCtx, cancel := context.WithTimeout(ctx, timeout) @@ -285,6 +286,10 @@ func (m *MTProto) waitForHealth(ctx context.Context) error { case <-m.lifetime.Done(): return errClosed case <-waitCtx.Done(): + // Shutdown also ends the contexts of the restarts and applies. + if m.lifetime.Err() != nil { + return errClosed + } if err := ctx.Err(); err != nil { return fmt.Errorf("stopped waiting for the telemt API: %w", err) } diff --git a/backend/mtproto/lifecycle_test.go b/backend/mtproto/lifecycle_test.go index 015c25df..0fe3f392 100644 --- a/backend/mtproto/lifecycle_test.go +++ b/backend/mtproto/lifecycle_test.go @@ -304,25 +304,45 @@ func TestWaitForReadyGivesUp(t *testing.T) { } } -// TestStartOutlivesRequestDeadline models the panel's 10 s Start call against -// a telemt that binds its API later: the start must complete anyway, so the -// panel's retry attaches to a running backend. -func TestStartOutlivesRequestDeadline(t *testing.T) { - exe := writeFakeTelemt(t, fakeTelemt{HealthDelay: 1500 * time.Millisecond}) +// TestCancelledStartStopsTelemt: the controller runs New under a context that +// outlives the panel's request (controller.StartOrAttach) and cancels it only +// to abort the start, on Disconnect. New must then stop waiting for telemt's +// API at once and leave no telemt running, not wait for telemtStartupTimeout. +func TestCancelledStartStopsTelemt(t *testing.T) { + lowerStartupTimeout(t, 20*time.Second) + record := filepath.Join(t.TempDir(), "record.json") + exe := writeFakeTelemt(t, fakeTelemt{NoAPI: true, RecordFile: record}) - ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + ctx, cancel := context.WithCancel(context.Background()) defer cancel() - mt, err := New(ctx, testNodeConfig(t, exe), testMTConfig(t), nil) - if err != nil { - t.Fatalf("New failed once the request deadline passed: %v", err) + type result struct { + mt *MTProto + err error } - t.Cleanup(mt.Shutdown) + done := make(chan result, 1) + go func() { + mt, err := New(ctx, testNodeConfig(t, exe), testMTConfig(t), nil) + done <- result{mt, err} + }() + + pid := readFakeTelemtRecord(t, record).PID + cancel() - if ctx.Err() == nil { - t.Fatal("the request deadline had not passed yet; the test proves nothing") + var r result + select { + case r = <-done: + case <-time.After(5 * time.Second): + t.Fatal("New still waits for telemt 5s after its context was cancelled") } - if !mt.Started() { - t.Error("backend not started") + if r.err == nil { + r.mt.Shutdown() + t.Fatal("New succeeded although its context was cancelled before telemt's API came up") + } + if !errors.Is(r.err, context.Canceled) { + t.Errorf("New error = %v, want one wrapping context.Canceled", r.err) + } + if err := syscall.Kill(pid, 0); !errors.Is(err, syscall.ESRCH) { + t.Errorf("telemt %d is still running after the cancelled start (kill 0: %v)", pid, err) } } diff --git a/controller/mtproto_start_unix_test.go b/controller/mtproto_start_unix_test.go index d48216ba..aec0c59e 100644 --- a/controller/mtproto_start_unix_test.go +++ b/controller/mtproto_start_unix_test.go @@ -11,6 +11,7 @@ import ( "path/filepath" "strconv" "strings" + "syscall" "testing" "time" @@ -255,3 +256,45 @@ func TestMTProtoRetryAttachesToAStartInProgress(t *testing.T) { t.Fatal("no running telemt after the retried Start") } } + +// Process shutdown calls Disconnect without LockControl. It must abort an +// MTProto start that waits for telemt's API instead of waiting up to the +// telemt startup timeout (90 s), and it must leave no telemt running. +func TestDisconnectAbortsAnMTProtoStartInProgress(t *testing.T) { + c, pidFile := newFakeTelemtController(t, "never") + + startErr := make(chan error, 1) + go func() { + c.LockControl() + defer c.UnlockControl() + startErr <- c.StartOrAttach(context.Background(), mtprotoStartRequest()) + }() + pid := readFakeTelemtPID(t, pidFile) + waitForStartInProgress(t, c) + + disconnected := make(chan struct{}) + go func() { + c.Disconnect() + close(disconnected) + }() + + select { + case err := <-startErr: + if err == nil { + t.Fatal("an MTProto start aborted by Disconnect reported success") + } + case <-time.After(10 * time.Second): + t.Fatal("the MTProto start still runs 10s after Disconnect") + } + select { + case <-disconnected: + case <-time.After(10 * time.Second): + t.Fatal("Disconnect still blocked 10s after the start ended") + } + if c.Backend() != nil { + t.Fatal("an aborted start left a backend attached") + } + if err := syscall.Kill(pid, 0); !errors.Is(err, syscall.ESRCH) { + t.Errorf("telemt %d still runs after the aborted start (kill 0: %v)", pid, err) + } +} From 9ff6ae0e918588b9aedecc1f73b89b7ab9dfe6ce Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Thu, 22 Oct 2026 21:53:50 +0330 Subject: [PATCH 24/40] test(controller): restart telemt when an MTProto Start changes the config Dev's StartOrAttach now offers a changed config to a running backend that implements backend.LiveConfigApplier before it restarts it (#105). Only xray does: telemt has no live path for its core config, so a Start with another MTProto config must still stop the old telemt and start a new one on the new config. Cover that against the fake telemt, and fail loudly if MTProto ever gains the interface without a review of what a live apply means for telemt. --- controller/mtproto_start_unix_test.go | 36 +++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/controller/mtproto_start_unix_test.go b/controller/mtproto_start_unix_test.go index aec0c59e..97a788ef 100644 --- a/controller/mtproto_start_unix_test.go +++ b/controller/mtproto_start_unix_test.go @@ -19,6 +19,7 @@ import ( "google.golang.org/protobuf/proto" "github.com/pasarguard/node/backend" + "github.com/pasarguard/node/backend/mtproto" "github.com/pasarguard/node/common" "github.com/pasarguard/node/config" "github.com/pasarguard/node/pkg/netutil" @@ -298,3 +299,38 @@ func TestDisconnectAbortsAnMTProtoStartInProgress(t *testing.T) { t.Errorf("telemt %d still runs after the aborted start (kill 0: %v)", pid, err) } } + +// telemt cannot take a changed core config live (MTProto does not implement +// backend.LiveConfigApplier), so a Start with another config restarts it on +// the new one instead of keeping the old telemt. +func TestMTProtoConfigChangeRestartsTelemt(t *testing.T) { + c, pidFile := newFakeTelemtController(t, "0s") + if _, live := any(&mtproto.MTProto{}).(backend.LiveConfigApplier); live { + t.Fatal("MTProto implements LiveConfigApplier: check that a live apply is correct for telemt") + } + + start := func(data *common.Backend) backend.Backend { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + c.LockControl() + defer c.UnlockControl() + if err := c.StartOrAttach(ctx, data); err != nil { + t.Fatalf("Start: %v", err) + } + return c.Backend() + } + + first := start(mtprotoStartRequest()) + oldPID := readFakeTelemtPID(t, pidFile) + second := start(mtprotoStartRequest()) // another listener port + if second == first { + t.Fatal("a Start with another MTProto config kept the running backend") + } + if !second.Started() { + t.Fatal("no running telemt after the config change") + } + if err := syscall.Kill(oldPID, 0); !errors.Is(err, syscall.ESRCH) { + t.Errorf("the old telemt %d still runs after the config change (kill 0: %v)", oldPID, err) + } +} From 087134dfdcf5dcff26af28c007cc93ff67fa6ccc Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Thu, 22 Oct 2026 22:07:55 +0330 Subject: [PATCH 25/40] test: pin the node answers the panel relies on for MTProto The panel reads three answers of the node as fixed: - "invalid backend type" from a node without the MTProto backend, which it turns into an upgrade message; - "the users are still being applied" and "context deadline exceeded" from a sync that is still applying, which it retries later; - a retry or repair Start that attaches to a running telemt, which since #117 needs GetSysStats to answer within the 3 s heartbeat budget. Assert the exact texts, check that the unknown backend type error stays unwrapped under the detached start (also when the request is gone), and check that GetSysStats answers while telemt is frozen (SIGSTOP, so its API cannot answer) and syncMu is held: it stays a local read of the process, so a stuck API or a full apply never turns a heartbeat into a restart. --- backend/mtproto/lifecycle_test.go | 34 +++++++++++++++++++++++++++ backend/mtproto/sync_test.go | 10 ++++++++ controller/mtproto_start_unix_test.go | 21 +++++++++++++++++ 3 files changed, 65 insertions(+) diff --git a/backend/mtproto/lifecycle_test.go b/backend/mtproto/lifecycle_test.go index 0fe3f392..2cd816ed 100644 --- a/backend/mtproto/lifecycle_test.go +++ b/backend/mtproto/lifecycle_test.go @@ -569,3 +569,37 @@ func TestCancelledContextNeverRestartsTelemt(t *testing.T) { t.Errorf("startLocked spawned telemt for a cancelled ctx (generation %d)", mt.process.Generation()) } } + +// TestGetSysStatsDoesNotWaitOnTelemt: the controller treats a same-config +// Start (the panel's retry or repair Start) as a heartbeat only when +// GetSysStats answers within its 3 s budget, and restarts the backend +// otherwise. GetSysStats must stay a local read of telemt's process: no +// telemt API call and no syncMu, so a stuck telemt API or a full apply in +// progress never turns a heartbeat into a restart that cuts every session. +func TestGetSysStatsDoesNotWaitOnTelemt(t *testing.T) { + mt := newTestBackend(t, testNodeConfig(t, writeFakeTelemt(t, fakeTelemt{})), nil) + pid := mt.process.PID() + + // Freeze telemt: its API cannot answer while its process still runs. + if err := syscall.Kill(pid, syscall.SIGSTOP); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = syscall.Kill(pid, syscall.SIGCONT) }) + // A full apply in progress holds syncMu. + mt.syncMu.Lock() + defer mt.syncMu.Unlock() + + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + started := time.Now() + stats, err := mt.GetSysStats(ctx) + if err != nil { + t.Fatalf("GetSysStats with telemt's API stuck and syncMu held = %v", err) + } + if elapsed := time.Since(started); elapsed > time.Second { + t.Errorf("GetSysStats took %s with telemt's API stuck and syncMu held", elapsed) + } + if stats.GetAlloc() == 0 { + t.Error("GetSysStats reported no memory for the running telemt") + } +} diff --git a/backend/mtproto/sync_test.go b/backend/mtproto/sync_test.go index 2ad30f3e..bb959c8c 100644 --- a/backend/mtproto/sync_test.go +++ b/backend/mtproto/sync_test.go @@ -810,6 +810,12 @@ func TestCallerDeadlineDoesNotStopApply(t *testing.T) { if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("SyncUsers = %v, want the caller's deadline", err) } + // The panel matches these texts to treat the sync as retry-later. + for _, want := range []string{"the users are still being applied", "context deadline exceeded"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("SyncUsers error %q does not contain %q", err, want) + } + } if elapsed := time.Since(started); elapsed > 1500*time.Millisecond { t.Errorf("the caller waited %s after its 1 s deadline", elapsed) } @@ -866,6 +872,10 @@ func TestQueuedCallerGivesUpAtDeadline(t *testing.T) { if !errors.Is(err, context.DeadlineExceeded) { t.Fatalf("queued UpdateUsers = %v, want its deadline", err) } + // The panel matches this text to treat the sync as retry-later. + if !strings.Contains(err.Error(), "context deadline exceeded") { + t.Errorf("queued UpdateUsers error %q does not contain %q", err, "context deadline exceeded") + } if elapsed := time.Since(started); elapsed > 1500*time.Millisecond { t.Errorf("the queued caller waited %s with a 500 ms deadline", elapsed) } diff --git a/controller/mtproto_start_unix_test.go b/controller/mtproto_start_unix_test.go index 97a788ef..110ef69c 100644 --- a/controller/mtproto_start_unix_test.go +++ b/controller/mtproto_start_unix_test.go @@ -334,3 +334,24 @@ func TestMTProtoConfigChangeRestartsTelemt(t *testing.T) { t.Errorf("the old telemt %d still runs after the config change (kill 0: %v)", oldPID, err) } } + +// The panel tells an old node from a broken one by the exact error text +// "invalid backend type", which nodes without the MTProto backend answer to +// an MTProto Start. The start runs detached from the request now (#116), +// and only a start that runs out of startTimeout gets its error wrapped, so +// that text must reach the caller unchanged, even when the request is gone. +func TestUnknownBackendTypeKeepsItsError(t *testing.T) { + c := New(config.NewTestConfig(t.TempDir(), uuid.New())) + t.Cleanup(c.Disconnect) + + gone, cancel := context.WithCancel(context.Background()) + cancel() + for name, ctx := range map[string]context.Context{"live request": context.Background(), "request gone": gone} { + c.LockControl() + err := c.StartOrAttach(ctx, &common.Backend{Type: common.BackendType(99), Config: "{}"}) + c.UnlockControl() + if err == nil || err.Error() != "invalid backend type" { + t.Errorf("%s: Start with an unknown backend type = %v, want exactly %q", name, err, "invalid backend type") + } + } +} From 2eb22c8c804dd94c5b3bbff142541183182cc29f Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Thu, 22 Oct 2026 22:26:02 +0330 Subject: [PATCH 26/40] fix(mtproto): mark sync errors of a busy or restarting telemt as retry-later Bridge 0.9.4 delivers user deltas through SyncUsersChunked and treats an INTERNAL "failed to update users" answer (REST: 500) as a rejection of the users it sent: it splits the batch in halves, sends again and holds each user it isolates with a backoff of 2 s up to 60 s (an add every 300 s after 10 minutes). MTProto answered that way for failures that have nothing to do with the users: the backend shut down while a request held it, telemt that does not start for a full apply, a reload whose restart failed, a caller that gave up while it queued behind an apply. A busy or restarting telemt therefore multiplied the requests and delayed every user of the batch. Add backend.RetryLaterError, a marker that keeps the error and its text, and wrap every MTProto sync failure in it except a render telemt read and refused (errReloadRejected): the users sent may have caused that one, so splitting the batch can find them. An invalid user synced alone stays a plain error too. The texts the panel matches ("the users are still being applied", "context deadline exceeded") are unchanged. Xray and WireGuard never return the marker. --- backend/errors.go | 31 ++++++++++++++ backend/errors_test.go | 29 +++++++++++++ backend/mtproto/sync.go | 25 ++++++++--- backend/mtproto/sync_test.go | 82 ++++++++++++++++++++++++++++++++++++ 4 files changed, 162 insertions(+), 5 deletions(-) create mode 100644 backend/errors.go create mode 100644 backend/errors_test.go diff --git a/backend/errors.go b/backend/errors.go new file mode 100644 index 00000000..740bc21d --- /dev/null +++ b/backend/errors.go @@ -0,0 +1,31 @@ +package backend + +import "errors" + +// RetryLaterError marks a user sync error that comes from the backend's +// state, not from the users sent: the backend is busy with an earlier +// change, its core is restarting, or it was shut down. The node answers it as +// unavailable (gRPC Unavailable, HTTP 503) with the same text, so a client +// retries the batch later instead of splitting it to find a user to blame. +type RetryLaterError struct { + Err error +} + +func (e *RetryLaterError) Error() string { return e.Err.Error() } + +func (e *RetryLaterError) Unwrap() error { return e.Err } + +// RetryLater marks err as a RetryLaterError; nil stays nil. +func RetryLater(err error) error { + if err == nil { + return nil + } + return &RetryLaterError{Err: err} +} + +// IsRetryLater reports whether err, or an error it wraps, is a +// RetryLaterError. +func IsRetryLater(err error) bool { + var retryLater *RetryLaterError + return errors.As(err, &retryLater) +} diff --git a/backend/errors_test.go b/backend/errors_test.go new file mode 100644 index 00000000..8f1d2f6f --- /dev/null +++ b/backend/errors_test.go @@ -0,0 +1,29 @@ +package backend + +import ( + "context" + "errors" + "fmt" + "testing" +) + +func TestRetryLaterKeepsTheErrorAndItsText(t *testing.T) { + cause := fmt.Errorf("the users are still being applied: %w", context.DeadlineExceeded) + err := RetryLater(cause) + + if !IsRetryLater(err) || !IsRetryLater(fmt.Errorf("failed to update users: %w", err)) { + t.Error("a RetryLater error, plain or wrapped, is not reported as retry-later") + } + if err.Error() != cause.Error() { + t.Errorf("RetryLater changed the text: %q, want %q", err, cause) + } + if !errors.Is(err, context.DeadlineExceeded) { + t.Error("RetryLater hides the error it marks") + } + if IsRetryLater(cause) || IsRetryLater(nil) { + t.Error("an unmarked error or nil is reported as retry-later") + } + if RetryLater(nil) != nil { + t.Error("RetryLater(nil) is not nil") + } +} diff --git a/backend/mtproto/sync.go b/backend/mtproto/sync.go index 29256662..314e193c 100644 --- a/backend/mtproto/sync.go +++ b/backend/mtproto/sync.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strings" + "github.com/pasarguard/node/backend" "github.com/pasarguard/node/common" ) @@ -31,7 +32,7 @@ func (m *MTProto) SyncUser(ctx context.Context, user *common.User) error { // SyncUsers replaces all users through one config render and reload. func (m *MTProto) SyncUsers(ctx context.Context, users []*common.User) error { if err := m.lockSync(ctx); err != nil { - return err + return syncError(err) } desired, _ := ActiveUsersFromCommon(users, m.config.InboundTag) @@ -40,7 +41,7 @@ func (m *MTProto) SyncUsers(ctx context.Context, users []*common.User) error { m.syncMu.Unlock() return nil } - return m.applyFullDetachedLocked(ctx, desired) + return syncError(m.applyFullDetachedLocked(ctx, desired)) } // UpdateUsers applies users on top of the current ones. Up to @@ -49,23 +50,37 @@ func (m *MTProto) SyncUsers(ctx context.Context, users []*common.User) error { // that did not change are not sent at all. func (m *MTProto) UpdateUsers(ctx context.Context, users []*common.User) error { if err := m.lockSync(ctx); err != nil { - return err + return syncError(err) } next, full := m.updateUsersLocked(ctx, users) if !full { m.syncMu.Unlock() return nil } - return m.applyFullDetachedLocked(ctx, next) + return syncError(m.applyFullDetachedLocked(ctx, next)) } // UpdateUsersAndRestart applies users on top of the current ones through a // config render and reload. func (m *MTProto) UpdateUsersAndRestart(ctx context.Context, users []*common.User) error { if err := m.lockSync(ctx); err != nil { + return syncError(err) + } + return syncError(m.applyFullDetachedLocked(ctx, m.buildNextDesiredLocked(users))) +} + +// syncError is how a sync reports err. A failure that comes from telemt's +// state (busy with an earlier apply, a caller that gave up while it queued or +// waited, telemt that does not start or answer, the backend shut down) is +// marked retry-later, so the node answers it as unavailable and the caller +// retries the batch. A render telemt read and refused (errReloadRejected) +// stays a plain error: the users sent may have caused it, so a caller may +// narrow the batch down. The text never changes. +func syncError(err error) error { + if err == nil || errors.Is(err, errReloadRejected) { return err } - return m.applyFullDetachedLocked(ctx, m.buildNextDesiredLocked(users)) + return backend.RetryLater(err) } // lockSync takes syncMu for a sync call. A caller waiting behind another diff --git a/backend/mtproto/sync_test.go b/backend/mtproto/sync_test.go index bb959c8c..7e83bf36 100644 --- a/backend/mtproto/sync_test.go +++ b/backend/mtproto/sync_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/pasarguard/node/backend" "github.com/pasarguard/node/common" "github.com/pasarguard/node/pkg/logstream" ) @@ -1224,3 +1225,84 @@ func TestAPIClientDeleteUserReadsTelemtAnswer(t *testing.T) { t.Error("u1 is still in the config file") } } + +// TestSyncErrorsTellRetryLaterFromRejection: a sync error that comes from +// telemt's state (busy with an earlier apply, a caller that gave up while it +// queued or waited, telemt that does not start, the backend shut down) is +// marked backend.RetryLaterError, which the node answers as unavailable, so +// the bridge retries the batch instead of splitting it to find a user to +// blame. A render telemt read and refused, which the users sent may have +// caused, and an invalid user synced alone stay rejections. The texts the +// panel matches do not change. +func TestSyncErrorsTellRetryLaterFromRejection(t *testing.T) { + users := []*common.User{mtUser("u2", "mtproto", otherSecret)} + cases := []struct { + name string + retryLater bool + text string + run func(t *testing.T, m *MTProto, api *fakeTelemtAPI) error + }{ + {"caller gave up during an apply", true, "the users are still being applied: context deadline exceeded", + func(t *testing.T, m *MTProto, api *fakeTelemtAPI) error { + lowerMarkerConfirmTimeout(t, 2*time.Second) + api.set(func(f *fakeTelemtAPI) { f.rejectReload = true }) + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + err := m.SyncUsers(ctx, users) + waitSettled(m) + return err + }}, + {"queued behind an apply", true, "context deadline exceeded", + func(t *testing.T, m *MTProto, _ *fakeTelemtAPI) error { + m.syncMu.Lock() + defer m.syncMu.Unlock() + ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond) + defer cancel() + return m.UpdateUsers(ctx, users) + }}, + {"caller already gone", true, "context canceled", + func(t *testing.T, m *MTProto, _ *fakeTelemtAPI) error { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + return m.UpdateUsersAndRestart(ctx, users) + }}, + {"telemt does not start", true, "failed to start telemt", + func(t *testing.T, m *MTProto, _ *fakeTelemtAPI) error { + // No telemt runs, and none can start: a full apply must + // start one first. + m.process.run = nil + return m.SyncUsers(context.Background(), users) + }}, + {"backend shut down", true, errClosed.Error(), + func(t *testing.T, m *MTProto, _ *fakeTelemtAPI) error { + m.cancel() // what Shutdown does first + return m.SyncUsers(context.Background(), users) + }}, + {"telemt refused the render", false, "telemt did not load the new config", + func(t *testing.T, m *MTProto, api *fakeTelemtAPI) error { + api.set(func(f *fakeTelemtAPI) { f.loadError = "failed to load config: TOML parse error at line 3" }) + return m.SyncUsers(context.Background(), users) + }}, + {"invalid user synced alone", false, "invalid secret", + func(t *testing.T, m *MTProto, _ *fakeTelemtAPI) error { + return m.SyncUser(context.Background(), mtUser("u3", "mtproto", "not-hex")) + }}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + api := newFakeTelemtAPI(t) + m := newSyncTestBackend(t, api, []*common.User{mtUser("u1", "mtproto", testSecret)}) + + err := tc.run(t, m, api) + if err == nil { + t.Fatal("the sync succeeded") + } + if got := backend.IsRetryLater(err); got != tc.retryLater { + t.Errorf("IsRetryLater(%q) = %v, want %v", err, got, tc.retryLater) + } + if !strings.Contains(err.Error(), tc.text) { + t.Errorf("error %q does not contain %q", err, tc.text) + } + }) + } +} From b301cd35b2d5a7db47fedd1d418e99243223b290 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Thu, 22 Oct 2026 22:32:56 +0330 Subject: [PATCH 27/40] fix(controller): answer retry-later sync errors as unavailable A sync error a backend marked retry-later (only MTProto does: telemt is busy with an earlier apply, restarting or shut down) now reaches the client as gRPC Unavailable or HTTP 503 instead of INTERNAL (SyncUser, SyncUsersChunked), Unknown (SyncUsers) or 500 (REST), with the same text. Bridge 0.9.4 retries such an answer as a whole batch with its 1 s to 30 s backoff, the same path a timed-out sync already takes, instead of splitting the batch and holding each user it isolates. The texts the panel matches ("the users are still being applied", "context deadline exceeded") stay in the answer, so the panel's manual sync keeps reporting a still-applying MTProto node as retry-later. Every other error keeps its code, so Xray and WireGuard answers do not change. Tests: the mapping in both services, and the rpc and rest handlers end to end against an MTProto backend on the fake telemt whose full apply is still waiting for telemt when the caller's deadline passes (the fake now catches SIGHUP like telemt). --- controller/export_unix_test.go | 10 ++ controller/mtproto_start_unix_test.go | 18 ++- controller/rest/sync_errors.go | 19 +++ controller/rest/sync_errors_test.go | 34 ++++++ controller/rest/user.go | 6 +- controller/rpc/sync_errors.go | 29 +++++ controller/rpc/sync_errors_test.go | 46 ++++++++ controller/rpc/user.go | 6 +- controller/sync_errors_unix_test.go | 162 ++++++++++++++++++++++++++ 9 files changed, 321 insertions(+), 9 deletions(-) create mode 100644 controller/export_unix_test.go create mode 100644 controller/rest/sync_errors.go create mode 100644 controller/rest/sync_errors_test.go create mode 100644 controller/rpc/sync_errors.go create mode 100644 controller/rpc/sync_errors_test.go create mode 100644 controller/sync_errors_unix_test.go diff --git a/controller/export_unix_test.go b/controller/export_unix_test.go new file mode 100644 index 00000000..878da918 --- /dev/null +++ b/controller/export_unix_test.go @@ -0,0 +1,10 @@ +//go:build !windows + +package controller + +// For the handler tests in package controller_test, which drive the rpc and +// rest services against the fake telemt. +var ( + FakeTelemtConfig = fakeTelemtConfig + MTProtoStartRequest = mtprotoStartRequest +) diff --git a/controller/mtproto_start_unix_test.go b/controller/mtproto_start_unix_test.go index 110ef69c..5326b762 100644 --- a/controller/mtproto_start_unix_test.go +++ b/controller/mtproto_start_unix_test.go @@ -8,6 +8,7 @@ import ( "fmt" "net/http" "os" + "os/signal" "path/filepath" "strconv" "strings" @@ -66,6 +67,9 @@ func runFakeTelemt(delay, pidFile string, args []string) int { return 2 } } + // Catch SIGHUP like telemt, whose hot reload it starts; by default it + // would end this process. + signal.Notify(make(chan os.Signal, 1), syscall.SIGHUP) // Never outlive a broken test by much. go func() { time.Sleep(2 * time.Minute) @@ -126,6 +130,16 @@ func fakeTelemtAPIListen(configPath string) (string, error) { // binds its API after healthDelay ("never" for a start that never ends on // its own) and writes its pid to the returned file. func newFakeTelemtController(t *testing.T, healthDelay string) (*Controller, string) { + t.Helper() + cfg, pidFile := fakeTelemtConfig(t, healthDelay) + c := New(cfg) + t.Cleanup(c.Disconnect) + return c, pidFile +} + +// fakeTelemtConfig returns a node config whose telemt is the fake (see +// newFakeTelemtController) and the file the fake writes its pid to. +func fakeTelemtConfig(t *testing.T, healthDelay string) (*config.Config, string) { t.Helper() testBinary, err := filepath.Abs(os.Args[0]) if err != nil { @@ -142,9 +156,7 @@ func newFakeTelemtController(t *testing.T, healthDelay string) (*Controller, str cfg := config.NewTestConfig(t.TempDir(), uuid.New()) cfg.TelemtExecutablePath = executable - c := New(cfg) - t.Cleanup(c.Disconnect) - return c, pidFile + return cfg, pidFile } func shellQuote(value string) string { diff --git a/controller/rest/sync_errors.go b/controller/rest/sync_errors.go new file mode 100644 index 00000000..8f388ccc --- /dev/null +++ b/controller/rest/sync_errors.go @@ -0,0 +1,19 @@ +package rest + +import ( + "net/http" + + "github.com/pasarguard/node/backend" +) + +// syncErrorStatus is the HTTP status of a failed user sync. A backend that is +// busy or restarting marks its error retry-later (backend.RetryLaterError): +// that is 503, so a client retries the batch as a whole. Anything else is +// 500, which bridge 0.9.4 reads as its users being refused and narrows down +// by splitting the batch. +func syncErrorStatus(err error) int { + if backend.IsRetryLater(err) { + return http.StatusServiceUnavailable + } + return http.StatusInternalServerError +} diff --git a/controller/rest/sync_errors_test.go b/controller/rest/sync_errors_test.go new file mode 100644 index 00000000..c5764e57 --- /dev/null +++ b/controller/rest/sync_errors_test.go @@ -0,0 +1,34 @@ +package rest + +import ( + "context" + "errors" + "fmt" + "net/http" + "testing" + + "github.com/pasarguard/node/backend" +) + +// Bridge 0.9.4 reads a 500 answer to /users/sync/chunked as its users being +// refused and splits the batch to find them; a 503 is a failure to retry as a +// whole. A backend that is busy or restarting marks its error retry-later +// (only MTProto does), and the node must answer that as 503 with the same +// body. Xray and WireGuard errors keep their 500. +func TestSyncErrorStatus(t *testing.T) { + plain := errors.New("failed to add user 1 to inbound vless-in: boom") // xray or WireGuard + busy := backend.RetryLater(fmt.Errorf("the users are still being applied: %w", context.DeadlineExceeded)) + + for name, tc := range map[string]struct { + err error + want int + }{ + "xray or wireguard error": {plain, http.StatusInternalServerError}, + "retry-later": {busy, http.StatusServiceUnavailable}, + "wrapped retry-later": {fmt.Errorf("apply: %w", busy), http.StatusServiceUnavailable}, + } { + if got := syncErrorStatus(tc.err); got != tc.want { + t.Errorf("%s: status = %d, want %d", name, got, tc.want) + } + } +} diff --git a/controller/rest/user.go b/controller/rest/user.go index 4f6bf3da..1df35249 100644 --- a/controller/rest/user.go +++ b/controller/rest/user.go @@ -38,7 +38,7 @@ func (s *Service) SyncUser(w http.ResponseWriter, r *http.Request) { if err = s.Backend().SyncUser(r.Context(), user); err != nil { log.Printf("Error syncing user: %v", err) - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, err.Error(), syncErrorStatus(err)) return } @@ -66,7 +66,7 @@ func (s *Service) SyncUsers(w http.ResponseWriter, r *http.Request) { } if err = s.Backend().SyncUsers(r.Context(), users.GetUsers()); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, err.Error(), syncErrorStatus(err)) return } @@ -130,7 +130,7 @@ func (s *Service) SyncUsersChunked(w http.ResponseWriter, r *http.Request) { } if err := controller.ApplyChunkedUserUpdate(r.Context(), s.Backend(), users); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + http.Error(w, err.Error(), syncErrorStatus(err)) return } diff --git a/controller/rpc/sync_errors.go b/controller/rpc/sync_errors.go new file mode 100644 index 00000000..c62453a1 --- /dev/null +++ b/controller/rpc/sync_errors.go @@ -0,0 +1,29 @@ +package rpc + +import ( + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/pasarguard/node/backend" +) + +// syncErrorCode is the code of a failed user sync. A backend that is busy or +// restarting marks its error retry-later (backend.RetryLaterError): that is +// Unavailable, so a client retries the batch as a whole. Anything else is +// Internal, which bridge 0.9.4 reads as its users being refused and narrows +// down by splitting the batch. +func syncErrorCode(err error) codes.Code { + if backend.IsRetryLater(err) { + return codes.Unavailable + } + return codes.Internal +} + +// syncUsersError is what SyncUsers answers for err: the error as it is +// (gRPC Unknown), or Unavailable with the same text when it is retry-later. +func syncUsersError(err error) error { + if backend.IsRetryLater(err) { + return status.Error(codes.Unavailable, err.Error()) + } + return err +} diff --git a/controller/rpc/sync_errors_test.go b/controller/rpc/sync_errors_test.go new file mode 100644 index 00000000..cc4ff862 --- /dev/null +++ b/controller/rpc/sync_errors_test.go @@ -0,0 +1,46 @@ +package rpc + +import ( + "context" + "errors" + "fmt" + "testing" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + "github.com/pasarguard/node/backend" +) + +// Bridge 0.9.4 reads an INTERNAL answer to SyncUsersChunked as its users +// being refused and splits the batch to find them; any other code is a +// failure to retry as a whole. A backend that is busy or restarting marks its +// error retry-later (only MTProto does), and the node must answer that as +// Unavailable with the same text. Xray and WireGuard errors keep their codes. +func TestSyncErrorCodes(t *testing.T) { + plain := errors.New("failed to add user 1 to inbound vless-in: boom") // xray or WireGuard + busy := backend.RetryLater(fmt.Errorf("the users are still being applied: %w", context.DeadlineExceeded)) + + for name, tc := range map[string]struct { + err error + want codes.Code + }{ + "xray or wireguard error": {plain, codes.Internal}, + "retry-later": {busy, codes.Unavailable}, + "wrapped retry-later": {fmt.Errorf("apply: %w", busy), codes.Unavailable}, + } { + if got := syncErrorCode(tc.err); got != tc.want { + t.Errorf("%s: SyncUser and SyncUsersChunked code = %s, want %s", name, got, tc.want) + } + } + + // SyncUsers answers the backend's error as it is, so gRPC sends it as + // Unknown; only a retry-later error becomes Unavailable, same text. + if got := syncUsersError(plain); got != plain { + t.Errorf("SyncUsers changed an xray or WireGuard error: %v", got) + } + got := status.Convert(syncUsersError(busy)) + if got.Code() != codes.Unavailable || got.Message() != busy.Error() { + t.Errorf("SyncUsers answered a retry-later error as %s %q, want Unavailable %q", got.Code(), got.Message(), busy.Error()) + } +} diff --git a/controller/rpc/user.go b/controller/rpc/user.go index 1a275c28..845eb42c 100644 --- a/controller/rpc/user.go +++ b/controller/rpc/user.go @@ -34,7 +34,7 @@ func (s *Service) SyncUser(stream grpc.ClientStreamingServer[common.User, common if err = backend.SyncUser(stream.Context(), user); err != nil { log.Printf("Error syncing user: %v", err) - return status.Errorf(codes.Internal, "failed to update user: %v", err) + return status.Errorf(syncErrorCode(err), "failed to update user: %v", err) } } } @@ -46,7 +46,7 @@ func (s *Service) SyncUsers(ctx context.Context, users *common.Users) (*common.E } if err := backend.SyncUsers(ctx, users.GetUsers()); err != nil { - return nil, err + return nil, syncUsersError(err) } return nil, nil @@ -88,7 +88,7 @@ func (s *Service) SyncUsersChunked(stream grpc.ClientStreamingServer[common.User } if err := controller.ApplyChunkedUserUpdate(stream.Context(), backend, users); err != nil { - return status.Errorf(codes.Internal, "failed to update users: %v", err) + return status.Errorf(syncErrorCode(err), "failed to update users: %v", err) } return stream.SendAndClose(&common.Empty{}) diff --git a/controller/sync_errors_unix_test.go b/controller/sync_errors_unix_test.go new file mode 100644 index 00000000..efa0b270 --- /dev/null +++ b/controller/sync_errors_unix_test.go @@ -0,0 +1,162 @@ +//go:build !windows + +package controller_test + +import ( + "bytes" + "context" + "encoding/binary" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" + + "github.com/pasarguard/node/common" + "github.com/pasarguard/node/controller" + "github.com/pasarguard/node/controller/rest" + "github.com/pasarguard/node/controller/rpc" +) + +// A busy MTProto node must answer a sync as unavailable (gRPC Unavailable, +// HTTP 503), not as INTERNAL or 500, which bridge 0.9.4 reads as its users +// being refused and splits the batch to find them. Here every sync changes +// the users, and the fake telemt never confirms a reload, so the apply is +// still waiting for telemt when the caller's deadline passes, like a full +// apply on a large node. The text the panel matches stays in the answer. +const stillApplying = "the users are still being applied" + +// syncCallTimeout is the caller's deadline for each sync below. +const syncCallTimeout = time.Second + +func changedUsers() []*common.User { + return []*common.User{{ + Email: "2", + Inbounds: []string{"mtproto"}, + Proxies: &common.Proxy{Mtproto: &common.Mtproto{Secret: "ffeeddccbbaa00112233445566778899"}}, + }} +} + +type startable interface { + LockControl() + UnlockControl() + StartOrAttach(context.Context, *common.Backend) error +} + +func startMTProto(t *testing.T, s startable) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + s.LockControl() + defer s.UnlockControl() + if err := s.StartOrAttach(ctx, controller.MTProtoStartRequest()); err != nil { + t.Fatalf("Start: %v", err) + } +} + +func newMTProtoRPCService(t *testing.T) *rpc.Service { + t.Helper() + cfg, _ := controller.FakeTelemtConfig(t, "0s") + s := rpc.New(cfg) + t.Cleanup(s.Disconnect) + startMTProto(t, s) + return s +} + +func newMTProtoRESTService(t *testing.T) *rest.Service { + t.Helper() + cfg, _ := controller.FakeTelemtConfig(t, "0s") + s := rest.New(cfg) + t.Cleanup(s.Disconnect) + startMTProto(t, s) + return s +} + +// chunkStream is the server side of a SyncUsersChunked call. +type chunkStream struct { + grpc.ServerStream + ctx context.Context + chunks []*common.UsersChunk +} + +func (s *chunkStream) Context() context.Context { return s.ctx } + +func (s *chunkStream) Recv() (*common.UsersChunk, error) { + if len(s.chunks) == 0 { + return nil, io.EOF + } + chunk := s.chunks[0] + s.chunks = s.chunks[1:] + return chunk, nil +} + +func (s *chunkStream) SendAndClose(*common.Empty) error { return nil } + +func assertUnavailable(t *testing.T, err error) { + t.Helper() + got := status.Convert(err) + if got.Code() != codes.Unavailable || !strings.Contains(got.Message(), stillApplying) { + t.Errorf("busy MTProto sync answered %s %q, want Unavailable with %q", got.Code(), got.Message(), stillApplying) + } +} + +func assert503(t *testing.T, recorder *httptest.ResponseRecorder) { + t.Helper() + if recorder.Code != http.StatusServiceUnavailable || !strings.Contains(recorder.Body.String(), stillApplying) { + t.Errorf("busy MTProto sync answered %d %q, want 503 with %q", recorder.Code, recorder.Body.String(), stillApplying) + } +} + +func TestBusyMTProtoSyncIsUnavailable(t *testing.T) { + t.Run("grpc SyncUsers", func(t *testing.T) { + s := newMTProtoRPCService(t) + ctx, cancel := context.WithTimeout(context.Background(), syncCallTimeout) + defer cancel() + _, err := s.SyncUsers(ctx, &common.Users{Users: changedUsers()}) + assertUnavailable(t, err) + }) + + t.Run("grpc SyncUsersChunked", func(t *testing.T) { + s := newMTProtoRPCService(t) + ctx, cancel := context.WithTimeout(context.Background(), syncCallTimeout) + defer cancel() + err := s.SyncUsersChunked(&chunkStream{ + ctx: ctx, + chunks: []*common.UsersChunk{{Users: changedUsers(), Index: 0, Last: true}}, + }) + assertUnavailable(t, err) + }) + + t.Run("rest SyncUsers", func(t *testing.T) { + s := newMTProtoRESTService(t) + body, err := proto.Marshal(&common.Users{Users: changedUsers()}) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), syncCallTimeout) + defer cancel() + recorder := httptest.NewRecorder() + s.SyncUsers(recorder, httptest.NewRequest(http.MethodPut, "/users/sync", bytes.NewReader(body)).WithContext(ctx)) + assert503(t, recorder) + }) + + t.Run("rest SyncUsersChunked", func(t *testing.T) { + s := newMTProtoRESTService(t) + payload, err := proto.Marshal(&common.UsersChunk{Users: changedUsers(), Index: 0, Last: true}) + if err != nil { + t.Fatal(err) + } + body := append(binary.AppendUvarint(nil, uint64(len(payload))), payload...) + ctx, cancel := context.WithTimeout(context.Background(), syncCallTimeout) + defer cancel() + recorder := httptest.NewRecorder() + s.SyncUsersChunked(recorder, httptest.NewRequest(http.MethodPut, "/users/sync/chunked", bytes.NewReader(body)).WithContext(ctx)) + assert503(t, recorder) + }) +} From d631cb461ce3589dfb95ed24a05bbdfe6a2cb371 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 26 Oct 2026 12:20:48 +0330 Subject: [PATCH 28/40] refactor(mtproto): render the telemt config with go-toml/v2 Replace the hand-written TOML encoder with github.com/pelletier/go-toml/v2, as asked in review. The test-only go-toml v1 dependency goes away. - JSON numbers from the panel config are converted to int64 or float64 first: the encoder writes a json.Number as a string. An integer outside the TOML 64-bit range is an error naming the field. - JSON nulls are still dropped from tables; a null array element is an error that names its path. - telemt expands any line "include = " as a file include before it parses TOML, in every table. The old encoder quoted such keys; go-toml writes them bare, so the rendered text is now checked with telemt's own line matcher and refused, and the username "include" is skipped like any other invalid user. - Test helpers and the fake telemts decode the config with go-toml instead of parsing double-quoted lines by hand. --- backend/mtproto/config.go | 6 + backend/mtproto/config_test.go | 36 +-- backend/mtproto/fake_telemt_test.go | 20 +- backend/mtproto/sync_test.go | 25 +-- backend/mtproto/toml.go | 307 +++++++------------------- backend/mtproto/toml_test.go | 158 ++++++++----- controller/mtproto_start_unix_test.go | 26 ++- go.mod | 2 +- go.sum | 2 + 9 files changed, 238 insertions(+), 344 deletions(-) diff --git a/backend/mtproto/config.go b/backend/mtproto/config.go index d7af06f5..14e42707 100644 --- a/backend/mtproto/config.go +++ b/backend/mtproto/config.go @@ -404,6 +404,12 @@ func activeUserFromCommon(user *common.User, inboundTag string) (*runtimeUser, b if !validUsernamePattern.MatchString(username) { return nil, false, fmt.Errorf("invalid mtproto username %q; expected [A-Za-z0-9_.-] and length 1..64", username) } + // Usernames are TOML keys, and telemt reads a key named include as a file + // include (see checkNoIncludeLines). Skipping the user here keeps one bad + // record from failing the whole render. + if username == "include" { + return nil, false, fmt.Errorf("invalid mtproto username %q; telemt reserves it", username) + } proxy := user.GetProxies().GetMtproto() if proxy == nil { diff --git a/backend/mtproto/config_test.go b/backend/mtproto/config_test.go index cca109f2..15fdbdee 100644 --- a/backend/mtproto/config_test.go +++ b/backend/mtproto/config_test.go @@ -220,7 +220,7 @@ func TestRenderForcesNodeKeys(t *testing.T) { "access.users.alice": testSecret, } for path, want := range expect { - if got := tree.GetPath(strings.Split(path, ".")); got != want { + if got := tomlAt(tree, strings.Split(path, ".")...); got != want { t.Errorf("%s = %#v, want %#v\n---\n%s", path, got, want, out) } } @@ -279,21 +279,21 @@ func TestConfigRender_InjectsManagedSections(t *testing.T) { if err != nil { t.Fatal(err) } - for _, want := range []string{ - "enabled = true", - `listen = "127.0.0.1:12345"`, - `auth_header = "Bearer testtoken"`, - `metrics_port = 23456`, - `metrics_listen = "127.0.0.1:23456"`, - "[access.users]", - `alice = "` + testSecret + `"`, - `tls_domain = "example.com"`, + tree := decodeTOML(t, out) + for path, want := range map[string]any{ + "server.api.enabled": true, + "server.api.listen": "127.0.0.1:12345", + "server.api.auth_header": "Bearer testtoken", + "server.metrics_port": int64(23456), + "server.metrics_listen": "127.0.0.1:23456", + "access.users.alice": testSecret, + "censorship.tls_domain": "example.com", } { - if !strings.Contains(out, want) { - t.Errorf("rendered config missing %q\n---\n%s", want, out) + if got := tomlAt(tree, strings.Split(path, ".")...); got != want { + t.Errorf("%s = %#v, want %#v\n---\n%s", path, got, want, out) } } - if strings.Contains(out, "user_enabled = {") || strings.Contains(out, "[access.user_enabled]") { + if tomlAt(tree, "access", "user_enabled") != nil { t.Errorf("real users must not disable anyone via access.user_enabled") } @@ -303,10 +303,11 @@ func TestConfigRender_InjectsManagedSections(t *testing.T) { if err != nil { t.Fatalf("render with marker failed: %v", err) } - if !strings.Contains(out, "pgm_0123456789abcdef") { + tree = decodeTOML(t, out) + if tomlAt(tree, "access", "users", "pgm_0123456789abcdef") != testSecret { t.Errorf("marker user missing from rendered config") } - if !strings.Contains(out, "[access.user_enabled]") || !strings.Contains(out, "pgm_0123456789abcdef = false") { + if tomlAt(tree, "access", "user_enabled", "pgm_0123456789abcdef") != false { t.Errorf("marker must be rendered disabled, got:\n%s", out) } @@ -346,6 +347,11 @@ func TestActiveUserFromCommon_Selection(t *testing.T) { if _, _, err := activeUserFromCommon(mtUser("has space", "mtproto", testSecret), "mtproto"); err == nil { t.Errorf("expected error for invalid username") } + + // include would render as a telemt file include -> error + if _, _, err := activeUserFromCommon(mtUser("include", "mtproto", testSecret), "mtproto"); err == nil { + t.Errorf("expected error for the username include") + } } func TestActiveUsersSkipsInvalidUsers(t *testing.T) { diff --git a/backend/mtproto/fake_telemt_test.go b/backend/mtproto/fake_telemt_test.go index 208f82cc..56e4ce8e 100644 --- a/backend/mtproto/fake_telemt_test.go +++ b/backend/mtproto/fake_telemt_test.go @@ -11,13 +11,14 @@ import ( "os" "os/signal" "path/filepath" - "strconv" "strings" "sync" "syscall" "testing" "time" + "github.com/pelletier/go-toml/v2" + "github.com/pasarguard/node/config" "github.com/pasarguard/node/pkg/netutil" ) @@ -314,17 +315,12 @@ func fakeTelemtAPIListen(configPath string) (string, error) { if err != nil { return "", err } - section := "" - for _, line := range strings.Split(string(data), "\n") { - line = strings.TrimSpace(line) - if strings.HasPrefix(line, "[") { - section = strings.Trim(line, "[]") - continue - } - key, value, ok := strings.Cut(line, "=") - if ok && section == "server.api" && strings.TrimSpace(key) == "listen" { - return strconv.Unquote(strings.TrimSpace(value)) - } + var tree map[string]any + if err := toml.Unmarshal(data, &tree); err != nil { + return "", err + } + if listen, ok := tomlAt(tree, "server", "api", "listen").(string); ok { + return listen, nil } return "", errors.New("server.api.listen not found in " + configPath) } diff --git a/backend/mtproto/sync_test.go b/backend/mtproto/sync_test.go index 7e83bf36..14a5b269 100644 --- a/backend/mtproto/sync_test.go +++ b/backend/mtproto/sync_test.go @@ -19,6 +19,8 @@ import ( "testing" "time" + "github.com/pelletier/go-toml/v2" + "github.com/pasarguard/node/backend" "github.com/pasarguard/node/common" "github.com/pasarguard/node/pkg/logstream" @@ -213,22 +215,15 @@ func (f *fakeTelemtAPI) editDiskLocked(edit func(users map[string]string)) { // parseConfigUsers reads [access.users] from a rendered config. func parseConfigUsers(config string) map[string]string { users := make(map[string]string) - section := "" - for _, line := range strings.Split(config, "\n") { - line = strings.TrimSpace(line) - if strings.HasPrefix(line, "[") { - section = strings.Trim(line, "[]") - continue - } - key, value, ok := strings.Cut(line, "=") - if !ok || section != "access.users" { - continue - } - secret, err := strconv.Unquote(strings.TrimSpace(value)) - if err != nil { - continue + var tree map[string]any + if err := toml.Unmarshal([]byte(config), &tree); err != nil { + return users + } + configured, _ := tomlAt(tree, "access", "users").(map[string]any) + for name, value := range configured { + if secret, ok := value.(string); ok { + users[name] = secret } - users[strings.Trim(strings.TrimSpace(key), `"`)] = secret } return users } diff --git a/backend/mtproto/toml.go b/backend/mtproto/toml.go index fd2ba9f3..01210a05 100644 --- a/backend/mtproto/toml.go +++ b/backend/mtproto/toml.go @@ -3,269 +3,112 @@ package mtproto import ( "encoding/json" "fmt" - "reflect" - "slices" - "strconv" "strings" + + "github.com/pelletier/go-toml/v2" ) +// renderTOML encodes root as TOML. root is the panel JSON (numbers as +// json.Number, null for "not set") with the node-managed values merged in. func renderTOML(root map[string]any) (string, error) { - var builder strings.Builder - if err := writeTOMLSection(&builder, nil, root, tomlHeaderNone); err != nil { + value, err := tomlValue(root, "") + if err != nil { return "", err } - return builder.String(), nil -} - -type tomlHeaderKind uint8 - -const ( - tomlHeaderNone tomlHeaderKind = iota - tomlHeaderTable - tomlHeaderArrayTable -) - -func writeTOMLSection(builder *strings.Builder, path []string, section map[string]any, headerKind tomlHeaderKind) error { - scalarKeys, tableKeys, arrayTableKeys := classifyTOMLSection(section) - - if headerKind != tomlHeaderNone { - writeSectionSpacing(builder) - headerText := joinPath(path) - switch headerKind { - case tomlHeaderTable: - builder.WriteString("[") - builder.WriteString(headerText) - builder.WriteString("]\n") - case tomlHeaderArrayTable: - builder.WriteString("[[") - builder.WriteString(headerText) - builder.WriteString("]]\n") - } + out, err := toml.Marshal(value) + if err != nil { + return "", fmt.Errorf("failed to render telemt config as TOML: %w", err) } - - for _, key := range scalarKeys { - inline, err := renderInlineValue(section[key]) - if err != nil { - return fmt.Errorf("failed to render TOML value for %s: %w", joinPath(appendPath(path, key)), err) - } - builder.WriteString(renderKey(key)) - builder.WriteString(" = ") - builder.WriteString(inline) - builder.WriteByte('\n') - } - - for _, key := range tableKeys { - child, _ := section[key].(map[string]any) - if err := writeTOMLSection(builder, appendPath(path, key), child, tomlHeaderTable); err != nil { - return err - } - } - - for _, key := range arrayTableKeys { - items, _ := section[key].([]any) - for index, rawItem := range items { - child, ok := rawItem.(map[string]any) - if !ok { - return fmt.Errorf("failed to render TOML array table %s[%d]: expected object element", joinPath(appendPath(path, key)), index) - } - if err := writeTOMLSection(builder, appendPath(path, key), child, tomlHeaderArrayTable); err != nil { - return err - } - } + rendered := string(out) + if err := checkNoIncludeLines(rendered); err != nil { + return "", err } - - return nil + return rendered, nil } -// classifyTOMLSection sorts the keys of section into scalars (inline -// values), tables and arrays of tables, each in key order. -func classifyTOMLSection(section map[string]any) (scalarKeys, tableKeys, arrayTableKeys []string) { - keys := make([]string, 0, len(section)) - for key := range section { - keys = append(keys, key) - } - slices.Sort(keys) - - scalarKeys = make([]string, 0, len(keys)) - tableKeys = make([]string, 0, len(keys)) - - for _, key := range keys { - switch value := section[key].(type) { - case nil: - // JSON null means "not set"; TOML has no null. - continue - case map[string]any: - tableKeys = append(tableKeys, key) - case []any: - if isArrayOfObjects(value) { - arrayTableKeys = append(arrayTableKeys, key) +// tomlValue copies value for the TOML encoder. JSON numbers become int64 or +// float64, since the encoder writes a json.Number as a string. A null map +// value means "not set" and is dropped; TOML has no null, so a null array +// element is an error. +func tomlValue(value any, path string) (any, error) { + switch typed := value.(type) { + case map[string]any: + out := make(map[string]any, len(typed)) + for key, item := range typed { + if item == nil { continue } - scalarKeys = append(scalarKeys, key) - default: - scalarKeys = append(scalarKeys, key) - } - } - - return scalarKeys, tableKeys, arrayTableKeys -} - -func isArrayOfObjects(values []any) bool { - if len(values) == 0 { - return false - } - for _, value := range values { - if _, ok := value.(map[string]any); !ok { - return false - } - } - return true -} - -func renderInlineValue(value any) (string, error) { - switch typed := value.(type) { - case nil: - return "", fmt.Errorf("null array elements are not valid in TOML") - case string: - return quoteTOMLString(typed), nil - case bool: - return strconv.FormatBool(typed), nil - case json.Number: - return typed.String(), nil - case []any: - rendered := make([]string, 0, len(typed)) - for _, item := range typed { - inline, err := renderInlineValue(item) + converted, err := tomlValue(item, appendTOMLPath(path, key)) if err != nil { - return "", err + return nil, err } - rendered = append(rendered, inline) - } - return "[" + strings.Join(rendered, ", ") + "]", nil - case map[string]any: - keys := make([]string, 0, len(typed)) - for key := range typed { - keys = append(keys, key) + out[key] = converted } - slices.Sort(keys) - - parts := make([]string, 0, len(keys)) - for _, key := range keys { - if typed[key] == nil { - continue + return out, nil + case []any: + out := make([]any, len(typed)) + for index, item := range typed { + itemPath := fmt.Sprintf("%s[%d]", path, index) + if item == nil { + return nil, fmt.Errorf("mtproto config field %s is null; TOML arrays cannot hold null", itemPath) } - inline, err := renderInlineValue(typed[key]) + converted, err := tomlValue(item, itemPath) if err != nil { - return "", err + return nil, err } - parts = append(parts, fmt.Sprintf("%s = %s", renderKey(key), inline)) + out[index] = converted } - return "{ " + strings.Join(parts, ", ") + " }", nil + return out, nil + case json.Number: + return tomlNumber(typed, path) default: - return renderPrimitiveValue(typed) + return value, nil } } -func renderPrimitiveValue(value any) (string, error) { - typedValue := reflect.ValueOf(value) - switch typedValue.Kind() { - case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: - return strconv.FormatInt(typedValue.Int(), 10), nil - case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - return strconv.FormatUint(typedValue.Uint(), 10), nil - case reflect.Float32: - return strconv.FormatFloat(typedValue.Float(), 'f', -1, 32), nil - case reflect.Float64: - return strconv.FormatFloat(typedValue.Float(), 'f', -1, 64), nil - default: - return "", fmt.Errorf("unsupported TOML value type %T", value) +// tomlNumber converts a JSON number to the TOML integer or float it spells. +// TOML integers are signed 64-bit, so a larger integer is an error rather than +// a float that telemt would reject for an integer field. +func tomlNumber(number json.Number, path string) (any, error) { + if integer, err := number.Int64(); err == nil { + return integer, nil } + if !strings.ContainsAny(number.String(), ".eE") { + return nil, fmt.Errorf("mtproto config field %s: integer %s is outside the TOML 64-bit range", path, number) + } + float, err := number.Float64() + if err != nil { + return nil, fmt.Errorf("mtproto config field %s: %s is not a valid TOML float", path, number) + } + return float, nil } -func renderKey(key string) string { - // telemt expands any line of the form `include = ""` as a file - // include before it parses TOML, in every table. A quoted key is not - // matched, so a key named include is always quoted. - if isBareKey(key) && key != "include" { +func appendTOMLPath(path, key string) string { + if path == "" { return key } - return quoteTOMLString(key) + return path + "." + key } -// quoteTOMLString renders value as a TOML basic string. strconv.Quote does not -// fit: it emits Go escapes such as \a, \v and \x01 that TOML rejects. -func quoteTOMLString(value string) string { - var builder strings.Builder - builder.Grow(len(value) + 2) - builder.WriteByte('"') - for _, r := range value { - switch r { - case '"': - builder.WriteString(`\"`) - case '\\': - builder.WriteString(`\\`) - case '\b': - builder.WriteString(`\b`) - case '\t': - builder.WriteString(`\t`) - case '\n': - builder.WriteString(`\n`) - case '\f': - builder.WriteString(`\f`) - case '\r': - builder.WriteString(`\r`) - default: - if r < 0x20 || r == 0x7f { - fmt.Fprintf(&builder, `\u%04X`, r) - continue - } - builder.WriteRune(r) +// checkNoIncludeLines refuses output that telemt would expand as a file +// include. Before it parses TOML, telemt reads every line that, trimmed, +// starts with "include" followed by "=" as an include of the named file, in +// any table (src/config/load/includes.rs). The encoder writes a key named +// include bare, so such a key would make telemt read other files on the node +// host. The check copies telemt's matcher rather than relying on how the +// encoder quotes keys. +func checkNoIncludeLines(rendered string) error { + table := "the top level" + for line := range strings.Lines(rendered) { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "[") { + table = trimmed + continue } - } - builder.WriteByte('"') - return builder.String() -} - -func isBareKey(key string) bool { - if key == "" { - return false - } - for _, r := range key { - if !(r >= 'a' && r <= 'z') && - !(r >= 'A' && r <= 'Z') && - !(r >= '0' && r <= '9') && - r != '_' && - r != '-' { - return false + rest, found := strings.CutPrefix(trimmed, "include") + if found && strings.HasPrefix(strings.TrimSpace(rest), "=") { + return fmt.Errorf("mtproto config must not use the key include (found in %s): telemt would read it as a file include", table) } } - return true -} - -func joinPath(path []string) string { - if len(path) == 0 { - return "" - } - parts := make([]string, 0, len(path)) - for _, segment := range path { - parts = append(parts, renderKey(segment)) - } - return strings.Join(parts, ".") -} - -func appendPath(path []string, key string) []string { - result := make([]string, 0, len(path)+1) - result = append(result, path...) - result = append(result, key) - return result -} - -func writeSectionSpacing(builder *strings.Builder) { - if builder.Len() == 0 { - return - } - if !strings.HasSuffix(builder.String(), "\n") { - builder.WriteByte('\n') - } - builder.WriteByte('\n') + return nil } diff --git a/backend/mtproto/toml_test.go b/backend/mtproto/toml_test.go index 40e27434..9d8b5537 100644 --- a/backend/mtproto/toml_test.go +++ b/backend/mtproto/toml_test.go @@ -5,20 +5,33 @@ import ( "strings" "testing" - "github.com/pelletier/go-toml" + "github.com/pelletier/go-toml/v2" ) -// decodeTOML parses rendered output with an independent TOML decoder, so the -// tests check real TOML validity rather than the renderer's own idea of it. -func decodeTOML(t *testing.T, rendered string) *toml.Tree { +// decodeTOML parses rendered output back, so the tests check the values the +// TOML means rather than the encoder's exact spelling. +func decodeTOML(t *testing.T, rendered string) map[string]any { t.Helper() - tree, err := toml.Load(rendered) - if err != nil { + var tree map[string]any + if err := toml.Unmarshal([]byte(rendered), &tree); err != nil { t.Fatalf("rendered TOML does not decode: %v\n---\n%s", err, rendered) } return tree } +// tomlAt returns the value at path in a decoded TOML tree, or nil. +func tomlAt(tree map[string]any, path ...string) any { + var current any = tree + for _, key := range path { + table, ok := current.(map[string]any) + if !ok { + return nil + } + current = table[key] + } + return current +} + func decodeJSONObject(t *testing.T, raw string) map[string]any { t.Helper() decoder := json.NewDecoder(strings.NewReader(raw)) @@ -33,34 +46,17 @@ func decodeJSONObject(t *testing.T, raw string) map[string]any { func TestTOMLEscapesControlCharacters(t *testing.T) { value := "bell\a vtab\v soh\x01 quote\" backslash\\ newline\n tab\t del\x7f esc\x1b unicodeé" key := "key\x01with\\backslash\ttab" + quotedKey := `say "hi"` out, err := renderTOML(map[string]any{ - "general": map[string]any{"value": value, key: "k"}, + "general": map[string]any{"value": value, key: "k", quotedKey: "q"}, }) if err != nil { t.Fatal(err) } - // go-toml v1.9.5 cannot decode \" inside a quoted key (TOML 1.0 allows - // it), so this key is checked on the rendered text only. - quoted, err := renderTOML(map[string]any{"quoted": map[string]any{`say "hi"`: "q"}}) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(quoted, `"say \"hi\"" = "q"`) { - t.Errorf("key with a quote is not rendered as an escaped basic string\n---\n%s", quoted) - } - - for _, want := range []string{`\u0007`, `\u000B`, `\u0001`, `\"`, `\\`, `\n`, `\u007F`, `\u001B`} { - if !strings.Contains(out, want) { - t.Errorf("rendered TOML missing escape %s\n---\n%s", want, out) - } - } - for _, invalid := range []string{`\a`, `\v`, `\x`} { - if strings.Contains(out, invalid) { - t.Errorf("rendered TOML contains non-TOML escape %s\n---\n%s", invalid, out) - } - } + // telemt reads the file line by line before it parses TOML (includes), so + // no value may spill onto a second line. for _, line := range strings.Split(out, "\n") { for _, r := range line { if (r < 0x20 && r != '\t') || r == 0x7f { @@ -70,11 +66,10 @@ func TestTOMLEscapesControlCharacters(t *testing.T) { } tree := decodeTOML(t, out) - if got := tree.GetPath([]string{"general", "value"}); got != value { - t.Errorf("value round trip = %q, want %q", got, value) - } - if got := tree.GetPath([]string{"general", key}); got != "k" { - t.Errorf("escaped key round trip = %v, want %q", got, "k") + for name, want := range map[string]string{"value": value, key: "k", quotedKey: "q"} { + if got := tomlAt(tree, "general", name); got != want { + t.Errorf("general[%q] round trip = %#v, want %q\n---\n%s", name, got, want, out) + } } } @@ -87,7 +82,7 @@ func TestTOMLDropsNullValues(t *testing.T) { t.Errorf("null values must be dropped\n---\n%s", out) } tree := decodeTOML(t, out) - if got := tree.GetPath([]string{"general", "x"}); got != int64(1) { + if got := tomlAt(tree, "general", "x"); got != int64(1) { t.Errorf("general.x = %#v, want 1", got) } @@ -101,8 +96,48 @@ func TestTOMLDropsNullValues(t *testing.T) { } // A null array element has no TOML meaning and is an error. - if out, err := renderTOML(decodeJSONObject(t, `{"general": {"list": [1, null]}}`)); err == nil { + out, err = renderTOML(decodeJSONObject(t, `{"general": {"list": [1, null]}}`)) + if err == nil { t.Errorf("null inside an array must be an error, got\n%s", out) + } else if !strings.Contains(err.Error(), "general.list[1]") { + t.Errorf("error %q does not name general.list[1]", err) + } +} + +func TestTOMLNumbers(t *testing.T) { + out, err := renderTOML(decodeJSONObject(t, `{"general": {"int": 443, "neg": -5, "float": 1.5, "exp": 1e5, "whole_float": 1.0, "max": 9223372036854775807}}`)) + if err != nil { + t.Fatal(err) + } + // The encoder writes a json.Number as a string; telemt needs numbers. + if !strings.Contains(out, "int = 443\n") { + t.Errorf("integer is not rendered as a bare TOML integer\n---\n%s", out) + } + + tree := decodeTOML(t, out) + expect := map[string]any{ + "int": int64(443), + "neg": int64(-5), + "float": 1.5, + "exp": float64(100000), + "whole_float": float64(1), + "max": int64(9223372036854775807), + } + for key, want := range expect { + if got := tomlAt(tree, "general", key); got != want { + t.Errorf("general.%s = %#v, want %#v\n---\n%s", key, got, want, out) + } + } + + for raw, wantInError := range map[string]string{ + `{"general": {"big": 9223372036854775808}}`: "general.big", + `{"general": {"huge": 1e400}}`: "general.huge", + } { + if out, err := renderTOML(decodeJSONObject(t, raw)); err == nil { + t.Errorf("%s: want an error, got\n%s", raw, out) + } else if !strings.Contains(err.Error(), wantInError) { + t.Errorf("%s: error %q does not name %s", raw, err, wantInError) + } } } @@ -116,38 +151,47 @@ func TestTOMLArrayOfTables(t *testing.T) { } tree := decodeTOML(t, out) - listeners, ok := tree.GetPath([]string{"server", "listeners"}).([]*toml.Tree) + listeners, ok := tomlAt(tree, "server", "listeners").([]any) if !ok || len(listeners) != 2 { - t.Fatalf("server.listeners = %#v, want two tables", tree.GetPath([]string{"server", "listeners"})) + t.Fatalf("server.listeners = %#v, want two tables", tomlAt(tree, "server", "listeners")) } - if listeners[0].Get("ip") != "0.0.0.0" || listeners[1].Get("ip") != "::" { - t.Errorf("listener ips = %v, %v", listeners[0].Get("ip"), listeners[1].Get("ip")) + for index, want := range []string{"0.0.0.0", "::"} { + listener, _ := listeners[index].(map[string]any) + if got := listener["ip"]; got != want { + t.Errorf("server.listeners[%d].ip = %#v, want %q", index, got, want) + } } - if got := tree.GetPath([]string{"server", "port"}); got != int64(443) { + if got := tomlAt(tree, "server", "port"); got != int64(443) { t.Errorf("server.port = %#v, want 443", got) } } -// telemt expands any line of the form `include = ""` as a textual -// include before it parses TOML (src/config/load/includes.rs), whatever table -// it sits in. A key named include must therefore never render bare. -func TestTOMLQuotesIncludeKeys(t *testing.T) { - out, err := renderTOML(map[string]any{ - "general": map[string]any{"include": "/etc/shadow"}, - "access": map[string]any{"users": map[string]any{"include": testSecret}}, - }) - if err != nil { - t.Fatal(err) - } - for _, line := range strings.Split(out, "\n") { - rest, found := strings.CutPrefix(strings.TrimSpace(line), "include") - if found && strings.HasPrefix(strings.TrimSpace(rest), "=") { - t.Errorf("line would be expanded by telemt as an include: %q", line) +// telemt expands any line of the form `include = ` as a textual include +// before it parses TOML (src/config/load/includes.rs), whatever table it sits +// in. A key named include must therefore never reach telemt. +func TestTOMLRejectsIncludeKeys(t *testing.T) { + cases := map[string]map[string]any{ + "[general]": {"general": map[string]any{"include": "/etc/shadow"}}, + "[access.users]": {"access": map[string]any{"users": map[string]any{"include": testSecret}}}, + "[[server.listeners]]": {"server": map[string]any{"listeners": []any{map[string]any{"include": "/etc/shadow"}}}}, + } + for table, root := range cases { + out, err := renderTOML(root) + if err == nil { + t.Errorf("%s: include key rendered without error\n---\n%s", table, out) + continue + } + if !strings.Contains(err.Error(), table) { + t.Errorf("%s: error %q does not name the table", table, err) } } - tree := decodeTOML(t, out) - if got := tree.GetPath([]string{"general", "include"}); got != "/etc/shadow" { - t.Errorf("general.include = %v, want the quoted key to round trip", got) + // Only the exact key matches, as in telemt. + out, err := renderTOML(map[string]any{"general": map[string]any{"included": "x", "inline": map[string]any{"nested": []any{"include = x"}}}}) + if err != nil { + t.Fatalf("keys that only start with include must render: %v", err) + } + if got := tomlAt(decodeTOML(t, out), "general", "included"); got != "x" { + t.Errorf("general.included = %#v, want x", got) } } diff --git a/controller/mtproto_start_unix_test.go b/controller/mtproto_start_unix_test.go index 5326b762..68fa4845 100644 --- a/controller/mtproto_start_unix_test.go +++ b/controller/mtproto_start_unix_test.go @@ -17,6 +17,7 @@ import ( "time" "github.com/google/uuid" + "github.com/pelletier/go-toml/v2" "google.golang.org/protobuf/proto" "github.com/pasarguard/node/backend" @@ -111,19 +112,20 @@ func fakeTelemtAPIListen(configPath string) (string, error) { if err != nil { return "", err } - section := "" - for _, line := range strings.Split(string(data), "\n") { - line = strings.TrimSpace(line) - if strings.HasPrefix(line, "[") { - section = strings.Trim(line, "[]") - continue - } - key, value, ok := strings.Cut(line, "=") - if ok && section == "server.api" && strings.TrimSpace(key) == "listen" { - return strconv.Unquote(strings.TrimSpace(value)) - } + var rendered struct { + Server struct { + API struct { + Listen string `toml:"listen"` + } `toml:"api"` + } `toml:"server"` + } + if err := toml.Unmarshal(data, &rendered); err != nil { + return "", err + } + if rendered.Server.API.Listen == "" { + return "", errors.New("server.api.listen not found in " + configPath) } - return "", errors.New("server.api.listen not found in " + configPath) + return rendered.Server.API.Listen, nil } // newFakeTelemtController returns a controller whose telemt is the fake. It diff --git a/go.mod b/go.mod index 7fd05fd2..71484063 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,7 @@ require ( github.com/google/uuid v1.6.0 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/joho/godotenv v1.5.1 - github.com/pelletier/go-toml v1.9.5 + github.com/pelletier/go-toml/v2 v2.4.3 github.com/shirou/gopsutil/v4 v4.26.9 github.com/vishvananda/netlink v1.3.1 github.com/xtls/xray-core v1.260327.1-0.20260930074004-b26a91de4f32 diff --git a/go.sum b/go.sum index 924b3e94..2a6ed6d2 100644 --- a/go.sum +++ b/go.sum @@ -96,6 +96,8 @@ github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCL github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= +github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pion/dtls/v3 v3.1.5 h1:9xJtVsHwMYeSjPp5Hh1FTis4DchnQWtnOa5o+6ygqfc= github.com/pion/dtls/v3 v3.1.5/go.mod h1:gz1K4jg6c+fq86oQMH4pilpCEOEPwmEr2jY+VcF/mkU= github.com/pion/logging v0.2.4 h1:tTew+7cmQ+Mc1pTBLKH2puKsOvhm32dROumOZ655zB8= From 0ca58f234d6f5375c5456d1e5252a01293dd85fd Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 26 Oct 2026 12:27:27 +0330 Subject: [PATCH 29/40] refactor(mtproto): use omitzero for the telemt user requests The create request now sends the limits as plain uint32 with omitzero instead of *int with omitempty, so the pointer and the int range conversion go away; a zero limit is still left out. The patch request keeps pointers without omitzero on the clearable fields: JSON Merge Patch needs nil sent as null to clear an override. --- backend/mtproto/client.go | 70 +++++++++++---------------------- backend/mtproto/mtproto_test.go | 27 +++++++++++++ 2 files changed, 50 insertions(+), 47 deletions(-) diff --git a/backend/mtproto/client.go b/backend/mtproto/client.go index c8033eaa..1fa3aebd 100644 --- a/backend/mtproto/client.go +++ b/backend/mtproto/client.go @@ -64,22 +64,24 @@ type telemtHealthReady struct { Reason string `json:"reason"` } +// telemtCreateUserRequest leaves out unset fields; a zero limit means no +// per-user limit. type telemtCreateUserRequest struct { Username string `json:"username"` - Secret string `json:"secret,omitempty"` - UserAdTag string `json:"user_ad_tag,omitempty"` - MaxTCPConns *int `json:"max_tcp_conns,omitempty"` - MaxUniqueIPs *int `json:"max_unique_ips,omitempty"` + Secret string `json:"secret,omitzero"` + UserAdTag string `json:"user_ad_tag,omitzero"` + MaxTCPConns uint32 `json:"max_tcp_conns,omitzero"` + MaxUniqueIPs uint32 `json:"max_unique_ips,omitzero"` } // telemtPatchUserRequest uses JSON Merge Patch: omitted fields are unchanged, -// JSON null removes the per-user override, and a value sets it. omitempty is -// therefore not used on the clearable fields. +// JSON null removes the per-user override, and a value sets it. The clearable +// fields are pointers without omitzero, so nil is sent as null. type telemtPatchUserRequest struct { - Secret string `json:"secret,omitempty"` + Secret string `json:"secret,omitzero"` UserAdTag *string `json:"user_ad_tag"` - MaxTCPConns *int `json:"max_tcp_conns"` - MaxUniqueIPs *int `json:"max_unique_ips"` + MaxTCPConns *uint32 `json:"max_tcp_conns"` + MaxUniqueIPs *uint32 `json:"max_unique_ips"` } type apiStatusError struct { @@ -163,11 +165,7 @@ func (c *apiClient) ActiveIPs(ctx context.Context) (map[string][]string, error) } func (c *apiClient) CreateUser(ctx context.Context, user *runtimeUser) error { - request, err := buildCreateUserRequest(user) - if err != nil { - return err - } - _, err = doAPIRequest[telemtUserInfo](ctx, c, http.MethodPost, "/v1/users", request) + _, err := doAPIRequest[telemtUserInfo](ctx, c, http.MethodPost, "/v1/users", buildCreateUserRequest(user)) return err } @@ -187,24 +185,14 @@ func (c *apiClient) DeleteUser(ctx context.Context, username string) error { return err } -func buildCreateUserRequest(user *runtimeUser) (*telemtCreateUserRequest, error) { - maxTCPConns, err := optionalPositiveInt(user.MaxTCPConns, "max_tcp_conns") - if err != nil { - return nil, err - } - maxUniqueIPs, err := optionalPositiveInt(user.MaxUniqueIPs, "max_unique_ips") - if err != nil { - return nil, err - } - - request := &telemtCreateUserRequest{ +func buildCreateUserRequest(user *runtimeUser) *telemtCreateUserRequest { + return &telemtCreateUserRequest{ Username: user.Username, Secret: user.Secret, UserAdTag: user.UserAdTag, - MaxTCPConns: maxTCPConns, - MaxUniqueIPs: maxUniqueIPs, + MaxTCPConns: user.MaxTCPConns, + MaxUniqueIPs: user.MaxUniqueIPs, } - return request, nil } func buildPatchUserRequest(user *runtimeUser) (*telemtPatchUserRequest, error) { @@ -212,19 +200,10 @@ func buildPatchUserRequest(user *runtimeUser) (*telemtPatchUserRequest, error) { return nil, errors.New("mtproto user is nil") } - maxTCPConns, err := optionalPositiveInt(user.MaxTCPConns, "max_tcp_conns") - if err != nil { - return nil, err - } - maxUniqueIPs, err := optionalPositiveInt(user.MaxUniqueIPs, "max_unique_ips") - if err != nil { - return nil, err - } - request := &telemtPatchUserRequest{ Secret: user.Secret, - MaxTCPConns: maxTCPConns, - MaxUniqueIPs: maxUniqueIPs, + MaxTCPConns: positiveOrNil(user.MaxTCPConns), + MaxUniqueIPs: positiveOrNil(user.MaxUniqueIPs), } if user.UserAdTag != "" { tag := user.UserAdTag @@ -233,16 +212,13 @@ func buildPatchUserRequest(user *runtimeUser) (*telemtPatchUserRequest, error) { return request, nil } -func optionalPositiveInt(value uint32, field string) (*int, error) { +// positiveOrNil returns nil for 0 (no per-user limit), which a patch sends as +// null to clear the override. +func positiveOrNil(value uint32) *uint32 { if value == 0 { - return nil, nil - } - maxInt := int(^uint(0) >> 1) - if uint64(value) > uint64(maxInt) { - return nil, fmt.Errorf("mtproto field %s exceeds native int range", field) + return nil } - converted := int(value) - return &converted, nil + return &value } func doAPIRequest[T any](ctx context.Context, client *apiClient, method, path string, body any) (*T, error) { diff --git a/backend/mtproto/mtproto_test.go b/backend/mtproto/mtproto_test.go index 495d0aa5..84e91852 100644 --- a/backend/mtproto/mtproto_test.go +++ b/backend/mtproto/mtproto_test.go @@ -75,3 +75,30 @@ func TestBuildPatchUserRequest_ClearsOptionalFields(t *testing.T) { t.Errorf("max_unique_ips = %v, want 2", setPayload.MaxUniqueIPs) } } + +func TestBuildCreateUserRequest_OmitsUnsetFields(t *testing.T) { + encoded, err := json.Marshal(buildCreateUserRequest(&runtimeUser{Username: "alice", Secret: testSecret})) + if err != nil { + t.Fatal(err) + } + if got, want := string(encoded), `{"username":"alice","secret":"`+testSecret+`"}`; got != want { + t.Errorf("create JSON = %s, want %s", got, want) + } + + tag := "aabbccddeeff00112233445566778899" + encoded, err = json.Marshal(buildCreateUserRequest(&runtimeUser{ + Username: "alice", + Secret: testSecret, + UserAdTag: tag, + MaxTCPConns: 4, + MaxUniqueIPs: 2, + })) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{`"user_ad_tag":"` + tag + `"`, `"max_tcp_conns":4`, `"max_unique_ips":2`} { + if !strings.Contains(string(encoded), want) { + t.Errorf("create JSON missing %s: %s", want, encoded) + } + } +} From 8cfb762c23fa12d11793661e3b76f4ab8d02fb94 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 26 Oct 2026 12:44:13 +0330 Subject: [PATCH 30/40] fix(mtproto): reject include keys at any depth of the panel config A key named include in any table becomes a telemt file include, and the render already refuses it; NewConfig now rejects it too, naming the path, so the panel gets the error up front instead of a failed Start. Also from review: - the TOML conversion refuses container types it does not walk, since they could hide a json.Number that would render as a string - a golden test pins one full render, so a go-toml upgrade that changes the output shows up - table test for checkNoIncludeLines against telemt's matcher, signed zero, and the request JSON without a secret - parseConfigUsers fails loudly when a config does not decode --- backend/mtproto/config.go | 32 +++++++++ backend/mtproto/config_test.go | 2 + backend/mtproto/mtproto_test.go | 16 +++++ backend/mtproto/sync_test.go | 4 +- backend/mtproto/toml.go | 7 ++ backend/mtproto/toml_test.go | 118 ++++++++++++++++++++++++++++++++ 6 files changed, 178 insertions(+), 1 deletion(-) diff --git a/backend/mtproto/config.go b/backend/mtproto/config.go index 14e42707..a48f31aa 100644 --- a/backend/mtproto/config.go +++ b/backend/mtproto/config.go @@ -6,6 +6,7 @@ import ( "fmt" "io" "log" + "maps" "regexp" "slices" "strings" @@ -152,6 +153,13 @@ func validateConfigRoot(root map[string]any) error { } } + // telemt reads a key named include in any table as a file include (see + // checkNoIncludeLines). The render refuses one too; rejecting it here + // tells the panel at once. + if path, found := findKey(root, "include", ""); found { + return fmt.Errorf("mtproto config must not define %s: telemt would read other files on the node host", path) + } + if _, err := listenPorts(root); err != nil { return err } @@ -460,6 +468,30 @@ func isSet(object map[string]any, key string) bool { return object != nil && object[key] != nil } +// findKey returns the path of the first set key named key at any depth of +// value, tables inside arrays included. +func findKey(value any, key, path string) (string, bool) { + switch typed := value.(type) { + case map[string]any: + for _, name := range slices.Sorted(maps.Keys(typed)) { + namePath := appendTOMLPath(path, name) + if name == key && typed[name] != nil { + return namePath, true + } + if found, ok := findKey(typed[name], key, namePath); ok { + return found, true + } + } + case []any: + for index, item := range typed { + if found, ok := findKey(item, key, fmt.Sprintf("%s[%d]", path, index)); ok { + return found, true + } + } + } + return "", false +} + func ensureObject(target map[string]any, key string) map[string]any { if existing, ok := target[key]; ok { if existingMap, ok := existing.(map[string]any); ok { diff --git a/backend/mtproto/config_test.go b/backend/mtproto/config_test.go index 15fdbdee..e796609b 100644 --- a/backend/mtproto/config_test.go +++ b/backend/mtproto/config_test.go @@ -58,6 +58,8 @@ func TestNewConfigRejectsNodeManagedKeys(t *testing.T) { key string }{ {`{"include": "x"}`, "include"}, + {`{"general": {"include": "x"}}`, "general.include"}, + {`{"server": {"listeners": [{"ip": "::", "include": "x"}]}}`, "server.listeners[0].include"}, {`{"server": {"api": {"read_only": true}}}`, "server.api"}, {`{"server": {"admin_api": {}}}`, "server.admin_api"}, {`{"server": {"metrics_port": 1}}`, "server.metrics_port"}, diff --git a/backend/mtproto/mtproto_test.go b/backend/mtproto/mtproto_test.go index 84e91852..27a99290 100644 --- a/backend/mtproto/mtproto_test.go +++ b/backend/mtproto/mtproto_test.go @@ -54,6 +54,15 @@ func TestBuildPatchUserRequest_ClearsOptionalFields(t *testing.T) { } } + // No secret: the patch leaves telemt's secret as it is. + noSecret, err := buildPatchUserRequest(&runtimeUser{Username: "alice"}) + if err != nil { + t.Fatal(err) + } + if encoded, _ := json.Marshal(noSecret); strings.Contains(string(encoded), "secret") { + t.Errorf("patch JSON without a secret sends one: %s", encoded) + } + tag := "aabbccddeeff00112233445566778899" setPayload, err := buildPatchUserRequest(&runtimeUser{ Username: "alice", @@ -84,6 +93,13 @@ func TestBuildCreateUserRequest_OmitsUnsetFields(t *testing.T) { if got, want := string(encoded), `{"username":"alice","secret":"`+testSecret+`"}`; got != want { t.Errorf("create JSON = %s, want %s", got, want) } + encoded, err = json.Marshal(buildCreateUserRequest(&runtimeUser{Username: "alice"})) + if err != nil { + t.Fatal(err) + } + if got, want := string(encoded), `{"username":"alice"}`; got != want { + t.Errorf("create JSON without a secret = %s, want %s", got, want) + } tag := "aabbccddeeff00112233445566778899" encoded, err = json.Marshal(buildCreateUserRequest(&runtimeUser{ diff --git a/backend/mtproto/sync_test.go b/backend/mtproto/sync_test.go index 14a5b269..1d4b3b62 100644 --- a/backend/mtproto/sync_test.go +++ b/backend/mtproto/sync_test.go @@ -217,7 +217,9 @@ func parseConfigUsers(config string) map[string]string { users := make(map[string]string) var tree map[string]any if err := toml.Unmarshal([]byte(config), &tree); err != nil { - return users + // Every config here is rendered by the node or rewritten by a fake; + // a decode failure is a bug, not a config without users. + panic(fmt.Sprintf("parseConfigUsers: config does not decode: %v\n---\n%s", err, config)) } configured, _ := tomlAt(tree, "access", "users").(map[string]any) for name, value := range configured { diff --git a/backend/mtproto/toml.go b/backend/mtproto/toml.go index 01210a05..340cd52b 100644 --- a/backend/mtproto/toml.go +++ b/backend/mtproto/toml.go @@ -3,6 +3,7 @@ package mtproto import ( "encoding/json" "fmt" + "reflect" "strings" "github.com/pelletier/go-toml/v2" @@ -62,6 +63,12 @@ func tomlValue(value any, path string) (any, error) { case json.Number: return tomlNumber(typed, path) default: + // Any other container could hide a json.Number from the conversion + // above, so it is refused rather than passed through. + switch reflect.ValueOf(value).Kind() { + case reflect.Map, reflect.Slice, reflect.Array, reflect.Struct, reflect.Pointer, reflect.Interface: + return nil, fmt.Errorf("mtproto config field %s has unsupported type %T", path, value) + } return value, nil } } diff --git a/backend/mtproto/toml_test.go b/backend/mtproto/toml_test.go index 9d8b5537..bdc68485 100644 --- a/backend/mtproto/toml_test.go +++ b/backend/mtproto/toml_test.go @@ -2,6 +2,7 @@ package mtproto import ( "encoding/json" + "math" "strings" "testing" @@ -166,6 +167,123 @@ func TestTOMLArrayOfTables(t *testing.T) { } } +func TestTOMLSignedZero(t *testing.T) { + out, err := renderTOML(decodeJSONObject(t, `{"general": {"int_zero": -0, "float_zero": -0.0}}`)) + if err != nil { + t.Fatal(err) + } + tree := decodeTOML(t, out) + if got := tomlAt(tree, "general", "int_zero"); got != int64(0) { + t.Errorf("general.int_zero = %#v, want 0", got) + } + if got, ok := tomlAt(tree, "general", "float_zero").(float64); !ok || got != 0 || !math.Signbit(got) { + t.Errorf("general.float_zero = %#v, want -0.0\n---\n%s", tomlAt(tree, "general", "float_zero"), out) + } +} + +func TestTOMLRefusesUnconvertedContainers(t *testing.T) { + // A typed container could hold a json.Number the conversion never sees. + if out, err := renderTOML(map[string]any{"server": map[string]any{"listeners": []map[string]any{{"port": json.Number("443")}}}}); err == nil { + t.Errorf("typed container rendered without error\n---\n%s", out) + } +} + +// TestRenderGolden pins the whole rendered file, so a go-toml upgrade that +// changes the output shows up here. +func TestRenderGolden(t *testing.T) { + cfg, err := NewConfig(`{"general": {"ad_tag": null, "modes": {"classic": false, "secure": false, "tls": true}}, "censorship": {"tls_domain": "example.com"}, "server": {"port": 443, "listeners": [{"ip": "0.0.0.0"}, {"ip": "::", "port": 8443}]}}`) + if err != nil { + t.Fatal(err) + } + out, err := cfg.Render(map[string]*runtimeUser{ + "1": {Username: "1", Secret: testSecret, MaxTCPConns: 4}, + "2": {Username: "2", Secret: otherSecret, UserAdTag: thirdSecret}, + }, &runtimeUser{Username: "pgm_0123456789abcdef", Secret: thirdSecret}, 12345, 23456, "Bearer token") + if err != nil { + t.Fatal(err) + } + + const want = `[access] +[access.user_ad_tags] +2 = '0123456789abcdef0123456789abcdef' + +[access.user_enabled] +pgm_0123456789abcdef = false + +[access.user_max_tcp_conns] +1 = 4 + +[access.users] +1 = '00112233445566778899aabbccddeeff' +2 = 'ffeeddccbbaa00112233445566778899' +pgm_0123456789abcdef = '0123456789abcdef0123456789abcdef' + +[censorship] +tls_domain = 'example.com' + +[general] +disable_colors = true + +[general.modes] +classic = false +secure = false +tls = true + +[general.telemetry] +user_enabled = true + +[logging] +destination = 'stderr' + +[server] +metrics_listen = '127.0.0.1:23456' +metrics_port = 23456 +metrics_whitelist = ['127.0.0.1/32', '::1/128'] +port = 443 + +[server.api] +auth_header = 'Bearer token' +enabled = true +listen = '127.0.0.1:12345' +read_only = false +whitelist = ['127.0.0.1/32', '::1/128'] + +[[server.listeners]] +ip = '0.0.0.0' + +[[server.listeners]] +ip = '::' +port = 8443 +` + if out != want { + t.Errorf("rendered config changed\n--- got\n%s\n--- want\n%s", out, want) + } +} + +// checkNoIncludeLines must match exactly the lines telemt's +// preprocess_includes expands: trimmed, "include", optional space, "=". +func TestCheckNoIncludeLines(t *testing.T) { + for line, expanded := range map[string]bool{ + "include = 'x'": true, + "include='x'": true, + "\tinclude = 'x'": true, + " include\t= 'x'": true, + "include = 'x'\r": true, + "include_x = 1": false, + "included = 1": false, + "Include = 1": false, + "'include' = 'x'": false, + `"include" = 'x'`: false, + "x = 'include = y'": false, + "[include]": false, + } { + err := checkNoIncludeLines("[general]\n" + line + "\n") + if got := err != nil; got != expanded { + t.Errorf("%q: refused = %v, want %v (err %v)", line, got, expanded, err) + } + } +} + // telemt expands any line of the form `include = ` as a textual include // before it parses TOML (src/config/load/includes.rs), whatever table it sits // in. A key named include must therefore never reach telemt. From 4579899e0dd5f8f6f1b31207fa5a71c0e86849da Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 26 Oct 2026 19:53:04 +0330 Subject: [PATCH 31/40] feat(mtproto): accept telemt WEB listeners and vhosts behind a private front Accept the web section and listeners with transport "web" on a loopback or private address and their own port. Vhosts may only use an http_upstream decoy on a loopback or private IP literal, must not define profiles, and may carry a user_profile template that the node keeps per vhost and strips from the config. web.debug and web.limits.max_profiles stay node managed. --- backend/mtproto/config.go | 34 +++-- backend/mtproto/config_test.go | 122 +++++++++++++++++- backend/mtproto/webconfig.go | 218 +++++++++++++++++++++++++++++++++ 3 files changed, 361 insertions(+), 13 deletions(-) create mode 100644 backend/mtproto/webconfig.go diff --git a/backend/mtproto/config.go b/backend/mtproto/config.go index a48f31aa..91b59d43 100644 --- a/backend/mtproto/config.go +++ b/backend/mtproto/config.go @@ -37,7 +37,8 @@ var ( // the node manages them, or they would reach past the node. rejectedConfigKeys = []rejectedConfigKey{ {path: []string{"include"}, reason: "telemt would read other files on the node host"}, - {path: []string{"web"}, reason: "the web transport can serve files from the node host"}, + {path: []string{"web", "debug"}, reason: "the WEB debug endpoint would expose node traffic records"}, + {path: []string{"web", "limits", "max_profiles"}, reason: "the node sizes it from the WEB profiles it renders"}, {path: []string{"server", "api"}, reason: "the node manages the telemt control API"}, {path: []string{"server", "admin_api"}, reason: "it is an alias of server.api, which the node manages"}, {path: []string{"server", "metrics_port"}, reason: "the node manages the metrics endpoint"}, @@ -53,6 +54,8 @@ var ( {"server"}, {"access"}, {"logging"}, + {"web"}, + {"web", "limits"}, } ) @@ -62,8 +65,9 @@ type rejectedConfigKey struct { } type Config struct { - base map[string]any - InboundTag string + base map[string]any + InboundTag string + webProfiles []webUserProfile } type runtimeUser struct { @@ -119,6 +123,12 @@ func NewConfig(raw string) (*Config, error) { return nil, err } + webProfiles, err := takeWebProfiles(cfg.base) + if err != nil { + return nil, err + } + cfg.webProfiles = webProfiles + return cfg, nil } @@ -167,10 +177,9 @@ func validateConfigRoot(root map[string]any) error { return validateListenerTransports(root) } -// validateListenerTransports allows only MTProxy listeners. telemt WEB -// listeners need [[web.vhosts]], and the node rejects the web section, so a -// web listener would only make telemt exit at startup. Call it after -// listenPorts, which checks the listener shapes. +// validateListenerTransports allows MTProxy listeners and WEB listeners that +// pass validateWebListener. Call it after listenPorts, which checks the +// listener shapes. func validateListenerTransports(root map[string]any) error { server, _ := objectAt(root, "server") listeners, _ := server["listeners"].([]any) @@ -179,8 +188,15 @@ func validateListenerTransports(root map[string]any) error { if !isSet(listener, "transport") { continue } - if transport, ok := listener["transport"].(string); !ok || transport != "mtproxy" { - return fmt.Errorf("mtproto config field server.listeners[%d].transport must be \"mtproxy\"; the node supports only MTProxy listeners", index) + transport, _ := listener["transport"].(string) + switch transport { + case "mtproxy": + case webTransport: + if err := validateWebListener(index, listener, server); err != nil { + return err + } + default: + return fmt.Errorf("mtproto config field server.listeners[%d].transport must be \"mtproxy\" (or \"web\" for a WEB listener behind an HTTPS front)", index) } } return nil diff --git a/backend/mtproto/config_test.go b/backend/mtproto/config_test.go index e796609b..c94b76d0 100644 --- a/backend/mtproto/config_test.go +++ b/backend/mtproto/config_test.go @@ -65,7 +65,6 @@ func TestNewConfigRejectsNodeManagedKeys(t *testing.T) { {`{"server": {"metrics_port": 1}}`, "server.metrics_port"}, {`{"server": {"metrics_listen": "0.0.0.0:9090"}}`, "server.metrics_listen"}, {`{"server": {"metrics_whitelist": ["0.0.0.0/0"]}}`, "server.metrics_whitelist"}, - {`{"web": {}}`, "web"}, {`{"access": {"user_rate_limits": {}}}`, "access.user_rate_limits"}, {`{"access": {"user_source_deny": {}}}`, "access.user_source_deny"}, {`{"access": {"users": {}}}`, "access.users"}, @@ -88,6 +87,11 @@ func TestNewConfigRejectsNodeManagedKeys(t *testing.T) { }) } + // An empty web object is not enabled and needs no listener. + if _, err := NewConfig(`{"web": {}}`); err != nil { + t.Errorf("an empty web object must be accepted, got %v", err) + } + // Keys that only share a prefix with a rejected one stay allowed. if _, err := NewConfig(`{"server": {"port": 443, "listen_backlog": 128}, "access": {"replay_check_len": 65536}}`); err != nil { t.Errorf("unrelated keys must be accepted, got %v", err) @@ -123,11 +127,9 @@ func TestNewConfigValidatesShapes(t *testing.T) { } } -// telemt WEB listeners need [[web.vhosts]], which the node rejects, so a web -// listener would only crash telemt at startup. func TestNewConfigRejectsNonMTProxyListeners(t *testing.T) { cases := map[string]string{ - "web": `{"server": {"listeners": [{"ip": "::", "transport": "web"}]}}`, + "socks": `{"server": {"listeners": [{"ip": "::", "transport": "socks"}]}}`, "upper case": `{"server": {"listeners": [{"ip": "::"}, {"ip": "0.0.0.0", "transport": "WEB"}]}}`, "not string": `{"server": {"listeners": [{"ip": "::", "transport": 1}]}}`, } @@ -421,3 +423,115 @@ func TestSyncUserReturnsValidationError(t *testing.T) { t.Errorf("SyncUser error = %v, want the secret validation error", err) } } + +const webTestListeners = `"listeners": [{"ip": "0.0.0.0"}, {"ip": "127.0.0.1", "port": 18080, "transport": "web", "web_trusted_proxy_cidrs": ["127.0.0.1/32"]}]` + +// webTestConfig builds a WEB config with one vhost; vhostExtra is appended to +// the vhost object. +func webTestConfig(vhostExtra string) string { + return `{"server": {"port": 443, ` + webTestListeners + `}, "web": {"enabled": true, "vhosts": [{"host": "proxy.example.com", "public_addr": "203.0.113.10:443", "decoy": {"mode": "http_upstream", "upstream": "http://127.0.0.1:18081"}` + vhostExtra + `}]}}` +} + +func TestNewConfigAcceptsWeb(t *testing.T) { + cfg, err := NewConfig(webTestConfig(`, "user_profile": {"secret_mode": "plain", "max_sessions": 8}`)) + if err != nil { + t.Fatalf("NewConfig failed: %v", err) + } + if !cfg.HasWeb() { + t.Errorf("HasWeb() = false, want true") + } + want := []webUserProfile{{SecretMode: "plain", MaxSessions: 8}} + if !slices.Equal(cfg.webProfiles, want) { + t.Errorf("webProfiles = %+v, want %+v", cfg.webProfiles, want) + } + vhost := cfg.base["web"].(map[string]any)["vhosts"].([]any)[0].(map[string]any) + if _, ok := vhost["user_profile"]; ok { + t.Errorf("user_profile must be stripped from the stored vhost: %v", vhost) + } + + cfg, err = NewConfig(webTestConfig("")) + if err != nil { + t.Fatalf("NewConfig without user_profile failed: %v", err) + } + want = []webUserProfile{{SecretMode: "dd"}} + if !slices.Equal(cfg.webProfiles, want) { + t.Errorf("default webProfiles = %+v, want %+v", cfg.webProfiles, want) + } + + cfg, err = NewConfig(`{"server": {"port": 443}}`) + if err != nil { + t.Fatal(err) + } + if cfg.HasWeb() || cfg.webProfiles != nil { + t.Errorf("a config without web must not report WEB: HasWeb=%v profiles=%v", cfg.HasWeb(), cfg.webProfiles) + } +} + +func TestNewConfigRejectsUnsafeWeb(t *testing.T) { + withListener := func(listener string) string { + return `{"server": {"port": 443, "listeners": [{"ip": "0.0.0.0"}, ` + listener + `]}, "web": {"enabled": true, "vhosts": [{"host": "proxy.example.com", "public_addr": "203.0.113.10:443", "decoy": {"mode": "http_upstream", "upstream": "http://127.0.0.1:18081"}}]}}` + } + withVhost := func(vhost string) string { + return `{"server": {"port": 443, ` + webTestListeners + `}, "web": {"enabled": true, "vhosts": [` + vhost + `]}}` + } + withWeb := func(web string) string { + return `{"server": {"port": 443, ` + webTestListeners + `}, "web": ` + web + `}` + } + const host = `"host": "proxy.example.com", "public_addr": "203.0.113.10:443"` + + cases := []struct { + name, raw, want string + }{ + {"listener ip any v4", withListener(`{"ip": "0.0.0.0", "port": 18080, "transport": "web"}`), "private"}, + {"listener ip any v6", withListener(`{"ip": "::", "port": 18080, "transport": "web"}`), "private"}, + {"listener ip public", withListener(`{"ip": "198.51.100.1", "port": 18080, "transport": "web"}`), "private"}, + {"listener ip missing", withListener(`{"port": 18080, "transport": "web"}`), "private"}, + {"listener without port", withListener(`{"ip": "127.0.0.1", "transport": "web"}`), "port"}, + {"static directory decoy", withVhost(`{` + host + `, "decoy": {"mode": "static_directory", "directory": "/srv"}}`), "static_directory"}, + {"decoy hostname", withVhost(`{` + host + `, "decoy": {"mode": "http_upstream", "upstream": "http://example.com"}}`), "private"}, + {"decoy public ip", withVhost(`{` + host + `, "decoy": {"mode": "http_upstream", "upstream": "http://8.8.8.8"}}`), "private"}, + {"decoy missing", withVhost(`{` + host + `}`), "decoy"}, + {"vhost profiles", withVhost(`{` + host + `, "decoy": {"mode": "http_upstream", "upstream": "http://127.0.0.1:1"}, "profiles": []}`), "must not define web.vhosts[0].profiles"}, + {"web debug", withWeb(`{"enabled": true, "debug": {}, "vhosts": [{` + host + `, "decoy": {"mode": "http_upstream", "upstream": "http://127.0.0.1:1"}}]}`), "must not define web.debug"}, + {"web max_profiles", withWeb(`{"enabled": true, "limits": {"max_profiles": 9}, "vhosts": [{` + host + `, "decoy": {"mode": "http_upstream", "upstream": "http://127.0.0.1:1"}}]}`), "must not define web.limits.max_profiles"}, + {"secret mode ee", webTestConfig(`, "user_profile": {"secret_mode": "ee"}`), "secret_mode"}, + {"negative limit", webTestConfig(`, "user_profile": {"max_sessions": -1}`), "max_sessions"}, + {"fraction limit", webTestConfig(`, "user_profile": {"max_streams": 1.5}`), "max_streams"}, + {"string limit", webTestConfig(`, "user_profile": {"max_streams_per_session": "4"}`), "max_streams_per_session"}, + {"enabled without listener", `{"server": {"port": 443}, "web": {"enabled": true, "vhosts": [{` + host + `, "decoy": {"mode": "http_upstream", "upstream": "http://127.0.0.1:1"}}]}}`, "web"}, + {"listener without enabled web", withWeb(`{"enabled": false, "vhosts": [{` + host + `, "decoy": {"mode": "http_upstream", "upstream": "http://127.0.0.1:1"}}]}`), "web"}, + {"listener without vhost", withWeb(`{"enabled": true, "vhosts": []}`), "web"}, + {"listener without web", `{"server": {"port": 443, ` + webTestListeners + `}}`, "web"}, + {"socks listener", `{"server": {"listeners": [{"ip": "::", "transport": "socks"}]}}`, "must be \"mtproxy\""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := NewConfig(tc.raw) + if err == nil { + t.Fatalf("NewConfig(%s) succeeded, want an error containing %q", tc.raw, tc.want) + } + if !strings.Contains(err.Error(), tc.want) { + t.Errorf("error %q does not contain %q", err, tc.want) + } + }) + } +} + +func TestWebListenerPortClash(t *testing.T) { + const web = `"web": {"enabled": true, "vhosts": [{"host": "proxy.example.com", "public_addr": "203.0.113.10:443", "decoy": {"mode": "http_upstream", "upstream": "http://127.0.0.1:18081"}}]}` + cases := map[string]string{ + "server port": `{"server": {"port": 443, "listeners": [{"ip": "127.0.0.1", "port": 443, "transport": "web"}]}, ` + web + `}`, + "native listener port": `{"server": {"port": 443, "listeners": [{"ip": "0.0.0.0", "port": 8443}, {"ip": "10.0.0.5", "port": 8443, "transport": "web"}]}, ` + web + `}`, + } + for name, raw := range cases { + t.Run(name, func(t *testing.T) { + _, err := NewConfig(raw) + if err == nil { + t.Fatalf("NewConfig(%s) succeeded, want a port clash error", raw) + } + if !strings.Contains(err.Error(), "port") { + t.Errorf("error %q does not mention the port", err) + } + }) + } +} diff --git a/backend/mtproto/webconfig.go b/backend/mtproto/webconfig.go new file mode 100644 index 00000000..b78c437e --- /dev/null +++ b/backend/mtproto/webconfig.go @@ -0,0 +1,218 @@ +package mtproto + +import ( + "encoding/json" + "errors" + "fmt" + "math" + "net" + "net/url" + "slices" +) + +const ( + webTransport = "web" + webSecretModeDD = "dd" + webSecretModePlain = "plain" +) + +// webUserProfileLimits are the optional numeric keys of a vhost user_profile. +var webUserProfileLimits = []string{"max_sessions", "max_streams", "max_streams_per_session"} + +// webUserProfile is the per-vhost template the node applies to every user it +// exposes through that vhost. A zero limit means unset. +type webUserProfile struct { + SecretMode string + MaxSessions uint32 + MaxStreams uint32 + MaxStreamsPerSession uint32 +} + +// HasWeb reports whether the config enables telemt's WEB mode with at least +// one vhost. +func (c *Config) HasWeb() bool { + if c == nil { + return false + } + web, _ := objectAt(c.base, "web") + enabled, _ := web["enabled"].(bool) + vhosts, _ := web["vhosts"].([]any) + return enabled && len(vhosts) > 0 +} + +// validateWebListener checks one listener with transport "web". It must sit +// on a loopback or private address behind the admin's HTTPS front, and on its +// own port: telemt cannot share a port between transports. +func validateWebListener(index int, listener map[string]any, server map[string]any) error { + ipText, _ := listener["ip"].(string) + ip := net.ParseIP(ipText) + if ip == nil || ip.IsUnspecified() || !(ip.IsLoopback() || ip.IsPrivate()) { + return fmt.Errorf("mtproto config field server.listeners[%d].ip must be a loopback or private address for a web listener; put an HTTPS front in front of it", index) + } + if !isSet(listener, "port") { + return fmt.Errorf("mtproto config field server.listeners[%d].port must be set for a web listener", index) + } + port, err := portValue(fmt.Sprintf("server.listeners[%d].port", index), listener["port"]) + if err != nil { + return err + } + + serverPort := telemtDefaultPort + if isSet(server, "port") { + if serverPort, err = portValue("server.port", server["port"]); err != nil { + return err + } + } + if port == serverPort { + return fmt.Errorf("mtproto config field server.listeners[%d].port %d must differ from server.port", index, port) + } + listeners, _ := server["listeners"].([]any) + for other, rawListener := range listeners { + native, _ := rawListener.(map[string]any) + if transport, _ := native["transport"].(string); transport == webTransport { + continue + } + nativePort := serverPort + if isSet(native, "port") { + if nativePort, err = portValue(fmt.Sprintf("server.listeners[%d].port", other), native["port"]); err != nil { + return err + } + } + if port == nativePort { + return fmt.Errorf("mtproto config field server.listeners[%d].port %d must differ from the port of MTProxy listener server.listeners[%d]", index, port, other) + } + } + return nil +} + +// takeWebProfiles validates the web section against the listeners, removes +// each vhost's user_profile from root and returns the profiles index-aligned +// with web.vhosts. Call it after validateConfigRoot. +func takeWebProfiles(root map[string]any) ([]webUserProfile, error) { + web, err := objectAt(root, "web") + if err != nil { + return nil, err + } + + enabled := false + if isSet(web, "enabled") { + var ok bool + if enabled, ok = web["enabled"].(bool); !ok { + return nil, errors.New("mtproto config field web.enabled must be a boolean") + } + } + + var vhosts []any + if isSet(web, "vhosts") { + var ok bool + if vhosts, ok = web["vhosts"].([]any); !ok { + return nil, errors.New("mtproto config field web.vhosts must be an array of objects") + } + } + + server, _ := objectAt(root, "server") + listeners, _ := server["listeners"].([]any) + hasWebListener := slices.ContainsFunc(listeners, func(raw any) bool { + listener, _ := raw.(map[string]any) + transport, _ := listener["transport"].(string) + return transport == webTransport + }) + switch { + case enabled && !hasWebListener: + return nil, errors.New("mtproto config field web.enabled needs a server.listeners entry with transport \"web\"") + case hasWebListener && (!enabled || len(vhosts) == 0): + return nil, errors.New("mtproto config web listeners need web.enabled set to true and at least one web.vhosts entry") + } + + if len(vhosts) == 0 { + return nil, nil + } + profiles := make([]webUserProfile, len(vhosts)) + for index, rawVhost := range vhosts { + vhost, ok := rawVhost.(map[string]any) + if !ok { + return nil, fmt.Errorf("mtproto config field web.vhosts[%d] must be an object", index) + } + if isSet(vhost, "profiles") { + return nil, fmt.Errorf("mtproto config must not define web.vhosts[%d].profiles: the node builds them from its users", index) + } + if err := validateWebDecoy(index, vhost["decoy"]); err != nil { + return nil, err + } + if profiles[index], err = webUserProfileValue(index, vhost["user_profile"]); err != nil { + return nil, err + } + } + for _, rawVhost := range vhosts { + delete(rawVhost.(map[string]any), "user_profile") + } + return profiles, nil +} + +// validateWebDecoy allows only an http_upstream decoy whose upstream host is a +// loopback or private IP literal: static_directory would serve files from the +// node host, and a public or named upstream would reach past the node. +func validateWebDecoy(index int, rawDecoy any) error { + field := fmt.Sprintf("web.vhosts[%d].decoy", index) + decoy, ok := rawDecoy.(map[string]any) + if !ok { + return fmt.Errorf("mtproto config field %s must be an object", field) + } + switch mode, _ := decoy["mode"].(string); mode { + case "http_upstream": + case "static_directory": + return fmt.Errorf("mtproto config field %s.mode must not be \"static_directory\": it would serve files from the node host", field) + default: + return fmt.Errorf("mtproto config field %s.mode must be \"http_upstream\"", field) + } + + upstream, _ := decoy["upstream"].(string) + parsed, err := url.Parse(upstream) + if err != nil || upstream == "" { + return fmt.Errorf("mtproto config field %s.upstream must be an http URL", field) + } + ip := net.ParseIP(parsed.Hostname()) + if ip == nil || ip.IsUnspecified() || !(ip.IsLoopback() || ip.IsPrivate()) { + return fmt.Errorf("mtproto config field %s.upstream must use a loopback or private IP address as its host", field) + } + return nil +} + +func webUserProfileValue(index int, raw any) (webUserProfile, error) { + profile := webUserProfile{SecretMode: webSecretModeDD} + if raw == nil { + return profile, nil + } + field := fmt.Sprintf("web.vhosts[%d].user_profile", index) + object, ok := raw.(map[string]any) + if !ok { + return profile, fmt.Errorf("mtproto config field %s must be an object", field) + } + for key := range object { + if key != "secret_mode" && !slices.Contains(webUserProfileLimits, key) { + return profile, fmt.Errorf("mtproto config field %s.%s is not supported", field, key) + } + } + + if isSet(object, "secret_mode") { + mode, _ := object["secret_mode"].(string) + if mode != webSecretModeDD && mode != webSecretModePlain { + return profile, fmt.Errorf("mtproto config field %s.secret_mode must be \"dd\" or \"plain\"", field) + } + profile.SecretMode = mode + } + + limits := []*uint32{&profile.MaxSessions, &profile.MaxStreams, &profile.MaxStreamsPerSession} + for i, key := range webUserProfileLimits { + if !isSet(object, key) { + continue + } + number, ok := object[key].(json.Number) + value, err := number.Int64() + if !ok || err != nil || value < 0 || value > math.MaxUint32 { + return profile, fmt.Errorf("mtproto config field %s.%s must be a non-negative integer", field, key) + } + *limits[i] = uint32(value) + } + return profile, nil +} From fe4db931914267020963ed4559505c7c2d3c6d3f Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 26 Oct 2026 20:01:49 +0330 Subject: [PATCH 32/40] feat(mtproto): render one telemt WEB profile per user --- backend/mtproto/config.go | 42 +++++++++++ backend/mtproto/config_test.go | 129 +++++++++++++++++++++++++++++++++ backend/mtproto/toml_test.go | 85 ++++++++++++++++++++++ 3 files changed, 256 insertions(+) diff --git a/backend/mtproto/config.go b/backend/mtproto/config.go index 91b59d43..971444cb 100644 --- a/backend/mtproto/config.go +++ b/backend/mtproto/config.go @@ -20,6 +20,11 @@ const ( loopbackV6CIDR = "::1/128" // telemtDefaultPort is the server.port telemt uses when the config omits it. telemtDefaultPort = 443 + // webMaxProfiles is the web.limits.max_profiles the node renders. telemt + // reads it only at start, so it is fixed well above any user count rather + // than sized per sync. Each profile reserves 64 bytes of telemt's WEB + // memory envelope, so this reserves 16 MiB. + webMaxProfiles = 262144 ) var ( @@ -324,10 +329,47 @@ func (c *Config) Render(users map[string]*runtimeUser, marker *runtimeUser, apiP if err := applyUsersToAccess(access, users, marker); err != nil { return "", err } + if c.HasWeb() { + applyWebProfiles(root, c.webProfiles, users, marker) + } return renderTOML(root) } +// applyWebProfiles gives every WEB vhost one profile per user, built from the +// vhost's user_profile template, and sets web.limits.max_profiles. With no +// users the marker gets the profile instead: telemt rejects an enabled vhost +// without profiles. +func applyWebProfiles(root map[string]any, profiles []webUserProfile, users map[string]*runtimeUser, marker *runtimeUser) { + usernames := slices.Sorted(maps.Keys(users)) + if len(usernames) == 0 && marker != nil { + usernames = []string{marker.Username} + } + + web := ensureObject(root, "web") + vhosts, _ := web["vhosts"].([]any) + for index, rawVhost := range vhosts { + vhost, _ := rawVhost.(map[string]any) + template := profiles[index] + rendered := make([]any, 0, len(usernames)) + for _, username := range usernames { + profile := map[string]any{"user": username, "secret_mode": template.SecretMode} + if template.MaxSessions > 0 { + profile["max_sessions"] = template.MaxSessions + } + if template.MaxStreams > 0 { + profile["max_streams"] = template.MaxStreams + } + if template.MaxStreamsPerSession > 0 { + profile["max_streams_per_session"] = template.MaxStreamsPerSession + } + rendered = append(rendered, profile) + } + vhost["profiles"] = rendered + } + ensureObject(web, "limits")["max_profiles"] = webMaxProfiles +} + func applyUsersToAccess(access map[string]any, users map[string]*runtimeUser, marker *runtimeUser) error { if len(users) == 0 && marker == nil { return errors.New("mtproto config requires at least one user or a marker user") diff --git a/backend/mtproto/config_test.go b/backend/mtproto/config_test.go index c94b76d0..82e1671c 100644 --- a/backend/mtproto/config_test.go +++ b/backend/mtproto/config_test.go @@ -535,3 +535,132 @@ func TestWebListenerPortClash(t *testing.T) { }) } } + +// webProfilesAt returns web.vhosts[index].profiles of a decoded render. +func webProfilesAt(t *testing.T, tree map[string]any, index int) []any { + t.Helper() + vhosts, ok := tomlAt(tree, "web", "vhosts").([]any) + if !ok || len(vhosts) <= index { + t.Fatalf("web.vhosts = %#v, want at least %d entries", tomlAt(tree, "web", "vhosts"), index+1) + } + vhost, _ := vhosts[index].(map[string]any) + profiles, _ := vhost["profiles"].([]any) + return profiles +} + +func TestRenderWebProfiles(t *testing.T) { + cfg, err := NewConfig(webTestConfig(`, "user_profile": {"secret_mode": "plain", "max_sessions": 8}`)) + if err != nil { + t.Fatal(err) + } + users := map[string]*runtimeUser{ + "2": {Username: "2", Secret: otherSecret}, + "1": {Username: "1", Secret: testSecret}, + } + marker := &runtimeUser{Username: "pgm_0123456789abcdef", Secret: thirdSecret} + out, err := cfg.Render(users, marker, 12345, 23456, "Bearer token") + if err != nil { + t.Fatal(err) + } + tree := decodeTOML(t, out) + + profiles := webProfilesAt(t, tree, 0) + want := []map[string]any{ + {"user": "1", "secret_mode": "plain", "max_sessions": int64(8)}, + {"user": "2", "secret_mode": "plain", "max_sessions": int64(8)}, + } + if len(profiles) != len(want) { + t.Fatalf("profiles = %#v, want %#v\n---\n%s", profiles, want, out) + } + for index, raw := range profiles { + profile, _ := raw.(map[string]any) + if !mapsEqual(profile, want[index]) { + t.Errorf("profiles[%d] = %#v, want %#v", index, profile, want[index]) + } + } + if got := tomlAt(tree, "web", "limits", "max_profiles"); got != int64(262144) { + t.Errorf("web.limits.max_profiles = %#v, want 262144", got) + } + if _, ok := tomlAt(tree, "web", "vhosts").([]any)[0].(map[string]any)["user_profile"]; ok { + t.Errorf("user_profile must not reach telemt\n---\n%s", out) + } +} + +func mapsEqual(a, b map[string]any) bool { + if len(a) != len(b) { + return false + } + for key, value := range a { + if other, ok := b[key]; !ok || other != value { + return false + } + } + return true +} + +func TestRenderWebMarkerProfileOnlyWithoutUsers(t *testing.T) { + cfg, err := NewConfig(webTestConfig("")) + if err != nil { + t.Fatal(err) + } + marker := &runtimeUser{Username: "pgm_0123456789abcdef", Secret: thirdSecret} + + // telemt rejects an enabled vhost without profiles, so the marker fills in + // while the node has no users. + out, err := cfg.Render(nil, marker, 12345, 23456, "Bearer token") + if err != nil { + t.Fatal(err) + } + profiles := webProfilesAt(t, decodeTOML(t, out), 0) + wantMarker := map[string]any{"user": marker.Username, "secret_mode": "dd"} + if len(profiles) != 1 || !mapsEqual(profiles[0].(map[string]any), wantMarker) { + t.Errorf("profiles without users = %#v, want only %#v", profiles, wantMarker) + } + + out, err = cfg.Render(map[string]*runtimeUser{"1": {Username: "1", Secret: testSecret}}, marker, 12345, 23456, "Bearer token") + if err != nil { + t.Fatal(err) + } + profiles = webProfilesAt(t, decodeTOML(t, out), 0) + wantUser := map[string]any{"user": "1", "secret_mode": "dd"} + if len(profiles) != 1 || !mapsEqual(profiles[0].(map[string]any), wantUser) { + t.Errorf("profiles with one user = %#v, want only %#v", profiles, wantUser) + } +} + +// TestRenderWebConfigSize checks that 50,000 users with one WEB vhost stay +// under telemt's 8 MiB config limit. Users carry a secret only and the vhost +// sets one profile limit, the common case; the logged bytes per user give the +// largest user count that fits. +func TestRenderWebConfigSize(t *testing.T) { + const userCount = 50000 + const sizeLimit = 8388608 + cfg, err := NewConfig(webTestConfig(`, "user_profile": {"secret_mode": "plain", "max_sessions": 8}`)) + if err != nil { + t.Fatal(err) + } + users := make(map[string]*runtimeUser, userCount) + for id := 1; id <= userCount; id++ { + username := strconv.Itoa(id) + users[username] = &runtimeUser{Username: username, Secret: testSecret} + } + marker := &runtimeUser{Username: "pgm_0123456789abcdef", Secret: thirdSecret} + + out, err := cfg.Render(users, marker, 12345, 23456, "Bearer token") + if err != nil { + t.Fatal(err) + } + empty, err := cfg.Render(nil, marker, 12345, 23456, "Bearer token") + if err != nil { + t.Fatal(err) + } + perUser := float64(len(out)-len(empty)) / userCount + t.Logf("%d users: %d bytes, %.1f bytes per user, about %d users fit in %d bytes", userCount, len(out), perUser, int(float64(sizeLimit-len(empty))/perUser), sizeLimit) + if len(out) >= sizeLimit { + t.Errorf("rendered config is %d bytes, want under %d", len(out), sizeLimit) + } + // Decoding a file this large takes seconds, so count the tables instead. + if got := strings.Count(out, "[[web.vhosts.profiles]]"); got != userCount { + t.Errorf("rendered %d profiles, want %d", got, userCount) + } +} diff --git a/backend/mtproto/toml_test.go b/backend/mtproto/toml_test.go index bdc68485..dab35728 100644 --- a/backend/mtproto/toml_test.go +++ b/backend/mtproto/toml_test.go @@ -313,3 +313,88 @@ func TestTOMLRejectsIncludeKeys(t *testing.T) { t.Errorf("general.included = %#v, want x", got) } } + +// TestRenderWebGolden pins a whole render with one WEB vhost. +func TestRenderWebGolden(t *testing.T) { + cfg, err := NewConfig(webTestConfig(`, "user_profile": {"secret_mode": "plain", "max_sessions": 8, "max_streams_per_session": 4}`)) + if err != nil { + t.Fatal(err) + } + out, err := cfg.Render(map[string]*runtimeUser{ + "1": {Username: "1", Secret: testSecret}, + "2": {Username: "2", Secret: otherSecret}, + }, &runtimeUser{Username: "pgm_0123456789abcdef", Secret: thirdSecret}, 12345, 23456, "Bearer token") + if err != nil { + t.Fatal(err) + } + + const want = `[access] +[access.user_enabled] +pgm_0123456789abcdef = false + +[access.users] +1 = '00112233445566778899aabbccddeeff' +2 = 'ffeeddccbbaa00112233445566778899' +pgm_0123456789abcdef = '0123456789abcdef0123456789abcdef' + +[general] +disable_colors = true + +[general.telemetry] +user_enabled = true + +[logging] +destination = 'stderr' + +[server] +metrics_listen = '127.0.0.1:23456' +metrics_port = 23456 +metrics_whitelist = ['127.0.0.1/32', '::1/128'] +port = 443 + +[server.api] +auth_header = 'Bearer token' +enabled = true +listen = '127.0.0.1:12345' +read_only = false +whitelist = ['127.0.0.1/32', '::1/128'] + +[[server.listeners]] +ip = '0.0.0.0' + +[[server.listeners]] +ip = '127.0.0.1' +port = 18080 +transport = 'web' +web_trusted_proxy_cidrs = ['127.0.0.1/32'] + +[web] +enabled = true + +[web.limits] +max_profiles = 262144 + +[[web.vhosts]] +host = 'proxy.example.com' +public_addr = '203.0.113.10:443' + +[web.vhosts.decoy] +mode = 'http_upstream' +upstream = 'http://127.0.0.1:18081' + +[[web.vhosts.profiles]] +max_sessions = 8 +max_streams_per_session = 4 +secret_mode = 'plain' +user = '1' + +[[web.vhosts.profiles]] +max_sessions = 8 +max_streams_per_session = 4 +secret_mode = 'plain' +user = '2' +` + if out != want { + t.Errorf("rendered config changed\n--- got\n%s\n--- want\n%s", out, want) + } +} From 7936ac0d2f6df995b28f015a4a2c8a016b54fff7 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 26 Oct 2026 20:07:49 +0330 Subject: [PATCH 33/40] fix(mtproto): apply user changes by render when WEB profiles are in use --- backend/mtproto/sync.go | 4 +++ backend/mtproto/sync_test.go | 49 +++++++++++++++++++++++++++++++++++- 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/backend/mtproto/sync.go b/backend/mtproto/sync.go index 314e193c..5688df19 100644 --- a/backend/mtproto/sync.go +++ b/backend/mtproto/sync.go @@ -118,6 +118,10 @@ func (m *MTProto) updateUsersLocked(ctx context.Context, users []*common.User) ( if len(changes) == 0 { return next, false } + // telemt's user API creates no WEB profile, and a profile must go before its user (telemt guide). + if m.config.HasWeb() { + return next, true + } if err := m.applyUserChangesLocked(ctx, changes); err != nil { // Some changes may have reached telemt: its users are unknown until // a full apply succeeds. diff --git a/backend/mtproto/sync_test.go b/backend/mtproto/sync_test.go index 1d4b3b62..ae8df285 100644 --- a/backend/mtproto/sync_test.go +++ b/backend/mtproto/sync_test.go @@ -419,8 +419,15 @@ func writeFakeError(w http.ResponseWriter, status int, code, message string) { // the fake is started on the initial render. func newSyncTestBackend(t *testing.T, api *fakeTelemtAPI, users []*common.User) *MTProto { t.Helper() + return newSyncTestBackendWithConfig(t, api, `{"inbound_tag": "mtproto", "server": {"port": 8443}}`, users) +} + +// newSyncTestBackendWithConfig is newSyncTestBackend with the panel config +// rawConfig. +func newSyncTestBackendWithConfig(t *testing.T, api *fakeTelemtAPI, rawConfig string, users []*common.User) *MTProto { + t.Helper() - mtConfig, err := NewConfig(`{"inbound_tag": "mtproto", "server": {"port": 8443}}`) + mtConfig, err := NewConfig(rawConfig) if err != nil { t.Fatal(err) } @@ -1067,6 +1074,46 @@ func TestLastRealUserGoesThroughAPI(t *testing.T) { } } +// TestWebConfigAlwaysRendersUsers: telemt's user API creates no WEB profile, +// so with WEB on even a one-user change goes through a full render, and the +// rendered profiles follow the user set. +func TestWebConfigAlwaysRendersUsers(t *testing.T) { + api := newFakeTelemtAPI(t) + m := newSyncTestBackendWithConfig(t, api, webTestConfig(""), []*common.User{mtUser("u1", "mtproto", testSecret)}) + ctx := context.Background() + + assertProfiles := func(want ...string) { + t.Helper() + var got []string + for _, raw := range webProfilesAt(t, decodeTOML(t, readConfig(t, api.configPath)), 0) { + profile, _ := raw.(map[string]any) + user, _ := profile["user"].(string) + got = append(got, user) + } + if !slices.Equal(got, want) { + t.Errorf("rendered profiles = %q, want %q", got, want) + } + } + + if err := m.SyncUser(ctx, mtUser("u2", "mtproto", testSecret)); err != nil { + t.Fatalf("SyncUser adding u2: %v", err) + } + assertCalls(t, api.takeCalls(), "GET /v1/users") + assertProfiles("u1", "u2") + if !api.connects("u2", testSecret) { + t.Error("u2 cannot connect") + } + + if err := m.SyncUser(ctx, untagged("u1")); err != nil { + t.Fatalf("SyncUser removing u1: %v", err) + } + assertCalls(t, api.takeCalls(), "GET /v1/users") + assertProfiles("u2") + if got := desiredNames(m); !slices.Equal(got, []string{"u2"}) { + t.Errorf("desired = %q, want u2", got) + } +} + func TestInvalidUserDoesNotFailBatch(t *testing.T) { api := newFakeTelemtAPI(t) m := newSyncTestBackend(t, api, []*common.User{mtUser("u1", "mtproto", testSecret)}) From 198b3eed8a8e13a9d34a7697c3a86ddeba8e38f4 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 26 Oct 2026 20:15:36 +0330 Subject: [PATCH 34/40] fix(mtproto): render web.limits.max_profiles within telemt's memory envelope telemt 3.5.14 reserves 64 bytes of its WEB memory envelope per max_profiles entry, and its default envelope (1280 MiB) has only about 4.1 MiB left after its other default reservations. With 262144 profiles (16 MiB) every WEB config failed at start with "web.limits memory reservations must fit memory_envelope_bytes within 4 GiB". Render 65536 (4 MiB), which telemt accepts with its default limits. A render with more profiles than that (users times vhosts) is rejected by telemt and the node restores the previous users, as for any rejected render. --- backend/mtproto/config.go | 10 ++++++---- backend/mtproto/config_test.go | 4 ++-- backend/mtproto/toml_test.go | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/backend/mtproto/config.go b/backend/mtproto/config.go index 971444cb..5c4b59d5 100644 --- a/backend/mtproto/config.go +++ b/backend/mtproto/config.go @@ -21,10 +21,12 @@ const ( // telemtDefaultPort is the server.port telemt uses when the config omits it. telemtDefaultPort = 443 // webMaxProfiles is the web.limits.max_profiles the node renders. telemt - // reads it only at start, so it is fixed well above any user count rather - // than sized per sync. Each profile reserves 64 bytes of telemt's WEB - // memory envelope, so this reserves 16 MiB. - webMaxProfiles = 262144 + // reads it only at start, so it is fixed rather than sized per sync. Each + // profile reserves 64 bytes of telemt's WEB memory envelope, and telemt's + // default envelope (1280 MiB) has about 4.1 MiB to spare once its other + // default reservations are counted: 65536 profiles take 4 MiB of it. A + // render with more profiles (users times vhosts) is rejected by telemt. + webMaxProfiles = 65536 ) var ( diff --git a/backend/mtproto/config_test.go b/backend/mtproto/config_test.go index 82e1671c..9e3149ed 100644 --- a/backend/mtproto/config_test.go +++ b/backend/mtproto/config_test.go @@ -578,8 +578,8 @@ func TestRenderWebProfiles(t *testing.T) { t.Errorf("profiles[%d] = %#v, want %#v", index, profile, want[index]) } } - if got := tomlAt(tree, "web", "limits", "max_profiles"); got != int64(262144) { - t.Errorf("web.limits.max_profiles = %#v, want 262144", got) + if got := tomlAt(tree, "web", "limits", "max_profiles"); got != int64(65536) { + t.Errorf("web.limits.max_profiles = %#v, want 65536", got) } if _, ok := tomlAt(tree, "web", "vhosts").([]any)[0].(map[string]any)["user_profile"]; ok { t.Errorf("user_profile must not reach telemt\n---\n%s", out) diff --git a/backend/mtproto/toml_test.go b/backend/mtproto/toml_test.go index dab35728..8c6e6671 100644 --- a/backend/mtproto/toml_test.go +++ b/backend/mtproto/toml_test.go @@ -372,7 +372,7 @@ web_trusted_proxy_cidrs = ['127.0.0.1/32'] enabled = true [web.limits] -max_profiles = 262144 +max_profiles = 65536 [[web.vhosts]] host = 'proxy.example.com' From 4e5b4960462ebd2af831984d06f49ebe7ba79cf4 Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 26 Oct 2026 20:15:37 +0330 Subject: [PATCH 35/40] test(mtproto): cover WEB decoy, profiles and empty start against telemt 3.5.14 startLiveTelemtWithConfig lets a live test pass its own panel config; startLiveTelemt keeps today's config through it. TestIntegrationWebDecoyAndProfiles runs a native and a WEB listener on 127.0.0.1 with one vhost and a local decoy: a plain request for the vhost reaches the decoy, GET /v1/config lists a profile per user and follows a full sync, and a native session stays open across that sync. TestIntegrationWebStartWithoutUsers starts with no real users: telemt accepts the disabled marker user's profile, so no fallback is needed. --- backend/mtproto/integration_test.go | 154 ++++++++++++++++++++++++++-- 1 file changed, 148 insertions(+), 6 deletions(-) diff --git a/backend/mtproto/integration_test.go b/backend/mtproto/integration_test.go index fad5d270..b8c8da9c 100644 --- a/backend/mtproto/integration_test.go +++ b/backend/mtproto/integration_test.go @@ -6,7 +6,10 @@ import ( "context" "errors" "fmt" + "io" "net" + "net/http" + "net/http/httptest" "os" "path/filepath" "slices" @@ -79,6 +82,21 @@ func telemtBinOrSkip(t *testing.T) string { // into generatedPath (a new temp dir when empty). The test runs in parallel // with the other integration tests. func startLiveTelemt(t *testing.T, generatedPath string, users ...*common.User) *liveTelemt { + t.Helper() + return startLiveTelemtWithConfig(t, generatedPath, func(nativePort int, dc string) string { + return fmt.Sprintf(`{ + "inbound_tag": "mtproto", + "general": {"use_middle_proxy": false, "modes": {"classic": true, "secure": true, "tls": false}}, + "server": {"port": %d, "listeners": [{"ip": "127.0.0.1"}]}, + "dc_overrides": {"%d": %q} + }`, nativePort, mtprotoTestDC, dc) + }, users...) +} + +// startLiveTelemtWithConfig is startLiveTelemt with the panel config built by +// configJSON from the MTProxy listener port and the fake DC address. The +// config must serve MTProxy on 127.0.0.1:nativePort. +func startLiveTelemtWithConfig(t *testing.T, generatedPath string, configJSON func(nativePort int, dc string) string, users ...*common.User) *liveTelemt { t.Helper() bin := telemtBinOrSkip(t) t.Parallel() @@ -90,12 +108,7 @@ func startLiveTelemt(t *testing.T, generatedPath string, users ...*common.User) } port := netutil.FindFreePort() - mtConfig, err := NewConfig(fmt.Sprintf(`{ - "inbound_tag": "mtproto", - "general": {"use_middle_proxy": false, "modes": {"classic": true, "secure": true, "tls": false}}, - "server": {"port": %d, "listeners": [{"ip": "127.0.0.1"}]}, - "dc_overrides": {"%d": %q} - }`, port, mtprotoTestDC, startFakeDC(t))) + mtConfig, err := NewConfig(configJSON(port, startFakeDC(t))) if err != nil { t.Fatalf("NewConfig: %v", err) } @@ -636,3 +649,132 @@ func TestIntegrationRejectedRenderRestoresUsers(t *testing.T) { l.expectAdmitted("alice", secretAlice) l.expectRefused("1000000", fmt.Sprintf("%032x", 1_000_000)) } + +// webVhostHost is the vhost of the WEB integration tests; requests name it +// in their Host header. +const webVhostHost = "proxy.example.com" + +// webDecoyBody is what the WEB tests' decoy upstream answers. +const webDecoyBody = "decoy-ok" + +// startLiveWebTelemt starts the real telemt with an MTProxy listener and a +// WEB listener on 127.0.0.1, one vhost for webVhostHost and a decoy upstream +// on 127.0.0.1 answering webDecoyBody. It returns the backend and the WEB +// listener address. +func startLiveWebTelemt(t *testing.T, users ...*common.User) (*liveTelemt, string) { + t.Helper() + telemtBinOrSkip(t) + decoy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = io.WriteString(w, webDecoyBody) + })) + t.Cleanup(decoy.Close) + webPort := netutil.FindFreePort() + + l := startLiveTelemtWithConfig(t, "", func(nativePort int, dc string) string { + return fmt.Sprintf(`{ + "inbound_tag": "mtproto", + "general": {"use_middle_proxy": false, "modes": {"classic": true, "secure": true, "tls": false}}, + "server": {"port": %d, "listeners": [ + {"ip": "127.0.0.1"}, + {"ip": "127.0.0.1", "port": %d, "transport": "web", "web_trusted_proxy_cidrs": ["127.0.0.1/32"]} + ]}, + "dc_overrides": {"%d": %q}, + "web": {"enabled": true, "vhosts": [{ + "host": %q, + "public_addr": "127.0.0.1:443", + "decoy": {"mode": "http_upstream", "upstream": %q} + }]} + }`, nativePort, webPort, mtprotoTestDC, dc, webVhostHost, decoy.URL) + }, users...) + return l, fmt.Sprintf("127.0.0.1:%d", webPort) +} + +// webProfileUsers returns the users of the WEB profiles telemt reports for +// webVhostHost in GET /v1/config. +func (l *liveTelemt) webProfileUsers() []string { + l.t.Helper() + type configData struct { + Web struct { + Vhosts []struct { + Host string `json:"host"` + Profiles []struct { + User string `json:"user"` + } `json:"profiles"` + } `json:"vhosts"` + } `json:"web"` + } + data, err := doAPIRequest[configData](l.ctx, l.mt.apiClient, http.MethodGet, "/v1/config", nil) + if err != nil { + l.t.Fatalf("GET /v1/config: %v", err) + } + var users []string + for _, vhost := range data.Web.Vhosts { + if vhost.Host != webVhostHost { + continue + } + for _, profile := range vhost.Profiles { + users = append(users, profile.User) + } + } + slices.Sort(users) + return users +} + +// waitWebProfileUsers waits until telemt reports WEB profiles for exactly +// want, sorted. +func (l *liveTelemt) waitWebProfileUsers(want ...string) { + l.t.Helper() + var got []string + if !waitUntil(liveWait, func() bool { + got = l.webProfileUsers() + return slices.Equal(got, want) + }) { + l.t.Fatalf("telemt WEB profiles of %s = %v, want %v", webVhostHost, got, want) + } +} + +// A WEB listener relays a plain request for the vhost to the decoy, telemt +// runs one WEB profile per user and follows a full sync, and a native +// session stays open across that sync. +func TestIntegrationWebDecoyAndProfiles(t *testing.T) { + u1 := mtUser("u1", "mtproto", secretAlice) + l, webAddr := startLiveWebTelemt(t, u1) + + request, err := http.NewRequestWithContext(l.ctx, http.MethodGet, "http://"+webAddr+"/", nil) + if err != nil { + t.Fatal(err) + } + request.Host = webVhostHost + request.Header.Set("X-Forwarded-For", "198.51.100.7") + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatalf("GET / on the WEB listener: %v", err) + } + body, err := io.ReadAll(response.Body) + _ = response.Body.Close() + if err != nil || response.StatusCode != http.StatusOK || string(body) != webDecoyBody { + t.Fatalf("GET / on the WEB listener = %d %q, %v; want 200 %q from the decoy", response.StatusCode, body, err, webDecoyBody) + } + t.Logf("WEB listener answered %d %q", response.StatusCode, body) + + l.waitWebProfileUsers("u1") + session := l.expectAdmitted("u1", secretAlice) + + if err := l.mt.SyncUsers(l.ctx, []*common.User{u1, mtUser("u2", "mtproto", secretBob)}); err != nil { + t.Fatalf("SyncUsers(u1, u2): %v", err) + } + l.waitWebProfileUsers("u1", "u2") + t.Logf("telemt WEB profiles after the sync: %v", l.webProfileUsers()) + l.expectOpen(session, "u1", 1) + l.expectAdmitted("u2", secretBob) +} + +// With no real users, telemt starts with WEB on and the marker's profile as +// the only one. +func TestIntegrationWebStartWithoutUsers(t *testing.T) { + l, _ := startLiveWebTelemt(t) + + marker := l.mt.markerUsername() + l.waitWebProfileUsers(marker) + t.Logf("telemt WEB profiles without users: %v", l.webProfileUsers()) +} From d9ce5a8c4796ed73d87658dab5b1570c2ee8aa4c Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 26 Oct 2026 20:28:04 +0330 Subject: [PATCH 36/40] test(mtproto): check WEB profiles in telemt's running config GET /v1/config reads telemt's config file on disk, so it showed only that the node rendered the profiles. The WEB tests now also send the bridge request Telegram Desktop sends for a dd link (capability HMAC-SHA256 over the context and host, keyed by the client secret) and check telemt's answer: the bridge page for a user with a running profile, the decoy once a sync removed it, and telemt's private 404 for the disabled marker, whose profile telemt runs but serves no bridge to. The /v1/config check stays, named as a check of the rendered file. --- backend/mtproto/integration_test.go | 215 +++++++++++++++++++++------- 1 file changed, 165 insertions(+), 50 deletions(-) diff --git a/backend/mtproto/integration_test.go b/backend/mtproto/integration_test.go index b8c8da9c..bb36e534 100644 --- a/backend/mtproto/integration_test.go +++ b/backend/mtproto/integration_test.go @@ -4,6 +4,10 @@ package mtproto import ( "context" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "encoding/hex" "errors" "fmt" "io" @@ -14,6 +18,7 @@ import ( "path/filepath" "slices" "strings" + "sync/atomic" "testing" "time" @@ -657,11 +662,27 @@ const webVhostHost = "proxy.example.com" // webDecoyBody is what the WEB tests' decoy upstream answers. const webDecoyBody = "decoy-ok" +// webCapabilityContext is the context telemt hashes into the WEB capability +// of a vhost without base_path (WEB_CAPABILITY_CONTEXT_V1 in telemt's +// src/config/load/runtime_web.rs). +const webCapabilityContext = "tdesktop-web-proxy-bridge-v1\n" + +// webBridgeContentType is the content type of telemt's WEB bridge page. +const webBridgeContentType = "text/html; charset=utf-8" + +// liveWeb is a backend on the real telemt with a WEB listener. +type liveWeb struct { + *liveTelemt + webAddr string + // requests numbers WEB requests so each comes from its own forwarded + // address: telemt caps live bootstraps per client address. + requests atomic.Uint32 +} + // startLiveWebTelemt starts the real telemt with an MTProxy listener and a // WEB listener on 127.0.0.1, one vhost for webVhostHost and a decoy upstream -// on 127.0.0.1 answering webDecoyBody. It returns the backend and the WEB -// listener address. -func startLiveWebTelemt(t *testing.T, users ...*common.User) (*liveTelemt, string) { +// on 127.0.0.1 answering webDecoyBody. +func startLiveWebTelemt(t *testing.T, users ...*common.User) *liveWeb { t.Helper() telemtBinOrSkip(t) decoy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { @@ -686,13 +707,94 @@ func startLiveWebTelemt(t *testing.T, users ...*common.User) (*liveTelemt, strin }]} }`, nativePort, webPort, mtprotoTestDC, dc, webVhostHost, decoy.URL) }, users...) - return l, fmt.Sprintf("127.0.0.1:%d", webPort) + return &liveWeb{liveTelemt: l, webAddr: fmt.Sprintf("127.0.0.1:%d", webPort)} } -// webProfileUsers returns the users of the WEB profiles telemt reports for -// webVhostHost in GET /v1/config. -func (l *liveTelemt) webProfileUsers() []string { - l.t.Helper() +// get sends GET target for webVhostHost to the WEB listener, as the HTTPS +// front would with one X-Forwarded-For address, and returns the status, the +// content type and the body. +func (w *liveWeb) get(target string) (int, string, string) { + w.t.Helper() + request, err := http.NewRequestWithContext(w.ctx, http.MethodGet, "http://"+w.webAddr+target, nil) + if err != nil { + w.t.Fatal(err) + } + request.Host = webVhostHost + request.Header.Set("X-Forwarded-For", fmt.Sprintf("198.51.100.%d", w.requests.Add(1)%250+1)) + response, err := http.DefaultClient.Do(request) + if err != nil { + w.t.Fatalf("GET %s on the WEB listener: %v", target, err) + } + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + if err != nil { + w.t.Fatalf("read GET %s on the WEB listener: %v", target, err) + } + return response.StatusCode, response.Header.Get("Content-Type"), string(body) +} + +// bridgeTarget is the bridge URL Telegram Desktop requests for a dd WEB +// link of secret on webVhostHost: the capability is HMAC-SHA256 keyed by +// the client secret (0xdd and the 16-byte secret) over the context and the +// host (derive_web_capability in telemt's src/config/load/runtime_web.rs). +func bridgeTarget(t *testing.T, secret string) string { + t.Helper() + raw, err := hex.DecodeString(secret) + if err != nil || len(raw) != 16 { + t.Fatalf("secret %q is not 16 hex bytes", secret) + } + mac := hmac.New(sha256.New, append([]byte{0xdd}, raw...)) + mac.Write([]byte(webCapabilityContext + webVhostHost)) + return "/?bridge=" + base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) +} + +// bridgeAnswer is how telemt's WEB listener answers a bridge request. +type bridgeAnswer string + +const ( + // bridgeServed: the capability is in telemt's running WEB config and + // telemt issued a bridge page for it. + bridgeServed bridgeAnswer = "bridge page" + // bridgeRefused: the capability is in telemt's running WEB config but + // telemt issued no bootstrap (a disabled user): telemt answers a + // private 404 instead of the decoy (has_internal_credential in telemt's + // src/web/http/decoy.rs). + bridgeRefused bridgeAnswer = "private 404" + // bridgeUnknown: telemt's running WEB config lacks the capability and + // the request reaches the decoy. + bridgeUnknown bridgeAnswer = "decoy" +) + +// waitBridge waits until telemt answers the bridge request for secret with +// want. +func (w *liveWeb) waitBridge(username, secret string, want bridgeAnswer) { + w.t.Helper() + target := bridgeTarget(w.t, secret) + var status int + var contentType, body string + if !waitUntil(liveWait, func() bool { + status, contentType, body = w.get(target) + switch want { + case bridgeServed: + return status == http.StatusOK && contentType == webBridgeContentType && body != webDecoyBody + case bridgeRefused: + return status == http.StatusNotFound && body == "not found\n" + default: + return status == http.StatusOK && body == webDecoyBody + } + }) { + w.t.Fatalf("WEB bridge request of %s: telemt answered %d %q with %d bytes, want the %s", username, status, contentType, len(body), want) + } + w.t.Logf("WEB bridge request of %s: %s (%d %q, %d bytes)", username, want, status, contentType, len(body)) +} + +// renderedProfileUsers returns the users of the WEB profiles for +// webVhostHost in the config file the node rendered, as telemt's +// GET /v1/config reads them. That route reads the file on disk +// (get_config_route in telemt's src/api/handler/fixed_routes.rs), not the +// running config: waitBridge checks that. +func (w *liveWeb) renderedProfileUsers() []string { + w.t.Helper() type configData struct { Web struct { Vhosts []struct { @@ -703,9 +805,9 @@ func (l *liveTelemt) webProfileUsers() []string { } `json:"vhosts"` } `json:"web"` } - data, err := doAPIRequest[configData](l.ctx, l.mt.apiClient, http.MethodGet, "/v1/config", nil) + data, err := doAPIRequest[configData](w.ctx, w.mt.apiClient, http.MethodGet, "/v1/config", nil) if err != nil { - l.t.Fatalf("GET /v1/config: %v", err) + w.t.Fatalf("GET /v1/config: %v", err) } var users []string for _, vhost := range data.Web.Vhosts { @@ -720,61 +822,74 @@ func (l *liveTelemt) webProfileUsers() []string { return users } -// waitWebProfileUsers waits until telemt reports WEB profiles for exactly -// want, sorted. -func (l *liveTelemt) waitWebProfileUsers(want ...string) { - l.t.Helper() - var got []string - if !waitUntil(liveWait, func() bool { - got = l.webProfileUsers() - return slices.Equal(got, want) - }) { - l.t.Fatalf("telemt WEB profiles of %s = %v, want %v", webVhostHost, got, want) +// expectRenderedProfiles checks that the rendered config file has WEB +// profiles for exactly want, sorted. +func (w *liveWeb) expectRenderedProfiles(want ...string) { + w.t.Helper() + if got := w.renderedProfileUsers(); !slices.Equal(got, want) { + w.t.Fatalf("rendered WEB profiles of %s = %v, want %v", webVhostHost, got, want) } + w.t.Logf("rendered WEB profiles: %v", want) +} + +// markerSecret returns the secret of the current marker user. +func markerSecret(m *MTProto) string { + m.stateMu.RLock() + defer m.stateMu.RUnlock() + return m.marker.Secret } // A WEB listener relays a plain request for the vhost to the decoy, telemt -// runs one WEB profile per user and follows a full sync, and a native -// session stays open across that sync. +// serves the bridge only to users with a WEB profile and follows full syncs, +// and a native session stays open across them. func TestIntegrationWebDecoyAndProfiles(t *testing.T) { u1 := mtUser("u1", "mtproto", secretAlice) - l, webAddr := startLiveWebTelemt(t, u1) + u2 := mtUser("u2", "mtproto", secretBob) + w := startLiveWebTelemt(t, u1) - request, err := http.NewRequestWithContext(l.ctx, http.MethodGet, "http://"+webAddr+"/", nil) - if err != nil { - t.Fatal(err) + if status, _, body := w.get("/"); status != http.StatusOK || body != webDecoyBody { + t.Fatalf("GET / on the WEB listener = %d %q, want 200 %q from the decoy", status, body, webDecoyBody) } - request.Host = webVhostHost - request.Header.Set("X-Forwarded-For", "198.51.100.7") - response, err := http.DefaultClient.Do(request) - if err != nil { - t.Fatalf("GET / on the WEB listener: %v", err) - } - body, err := io.ReadAll(response.Body) - _ = response.Body.Close() - if err != nil || response.StatusCode != http.StatusOK || string(body) != webDecoyBody { - t.Fatalf("GET / on the WEB listener = %d %q, %v; want 200 %q from the decoy", response.StatusCode, body, err, webDecoyBody) - } - t.Logf("WEB listener answered %d %q", response.StatusCode, body) + t.Logf("WEB listener answered GET / with the decoy %q", webDecoyBody) - l.waitWebProfileUsers("u1") - session := l.expectAdmitted("u1", secretAlice) + w.expectRenderedProfiles("u1") + w.waitBridge("u1", secretAlice, bridgeServed) + w.waitBridge("u2", secretBob, bridgeUnknown) + session := w.expectAdmitted("u1", secretAlice) - if err := l.mt.SyncUsers(l.ctx, []*common.User{u1, mtUser("u2", "mtproto", secretBob)}); err != nil { + if err := w.mt.SyncUsers(w.ctx, []*common.User{u1, u2}); err != nil { t.Fatalf("SyncUsers(u1, u2): %v", err) } - l.waitWebProfileUsers("u1", "u2") - t.Logf("telemt WEB profiles after the sync: %v", l.webProfileUsers()) - l.expectOpen(session, "u1", 1) - l.expectAdmitted("u2", secretBob) + w.expectRenderedProfiles("u1", "u2") + w.waitBridge("u2", secretBob, bridgeServed) + w.waitBridge("u1", secretAlice, bridgeServed) + w.expectOpen(session, "u1", 1) + w.expectAdmitted("u2", secretBob) + + if err := w.mt.SyncUsers(w.ctx, []*common.User{u1}); err != nil { + t.Fatalf("SyncUsers(u1): %v", err) + } + w.expectRenderedProfiles("u1") + w.waitBridge("u2", secretBob, bridgeUnknown) + w.waitBridge("u1", secretAlice, bridgeServed) + w.expectOpen(session, "u1", 1) } // With no real users, telemt starts with WEB on and the marker's profile as -// the only one. +// the only one: telemt runs it but serves no bridge to the disabled marker. +// A user added later gets a running WEB profile and the old marker's is +// gone. func TestIntegrationWebStartWithoutUsers(t *testing.T) { - l, _ := startLiveWebTelemt(t) + w := startLiveWebTelemt(t) - marker := l.mt.markerUsername() - l.waitWebProfileUsers(marker) - t.Logf("telemt WEB profiles without users: %v", l.webProfileUsers()) + marker, secret := w.mt.markerUsername(), markerSecret(w.mt) + w.expectRenderedProfiles(marker) + w.waitBridge(marker, secret, bridgeRefused) + + if err := w.mt.UpdateUsers(w.ctx, []*common.User{mtUser("alice", "mtproto", secretAlice)}); err != nil { + t.Fatalf("UpdateUsers(alice): %v", err) + } + w.expectRenderedProfiles("alice") + w.waitBridge("alice", secretAlice, bridgeServed) + w.waitBridge(marker, secret, bridgeUnknown) } From 12a86af2593335a8829b0825bbefdd8f580d837d Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 26 Oct 2026 20:34:37 +0330 Subject: [PATCH 37/40] docs(mtproto): explain WEB mode and an example HTTPS front --- README.md | 141 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 140 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b295ef02..d4f2521f 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,146 @@ The MTProto backend runs [telemt](https://github.com/telemt/telemt) as a separat - telemt needs outbound access to Telegram. On start it checks the Telegram data centers, which can take up to a minute. - Start can take up to 90 s while telemt probes Telegram, and the panel's retry attaches to the backend that is starting or running. - Above 4096 users with stats or configured users, telemt stops its per-user series and the node bills from telemt's API totals, reported as downlink (no uplink and downlink split). -- telemt refuses a config file above 8 MiB, which is roughly 150k to 190k users on one node. +- telemt refuses a config file above 8 MiB, which is about 196k users on one node without WEB mode (fewer with WEB, see below). + +## WEB mode +WEB mode carries MTProxy over real HTTPS (long polling or WebSocket) to your own domain on port 443. Links look like `tg://webproxy?server=proxy.example.com&secret=dd...` and have no port. +- Clients: Telegram Desktop's `WEB` proxy type, and Telegram iOS with the `https` carrier only. Android is not covered by telemt's docs and is untested. +- Secrets are `plain` or `dd`. Fake TLS (`ee`) is not used by WEB. +- telemt does not terminate TLS. A front (Caddy, nginx or HAProxy) holds the certificate on 443 and forwards plain HTTP/1.1 to telemt's WEB listener on a private address, with exactly one `X-Forwarded-For` address. pg-node runs the node with `network_mode: host`, so a front on the same host reaches `127.0.0.1:`. +- telemt's own guide is [`docs/WEB/WEB_PROXY.en.md`](https://github.com/telemt/telemt/blob/3.5.14/docs/WEB/WEB_PROXY.en.md). + +```text +Telegram client --HTTPS :443--> front (TLS) --HTTP/1.1--> 127.0.0.1:18080 (telemt WEB listener) --> Telegram +``` + +### Core config +What the node accepts in the MTProto core JSON: +- A `server.listeners[]` entry with `"transport": "web"`: + - `ip` is a loopback or private address (127/8, ::1, 10/8, 172.16/12, 192.168/16, fc00::/7). `0.0.0.0`, `::` and public addresses are refused. + - `port` is set, and differs from `server.port` and from every native listener's port. + - telemt's WEB listener keys pass through, for example `web_client_ip_source` and `web_trusted_proxy_cidrs`. +- A `web` section with `"enabled": true` and at least one `vhosts[]` entry: + - `host`: the public domain. `base_path` is optional. + - `public_addr`: the server's public IP on port 443, for example `203.0.113.10:443`. + - `decoy`: `{"mode": "http_upstream", "upstream": "http://[:port]"}`. Host names and `static_directory` are refused. The decoy is the site that ordinary visitors and probes see. + - `user_profile` (PasarGuard only, optional): `secret_mode` is `"dd"` (default) or `"plain"`, plus optional `max_sessions`, `max_streams` and `max_streams_per_session`. It applies to every user on that vhost. +- Refused: `web.vhosts[].profiles`, `web.debug`, `web.limits.max_profiles` and `include` anywhere. + +The node writes one telemt WEB profile per user per vhost and sets `web.limits.max_profiles` to 65536. With WEB on, every user change rewrites telemt's config and hot reloads it: open sessions of unchanged users stay, removed users are cut. + +telemt creates its default native listener only when no listeners are listed. Adding a WEB listener therefore turns native MTProto off unless you also list native listeners. The panel form writes them when WEB is enabled. If you edit the JSON by hand, add `{"ip": "0.0.0.0"}` and `{"ip": "::"}` (or your own). + +Minimal example: native MTProto on 8443, WEB on `127.0.0.1:18080`, a decoy site on `127.0.0.1:18081`: +```json +{ + "server": { + "port": 8443, + "listeners": [ + {"ip": "0.0.0.0"}, + {"ip": "::"}, + { + "ip": "127.0.0.1", + "port": 18080, + "transport": "web", + "web_client_ip_source": "x_forwarded_for", + "web_trusted_proxy_cidrs": ["127.0.0.1/32"] + } + ] + }, + "web": { + "enabled": true, + "carrier": "https", + "vhosts": [ + { + "host": "proxy.example.com", + "public_addr": "203.0.113.10:443", + "decoy": {"mode": "http_upstream", "upstream": "http://127.0.0.1:18081"}, + "user_profile": {"secret_mode": "dd"} + } + ] + } +} +``` +`"carrier": "https"` keeps iOS working. `https-lanes` needs public HTTP/2 and is not supported by iOS. + +### Front examples +Both forward the whole domain to telemt, overwrite `X-Forwarded-For` with one address, and turn off response buffering. Timeouts must stay above telemt's 25 s long poll and twice the WebSocket liveness interval; 65 s covers the defaults. Do not enable upstream retries. Keep request URLs and the `Authorization` header out of access logs: they carry the users' WEB credentials. + +Caddy (gets and renews the certificate itself): +```caddyfile +{ + servers { + protocols h1 h2 + } +} + +proxy.example.com { + reverse_proxy 127.0.0.1:18080 { + header_up X-Forwarded-For {remote_host} + flush_interval -1 + stream_close_delay 5m + } +} +``` +Do not add `header_up Host`. `protocols h1 h2` keeps HTTP/3 off, and it applies to every site on that Caddy server. + +nginx (the `map` goes in the `http` context): +```nginx +map $http_upgrade $telemt_connection_upgrade { + default upgrade; + '' ''; +} + +upstream telemt_web { + server 127.0.0.1:18080; + keepalive 64; +} + +server { + listen 443 ssl; + http2 on; + server_name proxy.example.com; + access_log off; + + ssl_certificate /etc/letsencrypt/live/proxy.example.com/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/proxy.example.com/privkey.pem; + + client_max_body_size 2m; + + location / { + proxy_pass http://telemt_web; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $telemt_connection_upgrade; + + proxy_connect_timeout 5s; + proxy_send_timeout 65s; + proxy_read_timeout 65s; + proxy_request_buffering off; + proxy_buffering off; + proxy_next_upstream off; + } +} +``` +`client_max_body_size` must be at least telemt's `web.limits.max_body_bytes`. + +Firewall the WEB listener port and the decoy port from outside, even on loopback. + +### Port 443 +The front needs public port 443, and native fake TLS usually wants 443 too. Either: +- move native MTProto to another port (8443 in the example above), or +- split 443 by host name in the front: nginx `stream` with `ssl_preread`, or HAProxy SNI routing. The WEB domain is terminated by the front and other names pass through to telemt's native listener. PasarGuard has not tested this split. + +### Several nodes +A WEB domain points at one server, and `public_addr` is that server's IP. With several MTProto nodes, use one core per node. + +### Capacity +- telemt holds at most 65536 WEB profiles (users times vhosts): 65536 users with one vhost, 32768 with two. +- The 8 MiB config file limit allows about 69k users with one WEB vhost (about 121.6 bytes per user), and about 36k if every user has every optional limit set (about 232.9 bytes per user). Native only is about 196k (about 42.8 bytes per user). Each extra vhost adds about 79 bytes per user. +- 65536 profiles fit telemt's default WEB memory envelope (1280 MiB, about 4 MiB free for profiles at 64 bytes each). If you raise other `web.limits` reservations or lower `web.limits.memory_envelope_bytes`, telemt refuses to start with `web.limits memory reservations must fit memory_envelope_bytes`. ## Xray and MTProto on one host One node runs one backend, so serving Xray (or WireGuard) and MTProto from one server takes two node instances, each added to the panel as its own node: From 9350e71049d83c88f59eebb7196e494ecd3511ae Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 26 Oct 2026 20:39:22 +0330 Subject: [PATCH 38/40] docs(mtproto): add tls_domain to the WEB example config --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d4f2521f..392cd9a1 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,7 @@ Minimal example: native MTProto on 8443, WEB on `127.0.0.1:18080`, a decoy site } ] }, + "censorship": {"tls_domain": "www.microsoft.com"}, "web": { "enabled": true, "carrier": "https", @@ -95,7 +96,7 @@ Minimal example: native MTProto on 8443, WEB on `127.0.0.1:18080`, a decoy site } } ``` -`"carrier": "https"` keeps iOS working. `https-lanes` needs public HTTP/2 and is not supported by iOS. +The panel requires `censorship.tls_domain` while fake TLS mode is on (telemt's default); it is the site that native fake TLS clients present. `"carrier": "https"` keeps iOS working. `https-lanes` needs public HTTP/2 and is not supported by iOS. ### Front examples Both forward the whole domain to telemt, overwrite `X-Forwarded-For` with one address, and turn off response buffering. Timeouts must stay above telemt's 25 s long poll and twice the WebSocket liveness interval; 65 s covers the defaults. Do not enable upstream retries. Keep request URLs and the `Authorization` header out of access logs: they carry the users' WEB credentials. From 26cff9afbe98f3d1f6cb6741cd6eac9766a424cc Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 26 Oct 2026 21:26:12 +0330 Subject: [PATCH 39/40] fix(mtproto): match telemt's decoy origin rule and keep decoys off telemt's own ports A WEB decoy upstream must now be a bare http origin (no https, path, query, fragment, credentials or port 0), as telemt's parse_origin requires, so the node refuses it before telemt fails at Start. Render also refuses a loopback decoy on the node's telemt API or metrics port, which would publish that endpoint on the WEB domain. Also fix the max_profiles rejection reason and the UpdateUsers comment, and say in the README that web_trusted_proxy_cidrs is required and where telemt's per-profile limits stop. --- README.md | 9 +++-- backend/mtproto/config.go | 6 ++- backend/mtproto/config_test.go | 65 +++++++++++++++++++++++++++++++ backend/mtproto/sync.go | 6 ++- backend/mtproto/webconfig.go | 70 ++++++++++++++++++++++++++++++++-- 5 files changed, 145 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 392cd9a1..d1c0de74 100644 --- a/README.md +++ b/README.md @@ -52,12 +52,13 @@ What the node accepts in the MTProto core JSON: - A `server.listeners[]` entry with `"transport": "web"`: - `ip` is a loopback or private address (127/8, ::1, 10/8, 172.16/12, 192.168/16, fc00::/7). `0.0.0.0`, `::` and public addresses are refused. - `port` is set, and differs from `server.port` and from every native listener's port. - - telemt's WEB listener keys pass through, for example `web_client_ip_source` and `web_trusted_proxy_cidrs`. + - `web_trusted_proxy_cidrs` is required: the networks your front connects from, for example `["127.0.0.1/32"]`. telemt refuses a WEB listener with an empty list or a `/0` network. + - telemt's other WEB listener keys pass through, for example `web_client_ip_source`. - A `web` section with `"enabled": true` and at least one `vhosts[]` entry: - `host`: the public domain. `base_path` is optional. - `public_addr`: the server's public IP on port 443, for example `203.0.113.10:443`. - - `decoy`: `{"mode": "http_upstream", "upstream": "http://[:port]"}`. Host names and `static_directory` are refused. The decoy is the site that ordinary visitors and probes see. - - `user_profile` (PasarGuard only, optional): `secret_mode` is `"dd"` (default) or `"plain"`, plus optional `max_sessions`, `max_streams` and `max_streams_per_session`. It applies to every user on that vhost. + - `decoy`: `{"mode": "http_upstream", "upstream": "http://[:port]"}`. The upstream is a bare http origin: no `https`, path, query, fragment or credentials. Host names, `static_directory`, and a loopback upstream on the node's telemt API or metrics port are refused. The decoy is the site that ordinary visitors and probes see. + - `user_profile` (PasarGuard only, optional): `secret_mode` is `"dd"` (default) or `"plain"`, plus optional `max_sessions`, `max_streams` and `max_streams_per_session`. It applies to every user on that vhost. telemt refuses at Start a limit of 0 and limits above its global WEB limits: `max_sessions` above `web.limits.max_sessions_global` (default 128), `max_streams` above `web.limits.max_streams_global` (default 4096), `max_streams_per_session` above `web.limits.max_streams_per_session` (default 128) or above `max_streams`. It also refuses more vhosts than `web.limits.max_vhosts` (default 8). - Refused: `web.vhosts[].profiles`, `web.debug`, `web.limits.max_profiles` and `include` anywhere. The node writes one telemt WEB profile per user per vhost and sets `web.limits.max_profiles` to 65536. With WEB on, every user change rewrites telemt's config and hot reloads it: open sessions of unchanged users stay, removed users are cut. @@ -173,7 +174,7 @@ A WEB domain points at one server, and `public_addr` is that server's IP. With s ### Capacity - telemt holds at most 65536 WEB profiles (users times vhosts): 65536 users with one vhost, 32768 with two. -- The 8 MiB config file limit allows about 69k users with one WEB vhost (about 121.6 bytes per user), and about 36k if every user has every optional limit set (about 232.9 bytes per user). Native only is about 196k (about 42.8 bytes per user). Each extra vhost adds about 79 bytes per user. +- The 8 MiB config file limit allows about 69k users with one WEB vhost (about 121.6 bytes per user), and about 36k if every user has every optional limit set (about 232.9 bytes per user). Native only is about 196k (about 42.8 bytes per user). Each extra vhost adds an estimated 79 bytes per user (not measured). - 65536 profiles fit telemt's default WEB memory envelope (1280 MiB, about 4 MiB free for profiles at 64 bytes each). If you raise other `web.limits` reservations or lower `web.limits.memory_envelope_bytes`, telemt refuses to start with `web.limits memory reservations must fit memory_envelope_bytes`. ## Xray and MTProto on one host diff --git a/backend/mtproto/config.go b/backend/mtproto/config.go index 5c4b59d5..a61e1dca 100644 --- a/backend/mtproto/config.go +++ b/backend/mtproto/config.go @@ -45,7 +45,7 @@ var ( rejectedConfigKeys = []rejectedConfigKey{ {path: []string{"include"}, reason: "telemt would read other files on the node host"}, {path: []string{"web", "debug"}, reason: "the WEB debug endpoint would expose node traffic records"}, - {path: []string{"web", "limits", "max_profiles"}, reason: "the node sizes it from the WEB profiles it renders"}, + {path: []string{"web", "limits", "max_profiles"}, reason: "the node sets it"}, {path: []string{"server", "api"}, reason: "the node manages the telemt control API"}, {path: []string{"server", "admin_api"}, reason: "it is an alias of server.api, which the node manages"}, {path: []string{"server", "metrics_port"}, reason: "the node manages the metrics endpoint"}, @@ -300,6 +300,10 @@ func (c *Config) Render(users map[string]*runtimeUser, marker *runtimeUser, apiP } } + if err := checkDecoyPorts(c.base, apiPort, metricsPort); err != nil { + return "", err + } + root := cloneMap(c.base) general := ensureObject(root, "general") telemetry := ensureObject(general, "telemetry") diff --git a/backend/mtproto/config_test.go b/backend/mtproto/config_test.go index 9e3149ed..88538a66 100644 --- a/backend/mtproto/config_test.go +++ b/backend/mtproto/config_test.go @@ -490,6 +490,14 @@ func TestNewConfigRejectsUnsafeWeb(t *testing.T) { {"static directory decoy", withVhost(`{` + host + `, "decoy": {"mode": "static_directory", "directory": "/srv"}}`), "static_directory"}, {"decoy hostname", withVhost(`{` + host + `, "decoy": {"mode": "http_upstream", "upstream": "http://example.com"}}`), "private"}, {"decoy public ip", withVhost(`{` + host + `, "decoy": {"mode": "http_upstream", "upstream": "http://8.8.8.8"}}`), "private"}, + {"decoy https", withVhost(`{` + host + `, "decoy": {"mode": "http_upstream", "upstream": "https://127.0.0.1:18081"}}`), "http origin"}, + {"decoy path", withVhost(`{` + host + `, "decoy": {"mode": "http_upstream", "upstream": "http://192.168.0.10/site"}}`), "http origin"}, + {"decoy query", withVhost(`{` + host + `, "decoy": {"mode": "http_upstream", "upstream": "http://127.0.0.1:18081/?a=1"}}`), "http origin"}, + {"decoy empty query", withVhost(`{` + host + `, "decoy": {"mode": "http_upstream", "upstream": "http://127.0.0.1:18081?"}}`), "http origin"}, + {"decoy fragment", withVhost(`{` + host + `, "decoy": {"mode": "http_upstream", "upstream": "http://127.0.0.1:18081/#top"}}`), "http origin"}, + {"decoy empty fragment", withVhost(`{` + host + `, "decoy": {"mode": "http_upstream", "upstream": "http://127.0.0.1:18081#"}}`), "http origin"}, + {"decoy userinfo", withVhost(`{` + host + `, "decoy": {"mode": "http_upstream", "upstream": "http://user:pass@127.0.0.1:18081"}}`), "http origin"}, + {"decoy port zero", withVhost(`{` + host + `, "decoy": {"mode": "http_upstream", "upstream": "http://127.0.0.1:0"}}`), "http origin"}, {"decoy missing", withVhost(`{` + host + `}`), "decoy"}, {"vhost profiles", withVhost(`{` + host + `, "decoy": {"mode": "http_upstream", "upstream": "http://127.0.0.1:1"}, "profiles": []}`), "must not define web.vhosts[0].profiles"}, {"web debug", withWeb(`{"enabled": true, "debug": {}, "vhosts": [{` + host + `, "decoy": {"mode": "http_upstream", "upstream": "http://127.0.0.1:1"}}]}`), "must not define web.debug"}, @@ -517,6 +525,63 @@ func TestNewConfigRejectsUnsafeWeb(t *testing.T) { } } +func TestNewConfigAcceptsDecoyOrigins(t *testing.T) { + for _, upstream := range []string{"http://127.0.0.1:18081", "http://127.0.0.1:18081/", "http://10.0.0.5", "http://[::1]:18081/"} { + raw := `{"server": {"port": 443, ` + webTestListeners + `}, "web": {"enabled": true, "vhosts": [{"host": "proxy.example.com", "public_addr": "203.0.113.10:443", "decoy": {"mode": "http_upstream", "upstream": "` + upstream + `"}}]}}` + if _, err := NewConfig(raw); err != nil { + t.Errorf("NewConfig with upstream %q failed: %v", upstream, err) + } + } +} + +// A loopback decoy on telemt's own API or metrics port would publish them on +// the WEB domain; Render refuses it because only Render knows those ports. +func TestRenderRejectsDecoyOnTelemtPorts(t *testing.T) { + users := map[string]*runtimeUser{"1": {Username: "1", Secret: testSecret}} + cases := map[string]struct { + upstream string + want string + }{ + "metrics port": {"http://127.0.0.1:23456", "metrics port"}, + "api port": {"http://127.0.0.1:12345/", "API port"}, + "v6 loopback": {"http://[::1]:23456", "metrics port"}, + "other loopback": {"http://127.0.0.2:12345", "API port"}, + "default port 80": {"http://127.0.0.1", "metrics port"}, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + raw := `{"server": {"port": 443, ` + webTestListeners + `}, "web": {"enabled": true, "vhosts": [{"host": "proxy.example.com", "public_addr": "203.0.113.10:443", "decoy": {"mode": "http_upstream", "upstream": "` + tc.upstream + `"}}]}}` + cfg, err := NewConfig(raw) + if err != nil { + t.Fatalf("NewConfig failed: %v", err) + } + metricsPort := 23456 + if name == "default port 80" { + metricsPort = 80 + } + _, err = cfg.Render(users, nil, 12345, metricsPort, "Bearer x") + if err == nil { + t.Fatalf("Render with upstream %q succeeded, want a port conflict", tc.upstream) + } + for _, part := range []string{"web.vhosts[0].decoy.upstream", tc.want} { + if !strings.Contains(err.Error(), part) { + t.Errorf("error %q does not contain %q", err, part) + } + } + }) + } + + // A private, non-loopback host on the same port is another machine. + raw := `{"server": {"port": 443, ` + webTestListeners + `}, "web": {"enabled": true, "vhosts": [{"host": "proxy.example.com", "public_addr": "203.0.113.10:443", "decoy": {"mode": "http_upstream", "upstream": "http://10.0.0.5:23456"}}]}}` + cfg, err := NewConfig(raw) + if err != nil { + t.Fatal(err) + } + if _, err := cfg.Render(users, nil, 12345, 23456, "Bearer x"); err != nil { + t.Errorf("Render with a private decoy on another host failed: %v", err) + } +} + func TestWebListenerPortClash(t *testing.T) { const web = `"web": {"enabled": true, "vhosts": [{"host": "proxy.example.com", "public_addr": "203.0.113.10:443", "decoy": {"mode": "http_upstream", "upstream": "http://127.0.0.1:18081"}}]}` cases := map[string]string{ diff --git a/backend/mtproto/sync.go b/backend/mtproto/sync.go index 5688df19..a76f9623 100644 --- a/backend/mtproto/sync.go +++ b/backend/mtproto/sync.go @@ -46,8 +46,10 @@ func (m *MTProto) SyncUsers(ctx context.Context, users []*common.User) error { // UpdateUsers applies users on top of the current ones. Up to // partialApplyMaxUsers changes go through telemt's user API; more, or an -// unconfirmed telemt state, go through a config render and reload. Users -// that did not change are not sent at all. +// unconfirmed telemt state, go through a config render and reload. With WEB +// on, every change goes through the render and reload, because telemt's user +// API cannot reach the WEB profiles. Users that did not change are not sent +// at all. func (m *MTProto) UpdateUsers(ctx context.Context, users []*common.User) error { if err := m.lockSync(ctx); err != nil { return syncError(err) diff --git a/backend/mtproto/webconfig.go b/backend/mtproto/webconfig.go index b78c437e..99ee642e 100644 --- a/backend/mtproto/webconfig.go +++ b/backend/mtproto/webconfig.go @@ -8,6 +8,8 @@ import ( "net" "net/url" "slices" + "strconv" + "strings" ) const ( @@ -151,7 +153,9 @@ func takeWebProfiles(root map[string]any) ([]webUserProfile, error) { // validateWebDecoy allows only an http_upstream decoy whose upstream host is a // loopback or private IP literal: static_directory would serve files from the -// node host, and a public or named upstream would reach past the node. +// node host, and a public or named upstream would reach past the node. The +// upstream must also be a bare http origin, as telemt's decoy_dns::parse_origin +// requires, so a value telemt refuses at Start is refused here first. func validateWebDecoy(index int, rawDecoy any) error { field := fmt.Sprintf("web.vhosts[%d].decoy", index) decoy, ok := rawDecoy.(map[string]any) @@ -167,9 +171,9 @@ func validateWebDecoy(index int, rawDecoy any) error { } upstream, _ := decoy["upstream"].(string) - parsed, err := url.Parse(upstream) - if err != nil || upstream == "" { - return fmt.Errorf("mtproto config field %s.upstream must be an http URL", field) + parsed, ok := parseHTTPOrigin(upstream) + if !ok { + return fmt.Errorf("mtproto config field %s.upstream must be an http origin without credentials, path, query, or fragment", field) } ip := net.ParseIP(parsed.Hostname()) if ip == nil || ip.IsUnspecified() || !(ip.IsLoopback() || ip.IsPrivate()) { @@ -178,6 +182,64 @@ func validateWebDecoy(index int, rawDecoy any) error { return nil } +// checkDecoyPorts refuses a decoy upstream on a loopback host and the node's +// telemt API or metrics port: the decoy connects from loopback, so the WEB +// domain would publish that endpoint. Only Render knows the two ports. +func checkDecoyPorts(base map[string]any, apiPort, metricsPort int) error { + web, _ := base["web"].(map[string]any) + vhosts, _ := web["vhosts"].([]any) + for index, rawVhost := range vhosts { + vhost, _ := rawVhost.(map[string]any) + decoy, _ := vhost["decoy"].(map[string]any) + upstream, _ := decoy["upstream"].(string) + origin, ok := parseHTTPOrigin(upstream) + if !ok { + continue + } + if ip := net.ParseIP(origin.Hostname()); ip == nil || !ip.IsLoopback() { + continue + } + field := fmt.Sprintf("web.vhosts[%d].decoy.upstream", index) + switch port := decoyOriginPort(origin); port { + case apiPort: + return fmt.Errorf("mtproto config field %s port %d conflicts with the node's telemt API port", field, port) + case metricsPort: + return fmt.Errorf("mtproto config field %s port %d conflicts with the node's telemt metrics port", field, port) + } + } + return nil +} + +// parseHTTPOrigin parses an http URL with a host, an optional port in +// 1-65535, a path of "" or "/", and no userinfo, query or fragment. +func parseHTTPOrigin(raw string) (*url.URL, bool) { + // url.Parse drops an empty "#", and "?" only sets ForceQuery, so both + // characters are checked on the raw string. + if strings.ContainsAny(raw, "?#") { + return nil, false + } + parsed, err := url.Parse(raw) + if err != nil || parsed.Scheme != "http" || parsed.Host == "" || parsed.User != nil || + (parsed.Path != "" && parsed.Path != "/") || parsed.RawPath != "" { + return nil, false + } + if port := parsed.Port(); port != "" { + if number, err := strconv.Atoi(port); err != nil || number < 1 || number > 65535 { + return nil, false + } + } + return parsed, true +} + +// decoyOriginPort is the TCP port an origin from parseHTTPOrigin connects to. +func decoyOriginPort(origin *url.URL) int { + if port := origin.Port(); port != "" { + number, _ := strconv.Atoi(port) + return number + } + return 80 +} + func webUserProfileValue(index int, raw any) (webUserProfile, error) { profile := webUserProfile{SecretMode: webSecretModeDD} if raw == nil { From 91e81980e8f653ded3a4d343fa920da72913ec0b Mon Sep 17 00:00:00 2001 From: T3ST3ST3R0N Date: Mon, 26 Oct 2026 22:03:11 +0330 Subject: [PATCH 40/40] fix(mtproto): check WEB profile totals and base paths before telemt does telemt counts WEB profiles across all vhosts and refuses a config above web.limits.max_profiles, and it refuses a vhost base_path that is not empty or slash-separated [A-Za-z0-9][A-Za-z0-9_-]* segments of at most 128 ASCII bytes. Render now refuses more than 65536 profiles before writing the file, naming the user and vhost counts, and NewConfig applies telemt's base_path rule. The README notes that `http2 on;` needs nginx 1.25.1 or newer. --- README.md | 6 ++- backend/mtproto/config.go | 15 +++++++ backend/mtproto/config_test.go | 72 ++++++++++++++++++++++++++++++++++ backend/mtproto/webconfig.go | 39 ++++++++++++++++++ 4 files changed, 130 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d1c0de74..3166ec26 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ What the node accepts in the MTProto core JSON: - `web_trusted_proxy_cidrs` is required: the networks your front connects from, for example `["127.0.0.1/32"]`. telemt refuses a WEB listener with an empty list or a `/0` network. - telemt's other WEB listener keys pass through, for example `web_client_ip_source`. - A `web` section with `"enabled": true` and at least one `vhosts[]` entry: - - `host`: the public domain. `base_path` is optional. + - `host`: the public domain. `base_path` is optional: empty, or up to 128 ASCII bytes of `/`-separated segments that each start with a letter or digit and hold only letters, digits, `-` and `_`, with no leading or trailing `/` (for example `tg` or `dobry-cola/super_app`, not `/tg`). - `public_addr`: the server's public IP on port 443, for example `203.0.113.10:443`. - `decoy`: `{"mode": "http_upstream", "upstream": "http://[:port]"}`. The upstream is a bare http origin: no `https`, path, query, fragment or credentials. Host names, `static_directory`, and a loopback upstream on the node's telemt API or metrics port are refused. The decoy is the site that ordinary visitors and probes see. - `user_profile` (PasarGuard only, optional): `secret_mode` is `"dd"` (default) or `"plain"`, plus optional `max_sessions`, `max_streams` and `max_streams_per_session`. It applies to every user on that vhost. telemt refuses at Start a limit of 0 and limits above its global WEB limits: `max_sessions` above `web.limits.max_sessions_global` (default 128), `max_streams` above `web.limits.max_streams_global` (default 4096), `max_streams_per_session` above `web.limits.max_streams_per_session` (default 128) or above `max_streams`. It also refuses more vhosts than `web.limits.max_vhosts` (default 8). @@ -160,6 +160,8 @@ server { } } ``` +`http2 on;` needs nginx 1.25.1 or newer; older nginx fails `nginx -t` with an unknown directive, so drop that line and use `listen 443 ssl http2;` instead. + `client_max_body_size` must be at least telemt's `web.limits.max_body_bytes`. Firewall the WEB listener port and the decoy port from outside, even on loopback. @@ -173,7 +175,7 @@ The front needs public port 443, and native fake TLS usually wants 443 too. Eith A WEB domain points at one server, and `public_addr` is that server's IP. With several MTProto nodes, use one core per node. ### Capacity -- telemt holds at most 65536 WEB profiles (users times vhosts): 65536 users with one vhost, 32768 with two. +- telemt holds at most 65536 WEB profiles (users times vhosts): 65536 users with one vhost, 32768 with two. Above that the node refuses the sync with an error and leaves telemt's config file unchanged. - The 8 MiB config file limit allows about 69k users with one WEB vhost (about 121.6 bytes per user), and about 36k if every user has every optional limit set (about 232.9 bytes per user). Native only is about 196k (about 42.8 bytes per user). Each extra vhost adds an estimated 79 bytes per user (not measured). - 65536 profiles fit telemt's default WEB memory envelope (1280 MiB, about 4 MiB free for profiles at 64 bytes each). If you raise other `web.limits` reservations or lower `web.limits.memory_envelope_bytes`, telemt refuses to start with `web.limits memory reservations must fit memory_envelope_bytes`. diff --git a/backend/mtproto/config.go b/backend/mtproto/config.go index a61e1dca..8e00678b 100644 --- a/backend/mtproto/config.go +++ b/backend/mtproto/config.go @@ -336,12 +336,27 @@ func (c *Config) Render(users map[string]*runtimeUser, marker *runtimeUser, apiP return "", err } if c.HasWeb() { + if err := checkWebProfileCount(len(c.webProfiles), len(users)); err != nil { + return "", err + } applyWebProfiles(root, c.webProfiles, users, marker) } return renderTOML(root) } +// checkWebProfileCount refuses a render with more WEB profiles than +// webMaxProfiles. telemt counts profiles across all vhosts and refuses the +// whole config above web.limits.max_profiles. Every vhost gets one profile per +// user; the marker's single profile per vhost, used only without users, never +// reaches the limit. +func checkWebProfileCount(vhostCount, userCount int) error { + if total := vhostCount * userCount; total > webMaxProfiles { + return fmt.Errorf("mtproto config would need %d WEB profiles (%d users x %d vhosts), above the %d the node allows", total, userCount, vhostCount, webMaxProfiles) + } + return nil +} + // applyWebProfiles gives every WEB vhost one profile per user, built from the // vhost's user_profile template, and sets web.limits.max_profiles. With no // users the marker gets the profile instead: telemt rejects an enabled vhost diff --git a/backend/mtproto/config_test.go b/backend/mtproto/config_test.go index 88538a66..fa0efcd8 100644 --- a/backend/mtproto/config_test.go +++ b/backend/mtproto/config_test.go @@ -2,6 +2,8 @@ package mtproto import ( "context" + "encoding/json" + "fmt" "slices" "strconv" "strings" @@ -729,3 +731,73 @@ func TestRenderWebConfigSize(t *testing.T) { t.Errorf("rendered %d profiles, want %d", got, userCount) } } + +// telemt counts WEB profiles across all vhosts and refuses a config above +// web.limits.max_profiles, so Render refuses it first and names the counts. +func TestRenderRejectsTooManyWebProfiles(t *testing.T) { + const twoVhosts = `{"server": {"port": 443, ` + webTestListeners + `}, "web": {"enabled": true, "vhosts": [` + + `{"host": "a.example.com", "public_addr": "203.0.113.10:443", "decoy": {"mode": "http_upstream", "upstream": "http://127.0.0.1:18081"}}, ` + + `{"host": "b.example.com", "public_addr": "203.0.113.10:443", "decoy": {"mode": "http_upstream", "upstream": "http://127.0.0.1:18081"}}]}}` + cfg, err := NewConfig(twoVhosts) + if err != nil { + t.Fatal(err) + } + usersOf := func(count int) map[string]*runtimeUser { + users := make(map[string]*runtimeUser, count) + for id := 1; id <= count; id++ { + username := strconv.Itoa(id) + users[username] = &runtimeUser{Username: username, Secret: testSecret} + } + return users + } + marker := &runtimeUser{Username: "pgm_0123456789abcdef", Secret: thirdSecret} + + _, err = cfg.Render(usersOf(webMaxProfiles/2+1), marker, 12345, 23456, "Bearer token") + if err == nil { + t.Fatal("Render with 65538 WEB profiles succeeded, want an error") + } + for _, part := range []string{"65538 WEB profiles", "32769 users x 2 vhosts", "65536"} { + if !strings.Contains(err.Error(), part) { + t.Errorf("error %q does not contain %q", err, part) + } + } + + out, err := cfg.Render(usersOf(webMaxProfiles/2), marker, 12345, 23456, "Bearer token") + if err != nil { + t.Fatalf("Render with exactly 65536 WEB profiles failed: %v", err) + } + if got := strings.Count(out, "[[web.vhosts.profiles]]"); got != webMaxProfiles { + t.Errorf("rendered %d profiles, want %d", got, webMaxProfiles) + } +} + +// base_path follows telemt's validate_web_base_path: empty, or at most 128 +// ASCII bytes of slash-separated [A-Za-z0-9][A-Za-z0-9_-]* segments. +func TestNewConfigWebBasePath(t *testing.T) { + withBasePath := func(value any) string { + encoded, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + return webTestConfig(`, "base_path": ` + string(encoded)) + } + long := strings.Repeat("a", 129) + for _, value := range []any{"/tg", "tg/", "/", "a//b", "-x", "_x", long, "télégram", "tg.app", "tg path", 7} { + t.Run(fmt.Sprintf("reject %v", value), func(t *testing.T) { + _, err := NewConfig(withBasePath(value)) + if err == nil { + t.Fatalf("NewConfig with base_path %#v succeeded, want an error", value) + } + if !strings.Contains(err.Error(), "web.vhosts[0].base_path") { + t.Errorf("error %q does not name web.vhosts[0].base_path", err) + } + }) + } + for _, value := range []string{"", "tg", "dobry-cola/super_app", "A1/b-2/c_3", strings.Repeat("a", 128)} { + t.Run("accept "+value, func(t *testing.T) { + if _, err := NewConfig(withBasePath(value)); err != nil { + t.Errorf("NewConfig with base_path %q failed: %v", value, err) + } + }) + } +} diff --git a/backend/mtproto/webconfig.go b/backend/mtproto/webconfig.go index 99ee642e..a7bf81f7 100644 --- a/backend/mtproto/webconfig.go +++ b/backend/mtproto/webconfig.go @@ -138,6 +138,9 @@ func takeWebProfiles(root map[string]any) ([]webUserProfile, error) { if isSet(vhost, "profiles") { return nil, fmt.Errorf("mtproto config must not define web.vhosts[%d].profiles: the node builds them from its users", index) } + if err := validateWebBasePath(index, vhost["base_path"]); err != nil { + return nil, err + } if err := validateWebDecoy(index, vhost["decoy"]); err != nil { return nil, err } @@ -151,6 +154,42 @@ func takeWebProfiles(root map[string]any) ([]webUserProfile, error) { return profiles, nil } +// webBasePathMaxBytes is telemt's length limit for a vhost base_path. +const webBasePathMaxBytes = 128 + +// validateWebBasePath applies telemt's validate_web_base_path: base_path is +// empty, or at most 128 ASCII bytes of slash-separated segments that each +// match [A-Za-z0-9][A-Za-z0-9_-]*, with no leading or trailing slash. telemt +// uses the value as is, so a value it refuses at Start is refused here first. +func validateWebBasePath(index int, rawBasePath any) error { + if rawBasePath == nil { + return nil + } + field := fmt.Sprintf("web.vhosts[%d].base_path", index) + basePath, ok := rawBasePath.(string) + if !ok { + return fmt.Errorf("mtproto config field %s must be a string", field) + } + if basePath == "" { + return nil + } + valid := len(basePath) <= webBasePathMaxBytes + for segment := range strings.SplitSeq(basePath, "/") { + valid = valid && segment != "" && isASCIIAlphanumeric(segment[0]) && + strings.IndexFunc(segment, func(r rune) bool { + return !(r < 0x80 && (isASCIIAlphanumeric(byte(r)) || r == '-' || r == '_')) + }) < 0 + } + if !valid { + return fmt.Errorf("mtproto config field %s must be empty or at most %d ASCII bytes of slash-separated [A-Za-z0-9][A-Za-z0-9_-]* segments, without a leading or trailing slash", field, webBasePathMaxBytes) + } + return nil +} + +func isASCIIAlphanumeric(b byte) bool { + return ('a' <= b && b <= 'z') || ('A' <= b && b <= 'Z') || ('0' <= b && b <= '9') +} + // validateWebDecoy allows only an http_upstream decoy whose upstream host is a // loopback or private IP literal: static_directory would serve files from the // node host, and a public or named upstream would reach past the node. The