diff --git a/cloud/relay/relay.go b/cloud/relay/relay.go index 0be2d0f9..0db72ec8 100644 --- a/cloud/relay/relay.go +++ b/cloud/relay/relay.go @@ -225,5 +225,4 @@ func (h *Host) Close() error { // Addr is the relay's, as a listener's. func (h *Host) Addr() net.Addr { return Addr("relay") } - var _ net.Listener = (*Host)(nil) diff --git a/cloud/server/relay.go b/cloud/server/relay.go index 45f6ba57..82b637bf 100644 --- a/cloud/server/relay.go +++ b/cloud/server/relay.go @@ -8,7 +8,7 @@ import ( "encoding/json" "errors" "io" - "net/http" + "net/http" "strings" "sync" "time" @@ -33,6 +33,11 @@ type RelayLimits struct { // BytesPerSecond is a team's rate, both ways together, all its // connections together; Burst is how much may go at once above it. BytesPerSecond, Burst int + // Ping is how often the relay pings the hosts' links and both sides of + // a pair: proxies on the way (Cloudflare's, after 100 seconds) cut a + // WebSocket that carries nothing, as a member watching a quiet + // terminal's would. + Ping time.Duration } const ( @@ -84,6 +89,9 @@ func (s *Server) relayLimits() RelayLimits { if l.Burst <= 0 { l.Burst = relayBurst } + if l.Ping <= 0 { + l.Ping = relayPing + } return l } @@ -134,7 +142,7 @@ func (s *Server) relayHostConn(w http.ResponseWriter, r *http.Request) { s.log.Info("relay host", "device", dev, "state", "gone") }() ctx := ws.CloseRead(r.Context()) // the host sends nothing on it - t := time.NewTicker(relayPing) + t := time.NewTicker(s.relayLimits().Ping) defer t.Stop() for { select { @@ -274,6 +282,7 @@ func (s *Server) relayConnect(w http.ResponseWriter, r *http.Request) { defer stop() m := websocket.NetConn(ctx, memberWS, websocket.MessageBinary) h := websocket.NetConn(ctx, hostWS, websocket.MessageBinary) + go keepPinging(ctx, stop, limits.Ping, memberWS, hostWS) var up, down int64 done := make(chan struct{}, 2) go func() { up = s.relayCopy(h, m, ts); stop(); done <- struct{}{} }() @@ -286,6 +295,33 @@ func (s *Server) relayConnect(w http.ResponseWriter, r *http.Request) { s.log.Info("relay visit", "team", team, "account", account, "host", host, "up", up, "down", down, "s", int(time.Since(call.start).Seconds())) } +// keepPinging pings both sides of a pair until ctx ends, so a quiet one +// isn't cut on the way; a side that doesn't answer ends the pair. Pings +// carry nothing of the visit: its records stay sealed end to end. +func keepPinging(ctx context.Context, stop func(), every time.Duration, sides ...*websocket.Conn) { + for _, ws := range sides { + // Each side apart: one slow to answer doesn't keep the other quiet. + go func() { + t := time.NewTicker(every) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + } + pctx, cancel := context.WithTimeout(ctx, 10*time.Second) + err := ws.Ping(pctx) + cancel() + if err != nil { + stop() + return + } + } + }() + } +} + // relayTap sees what the relay passes, for tests. var relayTap func([]byte) diff --git a/cloud/server/relay_test.go b/cloud/server/relay_test.go index 064acd2f..94b23859 100644 --- a/cloud/server/relay_test.go +++ b/cloud/server/relay_test.go @@ -221,3 +221,113 @@ func refusedWith(err error, status int) bool { var r *relaypkg.Refused return errors.As(err, &r) && r.Status == status } + +// idleCutter stands for a proxy on the way that cuts a connection which +// carried nothing either way for cut, as Cloudflare's does: it passes +// connections to target. +func idleCutter(t *testing.T, target string, cut time.Duration) string { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { l.Close() }) + go func() { + for { + in, err := l.Accept() + if err != nil { + return + } + out, err := net.Dial("tcp", target) + if err != nil { + in.Close() + continue + } + var mu sync.Mutex + last := time.Now() + pass := func(dst, src net.Conn) { + buf := make([]byte, 32<<10) + for { + n, err := src.Read(buf) + if n > 0 { + mu.Lock() + last = time.Now() + mu.Unlock() + dst.Write(buf[:n]) + } + if err != nil { + in.Close() + out.Close() + return + } + } + } + go pass(out, in) + go pass(in, out) + go func() { + for range time.Tick(cut / 10) { + mu.Lock() + idle := time.Since(last) + mu.Unlock() + if idle > cut { + in.Close() + out.Close() + return + } + } + }() + } + }() + return l.Addr().String() +} + +// A pair that carries nothing for longer than a proxy on the way lets a +// connection idle stays up: the relay pings both its sides. +func TestRelayQuietPairStaysUp(t *testing.T) { + r := startRelayTeam(t, Config{Relay: RelayLimits{Ping: 100 * time.Millisecond}}) + cut := "http://" + idleCutter(t, r.srv.Listener.Addr().String(), 500*time.Millisecond) + // Ada's host reaches the relay through the proxy too. + r.host.Close() + host := &relaypkg.Host{URL: cut, Token: r.adaTok, Key: r.key, Retry: 50 * time.Millisecond} + host.Start() + t.Cleanup(func() { host.Close() }) + go func() { + for { + c, err := host.Accept() + if err != nil { + return + } + go io.Copy(c, c) + } + }() + for up, _ := host.Status(); !up; up, _ = host.Status() { + time.Sleep(10 * time.Millisecond) + } + + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + c, err := relaypkg.Dial(ctx, "ws"+cut[len("http"):]+api.PathRelayConnect, r.ticket(t), r.key.Public()) + if err != nil { + t.Fatal(err) + } + defer c.Close() + // A member's client reads all along, as the app's does. + got := make(chan []byte, 1) + go func() { + back := make([]byte, 10) + n, _ := io.ReadFull(c, back) + got <- back[:n] + }() + time.Sleep(2 * time.Second) // four times what the proxy lets idle + msg := []byte("still here") + if _, err := c.Write(msg); err != nil { + t.Fatalf("the quiet pair was cut: %v", err) + } + select { + case back := <-got: + if !bytes.Equal(back, msg) { + t.Fatalf("the quiet pair was cut: %q", back) + } + case <-time.After(5 * time.Second): + t.Fatal("nothing came back") + } +} diff --git a/docs/cloud.md b/docs/cloud.md index e333762d..22642cd3 100644 --- a/docs/cloud.md +++ b/docs/cloud.md @@ -90,7 +90,7 @@ Tokens are random, kept hashed on the server, one per device; signing out or rem Two kinds of hosts serve a team: a **team host**, paired to the team by an admin, which stays when members come and go, and a member's **personal host shared** with the team, project by project, until they take it back. Members reach them through the cloud's relay (below); the cloud only says which hosts and projects a team has, and the daemon checks a member's token and role before letting them in. Sessions in a team's project show to the team (status, watching) unless marked private; taking over one needs its owner's approval. A session runs with the member's own agent account and git identity, lent from their vault for that session. An agent reads another session only when the user asks it to (`enowx-app peek`, docs/cli.md): its last messages and project, live from its host, never kept in the cloud; the user's own hosts, and the team's shared ones within what the member sees there. -A member reaches another's host through the cloud's **relay**, with no address of the host's and no port open on it: the host keeps a connection to the relay (`/v1/relay/host`, its own token) while it shares anything; a member's app connects with their ticket (`/v1/relay/connect`), the relay checks the ticket without using it (one pairing a ticket), tells the host, the host opens a connection for that member (`/v1/relay/accept/{id}`), and the relay passes bytes between the two. What passes is **end to end encrypted** (`cloud/relay`): the host has a key of its own (`team-relay.key` in its data folder), whose public half it announces with itself in the team's space, sealed, so the cloud can't put another there. The member's app opens a channel to that key alone: its fresh key, the host's fresh key and a proof that the host holds the announced one (X25519, HKDF-SHA256), then records sealed with ChaCha20-Poly1305 each way under a counter, so a record read, changed, dropped or replayed by the relay fails. The relay sees who connects to which host, when, and how many bytes; not a terminal, a file or a ticket's answer. It holds each team to a number of connections at once and a rate of bytes (32, and 4 MiB a second with bursts of 16 MiB, unless the server sets others), and an address to 30 connections a minute. A host that also has a Tailscale address keeps its team listener there too (port 7178), for members of an older enowX, which knows no relay; a member's app tries the relay first and that address after. The host's daemon serves them on its team listener, apart from its own socket: a connection there gets nothing, not even events, until it hands a ticket the cloud gave the member for that host (`team.auth`): before it, a connection has 10 seconds, frames of 64 KiB at most and one request (anything but its ticket, or a ticket refused, closes it), and the listener takes a few connections and tickets a minute from one address; the host redeems the ticket with its own token, so no member's token ever leaves their device, and serves the visitor as their role allows. It asks the cloud again every 2 minutes, and before a visitor starts a session, whether each is still the team's member and in which role: a member removed is let go, one made a viewer keeps what a viewer may. A visitor sees no more of the host than the shared projects: not its home, name or process, not how a session was started or what the vault lent it, and a session they don't see is as missing as one that isn't there. The host's own app lists who is connected (`host.visitors`). Its user shares a host from a team's Hosts tab (or `enowx-app team share TEAM PROJECT…`), picking which of its projects the team sees: the host then connects to the relay (and opens its team listener on its Tailscale address, port 7178, when it has one), and the host is announced in the team's space (`hosts/.json`, sealed and synced like its other items). The members' apps list it in the rail, under TEAM (`enowx-app team hosts`); opening it gets a ticket from the cloud through the member's own host, connects, and shows the shared projects and their sessions. A visitor's lists hold the shared projects alone, and their files read only within them, a link that leads out of a shared project refused. Taking a project back closes visitors' views of its sessions and ends their control of them; sharing nothing more with a team, or leaving it, closes its visitors' connections. A visitor **watches** a shared session live, its terminal read-only: what they type and their window's size reach it only once they **take over**, which they ask from the watching view (one ask a session at a time, a few a minute); the host's user is asked (Let them type, or Not now), sees in the session's header who watches it and who has control, and takes it back any time; a visitor who leaves gives back what they took over. A member (not a viewer) also **starts sessions of their own** in a shared project, from its page, once the host's user lets them: **Members start sessions here** in the team's Hosts tab, off until turned on (`allowSessions` of `host.share.set`). Such a session runs as the host's OS user, so an agent there can do what that user can, the daemon's socket included: turn it on only for people trusted with the machine. It is Claude Code or Codex, run as an agent account of theirs on the host (`Team · @login` in its Agent accounts, kept by their cloud account, signed in by them the first time), its home their own with nothing linked from the host's user's (no sign-in, settings, skills or conversations); with their git identity and a GitHub CLI sign-in of their own (`GH_CONFIG_DIR` in that account's home); and none of the host's credentials: not the variables its user set for sessions, not its model API, nothing the visitor passes, no project setup script; the vault lends nothing to it. The session is theirs to type into, and shows as theirs in the host's list. The host's user keeps a session of a shared project from the team from its menu (**Keep from the team**): it leaves the team's lists, watchers' views close, and a takeover of it ends. A member's agent asked to read another session (`enowx-app peek`) reaches the team's shared hosts too, seeing only what the member sees there, and each read goes in the team's log (`session.read`). +A member reaches another's host through the cloud's **relay**, with no address of the host's and no port open on it: the host keeps a connection to the relay (`/v1/relay/host`, its own token) while it shares anything; a member's app connects with their ticket (`/v1/relay/connect`), the relay checks the ticket without using it (one pairing a ticket), tells the host, the host opens a connection for that member (`/v1/relay/accept/{id}`), and the relay passes bytes between the two. What passes is **end to end encrypted** (`cloud/relay`): the host has a key of its own (`team-relay.key` in its data folder), whose public half it announces with itself in the team's space, sealed, so the cloud can't put another there. The member's app opens a channel to that key alone: its fresh key, the host's fresh key and a proof that the host holds the announced one (X25519, HKDF-SHA256), then records sealed with ChaCha20-Poly1305 each way under a counter, so a record read, changed, dropped or replayed by the relay fails. The relay sees who connects to which host, when, and how many bytes; not a terminal, a file or a ticket's answer. It holds each team to a number of connections at once and a rate of bytes (32, and 4 MiB a second with bursts of 16 MiB, unless the server sets others), and an address to 30 connections a minute. It pings the host's link and both sides of each pair every 30 seconds, so a pair that carries nothing for a while (a member watching a quiet terminal) isn't cut by a proxy on the way, such as Cloudflare's after 100 idle seconds; pings carry nothing of the visit. A host that also has a Tailscale address keeps its team listener there too (port 7178), for members of an older enowX, which knows no relay; a member's app tries the relay first and that address after. The host's daemon serves them on its team listener, apart from its own socket: a connection there gets nothing, not even events, until it hands a ticket the cloud gave the member for that host (`team.auth`): before it, a connection has 10 seconds, frames of 64 KiB at most and one request (anything but its ticket, or a ticket refused, closes it), and the listener takes a few connections and tickets a minute from one address; the host redeems the ticket with its own token, so no member's token ever leaves their device, and serves the visitor as their role allows. It asks the cloud again every 2 minutes, and before a visitor starts a session, whether each is still the team's member and in which role: a member removed is let go, one made a viewer keeps what a viewer may. A visitor sees no more of the host than the shared projects: not its home, name or process, not how a session was started or what the vault lent it, and a session they don't see is as missing as one that isn't there. The host's own app lists who is connected (`host.visitors`). Its user shares a host from a team's Hosts tab (or `enowx-app team share TEAM PROJECT…`), picking which of its projects the team sees: the host then connects to the relay (and opens its team listener on its Tailscale address, port 7178, when it has one), and the host is announced in the team's space (`hosts/.json`, sealed and synced like its other items). The members' apps list it in the rail, under TEAM (`enowx-app team hosts`); opening it gets a ticket from the cloud through the member's own host, connects, and shows the shared projects and their sessions. A visitor's lists hold the shared projects alone, and their files read only within them, a link that leads out of a shared project refused. Taking a project back closes visitors' views of its sessions and ends their control of them; sharing nothing more with a team, or leaving it, closes its visitors' connections. A visitor **watches** a shared session live, its terminal read-only: what they type and their window's size reach it only once they **take over**, which they ask from the watching view (one ask a session at a time, a few a minute); the host's user is asked (Let them type, or Not now), sees in the session's header who watches it and who has control, and takes it back any time; a visitor who leaves gives back what they took over. A member (not a viewer) also **starts sessions of their own** in a shared project, from its page, once the host's user lets them: **Members start sessions here** in the team's Hosts tab, off until turned on (`allowSessions` of `host.share.set`). Such a session runs as the host's OS user, so an agent there can do what that user can, the daemon's socket included: turn it on only for people trusted with the machine. It is Claude Code or Codex, run as an agent account of theirs on the host (`Team · @login` in its Agent accounts, kept by their cloud account, signed in by them the first time), its home their own with nothing linked from the host's user's (no sign-in, settings, skills or conversations); with their git identity and a GitHub CLI sign-in of their own (`GH_CONFIG_DIR` in that account's home); and none of the host's credentials: not the variables its user set for sessions, not its model API, nothing the visitor passes, no project setup script; the vault lends nothing to it. The session is theirs to type into, and shows as theirs in the host's list. The host's user keeps a session of a shared project from the team from its menu (**Keep from the team**): it leaves the team's lists, watchers' views close, and a takeover of it ends. A member's agent asked to read another session (`enowx-app peek`) reaches the team's shared hosts too, seeing only what the member sees there, and each read goes in the team's log (`session.read`). ## API @@ -160,7 +160,7 @@ docker run -p 8080:8080 -e ENOWX_CLOUD_URL=https://cloud.example.com -e ENOWX_CL | `ENOWX_CLOUD_R2_BUCKET` | a Cloudflare R2 bucket keeping teams' files (a chat's images, sealed); unset, they're kept in the data folder's `blobs` | | `ENOWX_CLOUD_R2_ACCOUNT_ID`, `ENOWX_CLOUD_R2_ACCESS_KEY_ID`, `ENOWX_CLOUD_R2_SECRET_ACCESS_KEY` | the bucket's account and an R2 API token's keys | | `ENOWX_CLOUD_DEFAULT_TEAM`, `ENOWX_CLOUD_DEFAULT_ROLE` | a team's ID everyone joins as they sign in, and the role they get (`viewer` by default); optional | -| `ENOWX_CLOUD_RELAY_TEAM_CONNS`, `ENOWX_CLOUD_RELAY_TEAM_RATE`, `ENOWX_CLOUD_RELAY_TEAM_BURST` | each team's share of the relay: connections at once (32), bytes a second (4 MiB) and a burst above it (16 MiB); optional. The relay's WebSockets need a proxy that passes `Upgrade` and keeps a connection open past a minute (the relay pings every 30 seconds) | +| `ENOWX_CLOUD_RELAY_TEAM_CONNS`, `ENOWX_CLOUD_RELAY_TEAM_RATE`, `ENOWX_CLOUD_RELAY_TEAM_BURST` | each team's share of the relay: connections at once (32), bytes a second (4 MiB) and a burst above it (16 MiB); optional. The relay's WebSockets need a proxy that passes `Upgrade` and keeps a connection open past a minute (the relay pings hosts' links and members' pairs every 30 seconds) | | `ENOWX_CLOUD_SMTP_URL` | `smtp://user:pass@host:587`, to send invites by email; optional | It speaks plain HTTP: put it behind a proxy that terminates TLS (Coolify's, Caddy, nginx).