diff --git a/Cargo.lock b/Cargo.lock index 1e6eee57..b4d321e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14988,6 +14988,24 @@ dependencies = [ "serde_json", ] +[[package]] +name = "raea-runtime-appscreen" +version = "0.1.0" +dependencies = [ + "bevy", + "blitz-traits", + "keyboard-types", + "raea-io-home", + "raea-page", + "raea-runtime-bundle", + "raea-runtime-dom", + "raea-runtime-js", + "raea-runtime-jsengine", + "raea-sys-fonts", + "raea-sys-page", + "serde_json", +] + [[package]] name = "raea-runtime-bake" version = "0.1.0" @@ -15283,6 +15301,7 @@ dependencies = [ "raea-io-net", "raea-page", "raea-render", + "raea-runtime-appscreen", "raea-runtime-input", "raea-runtime-react", "raea-runtime-twins", diff --git a/Cargo.toml b/Cargo.toml index 8279b020..ffd62da8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -167,6 +167,7 @@ members = [ "runtime/export", "runtime/live", "runtime/page", + "runtime/appscreen", "runtime/sound", "runtime/look", "runtime/scene", diff --git a/apps/game/src/export.rs b/apps/game/src/export.rs index 37ba8fd0..647266a8 100644 --- a/apps/game/src/export.rs +++ b/apps/game/src/export.rs @@ -101,10 +101,7 @@ pub fn export(args: &[String]) -> R { gather(&mut bundle, &dir, &game, &mut got)?; std::fs::write(&js, &bundle).map_err(|e| e.to_string())?; - // an app screen needs a player with the surface runtime (the `surfaces` feature), which the size-built one leaves out - if got.files.iter().any(|(rel, _)| rel.ends_with(raea_runtime_page::screen::BUNDLED)) && !player_has_surfaces(&player)? { - return Err(format!("this game has an app screen (), and the player {} can't run one: build it with them (cargo build --profile game -p raea-app-game --bin raea-game --features player,surfaces) and export again", player.display())); - } + // an app screen () draws on the player's own Blitz page (raea-runtime-appscreen): every player has one // 3. what play works out at open, worked out now from the package itself (so the package is known to load) let src = std::fs::read_to_string(&tsx).unwrap_or_default(); @@ -255,12 +252,6 @@ pub(crate) fn imported(dir: &Path, file: &Path) -> bool { false } -/// Whether the player at `bin` was built with the surface runtime (app screens): its shim's text is in it. -pub(crate) fn player_has_surfaces(bin: &Path) -> R { - let bytes = std::fs::read(bin).map_err(|e| format!("{}: {e}", bin.display()))?; - Ok(bytes.windows(14).any(|w| w == b"__surfaceFrame")) -} - pub(crate) fn default_name(tsx: &Path) -> String { let stem = tsx.file_stem().map(|s| s.to_string_lossy().to_string()).unwrap_or_default(); let base = if stem == "edit" { tsx.parent().and_then(|p| p.file_name()).map(|s| s.to_string_lossy().to_string()).unwrap_or(stem) } else { stem }; diff --git a/apps/game/src/gather.rs b/apps/game/src/gather.rs index a67533dc..510f4642 100644 --- a/apps/game/src/gather.rs +++ b/apps/game/src/gather.rs @@ -105,13 +105,21 @@ pub(crate) fn gather(bundle: &mut String, dir: &Path, game: &Path, got: &mut Gat let ext = file.extension().and_then(|x| x.to_str()).unwrap_or("").to_ascii_lowercase(); if SOURCE_EXTS.contains(&ext.as_str()) || file.file_name().is_some_and(|n| n == MANIFEST) { // an app screen's module (``, a component on react-dom): bundled now with its npm - // packages beside where it was, as `chat.tsx.surface.js`, which the player opens (it bundles nothing) + // packages beside where it was, as `chat.tsx.surface.js` and its CSS `chat.tsx.surface.css`, which the + // player opens (it bundles nothing; raea-runtime-appscreen code_of) if matches!(ext.as_str(), "tsx" | "jsx") && std::fs::read_to_string(&file).is_ok_and(|t| t.contains("export default") && !t.contains("registerRoot(")) && !imported(dir, &file) { if let Some(rel) = inside(&file) { - let rel = format!("{rel}{}", raea_runtime_page::screen::BUNDLED); - if !got.has(&rel) { - raea_runtime_react::scene::bundle_surface(&file, &game.join(&rel))?; - got.files.push((rel, file.clone())); + let (js_rel, css_rel) = (format!("{rel}{}", raea_runtime_page::screen::BUNDLED), format!("{rel}{}", raea_runtime_page::screen::BUNDLED_CSS)); + if !got.has(&js_rel) { + let (js, css) = raea_runtime_page::screen::build_app(&file)?; + for (r, text) in [(&js_rel, js), (&css_rel, css)] { + let out = game.join(r); + if let Some(d) = out.parent() { + std::fs::create_dir_all(d).map_err(|e| format!("{}: {e}", d.display()))?; + } + std::fs::write(&out, text).map_err(|e| format!("{}: {e}", out.display()))?; + got.files.push((r.clone(), file.clone())); + } } } } diff --git a/apps/play/src/offscreen.rs b/apps/play/src/offscreen.rs index f5062c93..d3163422 100644 --- a/apps/play/src/offscreen.rs +++ b/apps/play/src/offscreen.rs @@ -237,6 +237,10 @@ pub(crate) fn offscreen(mut run: Run, json: Option, flat_out: bool, snap if let Some((x, y, down)) = run.pointer { e["pointer"] = serde_json::json!({"x": x, "y": y, "down": down}); } + // what reached its app screens (the keys they took aren't the page's above), given them again in a render + if let Some(apps) = live.app_input() { + e["apps"] = apps; + } // the session's seed and start with its first frame: a render of the log draws the same Math.random and Date if logged.is_empty() { e["session"] = live.session(); diff --git a/apps/targets/src/native.rs b/apps/targets/src/native.rs index e281464b..925fe04d 100644 --- a/apps/targets/src/native.rs +++ b/apps/targets/src/native.rs @@ -64,7 +64,7 @@ pub fn host(args: &[String]) -> Result { let p = Path::new(a); if p.extension().is_some_and(|x| x == "tsx" || x == "ts" || x == "jsx") && p.is_file() { std::fs::create_dir_all(&work).map_err(|e| format!("{}: {e}", work.display()))?; - let (js, css) = raea_runtime_bundle::inproc::app::bundle(p, &raea_io_home::home())?; + let (js, css) = raea_runtime_bundle::inproc::app::bundle(p, &raea_io_home::home(), raea_runtime_bundle::inproc::app::WINDOW)?; let script = work.join(format!("{}.js", p.file_stem().unwrap_or_default().to_string_lossy())); std::fs::write(&script, js).map_err(|e| format!("{}: {e}", script.display()))?; handed.push(script.display().to_string()); diff --git a/apps/targets/src/web.rs b/apps/targets/src/web.rs index 6d349d2d..365c538e 100644 --- a/apps/targets/src/web.rs +++ b/apps/targets/src/web.rs @@ -24,7 +24,7 @@ pub struct Built { pub fn build(app: &Path, dir: &Path) -> Result { let app = raea_io_home::plain(app.canonicalize().map_err(|e| format!("{}: {e}", app.display()))?); std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?; - let (js, imported) = raea_runtime_bundle::inproc::app::bundle(&app, &raea_io_home::home())?; + let (js, imported) = raea_runtime_bundle::inproc::app::bundle(&app, &raea_io_home::home(), raea_runtime_bundle::inproc::app::WINDOW)?; let folder = app.parent().unwrap_or(Path::new(".")); let tw = raea_runtime_bundle::styles::tailwind(folder)?; let built = Built { js: dir.join("app.js"), css: dir.join("app.css") }; diff --git a/bench/render-play/appscreen.json b/bench/render-play/appscreen.json new file mode 100644 index 00000000..2f17747f --- /dev/null +++ b/bench/render-play/appscreen.json @@ -0,0 +1,7 @@ +{ + "0": ["Pointer:480:60"], + "6": ["Pointer:480:260"], + "10": ["Pointer:480:260:down"], + "12": ["Pointer:480:260"], + "21": ["Pointer:480:60"] +} diff --git a/bench/render-play/rows.tsv b/bench/render-play/rows.tsv index 0fca3eb0..9abdf51a 100644 --- a/bench/render-play/rows.tsv +++ b/bench/render-play/rows.tsv @@ -8,7 +8,8 @@ # React on the page with 3D in the world, picking and input replay are all here: the games (pong and platformer on # the page, arena3d's 3D drawn live), bevy-ts' live cases (keys, the pointer, a gamepad, the window and the World's # components through useQuery), the bridge's Engine reads (engine3d: a cube the World moves, React's marker reading it) -# and pick3d (on* on 3D parts: hover, press, leave); an 's own pointer events and (r3f live/pointer) +# and pick3d (on* on 3D parts: hover, press, leave); an 's own pointer events and (r3f live/pointer); +# an app screen's button clicked through the 3D, its own timer on the frame's clock (appscreen: blitz-bevy H12) sdk/react/bench/games/pong/edit.tsx keys:ArrowUp,ArrowDown,,ArrowUp 0.25 0.75 1.25 1.75 sdk/react/bench/games/platformer/edit.tsx keys:ArrowRight,ArrowRight+ArrowUp,ArrowRight, 0.25 0.75 1.25 1.75 sdk/react/bench/games/arena3d/edit.tsx keys:ArrowRight,ArrowUp,ArrowLeft+ArrowUp,ArrowDown, 0.25 0.75 1.25 1.75 @@ -22,3 +23,4 @@ sdk/raea/bench/bevy-ts/cases/text3d-readback.tsx script:sdk/raea/bench/bevy-ts/c bench/bridge/engine3d.tsx - 0.3 0.9 1.25 sdk/raea/bench/bevy/pick3d/edit.tsx script:bench/render-play/pick3d.json 0.1 0.2 0.3 0.4 0.6 0.9 sdk/r3f/test/live/pointer.tsx script:sdk/r3f/test/live/pointer.json 0.2 0.4 0.6 0.9 +sdk/raea/bench/bevy/appscreen/edit.tsx script:bench/render-play/appscreen.json 0.2 0.4 0.6 0.9 diff --git a/docs/manual/reference/sdk/3d.md b/docs/manual/reference/sdk/3d.md index 1124751f..df195378 100644 --- a/docs/manual/reference/sdk/3d.md +++ b/docs/manual/reference/sdk/3d.md @@ -2281,7 +2281,7 @@ The props of [`Screen`](#screen), tabled there. | `background` | `string` | behind the picture ("#000000"). A see-through one ("#ffffff00", "transparent", rgba(..., 0.5)) makes the screen see-through where its page paints nothing, so what's behind it in the scene shows (a flat screen: a layer of a page pulled apart in 3D); an opaque one keeps the screen opaque, which writes depth for DepthOfField | | `brightness` | `number` | a flat screen's (`crt={false}`) light: 1, the default, draws its page at the page's own colours, unlit (scene lights don't touch it); above 1 it glows, and blooms under a . A tone curve on the camera bends it like the rest of the frame: give the scene to keep the page's colours exact. A CRT's is `crt.brightness` | | `src` | `string` | a video to show instead of children: staticFile("clip.mp4"), filling the picture | -| `app` | `string` | an app to show instead of children: a module's path from the composition's folder ("chat.tsx"), its default export a React component given {width, height}. It runs on real react-dom in a document of its own, so npm's React libraries work on it (assistant-ui, Radix; installed with `raea add`). In play the pointer reaches it through the 3D, where it lands on the picture, and a press on it gives it the keys (the game doesn't get them) until a press elsewhere or Escape; in a render it runs on the frame's clock, so the render repeats | +| `app` | `string` | an app to show instead of children: a module's path from the composition's folder ("chat.tsx"), its default export a React component given {width, height}. It runs on react-dom (@raea/ui's) in a Blitz document of its own, so npm's React libraries work on it (assistant-ui, Radix; installed with `raea add`). In play the pointer reaches it through the 3D, where it lands on the picture, and a press on it gives it the keys (the game doesn't get them) until a press elsewhere or Escape; in a render it runs on the frame's clock, so the render repeats | | `children` | `ReactNode` | the nested composition or game | ### `Section` diff --git a/engine/rune/tests/common/mod.rs b/engine/rune/tests/common/mod.rs index 85a27a56..112f878b 100644 --- a/engine/rune/tests/common/mod.rs +++ b/engine/rune/tests/common/mod.rs @@ -12,7 +12,7 @@ fn home() -> PathBuf { /// `file`'s script pub fn script(file: &Path) -> String { - raea_runtime_bundle_lib::app::bundle(file, &home()).unwrap().0 + raea_runtime_bundle_lib::app::bundle(file, &home(), raea_runtime_bundle_lib::app::WINDOW).unwrap().0 } /// tests/apps/`name` diff --git a/runtime/appscreen/Cargo.toml b/runtime/appscreen/Cargo.toml new file mode 100644 index 00000000..fbc04834 --- /dev/null +++ b/runtime/appscreen/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "raea-runtime-appscreen" +version = "0.1.0" +edition = "2024" +license = "FSL-1.1-ALv2" +publish = false +description = "App screens (): a module on @raea/ui's react-dom in a Blitz document of its own at the screen's size, stepped once a frame on the frame's clock with the input routed to it" + +[dependencies] +bevy = { version = "=0.20.0", default-features = false, features = ["std", "default_app", "bevy_render", "bevy_window"] } +blitz-traits = "=0.3.0-beta.4" +keyboard-types = "0.8" +serde_json = "1" +raea-page = { path = "../../engine/page" } +raea-runtime-dom = { path = "../dom" } +raea-runtime-js = { path = "../js" } +raea-runtime-jsengine = { path = "../jsengine" } +raea-runtime-bundle = { path = "../bundle" } +raea-sys-page = { path = "../../systems/page" } +raea-sys-fonts = { path = "../../systems/fonts" } +raea-io-home = { path = "../../io/home" } diff --git a/runtime/appscreen/src/lib.rs b/runtime/appscreen/src/lib.rs new file mode 100644 index 00000000..acb0b53e --- /dev/null +++ b/runtime/appscreen/src/lib.rs @@ -0,0 +1,299 @@ +//! App screens (``, docs/blitz-bevy.feature H12): each one's module, its default export a React +//! component given `{width, height}`, runs on @raea/ui's react-dom in a Blitz document of its own at the screen's +//! size, on a page App of its own (raea-page's BlitzPage), so it is stepped exactly once a frame however often the +//! composition's page is synced or drawn. A step sends the input routed to it as Blitz's pointer, wheel and key +//! events, then runs one update that runs its JS (its frame, timers and commits) with the JS clock held at the +//! frame's time (raea-runtime-jsengine `at_clock`), and paints it. A render and its play step it on the same frames +//! with the same input, so the screen shows the same picture in both. Its picture goes where a nested 's +//! goes: its texture on the shared device in play (Engine::copy_screen), read back in a render. +//! +//! An app's own timers and performance.now follow the frames; its Date doesn't (a module reading the wall clock draws +//! another time in a render). + +use std::path::{Path, PathBuf}; + +use bevy::input::ButtonState; +use bevy::input::mouse::{MouseButton, MouseButtonInput, MouseScrollUnit, MouseWheel}; +use bevy::input::touch::TouchPhase; +use bevy::prelude::*; +use bevy::render::settings::RenderCreation; +use bevy::window::{CursorLeft, CursorMoved, WindowEvent}; +use blitz_traits::events::{BlitzKeyEvent, KeyState, UiEvent}; +use keyboard_types::{Code, Key, Location, Modifiers}; +use raea_page::motion::BlitzPage; +use raea_runtime_dom::host::{self, WindowInput}; + +type R = Result; + +/// What a prebuilt app screen's script is called beside its module (`kit.tsx` -> `kit.tsx.surface.js`): an exported +/// game's, which bundles nothing (apps/game gather.rs). Its CSS is beside it as [`BUNDLED_CSS`]. +pub const BUNDLED: &str = ".surface.js"; +pub const BUNDLED_CSS: &str = ".surface.css"; + +/// One frame's input for an app screen: the pointer on its picture (its px) and whether it's down, the keys held +/// for it (while it has the focus), and the wheel's turns over it (CSS px, down positive, as the DOM's deltaY). +#[derive(Default, Clone, Debug, PartialEq)] +pub struct Input { + pub pointer: Option<(f64, f64, bool)>, + pub keys: Vec, + pub wheel: Vec, +} + +/// What reached the app screens in a frame, from live play's routing (runtime/page screen.rs route) or a render's input +/// log of it: the screen with the focus and the keys held for it, the pointer on a screen (its name, x and y on its +/// picture, whether it's down) and the wheel's turns there (`Wheel:Y DY`). +#[derive(Default, Clone, Debug, PartialEq)] +pub struct Routed { + pub focus: Option, + pub keys: Vec, + pub pointer: Option<(String, f64, f64, bool)>, + pub wheel: Vec, +} + +impl Routed { + /// App screen `name`'s input: the pointer when it's on it (with the wheel turned there), the keys while it has the + /// focus (a DOM key's name; the live wire's marked keys, "Mouse:Right" or "Pad0:South", aren't keys to it). + pub fn input_of(&self, name: &str) -> Input { + let on = self.pointer.as_ref().filter(|p| p.0 == name); + let focused = self.focus.as_deref() == Some(name); + Input { + pointer: on.map(|p| (p.1, p.2, p.3)), + keys: self.keys.iter().filter(|k| focused && (k.chars().count() == 1 || !k.contains(':'))).cloned().collect(), + wheel: self.wheel.iter().filter(|_| on.is_some()).filter_map(|w| w.strip_prefix("Wheel:Y ")?.trim().parse().ok()).collect(), + } + } + + /// As the input log keeps it (`apps`); None when nothing reached an app screen. + pub fn logged(&self) -> Option { + let any = self.focus.is_some() || self.pointer.is_some() || !self.wheel.is_empty(); + any.then(|| serde_json::json!({ "focus": self.focus, "keys": self.keys, "pointer": self.pointer.as_ref().map(|p| serde_json::json!([p.0, p.1, p.2, p.3])), "wheel": self.wheel })) + } + + /// From a log's entry (`logged`'s); nothing reached them when there's none. + pub fn from_log(v: Option<&serde_json::Value>) -> Routed { + let Some(v) = v else { return Routed::default() }; + let texts = |k: &str| v[k].as_array().into_iter().flatten().filter_map(|s| s.as_str().map(String::from)).collect(); + let p = v["pointer"].as_array(); + let pointer = p.and_then(|p| Some((p.first()?.as_str()?.to_string(), p.get(1)?.as_f64()?, p.get(2)?.as_f64()?, p.get(3)?.as_bool()?))); + Routed { focus: v["focus"].as_str().map(String::from), keys: texts("keys"), pointer, wheel: texts("wheel") } + } +} + +/// Set for the one update a step runs the page's JS in (the draw's later updates, while faces and images load, only +/// paint). +#[derive(Resource, Default)] +struct StepNow(bool); + +fn step_once(world: &mut World) { + if std::mem::take(&mut world.resource_mut::().0) { + host::step(world); + } +} + +/// An app screen's document, open. +pub struct AppScreen { + page: BlitzPage, + root: Entity, + /// the module, as the composition names it + pub module: String, + pub size: (u32, u32), + window: WindowInput, + was: Input, + stepped: Option, +} + +/// Frame `f`'s time in ms at `fps`. +fn ms_at(f: i64, fps: f64) -> f64 { + f as f64 * 1000.0 / fps.max(1.0) +} + +impl AppScreen { + /// `module` (a path from `dir`, the composition's folder) opened `w` x `h` at frame `f` of `fps`: its script (see + /// [`code_of`]) mounted on a Blitz page on the device `render` makes (None: Bevy's own), in raea's bundled faces. + pub fn open(dir: &Path, module: &str, w: u32, h: u32, f: i64, fps: f64, render: Option) -> R { + let (code, css) = code_of(&dir.join(module))?; + let fonts = raea_io_home::home().join("core/fonts"); + let mut page = BlitzPage::new_on(w.max(1), h.max(1), Some(&fonts), render)?; + page.not_dumped(); + let app = page.app(); + app.init_resource::(); + app.add_systems(PreUpdate, step_once.in_set(raea_sys_page::PageSystems::Step)); + let world = app.world_mut(); + world.init_resource::(); + world.insert_resource(raea_runtime_js::UiFrame(f.max(0) as u64)); + let root = raea_runtime_jsengine::at_clock(ms_at(f, fps), || host::mount(world, w.max(1), h.max(1), &code, &[], Some(raea_sys_fonts::context(&fonts))))?; + // as raea's page host mounts an app (engine/page/host page.rs) + host::add_style(world, root, "html, body { height: 100%; }"); + if !css.trim().is_empty() { + host::add_style(world, root, &css); + } + page.show(root); + Ok(AppScreen { page, root, module: module.to_string(), size: (w, h), window: WindowInput::default(), was: Input::default(), stepped: None }) + } + + /// Stepped to frame `f` of `fps` with `input`: its events, then one update running its JS on the frame's clock, drawn. + /// Once a frame: stepped again at the same frame, nothing happens. + pub fn step(&mut self, f: i64, fps: f64, input: &Input) { + if self.stepped == Some(f) { + return; + } + self.stepped = Some(f); + let events = self.events(input); + self.was = input.clone(); + let root = self.root; + let page = &mut self.page; + raea_runtime_jsengine::at_clock(ms_at(f, fps), || { + page.at_frame(f, fps); + let world = page.app().world_mut(); + world.insert_resource(raea_runtime_js::UiFrame(f.max(0) as u64)); + for (press, ev) in events { + // each press a click of its own (a double click needs presses closer than a frame apart) + if press { + host::set_clicks(world, root, 1); + } + host::input(world, root, ev); + } + world.resource_mut::().0 = true; + page.draw(); + }); + } + + /// Its picture as last drawn: its texture on the page's device (sRGB-encoded premultiplied RGBA, as a nested + /// 's). + pub fn texture(&self) -> Option { + self.page.texture() + } + + /// Its picture as last drawn, read back: RGBA rows at its size. + pub fn rgba(&mut self) -> R> { + self.page.frame() + } + + /// The events that take the document from the last step's input to `now`'s, as Blitz takes them, each marked + /// when it's a press: the pointer's move, press and release (a release, then a leave, when it goes off the + /// picture), the wheel, the keys let go and the keys pressed. + fn events(&mut self, now: &Input) -> Vec<(bool, UiEvent)> { + let window = Entity::PLACEHOLDER; + let was = self.was.clone(); + let mut out = Vec::new(); + let mut put = |w: &mut WindowInput, ev: WindowEvent, press: bool| { + if let Some(e) = w.event(&ev) { + out.push((press, e)); + } + }; + let button = |state| WindowEvent::MouseButtonInput(MouseButtonInput { button: MouseButton::Left, state, window }); + let held = was.pointer.is_some_and(|p| p.2); + match now.pointer { + Some((x, y, down)) => { + if was.pointer.is_none_or(|(bx, by, _)| bx != x || by != y) { + put(&mut self.window, WindowEvent::CursorMoved(CursorMoved { window, position: Vec2::new(x as f32, y as f32), delta: None }), false); + } + if down && !held { + put(&mut self.window, button(ButtonState::Pressed), true); + } + if !down && held { + put(&mut self.window, button(ButtonState::Released), false); + } + } + None if was.pointer.is_some() => { + if held { + put(&mut self.window, button(ButtonState::Released), false); + } + put(&mut self.window, WindowEvent::CursorLeft(CursorLeft { window }), false); + } + None => {} + } + for dy in &now.wheel { + // Blitz's wheel is Bevy's way up (raea-runtime-dom gives the DOM -y) + put(&mut self.window, WindowEvent::MouseWheel(MouseWheel { unit: MouseScrollUnit::Pixel, x: 0.0, y: -(*dy as f32), window, phase: TouchPhase::Moved }), false); + } + let mods = mods_of(&now.keys); + for k in was.keys.iter().filter(|k| !now.keys.contains(k)) { + out.push((false, UiEvent::KeyUp(key_event(k, mods, false)))); + } + for k in now.keys.iter().filter(|k| !was.keys.contains(k)) { + out.push((false, UiEvent::KeyDown(key_event(k, mods, true)))); + } + out + } +} + +/// The modifiers the held keys (by their DOM names) make. +fn mods_of(keys: &[String]) -> Modifiers { + let mut m = Modifiers::empty(); + for (name, bit) in [("Shift", Modifiers::SHIFT), ("Control", Modifiers::CONTROL), ("Alt", Modifiers::ALT), ("Meta", Modifiers::META)] { + if keys.iter().any(|k| k == name) { + m |= bit; + } + } + m +} + +/// Key `name` (the DOM's: a character, or a named key like "Enter") pressed or let go: a character types itself +/// unless Control or Meta is held. +fn key_event(name: &str, modifiers: Modifiers, down: bool) -> BlitzKeyEvent { + let name = if name == "Space" { " " } else { name }; + let key = name.parse::().unwrap_or_else(|_| Key::Character(name.into())); + let types = matches!(key, Key::Character(_)) && !modifiers.intersects(Modifiers::CONTROL | Modifiers::META); + BlitzKeyEvent { + key, + code: Code::Unidentified, + modifiers, + location: Location::Standard, + is_auto_repeating: false, + is_composing: false, + state: if down { KeyState::Pressed } else { KeyState::Released }, + text: (types && down).then(|| blitz_traits::SmolStr::new(name)), + } +} + +/// The module's script and CSS: its prebuilt bundle beside it (an exported game's, [`BUNDLED`]), or [`build`] now. +pub fn code_of(file: &Path) -> R<(String, String)> { + let pre = PathBuf::from(format!("{}{BUNDLED}", file.display())); + if pre.is_file() { + let js = std::fs::read_to_string(&pre).map_err(|e| format!("{}: {e}", pre.display()))?; + let css = std::fs::read_to_string(format!("{}{BUNDLED_CSS}", file.display())).unwrap_or_default(); + return Ok((js, css)); + } + build(file) +} + +/// `file` bundled as an app screen (raea-runtime-bundle-lib's app bundle, its default export given the screen's size): +/// its script, and its CSS, what its modules import, then Tailwind's for the classes its folder uses, as `raea app` +/// builds an app's (apps/targets web.rs). +pub fn build(file: &Path) -> R<(String, String)> { + if !file.is_file() { + return Err(format!("{}: no such module", file.display())); + } + let (js, imported) = raea_runtime_bundle::inproc::app::bundle(file, &raea_io_home::home(), raea_runtime_bundle::inproc::app::SCREEN)?; + let tw = raea_runtime_bundle::styles::tailwind(file.parent().unwrap_or(Path::new(".")))?; + Ok((js, format!("{imported}\n{tw}"))) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn keys_type_characters_but_not_with_control() { + let a = key_event("a", Modifiers::empty(), true); + assert_eq!(a.text.as_deref(), Some("a")); + assert!(key_event("a", Modifiers::CONTROL, true).text.is_none()); + assert!(key_event("a", Modifiers::empty(), false).text.is_none()); + let enter = key_event("Enter", Modifiers::empty(), true); + assert!(matches!(enter.key, Key::Named(_)) && enter.text.is_none()); + assert_eq!(key_event("Space", Modifiers::empty(), true).text.as_deref(), Some(" ")); + assert_eq!(mods_of(&["Shift".into(), "a".into()]), Modifiers::SHIFT); + } + + #[test] + fn the_logged_input_comes_back_the_same() { + let r = Routed { focus: Some("kit".into()), keys: vec!["a".into(), "Pad0:South".into()], pointer: Some(("kit".into(), 12.5, 40.0, true)), wheel: vec!["Wheel:Y 120".into()] }; + let v = r.logged().unwrap(); + let back = Routed::from_log(Some(&v)); + assert_eq!(back, r); + assert_eq!(back.input_of("kit"), Input { pointer: Some((12.5, 40.0, true)), keys: vec!["a".into()], wheel: vec![120.0] }); + assert_eq!(back.input_of("other"), Input::default()); + assert_eq!(Routed::from_log(None).logged(), None); + } +} diff --git a/runtime/bundle/lib/src/app.rs b/runtime/bundle/lib/src/app.rs index f9d116e7..a7768233 100644 --- a/runtime/bundle/lib/src/app.rs +++ b/runtime/bundle/lib/src/app.rs @@ -5,10 +5,10 @@ use std::path::Path; /// The entry every app is mounted by: the DOM installed before the app's module runs, its default export mounted at -/// the root (an app that mounts itself with createRoot needs no default export) -fn entry(app: &Path) -> String { +/// the root with `props` (JS; an app's none) (an app that mounts itself with createRoot needs no default export) +fn entry(app: &Path, props: &str) -> String { format!( - "import \"@raea/ui/dom-globals\";\nimport {{ createElement }} from \"react\";\nimport {{ mount }} from \"@raea/ui/react\";\nimport * as M from {:?};\nconst App = (M as any).default;\nif (typeof App === \"function\") mount(createElement(App), 0, {{ dom: true }});\n", + "import \"@raea/ui/dom-globals\";\nimport {{ createElement }} from \"react\";\nimport {{ mount }} from \"@raea/ui/react\";\nimport * as M from {:?};\nconst App = (M as any).default;\nif (typeof App === \"function\") mount(createElement(App, {props}), 0, {{ dom: true }});\n", app.display().to_string() ) } @@ -36,8 +36,17 @@ fn ui(home: &Path) -> serde_json::Value { m.into() } -/// `app` bundled against `home`: its script and the CSS its modules import -pub fn bundle(app: &Path, home: &Path) -> Result<(String, String), String> { +/// What an app's default export is given, for `bundle`: nothing, as an app in a window of its own is (it reads the +/// window) +pub const WINDOW: &str = "null"; + +/// What an app screen's module (``, docs/blitz-bevy.feature H12) is given: `{width, height}`, the +/// screen's size (its document's, so one bundle serves a screen of any size) +pub const SCREEN: &str = "{ width: innerWidth, height: innerHeight }"; + +/// `app` bundled against `home`, its default export mounted with `props` (WINDOW or SCREEN): its script and the CSS its +/// modules import +pub fn bundle(app: &Path, home: &Path, props: &str) -> Result<(String, String), String> { // canonical, and on Windows without canonicalize()'s verbatim \\?\ prefix: Rolldown can't resolve an entry that // imports a path written with it (T1f: the Windows release check, 11-07) let app = raea_io_home::plain(app.canonicalize().map_err(|e| format!("{}: {e}", app.display()))?); @@ -49,7 +58,7 @@ pub fn bundle(app: &Path, home: &Path) -> Result<(String, String), String> { let _ = std::fs::remove_dir_all(&work); std::fs::create_dir_all(&work).map_err(|e| format!("{}: {e}", work.display()))?; let entry_file = work.join("entry.tsx"); - let wrote = std::fs::write(&entry_file, entry(&app)).and_then(|_| std::fs::write(work.join("tsconfig.json"), tsconfig(home))); + let wrote = std::fs::write(&entry_file, entry(&app, props)).and_then(|_| std::fs::write(work.join("tsconfig.json"), tsconfig(home))); let r = wrote.map_err(|e| e.to_string()).and_then(|_| { let opts = serde_json::json!({ "script": {"js": home.join("sdk/react/js"), "motion": home.join("sdk/raea/src"), "shell": home.join("sdk/shell/src/index.ts"), "aliases": ui(home), "stubs": []}, diff --git a/runtime/bundle/src/fresh.rs b/runtime/bundle/src/fresh.rs index 8fe4c109..59b320a8 100644 --- a/runtime/bundle/src/fresh.rs +++ b/runtime/bundle/src/fresh.rs @@ -25,28 +25,6 @@ fn spec(p: &Path) -> String { dunce::simplified(p).display().to_string() } -/// A react-dom page's bundle (a surface): the module at `src` (its default -/// export a React component) rendered by react-dom into #root with the page's width and height, again on each resize, -/// bundled with its packages (the project's node_modules, raea-io-npm's) into `out`. The bundle's text. A legacy-app panel's -/// and a ``'s. -pub fn bundle_surface(src: &Path, out: &Path) -> R { - let entry = format!( - r#"// raea writes this (motion.rs bundle_surface): {src} on react-dom, at the surface's size, again on each resize. -import {{ createElement }} from "react"; -import {{ createRoot }} from "react-dom/client"; -import Page from {src:?}; - -const root = createRoot(document.getElementById("root")!); -const draw = () => root.render(createElement(Page as any, {{ width: innerWidth, height: innerHeight }})); -addEventListener("resize", draw); -draw(); -"#, - src = spec(src) - ); - bundle_entry(&entry, out, &[])?; - std::fs::read_to_string(out).map_err(|e| format!("{}: {e}", out.display())) -} - /// `entry` (TypeScript source) bundled into `out` by raea-bundle's library (engine/bundle/src/script.rs's resolution); `extra` /// are bundle.ts's flags (--define K=V, --alias SPEC=PATH, --node-stub a,b). pub fn bundle_entry(entry: &str, out: &Path, extra: &[&str]) -> R<()> { diff --git a/runtime/export/src/render.rs b/runtime/export/src/render.rs index dfb65c64..05554466 100644 --- a/runtime/export/src/render.rs +++ b/runtime/export/src/render.rs @@ -35,6 +35,10 @@ pub fn render(args: &[String]) -> R { // the edit's words.json and perception.json beside it, as play loads them (a render without the flags lost the // captions its words drive; the corpus found 10 edits so); a flag still wins let mut o = comp_opts(args)?; + // an input log's `apps`, what reached the app screens in the play it came from, given them again frame by frame + // (docs/blitz-bevy.feature H12): the render draws what was played + let apps: std::collections::BTreeMap = o.input.as_ref().and_then(|v| v.as_object()).into_iter().flatten().filter_map(|(f, v)| Some((f.parse().ok()?, v.get("apps")?.clone()))).collect(); + *raea_runtime_page::screen::APP_LOG.lock().unwrap() = (!apps.is_empty()).then_some(apps); for (slot, file) in [(&mut o.words, "words.json"), (&mut o.perception, "perception.json")] { if slot.is_none() && dir.join(file).exists() { let v = json_file(&dir.join(file).display().to_string())?; diff --git a/runtime/jsengine/src/lib.rs b/runtime/jsengine/src/lib.rs index ce74bf36..94bfc11d 100644 --- a/runtime/jsengine/src/lib.rs +++ b/runtime/jsengine/src/lib.rs @@ -36,11 +36,21 @@ pub type Native = Box Result>; thread_local! { static T0: std::time::Instant = std::time::Instant::now(); + static HELD: std::cell::Cell> = const { std::cell::Cell::new(None) }; } -/// ms since this thread first asked: the clock a host's JS timers run on. +/// ms since this thread first asked: the clock a host's JS timers run on (or the time `at_clock` holds it at). pub fn now_ms() -> f64 { - T0.with(|t| t.elapsed().as_secs_f64() * 1e3) + HELD.with(|h| h.get()).unwrap_or_else(|| T0.with(|t| t.elapsed().as_secs_f64() * 1e3)) +} + +/// `f` run with this thread's JS clock held at `ms`: a root stepped on a composition's frames (an app screen, frame +/// over fps) sees the same timers and performance.now in a render and its play, so the render repeats. +pub fn at_clock(ms: f64, f: impl FnOnce() -> T) -> T { + let was = HELD.with(|h| h.replace(Some(ms))); + let out = f(); + HELD.with(|h| h.set(was)); + out } /// A native answering `now_ms()`, a host's `now` for its JS. diff --git a/runtime/live/src/lib.rs b/runtime/live/src/lib.rs index 93bfc77d..17471bab 100644 --- a/runtime/live/src/lib.rs +++ b/runtime/live/src/lib.rs @@ -19,6 +19,8 @@ mod pick3d; pub use pick3d::want_wire; pub use sound::warm_audio; mod draw; +mod react; +pub use react::needs_react; type R = Result; @@ -58,6 +60,9 @@ pub struct Live { frame: i64, /// the gamepads' marked keys the last frame was given beside its input ("Pad0:South"), for its input log pads: Vec, + /// the keys an app screen took from the last frame (the page didn't get them), left out of its input log, which + /// keeps them under `apps` (`app_input`) + taken: Vec, /// the pointer on the page's elements, as the last frame left it (pointer.rs) pointer: pointer::PagePointer, /// generated sounds' files, by what they were made from (looked up once) @@ -202,6 +207,9 @@ impl Live { self.lag = (lag, std::time::Instant::now()); self.frame = f; self.pads.clear(); + self.taken.clear(); + // the app screens step to this frame at the page's next sync (screen.rs step_all) + self.page.set_app_frame(f, self.fps); let t0 = std::time::Instant::now(); if self.baked.is_some() { // a baked timeline: its recorded commits, no JS; the clips it cuts to soon opened ahead; the page styled at @@ -224,6 +232,9 @@ impl Live { Some(e) => Some(raea_runtime_page::screen::route(&mut self.page, e, keys, pointer, (self.w, self.h))), None => None, }; + if let Some(r) = &routed { + self.taken = keys.iter().filter(|k| !r.contains(k)).cloned().collect(); + } let keys = routed.as_deref().unwrap_or(keys); // the pointer on the page's elements: their on* handlers, before the frame's step (pointer.rs) if let Some(p) = pointer { @@ -317,29 +328,6 @@ fn components_use_config(dir: &Path) -> bool { std::fs::read_dir(dir.join("components")).into_iter().flatten().flatten().any(|e| std::fs::read_to_string(e.path()).is_ok_and(|t| t.contains("useVideoConfig("))) } -/// Whether the edit's own sources (its folder, not node_modules) import React Three Fiber (`raea/r3f`, -/// `@react-three/...`): R3F runs a reconciler of its own on React, which live play's runtime (sdk/react/js/lite) isn't, -/// so such a play keeps React, as RAEA_LIVE_REACT=1 does (docs/r3f.feature R3F27) -pub fn needs_react(dir: &Path) -> bool { - fn walk(d: &Path, depth: usize) -> bool { - let Ok(rd) = std::fs::read_dir(d) else { return false }; - rd.flatten().any(|e| { - let name = e.file_name(); - let name = name.to_string_lossy(); - let p = e.path(); - if name.starts_with('.') || name == "node_modules" || name == "target" { - false - } else if p.is_dir() { - depth < 4 && walk(&p, depth + 1) - } else { - [".tsx", ".ts", ".jsx", ".js"].iter().any(|x| name.ends_with(x)) - && std::fs::read_to_string(&p).is_ok_and(|t| t.contains("raea/r3f") || t.contains("@react-three/")) - } - }) - } - walk(dir, 0) -} - /// What a composition's sound is made of: its bundle and every file in its project (path, size, time), hashed, so a /// sound worked out from them (the mastered track) is found again until one of them changes. fn sound_key(js: &Path, dir: &Path) -> u64 { @@ -389,7 +377,13 @@ impl Live { /// The keys the last frame stepped was given: `keys` and the gamepads marked beside them, as its input log records /// them, so a render of the log gives the game the same pads (bench/render-play's live-gamepad). pub fn played_keys(&self, keys: &[String]) -> Vec { - keys.iter().chain(&self.pads).cloned().collect() + keys.iter().chain(&self.pads).filter(|k| !self.taken.contains(k)).cloned().collect() + } + + /// What reached the app screens in the last frame stepped, as its input log keeps it (`apps`): a render of the + /// log gives them the same (runtime/page/src/screen.rs). + pub fn app_input(&self) -> Option { + raea_runtime_page::screen::logged(&self.page) } } @@ -475,20 +469,3 @@ mod frame_event_tests { } } } - -#[cfg(test)] -mod needs_react_tests { - #[test] - fn an_edit_importing_r3f_keeps_react_and_one_whose_packages_do_doesnt() { - let d = std::env::temp_dir().join(format!("raea-needs-react-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&d); - std::fs::create_dir_all(d.join("node_modules/drei")).unwrap(); - std::fs::write(d.join("edit.tsx"), "import { Composition } from \"raea\";").unwrap(); - std::fs::write(d.join("node_modules/drei/index.js"), "import \"@react-three/fiber\";").unwrap(); - assert!(!super::needs_react(&d)); - std::fs::create_dir_all(d.join("src/scene")).unwrap(); - std::fs::write(d.join("src/scene/ball.tsx"), "import { R3F } from \"raea/r3f\";").unwrap(); - assert!(super::needs_react(&d)); - let _ = std::fs::remove_dir_all(&d); - } -} diff --git a/runtime/live/src/open.rs b/runtime/live/src/open.rs index 318a278d..a1940840 100644 --- a/runtime/live/src/open.rs +++ b/runtime/live/src/open.rs @@ -65,7 +65,7 @@ impl Live { let (w, h) = if responsive { (stage.0.max(1), stage.1.max(1)) } else { (cw, ch) }; let page = raea_runtime_page::BevyPage::new(w, h, &dir)?; let brand = raea_runtime_react::scene::page_css(&dir, w, &js); - let mut live = Live { app, dir: dir.clone(), page, brand, dirty: true, mixer: None, mixer_failed: false, lag: (0.0, std::time::Instant::now()), w, h, fps, frames, responsive, js, master: None, master_wav: None, game_master: None, game_gain: None, scene_rx: None, scene: None, live3d: None, frame: 0, pads: Vec::new(), pointer: Default::default(), made: Default::default(), sound_buf: Vec::new(), holders: Vec::new(), baked: None, applied: -1, session: new_session() }; + let mut live = Live { app, dir: dir.clone(), page, brand, dirty: true, mixer: None, mixer_failed: false, lag: (0.0, std::time::Instant::now()), w, h, fps, frames, responsive, js, master: None, master_wav: None, game_master: None, game_gain: None, scene_rx: None, scene: None, live3d: None, frame: 0, pads: Vec::new(), taken: Vec::new(), pointer: Default::default(), made: Default::default(), sound_buf: Vec::new(), holders: Vec::new(), baked: None, applied: -1, session: new_session() }; if responsive { live.send("raea-size", json!({"w": w, "h": h})); } @@ -117,7 +117,7 @@ impl Live { raea_runtime_react::scene::one_thread(&mut app); let page = raea_runtime_page::BevyPage::new(w, h, &dir)?; let brand = raea_runtime_react::scene::page_css(&dir, w, &js); - let mut live = Live { app, dir: dir.clone(), page, brand, dirty: true, mixer: None, mixer_failed: false, lag: (0.0, std::time::Instant::now()), w, h, fps, frames, responsive: false, js, master: None, master_wav: None, game_master: None, game_gain: None, scene_rx: None, scene: None, live3d: None, frame: 0, pads: Vec::new(), pointer: Default::default(), made: Default::default(), sound_buf: Vec::new(), holders: Vec::new(), baked: None, applied: -1, session: new_session() }; + let mut live = Live { app, dir: dir.clone(), page, brand, dirty: true, mixer: None, mixer_failed: false, lag: (0.0, std::time::Instant::now()), w, h, fps, frames, responsive: false, js, master: None, master_wav: None, game_master: None, game_gain: None, scene_rx: None, scene: None, live3d: None, frame: 0, pads: Vec::new(), taken: Vec::new(), pointer: Default::default(), made: Default::default(), sound_buf: Vec::new(), holders: Vec::new(), baked: None, applied: -1, session: new_session() }; for b in &baked.init { live.app.world().resource::().feed(b.clone()); } diff --git a/runtime/live/src/react.rs b/runtime/live/src/react.rs new file mode 100644 index 00000000..bfb6a1d9 --- /dev/null +++ b/runtime/live/src/react.rs @@ -0,0 +1,44 @@ +//! Which live plays keep React (docs/r3f.feature R3F27). + +use std::path::Path; + +/// Whether the edit's own sources (its folder, not node_modules) import React Three Fiber (`raea/r3f`, +/// `@react-three/...`): R3F runs a reconciler of its own on React, which live play's runtime (sdk/react/js/lite) isn't, +/// so such a play keeps React, as RAEA_LIVE_REACT=1 does (docs/r3f.feature R3F27) +pub fn needs_react(dir: &Path) -> bool { + fn walk(d: &Path, depth: usize) -> bool { + let Ok(rd) = std::fs::read_dir(d) else { return false }; + rd.flatten().any(|e| { + let name = e.file_name(); + let name = name.to_string_lossy(); + let p = e.path(); + if name.starts_with('.') || name == "node_modules" || name == "target" { + false + } else if p.is_dir() { + depth < 4 && walk(&p, depth + 1) + } else { + [".tsx", ".ts", ".jsx", ".js"].iter().any(|x| name.ends_with(x)) + && std::fs::read_to_string(&p).is_ok_and(|t| t.contains("raea/r3f") || t.contains("@react-three/")) + } + }) + } + walk(dir, 0) +} + + +#[cfg(test)] +mod needs_react_tests { + #[test] + fn an_edit_importing_r3f_keeps_react_and_one_whose_packages_do_doesnt() { + let d = std::env::temp_dir().join(format!("raea-needs-react-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(d.join("node_modules/drei")).unwrap(); + std::fs::write(d.join("edit.tsx"), "import { Composition } from \"raea\";").unwrap(); + std::fs::write(d.join("node_modules/drei/index.js"), "import \"@react-three/fiber\";").unwrap(); + assert!(!super::needs_react(&d)); + std::fs::create_dir_all(d.join("src/scene")).unwrap(); + std::fs::write(d.join("src/scene/ball.tsx"), "import { R3F } from \"raea/r3f\";").unwrap(); + assert!(super::needs_react(&d)); + let _ = std::fs::remove_dir_all(&d); + } +} diff --git a/runtime/page/Cargo.toml b/runtime/page/Cargo.toml index 238e4542..7e4a2a9f 100644 --- a/runtime/page/Cargo.toml +++ b/runtime/page/Cargo.toml @@ -8,6 +8,7 @@ description = "A composition's page in Bevy: React's elements laid out and paint [dependencies] raea-runtime-view = { path = "../view" } +raea-runtime-appscreen = { path = "../appscreen" } raea-sys-frame = { path = "../../systems/frame" } raea-io-assets = { path = "../../io/assets" } raea-io-net = { path = "../../io/net" } diff --git a/runtime/page/src/bevy_page.rs b/runtime/page/src/bevy_page.rs index cbbacfdc..f3d9d881 100644 --- a/runtime/page/src/bevy_page.rs +++ b/runtime/page/src/bevy_page.rs @@ -98,11 +98,7 @@ impl BevyPage { } } crate::screen::retire(&mut self.apps, &self.app_screens); - for (name, _, _, app) in &self.app_screens { - if self.apps.failed.insert(name.clone(), app.clone()).is_none() { - eprintln!("raea: screen {name:?}: its app {app:?} isn't drawn yet: app screens move to raea-runtime-js roots of their own (ts-native P27)"); - } - } + crate::screen::step_all(&mut self.apps, &self.app_screens, &self.dir); self.page.place_videos(videos) } @@ -147,12 +143,21 @@ impl BevyPage { /// The page and each 's picture drawn and read back: the page's premultiplied RGBA, and each screen's name, /// size and RGBA. pub fn frame_all(&mut self) -> R<(Vec, Vec<(String, u32, u32, Vec)>)> { - self.page.frame_all() + let (px, mut pics) = self.page.frame_all()?; + pics.extend(crate::screen::pictures(&mut self.apps)?); + Ok((px, pics)) } - /// The page drawn at frame `f` of a composition at `fps`: its CSS animations and transitions where they are then. + /// The page drawn at frame `f` of a composition at `fps`: its CSS animations and transitions where they are then, + /// and its app screens stepped there. pub fn at_frame(&mut self, f: i64, fps: f64) { self.page.at_frame(f, fps); + self.set_app_frame(f, fps); + } + + /// The frame the app screens step to at the next sync (live play, whose page keeps its own CSS clock). + pub fn set_app_frame(&mut self, f: i64, fps: f64) { + self.apps.frame = Some((f, fps)); } /// The page drawn and read back: premultiplied RGBA rows, `w` x `h`. @@ -276,9 +281,10 @@ impl BevyPage { &mut self.apps } - /// Each 's picture as last drawn, by name: its texture on the shared device. + /// Each 's picture as last drawn, by name: its texture on the shared device (an app screen's too). pub fn screen_textures(&self) -> Vec<(String, wgpu::Texture)> { - self.page.sub_textures().into_iter().map(|(k, t, _, _)| (k, (*t).clone())).collect() + let apps = crate::screen::textures(&self.apps); + self.page.sub_textures().into_iter().map(|(k, t, _, _)| (k, t)).chain(apps).map(|(k, t)| (k, (*t).clone())).collect() } /// The page's picture as last drawn, on the shared device (`Rgba8UnormSrgb`, premultiplied). diff --git a/runtime/page/src/screen.rs b/runtime/page/src/screen.rs index d4e6a188..93752f55 100644 --- a/runtime/page/src/screen.rs +++ b/runtime/page/src/screen.rs @@ -5,28 +5,40 @@ //! (engine/render/src/engine/screen.rs). A render's screen equals its play's, and a game on a screen plays live with the //! player's input (the nested game reads the same useInput). //! -//! An app screen (``) shows a page of its own instead: the module's component on @raea/ui's -//! react-dom in a raea-runtime-js root of its own. In live play the pointer reaches it through the 3D (`route`: the camera's -//! ray to the screen, Engine::screen_at): a press on an app screen gives it the keys, which the game then doesn't get, -//! until a press anywhere else or Escape. +//! An app screen (``, docs/blitz-bevy.feature H12) shows a page of its own instead: the module's +//! component on @raea/ui's react-dom in a Blitz document of its own (raea-runtime-appscreen), stepped once a frame +//! (`step_all`, from the page's sync) and drawn into its picture, which goes to the 3D as a nested screen's does. In +//! live play the pointer reaches it through the 3D (`route`: the camera's ray to the screen, Engine::screen_at): a +//! press on an app screen gives it the keys, which the game then doesn't get, until a press anywhere else or Escape. +//! What reached the app screens each frame goes in the input log (Routed::logged), and a render of the log gives them +//! the same (`APP_LOG`), so the render draws what was played. -/// What an app's prebuilt bundle is called beside its module (`chat.tsx` -> `chat.tsx.surface.js`): an exported game's -/// (game.rs gather). -pub const BUNDLED: &str = ".surface.js"; +use std::collections::{BTreeMap, HashMap}; -/// The page's app screens: the input live play routed to them for their next step. +use raea_runtime_appscreen::{AppScreen, Routed}; +use serde_json::Value; + +/// What an app's prebuilt bundle is called beside its module (`chat.tsx` -> `chat.tsx.surface.js`, its CSS +/// `chat.tsx.surface.css`): an exported game's (game.rs gather). +pub use raea_runtime_appscreen::{BUNDLED, BUNDLED_CSS, build as build_app}; + +/// A render's input log, by frame: what reached the app screens in the play it logged (its `apps`), given them again +/// in place of live play's routing. Set by `raea render --input`; None in play. +pub static APP_LOG: std::sync::Mutex>> = std::sync::Mutex::new(None); + +/// The page's app screens: their documents, and the input live play routed to them for their next step. #[derive(Default)] pub(crate) struct Apps { + /// each app screen's document, by name + docs: HashMap, + /// the frame and fps the next step is at (BevyPage::at_frame, set_app_frame) + pub(crate) frame: Option<(i64, f64)>, /// screens whose app didn't open, with the module (said once) - pub(crate) failed: std::collections::HashMap, - /// the app screen the keys go to (the one last pressed on), and the keys held for it - focus: Option, - keys: Vec, - /// where the pointer is on an app screen (its picture's pixels) and whether it's down - pointer: Option<(String, f64, f64, bool)>, + pub(crate) failed: HashMap, + /// what reaches them this frame: the focus and its keys, the pointer and the wheel + pub(crate) r: Routed, + /// whether the pointer was down last frame (a press is its going down) down: bool, - /// the wheel's turns this frame (`Wheel:Y DY`), for the app screen under the pointer - wheel: Vec, } /// Live play's input split between the page and its app screens: the pointer where it lands on an app screen (the @@ -37,55 +49,101 @@ pub fn route(page: &mut crate::BevyPage, engine: &mut raea_render::engine::Engin let apps = page.app_screens(); let a = page.apps(); if apps.is_empty() { - a.focus = None; - a.pointer = None; + a.r.focus = None; + a.r.pointer = None; return keys.to_vec(); } let hit = pointer.and_then(|(x, y, _)| engine.screen_at(x / size.0.max(1) as f64, y / size.1.max(1) as f64)); let on = hit.and_then(|(name, u, v)| apps.iter().find(|s| s.0 == name).map(|s| (name, u * s.1 as f64, v * s.2 as f64))); let down = pointer.is_some_and(|p| p.2); if down && !a.down { - a.focus = on.as_ref().map(|o| o.0.clone()); + a.r.focus = on.as_ref().map(|o| o.0.clone()); } a.down = down; - a.pointer = on.map(|(name, x, y)| (name, x, y, down)); + let r = &mut a.r; + r.pointer = on.map(|(name, x, y)| (name, x, y, down)); // the wheel over an app screen scrolls it, and the page doesn't get it - a.wheel.clear(); - let keys: Vec = match &a.pointer { - Some(_) => keys.iter().filter(|k| !(k.starts_with("Wheel:Y ") && { a.wheel.push(k.to_string()); true })).cloned().collect(), + r.wheel.clear(); + let keys: Vec = match &r.pointer { + Some(_) => keys.iter().filter(|k| !(k.starts_with("Wheel:Y ") && { r.wheel.push(k.to_string()); true })).cloned().collect(), None => keys.to_vec(), }; - let keys = &keys[..]; let (mine, page_keys): (Vec, Vec) = keys.iter().cloned().partition(|k| !(k.starts_with("feed:") || k.starts_with("action:") || k.starts_with("space:"))); - match &a.focus { + match &r.focus { Some(_) if mine.iter().any(|k| k == "Escape") => { - a.focus = None; - a.keys.clear(); + r.focus = None; + r.keys.clear(); page_keys } Some(_) => { - a.keys = mine; + r.keys = mine; page_keys } None => { - a.keys.clear(); - keys.to_vec() + r.keys.clear(); + keys } } } +/// What reached the page's app screens this frame, for the input log (Routed::logged): None when nothing did. +pub fn logged(page: &crate::BevyPage) -> Option { + page.apps.r.logged() +} + /// The app screens that went let go of: their focus, pointer and failures forgotten, so a screen back again opens /// fresh. `screens`: each app screen now, its name, size and module. pub(crate) fn retire(a: &mut Apps, screens: &[(String, u32, u32, String)]) { let now = |name: &str, app: &str| screens.iter().any(|s| s.0 == name && s.3 == app); a.failed.retain(|name, app| now(name, app)); + // a document goes with its screen, or when the screen shows another module or size (opened again fresh) + a.docs.retain(|name, d| screens.iter().any(|s| &s.0 == name && s.3 == d.module && (s.1, s.2) == d.size)); let shown = |name: &str| screens.iter().any(|s| s.0 == name); - if a.focus.as_deref().is_some_and(|f| !shown(f)) { - a.focus = None; - a.keys.clear(); + if a.r.focus.as_deref().is_some_and(|f| !shown(f)) { + a.r.focus = None; + a.r.keys.clear(); } - if a.pointer.as_ref().is_some_and(|p| !shown(&p.0)) { - a.pointer = None; + if a.r.pointer.as_ref().is_some_and(|p| !shown(&p.0)) { + a.r.pointer = None; } } +/// Each app screen stepped to the page's frame with its input (`dir`: the composition's folder, its modules' from), +/// its document opened the first frame it's seen; one that won't open is said once and left blank. +pub(crate) fn step_all(a: &mut Apps, screens: &[(String, u32, u32, String)], dir: &std::path::Path) { + let (f, fps) = a.frame.unwrap_or((0, 30.0)); + if let Some(log) = APP_LOG.lock().unwrap().as_ref() { + a.r = Routed::from_log(log.get(&f)); + } + for (name, w, h, module) in screens { + if a.failed.contains_key(name) || *w == 0 || *h == 0 { + continue; + } + if !a.docs.contains_key(name) { + match AppScreen::open(dir, module, *w, *h, f, fps, Some(raea_runtime_view::render_creation())) { + Ok(d) => { + a.docs.insert(name.clone(), d); + } + Err(e) => { + eprintln!("raea: screen {name:?}: its app {module:?} didn't open: {e}"); + a.failed.insert(name.clone(), module.clone()); + continue; + } + } + } + let input = a.r.input_of(name); + if let Some(d) = a.docs.get_mut(name) { + d.step(f, fps, &input); + } + } +} + +/// Each app screen's picture as last drawn, read back: its name, size and RGBA. +pub(crate) fn pictures(a: &mut Apps) -> Result)>, String> { + a.docs.iter_mut().map(|(name, d)| Ok((name.clone(), d.size.0, d.size.1, d.rgba()?))).collect() +} + +/// Each app screen's picture as last drawn: its texture on the shared device. +pub(crate) fn textures(a: &Apps) -> Vec<(String, bevy::render::render_resource::Texture)> { + a.docs.iter().filter_map(|(name, d)| Some((name.clone(), d.texture()?))).collect() +} diff --git a/sdk/guides/craft/motion/apps.md b/sdk/guides/craft/motion/apps.md index ed3ee1ae..356195e7 100644 --- a/sdk/guides/craft/motion/apps.md +++ b/sdk/guides/craft/motion/apps.md @@ -10,25 +10,28 @@ the native one (Rune) chosen by `export const app = { target }`, is `raea app ru - Packages: `raea add @assistant-ui/react` writes `package.json` and `raea.lock` and lays out `node_modules`. raea's own package manager does it; no Node, npm or bun is needed, and install scripts never run. - A React library that needs a real DOM (assistant-ui, Radix, shadcn) runs as an app: a module whose default export is - a component taking `{ width, height }`, running on real react-dom in a document of its own. Place it: + a component taking `{ width, height }`, running on react-dom in a document of its own (@raea/ui's, on a Blitz page: + what raea feed runs on), with the CSS its modules import and Tailwind's for its folder. Place it: - on a screen in 3D: `` (beside the 3D, or as a Model's `screens` entry). The path is relative to the composition's folder; - as an app panel or tool: `export const surface = "dom"` in the panel's file. - Input in live play: the pointer reaches an app screen through the 3D (where it lands on the picture). Pressing on the screen gives it the keys, and the game gets none until a press elsewhere or Escape. -- Renders: an app runs on the frame clock (timers, `Date`, `performance.now`, `requestAnimationFrame` follow the frame), - so a render repeats frame for frame. -- Exported games: `export-game` bundles each app with its packages. The player must be built with them: - `cargo build --profile game -p raea-app-game --bin raea-game --features player,surfaces` (+6.7 MB); export refuses - an app screen with a player built without. +- Renders: an app screen is stepped once a frame on the frame clock (its timers and `performance.now` follow the frame), + so a render repeats frame for frame. A render of a play's input log (`render --input`) gives it what reached it in + the play, so the render draws what was played. +- Exported games: `export-game` bundles each app with its packages (`chat.tsx.surface.js` and `.surface.css`), and the + player draws it on its own Blitz page. Not yet: - An app as a flat 2D layer of a composition. It can be a 3D `` or an app panel only; for a flat look, put a `crt={false}` screen square to the camera. - A composition itself importing a DOM library. Only an app (above) can; the composition's own React is raea's lite runtime. -- Input to an app in a render: `render --input` replays the page's input, but an app screen gets none. Script what it - should show inside the app, on its own timers. +- `Date` in an app screen: it reads the wall clock, so a render shows another time than its play. Use timers or + `performance.now`. +- A drag that leaves an app screen: it ends at the edge (a browser's goes on). A double click on one. +- The page host's desktop calls in an app screen (the clipboard, links, notifications). - Anything in front of a screen blocking its clicks: the ray hits the nearest screen, through other objects. It also assumes the 3D fills the stage. - Network calls in an app during a render: nothing records and replays them, so a render that calls a model doesn't diff --git a/sdk/raea/bench/bevy/appscreen/check.sh b/sdk/raea/bench/bevy/appscreen/check.sh new file mode 100644 index 00000000..c08601fb --- /dev/null +++ b/sdk/raea/bench/bevy/appscreen/check.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# The appscreen fixture's measurements (docs/blitz-bevy.feature H12: , a module on its own Blitz document): +# edit.tsx played live offscreen with bench/render-play/appscreen.json's pointer (over the button from frame 6, pressed +# at 10 and let go at 12, off the screen from 21), and the play's input log rendered. On the render's frames: +# drawn the app is on the screen (its dark ground, the button blue) and neither run says it isn't drawn +# hover the button rings yellow while the pointer is over it (frame 9), and not once it's gone (frame 27) +# click the button is blue while pressed (frame 11) and green from the release on (frames 13 and 27) +# clock the bar its timer grows is there at frame 27 and not at frame 0 +# bash sdk/raea/bench/bevy/appscreen/check.sh [RAEA_BIN] exits 1 on any failure +here=$(cd "$(dirname "$0")" && pwd) +root=$(cd "$here/../../../../.." && pwd) +bin=${1:-${RAEA_BIN:-$root/target/release/raea}} +out=$(mktemp -d "${TMPDIR:-/tmp}/raea-appscreen-check-XXXXXX") +trap 'rm -rf "$out"' EXIT +export RAEA_SIGN=0 RAEA_MARK=0 RAEA_AUDIO_MUTE=1 +(cd "$here" && "$bin" play edit.tsx --offscreen --size 960x540 --seconds 1 --input-script "$root/bench/render-play/appscreen.json" --input-log "$out/input.json") > "$out/play.log" 2>&1 || { echo "appscreen: the play failed"; cat "$out/play.log"; exit 1; } +"$bin" render "$here/edit.tsx" --input "$out/input.json" -o "$out/" > "$out/render.log" 2>&1 || { echo "appscreen: the render failed"; cat "$out/render.log"; exit 1; } +status=0 +pass() { echo "appscreen: PASS $1"; } +fail() { echo "appscreen: FAIL $1"; status=1; } +# a pixel of frame F at X,Y as "R G B" +px() { ffmpeg -v error -i "$out/frame-$(printf %05d "$1").png" -vf "crop=1:1:$2:$3" -f rawvideo -pix_fmt rgb24 - | od -An -tu1 | tr -s ' ' | sed 's/^ //'; } +body() { px "$1" 420 290; } # the button's face, below its label +ring() { px "$1" 390 290; } # its border's left edge +bar() { px "$1" 341 379; } # the bar's first 40 px +green() { read -r r g b <<< "$1"; [ "$g" -gt 140 ] && [ "$g" -gt "$b" ] && [ "$g" -gt "$r" ]; } +blue() { read -r r g b <<< "$1"; [ "$b" -gt 200 ] && [ "$b" -gt "$g" ]; } +yellow() { read -r r g b <<< "$1"; [ "$r" -gt 200 ] && [ "$g" -gt 150 ] && [ "$b" -lt 120 ]; } +pink() { read -r r g b <<< "$1"; [ "$r" -gt 200 ] && [ "$g" -lt 140 ]; } +if grep -q "isn't drawn yet\|didn't open" "$out/play.log" "$out/render.log"; then fail "drawn (a run says: $(grep -h -m1 "isn't drawn yet\|didn't open" "$out/play.log" "$out/render.log"))"; else pass "neither run says the app screen isn't drawn"; fi +blue "$(body 3)" && pass "the button is on the screen, blue (frame 3: $(body 3))" || fail "the button isn't on the screen (frame 3: $(body 3))" +yellow "$(ring 9)" && pass "the button rings with the pointer over it (frame 9: $(ring 9))" || fail "no ring with the pointer over the button (frame 9: $(ring 9))" +blue "$(body 11)" && pass "pressed, not yet let go, it's still blue (frame 11: $(body 11))" || fail "the button changed before its release (frame 11: $(body 11))" +green "$(body 13)" && pass "the click reached it: green from the release (frame 13: $(body 13))" || fail "the click didn't reach the button (frame 13: $(body 13))" +green "$(body 27)" && pass "it stays clicked (frame 27: $(body 27))" || fail "the click didn't hold (frame 27: $(body 27))" +yellow "$(ring 27)" && fail "the ring stayed after the pointer left (frame 27: $(ring 27))" || pass "the ring went with the pointer (frame 27: $(ring 27))" +! pink "$(bar 0)" && pink "$(bar 27)" && pass "its timer's bar grew on the frame's clock (frame 0: $(bar 0), frame 27: $(bar 27))" || fail "the timer's bar (frame 0: $(bar 0), frame 27: $(bar 27))" +grep -q '"apps"' "$out/input.json" && pass "the play's log keeps what reached the app screen" || fail "the play's log has no apps entries" +exit $status diff --git a/sdk/raea/bench/bevy/appscreen/edit.tsx b/sdk/raea/bench/bevy/appscreen/edit.tsx new file mode 100644 index 00000000..99990522 --- /dev/null +++ b/sdk/raea/bench/bevy/appscreen/edit.tsx @@ -0,0 +1,21 @@ +// (docs/blitz-bevy.feature H12): an app screen, kit.tsx's component on @raea/ui's react-dom in a Blitz +// document of its own, on a flat screen in 3D (320 x 200, 2 units wide, filling x 295-665 and y 154-386 of the frame). +// Its button counts presses (blue "Press", then green "Pressed 1") and rings yellow while the pointer is over it; a pink +// bar along its foot grows on the app's own timer, so the app runs on the frame's clock. check.sh plays it with +// bench/render-play/appscreen.json's pointer and checks the render of the play's log; bench/render-play's row holds +// render = play. +import { AbsoluteFill, Composition, registerRoot } from "raea"; +import { Screen, ToneMapping } from "raea/3d"; +import { Camera3d } from "raea/camera"; +import { Transform } from "raea/transform"; + +function Scene() { + return ( + + + + + + ); +} +registerRoot(() => ); diff --git a/sdk/raea/bench/bevy/appscreen/kit.tsx b/sdk/raea/bench/bevy/appscreen/kit.tsx new file mode 100644 index 00000000..81c7dc67 --- /dev/null +++ b/sdk/raea/bench/bevy/appscreen/kit.tsx @@ -0,0 +1,26 @@ +// The app on the appscreen bench's screen (edit.tsx): a button that counts presses and rings while the pointer is over +// it, and a bar its own timer grows 8 px every 100 ms. +import { useEffect, useState } from "react"; + +export default function Kit({ width, height }: { width: number; height: number }) { + const [n, setN] = useState(0); + const [hot, setHot] = useState(false); + const [tick, setTick] = useState(0); + useEffect(() => { + const t = setInterval(() => setTick((v) => v + 1), 100); + return () => clearInterval(t); + }, []); + return ( +
+ +
+
+ ); +} diff --git a/sdk/render/src/screen.tsx b/sdk/render/src/screen.tsx index e93cb5bc..76fedc10 100644 --- a/sdk/render/src/screen.tsx +++ b/sdk/render/src/screen.tsx @@ -70,10 +70,10 @@ export interface ScreenSource { /** a video to show instead of children: staticFile("clip.mp4"), filling the picture */ src?: string; /** an app to show instead of children: a module's path from the composition's folder ("chat.tsx"), its default - * export a React component given {width, height}. It runs on real react-dom in a document of its own, so npm's React - * libraries work on it (assistant-ui, Radix; installed with `raea add`). In play the pointer reaches it through - * the 3D, where it lands on the picture, and a press on it gives it the keys (the game doesn't get them) until a - * press elsewhere or Escape; in a render it runs on the frame's clock, so the render repeats */ + * export a React component given {width, height}. It runs on react-dom (@raea/ui's) in a Blitz document of its + * own, so npm's React libraries work on it (assistant-ui, Radix; installed with `raea add`). In play the pointer + * reaches it through the 3D, where it lands on the picture, and a press on it gives it the keys (the game doesn't + * get them) until a press elsewhere or Escape; in a render it runs on the frame's clock, so the render repeats */ app?: string; /** the nested composition or game */ children?: ReactNode; diff --git a/skills b/skills index d6ba1129..80067cc2 160000 --- a/skills +++ b/skills @@ -1 +1 @@ -Subproject commit d6ba1129ef26f5a4936c65614d53ff0fcd087425 +Subproject commit 80067cc20a625174e00b653efc1707c59dad5c7d diff --git a/tools/layers.toml b/tools/layers.toml index c265d08a..77e264e9 100644 --- a/tools/layers.toml +++ b/tools/layers.toml @@ -150,6 +150,7 @@ raea-runtime-export = "runtime" raea-runtime-pagestore = "runtime" raea-runtime-cssurls = "runtime" raea-runtime-page = "runtime" +raea-runtime-appscreen = "runtime" raea-runtime-sound = "runtime" raea-runtime-view = "runtime" raea-runtime-world = "runtime"