Improve web text agent (#4561)

- Closes https://github.com/emilk/egui/issues/4060 - no longer aligned
to top
- Closes https://github.com/emilk/egui/issues/4479 - `canvas.style` is
not set anywhere anymore
- Closes https://github.com/emilk/egui/issues/2231 - same as #4060
- Closes https://github.com/emilk/egui/issues/3618 - there is now one
`<input>` per `eframe` app, and it's removed transitively by
`WebRunner::destroy -> AppRunner::drop -> TextAgent::drop`

This PR improves the text agent to make fewer assumptions about how
`egui` is embedded into the page:
- Text agent no longer sets the canvas position
- There is now a text agent for each instance of `WebRunner`
- The input element is now moved to the correct position, so the OS can
display the IME window in the correct place. Before it would typically
be outside of the viewport

The best way to test this is to build & server the web demo locally:
```
scripts/build_demo_web.sh && scripts/start_server.sh
```

Then open the EasyMark editor, and try using IME to input some emojis:
http://localhost:8888/#EasyMarkEditor

To open the emoji keyboard use:
- <kbd>win + .</kbd> on Windows
- <kbd>ctrl + cmd + space</kbd> on Mac

Tested on:
- [x] Windows
- [x] Linux
- [x] MacOS
- [x] Android
- [x] iOS

## Migration guide

The canvas no longer controls its own size/position on the page. This
means that those properties can now be controlled entirely via HTML and
CSS, and multiple separate `eframe` apps can coexist better on a single
page.

To match the old behavior, set the `canvas` width and height to 100% of
the `body` element:

```html
<html>
  <body>
    <canvas></canvas>
  </body>
</html>
```

```css
/* remove default margins and use full viewport */
html, body {
  margin: 0;
  width: 100%;
  height: 100%;
}

canvas {
  /* match parent element size */
  width: 100%;
  height: 100%;
}
```

Note that there is no need to set `position: absolute`/`left: 50%;
transform: translateX(-50%)`/etc., and setting those properties may
poorly affect the sharpness of `egui`-rendered text.

Because `eframe` no longer updates the canvas style in any way, it also
means that on mobile, the canvas no longer collapses upwards to make
space for a mobile keyboard. This should be solved in other ways:
https://github.com/emilk/egui/issues/4572
This commit is contained in:
Jan Procházka 2024-05-29 12:54:33 +02:00 committed by GitHub
parent 5eee463851
commit 514ee0c433
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 204 additions and 232 deletions

View File

@ -2,7 +2,7 @@ use egui::TexturesDelta;
use crate::{epi, App}; use crate::{epi, App};
use super::{now_sec, web_painter::WebPainter, NeedRepaint}; use super::{now_sec, text_agent::TextAgent, web_painter::WebPainter, NeedRepaint};
pub struct AppRunner { pub struct AppRunner {
#[allow(dead_code)] #[allow(dead_code)]
@ -14,7 +14,7 @@ pub struct AppRunner {
app: Box<dyn epi::App>, app: Box<dyn epi::App>,
pub(crate) needs_repaint: std::sync::Arc<NeedRepaint>, pub(crate) needs_repaint: std::sync::Arc<NeedRepaint>,
last_save_time: f64, last_save_time: f64,
pub(crate) ime: Option<egui::output::IMEOutput>, pub(crate) text_agent: TextAgent,
pub(crate) mutable_text_under_cursor: bool, pub(crate) mutable_text_under_cursor: bool,
// Output for the last run: // Output for the last run:
@ -35,6 +35,7 @@ impl AppRunner {
canvas_id: &str, canvas_id: &str,
web_options: crate::WebOptions, web_options: crate::WebOptions,
app_creator: epi::AppCreator, app_creator: epi::AppCreator,
text_agent: TextAgent,
) -> Result<Self, String> { ) -> Result<Self, String> {
let painter = super::ActiveWebPainter::new(canvas_id, &web_options).await?; let painter = super::ActiveWebPainter::new(canvas_id, &web_options).await?;
@ -119,7 +120,7 @@ impl AppRunner {
app, app,
needs_repaint, needs_repaint,
last_save_time: now_sec(), last_save_time: now_sec(),
ime: None, text_agent,
mutable_text_under_cursor: false, mutable_text_under_cursor: false,
textures_delta: Default::default(), textures_delta: Default::default(),
clipped_primitives: None, clipped_primitives: None,
@ -270,9 +271,11 @@ impl AppRunner {
self.mutable_text_under_cursor = mutable_text_under_cursor; self.mutable_text_under_cursor = mutable_text_under_cursor;
if self.ime != ime { if let Err(err) = self.text_agent.move_to(ime, self.canvas()) {
super::text_agent::move_text_cursor(ime, self.canvas()); log::error!(
self.ime = ime; "failed to update text agent position: {}",
super::string_from_js_value(&err)
);
} }
} }
} }

View File

@ -94,8 +94,8 @@ pub(crate) fn install_document_events(runner_ref: &WebRunner) -> Result<(), JsVa
if !modifiers.ctrl if !modifiers.ctrl
&& !modifiers.command && !modifiers.command
&& !should_ignore_key(&key) && !should_ignore_key(&key)
// When text agent is shown, it sends text event instead. // When text agent is focused, it is responsible for handling input events
&& text_agent::text_agent().hidden() && !runner.text_agent.has_focus()
{ {
runner.input.raw.events.push(egui::Event::Text(key)); runner.input.raw.events.push(egui::Event::Text(key));
} }
@ -375,10 +375,12 @@ pub(crate) fn install_canvas_events(runner_ref: &WebRunner) -> Result<(), JsValu
// event callback, which is why we run the app logic here and now: // event callback, which is why we run the app logic here and now:
runner.logic(); runner.logic();
runner
.text_agent
.set_focus(runner.mutable_text_under_cursor);
// Make sure we paint the output of the above logic call asap: // Make sure we paint the output of the above logic call asap:
runner.needs_repaint.repaint_asap(); runner.needs_repaint.repaint_asap();
text_agent::update_text_agent(runner);
} }
event.stop_propagation(); event.stop_propagation();
event.prevent_default(); event.prevent_default();
@ -467,13 +469,15 @@ pub(crate) fn install_canvas_events(runner_ref: &WebRunner) -> Result<(), JsValu
runner.input.raw.events.push(egui::Event::PointerGone); runner.input.raw.events.push(egui::Event::PointerGone);
push_touches(runner, egui::TouchPhase::End, &event); push_touches(runner, egui::TouchPhase::End, &event);
runner
.text_agent
.set_focus(runner.mutable_text_under_cursor);
runner.needs_repaint.repaint_asap(); runner.needs_repaint.repaint_asap();
event.stop_propagation(); event.stop_propagation();
event.prevent_default(); event.prevent_default();
} }
// Finally, focus or blur text agent to toggle mobile keyboard:
text_agent::update_text_agent(runner);
}, },
)?; )?;

View File

@ -53,6 +53,29 @@ pub(crate) fn string_from_js_value(value: &JsValue) -> String {
value.as_string().unwrap_or_else(|| format!("{value:#?}")) value.as_string().unwrap_or_else(|| format!("{value:#?}"))
} }
/// Returns the `Element` with active focus.
///
/// Elements can only be focused if they are:
/// - `<a>`/`<area>` with an `href` attribute
/// - `<input>`/`<select>`/`<textarea>`/`<button>` which aren't `disabled`
/// - any other element with a `tabindex` attribute
pub(crate) fn focused_element() -> Option<web_sys::Element> {
web_sys::window()?
.document()?
.active_element()?
.dyn_into()
.ok()
}
pub(crate) fn has_focus<T: JsCast>(element: &T) -> bool {
fn try_has_focus<T: JsCast>(element: &T) -> Option<bool> {
let element = element.dyn_ref::<web_sys::Element>()?;
let focused_element = focused_element()?;
Some(element == &focused_element)
}
try_has_focus(element).unwrap_or(false)
}
/// Current time in seconds (since undefined point in time). /// Current time in seconds (since undefined point in time).
/// ///
/// Monotonically increasing. /// Monotonically increasing.

View File

@ -1,237 +1,178 @@
//! The text agent is an `<input>` element used to trigger //! The text agent is a hidden `<input>` element used to capture
//! mobile keyboard and IME input. //! IME and mobile keyboard input events.
//!
use std::{cell::Cell, rc::Rc}; use std::cell::Cell;
use wasm_bindgen::prelude::*; use wasm_bindgen::prelude::*;
use super::{AppRunner, WebRunner}; use super::{AppRunner, WebRunner};
static AGENT_ID: &str = "egui_text_agent"; pub struct TextAgent {
input: web_sys::HtmlInputElement,
pub fn text_agent() -> web_sys::HtmlInputElement { prev_ime_output: Cell<Option<egui::output::IMEOutput>>,
web_sys::window()
.unwrap()
.document()
.unwrap()
.get_element_by_id(AGENT_ID)
.unwrap()
.dyn_into()
.unwrap()
} }
/// Text event handler, impl TextAgent {
pub fn install_text_agent(runner_ref: &WebRunner) -> Result<(), JsValue> { /// Attach the agent to the document.
let window = web_sys::window().unwrap(); pub fn attach(runner_ref: &WebRunner) -> Result<Self, JsValue> {
let document = window.document().unwrap(); let document = web_sys::window().unwrap().document().unwrap();
let body = document.body().expect("document should have a body");
// create an `<input>` element
let input = document let input = document
.create_element("input")? .create_element("input")?
.dyn_into::<web_sys::HtmlInputElement>()?; .dyn_into::<web_sys::HtmlInputElement>()?;
let input = std::rc::Rc::new(input); input.set_type("text");
input.set_id(AGENT_ID);
let is_composing = Rc::new(Cell::new(false)); // append it to `<body>` and hide it outside of the viewport
{
let style = input.style(); let style = input.style();
// Transparent style.set_property("opacity", "0")?;
style.set_property("opacity", "0").unwrap(); style.set_property("width", "1px")?;
// Hide under canvas style.set_property("height", "1px")?;
style.set_property("z-index", "-1").unwrap(); style.set_property("position", "absolute")?;
} style.set_property("top", "0")?;
// Set size as small as possible, in case user may click on it. style.set_property("left", "0")?;
input.set_size(1); document.body().unwrap().append_child(&input)?;
input.set_autofocus(true);
input.set_hidden(true);
// When IME is off // attach event listeners
runner_ref.add_event_listener(&input, "input", {
let input_clone = input.clone();
let is_composing = is_composing.clone();
move |_event: web_sys::InputEvent, runner| { let on_input = {
let text = input_clone.value(); let input = input.clone();
if !text.is_empty() && !is_composing.get() { move |event: web_sys::InputEvent, runner: &mut AppRunner| {
input_clone.set_value(""); let text = input.value();
runner.input.raw.events.push(egui::Event::Text(text)); // if `is_composing` is true, then user is using IME, for example: emoji, pinyin, kanji, hangul, etc.
// In that case, the browser emits both `input` and `compositionupdate` events,
// and we need to ignore the `input` event.
if !text.is_empty() && !event.is_composing() {
input.set_value("");
let event = egui::Event::Text(text);
runner.input.raw.events.push(event);
runner.needs_repaint.repaint_asap(); runner.needs_repaint.repaint_asap();
} }
} }
})?; };
{ let on_composition_start = {
// When IME is on, handle composition event let input = input.clone();
runner_ref.add_event_listener(&input, "compositionstart", { move |_: web_sys::CompositionEvent, runner: &mut AppRunner| {
let input_clone = input.clone(); input.set_value("");
let is_composing = is_composing.clone(); let event = egui::Event::Ime(egui::ImeEvent::Enabled);
runner.input.raw.events.push(event);
move |_event: web_sys::CompositionEvent, runner: &mut AppRunner| { // Repaint moves the text agent into place,
is_composing.set(true); // see `move_to` in `AppRunner::handle_platform_output`.
input_clone.set_value("");
let egui_event = egui::Event::Ime(egui::ImeEvent::Enabled);
runner.input.raw.events.push(egui_event);
runner.needs_repaint.repaint_asap(); runner.needs_repaint.repaint_asap();
} }
})?; };
runner_ref.add_event_listener( let on_composition_update = {
&input,
"compositionupdate",
move |event: web_sys::CompositionEvent, runner: &mut AppRunner| { move |event: web_sys::CompositionEvent, runner: &mut AppRunner| {
if let Some(text) = event.data() { let Some(text) = event.data() else { return };
let egui_event = egui::Event::Ime(egui::ImeEvent::Preedit(text)); let event = egui::Event::Ime(egui::ImeEvent::Preedit(text));
runner.input.raw.events.push(egui_event); runner.input.raw.events.push(event);
runner.needs_repaint.repaint_asap(); runner.needs_repaint.repaint_asap();
} }
}, };
)?;
runner_ref.add_event_listener(&input, "compositionend", {
let input_clone = input.clone();
let on_composition_end = {
let input = input.clone();
move |event: web_sys::CompositionEvent, runner: &mut AppRunner| { move |event: web_sys::CompositionEvent, runner: &mut AppRunner| {
is_composing.set(false); let Some(text) = event.data() else { return };
input_clone.set_value(""); input.set_value("");
let event = egui::Event::Ime(egui::ImeEvent::Commit(text));
if let Some(text) = event.data() { runner.input.raw.events.push(event);
let egui_event = egui::Event::Ime(egui::ImeEvent::Commit(text));
runner.input.raw.events.push(egui_event);
runner.needs_repaint.repaint_asap(); runner.needs_repaint.repaint_asap();
} }
} };
})?;
runner_ref.add_event_listener(&input, "input", on_input)?;
runner_ref.add_event_listener(&input, "compositionstart", on_composition_start)?;
runner_ref.add_event_listener(&input, "compositionupdate", on_composition_update)?;
runner_ref.add_event_listener(&input, "compositionend", on_composition_end)?;
Ok(Self {
input,
prev_ime_output: Default::default(),
})
} }
// When input lost focus, focus on it again. pub fn move_to(
// It is useful when user click somewhere outside canvas. &self,
let input_refocus = input.clone(); ime: Option<egui::output::IMEOutput>,
runner_ref.add_event_listener(&input, "focusout", move |_event: web_sys::MouseEvent, _| { canvas: &web_sys::HtmlCanvasElement,
// Delay 10 ms, and focus again. ) -> Result<(), JsValue> {
let input_refocus = input_refocus.clone(); // Mobile keyboards don't follow the text input it's writing to,
call_after_delay(std::time::Duration::from_millis(10), move || { // instead typically being fixed in place on the bottom of the screen,
input_refocus.focus().ok(); // so don't bother moving the text agent on mobile.
}); if is_mobile() {
})?; return Ok(());
}
body.append_child(&input)?; // Don't move the text agent unless the position actually changed:
if self.prev_ime_output.get() == ime {
return Ok(());
}
self.prev_ime_output.set(ime);
let Some(ime) = ime else { return Ok(()) };
let ime_pos = ime.cursor_rect.left_top();
let canvas_rect = canvas.get_bounding_client_rect();
let new_pos = ime_pos + egui::vec2(canvas_rect.left() as f32, canvas_rect.top() as f32);
let style = self.input.style();
style.set_property("top", &format!("{}px", new_pos.y))?;
style.set_property("left", &format!("{}px", new_pos.x))?;
Ok(()) Ok(())
} }
/// Focus or blur text agent to toggle mobile keyboard. pub fn set_focus(&self, on: bool) {
pub fn update_text_agent(runner: &AppRunner) -> Option<()> { if on {
use web_sys::HtmlInputElement; self.focus();
let window = web_sys::window()?;
let document = window.document()?;
let input: HtmlInputElement = document.get_element_by_id(AGENT_ID)?.dyn_into().unwrap();
let canvas_style = runner.canvas().style();
if runner.mutable_text_under_cursor {
let is_already_editing = input.hidden();
if is_already_editing {
input.set_hidden(false);
input.focus().ok()?;
// Move up canvas so that text edit is shown at ~30% of screen height.
// Only on touch screens, when keyboard popups.
if let Some(latest_touch_pos) = runner.input.latest_touch_pos {
let window_height = window.inner_height().ok()?.as_f64()? as f32;
let current_rel = latest_touch_pos.y / window_height;
// estimated amount of screen covered by keyboard
let keyboard_fraction = 0.5;
if current_rel > keyboard_fraction {
// below the keyboard
let target_rel = 0.3;
// Note: `delta` is negative, since we are moving the canvas UP
let delta = target_rel - current_rel;
let delta = delta.max(-keyboard_fraction); // Don't move it crazy much
let new_pos_percent = format!("{}%", (delta * 100.0).round());
canvas_style.set_property("position", "absolute").ok()?;
canvas_style.set_property("top", &new_pos_percent).ok()?;
}
}
}
} else { } else {
// Holding the runner lock while calling input.blur() causes a panic. self.blur();
// This is most probably caused by the browser running the event handler
// for the triggered blur event synchronously, meaning that the mutex
// lock does not get dropped by the time another event handler is called.
//
// Why this didn't exist before #1290 is a mystery to me, but it exists now
// and this apparently is the fix for it
//
// ¯\_(ツ)_/¯ - @DusterTheFirst
// So since we are inside a runner lock here, we just postpone the blur/hide:
call_after_delay(std::time::Duration::from_millis(0), move || {
input.blur().ok();
input.set_hidden(true);
canvas_style.set_property("position", "absolute").ok();
canvas_style.set_property("top", "0%").ok(); // move back to normal position
});
} }
Some(())
} }
fn call_after_delay(delay: std::time::Duration, f: impl FnOnce() + 'static) { pub fn has_focus(&self) -> bool {
use wasm_bindgen::prelude::*; super::has_focus(&self.input)
let window = web_sys::window().unwrap();
let closure = Closure::once(f);
let delay_ms = delay.as_millis() as _;
window
.set_timeout_with_callback_and_timeout_and_arguments_0(
closure.as_ref().unchecked_ref(),
delay_ms,
)
.unwrap();
closure.forget(); // We must forget it, or else the callback is canceled on drop
} }
/// If context is running under mobile device? fn focus(&self) {
fn is_mobile() -> Option<bool> { if self.has_focus() {
const MOBILE_DEVICE: [&str; 6] = ["Android", "iPhone", "iPad", "iPod", "webOS", "BlackBerry"]; return;
}
if let Err(err) = self.input.focus() {
log::error!("failed to set focus: {}", super::string_from_js_value(&err));
};
}
fn blur(&self) {
if !self.has_focus() {
return;
}
if let Err(err) = self.input.blur() {
log::error!("failed to set focus: {}", super::string_from_js_value(&err));
};
}
}
impl Drop for TextAgent {
fn drop(&mut self) {
self.input.remove();
}
}
/// Returns `true` if the app is likely running on a mobile device.
fn is_mobile() -> bool {
fn try_is_mobile() -> Option<bool> {
const MOBILE_DEVICE: [&str; 6] =
["Android", "iPhone", "iPad", "iPod", "webOS", "BlackBerry"];
let user_agent = web_sys::window()?.navigator().user_agent().ok()?; let user_agent = web_sys::window()?.navigator().user_agent().ok()?;
let is_mobile = MOBILE_DEVICE.iter().any(|&name| user_agent.contains(name)); let is_mobile = MOBILE_DEVICE.iter().any(|&name| user_agent.contains(name));
Some(is_mobile) Some(is_mobile)
} }
try_is_mobile().unwrap_or(false)
// Move text agent to text cursor's position, on desktop/laptop,
// candidate window moves following text element (agent),
// so it appears that the IME candidate window moves with text cursor.
// On mobile devices, there is no need to do that.
pub fn move_text_cursor(
ime: Option<egui::output::IMEOutput>,
canvas: &web_sys::HtmlCanvasElement,
) -> Option<()> {
let style = text_agent().style();
// Note: moving agent on mobile devices will lead to unpredictable scroll.
if is_mobile() == Some(false) {
ime.as_ref().and_then(|ime| {
let egui::Pos2 { x, y } = ime.cursor_rect.left_top();
let bounding_rect = text_agent().get_bounding_client_rect();
let y = (y + (canvas.scroll_top() + canvas.offset_top()) as f32)
.min(canvas.client_height() as f32 - bounding_rect.height() as f32);
let x = x + (canvas.scroll_left() + canvas.offset_left()) as f32;
// Canvas is translated 50% horizontally in html.
let x = (x - canvas.offset_width() as f32 / 2.0)
.min(canvas.client_width() as f32 - bounding_rect.width() as f32);
style.set_property("position", "absolute").ok()?;
style.set_property("top", &format!("{y}px")).ok()?;
style.set_property("left", &format!("{x}px")).ok()
})
} else {
style.set_property("position", "absolute").ok()?;
style.set_property("top", "0px").ok()?;
style.set_property("left", "0px").ok()
}
} }

View File

@ -4,7 +4,7 @@ use wasm_bindgen::prelude::*;
use crate::{epi, App}; use crate::{epi, App};
use super::{events, AppRunner, PanicHandler}; use super::{events, text_agent::TextAgent, AppRunner, PanicHandler};
/// This is how `eframe` runs your wepp application /// This is how `eframe` runs your wepp application
/// ///
@ -65,14 +65,15 @@ impl WebRunner {
let follow_system_theme = web_options.follow_system_theme; let follow_system_theme = web_options.follow_system_theme;
let runner = AppRunner::new(canvas_id, web_options, app_creator).await?; let text_agent = TextAgent::attach(self)?;
let runner = AppRunner::new(canvas_id, web_options, app_creator, text_agent).await?;
self.runner.replace(Some(runner)); self.runner.replace(Some(runner));
{ {
events::install_canvas_events(self)?; events::install_canvas_events(self)?;
events::install_document_events(self)?; events::install_document_events(self)?;
events::install_window_events(self)?; events::install_window_events(self)?;
super::text_agent::install_text_agent(self)?;
if follow_system_theme { if follow_system_theme {
events::install_color_scheme_change_event(self)?; events::install_color_scheme_change_event(self)?;

View File

@ -113,7 +113,7 @@ pub struct PlatformOutput {
/// Use by `eframe` web to show/hide mobile keyboard and IME agent. /// Use by `eframe` web to show/hide mobile keyboard and IME agent.
pub mutable_text_under_cursor: bool, pub mutable_text_under_cursor: bool,
/// This is et if, and only if, the user is currently editing text. /// This is set if, and only if, the user is currently editing text.
/// ///
/// Useful for IME. /// Useful for IME.
pub ime: Option<IMEOutput>, pub ime: Option<IMEOutput>,