skyler
/
egui
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Improve web text agent (#4561) 

- Closes https://github.com/emilk/egui/issues/4060 - no longer aligned
to top
- Closes https://github.com/emilk/egui/issues/4479 - `canvas.style` is
not set anywhere anymore
- Closes https://github.com/emilk/egui/issues/2231 - same as #4060
- Closes https://github.com/emilk/egui/issues/3618 - there is now one
`<input>` per `eframe` app, and it's removed transitively by
`WebRunner::destroy -> AppRunner::drop -> TextAgent::drop`

This PR improves the text agent to make fewer assumptions about how
`egui` is embedded into the page:
- Text agent no longer sets the canvas position
- There is now a text agent for each instance of `WebRunner`
- The input element is now moved to the correct position, so the OS can
display the IME window in the correct place. Before it would typically
be outside of the viewport

The best way to test this is to build & server the web demo locally:
```
scripts/build_demo_web.sh && scripts/start_server.sh
```

Then open the EasyMark editor, and try using IME to input some emojis:
http://localhost:8888/#EasyMarkEditor

To open the emoji keyboard use:
- <kbd>win + .</kbd> on Windows
- <kbd>ctrl + cmd + space</kbd> on Mac

Tested on:
- [x] Windows
- [x] Linux
- [x] MacOS
- [x] Android
- [x] iOS

## Migration guide

The canvas no longer controls its own size/position on the page. This
means that those properties can now be controlled entirely via HTML and
CSS, and multiple separate `eframe` apps can coexist better on a single
page.

To match the old behavior, set the `canvas` width and height to 100% of
the `body` element:

```html
<html>
  <body>
    <canvas></canvas>
  </body>
</html>
```

```css
/* remove default margins and use full viewport */
html, body {
  margin: 0;
  width: 100%;
  height: 100%;
}

canvas {
  /* match parent element size */
  width: 100%;
  height: 100%;
}
```

Note that there is no need to set `position: absolute`/`left: 50%;
transform: translateX(-50%)`/etc., and setting those properties may
poorly affect the sharpness of `egui`-rendered text.

Because `eframe` no longer updates the canvas style in any way, it also
means that on mobile, the canvas no longer collapses upwards to make
space for a mobile keyboard. This should be solved in other ways:
https://github.com/emilk/egui/issues/4572
This commit is contained in:
Jan Procházka 2024-05-29 12:54:33 +02:00 committed by GitHub
parent 5eee463851
commit 514ee0c433
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 204 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
crates/eframe/src/web/app_runner.rs
View File

@ -2,7 +2,7 @@ use egui::TexturesDelta;
use crate::{epi, App};
use super::{now_sec, web_painter::WebPainter, NeedRepaint};
use super::{now_sec, text_agent::TextAgent, web_painter::WebPainter, NeedRepaint};
pub struct AppRunner {
#[allow(dead_code)]
@ -14,7 +14,7 @@ pub struct AppRunner {
app: Box<dyn epi::App>,
pub(crate) needs_repaint: std::sync::Arc<NeedRepaint>,
last_save_time: f64,
pub(crate) ime: Option<egui::output::IMEOutput>,
pub(crate) text_agent: TextAgent,
pub(crate) mutable_text_under_cursor: bool,
// Output for the last run:
@ -35,6 +35,7 @@ impl AppRunner {
canvas_id: &str,
web_options: crate::WebOptions,
app_creator: epi::AppCreator,
text_agent: TextAgent,
) -> Result<Self, String> {
let painter = super::ActiveWebPainter::new(canvas_id, &web_options).await?;
@ -119,7 +120,7 @@ impl AppRunner {
app,
needs_repaint,
last_save_time: now_sec(),
ime: None,
text_agent,
mutable_text_under_cursor: false,
textures_delta: Default::default(),
clipped_primitives: None,
@ -270,9 +271,11 @@ impl AppRunner {
self.mutable_text_under_cursor = mutable_text_under_cursor;
if self.ime != ime {
super::text_agent::move_text_cursor(ime, self.canvas());
self.ime = ime;
if let Err(err) = self.text_agent.move_to(ime, self.canvas()) {
log::error!(
"failed to update text agent position: {}",
super::string_from_js_value(&err)
);
}
}
}

18
crates/eframe/src/web/events.rs
View File

@ -94,8 +94,8 @@ pub(crate) fn install_document_events(runner_ref: &WebRunner) -> Result<(), JsVa
if !modifiers.ctrl
&& !modifiers.command
&& !should_ignore_key(&key)
// When text agent is shown, it sends text event instead.
&& text_agent::text_agent().hidden()
// When text agent is focused, it is responsible for handling input events
&& !runner.text_agent.has_focus()
{
runner.input.raw.events.push(egui::Event::Text(key));
}
@ -375,10 +375,12 @@ pub(crate) fn install_canvas_events(runner_ref: &WebRunner) -> Result<(), JsValu
// event callback, which is why we run the app logic here and now:
runner.logic();
runner
.text_agent
.set_focus(runner.mutable_text_under_cursor);
// Make sure we paint the output of the above logic call asap:
runner.needs_repaint.repaint_asap();
text_agent::update_text_agent(runner);
}
event.stop_propagation();
event.prevent_default();
@ -467,13 +469,15 @@ pub(crate) fn install_canvas_events(runner_ref: &WebRunner) -> Result<(), JsValu
runner.input.raw.events.push(egui::Event::PointerGone);
push_touches(runner, egui::TouchPhase::End, &event);
runner
.text_agent
.set_focus(runner.mutable_text_under_cursor);
runner.needs_repaint.repaint_asap();
event.stop_propagation();
event.prevent_default();
}
// Finally, focus or blur text agent to toggle mobile keyboard:
text_agent::update_text_agent(runner);
},
)?;

23
crates/eframe/src/web/mod.rs
View File

@ -53,6 +53,29 @@ pub(crate) fn string_from_js_value(value: &JsValue) -> String {
value.as_string().unwrap_or_else(|| format!("{value:#?}"))
}
/// Returns the `Element` with active focus.
///
/// Elements can only be focused if they are:
/// - `<a>`/`<area>` with an `href` attribute
/// - `<input>`/`<select>`/`<textarea>`/`<button>` which aren't `disabled`
/// - any other element with a `tabindex` attribute
pub(crate) fn focused_element() -> Option<web_sys::Element> {
web_sys::window()?
.document()?
.active_element()?
.dyn_into()
.ok()
}
pub(crate) fn has_focus<T: JsCast>(element: &T) -> bool {
fn try_has_focus<T: JsCast>(element: &T) -> Option<bool> {
let element = element.dyn_ref::<web_sys::Element>()?;
let focused_element = focused_element()?;
Some(element == &focused_element)
}
try_has_focus(element).unwrap_or(false)
}
/// Current time in seconds (since undefined point in time).
///
/// Monotonically increasing.

371
crates/eframe/src/web/text_agent.rs
View File

@ -1,237 +1,178 @@
//! The text agent is an `<input>` element used to trigger
//! mobile keyboard and IME input.
//!
use std::{cell::Cell, rc::Rc};
//! The text agent is a hidden `<input>` element used to capture
//! IME and mobile keyboard input events.
use std::cell::Cell;
use wasm_bindgen::prelude::*;
use super::{AppRunner, WebRunner};
static AGENT_ID: &str = "egui_text_agent";
pub fn text_agent() -> web_sys::HtmlInputElement {
web_sys::window()
.unwrap()
.document()
.unwrap()
.get_element_by_id(AGENT_ID)
.unwrap()
.dyn_into()
.unwrap()
pub struct TextAgent {
input: web_sys::HtmlInputElement,
prev_ime_output: Cell<Option<egui::output::IMEOutput>>,
}
/// Text event handler,
pub fn install_text_agent(runner_ref: &WebRunner) -> Result<(), JsValue> {
let window = web_sys::window().unwrap();
let document = window.document().unwrap();
let body = document.body().expect("document should have a body");
let input = document
.create_element("input")?
.dyn_into::<web_sys::HtmlInputElement>()?;
let input = std::rc::Rc::new(input);
input.set_id(AGENT_ID);
let is_composing = Rc::new(Cell::new(false));
{
impl TextAgent {
/// Attach the agent to the document.
pub fn attach(runner_ref: &WebRunner) -> Result<Self, JsValue> {
let document = web_sys::window().unwrap().document().unwrap();
// create an `<input>` element
let input = document
.create_element("input")?
.dyn_into::<web_sys::HtmlInputElement>()?;
input.set_type("text");
// append it to `<body>` and hide it outside of the viewport
let style = input.style();
// Transparent
style.set_property("opacity", "0").unwrap();
// Hide under canvas
style.set_property("z-index", "-1").unwrap();
}
// Set size as small as possible, in case user may click on it.
input.set_size(1);
input.set_autofocus(true);
input.set_hidden(true);
style.set_property("opacity", "0")?;
style.set_property("width", "1px")?;
style.set_property("height", "1px")?;
style.set_property("position", "absolute")?;
style.set_property("top", "0")?;
style.set_property("left", "0")?;
document.body().unwrap().append_child(&input)?;
// When IME is off
runner_ref.add_event_listener(&input, "input", {
let input_clone = input.clone();
let is_composing = is_composing.clone();
// attach event listeners
move |_event: web_sys::InputEvent, runner| {
let text = input_clone.value();
if !text.is_empty() && !is_composing.get() {
input_clone.set_value("");
runner.input.raw.events.push(egui::Event::Text(text));
runner.needs_repaint.repaint_asap();
}
}
})?;
{
// When IME is on, handle composition event
runner_ref.add_event_listener(&input, "compositionstart", {
let input_clone = input.clone();
let is_composing = is_composing.clone();
move |_event: web_sys::CompositionEvent, runner: &mut AppRunner| {
is_composing.set(true);
input_clone.set_value("");
let egui_event = egui::Event::Ime(egui::ImeEvent::Enabled);
runner.input.raw.events.push(egui_event);
runner.needs_repaint.repaint_asap();
}
})?;
runner_ref.add_event_listener(
&input,
"compositionupdate",
move |event: web_sys::CompositionEvent, runner: &mut AppRunner| {
if let Some(text) = event.data() {
let egui_event = egui::Event::Ime(egui::ImeEvent::Preedit(text));
runner.input.raw.events.push(egui_event);
runner.needs_repaint.repaint_asap();
}
},
)?;
runner_ref.add_event_listener(&input, "compositionend", {
let input_clone = input.clone();
move |event: web_sys::CompositionEvent, runner: &mut AppRunner| {
is_composing.set(false);
input_clone.set_value("");
if let Some(text) = event.data() {
let egui_event = egui::Event::Ime(egui::ImeEvent::Commit(text));
runner.input.raw.events.push(egui_event);
let on_input = {
let input = input.clone();
move |event: web_sys::InputEvent, runner: &mut AppRunner| {
let text = input.value();
// if `is_composing` is true, then user is using IME, for example: emoji, pinyin, kanji, hangul, etc.
// In that case, the browser emits both `input` and `compositionupdate` events,
// and we need to ignore the `input` event.
if !text.is_empty() && !event.is_composing() {
input.set_value("");
let event = egui::Event::Text(text);
runner.input.raw.events.push(event);
runner.needs_repaint.repaint_asap();
}
}
})?;
}
};
// When input lost focus, focus on it again.
// It is useful when user click somewhere outside canvas.
let input_refocus = input.clone();
runner_ref.add_event_listener(&input, "focusout", move |_event: web_sys::MouseEvent, _| {
// Delay 10 ms, and focus again.
let input_refocus = input_refocus.clone();
call_after_delay(std::time::Duration::from_millis(10), move || {
input_refocus.focus().ok();
});
})?;
body.append_child(&input)?;
Ok(())
}
/// Focus or blur text agent to toggle mobile keyboard.
pub fn update_text_agent(runner: &AppRunner) -> Option<()> {
use web_sys::HtmlInputElement;
let window = web_sys::window()?;
let document = window.document()?;
let input: HtmlInputElement = document.get_element_by_id(AGENT_ID)?.dyn_into().unwrap();
let canvas_style = runner.canvas().style();
if runner.mutable_text_under_cursor {
let is_already_editing = input.hidden();
if is_already_editing {
input.set_hidden(false);
input.focus().ok()?;
// Move up canvas so that text edit is shown at ~30% of screen height.
// Only on touch screens, when keyboard popups.
if let Some(latest_touch_pos) = runner.input.latest_touch_pos {
let window_height = window.inner_height().ok()?.as_f64()? as f32;
let current_rel = latest_touch_pos.y / window_height;
// estimated amount of screen covered by keyboard
let keyboard_fraction = 0.5;
if current_rel > keyboard_fraction {
// below the keyboard
let target_rel = 0.3;
// Note: `delta` is negative, since we are moving the canvas UP
let delta = target_rel - current_rel;
let delta = delta.max(-keyboard_fraction); // Don't move it crazy much
let new_pos_percent = format!("{}%", (delta * 100.0).round());
canvas_style.set_property("position", "absolute").ok()?;
canvas_style.set_property("top", &new_pos_percent).ok()?;
}
let on_composition_start = {
let input = input.clone();
move |_: web_sys::CompositionEvent, runner: &mut AppRunner| {
input.set_value("");
let event = egui::Event::Ime(egui::ImeEvent::Enabled);
runner.input.raw.events.push(event);
// Repaint moves the text agent into place,
// see `move_to` in `AppRunner::handle_platform_output`.
runner.needs_repaint.repaint_asap();
}
}
} else {
// Holding the runner lock while calling input.blur() causes a panic.
// This is most probably caused by the browser running the event handler
// for the triggered blur event synchronously, meaning that the mutex
// lock does not get dropped by the time another event handler is called.
//
// Why this didn't exist before #1290 is a mystery to me, but it exists now
// and this apparently is the fix for it
//
// ¯\_(ツ)_/¯ - @DusterTheFirst
};
// So since we are inside a runner lock here, we just postpone the blur/hide:
let on_composition_update = {
move |event: web_sys::CompositionEvent, runner: &mut AppRunner| {
let Some(text) = event.data() else { return };
let event = egui::Event::Ime(egui::ImeEvent::Preedit(text));
runner.input.raw.events.push(event);
runner.needs_repaint.repaint_asap();
}
};
call_after_delay(std::time::Duration::from_millis(0), move || {
input.blur().ok();
input.set_hidden(true);
canvas_style.set_property("position", "absolute").ok();
canvas_style.set_property("top", "0%").ok(); // move back to normal position
});
}
Some(())
}
let on_composition_end = {
let input = input.clone();
move |event: web_sys::CompositionEvent, runner: &mut AppRunner| {
let Some(text) = event.data() else { return };
input.set_value("");
let event = egui::Event::Ime(egui::ImeEvent::Commit(text));
runner.input.raw.events.push(event);
runner.needs_repaint.repaint_asap();
}
};
fn call_after_delay(delay: std::time::Duration, f: impl FnOnce() + 'static) {
use wasm_bindgen::prelude::*;
let window = web_sys::window().unwrap();
let closure = Closure::once(f);
let delay_ms = delay.as_millis() as _;
window
.set_timeout_with_callback_and_timeout_and_arguments_0(
closure.as_ref().unchecked_ref(),
delay_ms,
)
.unwrap();
closure.forget(); // We must forget it, or else the callback is canceled on drop
}
runner_ref.add_event_listener(&input, "input", on_input)?;
runner_ref.add_event_listener(&input, "compositionstart", on_composition_start)?;
runner_ref.add_event_listener(&input, "compositionupdate", on_composition_update)?;
runner_ref.add_event_listener(&input, "compositionend", on_composition_end)?;
/// If context is running under mobile device?
fn is_mobile() -> Option<bool> {
const MOBILE_DEVICE: [&str; 6] = ["Android", "iPhone", "iPad", "iPod", "webOS", "BlackBerry"];
let user_agent = web_sys::window()?.navigator().user_agent().ok()?;
let is_mobile = MOBILE_DEVICE.iter().any(|&name| user_agent.contains(name));
Some(is_mobile)
}
// Move text agent to text cursor's position, on desktop/laptop,
// candidate window moves following text element (agent),
// so it appears that the IME candidate window moves with text cursor.
// On mobile devices, there is no need to do that.
pub fn move_text_cursor(
ime: Option<egui::output::IMEOutput>,
canvas: &web_sys::HtmlCanvasElement,
) -> Option<()> {
let style = text_agent().style();
// Note: moving agent on mobile devices will lead to unpredictable scroll.
if is_mobile() == Some(false) {
ime.as_ref().and_then(|ime| {
let egui::Pos2 { x, y } = ime.cursor_rect.left_top();
let bounding_rect = text_agent().get_bounding_client_rect();
let y = (y + (canvas.scroll_top() + canvas.offset_top()) as f32)
.min(canvas.client_height() as f32 - bounding_rect.height() as f32);
let x = x + (canvas.scroll_left() + canvas.offset_left()) as f32;
// Canvas is translated 50% horizontally in html.
let x = (x - canvas.offset_width() as f32 / 2.0)
.min(canvas.client_width() as f32 - bounding_rect.width() as f32);
style.set_property("position", "absolute").ok()?;
style.set_property("top", &format!("{y}px")).ok()?;
style.set_property("left", &format!("{x}px")).ok()
Ok(Self {
input,
prev_ime_output: Default::default(),
})
} else {
style.set_property("position", "absolute").ok()?;
style.set_property("top", "0px").ok()?;
style.set_property("left", "0px").ok()
}
pub fn move_to(
&self,
ime: Option<egui::output::IMEOutput>,
canvas: &web_sys::HtmlCanvasElement,
) -> Result<(), JsValue> {
// Mobile keyboards don't follow the text input it's writing to,
// instead typically being fixed in place on the bottom of the screen,
// so don't bother moving the text agent on mobile.
if is_mobile() {
return Ok(());
}
// Don't move the text agent unless the position actually changed:
if self.prev_ime_output.get() == ime {
return Ok(());
}
self.prev_ime_output.set(ime);
let Some(ime) = ime else { return Ok(()) };
let ime_pos = ime.cursor_rect.left_top();
let canvas_rect = canvas.get_bounding_client_rect();
let new_pos = ime_pos + egui::vec2(canvas_rect.left() as f32, canvas_rect.top() as f32);
let style = self.input.style();
style.set_property("top", &format!("{}px", new_pos.y))?;
style.set_property("left", &format!("{}px", new_pos.x))?;
Ok(())
}
pub fn set_focus(&self, on: bool) {
if on {
self.focus();
} else {
self.blur();
}
}
pub fn has_focus(&self) -> bool {
super::has_focus(&self.input)
}
fn focus(&self) {
if self.has_focus() {
return;
}
if let Err(err) = self.input.focus() {
log::error!("failed to set focus: {}", super::string_from_js_value(&err));
};
}
fn blur(&self) {
if !self.has_focus() {
return;
}
if let Err(err) = self.input.blur() {
log::error!("failed to set focus: {}", super::string_from_js_value(&err));
};
}
}
impl Drop for TextAgent {
fn drop(&mut self) {
self.input.remove();
}
}
/// Returns `true` if the app is likely running on a mobile device.
fn is_mobile() -> bool {
fn try_is_mobile() -> Option<bool> {
const MOBILE_DEVICE: [&str; 6] =
["Android", "iPhone", "iPad", "iPod", "webOS", "BlackBerry"];
let user_agent = web_sys::window()?.navigator().user_agent().ok()?;
let is_mobile = MOBILE_DEVICE.iter().any(|&name| user_agent.contains(name));
Some(is_mobile)
}
try_is_mobile().unwrap_or(false)
}

7
crates/eframe/src/web/web_runner.rs
View File

@ -4,7 +4,7 @@ use wasm_bindgen::prelude::*;
use crate::{epi, App};
use super::{events, AppRunner, PanicHandler};
use super::{events, text_agent::TextAgent, AppRunner, PanicHandler};
/// This is how `eframe` runs your wepp application
///
@ -65,14 +65,15 @@ impl WebRunner {
let follow_system_theme = web_options.follow_system_theme;
let runner = AppRunner::new(canvas_id, web_options, app_creator).await?;
let text_agent = TextAgent::attach(self)?;
let runner = AppRunner::new(canvas_id, web_options, app_creator, text_agent).await?;
self.runner.replace(Some(runner));
{
events::install_canvas_events(self)?;
events::install_document_events(self)?;
events::install_window_events(self)?;
super::text_agent::install_text_agent(self)?;
if follow_system_theme {
events::install_color_scheme_change_event(self)?;

2
crates/egui/src/data/output.rs
View File

@ -113,7 +113,7 @@ pub struct PlatformOutput {
/// Use by `eframe` web to show/hide mobile keyboard and IME agent.
pub mutable_text_under_cursor: bool,
/// This is et if, and only if, the user is currently editing text.
/// This is set if, and only if, the user is currently editing text.
///
/// Useful for IME.
pub ime: Option<IMEOutput>,