Report egui::Window contents as children to accesskit (#5240)

Previously, all widgets would be listed in accesskit as children of the
toplevel window.
With this change, they will be reported as child of the `egui::Window`
they are in, Which should increase parseability of the ui for
screenreaders and integration tests.

Added an accesskit test to check that it is indeed working.

Co-authored-by: Wybe Westra <w.westra@kwantcontrols.nl>

crates/egui/src/containers/area.rs

@@ -569,6 +569,14 @@ impl Prepared {
         ui
     }
 
+    pub(crate) fn with_widget_info(&self, make_info: impl Fn() -> crate::WidgetInfo) {
+        self.move_response.widget_info(make_info);
+    }
+
+    pub(crate) fn id(&self) -> Id {
+        self.move_response.id
+    }
+
     #[allow(clippy::needless_pass_by_value)] // intentional to swallow up `content_ui`.
     pub(crate) fn end(self, ctx: &Context, content_ui: Ui) -> Response {
         let Self {

crates/egui/src/containers/collapsing_header.rs

@@ -87,6 +87,14 @@ impl CollapsingState {
     ) -> Response {
         let (_id, rect) = ui.allocate_space(button_size);
         let response = ui.interact(rect, self.id, Sense::click());
+        response.widget_info(|| {
+            WidgetInfo::labeled(
+                WidgetType::Button,
+                ui.is_enabled(),
+                if self.is_open() { "Hide" } else { "Show" },
+            )
+        });
+
         if response.clicked() {
             self.toggle(ui);
         }

crates/egui/src/containers/window.rs

@@ -5,7 +5,7 @@ use std::sync::Arc;
 use crate::collapsing_header::CollapsingState;
 use crate::{
     Align, Align2, Context, CursorIcon, Id, InnerResponse, LayerId, NumExt, Order, Response, Sense,
-    TextStyle, Ui, UiKind, Vec2b, WidgetRect, WidgetText,
+    TextStyle, Ui, UiKind, Vec2b, WidgetInfo, WidgetRect, WidgetText, WidgetType,
 };
 use epaint::{emath, pos2, vec2, Galley, Pos2, Rect, RectShape, Rounding, Shape, Stroke, Vec2};
 
@@ -466,6 +466,8 @@ impl<'open> Window<'open> {
         let on_top = Some(area_layer_id) == ctx.top_layer_id();
         let mut area = area.begin(ctx);
 
+        area.with_widget_info(|| WidgetInfo::labeled(WidgetType::Window, true, title.text()));
+
         // Calculate roughly how much larger the window size is compared to the inner rect
         let (title_bar_height, title_content_spacing) = if with_title_bar {
             let style = ctx.style();
@@ -489,8 +491,9 @@ impl<'open> Window<'open> {
 
         // First check for resize to avoid frame delay:
         let last_frame_outer_rect = area.state().rect();
-        let resize_interaction =
+        let resize_interaction = ctx.with_accessibility_parent(area.id(), || {
-            resize_interaction(ctx, possible, area_layer_id, last_frame_outer_rect);
+            resize_interaction(ctx, possible, area_layer_id, last_frame_outer_rect)
+        });
 
         let margins = window_frame.outer_margin.sum()
             + window_frame.inner_margin.sum()
@@ -514,107 +517,109 @@ impl<'open> Window<'open> {
         }
 
         let content_inner = {
-            // BEGIN FRAME --------------------------------
+            ctx.with_accessibility_parent(area.id(), || {
-            let frame_stroke = window_frame.stroke;
+                // BEGIN FRAME --------------------------------
-            let mut frame = window_frame.begin(&mut area_content_ui);
+                let frame_stroke = window_frame.stroke;
+                let mut frame = window_frame.begin(&mut area_content_ui);
 
-            let show_close_button = open.is_some();
+                let show_close_button = open.is_some();
 
-            let where_to_put_header_background = &area_content_ui.painter().add(Shape::Noop);
+                let where_to_put_header_background = &area_content_ui.painter().add(Shape::Noop);
 
-            // Backup item spacing before the title bar
+                // Backup item spacing before the title bar
-            let item_spacing = frame.content_ui.spacing().item_spacing;
+                let item_spacing = frame.content_ui.spacing().item_spacing;
-            // Use title bar spacing as the item spacing before the content
+                // Use title bar spacing as the item spacing before the content
-            frame.content_ui.spacing_mut().item_spacing.y = title_content_spacing;
+                frame.content_ui.spacing_mut().item_spacing.y = title_content_spacing;
 
-            let title_bar = if with_title_bar {
+                let title_bar = if with_title_bar {
-                let title_bar = TitleBar::new(
+                    let title_bar = TitleBar::new(
-                    &mut frame.content_ui,
+                        &mut frame.content_ui,
-                    title,
+                        title,
-                    show_close_button,
+                        show_close_button,
-                    &mut collapsing,
+                        &mut collapsing,
-                    collapsible,
+                        collapsible,
-                );
-                resize.min_size.x = resize.min_size.x.at_least(title_bar.rect.width()); // Prevent making window smaller than title bar width
-                Some(title_bar)
-            } else {
-                None
-            };
-
-            // Remove item spacing after the title bar
-            frame.content_ui.spacing_mut().item_spacing.y = 0.0;
-
-            let (content_inner, mut content_response) = collapsing
-                .show_body_unindented(&mut frame.content_ui, |ui| {
-                    // Restore item spacing for the content
-                    ui.spacing_mut().item_spacing.y = item_spacing.y;
-
-                    resize.show(ui, |ui| {
-                        if scroll.is_any_scroll_enabled() {
-                            scroll.show(ui, add_contents).inner
-                        } else {
-                            add_contents(ui)
-                        }
-                    })
-                })
-                .map_or((None, None), |ir| (Some(ir.inner), Some(ir.response)));
-
-            let outer_rect = frame.end(&mut area_content_ui).rect;
-            paint_resize_corner(
-                &area_content_ui,
-                &possible,
-                outer_rect,
-                frame_stroke,
-                window_frame.rounding,
-            );
-
-            // END FRAME --------------------------------
-
-            if let Some(title_bar) = title_bar {
-                let mut title_rect = Rect::from_min_size(
-                    outer_rect.min,
-                    Vec2 {
-                        x: outer_rect.size().x,
-                        y: title_bar_height,
-                    },
-                );
-
-                title_rect = area_content_ui.painter().round_rect_to_pixels(title_rect);
-
-                if on_top && area_content_ui.visuals().window_highlight_topmost {
-                    let mut round = window_frame.rounding;
-
-                    if !is_collapsed {
-                        round.se = 0.0;
-                        round.sw = 0.0;
-                    }
-
-                    area_content_ui.painter().set(
-                        *where_to_put_header_background,
-                        RectShape::filled(title_rect, round, header_color),
                     );
+                    resize.min_size.x = resize.min_size.x.at_least(title_bar.rect.width()); // Prevent making window smaller than title bar width
+                    Some(title_bar)
+                } else {
+                    None
                 };
 
-                // Fix title bar separator line position
+                // Remove item spacing after the title bar
-                if let Some(response) = &mut content_response {
+                frame.content_ui.spacing_mut().item_spacing.y = 0.0;
-                    response.rect.min.y = outer_rect.min.y + title_bar_height;
+
+                let (content_inner, mut content_response) = collapsing
+                    .show_body_unindented(&mut frame.content_ui, |ui| {
+                        // Restore item spacing for the content
+                        ui.spacing_mut().item_spacing.y = item_spacing.y;
+
+                        resize.show(ui, |ui| {
+                            if scroll.is_any_scroll_enabled() {
+                                scroll.show(ui, add_contents).inner
+                            } else {
+                                add_contents(ui)
+                            }
+                        })
+                    })
+                    .map_or((None, None), |ir| (Some(ir.inner), Some(ir.response)));
+
+                let outer_rect = frame.end(&mut area_content_ui).rect;
+                paint_resize_corner(
+                    &area_content_ui,
+                    &possible,
+                    outer_rect,
+                    frame_stroke,
+                    window_frame.rounding,
+                );
+
+                // END FRAME --------------------------------
+
+                if let Some(title_bar) = title_bar {
+                    let mut title_rect = Rect::from_min_size(
+                        outer_rect.min,
+                        Vec2 {
+                            x: outer_rect.size().x,
+                            y: title_bar_height,
+                        },
+                    );
+
+                    title_rect = area_content_ui.painter().round_rect_to_pixels(title_rect);
+
+                    if on_top && area_content_ui.visuals().window_highlight_topmost {
+                        let mut round = window_frame.rounding;
+
+                        if !is_collapsed {
+                            round.se = 0.0;
+                            round.sw = 0.0;
+                        }
+
+                        area_content_ui.painter().set(
+                            *where_to_put_header_background,
+                            RectShape::filled(title_rect, round, header_color),
+                        );
+                    };
+
+                    // Fix title bar separator line position
+                    if let Some(response) = &mut content_response {
+                        response.rect.min.y = outer_rect.min.y + title_bar_height;
+                    }
+
+                    title_bar.ui(
+                        &mut area_content_ui,
+                        title_rect,
+                        &content_response,
+                        open,
+                        &mut collapsing,
+                        collapsible,
+                    );
                 }
 
-                title_bar.ui(
+                collapsing.store(ctx);
-                    &mut area_content_ui,
-                    title_rect,
-                    &content_response,
-                    open,
-                    &mut collapsing,
-                    collapsible,
-                );
-            }
 
-            collapsing.store(ctx);
+                paint_frame_interaction(&area_content_ui, outer_rect, resize_interaction);
 
-            paint_frame_interaction(&area_content_ui, outer_rect, resize_interaction);
+                content_inner
-
+            })
-            content_inner
         };
 
         let full_response = area.end(ctx, area_content_ui);
@@ -1192,6 +1197,9 @@ impl TitleBar {
 fn close_button(ui: &mut Ui, rect: Rect) -> Response {
     let close_id = ui.auto_id_with("window_close_button");
     let response = ui.interact(rect, close_id, Sense::click());
+    response
+        .widget_info(|| WidgetInfo::labeled(WidgetType::Button, ui.is_enabled(), "Close window"));
+
     ui.expand_to_include_rect(response.rect);
 
     let visuals = ui.style().interact(&response);

crates/egui/src/data/output.rs

@@ -675,6 +675,7 @@ impl WidgetInfo {
             WidgetType::ImageButton => "image button",
             WidgetType::CollapsingHeader => "collapsing header",
             WidgetType::ProgressIndicator => "progress indicator",
+            WidgetType::Window => "window",
             WidgetType::Label | WidgetType::Other => "",
         };
 

crates/egui/src/lib.rs

@@ -656,6 +656,8 @@ pub enum WidgetType {
 
     ProgressIndicator,
 
+    Window,
+
     /// If you cannot fit any of the above slots.
     ///
     /// If this is something you think should be added, file an issue.

crates/egui/src/response.rs

@@ -1032,6 +1032,7 @@ impl Response {
             WidgetType::DragValue => Role::SpinButton,
             WidgetType::ColorButton => Role::ColorWell,
             WidgetType::ProgressIndicator => Role::ProgressIndicator,
+            WidgetType::Window => Role::Window,
             WidgetType::Other => Role::Unknown,
         });
         if !info.enabled {

crates/egui/tests/accesskit.rs

@@ -1,8 +1,8 @@
 //! Tests the accesskit accessibility output of egui.
 #![cfg(feature = "accesskit")]
 
-use accesskit::{Role, TreeUpdate};
+use accesskit::{NodeId, Role, TreeUpdate};
-use egui::{CentralPanel, Context, RawInput};
+use egui::{CentralPanel, Context, RawInput, Window};
 
 /// Baseline test that asserts there are no spurious nodes in the
 /// accesskit output when the ui is empty.
@@ -130,8 +130,30 @@ fn multiple_disabled_widgets() {
     );
 }
 
+#[test]
+fn window_children() {
+    let output = accesskit_output_single_egui_frame(|ctx| {
+        let mut open = true;
+        Window::new("test window")
+            .open(&mut open)
+            .resizable(false)
+            .show(ctx, |ui| {
+                let _ = ui.button("A button");
+            });
+    });
+
+    let root = output.tree.as_ref().map(|tree| tree.root).unwrap();
+
+    let window_id = assert_window_exists(&output, "test window", root);
+    assert_button_exists(&output, "A button", window_id);
+    assert_button_exists(&output, "Close window", window_id);
+    assert_button_exists(&output, "Hide", window_id);
+}
+
 fn accesskit_output_single_egui_frame(run_ui: impl FnMut(&Context)) -> TreeUpdate {
     let ctx = Context::default();
+    // Disable animations, so we do not need to wait for animations to end to see the result.
+    ctx.style_mut(|style| style.animation_time = 0.0);
     ctx.enable_accesskit();
 
     let output = ctx.run(RawInput::default(), run_ui);
@@ -141,3 +163,45 @@ fn accesskit_output_single_egui_frame(run_ui: impl FnMut(&Context)) -> TreeUpdat
         .accesskit_update
         .expect("Missing accesskit update")
 }
+
+#[track_caller]
+fn assert_button_exists(tree: &TreeUpdate, name: &str, parent: NodeId) {
+    let (node_id, _) = tree
+        .nodes
+        .iter()
+        .find(|(_, node)| {
+            !node.is_hidden() && node.role() == Role::Button && node.name() == Some(name)
+        })
+        .expect("No visible button with that name exists.");
+
+    assert_parent_child(tree, parent, *node_id);
+}
+
+#[track_caller]
+fn assert_window_exists(tree: &TreeUpdate, title: &str, parent: NodeId) -> NodeId {
+    let (node_id, _) = tree
+        .nodes
+        .iter()
+        .find(|(_, node)| {
+            !node.is_hidden() && node.role() == Role::Window && node.name() == Some(title)
+        })
+        .expect("No visible window with that title exists.");
+
+    assert_parent_child(tree, parent, *node_id);
+
+    *node_id
+}
+
+#[track_caller]
+fn assert_parent_child(tree: &TreeUpdate, parent: NodeId, child: NodeId) {
+    let (_, parent) = tree
+        .nodes
+        .iter()
+        .find(|(id, _)| id == &parent)
+        .expect("Parent does not exist.");
+
+    assert!(
+        parent.children().contains(&child),
+        "Node is not a child of the given parent."
+    );
+}