fix: better heuristic for clickable detection (#51)

Author: mtraworld

README.md

@@ -72,7 +72,6 @@ This is an intentional design choice to keep the project lean, maintainable, and
 - Sort of working in Mision Control, but it still shows hints from the frontmost app. How can we know that we are in mission control and ignore the frontmost app?
 - Firefox seems fine but still the first activation shows very minimal hint after setting it up, same as electron apps
 - Research if there's a good way to implemet visual hint mode where we can select text? Doing this with accessibility seems a little hard and vague
-- Current `isClickable` heuristic implementation doesn't looks good enough to cover most cases, think of a way that we can determine if the target is actually clickable with a reliable way
 - Add more actions to the menubar like `status`, `stop`, `start`, `current version`
 - Test suites, but am lazy for it
 - Implements launch agent with `start-service` and `stop-service`? Though I am fine just doing it in my nix config directly

configs/default-config.toml

@@ -57,17 +57,14 @@ clickable_roles = [
     "AXPopUpButton",
     "AXTextField",
     "AXSlider",
-    "AXTabGroup",
     "AXTabButton",
     "AXSwitch",
-    "AXToolbar",
     "AXDisclosureTriangle",
     "AXTextArea",
     "AXMenuButton",
     "AXMenuItem",
-    "AXGroup",
-    "AXImage",
     "AXCell",
+    "AXRow",
 ]
 
 # Global accessibility roles that are treated as scrollable
@@ -89,6 +86,13 @@ scrollable_roles = [
 # # You can also add scrollable roles scoped to the app:
 # # additional_scrollable_roles = ["AXScrollArea"]
 
+# For example, in `Chrome`, we can add support for tabgroups.
+# [[accessibility.app_configs]]
+# bundle_id = "com.google.Chrome"
+# additional_clickable_roles = ["AXTabGroup"]
+
+# You can do and customise alot more when the defaults or global are not good enough.
+
 [accessibility.electron_support]
 ## Electron & Chromium support
 # Enables manual accessibility toggles for Electron-like apps.

internal/accessibility/query.go

@@ -67,10 +67,10 @@ func GetClickableElements() ([]*TreeNode, error) {
 	opts := DefaultTreeOptions()
 	if isElectron {
 		// For Electron apps, go deeper to find web content
-		opts.MaxDepth = 25
+		opts.MaxDepth = 40
 		logger.Debug("Detected Electron app, using deeper tree traversal for scroll areas")
 	} else {
-		opts.MaxDepth = 10
+		opts.MaxDepth = 25
 	}
 	opts.FilterFunc = func(info *ElementInfo) bool {
 		// Filter out very small elements
@@ -113,10 +113,10 @@ func GetScrollableElements() ([]*TreeNode, error) {
 	opts := DefaultTreeOptions()
 	if isElectron {
 		// For Electron apps, go deeper to find web content
-		opts.MaxDepth = 15
+		opts.MaxDepth = 20
 		logger.Debug("Detected Electron app, using deeper tree traversal for scroll areas")
 	} else {
-		opts.MaxDepth = 5
+		opts.MaxDepth = 10
 	}
 
 	tree, err := BuildTree(window, opts)
@@ -142,7 +142,7 @@ func GetMenuBarClickableElements() ([]*TreeNode, error) {
 	defer menubar.Release()
 
 	opts := DefaultTreeOptions()
-	opts.MaxDepth = 8
+	opts.MaxDepth = 10
 	// Filter out tiny elements
 	opts.FilterFunc = func(info *ElementInfo) bool {
 		if info.Size.X < 6 || info.Size.Y < 6 {
@@ -172,7 +172,7 @@ func GetDockClickableElements() ([]*TreeNode, error) {
 	opts := DefaultTreeOptions()
 	opts.IncludeOutOfBounds = true
 	opts.CheckOcclusion = false
-	opts.MaxDepth = 8
+	opts.MaxDepth = 10
 	opts.FilterFunc = func(info *ElementInfo) bool {
 		if info.Size.X < 6 || info.Size.Y < 6 {
 			return false
@@ -201,7 +201,7 @@ func GetNCClickableElements() ([]*TreeNode, error) {
 	opts := DefaultTreeOptions()
 	opts.IncludeOutOfBounds = true
 	opts.CheckOcclusion = false
-	opts.MaxDepth = 8
+	opts.MaxDepth = 10
 	opts.FilterFunc = func(info *ElementInfo) bool {
 		if info.Size.X < 6 || info.Size.Y < 6 {
 			return false

internal/bridge/accessibility.m

@@ -506,99 +506,94 @@ int hasClickAction(void* element) {
     if (!element) return 0;
     AXUIElementRef axElement = (AXUIElementRef)element;
 
-    // Check explicit actions
+    // Ignore hidden or disabled elements early
+    CFBooleanRef hidden = NULL;
+    if (AXUIElementCopyAttributeValue(axElement, kAXHiddenAttribute, (CFTypeRef *)&hidden) == kAXErrorSuccess && hidden) {
+        if (CFBooleanGetValue(hidden)) {
+            CFRelease(hidden);
+            return 0;
+        }
+        CFRelease(hidden);
+    }
+
+    CFBooleanRef enabled = NULL;
+    bool isEnabled = true; // default to true if attribute not present
+    if (AXUIElementCopyAttributeValue(axElement, kAXEnabledAttribute, (CFTypeRef *)&enabled) == kAXErrorSuccess && enabled) {
+        isEnabled = CFBooleanGetValue(enabled);
+        CFRelease(enabled);
+    }
+    if (!isEnabled) return 0;
+
+    // Get role for role-specific fallbacks
+    CFStringRef role = NULL;
+    if (AXUIElementCopyAttributeValue(axElement, kAXRoleAttribute, (CFTypeRef *)&role) != kAXErrorSuccess) {
+        role = NULL;
+    }
+
+    // Explicit actions are the strongest signal
     CFArrayRef actions = NULL;
     if (AXUIElementCopyActionNames(axElement, &actions) == kAXErrorSuccess && actions) {
         CFIndex count = CFArrayGetCount(actions);
         for (CFIndex i = 0; i < count; i++) {
             CFStringRef action = (CFStringRef)CFArrayGetValueAtIndex(actions, i);
             if (CFStringCompare(action, kAXPressAction, 0) == kCFCompareEqualTo ||
+                CFStringCompare(action, CFSTR("AXShowMenu"), 0) == kCFCompareEqualTo ||
                 CFStringCompare(action, CFSTR("AXConfirm"), 0) == kCFCompareEqualTo ||
                 CFStringCompare(action, CFSTR("AXPick"), 0) == kCFCompareEqualTo ||
-                CFStringCompare(action, CFSTR("AXShowMenu"), 0) == kCFCompareEqualTo) {
+                CFStringCompare(action, CFSTR("AXRaise"), 0) == kCFCompareEqualTo) {
                 CFRelease(actions);
+                if (role) CFRelease(role);
                 return 1;
             }
         }
         CFRelease(actions);
     }
 
-    // Focusable + enabled
-    CFBooleanRef enabled = NULL;
-    if (AXUIElementCopyAttributeValue(axElement, kAXEnabledAttribute, (CFTypeRef *)&enabled) == kAXErrorSuccess && enabled) {
-        if (CFBooleanGetValue(enabled)) {
-            CFRelease(enabled);
-            CFBooleanRef focusable = NULL;
-            if (AXUIElementCopyAttributeValue(axElement, kAXFocusableAttribute, (CFTypeRef *)&focusable) == kAXErrorSuccess && focusable) {
-                int isFocus = CFBooleanGetValue(focusable);
-                CFRelease(focusable);
-                if (isFocus) return 1;
-            }
-        } else CFRelease(enabled);
-    }
-
-    // Has title or label
-    CFTypeRef title = NULL;
-    if (AXUIElementCopyAttributeValue(axElement, kAXTitleAttribute, &title) == kAXErrorSuccess && title) {
-        CFRelease(title);
-        return 1;
-    }
-
-    // Has description or help
-    CFTypeRef desc = NULL;
-    if (AXUIElementCopyAttributeValue(axElement, kAXDescriptionAttribute, &desc) == kAXErrorSuccess && desc) {
-        CFRelease(desc);
+    // Some elements support AXPress even if not listed in action names
+    CFStringRef pressDesc = NULL;
+    if (AXUIElementCopyActionDescription(axElement, kAXPressAction, &pressDesc) == kAXErrorSuccess && pressDesc) {
+        CFRelease(pressDesc);
+        if (role) CFRelease(role);
         return 1;
     }
-
-    // Has value or selected state
-    CFTypeRef value = NULL;
-    if (AXUIElementCopyAttributeValue(axElement, kAXValueAttribute, &value) == kAXErrorSuccess && value) {
-        CFRelease(value);
-        return 1;
-    }
-
-    CFTypeRef selected = NULL;
-    if (AXUIElementCopyAttributeValue(axElement, kAXSelectedAttribute, &selected) == kAXErrorSuccess && selected) {
-        CFRelease(selected);
-        return 1;
+    if (pressDesc) CFRelease(pressDesc);
+
+    // Focusable and enabled controls are clickable (e.g., text fields)
+    CFBooleanRef focusable = NULL;
+    if (AXUIElementCopyAttributeValue(axElement, kAXFocusableAttribute, (CFTypeRef *)&focusable) == kAXErrorSuccess && focusable) {
+        if (CFBooleanGetValue(focusable)) {
+            CFRelease(focusable);
+            // Ensure the element has a hittable rect and is not occluded by other apps
+            CGPoint center;
+            pid_t pid;
+            bool visible = true;
+            if (getElementCenter((void*)axElement, &center) && AXUIElementGetPid(axElement, &pid) == kAXErrorSuccess) {
+                visible = isPointVisible(center, pid);
+            }
+            if (role) CFRelease(role);
+            return visible ? 1 : 0;
+        }
+        CFRelease(focusable);
     }
 
-    // Has visual bounds
-    CFTypeRef position = NULL;
-    CFTypeRef size = NULL;
-    if (AXUIElementCopyAttributeValue(axElement, kAXPositionAttribute, &position) == kAXErrorSuccess &&
-        AXUIElementCopyAttributeValue(axElement, kAXSizeAttribute, &size) == kAXErrorSuccess &&
-        position && size) {
-        CFRelease(position);
-        CFRelease(size);
-        return 1;
+    // Role-specific fallback for links: Treat as clickable if they expose a URL
+    if (role && CFStringCompare(role, kAXLinkRole, 0) == kCFCompareEqualTo) {
+        CFTypeRef urlAttr = NULL;
+        if (AXUIElementCopyAttributeValue(axElement, kAXURLAttribute, &urlAttr) == kAXErrorSuccess && urlAttr) {
+            CFRelease(urlAttr);
+            CFRelease(role);
+            return 1;
+        }
+        if (urlAttr) CFRelease(urlAttr);
     }
-    if (position) CFRelease(position);
-    if (size) CFRelease(size);
 
-    // Expanded/collapsible state
-    CFTypeRef expanded = NULL;
-    if (AXUIElementCopyAttributeValue(axElement, kAXExpandedAttribute, &expanded) == kAXErrorSuccess && expanded) {
-        CFRelease(expanded);
-        return 1;
-    }
+    if (role) CFRelease(role);
 
-    // Visible + leaf node
-    CFBooleanRef visible = NULL;
-    if (AXUIElementCopyAttributeValue(axElement, kAXVisibleAttribute, (CFTypeRef *)&visible) == kAXErrorSuccess && visible) {
-        if (CFBooleanGetValue(visible)) {
-            CFArrayRef children = NULL;
-            if (AXUIElementCopyAttributeValue(axElement, kAXChildrenAttribute, (CFTypeRef *)&children) == kAXErrorSuccess) {
-                if (children && CFArrayGetCount(children) == 0) {
-                    CFRelease(children);
-                    CFRelease(visible);
-                    return 1;
-                }
-                if (children) CFRelease(children);
-            }
-        }
-        CFRelease(visible);
+    // final check, ensure a visible bounding box and not occluded
+    CGPoint center;
+    pid_t pid;
+    if (getElementCenter((void*)axElement, &center) && AXUIElementGetPid(axElement, &pid) == kAXErrorSuccess) {
+        return isPointVisible(center, pid) ? 1 : 0;
     }
 
     return 0;

internal/config/config.go

@@ -126,17 +126,14 @@ func DefaultConfig() *Config {
 				"AXPopUpButton",
 				"AXTextField",
 				"AXSlider",
-				"AXTabGroup",
 				"AXTabButton",
 				"AXSwitch",
-				"AXToolbar",
 				"AXDisclosureTriangle",
 				"AXTextArea",
 				"AXMenuButton",
 				"AXMenuItem",
-				"AXGroup",
-				"AXImage",
 				"AXCell",
+				"AXRow",
 			},
 			ScrollableRoles: []string{
 				"AXScrollArea",