feat: add occlusion detection with Dock exception (#34)

Author: mtraworld

internal/accessibility/element.go

@@ -212,21 +212,34 @@ func (e *Element) GetInfo() (*ElementInfo, error) {
 	return info, nil
 }
 
-// GetChildren returns all child elements
+// GetChildren returns all child elements with optional occlusion checking
 func (e *Element) GetChildren() ([]*Element, error) {
 	if e.ref == nil {
 		return nil, fmt.Errorf("element is nil")
 	}
 
-	return e.getChildrenInternal(true)
+	return e.getChildrenInternal(true, false) // visible only, but no occlusion check by default
 }
 
-func (e *Element) getChildrenInternal(visibleOnly bool) ([]*Element, error) {
+// GetChildrenWithOcclusionCheck returns child elements with strict occlusion filtering
+func (e *Element) GetChildrenWithOcclusionCheck(visibleOnly, checkOcclusion bool) ([]*Element, error) {
+	if e.ref == nil {
+		return nil, fmt.Errorf("element is nil")
+	}
+
+	return e.getChildrenInternal(visibleOnly, checkOcclusion)
+}
+
+func (e *Element) getChildrenInternal(visibleOnly bool, checkOcclusion bool) ([]*Element, error) {
 	var count C.int
 	var rawChildren unsafe.Pointer
 
 	if visibleOnly {
-		rawChildren = unsafe.Pointer(C.getVisibleChildren(e.ref, &count))
+		occlusionFlag := C.int(0)
+		if checkOcclusion {
+			occlusionFlag = C.int(1)
+		}
+		rawChildren = unsafe.Pointer(C.getVisibleChildren(e.ref, &count, occlusionFlag))
 	} else {
 		rawChildren = unsafe.Pointer(C.getChildren(e.ref, &count))
 	}

internal/accessibility/query.go

@@ -144,6 +144,7 @@ func GetDockClickableElements() ([]*TreeNode, error) {
 
 	opts := DefaultTreeOptions()
 	opts.IncludeOutOfBounds = true
+	opts.CheckOcclusion = false
 	opts.MaxDepth = 8
 	opts.FilterFunc = func(info *ElementInfo) bool {
 		if info.Size.X < 6 || info.Size.Y < 6 {

internal/accessibility/tree.go

@@ -20,6 +20,7 @@ type TreeOptions struct {
 	MaxDepth           int
 	FilterFunc         func(*ElementInfo) bool
 	IncludeOutOfBounds bool
+	CheckOcclusion     bool
 }
 
 // DefaultTreeOptions returns default tree traversal options
@@ -28,6 +29,7 @@ func DefaultTreeOptions() TreeOptions {
 		MaxDepth:           10,
 		FilterFunc:         nil,
 		IncludeOutOfBounds: false,
+		CheckOcclusion:     false,
 	}
 }
 
@@ -64,7 +66,7 @@ func buildTreeRecursive(parent *TreeNode, depth int, opts TreeOptions, windowBou
 	if depth >= opts.MaxDepth {
 		return
 	}
-	children, err := parent.Element.GetChildren()
+	children, err := parent.Element.GetChildrenWithOcclusionCheck(true, opts.CheckOcclusion)
 	if err != nil || len(children) == 0 {
 		return
 	}

internal/bridge/accessibility.h

@@ -28,7 +28,7 @@ void freeElementInfo(ElementInfo* info);
 void* getElementAtPosition(CGPoint position);
 int getChildrenCount(void* element);
 void** getChildren(void* element, int* count);
-void** getVisibleChildren(void* element, int* count);
+void** getVisibleChildren(void* element, int* count, int checkOcclusion);
 int performClick(void* element);
 int performRightClick(void* element);
 int performDoubleClick(void* element);

internal/bridge/accessibility.m

@@ -35,34 +35,219 @@ int setApplicationAttribute(int pid, const char* attribute, int value) {
     return (error == kAXErrorSuccess) ? 1 : 0;
 }
 
-void** getVisibleChildren(void* element, int* count) {
+// Helper function to check if a point is actually visible (not occluded by other windows)
+static bool isPointVisible(CGPoint point, pid_t elementPid) {
+    // Use AXUIElementCopyElementAtPosition to check what's at this point
+    AXUIElementRef systemWide = AXUIElementCreateSystemWide();
+    if (!systemWide) return true;
+
+    AXUIElementRef elementAtPoint = NULL;
+    AXError error = AXUIElementCopyElementAtPosition(systemWide, point.x, point.y, &elementAtPoint);
+    CFRelease(systemWide);
+
+    if (error != kAXErrorSuccess || !elementAtPoint) {
+        return true; // Assume visible if we can't check
+    }
+
+    // Get the PID of the element at this point
+    pid_t pidAtPoint;
+    bool isVisible = false;
+
+    if (AXUIElementGetPid(elementAtPoint, &pidAtPoint) == kAXErrorSuccess) {
+        // The point is visible if it belongs to the same application
+        isVisible = (pidAtPoint == elementPid);
+    }
+
+    CFRelease(elementAtPoint);
+    return isVisible;
+}
+
+// Helper function to check if an element is occluded by checking multiple sample points
+static bool isElementOccluded(CGRect elementRect, pid_t elementPid) {
+    // Sample 5 points: center and 4 corners (slightly inset)
+    CGFloat inset = 2.0; // Inset from edges to avoid border issues
+
+    CGPoint samplePoints[5] = {
+        // Center
+        CGPointMake(elementRect.origin.x + elementRect.size.width / 2,
+                   elementRect.origin.y + elementRect.size.height / 2),
+        // Top-left
+        CGPointMake(elementRect.origin.x + inset,
+                   elementRect.origin.y + inset),
+        // Top-right
+        CGPointMake(elementRect.origin.x + elementRect.size.width - inset,
+                   elementRect.origin.y + inset),
+        // Bottom-left
+        CGPointMake(elementRect.origin.x + inset,
+                   elementRect.origin.y + elementRect.size.height - inset),
+        // Bottom-right
+        CGPointMake(elementRect.origin.x + elementRect.size.width - inset,
+                   elementRect.origin.y + elementRect.size.height - inset)
+    };
+
+    // Element is considered visible if at least 2 sample points are visible
+    int visiblePoints = 0;
+    for (int i = 0; i < 5; i++) {
+        if (isPointVisible(samplePoints[i], elementPid)) {
+            visiblePoints++;
+            if (visiblePoints >= 2) {
+                return false; // Not occluded
+            }
+        }
+    }
+
+    return true; // Occluded (less than 2 points visible)
+}
+
+// Get children that are actually visible on screen
+// checkOcclusion: if true, filters out elements covered by other windows
+void** getVisibleChildren(void* element, int* count, int checkOcclusion) {
     if (!element || !count) return NULL;
 
     AXUIElementRef axElement = (AXUIElementRef)element;
-    CFTypeRef visibleChildrenValue = NULL;
 
-    if (AXUIElementCopyAttributeValue(axElement, kAXVisibleChildrenAttribute, &visibleChildrenValue) == kAXErrorSuccess && visibleChildrenValue) {
-        if (CFGetTypeID(visibleChildrenValue) == CFArrayGetTypeID()) {
-            CFArrayRef children = (CFArrayRef)visibleChildrenValue;
-            CFIndex childCount = CFArrayGetCount(children);
-            *count = (int)childCount;
+    // Get the PID of the element's application
+    pid_t elementPid;
+    if (AXUIElementGetPid(axElement, &elementPid) != kAXErrorSuccess) {
+        *count = 0;
+        return NULL;
+    }
+
+    // Check if this is the Dock
+    NSRunningApplication* app = [NSRunningApplication runningApplicationWithProcessIdentifier:elementPid];
+    BOOL isDock = app && [[app bundleIdentifier] isEqualToString:@"com.apple.dock"];
 
-            void** result = (void**)malloc(childCount * sizeof(void*));
+    // First, get all children
+    int totalCount = 0;
+    void** allChildren = getChildren(element, &totalCount);
 
-            for (CFIndex i = 0; i < childCount; i++) {
-                AXUIElementRef child = (AXUIElementRef)CFArrayGetValueAtIndex(children, i);
-                CFRetain(child);
-                result[i] = (void*)child;
+    if (!allChildren || totalCount == 0) {
+        *count = 0;
+        return NULL;
+    }
+
+    // For the Dock, skip all filtering and return all children
+    if (isDock) {
+        *count = totalCount;
+        return allChildren;
+    }
+
+    // For non-Dock apps, continue with normal filtering...
+
+    // Try to get the parent element's bounds (if available)
+    CGRect parentBounds = CGRectZero;
+    bool hasParentBounds = false;
+
+    CFTypeRef posValue = NULL, sizeValue = NULL;
+
+    if (AXUIElementCopyAttributeValue(axElement, kAXPositionAttribute, &posValue) == kAXErrorSuccess && posValue) {
+        if (AXValueGetValue(posValue, kAXValueCGPointType, &parentBounds.origin)) {
+            if (AXUIElementCopyAttributeValue(axElement, kAXSizeAttribute, &sizeValue) == kAXErrorSuccess && sizeValue) {
+                if (AXValueGetValue(sizeValue, kAXValueCGSizeType, &parentBounds.size)) {
+                    hasParentBounds = true;
+                }
+                CFRelease(sizeValue);
             }
+        }
+        CFRelease(posValue);
+    }
+
+    // Get screen bounds for all displays
+    uint32_t displayCount;
+    CGDirectDisplayID displays[32];
+    CGGetActiveDisplayList(32, displays, &displayCount);
+
+    // Create a union of all display bounds
+    CGRect allScreensBounds = CGRectZero;
+    for (uint32_t i = 0; i < displayCount; i++) {
+        CGRect displayBounds = CGDisplayBounds(displays[i]);
+        if (i == 0) {
+            allScreensBounds = displayBounds;
+        } else {
+            allScreensBounds = CGRectUnion(allScreensBounds, displayBounds);
+        }
+    }
+
+    // Temporary array to hold visible children
+    void** visibleChildren = (void**)malloc(totalCount * sizeof(void*));
+    int visibleCount = 0;
+
+    for (int i = 0; i < totalCount; i++) {
+        AXUIElementRef child = (AXUIElementRef)allChildren[i];
+
+        // Get child's position and size
+        CGPoint childPos = CGPointZero;
+        CGSize childSize = CGSizeZero;
+        bool hasPosition = false, hasSize = false;
+
+        CFTypeRef childPosValue = NULL;
+        if (AXUIElementCopyAttributeValue(child, kAXPositionAttribute, &childPosValue) == kAXErrorSuccess && childPosValue) {
+            if (AXValueGetValue(childPosValue, kAXValueCGPointType, &childPos)) {
+                hasPosition = true;
+            }
+            CFRelease(childPosValue);
+        }
+
+        CFTypeRef childSizeValue = NULL;
+        if (AXUIElementCopyAttributeValue(child, kAXSizeAttribute, &childSizeValue) == kAXErrorSuccess && childSizeValue) {
+            if (AXValueGetValue(childSizeValue, kAXValueCGSizeType, &childSize)) {
+                hasSize = true;
+            }
+            CFRelease(childSizeValue);
+        }
+
+        // If we can't get position/size, include it anyway (might be a container)
+        if (!hasPosition && !hasSize) {
+            visibleChildren[visibleCount++] = (void*)child;
+            continue;
+        }
 
-            CFRelease(visibleChildrenValue);
-            return result;
+        // Check if child has non-zero size (if size is available)
+        if (hasSize && (childSize.width <= 0 || childSize.height <= 0)) {
+            CFRelease(child);
+            continue;
         }
 
-        CFRelease(visibleChildrenValue);
+        // If we have position, do spatial checks
+        if (hasPosition && hasSize) {
+            CGRect childRect = CGRectMake(childPos.x, childPos.y, childSize.width, childSize.height);
+
+            // Check if child intersects with any screen bounds
+            if (!CGRectIntersectsRect(childRect, allScreensBounds)) {
+                CFRelease(child);
+                continue;
+            }
+
+            // If parent has bounds, check if child intersects with parent
+            if (hasParentBounds && !CGRectIntersectsRect(childRect, parentBounds)) {
+                CFRelease(child);
+                continue;
+            }
+
+            // Only check occlusion if requested
+            if (checkOcclusion && isElementOccluded(childRect, elementPid)) {
+                CFRelease(child);
+                continue;
+            }
+        }
+
+        // This child is visible, add it to our result
+        visibleChildren[visibleCount++] = (void*)child;
     }
 
-    return getChildren(element, count);
+    free(allChildren);
+
+    if (visibleCount == 0) {
+        free(visibleChildren);
+        *count = 0;
+        return NULL;
+    }
+
+    // Resize array to actual visible count
+    void** result = (void**)realloc(visibleChildren, visibleCount * sizeof(void*));
+    *count = visibleCount;
+
+    return result ? result : visibleChildren;
 }
 
 // Get system-wide accessibility element