change apply_exclusive() to closer match layer shell protocol

With these changes, sway will respect positive exclusive zones of layer
surfaces anchored to one or three sides.

This matches the protocol, which states that a positive exclusive zone
should be respected, "if the surface is anchored to one edge or an
edge and both perpendicular edges". If the surfaces is "anchored to
only two perpendicular edges (a corner), anchored to only two
parallel edges or anchored to all edges a positive value will be
treated the same as zero".

1 files changed, 17 insertions, 6 deletions

diff --git a/sway/desktop/layer_shell.c b/sway/desktop/layer_shell.c
index b1ee8ae3..60a8f06b 100644
--- a/sway/desktop/layer_shell.c
+++ b/sway/desktop/layer_shell.c
@@ -25,13 +25,16 @@ static void apply_exclusive(struct wlr_box *usable_area,
                 return;
         }
         struct {
-                uint32_t anchors;
+                uint32_t singular_anchor;
+                uint32_t anchor_triplet;
                 int *positive_axis;
                 int *negative_axis;
                 int margin;
         } edges[] = {
+                // Top
                 {
-                        .anchors =
+                        .singular_anchor = ZWLR_LAYER_SURFACE_V1_ANCHOR_TOP,
+                        .anchor_triplet =
                                 ZWLR_LAYER_SURFACE_V1_ANCHOR_LEFT |
                                 ZWLR_LAYER_SURFACE_V1_ANCHOR_RIGHT |
                                 ZWLR_LAYER_SURFACE_V1_ANCHOR_TOP,
@@ -39,8 +42,10 @@ static void apply_exclusive(struct wlr_box *usable_area,
                         .negative_axis = &usable_area->height,
                         .margin = margin_top,
                 },
+                // Bottom
                 {
-                        .anchors =
+                        .singular_anchor = ZWLR_LAYER_SURFACE_V1_ANCHOR_BOTTOM,
+                        .anchor_triplet =
                                 ZWLR_LAYER_SURFACE_V1_ANCHOR_LEFT |
                                 ZWLR_LAYER_SURFACE_V1_ANCHOR_RIGHT |
                                 ZWLR_LAYER_SURFACE_V1_ANCHOR_BOTTOM,
@@ -48,8 +53,10 @@ static void apply_exclusive(struct wlr_box *usable_area,
                         .negative_axis = &usable_area->height,
                         .margin = margin_bottom,
                 },
+                // Left
                 {
-                        .anchors =
+                        .singular_anchor = ZWLR_LAYER_SURFACE_V1_ANCHOR_LEFT,
+                        .anchor_triplet =
                                 ZWLR_LAYER_SURFACE_V1_ANCHOR_LEFT |
                                 ZWLR_LAYER_SURFACE_V1_ANCHOR_TOP |
                                 ZWLR_LAYER_SURFACE_V1_ANCHOR_BOTTOM,
@@ -57,8 +64,10 @@ static void apply_exclusive(struct wlr_box *usable_area,
                         .negative_axis = &usable_area->width,
                         .margin = margin_left,
                 },
+                // Right
                 {
-                        .anchors =
+                        .singular_anchor = ZWLR_LAYER_SURFACE_V1_ANCHOR_RIGHT,
+                        .anchor_triplet =
                                 ZWLR_LAYER_SURFACE_V1_ANCHOR_RIGHT |
                                 ZWLR_LAYER_SURFACE_V1_ANCHOR_TOP |
                                 ZWLR_LAYER_SURFACE_V1_ANCHOR_BOTTOM,
@@ -68,13 +77,15 @@ static void apply_exclusive(struct wlr_box *usable_area,
                 },
         };
         for (size_t i = 0; i < sizeof(edges) / sizeof(edges[0]); ++i) {
-                if ((anchor & edges[i].anchors) == edges[i].anchors && exclusive + edges[i].margin > 0) {
+                if ((anchor  == edges[i].singular_anchor || anchor == edges[i].anchor_triplet)
+                                && exclusive + edges[i].margin > 0) {
                         if (edges[i].positive_axis) {
                                 *edges[i].positive_axis += exclusive + edges[i].margin;
                         }
                         if (edges[i].negative_axis) {
                                 *edges[i].negative_axis -= exclusive + edges[i].margin;
                         }
+                        break;
                 }
         }
 }