[fix] corner cases win nb_children < nb_master|nb_col

1 files changed, 40 insertions, 21 deletions

diff --git a/sway/layout.c b/sway/layout.c
index 5f8da9e6..9661f505 100644
--- a/sway/layout.c
+++ b/sway/layout.c
@@ -1505,11 +1505,32 @@ bool is_auto_layout(enum swayc_layouts layout) {
 }
 
 /**
+ * Return the number of master elements in a container
+ */
+static inline size_t auto_master_count(swayc_t *container) {
+        return MIN(container->nb_master, container->children->length);
+}
+
+/**
  * Return the number of children in the slave groups. This corresponds to the children
  * that are not members of the master group.
  */
 static inline size_t auto_slave_count(swayc_t *container) {
-        return container->children->length - container->nb_master;
+        return container->children->length - auto_master_count(container);
+}
+
+/**
+ * Return the number of slave groups in the container.
+ */
+size_t auto_slave_group_count(swayc_t *container) {
+        return MIN(container->nb_slave_groups, auto_slave_count(container));
+}
+
+/**
+ * Return the combined number of master and slave groups in the container.
+ */
+size_t auto_group_count(swayc_t *container) {
+        return auto_slave_group_count(container) + (container->nb_master ? 1 : 0);
 }
 
 /**
@@ -1521,10 +1542,12 @@ int auto_group_start_index(swayc_t *container, int index) {
                 || (size_t) index < container->nb_master) {
                 return 0;
         } else {
-                size_t grp_sz = auto_slave_count(container) / container->nb_slave_groups;
-                size_t remainder = auto_slave_count(container) % container->nb_slave_groups;
+                size_t nb_slaves = auto_slave_count(container);
+                size_t nb_slave_grp = auto_slave_group_count(container);
+                size_t grp_sz = nb_slaves / nb_slave_grp;
+                size_t remainder = nb_slaves % nb_slave_grp;
+                int idx2 = (nb_slave_grp - remainder) * grp_sz + container->nb_master;
                 int start_idx;
-                int idx2 = (container->nb_slave_groups - remainder) * grp_sz + container->nb_master;
                 if (index < idx2) {
                         start_idx = ((index - container->nb_master) / grp_sz) * grp_sz + container->nb_master;
                 } else {
@@ -1545,11 +1568,13 @@ int auto_group_end_index(swayc_t *container, int index) {
         } else {
                 int nxt_idx;
                 if ((size_t)index < container->nb_master) {
-                        nxt_idx = container->nb_master;
+                        nxt_idx = auto_master_count(container);
                 } else {
-                        size_t grp_sz = auto_slave_count(container) / container->nb_slave_groups;
-                        size_t remainder = auto_slave_count(container) % container->nb_slave_groups;
-                        int idx2 = (container->nb_slave_groups - remainder) * grp_sz + container->nb_master;
+                        size_t nb_slaves = auto_slave_count(container);
+                        size_t nb_slave_grp = auto_slave_group_count(container);
+                        size_t grp_sz = nb_slaves / nb_slave_grp;
+                        size_t remainder = nb_slaves % nb_slave_grp;
+                        int idx2 = (nb_slave_grp - remainder) * grp_sz + container->nb_master;
                         if (index < idx2) {
                                 nxt_idx = ((index - container->nb_master) / grp_sz + 1) * grp_sz + container->nb_master;
                         } else {
@@ -1561,13 +1586,6 @@ int auto_group_end_index(swayc_t *container, int index) {
 }
 
 /**
- * Return the combined number of master and slave groups in the container.
- */
-size_t auto_group_count(swayc_t *container) {
-        return MIN(container->nb_slave_groups, auto_slave_count(container)) + (container->nb_master ? 1 : 0);
-}
-
-/**
  * return the index of the Group containing <index>th child of <container>.
  * The index is the order of the group along the container's major axis (starting at 0).
  */
@@ -1576,22 +1594,23 @@ size_t auto_group_index(swayc_t *container, int index) {
                 return 0;
         }
         bool master_first = (container->layout == L_AUTO_LEFT || container->layout == L_AUTO_TOP);
-        int nb_slaves = auto_slave_count(container);
+        size_t nb_slaves = auto_slave_count(container);
         if ((size_t) index < container->nb_master) {
                 if (master_first || nb_slaves <= 0) {
                         return 0;
                 } else {
-                        return MIN(container->nb_slave_groups, nb_slaves);
+                        return auto_slave_group_count(container);
                 }
         } else {
-                size_t grp_sz = auto_slave_count(container) / container->nb_slave_groups;
-                size_t remainder = auto_slave_count(container) % container->nb_slave_groups;
+                size_t nb_slave_grp = auto_slave_group_count(container);
+                size_t grp_sz = nb_slaves / nb_slave_grp;
+                size_t remainder = nb_slaves % nb_slave_grp;
+                int idx2 = (nb_slave_grp - remainder) * grp_sz + container->nb_master;
                 size_t grp_idx;
-                int idx2 = (container->nb_slave_groups - remainder) * grp_sz + container->nb_master;
                 if (index < idx2) {
                         grp_idx = (index - container->nb_master) / grp_sz;
                 } else {
-                        grp_idx = (container->nb_slave_groups - remainder) + (index - idx2) / (grp_sz + 1) ;
+                        grp_idx = (nb_slave_grp - remainder) + (index - idx2) / (grp_sz + 1) ;
                 }
                 return grp_idx + (master_first ? 1 : 0);
         }