Fix output scale auto-detection

Closes: https://github.com/swaywm/sway/issues/4960

1 files changed, 36 insertions, 14 deletions

diff --git a/sway/config/output.c b/sway/config/output.c
index ec21b512..e1925c93 100644
--- a/sway/config/output.c
+++ b/sway/config/output.c
@@ -283,8 +283,28 @@ static bool phys_size_is_aspect_ratio(struct wlr_output *output) {
 #define MM_PER_INCH 25.4
 
 static int compute_default_scale(struct wlr_output *output) {
-        int width, height;
-        wlr_output_transformed_resolution(output, &width, &height);
+        struct wlr_box box = { .width = output->width, .height = output->height };
+        if (output->pending.committed & WLR_OUTPUT_STATE_MODE) {
+                switch (output->pending.mode_type) {
+                case WLR_OUTPUT_STATE_MODE_FIXED:
+                        box.width = output->pending.mode->width;
+                        box.height = output->pending.mode->height;
+                        break;
+                case WLR_OUTPUT_STATE_MODE_CUSTOM:
+                        box.width = output->pending.custom_mode.width;
+                        box.height = output->pending.custom_mode.height;
+                        break;
+                }
+        }
+        enum wl_output_transform transform = output->transform;
+        if (output->pending.committed & WLR_OUTPUT_STATE_TRANSFORM) {
+                transform = output->pending.transform;
+        }
+        wlr_box_transform(&box, &box, transform, box.width, box.height);
+
+        int width = box.width;
+        int height = box.height;
+
         if (height < HIDPI_MIN_HEIGHT) {
                 return 1;
         }
@@ -349,18 +369,6 @@ bool apply_output_config(struct output_config *oc, struct sway_output *output) {
                 output->current_mode = wlr_output->pending.mode;
         }
 
-        float scale;
-        if (oc && oc->scale > 0) {
-                scale = oc->scale;
-        } else {
-                scale = compute_default_scale(wlr_output);
-                sway_log(SWAY_DEBUG, "Auto-detected output scale: %f", scale);
-        }
-        if (scale != wlr_output->scale) {
-                sway_log(SWAY_DEBUG, "Set %s scale to %f", oc->name, scale);
-                wlr_output_set_scale(wlr_output, scale);
-        }
-
         if (oc) {
                 enum scale_filter_mode scale_filter_old = output->scale_filter;
                 switch (oc->scale_filter) {
@@ -393,6 +401,20 @@ bool apply_output_config(struct output_config *oc, struct sway_output *output) {
                 wlr_output_set_transform(wlr_output, oc->transform);
         }
 
+        // Apply the scale last before the commit, because the scale auto-detection
+        // reads the pending output size
+        float scale;
+        if (oc && oc->scale > 0) {
+                scale = oc->scale;
+        } else {
+                scale = compute_default_scale(wlr_output);
+                sway_log(SWAY_DEBUG, "Auto-detected output scale: %f", scale);
+        }
+        if (scale != wlr_output->scale) {
+                sway_log(SWAY_DEBUG, "Set %s scale to %f", oc->name, scale);
+                wlr_output_set_scale(wlr_output, scale);
+        }
+
         sway_log(SWAY_DEBUG, "Committing output %s", wlr_output->name);
         if (!wlr_output_commit(wlr_output)) {
                 // Failed to modeset, maybe the output is missing a CRTC. Leave the