config/output: Search for output config fallbacks

The original sway output config implementation enabled one output at a
time, testing modes, render formats and VRR support as it went along.
While this sort of fallback is easy to do, it has the downside of not
considering the effect of neighbor outputs on the configuration
viability.

With backend-wide commits, we can now better consider the effect of
neighbor outputs, but to handle the fact that we commit all outputs at
once we need to perform a more elaborate search of viable
configurations.

Implement a recursive configuration search for when the primary
configuration failed to apply.

3 files changed, 261 insertions, 55 deletions

diff --git a/include/sway/config.h b/include/sway/config.h
index 0be1cd22..5ccc3e77 100644
--- a/include/sway/config.h
+++ b/include/sway/config.h
@@ -689,10 +689,13 @@ const char *sway_output_scale_filter_to_string(enum scale_filter_mode scale_filt
 struct output_config *new_output_config(const char *name);
 
 bool apply_output_configs(struct matched_output_config *configs,
-                size_t configs_len, bool test_only);
+                size_t configs_len, bool test_only, bool degrade_to_off);
 
 void apply_all_output_configs(void);
 
+void sort_output_configs_by_priority(struct matched_output_config *configs,
+                size_t configs_len);
+
 /**
  * store_output_config stores a new output config. An output may be matched by
  * three different config types, in order of precedence: Identifier, name and
diff --git a/sway/config/output.c b/sway/config/output.c
index 7e037676..3ec5d77b 100644
--- a/sway/config/output.c
+++ b/sway/config/output.c
@@ -386,22 +386,18 @@ static int compute_default_scale(struct wlr_output *output,
         return 2;
 }
 
-/* Lists of formats to try, in order, when a specific render bit depth has
- * been asked for. The second to last format in each list should always
- * be XRGB8888, as a reliable backup in case the others are not available;
- * the last should be DRM_FORMAT_INVALID, to indicate the end of the list. */
-static const uint32_t *bit_depth_preferences[] = {
-        [RENDER_BIT_DEPTH_8] = (const uint32_t []){
-                DRM_FORMAT_XRGB8888,
-                DRM_FORMAT_INVALID,
-        },
-        [RENDER_BIT_DEPTH_10] = (const uint32_t []){
-                DRM_FORMAT_XRGB2101010,
-                DRM_FORMAT_XBGR2101010,
-                DRM_FORMAT_XRGB8888,
-                DRM_FORMAT_INVALID,
-        },
-};
+static bool render_format_is_10bit(uint32_t render_format) {
+        return render_format == DRM_FORMAT_XRGB2101010 ||
+                render_format == DRM_FORMAT_XBGR2101010;
+}
+
+static bool render_format_is_bgr(uint32_t fmt) {
+        return fmt == DRM_FORMAT_XBGR2101010 || fmt == DRM_FORMAT_XBGR8888;
+}
+
+static bool output_config_is_disabling(struct output_config *oc) {
+        return oc && (!oc->enabled || oc->power == 0);
+}
 
 static void queue_output_config(struct output_config *oc,
                 struct sway_output *output, struct wlr_output_state *pending) {
@@ -411,7 +407,7 @@ static void queue_output_config(struct output_config *oc,
 
         struct wlr_output *wlr_output = output->wlr_output;
 
-        if (oc && (!oc->enabled || oc->power == 0)) {
+        if (output_config_is_disabling(oc)) {
                 sway_log(SWAY_DEBUG, "Turning off output %s", wlr_output->name);
                 wlr_output_state_set_enabled(pending, false);
                 return;
@@ -434,22 +430,6 @@ static void queue_output_config(struct output_config *oc,
                 struct wlr_output_mode *preferred_mode =
                         wlr_output_preferred_mode(wlr_output);
                 wlr_output_state_set_mode(pending, preferred_mode);
-
-                if (!wlr_output_test_state(wlr_output, pending)) {
-                        sway_log(SWAY_DEBUG, "Preferred mode rejected, "
-                                "falling back to another mode");
-                        struct wlr_output_mode *mode;
-                        wl_list_for_each(mode, &wlr_output->modes, link) {
-                                if (mode == preferred_mode) {
-                                        continue;
-                                }
-
-                                wlr_output_state_set_mode(pending, mode);
-                                if (wlr_output_test_state(wlr_output, pending)) {
-                                        break;
-                                }
-                        }
-                }
         }
 
         if (oc && (oc->subpixel != WL_OUTPUT_SUBPIXEL_UNKNOWN || config->reloading)) {
@@ -500,25 +480,17 @@ static void queue_output_config(struct output_config *oc,
                 sway_log(SWAY_DEBUG, "Set %s adaptive sync to %d", wlr_output->name,
                         oc->adaptive_sync);
                 wlr_output_state_set_adaptive_sync_enabled(pending, oc->adaptive_sync == 1);
-                if (oc->adaptive_sync == 1 && !wlr_output_test_state(wlr_output, pending)) {
-                        sway_log(SWAY_DEBUG, "Adaptive sync failed, ignoring");
-                        wlr_output_state_set_adaptive_sync_enabled(pending, false);
-                }
         }
 
         if (oc && oc->render_bit_depth != RENDER_BIT_DEPTH_DEFAULT) {
-                const uint32_t *fmts = bit_depth_preferences[oc->render_bit_depth];
-                assert(fmts);
-
-                for (size_t i = 0; fmts[i] != DRM_FORMAT_INVALID; i++) {
-                        wlr_output_state_set_render_format(pending, fmts[i]);
-                        if (wlr_output_test_state(wlr_output, pending)) {
-                                break;
-                        }
-
-                        sway_log(SWAY_DEBUG, "Preferred output format 0x%08x "
-                                "failed to work, falling back to next in "
-                                "list, 0x%08x", fmts[i], fmts[i + 1]);
+                if (oc->render_bit_depth == RENDER_BIT_DEPTH_10 &&
+                        render_format_is_10bit(output->wlr_output->render_format)) {
+                        // 10-bit was set successfully before, try to save some tests by reusing the format
+                        wlr_output_state_set_render_format(pending, output->wlr_output->render_format);
+                } else if (oc->render_bit_depth == RENDER_BIT_DEPTH_10) {
+                        wlr_output_state_set_render_format(pending, DRM_FORMAT_XRGB2101010);
+                } else {
+                        wlr_output_state_set_render_format(pending, DRM_FORMAT_XRGB8888);
                 }
         }
 }
@@ -649,8 +621,227 @@ struct output_config *find_output_config(struct sway_output *sway_output) {
         return result;
 }
 
+static bool config_has_auto_mode(struct output_config *oc) {
+        if (!oc) {
+                return true;
+        }
+        if (oc->drm_mode.type != 0 && oc->drm_mode.type != (uint32_t)-1) {
+                return true;
+        } else if (oc->width > 0 && oc->height > 0) {
+                return true;
+        }
+        return false;
+}
+
+struct search_context {
+        struct wlr_output_swapchain_manager *swapchain_mgr;
+        struct wlr_backend_output_state *states;
+        struct matched_output_config *configs;
+        size_t configs_len;
+        bool degrade_to_off;
+};
+
+static bool search_valid_config(struct search_context *ctx, size_t output_idx);
+
+static void reset_output_state(struct wlr_output_state *state) {
+        wlr_output_state_finish(state);
+        wlr_output_state_init(state);
+        state->committed = 0;
+}
+
+static void clear_later_output_states(struct wlr_backend_output_state *states,
+                size_t configs_len, size_t output_idx) {
+
+        // Clear and disable all output states after this one to avoid conflict
+        // with previous tests.
+        for (size_t idx = output_idx+1; idx < configs_len; idx++) {
+                struct wlr_backend_output_state *backend_state = &states[idx];
+                struct wlr_output_state *state = &backend_state->base;
+
+                reset_output_state(state);
+                wlr_output_state_set_enabled(state, false);
+        }
+}
+
+static bool search_finish(struct search_context *ctx, size_t output_idx) {
+        clear_later_output_states(ctx->states, ctx->configs_len, output_idx);
+        return wlr_output_swapchain_manager_prepare(ctx->swapchain_mgr, ctx->states, ctx->configs_len) &&
+                search_valid_config(ctx, output_idx+1);
+}
+
+static bool search_adaptive_sync(struct search_context *ctx, size_t output_idx) {
+        struct matched_output_config *cfg = &ctx->configs[output_idx];
+        struct wlr_backend_output_state *backend_state = &ctx->states[output_idx];
+        struct wlr_output_state *state = &backend_state->base;
+
+        if (cfg->config && cfg->config->adaptive_sync == 1) {
+                wlr_output_state_set_adaptive_sync_enabled(state, true);
+                if (search_finish(ctx, output_idx)) {
+                        return true;
+                }
+        }
+        if (!cfg->config || cfg->config->adaptive_sync != -1) {
+                sway_log(SWAY_DEBUG, "Trying with adaptive sync disabled for: %s",
+                        backend_state->output->name);
+                wlr_output_state_set_adaptive_sync_enabled(state, false);
+                if (search_finish(ctx, output_idx)) {
+                        return true;
+                }
+        }
+        // If adaptive sync has not been set, or fallback in case we are on a
+        // backend that cannot disable adaptive sync such as the wayland backend.
+        sway_log(SWAY_DEBUG, "Trying with adaptive sync unset for: %s",
+                backend_state->output->name);
+        state->committed &= ~WLR_OUTPUT_STATE_ADAPTIVE_SYNC_ENABLED;
+        return search_finish(ctx, output_idx);
+}
+
+static bool search_mode(struct search_context *ctx, size_t output_idx) {
+        struct matched_output_config *cfg = &ctx->configs[output_idx];
+        struct wlr_backend_output_state *backend_state = &ctx->states[output_idx];
+        struct wlr_output_state *state = &backend_state->base;
+        struct wlr_output *wlr_output = backend_state->output;
+
+        if (!config_has_auto_mode(cfg->config)) {
+                return search_adaptive_sync(ctx, output_idx);
+        }
+
+        struct wlr_output_mode *preferred_mode = wlr_output_preferred_mode(wlr_output);
+        if (preferred_mode) {
+                sway_log(SWAY_DEBUG, "Trying with preferred mode for: %s", backend_state->output->name);
+                wlr_output_state_set_mode(state, preferred_mode);
+                if (search_adaptive_sync(ctx, output_idx)) {
+                        return true;
+                }
+        }
+
+        if (wl_list_empty(&wlr_output->modes)) {
+                state->committed &= ~WLR_OUTPUT_STATE_MODE;
+                return search_adaptive_sync(ctx, output_idx);
+        }
+
+        struct wlr_output_mode *mode;
+        wl_list_for_each(mode, &backend_state->output->modes, link) {
+                if (mode == preferred_mode) {
+                        continue;
+                }
+                sway_log(SWAY_DEBUG, "Trying with mode %dx%d@%dmHz for: %s",
+                        mode->width, mode->height, mode->refresh, backend_state->output->name);
+                wlr_output_state_set_mode(state, mode);
+                if (search_adaptive_sync(ctx, output_idx)) {
+                        return true;
+                }
+        }
+
+        return false;
+}
+
+static bool search_render_format(struct search_context *ctx, size_t output_idx) {
+        struct matched_output_config *cfg = &ctx->configs[output_idx];
+        struct wlr_backend_output_state *backend_state = &ctx->states[output_idx];
+        struct wlr_output_state *state = &backend_state->base;
+        struct wlr_output *wlr_output = backend_state->output;
+
+        uint32_t fmts[] = {
+                DRM_FORMAT_XRGB2101010,
+                DRM_FORMAT_XBGR2101010,
+                DRM_FORMAT_XRGB8888,
+                DRM_FORMAT_INVALID,
+        };
+        if (render_format_is_bgr(wlr_output->render_format)) {
+                // Start with BGR in the unlikely event that we previously required it.
+                fmts[0] = DRM_FORMAT_XBGR2101010;
+                fmts[1] = DRM_FORMAT_XRGB2101010;
+        }
+
+        const struct wlr_drm_format_set *primary_formats =
+                wlr_output_get_primary_formats(wlr_output, WLR_BUFFER_CAP_DMABUF);
+        bool need_10bit = cfg->config && cfg->config->render_bit_depth == RENDER_BIT_DEPTH_10;
+        for (size_t idx = 0; fmts[idx] != DRM_FORMAT_INVALID; idx++) {
+                if (!need_10bit && render_format_is_10bit(fmts[idx])) {
+                        continue;
+                }
+                if (!wlr_drm_format_set_get(primary_formats, fmts[idx])) {
+                        // This is not a supported format for this output
+                        continue;
+                }
+                sway_log(SWAY_DEBUG, "Trying with render format %d for: %s", fmts[idx],
+                        wlr_output->name);
+                wlr_output_state_set_render_format(state, fmts[idx]);
+                if (search_mode(ctx, output_idx)) {
+                        return true;
+                }
+        }
+        return false;
+}
+
+static bool search_valid_config(struct search_context *ctx, size_t output_idx) {
+        if (output_idx >= ctx->configs_len) {
+                // We reached the end of the search, all good!
+                return true;
+        }
+
+        struct matched_output_config *cfg = &ctx->configs[output_idx];
+        struct wlr_backend_output_state *backend_state = &ctx->states[output_idx];
+        struct wlr_output_state *state = &backend_state->base;
+
+        sway_log(SWAY_DEBUG, "Finding valid config for: %s",
+                backend_state->output->name);
+
+        if (!output_config_is_disabling(cfg->config)) {
+                // Search through our possible configurations, doing a depth-first
+                // through render_format, modes, adaptive_sync and the next output's
+                // config.
+                queue_output_config(cfg->config, cfg->output, &backend_state->base);
+                if (search_render_format(ctx, output_idx)) {
+                        return true;
+                } else if (!ctx->degrade_to_off) {
+                        return false;
+                }
+                // We could not get anything to work, try to disable this output to see
+                // if we can at least make the outputs before us work.
+                sway_log(SWAY_DEBUG, "Trying with disabled output for: %s",
+                        backend_state->output->name);
+                reset_output_state(state);
+        }
+
+        wlr_output_state_set_enabled(state, false);
+        return search_finish(ctx, output_idx);
+}
+
+static int compare_matched_output_config_priority(const void *a, const void *b) {
+
+        const struct matched_output_config *amc = a;
+        const struct matched_output_config *bmc = b;
+        bool a_disabling = output_config_is_disabling(amc->config);
+        bool b_disabling = output_config_is_disabling(bmc->config);
+        bool a_enabled = amc->output->enabled;
+        bool b_enabled = bmc->output->enabled;
+
+        // We want to give priority to existing enabled outputs. To do so, we want
+        // the configuration order to be:
+        // 1. Existing, enabled outputs
+        // 2. Outputs that need to be enabled
+        // 3. Disabled or disabling outputs
+        if (a_enabled && !a_disabling) {
+                return -1;
+        } else if (b_enabled && !b_disabling) {
+                return 1;
+        } else if (b_disabling && !a_disabling) {
+                return -1;
+        } else if (a_disabling && !b_disabling) {
+                return 1;
+        }
+        return 0;
+}
+
+void sort_output_configs_by_priority(struct matched_output_config *configs,
+                size_t configs_len) {
+        qsort(configs, configs_len, sizeof(*configs), compare_matched_output_config_priority);
+}
+
 bool apply_output_configs(struct matched_output_config *configs,
-                size_t configs_len, bool test_only) {
+                size_t configs_len, bool test_only, bool degrade_to_off) {
         struct wlr_backend_output_state *states = calloc(configs_len, sizeof(*states));
         if (!states) {
                 return false;
@@ -674,8 +865,18 @@ bool apply_output_configs(struct matched_output_config *configs,
 
         bool ok = wlr_output_swapchain_manager_prepare(&swapchain_mgr, states, configs_len);
         if (!ok) {
-                sway_log(SWAY_ERROR, "Swapchain prepare failed");
-                goto out;
+                sway_log(SWAY_ERROR, "Requested backend configuration failed, searching for valid fallbacks");
+                struct search_context ctx = {
+                        .swapchain_mgr = &swapchain_mgr,
+                        .states = states,
+                        .configs = configs,
+                        .configs_len = configs_len,
+                        .degrade_to_off = degrade_to_off,
+                };
+                if (!search_valid_config(&ctx, 0)) {
+                        sway_log(SWAY_ERROR, "Search for valid config failed");
+                        goto out;
+                }
         }
 
         if (test_only) {
@@ -761,7 +962,8 @@ void apply_all_output_configs(void) {
                 config->config = find_output_config(sway_output);
         }
 
-        apply_output_configs(configs, configs_len, false);
+        sort_output_configs_by_priority(configs, configs_len);
+        apply_output_configs(configs, configs_len, false, true);
         for (size_t idx = 0; idx < configs_len; idx++) {
                 struct matched_output_config *cfg = &configs[idx];
                 free_output_config(cfg->config);
diff --git a/sway/desktop/output.c b/sway/desktop/output.c
index 70987feb..2722e556 100644
--- a/sway/desktop/output.c
+++ b/sway/desktop/output.c
@@ -609,7 +609,8 @@ static void output_manager_apply(struct sway_server *server,
                 }
         }
 
-        bool ok = apply_output_configs(configs, configs_len, test_only);
+        sort_output_configs_by_priority(configs, configs_len);
+        bool ok = apply_output_configs(configs, configs_len, test_only, false);
         for (size_t idx = 0; idx < configs_len; idx++) {
                 struct matched_output_config *cfg = &configs[idx];