[Impeller] experimental canvas bdf support. (flutter/engine#53597)

Almost working bdf for experimentcal canvas.

Currently there are some problems with the clip depth, and positioning of the backdrop filter. I think I am not taking the blur transform into account at the very least.

FYI @bdero

engine/src/flutter/impeller/aiks/experimental_canvas.cc

@@ -84,8 +84,7 @@ static std::unique_ptr<EntityPassTarget> CreateRenderTarget(
             .load_action = LoadAction::kDontCare,
             .store_action = StoreAction::kMultisampleResolve,
             .clear_color = clear_color},
-        /*stencil_attachment_config=*/
-        kDefaultStencilConfig);
+        /*stencil_attachment_config=*/kDefaultStencilConfig);
   } else {
     target = renderer.GetRenderTargetCache()->CreateOffscreen(
         *context,  // context
@@ -98,7 +97,7 @@ static std::unique_ptr<EntityPassTarget> CreateRenderTarget(
             .store_action = StoreAction::kDontCare,
             .clear_color = clear_color,
         },                     // color_attachment_config
-        kDefaultStencilConfig  // stencil_attachment_config
+        kDefaultStencilConfig  //
     );
   }
 
@@ -172,21 +171,23 @@ void ExperimentalCanvas::SetupRenderPass() {
   // a second save layer with the same dimensions as the onscreen. When
   // rendering is completed, we must blit this saveLayer to the onscreen.
   if (requires_readback_) {
-    entity_pass_targets_.push_back(CreateRenderTarget(
-        renderer_, color0.texture->GetSize(), /*mip_count=*/1,
-        /*clear_color=*/Color::BlackTransparent()));
+    auto entity_pass_target = CreateRenderTarget(
+        renderer_,                  //
+        color0.texture->GetSize(),  //
+        // Note: this is incorrect, we also need to know what kind of filter.
+        /*mip_count=*/4,  //
+        /*clear_color=*/Color::BlackTransparent());
+    render_passes_.push_back(
+        LazyRenderingConfig(renderer_, std::move(entity_pass_target)));
   } else {
-    entity_pass_targets_.push_back(std::make_unique<EntityPassTarget>(
-        render_target_,
-        renderer_.GetDeviceCapabilities().SupportsReadFromResolve(),
-        renderer_.GetDeviceCapabilities().SupportsImplicitResolvingMSAA()));
+    auto entity_pass_target = std::make_unique<EntityPassTarget>(
+        render_target_,                                                    //
+        renderer_.GetDeviceCapabilities().SupportsReadFromResolve(),       //
+        renderer_.GetDeviceCapabilities().SupportsImplicitResolvingMSAA()  //
+    );
+    render_passes_.push_back(
+        LazyRenderingConfig(renderer_, std::move(entity_pass_target)));
   }
-
-  auto inline_pass = std::make_unique<InlinePassContext>(
-      renderer_, *entity_pass_targets_.back(), 0);
-  inline_pass_contexts_.emplace_back(std::move(inline_pass));
-  auto result = inline_pass_contexts_.back()->GetRenderPass(0u);
-  render_passes_.push_back(result.pass);
 }
 
 void ExperimentalCanvas::Save(uint32_t total_content_depth) {
@@ -210,16 +211,148 @@ void ExperimentalCanvas::SaveLayer(
     ContentBoundsPromise bounds_promise,
     uint32_t total_content_depth,
     bool can_distribute_opacity) {
+  // Can we always guarantee that we get a bounds? Does a lack of bounds
+  // indicate something?
+  if (!bounds.has_value()) {
+    bounds = Rect::MakeSize(render_target_.GetRenderTargetSize());
+  }
+
+  // SaveLayer is a no-op, depending on the bounds promise. Should/Can DL elide
+  // this?
+  if (bounds->IsEmpty()) {
+    Save(total_content_depth);
+    return;
+  }
+
+  // The maximum coverage of the subpass. Subpasses textures should never
+  // extend outside the parent pass texture or the current clip coverage.
+  Rect coverage_limit = Rect::MakeOriginSize(
+      GetGlobalPassPosition(),
+      Size(render_passes_.back().inline_pass_context->GetTexture()->GetSize()));
+
+  // BDF No-op. need to do some precomputation to ensure this is fully skipped.
+  if (backdrop_filter) {
+    if (!clip_coverage_stack_.HasCoverage()) {
+      Save(total_content_depth);
+      return;
+    }
+    auto maybe_clip_coverage = clip_coverage_stack_.CurrentClipCoverage();
+    if (!maybe_clip_coverage.has_value()) {
+      Save(total_content_depth);
+      return;
+    }
+    auto clip_coverage = maybe_clip_coverage.value();
+    if (clip_coverage.IsEmpty() ||
+        !coverage_limit.IntersectsWithRect(clip_coverage)) {
+      Save(total_content_depth);
+      return;
+    }
+  }
+
   if (can_distribute_opacity && !backdrop_filter &&
       Paint::CanApplyOpacityPeephole(paint)) {
     Save(total_content_depth);
     transform_stack_.back().distributed_opacity *= paint.color.alpha;
     return;
   }
-  // Can we always guarantee that we get a bounds? Does a lack of bounds
-  // indicate something?
-  if (!bounds.has_value()) {
-    bounds = Rect::MakeSize(render_target_.GetRenderTargetSize());
+
+  // Backdrop filter state, ignored if there is no BDF.
+  std::shared_ptr<FilterContents> backdrop_filter_contents;
+  Point local_position = {0, 0};
+  if (backdrop_filter) {
+    auto current_clip_coverage = clip_coverage_stack_.CurrentClipCoverage();
+    if (current_clip_coverage.has_value()) {
+      local_position =
+          current_clip_coverage->GetOrigin() - GetGlobalPassPosition();
+    }
+    EntityPass::BackdropFilterProc backdrop_filter_proc =
+        [backdrop_filter = backdrop_filter->Clone()](
+            const FilterInput::Ref& input, const Matrix& effect_transform,
+            Entity::RenderingMode rendering_mode) {
+          auto filter = backdrop_filter->WrapInput(input);
+          filter->SetEffectTransform(effect_transform);
+          filter->SetRenderingMode(rendering_mode);
+          return filter;
+        };
+
+    auto rendering_config = std::move(render_passes_.back());
+    render_passes_.pop_back();
+
+    // If the very first thing we render in this EntityPass is a subpass that
+    // happens to have a backdrop filter, than that backdrop filter will end
+    // may wind up sampling from the raw, uncleared texture that came straight
+    // out of the texture cache. By calling `pass_context.GetRenderPass` here,
+    // we force the texture to pass through at least one RenderPass with the
+    // correct clear configuration before any sampling occurs.
+    rendering_config.inline_pass_context->GetRenderPass(0);
+
+    ISize restore_size =
+        rendering_config.inline_pass_context->GetTexture()->GetSize();
+
+    std::shared_ptr<Texture> input_texture =
+        rendering_config.entity_pass_target->Flip(
+            *renderer_.GetContext()->GetResourceAllocator());
+
+    backdrop_filter_contents = backdrop_filter_proc(
+        FilterInput::Make(std::move(input_texture)),
+        transform_stack_.back().transform.Basis(),
+        // When the subpass has a translation that means the math with
+        // the snapshot has to be different.
+        transform_stack_.back().transform.HasTranslation()
+            ? Entity::RenderingMode::kSubpassPrependSnapshotTransform
+            : Entity::RenderingMode::kSubpassAppendSnapshotTransform);
+
+    // The subpass will need to read from the current pass texture when
+    // rendering the backdrop, so if there's an active pass, end it prior to
+    // rendering the subpass.
+    rendering_config.inline_pass_context->EndPass();
+
+    // Create a new render pass that the backdrop filter contents will be
+    // restored to in order to continue rendering.
+    render_passes_.push_back(LazyRenderingConfig(
+        renderer_, std::move(rendering_config.entity_pass_target)));
+    // Eagerly restore the BDF contents.
+
+    // If the pass context returns a backdrop texture, we need to draw it to the
+    // current pass. We do this because it's faster and takes significantly less
+    // memory than storing/loading large MSAA textures. Also, it's not possible
+    // to blit the non-MSAA resolve texture of the previous pass to MSAA
+    // textures (let alone a transient one).
+    Rect size_rect = Rect::MakeSize(restore_size);
+    auto msaa_backdrop_contents = TextureContents::MakeRect(size_rect);
+    msaa_backdrop_contents->SetStencilEnabled(false);
+    msaa_backdrop_contents->SetLabel("MSAA backdrop");
+    msaa_backdrop_contents->SetSourceRect(size_rect);
+    msaa_backdrop_contents->SetTexture(
+        rendering_config.inline_pass_context->GetTexture());
+
+    Entity msaa_backdrop_entity;
+    msaa_backdrop_entity.SetContents(std::move(msaa_backdrop_contents));
+    msaa_backdrop_entity.SetBlendMode(BlendMode::kSource);
+    msaa_backdrop_entity.SetClipDepth(std::numeric_limits<uint32_t>::max());
+    if (!msaa_backdrop_entity.Render(renderer_,
+                                     *render_passes_.back()
+                                          .inline_pass_context->GetRenderPass(0)
+                                          .pass)) {
+      VALIDATION_LOG << "Failed to render MSAA backdrop filter entity.";
+      return;
+    }
+
+    // Restore any clips that were recorded before the backdrop filter was
+    // applied.
+    auto& replay_entities = clip_coverage_stack_.GetReplayEntities();
+    for (const auto& replay : replay_entities) {
+      SetClipScissor(
+          clip_coverage_stack_.CurrentClipCoverage(),
+          *render_passes_.back().inline_pass_context->GetRenderPass(0).pass,
+          GetGlobalPassPosition());
+      if (!replay.entity.Render(renderer_,
+                                *render_passes_.back()
+                                     .inline_pass_context->GetRenderPass(0)
+                                     .pass)) {
+        VALIDATION_LOG << "Failed to render entity for clip restore.";
+      }
+    }
   }
 
   // When applying a save layer, absorb any pending distributed opacity.
@@ -227,13 +360,25 @@ void ExperimentalCanvas::SaveLayer(
   paint_copy.color.alpha *= transform_stack_.back().distributed_opacity;
   transform_stack_.back().distributed_opacity = 1.0;
 
+  // Backdrop Filter must expand bounds to at least the clip stack, otherwise
+  // the coverage of the parent render pass.
   Rect subpass_coverage = bounds->TransformBounds(GetCurrentTransform());
-  auto target =
-      CreateRenderTarget(renderer_,
-                         ISize::MakeWH(subpass_coverage.GetSize().width,
-                                       subpass_coverage.GetSize().height),
-                         1u, Color::BlackTransparent());
-  entity_pass_targets_.push_back(std::move(target));
+  if (backdrop_filter_contents) {
+    FML_CHECK(clip_coverage_stack_.HasCoverage());
+    // We should never hit this case as we check the intersection above.
+    // NOLINTBEGIN(bugprone-unchecked-optional-access)
+    subpass_coverage =
+        coverage_limit
+            .Intersection(clip_coverage_stack_.CurrentClipCoverage().value())
+            .value();
+    // NOLINTEND(bugprone-unchecked-optional-access)
+  }
+
+  render_passes_.push_back(LazyRenderingConfig(
+      renderer_,                                             //
+      CreateRenderTarget(renderer_,                          //
+                         ISize(subpass_coverage.GetSize()),  //
+                         1u, Color::BlackTransparent())));
   save_layer_state_.push_back(SaveLayerState{paint_copy, subpass_coverage});
 
   CanvasStackEntry entry;
@@ -247,19 +392,25 @@ void ExperimentalCanvas::SaveLayer(
   entry.rendering_mode = Entity::RenderingMode::kSubpassAppendSnapshotTransform;
   transform_stack_.emplace_back(entry);
 
-  auto inline_pass = std::make_unique<InlinePassContext>(
-      renderer_, *entity_pass_targets_.back(), 0);
-  inline_pass_contexts_.emplace_back(std::move(inline_pass));
-
-  auto result = inline_pass_contexts_.back()->GetRenderPass(0u);
-  render_passes_.push_back(result.pass);
-
   // Start non-collapsed subpasses with a fresh clip coverage stack limited by
   // the subpass coverage. This is important because image filters applied to
   // save layers may transform the subpass texture after it's rendered,
   // causing parent clip coverage to get misaligned with the actual area that
   // the subpass will affect in the parent pass.
   clip_coverage_stack_.PushSubpass(subpass_coverage, GetClipHeight());
+
+  if (backdrop_filter_contents) {
+    // Render the backdrop entity.
+    Entity backdrop_entity;
+    backdrop_entity.SetContents(std::move(backdrop_filter_contents));
+    backdrop_entity.SetTransform(
+        Matrix::MakeTranslation(Vector3(-local_position)));
+    backdrop_entity.SetClipDepth(std::numeric_limits<uint32_t>::max());
+
+    backdrop_entity.Render(
+        renderer_,
+        *render_passes_.back().inline_pass_context->GetRenderPass(0).pass);
+  }
 }
 
 bool ExperimentalCanvas::Restore() {
@@ -289,26 +440,44 @@ bool ExperimentalCanvas::Restore() {
           Entity::RenderingMode::kSubpassAppendSnapshotTransform ||
       transform_stack_.back().rendering_mode ==
           Entity::RenderingMode::kSubpassPrependSnapshotTransform) {
-    auto inline_pass = std::move(inline_pass_contexts_.back());
+    auto lazy_render_pass = std::move(render_passes_.back());
+    render_passes_.pop_back();
+    // Force the render pass to be constructed if it never was.
+    lazy_render_pass.inline_pass_context->GetRenderPass(0);
 
     SaveLayerState save_layer_state = save_layer_state_.back();
     save_layer_state_.pop_back();
 
     std::shared_ptr<Contents> contents =
         PaintPassDelegate(save_layer_state.paint)
-            .CreateContentsForSubpassTarget(inline_pass->GetTexture(),
-                                            transform_stack_.back().transform);
+            .CreateContentsForSubpassTarget(
+                lazy_render_pass.inline_pass_context->GetTexture(),
+                transform_stack_.back().transform);
 
-    inline_pass->EndPass();
-    render_passes_.pop_back();
-    inline_pass_contexts_.pop_back();
+    lazy_render_pass.inline_pass_context->EndPass();
+
+    // Round the subpass texture position for pixel alignment with the parent
+    // pass render target. By default, we draw subpass textures with nearest
+    // sampling, so aligning here is important for avoiding visual nearest
+    // sampling errors caused by limited floating point precision when
+    // straddling a half pixel boundary.
+    //
+    // We do this in lieu of expanding/rounding out the subpass coverage in
+    // order to keep the bounds wrapping consistently tight around subpass
+    // elements. Which is necessary to avoid intense flickering in cases
+    // where a subpass texture has a large blur filter with clamp sampling.
+    //
+    // See also this bug: https://github.com/flutter/flutter/issues/144213
+    Point subpass_texture_position =
+        (save_layer_state.coverage.GetOrigin() - GetGlobalPassPosition())
+            .Round();
 
     Entity element_entity;
     element_entity.SetClipDepth(++current_depth_);
     element_entity.SetContents(std::move(contents));
     element_entity.SetBlendMode(save_layer_state.paint.blend_mode);
-    element_entity.SetTransform(Matrix::MakeTranslation(
-        Vector3(save_layer_state.coverage.GetOrigin())));
+    element_entity.SetTransform(
+        Matrix::MakeTranslation(Vector3(subpass_texture_position)));
 
     if (element_entity.GetBlendMode() > Entity::kLastPipelineBlendMode) {
       if (renderer_.GetDeviceCapabilities().SupportsFramebufferFetch()) {
@@ -319,7 +488,10 @@ bool ExperimentalCanvas::Restore() {
       }
     }
 
-    element_entity.Render(renderer_, *render_passes_.back());
+    element_entity.Render(
+        renderer_,                                                         //
+        *render_passes_.back().inline_pass_context->GetRenderPass(0).pass  //
+    );
     clip_coverage_stack_.PopSubpass();
     transform_stack_.pop_back();
 
@@ -364,15 +536,20 @@ bool ExperimentalCanvas::Restore() {
 
     if (clip_state_result.clip_did_change) {
       // We only need to update the pass scissor if the clip state has changed.
-      SetClipScissor(clip_coverage_stack_.CurrentClipCoverage(),
-                     *render_passes_.back(), GetGlobalPassPosition());
+      SetClipScissor(
+          clip_coverage_stack_.CurrentClipCoverage(),                         //
+          *render_passes_.back().inline_pass_context->GetRenderPass(0).pass,  //
+          GetGlobalPassPosition()                                             //
+      );
     }
 
     if (!clip_state_result.should_render) {
       return true;
     }
 
-    entity.Render(renderer_, *render_passes_.back());
+    entity.Render(
+        renderer_,
+        *render_passes_.back().inline_pass_context->GetRenderPass(0).pass);
   }
 
   return true;
@@ -441,7 +618,16 @@ void ExperimentalCanvas::AddRenderEntityToCurrentPass(Entity entity,
     }
   }
 
-  entity.Render(renderer_, *render_passes_.back());
+  InlinePassContext::RenderPassResult result =
+      render_passes_.back().inline_pass_context->GetRenderPass(0);
+  if (!result.pass) {
+    // Failure to produce a render pass should be explained by specific errors
+    // in `InlinePassContext::GetRenderPass()`, so avoid log spam and don't
+    // append a validation log here.
+    return;
+  }
+
+  entity.Render(renderer_, *result.pass);
 }
 
 void ExperimentalCanvas::AddClipEntityToCurrentPass(Entity entity) {
@@ -482,21 +668,27 @@ void ExperimentalCanvas::AddClipEntityToCurrentPass(Entity entity) {
 
   if (clip_state_result.clip_did_change) {
     // We only need to update the pass scissor if the clip state has changed.
-    SetClipScissor(clip_coverage_stack_.CurrentClipCoverage(),
-                   *render_passes_.back(), GetGlobalPassPosition());
+    SetClipScissor(
+        clip_coverage_stack_.CurrentClipCoverage(),
+        *render_passes_.back().inline_pass_context->GetRenderPass(0).pass,
+        GetGlobalPassPosition());
   }
 
   if (!clip_state_result.should_render) {
     return;
   }
 
-  entity.Render(renderer_, *render_passes_.back());
+  entity.Render(
+      renderer_,
+      *render_passes_.back().inline_pass_context->GetRenderPass(0).pass);
 }
 
 bool ExperimentalCanvas::BlitToOnscreen() {
   auto command_buffer = renderer_.GetContext()->CreateCommandBuffer();
   command_buffer->SetLabel("EntityPass Root Command Buffer");
-  auto offscreen_target = entity_pass_targets_.back()->GetRenderTarget();
+  auto offscreen_target = render_passes_.back()
+                              .inline_pass_context->GetPassTarget()
+                              .GetRenderTarget();
 
   if (renderer_.GetContext()
           ->GetCapabilities()
@@ -551,8 +743,8 @@ bool ExperimentalCanvas::BlitToOnscreen() {
 }
 
 void ExperimentalCanvas::EndReplay() {
-  FML_DCHECK(inline_pass_contexts_.size() == 1u);
-  inline_pass_contexts_.back()->EndPass();
+  FML_DCHECK(render_passes_.size() == 1u);
+  render_passes_.back().inline_pass_context->EndPass();
 
   // If requires_readback_ was true, then we rendered to an offscreen texture
   // instead of to the onscreen provided in the render target. Now we need to
@@ -562,7 +754,6 @@ void ExperimentalCanvas::EndReplay() {
   }
 
   render_passes_.clear();
-  inline_pass_contexts_.clear();
   renderer_.GetRenderTargetCache()->End();
 
   Reset();

engine/src/flutter/impeller/aiks/experimental_canvas.h

@@ -18,6 +18,18 @@
 
 namespace impeller {
 
+struct LazyRenderingConfig {
+  std::unique_ptr<EntityPassTarget> entity_pass_target;
+  std::unique_ptr<InlinePassContext> inline_pass_context;
+
+  LazyRenderingConfig(ContentContext& renderer,
+                      std::unique_ptr<EntityPassTarget> p_entity_pass_target)
+      : entity_pass_target(std::move(p_entity_pass_target)) {
+    inline_pass_context =
+        std::make_unique<InlinePassContext>(renderer, *entity_pass_target, 0);
+  }
+};
+
 /// This Canvas attempts to translate from display lists to draw calls directly.
 ///
 /// It's not fully implemented yet but if successful it will be replacing the
@@ -78,10 +90,8 @@ class ExperimentalCanvas : public Canvas {
   RenderTarget& render_target_;
   const bool requires_readback_;
   EntityPassClipStack clip_coverage_stack_;
-  std::vector<std::unique_ptr<InlinePassContext>> inline_pass_contexts_;
-  std::vector<std::unique_ptr<EntityPassTarget>> entity_pass_targets_;
+  std::vector<LazyRenderingConfig> render_passes_;
   std::vector<SaveLayerState> save_layer_state_;
-  std::vector<std::shared_ptr<RenderPass>> render_passes_;
 
   void SetupRenderPass();
 

engine/src/flutter/impeller/aiks/image_filter.h

@@ -87,6 +87,8 @@ class ImageFilter {
   virtual std::shared_ptr<ImageFilter> Clone() const = 0;
 
   virtual void Visit(ImageFilterVisitor& visitor) = 0;
+
+  virtual int GetRequiredMipCount() const { return 1; }
 };
 
 /*******************************************************************************
@@ -112,6 +114,8 @@ class BlurImageFilter : public ImageFilter {
   // |ImageFilter|
   void Visit(ImageFilterVisitor& visitor) override { visitor.Visit(*this); }
 
+  int GetRequiredMipCount() const override { return 4; }
+
  private:
   Sigma sigma_x_;
   Sigma sigma_y_;

engine/src/flutter/impeller/display_list/dl_dispatcher.cc

@@ -1313,10 +1313,12 @@ void TextFrameDispatcher::drawTextFrame(
   if (text_frame->HasColor()) {
     properties.color = paint_.color;
   }
-  renderer_.GetLazyGlyphAtlas()->AddTextFrame(*text_frame,                    //
-                                              matrix_.GetMaxBasisLengthXY(),  //
-                                              Point(x, y),                    //
-                                              properties                      //
+  auto scale =
+      (matrix_ * Matrix::MakeTranslation(Point(x, y))).GetMaxBasisLengthXY();
+  renderer_.GetLazyGlyphAtlas()->AddTextFrame(*text_frame,  //
+                                              scale,        //
+                                              Point(x, y),  //
+                                              properties    //
   );
 }
 

engine/src/flutter/impeller/display_list/dl_playground.cc

@@ -56,8 +56,8 @@ bool DlPlayground::OpenPlaygroundHere(DisplayListPlaygroundCallback callback) {
 
         ExperimentalDlDispatcher impeller_dispatcher(
             context.GetContentContext(), render_target,
-            display_list->root_has_backdrop_filter(),
-            display_list->max_root_blend_mode(), IRect::MakeMaximum());
+            list->root_has_backdrop_filter(), list->max_root_blend_mode(),
+            IRect::MakeMaximum());
         list->Dispatch(impeller_dispatcher);
         impeller_dispatcher.FinishRecording();
         context.GetContentContext().GetTransientsBuffer().Reset();

engine/src/flutter/impeller/entity/entity_pass_clip_stack.cc

@@ -4,7 +4,6 @@
 
 #include "impeller/entity/entity_pass_clip_stack.h"
 #include "impeller/entity/contents/clip_contents.h"
-#include "impeller/entity/contents/content_context.h"
 #include "impeller/entity/entity.h"
 
 namespace impeller {

engine/src/flutter/shell/gpu/gpu_surface_metal_impeller.mm

@@ -293,7 +293,9 @@ std::unique_ptr<SurfaceFrame> GPUSurfaceMetalImpeller::AcquireFrameFromMTLTextur
             fml::MakeCopyable([aiks_context, &display_list, &cull_rect,
                                &sk_cull_rect](impeller::RenderTarget& render_target) -> bool {
               impeller::ExperimentalDlDispatcher impeller_dispatcher(
-                  aiks_context->GetContentContext(), render_target, cull_rect);
+                  aiks_context->GetContentContext(), render_target,
+                  display_list->root_has_backdrop_filter(), display_list->max_root_blend_mode(),
+                  cull_rect);
               display_list->Dispatch(impeller_dispatcher, sk_cull_rect);
               impeller_dispatcher.FinishRecording();
               aiks_context->GetContentContext().GetTransientsBuffer().Reset();