RenderLayer::updateLayerPositions() doesn't propagate the ancestor flags correctly
https://bugs.webkit.org/show_bug.cgi?id=201115

Reviewed by Zalan Bujtas.

When an updateLayerPositions() traversal starts at a non-root layer, we failed to populate
the ancestor-related UpdateLayerPositionsFlag flags, leaving layers with missing flags
(e.g. the m_hasTransformedAncestor flag). This is detected by the patch in bug 201066.

Fix by having updateLayerPositionsAfterStyleChange() and updateLayerPositionsAfterLayout()
initialize the flags from the parent layer.

This is a behavior change not detected by any existing test, but will be exercised once
the patch from bug 201066 lands.

* rendering/RenderLayer.cpp:
(WebCore::RenderLayer::flagsForUpdateLayerPositions):
(WebCore::RenderLayer::updateLayerPositionsAfterStyleChange):
(WebCore::RenderLayer::updateLayerPositionsAfterLayout):
(WebCore::outputPaintOrderTreeLegend):
(WebCore::outputPaintOrderTreeRecursive): Log hasTransformedAncestor().
* rendering/RenderLayer.h:

git-svn-id: http://svn.webkit.org/repository/webkit/trunk@249088 268f45cc-cd09-0410-ab3c-d52691b4dbfc
diff --git a/Source/WebCore/ChangeLog b/Source/WebCore/ChangeLog
index bb1693b..7b34011 100644
--- a/Source/WebCore/ChangeLog
+++ b/Source/WebCore/ChangeLog
@@ -1,3 +1,28 @@
+2019-08-23  Simon Fraser  <simon.fraser@apple.com>
+
+        RenderLayer::updateLayerPositions() doesn't propagate the ancestor flags correctly
+        https://bugs.webkit.org/show_bug.cgi?id=201115
+
+        Reviewed by Zalan Bujtas.
+
+        When an updateLayerPositions() traversal starts at a non-root layer, we failed to populate
+        the ancestor-related UpdateLayerPositionsFlag flags, leaving layers with missing flags
+        (e.g. the m_hasTransformedAncestor flag). This is detected by the patch in bug 201066.
+
+        Fix by having updateLayerPositionsAfterStyleChange() and updateLayerPositionsAfterLayout()
+        initialize the flags from the parent layer.
+
+        This is a behavior change not detected by any existing test, but will be exercised once
+        the patch from bug 201066 lands.
+
+        * rendering/RenderLayer.cpp:
+        (WebCore::RenderLayer::flagsForUpdateLayerPositions):
+        (WebCore::RenderLayer::updateLayerPositionsAfterStyleChange):
+        (WebCore::RenderLayer::updateLayerPositionsAfterLayout):
+        (WebCore::outputPaintOrderTreeLegend):
+        (WebCore::outputPaintOrderTreeRecursive): Log hasTransformedAncestor().
+        * rendering/RenderLayer.h:
+
 2019-08-24  Zalan Bujtas  <zalan@apple.com>
 
         [LFC][TFC] Add section renderers to the layout tree (THEAD/TBODY/TFOOT)
diff --git a/Source/WebCore/rendering/RenderLayer.cpp b/Source/WebCore/rendering/RenderLayer.cpp
index e741d7b..556136b 100644
--- a/Source/WebCore/rendering/RenderLayer.cpp
+++ b/Source/WebCore/rendering/RenderLayer.cpp
@@ -875,15 +875,36 @@
     return m_filters && m_filters->hasFilterThatMovesPixels();
 }
 
+OptionSet<RenderLayer::UpdateLayerPositionsFlag> RenderLayer::flagsForUpdateLayerPositions(RenderLayer& startingLayer)
+{
+    OptionSet<UpdateLayerPositionsFlag> flags = { CheckForRepaint };
+
+    if (auto* parent = startingLayer.parent()) {
+        if (parent->hasTransformedAncestor() || parent->transform())
+            flags.add(SeenTransformedLayer);
+
+        if (parent->has3DTransformedAncestor() || (parent->transform() && !parent->transform()->isAffine()))
+            flags.add(Seen3DTransformedLayer);
+
+        if (parent->behavesAsFixed() || (parent->renderer().isFixedPositioned() && !parent->hasTransformedAncestor()))
+            flags.add(SeenFixedLayer);
+
+        if (parent->hasCompositedScrollingAncestor() || parent->hasCompositedScrollableOverflow())
+            flags.add(SeenCompositedScrollingLayer);
+    }
+
+    return flags;
+}
+
 void RenderLayer::updateLayerPositionsAfterStyleChange()
 {
-    updateLayerPositions(nullptr, RenderLayer::updateLayerPositionsDefaultFlags());
+    updateLayerPositions(nullptr, flagsForUpdateLayerPositions(*this));
 }
 
 void RenderLayer::updateLayerPositionsAfterLayout(bool isRelayoutingSubtree, bool didFullRepaint)
 {
     auto updateLayerPositionFlags = [&](bool isRelayoutingSubtree, bool didFullRepaint) {
-        auto flags = RenderLayer::updateLayerPositionsDefaultFlags();
+        auto flags = flagsForUpdateLayerPositions(*this);
         if (didFullRepaint) {
             flags.remove(RenderLayer::CheckForRepaint);
             flags.add(RenderLayer::NeedsFullRepaintInBacking);
@@ -6974,7 +6995,7 @@
 static void outputPaintOrderTreeLegend(TextStream& stream)
 {
     stream.nextLine();
-    stream << "(S)tacking Context/(F)orced SC/O(P)portunistic SC, (N)ormal flow only, (O)verflow clip, (A)lpha (opacity or mask), has (B)lend mode, (I)solates blending, (T)ransform-ish, (F)ilter, Fi(X)ed position, Behaves as fi(x)ed, (C)omposited, (P)rovides backing/uses (p)rovided backing/paints to (a)ncestor, (c)omposited descendant, (s)scrolling ancestor\n"
+    stream << "(S)tacking Context/(F)orced SC/O(P)portunistic SC, (N)ormal flow only, (O)verflow clip, (A)lpha (opacity or mask), has (B)lend mode, (I)solates blending, (T)ransform-ish, (F)ilter, Fi(X)ed position, Behaves as fi(x)ed, (C)omposited, (P)rovides backing/uses (p)rovided backing/paints to (a)ncestor, (c)omposited descendant, (s)scrolling ancestor, (t)transformed ancestor\n"
         "Dirty (z)-lists, Dirty (n)ormal flow lists\n"
         "Traversal needs: requirements (t)raversal on descendants, (b)acking or hierarchy traversal on descendants, (r)equirements traversal on all descendants, requirements traversal on all (s)ubsequent layers, (h)ierarchy traversal on all descendants, update of paint (o)rder children\n"
         "Update needs:    post-(l)ayout requirements, (g)eometry, (k)ids geometry, (c)onfig, layer conne(x)ion, (s)crolling tree\n";
@@ -7021,6 +7042,7 @@
     stream << compositedPaintingDestinationString();
     stream << (layer.hasCompositingDescendant() ? "c" : "-");
     stream << (layer.hasCompositedScrollingAncestor() ? "s" : "-");
+    stream << (layer.hasTransformedAncestor() ? "t" : "-");
 
     stream << " ";
 
diff --git a/Source/WebCore/rendering/RenderLayer.h b/Source/WebCore/rendering/RenderLayer.h
index 32e4f75..79c380e 100644
--- a/Source/WebCore/rendering/RenderLayer.h
+++ b/Source/WebCore/rendering/RenderLayer.h
@@ -784,6 +784,7 @@
     FloatPoint perspectiveOrigin() const;
     bool preserves3D() const { return renderer().style().transformStyle3D() == TransformStyle3D::Preserve3D; }
     bool has3DTransform() const { return m_transform && !m_transform->isAffine(); }
+    bool hasTransformedAncestor() const { return m_hasTransformedAncestor; }
 
     void filterNeedsRepaint();
     bool hasFilter() const { return renderer().hasFilter(); }
@@ -988,12 +989,12 @@
         Seen3DTransformedLayer              = 1 << 6,
         SeenCompositedScrollingLayer        = 1 << 7,
     };
-    static constexpr OptionSet<UpdateLayerPositionsFlag> updateLayerPositionsDefaultFlags() { return { CheckForRepaint }; }
+    static OptionSet<UpdateLayerPositionsFlag> flagsForUpdateLayerPositions(RenderLayer& startingLayer);
 
     // Returns true if the position changed.
     bool updateLayerPosition(OptionSet<UpdateLayerPositionsFlag>* = nullptr);
 
-    void updateLayerPositions(RenderGeometryMap* = nullptr, OptionSet<UpdateLayerPositionsFlag> = updateLayerPositionsDefaultFlags());
+    void updateLayerPositions(RenderGeometryMap*, OptionSet<UpdateLayerPositionsFlag>);
 
     enum UpdateLayerPositionsAfterScrollFlag {
         IsOverflowScroll                        = 1 << 0,
@@ -1127,8 +1128,6 @@
     void setAncestorChainHasVisibleDescendant();
 
     bool has3DTransformedDescendant() const { return m_has3DTransformedDescendant; }
-
-    bool hasTransformedAncestor() const { return m_hasTransformedAncestor; }
     bool has3DTransformedAncestor() const { return m_has3DTransformedAncestor; }
 
     void dirty3DTransformedDescendantStatus();