Don't coerce pointers to compositor layer mappings to booleans.

Source/core/rendering/RenderLayer.cpp

 /*
  * Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012 Apple Inc. All rights reserved.
  *
  * Portions are Copyright (C) 1998 Netscape Communications Corporation.
  *
  * Other contributors:
  *   Robert O'Callahan <roc+@cs.cmu.edu>
  *   David Baron <dbaron@fas.harvard.edu>
  *   Christian Biesinger <cbiesinger@web.de>
  *   Randall Jesup <rjesup@wgate.com>
@@ skipping 270 matching lines @@
         m_enclosingPaginationLayer = 0;
     }
 
     repainter().repaintAfterLayout(geometryMap, flags & CheckForRepaint);
 
     // Go ahead and update the reflection's position and size.
     if (m_reflectionInfo)
         m_reflectionInfo->reflection()->layout();
 
     // Clear the IsCompositingUpdateRoot flag once we've found the first compositing layer in this update.
+    // Layers are positioned in the space of the composited layer they paint into, so we only clear
+    // IsCompositingUpdateRoot when we reach a layer that paints into its own backing.
     bool isUpdateRoot = (flags & IsCompositingUpdateRoot);
-    if (compositedLayerMapping())
+    if (compositingState() == PaintsIntoOwnBacking)
         flags &= ~IsCompositingUpdateRoot;
 
     if (useRegionBasedColumns() && renderer()->isInFlowRenderFlowThread()) {
         updatePagination();
         flags |= UpdatePagination;
     }
 
     if (renderer()->hasColumns())
         flags |= UpdatePagination;
 
     for (RenderLayer* child = firstChild(); child; child = child->nextSibling())
         child->updateLayerPositions(geometryMap, flags);
 
-    if ((flags & UpdateCompositingLayers) && compositedLayerMapping()) {
+    if ((flags & UpdateCompositingLayers) && compositingState() == PaintsIntoOwnBacking) {
         CompositedLayerMapping::UpdateAfterLayoutFlags updateFlags = CompositedLayerMapping::CompositingChildrenOnly;
         if (flags & NeedsFullRepaintInBacking)
             updateFlags |= CompositedLayerMapping::NeedsFullRepaint;
         if (isUpdateRoot)
             updateFlags |= CompositedLayerMapping::IsUpdateRoot;
         compositedLayerMapping()->updateAfterLayout(updateFlags);
     }
 
     if (geometryMap)
         geometryMap->popMappingsToAncestor(parent());
@@ skipping 211 matching lines @@
 
     bool hadBlendMode = m_blendMode != BlendModeNormal;
     BlendMode newBlendMode = renderer()->style()->blendMode();
     if (newBlendMode != m_blendMode) {
         m_blendMode = newBlendMode;
 
         // Only update the flag if a blend mode is set or unset.
         if (!hadBlendMode || !hasBlendMode())
             dirtyAncestorChainBlendedDescendantStatus();
 
-        if (compositedLayerMapping())
+        // The compositor can only apply one blend mode per composited layer.
+        if (compositingState() == PaintsIntoOwnBacking)
             compositedLayerMapping()->setBlendMode(newBlendMode);
     }
 }
 
 void RenderLayer::updateTransform()
 {
     // hasTransform() on the renderer is also true when there is transform-style: preserve-3d or perspective set,
     // so check style too.
     bool hasTransform = renderer()->hasTransform() && renderer()->style()->hasTransform();
     bool had3DTransform = has3DTransform();
@@ skipping 86 matching lines @@
 {
     const Settings* settings = renderer()->document().settings();
     return settings && settings->regionBasedColumnsEnabled();
 }
 
 void RenderLayer::updatePagination()
 {
     m_isPaginated = false;
     m_enclosingPaginationLayer = 0;
 
-    if (compositedLayerMapping() || !parent())
+    if (compositingState() == PaintsIntoOwnBacking || !parent())

[shawnsingh, 2013/11/05 10:20:27]

I'm really not sure about this one.  If you feel certain, can you please
elaborate why we shouldn't include HasOwnBackingButPaintsIntoAncestor here?

         return; // FIXME: We will have to deal with paginated compositing layers someday.
                 // FIXME: For now the RenderView can't be paginated.  Eventually printing will move to a model where it is though.
 
     // The main difference between the paginated booleans for the old column code and the new column code
     // is that each paginated layer has to paint on its own with the new code. There is no
     // recurring into child layers. This means that the m_isPaginated bits for the new column code can't just be set on
     // "roots" that get split and paint all their descendants. Instead each layer has to be checked individually and
     // genuinely know if it is going to have to split itself up when painting only its contents (and not any other descendant
     // layers). We track an enclosingPaginationLayer instead of using a simple bit, since we want to be able to get back
     // to that layer easily.
@@ skipping 330 matching lines @@
         if (positionedParent->renderer()->hasOverflowClip()) {
             LayoutSize offset = positionedParent->renderBox()->scrolledContentOffset();
             localPoint -= offset;
         }
 
         if (renderer()->isOutOfFlowPositioned() && positionedParent->renderer()->isInFlowPositioned() && positionedParent->renderer()->isRenderInline()) {
             LayoutSize offset = toRenderInline(positionedParent->renderer())->offsetForInFlowPositionedInline(toRenderBox(renderer()));
             localPoint += offset;
         }
     } else if (parent()) {
-        if (compositedLayerMapping()) {
+        // Rationale: composited layers need to be positioned in their correct column. If we've been squashed into another layer,
+        // it will already be positioned into the column-correct location. If we also adjust our position, we'll be double
+        // compensating.
+        if (compositingState() == PaintsIntoOwnBacking) {

[shawnsingh, 2013/11/05 10:20:27]

What about HasOwnBackingButPaintsIntoAncestor ?   composited layer positioning
doesn't care about whether contents paint into it or an ancestor, right?

[Ian Vollick, 2013/11/08 03:35:12]

That was the point I was trying to address with this comment: if we paint into a
composited layer that is already in its column-correct location, executing this
code will be a double compensation. The code below is quite hairy and I may be
misunderstanding it; but that's my current impression.

             // FIXME: Composited layers ignore pagination, so about the best we can do is make sure they're offset into the appropriate column.
             // They won't split across columns properly.
             LayoutSize columnOffset;
             if (!parent()->renderer()->hasColumns() && parent()->renderer()->isRoot() && renderer()->view()->hasColumns())
                 renderer()->view()->adjustForColumns(columnOffset, localPoint);
             else
                 parent()->renderer()->adjustForColumns(columnOffset, localPoint);
 
             localPoint += columnOffset;
         }
@@ skipping 99 matching lines @@
 {
     return layer->stackingNode()->isNormalFlowOnly() ? layer->parent() : (layer->stackingNode()->ancestorStackingContainerNode() ? layer->stackingNode()->ancestorStackingContainerNode()->layer() : 0);
 }
 
 // FIXME: having two different functions named enclosingCompositingLayer and enclosingCompositingLayerForRepaint
 // is error-prone and misleading for reading code that uses these functions - especially compounded with
 // the includeSelf option. It is very likely that some call sites of this function actually mean to use
 // enclosingCompositingLayerForRepaint().
 RenderLayer* RenderLayer::enclosingCompositingLayer(bool includeSelf) const
 {
-    if (includeSelf && compositedLayerMapping())
+    if (includeSelf && compositingState() != NotComposited)
         return const_cast<RenderLayer*>(this);
 
     for (const RenderLayer* curr = compositingContainer(this); curr; curr = compositingContainer(curr)) {
-        if (curr->compositedLayerMapping())
+        if (curr->compositingState() != NotComposited)
             return const_cast<RenderLayer*>(curr);
     }
 
     return 0;
 }
 
 RenderLayer* RenderLayer::enclosingCompositingLayerForRepaint(bool includeSelf) const
 {
     if (includeSelf && compositingState() == PaintsIntoOwnBacking)
         return const_cast<RenderLayer*>(this);
@@ skipping 53 matching lines @@
     for (const RenderLayer* curr = this; curr; curr = curr->parent()) {
         RenderLayerModelObject* renderer = curr->renderer();
         if (renderer->style()->hasFilterOutsets())
             return true;
     }
     return false;
 }
 
 RenderLayer* RenderLayer::clippingRootForPainting() const
 {
-    if (compositedLayerMapping())
+    if (compositingState() == PaintsIntoOwnBacking)

[shawnsingh, 2013/11/05 10:20:27]

Should this also include HasOwnBackingButPaintsIntoAncestor ?  I can imagine a
composited layer that paints into ancestor but still has a clip.  That's one of
the common cases where a composited layer paints into its ancestor in the first
place.

         return const_cast<RenderLayer*>(this);
 
     const RenderLayer* current = this;
     while (current) {
         if (current->isRootLayer())
             return const_cast<RenderLayer*>(current);
 
         current = compositingContainer(current);
         ASSERT(current);
         if (current->transform()
@@ skipping 18 matching lines @@
 bool RenderLayer::isTransparent() const
 {
     if (renderer()->node() && renderer()->node()->isSVGElement())
         return false;
 
     return renderer()->isTransparent() || renderer()->hasMask();
 }
 
 RenderLayer* RenderLayer::transparentPaintingAncestor()
 {
-    if (compositedLayerMapping())
+    // The compositor will handle applying opacity for true composited layers.
+    // If we squish into a layer that is not transparent, we won't be able to
+    // rely on it to do this.
+    if (compositingState() == PaintsIntoOwnBacking)

[shawnsingh, 2013/11/05 10:20:27]

Again, I think we need to include HasOwnBackingbutPaintsIntoAncestor here...
it's about the composited layer hierarchy, not about who paints into where...

The scenario here is a composited layer with a non-composited ancestor that has
an opacity.  The composited layer has to walk all the way up to the next
composited layer to accumulate the correct opacity that it must inherit from all
non-composited layers.  The walk stops at the first layer that shows up in the
composited layer tree - regardless of whether it paints into its own backing or
not.

         return 0;
 
     for (RenderLayer* curr = parent(); curr; curr = curr->parent()) {
-        if (curr->compositedLayerMapping())
+        if (curr->compositingState() == PaintsIntoOwnBacking)
             return 0;
         if (curr->isTransparent())
             return curr;
     }
     return 0;
 }
 
 enum TransparencyClipBoxBehavior {
     PaintingTransparencyClipBox,
     HitTestingTransparencyClipBox
@@ skipping 515 matching lines @@
 void RenderLayer::updateScrollableArea()
 {
     if (requiresScrollableArea())
         m_scrollableArea = adoptPtr(new RenderLayerScrollableArea(renderBox()));
     else
         m_scrollableArea = nullptr;
 }
 
 PassOwnPtr<Vector<FloatRect> > RenderLayer::collectTrackedRepaintRects() const
 {
-    if (CompositedLayerMapping* mapping = compositedLayerMapping())
-        return mapping->collectTrackedRepaintRects();
+    if (compositingState() == PaintsIntoOwnBacking)

[shawnsingh, 2013/11/05 10:20:27]

So yeah, the theme of my review - did you remember to use
HasOwnBackingButPaintsIntoAncestor? =P  Or am I missing something that makes you
confident we don't need it in so many places? 

+        return compositedLayerMapping()->collectTrackedRepaintRects();
     return nullptr;
 }
 
 bool RenderLayer::hasOverflowControls() const
 {
     return m_scrollableArea && (m_scrollableArea->hasScrollbar() || m_scrollableArea->hasScrollCorner() || renderer()->style()->resize() != RESIZE_NONE);
 }
 
 void RenderLayer::paint(GraphicsContext* context, const LayoutRect& damageRect, PaintBehavior paintBehavior, RenderObject* paintingRoot, RenderRegion* region, PaintLayerFlags paintFlags)
 {
@@ skipping 1797 matching lines @@
         }
     }
 
     // FIXME: should probably just pass 'flags' down to descendants.
     CalculateLayerBoundsFlags descendantFlags = DefaultCalculateLayerBoundsFlags | (flags & ExcludeHiddenDescendants) | (flags & IncludeCompositedDescendants);
 
     const_cast<RenderLayer*>(this)->stackingNode()->updateLayerListsIfNeeded();
 
     if (m_reflectionInfo) {
         RenderLayer* reflectionLayer = m_reflectionInfo->reflectionLayer();
-        if (!reflectionLayer->compositedLayerMapping()) {
+        // If the reflection layer paints either into its compositing ancestor
+        // or is not composited all, we have to consider its effect on our bounds.
+        if (reflectionLayer->compositingState() != PaintsIntoOwnBacking) {
             IntRect childUnionBounds = reflectionLayer->calculateLayerBounds(this, 0, descendantFlags);
             unionBounds.unite(childUnionBounds);
         }
     }
 
     ASSERT(m_stackingNode->isStackingContainer() || (!m_stackingNode->posZOrderList() || !m_stackingNode->posZOrderList()->size()));
 
 #if !ASSERT_DISABLED
     LayerListMutationDetector mutationChecker(const_cast<RenderLayer*>(this)->stackingNode());
 #endif
 
-    // FIXME: Descendants that are composited should not necessarily be skipped, if they don't paint into their own
-    // separate backing. Instead, they ought to contribute to the bounds of the layer we're trying to compute.
-    // This applies to all z-order lists below.
     RenderLayerStackingNodeIterator iterator(*m_stackingNode.get(), AllChildren);
     while (RenderLayerStackingNode* node = iterator.next()) {
-        if (flags & IncludeCompositedDescendants || !node->layer()->compositedLayerMapping()) {
+        if (flags & IncludeCompositedDescendants || node->layer()->compositingState() != PaintsIntoOwnBacking) {

[shawnsingh, 2013/11/05 10:20:27]

This was another place where I put a FIXME because it's very likely a bug that
we should repro with a layout test before fixing.  Could we add a layout test
that repros a bug before your patch?

[Ian Vollick, 2013/11/08 03:35:12]

Sure. I've reinstated the FIXME and matched the existing, buggy behavior.

             IntRect childUnionBounds = node->layer()->calculateLayerBounds(this, 0, descendantFlags);
             unionBounds.unite(childUnionBounds);
         }
     }
 
     // FIXME: We can optimize the size of the composited layers, by not enlarging
     // filtered areas with the outsets if we know that the filter is going to render in hardware.
     // https://bugs.webkit.org/show_bug.cgi?id=81239
     if (flags & IncludeLayerFilterOutsets)
         renderer->style()->filterOutsets().expandRect(unionBounds);
@@ skipping 22 matching lines @@
     if (!m_compositedLayerMapping)
         return NotComposited;
 
     if (m_compositedLayerMapping && compositedLayerMapping()->paintsIntoCompositedAncestor())
         return HasOwnBackingButPaintsIntoAncestor;
 
     ASSERT(m_compositedLayerMapping);
     return PaintsIntoOwnBacking;
 }
 
-CompositedLayerMapping* RenderLayer::ensureCompositedLayerMapping()
+CompositedLayerMappingPtr RenderLayer::ensureCompositedLayerMapping()
 {
     if (!m_compositedLayerMapping) {
         m_compositedLayerMapping = adoptPtr(new CompositedLayerMapping(this));
         compositor()->layerBecameComposited(this);
 
         updateOrRemoveFilterEffectRenderer();
 
         if (RuntimeEnabledFeatures::cssCompositingEnabled())
             compositedLayerMapping()->setBlendMode(m_blendMode);
     }
@@ skipping 66 matching lines @@
         || listBackgroundIsKnownToBeOpaqueInRect(m_stackingNode->normalFlowList(), localRect);
 }
 
 bool RenderLayer::listBackgroundIsKnownToBeOpaqueInRect(const Vector<RenderLayerStackingNode*>* list, const LayoutRect& localRect) const
 {
     if (!list || list->isEmpty())
         return false;
 
     for (Vector<RenderLayerStackingNode*>::const_reverse_iterator iter = list->rbegin(); iter != list->rend(); ++iter) {
         const RenderLayer* childLayer = (*iter)->layer();
-        if (childLayer->compositedLayerMapping())
+        if (childLayer->compositingState() == PaintsIntoOwnBacking)
             continue;
 
         if (!childLayer->canUseConvertToLayerCoords())
             continue;
 
         LayoutPoint childOffset;
         LayoutRect childLocalRect(localRect);
         childLayer->convertToLayerCoords(this, childOffset);
         childLocalRect.moveBy(-childOffset);
 
@@ skipping 152 matching lines @@
 
 inline bool RenderLayer::needsCompositingLayersRebuiltForClip(const RenderStyle* oldStyle, const RenderStyle* newStyle) const
 {
     ASSERT(newStyle);
     return oldStyle && (oldStyle->clip() != newStyle->clip() || oldStyle->hasClip() != newStyle->hasClip());
 }
 
 inline bool RenderLayer::needsCompositingLayersRebuiltForOverflow(const RenderStyle* oldStyle, const RenderStyle* newStyle) const
 {
     ASSERT(newStyle);
-    return !compositedLayerMapping() && oldStyle && (oldStyle->overflowX() != newStyle->overflowX()) && m_stackingNode->ancestorStackingContainerNode()->layer()->hasCompositingDescendant();
+    return compositingState() == NotComposited && oldStyle && (oldStyle->overflowX() != newStyle->overflowX()) && m_stackingNode->ancestorStackingContainerNode()->layer()->hasCompositingDescendant();
 }
 
 inline bool RenderLayer::needsCompositingLayersRebuiltForFilters(const RenderStyle* oldStyle, const RenderStyle* newStyle, bool didPaintWithFilters) const
 {
     if (!hasOrHadFilters(oldStyle, newStyle))
         return false;
 
     if (renderer()->animation().isRunningAcceleratedAnimationOnRenderer(renderer(), CSSPropertyWebkitFilter)) {
         // When the compositor is performing the filter animation, we shouldn't touch the compositing layers.
         // All of the layers above us should have been promoted to compositing layers already.
@@ skipping 23 matching lines @@
 }
 
 void RenderLayer::updateFilters(const RenderStyle* oldStyle, const RenderStyle* newStyle)
 {
     if (!hasOrHadFilters(oldStyle, newStyle))
         return;
 
     updateOrRemoveFilterClients();
     // During an accelerated animation, both WebKit and the compositor animate properties.
     // However, WebKit shouldn't ask the compositor to update its filters if the compositor is performing the animation.
-    bool shouldUpdateFilters = compositedLayerMapping() && !renderer()->animation().isRunningAcceleratedAnimationOnRenderer(renderer(), CSSPropertyWebkitFilter);
+    bool shouldUpdateFilters = compositingState() == PaintsIntoOwnBacking && !renderer()->animation().isRunningAcceleratedAnimationOnRenderer(renderer(), CSSPropertyWebkitFilter);
     if (shouldUpdateFilters)
         compositedLayerMapping()->updateFilters(renderer()->style());
     updateOrRemoveFilterEffectRenderer();
 }
 
 void RenderLayer::styleChanged(StyleDifference, const RenderStyle* oldStyle)
 {
     m_stackingNode->updateIsNormalFlowOnly();
 
     if (m_scrollableArea)
@@ skipping 18 matching lines @@
     if (paintsWithFilters())
         didPaintWithFilters = true;
     updateFilters(oldStyle, renderer()->style());
 
     const RenderStyle* newStyle = renderer()->style();
     if (compositor()->updateLayerCompositingState(this)
         || needsCompositingLayersRebuiltForClip(oldStyle, newStyle)
         || needsCompositingLayersRebuiltForOverflow(oldStyle, newStyle)
         || needsCompositingLayersRebuiltForFilters(oldStyle, newStyle, didPaintWithFilters))
         compositor()->setCompositingLayersNeedRebuild();
-    else if (compositedLayerMapping())
+    else if (compositingState() == PaintsIntoOwnBacking)

[shawnsingh, 2013/11/05 10:20:27]

I'm pretty sure we would need HasOwnBackingButPaintsIntoAncestor here, too?

         compositedLayerMapping()->updateGraphicsLayerGeometry();
 }
 
 bool RenderLayer::scrollsOverflow() const
 {
     if (RenderLayerScrollableArea* scrollableArea = this->scrollableArea())
         return scrollableArea->scrollsOverflow();
 
     return false;
 }
@@ skipping 157 matching lines @@
     }
 }
 
 void showLayerTree(const WebCore::RenderObject* renderer)
 {
     if (!renderer)
         return;
     showLayerTree(renderer->enclosingLayer());
 }
 #endif