Merge "Implementing detector of view flashes" into udc-qpr-dev

tests/src/com/android/launcher3/util/viewcapture_analysis/AlphaJumpDetector.java

@@ -16,11 +16,8 @@
 package com.android.launcher3.util.viewcapture_analysis;
 
 import com.android.launcher3.util.viewcapture_analysis.ViewCaptureAnalyzer.AnalysisNode;
-import com.android.launcher3.util.viewcapture_analysis.ViewCaptureAnalyzer.AnomalyDetector;
 
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 
 /**
  * Anomaly detector that triggers an error when alpha of a view changes too rapidly.
@@ -34,8 +31,7 @@ final class AlphaJumpDetector extends AnomalyDetector {
     private static final String RECENTS_DRAG_LAYER =
             CONTENT + "LauncherRootView:id/launcher|RecentsDragLayer:id/drag_layer|";
 
-    // Paths of nodes that are excluded from analysis.
-    private static final Iterable<String> PATHS_TO_IGNORE = List.of(
+    private static final IgnoreNode IGNORED_NODES_ROOT = buildIgnoreNodesTree(List.of(
             CONTENT
                     + "AddItemDragLayer:id/add_item_drag_layer|AddItemWidgetsBottomSheet:id"
                     + "/add_item_bottom_sheet|LinearLayout:id/add_item_bottom_sheet_content"
@@ -135,38 +131,7 @@ final class AlphaJumpDetector extends AnomalyDetector {
                     + "NexusOverviewActionsView:id/overview_actions_view"
                     + "|LinearLayout:id/action_buttons|Button:id/action_split",
             DRAG_LAYER + "IconView"
-    );
-
-    /**
-     * Element of the tree of ignored nodes.
-     * If the "children" map is empty, then this node should be ignored, i.e. alpha jumps analysis
-     * shouldn't run for it.
-     * I.e. ignored nodes correspond to the leaves in the ignored nodes tree.
-     */
-    private static class IgnoreNode {
-        // Map from child node identities to ignore-nodes for these children.
-        public final Map<String, IgnoreNode> children = new HashMap<>();
-    }
-
-    private static final IgnoreNode IGNORED_NODES_ROOT = buildIgnoreNodesTree();
-
-    // Converts the list of full paths of nodes to ignore to a more efficient tree of ignore-nodes.
-    private static IgnoreNode buildIgnoreNodesTree() {
-        final IgnoreNode root = new IgnoreNode();
-        for (String pathToIgnore : PATHS_TO_IGNORE) {
-            // Scan the diag path of an ignored node and add its elements into the tree.
-            IgnoreNode currentIgnoreNode = root;
-            for (String part : pathToIgnore.split("\\|")) {
-                // Ensure that the child of the node is added to the tree.
-                IgnoreNode child = currentIgnoreNode.children.get(part);
-                if (child == null) {
-                    currentIgnoreNode.children.put(part, child = new IgnoreNode());
-                }
-                currentIgnoreNode = child;
-            }
-        }
-        return root;
-    }
+    ));
 
     // Minimal increase or decrease of view's alpha between frames that triggers the error.
     private static final float ALPHA_JUMP_THRESHOLD = 1f;
@@ -213,7 +178,7 @@ final class AlphaJumpDetector extends AnomalyDetector {
     }
 
     @Override
-    String detectAnomalies(AnalysisNode oldInfo, AnalysisNode newInfo, int frameN) {
+    String detectAnomalies(AnalysisNode oldInfo, AnalysisNode newInfo, int frameN, long timestamp) {
         // If the view was previously seen, proceed with analysis only if it was present in the
         // view hierarchy in the previous frame.
         if (oldInfo != null && oldInfo.frameN != frameN) return null;
@@ -229,9 +194,8 @@ final class AlphaJumpDetector extends AnomalyDetector {
         if (alphaDeltaAbs >= ALPHA_JUMP_THRESHOLD) {
             nodeData.ignoreAlphaJumps = true; // No need to report alpha jump in children.
             return String.format(
-                    "Alpha jump detected in ViewCapture data: alpha change: %s (%s -> %s)"
-                            + ", threshold: %s, %s", // ----------- no need to include view?
-                    alphaDeltaAbs, oldAlpha, newAlpha, ALPHA_JUMP_THRESHOLD, latestInfo);
+                    "Alpha jump detected: alpha change: %s (%s -> %s), threshold: %s",
+                    alphaDeltaAbs, oldAlpha, newAlpha, ALPHA_JUMP_THRESHOLD);
         }
         return null;
     }

tests/src/com/android/launcher3/util/viewcapture_analysis/AnomalyDetector.java

@@ -0,0 +1,84 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.launcher3.util.viewcapture_analysis;
+
+import androidx.annotation.NonNull;
+import androidx.annotation.Nullable;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Detector of one kind of anomaly.
+ */
+abstract class AnomalyDetector {
+    // Index of this detector in ViewCaptureAnalyzer.ANOMALY_DETECTORS
+    public int detectorOrdinal;
+
+    /**
+     * Element of the tree of ignored nodes.
+     * If the "children" map is empty, then this node should be ignored, i.e. the analysis shouldn't
+     * run for it.
+     * I.e. ignored nodes correspond to the leaves in the ignored nodes tree.
+     */
+    protected static class IgnoreNode {
+        // Map from child node identities to ignore-nodes for these children.
+        public final Map<String, IgnoreNode> children = new HashMap<>();
+    }
+
+    // Converts the list of full paths of nodes to ignore to a more efficient tree of ignore-nodes.
+    protected static IgnoreNode buildIgnoreNodesTree(Iterable<String> pathsToIgnore) {
+        final IgnoreNode root = new IgnoreNode();
+        for (String pathToIgnore : pathsToIgnore) {
+            // Scan the diag path of an ignored node and add its elements into the tree.
+            IgnoreNode currentIgnoreNode = root;
+            for (String part : pathToIgnore.split("\\|")) {
+                // Ensure that the child of the node is added to the tree.
+                IgnoreNode child = currentIgnoreNode.children.get(part);
+                if (child == null) {
+                    currentIgnoreNode.children.put(part, child = new IgnoreNode());
+                }
+                currentIgnoreNode = child;
+            }
+        }
+        return root;
+    }
+
+    /**
+     * Initializes fields of the node that are specific to the anomaly detected by this
+     * detector.
+     */
+    abstract void initializeNode(@NonNull ViewCaptureAnalyzer.AnalysisNode info);
+
+    /**
+     * Detects anomalies by looking at the last occurrence of a view, and the current one.
+     * null value means that the view. 'oldInfo' and 'newInfo' cannot be both null.
+     * If an anomaly is detected, an exception will be thrown.
+     *
+     * @param oldInfo the view, as seen in the last frame that contained it in the view
+     *                hierarchy before 'currentFrame'. 'null' means that the view is first seen
+     *                in the 'currentFrame'.
+     * @param newInfo the view in the view hierarchy of the 'currentFrame'. 'null' means that
+     *                the view is not present in the 'currentFrame', but was present in the previous
+     *                frame.
+     * @param frameN  number of the current frame.
+     * @return Anomaly diagnostic message if an anomaly has been detected; null otherwise.
+     */
+    abstract String detectAnomalies(
+            @Nullable ViewCaptureAnalyzer.AnalysisNode oldInfo,
+            @Nullable ViewCaptureAnalyzer.AnalysisNode newInfo, int frameN,
+            long frameTimeNs);
+}

tests/src/com/android/launcher3/util/viewcapture_analysis/FlashDetector.java

@@ -0,0 +1,173 @@
+/*
+ * Copyright (C) 2023 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.launcher3.util.viewcapture_analysis;
+
+import static org.junit.Assert.assertTrue;
+
+import com.android.launcher3.util.viewcapture_analysis.ViewCaptureAnalyzer.AnalysisNode;
+
+import java.util.List;
+
+/**
+ * Anomaly detector that triggers an error when a view flashes, i.e. appears or disappears for a too
+ * short period of time.
+ */
+final class FlashDetector extends AnomalyDetector {
+    // Maximum time period of a view visibility or invisibility that is recognized as a flash.
+    private static final int FLASH_DURATION_MS = 300;
+
+    // Commonly used parts of the paths to ignore.
+    private static final String CONTENT = "DecorView|LinearLayout|FrameLayout:id/content|";
+    private static final String DRAG_LAYER =
+            CONTENT + "LauncherRootView:id/launcher|DragLayer:id/drag_layer|";
+    private static final String RECENTS_DRAG_LAYER =
+            CONTENT + "LauncherRootView:id/launcher|RecentsDragLayer:id/drag_layer|";
+
+    private static final IgnoreNode IGNORED_NODES_ROOT = buildIgnoreNodesTree(List.of(
+            CONTENT + "LauncherRootView:id/launcher|FloatingIconView",
+            DRAG_LAYER
+                    + "SearchContainerView:id/apps_view|AllAppsRecyclerView:id/apps_list_view"
+                    + "|BubbleTextView:id/icon",
+            DRAG_LAYER + "LauncherDragView|ImageView",
+            DRAG_LAYER + "LauncherRecentsView:id/overview_panel|TaskView|TextView",
+            DRAG_LAYER
+                    + "LauncherAllAppsContainerView:id/apps_view|AllAppsRecyclerView:id"
+                    + "/apps_list_view|BubbleTextView:id/icon",
+            DRAG_LAYER + "LauncherDragView|View",
+            CONTENT
+                    + "AddItemDragLayer:id/add_item_drag_layer|AddItemWidgetsBottomSheet:id"
+                    + "/add_item_bottom_sheet|LinearLayout:id/add_item_bottom_sheet_content"
+                    + "|ScrollView:id/widget_preview_scroll_view|WidgetCell:id/widget_cell"
+                    + "|WidgetCellPreview:id/widget_preview_container|WidgetImageView:id"
+                    + "/widget_preview",
+            CONTENT
+                    + "AddItemDragLayer:id/add_item_drag_layer|AddItemWidgetsBottomSheet:id"
+                    + "/add_item_bottom_sheet|LinearLayout:id/add_item_bottom_sheet_content"
+                    + "|ScrollView:id/widget_preview_scroll_view|WidgetCell:id/widget_cell"
+                    + "|WidgetCellPreview:id/widget_preview_container|ImageView:id/widget_badge",
+            RECENTS_DRAG_LAYER + "FallbackRecentsView:id/overview_panel|TaskView|IconView:id/icon",
+            DRAG_LAYER
+                    + "SearchContainerView:id/apps_view|UniversalSearchInputView:id"
+                    + "/search_container_all_apps|View:id/ripple"
+    ));
+
+    // Per-AnalysisNode data that's specific to this detector.
+    private static class NodeData {
+        public boolean ignoreFlashes;
+
+        // If ignoreNode is null, then this AnalysisNode node will be ignored if its parent is
+        // ignored.
+        // Otherwise, this AnalysisNode will be ignored if ignoreNode is a leaf i.e. has no
+        // children.
+        public IgnoreNode ignoreNode;
+    }
+
+    private NodeData getNodeData(AnalysisNode info) {
+        return (NodeData) info.detectorsData[detectorOrdinal];
+    }
+
+    @Override
+    void initializeNode(AnalysisNode info) {
+        final NodeData nodeData = new NodeData();
+        info.detectorsData[detectorOrdinal] = nodeData;
+
+        // If the parent view ignores flashes, its descendants will too.
+        final boolean parentIgnoresFlashes = info.parent != null && getNodeData(
+                info.parent).ignoreFlashes;
+        if (parentIgnoresFlashes) {
+            nodeData.ignoreFlashes = true;
+            return;
+        }
+
+        // Parent view doesn't ignore flashes.
+        // Initialize this AnalysisNode's ignore-node with the corresponding child of the
+        // ignore-node of the parent, if present.
+        final IgnoreNode parentIgnoreNode = info.parent != null
+                ? getNodeData(info.parent).ignoreNode
+                : IGNORED_NODES_ROOT;
+        nodeData.ignoreNode = parentIgnoreNode != null
+                ? parentIgnoreNode.children.get(info.nodeIdentity) : null;
+        // AnalysisNode will be ignored if the corresponding ignore-node is a leaf.
+        nodeData.ignoreFlashes =
+                nodeData.ignoreNode != null && nodeData.ignoreNode.children.isEmpty();
+    }
+
+    @Override
+    String detectAnomalies(AnalysisNode oldInfo, AnalysisNode newInfo, int frameN,
+            long frameTimeNs) {
+        // Should we check when a view was visible for a short period, then its alpha became 0?
+        // Then 'lastVisible' time should be the last one still visible?
+        // Check only transitions of alpha between 0 and 1?
+
+        // If this is the first time ever when we see the view, there have been no flashes yet.
+        if (oldInfo == null) return null;
+
+        // A flash requires a view to go from the full visibility to no-visibility and then back,
+        // or vice versa.
+        // If the last time the view was seen before the current frame, it didn't have full
+        // visibility; no flash can possibly be detected at the current frame.
+        if (oldInfo.alpha < 1) return null;
+
+        final AnalysisNode latestInfo = newInfo != null ? newInfo : oldInfo;
+        final NodeData nodeData = getNodeData(latestInfo);
+        if (nodeData.ignoreFlashes) return null;
+
+        // Once the view becomes invisible, see for how long it was visible prior to that. If it
+        // was visible only for a short interval of time, it's a flash.
+        if (
+            // View is invisible in the current frame
+                newInfo == null
+                        // When the view became visible last time, it was a transition from
+                        // no-visibility to full visibility.
+                        && oldInfo.timeBecameVisibleNs != -1) {
+            final long wasVisibleTimeMs = (frameTimeNs - oldInfo.timeBecameVisibleNs) / 1000000;
+
+            if (wasVisibleTimeMs <= FLASH_DURATION_MS) {
+                nodeData.ignoreFlashes = true; // No need to report flashes in children.
+                return
+                        String.format(
+                                "View was visible for a too short period of time %dms, which is a"
+                                        + " flash",
+                                wasVisibleTimeMs
+                        );
+            }
+        }
+
+        // Once a view becomes visible, see for how long it was invisible prior to that. If it
+        // was invisible only for a short interval of time, it's a flash.
+        if (
+            // The view is fully visible now
+                newInfo != null && newInfo.alpha >= 1
+                        // The view wasn't visible in the previous frame
+                        && frameN != oldInfo.frameN + 1) {
+            // We can assert the below condition because at this point, we know that
+            // oldInfo.alpha >= 1, i.e. it disappeared abruptly.
+            assertTrue("oldInfo.timeBecameInvisibleNs must not be -1",
+                    oldInfo.timeBecameInvisibleNs != -1);
+
+            final long wasInvisibleTimeMs = (frameTimeNs - oldInfo.timeBecameInvisibleNs) / 1000000;
+            if (wasInvisibleTimeMs <= FLASH_DURATION_MS) {
+                nodeData.ignoreFlashes = true; // No need to report flashes in children.
+                return
+                        String.format(
+                                "View was invisible for a too short period of time %dms, which "
+                                        + "is a flash",
+                                wasInvisibleTimeMs);
+            }
+        }
+        return null;
+    }
+}

tests/src/com/android/launcher3/util/viewcapture_analysis/ViewCaptureAnalyzer.java

@@ -17,9 +17,6 @@ package com.android.launcher3.util.viewcapture_analysis;
 
 import static android.view.View.VISIBLE;
 
-import androidx.annotation.NonNull;
-import androidx.annotation.Nullable;
-
 import com.android.app.viewcapture.data.ExportedData;
 import com.android.app.viewcapture.data.FrameData;
 import com.android.app.viewcapture.data.ViewNode;
@@ -36,40 +33,10 @@ import java.util.Map;
 public class ViewCaptureAnalyzer {
     private static final String SCRIM_VIEW_CLASS = "com.android.launcher3.views.ScrimView";
 
-    /**
-     * Detector of one kind of anomaly.
-     */
-    abstract static class AnomalyDetector {
-        // Index of this detector in ViewCaptureAnalyzer.ANOMALY_DETECTORS
-        public int detectorOrdinal;
-
-        /**
-         * Initializes fields of the node that are specific to the anomaly detected by this
-         * detector.
-         */
-        abstract void initializeNode(@NonNull AnalysisNode info);
-
-        /**
-         * Detects anomalies by looking at the last occurrence of a view, and the current one.
-         * null value means that the view. 'oldInfo' and 'newInfo' cannot be both null.
-         * If an anomaly is detected, an exception will be thrown.
-         *
-         * @param oldInfo the view, as seen in the last frame that contained it in the view
-         *                hierarchy before 'currentFrame'. 'null' means that the view is first seen
-         *                in the 'currentFrame'.
-         * @param newInfo the view in the view hierarchy of the 'currentFrame'. 'null' means that
-         *                the view is not present in the 'currentFrame', but was present in earlier
-         *                frames.
-         * @param frameN  number of the current frame.
-         * @return Anomaly diagnostic message if an anomaly has been detected; null otherwise.
-         */
-        abstract String detectAnomalies(
-                @Nullable AnalysisNode oldInfo, @Nullable AnalysisNode newInfo, int frameN);
-    }
-
     // All detectors. They will be invoked in the order listed here.
     private static final AnomalyDetector[] ANOMALY_DETECTORS = {
-            new AlphaJumpDetector()
+            new AlphaJumpDetector(),
+            new FlashDetector()
     };
 
     static {
@@ -89,9 +56,21 @@ public class ViewCaptureAnalyzer {
         // Visible scale and alpha, build recursively from the ancestor list.
         public float scaleX;
         public float scaleY;
-        public float alpha;
+        public float alpha; // Always > 0
 
         public int frameN;
+
+        // Timestamp of the frame when this view became abruptly visible, i.e. its alpha became 1
+        // the next frame after it was 0 or the view wasn't visible.
+        // If the view is currently invisible or the last appearance wasn't abrupt, the value is -1.
+        public long timeBecameVisibleNs;
+
+        // Timestamp of the frame when this view became abruptly invisible last time, i.e. its
+        // alpha became 0, or view disappeared, after being 1 in the previous frame.
+        // If the view is currently visible or the last disappearance wasn't abrupt, the value is
+        // -1.
+        public long timeBecameInvisibleNs;
+
         public ViewNode viewCaptureNode;
 
         // Class name + resource id
@@ -143,7 +122,9 @@ public class ViewCaptureAnalyzer {
             Map<Integer, AnalysisNode> lastSeenNodes, int scrimClassIndex,
             Map<String, String> anomalies) {
         // Analyze the node tree starting from the root.
+        long frameTimeNs = frame.getTimestamp();
         analyzeView(
+                frameTimeNs,
                 frame.getNode(),
                 /* parent = */ null,
                 frameN,
@@ -154,7 +135,7 @@ public class ViewCaptureAnalyzer {
                 scrimClassIndex,
                 anomalies);
 
-        // Analyze transitions when a view visible in the last frame become invisible in the
+        // Analyze transitions when a view visible in the previous frame became invisible in the
         // current one.
         for (AnalysisNode info : lastSeenNodes.values()) {
             if (info.frameN == frameN - 1) {
@@ -166,14 +147,18 @@ public class ViewCaptureAnalyzer {
                                             frameN,
                                             /* oldInfo = */ info,
                                             /* newInfo = */ null,
-                                            anomalies)
+                                            anomalies,
+                                            frameTimeNs)
                     );
                 }
+                info.timeBecameInvisibleNs = info.alpha == 1 ? frameTimeNs : -1;
+                info.timeBecameVisibleNs = -1;
             }
         }
     }
 
-    private static void analyzeView(ViewNode viewCaptureNode, AnalysisNode parent, int frameN,
+    private static void analyzeView(long frameTimeNs, ViewNode viewCaptureNode, AnalysisNode parent,
+            int frameN,
             float leftShift, float topShift, ExportedData viewCaptureData,
             Map<Integer, AnalysisNode> lastSeenNodes, int scrimClassIndex,
             Map<String, String> anomalies) {
@@ -211,17 +196,31 @@ public class ViewCaptureAnalyzer {
         newAnalysisNode.scaleY = scaleY;
         newAnalysisNode.alpha = alpha;
         newAnalysisNode.frameN = frameN;
+        newAnalysisNode.timeBecameInvisibleNs = -1;
         newAnalysisNode.viewCaptureNode = viewCaptureNode;
         Arrays.stream(ANOMALY_DETECTORS).forEach(
                 detector -> detector.initializeNode(newAnalysisNode));
 
-        // Detect anomalies for the view
         final AnalysisNode oldAnalysisNode = lastSeenNodes.get(hashcode); // may be null
+
+        if (oldAnalysisNode != null && oldAnalysisNode.frameN + 1 == frameN) {
+            // If this view was present in the previous frame, keep the time when it became visible.
+            newAnalysisNode.timeBecameVisibleNs = oldAnalysisNode.timeBecameVisibleNs;
+        } else {
+            // If the view is becoming visible after being invisible, initialize the time when it
+            // became visible with a new value.
+            // If the view became visible abruptly, i.e. alpha jumped from 0 to 1 between the
+            // previous and the current frames, then initialize with the time of the current
+            // frame. Otherwise, use -1.
+            newAnalysisNode.timeBecameVisibleNs = newAnalysisNode.alpha >= 1 ? frameTimeNs : -1;
+        }
+
+        // Detect anomalies for the view.
         if (frameN != 0 && !viewCaptureNode.getWillNotDraw()) {
             Arrays.stream(ANOMALY_DETECTORS).forEach(
                     detector ->
                             detectAnomaly(detector, frameN, oldAnalysisNode, newAnalysisNode,
-                                    anomalies)
+                                    anomalies, frameTimeNs)
             );
         }
         lastSeenNodes.put(hashcode, newAnalysisNode);
@@ -236,20 +235,22 @@ public class ViewCaptureAnalyzer {
             // transparent.
             if (child.getClassnameIndex() == scrimClassIndex) break;
 
-            analyzeView(child, newAnalysisNode, frameN, leftShiftForChildren, topShiftForChildren,
+            analyzeView(frameTimeNs, child, newAnalysisNode, frameN, leftShiftForChildren,
+                    topShiftForChildren,
                     viewCaptureData, lastSeenNodes, scrimClassIndex, anomalies);
         }
     }
 
     private static void detectAnomaly(AnomalyDetector detector, int frameN,
             AnalysisNode oldAnalysisNode, AnalysisNode newAnalysisNode,
-            Map<String, String> anomalies) {
+            Map<String, String> anomalies, long frameTimeNs) {
         final String maybeAnomaly =
-                detector.detectAnomalies(oldAnalysisNode, newAnalysisNode, frameN);
+                detector.detectAnomalies(oldAnalysisNode, newAnalysisNode, frameN, frameTimeNs);
         if (maybeAnomaly != null) {
-            final String viewDiagPath = diagPathFromRoot(newAnalysisNode);
+            AnalysisNode latestInfo = newAnalysisNode != null ? newAnalysisNode : oldAnalysisNode;
+            final String viewDiagPath = diagPathFromRoot(latestInfo);
             if (!anomalies.containsKey(viewDiagPath)) {
-                anomalies.put(viewDiagPath, maybeAnomaly);
+                anomalies.put(viewDiagPath, String.format("%s, %s", maybeAnomaly, latestInfo));
             }
         }
     }