Optimize memory utilization for large # of icons

Optimize memory utilization for large # of icons by dynaically loading / unloading icons as they are scrolled into view.

Assets/Scripts/AppEntry.cs

@@ -1,32 +1,157 @@
-using System.Collections;
+using System;
+using System.Collections;
 using System.Collections.Generic;
+using System.Threading.Tasks;
 using TMPro;
 using UnityEngine;
+using UnityEngine.UI;
 
-public class AppEntry : MonoBehaviour
+namespace QuestAppLauncher
 {
-    // Sprite gameobject
-    public GameObject sprite;
-
-    // TMP text
-    public TextMeshProUGUI text;
-
-    // App entry contents
-    public string packageId;
-    public string appName;
-    public string externalIconPath;
-    public int installedApkIndex;
-    public bool isRenameMode;
-
-    // Start is called before the first frame update
-    void Start()
+    public class AppEntry : MonoBehaviour
     {
-        
-    }
+        // Sprite gameobject
+        public GameObject sprite = null;
+        public Image image = null;
+        public AspectRatioFitter aspectRatioFitter = null;
 
-    // Update is called once per frame
-    void Update()
-    {
-        
+        // TMP text
+        public TextMeshProUGUI text;
+
+        // Scroll view transform
+        public RectTransform scrollViewTransform;
+
+        // App entry contents
+        public string packageId;
+        public string appName;
+        public string externalIconPath = null;
+        public int installedApkIndex = -1;
+        public bool isRenameMode;
+        public bool dynamicallyLoadIcon;
+
+        const int MAX_FRAME_DELAYS = 35;
+        private float lastPos = -1f;
+        private bool isIconLoaded = false;
+        private int curFrameDelay = 0;
+        private int maxFrameDelays = 0;
+        private float maxDistance = 0f;
+        private System.Random rand = new System.Random();
+
+        // Start is called before the first frame update
+        void Start()
+        {
+            // Calculate max distance outside the visible scroll area, beyond which we'll unload the icon to reduce memory
+            this.maxDistance = 1.5f * this.scrollViewTransform.rect.size.y;
+
+            // Calculate the maximum frame delays when populating this icon; used to stagger loading of icons in the visible scroll area
+            this.maxFrameDelays = this.rand.Next(MAX_FRAME_DELAYS);
+        }
+
+        // Update is called once per frame
+        async void Update()
+        {
+            if (!this.dynamicallyLoadIcon || null == this.scrollViewTransform ||
+                (this.externalIconPath == null && this.installedApkIndex == -1))
+            {
+                return;
+            }
+
+            var localPos = this.scrollViewTransform.InverseTransformPoint(this.transform.position).y;
+            if (Mathf.Abs(localPos) < this.maxDistance)
+            {
+                // Don't bother loading icons while we're scrolling
+                if (this.lastPos == -1)
+                {
+                    this.lastPos = localPos;
+                    return;
+                }
+
+                if (this.lastPos != localPos)
+                {
+                    this.lastPos = localPos;
+                    return;
+
+                }
+
+                // We're within max distance, so show the icon after some delay (to stagger loading of icons)
+                if (this.curFrameDelay < this.maxFrameDelays)
+                {
+                    this.curFrameDelay++;
+                    return;
+                }
+                this.maxFrameDelays = this.rand.Next(MAX_FRAME_DELAYS);
+                this.curFrameDelay = 0;
+
+                if (!this.isIconLoaded)
+                {
+                    UnloadIcon();
+                    await LoadIcon();
+                    this.isIconLoaded = true;
+                }
+            }
+            else if (this.isIconLoaded)
+            {
+                // We're outside max distance, so unload the icon
+                UnloadIcon();
+                this.isIconLoaded = false;
+            }
+        }
+
+        public async Task LoadIcon()
+        {
+            var result = await AppProcessor.GetAppIconAsync(this.externalIconPath, this.installedApkIndex, 1024 * 1024);
+            var image = result.Item1;
+            var imageWidth = result.Item2;
+            var imageHeight = result.Item3;
+
+            Texture2D texture = null;
+
+            // Set the icon image
+            if (imageWidth == 0 || imageHeight == 0)
+            {
+                texture = new Texture2D(2, 2, TextureFormat.RGBA32, false);
+                texture.filterMode = FilterMode.Trilinear;
+                texture.anisoLevel = 16;
+                texture.LoadImage(image);
+            }
+            else
+            {
+                texture = new Texture2D(imageWidth, imageHeight, TextureFormat.ARGB32, false);
+                texture.filterMode = FilterMode.Trilinear;
+                texture.anisoLevel = 16;
+                texture.LoadRawTextureData(image);
+                texture.Apply();
+            }
+
+            var rect = new Rect(0, 0, texture.width, texture.height);
+            this.image.sprite = Sprite.Create(texture, rect, new Vector2(0.5f, 0.5f));
+            this.image.color = Color.white;
+
+            // Preserve icon's aspect ratio
+            this.aspectRatioFitter.aspectRatio = (float)texture.width / (float)texture.height;
+        }
+
+        private void UnloadIcon()
+        {
+            // We always manually create the sprite's texture object,
+            // so we must explicitly destroy it.
+            if (null != this.image.sprite)
+            {
+                if (null != this.image.sprite.texture)
+                {
+                    DestroyImmediate(image.sprite.texture);
+                }
+
+                DestroyImmediate(this.image.sprite);
+                this.image.sprite = null;
+                this.image.color = Color.clear;
+            }
+        }
+
+        private void OnDestroy()
+        {
+            Debug.Log("AppEntry.OnDestroy");
+            UnloadIcon();
+        }
     }
-}
+}

Assets/Scripts/AppProcessor.cs

@@ -482,17 +482,114 @@ namespace QuestAppLauncher
             }
         }
 
-        public static byte[] GetAppIcon(string iconPath, int appIndex)
+        /// <summary>
+        /// Loads an image from specified path. Scaled down if image is larger than max pixels.
+        /// </summary>
+        /// <param name="path">Image path</param>
+        /// <param name="maxPixels">Max pixels</param>
+        /// <returns>Image and dimensions</returns>
+        public static async Task<(byte[], int, int)> LoadRawImageAsync(string path, int maxPixels)
         {
-            byte[] bytesIcon = null;
-            bool useApkIcon = true;
-            if (null != iconPath)
+            int imageHeight = 0;
+            int imageWidth = 0;
+            byte[] image = null;
+
+            await Task.Run(() =>
             {
-                // Use overridden icon
+                AndroidJNI.AttachCurrentThread();
+
                 try
                 {
-                    bytesIcon = File.ReadAllBytes(iconPath);
-                    useApkIcon = false;
+                    LoadRawImage(path, maxPixels, out image, out imageWidth, out imageHeight);
+                }
+                finally
+                {
+                    AndroidJNI.DetachCurrentThread();
+                }
+            });
+
+            return (image, imageWidth, imageHeight);
+        }
+
+        /// <summary>
+        /// Loads an image from specified path. Scaled down if image is larger than max pixels.
+        /// </summary>
+        /// <param name="path">Image path</param>
+        /// <param name="maxPixels">Max pixels</param>
+        /// <param name="image">Output raw image byte array</param>
+        /// <param name="imageWidth">Output width</param>
+        /// <param name="imageHeight">Output height</param>
+        public static void LoadRawImage(string path, int maxPixels, out byte[] image, out int imageWidth, out int imageHeight)
+        {
+            image = null;
+            imageWidth = 0;
+            imageHeight = 0;
+
+            try
+            {
+                using (AndroidJavaClass unity = new AndroidJavaClass("com.unity3d.player.UnityPlayer"))
+                using (AndroidJavaObject currentActivity = unity.GetStatic<AndroidJavaObject>("currentActivity"))
+                {
+                    // Call Android plugin to load the raw image.
+                    var jo = currentActivity.CallStatic<AndroidJavaObject>("loadRawImage", path, maxPixels);
+                    if (null == jo)
+                    {
+                        return;
+                    }
+
+                    // Get the width, height and raw image data.
+                    imageWidth = jo.Get<int>("width");
+                    imageHeight = jo.Get<int>("height");
+                    var rawImage = (byte[])(Array)jo.Get<sbyte[]>("rawImage");
+
+                    // The image is in ARGB_8888 format (Alpha, Red, Green, Blue - each 1 byte). In addition, (0, 0) coordinates are bottom-left.
+                    // Unity expects RGBA (with Alpha as the last byte) and origin at top-left. So we need to compensate for both.
+
+                    // Shift alpha
+                    for (var i = 0; i < rawImage.Length / 4; i++)
+                    {
+                        var tmp = rawImage[i * 4 + 3];
+                        rawImage[i * 4 + 3] = rawImage[i * 4 + 2];
+                        rawImage[i * 4 + 2] = rawImage[i * 4 + 1];
+                        rawImage[i * 4 + 1] = rawImage[i * 4];
+                        rawImage[i * 4] = tmp;
+                    }
+
+                    // Swap rows
+                    var row = new byte[imageWidth * 4];
+                    for (var i = 0; i < imageHeight / 2; i++)
+                    {
+                        Buffer.BlockCopy(rawImage, i * imageWidth * 4, row, 0, imageWidth * 4);
+                        Buffer.BlockCopy(rawImage, (imageHeight - i - 1) * imageWidth * 4, rawImage, i * imageWidth * 4, imageWidth * 4);
+                        Buffer.BlockCopy(row, 0, rawImage, (imageHeight - i - 1) * imageWidth * 4, imageWidth * 4);
+                    }
+
+                    image = rawImage;
+                }
+            }
+            catch (Exception e)
+            {
+                // Fall back to using the apk icon
+                Debug.LogFormat("Error decoding image [{0}]: {1}", path, e.Message);
+            }
+        }
+
+        /// <summary>
+        /// Loads an app icon asynchronously, either from specified external icon path or, if path is not provided or fails to load,
+        /// falls back to loading the icon from the apk itself.
+        /// </summary>
+        /// <param name="iconPath">External icon path</param>
+        /// <param name="appIndex">App index (used when falling back to loading icon from APK)</param>
+        /// <param name="maxPixels">Max pixels - image will be scaled down if larger than this size</param>
+        /// <returns>Icon bytes and dimensions</returns>
+        public static async Task<(byte[], int, int)> GetAppIconAsync(string iconPath, int appIndex, int maxPixels)
+        {
+            if (null != iconPath)
+            {
+                // Load icon from file path
+                try
+                {
+                    return await LoadRawImageAsync(iconPath, maxPixels);
                 }
                 catch (Exception e)
                 {
@@ -501,17 +598,27 @@ namespace QuestAppLauncher
                 }
             }
 
-            if (useApkIcon)
+            byte[] bytesIcon = null;
+            await Task.Run(() =>
             {
-                // Use built-in icon from the apk
-                using (AndroidJavaClass unity = new AndroidJavaClass("com.unity3d.player.UnityPlayer"))
-                using (AndroidJavaObject currentActivity = unity.GetStatic<AndroidJavaObject>("currentActivity"))
-                {
-                    bytesIcon = (byte[])(Array)currentActivity.Call<sbyte[]>("getIcon", appIndex);
-                }
-            }
+                AndroidJNI.AttachCurrentThread();
 
-            return bytesIcon;
+                try
+                {
+                    // Use built-in icon from the apk
+                    using (AndroidJavaClass unity = new AndroidJavaClass("com.unity3d.player.UnityPlayer"))
+                    using (AndroidJavaObject currentActivity = unity.GetStatic<AndroidJavaObject>("currentActivity"))
+                    {
+                        bytesIcon = (byte[])(Array)currentActivity.Call<sbyte[]>("getIcon", appIndex);
+                    }
+                }
+                finally
+                {
+                    AndroidJNI.DetachCurrentThread();
+                }
+            });
+
+            return (bytesIcon, 0, 0);
         }
 
         /// <summary>

Assets/Scripts/GridPopulation.cs

@@ -28,6 +28,11 @@ namespace QuestAppLauncher
             }
         }
 
+        // Max icons to initially populate. If we have more than this number,
+        // then we fall back into dynamically loading icons as we scroll to conserve memory.
+        const int MaxIconsOnInitialPopulation = 150;
+
+
         // Grid container game object
         public GameObject panelContainer;
 
@@ -211,29 +216,31 @@ namespace QuestAppLauncher
                 ResizeTabContent(this.rightTabContainer.transform, size, rightTabs.Count, false);
             }
 
+            bool loadIcons = apps.Count <= MaxIconsOnInitialPopulation;
+
             // Populate grid with app information (name & icon)
             // Sort by custom comparer
             foreach (var app in apps.OrderBy(key => key.Value, new AppComparer()))
             {
                 // Add to all tab
-                await AddCellToGridAsync(app.Value, gridContents[AppProcessor.Tab_All].transform);
+                await AddCellToGridAsync(app.Value, gridContents[AppProcessor.Tab_All].transform, loadIcons);
 
                 // Add to auto (built-in) tabs
                 if (gridContents.ContainsKey(app.Value.AutoTabName))
                 {
-                    await AddCellToGridAsync(app.Value, gridContents[app.Value.AutoTabName].transform);
+                    await AddCellToGridAsync(app.Value, gridContents[app.Value.AutoTabName].transform, loadIcons);
                 }
 
                 // Add to tab1
                 if (null != app.Value.Tab1Name && gridContents.ContainsKey(app.Value.Tab1Name))
                 {
-                    await AddCellToGridAsync(app.Value, gridContents[app.Value.Tab1Name].transform);
+                    await AddCellToGridAsync(app.Value, gridContents[app.Value.Tab1Name].transform, loadIcons);
                 }
 
                 // Add to tab2
                 if (null != app.Value.Tab2Name && gridContents.ContainsKey(app.Value.Tab2Name))
                 {
-                    await AddCellToGridAsync(app.Value, gridContents[app.Value.Tab2Name].transform);
+                    await AddCellToGridAsync(app.Value, gridContents[app.Value.Tab2Name].transform, loadIcons);
                 }
             }
         }
@@ -252,11 +259,13 @@ namespace QuestAppLauncher
             // Set panel size
             SetGridSize(this.renamePanelContainer, gridContent, config.gridSize.rows, config.gridSize.cols);
 
+            bool loadIcons = apps.Count <= MaxIconsOnInitialPopulation;
+
             // Populate grid with app information (name & icon)
             // Sort alphabetically
             foreach (var app in apps.OrderBy(key => key.Value.AppName, StringComparer.InvariantCultureIgnoreCase))
             {
-                await AddCellToGridAsync(app.Value, gridContent.transform, true);
+                await AddCellToGridAsync(app.Value, gridContent.transform, loadIcons, true);
             }
         }
 
@@ -359,7 +368,7 @@ namespace QuestAppLauncher
             }
         }
 
-        private async Task AddCellToGridAsync(ProcessedApp app, Transform transform, bool isRenameMode = false)
+        private async Task AddCellToGridAsync(ProcessedApp app, Transform transform, bool loadIcon, bool isRenameMode = false)
         {
             if (app.Index == -1 && string.IsNullOrEmpty(app.IconPath))
             {
@@ -367,46 +376,25 @@ namespace QuestAppLauncher
                 return;
             }
 
-            // Get app icon in background
-            var bytesIcon = await Task.Run(() =>
-            {
-                AndroidJNI.AttachCurrentThread();
-
-                try
-                {
-                    return AppProcessor.GetAppIcon(app.IconPath, app.Index);
-                }
-                finally
-                {
-                    AndroidJNI.DetachCurrentThread();
-                }
-            });
-
             // Create new instances of our app info prefabCell
             var newObj = (GameObject)Instantiate(this.prefabCell, transform);
 
             // Set app entry info
             var appEntry = newObj.GetComponent<AppEntry>();
+            appEntry.scrollViewTransform = transform.parent.parent.gameObject.GetComponent<RectTransform>();
             appEntry.packageId = app.PackageName;
             appEntry.appName = app.AppName;
             appEntry.isRenameMode = isRenameMode;
             appEntry.installedApkIndex = app.Index;
             appEntry.externalIconPath = app.IconPath;
 
-            // Set the icon image
-            if (null != bytesIcon)
-            {
-                var texture = new Texture2D(2, 2, TextureFormat.RGBA32, false);
-                texture.filterMode = FilterMode.Trilinear;
-                texture.anisoLevel = 16;
-                texture.LoadImage(bytesIcon);
-                var rect = new Rect(0, 0, texture.width, texture.height);
-                var image = appEntry.sprite.GetComponent<Image>();
-                image.sprite = Sprite.Create(texture, rect, new Vector2(0.5f, 0.5f));
+            // Dynamically load icon if we're not loading the icon now
+            appEntry.dynamicallyLoadIcon = !loadIcon;
 
-                // Preserve icon's aspect ratio
-                var aspectRatioFitter = appEntry.sprite.GetComponent<AspectRatioFitter>();
-                aspectRatioFitter.aspectRatio = (float)texture.width / (float)texture.height;
+            // Set the icon image
+            if (loadIcon)
+            {
+                await appEntry.LoadIcon();
             }
 
             // Set app name in text

Assets/Scripts/SkyboxHandler.cs

@@ -12,9 +12,8 @@ namespace QuestAppLauncher
     public class SkyboxHandler : MonoBehaviour
     {
         // Max pixles of skybox image for both equirectangular and cubemap images.
-        // We calculate this to be: 2048px x 2048px x 6 faces (for cubemap).
         // Anything larger we'll scale down. We restrict it primarily due to memory constraints.
-        const int MaxPixels = 2048 * 2048 * 6;
+        const int MaxPixels = 4096 * 4096;
 
         // Skybox selected callback
         public Action<string> OnSkyboxSelected;
@@ -142,66 +141,26 @@ namespace QuestAppLauncher
                 return;
             }
 
-            // Read the image
-            int imageHeight = 0;
-            int imageWidth = 0;
-            var image = await Task.Run(() =>
+            // Destroy existing skybox
+            if (null != RenderSettings.skybox && RenderSettings.skybox != this.defaultSkybox)
             {
-                AndroidJNI.AttachCurrentThread();
-
-                try
+                // Destroy texture
+                var oldTexture = RenderSettings.skybox.GetTexture("_Tex");
+                if (null != oldTexture)
                 {
-                    using (AndroidJavaClass unity = new AndroidJavaClass("com.unity3d.player.UnityPlayer"))
-                    using (AndroidJavaObject currentActivity = unity.GetStatic<AndroidJavaObject>("currentActivity"))
-                    {
-                        // Call Android plugin to load the raw image.
-                        var jo = currentActivity.CallStatic<AndroidJavaObject>("loadRawImage", MakeAbsoluteSkymapPath(skyboxPath), MaxPixels);
-                        if (null == jo)
-                        {
-                            return null;
-                        }
-
-                        // Get the width, height and raw image data.
-                        imageWidth = jo.Get<int>("width");
-                        imageHeight = jo.Get<int>("height");
-                        var rawImage = (byte[])(Array)jo.Get<sbyte[]>("rawImage");
-
-                        // The image is in ARGB_8888 format (Alpha, Red, Green, Blue - each 1 byte). In addition, (0, 0) coordinates are bottom-left.
-                        // Unity expects RGBA (with Alpha as the last byte) and origin at top-left. So we need to compensate for both.
-
-                        // Shift alpha
-                        for (var i = 0; i < rawImage.Length / 4; i++)
-                        {
-                            var tmp = rawImage[i * 4 + 3];
-                            rawImage[i * 4 + 3] = rawImage[i * 4 + 2];
-                            rawImage[i * 4 + 2] = rawImage[i * 4 + 1];
-                            rawImage[i * 4 + 1] = rawImage[i * 4];
-                            rawImage[i * 4] = tmp;
-                        }
-
-                        // Swap rows
-                        var row = new byte[imageWidth * 4];
-                        for (var i = 0; i < imageHeight / 2; i++)
-                        {
-                            Buffer.BlockCopy(rawImage, i * imageWidth * 4, row, 0, imageWidth * 4);
-                            Buffer.BlockCopy(rawImage, (imageHeight - i - 1) * imageWidth * 4, rawImage, i * imageWidth * 4, imageWidth * 4);
-                            Buffer.BlockCopy(row, 0, rawImage, (imageHeight - i - 1) * imageWidth * 4, imageWidth * 4);
-                        }
-
-                        return rawImage;
-                    }
+                    DestroyImmediate(oldTexture);
                 }
-                catch (Exception e)
-                {
-                    // Fall back to using the apk icon
-                    Debug.LogFormat("Error decoding image [{0}]: {1}", skyboxPath, e.Message);
-                    return null;
-                }
-                finally
-                {
-                    AndroidJNI.DetachCurrentThread();
-                }
-            });
+
+                // Destroy material
+                DestroyImmediate(RenderSettings.skybox);
+                RenderSettings.skybox = null;
+            }
+
+            // Read the image
+            var result = await AppProcessor.LoadRawImageAsync(MakeAbsoluteSkymapPath(skyboxPath), MaxPixels);
+            var image = result.Item1;
+            var imageWidth = result.Item2;
+            var imageHeight = result.Item3;
 
             if (null == image)
             {
@@ -210,23 +169,27 @@ namespace QuestAppLauncher
                 return;
             }
 
+            Texture2D texture = null;
+            Material material = null;
+            bool destroyTexture = false;
+
             try
             {
                 // Load the image into a 2D texture. We decode in background thread (above) in Java and load the raw image here
                 // because Texture2D.LoadImage on the main thread can cause significant freezes since it is not async.
-                var texture = new Texture2D(imageWidth, imageHeight, TextureFormat.ARGB32, false);
+                texture = new Texture2D(imageWidth, imageHeight, TextureFormat.ARGB32, false);
                 texture.filterMode = FilterMode.Trilinear;
                 texture.anisoLevel = 16;
                 texture.LoadRawTextureData(image);
                 texture.Apply();
 
-                Material material;
                 if (4 * texture.height == 3 * texture.width)
                 {
                     // Texture is a horizontal cross cube map (4:3 aspect ratio).
                     // Load cubemap shader. Also rotate x-axis by 180 degrees to compensate for platform-specific rendering differences
                     // (see https://docs.unity3d.com/Manual/SL-PlatformDifferences.html).
                     Debug.LogFormat("Setting horizontal-cross cubemap skybox");
+                    destroyTexture = true;
                     material = new Material(Shader.Find("skybox/cube"));
                     material.SetFloat("_RotationX", 180);
                     material.SetTexture("_Tex", CubemapFromHorizCrossTexture2D(texture));
@@ -237,6 +200,7 @@ namespace QuestAppLauncher
                     // Load cubemap shader. Also rotate x-axis by 180 degrees to compensate for platform-specific rendering differences
                     // (see https://docs.unity3d.com/Manual/SL-PlatformDifferences.html).
                     Debug.LogFormat("Setting horizontal cubemap skybox");
+                    destroyTexture = true;
                     material = new Material(Shader.Find("skybox/cube"));
                     material.SetFloat("_RotationX", 180);
                     material.SetTexture("_Tex", CubemapFromHorizTexture2D(texture));
@@ -258,6 +222,13 @@ namespace QuestAppLauncher
                 Debug.LogFormat("Exception: {0}", e.Message);
                 SetDefaultSkybox();
             }
+            finally
+            {
+                if (destroyTexture && null != texture)
+                {
+                    DestroyImmediate(texture);
+                }
+            }
         }
 
         /// <summary>
@@ -292,7 +263,7 @@ namespace QuestAppLauncher
         /// <returns></returns>
         private static Cubemap CubemapFromHorizTexture2D(Texture2D texture)
         {
-            int cubedim = texture.width / 4;
+            int cubedim = texture.height;
             Cubemap cube = new Cubemap(cubedim, TextureFormat.ARGB32, false);
             cube.SetPixels(texture.GetPixels(0, 0, cubedim, cubedim), CubemapFace.PositiveX);
             cube.SetPixels(texture.GetPixels(cubedim, 0, cubedim, cubedim), CubemapFace.NegativeX);