Consistent "low storage" behavior.

Fix several bugs related to storage accounting.  Since getDataBytes()
already includes cached data, we need to subtract it to avoid blaming
apps for it.

We also need to blame app code on someone, so we blame it on the
current user.  StorageStatsManager was fixed awhile back to only
return the app code size on the requested storage volume, so we can
remove the system app checks.

Subtract "appBytes" from external storage accounting, since it's
already been blamed elsewhere against specific apps.

Pass along storage results from all users on the device, and subtract
them all when estimating size of "system" data.  To avoid embarrassing
estimation bugs, make sure that "system" data is at least 1GB.

Bug: 38008706
Test: cts-tradefed run commandAndExit cts-dev -m CtsJobSchedulerTestCases -t android.jobscheduler.cts.StorageConstraintTest
Test: cts-tradefed run commandAndExit cts-dev -m CtsAppSecurityHostTestCases -t android.appsecurity.cts.StorageHostTest
Change-Id: Ide1e6d0690e5ad4e751c87891f63ba1036434619

src/com/android/settings/deviceinfo/StorageDashboardFragment.java

@@ -131,7 +131,7 @@ public class StorageDashboardFragment extends DashboardFragment
             }
         }
 
-        mPreferenceController.onLoadFinished(mAppsResult.get(UserHandle.myUserId()));
+        mPreferenceController.onLoadFinished(mAppsResult, UserHandle.myUserId());
         updateSecondaryUserControllers(mSecondaryUsers, mAppsResult);
 
         // setLoading always causes a flicker, so let's avoid doing it.

src/com/android/settings/deviceinfo/StorageProfileFragment.java

@@ -120,7 +120,8 @@ public class StorageProfileFragment extends DashboardFragment
     @Override
     public void onLoadFinished(Loader<SparseArray<AppsStorageResult>> loader,
             SparseArray<AppsStorageResult> result) {
-        mPreferenceController.onLoadFinished(scrubAppsFromResult(result.get(mUserId)));
+        scrubAppsFromResult(result.get(mUserId));
+        mPreferenceController.onLoadFinished(result, mUserId);
     }
 
     @Override

src/com/android/settings/deviceinfo/storage/StorageAsyncLoader.java

@@ -22,8 +22,8 @@ import static android.content.pm.ApplicationInfo.CATEGORY_VIDEO;
 
 import android.content.Context;
 import android.content.pm.ApplicationInfo;
-import android.content.pm.UserInfo;
 import android.content.pm.PackageManager.NameNotFoundException;
+import android.content.pm.UserInfo;
 import android.os.UserHandle;
 import android.util.Log;
 import android.util.SparseArray;
@@ -87,45 +87,43 @@ public class StorageAsyncLoader
                 stats = mStatsManager.getStatsForPackage(mUuid, app.packageName, myUser);
             } catch (NameNotFoundException | IOException e) {
                 // This may happen if the package was removed during our calculation.
-                Log.w("App unexpectedly not found", e);
+                Log.w(TAG, "App unexpectedly not found", e);
                 continue;
             }
 
-            long attributedAppSizeInBytes = stats.getDataBytes();
+            long blamedSize = stats.getDataBytes() - stats.getCacheBytes();
-            // This matches how the package manager calculates sizes -- by zeroing out code sizes of
+
-            // system apps which are not updated. My initial tests suggest that this results in the
+            // Only count app code against the current user; we don't want
-            // original code size being counted for updated system apps when they shouldn't, but
+            // double-counting on multi-user devices.
-            // I am not sure how to avoid this problem without specifically going in to find that
+            if (userId == UserHandle.myUserId()) {
-            // code size.
+                blamedSize += stats.getCodeBytes();
-            if (!app.isSystemApp() || app.isUpdatedSystemApp()) {
-                attributedAppSizeInBytes += stats.getCodeBytes();
-            } else {
-                result.systemSize += stats.getCodeBytes();
             }
+
             switch (app.category) {
                 case CATEGORY_GAME:
-                    result.gamesSize += attributedAppSizeInBytes;
+                    result.gamesSize += blamedSize;
                     break;
                 case CATEGORY_AUDIO:
-                    result.musicAppsSize += attributedAppSizeInBytes;
+                    result.musicAppsSize += blamedSize;
                     break;
                 case CATEGORY_VIDEO:
-                    result.videoAppsSize += attributedAppSizeInBytes;
+                    result.videoAppsSize += blamedSize;
                     break;
                 default:
                     // The deprecated game flag does not set the category.
                     if ((app.flags & ApplicationInfo.FLAG_IS_GAME) != 0) {
-                        result.gamesSize += attributedAppSizeInBytes;
+                        result.gamesSize += blamedSize;
                         break;
                     }
-                    result.otherAppsSize += attributedAppSizeInBytes;
+                    result.otherAppsSize += blamedSize;
                     break;
             }
         }
 
         Log.d(TAG, "Loading external stats");
         try {
-            result.externalStats = mStatsManager.getExternalStorageStats(mUuid, UserHandle.of(userId));
+            result.externalStats = mStatsManager.getExternalStorageStats(mUuid,
+                    UserHandle.of(userId));
         } catch (IOException e) {
             Log.w(TAG, e);
         }
@@ -142,7 +140,6 @@ public class StorageAsyncLoader
         public long musicAppsSize;
         public long videoAppsSize;
         public long otherAppsSize;
-        public long systemSize;
         public StorageStatsSource.ExternalStorageStats externalStats;
     }
 

src/com/android/settings/deviceinfo/storage/StorageItemPreferenceController.java

@@ -23,6 +23,7 @@ import android.content.Intent;
 import android.content.pm.PackageManager;
 import android.content.res.TypedArray;
 import android.graphics.drawable.Drawable;
+import android.net.TrafficStats;
 import android.os.Bundle;
 import android.os.UserHandle;
 import android.os.storage.VolumeInfo;
@@ -30,6 +31,7 @@ import android.support.annotation.VisibleForTesting;
 import android.support.v7.preference.Preference;
 import android.support.v7.preference.PreferenceScreen;
 import android.util.Log;
+import android.util.SparseArray;
 
 import com.android.internal.logging.nano.MetricsProto.MetricsEvent;
 import com.android.settings.R;
@@ -237,7 +239,10 @@ public class StorageItemPreferenceController extends PreferenceController {
         setFilesPreferenceVisibility();
     }
 
-    public void onLoadFinished(StorageAsyncLoader.AppsStorageResult data) {
+    public void onLoadFinished(SparseArray<StorageAsyncLoader.AppsStorageResult> result,
+            int userId) {
+        final StorageAsyncLoader.AppsStorageResult data = result.get(userId);
+
         // TODO(b/35927909): Figure out how to split out apps which are only installed for work
         //       profiles in order to attribute those app's code bytes only to that profile.
         mPhotoPreference.setStorageSize(
@@ -248,23 +253,30 @@ public class StorageItemPreferenceController extends PreferenceController {
         mMoviesPreference.setStorageSize(data.videoAppsSize, mTotalSize);
         mAppPreference.setStorageSize(data.otherAppsSize, mTotalSize);
 
-        long unattributedExternalBytes =
+        long otherExternalBytes =
                 data.externalStats.totalBytes
                         - data.externalStats.audioBytes
                         - data.externalStats.videoBytes
-                        - data.externalStats.imageBytes;
+                        - data.externalStats.imageBytes
-        mFilePreference.setStorageSize(unattributedExternalBytes, mTotalSize);
+                        - data.externalStats.appBytes;
+        mFilePreference.setStorageSize(otherExternalBytes, mTotalSize);
 
-        // We define the system size as everything we can't classify.
         if (mSystemPreference != null) {
-            mSystemPreference.setStorageSize(
+            // Everything else that hasn't already been attributed is tracked as
-                    mUsedBytes
+            // belonging to system.
-                            - data.externalStats.totalBytes
+            long attributedSize = 0;
-                            - data.musicAppsSize
+            for (int i = 0; i < result.size(); i++) {
-                            - data.gamesSize
+                final StorageAsyncLoader.AppsStorageResult otherData = result.valueAt(i);
-                            - data.videoAppsSize
+                attributedSize += otherData.gamesSize
-                            - data.otherAppsSize,
+                        + otherData.musicAppsSize
-                    mTotalSize);
+                        + otherData.videoAppsSize
+                        + otherData.otherAppsSize;
+                attributedSize += otherData.externalStats.totalBytes
+                        - otherData.externalStats.appBytes;
+            }
+
+            final long systemSize = Math.max(TrafficStats.GB_IN_BYTES, mUsedBytes - attributedSize);
+            mSystemPreference.setStorageSize(systemSize, mTotalSize);
         }
     }
 

tests/robotests/src/com/android/settings/applications/MusicViewHolderControllerTest.java

@@ -83,7 +83,7 @@ public class MusicViewHolderControllerTest {
     @Test
     public void storageShouldRepresentStorageStatsQuery() throws Exception {
         when(mSource.getExternalStorageStats(any(String.class), any(UserHandle.class))).thenReturn(
-                new StorageStatsSource.ExternalStorageStats(1, 1, 0, 0));
+                new StorageStatsSource.ExternalStorageStats(1, 1, 0, 0, 0));
 
         mController.queryStats();
         mController.setupView(mHolder);

tests/robotests/src/com/android/settings/deviceinfo/StorageProfileFragmentTest.java

@@ -17,6 +17,7 @@ package com.android.settings.deviceinfo;
 
 import static com.google.common.truth.Truth.assertThat;
 
+import static org.mockito.Matchers.anyInt;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
 
@@ -31,11 +32,16 @@ import com.android.settingslib.applications.StorageStatsSource;
 import org.junit.Test;
 import org.junit.runner.RunWith;
 import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.MockitoAnnotations;
 import org.robolectric.annotation.Config;
 
 @RunWith(SettingsRobolectricTestRunner.class)
 @Config(manifest = TestConfig.MANIFEST_PATH, sdk = TestConfig.SDK_VERSION)
 public class StorageProfileFragmentTest {
+    @Captor
+    private ArgumentCaptor<SparseArray<StorageAsyncLoader.AppsStorageResult>> mCaptor;
+
     @Test
     public void verifyAppSizesAreZeroedOut() {
         StorageItemPreferenceController controller = mock(StorageItemPreferenceController.class);
@@ -45,18 +51,17 @@ public class StorageProfileFragmentTest {
         result.otherAppsSize = 200;
         result.gamesSize = 300;
         result.videoAppsSize = 400;
-        result.externalStats = new StorageStatsSource.ExternalStorageStats(6, 1, 2, 3);
+        result.externalStats = new StorageStatsSource.ExternalStorageStats(6, 1, 2, 3, 0);
         SparseArray<StorageAsyncLoader.AppsStorageResult> resultsArray = new SparseArray<>();
         resultsArray.put(0, result);
         fragment.setPreferenceController(controller);
 
         fragment.onLoadFinished(null, resultsArray);
 
-        ArgumentCaptor<StorageAsyncLoader.AppsStorageResult> resultCaptor = ArgumentCaptor.forClass(
+        MockitoAnnotations.initMocks(this);
-                StorageAsyncLoader.AppsStorageResult.class);
+        verify(controller).onLoadFinished(mCaptor.capture(), anyInt());
-        verify(controller).onLoadFinished(resultCaptor.capture());
 
-        StorageAsyncLoader.AppsStorageResult extractedResult = resultCaptor.getValue();
+        StorageAsyncLoader.AppsStorageResult extractedResult = mCaptor.getValue().get(0);
         assertThat(extractedResult.musicAppsSize).isEqualTo(0);
         assertThat(extractedResult.videoAppsSize).isEqualTo(0);
         assertThat(extractedResult.otherAppsSize).isEqualTo(0);

tests/robotests/src/com/android/settings/deviceinfo/storage/SecondaryUserControllerTest.java

@@ -168,7 +168,7 @@ public class SecondaryUserControllerTest {
                         MEGABYTE_IN_BYTES * 30,
                         MEGABYTE_IN_BYTES * 10,
                         MEGABYTE_IN_BYTES * 10,
-                        MEGABYTE_IN_BYTES * 10);
+                        MEGABYTE_IN_BYTES * 10, 0);
         result.put(10, userResult);
 
         mController.handleResult(result);

tests/robotests/src/com/android/settings/deviceinfo/storage/StorageItemPreferenceControllerTest.java

@@ -28,7 +28,6 @@ import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
 
-
 import android.app.Fragment;
 import android.content.Context;
 import android.content.Intent;
@@ -36,6 +35,7 @@ import android.graphics.drawable.Drawable;
 import android.os.UserHandle;
 import android.os.storage.VolumeInfo;
 import android.support.v7.preference.PreferenceScreen;
+import android.util.SparseArray;
 import android.view.LayoutInflater;
 import android.view.View;
 import android.widget.LinearLayout;
@@ -275,22 +275,22 @@ public class StorageItemPreferenceControllerTest {
         result.videoAppsSize = MEGABYTE_IN_BYTES * 160;
         result.musicAppsSize = MEGABYTE_IN_BYTES * 40;
         result.otherAppsSize = MEGABYTE_IN_BYTES * 90;
-        result.systemSize = MEGABYTE_IN_BYTES * 100; // This value is ignored and overridden now.
         result.externalStats =
                 new StorageStatsSource.ExternalStorageStats(
                         MEGABYTE_IN_BYTES * 500, // total
                         MEGABYTE_IN_BYTES * 100, // audio
                         MEGABYTE_IN_BYTES * 150, // video
-                        MEGABYTE_IN_BYTES * 200); // image
+                        MEGABYTE_IN_BYTES * 200, 0); // image
 
-        mController.onLoadFinished(result);
+        SparseArray<StorageAsyncLoader.AppsStorageResult> results = new SparseArray<>();
+        results.put(0, result);
+        mController.onLoadFinished(results, 0);
 
         assertThat(audio.getSummary().toString()).isEqualTo("0.14GB");
         assertThat(image.getSummary().toString()).isEqualTo("0.35GB");
         assertThat(games.getSummary().toString()).isEqualTo("0.08GB");
         assertThat(movies.getSummary().toString()).isEqualTo("0.16GB");
         assertThat(apps.getSummary().toString()).isEqualTo("0.09GB");
-        assertThat(system.getSummary().toString()).isEqualTo("0.10GB");
         assertThat(files.getSummary().toString()).isEqualTo("0.05GB");
     }
 

tests/robotests/src/com/android/settings/deviceinfo/storage/UserProfileControllerTest.java

@@ -114,7 +114,7 @@ public class UserProfileControllerTest {
                         99 * MEGABYTE_IN_BYTES,
                         33 * MEGABYTE_IN_BYTES,
                         33 * MEGABYTE_IN_BYTES,
-                        33 * MEGABYTE_IN_BYTES);
+                        33 * MEGABYTE_IN_BYTES, 0);
         result.put(10, userResult);
 
         mController.handleResult(result);

tests/unit/src/com/android/settings/deviceinfo/storage/StorageAsyncLoaderTest.java

@@ -133,9 +133,9 @@ public class StorageAsyncLoaderTest {
         info.id = SECONDARY_USER_ID;
         mUsers.add(info);
         when(mSource.getExternalStorageStats(anyString(), eq(UserHandle.SYSTEM)))
-                .thenReturn(new StorageStatsSource.ExternalStorageStats(9, 2, 3, 4));
+                .thenReturn(new StorageStatsSource.ExternalStorageStats(9, 2, 3, 4, 0));
         when(mSource.getExternalStorageStats(anyString(), eq(new UserHandle(SECONDARY_USER_ID))))
-                .thenReturn(new StorageStatsSource.ExternalStorageStats(10, 3, 3, 4));
+                .thenReturn(new StorageStatsSource.ExternalStorageStats(10, 3, 3, 4, 0));
 
         SparseArray<StorageAsyncLoader.AppsStorageResult> result = mLoader.loadInBackground();
 
@@ -144,19 +144,6 @@ public class StorageAsyncLoaderTest {
         assertThat(result.get(SECONDARY_USER_ID).externalStats.totalBytes).isEqualTo(10L);
     }
 
-    @Test
-    public void testSystemAppsBaseSizeIsAddedToSystem() throws Exception {
-        ApplicationInfo systemApp =
-                addPackage(PACKAGE_NAME_1, 100, 1, 10, ApplicationInfo.CATEGORY_UNDEFINED);
-        systemApp.flags = ApplicationInfo.FLAG_SYSTEM;
-
-        SparseArray<StorageAsyncLoader.AppsStorageResult> result = mLoader.loadInBackground();
-
-        assertThat(result.size()).isEqualTo(1);
-        assertThat(result.get(PRIMARY_USER_ID).otherAppsSize).isEqualTo(10L);
-        assertThat(result.get(PRIMARY_USER_ID).systemSize).isEqualTo(1L);
-    }
-
     @Test
     public void testUpdatedSystemAppCodeSizeIsCounted() throws Exception {
         ApplicationInfo systemApp =