diff --git a/src/com/android/settings/bluetooth/CachedBluetoothDevice.java b/src/com/android/settings/bluetooth/CachedBluetoothDevice.java
index a7104df6439..3b64ade0b97 100755
--- a/src/com/android/settings/bluetooth/CachedBluetoothDevice.java
+++ b/src/com/android/settings/bluetooth/CachedBluetoothDevice.java
@@ -539,7 +539,7 @@ final class CachedBluetoothDevice implements Comparable<CachedBluetoothDevice> {
         if (bondState == BluetoothDevice.BOND_NONE) {
             mProfiles.clear();
             mConnectAfterPairing = false;  // cancel auto-connect
-            setPhonebookPermissionChoice(ACCESS_ALLOWED);
+            setPhonebookPermissionChoice(ACCESS_UNKNOWN);
             setMessagePermissionChoice(ACCESS_UNKNOWN);
             mPhonebookRejectedTimes = 0;
             savePhonebookRejectTimes();