path: root/services/core/java/com/android/server/RescueParty.java

/*
 * Copyright (C) 2017 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package com.android.server;

import static android.provider.DeviceConfig.Properties;

import static com.android.server.pm.PackageManagerServiceUtils.logCriticalInfo;

import android.annotation.IntDef;
import android.annotation.NonNull;
import android.annotation.Nullable;
import android.content.ContentResolver;
import android.content.Context;
import android.content.pm.ApplicationInfo;
import android.content.pm.PackageManager;
import android.content.pm.VersionedPackage;
import android.crashrecovery.flags.Flags;
import android.os.Build;
import android.os.Environment;
import android.os.FileUtils;
import android.os.PowerManager;
import android.os.RecoverySystem;
import android.os.SystemClock;
import android.os.SystemProperties;
import android.os.UserHandle;
import android.provider.DeviceConfig;
import android.provider.Settings;
import android.sysprop.CrashRecoveryProperties;
import android.text.TextUtils;
import android.util.ArraySet;
import android.util.Log;
import android.util.Slog;

import com.android.internal.annotations.GuardedBy;
import com.android.internal.annotations.VisibleForTesting;
import com.android.internal.util.ArrayUtils;
import com.android.server.PackageWatchdog.FailureReasons;
import com.android.server.PackageWatchdog.PackageHealthObserver;
import com.android.server.PackageWatchdog.PackageHealthObserverImpact;
import com.android.server.am.SettingsToPropertiesMapper;
import com.android.server.crashrecovery.proto.CrashRecoveryStatsLog;

import java.io.File;
import java.lang.annotation.Retention;
import java.lang.annotation.RetentionPolicy;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;

/**
 * Utilities to help rescue the system from crash loops. Callers are expected to
 * report boot events and persistent app crashes, and if they happen frequently
 * enough this class will slowly escalate through several rescue operations
 * before finally rebooting and prompting the user if they want to wipe data as
 * a last resort.
 *
 * @hide
 */
public class RescueParty {
    @VisibleForTesting
    static final String PROP_ENABLE_RESCUE = "persist.sys.enable_rescue";
    @VisibleForTesting
    static final int LEVEL_NONE = 0;
    @VisibleForTesting
    static final int LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS = 1;
    @VisibleForTesting
    static final int LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES = 2;
    @VisibleForTesting
    static final int LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS = 3;
    @VisibleForTesting
    static final int LEVEL_WARM_REBOOT = 4;
    @VisibleForTesting
    static final int LEVEL_FACTORY_RESET = 5;
    @VisibleForTesting
    static final int RESCUE_LEVEL_NONE = 0;
    @VisibleForTesting
    static final int RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET = 1;
    @VisibleForTesting
    static final int RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET = 2;
    @VisibleForTesting
    static final int RESCUE_LEVEL_WARM_REBOOT = 3;
    @VisibleForTesting
    static final int RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS = 4;
    @VisibleForTesting
    static final int RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES = 5;
    @VisibleForTesting
    static final int RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS = 6;
    @VisibleForTesting
    static final int RESCUE_LEVEL_FACTORY_RESET = 7;

    @IntDef(prefix = { "RESCUE_LEVEL_" }, value = {
        RESCUE_LEVEL_NONE,
        RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET,
        RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET,
        RESCUE_LEVEL_WARM_REBOOT,
        RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS,
        RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES,
        RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS,
        RESCUE_LEVEL_FACTORY_RESET
    })
    @Retention(RetentionPolicy.SOURCE)
    @interface RescueLevels {}

    @VisibleForTesting
    static final String RESCUE_NON_REBOOT_LEVEL_LIMIT = "persist.sys.rescue_non_reboot_level_limit";
    @VisibleForTesting
    static final int DEFAULT_RESCUE_NON_REBOOT_LEVEL_LIMIT = RESCUE_LEVEL_WARM_REBOOT - 1;
    @VisibleForTesting
    static final String TAG = "RescueParty";
    @VisibleForTesting
    static final long DEFAULT_OBSERVING_DURATION_MS = TimeUnit.DAYS.toMillis(2);
    @VisibleForTesting
    static final int DEVICE_CONFIG_RESET_MODE = Settings.RESET_MODE_TRUSTED_DEFAULTS;
    // The DeviceConfig namespace containing all RescueParty switches.
    @VisibleForTesting
    static final String NAMESPACE_CONFIGURATION = "configuration";
    @VisibleForTesting
    static final String NAMESPACE_TO_PACKAGE_MAPPING_FLAG =
            "namespace_to_package_mapping";
    @VisibleForTesting
    static final long DEFAULT_FACTORY_RESET_THROTTLE_DURATION_MIN = 1440;

    private static final String NAME = "rescue-party-observer";

    private static final String PROP_DISABLE_RESCUE = "persist.sys.disable_rescue";
    private static final String PROP_VIRTUAL_DEVICE = "ro.hardware.virtual_device";
    private static final String PROP_DEVICE_CONFIG_DISABLE_FLAG =
            "persist.device_config.configuration.disable_rescue_party";
    private static final String PROP_DISABLE_FACTORY_RESET_FLAG =
            "persist.device_config.configuration.disable_rescue_party_factory_reset";
    private static final String PROP_THROTTLE_DURATION_MIN_FLAG =
            "persist.device_config.configuration.rescue_party_throttle_duration_min";

    private static final int PERSISTENT_MASK = ApplicationInfo.FLAG_PERSISTENT
            | ApplicationInfo.FLAG_SYSTEM;

    /** Register the Rescue Party observer as a Package Watchdog health observer */
    public static void registerHealthObserver(Context context) {
        PackageWatchdog.getInstance(context).registerHealthObserver(
                RescuePartyObserver.getInstance(context));
    }

    private static boolean isDisabled() {
        // Check if we're explicitly enabled for testing
        if (SystemProperties.getBoolean(PROP_ENABLE_RESCUE, false)) {
            return false;
        }

        // We're disabled if the DeviceConfig disable flag is set to true.
        // This is in case that an emergency rollback of the feature is needed.
        if (SystemProperties.getBoolean(PROP_DEVICE_CONFIG_DISABLE_FLAG, false)) {
            Slog.v(TAG, "Disabled because of DeviceConfig flag");
            return true;
        }

        // We're disabled on all engineering devices
        if (Build.TYPE.equals("eng")) {
            Slog.v(TAG, "Disabled because of eng build");
            return true;
        }

        // We're disabled on userdebug devices connected over USB, since that's
        // a decent signal that someone is actively trying to debug the device,
        // or that it's in a lab environment.
        if (Build.TYPE.equals("userdebug") && isUsbActive()) {
            Slog.v(TAG, "Disabled because of active USB connection");
            return true;
        }

        // One last-ditch check
        if (SystemProperties.getBoolean(PROP_DISABLE_RESCUE, false)) {
            Slog.v(TAG, "Disabled because of manual property");
            return true;
        }

        return false;
    }

    /**
     * Check if we're currently attempting to reboot for a factory reset. This method must
     * return true if RescueParty tries to reboot early during a boot loop, since the device
     * will not be fully booted at this time.
     */
    public static boolean isRecoveryTriggeredReboot() {
        return isFactoryResetPropertySet() || isRebootPropertySet();
    }

    static boolean isFactoryResetPropertySet() {
        return CrashRecoveryProperties.attemptingFactoryReset().orElse(false);
    }

    static boolean isRebootPropertySet() {
        return CrashRecoveryProperties.attemptingReboot().orElse(false);
    }

    protected static long getLastFactoryResetTimeMs() {
        return CrashRecoveryProperties.lastFactoryResetTimeMs().orElse(0L);
    }

    protected static int getMaxRescueLevelAttempted() {
        return CrashRecoveryProperties.maxRescueLevelAttempted().orElse(LEVEL_NONE);
    }

    protected static void setFactoryResetProperty(boolean value) {
        CrashRecoveryProperties.attemptingFactoryReset(value);
    }
    protected static void setRebootProperty(boolean value) {
        CrashRecoveryProperties.attemptingReboot(value);
    }

    protected static void setLastFactoryResetTimeMs(long value) {
        CrashRecoveryProperties.lastFactoryResetTimeMs(value);
    }

    protected static void setMaxRescueLevelAttempted(int level) {
        CrashRecoveryProperties.maxRescueLevelAttempted(level);
    }

    /**
     * Called when {@code SettingsProvider} has been published, which is a good
     * opportunity to reset any settings depending on our rescue level.
     */
    public static void onSettingsProviderPublished(Context context) {
        handleNativeRescuePartyResets();
        ContentResolver contentResolver = context.getContentResolver();
        DeviceConfig.setMonitorCallback(
                contentResolver,
                Executors.newSingleThreadExecutor(),
                new RescuePartyMonitorCallback(context));
    }


    /**
     * Called when {@code RollbackManager} performs Mainline module rollbacks,
     * to avoid rolled back modules consuming flag values only expected to work
     * on modules of newer versions.
     */
    public static void resetDeviceConfigForPackages(List<String> packageNames) {
        if (packageNames == null) {
            return;
        }
        Set<String> namespacesToReset = new ArraySet<String>();
        Iterator<String> it = packageNames.iterator();
        RescuePartyObserver rescuePartyObserver = RescuePartyObserver.getInstanceIfCreated();
        // Get runtime package to namespace mapping if created.
        if (rescuePartyObserver != null) {
            while (it.hasNext()) {
                String packageName = it.next();
                Set<String> runtimeAffectedNamespaces =
                        rescuePartyObserver.getAffectedNamespaceSet(packageName);
                if (runtimeAffectedNamespaces != null) {
                    namespacesToReset.addAll(runtimeAffectedNamespaces);
                }
            }
        }
        // Get preset package to namespace mapping if created.
        Set<String> presetAffectedNamespaces = getPresetNamespacesForPackages(
                packageNames);
        if (presetAffectedNamespaces != null) {
            namespacesToReset.addAll(presetAffectedNamespaces);
        }

        // Clear flags under the namespaces mapped to these packages.
        // Using setProperties since DeviceConfig.resetToDefaults bans the current flag set.
        Iterator<String> namespaceIt = namespacesToReset.iterator();
        while (namespaceIt.hasNext()) {
            String namespaceToReset = namespaceIt.next();
            Properties properties = new Properties.Builder(namespaceToReset).build();
            try {
                if (!DeviceConfig.setProperties(properties)) {
                    logCriticalInfo(Log.ERROR, "Failed to clear properties under "
                            + namespaceToReset
                            + ". Running `device_config get_sync_disabled_for_tests` will confirm"
                            + " if config-bulk-update is enabled.");
                }
            } catch (DeviceConfig.BadConfigException exception) {
                logCriticalInfo(Log.WARN, "namespace " + namespaceToReset
                        + " is already banned, skip reset.");
            }
        }
    }

    private static Set<String> getPresetNamespacesForPackages(List<String> packageNames) {
        Set<String> resultSet = new ArraySet<String>();
        try {
            String flagVal = DeviceConfig.getString(NAMESPACE_CONFIGURATION,
                    NAMESPACE_TO_PACKAGE_MAPPING_FLAG, "");
            String[] mappingEntries = flagVal.split(",");
            for (int i = 0; i < mappingEntries.length; i++) {
                if (TextUtils.isEmpty(mappingEntries[i])) {
                    continue;
                }
                String[] splittedEntry = mappingEntries[i].split(":");
                if (splittedEntry.length != 2) {
                    throw new RuntimeException("Invalid mapping entry: " + mappingEntries[i]);
                }
                String namespace = splittedEntry[0];
                String packageName = splittedEntry[1];

                if (packageNames.contains(packageName)) {
                    resultSet.add(namespace);
                }
            }
        } catch (Exception e) {
            resultSet.clear();
            Slog.e(TAG, "Failed to read preset package to namespaces mapping.", e);
        } finally {
            return resultSet;
        }
    }

    @VisibleForTesting
    static long getElapsedRealtime() {
        return SystemClock.elapsedRealtime();
    }

    private static class RescuePartyMonitorCallback implements DeviceConfig.MonitorCallback {
        Context mContext;

        RescuePartyMonitorCallback(Context context) {
            this.mContext = context;
        }

        public void onNamespaceUpdate(@NonNull String updatedNamespace) {
            startObservingPackages(mContext, updatedNamespace);
        }

        public void onDeviceConfigAccess(@NonNull String callingPackage,
                @NonNull String namespace) {
            RescuePartyObserver.getInstance(mContext).recordDeviceConfigAccess(
                            callingPackage,
                            namespace);
        }
    }

    private static void startObservingPackages(Context context, @NonNull String updatedNamespace) {
        RescuePartyObserver rescuePartyObserver = RescuePartyObserver.getInstance(context);
        Set<String> callingPackages = rescuePartyObserver.getCallingPackagesSet(updatedNamespace);
        if (callingPackages == null) {
            return;
        }
        List<String> callingPackageList = new ArrayList<>();
        callingPackageList.addAll(callingPackages);
        Slog.i(TAG, "Starting to observe: " + callingPackageList + ", updated namespace: "
                + updatedNamespace);
        PackageWatchdog.getInstance(context).startObservingHealth(
                rescuePartyObserver,
                callingPackageList,
                DEFAULT_OBSERVING_DURATION_MS);
    }

    private static void handleNativeRescuePartyResets() {
        if (SettingsToPropertiesMapper.isNativeFlagsResetPerformed()) {
            String[] resetNativeCategories = SettingsToPropertiesMapper.getResetNativeCategories();
            for (int i = 0; i < resetNativeCategories.length; i++) {
                // Don't let RescueParty reset the namespace for RescueParty switches.
                if (NAMESPACE_CONFIGURATION.equals(resetNativeCategories[i])) {
                    continue;
                }
                DeviceConfig.resetToDefaults(DEVICE_CONFIG_RESET_MODE,
                        resetNativeCategories[i]);
            }
        }
    }

    private static int getMaxRescueLevel(boolean mayPerformReboot) {
        if (Flags.recoverabilityDetection()) {
            if (!mayPerformReboot
                    || SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)) {
                return SystemProperties.getInt(RESCUE_NON_REBOOT_LEVEL_LIMIT,
                        DEFAULT_RESCUE_NON_REBOOT_LEVEL_LIMIT);
            }
            return RESCUE_LEVEL_FACTORY_RESET;
        } else {
            if (!mayPerformReboot
                    || SystemProperties.getBoolean(PROP_DISABLE_FACTORY_RESET_FLAG, false)) {
                return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS;
            }
            return LEVEL_FACTORY_RESET;
        }
    }

    /**
     * Get the rescue level to perform if this is the n-th attempt at mitigating failure.
     *
     * @param mitigationCount: the mitigation attempt number (1 = first attempt etc.)
     * @param mayPerformReboot: whether or not a reboot and factory reset may be performed
     *                          for the given failure.
     * @return the rescue level for the n-th mitigation attempt.
     */
    private static int getRescueLevel(int mitigationCount, boolean mayPerformReboot) {
        if (mitigationCount == 1) {
            return LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS;
        } else if (mitigationCount == 2) {
            return LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES;
        } else if (mitigationCount == 3) {
            return LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS;
        } else if (mitigationCount == 4) {
            return Math.min(getMaxRescueLevel(mayPerformReboot), LEVEL_WARM_REBOOT);
        } else if (mitigationCount >= 5) {
            return Math.min(getMaxRescueLevel(mayPerformReboot), LEVEL_FACTORY_RESET);
        } else {
            Slog.w(TAG, "Expected positive mitigation count, was " + mitigationCount);
            return LEVEL_NONE;
        }
    }

    /**
     * Get the rescue level to perform if this is the n-th attempt at mitigating failure.
     * When failedPackage is null then 1st and 2nd mitigation counts are redundant (scoped and
     * all device config reset). Behaves as if one mitigation attempt was already done.
     *
     * @param mitigationCount the mitigation attempt number (1 = first attempt etc.).
     * @param mayPerformReboot whether or not a reboot and factory reset may be performed
     * for the given failure.
     * @param failedPackage in case of bootloop this is null.
     * @return the rescue level for the n-th mitigation attempt.
     */
    private static @RescueLevels int getRescueLevel(int mitigationCount, boolean mayPerformReboot,
            @Nullable VersionedPackage failedPackage) {
        // Skipping RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET since it's not defined without a failed
        // package.
        if (failedPackage == null && mitigationCount > 0) {
            mitigationCount += 1;
        }
        if (mitigationCount == 1) {
            return RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET;
        } else if (mitigationCount == 2) {
            return RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET;
        } else if (mitigationCount == 3) {
            return Math.min(getMaxRescueLevel(mayPerformReboot), RESCUE_LEVEL_WARM_REBOOT);
        } else if (mitigationCount == 4) {
            return Math.min(getMaxRescueLevel(mayPerformReboot),
                                RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS);
        } else if (mitigationCount == 5) {
            return Math.min(getMaxRescueLevel(mayPerformReboot),
                                RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES);
        } else if (mitigationCount == 6) {
            return Math.min(getMaxRescueLevel(mayPerformReboot),
                                RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS);
        } else if (mitigationCount >= 7) {
            return Math.min(getMaxRescueLevel(mayPerformReboot), RESCUE_LEVEL_FACTORY_RESET);
        } else {
            return RESCUE_LEVEL_NONE;
        }
    }

    private static void executeRescueLevel(Context context, @Nullable String failedPackage,
            int level) {
        Slog.w(TAG, "Attempting rescue level " + levelToString(level));
        try {
            executeRescueLevelInternal(context, level, failedPackage);
            EventLogTags.writeRescueSuccess(level);
            String successMsg = "Finished rescue level " + levelToString(level);
            if (!TextUtils.isEmpty(failedPackage)) {
                successMsg += " for package " + failedPackage;
            }
            logCriticalInfo(Log.DEBUG, successMsg);
        } catch (Throwable t) {
            logRescueException(level, failedPackage, t);
        }
    }

    private static void executeRescueLevelInternal(Context context, int level, @Nullable
            String failedPackage) throws Exception {
        if (Flags.recoverabilityDetection()) {
            executeRescueLevelInternalNew(context, level, failedPackage);
        } else {
            executeRescueLevelInternalOld(context, level, failedPackage);
        }
    }

    private static void executeRescueLevelInternalOld(Context context, int level, @Nullable
            String failedPackage) throws Exception {

        // Note: DeviceConfig reset is disabled currently and would be enabled using the flag,
        // after we have figured out a way to reset flags without interfering with trunk
        // development. TODO: b/287618292 For enabling flag resets.
        if (!Flags.allowRescuePartyFlagResets() && level <= LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS) {
            return;
        }

        CrashRecoveryStatsLog.write(CrashRecoveryStatsLog.RESCUE_PARTY_RESET_REPORTED,
                level, levelToString(level));
        // Try our best to reset all settings possible, and once finished
        // rethrow any exception that we encountered
        Exception res = null;
        switch (level) {
            case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
                try {
                    resetAllSettingsIfNecessary(context, Settings.RESET_MODE_UNTRUSTED_DEFAULTS,
                            level);
                } catch (Exception e) {
                    res = e;
                }
                try {
                    resetDeviceConfig(context, /*isScoped=*/true, failedPackage);
                } catch (Exception e) {
                    res = e;
                }
                break;
            case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
                try {
                    resetAllSettingsIfNecessary(context, Settings.RESET_MODE_UNTRUSTED_CHANGES,
                            level);
                } catch (Exception e) {
                    res = e;
                }
                try {
                    resetDeviceConfig(context, /*isScoped=*/true, failedPackage);
                } catch (Exception e) {
                    res = e;
                }
                break;
            case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
                try {
                    resetAllSettingsIfNecessary(context, Settings.RESET_MODE_TRUSTED_DEFAULTS,
                            level);
                } catch (Exception e) {
                    res = e;
                }
                try {
                    resetDeviceConfig(context, /*isScoped=*/false, failedPackage);
                } catch (Exception e) {
                    res = e;
                }
                break;
            case LEVEL_WARM_REBOOT:
                executeWarmReboot(context, level, failedPackage);
                break;
            case LEVEL_FACTORY_RESET:
                // Before the completion of Reboot, if any crash happens then PackageWatchdog
                // escalates to next level i.e. factory reset, as they happen in separate threads.
                // Adding a check to prevent factory reset to execute before above reboot completes.
                // Note: this reboot property is not persistent resets after reboot is completed.
                if (isRebootPropertySet()) {
                    return;
                }
                executeFactoryReset(context, level, failedPackage);
                break;
        }

        if (res != null) {
            throw res;
        }
    }

    private static void executeRescueLevelInternalNew(Context context, @RescueLevels int level,
            @Nullable String failedPackage) throws Exception {
        CrashRecoveryStatsLog.write(CrashRecoveryStatsLog.RESCUE_PARTY_RESET_REPORTED,
                level, levelToString(level));
        switch (level) {
            case RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET:
                // Enable deviceConfig reset behind flag
                if (Flags.allowRescuePartyFlagResets()) {
                    resetDeviceConfig(context, /*isScoped=*/true, failedPackage);
                }
                break;
            case RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET:
                // Enable deviceConfig reset behind flag
                if (Flags.allowRescuePartyFlagResets()) {
                    resetDeviceConfig(context, /*isScoped=*/false, failedPackage);
                }
                break;
            case RESCUE_LEVEL_WARM_REBOOT:
                executeWarmReboot(context, level, failedPackage);
                break;
            case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
                resetAllSettingsIfNecessary(context, Settings.RESET_MODE_UNTRUSTED_DEFAULTS, level);
                break;
            case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
                resetAllSettingsIfNecessary(context, Settings.RESET_MODE_UNTRUSTED_CHANGES, level);
                break;
            case RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
                resetAllSettingsIfNecessary(context, Settings.RESET_MODE_TRUSTED_DEFAULTS, level);
                break;
            case RESCUE_LEVEL_FACTORY_RESET:
                // Before the completion of Reboot, if any crash happens then PackageWatchdog
                // escalates to next level i.e. factory reset, as they happen in separate threads.
                // Adding a check to prevent factory reset to execute before above reboot completes.
                // Note: this reboot property is not persistent resets after reboot is completed.
                if (isRebootPropertySet()) {
                    return;
                }
                executeFactoryReset(context, level, failedPackage);
                break;
        }
    }

    private static void executeWarmReboot(Context context, int level,
            @Nullable String failedPackage) {
        // Request the reboot from a separate thread to avoid deadlock on PackageWatchdog
        // when device shutting down.
        setRebootProperty(true);
        Runnable runnable = () -> {
            try {
                PowerManager pm = context.getSystemService(PowerManager.class);
                if (pm != null) {
                    pm.reboot(TAG);
                }
            } catch (Throwable t) {
                logRescueException(level, failedPackage, t);
            }
        };
        Thread thread = new Thread(runnable);
        thread.start();
    }

    private static void executeFactoryReset(Context context, int level,
            @Nullable String failedPackage) {
        setFactoryResetProperty(true);
        long now = System.currentTimeMillis();
        setLastFactoryResetTimeMs(now);
        Runnable runnable = new Runnable() {
            @Override
            public void run() {
                try {
                    RecoverySystem.rebootPromptAndWipeUserData(context, TAG);
                } catch (Throwable t) {
                    logRescueException(level, failedPackage, t);
                }
            }
        };
        Thread thread = new Thread(runnable);
        thread.start();
    }


    private static String getCompleteMessage(Throwable t) {
        final StringBuilder builder = new StringBuilder();
        builder.append(t.getMessage());
        while ((t = t.getCause()) != null) {
            builder.append(": ").append(t.getMessage());
        }
        return builder.toString();
    }

    private static void logRescueException(int level, @Nullable String failedPackageName,
            Throwable t) {
        final String msg = getCompleteMessage(t);
        EventLogTags.writeRescueFailure(level, msg);
        String failureMsg = "Failed rescue level " + levelToString(level);
        if (!TextUtils.isEmpty(failedPackageName)) {
            failureMsg += " for package " + failedPackageName;
        }
        logCriticalInfo(Log.ERROR, failureMsg + ": " + msg);
    }

    private static int mapRescueLevelToUserImpact(int rescueLevel) {
        if (Flags.recoverabilityDetection()) {
            switch (rescueLevel) {
                case RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET:
                    return PackageHealthObserverImpact.USER_IMPACT_LEVEL_10;
                case RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET:
                    return PackageHealthObserverImpact.USER_IMPACT_LEVEL_20;
                case RESCUE_LEVEL_WARM_REBOOT:
                    return PackageHealthObserverImpact.USER_IMPACT_LEVEL_50;
                case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
                    return PackageHealthObserverImpact.USER_IMPACT_LEVEL_71;
                case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
                    return PackageHealthObserverImpact.USER_IMPACT_LEVEL_75;
                case RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
                    return PackageHealthObserverImpact.USER_IMPACT_LEVEL_80;
                case RESCUE_LEVEL_FACTORY_RESET:
                    return PackageHealthObserverImpact.USER_IMPACT_LEVEL_100;
                default:
                    return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
            }
        } else {
            switch (rescueLevel) {
                case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
                case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
                    return PackageHealthObserverImpact.USER_IMPACT_LEVEL_10;
                case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
                case LEVEL_WARM_REBOOT:
                    return PackageHealthObserverImpact.USER_IMPACT_LEVEL_50;
                case LEVEL_FACTORY_RESET:
                    return PackageHealthObserverImpact.USER_IMPACT_LEVEL_100;
                default:
                    return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
            }
        }
    }

    private static void resetAllSettingsIfNecessary(Context context, int mode,
            int level) throws Exception {
        // No need to reset Settings again if they are already reset in the current level once.
        if (getMaxRescueLevelAttempted() >= level) {
            return;
        }
        setMaxRescueLevelAttempted(level);
        // Try our best to reset all settings possible, and once finished
        // rethrow any exception that we encountered
        Exception res = null;
        final ContentResolver resolver = context.getContentResolver();
        try {
            Settings.Global.resetToDefaultsAsUser(resolver, null, mode,
                    UserHandle.SYSTEM.getIdentifier());
        } catch (Exception e) {
            res = new RuntimeException("Failed to reset global settings", e);
        }
        for (int userId : getAllUserIds()) {
            try {
                Settings.Secure.resetToDefaultsAsUser(resolver, null, mode, userId);
            } catch (Exception e) {
                res = new RuntimeException("Failed to reset secure settings for " + userId, e);
            }
        }
        if (res != null) {
            throw res;
        }
    }

    private static void resetDeviceConfig(Context context, boolean isScoped,
            @Nullable String failedPackage) throws Exception {
        final ContentResolver resolver = context.getContentResolver();
        try {
            if (!isScoped || failedPackage == null) {
                resetAllAffectedNamespaces(context);
            } else {
                performScopedReset(context, failedPackage);
            }
        } catch (Exception e) {
            throw new RuntimeException("Failed to reset config settings", e);
        }
    }

    private static void resetAllAffectedNamespaces(Context context) {
        RescuePartyObserver rescuePartyObserver = RescuePartyObserver.getInstance(context);
        Set<String> allAffectedNamespaces = rescuePartyObserver.getAllAffectedNamespaceSet();

        Slog.w(TAG,
                "Performing reset for all affected namespaces: "
                        + Arrays.toString(allAffectedNamespaces.toArray()));
        Iterator<String> it = allAffectedNamespaces.iterator();
        while (it.hasNext()) {
            String namespace = it.next();
            // Don't let RescueParty reset the namespace for RescueParty switches.
            if (NAMESPACE_CONFIGURATION.equals(namespace)) {
                continue;
            }
            DeviceConfig.resetToDefaults(DEVICE_CONFIG_RESET_MODE, namespace);
        }
    }

    private static void performScopedReset(Context context, @NonNull String failedPackage) {
        RescuePartyObserver rescuePartyObserver = RescuePartyObserver.getInstance(context);
        Set<String> affectedNamespaces = rescuePartyObserver.getAffectedNamespaceSet(
                failedPackage);
        // If we can't find namespaces affected for current package,
        // skip this round of reset.
        if (affectedNamespaces != null) {
            Slog.w(TAG,
                    "Performing scoped reset for package: " + failedPackage
                            + ", affected namespaces: "
                            + Arrays.toString(affectedNamespaces.toArray()));
            Iterator<String> it = affectedNamespaces.iterator();
            while (it.hasNext()) {
                String namespace = it.next();
                // Don't let RescueParty reset the namespace for RescueParty switches.
                if (NAMESPACE_CONFIGURATION.equals(namespace)) {
                    continue;
                }
                DeviceConfig.resetToDefaults(DEVICE_CONFIG_RESET_MODE, namespace);
            }
        }
    }

    /**
     * Handle mitigation action for package failures. This observer will be register to Package
     * Watchdog and will receive calls about package failures. This observer is persistent so it
     * may choose to mitigate failures for packages it has not explicitly asked to observe.
     */
    public static class RescuePartyObserver implements PackageHealthObserver {

        private final Context mContext;
        private final Map<String, Set<String>> mCallingPackageNamespaceSetMap = new HashMap<>();
        private final Map<String, Set<String>> mNamespaceCallingPackageSetMap = new HashMap<>();

        @GuardedBy("RescuePartyObserver.class")
        static RescuePartyObserver sRescuePartyObserver;

        private RescuePartyObserver(Context context) {
            mContext = context;
        }

        /** Creates or gets singleton instance of RescueParty. */
        public static RescuePartyObserver getInstance(Context context) {
            synchronized (RescuePartyObserver.class) {
                if (sRescuePartyObserver == null) {
                    sRescuePartyObserver = new RescuePartyObserver(context);
                }
                return sRescuePartyObserver;
            }
        }

        /** Gets singleton instance. It returns null if the instance is not created yet.*/
        @Nullable
        public static RescuePartyObserver getInstanceIfCreated() {
            synchronized (RescuePartyObserver.class) {
                return sRescuePartyObserver;
            }
        }

        @VisibleForTesting
        static void reset() {
            synchronized (RescuePartyObserver.class) {
                sRescuePartyObserver = null;
            }
        }

        @Override
        public int onHealthCheckFailed(@Nullable VersionedPackage failedPackage,
                @FailureReasons int failureReason, int mitigationCount) {
            if (!isDisabled() && (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH
                    || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING)) {
                if (Flags.recoverabilityDetection()) {
                    return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount,
                            mayPerformReboot(failedPackage), failedPackage));
                } else {
                    return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount,
                        mayPerformReboot(failedPackage)));
                }
            } else {
                return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
            }
        }

        @Override
        public boolean execute(@Nullable VersionedPackage failedPackage,
                @FailureReasons int failureReason, int mitigationCount) {
            if (isDisabled()) {
                return false;
            }
            if (failureReason == PackageWatchdog.FAILURE_REASON_APP_CRASH
                    || failureReason == PackageWatchdog.FAILURE_REASON_APP_NOT_RESPONDING) {
                final int level = Flags.recoverabilityDetection() ? getRescueLevel(mitigationCount,
                        mayPerformReboot(failedPackage), failedPackage)
                        : getRescueLevel(mitigationCount,
                                mayPerformReboot(failedPackage));
                executeRescueLevel(mContext,
                        failedPackage == null ? null : failedPackage.getPackageName(), level);
                return true;
            } else {
                return false;
            }
        }

        @Override
        public boolean isPersistent() {
            return true;
        }

        @Override
        public boolean mayObservePackage(String packageName) {
            PackageManager pm = mContext.getPackageManager();
            try {
                // A package is a module if this is non-null
                if (pm.getModuleInfo(packageName, 0) != null) {
                    return true;
                }
            } catch (PackageManager.NameNotFoundException | IllegalStateException ignore) {
            }

            return isPersistentSystemApp(packageName);
        }

        @Override
        public int onBootLoop(int mitigationCount) {
            if (isDisabled()) {
                return PackageHealthObserverImpact.USER_IMPACT_LEVEL_0;
            }
            if (Flags.recoverabilityDetection()) {
                return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount,
                        true, /*failedPackage=*/ null));
            } else {
                return mapRescueLevelToUserImpact(getRescueLevel(mitigationCount, true));
            }
        }

        @Override
        public boolean executeBootLoopMitigation(int mitigationCount) {
            if (isDisabled()) {
                return false;
            }
            boolean mayPerformReboot = !shouldThrottleReboot();
            final int level = Flags.recoverabilityDetection() ? getRescueLevel(mitigationCount,
                        mayPerformReboot, /*failedPackage=*/ null)
                        : getRescueLevel(mitigationCount, mayPerformReboot);
            executeRescueLevel(mContext, /*failedPackage=*/ null, level);
            return true;
        }

        @Override
        public String getName() {
            return NAME;
        }

        /**
         * Returns {@code true} if the failing package is non-null and performing a reboot or
         * prompting a factory reset is an acceptable mitigation strategy for the package's
         * failure, {@code false} otherwise.
         */
        private boolean mayPerformReboot(@Nullable VersionedPackage failingPackage) {
            if (failingPackage == null) {
                return false;
            }
            if (shouldThrottleReboot())  {
                return false;
            }

            return isPersistentSystemApp(failingPackage.getPackageName());
        }

        /**
         * Returns {@code true} if Rescue Party is allowed to attempt a reboot or factory reset.
         * Will return {@code false} if a factory reset was already offered recently.
         */
        private boolean shouldThrottleReboot() {
            Long lastResetTime = getLastFactoryResetTimeMs();
            long now = System.currentTimeMillis();
            long throttleDurationMin = SystemProperties.getLong(PROP_THROTTLE_DURATION_MIN_FLAG,
                    DEFAULT_FACTORY_RESET_THROTTLE_DURATION_MIN);
            return now < lastResetTime + TimeUnit.MINUTES.toMillis(throttleDurationMin);
        }

        private boolean isPersistentSystemApp(@NonNull String packageName) {
            PackageManager pm = mContext.getPackageManager();
            try {
                ApplicationInfo info = pm.getApplicationInfo(packageName, 0);
                return (info.flags & PERSISTENT_MASK) == PERSISTENT_MASK;
            } catch (PackageManager.NameNotFoundException e) {
                return false;
            }
        }

        private synchronized void recordDeviceConfigAccess(@NonNull String callingPackage,
                @NonNull String namespace) {
            // Record it in calling packages to namespace map
            Set<String> namespaceSet = mCallingPackageNamespaceSetMap.get(callingPackage);
            if (namespaceSet == null) {
                namespaceSet = new ArraySet<>();
                mCallingPackageNamespaceSetMap.put(callingPackage, namespaceSet);
            }
            namespaceSet.add(namespace);
            // Record it in namespace to calling packages map
            Set<String> callingPackageSet = mNamespaceCallingPackageSetMap.get(namespace);
            if (callingPackageSet == null) {
                callingPackageSet = new ArraySet<>();
            }
            callingPackageSet.add(callingPackage);
            mNamespaceCallingPackageSetMap.put(namespace, callingPackageSet);
        }

        private synchronized Set<String> getAffectedNamespaceSet(String failedPackage) {
            return mCallingPackageNamespaceSetMap.get(failedPackage);
        }

        private synchronized Set<String> getAllAffectedNamespaceSet() {
            return new HashSet<String>(mNamespaceCallingPackageSetMap.keySet());
        }

        private synchronized Set<String> getCallingPackagesSet(String namespace) {
            return mNamespaceCallingPackageSetMap.get(namespace);
        }
    }

    private static int[] getAllUserIds() {
        int systemUserId = UserHandle.SYSTEM.getIdentifier();
        int[] userIds = { systemUserId };
        try {
            for (File file : FileUtils.listFilesOrEmpty(Environment.getDataSystemDeDirectory())) {
                try {
                    final int userId = Integer.parseInt(file.getName());
                    if (userId != systemUserId) {
                        userIds = ArrayUtils.appendInt(userIds, userId);
                    }
                } catch (NumberFormatException ignored) {
                }
            }
        } catch (Throwable t) {
            Slog.w(TAG, "Trouble discovering users", t);
        }
        return userIds;
    }

    /**
     * Hacky test to check if the device has an active USB connection, which is
     * a good proxy for someone doing local development work.
     */
    private static boolean isUsbActive() {
        if (SystemProperties.getBoolean(PROP_VIRTUAL_DEVICE, false)) {
            Slog.v(TAG, "Assuming virtual device is connected over USB");
            return true;
        }
        try {
            final String state = FileUtils
                    .readTextFile(new File("/sys/class/android_usb/android0/state"), 128, "");
            return "CONFIGURED".equals(state.trim());
        } catch (Throwable t) {
            Slog.w(TAG, "Failed to determine if device was on USB", t);
            return false;
        }
    }

    private static String levelToString(int level) {
        if (Flags.recoverabilityDetection()) {
            switch (level) {
                case RESCUE_LEVEL_NONE:
                    return "NONE";
                case RESCUE_LEVEL_SCOPED_DEVICE_CONFIG_RESET:
                    return "SCOPED_DEVICE_CONFIG_RESET";
                case RESCUE_LEVEL_ALL_DEVICE_CONFIG_RESET:
                    return "ALL_DEVICE_CONFIG_RESET";
                case RESCUE_LEVEL_WARM_REBOOT:
                    return "WARM_REBOOT";
                case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
                    return "RESET_SETTINGS_UNTRUSTED_DEFAULTS";
                case RESCUE_LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
                    return "RESET_SETTINGS_UNTRUSTED_CHANGES";
                case RESCUE_LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
                    return "RESET_SETTINGS_TRUSTED_DEFAULTS";
                case RESCUE_LEVEL_FACTORY_RESET:
                    return "FACTORY_RESET";
                default:
                    return Integer.toString(level);
            }
        } else {
            switch (level) {
                case LEVEL_NONE:
                    return "NONE";
                case LEVEL_RESET_SETTINGS_UNTRUSTED_DEFAULTS:
                    return "RESET_SETTINGS_UNTRUSTED_DEFAULTS";
                case LEVEL_RESET_SETTINGS_UNTRUSTED_CHANGES:
                    return "RESET_SETTINGS_UNTRUSTED_CHANGES";
                case LEVEL_RESET_SETTINGS_TRUSTED_DEFAULTS:
                    return "RESET_SETTINGS_TRUSTED_DEFAULTS";
                case LEVEL_WARM_REBOOT:
                    return "WARM_REBOOT";
                case LEVEL_FACTORY_RESET:
                    return "FACTORY_RESET";
                default:
                    return Integer.toString(level);
            }
        }
    }
}