Merge changes from topic 'secondary-dex'

* changes:
  Fix the ownership of compiler artifacts for secondary dex files
  DEXOPT_FORCE may be passed for primary apk as well.
  [Installd] Clean up logic for secondary dex oat files
  Add installd logic for compiling secondary dex files

8 files changed, 487 insertions, 104 deletions

diff --git a/cmds/installd/InstalldNativeService.cpp b/cmds/installd/InstalldNativeService.cpp
index cdef7e1f07..f2ea0d30ab 100644
--- a/cmds/installd/InstalldNativeService.cpp
+++ b/cmds/installd/InstalldNativeService.cpp
@@ -79,10 +79,6 @@ static constexpr const char* CODE_CACHE_DIR_POSTFIX = "/code_cache";
 static constexpr const char* IDMAP_PREFIX = "/data/resource-cache/";
 static constexpr const char* IDMAP_SUFFIX = "@idmap";
 
-// NOTE: keep in sync with StorageManager
-static constexpr int FLAG_STORAGE_DE = 1 << 0;
-static constexpr int FLAG_STORAGE_CE = 1 << 1;
-
 // NOTE: keep in sync with Installer
 static constexpr int FLAG_CLEAR_CACHE_ONLY = 1 << 8;
 static constexpr int FLAG_CLEAR_CODE_CACHE_ONLY = 1 << 9;
@@ -285,73 +281,6 @@ static int prepare_app_dir(const std::string& path, mode_t target_mode, uid_t ui
     return 0;
 }
 
-/**
- * Prepare an app cache directory, which offers to fix-up the GID and
- * directory mode flags during a platform upgrade.
- */
-static int prepare_app_cache_dir(const std::string& parent, const char* name, mode_t target_mode,
-        uid_t uid, gid_t gid) {
-    auto path = StringPrintf("%s/%s", parent.c_str(), name);
-    struct stat st;
-    if (stat(path.c_str(), &st) != 0) {
-        if (errno == ENOENT) {
-            // This is fine, just create it
-            if (fs_prepare_dir_strict(path.c_str(), target_mode, uid, gid) != 0) {
-                PLOG(ERROR) << "Failed to prepare " << path;
-                return -1;
-            } else {
-                return 0;
-            }
-        } else {
-            PLOG(ERROR) << "Failed to stat " << path;
-            return -1;
-        }
-    }
-
-    mode_t actual_mode = st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISGID);
-    if (st.st_uid != uid) {
-        // Mismatched UID is real trouble; we can't recover
-        LOG(ERROR) << "Mismatched UID at " << path << ": found " << st.st_uid
-                << " but expected " << uid;
-        return -1;
-    } else if (st.st_gid == gid && actual_mode == target_mode) {
-        // Everything looks good!
-        return 0;
-    }
-
-    // Directory is owned correctly, but GID or mode mismatch means it's
-    // probably a platform upgrade so we need to fix them
-    FTS *fts;
-    FTSENT *p;
-    char *argv[] = { (char*) path.c_str(), nullptr };
-    if (!(fts = fts_open(argv, FTS_PHYSICAL | FTS_XDEV, NULL))) {
-        PLOG(ERROR) << "Failed to fts_open " << path;
-        return -1;
-    }
-    while ((p = fts_read(fts)) != NULL) {
-        switch (p->fts_info) {
-        case FTS_DP:
-            if (chmod(p->fts_accpath, target_mode) != 0) {
-                PLOG(WARNING) << "Failed to chmod " << p->fts_path;
-            }
-            // Intentional fall through to also set GID
-        case FTS_F:
-            if (chown(p->fts_accpath, -1, gid) != 0) {
-                PLOG(WARNING) << "Failed to chown " << p->fts_path;
-            }
-            break;
-        case FTS_SL:
-        case FTS_SLNONE:
-            if (lchown(p->fts_accpath, -1, gid) != 0) {
-                PLOG(WARNING) << "Failed to chown " << p->fts_path;
-            }
-            break;
-        }
-    }
-    fts_close(fts);
-    return 0;
-}
-
 binder::Status InstalldNativeService::createAppData(const std::unique_ptr<std::string>& uuid,
         const std::string& packageName, int32_t userId, int32_t flags, int32_t appId,
         const std::string& seInfo, int32_t targetSdkVersion, int64_t* _aidl_return) {
@@ -2013,6 +1942,20 @@ binder::Status InstalldNativeService::deleteOdex(const std::string& apkPath,
     return res ? ok() : error();
 }
 
+binder::Status InstalldNativeService::reconcileSecondaryDexFile(
+        const std::string& dexPath, const std::string& packageName, int32_t uid,
+        const std::vector<std::string>& isas, const std::unique_ptr<std::string>& volumeUuid,
+        int32_t storage_flag, bool* _aidl_return) {
+    ENFORCE_UID(AID_SYSTEM);
+    CHECK_ARGUMENT_UUID(volumeUuid);
+    CHECK_ARGUMENT_PACKAGE_NAME(packageName);
+
+    std::lock_guard<std::recursive_mutex> lock(mLock);
+    bool result = android::installd::reconcile_secondary_dex_file(
+            dexPath, packageName, uid, isas, volumeUuid, storage_flag, _aidl_return);
+    return result ? ok() : error();
+}
+
 binder::Status InstalldNativeService::invalidateMounts() {
     ENFORCE_UID(AID_SYSTEM);
     std::lock_guard<std::recursive_mutex> lock(mLock);
diff --git a/cmds/installd/InstalldNativeService.h b/cmds/installd/InstalldNativeService.h
index 0a9f12f311..37e0090c01 100644
--- a/cmds/installd/InstalldNativeService.h
+++ b/cmds/installd/InstalldNativeService.h
@@ -105,6 +105,9 @@ public:
             const std::string& outputPath);
     binder::Status deleteOdex(const std::string& apkPath, const std::string& instructionSet,
             const std::string& outputPath);
+    binder::Status reconcileSecondaryDexFile(const std::string& dexPath,
+        const std::string& packageName, int32_t uid, const std::vector<std::string>& isa,
+        const std::unique_ptr<std::string>& volumeUuid, int32_t storage_flag, bool* _aidl_return);
 
     binder::Status invalidateMounts();
 
diff --git a/cmds/installd/binder/android/os/IInstalld.aidl b/cmds/installd/binder/android/os/IInstalld.aidl
index aa5e4f23fe..4dbfa91207 100644
--- a/cmds/installd/binder/android/os/IInstalld.aidl
+++ b/cmds/installd/binder/android/os/IInstalld.aidl
@@ -71,5 +71,9 @@ interface IInstalld {
     void deleteOdex(@utf8InCpp String apkPath, @utf8InCpp String instructionSet,
             @utf8InCpp String outputPath);
 
+    boolean reconcileSecondaryDexFile(@utf8InCpp String dexPath, @utf8InCpp String pkgName,
+        int uid, in @utf8InCpp String[] isas, @nullable @utf8InCpp String volume_uuid,
+        int storage_flag);
+
     void invalidateMounts();
 }
diff --git a/cmds/installd/dexopt.cpp b/cmds/installd/dexopt.cpp
index 5d84157436..0fb207bb44 100644
--- a/cmds/installd/dexopt.cpp
+++ b/cmds/installd/dexopt.cpp
@@ -31,6 +31,7 @@
 #include <android-base/stringprintf.h>
 #include <android-base/strings.h>
 #include <android-base/unique_fd.h>
+#include <cutils/fs.h>
 #include <cutils/properties.h>
 #include <cutils/sched_policy.h>
 #include <log/log.h>               // TODO: Move everything to base/logging.
@@ -853,13 +854,16 @@ static int open_output_file(const char* file_name, bool recreate, int permission
     return open(file_name, flags, permissions);
 }
 
-static bool set_permissions_and_ownership(int fd, bool is_public, int uid, const char* path) {
+static bool set_permissions_and_ownership(
+        int fd, bool is_public, int uid, const char* path, bool is_secondary_dex) {
+    // Primary apks are owned by the system. Secondary dex files are owned by the app.
+    int owning_uid = is_secondary_dex ? uid : AID_SYSTEM;
     if (fchmod(fd,
                S_IRUSR|S_IWUSR|S_IRGRP |
                (is_public ? S_IROTH : 0)) < 0) {
         ALOGE("installd cannot chmod '%s' during dexopt\n", path);
         return false;
-    } else if (fchown(fd, AID_SYSTEM, uid) < 0) {
+    } else if (fchown(fd, owning_uid, uid) < 0) {
         ALOGE("installd cannot chown '%s' during dexopt\n", path);
         return false;
     }
@@ -875,7 +879,7 @@ static bool IsOutputDalvikCache(const char* oat_dir) {
 }
 
 static bool create_oat_out_path(const char* apk_path, const char* instruction_set,
-            const char* oat_dir, /*out*/ char* out_oat_path) {
+            const char* oat_dir, bool is_secondary_dex, /*out*/ char* out_oat_path) {
     // Early best-effort check whether we can fit the the path into our buffers.
     // Note: the cache path will require an additional 5 bytes for ".swap", but we'll try to run
     // without a swap file, if necessary. Reference profiles file also add an extra ".prof"
@@ -886,7 +890,8 @@ static bool create_oat_out_path(const char* apk_path, const char* instruction_se
     }
 
     if (!IsOutputDalvikCache(oat_dir)) {
-        if (validate_apk_path(oat_dir)) {
+        // Oat dirs for secondary dex files are already validated.
+        if (!is_secondary_dex && validate_apk_path(oat_dir)) {
             ALOGE("cannot validate apk path with oat_dir '%s'\n", oat_dir);
             return false;
         }
@@ -1007,10 +1012,11 @@ class Dex2oatFileWrapper {
 
 // (re)Creates the app image if needed.
 Dex2oatFileWrapper maybe_open_app_image(const char* out_oat_path, bool profile_guided,
-        bool is_public, int uid) {
+        bool is_public, int uid, bool is_secondary_dex) {
     // Use app images only if it is enabled (by a set image format) and we are compiling
     // profile-guided (so the app image doesn't conservatively contain all classes).
-    if (!profile_guided) {
+    // Note that we don't create an image for secondary dex files.
+    if (is_secondary_dex || !profile_guided) {
         return Dex2oatFileWrapper();
     }
 
@@ -1041,7 +1047,7 @@ Dex2oatFileWrapper maybe_open_app_image(const char* out_oat_path, bool profile_g
             }
         }
     } else if (!set_permissions_and_ownership(
-                wrapper_fd.get(), is_public, uid, image_path.c_str())) {
+                wrapper_fd.get(), is_public, uid, image_path.c_str(), is_secondary_dex)) {
         ALOGE("installd cannot set owner '%s' for image during dexopt\n", image_path.c_str());
         wrapper_fd.reset(-1);
     }
@@ -1079,10 +1085,11 @@ base::unique_fd maybe_open_dexopt_swap_file(const char* out_oat_path) {
 // Opens the reference profiles if needed.
 // Note that the reference profile might not exist so it's OK if the fd will be -1.
 Dex2oatFileWrapper maybe_open_reference_profile(const char* pkgname, bool profile_guided,
-        bool is_public, int uid) {
+        bool is_public, int uid, bool is_secondary_dex) {
     // Public apps should not be compiled with profile information ever. Same goes for the special
     // package '*' used for the system server.
-    if (profile_guided && !is_public && (pkgname[0] != '*')) {
+    // TODO(calin): add support for writing profiles for secondary dex files
+    if (profile_guided && !is_secondary_dex && !is_public && (pkgname[0] != '*')) {
         // Open reference profile in read only mode as dex2oat does not get write permissions.
         const std::string pkgname_str(pkgname);
         return Dex2oatFileWrapper(
@@ -1098,7 +1105,7 @@ Dex2oatFileWrapper maybe_open_reference_profile(const char* pkgname, bool profil
 // Opens the vdex files and assigns the input fd to in_vdex_wrapper_fd and the output fd to
 // out_vdex_wrapper_fd. Returns true for success or false in case of errors.
 bool open_vdex_files(const char* apk_path, const char* out_oat_path, int dexopt_needed,
-        const char* instruction_set, bool is_public, int uid,
+        const char* instruction_set, bool is_public, int uid, bool is_secondary_dex,
         Dex2oatFileWrapper* in_vdex_wrapper_fd,
         Dex2oatFileWrapper* out_vdex_wrapper_fd) {
     CHECK(in_vdex_wrapper_fd != nullptr);
@@ -1161,7 +1168,7 @@ bool open_vdex_files(const char* apk_path, const char* out_oat_path, int dexopt_
         }
     }
     if (!set_permissions_and_ownership(out_vdex_wrapper_fd->get(), is_public, uid,
-            out_vdex_path_str.c_str())) {
+            out_vdex_path_str.c_str(), is_secondary_dex)) {
         ALOGE("installd cannot set owner '%s' for vdex during dexopt\n", out_vdex_path_str.c_str());
         return false;
     }
@@ -1173,8 +1180,9 @@ bool open_vdex_files(const char* apk_path, const char* out_oat_path, int dexopt_
 // Opens the output oat file for the given apk.
 // If successful it stores the output path into out_oat_path and returns true.
 Dex2oatFileWrapper open_oat_out_file(const char* apk_path, const char* oat_dir,
-        bool is_public, int uid, const char* instruction_set, char* out_oat_path) {
-    if (!create_oat_out_path(apk_path, instruction_set, oat_dir, out_oat_path)) {
+        bool is_public, int uid, const char* instruction_set, bool is_secondary_dex,
+        char* out_oat_path) {
+    if (!create_oat_out_path(apk_path, instruction_set, oat_dir, is_secondary_dex, out_oat_path)) {
         return Dex2oatFileWrapper();
     }
     const std::string out_oat_path_str(out_oat_path);
@@ -1182,8 +1190,9 @@ Dex2oatFileWrapper open_oat_out_file(const char* apk_path, const char* oat_dir,
             open_output_file(out_oat_path, /*recreate*/true, /*permissions*/0644),
             [out_oat_path_str]() { unlink(out_oat_path_str.c_str()); });
     if (wrapper_fd.get() < 0) {
-        ALOGE("installd cannot open '%s' for output during dexopt\n", out_oat_path);
-    } else if (!set_permissions_and_ownership(wrapper_fd.get(), is_public, uid, out_oat_path)) {
+        PLOG(ERROR) << "installd cannot open output during dexopt" <<  out_oat_path;
+    } else if (!set_permissions_and_ownership(
+                wrapper_fd.get(), is_public, uid, out_oat_path, is_secondary_dex)) {
         ALOGE("installd cannot set owner '%s' for output during dexopt\n", out_oat_path);
         wrapper_fd.reset(-1);
     }
@@ -1207,9 +1216,189 @@ void update_out_oat_access_times(const char* apk_path, const char* out_oat_path)
     }
 }
 
-int dexopt(const char* apk_path, uid_t uid, const char* pkgname, const char* instruction_set,
-        int dexopt_needed, const char* oat_dir, int dexopt_flags,const char* compiler_filter,
-        const char* volume_uuid ATTRIBUTE_UNUSED, const char* shared_libraries) {
+// Runs (execv) dexoptanalyzer on the given arguments.
+static void exec_dexoptanalyzer(const char* dex_file, const char* instruction_set,
+        const char* compiler_filter) {
+    static const char* DEXOPTANALYZER_BIN = "/system/bin/dexoptanalyzer";
+    static const unsigned int MAX_INSTRUCTION_SET_LEN = 7;
+
+    if (strlen(instruction_set) >= MAX_INSTRUCTION_SET_LEN) {
+        ALOGE("Instruction set %s longer than max length of %d",
+              instruction_set, MAX_INSTRUCTION_SET_LEN);
+        return;
+    }
+
+    char dex_file_arg[strlen("--dex-file=") + PKG_PATH_MAX];
+    char isa_arg[strlen("--isa=") + MAX_INSTRUCTION_SET_LEN];
+    char compiler_filter_arg[strlen("--compiler-filter=") + kPropertyValueMax];
+
+    sprintf(dex_file_arg, "--dex-file=%s", dex_file);
+    sprintf(isa_arg, "--isa=%s", instruction_set);
+    sprintf(compiler_filter_arg, "--compiler-filter=%s", compiler_filter);
+
+    // program name, dex file, isa, filter, the final NULL
+    const char* argv[5];
+    int i = 0;
+    argv[i++] = DEXOPTANALYZER_BIN;
+    argv[i++] = dex_file_arg;
+    argv[i++] = isa_arg;
+    argv[i++] = compiler_filter_arg;
+    argv[i] = NULL;
+
+    execv(DEXOPTANALYZER_BIN, (char * const *)argv);
+    ALOGE("execv(%s) failed: %s\n", DEXOPTANALYZER_BIN, strerror(errno));
+}
+
+// Prepares the oat dir for the secondary dex files.
+static bool prepare_secondary_dex_oat_dir(const char* dex_path, int uid,
+         const char* instruction_set, std::string* oat_dir_out) {
+    std::string apk_path_str(dex_path);
+    unsigned long dirIndex = apk_path_str.rfind('/');
+    if (dirIndex == std::string::npos) {
+        LOG(ERROR ) << "Unexpected dir structure for secondary dex " << dex_path;
+        return false;
+    }
+    std::string apk_dir = apk_path_str.substr(0, dirIndex);
+
+    // Assign the gid to the cache gid so that the oat file storage
+    // is counted towards the app cache.
+    int32_t cache_gid = multiuser_get_cache_gid(
+            multiuser_get_user_id(uid), multiuser_get_app_id(uid));
+    // If UID doesn't have a specific cache GID, use UID value
+    if (cache_gid == -1) {
+        cache_gid = uid;
+    }
+
+    // Create oat file output directory.
+    if (prepare_app_cache_dir(apk_dir, "oat", 02711, uid, cache_gid) != 0) {
+        LOG(ERROR) << "Could not prepare oat dir for secondary dex: " << dex_path;
+        return false;
+    }
+
+    char oat_dir[PKG_PATH_MAX];
+    snprintf(oat_dir, PKG_PATH_MAX, "%s/oat", apk_dir.c_str());
+    oat_dir_out->assign(oat_dir);
+
+    // Create oat/isa output directory.
+    if (prepare_app_cache_dir(*oat_dir_out, instruction_set, 02711, uid, cache_gid) != 0) {
+        LOG(ERROR) << "Could not prepare oat/isa dir for secondary dex: " << dex_path;
+        return false;
+    }
+
+    return true;
+}
+
+static int constexpr DEXOPTANALYZER_BIN_EXEC_ERROR = 200;
+
+// Verifies the result of dexoptanalyzer executed for the apk_path.
+// If the result is valid returns true and sets dexopt_needed_out to a valid value.
+// Returns false for errors or unexpected result values.
+static bool process_dexoptanalyzer_result(const char* dex_path, int result,
+            int* dexopt_needed_out) {
+    // The result values are defined in dexoptanalyzer.
+    switch (result) {
+        case 0:  // no_dexopt_needed
+            *dexopt_needed_out = NO_DEXOPT_NEEDED; return true;
+        case 1:  // dex2oat_from_scratch
+            *dexopt_needed_out = DEX2OAT_FROM_SCRATCH; return true;
+        case 5:  // dex2oat_for_bootimage_odex
+            *dexopt_needed_out = -DEX2OAT_FOR_BOOT_IMAGE; return true;
+        case 6:  // dex2oat_for_filter_odex
+            *dexopt_needed_out = -DEX2OAT_FOR_FILTER; return true;
+        case 7:  // dex2oat_for_relocation_odex
+            *dexopt_needed_out = -DEX2OAT_FOR_RELOCATION; return true;
+        case 2:  // dex2oat_for_bootimage_oat
+        case 3:  // dex2oat_for_filter_oat
+        case 4:  // dex2oat_for_relocation_oat
+            LOG(ERROR) << "Dexoptnalyzer return the status of an oat file."
+                    << " Expected odex file status for secondary dex " << dex_path
+                    << " : dexoptanalyzer result=" << result;
+            return false;
+        default:
+            LOG(ERROR) << "Unexpected result for dexoptanalyzer " << dex_path
+                    << " exec_dexoptanalyzer result=" << result;
+            return false;
+    }
+}
+
+// Processes the dex_path as a secondary dex files and return true if the path dex file should
+// be compiled. Returns false for errors (logged) or true if the secondary dex path was process
+// successfully.
+// When returning true, dexopt_needed_out is assigned a valid OatFileAsssitant::DexOptNeeded
+// code and aot_dir_out is assigned the oat dir path where the oat file should be stored.
+static bool process_secondary_dex_dexopt(const char* dex_path, const char* pkgname,
+        int dexopt_flags, const char* volume_uuid, int uid, const char* instruction_set,
+        const char* compiler_filter, int* dexopt_needed_out, std::string* aot_dir_out) {
+    int storage_flag;
+
+    if ((dexopt_flags & DEXOPT_STORAGE_CE) != 0) {
+        storage_flag = FLAG_STORAGE_CE;
+        if ((dexopt_flags & DEXOPT_STORAGE_DE) != 0) {
+            LOG(ERROR) << "Ambiguous secondary dex storage flag. Both, CE and DE, flags are set";
+            return false;
+        }
+    } else if ((dexopt_flags & DEXOPT_STORAGE_DE) != 0) {
+        storage_flag = FLAG_STORAGE_DE;
+    } else {
+        LOG(ERROR) << "Secondary dex storage flag must be set";
+        return false;
+    }
+
+    if (!validate_secondary_dex_path(pkgname, dex_path, volume_uuid, uid, storage_flag)) {
+        LOG(ERROR) << "Could not validate secondary dex path " << dex_path;
+        return false;
+    }
+
+    // Check if the path exist. If not, there's nothing to do.
+    if (access(dex_path, F_OK) != 0) {
+        if (errno == ENOENT) {
+            // Secondary dex files might be deleted any time by the app.
+            // Nothing to do if that's the case
+            ALOGV("Secondary dex does not exist %s", dex_path);
+            return NO_DEXOPT_NEEDED;
+        } else {
+            PLOG(ERROR) << "Could not access secondary dex " << dex_path;
+        }
+    }
+
+    // Prepare the oat directories.
+    if (!prepare_secondary_dex_oat_dir(dex_path, uid, instruction_set, aot_dir_out)) {
+        return false;
+    }
+
+    pid_t pid = fork();
+    if (pid == 0) {
+        // child -- drop privileges before continuing.
+        drop_capabilities(uid);
+        // Run dexoptanalyzer to get dexopt_needed code.
+        exec_dexoptanalyzer(dex_path, instruction_set, compiler_filter);
+        exit(DEXOPTANALYZER_BIN_EXEC_ERROR);
+    }
+
+    /* parent */
+
+    int result = wait_child(pid);
+    if (!WIFEXITED(result)) {
+        LOG(ERROR) << "dexoptanalyzer failed for path " << dex_path << ": " << result;
+        return false;
+    }
+    result = WEXITSTATUS(result);
+    bool success = process_dexoptanalyzer_result(dex_path, result, dexopt_needed_out);
+    // Run dexopt only if needed or forced.
+    // Note that dexoptanalyzer is executed even if force compilation is enabled.
+    // We ignore its valid dexopNeeded result, but still check (in process_dexoptanalyzer_result)
+    // that we only get results for odex files (apk_dir/oat/isa/code.odex) and not
+    // for oat files from dalvik-cache.
+    if (success && ((dexopt_flags & DEXOPT_FORCE) != 0)) {
+        *dexopt_needed_out = DEX2OAT_FROM_SCRATCH;
+    }
+
+    return success;
+}
+
+int dexopt(const char* dex_path, uid_t uid, const char* pkgname, const char* instruction_set,
+        int dexopt_needed, const char* oat_dir, int dexopt_flags, const char* compiler_filter,
+        const char* volume_uuid, const char* shared_libraries) {
     CHECK(pkgname != nullptr);
     CHECK(pkgname[0] != 0);
     if ((dexopt_flags & ~DEXOPT_MASK) != 0) {
@@ -1221,18 +1410,38 @@ int dexopt(const char* apk_path, uid_t uid, const char* pkgname, const char* ins
     bool debuggable = (dexopt_flags & DEXOPT_DEBUGGABLE) != 0;
     bool boot_complete = (dexopt_flags & DEXOPT_BOOTCOMPLETE) != 0;
     bool profile_guided = (dexopt_flags & DEXOPT_PROFILE_GUIDED) != 0;
+    bool is_secondary_dex = (dexopt_flags & DEXOPT_SECONDARY_DEX) != 0;
+
+    // Check if we're dealing with a secondary dex file and if we need to compile it.
+    std::string oat_dir_str;
+    if (is_secondary_dex) {
+        if (process_secondary_dex_dexopt(dex_path, pkgname, dexopt_flags, volume_uuid, uid,
+                instruction_set, compiler_filter, &dexopt_needed, &oat_dir_str)) {
+            oat_dir = oat_dir_str.c_str();
+            if (dexopt_needed == NO_DEXOPT_NEEDED) {
+                return 0;  // Nothing to do, report success.
+            }
+        } else {
+            return -1;  // We had an error, logged in the process method.
+        }
+    } else {
+        // Currently these flags are only use for secondary dex files.
+        // Verify that they are not set for primary apks.
+        CHECK((dexopt_flags & DEXOPT_STORAGE_CE) == 0);
+        CHECK((dexopt_flags & DEXOPT_STORAGE_DE) == 0);
+    }
 
     // Open the input file.
-    base::unique_fd input_fd(open(apk_path, O_RDONLY, 0));
+    base::unique_fd input_fd(open(dex_path, O_RDONLY, 0));
     if (input_fd.get() < 0) {
-        ALOGE("installd cannot open '%s' for input during dexopt\n", apk_path);
+        ALOGE("installd cannot open '%s' for input during dexopt\n", dex_path);
         return -1;
     }
 
     // Create the output OAT file.
     char out_oat_path[PKG_PATH_MAX];
-    Dex2oatFileWrapper out_oat_fd = open_oat_out_file(apk_path, oat_dir, is_public, uid,
-            instruction_set, out_oat_path);
+    Dex2oatFileWrapper out_oat_fd = open_oat_out_file(dex_path, oat_dir, is_public, uid,
+            instruction_set, is_secondary_dex, out_oat_path);
     if (out_oat_fd.get() < 0) {
         return -1;
     }
@@ -1240,8 +1449,8 @@ int dexopt(const char* apk_path, uid_t uid, const char* pkgname, const char* ins
     // Open vdex files.
     Dex2oatFileWrapper in_vdex_fd;
     Dex2oatFileWrapper out_vdex_fd;
-    if (!open_vdex_files(apk_path, out_oat_path, dexopt_needed, instruction_set, is_public, uid,
-            &in_vdex_fd, &out_vdex_fd)) {
+    if (!open_vdex_files(dex_path, out_oat_path, dexopt_needed, instruction_set, is_public, uid,
+            is_secondary_dex, &in_vdex_fd, &out_vdex_fd)) {
         return -1;
     }
 
@@ -1250,13 +1459,13 @@ int dexopt(const char* apk_path, uid_t uid, const char* pkgname, const char* ins
 
     // Create the app image file if needed.
     Dex2oatFileWrapper image_fd =
-            maybe_open_app_image(out_oat_path, profile_guided, is_public, uid);
+            maybe_open_app_image(out_oat_path, profile_guided, is_public, uid, is_secondary_dex);
 
     // Open the reference profile if needed.
     Dex2oatFileWrapper reference_profile_fd =
-            maybe_open_reference_profile(pkgname, profile_guided, is_public, uid);
+        maybe_open_reference_profile(pkgname, profile_guided, is_public, uid, is_secondary_dex);
 
-    ALOGV("DexInv: --- BEGIN '%s' ---\n", apk_path);
+    ALOGV("DexInv: --- BEGIN '%s' ---\n", dex_path);
 
     pid_t pid = fork();
     if (pid == 0) {
@@ -1270,7 +1479,7 @@ int dexopt(const char* apk_path, uid_t uid, const char* pkgname, const char* ins
         }
 
         // Pass dex2oat the relative path to the input file.
-        const char *input_file_name = get_location_from_path(apk_path);
+        const char *input_file_name = get_location_from_path(dex_path);
         run_dex2oat(input_fd.get(),
                     out_oat_fd.get(),
                     in_vdex_fd.get(),
@@ -1290,14 +1499,14 @@ int dexopt(const char* apk_path, uid_t uid, const char* pkgname, const char* ins
     } else {
         int res = wait_child(pid);
         if (res == 0) {
-            ALOGV("DexInv: --- END '%s' (success) ---\n", apk_path);
+            ALOGV("DexInv: --- END '%s' (success) ---\n", dex_path);
         } else {
-            ALOGE("DexInv: --- END '%s' --- status=0x%04x, process failed\n", apk_path, res);
+            ALOGE("DexInv: --- END '%s' --- status=0x%04x, process failed\n", dex_path, res);
             return -1;
         }
     }
 
-    update_out_oat_access_times(apk_path, out_oat_path);
+    update_out_oat_access_times(dex_path, out_oat_path);
 
     // We've been successful, don't delete output.
     out_oat_fd.SetCleanup(false);
@@ -1308,6 +1517,115 @@ int dexopt(const char* apk_path, uid_t uid, const char* pkgname, const char* ins
     return 0;
 }
 
+// Try to remove the given directory. Log an error if the directory exists
+// and is empty but could not be removed.
+static bool rmdir_if_empty(const char* dir) {
+    if (rmdir(dir) == 0) {
+        return true;
+    }
+    if (errno == ENOENT || errno == ENOTEMPTY) {
+        return true;
+    }
+    PLOG(ERROR) << "Failed to remove dir: " << dir;
+    return false;
+}
+
+// Try to unlink the given file. Log an error if the file exists and could not
+// be unlinked.
+static bool unlink_if_exists(const std::string& file) {
+    if (unlink(file.c_str()) == 0) {
+        return true;
+    }
+    if (errno == ENOENT) {
+        return true;
+
+    }
+    PLOG(ERROR) << "Could not unlink: " << file;
+    return false;
+}
+
+// Create the oat file structure for the secondary dex 'dex_path' and assign
+// the individual path component to the 'out_' parameters.
+static bool create_secondary_dex_oat_layout(const std::string& dex_path, const std::string& isa,
+        /*out*/char* out_oat_dir, /*out*/char* out_oat_isa_dir, /*out*/char* out_oat_path) {
+    size_t dirIndex = dex_path.rfind('/');
+    if (dirIndex == std::string::npos) {
+        LOG(ERROR) << "Unexpected dir structure for dex file " << dex_path;
+        return false;
+    }
+    // TODO(calin): we have similar computations in at lest 3 other places
+    // (InstalldNativeService, otapropt and dexopt). Unify them and get rid of snprintf by
+    // use string append.
+    std::string apk_dir = dex_path.substr(0, dirIndex);
+    snprintf(out_oat_dir, PKG_PATH_MAX, "%s/oat", apk_dir.c_str());
+    snprintf(out_oat_isa_dir, PKG_PATH_MAX, "%s/%s", out_oat_dir, isa.c_str());
+
+    if (!create_oat_out_path(dex_path.c_str(), isa.c_str(), out_oat_dir,
+            /*is_secondary_dex*/ true, out_oat_path)) {
+        LOG(ERROR) << "Could not create oat path for secondary dex " << dex_path;
+        return false;
+    }
+    return true;
+}
+
+// Reconcile the secondary dex 'dex_path' and its generated oat files.
+// Return true if all the parameters are valid and the secondary dex file was
+//   processed successfully (i.e. the dex_path either exists, or if not, its corresponding
+//   oat/vdex/art files where deleted successfully). In this case, out_secondary_dex_exists
+//   will be true if the secondary dex file still exists. If the secondary dex file does not exist,
+//   the method cleans up any previously generated compiler artifacts (oat, vdex, art).
+// Return false if there were errors during processing. In this case
+//   out_secondary_dex_exists will be set to false.
+bool reconcile_secondary_dex_file(const std::string& dex_path,
+        const std::string& pkgname, int uid, const std::vector<std::string>& isas,
+        const std::unique_ptr<std::string>& volume_uuid, int storage_flag,
+        /*out*/bool* out_secondary_dex_exists) {
+    // Set out to false to start with, just in case we have validation errors.
+    *out_secondary_dex_exists = false;
+    if (isas.size() == 0) {
+        LOG(ERROR) << "reconcile_secondary_dex_file called with empty isas vector";
+        return false;
+    }
+
+    const char* volume_uuid_cstr = volume_uuid == nullptr ? nullptr : volume_uuid->c_str();
+    if (!validate_secondary_dex_path(pkgname.c_str(), dex_path.c_str(), volume_uuid_cstr,
+            uid, storage_flag)) {
+        LOG(ERROR) << "Could not validate secondary dex path " << dex_path;
+        return false;
+    }
+
+    if (access(dex_path.c_str(), F_OK) == 0) {
+        // The path exists, nothing to do. The odex files (if any) will be left untouched.
+        *out_secondary_dex_exists = true;
+        return true;
+    } else if (errno != ENOENT) {
+        PLOG(ERROR) << "Failed to check access to secondary dex " << dex_path;
+        return false;
+    }
+
+    // The secondary dex does not exist anymore. Clear any generated files.
+    char oat_path[PKG_PATH_MAX];
+    char oat_dir[PKG_PATH_MAX];
+    char oat_isa_dir[PKG_PATH_MAX];
+    bool result = true;
+    for (size_t i = 0; i < isas.size(); i++) {
+        if (!create_secondary_dex_oat_layout(dex_path, isas[i], oat_dir, oat_isa_dir, oat_path)) {
+            LOG(ERROR) << "Could not create secondary odex layout: " << dex_path;
+            result = false;
+            continue;
+        }
+        result = unlink_if_exists(oat_path) && result;
+        result = unlink_if_exists(create_vdex_filename(oat_path)) && result;
+        result = unlink_if_exists(create_image_filename(oat_path)) && result;
+
+        // Try removing the directories as well, they might be empty.
+        result = rmdir_if_empty(oat_isa_dir) && result;
+        result = rmdir_if_empty(oat_dir) && result;
+    }
+
+    return result;
+}
+
 // Helper for move_ab, so that we can have common failure-case cleanup.
 static bool unlink_and_rename(const char* from, const char* to) {
     // Check whether "from" exists, and if so whether it's regular. If it is, unlink. Otherwise,
@@ -1439,7 +1757,8 @@ bool move_ab(const char* apk_path, const char* instruction_set, const char* oat_
 bool delete_odex(const char* apk_path, const char* instruction_set, const char* oat_dir) {
     // Delete the oat/odex file.
     char out_path[PKG_PATH_MAX];
-    if (!create_oat_out_path(apk_path, instruction_set, oat_dir, out_path)) {
+    if (!create_oat_out_path(apk_path, instruction_set, oat_dir,
+            /*is_secondary_dex*/ false, out_path)) {
         return false;
     }
 
diff --git a/cmds/installd/dexopt.h b/cmds/installd/dexopt.h
index 1115c78cbd..7bb6eeef73 100644
--- a/cmds/installd/dexopt.h
+++ b/cmds/installd/dexopt.h
@@ -25,6 +25,7 @@ namespace android {
 namespace installd {
 
 /* dexopt needed flags matching those in dalvik.system.DexFile */
+static constexpr int NO_DEXOPT_NEEDED            = 0;
 static constexpr int DEX2OAT_FROM_SCRATCH        = 1;
 static constexpr int DEX2OAT_FOR_BOOT_IMAGE      = 2;
 static constexpr int DEX2OAT_FOR_FILTER          = 3;
@@ -44,6 +45,11 @@ bool dump_profiles(int32_t uid, const char* pkgname, const char* code_paths);
 
 bool delete_odex(const char* apk_path, const char* instruction_set, const char* output_path);
 
+bool reconcile_secondary_dex_file(const std::string& dex_path,
+        const std::string& pkgname, int uid, const std::vector<std::string>& isas,
+        const std::unique_ptr<std::string>& volumeUuid, int storage_flag,
+        /*out*/bool* out_secondary_dex_exists);
+
 int dexopt(const char *apk_path, uid_t uid, const char *pkgName, const char *instruction_set,
         int dexopt_needed, const char* oat_dir, int dexopt_flags, const char* compiler_filter,
         const char* volume_uuid, const char* shared_libraries);
diff --git a/cmds/installd/installd_constants.h b/cmds/installd/installd_constants.h
index 401e581e45..d8a754cc9b 100644
--- a/cmds/installd/installd_constants.h
+++ b/cmds/installd/installd_constants.h
@@ -42,6 +42,14 @@ constexpr int DEXOPT_SAFEMODE       = 1 << 2;
 constexpr int DEXOPT_DEBUGGABLE     = 1 << 3;
 constexpr int DEXOPT_BOOTCOMPLETE   = 1 << 4;
 constexpr int DEXOPT_PROFILE_GUIDED = 1 << 5;
+constexpr int DEXOPT_SECONDARY_DEX  = 1 << 6;
+// DEXOPT_FORCE, DEXOPT_STORAGE_CE, DEXOPT_STORAGE_DE are exposed for secondary
+// dex files only. Primary apks are analyzed in PackageManager and installd
+// does not need to know if the compilation is forced or on what kind of storage
+// the dex files are.
+constexpr int DEXOPT_FORCE          = 1 << 7;
+constexpr int DEXOPT_STORAGE_CE     = 1 << 8;
+constexpr int DEXOPT_STORAGE_DE     = 1 << 9;
 
 /* all known values for dexopt flags */
 constexpr int DEXOPT_MASK =
@@ -49,7 +57,15 @@ constexpr int DEXOPT_MASK =
     | DEXOPT_SAFEMODE
     | DEXOPT_DEBUGGABLE
     | DEXOPT_BOOTCOMPLETE
-    | DEXOPT_PROFILE_GUIDED;
+    | DEXOPT_PROFILE_GUIDED
+    | DEXOPT_SECONDARY_DEX
+    | DEXOPT_FORCE
+    | DEXOPT_STORAGE_CE
+    | DEXOPT_STORAGE_DE;
+
+// NOTE: keep in sync with StorageManager
+constexpr int FLAG_STORAGE_DE = 1 << 0;
+constexpr int FLAG_STORAGE_CE = 1 << 1;
 
 #define ARRAY_SIZE(a) (sizeof(a) / sizeof(*(a)))
 
diff --git a/cmds/installd/utils.cpp b/cmds/installd/utils.cpp
index 0b1cd7e99d..1abf2b6460 100644
--- a/cmds/installd/utils.cpp
+++ b/cmds/installd/utils.cpp
@@ -1171,6 +1171,25 @@ int validate_system_app_path(const char* path) {
     return -1;
 }
 
+bool validate_secondary_dex_path(const char* pkgname, const char* path,
+        const char* volume_uuid, int uid, int storage_flag) {
+    CHECK(storage_flag == FLAG_STORAGE_CE || storage_flag == FLAG_STORAGE_DE);
+
+    std::string app_private_dir = storage_flag == FLAG_STORAGE_CE
+        ? create_data_user_ce_package_path(volume_uuid, multiuser_get_user_id(uid), pkgname)
+        : create_data_user_de_package_path(volume_uuid, multiuser_get_user_id(uid), pkgname);
+    dir_rec_t dir;
+    if (get_path_from_string(&dir, app_private_dir.c_str()) != 0) {
+        LOG(WARNING) << "Could not get dir rec for " << app_private_dir;
+        return false;
+    }
+    // Usually secondary dex files have a nested directory structure.
+    // Pick at most 10 subdirectories when validating (arbitrary value).
+    // If the secondary dex file is >10 directory nested then validation will
+    // fail and the file will not be compiled.
+    return validate_path(&dir, path, /*max_subdirs*/ 10) == 0;
+}
+
 /**
  * Get the contents of a environment variable that contains a path. Caller
  * owns the string that is inserted into the directory record. Returns
@@ -1370,5 +1389,73 @@ int wait_child(pid_t pid)
     }
 }
 
+/**
+ * Prepare an app cache directory, which offers to fix-up the GID and
+ * directory mode flags during a platform upgrade.
+ * The app cache directory path will be 'parent'/'name'.
+ */
+int prepare_app_cache_dir(const std::string& parent, const char* name, mode_t target_mode,
+        uid_t uid, gid_t gid) {
+    auto path = StringPrintf("%s/%s", parent.c_str(), name);
+    struct stat st;
+    if (stat(path.c_str(), &st) != 0) {
+        if (errno == ENOENT) {
+            // This is fine, just create it
+            if (fs_prepare_dir_strict(path.c_str(), target_mode, uid, gid) != 0) {
+                PLOG(ERROR) << "Failed to prepare " << path;
+                return -1;
+            } else {
+                return 0;
+            }
+        } else {
+            PLOG(ERROR) << "Failed to stat " << path;
+            return -1;
+        }
+    }
+
+    mode_t actual_mode = st.st_mode & (S_IRWXU | S_IRWXG | S_IRWXO | S_ISGID);
+    if (st.st_uid != uid) {
+        // Mismatched UID is real trouble; we can't recover
+        LOG(ERROR) << "Mismatched UID at " << path << ": found " << st.st_uid
+                << " but expected " << uid;
+        return -1;
+    } else if (st.st_gid == gid && actual_mode == target_mode) {
+        // Everything looks good!
+        return 0;
+    }
+
+    // Directory is owned correctly, but GID or mode mismatch means it's
+    // probably a platform upgrade so we need to fix them
+    FTS *fts;
+    FTSENT *p;
+    char *argv[] = { (char*) path.c_str(), nullptr };
+    if (!(fts = fts_open(argv, FTS_PHYSICAL | FTS_XDEV, NULL))) {
+        PLOG(ERROR) << "Failed to fts_open " << path;
+        return -1;
+    }
+    while ((p = fts_read(fts)) != NULL) {
+        switch (p->fts_info) {
+        case FTS_DP:
+            if (chmod(p->fts_accpath, target_mode) != 0) {
+                PLOG(WARNING) << "Failed to chmod " << p->fts_path;
+            }
+            // Intentional fall through to also set GID
+        case FTS_F:
+            if (chown(p->fts_accpath, -1, gid) != 0) {
+                PLOG(WARNING) << "Failed to chown " << p->fts_path;
+            }
+            break;
+        case FTS_SL:
+        case FTS_SLNONE:
+            if (lchown(p->fts_accpath, -1, gid) != 0) {
+                PLOG(WARNING) << "Failed to chown " << p->fts_path;
+            }
+            break;
+        }
+    }
+    fts_close(fts);
+    return 0;
+}
+
 }  // namespace installd
 }  // namespace android
diff --git a/cmds/installd/utils.h b/cmds/installd/utils.h
index 5e396c7f37..6cd5aee099 100644
--- a/cmds/installd/utils.h
+++ b/cmds/installd/utils.h
@@ -148,6 +148,8 @@ void clear_cache_files(const std::string& data_path, cache_t* cache, int64_t fre
 void finish_cache_collection(cache_t* cache);
 
 int validate_system_app_path(const char* path);
+bool validate_secondary_dex_path(const char* pkgname, const char* path,
+        const char* volume_uuid, int uid, int storage_flag);
 
 int get_path_from_env(dir_rec_t* rec, const char* var);
 
@@ -167,6 +169,9 @@ int ensure_config_user_dirs(userid_t userid);
 
 int wait_child(pid_t pid);
 
+int prepare_app_cache_dir(const std::string& parent, const char* name, mode_t target_mode,
+        uid_t uid, gid_t gid);
+
 }  // namespace installd
 }  // namespace android