fec: accept multiple input files for encoding

Make it possible to encode input files in pieces (not with -m).

Bug: 26251929
Change-Id: I09349368925e5e2f2a7961a712a1e933eb0b9ec8

3 files changed, 147 insertions, 88 deletions

diff --git a/verity/fec/image.cpp b/verity/fec/image.cpp
index 95bc88ee..a378c932 100644
--- a/verity/fec/image.cpp
+++ b/verity/fec/image.cpp
@@ -55,12 +55,12 @@ static void mmap_image_free(image *ctx)
 {
     if (ctx->input) {
         munmap(ctx->input, (size_t)ctx->inp_size);
-        TEMP_FAILURE_RETRY(close(ctx->inp_fd));
+        close(ctx->inp_fd);
     }
 
     if (ctx->fec_mmap_addr) {
         munmap(ctx->fec_mmap_addr, FEC_BLOCKSIZE + ctx->fec_size);
-        TEMP_FAILURE_RETRY(close(ctx->fec_fd));
+        close(ctx->fec_fd);
     }
 
     if (!ctx->inplace && ctx->output) {
@@ -129,8 +129,15 @@ static void calculate_rounds(uint64_t size, image *ctx)
     ctx->rounds = fec_div_round_up(ctx->blocks, ctx->rs_n);
 }
 
-static void mmap_image_load(int fd, image *ctx, bool output_needed)
+static void mmap_image_load(const std::vector<int>& fds, image *ctx,
+        bool output_needed)
 {
+    if (fds.size() != 1) {
+        FATAL("multiple input files not supported with mmap\n");
+    }
+
+    int fd = fds.front();
+
     calculate_rounds(get_size(fd), ctx);
 
     /* check that we can memory map the file; on 32-bit platforms we are
@@ -194,33 +201,43 @@ static int process_chunk(void *priv, const void *data, int len)
 }
 #endif
 
-static void file_image_load(int fd, image *ctx)
+static void file_image_load(const std::vector<int>& fds, image *ctx)
 {
-    uint64_t len = 0;
+    uint64_t size = 0;
+#ifndef IMAGE_NO_SPARSE
+    std::vector<struct sparse_file *> files;
+#endif
+
+    for (auto fd : fds) {
+        uint64_t len = 0;
 
 #ifdef IMAGE_NO_SPARSE
-    if (ctx->sparse) {
-        FATAL("sparse files not supported\n");
-    }
+        if (ctx->sparse) {
+            FATAL("sparse files not supported\n");
+        }
 
-    len = get_size(fd);
+        len = get_size(fd);
 #else
-    struct sparse_file *file;
+        struct sparse_file *file;
 
-    if (ctx->sparse) {
-        file = sparse_file_import(fd, false, false);
-    } else {
-        file = sparse_file_import_auto(fd, false, ctx->verbose);
-    }
+        if (ctx->sparse) {
+            file = sparse_file_import(fd, false, false);
+        } else {
+            file = sparse_file_import_auto(fd, false, ctx->verbose);
+        }
 
-    if (!file) {
-        FATAL("failed to read file %s\n", ctx->fec_filename);
-    }
+        if (!file) {
+            FATAL("failed to read file %s\n", ctx->fec_filename);
+        }
 
-    len = sparse_file_len(file, false, false);
+        len = sparse_file_len(file, false, false);
+        files.push_back(file);
 #endif /* IMAGE_NO_SPARSE */
 
-    calculate_rounds(len, ctx);
+        size += len;
+    }
+
+    calculate_rounds(size, ctx);
 
     if (ctx->verbose) {
         INFO("allocating %" PRIu64 " bytes of memory\n", ctx->inp_size);
@@ -234,21 +251,33 @@ static void file_image_load(int fd, image *ctx)
 
     memset(ctx->input, 0, ctx->inp_size);
     ctx->output = ctx->input;
+    ctx->pos = 0;
 
 #ifdef IMAGE_NO_SPARSE
-    if (!android::base::ReadFully(fd, ctx->input, ctx->inp_size)) {
-        FATAL("failed to read: %s\n", strerror(errno));
+    for (auto fd : fds) {
+        uint64_t len = get_size(fd);
+
+        if (!android::base::ReadFully(fd, &ctx->input[ctx->pos], len)) {
+            FATAL("failed to read: %s\n", strerror(errno));
+        }
+
+        ctx->pos += len;
+        close(fd);
     }
 #else
-    ctx->pos = 0;
-    sparse_file_callback(file, false, false, process_chunk, ctx);
-    sparse_file_destroy(file);
-#endif
+    for (auto file : files) {
+        sparse_file_callback(file, false, false, process_chunk, ctx);
+        sparse_file_destroy(file);
+    }
 
-    TEMP_FAILURE_RETRY(close(fd));
+    for (auto fd : fds) {
+        close(fd);
+    }
+#endif
 }
 
-bool image_load(const char *filename, image *ctx, bool output_needed)
+bool image_load(const std::vector<std::string>& filenames, image *ctx,
+        bool output_needed)
 {
     assert(ctx->roots > 0 && ctx->roots < FEC_RSM);
     ctx->rs_n = FEC_RSM - ctx->roots;
@@ -259,40 +288,47 @@ bool image_load(const char *filename, image *ctx, bool output_needed)
         flags = O_RDWR;
     }
 
-    int fd = TEMP_FAILURE_RETRY(open(filename, flags | O_LARGEFILE));
+    std::vector<int> fds;
 
-    if (fd < 0) {
-        FATAL("failed to open file '%s': %s\n", filename, strerror(errno));
+    for (auto fn : filenames) {
+        int fd = TEMP_FAILURE_RETRY(open(fn.c_str(), flags | O_LARGEFILE));
+
+        if (fd < 0) {
+            FATAL("failed to open file '%s': %s\n", fn.c_str(), strerror(errno));
+        }
+
+        fds.push_back(fd);
     }
 
     if (ctx->mmap) {
-        mmap_image_load(fd, ctx, output_needed);
+        mmap_image_load(fds, ctx, output_needed);
     } else {
-        file_image_load(fd, ctx);
+        file_image_load(fds, ctx);
     }
 
     return true;
 }
 
-bool image_save(const char *filename, image *ctx)
+bool image_save(const std::string& filename, image *ctx)
 {
     if (ctx->inplace && ctx->mmap) {
         return true; /* nothing to do */
     }
 
     /* TODO: support saving as a sparse file */
-    int fd = TEMP_FAILURE_RETRY(open(filename, O_WRONLY | O_CREAT | O_TRUNC,
-                0666));
+    int fd = TEMP_FAILURE_RETRY(open(filename.c_str(),
+                O_WRONLY | O_CREAT | O_TRUNC, 0666));
 
     if (fd < 0) {
-        FATAL("failed to open file '%s: %s'\n", filename, strerror(errno));
+        FATAL("failed to open file '%s: %s'\n", filename.c_str(),
+            strerror(errno));
     }
 
     if (!android::base::WriteFully(fd, ctx->output, ctx->inp_size)) {
         FATAL("failed to write to output: %s\n", strerror(errno));
     }
 
-    TEMP_FAILURE_RETRY(close(fd));
+    close(fd);
     return true;
 }
 
@@ -347,11 +383,11 @@ static void file_image_ecc_new(image *ctx)
     }
 }
 
-bool image_ecc_new(const char *filename, image *ctx)
+bool image_ecc_new(const std::string& filename, image *ctx)
 {
     assert(ctx->rounds > 0); /* image_load should be called first */
 
-    ctx->fec_filename = filename;
+    ctx->fec_filename = filename.c_str();
     ctx->fec_size = ctx->rounds * ctx->roots * FEC_BLOCKSIZE;
 
     if (ctx->mmap) {
@@ -363,16 +399,17 @@ bool image_ecc_new(const char *filename, image *ctx)
     return true;
 }
 
-bool image_ecc_load(const char *filename, image *ctx)
+bool image_ecc_load(const std::string& filename, image *ctx)
 {
-    int fd = TEMP_FAILURE_RETRY(open(filename, O_RDONLY));
+    int fd = TEMP_FAILURE_RETRY(open(filename.c_str(), O_RDONLY));
 
     if (fd < 0) {
-        FATAL("failed to open file '%s': %s\n", filename, strerror(errno));
+        FATAL("failed to open file '%s': %s\n", filename.c_str(),
+            strerror(errno));
     }
 
     if (lseek64(fd, -FEC_BLOCKSIZE, SEEK_END) < 0) {
-        FATAL("failed to seek to header in '%s': %s\n", filename,
+        FATAL("failed to seek to header in '%s': %s\n", filename.c_str(),
             strerror(errno));
     }
 
@@ -383,27 +420,29 @@ bool image_ecc_load(const char *filename, image *ctx)
 
     if (!android::base::ReadFully(fd, header, sizeof(header))) {
         FATAL("failed to read %zd bytes from '%s': %s\n", sizeof(header),
-            filename, strerror(errno));
+            filename.c_str(), strerror(errno));
     }
 
     if (p->magic != FEC_MAGIC) {
-        FATAL("invalid magic in '%s': %08x\n", filename, p->magic);
+        FATAL("invalid magic in '%s': %08x\n", filename.c_str(), p->magic);
     }
 
     if (p->version != FEC_VERSION) {
-        FATAL("unsupported version in '%s': %u\n", filename, p->version);
+        FATAL("unsupported version in '%s': %u\n", filename.c_str(),
+            p->version);
     }
 
     if (p->size != sizeof(fec_header)) {
-        FATAL("unexpected header size in '%s': %u\n", filename, p->size);
+        FATAL("unexpected header size in '%s': %u\n", filename.c_str(),
+            p->size);
     }
 
     if (p->roots == 0 || p->roots >= FEC_RSM) {
-        FATAL("invalid roots in '%s': %u\n", filename, p->roots);
+        FATAL("invalid roots in '%s': %u\n", filename.c_str(), p->roots);
     }
 
     if (p->fec_size % p->roots || p->fec_size % FEC_BLOCKSIZE) {
-        FATAL("invalid length in '%s': %u\n", filename, p->fec_size);
+        FATAL("invalid length in '%s': %u\n", filename.c_str(), p->fec_size);
     }
 
     ctx->roots = (int)p->roots;
@@ -416,19 +455,19 @@ bool image_ecc_load(const char *filename, image *ctx)
     }
 
     if (p->fec_size != ctx->fec_size) {
-        FATAL("inconsistent header in '%s'\n", filename);
+        FATAL("inconsistent header in '%s'\n", filename.c_str());
     }
 
     if (lseek64(fd, 0, SEEK_SET) < 0) {
-        FATAL("failed to rewind '%s': %s", filename, strerror(errno));
+        FATAL("failed to rewind '%s': %s", filename.c_str(), strerror(errno));
     }
 
     if (!ctx->mmap && !android::base::ReadFully(fd, ctx->fec, ctx->fec_size)) {
         FATAL("failed to read %u bytes from '%s': %s\n", ctx->fec_size,
-            filename, strerror(errno));
+            filename.c_str(), strerror(errno));
     }
 
-    TEMP_FAILURE_RETRY(close(fd));
+    close(fd);
 
     uint8_t hash[SHA256_DIGEST_LENGTH];
     SHA256(ctx->fec, ctx->fec_size, hash);
@@ -483,7 +522,7 @@ bool image_ecc_save(image *ctx)
             FATAL("failed to write to output: %s\n", strerror(errno));
         }
 
-        TEMP_FAILURE_RETRY(close(fd));
+        close(fd);
     }
 
     return true;
diff --git a/verity/fec/image.h b/verity/fec/image.h
index f235eb58..f0211fdf 100644
--- a/verity/fec/image.h
+++ b/verity/fec/image.h
@@ -18,6 +18,8 @@
 #define __FEC_H__
 
 #include <utils/Compat.h>
+#include <string>
+#include <vector>
 #include <fec/io.h>
 #include <fec/ecc.h>
 
@@ -77,12 +79,12 @@ struct image_proc_ctx {
     void *rs;
 };
 
-extern bool image_load(const char *filename, image *ctx,
+extern bool image_load(const std::vector<std::string>& filename, image *ctx,
         bool output_needed);
-extern bool image_save(const char *filename, image *ctx);
+extern bool image_save(const std::string& filename, image *ctx);
 
-extern bool image_ecc_new(const char *filename, image *ctx);
-extern bool image_ecc_load(const char *filename, image *ctx);
+extern bool image_ecc_new(const std::string& filename, image *ctx);
+extern bool image_ecc_load(const std::string& filename, image *ctx);
 extern bool image_ecc_save(image *ctx);
 
 extern bool image_process(image_proc_func f, image *ctx);
diff --git a/verity/fec/main.cpp b/verity/fec/main.cpp
index ee844f55..50d807e2 100644
--- a/verity/fec/main.cpp
+++ b/verity/fec/main.cpp
@@ -28,9 +28,8 @@ extern "C" {
 #include <stdbool.h>
 #include <stdlib.h>
 #include <string.h>
+
 #include <android-base/file.h>
-#include <fec/io.h>
-#include <fec/ecc.h>
 #include "image.h"
 
 enum {
@@ -141,7 +140,7 @@ static int print_size(image& ctx)
     return 0;
 }
 
-static int get_start(int mode, const char *filename)
+static int get_start(int mode, const std::string& filename)
 {
     fec::io fh(filename, O_RDONLY, FEC_VERITY_DISABLE);
 
@@ -170,14 +169,14 @@ static int get_start(int mode, const char *filename)
     return 0;
 }
 
-static int encode(image& ctx, const char *inp_filename,
-        const char *fec_filename)
+static int encode(image& ctx, const std::vector<std::string>& inp_filenames,
+        const std::string& fec_filename)
 {
     if (ctx.inplace) {
         FATAL("invalid parameters: inplace can only used when decoding\n");
     }
 
-    if (!image_load(inp_filename, &ctx, false)) {
+    if (!image_load(inp_filenames, &ctx, false)) {
         FATAL("failed to read input\n");
     }
 
@@ -185,8 +184,14 @@ static int encode(image& ctx, const char *inp_filename,
         FATAL("failed to allocate ecc\n");
     }
 
-    INFO("encoding RS(255, %d) for '%s' to '%s'\n", ctx.rs_n, inp_filename,
-        fec_filename);
+    INFO("encoding RS(255, %d) to '%s' for input files:\n", ctx.rs_n,
+        fec_filename.c_str());
+
+    size_t n = 1;
+
+    for (auto fn : inp_filenames) {
+        INFO("\t%zu: '%s'\n", n++, fn.c_str());
+    }
 
     if (ctx.verbose) {
         INFO("\traw fec size: %u\n", ctx.fec_size);
@@ -206,28 +211,31 @@ static int encode(image& ctx, const char *inp_filename,
     return 0;
 }
 
-static int decode(image& ctx, const char *inp_filename,
-        const char *fec_filename, const char *out_filename)
+static int decode(image& ctx, const std::vector<std::string>& inp_filenames,
+        const std::string& fec_filename, std::string& out_filename)
 {
+    const std::string& inp_filename = inp_filenames.front();
+
     if (ctx.inplace && ctx.sparse) {
         FATAL("invalid parameters: inplace cannot be used with sparse "
             "files\n");
     }
 
     if (!image_ecc_load(fec_filename, &ctx) ||
-            !image_load(inp_filename, &ctx, !!out_filename)) {
+            !image_load(inp_filenames, &ctx, !out_filename.empty())) {
         FATAL("failed to read input\n");
     }
 
     if (ctx.inplace) {
-        INFO("correcting '%s' using RS(255, %d) from '%s'\n", inp_filename,
-            ctx.rs_n, fec_filename);
+        INFO("correcting '%s' using RS(255, %d) from '%s'\n",
+            inp_filename.c_str(), ctx.rs_n, fec_filename.c_str());
 
         out_filename = inp_filename;
     } else {
         INFO("decoding '%s' to '%s' using RS(255, %d) from '%s'\n",
-            inp_filename, out_filename ? out_filename : "<none>", ctx.rs_n,
-            fec_filename);
+            inp_filename.c_str(),
+            out_filename.empty() ? out_filename.c_str() : "<none>", ctx.rs_n,
+            fec_filename.c_str());
     }
 
     if (ctx.verbose) {
@@ -246,7 +254,7 @@ static int decode(image& ctx, const char *inp_filename,
         INFO("no errors found\n");
     }
 
-    if (out_filename && !image_save(out_filename, &ctx)) {
+    if (!out_filename.empty() && !image_save(out_filename, &ctx)) {
         FATAL("failed to write output\n");
     }
 
@@ -256,9 +264,9 @@ static int decode(image& ctx, const char *inp_filename,
 
 int main(int argc, char **argv)
 {
-    char *fec_filename = NULL;
-    char *inp_filename = NULL;
-    char *out_filename = NULL;
+    std::string fec_filename;
+    std::string out_filename;
+    std::vector<std::string> inp_filenames;
     int mode = MODE_ENCODE;
     image ctx;
 
@@ -327,14 +335,14 @@ int main(int argc, char **argv)
                 return usage();
             }
             mode = MODE_GETECCSTART;
-            inp_filename = optarg;
+            inp_filenames.push_back(optarg);
             break;
         case 'V':
             if (mode != MODE_ENCODE) {
                 return usage();
             }
             mode = MODE_GETVERITYSTART;
-            inp_filename = optarg;
+            inp_filenames.push_back(optarg);
             break;
         case 'v':
             ctx.verbose = true;
@@ -352,19 +360,29 @@ int main(int argc, char **argv)
     assert(ctx.roots > 0 && ctx.roots < FEC_RSM);
 
     /* check for input / output parameters */
-    if (mode == MODE_ENCODE || mode == MODE_DECODE) {
+    if (mode == MODE_ENCODE) {
+        /* allow multiple input files */
+        for (int i = 0; i < (argc - 1); ++i) {
+            inp_filenames.push_back(argv[i]);
+        }
+
+        if (inp_filenames.empty()) {
+            return usage();
+        }
+
+        /* the last one is the output file */
+        fec_filename = argv[argc - 1];
+    } else if (mode == MODE_DECODE) {
         if (argc < 2 || argc > 3) {
             return usage();
         } else if (argc == 3) {
-            if (mode != MODE_DECODE || ctx.inplace) {
+            if (ctx.inplace) {
                 return usage();
             }
             out_filename = argv[2];
-        } else {
-            out_filename = NULL;
         }
 
-        inp_filename = argv[0];
+        inp_filenames.push_back(argv[0]);
         fec_filename = argv[1];
     }
 
@@ -373,11 +391,11 @@ int main(int argc, char **argv)
         return print_size(ctx);
     case MODE_GETECCSTART:
     case MODE_GETVERITYSTART:
-        return get_start(mode, inp_filename);
+        return get_start(mode, inp_filenames.front());
     case MODE_ENCODE:
-        return encode(ctx, inp_filename, fec_filename);
+        return encode(ctx, inp_filenames, fec_filename);
     case MODE_DECODE:
-        return decode(ctx, inp_filename, fec_filename, out_filename);
+        return decode(ctx, inp_filenames, fec_filename, out_filename);
     default:
         abort();
     }