IOshark portability across 32- and 64-bit, little/big endian devices.

Make IOshark portable across 32-bit and 64-bit devices, as well as
little endian and big endian architectures. This is achieved by
locking down the on-disk format of IOshark workload files. We can now
generate the workload files on a 64-bit host and use those workload
files to run the benchmark on a 32-bit userspace device (such as
fugu). The change also has minor fixes to make IOshark work on taimen
and walleye.

Test: Test across 64- and 32-bit devices. Generate workload files on a
64 bit device and test on 32-bit devices (as well as 64-bit devices).

Change-Id: I700ca5ae0da0ab029d977c906a63f1cd20a78a14
Signed-off-by: Mohan Srinivasan <srmohan@google.com>

12 files changed, 580 insertions, 68 deletions

diff --git a/ioshark/Android.mk b/ioshark/Android.mk
index a647d755..b15d64a0 100644
--- a/ioshark/Android.mk
+++ b/ioshark/Android.mk
@@ -15,7 +15,6 @@
 LOCAL_PATH:= $(call my-dir)
 
 include $(CLEAR_VARS)
-#LOCAL_32_BIT_ONLY = true
 LOCAL_MODULE_HOST_OS := linux
 LOCAL_SRC_FILES := ioshark_bench.c ioshark_bench_subr.c ioshark_bench_mmap.c
 LOCAL_CFLAGS := -g -O2 -Wall  -Werror
@@ -25,7 +24,6 @@ LOCAL_MODULE_PATH := $(TARGET_OUT_OPTIONAL_EXECUTABLES)
 include $(BUILD_EXECUTABLE)
 
 include $(CLEAR_VARS)
-#LOCAL_32_BIT_ONLY = true
 LOCAL_MODULE_HOST_OS := linux
 LOCAL_SRC_FILES := compile_ioshark.c compile_ioshark_subr.c
 LOCAL_CFLAGS := -g -O2 -Wall -Werror -D_GNU_SOURCE
@@ -34,7 +32,6 @@ LOCAL_MODULE_TAGS := debug
 include $(BUILD_HOST_EXECUTABLE)
 
 include $(CLEAR_VARS)
-#LOCAL_32_BIT_ONLY = true
 LOCAL_MODULE_HOST_OS := linux
 LOCAL_SRC_FILES := dump_ioshark_filenames.c
 LOCAL_CFLAGS := -g -O2 -Wall -Werror -D_GNU_SOURCE
@@ -42,3 +39,10 @@ LOCAL_MODULE := dump_ioshark_filenames
 LOCAL_MODULE_TAGS := debug
 include $(BUILD_HOST_EXECUTABLE)
 
+include $(CLEAR_VARS)
+LOCAL_MODULE_HOST_OS := linux
+LOCAL_SRC_FILES := convert_format.c
+LOCAL_CFLAGS := -g -O2 -Wall -Werror -D_GNU_SOURCE
+LOCAL_MODULE := convert_format
+LOCAL_MODULE_TAGS := debug
+include $(BUILD_HOST_EXECUTABLE)
diff --git a/ioshark/README b/ioshark/README
index f71a2600..9a8eddc5 100644
--- a/ioshark/README
+++ b/ioshark/README
@@ -30,3 +30,75 @@ will launch as many threads as there are input files, so 1 thread/file.
 -s : One line summary.
 -q : Don't create the files in read-only partitions like /system and
 /vendor. Instead do reads on those files.
+
+FILE FORMAT :
+-----------
+
+Each IOshark workload file is composed of the following
+
+Header
+File State : Table of File Entries. Each entry describes a file
+File Op : Table of File Operations. One entry describes one operation
+
+Each of the above is described below :
+
+Note : Everything is in Big Endian byte order.
+
+Header {
+       /* IOshark version number */
+       u_int64_t	ioshark_version;
+       /* Total number of files used in this IOshark workload file */
+       u_int64_t	num_files;
+       /* Total number of IO operations in this IOshark workload file */
+       u_int64_t       num_io_operations;
+}
+
+File State {
+       u_int64_t	fileno;
+       u_int64_t	size;
+       u_int64_t 	global_filename_ix;
+}
+
+File Op {
+	/* delta us between previous file op and this */
+	u_int64_t		delta_us;
+#define file_op			file_op_union.file_op_u
+	union {
+		enum file_op		file_op_u;
+		int32_t			enum_size;
+	} file_op_union;
+	u_int64_t		fileno;
+	union {
+		struct lseek_args {
+#define lseek_offset	u.lseek_a.offset
+#define lseek_action	u.lseek_a.action
+			u_int64_t	offset;
+			int32_t		action;
+		} lseek_a;
+		struct prw_args {
+#define prw_offset	u.prw_a.offset
+#define prw_len		u.prw_a.len
+			u_int64_t	offset;
+			u_int64_t	len;
+		} prw_a;
+#define rw_len		u.rw_a.len
+		struct rw_args {
+			u_int64_t	len;
+		} rw_a;
+#define mmap_offset	u.mmap_a.offset
+#define mmap_len	u.mmap_a.len
+#define mmap_prot	u.mmap_a.prot
+		struct mmap_args {
+			u_int64_t	offset;
+			u_int64_t	len;
+			int32_t		prot;
+	} mmap_a;
+#define open_flags	u.open_a.flags
+#define open_mode	u.open_a.mode
+		struct open_args {
+			int32_t		flags;
+			int32_t		mode;
+		} open_a;
+	} u;
+
+}
diff --git a/ioshark/compile-only.sh b/ioshark/compile-only.sh
index 634b4741..7a6b3cd2 100644
--- a/ioshark/compile-only.sh
+++ b/ioshark/compile-only.sh
@@ -110,7 +110,7 @@ merge_compile()
 
 rm -f *.wl
 rm -f parsed*
-rm ioshark_filenames
+rm -f ioshark_filenames
 
 # Pre-process the ftrace file
 prep_fstrace fstrace
diff --git a/ioshark/compile_ioshark.c b/ioshark/compile_ioshark.c
index f35d595a..13755299 100644
--- a/ioshark/compile_ioshark.c
+++ b/ioshark/compile_ioshark.c
@@ -571,18 +571,18 @@ int main(int argc, char **argv)
 		get_tracetype(s, trace_type);
 		if (strcmp(trace_type, "strace") == 0) {
 			get_syscall(s, syscall);
-			disk_file_op->file_op = map_syscall(syscall);
+			disk_file_op->ioshark_io_op = map_syscall(syscall);
 		} else
-			disk_file_op->file_op = map_syscall("ftrace");
-		get_pathname(s, path, disk_file_op->file_op);
+			disk_file_op->ioshark_io_op = map_syscall("ftrace");
+		get_pathname(s, path, disk_file_op->ioshark_io_op);
 		db_node = files_db_add(path);
 		disk_file_op->fileno = files_db_get_fileno(db_node);
-		switch (disk_file_op->file_op) {
+		switch (disk_file_op->ioshark_io_op) {
 		case IOSHARK_LLSEEK:
 		case IOSHARK_LSEEK:
 			get_lseek_offset_action(s,
-					disk_file_op->file_op,
-					&disk_file_op->lseek_offset,
+					disk_file_op->ioshark_io_op,
+					(off_t *)&disk_file_op->lseek_offset,
 					lseek_action_str);
 			disk_file_op->lseek_action =
 				map_lseek_action(lseek_action_str);
@@ -593,7 +593,7 @@ int main(int argc, char **argv)
 		case IOSHARK_PREAD64:
 		case IOSHARK_PWRITE64:
 			get_prw64_offset_len(s,
-					     &disk_file_op->prw_offset,
+					     (off_t *)&disk_file_op->prw_offset,
 					     (u_int64_t *)&disk_file_op->prw_len);
 			files_db_update_size(db_node,
 					     disk_file_op->prw_offset +
@@ -608,8 +608,8 @@ int main(int argc, char **argv)
 		case IOSHARK_MMAP:
 		case IOSHARK_MMAP2:
 			get_mmap_offset_len_prot(s,
-				    &disk_file_op->mmap_prot,
-				    &disk_file_op->mmap_offset,
+			            (int *)&disk_file_op->mmap_prot,
+				    (off_t *)&disk_file_op->mmap_offset,
 				    (u_int64_t *)&disk_file_op->mmap_len);
 			files_db_update_size(db_node,
 				     disk_file_op->mmap_offset +
@@ -629,9 +629,9 @@ int main(int argc, char **argv)
 			break;
 		case IOSHARK_MAPPED_PREAD:
 			/* Convert a mmap'ed read into a PREAD64 */
-			disk_file_op->file_op = IOSHARK_PREAD64;
+			disk_file_op->ioshark_io_op = IOSHARK_PREAD64;
 			get_ftrace_offset_len(s,
-					      &disk_file_op->prw_offset,
+					      (off_t *)&disk_file_op->prw_offset,
 					      (u_int64_t *)&disk_file_op->prw_len);
 			files_db_update_size(db_node,
 					     disk_file_op->prw_offset +
@@ -663,7 +663,7 @@ int main(int argc, char **argv)
 	}
 	header.num_io_operations = num_io_operations;
 	header.num_files = files_db_get_total_obj();
-	if (fwrite(&header, sizeof(struct ioshark_header), 1, fp) != 1) {
+	if (ioshark_write_header(fp, &header) != 1) {
 		fprintf(stderr, "%s Write error trace.outfile\n",
 			progname);
 		exit(EXIT_FAILURE);
@@ -673,8 +673,7 @@ int main(int argc, char **argv)
 		struct in_mem_file_op *temp;
 
 		disk_file_op = &in_mem_file_op_head->disk_file_op;
-		if (fwrite(disk_file_op,
-			   sizeof(struct ioshark_file_operation), 1, fp) != 1) {
+		if (ioshark_write_file_op(fp, disk_file_op) != 1) {
 			fprintf(stderr, "%s Write error trace.outfile\n",
 				progname);
 			exit(EXIT_FAILURE);
diff --git a/ioshark/compile_ioshark.h b/ioshark/compile_ioshark.h
index 9e11bd20..5dd8b7b7 100644
--- a/ioshark/compile_ioshark.h
+++ b/ioshark/compile_ioshark.h
@@ -68,7 +68,6 @@ files_db_get_filename(void *node)
 	return (((struct files_db_s *)node)->filename);
 }
 
-
 void *files_db_create_handle(void);
 void files_db_write_objects(FILE *fp);
 void *files_db_add(char *filename);
@@ -77,5 +76,6 @@ int files_db_get_total_obj(void);
 void init_filename_cache(void);
 void store_filename_cache(void);
 
-
-
+int ioshark_write_header(FILE *fp, struct ioshark_header *header);
+int ioshark_write_file_state(FILE *fp, struct ioshark_file_state *state);
+int ioshark_write_file_op(FILE *fp, struct ioshark_file_operation *file_op);
diff --git a/ioshark/compile_ioshark_subr.c b/ioshark/compile_ioshark_subr.c
index 5cc07c66..9f2028a1 100644
--- a/ioshark/compile_ioshark_subr.c
+++ b/ioshark/compile_ioshark_subr.c
@@ -24,6 +24,7 @@
 #include <sys/errno.h>
 #include "ioshark.h"
 #include "compile_ioshark.h"
+#include <endian.h>
 
 extern char *progname;
 
@@ -48,7 +49,7 @@ files_db_write_objects(FILE *fp)
 			st.size = db_node->size;
 			st.global_filename_ix =
 				db_node->global_filename_ix;
-			if (fwrite(&st, sizeof(st), 1, fp) != 1) {
+			if (ioshark_write_file_state(fp, &st) != 1) {
 				fprintf(stderr,
 					"%s Write error trace.outfile\n",
 					progname);
@@ -213,5 +214,67 @@ store_filename_cache(void)
 	free(filename_cache);
 }
 
+int
+ioshark_write_header(FILE *fp, struct ioshark_header *header)
+{
+	header->version = htobe64(header->version);
+	header->num_files = htobe64(header->num_files);
+	header->num_io_operations = htobe64(header->num_io_operations);
+	return fwrite(header, sizeof(struct ioshark_header), 1, fp);
+}
+
+int
+ioshark_write_file_state(FILE *fp, struct ioshark_file_state *state)
+{
+	state->fileno = htobe64(state->fileno);
+	state->size = htobe64(state->size);
+	state->global_filename_ix = htobe64(state->global_filename_ix);
+	return fwrite(state, sizeof(struct ioshark_file_state), 1, fp);
+}
 
+int
+ioshark_write_file_op(FILE *fp, struct ioshark_file_operation *file_op)
+{
+	enum file_op op = file_op->ioshark_io_op;
 
+	file_op->delta_us = htobe64(file_op->delta_us);
+	file_op->op_union.enum_size = htobe32(file_op->op_union.enum_size);
+	file_op->fileno = htobe64(file_op->fileno);
+	switch (op) {
+	case IOSHARK_LSEEK:
+	case IOSHARK_LLSEEK:
+		file_op->lseek_offset = htobe64(file_op->lseek_offset);
+		file_op->lseek_action = htobe32(file_op->lseek_action);
+		break;
+	case IOSHARK_PREAD64:
+	case IOSHARK_PWRITE64:
+		file_op->prw_offset = htobe64(file_op->prw_offset);
+		file_op->prw_len = htobe64(file_op->prw_len);
+		break;
+	case IOSHARK_READ:
+	case IOSHARK_WRITE:
+		file_op->rw_len = htobe64(file_op->rw_len);
+		break;
+	case IOSHARK_MMAP:
+	case IOSHARK_MMAP2:
+		file_op->mmap_offset = htobe64(file_op->mmap_offset);
+		file_op->mmap_len = htobe64(file_op->mmap_len);
+		file_op->mmap_prot = htobe32(file_op->mmap_prot);
+		break;
+	case IOSHARK_OPEN:
+		file_op->open_flags = htobe32(file_op->open_flags);
+		file_op->open_mode = htobe32(file_op->open_mode);
+		break;
+	case IOSHARK_FSYNC:
+	case IOSHARK_FDATASYNC:
+		break;
+	case IOSHARK_CLOSE:
+		break;
+	default:
+		fprintf(stderr, "%s: unknown FILE_OP %d\n",
+			progname, op);
+		exit(EXIT_FAILURE);
+		break;
+	}
+	return fwrite(file_op, sizeof(struct ioshark_file_operation), 1, fp);
+}
diff --git a/ioshark/convert_format.c b/ioshark/convert_format.c
new file mode 100644
index 00000000..3436ca77
--- /dev/null
+++ b/ioshark/convert_format.c
@@ -0,0 +1,283 @@
+/*
+ * Copyright (C) 2016 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <stdio.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <signal.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/errno.h>
+#include <fcntl.h>
+#include <ctype.h>
+#include <endian.h>
+#include "ioshark.h"
+
+char *progname;
+
+struct ioshark_header_old {
+	int	num_files;
+	int	num_io_operations;
+};
+
+struct ioshark_file_operation_old {
+	/* delta us between previous file op and this */
+	u_int64_t		delta_us;
+	enum file_op		file_op;
+	int			fileno;
+	union {
+		struct lseek_args_old {
+#define lseek_offset_old	u.lseek_a.offset
+#define lseek_action_old	u.lseek_a.action
+			off_t	offset;
+			int action;
+		} lseek_a;
+		struct prw_args_old {
+#define prw_offset_old		u.prw_a.offset
+#define prw_len_old		u.prw_a.len
+			off_t	offset;
+			size_t	len;
+		} prw_a;
+#define rw_len_old		u.rw_a.len
+		struct rw_args_old {
+			size_t	len;
+		} rw_a;
+#define mmap_offset_old		u.mmap_a.offset
+#define mmap_len_old		u.mmap_a.len
+#define mmap_prot_old		u.mmap_a.prot
+		struct mmap_args_old {
+			off_t	offset;
+			size_t	len;
+			int	prot;
+	} mmap_a;
+#define open_flags_old		u.open_a.flags
+#define open_mode_old		u.open_a.mode
+		struct open_args_old {
+			int	flags;
+			mode_t	mode;
+		} open_a;
+	} u;
+};
+
+struct ioshark_file_state_old {
+	int	fileno;	/* 1..num_files, with files name ioshark.<fileno> */
+	size_t	size;
+	int	global_filename_ix;
+};
+
+void usage(void)
+{
+	fprintf(stderr, "%s in_file out_file\n", progname);
+}
+
+int main(int argc, char **argv)
+{
+	FILE *old_fp, *new_fp;
+	char *infile, *outfile;
+	struct ioshark_header new_header;
+	struct ioshark_file_operation new_disk_file_op;
+	struct ioshark_header_old old_header;
+	struct ioshark_file_operation_old old_disk_file_op;
+	struct ioshark_file_state new_file_state;
+	struct ioshark_file_state_old old_file_state;
+	struct stat st;
+	int i;
+	u_int64_t aggr_old_file_size = 0;
+	u_int64_t aggr_new_file_size = 0;
+
+	progname = argv[0];
+	if (argc != 3) {
+		usage();
+		exit(EXIT_FAILURE);
+	}
+	infile = argv[1];
+	outfile = argv[2];
+	if (stat(infile, &st) < 0) {
+		fprintf(stderr, "%s Can't stat %s\n",
+			progname, infile);
+		exit(EXIT_FAILURE);
+	}
+	if (st.st_size == 0) {
+		fprintf(stderr, "%s Empty file %s\n",
+			progname, infile);
+		exit(EXIT_FAILURE);
+	}
+	old_fp = fopen(infile, "r");
+	if (old_fp == NULL) {
+		fprintf(stderr, "%s Can't open %s\n",
+			progname, infile);
+		exit(EXIT_FAILURE);
+	}
+	new_fp = fopen(outfile, "w+");
+	if (new_fp == NULL) {
+		fprintf(stderr, "%s Can't open outfile\n",
+			progname);
+		exit(EXIT_FAILURE);
+	}
+	/* Convert header */
+	if (fread(&old_header, sizeof(struct ioshark_header_old),
+		  1, old_fp) != 1) {
+		fprintf(stderr,
+			"%s Read error Header\n",
+			progname);
+		exit(EXIT_FAILURE);
+	}
+	new_header.version = IOSHARK_VERSION;
+	new_header.num_files = old_header.num_files;
+	new_header.num_io_operations = old_header.num_io_operations;
+	new_header.version = htobe64(new_header.version);
+	new_header.num_files = htobe64(new_header.num_files);
+	new_header.num_io_operations =
+		htobe64(new_header.num_io_operations);
+	if (fwrite(&new_header, sizeof(struct ioshark_header),
+		   1, new_fp) != 1) {
+		fprintf(stderr,
+			"%s Write error Header\n",
+			progname);
+		exit(EXIT_FAILURE);
+	}
+	for (i = 0 ; i < old_header.num_files ; i++) {
+		if (fread(&old_file_state,
+			  sizeof(struct ioshark_file_state_old),
+			  1, old_fp) != 1) {
+			fprintf(stderr,
+				"%s Read error file state\n",
+				progname);
+			exit(EXIT_FAILURE);
+		}
+		new_file_state.fileno = old_file_state.fileno;
+		new_file_state.size = old_file_state.size;
+		aggr_old_file_size += old_file_state.size;
+		new_file_state.global_filename_ix =
+			old_file_state.global_filename_ix;
+		new_file_state.fileno = htobe64(new_file_state.fileno);
+		new_file_state.size = htobe64(new_file_state.size);
+		aggr_new_file_size += be64toh(new_file_state.size);
+		new_file_state.global_filename_ix =
+			htobe64(new_file_state.global_filename_ix);
+		if (fwrite(&new_file_state,
+			   sizeof(struct ioshark_file_state), 1, new_fp) != 1) {
+			fprintf(stderr,
+				"%s Write error file state\n",
+				progname);
+			exit(EXIT_FAILURE);
+		}
+	}
+	if (aggr_new_file_size != aggr_old_file_size) {
+		fprintf(stderr,
+			"%s Aggr file size mismath %lu != %lu\n",
+			progname, aggr_new_file_size, aggr_old_file_size);
+		exit(EXIT_FAILURE);
+	}
+
+	for (i = 0 ; i < old_header.num_io_operations ; i++) {
+		enum file_op op;
+
+		if (fread(&old_disk_file_op,
+			  sizeof(struct ioshark_file_operation_old),
+			  1, old_fp) != 1)  {
+			fprintf(stderr,
+				"%s Read error file op\n",
+				progname);
+			exit(EXIT_FAILURE);
+		}
+		op = old_disk_file_op.file_op;
+		new_disk_file_op.delta_us = old_disk_file_op.delta_us;
+		new_disk_file_op.delta_us =
+			htobe64(new_disk_file_op.delta_us);
+		new_disk_file_op.ioshark_io_op = op;
+		new_disk_file_op.op_union.enum_size =
+			htobe32(new_disk_file_op.op_union.enum_size);
+		new_disk_file_op.fileno = old_disk_file_op.fileno;
+		new_disk_file_op.fileno = htobe64(new_disk_file_op.fileno);
+		switch (op) {
+		case IOSHARK_LSEEK:
+		case IOSHARK_LLSEEK:
+			new_disk_file_op.lseek_offset =
+				old_disk_file_op.lseek_offset_old;
+			new_disk_file_op.lseek_action =
+				old_disk_file_op.lseek_action_old;
+			new_disk_file_op.lseek_offset =
+				htobe64(new_disk_file_op.lseek_offset);
+			new_disk_file_op.lseek_action =
+				htobe32(new_disk_file_op.lseek_action);
+			break;
+		case IOSHARK_PREAD64:
+		case IOSHARK_PWRITE64:
+			new_disk_file_op.prw_offset =
+				old_disk_file_op.prw_offset_old;
+			new_disk_file_op.prw_len =
+				old_disk_file_op.prw_len_old;
+			new_disk_file_op.prw_offset =
+				htobe64(new_disk_file_op.prw_offset);
+			new_disk_file_op.prw_len =
+				htobe64(new_disk_file_op.prw_len);
+			break;
+		case IOSHARK_READ:
+		case IOSHARK_WRITE:
+			new_disk_file_op.rw_len =
+				old_disk_file_op.rw_len_old;
+			new_disk_file_op.rw_len =
+				htobe64(new_disk_file_op.rw_len);
+			break;
+		case IOSHARK_MMAP:
+		case IOSHARK_MMAP2:
+			new_disk_file_op.mmap_offset =
+				old_disk_file_op.mmap_offset_old;
+			new_disk_file_op.mmap_len =
+				old_disk_file_op.mmap_len_old;
+			new_disk_file_op.mmap_prot =
+				old_disk_file_op.mmap_prot;
+			new_disk_file_op.mmap_offset =
+				htobe64(new_disk_file_op.mmap_offset);
+			new_disk_file_op.mmap_len =
+				htobe64(new_disk_file_op.mmap_len);
+			new_disk_file_op.mmap_prot =
+				htobe32(new_disk_file_op.mmap_prot);
+			break;
+		case IOSHARK_OPEN:
+			new_disk_file_op.open_flags =
+				old_disk_file_op.open_flags_old;
+			new_disk_file_op.open_mode =
+				old_disk_file_op.open_mode_old;
+			new_disk_file_op.open_flags =
+				htobe32(new_disk_file_op.open_flags);
+			new_disk_file_op.open_mode =
+				htobe32(new_disk_file_op.open_mode);
+			break;
+		case IOSHARK_FSYNC:
+		case IOSHARK_FDATASYNC:
+			break;
+		case IOSHARK_CLOSE:
+			break;
+		default:
+			fprintf(stderr, "%s: unknown FILE_OP %d\n",
+				progname, op);
+			exit(EXIT_FAILURE);
+			break;
+		}
+		if (fwrite(&new_disk_file_op,
+			   sizeof(struct ioshark_file_operation),
+			   1, new_fp) != 1) {
+			fprintf(stderr,
+				"%s Write error file op\n",
+				progname);
+			exit(EXIT_FAILURE);
+		}
+	}
+}
diff --git a/ioshark/ioshark.h b/ioshark/ioshark.h
index fd9caca4..33e9f98c 100644
--- a/ioshark/ioshark.h
+++ b/ioshark/ioshark.h
@@ -21,14 +21,18 @@
  * 3) Table of IO operations to perform on the files
  */
 
+#pragma pack(1)
+
 /*
  * The parsed workload file starts off with the header, which
  * contains the count of the total # of files that are operated on.
  * and the total number of IO operations.
  */
 struct ioshark_header {
-	int	num_files;
-	int	num_io_operations;
+#define IOSHARK_VERSION			2
+	u_int64_t	version;
+	u_int64_t	num_files;
+	u_int64_t	num_io_operations;
 };
 
 /*
@@ -38,9 +42,9 @@ struct ioshark_header {
  * Before the tests starts, these files are pre-created.
  */
 struct ioshark_file_state {
-	int	fileno;	/* 1..num_files, with files name ioshark.<fileno> */
-	size_t	size;
-	int	global_filename_ix;
+	u_int64_t	fileno;	/* 1..num_files, with files name ioshark.<fileno> */
+	u_int64_t	size;
+	u_int64_t	global_filename_ix;
 };
 
 enum file_op {
@@ -66,44 +70,48 @@ enum file_op {
 #define IOSHARK_PROT_WRITE	0x2
 
 /*
- * Next we have the table of IO operatiosn to perform. Each
+ * Next we have the table of IO operations to perform. Each
  * IO operation is described by this entry.
  */
 struct ioshark_file_operation {
 	/* delta us between previous file op and this */
-	u_int64_t		delta_us;
-	enum file_op		file_op;
-	int			fileno;
+	u_int64_t			delta_us;
+#define ioshark_io_op			op_union.file_op_u
+	union {
+		enum file_op		file_op_u;
+		u_int32_t		enum_size;
+	} op_union;
+	u_int64_t			fileno;
 	union {
 		struct lseek_args {
 #define lseek_offset	u.lseek_a.offset
 #define lseek_action	u.lseek_a.action
-			off_t	offset;
-			int action;
+			u_int64_t	offset;
+			u_int32_t	action;
 		} lseek_a;
 		struct prw_args {
 #define prw_offset	u.prw_a.offset
 #define prw_len		u.prw_a.len
-			off_t	offset;
-			size_t	len;
+			u_int64_t	offset;
+			u_int64_t	len;
 		} prw_a;
 #define rw_len		u.rw_a.len
 		struct rw_args {
-			size_t	len;
+			u_int64_t	len;
 		} rw_a;
 #define mmap_offset	u.mmap_a.offset
 #define mmap_len	u.mmap_a.len
 #define mmap_prot	u.mmap_a.prot
 		struct mmap_args {
-			off_t	offset;
-			size_t	len;
-			int	prot;
+			u_int64_t	offset;
+			u_int64_t	len;
+			u_int32_t	prot;
 	} mmap_a;
 #define open_flags	u.open_a.flags
 #define open_mode	u.open_a.mode
 		struct open_args {
-			int	flags;
-			mode_t	mode;
+			u_int32_t	flags;
+			u_int32_t	mode;
 		} open_a;
 	} u;
 };
@@ -117,3 +125,5 @@ struct ioshark_filename_struct
 {
 	char path[MAX_IOSHARK_PATHLEN];
 };
+
+#pragma options align=reset
diff --git a/ioshark/ioshark_bench.c b/ioshark/ioshark_bench.c
index 7593394e..80600425 100644
--- a/ioshark/ioshark_bench.c
+++ b/ioshark/ioshark_bench.c
@@ -29,6 +29,7 @@
 #include <pthread.h>
 #include <sys/statfs.h>
 #include <sys/resource.h>
+#include <inttypes.h>
 #include "ioshark.h"
 #define IOSHARK_MAIN
 #include "ioshark_bench.h"
@@ -173,8 +174,7 @@ create_files(struct thread_state_s *state)
 
 	memset(&rw_bytes, 0, sizeof(struct rw_bytes_s));
 	for (i = 0 ; i < state->num_files ; i++) {
-		if (fread(&file_state, sizeof(struct ioshark_file_state),
-			  1, state->fp) != 1) {
+		if (ioshark_read_file_state(state->fp, &file_state) != 1) {
 			fprintf(stderr, "%s read error tracefile\n",
 				progname);
 			exit(EXIT_FAILURE);
@@ -190,7 +190,7 @@ create_files(struct thread_state_s *state)
 			assert(filename != NULL);
 		if (quick_mode == 0 ||
 		    is_readonly_mount(filename, file_state.size) == 0) {
-			sprintf(path, "file.%d.%d",
+			sprintf(path, "file.%d.%"PRIu64"",
 				(int)(state - thread_state),
 				file_state.fileno);
 			create_file(path, file_state.size,
@@ -216,9 +216,9 @@ do_one_io(void *db_node,
 	  struct rw_bytes_s *rw_bytes,
 	  char **bufp, int *buflen)
 {
-	assert(file_op->file_op < IOSHARK_MAX_FILE_OP);
-	op_counts[file_op->file_op]++;
-	switch (file_op->file_op) {
+	assert(file_op->ioshark_io_op < IOSHARK_MAX_FILE_OP);
+	op_counts[file_op->ioshark_io_op]++;
+	switch (file_op->ioshark_io_op) {
 	int ret;
 	char *p;
 	int fd;
@@ -230,7 +230,7 @@ do_one_io(void *db_node,
 			    file_op->lseek_action);
 		if (ret < 0) {
 			fprintf(stderr,
-				"%s: lseek(%s %lu %d) returned error %d\n",
+				"%s: lseek(%s %"PRIu64" %d) returned error %d\n",
 				progname, files_db_get_filename(db_node),
 				file_op->lseek_offset,
 				file_op->lseek_action, errno);
@@ -244,7 +244,7 @@ do_one_io(void *db_node,
 		rw_bytes->bytes_read += file_op->prw_len;
 		if (ret < 0) {
 			fprintf(stderr,
-				"%s: pread(%s %zu %lu) error %d\n",
+				"%s: pread(%s %"PRIu64" %"PRIu64") error %d\n",
 				progname,
 				files_db_get_filename(db_node),
 				file_op->prw_len,
@@ -259,7 +259,7 @@ do_one_io(void *db_node,
 		rw_bytes->bytes_written += file_op->prw_len;
 		if (ret < 0) {
 			fprintf(stderr,
-				"%s: pwrite(%s %zu %lu) error %d\n",
+				"%s: pwrite(%s %"PRIu64" %"PRIu64") error %d\n",
 				progname,
 				files_db_get_filename(db_node),
 				file_op->prw_len,
@@ -274,7 +274,7 @@ do_one_io(void *db_node,
 		rw_bytes->bytes_read += file_op->rw_len;
 		if (ret < 0) {
 			fprintf(stderr,
-				"%s: read(%s %zu) error %d\n",
+				"%s: read(%s %"PRIu64") error %d\n",
 				progname,
 				files_db_get_filename(db_node),
 				file_op->rw_len,
@@ -289,7 +289,7 @@ do_one_io(void *db_node,
 		rw_bytes->bytes_written += file_op->rw_len;
 		if (ret < 0) {
 			fprintf(stderr,
-				"%s: write(%s %zu) error %d\n",
+				"%s: write(%s %"PRIu64") error %d\n",
 				progname,
 				files_db_get_filename(db_node),
 				file_op->rw_len,
@@ -350,7 +350,7 @@ do_one_io(void *db_node,
 		break;
 	case IOSHARK_FSYNC:
 	case IOSHARK_FDATASYNC:
-		if (file_op->file_op == IOSHARK_FSYNC) {
+		if (file_op->ioshark_io_op == IOSHARK_FSYNC) {
 			ret = fsync(files_db_get_fd(db_node));
 			if (ret < 0) {
 				fprintf(stderr,
@@ -385,7 +385,7 @@ do_one_io(void *db_node,
 		break;
 	default:
 		fprintf(stderr, "%s: unknown FILE_OP %d\n",
-			progname, file_op->file_op);
+			progname, file_op->ioshark_io_op);
 		exit(EXIT_FAILURE);
 		break;
 	}
@@ -406,7 +406,7 @@ do_io(struct thread_state_s *state)
 	struct rw_bytes_s rw_bytes;
 
 	rewind(state->fp);
-	if (fread(&header, sizeof(struct ioshark_header), 1, state->fp) != 1) {
+	if (ioshark_read_header(state->fp, &header) != 1) {
 		fprintf(stderr, "%s read error %s\n",
 			progname, state->filename);
 		exit(EXIT_FAILURE);
@@ -424,9 +424,8 @@ do_io(struct thread_state_s *state)
 	/*
 	 * Loop over all the IOs, and launch each
 	 */
-	for (i = 0 ; i < header.num_io_operations ; i++) {
-		if (fread(&file_op, sizeof(struct ioshark_file_operation),
-			  1, state->fp) != 1) {
+	for (i = 0 ; i < (int)header.num_io_operations ; i++) {
+		if (ioshark_read_file_op(state->fp, &file_op) != 1) {
 			fprintf(stderr, "%s read error trace.outfile\n",
 				progname);
 			exit(EXIT_FAILURE);
@@ -442,11 +441,14 @@ do_io(struct thread_state_s *state)
 						   file_op.fileno);
 		if (db_node == NULL) {
 			fprintf(stderr,
-				"%s Can't lookup fileno %d, fatal error\n",
+				"%s Can't lookup fileno %"PRIu64", fatal error\n",
 				progname, file_op.fileno);
+			fprintf(stderr,
+				"%s state filename %s, i %d\n",
+				progname, state->filename, i);
 			exit(EXIT_FAILURE);
 		}
-		if (file_op.file_op != IOSHARK_OPEN &&
+		if (file_op.ioshark_io_op != IOSHARK_OPEN &&
 		    files_db_get_fd(db_node) == -1) {
 			int openflags;
 
@@ -498,7 +500,7 @@ do_create(struct thread_state_s *state)
 {
 	struct ioshark_header header;
 
-	if (fread(&header, sizeof(struct ioshark_header), 1, state->fp) != 1) {
+	if (ioshark_read_header(state->fp, &header) != 1) {
 		fprintf(stderr, "%s read error %s\n",
 			progname, state->filename);
 		exit(EXIT_FAILURE);
@@ -548,15 +550,13 @@ get_start_end(int *start_ix)
 				progname, infile);
 			exit(EXIT_FAILURE);
 		}
-		if (fread(&header, sizeof(struct ioshark_header),
-			  1, fp) != 1) {
+		if (ioshark_read_header(fp, &header) != 1) {
 			fprintf(stderr, "%s read error %s\n",
 				progname, infile);
 			exit(EXIT_FAILURE);
 		}
-		for (j = 0 ; j < header.num_files ; j++) {
-			if (fread(&file_state, sizeof(struct ioshark_file_state),
-				  1, fp) != 1) {
+		for (j = 0 ; j < (int)header.num_files ; j++) {
+			if (ioshark_read_file_state(fp, &file_state) != 1) {
 				fprintf(stderr, "%s read error tracefile\n",
 					progname);
 				exit(EXIT_FAILURE);
diff --git a/ioshark/ioshark_bench.h b/ioshark/ioshark_bench.h
index d0d09626..8ae8597d 100644
--- a/ioshark/ioshark_bench.h
+++ b/ioshark/ioshark_bench.h
@@ -152,3 +152,7 @@ char *get_ro_filename(int ix);
 void init_filename_cache(void);
 void free_filename_cache(void);
 int is_readonly_mount(char *filename, size_t size);
+
+int ioshark_read_header(FILE *fp, struct ioshark_header *header);
+int ioshark_read_file_state(FILE *fp, struct ioshark_file_state *state);
+int ioshark_read_file_op(FILE *fp, struct ioshark_file_operation *file_op);
diff --git a/ioshark/ioshark_bench_subr.c b/ioshark/ioshark_bench_subr.c
index e8e93c9c..e7ca3f90 100644
--- a/ioshark/ioshark_bench_subr.c
+++ b/ioshark/ioshark_bench_subr.c
@@ -29,8 +29,11 @@
 #include <sys/vfs.h>
 #include <sys/statvfs.h>
 #include <sys/mman.h>
+#include <inttypes.h>
 #include "ioshark.h"
 #include "ioshark_bench.h"
+#define _BSD_SOURCE
+#include <endian.h>
 
 extern char *progname;
 extern int verbose, summary_mode;
@@ -399,10 +402,15 @@ get_blockdev_name(char *bdev)
 	/* strncmp needed because of the trailing '\n' */
 	if (strncmp(dev_name, "bullhead", strlen("bullhead")) == 0 ||
 	    strncmp(dev_name, "angler", strlen("angler")) == 0 ||
-	    strncmp(dev_name, "shamu", strlen("shamu")) == 0) {
+	    strncmp(dev_name, "shamu", strlen("shamu")) == 0 ||
+	    strncmp(dev_name, "aosp_gobo", strlen("aosp_gobo")) == 0 ||
+	    strncmp(dev_name, "full_k37_y33_gms", strlen("full_k37_y33_gms")) == 0 ||
+	    strncmp(dev_name, "fugu", strlen("fugu")) == 0) {
 		strcpy(bdev, "mmcblk0");
 	} else if (strncmp(dev_name, "marlin", strlen("marlin")) == 0 ||
-		   strncmp(dev_name, "sailfish", strlen("sailfish")) == 0) {
+		   strncmp(dev_name, "sailfish", strlen("sailfish")) == 0 ||
+		   strncmp(dev_name, "taimen", strlen("taimen")) == 0 ||
+		   strncmp(dev_name, "walleye", strlen("walleye")) == 0) {
 		strcpy(bdev, "sda");
 	} else if (blockdev_name != NULL) {
 		strcpy(bdev, blockdev_name);
@@ -646,3 +654,72 @@ is_readonly_mount(char *filename, size_t size)
 	else
 		return 1;
 }
+
+int
+ioshark_read_header(FILE *fp, struct ioshark_header *header)
+{
+	if (fread(header, sizeof(struct ioshark_header), 1, fp) != 1)
+		return -1;
+	header->version = be64toh(header->version);
+	header->num_files = be64toh(header->num_files);
+	header->num_io_operations = be64toh(header->num_io_operations);
+	return 1;
+}
+
+int
+ioshark_read_file_state(FILE *fp, struct ioshark_file_state *state)
+{
+	if (fread(state, sizeof(struct ioshark_file_state), 1, fp) != 1)
+		return -1;
+	state->fileno = be64toh(state->fileno);
+	state->size = be64toh(state->size);
+	state->global_filename_ix = be64toh(state->global_filename_ix);
+	return 1;
+}
+
+int
+ioshark_read_file_op(FILE *fp, struct ioshark_file_operation *file_op)
+{
+	if (fread(file_op, sizeof(struct ioshark_file_operation), 1, fp) != 1)
+		return -1;
+	file_op->delta_us = be64toh(file_op->delta_us);
+	file_op->op_union.enum_size = be32toh(file_op->op_union.enum_size);
+	file_op->fileno = be64toh(file_op->fileno);
+	switch (file_op->ioshark_io_op) {
+	case IOSHARK_LSEEK:
+	case IOSHARK_LLSEEK:
+		file_op->lseek_offset = be64toh(file_op->lseek_offset);
+		file_op->lseek_action = be32toh(file_op->lseek_action);
+		break;
+	case IOSHARK_PREAD64:
+	case IOSHARK_PWRITE64:
+		file_op->prw_offset = be64toh(file_op->prw_offset);
+		file_op->prw_len = be64toh(file_op->prw_len);
+		break;
+	case IOSHARK_READ:
+	case IOSHARK_WRITE:
+		file_op->rw_len = be64toh(file_op->rw_len);
+		break;
+	case IOSHARK_MMAP:
+	case IOSHARK_MMAP2:
+		file_op->mmap_offset = be64toh(file_op->mmap_offset);
+		file_op->mmap_len = be64toh(file_op->mmap_len);
+		file_op->mmap_prot = be32toh(file_op->mmap_prot);
+		break;
+	case IOSHARK_OPEN:
+		file_op->open_flags = be32toh(file_op->open_flags);
+		file_op->open_mode = be32toh(file_op->open_mode);
+		break;
+	case IOSHARK_FSYNC:
+	case IOSHARK_FDATASYNC:
+		break;
+	case IOSHARK_CLOSE:
+		break;
+	default:
+		fprintf(stderr, "%s: unknown FILE_OP %d\n",
+			progname, file_op->ioshark_io_op);
+		exit(EXIT_FAILURE);
+		break;
+	}
+	return 1;
+}
diff --git a/ioshark/wl.tar b/ioshark/wl.tar
index a004ad12..f44584b4 100644
--- a/ioshark/wl.tar
+++ b/ioshark/wl.tar