diff options
| author | Bernhard Rosenkränzer <Bernhard.Rosenkranzer@linaro.org> | 2017-11-28 20:31:42 +0100 |
|---|---|---|
| committer | Bernhard Rosenkränzer <Bernhard.Rosenkranzer@linaro.org> | 2017-11-28 20:58:44 +0100 |
| commit | 13249375a548006124dc82f35c59b8e480119d29 (patch) | |
| tree | dcb2fd5690bef5fdb5fea324044b769427f83faf | |
| parent | cc4a1183327e58f4a703dd23b49b4c659a2b2f51 (diff) | |
| parent | 9f50b4106bd1d6fa1c325900d1fb286832ccc5e8 (diff) | |
| download | fio-linaro-upstream-3.2.tar.gz | |
Merge update to 3.2linaro-upstream-3.2
Change-Id: Icbce3fd0066660444c3cb577e6e47503b76908f9
79 files changed, 1121 insertions, 378 deletions
@@ -77,6 +77,7 @@ LOCAL_CFLAGS += -DFIO_VERSION="\"fio-3.1\"" \ -DCONFIG_IPV6 \ -DCONFIG_LINUX_FALLOCATE \ -DCONFIG_LITTLE_ENDIAN \ + -DCONFIG_HAVE_MKDIR_TWO \ -DCONFIG_POSIX_FALLOCATE \ -DCONFIG_RLIMIT_MEMLOCK \ -DCONFIG_RUSAGE_THREAD \ @@ -1,12 +1,12 @@ - GNU GENERAL PUBLIC LICENSE - Version 2, June 1991 + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 - Copyright (C) 1989, 1991 Free Software Foundation, Inc. - 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Copyright (C) 1989, 1991 Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. - Preamble + Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public @@ -15,7 +15,7 @@ software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by -the GNU Library General Public License instead.) You can apply it to +the GNU Lesser General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not @@ -55,8 +55,8 @@ patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. - - GNU GENERAL PUBLIC LICENSE + + GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains @@ -110,7 +110,7 @@ above, provided that you also meet all of these conditions: License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) - + These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in @@ -168,7 +168,7 @@ access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. - + 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is @@ -225,7 +225,7 @@ impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. - + 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License @@ -255,7 +255,7 @@ make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. - NO WARRANTY + NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN @@ -277,9 +277,9 @@ YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - END OF TERMS AND CONDITIONS - - How to Apply These Terms to Your New Programs + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it @@ -303,10 +303,9 @@ the "copyright" line and a pointer to where the full notice is found. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - + You should have received a copy of the GNU General Public License along + with this program; if not, write to the Free Software Foundation, Inc., + 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. Also add information on how to contact you by electronic and paper mail. @@ -336,5 +335,5 @@ necessary. Here is a sample; alter the names: This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General +library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. diff --git a/FIO-VERSION-GEN b/FIO-VERSION-GEN index 8c075cbd..22f44044 100755 --- a/FIO-VERSION-GEN +++ b/FIO-VERSION-GEN @@ -1,7 +1,7 @@ #!/bin/sh GVF=FIO-VERSION-FILE -DEF_VER=fio-3.1 +DEF_VER=fio-3.2 LF=' ' @@ -217,6 +217,9 @@ Command line options .. option:: --max-jobs=nr Set the maximum number of threads/processes to support to `nr`. + NOTE: On Linux, it may be necessary to increase the shared-memory + limit (:file:`/proc/sys/kernel/shmmax`) if fio runs into errors while + creating jobs. .. option:: --server=args @@ -230,7 +233,7 @@ Command line options .. option:: --client=hostname Instead of running the jobs locally, send and run them on the given `hostname` - or set of `hostname`s. See `Client/Server`_ section. + or set of `hostname`\s. See `Client/Server`_ section. .. option:: --remote-config=file @@ -792,6 +795,13 @@ Target file/device named :file:`testfiles.4`. The default of :file:`$jobname.$jobnum.$filenum` will be used if no other format specifier is given. + If you specify a path then the directories will be created up to the + main directory for the file. So for example if you specify + ``filename_format=a/b/c/$jobnum`` then the directories a/b/c will be + created before the file setup part of the job. If you specify + :option:`directory` then the path will be relative that directory, + otherwise it is treated as the absolute path. + .. option:: unique_filename=bool To avoid collisions between networked clients, fio defaults to prefixing any @@ -1118,13 +1128,20 @@ I/O type .. option:: offset=int Start I/O at the provided offset in the file, given as either a fixed size in - bytes or a percentage. If a percentage is given, the next ``blockalign``-ed - offset will be used. Data before the given offset will not be touched. This + bytes or a percentage. If a percentage is given, the generated offset will be + aligned to the minimum ``blocksize`` or to the value of ``offset_align`` if + provided. Data before the given offset will not be touched. This effectively caps the file size at `real_size - offset`. Can be combined with :option:`size` to constrain the start and end range of the I/O workload. A percentage can be specified by a number between 1 and 100 followed by '%', for example, ``offset=20%`` to specify 20%. +.. option:: offset_align=int + + If set to non-zero value, the byte offset generated by a percentage ``offset`` + is aligned upwards to this value. Defaults to 0 meaning that a percentage + offset is aligned to the minimum block size. + .. option:: offset_increment=int If this is provided, then the real offset becomes `offset + offset_increment @@ -1436,8 +1453,8 @@ Buffers and memory mix of random data and a fixed pattern. The fixed pattern is either zeros, or the pattern specified by :option:`buffer_pattern`. If the pattern option is used, it might skew the compression ratio slightly. Note that this is per - block size unit, for file/disk wide compression level that matches this - setting, you'll also want to set :option:`refill_buffers`. + block size unit, see :option:`buffer_compress_chunk` for setting a finer + granularity of compression regions. .. option:: buffer_compress_chunk=int @@ -1446,7 +1463,8 @@ Buffers and memory will provide :option:`buffer_compress_percentage` of blocksize random data, followed by the remaining zeroed. With this set to some chunk size smaller than the block size, fio can alternate random and zeroed data throughout the - I/O buffer. + I/O buffer. This is particularly useful when bigger block sizes are used + for a job. Defaults to 512. .. option:: buffer_pattern=str @@ -1705,7 +1723,7 @@ I/O engine Doesn't transfer any data, but burns CPU cycles according to the :option:`cpuload` and :option:`cpuchunks` options. Setting :option:`cpuload`\=85 will cause that job to do nothing but burn 85% - of the CPU. In case of SMP machines, use :option:`numjobs`=<nr_of_cpu> + of the CPU. In case of SMP machines, use :option:`numjobs`\=<nr_of_cpu> to get desired CPU usage, as the cpuload only loads a single CPU at the desired rate. A job never finishes unless there is at least one non-cpuio job. @@ -1797,6 +1815,10 @@ I/O engine absolute or relative. See :file:`engines/skeleton_external.c` for details of writing an external I/O engine. + **filecreate** + Simply create the files and do no I/O to them. You still need to + set `filesize` so that all the accounting still occurs, but no + actual I/O will be done other than creating the file. I/O engine specific parameters ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ @@ -2039,7 +2061,7 @@ I/O depth changing data and the overlapping region has a non-zero size. Setting ``serialize_overlap`` tells fio to avoid provoking this behavior by explicitly serializing in-flight I/Os that have a non-zero overlap. Note that setting - this option can reduce both performance and the `:option:iodepth` achieved. + this option can reduce both performance and the :option:`iodepth` achieved. Additionally this option does not work when :option:`io_submit_mode` is set to offload. Default: false. @@ -2701,47 +2723,46 @@ Measurements and reporting .. option:: write_bw_log=str If given, write a bandwidth log for this job. Can be used to store data of - the bandwidth of the jobs in their lifetime. The included - :command:`fio_generate_plots` script uses :command:`gnuplot` to turn these - text files into nice graphs. See :option:`write_lat_log` for behavior of - given filename. For this option, the postfix is :file:`_bw.x.log`, where `x` - is the index of the job (`1..N`, where `N` is the number of jobs). If - :option:`per_job_logs` is false, then the filename will not include the job - index. See `Log File Formats`_. + the bandwidth of the jobs in their lifetime. -.. option:: write_lat_log=str + If no str argument is given, the default filename of + :file:`jobname_type.x.log` is used. Even when the argument is given, fio + will still append the type of log. So if one specifies:: - Same as :option:`write_bw_log`, except that this option stores I/O - submission, completion, and total latencies instead. If no filename is given - with this option, the default filename of :file:`jobname_type.log` is - used. Even if the filename is given, fio will still append the type of - log. So if one specifies:: + write_bw_log=foo - write_lat_log=foo + The actual log name will be :file:`foo_bw.x.log` where `x` is the index + of the job (`1..N`, where `N` is the number of jobs). If + :option:`per_job_logs` is false, then the filename will not include the + `.x` job index. + + The included :command:`fio_generate_plots` script uses :command:`gnuplot` to turn these + text files into nice graphs. See `Log File Formats`_ for how data is + structured within the file. + +.. option:: write_lat_log=str - The actual log names will be :file:`foo_slat.x.log`, :file:`foo_clat.x.log`, - and :file:`foo_lat.x.log`, where `x` is the index of the job (`1..N`, where `N` - is the number of jobs). This helps :command:`fio_generate_plots` find the - logs automatically. If :option:`per_job_logs` is false, then the filename - will not include the job index. See `Log File Formats`_. + Same as :option:`write_bw_log`, except this option creates I/O + submission (e.g., `file:`name_slat.x.log`), completion (e.g., + `file:`name_clat.x.log`), and total (e.g., `file:`name_lat.x.log`) + latency files instead. See :option:`write_bw_log` for details about + the filename format and `Log File Formats`_ for how data is structured + within the files. .. option:: write_hist_log=str - Same as :option:`write_lat_log`, but writes I/O completion latency - histograms. If no filename is given with this option, the default filename - of :file:`jobname_clat_hist.x.log` is used, where `x` is the index of the - job (`1..N`, where `N` is the number of jobs). Even if the filename is given, - fio will still append the type of log. If :option:`per_job_logs` is false, - then the filename will not include the job index. See `Log File Formats`_. + Same as :option:`write_bw_log` but writes an I/O completion latency + histogram file (e.g., `file:`name_hist.x.log`) instead. Note that this + file will be empty unless :option:`log_hist_msec` has also been set. + See :option:`write_bw_log` for details about the filename format and + `Log File Formats`_ for how data is structured within the file. .. option:: write_iops_log=str - Same as :option:`write_bw_log`, but writes IOPS. If no filename is given - with this option, the default filename of :file:`jobname_type.x.log` is - used, where `x` is the index of the job (`1..N`, where `N` is the number of - jobs). Even if the filename is given, fio will still append the type of - log. If :option:`per_job_logs` is false, then the filename will not include - the job index. See `Log File Formats`_. + Same as :option:`write_bw_log`, but writes an IOPS file (e.g. + `file:`name_iops.x.log`) instead. See :option:`write_bw_log` for + details about the filename format and `Log File Formats`_ for how data + is structured within the file. .. option:: log_avg_msec=int @@ -2759,15 +2780,16 @@ Measurements and reporting :option:`log_avg_msec` is inaccurate. Setting this option makes fio log histogram entries over the specified period of time, reducing log sizes for high IOPS devices while retaining percentile accuracy. See - :option:`log_hist_coarseness` as well. Defaults to 0, meaning histogram - logging is disabled. + :option:`log_hist_coarseness` and :option:`write_hist_log` as well. + Defaults to 0, meaning histogram logging is disabled. .. option:: log_hist_coarseness=int Integer ranging from 0 to 6, defining the coarseness of the resolution of the histogram logs enabled with :option:`log_hist_msec`. For each increment in coarseness, fio outputs half as many bins. Defaults to 0, for which - histogram logs contain 1216 latency bins. See `Log File Formats`_. + histogram logs contain 1216 latency bins. See :option:`write_hist_log` + and `Log File Formats`_. .. option:: log_max_value=bool @@ -2867,7 +2889,7 @@ Measurements and reporting .. option:: lat_percentiles=bool - Enable the reporting of percentiles of IO latencies. This is similar + Enable the reporting of percentiles of I/O latencies. This is similar to :option:`clat_percentiles`, except that this includes the submission latency. This option is mutually exclusive with :option:`clat_percentiles`. @@ -3237,7 +3259,7 @@ writes in the example above). In the order listed, they denote: short or dropped. **IO latency** - These values are for `--latency-target` and related options. When + These values are for :option:`latency_target` and related options. When these options are engaged, this section describes the I/O depth required to meet the specified latency target. @@ -42,7 +42,7 @@ SOURCE := $(sort $(patsubst $(SRCDIR)/%,%,$(wildcard $(SRCDIR)/crc/*.c)) \ eta.c verify.c memory.c io_u.c parse.c mutex.c options.c \ smalloc.c filehash.c profile.c debug.c engines/cpu.c \ engines/mmap.c engines/sync.c engines/null.c engines/net.c \ - engines/ftruncate.c \ + engines/ftruncate.c engines/filecreate.c \ server.c client.c iolog.c backend.c libfio.c flow.c cconv.c \ gettime-thread.c helpers.c json.c idletime.c td_error.c \ profiles/tiobench.c profiles/act.c io_u_queue.c filelock.c \ diff --git a/appveyor.yml b/appveyor.yml index 39f50a80..844afa59 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,16 +1,21 @@ -clone_depth: 50 +clone_depth: 1 environment: + CYG_MIRROR: http://cygwin.mirror.constant.com + CYG_ROOT: C:\cygwin64 MAKEFLAGS: -j 2 matrix: - platform: x86_64 BUILD_ARCH: x64 - CYG_ROOT: C:\cygwin64 + PACKAGE_ARCH: x86_64 CONFIGURE_OPTIONS: - platform: x86 BUILD_ARCH: x86 - CYG_ROOT: C:\cygwin + PACKAGE_ARCH: i686 CONFIGURE_OPTIONS: --build-32bit-win +install: + - '%CYG_ROOT%\setup-x86_64.exe --quiet-mode --no-shortcuts --only-site --site "%CYG_MIRROR%" --packages "mingw64-%PACKAGE_ARCH%-zlib" > NULL' + build_script: - SET PATH=%CYG_ROOT%\bin;%PATH% - 'bash.exe -lc "cd \"${APPVEYOR_BUILD_FOLDER}\" && ./configure --extra-cflags=\"-Werror\" ${CONFIGURE_OPTIONS} && make.exe' @@ -18,7 +18,7 @@ * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * */ #include <unistd.h> @@ -499,7 +499,6 @@ int io_queue_event(struct thread_data *td, struct io_u *io_u, int *ret, if (ddir_rw(io_u->ddir)) td->ts.short_io_u[io_u->ddir]++; - f = io_u->file; if (io_u->offset == f->real_file_size) goto sync_done; @@ -1204,9 +1203,9 @@ static int init_io_u(struct thread_data *td) data_xfer = 0; err = 0; - err += io_u_rinit(&td->io_u_requeues, td->o.iodepth); - err += io_u_qinit(&td->io_u_freelist, td->o.iodepth); - err += io_u_qinit(&td->io_u_all, td->o.iodepth); + err += !io_u_rinit(&td->io_u_requeues, td->o.iodepth); + err += !io_u_qinit(&td->io_u_freelist, td->o.iodepth); + err += !io_u_qinit(&td->io_u_all, td->o.iodepth); if (err) { log_err("fio: failed setting up IO queues\n"); @@ -1505,7 +1504,7 @@ static void *thread_main(void *data) struct sk_out *sk_out = fd->sk_out; uint64_t bytes_done[DDIR_RWDIR_CNT]; int deadlock_loop_cnt; - int clear_state; + bool clear_state, did_some_io; int ret; sk_out_assign(sk_out); @@ -1693,16 +1692,14 @@ static void *thread_main(void *data) if (td_io_init(td)) goto err; - if (init_random_map(td)) + if (!init_random_map(td)) goto err; if (o->exec_prerun && exec_string(o, o->exec_prerun, (const char *)"prerun")) goto err; - if (o->pre_read) { - if (pre_read_files(td) < 0) - goto err; - } + if (o->pre_read && !pre_read_files(td)) + goto err; fio_verify_init(td); @@ -1726,7 +1723,8 @@ static void *thread_main(void *data) } memset(bytes_done, 0, sizeof(bytes_done)); - clear_state = 0; + clear_state = false; + did_some_io = false; while (keep_running(td)) { uint64_t verify_bytes; @@ -1765,7 +1763,7 @@ static void *thread_main(void *data) if (td->runstate >= TD_EXITED) break; - clear_state = 1; + clear_state = true; /* * Make sure we've successfully updated the rusage stats @@ -1804,6 +1802,9 @@ static void *thread_main(void *data) td_ioengine_flagged(td, FIO_UNIDIR)) continue; + if (ddir_rw_sum(bytes_done)) + did_some_io = true; + clear_io_state(td, 0); fio_gettime(&td->start, NULL); @@ -1830,6 +1831,7 @@ static void *thread_main(void *data) * (Are we not missing other flags that can be ignored ?) */ if ((td->o.size || td->o.io_size) && !ddir_rw_sum(bytes_done) && + !did_some_io && !td->o.create_only && !(td_ioengine_flagged(td, FIO_NOIO) || td_ioengine_flagged(td, FIO_DISKLESSIO))) log_err("%s: No I/O performed by %s, " @@ -1925,11 +1927,7 @@ static void reap_threads(unsigned int *nr_running, uint64_t *t_rate, for_each_td(td, i) { int flags = 0; - /* - * ->io_ops is NULL for a thread that has closed its - * io engine - */ - if (td->io_ops && !strcmp(td->io_ops->name, "cpuio")) + if (!strcmp(td->o.ioengine, "cpuio")) cputhreads++; else realthreads++; @@ -2342,6 +2340,7 @@ reap: fio_terminate_threads(TERMINATE_ALL); fio_abort = 1; nr_started--; + free(fd); break; } dprint(FD_MUTEX, "done waiting on startup_mutex\n"); @@ -500,10 +500,8 @@ int load_blktrace(struct thread_data *td, const char *filename, int need_swap) handle_trace(td, &t, ios, rw_bs); } while (1); - for (i = 0; i < td->files_index; i++) { - f = td->files[i]; + for_each_file(td, f, i) trace_add_open_close_event(td, f->fileno, FIO_LOG_CLOSE_FILE); - } fifo_free(fifo); close(fd); @@ -105,6 +105,7 @@ void convert_thread_options_to_cpu(struct thread_options *o, o->file_size_low = le64_to_cpu(top->file_size_low); o->file_size_high = le64_to_cpu(top->file_size_high); o->start_offset = le64_to_cpu(top->start_offset); + o->start_offset_align = le64_to_cpu(top->start_offset_align); o->start_offset_percent = le32_to_cpu(top->start_offset_percent); for (i = 0; i < DDIR_RWDIR_CNT; i++) { @@ -548,6 +549,7 @@ void convert_thread_options_to_net(struct thread_options_pack *top, top->file_size_low = __cpu_to_le64(o->file_size_low); top->file_size_high = __cpu_to_le64(o->file_size_high); top->start_offset = __cpu_to_le64(o->start_offset); + top->start_offset_align = __cpu_to_le64(o->start_offset_align); top->start_offset_percent = __cpu_to_le32(o->start_offset_percent); top->trim_backlog = __cpu_to_le64(o->trim_backlog); top->offset_increment = __cpu_to_le64(o->offset_increment); @@ -1312,14 +1312,16 @@ static void client_flush_hist_samples(FILE *f, int hist_coarseness, void *sample static int fio_client_handle_iolog(struct fio_client *client, struct fio_net_cmd *cmd) { - struct cmd_iolog_pdu *pdu; + struct cmd_iolog_pdu *pdu = NULL; bool store_direct; - char *log_pathname; + char *log_pathname = NULL; + int ret = 0; pdu = convert_iolog(cmd, &store_direct); if (!pdu) { log_err("fio: failed converting IO log\n"); - return 1; + ret = 1; + goto out; } /* allocate buffer big enough for next sprintf() call */ @@ -1327,7 +1329,8 @@ static int fio_client_handle_iolog(struct fio_client *client, strlen(client->hostname)); if (!log_pathname) { log_err("fio: memory allocation of unique pathname failed\n"); - return -1; + ret = -1; + goto out; } /* generate a unique pathname for the log file using hostname */ sprintf(log_pathname, "%s.%s", pdu->name, client->hostname); @@ -1342,7 +1345,8 @@ static int fio_client_handle_iolog(struct fio_client *client, if (fd < 0) { log_err("fio: open log %s: %s\n", log_pathname, strerror(errno)); - return 1; + ret = 1; + goto out; } sz = cmd->pdu_len - sizeof(*pdu); @@ -1351,17 +1355,19 @@ static int fio_client_handle_iolog(struct fio_client *client, if (ret != sz) { log_err("fio: short write on compressed log\n"); - return 1; + ret = 1; + goto out; } - return 0; + ret = 0; } else { FILE *f; f = fopen((const char *) log_pathname, "w"); if (!f) { log_err("fio: fopen log %s : %s\n", log_pathname, strerror(errno)); - return 1; + ret = 1; + goto out; } if (pdu->log_type == IO_LOG_TYPE_HIST) { @@ -1372,8 +1378,17 @@ static int fio_client_handle_iolog(struct fio_client *client, pdu->nr_samples * sizeof(struct io_sample)); } fclose(f); - return 0; + ret = 0; } + +out: + if (pdu && pdu != (void *) cmd->payload) + free(pdu); + + if (log_pathname) + free(log_pathname); + + return ret; } static void handle_probe(struct fio_client *client, struct fio_net_cmd *cmd) @@ -1849,10 +1864,12 @@ static void request_client_etas(struct client_ops *ops) static int handle_cmd_timeout(struct fio_client *client, struct fio_net_cmd_reply *reply) { + uint16_t reply_opcode = reply->opcode; + flist_del(&reply->list); free(reply); - if (reply->opcode != FIO_NET_CMD_SEND_ETA) + if (reply_opcode != FIO_NET_CMD_SEND_ETA) return 1; log_info("client <%s>: timeout on SEND_ETA\n", client->hostname); @@ -225,7 +225,20 @@ if test "$show_help" = "yes" ; then fi cross_prefix=${cross_prefix-${CROSS_COMPILE}} -cc="${CC-${cross_prefix}gcc}" +# Preferred compiler (can be overriden later after we know the platform): +# ${CC} (if set) +# ${cross_prefix}gcc (if cross-prefix specified) +# gcc if available +# clang if available +if test -z "${CC}${cross_prefix}"; then + if has gcc; then + cc=gcc + elif has clang; then + cc=clang + fi +else + cc="${CC-${cross_prefix}gcc}" +fi if check_define __ANDROID__ ; then targetos="Android" @@ -301,16 +314,16 @@ SunOS) CYGWIN*) # We still force some options, so keep this message here. echo "Forcing some known good options on Windows" - if test -z "$CC" ; then + if test -z "${CC}${cross_prefix}"; then if test ! -z "$build_32bit_win" && test "$build_32bit_win" = "yes"; then - CC="i686-w64-mingw32-gcc" + cc="i686-w64-mingw32-gcc" if test -e "../zlib/contrib/vstudio/vc14/x86/ZlibStatReleaseWithoutAsm/zlibstat.lib"; then echo "Building with zlib support" output_sym "CONFIG_ZLIB" echo "LIBS=../zlib/contrib/vstudio/vc14/x86/ZlibStatReleaseWithoutAsm/zlibstat.lib" >> $config_host_mak fi else - CC="x86_64-w64-mingw32-gcc" + cc="x86_64-w64-mingw32-gcc" if test -e "../zlib/contrib/vstudio/vc14/x64/ZlibStatReleaseWithoutAsm/zlibstat.lib"; then echo "Building with zlib support" output_sym "CONFIG_ZLIB" @@ -330,6 +343,8 @@ CYGWIN*) # Flags below are still necessary mostly for MinGW. socklen_t="yes" sfaa="yes" + sync_sync="yes" + cmp_swap="yes" rusage_thread="yes" fdatasync="yes" clock_gettime="yes" # clock_monotonic probe has dependency on this @@ -340,11 +355,25 @@ CYGWIN*) tls_thread="yes" static_assert="yes" ipv6="yes" - echo "CC=$CC" >> $config_host_mak + mkdir_two="no" echo "BUILD_CFLAGS=$CFLAGS -I../zlib -include config-host.h -D_GNU_SOURCE" >> $config_host_mak ;; esac +# Now we know the target platform we can have another guess at the preferred +# compiler when it wasn't explictly set +if test -z "${CC}${cross_prefix}"; then + if test "$targetos" = "FreeBSD" || test "$targetos" = "Darwin"; then + if has clang; then + cc=clang + fi + fi +fi +if test -z "$cc"; then + echo "configure: failed to find compiler" + exit 1 +fi + if test ! -z "$cpu" ; then # command line argument : @@ -415,18 +444,6 @@ case "$cpu" in ;; esac -if test -z "$CC" ; then - if test "$targetos" = "FreeBSD"; then - if has clang; then - CC=clang - else - CC=gcc - fi - fi -fi - -cc="${CC-${cross_prefix}gcc}" - ########################################## # check cross compile @@ -692,6 +709,44 @@ fi print_config "__sync_fetch_and_add" "$sfaa" ########################################## +# __sync_synchronize() test +if test "$sync_sync" != "yes" ; then + sync_sync="no" +fi +cat > $TMPC << EOF +#include <inttypes.h> + +int main(int argc, char **argv) +{ + __sync_synchronize(); + return 0; +} +EOF +if compile_prog "" "" "__sync_synchronize()" ; then + sync_sync="yes" +fi +print_config "__sync_synchronize" "$sync_sync" + +########################################## +# __sync_val_compare_and_swap() test +if test "$cmp_swap" != "yes" ; then + cmp_swap="no" +fi +cat > $TMPC << EOF +#include <inttypes.h> + +int main(int argc, char **argv) +{ + int x = 0; + return __sync_val_compare_and_swap(&x, 1, 2); +} +EOF +if compile_prog "" "" "__sync_val_compare_and_swap()" ; then + cmp_swap="yes" +fi +print_config "__sync_val_compare_and_swap" "$cmp_swap" + +########################################## # libverbs probe if test "$libverbs" != "yes" ; then libverbs="no" @@ -2033,6 +2088,22 @@ if test "$enable_cuda" = "yes" && compile_prog "" "-lcuda" "cuda"; then fi print_config "cuda" "$cuda" +########################################## +# mkdir() probe. mingw apparently has a one-argument mkdir :/ +mkdir_two="no" +cat > $TMPC << EOF +#include <sys/stat.h> +#include <sys/types.h> +int main(int argc, char **argv) +{ + return mkdir("/tmp/bla", 0600); +} +EOF +if compile_prog "" "" "mkdir(a, b)"; then + mkdir_two="yes" +fi +print_config "mkdir(a, b)" "$mkdir_two" + ############################################################################# if test "$wordsize" = "64" ; then @@ -2077,6 +2148,12 @@ fi if test "$sfaa" = "yes" ; then output_sym "CONFIG_SFAA" fi +if test "$sync_sync" = "yes" ; then + output_sym "CONFIG_SYNC_SYNC" +fi +if test "$cmp_swap" = "yes" ; then + output_sym "CONFIG_CMP_SWAP" +fi if test "$libverbs" = "yes" -a "$rdmacm" = "yes" ; then output_sym "CONFIG_RDMA" fi @@ -2261,6 +2338,9 @@ fi if test "$cuda" = "yes" ; then output_sym "CONFIG_CUDA" fi +if test "$mkdir_two" = "yes" ; then + output_sym "CONFIG_HAVE_MKDIR_TWO" +fi echo "LIBS+=$LIBS" >> $config_host_mak echo "GFIO_LIBS+=$GFIO_LIBS" >> $config_host_mak diff --git a/crc/crc32.c b/crc/crc32.c index 657031d4..4140a8d4 100644 --- a/crc/crc32.c +++ b/crc/crc32.c @@ -13,7 +13,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, - Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ + Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #include <inttypes.h> #include "crc32.h" diff --git a/crc/crc32.h b/crc/crc32.h index 674057b2..a37d7ada 100644 --- a/crc/crc32.h +++ b/crc/crc32.h @@ -13,7 +13,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, - Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ + Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef CRC32_H #define CRC32_H diff --git a/crc/crc32c.h b/crc/crc32c.h index d513f3aa..be03c1a2 100644 --- a/crc/crc32c.h +++ b/crc/crc32c.h @@ -13,7 +13,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, - Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ + Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef CRC32C_H #define CRC32C_H diff --git a/doc/conf.py b/doc/conf.py index 4102140a..d4dd9d20 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # fio documentation build configuration file, created by diff --git a/engines/filecreate.c b/engines/filecreate.c new file mode 100644 index 00000000..0c3bcdd6 --- /dev/null +++ b/engines/filecreate.c @@ -0,0 +1,119 @@ +/* + * filecreate engine + * + * IO engine that doesn't do any IO, just creates files and tracks the latency + * of the file creation. + */ +#include <stdio.h> +#include <unistd.h> +#include <fcntl.h> +#include <errno.h> + +#include "../fio.h" +#include "../filehash.h" + +struct fc_data { + enum fio_ddir stat_ddir; +}; + +static int open_file(struct thread_data *td, struct fio_file *f) +{ + struct timespec start; + int do_lat = !td->o.disable_lat; + + dprint(FD_FILE, "fd open %s\n", f->file_name); + + if (f->filetype != FIO_TYPE_FILE) { + log_err("fio: only files are supported fallocate \n"); + return 1; + } + if (!strcmp(f->file_name, "-")) { + log_err("fio: can't read/write to stdin/out\n"); + return 1; + } + + if (do_lat) + fio_gettime(&start, NULL); + + f->fd = open(f->file_name, O_CREAT|O_RDWR, 0600); + + if (f->fd == -1) { + char buf[FIO_VERROR_SIZE]; + int e = errno; + + snprintf(buf, sizeof(buf), "open(%s)", f->file_name); + td_verror(td, e, buf); + return 1; + } + + if (do_lat) { + struct fc_data *data = td->io_ops_data; + uint64_t nsec; + + nsec = ntime_since_now(&start); + add_clat_sample(td, data->stat_ddir, nsec, 0, 0); + } + + return 0; +} + +static int queue_io(struct thread_data *td, struct io_u fio_unused *io_u) +{ + return FIO_Q_COMPLETED; +} + +/* + * Ensure that we at least have a block size worth of IO to do for each + * file. If the job file has td->o.size < nr_files * block_size, then + * fio won't do anything. + */ +static int get_file_size(struct thread_data *td, struct fio_file *f) +{ + f->real_file_size = td_min_bs(td); + return 0; +} + +static int init(struct thread_data *td) +{ + struct fc_data *data; + + data = calloc(1, sizeof(*data)); + + if (td_read(td)) + data->stat_ddir = DDIR_READ; + else if (td_write(td)) + data->stat_ddir = DDIR_WRITE; + + td->io_ops_data = data; + return 0; +} + +static void cleanup(struct thread_data *td) +{ + struct fc_data *data = td->io_ops_data; + + free(data); +} + +static struct ioengine_ops ioengine = { + .name = "filecreate", + .version = FIO_IOOPS_VERSION, + .init = init, + .cleanup = cleanup, + .queue = queue_io, + .get_file_size = get_file_size, + .open_file = open_file, + .close_file = generic_close_file, + .flags = FIO_DISKLESSIO | FIO_SYNCIO | FIO_FAKEIO | + FIO_NOSTATS | FIO_NOFILEHASH, +}; + +static void fio_init fio_filecreate_register(void) +{ + register_ioengine(&ioengine); +} + +static void fio_exit fio_filecreate_unregister(void) +{ + unregister_ioengine(&ioengine); +} diff --git a/engines/null.c b/engines/null.c index 812cadfe..8a4d106b 100644 --- a/engines/null.c +++ b/engines/null.c @@ -6,7 +6,11 @@ * * It also can act as external C++ engine - compiled with: * - * g++ -O2 -g -shared -rdynamic -fPIC -o null.so null.c -DFIO_EXTERNAL_ENGINE + * g++ -O2 -g -shared -rdynamic -fPIC -o cpp_null null.c -DFIO_EXTERNAL_ENGINE + * + * to test it execute: + * + * LD_LIBRARY_PATH=./engines ./fio examples/cpp_null.fio * */ #include <stdio.h> @@ -23,20 +27,17 @@ struct null_data { int events; }; -static struct io_u *fio_null_event(struct thread_data *td, int event) +static struct io_u *null_event(struct null_data *nd, int event) { - struct null_data *nd = (struct null_data *) td->io_ops_data; - return nd->io_us[event]; } -static int fio_null_getevents(struct thread_data *td, unsigned int min_events, - unsigned int fio_unused max, - const struct timespec fio_unused *t) +static int null_getevents(struct null_data *nd, unsigned int min_events, + unsigned int fio_unused max, + const struct timespec fio_unused *t) { - struct null_data *nd = (struct null_data *) td->io_ops_data; int ret = 0; - + if (min_events) { ret = nd->events; nd->events = 0; @@ -45,10 +46,8 @@ static int fio_null_getevents(struct thread_data *td, unsigned int min_events, return ret; } -static int fio_null_commit(struct thread_data *td) +static int null_commit(struct thread_data *td, struct null_data *nd) { - struct null_data *nd = (struct null_data *) td->io_ops_data; - if (!nd->events) { #ifndef FIO_EXTERNAL_ENGINE io_u_mark_submit(td, nd->queued); @@ -60,10 +59,9 @@ static int fio_null_commit(struct thread_data *td) return 0; } -static int fio_null_queue(struct thread_data *td, struct io_u *io_u) +static int null_queue(struct thread_data *td, struct null_data *nd, + struct io_u *io_u) { - struct null_data *nd = (struct null_data *) td->io_ops_data; - fio_ro_check(td, io_u); if (td->io_ops->flags & FIO_SYNCIO) @@ -75,25 +73,23 @@ static int fio_null_queue(struct thread_data *td, struct io_u *io_u) return FIO_Q_QUEUED; } -static int fio_null_open(struct thread_data fio_unused *td, - struct fio_file fio_unused *f) +static int null_open(struct null_data fio_unused *nd, + struct fio_file fio_unused *f) { return 0; } -static void fio_null_cleanup(struct thread_data *td) +static void null_cleanup(struct null_data *nd) { - struct null_data *nd = (struct null_data *) td->io_ops_data; - if (nd) { free(nd->io_us); free(nd); } } -static int fio_null_init(struct thread_data *td) +static int null_init(struct thread_data *td, struct null_data **nd_ptr) { - struct null_data *nd = (struct null_data *) malloc(sizeof(*nd)); + struct null_data *nd = (struct null_data *) malloc(sizeof(**nd_ptr)); memset(nd, 0, sizeof(*nd)); @@ -103,11 +99,49 @@ static int fio_null_init(struct thread_data *td) } else td->io_ops->flags |= FIO_SYNCIO; - td->io_ops_data = nd; + *nd_ptr = nd; return 0; } #ifndef __cplusplus + +static struct io_u *fio_null_event(struct thread_data *td, int event) +{ + return null_event((struct null_data *)td->io_ops_data, event); +} + +static int fio_null_getevents(struct thread_data *td, unsigned int min_events, + unsigned int max, const struct timespec *t) +{ + struct null_data *nd = (struct null_data *)td->io_ops_data; + return null_getevents(nd, min_events, max, t); +} + +static int fio_null_commit(struct thread_data *td) +{ + return null_commit(td, (struct null_data *)td->io_ops_data); +} + +static int fio_null_queue(struct thread_data *td, struct io_u *io_u) +{ + return null_queue(td, (struct null_data *)td->io_ops_data, io_u); +} + +static int fio_null_open(struct thread_data *td, struct fio_file *f) +{ + return null_open((struct null_data *)td->io_ops_data, f); +} + +static void fio_null_cleanup(struct thread_data *td) +{ + null_cleanup((struct null_data *)td->io_ops_data); +} + +static int fio_null_init(struct thread_data *td) +{ + return null_init(td, (struct null_data **)&td->io_ops_data); +} + static struct ioengine_ops ioengine = { .name = "null", .version = FIO_IOOPS_VERSION, @@ -134,7 +168,91 @@ static void fio_exit fio_null_unregister(void) #else #ifdef FIO_EXTERNAL_ENGINE + +struct NullData { + NullData(struct thread_data *td) + { + null_init(td, &impl_); + } + + ~NullData() + { + null_cleanup(impl_); + } + + static NullData *get(struct thread_data *td) + { + return reinterpret_cast<NullData *>(td->io_ops_data); + } + + io_u *fio_null_event(struct thread_data *, int event) + { + return null_event(impl_, event); + } + + int fio_null_getevents(struct thread_data *, unsigned int min_events, + unsigned int max, const struct timespec *t) + { + return null_getevents(impl_, min_events, max, t); + } + + int fio_null_commit(struct thread_data *td) + { + return null_commit(td, impl_); + } + + int fio_null_queue(struct thread_data *td, struct io_u *io_u) + { + return null_queue(td, impl_, io_u); + } + + int fio_null_open(struct thread_data *, struct fio_file *f) + { + return null_open(impl_, f); + } + + struct null_data *impl_; +}; + extern "C" { + +static struct io_u *fio_null_event(struct thread_data *td, int event) +{ + return NullData::get(td)->fio_null_event(td, event); +} + +static int fio_null_getevents(struct thread_data *td, unsigned int min_events, + unsigned int max, const struct timespec *t) +{ + return NullData::get(td)->fio_null_getevents(td, min_events, max, t); +} + +static int fio_null_commit(struct thread_data *td) +{ + return NullData::get(td)->fio_null_commit(td); +} + +static int fio_null_queue(struct thread_data *td, struct io_u *io_u) +{ + return NullData::get(td)->fio_null_queue(td, io_u); +} + +static int fio_null_open(struct thread_data *td, struct fio_file *f) +{ + return NullData::get(td)->fio_null_open(td, f); +} + +static int fio_null_init(struct thread_data *td) +{ + td->io_ops_data = new NullData(td); + return 0; +} + +static void fio_null_cleanup(struct thread_data *td) +{ + delete NullData::get(td); +} + static struct ioengine_ops ioengine; void get_ioengine(struct ioengine_ops **ioengine_ptr) { diff --git a/engines/pmemblk.c b/engines/pmemblk.c index 52af9eda..5d219155 100644 --- a/engines/pmemblk.c +++ b/engines/pmemblk.c @@ -14,8 +14,8 @@ * * You should have received a copy of the GNU General Public * License along with this program; if not, write to the Free - * Software Foundation, Inc., 59 Temple Place, Suite 330, - * Boston, MA 02111-1307 USA + * Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. */ /* diff --git a/engines/windowsaio.c b/engines/windowsaio.c index 314eaadf..94393931 100644 --- a/engines/windowsaio.c +++ b/engines/windowsaio.c @@ -94,15 +94,13 @@ static int fio_windowsaio_init(struct thread_data *td) if (!rc) ctx = malloc(sizeof(struct thread_ctx)); - if (!rc && ctx == NULL) - { + if (!rc && ctx == NULL) { log_err("windowsaio: failed to allocate memory for thread context structure\n"); CloseHandle(hFile); rc = 1; } - if (!rc) - { + if (!rc) { DWORD threadid; ctx->iocp = hFile; @@ -142,6 +140,44 @@ static void fio_windowsaio_cleanup(struct thread_data *td) } } +static int windowsaio_invalidate_cache(struct fio_file *f) +{ + DWORD error; + DWORD isharemode = (FILE_SHARE_DELETE | FILE_SHARE_READ | + FILE_SHARE_WRITE); + HANDLE ihFile; + int rc = 0; + + /* + * Encourage Windows to drop cached parts of a file by temporarily + * opening it for non-buffered access. Note: this will only work when + * the following is the only thing with the file open on the whole + * system. + */ + dprint(FD_IO, "windowaio: attempt invalidate cache for %s\n", + f->file_name); + ihFile = CreateFile(f->file_name, 0, isharemode, NULL, OPEN_EXISTING, + FILE_FLAG_NO_BUFFERING, NULL); + + if (ihFile != INVALID_HANDLE_VALUE) { + if (!CloseHandle(ihFile)) { + error = GetLastError(); + log_info("windowsaio: invalidation fd close %s " + "failed: error %d\n", f->file_name, error); + rc = 1; + } + } else { + error = GetLastError(); + if (error != ERROR_FILE_NOT_FOUND) { + log_info("windowsaio: cache invalidation of %s failed: " + "error %d\n", f->file_name, error); + rc = 1; + } + } + + return rc; +} + static int fio_windowsaio_open_file(struct thread_data *td, struct fio_file *f) { int rc = 0; @@ -200,6 +236,11 @@ static int fio_windowsaio_open_file(struct thread_data *td, struct fio_file *f) else openmode = OPEN_EXISTING; + /* If we're going to use direct I/O, Windows will try and invalidate + * its cache at that point so there's no need to do it here */ + if (td->o.invalidate_cache && !td->o.odirect) + windowsaio_invalidate_cache(f); + f->hFile = CreateFile(f->file_name, access, sharemode, NULL, openmode, flags, NULL); @@ -305,7 +346,8 @@ static int fio_windowsaio_getevents(struct thread_data *td, unsigned int min, break; } - if (dequeued >= min || (t != NULL && timeout_expired(start_count, end_count))) + if (dequeued >= min || + (t != NULL && timeout_expired(start_count, end_count))) break; } while (1); @@ -328,10 +370,12 @@ static int fio_windowsaio_queue(struct thread_data *td, struct io_u *io_u) switch (io_u->ddir) { case DDIR_WRITE: - success = WriteFile(io_u->file->hFile, io_u->xfer_buf, io_u->xfer_buflen, NULL, lpOvl); + success = WriteFile(io_u->file->hFile, io_u->xfer_buf, + io_u->xfer_buflen, NULL, lpOvl); break; case DDIR_READ: - success = ReadFile(io_u->file->hFile, io_u->xfer_buf, io_u->xfer_buflen, NULL, lpOvl); + success = ReadFile(io_u->file->hFile, io_u->xfer_buf, + io_u->xfer_buflen, NULL, lpOvl); break; case DDIR_SYNC: case DDIR_DATASYNC: @@ -343,13 +387,11 @@ static int fio_windowsaio_queue(struct thread_data *td, struct io_u *io_u) } return FIO_Q_COMPLETED; - break; case DDIR_TRIM: log_err("windowsaio: manual TRIM isn't supported on Windows\n"); io_u->error = 1; io_u->resid = io_u->xfer_buflen; return FIO_Q_COMPLETED; - break; default: assert(0); break; @@ -380,7 +422,11 @@ static DWORD WINAPI IoCompletionRoutine(LPVOID lpParameter) wd = ctx->wd; do { - if (!GetQueuedCompletionStatus(ctx->iocp, &bytes, &ulKey, &ovl, 250) && ovl == NULL) + BOOL ret; + + ret = GetQueuedCompletionStatus(ctx->iocp, &bytes, &ulKey, + &ovl, 250); + if (!ret && ovl == NULL) continue; fov = CONTAINING_RECORD(ovl, struct fio_overlapped, o); diff --git a/examples/cpp_null.fio b/examples/cpp_null.fio new file mode 100644 index 00000000..436ed90a --- /dev/null +++ b/examples/cpp_null.fio @@ -0,0 +1,10 @@ +[global] +bs=4k +gtod_reduce=1 + +[null] +ioengine=cpp_null +size=100g +rw=randread +norandommap +time_based=0 diff --git a/examples/filecreate-ioengine.fio b/examples/filecreate-ioengine.fio new file mode 100644 index 00000000..ec7caad5 --- /dev/null +++ b/examples/filecreate-ioengine.fio @@ -0,0 +1,35 @@ +# Example filecreate job +# +# create_on_open is needed so that the open happens during the run and not the +# setup. +# +# openfiles needs to be set so that you do not exceed the maximum allowed open +# files. +# +# filesize needs to be set to a non zero value so fio will actually run, but the +# IO will not really be done and the write latency numbers will only reflect the +# open times. +[global] +create_on_open=1 +nrfiles=31250 +ioengine=filecreate +fallocate=none +filesize=4k +openfiles=1 + +[t0] +[t1] +[t2] +[t3] +[t4] +[t5] +[t6] +[t7] +[t8] +[t9] +[t10] +[t11] +[t12] +[t13] +[t14] +[t15] diff --git a/examples/fio-rand-RW.job b/examples/fio-rand-RW.job new file mode 100644 index 00000000..0df0bc17 --- /dev/null +++ b/examples/fio-rand-RW.job @@ -0,0 +1,18 @@ +; fio-rand-RW.job for fiotest + +[global] +name=fio-rand-RW +filename=fio-rand-RW +rw=randrw +rwmixread=60 +rwmixwrite=40 +bs=4K +direct=0 +numjobs=4 +time_based=1 +runtime=900 + +[file1] +size=10G +ioengine=libaio +iodepth=16 diff --git a/examples/fio-rand-read.job b/examples/fio-rand-read.job new file mode 100644 index 00000000..bc154668 --- /dev/null +++ b/examples/fio-rand-read.job @@ -0,0 +1,16 @@ +; fio-rand-read.job for fiotest + +[global] +name=fio-rand-read +filename=fio-rand-read +rw=randread +bs=4K +direct=0 +numjobs=1 +time_based=1 +runtime=900 + +[file1] +size=10G +ioengine=libaio +iodepth=16 diff --git a/examples/fio-rand-write.job b/examples/fio-rand-write.job new file mode 100644 index 00000000..bd1b73a9 --- /dev/null +++ b/examples/fio-rand-write.job @@ -0,0 +1,16 @@ +; fio-rand-write.job for fiotest + +[global] +name=fio-rand-write +filename=fio-rand-write +rw=randwrite +bs=4K +direct=0 +numjobs=4 +time_based=1 +runtime=900 + +[file1] +size=10G +ioengine=libaio +iodepth=16 diff --git a/examples/fio-seq-RW.job b/examples/fio-seq-RW.job new file mode 100644 index 00000000..8f7090f3 --- /dev/null +++ b/examples/fio-seq-RW.job @@ -0,0 +1,18 @@ +; fio-seq-RW.job for fiotest + +[global] +name=fio-seq-RW +filename=fio-seq-RW +rw=rw +rwmixread=60 +rwmixwrite=40 +bs=256K +direct=0 +numjobs=4 +time_based=1 +runtime=900 + +[file1] +size=10G +ioengine=libaio +iodepth=16 diff --git a/examples/fio-seq-read.job b/examples/fio-seq-read.job new file mode 100644 index 00000000..74b1b308 --- /dev/null +++ b/examples/fio-seq-read.job @@ -0,0 +1,14 @@ +[global] +name=fio-seq-reads +filename=fio-seq-reads +rw=read +bs=256K +direct=0 +numjobs=1 +time_based=1 +runtime=900 + +[file1] +size=10G +ioengine=libaio +iodepth=16 diff --git a/examples/fio-seq-write.job b/examples/fio-seq-write.job new file mode 100644 index 00000000..b291a15a --- /dev/null +++ b/examples/fio-seq-write.job @@ -0,0 +1,16 @@ +; fio-seq-write.job for fiotest + +[global] +name=fio-seq-write +filename=fio-seq-write +rw=write +bs=256K +direct=0 +numjobs=1 +time_based=1 +runtime=900 + +[file1] +size=10G +ioengine=libaio +iodepth=16 diff --git a/exp/expression-parser.l b/exp/expression-parser.l index 50bd3832..692c6cc3 100644 --- a/exp/expression-parser.l +++ b/exp/expression-parser.l @@ -14,7 +14,7 @@ * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * */ diff --git a/exp/expression-parser.y b/exp/expression-parser.y index d664b8ed..04a6e07a 100644 --- a/exp/expression-parser.y +++ b/exp/expression-parser.y @@ -14,7 +14,7 @@ * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * */ diff --git a/exp/test-expression-parser.c b/exp/test-expression-parser.c index bf3fb3ed..e22f24dc 100644 --- a/exp/test-expression-parser.c +++ b/exp/test-expression-parser.c @@ -15,7 +15,7 @@ * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * */ @@ -15,7 +15,7 @@ * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * */ @@ -17,7 +17,7 @@ * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * */ #include "minmax.h" @@ -188,11 +188,17 @@ extern void close_and_free_files(struct thread_data *); extern uint64_t get_start_offset(struct thread_data *, struct fio_file *); extern int __must_check setup_files(struct thread_data *); extern int __must_check file_invalidate_cache(struct thread_data *, struct fio_file *); +#ifdef __cplusplus +extern "C" { +#endif extern int __must_check generic_open_file(struct thread_data *, struct fio_file *); extern int __must_check generic_close_file(struct thread_data *, struct fio_file *); extern int __must_check generic_get_file_size(struct thread_data *, struct fio_file *); +#ifdef __cplusplus +} +#endif extern int __must_check file_lookup_open(struct fio_file *f, int flags); -extern int __must_check pre_read_files(struct thread_data *); +extern bool __must_check pre_read_files(struct thread_data *); extern unsigned long long get_rand_file_size(struct thread_data *td); extern int add_file(struct thread_data *, const char *, int, int); extern int add_file_exclusive(struct thread_data *, const char *); @@ -203,7 +209,7 @@ extern void lock_file(struct thread_data *, struct fio_file *, enum fio_ddir); extern void unlock_file(struct thread_data *, struct fio_file *); extern void unlock_file_all(struct thread_data *, struct fio_file *); extern int add_dir_files(struct thread_data *, const char *); -extern int init_random_map(struct thread_data *); +extern bool init_random_map(struct thread_data *); extern void dup_files(struct thread_data *, struct thread_data *); extern int get_fileno(struct thread_data *, const char *); extern void free_release_files(struct thread_data *); diff --git a/filesetup.c b/filesetup.c index 891a55a1..4d29b70a 100644 --- a/filesetup.c +++ b/filesetup.c @@ -38,7 +38,7 @@ static inline void clear_error(struct thread_data *td) td->verror[0] = '\0'; } -static inline int native_fallocate(struct thread_data *td, struct fio_file *f) +static int native_fallocate(struct thread_data *td, struct fio_file *f) { bool success; @@ -49,32 +49,29 @@ static inline int native_fallocate(struct thread_data *td, struct fio_file *f) !success ? "un": ""); if (success) - return 0; + return false; if (errno == ENOSYS) dprint(FD_FILE, "native fallocate is not implemented\n"); - return -1; + return true; } static void fallocate_file(struct thread_data *td, struct fio_file *f) { - int r; - if (td->o.fill_device) return; switch (td->o.fallocate_mode) { case FIO_FALLOCATE_NATIVE: - r = native_fallocate(td, f); - if (r != 0 && errno != ENOSYS) - log_err("fio: native_fallocate call failed: %s\n", - strerror(errno)); + native_fallocate(td, f); break; case FIO_FALLOCATE_NONE: break; #ifdef CONFIG_POSIX_FALLOCATE - case FIO_FALLOCATE_POSIX: + case FIO_FALLOCATE_POSIX: { + int r; + dprint(FD_FILE, "posix_fallocate file %s size %llu\n", f->file_name, (unsigned long long) f->real_file_size); @@ -83,9 +80,12 @@ static void fallocate_file(struct thread_data *td, struct fio_file *f) if (r > 0) log_err("fio: posix_fallocate fails: %s\n", strerror(r)); break; + } #endif /* CONFIG_POSIX_FALLOCATE */ #ifdef CONFIG_LINUX_FALLOCATE - case FIO_FALLOCATE_KEEP_SIZE: + case FIO_FALLOCATE_KEEP_SIZE: { + int r; + dprint(FD_FILE, "fallocate(FALLOC_FL_KEEP_SIZE) " "file %s size %llu\n", f->file_name, (unsigned long long) f->real_file_size); @@ -95,6 +95,7 @@ static void fallocate_file(struct thread_data *td, struct fio_file *f) td_verror(td, errno, "fallocate"); break; + } #endif /* CONFIG_LINUX_FALLOCATE */ default: log_err("fio: unknown fallocate mode: %d\n", td->o.fallocate_mode); @@ -258,24 +259,25 @@ err: return 1; } -static int pre_read_file(struct thread_data *td, struct fio_file *f) +static bool pre_read_file(struct thread_data *td, struct fio_file *f) { - int ret = 0, r, did_open = 0, old_runstate; + int r, did_open = 0, old_runstate; unsigned long long left; unsigned int bs; + bool ret = true; char *b; if (td_ioengine_flagged(td, FIO_PIPEIO) || td_ioengine_flagged(td, FIO_NOIO)) - return 0; + return true; if (f->filetype == FIO_TYPE_CHAR) - return 0; + return true; if (!fio_file_open(f)) { if (td->io_ops->open_file(td, f)) { log_err("fio: cannot pre-read, failed to open file\n"); - return 1; + return false; } did_open = 1; } @@ -290,7 +292,7 @@ static int pre_read_file(struct thread_data *td, struct fio_file *f) b = malloc(bs); if (!b) { td_verror(td, errno, "malloc"); - ret = 1; + ret = false; goto error; } memset(b, 0, bs); @@ -298,7 +300,7 @@ static int pre_read_file(struct thread_data *td, struct fio_file *f) if (lseek(f->fd, f->file_offset, SEEK_SET) < 0) { td_verror(td, errno, "lseek"); log_err("fio: failed to lseek pre-read file\n"); - ret = 1; + ret = false; goto error; } @@ -869,12 +871,10 @@ uint64_t get_start_offset(struct thread_data *td, struct fio_file *f) if (o->start_offset_percent > 0) { /* - * if blockalign is provided, find the min across read, write, - * and trim + * if offset_align is provided, set initial offset */ - if (fio_option_is_set(o, ba)) { - align_bs = (unsigned long long) min(o->ba[DDIR_READ], o->ba[DDIR_WRITE]); - align_bs = min((unsigned long long) o->ba[DDIR_TRIM], align_bs); + if (fio_option_is_set(o, start_offset_align)) { + align_bs = o->start_offset_align; } else { /* else take the minimum block size */ align_bs = td_min_bs(td); @@ -1179,7 +1179,7 @@ err_out: return 1; } -int pre_read_files(struct thread_data *td) +bool pre_read_files(struct thread_data *td) { struct fio_file *f; unsigned int i; @@ -1187,14 +1187,14 @@ int pre_read_files(struct thread_data *td) dprint(FD_FILE, "pre_read files\n"); for_each_file(td, f, i) { - if (pre_read_file(td, f)) - return -1; + if (!pre_read_file(td, f)) + return false; } - return 0; + return true; } -static int __init_rand_distribution(struct thread_data *td, struct fio_file *f) +static void __init_rand_distribution(struct thread_data *td, struct fio_file *f) { unsigned int range_size, seed; unsigned long nranges; @@ -1215,18 +1215,16 @@ static int __init_rand_distribution(struct thread_data *td, struct fio_file *f) pareto_init(&f->zipf, nranges, td->o.pareto_h.u.f, seed); else if (td->o.random_distribution == FIO_RAND_DIST_GAUSS) gauss_init(&f->gauss, nranges, td->o.gauss_dev.u.f, seed); - - return 1; } -static int init_rand_distribution(struct thread_data *td) +static bool init_rand_distribution(struct thread_data *td) { struct fio_file *f; unsigned int i; int state; if (td->o.random_distribution == FIO_RAND_DIST_RANDOM) - return 0; + return false; state = td_bump_runstate(td, TD_SETTING_UP); @@ -1234,8 +1232,7 @@ static int init_rand_distribution(struct thread_data *td) __init_rand_distribution(td, f); td_restore_runstate(td, state); - - return 1; + return true; } /* @@ -1275,16 +1272,16 @@ static int check_rand_gen_limits(struct thread_data *td, struct fio_file *f, return 0; } -int init_random_map(struct thread_data *td) +bool init_random_map(struct thread_data *td) { unsigned long long blocks; struct fio_file *f; unsigned int i; if (init_rand_distribution(td)) - return 0; + return true; if (!td_random(td)) - return 0; + return true; for_each_file(td, f, i) { uint64_t fsize = min(f->real_file_size, f->io_size); @@ -1292,7 +1289,7 @@ int init_random_map(struct thread_data *td) blocks = fsize / (unsigned long long) td->o.rw_min_bs; if (check_rand_gen_limits(td, f, blocks)) - return 1; + return false; if (td->o.random_generator == FIO_RAND_GEN_LFSR) { unsigned long seed; @@ -1317,14 +1314,14 @@ int init_random_map(struct thread_data *td) " a large number of jobs, try the 'norandommap'" " option or set 'softrandommap'. Or give" " a larger --alloc-size to fio.\n"); - return 1; + return false; } log_info("fio: file %s failed allocating random map. Running " "job without.\n", f->file_name); } - return 0; + return true; } void close_files(struct thread_data *td) @@ -1342,6 +1339,7 @@ void close_and_free_files(struct thread_data *td) { struct fio_file *f; unsigned int i; + bool use_free = td_ioengine_flagged(td, FIO_NOFILEHASH); dprint(FD_FILE, "close files\n"); @@ -1361,13 +1359,19 @@ void close_and_free_files(struct thread_data *td) td_io_unlink_file(td, f); } - sfree(f->file_name); + if (use_free) + free(f->file_name); + else + sfree(f->file_name); f->file_name = NULL; if (fio_file_axmap(f)) { axmap_free(f->io_axmap); f->io_axmap = NULL; } - sfree(f); + if (use_free) + free(f); + else + sfree(f); } td->o.filename = NULL; @@ -1481,7 +1485,10 @@ static struct fio_file *alloc_new_file(struct thread_data *td) { struct fio_file *f; - f = smalloc(sizeof(*f)); + if (td_ioengine_flagged(td, FIO_NOFILEHASH)) + f = calloc(1, sizeof(*f)); + else + f = smalloc(sizeof(*f)); if (!f) { assert(0); return NULL; @@ -1513,6 +1520,42 @@ bool exists_and_not_regfile(const char *filename) return true; } +static bool create_work_dirs(struct thread_data *td, const char *fname) +{ + char path[PATH_MAX]; + char *start, *end; + + if (td->o.directory) { + snprintf(path, PATH_MAX, "%s%c%s", td->o.directory, + FIO_OS_PATH_SEPARATOR, fname); + start = strstr(path, fname); + } else { + snprintf(path, PATH_MAX, "%s", fname); + start = path; + } + + end = start; + while ((end = strchr(end, FIO_OS_PATH_SEPARATOR)) != NULL) { + if (end == start) + break; + *end = '\0'; + errno = 0; +#ifdef CONFIG_HAVE_MKDIR_TWO + if (mkdir(path, 0600) && errno != EEXIST) { +#else + if (mkdir(path) && errno != EEXIST) { +#endif + log_err("fio: failed to create dir (%s): %d\n", + start, errno); + return false; + } + *end = FIO_OS_PATH_SEPARATOR; + end++; + } + td->flags |= TD_F_DIRS_CREATED; + return true; +} + int add_file(struct thread_data *td, const char *fname, int numjob, int inc) { int cur_files = td->files_index; @@ -1528,6 +1571,11 @@ int add_file(struct thread_data *td, const char *fname, int numjob, int inc) sprintf(file_name + len, "%s", fname); + if (strchr(fname, FIO_OS_PATH_SEPARATOR) && + !(td->flags & TD_F_DIRS_CREATED) && + !create_work_dirs(td, fname)) + return 1; + /* clean cloned siblings using existing files */ if (numjob && is_already_allocated(file_name) && !exists_and_not_regfile(fname)) @@ -1564,7 +1612,10 @@ int add_file(struct thread_data *td, const char *fname, int numjob, int inc) if (td->io_ops && td_ioengine_flagged(td, FIO_DISKLESSIO)) f->real_file_size = -1ULL; - f->file_name = smalloc_strdup(file_name); + if (td_ioengine_flagged(td, FIO_NOFILEHASH)) + f->file_name = strdup(file_name); + else + f->file_name = smalloc_strdup(file_name); if (!f->file_name) assert(0); @@ -1588,7 +1639,8 @@ int add_file(struct thread_data *td, const char *fname, int numjob, int inc) if (f->filetype == FIO_TYPE_FILE) td->nr_normal_files++; - set_already_allocated(file_name); + if (td->o.numjobs > 1) + set_already_allocated(file_name); if (inc) td->o.nr_files++; @@ -1689,10 +1741,10 @@ void unlock_file_all(struct thread_data *td, struct fio_file *f) unlock_file(td, f); } -static int recurse_dir(struct thread_data *td, const char *dirname) +static bool recurse_dir(struct thread_data *td, const char *dirname) { struct dirent *dir; - int ret = 0; + bool ret = false; DIR *D; D = opendir(dirname); @@ -1701,7 +1753,7 @@ static int recurse_dir(struct thread_data *td, const char *dirname) snprintf(buf, FIO_VERROR_SIZE, "opendir(%s)", dirname); td_verror(td, errno, buf); - return 1; + return true; } while ((dir = readdir(D)) != NULL) { @@ -1711,12 +1763,12 @@ static int recurse_dir(struct thread_data *td, const char *dirname) if (!strcmp(dir->d_name, ".") || !strcmp(dir->d_name, "..")) continue; - sprintf(full_path, "%s%s%s", dirname, FIO_OS_PATH_SEPARATOR, dir->d_name); + sprintf(full_path, "%s%c%s", dirname, FIO_OS_PATH_SEPARATOR, dir->d_name); if (lstat(full_path, &sb) == -1) { if (errno != ENOENT) { td_verror(td, errno, "stat"); - ret = 1; + ret = true; break; } } @@ -1768,7 +1820,10 @@ void dup_files(struct thread_data *td, struct thread_data *org) __f = alloc_new_file(td); if (f->file_name) { - __f->file_name = smalloc_strdup(f->file_name); + if (td_ioengine_flagged(td, FIO_NOFILEHASH)) + __f->file_name = strdup(f->file_name); + else + __f->file_name = smalloc_strdup(f->file_name); if (!__f->file_name) assert(0); @@ -113,6 +113,8 @@ All fio parser warnings are fatal, causing fio to exit with an error. .TP .BI \-\-max\-jobs \fR=\fPnr Set the maximum number of threads/processes to support to \fInr\fR. +NOTE: On Linux, it may be necessary to increase the shared-memory limit +(`/proc/sys/kernel/shmmax') if fio runs into errors while creating jobs. .TP .BI \-\-server \fR=\fPargs Start a backend server, with \fIargs\fR specifying what to listen to. @@ -576,6 +578,12 @@ fio generate filenames that are shared between the two. For instance, if `testfiles.$filenum' is specified, file number 4 for any job will be named `testfiles.4'. The default of `$jobname.$jobnum.$filenum' will be used if no other format specifier is given. +.P +If you specify a path then the directories will be created up to the main +directory for the file. So for example if you specify `a/b/c/$jobnum` then the +directories a/b/c will be created before the file setup part of the job. If you +specify \fBdirectory\fR then the path will be relative that directory, otherwise +it is treated as the absolute path. .RE .TP .BI unique_filename \fR=\fPbool @@ -905,13 +913,19 @@ should be associated with them. .TP .BI offset \fR=\fPint Start I/O at the provided offset in the file, given as either a fixed size in -bytes or a percentage. If a percentage is given, the next \fBblockalign\fR\-ed -offset will be used. Data before the given offset will not be touched. This +bytes or a percentage. If a percentage is given, the generated offset will be +aligned to the minimum \fBblocksize\fR or to the value of \fBoffset_align\fR if +provided. Data before the given offset will not be touched. This effectively caps the file size at `real_size \- offset'. Can be combined with \fBsize\fR to constrain the start and end range of the I/O workload. A percentage can be specified by a number between 1 and 100 followed by '%', for example, `offset=20%' to specify 20%. .TP +.BI offset_align \fR=\fPint +If set to non-zero value, the byte offset generated by a percentage \fBoffset\fR +is aligned upwards to this value. Defaults to 0 meaning that a percentage +offset is aligned to the minimum block size. +.TP .BI offset_increment \fR=\fPint If this is provided, then the real offset becomes `\fBoffset\fR + \fBoffset_increment\fR * thread_number', where the thread number is a counter that starts at 0 and @@ -1228,8 +1242,8 @@ WRITEs) that compresses to the specified level. Fio does this by providing a mix of random data and a fixed pattern. The fixed pattern is either zeros, or the pattern specified by \fBbuffer_pattern\fR. If the pattern option is used, it might skew the compression ratio slightly. Note that this is per -block size unit, for file/disk wide compression level that matches this -setting, you'll also want to set \fBrefill_buffers\fR. +block size unit, see \fBbuffer_compress_chunk\fR for setting a finer granularity +of compressible regions. .TP .BI buffer_compress_chunk \fR=\fPint See \fBbuffer_compress_percentage\fR. This setting allows fio to manage @@ -1237,7 +1251,8 @@ how big the ranges of random data and zeroed data is. Without this set, fio will provide \fBbuffer_compress_percentage\fR of blocksize random data, followed by the remaining zeroed. With this set to some chunk size smaller than the block size, fio can alternate random and zeroed data throughout the -I/O buffer. +I/O buffer. This is particularly useful when bigger block sizes are used +for a job. Defaults to 512. .TP .BI buffer_pattern \fR=\fPstr If set, fio will fill the I/O buffers with this pattern or with the contents @@ -1577,6 +1592,11 @@ the engine filename, e.g. `ioengine=external:/tmp/foo.o' to load ioengine `foo.o' in `/tmp'. The path can be either absolute or relative. See `engines/skeleton_external.c' in the fio source for details of writing an external I/O engine. +.TP +.B filecreate +Simply create the files and do no I/O to them. You still need to set +\fBfilesize\fR so that all the accounting still occurs, but no actual I/O will be +done other than creating the file. .SS "I/O engine specific parameters" In addition, there are some parameters which are only valid when a specific \fBioengine\fR is in use. These are used identically to normal parameters, @@ -2401,48 +2421,48 @@ the final stat output. .TP .BI write_bw_log \fR=\fPstr If given, write a bandwidth log for this job. Can be used to store data of -the bandwidth of the jobs in their lifetime. The included -\fBfio_generate_plots\fR script uses gnuplot to turn these -text files into nice graphs. See \fBwrite_lat_log\fR for behavior of -given filename. For this option, the postfix is `_bw.x.log', where `x' -is the index of the job (1..N, where N is the number of jobs). If -\fBper_job_logs\fR is false, then the filename will not include the job -index. See \fBLOG FILE FORMATS\fR section. -.TP -.BI write_lat_log \fR=\fPstr -Same as \fBwrite_bw_log\fR, except that this option stores I/O -submission, completion, and total latencies instead. If no filename is given -with this option, the default filename of `jobname_type.log' is -used. Even if the filename is given, fio will still append the type of -log. So if one specifies: +the bandwidth of the jobs in their lifetime. .RS +.P +If no str argument is given, the default filename of +`jobname_type.x.log' is used. Even when the argument is given, fio +will still append the type of log. So if one specifies: .RS .P -write_lat_log=foo +write_bw_log=foo .RE .P -The actual log names will be `foo_slat.x.log', `foo_clat.x.log', -and `foo_lat.x.log', where `x' is the index of the job (1..N, where N -is the number of jobs). This helps \fBfio_generate_plots\fR find the -logs automatically. If \fBper_job_logs\fR is false, then the filename -will not include the job index. See \fBLOG FILE FORMATS\fR section. +The actual log name will be `foo_bw.x.log' where `x' is the index +of the job (1..N, where N is the number of jobs). If +\fBper_job_logs\fR is false, then the filename will not include the +`.x` job index. +.P +The included \fBfio_generate_plots\fR script uses gnuplot to turn these +text files into nice graphs. See the \fBLOG FILE FORMATS\fR section for how data is +structured within the file. .RE .TP +.BI write_lat_log \fR=\fPstr +Same as \fBwrite_bw_log\fR, except this option creates I/O +submission (e.g., `name_slat.x.log'), completion (e.g., +`name_clat.x.log'), and total (e.g., `name_lat.x.log') latency +files instead. See \fBwrite_bw_log\fR for details about the +filename format and the \fBLOG FILE FORMATS\fR section for how data is structured +within the files. +.TP .BI write_hist_log \fR=\fPstr -Same as \fBwrite_lat_log\fR, but writes I/O completion latency -histograms. If no filename is given with this option, the default filename -of `jobname_clat_hist.x.log' is used, where `x' is the index of the -job (1..N, where N is the number of jobs). Even if the filename is given, -fio will still append the type of log. If \fBper_job_logs\fR is false, -then the filename will not include the job index. See \fBLOG FILE FORMATS\fR section. +Same as \fBwrite_bw_log\fR but writes an I/O completion latency +histogram file (e.g., `name_hist.x.log') instead. Note that this +file will be empty unless \fBlog_hist_msec\fR has also been set. +See \fBwrite_bw_log\fR for details about the filename format and +the \fBLOG FILE FORMATS\fR section for how data is structured +within the file. .TP .BI write_iops_log \fR=\fPstr -Same as \fBwrite_bw_log\fR, but writes IOPS. If no filename is given -with this option, the default filename of `jobname_type.x.log' is -used, where `x' is the index of the job (1..N, where N is the number of -jobs). Even if the filename is given, fio will still append the type of -log. If \fBper_job_logs\fR is false, then the filename will not include -the job index. See \fBLOG FILE FORMATS\fR section. +Same as \fBwrite_bw_log\fR, but writes an IOPS file (e.g. +`name_iops.x.log') instead. See \fBwrite_bw_log\fR for +details about the filename format and the \fBLOG FILE FORMATS\fR section for how data +is structured within the file. .TP .BI log_avg_msec \fR=\fPint By default, fio will log an entry in the iops, latency, or bw log for every @@ -2458,8 +2478,8 @@ histograms. Computing latency percentiles from averages of intervals using \fBlog_avg_msec\fR is inaccurate. Setting this option makes fio log histogram entries over the specified period of time, reducing log sizes for high IOPS devices while retaining percentile accuracy. See -\fBlog_hist_coarseness\fR as well. Defaults to 0, meaning histogram -logging is disabled. +\fBlog_hist_coarseness\fR and \fBwrite_hist_log\fR as well. +Defaults to 0, meaning histogram logging is disabled. .TP .BI log_hist_coarseness \fR=\fPint Integer ranging from 0 to 6, defining the coarseness of the resolution of @@ -2549,7 +2569,7 @@ Enable the reporting of percentiles of completion latencies. This option is mutually exclusive with \fBlat_percentiles\fR. .TP .BI lat_percentiles \fR=\fPbool -Enable the reporting of percentiles of IO latencies. This is similar to +Enable the reporting of percentiles of I/O latencies. This is similar to \fBclat_percentiles\fR, except that this includes the submission latency. This option is mutually exclusive with \fBclat_percentiles\fR. .TP @@ -2897,7 +2917,7 @@ The number of \fBread/write/trim\fR requests issued, and how many of them were short or dropped. .TP .B IO latency -These values are for \fBlatency-target\fR and related options. When +These values are for \fBlatency_target\fR and related options. When these options are engaged, this section describes the I/O depth required to meet the specified latency target. .RE @@ -18,7 +18,7 @@ * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * */ #include <unistd.h> @@ -72,22 +72,42 @@ enum { }; enum { - TD_F_VER_BACKLOG = 1U << 0, - TD_F_TRIM_BACKLOG = 1U << 1, - TD_F_READ_IOLOG = 1U << 2, - TD_F_REFILL_BUFFERS = 1U << 3, - TD_F_SCRAMBLE_BUFFERS = 1U << 4, - TD_F_VER_NONE = 1U << 5, - TD_F_PROFILE_OPS = 1U << 6, - TD_F_COMPRESS = 1U << 7, - TD_F_RESERVED = 1U << 8, /* not used */ - TD_F_COMPRESS_LOG = 1U << 9, - TD_F_VSTATE_SAVED = 1U << 10, - TD_F_NEED_LOCK = 1U << 11, - TD_F_CHILD = 1U << 12, - TD_F_NO_PROGRESS = 1U << 13, - TD_F_REGROW_LOGS = 1U << 14, - TD_F_MMAP_KEEP = 1U << 15, + __TD_F_VER_BACKLOG = 0, + __TD_F_TRIM_BACKLOG, + __TD_F_READ_IOLOG, + __TD_F_REFILL_BUFFERS, + __TD_F_SCRAMBLE_BUFFERS, + __TD_F_VER_NONE, + __TD_F_PROFILE_OPS, + __TD_F_COMPRESS, + __TD_F_COMPRESS_LOG, + __TD_F_VSTATE_SAVED, + __TD_F_NEED_LOCK, + __TD_F_CHILD, + __TD_F_NO_PROGRESS, + __TD_F_REGROW_LOGS, + __TD_F_MMAP_KEEP, + __TD_F_DIRS_CREATED, + __TD_F_LAST, /* not a real bit, keep last */ +}; + +enum { + TD_F_VER_BACKLOG = 1U << __TD_F_VER_BACKLOG, + TD_F_TRIM_BACKLOG = 1U << __TD_F_TRIM_BACKLOG, + TD_F_READ_IOLOG = 1U << __TD_F_READ_IOLOG, + TD_F_REFILL_BUFFERS = 1U << __TD_F_REFILL_BUFFERS, + TD_F_SCRAMBLE_BUFFERS = 1U << __TD_F_SCRAMBLE_BUFFERS, + TD_F_VER_NONE = 1U << __TD_F_VER_NONE, + TD_F_PROFILE_OPS = 1U << __TD_F_PROFILE_OPS, + TD_F_COMPRESS = 1U << __TD_F_COMPRESS, + TD_F_COMPRESS_LOG = 1U << __TD_F_COMPRESS_LOG, + TD_F_VSTATE_SAVED = 1U << __TD_F_VSTATE_SAVED, + TD_F_NEED_LOCK = 1U << __TD_F_NEED_LOCK, + TD_F_CHILD = 1U << __TD_F_CHILD, + TD_F_NO_PROGRESS = 1U << __TD_F_NO_PROGRESS, + TD_F_REGROW_LOGS = 1U << __TD_F_REGROW_LOGS, + TD_F_MMAP_KEEP = 1U << __TD_F_MMAP_KEEP, + TD_F_DIRS_CREATED = 1U << __TD_F_DIRS_CREATED, }; enum { @@ -591,12 +611,6 @@ enum { #define TD_ENG_FLAG_SHIFT 16 #define TD_ENG_FLAG_MASK ((1U << 16) - 1) -static inline enum fio_ioengine_flags td_ioengine_flags(struct thread_data *td) -{ - return (enum fio_ioengine_flags) - ((td->flags >> TD_ENG_FLAG_SHIFT) & TD_ENG_FLAG_MASK); -} - static inline void td_set_ioengine_flags(struct thread_data *td) { td->flags = (~(TD_ENG_FLAG_MASK << TD_ENG_FLAG_SHIFT) & td->flags) | @@ -5,6 +5,7 @@ struct thread_data; extern uint64_t ntime_since(const struct timespec *, const struct timespec *); +extern uint64_t ntime_since_now(const struct timespec *); extern uint64_t utime_since(const struct timespec *, const struct timespec *); extern uint64_t utime_since_now(const struct timespec *); extern uint64_t mtime_since(const struct timespec *, const struct timespec *); @@ -16,13 +16,17 @@ static struct fio_mutex *flow_lock; int flow_threshold_exceeded(struct thread_data *td) { struct fio_flow *flow = td->flow; - int sign; + long long flow_counter; if (!flow) return 0; - sign = td->o.flow > 0 ? 1 : -1; - if (sign * flow->flow_counter > td->o.flow_watermark) { + if (td->o.flow > 0) + flow_counter = flow->flow_counter; + else + flow_counter = -flow->flow_counter; + + if (flow_counter > td->o.flow_watermark) { if (td->o.flow_sleep) { io_u_quiesce(td); usleep(td->o.flow_sleep); @@ -1099,7 +1099,7 @@ static void gfio_show_clat_percentiles(struct gfio_client *gc, int ddir) { unsigned int *io_u_plat = ts->io_u_plat[ddir]; - unsigned long nr = ts->clat_stat[ddir].samples; + unsigned long long nr = ts->clat_stat[ddir].samples; fio_fp64_t *plist = ts->percentile_list; unsigned int len, scale_down; unsigned long long *ovals, minv, maxv; @@ -15,7 +15,7 @@ #if defined(ARCH_HAVE_CPU_CLOCK) #ifndef ARCH_CPU_CLOCK_CYCLES_PER_USEC -static unsigned long cycles_per_msec; +static unsigned long long cycles_per_msec; static unsigned long long cycles_start; static unsigned long long clock_mult; static unsigned long long max_cycles_mask; @@ -448,6 +448,14 @@ uint64_t ntime_since(const struct timespec *s, const struct timespec *e) return nsec + (sec * 1000000000LL); } +uint64_t ntime_since_now(const struct timespec *s) +{ + struct timespec now; + + fio_gettime(&now, NULL); + return ntime_since(s, &now); +} + uint64_t utime_since(const struct timespec *s, const struct timespec *e) { int64_t sec, usec; @@ -540,7 +548,7 @@ uint64_t time_since_now(const struct timespec *s) } #if defined(FIO_HAVE_CPU_AFFINITY) && defined(ARCH_HAVE_CPU_CLOCK) && \ - defined(CONFIG_SFAA) + defined(CONFIG_SYNC_SYNC) && defined(CONFIG_CMP_SWAP) #define CLOCK_ENTRIES_DEBUG 100000 #define CLOCK_ENTRIES_TEST 1000 @@ -562,9 +570,10 @@ struct clock_thread { struct clock_entry *entries; }; -static inline uint32_t atomic32_inc_return(uint32_t *seq) +static inline uint32_t atomic32_compare_and_swap(uint32_t *ptr, uint32_t old, + uint32_t new) { - return 1 + __sync_fetch_and_add(seq, 1); + return __sync_val_compare_and_swap(ptr, old, new); } static void *clock_thread_fn(void *data) @@ -572,7 +581,6 @@ static void *clock_thread_fn(void *data) struct clock_thread *t = data; struct clock_entry *c; os_cpu_mask_t cpu_mask; - uint32_t last_seq; unsigned long long first; int i; @@ -596,7 +604,6 @@ static void *clock_thread_fn(void *data) pthread_mutex_unlock(&t->started); first = get_cpu_clock(); - last_seq = 0; c = &t->entries[0]; for (i = 0; i < t->nr_entries; i++, c++) { uint32_t seq; @@ -604,11 +611,15 @@ static void *clock_thread_fn(void *data) c->cpu = t->cpu; do { - seq = atomic32_inc_return(t->seq); - if (seq < last_seq) + seq = *t->seq; + if (seq == UINT_MAX) break; + __sync_synchronize(); tsc = get_cpu_clock(); - } while (seq != *t->seq); + } while (seq != atomic32_compare_and_swap(t->seq, seq, seq + 1)); + + if (seq == UINT_MAX) + break; c->seq = seq; c->tsc = tsc; @@ -626,7 +637,7 @@ static void *clock_thread_fn(void *data) * The most common platform clock breakage is returning zero * indefinitely. Check for that and return failure. */ - if (!t->entries[i - 1].tsc && !t->entries[0].tsc) + if (i > 1 && !t->entries[i - 1].tsc && !t->entries[0].tsc) goto err; fio_cpuset_exit(&cpu_mask); @@ -18,7 +18,7 @@ * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * */ #include <locale.h> @@ -17,7 +17,7 @@ * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * */ #include <string.h> @@ -855,8 +855,10 @@ static int fixup_options(struct thread_data *td) if (o->compress_percentage == 100) { o->zero_buffers = 1; o->compress_percentage = 0; - } else if (!fio_option_is_set(o, refill_buffers)) + } else if (!fio_option_is_set(o, refill_buffers)) { o->refill_buffers = 1; + td->flags |= TD_F_REFILL_BUFFERS; + } } /* @@ -323,6 +323,17 @@ fetch: goto fetch; } +static void loop_cache_invalidate(struct thread_data *td, struct fio_file *f) +{ + struct thread_options *o = &td->o; + + if (o->invalidate_cache && !o->odirect) { + int fio_unused ret; + + ret = file_invalidate_cache(td, f); + } +} + static int get_next_rand_block(struct thread_data *td, struct fio_file *f, enum fio_ddir ddir, uint64_t *b) { @@ -334,6 +345,7 @@ static int get_next_rand_block(struct thread_data *td, struct fio_file *f, fio_file_reset(td, f); if (!get_next_rand_offset(td, f, ddir, b)) return 0; + loop_cache_invalidate(td, f); } dprint(FD_IO, "%s: rand offset failed, last=%llu, size=%llu\n", @@ -349,15 +361,14 @@ static int get_next_seq_offset(struct thread_data *td, struct fio_file *f, assert(ddir_rw(ddir)); + /* + * If we reach the end for a time based run, reset us back to 0 + * and invalidate the cache, if we need to. + */ if (f->last_pos[ddir] >= f->io_size + get_start_offset(td, f) && o->time_based) { - struct thread_options *o = &td->o; - uint64_t io_size = f->io_size + (f->io_size % o->min_bs[ddir]); - - if (io_size > f->last_pos[ddir]) - f->last_pos[ddir] = 0; - else - f->last_pos[ddir] = f->last_pos[ddir] - io_size; + f->last_pos[ddir] = f->file_offset; + loop_cache_invalidate(td, f); } if (f->last_pos[ddir] < f->real_file_size) { @@ -662,7 +673,7 @@ int io_u_quiesce(struct thread_data *td) static enum fio_ddir rate_ddir(struct thread_data *td, enum fio_ddir ddir) { enum fio_ddir odir = ddir ^ 1; - long usec; + uint64_t usec; uint64_t now; assert(ddir_rw(ddir)); @@ -1779,7 +1790,7 @@ static void account_io_completion(struct thread_data *td, struct io_u *io_u, if (td->parent) td = td->parent; - if (!td->o.stats) + if (!td->o.stats || td_ioengine_flagged(td, FIO_NOSTATS)) return; if (no_reduce) diff --git a/io_u_queue.c b/io_u_queue.c index 9994c787..8cf4c8c3 100644 --- a/io_u_queue.c +++ b/io_u_queue.c @@ -1,15 +1,15 @@ #include <stdlib.h> #include "io_u_queue.h" -int io_u_qinit(struct io_u_queue *q, unsigned int nr) +bool io_u_qinit(struct io_u_queue *q, unsigned int nr) { q->io_us = calloc(nr, sizeof(struct io_u *)); if (!q->io_us) - return 1; + return false; q->nr = 0; q->max = nr; - return 0; + return true; } void io_u_qexit(struct io_u_queue *q) @@ -17,7 +17,7 @@ void io_u_qexit(struct io_u_queue *q) free(q->io_us); } -int io_u_rinit(struct io_u_ring *ring, unsigned int nr) +bool io_u_rinit(struct io_u_ring *ring, unsigned int nr) { ring->max = nr + 1; if (ring->max & (ring->max - 1)) { @@ -32,10 +32,10 @@ int io_u_rinit(struct io_u_ring *ring, unsigned int nr) ring->ring = calloc(ring->max, sizeof(struct io_u *)); if (!ring->ring) - return 1; + return false; ring->head = ring->tail = 0; - return 0; + return true; } void io_u_rexit(struct io_u_ring *ring) diff --git a/io_u_queue.h b/io_u_queue.h index 118e5934..b5b8d2fa 100644 --- a/io_u_queue.h +++ b/io_u_queue.h @@ -2,6 +2,7 @@ #define FIO_IO_U_QUEUE #include <assert.h> +#include "lib/types.h" struct io_u; @@ -42,7 +43,7 @@ static inline int io_u_qempty(const struct io_u_queue *q) #define io_u_qiter(q, io_u, i) \ for (i = 0; i < (q)->nr && (io_u = (q)->io_us[i]); i++) -int io_u_qinit(struct io_u_queue *q, unsigned int nr); +bool io_u_qinit(struct io_u_queue *q, unsigned int nr); void io_u_qexit(struct io_u_queue *q); struct io_u_ring { @@ -52,7 +53,7 @@ struct io_u_ring { struct io_u **ring; }; -int io_u_rinit(struct io_u_ring *ring, unsigned int nr); +bool io_u_rinit(struct io_u_ring *ring, unsigned int nr); void io_u_rexit(struct io_u_ring *ring); static inline void io_u_rpush(struct io_u_ring *r, struct io_u *io_u) diff --git a/ioengines.c b/ioengines.c index 9638d804..1bfc06f9 100644 --- a/ioengines.c +++ b/ioengines.c @@ -133,8 +133,10 @@ static struct ioengine_ops *__load_ioengine(const char *name) /* * linux libaio has alias names, so convert to what we want */ - if (!strncmp(engine, "linuxaio", 8) || !strncmp(engine, "aio", 3)) + if (!strncmp(engine, "linuxaio", 8) || !strncmp(engine, "aio", 3)) { + dprint(FD_IO, "converting ioengine name: %s -> libaio\n", name); strcpy(engine, "libaio"); + } dprint(FD_IO, "load ioengine %s\n", engine); return find_ioengine(engine); @@ -436,6 +438,7 @@ int td_io_open_file(struct thread_data *td, struct fio_file *f) { assert(!fio_file_open(f)); assert(f->fd == -1); + assert(td->io_ops->open_file); if (td->io_ops->open_file(td, f)) { if (td->error == EINVAL && td->o.odirect) diff --git a/ioengines.h b/ioengines.h index 177cbc05..32b18eda 100644 --- a/ioengines.h +++ b/ioengines.h @@ -59,6 +59,8 @@ enum fio_ioengine_flags { FIO_MEMALIGN = 1 << 9, /* engine wants aligned memory */ FIO_BIT_BASED = 1 << 10, /* engine uses a bit base (e.g. uses Kbit as opposed to KB) */ FIO_FAKEIO = 1 << 11, /* engine pretends to do IO */ + FIO_NOSTATS = 1 << 12, /* don't do IO stats */ + FIO_NOFILEHASH = 1 << 13, /* doesn't hash the files for lookup later. */ }; /* diff --git a/lib/rbtree.c b/lib/rbtree.c index 883bc723..00a5a90b 100644 --- a/lib/rbtree.c +++ b/lib/rbtree.c @@ -15,7 +15,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. linux/lib/rbtree.c */ diff --git a/lib/rbtree.h b/lib/rbtree.h index c6cfe4a9..f31fc56d 100644 --- a/lib/rbtree.h +++ b/lib/rbtree.h @@ -14,7 +14,7 @@ You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software - Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. linux/include/linux/rbtree.h diff --git a/lib/types.h b/lib/types.h index 287a3b42..bb245068 100644 --- a/lib/types.h +++ b/lib/types.h @@ -1,7 +1,7 @@ #ifndef FIO_TYPES_H #define FIO_TYPES_H -#ifndef CONFIG_HAVE_BOOL +#if !defined(CONFIG_HAVE_BOOL) && !defined(__cplusplus) typedef int bool; #ifndef false #define false 0 @@ -18,7 +18,7 @@ * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * */ @@ -365,6 +365,8 @@ int initialize_fio(char *envp[]) compiletime_assert((offsetof(struct thread_options_pack, latency_percentile) % 8) == 0, "latency_percentile"); compiletime_assert((offsetof(struct jobs_eta, m_rate) % 8) == 0, "m_rate"); + compiletime_assert(__TD_F_LAST <= TD_ENG_FLAG_SHIFT, "TD_ENG_FLAG_SHIFT"); + err = endian_check(); if (err) { log_err("fio: endianness settings appear wrong.\n"); @@ -1843,6 +1843,10 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .help = "DAX Device based IO engine", }, #endif + { + .ival = "filecreate", + .help = "File creation engine", + }, { .ival = "external", .help = "Load external engine (append name)", .cb = str_ioengine_external_cb, @@ -2015,6 +2019,17 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .group = FIO_OPT_G_INVALID, }, { + .name = "offset_align", + .lname = "IO offset alignment", + .type = FIO_OPT_INT, + .off1 = offsetof(struct thread_options, start_offset_align), + .help = "Start IO from this offset alignment", + .def = "0", + .interval = 512, + .category = FIO_OPT_C_IO, + .group = FIO_OPT_G_INVALID, + }, + { .name = "offset_increment", .lname = "IO offset increment", .type = FIO_OPT_STR_VAL, @@ -3237,8 +3252,8 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .type = FIO_OPT_INT, .off1 = offsetof(struct thread_options, nice), .help = "Set job CPU nice value", - .minval = -19, - .maxval = 20, + .minval = -20, + .maxval = 19, .def = "0", .interval = 1, .category = FIO_OPT_C_GENERAL, @@ -4052,6 +4067,7 @@ struct fio_option fio_options[FIO_MAX_OPTS] = { .parent = "buffer_compress_percentage", .hide = 1, .help = "Size of compressible region in buffer", + .def = "512", .interval = 256, .category = FIO_OPT_C_IO, .group = FIO_OPT_G_IO_BUF, diff --git a/os/os-solaris.h b/os/os-solaris.h index 45268b23..2f137234 100644 --- a/os/os-solaris.h +++ b/os/os-solaris.h @@ -12,12 +12,15 @@ #include <sys/mman.h> #include <sys/dkio.h> #include <sys/byteorder.h> +#include <sys/statvfs.h> +#include <pthread.h> #include "../file.h" #define FIO_HAVE_CPU_AFFINITY #define FIO_HAVE_CHARDEV_SIZE #define FIO_USE_GENERIC_BDEV_SIZE +#define FIO_HAVE_FS_STAT #define FIO_USE_GENERIC_INIT_RANDOM_STATE #define FIO_HAVE_GETTID @@ -65,7 +68,27 @@ static inline int blockdev_invalidate_cache(struct fio_file *f) static inline unsigned long long os_phys_mem(void) { - return 0; + long pagesize, pages; + + pagesize = sysconf(_SC_PAGESIZE); + pages = sysconf(_SC_PHYS_PAGES); + if (pages == -1 || pagesize == -1) + return 0; + + return (unsigned long long) pages * (unsigned long long) pagesize; +} + +static inline unsigned long long get_fs_free_size(const char *path) +{ + unsigned long long ret; + struct statvfs s; + + if (statvfs(path, &s) < 0) + return -1ULL; + + ret = s.f_frsize; + ret *= (unsigned long long) s.f_bfree; + return ret; } static inline void os_random_seed(unsigned long seed, os_random_state_t *rs) diff --git a/os/os-windows.h b/os/os-windows.h index 36b421ee..520da19a 100644 --- a/os/os-windows.h +++ b/os/os-windows.h @@ -37,7 +37,7 @@ int rand_r(unsigned *); #define FIO_PREFERRED_ENGINE "windowsaio" #define FIO_PREFERRED_CLOCK_SOURCE CS_CGETTIME -#define FIO_OS_PATH_SEPARATOR "\\" +#define FIO_OS_PATH_SEPARATOR '\\' #define FIO_MAX_CPUS MAXIMUM_PROCESSORS @@ -155,7 +155,7 @@ extern int fio_cpus_split(os_cpu_mask_t *mask, unsigned int cpu); #endif #ifndef FIO_OS_PATH_SEPARATOR -#define FIO_OS_PATH_SEPARATOR "/" +#define FIO_OS_PATH_SEPARATOR '/' #endif #ifndef FIO_PREFERRED_CLOCK_SOURCE diff --git a/os/windows/install.wxs b/os/windows/install.wxs index 58244c56..6dfe231d 100755 --- a/os/windows/install.wxs +++ b/os/windows/install.wxs @@ -10,7 +10,7 @@ <Product Id="*" Codepage="1252" Language="1033" Manufacturer="fio" Name="fio" - UpgradeCode="2338A332-5511-43CF-B9BD-5C60496CCFCC" Version="3.1"> + UpgradeCode="2338A332-5511-43CF-B9BD-5C60496CCFCC" Version="3.2"> <Package Description="Flexible IO Tester" InstallerVersion="301" Keywords="Installer,MSI,Database" diff --git a/oslib/libmtd.c b/oslib/libmtd.c index 24e9db9c..385b9d2f 100644 --- a/oslib/libmtd.c +++ b/oslib/libmtd.c @@ -14,7 +14,7 @@ * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * * Author: Artem Bityutskiy * @@ -1002,7 +1002,6 @@ int mtd_torture(libmtd_t desc, const struct mtd_dev_info *mtd, int fd, int eb) } } - err = 0; normsg("PEB %d passed torture test, do not mark it a bad", eb); out: diff --git a/oslib/libmtd.h b/oslib/libmtd.h index b5fd3f35..a0c90dcb 100644 --- a/oslib/libmtd.h +++ b/oslib/libmtd.h @@ -13,7 +13,7 @@ * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * * Author: Artem Bityutskiy * diff --git a/oslib/libmtd_common.h b/oslib/libmtd_common.h index 35628fea..87f93b61 100644 --- a/oslib/libmtd_common.h +++ b/oslib/libmtd_common.h @@ -13,7 +13,7 @@ * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ /* Imported from mtd-utils by dehrenberg */ diff --git a/oslib/libmtd_int.h b/oslib/libmtd_int.h index cbe2ff5c..a08e5745 100644 --- a/oslib/libmtd_int.h +++ b/oslib/libmtd_int.h @@ -14,7 +14,7 @@ * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * * Author: Artem Bityutskiy * diff --git a/oslib/libmtd_legacy.c b/oslib/libmtd_legacy.c index 38dc2b71..137e80a9 100644 --- a/oslib/libmtd_legacy.c +++ b/oslib/libmtd_legacy.c @@ -13,7 +13,7 @@ * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. * * Author: Artem Bityutskiy * diff --git a/oslib/libmtd_xalloc.h b/oslib/libmtd_xalloc.h index 532b80ff..6ac595ad 100644 --- a/oslib/libmtd_xalloc.h +++ b/oslib/libmtd_xalloc.h @@ -21,7 +21,7 @@ * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ #ifndef __MTD_UTILS_XALLOC_H__ @@ -556,8 +556,8 @@ static int __handle_option(struct fio_option *o, const char *ptr, void *data, return 1; } if (o->minval && ull < o->minval) { - log_err("min value out of range: %llu" - " (%u min)\n", ull, o->minval); + log_err("min value out of range: %lld" + " (%d min)\n", ull, o->minval); return 1; } if (o->posval[0].ival) { @@ -135,11 +135,11 @@ static int double_cmp(const void *a, const void *b) return cmp; } -unsigned int calc_clat_percentiles(unsigned int *io_u_plat, unsigned long nr, +unsigned int calc_clat_percentiles(unsigned int *io_u_plat, unsigned long long nr, fio_fp64_t *plist, unsigned long long **output, unsigned long long *maxv, unsigned long long *minv) { - unsigned long sum = 0; + unsigned long long sum = 0; unsigned int len, i, j = 0; unsigned int oval_len = 0; unsigned long long *ovals = NULL; @@ -198,7 +198,7 @@ unsigned int calc_clat_percentiles(unsigned int *io_u_plat, unsigned long nr, /* * Find and display the p-th percentile of clat */ -static void show_clat_percentiles(unsigned int *io_u_plat, unsigned long nr, +static void show_clat_percentiles(unsigned int *io_u_plat, unsigned long long nr, fio_fp64_t *plist, unsigned int precision, bool is_clat, struct buf_output *out) { @@ -957,13 +957,13 @@ static void add_ddir_status_json(struct thread_stat *ts, struct group_run_stats *rs, int ddir, struct json_object *parent) { unsigned long long min, max, minv, maxv; - unsigned long long bw; + unsigned long long bw_bytes, bw; unsigned long long *ovals = NULL; double mean, dev, iops; unsigned int len; int i; const char *ddirname[] = {"read", "write", "trim"}; - struct json_object *dir_object, *tmp_object, *percentile_object, *clat_bins_object; + struct json_object *dir_object, *tmp_object, *percentile_object, *clat_bins_object = NULL; char buf[120]; double p_of_agg = 100.0; @@ -976,17 +976,20 @@ static void add_ddir_status_json(struct thread_stat *ts, json_object_add_value_object(parent, ts->unified_rw_rep ? "mixed" : ddirname[ddir], dir_object); + bw_bytes = 0; bw = 0; iops = 0.0; if (ts->runtime[ddir]) { uint64_t runt = ts->runtime[ddir]; - bw = ((1000 * ts->io_bytes[ddir]) / runt) / 1024; /* KiB/s */ + bw_bytes = ((1000 * ts->io_bytes[ddir]) / runt); /* Bytes/s */ + bw = bw_bytes / 1024; /* KiB/s */ iops = (1000.0 * (uint64_t) ts->total_io_u[ddir]) / runt; } json_object_add_value_int(dir_object, "io_bytes", ts->io_bytes[ddir]); json_object_add_value_int(dir_object, "io_kbytes", ts->io_bytes[ddir] >> 10); + json_object_add_value_int(dir_object, "bw_bytes", bw_bytes); json_object_add_value_int(dir_object, "bw", bw); json_object_add_value_float(dir_object, "iops", iops); json_object_add_value_int(dir_object, "runtime", ts->runtime[ddir]); @@ -1037,7 +1040,9 @@ static void add_ddir_status_json(struct thread_stat *ts, if (output_format & FIO_OUTPUT_JSON_PLUS) { clat_bins_object = json_create_object(); - json_object_add_value_object(tmp_object, "bins", clat_bins_object); + if (ts->clat_percentiles) + json_object_add_value_object(tmp_object, "bins", clat_bins_object); + for(i = 0; i < FIO_IO_U_PLAT_NR; i++) { if (ts->io_u_plat[ddir][i]) { snprintf(buf, sizeof(buf), "%llu", plat_idx_to_val(i)); @@ -1056,6 +1061,9 @@ static void add_ddir_status_json(struct thread_stat *ts, json_object_add_value_int(tmp_object, "max", max); json_object_add_value_float(tmp_object, "mean", mean); json_object_add_value_float(tmp_object, "stddev", dev); + if (output_format & FIO_OUTPUT_JSON_PLUS && ts->lat_percentiles) + json_object_add_value_object(tmp_object, "bins", clat_bins_object); + if (ovals) free(ovals); @@ -24,6 +24,16 @@ struct group_run_stats { #define FIO_IO_U_LAT_M_NR 12 /* + * Constants for clat percentiles + */ +#define FIO_IO_U_PLAT_BITS 6 +#define FIO_IO_U_PLAT_VAL (1 << FIO_IO_U_PLAT_BITS) +#define FIO_IO_U_PLAT_GROUP_NR 29 +#define FIO_IO_U_PLAT_NR (FIO_IO_U_PLAT_GROUP_NR * FIO_IO_U_PLAT_VAL) +#define FIO_IO_U_LIST_MAX_LEN 20 /* The size of the default and user-specified + list of percentiles */ + +/* * Aggregate clat samples to report percentile(s) of them. * * EXECUTIVE SUMMARY @@ -34,7 +44,7 @@ struct group_run_stats { * * FIO_IO_U_PLAT_GROUP_NR and FIO_IO_U_PLAT_BITS determine the maximum * range being tracked for latency samples. The maximum value tracked - * accurately will be 2^(GROUP_NR + PLAT_BITS -1) microseconds. + * accurately will be 2^(GROUP_NR + PLAT_BITS - 1) nanoseconds. * * FIO_IO_U_PLAT_GROUP_NR and FIO_IO_U_PLAT_BITS determine the memory * requirement of storing those aggregate counts. The memory used will @@ -98,22 +108,15 @@ struct group_run_stats { * 3 8 2 [256,511] 64 * 4 9 3 [512,1023] 64 * ... ... ... [...,...] ... - * 18 23 17 [8838608,+inf]** 64 + * 28 33 27 [8589934592,+inf]** 64 * * * Special cases: when n < (M-1) or when n == (M-1), in both cases, * the value cannot be rounded off. Use all bits of the sample as * index. * - * ** If a sample's MSB is greater than 23, it will be counted as 23. + * ** If a sample's MSB is greater than 33, it will be counted as 33. */ -#define FIO_IO_U_PLAT_BITS 6 -#define FIO_IO_U_PLAT_VAL (1 << FIO_IO_U_PLAT_BITS) -#define FIO_IO_U_PLAT_GROUP_NR 29 -#define FIO_IO_U_PLAT_NR (FIO_IO_U_PLAT_GROUP_NR * FIO_IO_U_PLAT_VAL) -#define FIO_IO_U_LIST_MAX_LEN 20 /* The size of the default and user-specified - list of percentiles */ - /* * Trim cycle count measurements */ @@ -290,7 +293,7 @@ extern void init_thread_stat(struct thread_stat *ts); extern void init_group_run_stat(struct group_run_stats *gs); extern void eta_to_str(char *str, unsigned long eta_sec); extern bool calc_lat(struct io_stat *is, unsigned long long *min, unsigned long long *max, double *mean, double *dev); -extern unsigned int calc_clat_percentiles(unsigned int *io_u_plat, unsigned long nr, fio_fp64_t *plist, unsigned long long **output, unsigned long long *maxv, unsigned long long *minv); +extern unsigned int calc_clat_percentiles(unsigned int *io_u_plat, unsigned long long nr, fio_fp64_t *plist, unsigned long long **output, unsigned long long *maxv, unsigned long long *minv); extern void stat_calc_lat_n(struct thread_stat *ts, double *io_u_lat); extern void stat_calc_lat_m(struct thread_stat *ts, double *io_u_lat); extern void stat_calc_lat_u(struct thread_stat *ts, double *io_u_lat); diff --git a/t/gen-rand.c b/t/gen-rand.c index 6c31f925..4e9d39c6 100644 --- a/t/gen-rand.c +++ b/t/gen-rand.c @@ -63,6 +63,6 @@ int main(int argc, char *argv[]) } printf("Passes=%lu, Fail=%lu\n", pass, fail); - + free(buckets); return 0; } diff --git a/thread_options.h b/thread_options.h index 1813cdc7..5a037bfe 100644 --- a/thread_options.h +++ b/thread_options.h @@ -78,6 +78,7 @@ struct thread_options { unsigned long long file_size_low; unsigned long long file_size_high; unsigned long long start_offset; + unsigned long long start_offset_align; unsigned int bs[DDIR_RWDIR_CNT]; unsigned int ba[DDIR_RWDIR_CNT]; @@ -355,6 +356,7 @@ struct thread_options_pack { uint64_t file_size_low; uint64_t file_size_high; uint64_t start_offset; + uint64_t start_offset_align; uint32_t bs[DDIR_RWDIR_CNT]; uint32_t ba[DDIR_RWDIR_CNT]; diff --git a/tools/fio_generate_plots b/tools/fio_generate_plots index a47bfa5c..8872206e 100755 --- a/tools/fio_generate_plots +++ b/tools/fio_generate_plots @@ -93,20 +93,26 @@ plot () { i=0 - for x in *_"$FILETYPE".log + for x in *_"$FILETYPE".log *_"$FILETYPE".*.log do - i=$((i+1)) - PT=$(echo $x | sed s/_"$FILETYPE".log//g) - if [ ! -z "$PLOT_LINE" ] - then - PLOT_LINE=$PLOT_LINE", " + if [ -e "$x" ]; then + i=$((i+1)) + PT=$(echo $x | sed 's/\(.*\)_'$FILETYPE'\(.*\).log$/\1\2/') + if [ ! -z "$PLOT_LINE" ] + then + PLOT_LINE=$PLOT_LINE", " + fi + + DEPTH=$(echo $PT | cut -d "-" -f 4) + PLOT_LINE=$PLOT_LINE"'$x' using (\$1/1000):(\$2/$SCALE) title \"Queue depth $DEPTH\" with lines ls $i" fi - - DEPTH=$(echo $PT | cut -d "-" -f 4) - PLOT_LINE=$PLOT_LINE"'$x' using (\$1/1000):(\$2/$SCALE) title \"Queue depth $DEPTH\" with lines ls $i" - done + if [ $i -eq 0 ]; then + echo "No log files found" + exit 1 + fi + OUTPUT="set output \"$TITLE-$FILETYPE.svg\" " echo " $PLOT_TITLE ; $YAXIS ; $DEFAULT_OPTS ; show style lines ; $OUTPUT ; plot " $PLOT_LINE | $GNUPLOT - diff --git a/tools/fio_jsonplus_clat2csv b/tools/fio_jsonplus_clat2csv index d4ac16e4..e63d6d8b 100755 --- a/tools/fio_jsonplus_clat2csv +++ b/tools/fio_jsonplus_clat2csv @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2.7 # # fio_jsonplus_clat2csv # @@ -107,8 +107,16 @@ def main(): prev_ddir = None for ddir in ddir_set: + if 'bins' in jsondata['jobs'][jobnum][ddir]['clat_ns']: + bins_loc = 'clat_ns' + elif 'bins' in jsondata['jobs'][jobnum][ddir]['lat_ns']: + bins_loc = 'lat_ns' + else: + raise RuntimeError("Latency bins not found. " + "Are you sure you are using json+ output?") + bins[ddir] = [[int(key), value] for key, value in - jsondata['jobs'][jobnum][ddir]['clat_ns'] + jsondata['jobs'][jobnum][ddir][bins_loc] ['bins'].iteritems()] bins[ddir] = sorted(bins[ddir], key=lambda bin: bin[0]) @@ -123,7 +131,7 @@ def main(): outfile = stub + '_job' + str(jobnum) + ext with open(outfile, 'w') as output: - output.write("clat_nsec, ") + output.write("{0}ec, ".format(bins_loc)) ddir_list = list(ddir_set) for ddir in ddir_list: output.write("{0}_count, {0}_cumulative, {0}_percentile, ". diff --git a/tools/fiologparser.py b/tools/fiologparser.py index 5a95009e..8549859f 100755 --- a/tools/fiologparser.py +++ b/tools/fiologparser.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python2.7 # # fiologparser.py # @@ -218,4 +218,3 @@ if __name__ == '__main__': print_all_stats(ctx, series) else: print_default(ctx, series) - diff --git a/tools/genfio b/tools/genfio index 68004520..286d814d 100755 --- a/tools/genfio +++ b/tools/genfio @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/usr/bin/bash # # Copyright (C) 2013 eNovance SAS <licensing@enovance.com> # Author: Erwan Velu <erwan@enovance.com> @@ -17,7 +17,7 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. BLK_SIZE= BLOCK_SIZE=4k diff --git a/tools/hist/fiologparser_hist.py b/tools/hist/fiologparser_hist.py index ad97a54d..2e05b92c 100755 --- a/tools/hist/fiologparser_hist.py +++ b/tools/hist/fiologparser_hist.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/python2.7 """ Utility for converting *_clat_hist* files generated by fio into latency statistics. diff --git a/tools/hist/fiologparser_hist.py.1 b/tools/hist/fiologparser_hist.py.1 index ed22c747..5dfacfe9 100644 --- a/tools/hist/fiologparser_hist.py.1 +++ b/tools/hist/fiologparser_hist.py.1 @@ -17,7 +17,7 @@ end-time, samples, min, avg, median, 90%, 95%, 99%, max 1000, 15, 192, 1678.107, 1788.859, 1856.076, 1880.040, 1899.208, 1888.000 2000, 43, 152, 1642.368, 1714.099, 1816.659, 1845.552, 1888.131, 1888.000 4000, 39, 1152, 1546.962, 1545.785, 1627.192, 1640.019, 1691.204, 1744 -... +\[char46].. .fi .PP diff --git a/tools/hist/half-bins.py b/tools/hist/half-bins.py index d592af00..1bba8ff7 100755 --- a/tools/hist/half-bins.py +++ b/tools/hist/half-bins.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2.7 +#!/usr/bin/python2.7 """ Cut the number bins in half in fio histogram output. Example usage: $ half-bins.py -c 2 output_clat_hist.1.log > smaller_clat_hist.1.log @@ -35,4 +35,3 @@ if __name__ == '__main__': 'e.g. coarseness of 4 merges each 2^4 = 16 consecutive ' 'bins.') main(p.parse_args()) - diff --git a/tools/plot/fio2gnuplot b/tools/plot/fio2gnuplot index a703ae33..5d31f13a 100755 --- a/tools/plot/fio2gnuplot +++ b/tools/plot/fio2gnuplot @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python2.7 # # Copyright (C) 2013 eNovance SAS <licensing@enovance.com> # Author: Erwan Velu <erwan@enovance.com> @@ -17,7 +17,7 @@ # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. import os import fnmatch diff --git a/unit_tests/steadystate_tests.py b/unit_tests/steadystate_tests.py index 91c79a45..5a74f956 100755 --- a/unit_tests/steadystate_tests.py +++ b/unit_tests/steadystate_tests.py @@ -1,10 +1,10 @@ -#!/usr/bin/python +#!/usr/bin/python2.7 # # steadystate_tests.py # # Test option parsing and functonality for fio's steady state detection feature. # -# steadystate_tests.py ./fio file-for-read-testing file-for-write-testing +# steadystate_tests.py --read file-for-read-testing --write file-for-write-testing ./fio # # REQUIREMENTS # Python 2.6+ @@ -252,7 +252,7 @@ static void dump_buf(char *buf, unsigned int len, unsigned long long offset, memset(fname, 0, sizeof(fname)); if (aux_path) - sprintf(fname, "%s%s", aux_path, FIO_OS_PATH_SEPARATOR); + sprintf(fname, "%s%c", aux_path, FIO_OS_PATH_SEPARATOR); strncpy(fname + strlen(fname), basename(ptr), buf_left - 1); @@ -1726,7 +1726,7 @@ void verify_save_state(int mask) char prefix[PATH_MAX]; if (aux_path) - sprintf(prefix, "%s%slocal", aux_path, FIO_OS_PATH_SEPARATOR); + sprintf(prefix, "%s%clocal", aux_path, FIO_OS_PATH_SEPARATOR); else strcpy(prefix, "local"); |