/*
 * Copyright (c) 1998-2001 Apple Computer, Inc. All rights reserved.
 *
 * @APPLE_LICENSE_HEADER_START@
 *
 * The contents of this file constitute Original Code as defined in and
 * are subject to the Apple Public Source License Version 2.0 (the
 * "License").  You may not use this file except in compliance with the
 * License.  Please obtain a copy of the License at
 * http://www.opensource.apple.com/apsl/ and read it before using this file.
 *
 * This Original Code and all software distributed under the License are
 * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
 * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
 * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT.  Please see the
 * License for the specific language governing rights and limitations
 * under the License.
 *
 * @APPLE_LICENSE_HEADER_END@
 *
 *	File:	fsx.c
 *	Author:	Avadis Tevanian, Jr.
 *
 *	File system exerciser. 
 *
 *	Rewrite and enhancements 1998-2001 Conrad Minshall -- conrad@mac.com
 *
 *	Various features from Joe Sokol, Pat Dirks, and Clark Warner.
 *
 *	Small changes to work under Linux -- davej@suse.de
 *
 *	Sundry porting patches from Guy Harris 12/2001
 *
 *	Checks for mmap last-page zero fill.
 *
 *	Updated license to APSL 2.0, 2004/7/27 - Jordan Hubbard
 *  
 *  Modified to test file system symantics using MPI. 07/2008 - Sumit Narayan
 *
 * $FreeBSD: src/tools/regression/fsx/fsx.c,v 1.5 2007/06/26 13:51:53 delphij Exp $
 *
 */

#include <stdint.h>
#include <sys/types.h>
#include <sys/stat.h>
#ifdef _UWIN
# include <sys/param.h>
# include <limits.h>
# include <time.h>
# include <strings.h>
#endif
#include <fcntl.h>
#include <sys/mman.h>
#ifndef MAP_FILE
# define MAP_FILE 0
#endif
#include <limits.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <stdarg.h>
#include <errno.h>

#define NUMPRINTCOLUMNS 32	/* # columns of data to print on each line */

#define FSX_MPI /* Enable tests using MPI */

#ifdef FSX_MPI
#include <mpi.h>
int rank;
#endif

/*
 *	A log entry is an operation and a bunch of arguments.
 */

struct log_entry {
	  int	operation;
	  int	args[3];
};

#define	LOGSIZE	1000

struct log_entry	oplog[LOGSIZE];	/* the log */
int			logptr = 0;	/* current position in log */
int			logcount = 0;	/* total ops */

/*
 *	Define operations
 */

#define	OP_READ		1
#define OP_WRITE	2
#define OP_TRUNCATE	3
#define OP_CLOSEOPEN	4
#define OP_MAPREAD	5
#define OP_MAPWRITE	6
#define OP_SKIPPED	7

int page_size;
int page_mask;

char	*original_buf;			/* a pointer to the original data */
char	*good_buf;			/* a pointer to the correct data */
char	*temp_buf;			/* a pointer to the current data */
char	*fname;				/* name of our test file */
int	fd;				/* fd for our test file */

off_t		file_size = 0;
off_t		biggest = 0;
char		state[256];
unsigned long	testcalls = 0;		/* calls to function "test" */

unsigned long	simulatedopcount = 0;	/* -b flag */
int	closeprob = 0;			/* -c flag */
int	debug = 0;			/* -d flag */
unsigned long	debugstart = 0;		/* -D flag */
unsigned long	maxfilelen = 256 * 1024;	/* -l flag */
int	sizechecks = 1;			/* -n flag disables them */
int	maxoplen = 64 * 1024;		/* -o flag */
int	quiet = 0;			/* -q flag */
unsigned long progressinterval = 0;	/* -p flag */
int	readbdy = 1;			/* -r flag */
int	style = 0;			/* -s flag */
int	truncbdy = 1;			/* -t flag */
int	writebdy = 1;			/* -w flag */
long	monitorstart = -1;		/* -m flag */
long	monitorend = -1;		/* -m flag */
int	lite = 0;			/* -L flag */
long	numops = -1;			/* -N flag */
int	randomoplen = 1;		/* -O flag disables it */
int	seed = 1;			/* -S flag */
int     mapped_writes = 1;	      /* -W flag disables */
int 	mapped_reads = 1;		/* -R flag disables it */
int	fsxgoodfd = 0;
FILE *	fsxlogf = NULL;
int badoff = -1;
int closeopen = 0;


void
vwarnc(code, fmt, ap)
   int code;
   const char *fmt;
   va_list ap;
{
   fprintf(stderr, "fsx: ");
   if (fmt != NULL) {
	  vfprintf(stderr, fmt, ap);
	  fprintf(stderr, ": ");
   }
   fprintf(stderr, "%s\n", strerror(code));
}


void
warn(const char * fmt, ...)
{
   va_list ap;
   va_start(ap, fmt);
   vwarnc(errno, fmt, ap);
   va_end(ap);
}


void
prt(char *fmt, ...)
{
   va_list args;
   va_start(args, fmt);
   vfprintf(stdout, fmt, args);

   va_end(args);

   if (fsxlogf) {
	  va_start(args, fmt);
	  vfprintf(fsxlogf, fmt, args);
	  va_end(args);
   }
}

void
prterr(char *prefix)
{
   prt("%s%s%s\n", prefix, prefix ? ": " : "", strerror(errno));
}


void
log4(int operation, int arg0, int arg1, int arg2)
{
   struct log_entry *le;

   le = &oplog[logptr];
   le->operation = operation;
   if (closeopen)
	  le->operation = ~ le->operation;
   le->args[0] = arg0;
   le->args[1] = arg1;
   le->args[2] = arg2;
   logptr++;
   logcount++;
   if (logptr >= LOGSIZE)
	  logptr = 0;
}


void
logdump(void)
{
   int	i, count, down;
   struct log_entry	*lp;

   prt("LOG DUMP (%d total operations):\n", logcount);
   if (logcount < LOGSIZE) {
	  i = 0;
	  count = logcount;
   } else {
	  i = logptr;
	  count = LOGSIZE;
   }
   for ( ; count > 0; count--) {
	  int opnum;

	  opnum = i+1 + (logcount/LOGSIZE)*LOGSIZE;
	  prt("%d(%d mod 256): ", opnum, opnum%256);
	  lp = &oplog[i];
	  if ((closeopen = lp->operation < 0))
		 lp->operation = ~ lp->operation;
			
	  switch (lp->operation) {
		 case OP_MAPREAD:
			prt("MAPREAD\t0x%x thru 0x%x\t(0x%x bytes)",
			    lp->args[0], lp->args[0] + lp->args[1] - 1,
			    lp->args[1]);
			if (badoff >= lp->args[0] && badoff <
				lp->args[0] + lp->args[1])
			   prt("\t***RRRR***");
			break;
		 case OP_MAPWRITE:
			prt("MAPWRITE 0x%x thru 0x%x\t(0x%x bytes)",
			    lp->args[0], lp->args[0] + lp->args[1] - 1,
			    lp->args[1]);
			if (badoff >= lp->args[0] && badoff <
				lp->args[0] + lp->args[1])
			   prt("\t******WWWW");
			break;
		 case OP_READ:
			prt("READ\t0x%x thru 0x%x\t(0x%x bytes)",
			    lp->args[0], lp->args[0] + lp->args[1] - 1,
			    lp->args[1]);
			if (badoff >= lp->args[0] &&
			    badoff < lp->args[0] + lp->args[1])
			   prt("\t***RRRR***");
			break;
		 case OP_WRITE:
			prt("WRITE\t0x%x thru 0x%x\t(0x%x bytes)",
			    lp->args[0], lp->args[0] + lp->args[1] - 1,
			    lp->args[1]);
			if (lp->args[0] > lp->args[2])
			   prt(" HOLE");
			else if (lp->args[0] + lp->args[1] > lp->args[2])
			   prt(" EXTEND");
			if ((badoff >= lp->args[0] || badoff >=lp->args[2]) &&
			    badoff < lp->args[0] + lp->args[1])
			   prt("\t***WWWW");
			break;
		 case OP_TRUNCATE:
			down = lp->args[0] < lp->args[1];
			prt("TRUNCATE %s\tfrom 0x%x to 0x%x",
			    down ? "DOWN" : "UP", lp->args[1], lp->args[0]);
			if (badoff >= lp->args[!down] &&
			    badoff < lp->args[!!down])
			   prt("\t******WWWW");
			break;
		 case OP_SKIPPED:
			prt("SKIPPED (no operation)");
			break;
		 default:
			prt("BOGUS LOG ENTRY (operation code = %d)!",
			    lp->operation);
	  }
	  if (closeopen)
		 prt("\n\t\tCLOSE/OPEN");
	  prt("\n");
	  i++;
	  if (i == LOGSIZE)
		 i = 0;
   }
}


void
save_buffer(char *buffer, off_t bufferlength, int fd)
{
   off_t ret;
   ssize_t byteswritten;

   if (fd <= 0 || bufferlength == 0)
	  return;

   if (bufferlength > SSIZE_MAX) {
	  prt("fsx flaw: overflow in save_buffer\n");
	  exit(67);
   }
   if (lite) {
	  off_t size_by_seek = lseek(fd, (off_t)0, SEEK_END);
	  if (size_by_seek == (off_t)-1)
		 prterr("save_buffer: lseek eof");
	  else if (bufferlength > size_by_seek) {
		 warn("save_buffer: .fsxgood file too short... will save 0x%llx bytes instead of 0x%llx\n", (unsigned long long)size_by_seek,
			  (unsigned long long)bufferlength);
		 bufferlength = size_by_seek;
	  }
   }

   ret = lseek(fd, (off_t)0, SEEK_SET);
   if (ret == (off_t)-1)
	  prterr("save_buffer: lseek 0");
	
   byteswritten = write(fd, buffer, (size_t)bufferlength);
   if (byteswritten != bufferlength) {
	  if (byteswritten == -1)
		 prterr("save_buffer write");
	  else
		 warn("save_buffer: short write, 0x%x bytes instead of 0x%llx\n",
			  (unsigned)byteswritten,
			  (unsigned long long)bufferlength);
   }
}


void
report_failure(int status)
{
   logdump();
	
   if (fsxgoodfd) {
	  if (good_buf) {
		 save_buffer(good_buf, file_size, fsxgoodfd);
		 prt("Correct content saved for comparison\n");
		 prt("(maybe hexdump \"%s\" vs \"%s.fsxgood\")\n",
			 fname, fname);
	  }
	  close(fsxgoodfd);
   }
   exit(status);
}


#define short_at(cp) ((unsigned short)((*((unsigned char *)(cp)) << 8) | \
					*(((unsigned char *)(cp)) + 1)))

void
check_buffers(unsigned offset, unsigned size)
{
   unsigned char c, t;
   unsigned i = 0;
   unsigned n = 0;
   unsigned op = 0;
   unsigned bad = 0;

   if (memcmp(good_buf + offset, temp_buf, size) != 0) {
	  prt("READ BAD DATA: offset = 0x%x, size = 0x%x\n",
		  offset, size);
	  prt("OFFSET\tGOOD\tBAD\tRANGE\n");
	  while (size > 0) {
		 c = good_buf[offset];
		 t = temp_buf[i];
		 if (c != t) {
			if (n == 0) {
			   bad = short_at(&temp_buf[i]);
			   prt("0x%5x\t0x%04x\t0x%04x", offset,
				   short_at(&good_buf[offset]), bad);
			   op = temp_buf[offset & 1 ? i+1 : i];
			}
			n++;
			badoff = offset;
		 }
		 offset++;
		 i++;
		 size--;
	  }
	  if (n) {
		 prt("\t0x%5x\n", n);
		 if (bad)
			prt("operation# (mod 256) for the bad data may be %u\n", ((unsigned)op & 0xff));
		 else
			prt("operation# (mod 256) for the bad data unknown, check HOLE and EXTEND ops\n");
	  } else
		 prt("????????????????\n");
	  report_failure(110);
   }
}

void
check_size(void)
{
   struct stat	statbuf;
   off_t	size_by_seek;

   if (fstat(fd, &statbuf)) {
	  prterr("check_size: fstat");
	  statbuf.st_size = -1;
   }
   size_by_seek = lseek(fd, (off_t)0, SEEK_END);
   if (file_size != statbuf.st_size || file_size != size_by_seek) {
	  prt("Size error: expected 0x%llx stat 0x%llx seek 0x%llx\n",
		  (unsigned long long)file_size,
		  (unsigned long long)statbuf.st_size,
		  (unsigned long long)size_by_seek);
	  report_failure(120);
   }
}


void
check_trunc_hack(void)
{
   struct stat statbuf;

   ftruncate(fd, (off_t)0);
   ftruncate(fd, (off_t)100000);
   fstat(fd, &statbuf);
   if (statbuf.st_size != (off_t)100000) {
	  prt("no extend on truncate! not posix!\n");
	  exit(130);
   }
   ftruncate(fd, (off_t)0);
}


void
doread(unsigned offset, unsigned size)
{
   off_t ret;
   unsigned iret;

   offset -= offset % readbdy;
   if (size == 0) {
	  if (!quiet && testcalls > simulatedopcount)
		 prt("skipping zero size read\n");
	  log4(OP_SKIPPED, OP_READ, offset, size);
	  return;
   }
   if (size + offset > file_size) {
	  if (!quiet && testcalls > simulatedopcount)
		 prt("skipping seek/read past end of file\n");
	  log4(OP_SKIPPED, OP_READ, offset, size);
	  return;
   }

   log4(OP_READ, offset, size, 0);

   if (testcalls <= simulatedopcount)
	  return;

   if (!quiet && ((progressinterval &&
				   testcalls % progressinterval == 0) ||
				  (debug &&
				   (monitorstart == -1 ||
					(offset + size > monitorstart &&
					 (monitorend == -1 || offset <= monitorend))))))
	  prt("%lu read\t0x%x thru\t0x%x\t(0x%x bytes)\n", testcalls,
		  offset, offset + size - 1, size);
   ret = lseek(fd, (off_t)offset, SEEK_SET);
   if (ret == (off_t)-1) {
	  prterr("doread: lseek");
	  report_failure(140);
   }
   iret = read(fd, temp_buf, size);
   if (iret != size) {
	  if (iret == -1)
		 prterr("doread: read");
	  else
		 prt("short read: 0x%x bytes instead of 0x%x\n",
			 iret, size);
	  report_failure(141);
   }
   check_buffers(offset, size);
}


void
check_eofpage(char *s, unsigned offset, char *p, int size)
{
   uintptr_t last_page, should_be_zero;

   if (offset + size <= (file_size & ~page_mask))
	  return;
   /*
	* we landed in the last page of the file
	* test to make sure the VM system provided 0's 
	* beyond the true end of the file mapping
	* (as required by mmap def in 1996 posix 1003.1)
	*/
   last_page = ((uintptr_t)p + (offset & page_mask) + size) & ~page_mask;

   for (should_be_zero = last_page + (file_size & page_mask);
		should_be_zero < last_page + page_size;
		should_be_zero++)
	  if (*(char *)should_be_zero) {
		 prt("Mapped %s: non-zero data past EOF (0x%llx) page offset 0x%x is 0x%04x\n",
			 s, file_size - 1, should_be_zero & page_mask,
			 short_at(should_be_zero));
		 report_failure(205);
	  }
}


void
domapread(unsigned offset, unsigned size)
{
   unsigned pg_offset;
   unsigned map_size;
   char    *p;

   offset -= offset % readbdy;
   if (size == 0) {
	  if (!quiet && testcalls > simulatedopcount)
		 prt("skipping zero size read\n");
	  log4(OP_SKIPPED, OP_MAPREAD, offset, size);
	  return;
   }
   if (size + offset > file_size) {
	  if (!quiet && testcalls > simulatedopcount)
		 prt("skipping seek/read past end of file\n");
	  log4(OP_SKIPPED, OP_MAPREAD, offset, size);
	  return;
   }

   log4(OP_MAPREAD, offset, size, 0);

   if (testcalls <= simulatedopcount)
	  return;

   if (!quiet && ((progressinterval &&
				   testcalls % progressinterval == 0) ||
				  (debug &&
				   (monitorstart == -1 ||
					(offset + size > monitorstart &&
					 (monitorend == -1 || offset <= monitorend))))))
	  prt("%lu mapread\t0x%x thru\t0x%x\t(0x%x bytes)\n", testcalls,
		  offset, offset + size - 1, size);

   pg_offset = offset & page_mask;
   map_size  = pg_offset + size;

   if ((p = (char *)mmap(0, map_size, PROT_READ, MAP_FILE | MAP_SHARED, fd,
						 (off_t)(offset - pg_offset))) == (char *)-1) {
	  prterr("domapread: mmap");
	  report_failure(190);
   }
   memcpy(temp_buf, p + pg_offset, size);

   check_eofpage("Read", offset, p, size);

   if (munmap(p, map_size) != 0) {
	  prterr("domapread: munmap");
	  report_failure(191);
   }

   check_buffers(offset, size);
}


void
gendata(char *original_buf, char *good_buf, unsigned offset, unsigned size)
{
   while (size--) {
	  good_buf[offset] = testcalls % 256; 
	  if (offset % 2)
		 good_buf[offset] += original_buf[offset];
	  offset++;
   }
}


void
dowrite(unsigned offset, unsigned size)
{
   off_t ret;
   unsigned iret;

   offset -= offset % writebdy;
   if (size == 0) {
	  if (!quiet && testcalls > simulatedopcount)
		 prt("skipping zero size write\n");
	  log4(OP_SKIPPED, OP_WRITE, offset, size);
	  return;
   }

   log4(OP_WRITE, offset, size, file_size);

   gendata(original_buf, good_buf, offset, size);
   if (file_size < offset + size) {
	  if (file_size < offset)
		 memset(good_buf + file_size, '\0', offset - file_size);
	  file_size = offset + size;
	  if (lite) {
		 warn("Lite file size bug in fsx!");
		 report_failure(149);
	  }
   }

   if (testcalls <= simulatedopcount)
	  return;

   if (!quiet && ((progressinterval &&
				   testcalls % progressinterval == 0) ||
				  (debug &&
				   (monitorstart == -1 ||
					(offset + size > monitorstart &&
					 (monitorend == -1 || offset <= monitorend))))))
	  prt("%lu write\t0x%x thru\t0x%x\t(0x%x bytes)\n", testcalls,
		  offset, offset + size - 1, size);
   ret = lseek(fd, (off_t)offset, SEEK_SET);
   if (ret == (off_t)-1) {
	  prterr("dowrite: lseek");
	  report_failure(150);
   }

#ifdef FSX_MPI
   if(rank == 0)
   {
#endif
   iret = write(fd, good_buf + offset, size);
   if (iret != size) {
	  if (iret == -1)
		 prterr("dowrite: write");
	  else
		 prt("short write: 0x%x bytes instead of 0x%x\n",
			 iret, size);
	  report_failure(151);
   }
#ifdef FSX_MPI
   }
#endif
}

void
domapwrite(unsigned offset, unsigned size)
{
   unsigned pg_offset;
   unsigned map_size;
   off_t    cur_filesize;
   char    *p;

   offset -= offset % writebdy;
   if (size == 0) {
	  if (!quiet && testcalls > simulatedopcount)
		 prt("skipping zero size write\n");
	  log4(OP_SKIPPED, OP_MAPWRITE, offset, size);
	  return;
   }
   cur_filesize = file_size;

   log4(OP_MAPWRITE, offset, size, 0);

   gendata(original_buf, good_buf, offset, size);
   if (file_size < offset + size) {
	  if (file_size < offset)
		 memset(good_buf + file_size, '\0', offset - file_size);
	  file_size = offset + size;
	  if (lite) {
		 warn("Lite file size bug in fsx!");
		 report_failure(200);
	  }
   }

   if (testcalls <= simulatedopcount)
	  return;

   if (!quiet && ((progressinterval &&
				   testcalls % progressinterval == 0) ||
				  (debug &&
				   (monitorstart == -1 ||
					(offset + size > monitorstart &&
					 (monitorend == -1 || offset <= monitorend))))))
	  prt("%lu mapwrite\t0x%x thru\t0x%x\t(0x%x bytes)\n", testcalls,
		  offset, offset + size - 1, size);

   if (file_size > cur_filesize) {
	  if (ftruncate(fd, file_size) == -1) {
		 prterr("domapwrite: ftruncate");
		 exit(201);
	  }
   }
   pg_offset = offset & page_mask;
   map_size  = pg_offset + size;

   if ((p = (char *)mmap(0, map_size, PROT_READ | PROT_WRITE,
						 MAP_FILE | MAP_SHARED, fd,
						 (off_t)(offset - pg_offset))) == (char *)-1) {
	  prterr("domapwrite: mmap");
	  report_failure(202);
   }
   memcpy(p + pg_offset, good_buf + offset, size);
   if (msync(p, map_size, 0) != 0) {
	  prterr("domapwrite: msync");
	  report_failure(203);
   }

   check_eofpage("Write", offset, p, size);

   if (munmap(p, map_size) != 0) {
	  prterr("domapwrite: munmap");
	  report_failure(204);
   }
}

void
dotruncate(unsigned size)
{
   int oldsize = file_size;

   size -= size % truncbdy;
   if (size > biggest) {
	  biggest = size;
#ifdef FSX_MPI
	  if(rank == 0)
#endif
	  if (!quiet && testcalls > simulatedopcount)
		 prt("truncating to largest ever: 0x%x\n", size);
   }

   log4(OP_TRUNCATE, size, (unsigned)file_size, 0);

   if (size > file_size)
	  memset(good_buf + file_size, '\0', size - file_size);
   file_size = size;

   if (testcalls <= simulatedopcount)
	  return;

#ifdef FSX_MPI
   if(rank == 0)
   {
#endif	
	  if ((progressinterval && testcalls % progressinterval == 0) ||
		  (debug && (monitorstart == -1 || monitorend == -1 ||
					 size <= monitorend)))
		 prt("%lu trunc\tfrom 0x%x to 0x%x\n", testcalls, oldsize, size);
	  if (ftruncate(fd, (off_t)size) == -1) {
		 prt("ftruncate1: %x\n", size);
		 prterr("dotruncate: ftruncate");
		 report_failure(160);
	  }
#ifdef FSX_MPI
   }
   MPI_Bcast(&file_size, 1, MPI_LONG, 0, MPI_COMM_WORLD);
#endif
}


void
writefileimage()
{
   ssize_t iret;

   if (lseek(fd, (off_t)0, SEEK_SET) == (off_t)-1) {
	  prterr("writefileimage: lseek");
	  report_failure(171);
   }
   iret = write(fd, good_buf, file_size);
   if ((off_t)iret != file_size) {
	  if (iret == -1)
		 prterr("writefileimage: write");
	  else
		 prt("short write: 0x%x bytes instead of 0x%llx\n",
			 iret, (unsigned long long)file_size);
	  report_failure(172);
   }
   if (lite ? 0 : ftruncate(fd, file_size) == -1) {
	  prt("ftruncate2: %llx\n", (unsigned long long)file_size);
	  prterr("writefileimage: ftruncate");
	  report_failure(173);
   }
}


void
docloseopen(void)
{ 
   if (testcalls <= simulatedopcount)
	  return;

   if (debug)
	  prt("%lu close/open\n", testcalls);
   if (close(fd)) {
	  prterr("docloseopen: close");
	  report_failure(180);
   }
   fd = open(fname, O_RDWR, 0);
   if (fd < 0) {
	  prterr("docloseopen: open");
	  report_failure(181);
   }
}


void
test(void)
{
   unsigned long	offset;
   unsigned long	size = maxoplen;
   unsigned long	rv = random();
   unsigned long	op = rv % (3 + !lite + mapped_writes);

   /* turn off the map read if necessary */

   if (op == 2 && !mapped_reads)
	  op = 0;

#ifdef FSX_MPI
   MPI_Bcast(&op, 1, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD);
   if(rank == 0)
#endif
	  if (simulatedopcount > 0 && testcalls == simulatedopcount)
		 writefileimage();

   testcalls++;

   if (closeprob)
   {
	  closeopen = (rv >> 3) < (1 << 28) / closeprob;
#ifdef FSX_MPI
	  MPI_Bcast(&closeopen, 1, MPI_INT, 0, MPI_COMM_WORLD);
#endif
   }
   if (debugstart > 0 && testcalls >= debugstart)
	  debug = 1;

   if (!quiet && testcalls < simulatedopcount && testcalls % 100000 == 0)
	  prt("%lu...\n", testcalls);

   /*
	* READ:	op = 0
	* WRITE:	op = 1
	* MAPREAD:     op = 2
	* TRUNCATE:	op = 3
	* MAPWRITE:    op = 3 or 4
	*/

   if (lite ? 0 : op == 3 && style == 0) /* vanilla truncate? */
	  dotruncate(random() % maxfilelen);
   else {
	  if (randomoplen)
		 size = random() % (maxoplen+1);
	  if (lite ? 0 : op == 3)
		 dotruncate(size);
	  else {
		 offset = random();
		 if (op == 1 || op == (lite ? 3 : 4)) {
			offset %= maxfilelen;
			if (offset + size > maxfilelen)
			   size = maxfilelen - offset;
			if (op != 1)
			   domapwrite(offset, size);
			else
			{
#ifdef FSX_MPI
			   MPI_Bcast(&offset, 1, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD);
			   MPI_Bcast(&size, 1, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD);
#endif
			   dowrite(offset, size);
			}

#ifdef FSX_MPI
			MPI_Barrier(MPI_COMM_WORLD);
			if(rank != 0 && size != 0)
			   doread(offset, size);
#endif
		 } else {
			if (file_size)
			   offset %= file_size;
			else
			   offset = 0;
			if (offset + size > file_size)
			   size = file_size - offset;
			if (op != 0)
			   domapread(offset, size);
			else
			{
#ifdef FSX_MPI
			   MPI_Bcast(&offset, 1, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD);
			   MPI_Bcast(&size, 1, MPI_UNSIGNED_LONG, 0, MPI_COMM_WORLD);
#endif
			   doread(offset, size);
			}
		 }
	  }
   }
   if (sizechecks && testcalls > simulatedopcount)
   {
#ifdef FSX_MPI
	  MPI_Barrier(MPI_COMM_WORLD);
#endif
	  check_size();
   }
   if (closeopen)
	  docloseopen();
}


void
cleanup(sig)
   int	sig;
{
   if (sig)
	  prt("signal %d\n", sig);
   prt("testcalls = %lu\n", testcalls);
   exit(sig);
}


void
usage(void)
{
   fprintf(stdout, "usage: %s",
		   "fsx [-dnqLOW] [-b opnum] [-c Prob] [-l flen] [-m start:end] [-o oplen] [-p progressinterval] [-r readbdy] [-s style] [-t truncbdy] [-w writebdy] [-D startingop] [-N numops] [-P dirpath] [-S seed] fname\n\
	-b opnum: beginning operation number (default 1)\n\
	-c P: 1 in P chance of file close+open at each op (default infinity)\n\
	-d: debug output for all operations\n\
	-l flen: the upper bound on file size (default 262144)\n\
	-m startop:endop: monitor (print debug output) specified byte range (default 0:infinity)\n\
	-n: no verifications of file size\n\
	-o oplen: the upper bound on operation size (default 65536)\n\
	-p progressinterval: debug output at specified operation interval\n\
	-q: quieter operation\n\
	-r readbdy: 4096 would make reads page aligned (default 1)\n\
	-s style: 1 gives smaller truncates (default 0)\n\
	-t truncbdy: 4096 would make truncates page aligned (default 1)\n\
	-w writebdy: 4096 would make writes page aligned (default 1)\n\
	-D startingop: debug output starting at specified operation\n\
	-L: fsxLite - no file creations & no file size changes\n\
	-N numops: total # operations to do (default infinity)\n\
	-O: use oplen (see -o flag) for every op (default random)\n\
	-P dirpath: save .fsxlog and .fsxgood files in dirpath (default ./)\n\
	-S seed: for random # generator (default 1) 0 gets timestamp\n\
	-W: mapped write operations DISabled\n\
	-R: mapped read operations DISabled)\n\
	fname: this filename is REQUIRED (no default)\n");
   exit(90);
}


int
getnum(char *s, char **e)
{
   int ret = -1;

   *e = (char *) 0;
   ret = strtol(s, e, 0);
   if (*e)
	  switch (**e) {
		 case 'b':
		 case 'B':
			ret *= 512;
			*e = *e + 1;
			break;
		 case 'k':
		 case 'K':
			ret *= 1024;
			*e = *e + 1;
			break;
		 case 'm':
		 case 'M':
			ret *= 1024*1024;
			*e = *e + 1;
			break;
		 case 'w':
		 case 'W':
			ret *= 4;
			*e = *e + 1;
			break;
	  }
   return (ret);
}


int
main(int argc, char **argv)
{
   int	i, ch;
   char	*endp;
   char goodfile[1024];
   char logfile[1024];

#ifdef FSX_MPI
   int num_processors, ss;

   MPI_Init(&argc, &argv);
   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
#endif

   goodfile[0] = 0;
   logfile[0] = 0;

   page_size = getpagesize();
   page_mask = page_size - 1;

   setvbuf(stdout, (char *)0, _IOLBF, 0); /* line buffered stdout */

   while ((ch = getopt(argc, argv, "b:c:dl:m:no:p:qr:s:t:w:D:LN:OP:RS:W"))
		  != EOF)
	  switch (ch) {
		 case 'b':
			simulatedopcount = getnum(optarg, &endp);
			if (!quiet)
			   fprintf(stdout, "Will begin at operation %ld\n",
					   simulatedopcount);
			if (simulatedopcount == 0)
			   usage();
			simulatedopcount -= 1;
			break;
		 case 'c':
			closeprob = getnum(optarg, &endp);
			if (!quiet)
			   fprintf(stdout,
					   "Chance of close/open is 1 in %d\n",
					   closeprob);
			if (closeprob <= 0)
			   usage();
			break;
		 case 'd':
			debug = 1;
			break;
		 case 'l':
			maxfilelen = getnum(optarg, &endp);
			if (maxfilelen <= 0)
			   usage();
			break;
		 case 'm':
			monitorstart = getnum(optarg, &endp);
			if (monitorstart < 0)
			   usage();
			if (!endp || *endp++ != ':')
			   usage();
			monitorend = getnum(endp, &endp);
			if (monitorend < 0)
			   usage();
			if (monitorend == 0)
			   monitorend = -1; /* aka infinity */
			debug = 1;
		 case 'n':
			sizechecks = 0;
			break;
		 case 'o':
			maxoplen = getnum(optarg, &endp);
			if (maxoplen <= 0)
			   usage();
			break;
		 case 'p':
			progressinterval = getnum(optarg, &endp);
			if (progressinterval < 0)
			   usage();
			break;
		 case 'q':
			quiet = 1;
			break;
		 case 'r':
			readbdy = getnum(optarg, &endp);
			if (readbdy <= 0)
			   usage();
			break;
		 case 's':
			style = getnum(optarg, &endp);
			if (style < 0 || style > 1)
			   usage();
			break;
		 case 't':
			truncbdy = getnum(optarg, &endp);
			if (truncbdy <= 0)
			   usage();
			break;
		 case 'w':
			writebdy = getnum(optarg, &endp);
			if (writebdy <= 0)
			   usage();
			break;
		 case 'D':
			debugstart = getnum(optarg, &endp);
			if (debugstart < 1)
			   usage();
			break;
		 case 'L':
			lite = 1;
			break;
		 case 'N':
			numops = getnum(optarg, &endp);
			if (numops < 0)
			   usage();
			break;
		 case 'O':
			randomoplen = 0;
			break;
		 case 'P':
			strncpy(goodfile, optarg, sizeof(goodfile));
			strcat(goodfile, "/");
			strncpy(logfile, optarg, sizeof(logfile));
			strcat(logfile, "/");
			break;
		 case 'R':
			mapped_reads = 0;
			if (!quiet)
			   fprintf(stdout, "mapped reads DISABLED\n");
			break;
		 case 'S':
			seed = getnum(optarg, &endp);
			if (seed == 0)
			   seed = time(0) % 10000;
			if (!quiet)
			   fprintf(stdout, "Seed set to %d\n", seed);
			if (seed < 0)
			   usage();
			break;
		 case 'W':
			mapped_writes = 0;
			if (!quiet)
			   fprintf(stdout, "mapped writes DISABLED\n");
			break;

		 default:
			usage();
			/* NOTREACHED */
	  }
   argc -= optind;
   argv += optind;
   if (argc != 1)
	  usage();
   fname = argv[0];

   signal(SIGHUP,	cleanup);
   signal(SIGINT,	cleanup);
   signal(SIGPIPE,	cleanup);
   signal(SIGALRM,	cleanup);
   signal(SIGTERM,	cleanup);
   signal(SIGXCPU,	cleanup);
   signal(SIGXFSZ,	cleanup);
   signal(SIGVTALRM,	cleanup);
   signal(SIGUSR1,	cleanup);
   signal(SIGUSR2,	cleanup);

   initstate(seed, state, 256);
   setstate(state);
#ifdef FSX_MPI
   if(rank == 0)
   {
#endif
	  fd = open(fname, O_RDWR|(lite ? 0 : O_CREAT|O_TRUNC), 0666);
	  if (fd < 0) {
		 prterr(fname);
		 exit(91);
	  }
	  strncat(goodfile, fname, 256);
	  strcat (goodfile, ".fsxgood");
	  fsxgoodfd = open(goodfile, O_RDWR|O_CREAT|O_TRUNC, 0666);
	  if (fsxgoodfd < 0) {
		 prterr(goodfile);
		 exit(92);
	  }
	  strncat(logfile, fname, 256);
	  strcat (logfile, ".fsxlog");
	  fsxlogf = fopen(logfile, "w");
	  if (fsxlogf == NULL) {
		 prterr(logfile);
		 exit(93);
	  }
	  if (lite) {
		 off_t ret;
		 file_size = maxfilelen = lseek(fd, (off_t)0, SEEK_END);
		 if (file_size == (off_t)-1) {
			prterr(fname);
			warn("main: lseek eof");
			exit(94);
		 }
		 ret = lseek(fd, (off_t)0, SEEK_SET);
		 if (ret == (off_t)-1) {
			prterr(fname);
			warn("main: lseek 0");
			exit(95);
		 }
	  }
	  original_buf = (char *) malloc(maxfilelen);
	  for (i = 0; i < maxfilelen; i++)
		 original_buf[i] = random() % 256;
	  good_buf = (char *) malloc(maxfilelen);
	  memset(good_buf, '\0', maxfilelen);
	  temp_buf = (char *) malloc(maxoplen);
	  memset(temp_buf, '\0', maxoplen);
	  if (lite) {	/* zero entire existing file */
		 ssize_t written;
		  
		 written = write(fd, good_buf, (size_t)maxfilelen);
		 if (written != maxfilelen) {
			if (written == -1) {
			   prterr(fname);
			   warn("main: error on write");
			} else
			   warn("main: short write, 0x%x bytes instead of 0x%x\n",
					(unsigned)written, maxfilelen);
			exit(98);
		 }
	  } else 
		 check_trunc_hack();
#ifdef FSX_MPI
   }
   else
   {
	  original_buf = (char *) malloc(maxfilelen);
	  good_buf = (char *) malloc(maxfilelen);
	  memset(good_buf, '\0', maxfilelen);
	  temp_buf = (char *) malloc(maxoplen);
	  memset(temp_buf, '\0', maxoplen);
   }

   /* Broadcast the original buffer. */
   MPI_Bcast(original_buf, maxfilelen, MPI_CHAR, 0, MPI_COMM_WORLD);

   if(rank != 0)
   {
	  fd = open(fname, O_RDWR| O_TRUNC, 0666);
	  if(fd < 0)
	  {
		 prterr(fname);
		 exit(91);
	  }

	  strncat(goodfile, fname, 256);
	  strcat (goodfile, ".fsxgood");
	  fsxgoodfd = open(goodfile, O_RDWR | O_TRUNC, 0666);
	  if (fsxgoodfd < 0) {
		 prterr(goodfile);
		 exit(92);
	  }

	  strncat(logfile, fname, 256);
	  strcat (logfile, ".fsxlog");
	  fsxlogf = fopen(logfile, "w");
	  if (fsxlogf == NULL) {
		 prterr(logfile);
		 exit(93);
	  }
   }

   /* Barrier - let all process have the file open. */
   MPI_Barrier(MPI_COMM_WORLD);
#endif

   while (numops == -1 || numops--)
   {
#ifdef FSX_MPI
	  MPI_Barrier(MPI_COMM_WORLD);
#endif
	  test();
   }
	
   if (close(fd)) {
	  prterr("close");
	  report_failure(99);
   }

#ifdef FSX_MPI
   if(rank == 0)
#endif
	  prt("All operations completed A-OK!\n");

#ifdef FSX_MPI
   MPI_Barrier(MPI_COMM_WORLD);
   MPI_Finalize();
#endif

   exit(0);
   return 0;
}