diff --git a/NEWS b/NEWS index 627b324..a0b58d2 100644 --- a/NEWS +++ b/NEWS @@ -1,7 +1,14 @@ HIO NEWS ======== -Last updated 2016-06-18 +Last updated 2016-07-19 + +Release hio.1.3.0.5 + +This release adds additional performance instrumentation and diagnostic +messages on test failure. A new action "dbuf" is added to xexec to specify +the write data pattern used for data compare. The test cases have been updated +to specify the pattern previously used by default. Release hio.1.3.0.4 @@ -32,7 +39,7 @@ at https://github.com/hpc/libhio . We're still working out our release procedures, so for now the release is designated by the tag hio.1.3.0.0. Changes since the rc version are primarily fixes and performance enhancements -to optimixed mode. Additionally, there are a number of minor test fixes and +to optimised mode. Additionally, there are a number of minor test fixes and functional enhancements. Release hio.1.3.0.0-rc1: diff --git a/test/cw_misc.c b/test/cw_misc.c index 1385ed4..544be97 100644 --- a/test/cw_misc.c +++ b/test/cw_misc.c @@ -335,7 +335,7 @@ void hex_dump(void *data, int size) { skipped = 0; } /* print rest of buffer if not empty */ - printf("[%4.4s] %-50.50s %s\n", addrstr, hexstr, charstr); + printf("[%6.6s] %-50.50s %s\n", addrstr, hexstr, charstr); } } @@ -527,7 +527,7 @@ I64 GetCPUaffinity(void) { // eng_not(buf, sizeof(buf), 0.00835, "D5.4", "Sec") returns "8.3500 mSec" //------------------------------------------------------------------------------- char * eng_not(char * buf, size_t len, double val, char * format, char * unit) { - static const char *pref[] = {"y", "z", "a", "f", "p", "n", "u", "m", "", + static const char *pref[] = {"y", "z", "a", "f", "p", "n", "u", "m", " ", "k", "M", "G", "T", "P", "E", "Z", "Y"}; int zofs = 8; // pref[8] is the empty prefix "" for multiplier 1.0 double lval, mant; diff --git a/test/cw_misc.h b/test/cw_misc.h index 2a6aecc..16c1e17 100644 --- a/test/cw_misc.h +++ b/test/cw_misc.h @@ -27,6 +27,8 @@ typedef int64_t I64; typedef uint32_t U32; typedef int32_t I32; #define DIM1(array) ( sizeof(array) / sizeof(array[0]) ) +#define BDYUP(x, pwr2) ((unsigned)(x+pwr2-1) & ~((unsigned)pwr2-1)) +#define BDYDN(x, pwr2) ((unsigned)(x ) & ~((unsigned)pwr2-1)) #ifndef MAX #define MAX(a,b) ( (a) > (b) ? (a) : (b) ) #endif diff --git a/test/run02 b/test/run02 index c41428f..57fe9ac 100755 --- a/test/run02 +++ b/test/run02 @@ -19,9 +19,10 @@ batch_sub $(( $ranks * $blksz * $nblk )) cmdw=" name run02w v $verbose_lev d $debug_lev mi 0 /@@ Read and write N-N test case with read data value checking @/ + dbuf RAND22P 20Mi hi MYCTX $HIO_TEST_ROOTS hda NTNDS 97 WRITE,CREAT UNIQUE hdo - heo MYEL WRITE,CREAT,TRUNC 20Mi + heo MYEL WRITE,CREAT,TRUNC hvp c. . lc $nblk hew 0 $blksz @@ -32,9 +33,10 @@ cmdw=" cmdr=" name run02r v $verbose_lev d $debug_lev mi 32 /@@ Read and write N-N test case with read data value checking @/ + dbuf RAND22P 20Mi hi MYCTX $HIO_TEST_ROOTS hda NTNDS 97 READ UNIQUE hdo - heo MYEL READ 20Mi + heo MYEL READ hvp c. . hck on lc $nblk diff --git a/test/run03 b/test/run03 index 684c3d8..789d7f9 100755 --- a/test/run03 +++ b/test/run03 @@ -19,10 +19,11 @@ batch_sub $(( $ranks * $blksz * $nblkpseg * $nseg )) cmdw=" name run03w v $verbose_lev d $debug_lev mi 0 /@@ Write N-1 test case @/ + dbuf RAND22P 20Mi hi MY_CTX $HIO_TEST_ROOTS hdu NT1_DS 98 ALL hda NT1_DS 98 WRITE,CREAT SHARED hdo - heo MY_EL WRITE,CREAT,TRUNC 20Mi + heo MY_EL WRITE,CREAT,TRUNC hvp c. . lc $nseg hsegr 0 $segsz 0 @@ -36,9 +37,10 @@ cmdw=" cmdr=" name run03r v $verbose_lev d $debug_lev mi 32 /@@ Read N-1 test case with data checking @/ + dbuf RAND22P 20Mi hi MY_CTX $HIO_TEST_ROOTS hda NT1_DS 98 READ SHARED hdo - heo MY_EL READ 20Mi + heo MY_EL READ hvp c. . hck on lc $nseg diff --git a/test/run04 b/test/run04 index 9857468..d5e4d09 100755 --- a/test/run04 +++ b/test/run04 @@ -27,7 +27,7 @@ fails=" /@@ 1. Basic successful function @/ hi MY_CTX $HIO_TEST_ROOTS hda NTN_DS 97 WRITE,CREAT UNIQUE hdo - heo MY_EL WRITE,CREAT,TRUNC 20Mi + heo MY_EL WRITE,CREAT,TRUNC hvp c. . hec hdc hdf hf @@ -61,7 +61,7 @@ fails=" /@@ 6. HEO without HDO @/ hxrc ERR_BAD_PARAM - heo MY_EL WRITE,CREAT,TRUNC 20Mi + heo MY_EL WRITE,CREAT,TRUNC /@@ 7. HI, HDA READ HDO non-existent DS ID @/ hi MY_CTX $HIO_TEST_ROOTS diff --git a/test/run05 b/test/run05 index 301c5a3..8b87b45 100755 --- a/test/run05 +++ b/test/run05 @@ -18,9 +18,10 @@ batch_sub $(( $ranks * $blksz * $nblk )) cmdw=" name run05w v $verbose_lev d $debug_lev mi 0 /@@ Create and reuse N-N dataset - i.e., ensure TRUNC works @/ + dbuf RAND22P 20Mi hi MY_CTX $HIO_TEST_ROOTS hda NTN_DS 97 WRITE,CREAT,TRUNC UNIQUE hdo - heo MY_EL WRITE,CREAT,TRUNC 20Mi + heo MY_EL WRITE,CREAT,TRUNC hvp c. . lc $nblk hew 0 $blksz @@ -31,9 +32,10 @@ cmdw=" cmdr=" name run05r v $verbose_lev d $debug_lev mi 0 /@@ Create and reuse N-N dataset - i.e., ensure TRUNC works @/ + dbuf RAND22P 20Mi hi MY_CTX $HIO_TEST_ROOTS hda NTN_DS 97 READ UNIQUE hdo - heo MY_EL READ 20Mi + heo MY_EL READ hvp c. . hck on lc $nblk diff --git a/test/run07 b/test/run07 index 8d8e910..518bc73 100755 --- a/test/run07 +++ b/test/run07 @@ -26,11 +26,12 @@ batch_sub $(( $ranks * $blksz * $nblk )) cmdw=" name run07w v $verbose_lev d $debug_lev mi 0 /@@ Read and write N-N test case with read data value checking @/ + dbuf RAND22P 20Mi hi MYCTX $HIO_TEST_ROOTS hda NTNDS 97 WRITE,CREAT UNIQUE hdo hvsd datawarp_stage_mode immediate hvp c. . - heo MY_EL WRITE,CREAT,TRUNC 20Mi + heo MY_EL WRITE,CREAT,TRUNC lc $nblk hew 0 $blksz le @@ -40,10 +41,11 @@ cmdw=" cmdr=" name run07r v $verbose_lev d $debug_lev mi 32 /@@ Read and write N-N test case with read data value checking @/ + dbuf RAND22P 20Mi hi MYCTX $HIO_TEST_ROOTS hvp c. . hda NTNDS 97 READ UNIQUE hdo - heo MY_EL READ 20Mi + heo MY_EL READ hvp c. . hck on lc $nblk diff --git a/test/run08 b/test/run08 index d03cdde..3290c02 100755 --- a/test/run08 +++ b/test/run08 @@ -26,12 +26,13 @@ batch_sub $(( 3 * $ranks * $blksz * $nblk )) cmdw=" name run07w v $verbose_lev d $debug_lev mi 0 /@@ Read and write N-N test case with read data value checking @/ + dbuf RAND22P 20Mi hi MYCTX $HIO_TEST_ROOTS hda NTNDS 97 WRITE,CREAT UNIQUE hdo hvsd datawarp_stage_mode immediate hvp c. . - heo MY_EL WRITE,CREAT,TRUNC 20Mi + heo MY_EL WRITE,CREAT,TRUNC lc $nblk hew 0 $blksz le @@ -40,7 +41,7 @@ cmdw=" hda NTNDS 98 WRITE,CREAT UNIQUE hdo hvsd datawarp_stage_mode immediate hvp c. . - heo MY_EL WRITE,CREAT,TRUNC 20Mi + heo MY_EL WRITE,CREAT,TRUNC lc $nblk hew 0 $blksz le @@ -49,7 +50,7 @@ cmdw=" hda NTNDS 99 WRITE,CREAT UNIQUE hdo hvsd datawarp_stage_mode immediate hvp c. . - heo MY_EL WRITE,CREAT,TRUNC 20Mi + heo MY_EL WRITE,CREAT,TRUNC lc $nblk hew 0 $blksz le @@ -61,9 +62,10 @@ cmdw=" cmdr=" name run07r v $verbose_lev d $debug_lev mi 32 /@@ Read and write N-N test case with read data value checking @/ + dbuf RAND22P 20Mi hi MYCTX $HIO_TEST_ROOTS hda NTNDS 99 READ UNIQUE hdo - heo MY_EL READ 20Mi + heo MY_EL READ hvp c. . hck on lc $nblk diff --git a/test/run09 b/test/run09 index 9eb820e..20c4cff 100755 --- a/test/run09 +++ b/test/run09 @@ -27,12 +27,13 @@ batch_sub $(( 3 * $ranks * $blksz * $nblk )) cmdw=" name run09w v $verbose_lev d $debug_lev mi 0 /@@ Read and write N-N test case with read data value checking @/ + dbuf RAND22P 20Mi hi MYCTX $HIO_TEST_ROOTS hda NTNDS 98 WRITE,CREAT UNIQUE hdo hvsd datawarp_stage_mode immediate hvp c. . - heo MY_EL WRITE,CREAT,TRUNC 20Mi + heo MY_EL WRITE,CREAT,TRUNC lc $nblk hew 0 $blksz le @@ -43,7 +44,7 @@ cmdw=" hda NTNDS 97 WRITE,CREAT UNIQUE hdo hvsd datawarp_stage_mode immediate hvp c. . - heo MY_EL WRITE,CREAT,TRUNC 20Mi + heo MY_EL WRITE,CREAT,TRUNC lc $nblk hew 0 $blksz le @@ -54,7 +55,7 @@ cmdw=" hda NTNDS 96 WRITE,CREAT UNIQUE hdo hvsd datawarp_stage_mode immediate hvp c. . - heo MY_EL WRITE,CREAT,TRUNC 20Mi + heo MY_EL WRITE,CREAT,TRUNC lc $nblk hew 0 $blksz le @@ -66,12 +67,13 @@ cmdw=" cmdr=" name run09r v $verbose_lev d $debug_lev mi 32 /@@ Read and write N-N test case with read data value checking @/ + dbuf RAND22P 20Mi hi MYCTX $HIO_TEST_ROOTS hxdi 97 hvp c. . hda NTNDS 97 READ UNIQUE hdo - heo MY_EL READ 20Mi + heo MY_EL READ hvp c. . hck on lc $nblk @@ -81,7 +83,7 @@ cmdr=" hxdi 98 hda NTNDS ID_HIGHEST READ UNIQUE - heo MY_EL READ 20Mi + heo MY_EL READ hvp c. . hck on lc $nblk @@ -91,7 +93,7 @@ cmdr=" qof 2 hxdi 96 hda NTNDS ID_NEWEST READ UNIQUE hdo - heo MY_EL READ 20Mi + heo MY_EL READ hvp c. . hck on lc $nblk diff --git a/test/run10 b/test/run10 index 44754f0..f0b226b 100755 --- a/test/run10 +++ b/test/run10 @@ -33,9 +33,10 @@ pfs="/ufs/home/dwphase2/cornell" cmdwc=" /@@ Write cache @/ name run10wc v $verbose_lev d $debug_lev mi 0 + dbuf RAND22P 20Mi hi MY_CTX posix:$cache hda NTN_DS 97 WRITE,CREAT UNIQUE hdo - heo MY_EL WRITE,CREAT,TRUNC 20Mi + heo MY_EL WRITE,CREAT,TRUNC hvp c. . lc $nblk hew -$blk $blk @@ -46,9 +47,10 @@ cmdwc=" cmdrc=" /@@ Read cache @/ name run102rc v $verbose_lev d $debug_lev mi 0 + dbuf RAND22P 20Mi hi MY_CTX posix:$cache hda NTN_DS 97 READ UNIQUE hdo - heo MY_EL READ 20Mi + heo MY_EL READ hvp c. . hck on lc $nblk @@ -60,9 +62,10 @@ cmdrc=" cmdwp=" /@@ Write backing FS @/ name run10wp v $verbose_lev d $debug_lev mi 0 + dbuf RAND22P 20Mi hi MY_CTX posix:$pfs hda NTN_DS 97 WRITE,CREAT UNIQUE hdo - heo MY_EL WRITE,CREAT,TRUNC 20Mi + heo MY_EL WRITE,CREAT,TRUNC hvp c. . lc $nblk hew -$blk $blk @@ -73,9 +76,10 @@ cmdwp=" cmdrp=" /@@ Read backing FS @/ name run102rp v $verbose_lev d $debug_lev mi 0 + dbuf RAND22P 20Mi hi MY_CTX posix:$pfs hda NTN_DS 97 READ UNIQUE hdo - heo MY_EL READ 20Mi + heo MY_EL READ hvp c. . hck on lc $nblk diff --git a/test/run12 b/test/run12 index 255a28e..acee5d5 100755 --- a/test/run12 +++ b/test/run12 @@ -22,9 +22,10 @@ batch_sub $(( $ranks * $blkszM * $nblkM )) cmdw=" name run12w v $verbose_lev d $debug_lev mi 0 /@@ Read and write random N-N test case with read data value checking @/ + dbuf RAND22P 20Mi hi MY_CTX $HIO_TEST_ROOTS hda NTN_DS 97 WRITE,CREAT UNIQUE hdo - heo MY_EL WRITE,CREAT,TRUNC 20Mi + heo MY_EL WRITE,CREAT,TRUNC hvp c. . srr 100 lcr $nblk $nblkM @@ -36,9 +37,10 @@ cmdw=" cmdr=" name run12r v $verbose_lev d $debug_lev mi 32 /@@ Read and write random N-N test case with read data value checking @/ + dbuf RAND22P 20Mi hi MY_CTX $HIO_TEST_ROOTS hda NTN_DS 97 READ UNIQUE hdo - heo MY_EL READ 20Mi + heo MY_EL READ hvp c. . hck on srr 100 diff --git a/test/run20 b/test/run20 index eae16f1..bfb94b5 100755 --- a/test/run20 +++ b/test/run20 @@ -20,10 +20,11 @@ batch_sub $(( 3 * $ranks * $blksz * $nblk )) cmdw=" name run20w v $verbose_lev d $debug_lev mi 0 /@@ Read and write N-N multi DS test case @/ + dbuf RAND22P 20Mi hi MY_CTX $HIO_TEST_ROOTS - hda NTN_DS 98 WRITE,CREAT UNIQUE hdo - heo MY_EL WRITE,CREAT,TRUNC 20Mi + hda NTN_DS 97 WRITE,CREAT UNIQUE hdo + heo MY_EL WRITE,CREAT,TRUNC hvp c. . lc $nblk hew 0 $blksz @@ -31,8 +32,8 @@ cmdw=" hec hdc hdf s 1 - hda NTN_DS 99 WRITE,CREAT UNIQUE hdo - heo MY_EL WRITE,CREAT,TRUNC 20Mi + hda NTN_DS 98 WRITE,CREAT UNIQUE hdo + heo MY_EL WRITE,CREAT,TRUNC hvp c. . lc $nblk hew 0 $blksz @@ -40,8 +41,8 @@ cmdw=" hec hdc hdf s 1 - hda NTN_DS 97 WRITE,CREAT UNIQUE hdo - heo MY_EL WRITE,CREAT,TRUNC 20Mi + hda NTN_DS 99 WRITE,CREAT UNIQUE hdo + heo MY_EL WRITE,CREAT,TRUNC hvp c. . lc $nblk hew 0 $blksz @@ -54,11 +55,12 @@ cmdw=" cmdr=" name run20r v $verbose_lev d $debug_lev mi 32 /@@ Read and write N-N multi DS test case @/ + dbuf RAND22P 20Mi hi MY_CTX $HIO_TEST_ROOTS - hxdi 98 - hda NTN_DS 98 READ UNIQUE hdo - heo MY_EL READ 20Mi + hxdi 99 + hda NTN_DS 99 READ UNIQUE hdo + heo MY_EL READ hvp c. . hck on lc $nblk @@ -66,9 +68,9 @@ cmdr=" le hec hdc hdf - hxdi 97 + hxdi 99 hda NTN_DS ID_NEWEST READ UNIQUE hdo - heo MY_EL READ 20Mi + heo MY_EL READ hvp c. . hck on lc $nblk @@ -78,7 +80,7 @@ cmdr=" hxdi 99 hda NTN_DS ID_HIGHEST READ UNIQUE hdo - heo MY_EL READ 20Mi + heo MY_EL READ hvp c. . hck on lc $nblk diff --git a/test/run_combo b/test/run_combo index 1716742..4522df4 100755 --- a/test/run_combo +++ b/test/run_combo @@ -38,7 +38,7 @@ synexit() { echo "" echo " Options:" echo " -t [...] Tests to invoke" - echo " -s Per rank run sizes t|s|m|l|x|y|z (default s)" + echo " -s Per rank run sizes u|t|s|m|l|x|y|z (default s)" echo " [,][,] where:" echo " totsz - total size per rank in MiB" echo " blksz - size of each write/read in KiB" diff --git a/test/run_setup b/test/run_setup index f33056b..1e3dd7b 100755 --- a/test/run_setup +++ b/test/run_setup @@ -52,7 +52,7 @@ synexit() { echo " [-u ]" echo "" echo " Options:" - echo " -s Per rank run size t|s|m|l|x|y|z (default s)" + echo " -s Per rank run size u|t|s|m|l|x|y|z (default s)" echo " [,][,] where:" echo " totsz - total size per rank in MiB" echo " blksz - size of each write/read in KiB" @@ -576,6 +576,7 @@ if [[ $batch -eq 0 ]]; then addifd "/lustre/scratch5/$LOGNAME" addifd "/lustre/trscratch1/$LOGNAME" addifd "/lustre/trscratch2/$LOGNAME" + addifd "/lustre/ttscratch1/$LOGNAME" addifd "/scratch1/$LOGNAME" addifd "/scratch1/users/$LOGNAME" addifd "/lustre/scratch1/yellow/$LOGNAME" diff --git a/test/xexec.c b/test/xexec.c index 9ad38c0..8f3b366 100644 --- a/test/xexec.c +++ b/test/xexec.c @@ -159,6 +159,9 @@ char * help = " dls Issue dlsym for specified symbol in most recently opened library\n" " dlc Issue dlclose for most recently opened library\n" #endif // DLFCN + " dbuf RAND22 | RAND22P | OFS20 Allocates and initializes data buffer\n" + " with pattern used for read data check. Write and read patterns\n" + " must match.\n" #ifdef HIO " hi Init hio context\n" " hda Dataset allocate\n" @@ -297,6 +300,13 @@ ACTION * actv; MSG_CONTEXT my_msg_context; #define MY_MSG_CTX (&my_msg_context) +// Common read / write buffer pointers, etc. Set by dbuf action. +static void * wbuf_ptr = NULL; +static void * rbuf_ptr = NULL; +static size_t rwbuf_len = 0; // length not counting pattern overun area +static U64 wbuf_hash_mod; // Element hash modulus +static U64 wbuf_bdy; + //---------------------------------------------------------------------------- // Common subroutines and macros //---------------------------------------------------------------------------- @@ -393,27 +403,24 @@ void lfsr_22_byte(unsigned char * p, U64 len) { } } -#if 0 void lfsr_22_byte_init(void) { srandom(15485863); // The 1 millionth prime for (int i = 0; i (*vo).max.val ) { + (*vo).max = (*vi).max; + } + } +} + void prt_mmmst(double val, char * desc, char * unit) { #ifdef MPI if (mpi_size > 0) { - double sum, mean, min, max, diff, sumdiff, sd; + double sum, mean, diff, sumdiff, sd; + struct min_max_s mm, mmr; + MPI_Datatype mmtype; + MPI_Op mmw_op; + + // Set up MPI data type and op + MPI_CK( MPI_Type_contiguous(sizeof(struct min_max_s), MPI_BYTE, &mmtype) ); + MPI_CK( MPI_Type_commit(&mmtype) ); + MPI_CK( MPI_Op_create(min_max_who, true, & mmw_op) ); + // Use two pass method to calculate variance - MPI_CK(MPI_Allreduce(&val, &sum, 1, MPI_DOUBLE, MPI_SUM, mpi_comm)); + MPI_CK( MPI_Allreduce(&val, &sum, 1, MPI_DOUBLE, MPI_SUM, mpi_comm) ); mean = sum/(double)mpi_size; diff = (val - mean) * (val - mean); - MPI_CK(MPI_Reduce(&diff, &sumdiff, 1, MPI_DOUBLE, MPI_SUM, 0, mpi_comm)); + MPI_CK( MPI_Reduce(&diff, &sumdiff, 1, MPI_DOUBLE, MPI_SUM, 0, mpi_comm) ); sd = sqrt(sumdiff / mpi_size); + // Get min, max - MPI_CK(MPI_Reduce(&val, &min, 1, MPI_DOUBLE, MPI_MIN, 0, mpi_comm)); - MPI_CK(MPI_Reduce(&val, &max, 1, MPI_DOUBLE, MPI_MAX, 0, mpi_comm)); + mm.min.val = mm.max.val = val; + strncpy(mm.min.id, id_string, sizeof(mm.min.id)); + mm.min.id[sizeof(mm.min.id)-1] = '\0'; + char * p = strchr(mm.min.id, ' '); + if (p) *p = '\0'; + strncpy(mm.max.id, mm.min.id, sizeof(mm.max.id)); + MPI_CK( MPI_Reduce(&mm, &mmr, 1, mmtype, mmw_op, 0, mpi_comm) ); + MPI_CK( MPI_Type_free(&mmtype) ); + MPI_CK( MPI_Op_free(&mmw_op) ); + char b1[32], b2[32], b3[32], b4[32], b5[32]; - if (0 == myrank) VERB1("%s mean/min/max/s/tot: %s %s %s %s %s %s", desc, + if (0 == myrank) VERB1("%s mean/min/max/s/tot: %s %s %s %s %s %s min/max: %s %s", desc, eng_not(b1, sizeof(b1), mean, "D6.4", ""), - eng_not(b2, sizeof(b2), min, "D6.4", ""), - eng_not(b3, sizeof(b3), max, "D6.4", ""), + eng_not(b2, sizeof(b2), mmr.min.val, "D6.4", ""), + eng_not(b3, sizeof(b3), mmr.max.val, "D6.4", ""), eng_not(b4, sizeof(b4), sd, "D6.4", ""), - eng_not(b5, sizeof(b5), sum, "D6.4", ""), unit); + eng_not(b5, sizeof(b5), sum, "D6.4", ""), unit, + mmr.min.id, mmr.max.id); } #endif @@ -548,12 +598,15 @@ ACTION_RUN(ztest_run) { char * s2 = V2.s; char * s3 = V3.s; - VERB0("nothing to see here, move along"); + VERB0("Nothing to see here, move along"); //char buf[32]; //printf("eng_not(%lg, %s, %s) --> \"%s\"\n", d1, s2, s3, eng_not(buf, sizeof(buf), d1, s2, s3)); - //lfsr_test(); + //lfsr_test(); + + //prt_mmmst((double) myrank, "rank", ""); + } //---------------------------------------------------------------------------- @@ -789,6 +842,75 @@ ACTION_RUN(le_run) { } } +//---------------------------------------------------------------------------- +// dbuf action handler +//---------------------------------------------------------------------------- +enum dbuf_type {RAND22P, RAND22, OFS20}; +ENUM_START(etab_dbuf) // Write buffer type +ENUM_NAME("RAND22P", RAND22P) +ENUM_NAME("RAND22", RAND22) +ENUM_NAME("OFS20", OFS20) +ENUM_END(etab_dbuf, 0, NULL) + +#define PRIME_LT_64KI 65521 // A prime a little less than 2**16 +#define PRIME_75PCT_2MI 786431 // A prime about 75% of 2**20 +#define PRIME_LT_2MI 1048573 // A prime a little less that 2**20 + +void dbuf_init(enum dbuf_type type, U64 size) { + rwbuf_len = size; + int rc; + size_t extra; + + FREEX(wbuf_ptr); + FREEX(rbuf_ptr); + + switch (type) { + case RAND22P: + extra = LFSR_22_CYCLE; + wbuf_bdy = 1; + wbuf_hash_mod = PRIME_LT_64KI; + rc = posix_memalign((void * *)&wbuf_ptr, 4096, rwbuf_len + extra); + if (rc) ERRX("wbuf posix_memalign %d bytes failed: %s", rwbuf_len + extra, strerror(rc)); + lfsr_22_byte_init_p(); + lfsr_22_byte(wbuf_ptr, rwbuf_len + extra); + break; + case RAND22: + extra = LFSR_22_CYCLE; + wbuf_bdy = 1; + wbuf_hash_mod = PRIME_LT_2MI; + rc = posix_memalign((void * *)&wbuf_ptr, 4096, rwbuf_len + extra); + if (rc) ERRX("wbuf posix_memalign %d bytes failed: %s", rwbuf_len + extra, strerror(rc)); + lfsr_22_byte_init(); + lfsr_22_byte(wbuf_ptr, rwbuf_len + extra); + break; + case OFS20: + extra = 1024 * 1024; + wbuf_bdy = sizeof(uint32_t); + wbuf_hash_mod = 1024 * 1024; + rc = posix_memalign((void * *)&wbuf_ptr, 4096, rwbuf_len + extra); + if (rc) ERRX("wbuf posix_memalign %d bytes failed: %s", rwbuf_len + extra, strerror(rc)); + for (long i = 0; i<(rwbuf_len+extra)/sizeof(uint32_t); ++i) { + ((uint32_t *)wbuf_ptr)[i] = i * sizeof(uint32_t); + } + break; + } + + rbuf_ptr = MALLOCX(rwbuf_len); + + DBG3("dbuf_init type: %s size: %lld wbuf_ptr: 0x%lX", \ + enum_name(MY_MSG_CTX, &etab_dbuf, type), size, wbuf_ptr); + IFDBG3( hex_dump(wbuf_ptr, 32) ); + +} + +ACTION_CHECK(dbuf_check) { + rwbuf_len = V1.u; +} + +ACTION_RUN(dbuf_run) { + dbuf_init(V0.i, V1.u); +} + //---------------------------------------------------------------------------- // o, e (stdout, stderr) action handlers //---------------------------------------------------------------------------- @@ -1462,24 +1584,19 @@ static hio_dataset_mode_t hio_dataset_mode; static char * hio_element_name; static U64 hio_element_hash; #define EL_HASH_MODULUS 65521 // A prime a little less than 2**16 -//#define EL_HASH_MODULUS 786431 // A prime about 75% of 2**20 -//#define EL_HASH_MODULUS 1048573 // A prime a little less that 2**20 static hio_return_t hio_rc_exp = HIO_SUCCESS; static I64 hio_cnt_exp = HIO_CNT_REQ; static I64 hio_dsid_exp = -999; static int hio_dsid_exp_set = 0; static int hio_fail = 0; -static void * wbuf = NULL, *rbuf = NULL; -static U64 bufsz = 0; static int hio_check = 0; static U64 hio_e_ofs; static I64 hseg_start = 0; static U64 rw_count[2]; -static ETIMER hio_hdaf_tmr, hio_api_tmr; +static ETIMER hio_hdaf_tmr, local_tmr; double hio_hda_time, hio_hdo_time, hio_heo_time, hio_hew_time, hio_her_time, hio_hec_time, hio_hdc_time, hio_hdf_time, hio_exc_time; - #define HRC_TEST(API_NAME) { \ local_fails += (hio_fail = (hrc != hio_rc_exp && hio_rc_exp != HIO_ANY) ? 1: 0); \ if (hio_fail || MY_MSG_CTX->verbose_level >= 3) { \ @@ -1542,9 +1659,9 @@ ACTION_RUN(hda_run) { DBG2("Calling hio_datset_alloc(context, &dataset, %s, %lld, %d(%s), %d(%s))", hio_dataset_name, hio_ds_id_req, hio_dataset_flags, V2.s, hio_dataset_mode, V3.s); ETIMER_START(&hio_hdaf_tmr); - ETIMER_START(&hio_api_tmr); + ETIMER_START(&local_tmr); hrc = hio_dataset_alloc (context, &dataset, hio_dataset_name, hio_ds_id_req, hio_dataset_flags, hio_dataset_mode); - hio_hda_time += ETIMER_ELAPSED(&hio_api_tmr); + hio_hda_time += ETIMER_ELAPSED(&local_tmr); DBG2("hda_run: dataset: %p", dataset); HRC_TEST(hio_dataset_alloc); } @@ -1554,9 +1671,9 @@ ACTION_RUN(hda_run) { ACTION_RUN(hdo_run) { hio_return_t hrc; DBG2("calling hio_dataset_open(%p)", dataset); - ETIMER_START(&hio_api_tmr); + ETIMER_START(&local_tmr); hrc = hio_dataset_open (dataset); - hio_hdo_time += ETIMER_ELAPSED(&hio_api_tmr); + hio_hdo_time += ETIMER_ELAPSED(&local_tmr); HRC_TEST(hio_dataset_open); if (HIO_SUCCESS == hrc) { hrc = hio_dataset_get_id(dataset, &hio_ds_id_act); @@ -1576,38 +1693,30 @@ ACTION_RUN(hdo_run) { } ACTION_CHECK(heo_check) { - bufsz = V2.u; + if (rwbuf_len == 0) rwbuf_len = 20 * 1024 * 1024; } ACTION_RUN(heo_run) { hio_return_t hrc; hio_element_name = V0.s; int flag_i = V1.i; - bufsz = V2.u; - ETIMER_START(&hio_api_tmr); + ETIMER_START(&local_tmr); hrc = hio_element_open (dataset, &element, hio_element_name, flag_i); - hio_heo_time += ETIMER_ELAPSED(&hio_api_tmr); + hio_heo_time += ETIMER_ELAPSED(&local_tmr); HRC_TEST(hio_element_open) - ETIMER_START(&hio_api_tmr); - - int rc = posix_memalign((void * *)&wbuf, 4096, bufsz + LFSR_22_CYCLE); - if (rc) ERRX("wbuf posix_memalign %d bytes failed: %s", bufsz + LFSR_22_CYCLE, strerror(rc)); - lfsr_22_byte_init(); - lfsr_22_byte(wbuf, bufsz+LFSR_22_CYCLE); - - rbuf = MALLOCX(bufsz); - hio_e_ofs = 0; + ETIMER_START(&local_tmr); + if (! wbuf_ptr ) dbuf_init(RAND22P, 20 * 1024 * 1024); - // Calculate element hash which is added to the offset for read verification char * hash_str = ALLOC_PRINTF("%s %s %d %s %d", hio_context_name, hio_dataset_name, hio_ds_id_act, hio_element_name, (HIO_SET_ELEMENT_UNIQUE == hio_dataset_mode) ? myrank: 0); - hio_element_hash = crc32(0, hash_str, strlen(hash_str)) % EL_HASH_MODULUS; + hio_element_hash = BDYDN(crc32(0, hash_str, strlen(hash_str)) % wbuf_hash_mod, wbuf_bdy); DBG4("heo hash: \"%s\" 0x%04X", hash_str, hio_element_hash); FREEX(hash_str); - hio_exc_time += ETIMER_ELAPSED(&hio_api_tmr); + hio_e_ofs = 0; + hio_exc_time += ETIMER_ELAPSED(&local_tmr); } ACTION_RUN(hso_run) { @@ -1623,7 +1732,7 @@ ACTION_RUN(hck_run) { ACTION_CHECK(hew_check) { U64 size = V1.u; - if (size > bufsz) ERRX("%s; size > bufsz", A.desc); + if (size > rwbuf_len) ERRX("%s; size > rwbuf_len", A.desc); } ACTION_RUN(hsega_run) { @@ -1653,11 +1762,11 @@ ACTION_RUN(hew_run) { ofs_abs = hio_e_ofs + ofs_param; DBG2("hew el_ofs: %lld ofs_param: %lld ofs_abs: %lld len: %lld", hio_e_ofs, ofs_param, ofs_abs, hreq); hio_e_ofs = ofs_abs + hreq; - ETIMER_START(&hio_api_tmr); - void * expected = wbuf + ( (ofs_abs + hio_element_hash) % LFSR_22_CYCLE); - DBG2("hew: hio_element_hash: 0x%lX wbuf: 0x%lX expexted-wbuf: 0x%lX", hio_element_hash, wbuf, expected - wbuf); + void * expected = wbuf_ptr + ( (ofs_abs + hio_element_hash) % LFSR_22_CYCLE); + DBG2("hew: hio_element_hash: 0x%lX wbuf_ptr: 0x%lX expected-wbuf_ptr: 0x%lX", hio_element_hash, wbuf_ptr, expected - wbuf_ptr); + ETIMER_START(&local_tmr); hcnt = hio_element_write (element, ofs_abs, 0, expected, 1, hreq); - hio_hew_time += ETIMER_ELAPSED(&hio_api_tmr); + hio_hew_time += ETIMER_ELAPSED(&local_tmr); HCNT_TEST(hio_element_write) rw_count[1] += hcnt; } @@ -1671,7 +1780,7 @@ ACTION_RUN(hewr_run) { ACTION_CHECK(her_check) { U64 size = V1.u; - if (size > bufsz) ERRX("%s; size > bufsz", A.desc); + if (size > rwbuf_len) ERRX("%s; size > rwbuf_len", A.desc); } ACTION_RUN(her_run) { @@ -1683,45 +1792,48 @@ ACTION_RUN(her_run) { ofs_abs = hio_e_ofs + ofs_param; DBG2("her el_ofs: %lld ofs_param: %lld ofs_abs: %lld len: %lld", hio_e_ofs, ofs_param, ofs_abs, hreq); hio_e_ofs = ofs_abs + hreq; - ETIMER_START(&hio_api_tmr); - hcnt = hio_element_read (element, ofs_abs, 0, rbuf, 1, hreq); - hio_her_time += ETIMER_ELAPSED(&hio_api_tmr); + ETIMER_START(&local_tmr); + hcnt = hio_element_read (element, ofs_abs, 0, rbuf_ptr, 1, hreq); + hio_her_time += ETIMER_ELAPSED(&local_tmr); HCNT_TEST(hio_element_read) rw_count[0] += hcnt; if (hio_check) { - void * expected = wbuf + ( (ofs_abs + hio_element_hash) % LFSR_22_CYCLE); + ETIMER_START(&local_tmr); + void * expected = wbuf_ptr + ( (ofs_abs + hio_element_hash) % LFSR_22_CYCLE); void * mis_comp; - DBG2("her: hio_element_hash: 0x%lX wbuf: 0x%lX expected-wbuf: 0x%lX", hio_element_hash, wbuf, expected - wbuf); + DBG2("her: hio_element_hash: 0x%lX wbuf_ptr: 0x%lX expected-wbuf_ptr: 0x%lX", hio_element_hash, wbuf_ptr, expected - wbuf_ptr); // Force error for unit test - //*(char *)(rbuf+27) = '\0'; - if ( ! (mis_comp = memdiff(rbuf, expected, hreq)) ) { + // *(char *)(rbuf_ptr+16) = '\0'; + if ( ! (mis_comp = memdiff(rbuf_ptr, expected, hreq)) ) { VERB3("hio_element_read data check successful"); } else { // Read data miscompare - dump lots of data about it local_fails++; - I64 offset = (char *)mis_comp - (char *)rbuf; + I64 offset = (char *)mis_comp - (char *)rbuf_ptr; I64 dump_start = MAX(0, offset - 16) & (~15); VERB0("Error: hio_element_read data check miscompare"); VERB0("Element read offset: 0x%llX %lld", ofs_abs, ofs_abs); VERB0("Element read length: 0x%llX %lld", hreq, hreq); - VERB0(" Read addresss: 0x%llX", rbuf); + VERB0(" Read addresss: 0x%llX", rbuf_ptr); VERB0(" Miscompare address: 0x%llX", mis_comp); - VERB0(" Miscompare offset: 0x%llX %lld", mis_comp-rbuf, mis_comp-rbuf); + VERB0(" Miscompare offset: 0x%llX %lld", mis_comp-rbuf_ptr, mis_comp-rbuf_ptr); + + VERB0("Debug: wbuf_ptr addr: 0x%lX:", wbuf_ptr); hex_dump(wbuf_ptr, 32); VERB0("Miscompare expected data at offset 0x%llX %lld follows:", dump_start, dump_start); - hex_dump( wbuf + ( (ofs_abs + hio_element_hash) % LFSR_22_CYCLE) + dump_start, 96); + hex_dump( wbuf_ptr + ( (ofs_abs + hio_element_hash) % LFSR_22_CYCLE) + dump_start, 96); VERB0("Miscompare actual data at offset 0x%llX %lld follows:", dump_start, dump_start); - hex_dump( rbuf + dump_start, 96); + hex_dump( rbuf_ptr + dump_start, 96); VERB0("XOR of Expected addr: 0x%lX Miscomp addr: 0x%lX", expected, mis_comp); - char * xorbuf = MALLOCX(hreq); + char * xorbuf_ptr = MALLOCX(hreq); for (int i=0; i