[gentoo-cluster] Iozone on multiple nodes using ssh

[Chuanwen Wu]

Hi,
I am trying to use iozone to test my cluster with ssh but not rsh, but
I still can't running iozone on multiple nodes.

I have two nodes node73 and node74 in my LAN. And each node is able to
execute commands on another one without being challenged for a
password with user "dnfs":
dnfs@node73 ~ $ ssh node74 date
Thu Jan  8 22:44:55 CST 2009

dnfs@Gnode74 ~ $ ssh node73 date
Thu Jan  8 22:47:19 CST 2009

Iozone use rsh to execute commands on the clients, and now I use ssh
to replace rsh:
dnfs@node73 ~ $ cat ~/.bashrc
# /etc/skel/.bashrc
[...]
# Put your fun stuff here.
export RSH=ssh

Here is my clientlist, which contains the nodes' information:
dnfs@node73 ~ $ cat clientlist
node74 /home/dnfs /tmp/iozone

Then I run iozone on node73:
/***********************************************************/
dnfs@node73 ~ $ iozone -s 1m -Rb log.xls  -t 1 -+m clientlist
        Iozone: Performance Test of File I/O
                Version $Revision: 3.242 $
                Compiled for 64 bit mode.
                Build: linux-AMD64

        Contributors:William Norcott, Don Capps, Isom Crawford, Kirby Collins
                     Al Slater, Scott Rhine, Mike Wisner, Ken Goss
                     Steve Landherr, Brad Smith, Mark Kelly, Dr. Alain CYR,
                     Randy Dunlap, Mark Montague, Dan Million,
                     Jean-Marc Zucconi, Jeff Blomberg,
                     Erik Habbinga, Kris Strecker, Walter Wong.

        Run began: Thu Jan  8 23:11:13 2009

        File size set to 1024 KB
        Excel chart generation enabled
        Network distribution mode enabled.
        Command line used: iozone -s 1m -Rb log.xls -t 1 -+m clientlist
        Output is in Kbytes/sec
        Time Resolution = 0.000001 seconds.
        Processor cache size set to 1024 Kbytes.
        Processor cache line size set to 32 bytes.
        File stride size set to 17 * record size.
        Throughput test with 1 process
        Each process writes a 1024 Kbyte file in 4 Kbyte records
/******************************************************/
Then iozone stoped here.

I still use strace to see what happend:
/******************************************************/
dnfs@node73 ~ $ strace iozone -s 1m -Rb log.xls  -t 1 -+m clientlist
execve("/usr/bin/iozone", ["iozone", "-s", "1m", "-Rb", "log.xls",
"-t", "1", "-+m", "clientlist"], [/* 45 vars */]) = 0
brk(0)                                  = 0x7cb000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7f67f3639000
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7f67f3638000
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
open("/etc/ld.so.cache", O_RDONLY)      = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=48606, ...}) = 0
mmap(NULL, 48606, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7f67f362c000
close(3)                                = 0
open("/lib/librt.so.1", O_RDONLY)       = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\300\"\0\0\0\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=35688, ...}) = 0
mmap(NULL, 2132968, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f67f3217000
mprotect(0x7f67f321f000, 2093056, PROT_NONE) = 0
mmap(0x7f67f341e000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x7000) = 0x7f67f341e000
close(3)                                = 0
open("/lib/libpthread.so.0", O_RDONLY)  = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\240W\0\0\0\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=131577, ...}) = 0
mmap(NULL, 2204528, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f67f2ffc000
mprotect(0x7f67f3011000, 2097152, PROT_NONE) = 0
mmap(0x7f67f3211000, 8192, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x15000) = 0x7f67f3211000
mmap(0x7f67f3213000, 13168, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f67f3213000
close(3)                                = 0
open("/lib/libc.so.6", O_RDONLY)        = 3
read(3, "\177ELF\2\1\1\0\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\220\334\1\0\0\0\0\0"...,
832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=1293456, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7f67f362b000
mmap(NULL, 3399928, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3,
0) = 0x7f67f2cbd000
mprotect(0x7f67f2df3000, 2093056, PROT_NONE) = 0
mmap(0x7f67f2ff2000, 20480, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x135000) = 0x7f67f2ff2000
mmap(0x7f67f2ff7000, 16632, PROT_READ|PROT_WRITE,
MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7f67f2ff7000
close(3)                                = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7f67f362a000
arch_prctl(ARCH_SET_FS, 0x7f67f362a6f0) = 0
mprotect(0x7f67f2ff2000, 16384, PROT_READ) = 0
mprotect(0x7f67f3211000, 4096, PROT_READ) = 0
mprotect(0x7f67f341e000, 4096, PROT_READ) = 0
mprotect(0x62a000, 4096, PROT_READ)     = 0
mprotect(0x7f67f363a000, 4096, PROT_READ) = 0
munmap(0x7f67f362c000, 48606)           = 0
set_tid_address(0x7f67f362a780)         = 31886
set_robust_list(0x7f67f362a790, 0x18)   = 0
rt_sigaction(SIGRTMIN, {0x7f67f3001310, [], SA_RESTORER|SA_SIGINFO,
0x7f67f3009ec0}, NULL, 8) = 0
rt_sigaction(SIGRT_1, {0x7f67f3001390, [],
SA_RESTORER|SA_RESTART|SA_SIGINFO, 0x7f67f3009ec0}, NULL, 8) = 0
rt_sigprocmask(SIG_UNBLOCK, [RTMIN RT_1], NULL, 8) = 0
getrlimit(RLIMIT_STACK, {rlim_cur=8192*1024, rlim_max=RLIM_INFINITY}) = 0
uname({sys="Linux", node="Gentoo-F312-73", ...}) = 0
brk(0)                                  = 0x7cb000
brk(0x7ec000)                           = 0x7ec000
open("/etc/localtime", O_RDONLY)        = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=405, ...}) = 0
fstat(3, {st_mode=S_IFREG|0644, st_size=405, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7f67f3637000
read(3, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\3\0\0\0\3\0\0\0\0"...,
4096) = 405
lseek(3, -240, SEEK_CUR)                = 165
read(3, "TZif2\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\3\0\0\0\3\0\0\0\0"...,
4096) = 240
close(3)                                = 0
munmap(0x7f67f3637000, 4096)            = 0
rt_sigaction(SIGINT, {0x408b2c, [INT], SA_RESTORER|SA_RESTART,
0x7f67f2ced430}, {SIG_DFL}, 8) = 0
rt_sigaction(SIGTERM, {0x408b2c, [TERM], SA_RESTORER|SA_RESTART,
0x7f67f2ced430}, {SIG_DFL}, 8) = 0
mmap(NULL, 18878464, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS,
-1, 0) = 0x7f67f1abc000
open("clientlist", O_RDONLY)            = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=30, ...}) = 0
mmap(NULL, 4096, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1,
0) = 0x7f67f3637000
read(3, "node74 /home/dnfs /tmp/iozone\n", 4096) = 30
read(3, "", 4096)                       = 0
close(3)                                = 0
munmap(0x7f67f3637000, 4096)            = 0
write(1, "\tIozone: Performance Test of Fil"..., 38     Iozone:
Performance Test of File I/O
) = 38
write(1, "\t        Version $Revision: 3.24"..., 64
Version $Revision: 3.242 $
                Compiled for 64 bit mode.
) = 64
write(1, "\t\tBuild: linux-AMD64 \n\n", 23              Build: linux-AMD64

) = 23
write(1, "\tContributors:William Norcott, D"..., 71
Contributors:William Norcott, Don Capps, Isom Crawford, Kirby Collins
) = 71
write(1, "\t             Al Slater, Scott R"..., 60
Al Slater, Scott Rhine, Mike Wisner, Ken Goss
) = 60
write(1, "\t             Steve Landherr, Br"..., 69
Steve Landherr, Brad Smith, Mark Kelly, Dr. Alain CYR,
) = 69
write(1, "\t             Randy Dunlap, Mark"..., 57
Randy Dunlap, Mark Montague, Dan Million,
) = 57
write(1, "\t             Jean-Marc Zucconi,"..., 48
Jean-Marc Zucconi, Jeff Blomberg,
) = 48
write(1, "\t             Erik Habbinga, Kri"..., 58
Erik Habbinga, Kris Strecker, Walter Wong.

) = 58
write(1, "\tRun began: Thu Jan  8 23:12:20 "..., 38     Run began: Thu
Jan  8 23:12:20 2009

) = 38
write(1, "\tFile size set to 1024 KB\n", 26     File size set to 1024 KB
) = 26
write(1, "\tExcel chart generation enabled\n", 32       Excel chart
generation enabled
) = 32
write(1, "\tNetwork distribution mode enabl"..., 36     Network
distribution mode enabled.
) = 36
write(1, "\tCommand line used:", 19     Command line used    = 19
write(1, " iozone", 7 iozone)                  = 7
write(1, " -s", 3 -s)                      = 3
write(1, " 1m", 3 1m)                      = 3
write(1, " -Rb", 4 -Rb)                     = 4
write(1, " log.xls", 8 log.xls)                 = 8
write(1, " -t", 3 -t)                      = 3
write(1, " 1", 2 1)                       = 2
write(1, " -+m", 4 -+m)                     = 4
write(1, " clientlist", 11 clientlist)             = 11
write(1, "\n", 1
)                       = 1
write(1, "\tOutput is in Kbytes/sec", 24        Output is in Kbytes/sec) = 24
write(1, "\n", 1
)                       = 1
write(1, "\tTime Resolution = 0.000001 seco"..., 37     Time
Resolution = 0.000001 seconds.
) = 37
write(1, "\tProcessor cache size set to 102"..., 42     Processor
cache size set to 1024 Kbytes.
) = 42
write(1, "\tProcessor cache line size set t"..., 44     Processor
cache line size set to 32 bytes.
) = 44
write(1, "\tFile stride size set to 17 * re"..., 43     File stride
size set to 17 * record size.
) = 43
write(1, "\tThroughput test with 1 process\n", 32       Throughput
test with 1 process
) = 32
shmget(IPC_PRIVATE, 16384, IPC_CREAT|0666) = 1376256
shmat(1376256, 0, 0)                    = ?
shmctl(1376256, IPC_RMID, 0)            = 0
write(1, "\tEach process writes a 1024 Kbyt"..., 58     Each process
writes a 1024 Kbyte file in 4 Kbyte records
) = 58
socket(PF_INET, SOCK_DGRAM, IPPROTO_IP) = 3
setsockopt(3, SOL_SOCKET, SO_RCVBUF, [262144], 4) = 0
bind(3, {sa_family=AF_INET, sin_port=htons(20000),
sin_addr=inet_addr("0.0.0.0")}, 16) = 0
rt_sigaction(SIGINT, {SIG_IGN}, {0x408b2c, [INT],
SA_RESTORER|SA_RESTART, 0x7f67f2ced430}, 8) = 0
rt_sigaction(SIGQUIT, {SIG_IGN}, {SIG_DFL}, 8) = 0
rt_sigprocmask(SIG_BLOCK, [CHLD], [], 8) = 0
clone(child_stack=0, flags=CLONE_PARENT_SETTID|SIGCHLD,
parent_tidptr=0x7ffffb6398b8) = 31887
wait4(31887, [{WIFEXITED(s) && WEXITSTATUS(s) == 0}], 0, NULL) = 31887
rt_sigaction(SIGINT, {0x408b2c, [INT], SA_RESTORER|SA_RESTART,
0x7f67f2ced430}, NULL, 8) = 0
rt_sigaction(SIGQUIT, {SIG_DFL}, NULL, 8) = 0
rt_sigprocmask(SIG_SETMASK, [], NULL, 8) = 0
--- SIGCHLD (Child exited) @ 0 (0) ---
recvfrom(3,
/******************************************************/

I am not familiar with strace but I guess iozone was watiing to
receive something from node74.

Anyone ever use iozone across multiple nodes?
Any help will be very appreciated!

PS:
I can run iozone to test local fs without any problem.

-- 
wcw