[top] [TitleIndex] [WordIndex]

s/diskbench

Disk I/O Benchmarks

I have a tar file in http://sage.math.washington.edu/home/wstein/tmp/bench/, which is a sage-3.3.alpha3 build for sage.math.

The benchmark is to do the following tests, which reflect real-world usage that users of sage.math care about.

FILE=test_tar


if [ -d $FILE ]; then
    echo "Deleting left around $FILE"
    rm -rf $FILE
fi


sync
echo "Extract tarball"
time tar xf $FILE.tar
if [ $? -ne 0 ]; then
    exit 1
fi
time sync

echo "Remove tarball"
time rm $FILE.tar
if [ $? -ne 0 ]; then
    exit 1
fi
time sync

echo "Create tarball"
time tar cf $FILE.tar $FILE
if [ $? -ne 0 ]; then
    exit 1
fi
time sync

echo "Check size of tarball"
time du -sh $FILE
if [ $? -ne 0 ]; then
    exit 1
fi
time sync

echo "Find test"
cd $FILE
time find . -name sage_object.pyx -print
cd ..
if [ $? -ne 0 ]; then
    exit 1
fi
time sync

echo "Copy extracted tarball"
time cp -r $FILE "$FILE"2
if [ $? -ne 0 ]; then
    exit 1
fi
time sync

echo "Remove copied tarball"
time rm -rf "$FILE"2
if [ $? -ne 0 ]; then
    exit 1
fi
time sync


if [ `uname` = "Linux" ]; then
    echo "Testing doctesting of sage_object.pyx"
    cd $FILE
    export DOT_SAGE=`pwd`/dot_sage
    echo "Doctesting entire sage tree using 20 threads."
    time ./sage -tp 20 devel/sage/sage  > /dev/null 2>/dev/null
fi;

RESULTS:

Using a RAMDISK on X4450

RUN 1

wstein@sage:/bench$ time ./bench
Deleting left around test_tar
Extract tarball

real    0m4.280s
user    0m0.180s
sys     0m4.100s

real    0m0.087s
user    0m0.010s
sys     0m0.010s
Remove tarball

real    0m0.316s
user    0m0.000s
sys     0m0.320s

real    0m0.103s
user    0m0.000s
sys     0m0.000s
Create tarball

real    0m4.151s
user    0m0.150s
sys     0m4.000s

real    0m0.041s
user    0m0.000s
sys     0m0.000s
Check size of tarball
1.8G    test_tar

real    0m0.105s
user    0m0.010s
sys     0m0.100s

real    0m0.044s
user    0m0.000s
sys     0m0.000s
Find test
./devel/sage-main/build/sage/structure/sage_object.pyx
./devel/sage-main/sage/structure/sage_object.pyx

real    0m0.089s
user    0m0.050s
sys     0m0.040s

real    0m0.021s
user    0m0.000s
sys     0m0.000s
Copy extracted tarball

real    0m4.239s
user    0m0.100s
sys     0m4.140s

real    0m0.048s
user    0m0.000s
sys     0m0.000s
Remove copied tarball

real    0m0.549s
user    0m0.020s
sys     0m0.530s

real    0m0.032s
user    0m0.000s
sys     0m0.000s
Testing doctesting of sage_object.pyx
Doctesting entire sage tree using 20 threads.

real    2m53.640s
user    37m27.420s
sys     9m45.210s

real    3m8.389s
user    37m27.960s
sys     9m58.990s

RUN 2

wstein@sage:/bench$ time ./bench 
Deleting left around test_tar
Extract tarball

real    0m4.192s
user    0m0.230s
sys     0m3.960s

real    0m0.045s
user    0m0.000s
sys     0m0.000s
Remove tarball

real    0m0.306s
user    0m0.000s
sys     0m0.310s

real    0m0.031s
user    0m0.000s
sys     0m0.000s
Create tarball

real    0m4.078s
user    0m0.140s
sys     0m3.940s

real    0m0.028s
user    0m0.000s
sys     0m0.000s
Check size of tarball
1.8G    test_tar

real    0m0.105s
user    0m0.030s
sys     0m0.070s

real    0m0.055s
user    0m0.000s
sys     0m0.000s
Find test
./devel/sage-main/sage/structure/sage_object.pyx
./devel/sage-main/build/sage/structure/sage_object.pyx

real    0m0.086s
user    0m0.060s
sys     0m0.020s

real    0m0.046s
user    0m0.000s
sys     0m0.000s
Copy extracted tarball

real    0m4.150s
user    0m0.160s
sys     0m3.990s

real    0m0.018s
user    0m0.000s
sys     0m0.000s
Remove copied tarball

real    0m0.528s
user    0m0.010s
sys     0m0.520s

real    0m0.051s
user    0m0.000s
sys     0m0.000s
Testing doctesting of sage_object.pyx
Doctesting entire sage tree using 20 threads.

real    3m22.634s
user    37m25.810s
sys     9m48.960s

real    3m37.004s
user    37m26.440s
sys     10m2.310s

Using builtin SAS disk on X4450

RUN 1

wstein@sage:/tmp/bench$ time ./bench
Extract tarball

real    0m8.415s
user    0m0.230s
sys     0m6.650s

real    0m36.919s
user    0m0.000s
sys     0m1.250s
Remove tarball

real    0m0.679s
user    0m0.000s
sys     0m0.680s

real    0m2.667s
user    0m0.000s
sys     0m0.000s
Create tarball

real    0m5.978s
user    0m0.190s
sys     0m5.790s

real    0m24.290s
user    0m0.000s
sys     0m0.880s
Check size of tarball
1.8G    test_tar

real    0m0.139s
user    0m0.010s
sys     0m0.130s

real    0m0.428s
user    0m0.010s
sys     0m0.000s
Find test
./devel/sage-main/build/sage/structure/sage_object.pyx
./devel/sage-main/sage/structure/sage_object.pyx

real    0m0.096s
user    0m0.030s
sys     0m0.070s

real    0m0.018s
user    0m0.000s
sys     0m0.000s
Copy extracted tarball

real    0m14.928s
user    0m0.150s
sys     0m6.160s

real    0m26.526s
user    0m0.000s
sys     0m1.030s
Remove copied tarball

real    0m1.379s
user    0m0.040s
sys     0m1.340s

real    0m0.740s
user    0m0.000s
sys     0m0.000s

RUN 2

wstein@sage:/tmp/bench$ time ./bench
Extract tarball

real    0m8.654s
user    0m0.300s
sys     0m6.240s

real    0m39.338s
user    0m0.000s
sys     0m1.350s
Remove tarball

real    0m0.662s
user    0m0.010s
sys     0m0.660s

real    0m0.203s
user    0m0.000s
sys     0m0.000s
Create tarball

real    0m5.919s
user    0m0.210s
sys     0m5.710s

real    0m24.731s
user    0m0.000s
sys     0m1.000s
Check size of tarball
1.8G    test_tar

real    0m0.135s
user    0m0.010s
sys     0m0.120s

real    0m0.389s
user    0m0.000s
sys     0m0.000s
Find test
./devel/sage-main/build/sage/structure/sage_object.pyx
./devel/sage-main/sage/structure/sage_object.pyx

real    0m0.096s
user    0m0.040s
sys     0m0.060s

real    0m0.026s
user    0m0.000s
sys     0m0.000s
Copy extracted tarball

real    0m23.445s
user    0m0.100s
sys     0m6.230s

real    0m22.188s
user    0m0.000s
sys     0m1.060s
Remove copied tarball

real    0m1.462s
user    0m0.040s
sys     0m1.350s

real    0m0.723s
user    0m0.000s
sys     0m0.020s


Using the USB disk on X4450 locally (/wd)

wstein@sage:~/tmp/bench$ ./bench
Extract tarball

real    0m6.651s
user    0m0.210s
sys     0m6.370s

real    1m23.883s
user    0m0.000s
sys     0m1.550s
Remove tarball

real    0m0.650s
user    0m0.000s
sys     0m0.620s

real    0m0.234s
user    0m0.000s
sys     0m0.000s
Create tarball

real    0m5.837s
user    0m0.120s
sys     0m5.700s

real    1m13.255s
user    0m0.000s
sys     0m1.140s
Check size of tarball
1.8G    test_tar

real    0m0.136s
user    0m0.010s
sys     0m0.130s

real    0m0.415s
user    0m0.000s
sys     0m0.000s
Find test
./devel/sage-main/build/sage/structure/sage_object.pyx
./devel/sage-main/sage/structure/sage_object.pyx

real    0m0.099s
user    0m0.060s
sys     0m0.040s

real    0m0.038s
user    0m0.000s
sys     0m0.000s
Copy extracted tarball

real    0m6.640s
user    0m0.220s
sys     0m6.080s

real    1m23.672s
user    0m0.000s
sys     0m1.320s
Remove copied tarball

real    0m1.360s
user    0m0.010s
sys     0m1.350s

real    0m0.830s
user    0m0.000s
sys     0m0.010s


doctest sage_object.pyx:
wstein@sage:~/tmp/bench/test_tar$ time ./sage -t devel/sage/sage/structure/sage_object.pyx >/dev/null 2>/dev/null

real    0m3.003s
user    0m2.350s
sys     0m0.560s

doctest all sage with -tp 20:
real    2m50.116s
user    37m8.430s
sys     9m50.840s

Using the USB disk (/wd) served from sage.math over the network via NFS on geom.math

wstein@boxen:~/tmp/bench$ ./bench
Extract tarball

real    13m25.195s
user    0m0.380s
sys     0m15.210s

real    0m1.169s
user    0m0.000s
sys     0m0.000s
Remove tarball

real    0m1.019s
user    0m0.000s
sys     0m0.280s

real    0m1.221s
user    0m0.000s
sys     0m0.000s
Create tarball

real    1m46.367s
user    0m0.200s
sys     0m7.300s

real    0m2.145s
user    0m0.000s
sys     0m0.000s
Check size of tarball
1.9G    test_tar

real    0m7.033s
user    0m0.060s
sys     0m1.250s

real    0m0.670s
user    0m0.000s
sys     0m0.010s
Find test
./devel/sage-main/build/sage/structure/sage_object.pyx
./devel/sage-main/sage/structure/sage_object.pyx

real    0m1.260s
user    0m0.110s
sys     0m0.320s

real    0m1.256s
user    0m0.000s
sys     0m0.020s
Copy extracted tarball

real    9m43.929s
user    0m0.330s
sys     0m13.360s

real    0m2.293s
user    0m0.000s
sys     0m0.000s
Remove copied tarball

real    2m51.147s
user    0m0.060s
sys     0m2.500s

real    0m1.995s
user    0m0.000s
sys     0m0.000s

Testing doctesting of sage_object.pyx

wstein@boxen:~/tmp/bench/test_tar$ sync; time ./sage -tp 20 devel/sage/sage  > /dev/null 2>/dev/null
real    26m51.001s
user    59m12.220s
sys     443m32.040s

???

Using X4540 root filesystem ZFS

wstein@disk:~/bench$ ./bench
Extract tarball

real    1m23.932s
user    0m0.891s
sys     0m16.022s

real    0m5.381s
user    0m0.001s
sys     0m0.004s
Remove tarball

real    0m0.049s
user    0m0.001s
sys     0m0.012s

real    0m1.307s
user    0m0.001s
sys     0m0.004s
Create tarball

real    1m42.422s
user    0m0.870s
sys     0m11.583s

real    0m4.807s
user    0m0.001s
sys     0m0.004s
Check size of tarball
1.9G    tar_test

real    0m6.250s
user    0m0.097s
sys     0m0.586s

real    0m1.206s
user    0m0.001s
sys     0m0.004s
Find test
./devel/sage-main/sage/structure/sage_object.pyx
./devel/sage-main/build/sage/structure/sage_object.pyx

real    0m6.459s
user    0m0.116s
sys     0m0.608s

real    0m1.187s
user    0m0.001s
sys     0m0.004s
Copy extracted tarball

real    1m53.577s
user    0m0.594s
sys     0m11.629s

real    0m6.029s
user    0m0.001s
sys     0m0.003s
Remove copied tarball

real    0m8.951s
user    0m0.069s
sys     0m1.961s

real    0m1.829s
user    0m0.001s
sys     0m0.004s

Using X4540's original tank pool (the 47 disk raidz1)

wstein@disk:/tank/home/wstein/build/b$ ./bench
Extract tarball

real    9m29.873s
user    0m0.990s
sys     1m5.320s

real    0m1.412s
user    0m0.001s
sys     0m0.003s
Remove tarball

real    0m0.198s
user    0m0.001s
sys     0m0.014s

real    0m1.219s
user    0m0.001s
sys     0m0.004s
Create tarball

real    5m4.167s
user    0m0.960s
sys     0m24.742s

real    0m1.554s
user    0m0.001s
sys     0m0.004s
Check size of tarball
1.9G    tar_test

real    0m7.436s
user    0m0.085s
sys     0m0.607s

real    0m1.184s
user    0m0.001s
sys     0m0.003s
Find test
./devel/sage-main/build/sage/structure/sage_object.pyx
./devel/sage-main/sage/structure/sage_object.pyx

real    0m7.154s
user    0m0.094s
sys     0m0.610s

real    0m1.369s
user    0m0.001s
sys     0m0.003s
Copy extracted tarball

real    6m41.956s
user    0m0.603s
sys     0m22.431s

real    0m1.703s
user    0m0.001s
sys     0m0.003s
Remove copied tarball

real    0m13.438s
user    0m0.086s
sys     0m2.427s

real    0m1.516s
user    0m0.001s
sys     0m0.003s

A new ZFS Raidz2 pool on disk.math

root@disk:~# zpool create -f tank raidz2 c4t1d0 c4t2d0 c4t3d0

wstein@disk:/tank/bench$ ./bench 
Extract tarball

real    0m44.739s
user    0m0.748s
sys     0m11.993s

real    0m5.136s
user    0m0.001s
sys     0m0.004s
Remove tarball
rm: remove write-protected regular file `tar_test.tar'? yes

real    0m5.731s
user    0m0.001s
sys     0m0.021s

real    0m0.126s
user    0m0.001s
sys     0m0.004s
Create tarball

real    1m18.808s
user    0m0.641s
sys     0m8.777s

real    0m2.828s
user    0m0.001s
sys     0m0.003s
Check size of tarball
1.9G    tar_test

real    0m6.500s
user    0m0.058s
sys     0m0.412s

real    0m0.005s
user    0m0.001s
sys     0m0.003s
Find test
./devel/sage-main/sage/structure/sage_object.pyx
./devel/sage-main/build/sage/structure/sage_object.pyx

real    0m6.801s
user    0m0.063s
sys     0m0.442s

real    0m0.005s
user    0m0.001s
sys     0m0.004s
Copy extracted tarball

real    1m25.806s
user    0m0.609s
sys     0m11.100s

real    0m5.224s
user    0m0.001s
sys     0m0.004s
Remove copied tarball

real    0m10.913s
user    0m0.058s
sys     0m1.605s

real    0m0.462s
user    0m0.001s
sys     0m0.004s

Do it again but with a spare:

zpool add -f tank spare c4t4d0

...

Remove tarball

real    0m0.076s
user    0m0.001s
sys     0m0.019s

real    0m0.123s
user    0m0.001s
sys     0m0.004s
Create tarball

real    1m19.117s
user    0m0.601s
sys     0m8.397s

real    0m2.783s

user    0m0.001s
sys     0m0.003s

Check size of tarball
1.9G    tar_test

real    0m6.651s
user    0m0.097s
sys     0m0.581s

real    0m0.005s
user    0m0.001s
sys     0m0.004s
Find test
./devel/sage-main/sage/structure/sage_object.pyx
./devel/sage-main/build/sage/structure/sage_object.pyx

real    0m6.898s
user    0m0.112s
sys     0m0.623s

real    0m0.005s
user    0m0.001s
sys     0m0.004s
Copy extracted tarball

real    1m27.787s
user    0m0.617s
sys     0m11.275s

real    0m6.213s
user    0m0.001s
sys     0m0.004s
Remove copied tarball

real    0m11.173s
user    0m0.085s
sys     0m2.242s

real    0m0.467s
user    0m0.001s
sys     0m0.003s

Double the number of disks and include the spare

root@disk:~# zpool destroy tank
root@disk:~# zpool create -f tank raidz2 c4t1d0 c4t2d0 c4t3d0 c4t4d0 c4t5d0 c4t6d0
root@disk:~# zpool add -f tank spare c4t7d0


wstein@disk:/tank/bench$ ./bench 
Extract tarball

real    0m22.055s
user    0m0.511s
sys     0m9.266s

real    0m1.945s
user    0m0.000s
sys     0m0.002s
Remove tarball

real    0m0.131s
user    0m0.001s
sys     0m0.015s

real    0m0.135s
user    0m0.001s
sys     0m0.003s
Create tarball

real    0m39.516s
user    0m0.795s
sys     0m11.394s

real    0m0.793s
user    0m0.000s
sys     0m0.002s
Check size of tarball
1.9G    tar_test

real    0m5.779s
user    0m0.057s
sys     0m0.415s

real    0m0.003s
user    0m0.000s
sys     0m0.002s
Find test
./devel/sage-main/build/sage/structure/sage_object.pyx
./devel/sage-main/sage/structure/sage_object.pyx

real    0m5.972s
user    0m0.075s
sys     0m0.506s

real    0m0.003s
user    0m0.000s
sys     0m0.002s
Copy extracted tarball

real    0m45.943s
user    0m0.359s
sys     0m8.232s

real    0m1.371s
user    0m0.001s
sys     0m0.003s
Remove copied tarball

real    0m10.195s
user    0m0.049s
sys     0m1.391s

real    0m0.465s
user    0m0.000s
sys     0m0.002s

Same as above, but from an NFS mount on sage.math!

Extract tarball

real    27m17.589s
user    0m0.650s
sys     0m12.800s

real    0m0.709s
user    0m0.000s
sys     0m0.010s
Remove tarball

real    0m0.370s
user    0m0.000s
sys     0m0.270s

real    0m0.029s
user    0m0.000s
sys     0m0.000s
Create tarball

real    0m43.244s
user    0m0.300s
sys     0m8.180s

real    0m0.558s
user    0m0.000s
sys     0m0.000s
Check size of tarball
1.9G    tar_test

real    0m4.702s
user    0m0.050s
sys     0m0.980s

real    0m0.018s
user    0m0.000s
sys     0m0.000s
Find test
./devel/sage-main/build/sage/structure/sage_object.pyx
./devel/sage-main/sage/structure/sage_object.pyx

real    0m1.006s
user    0m0.040s
sys     0m0.320s

real    0m0.034s
user    0m0.000s
sys     0m0.000s
Copy extracted tarball

real    20m37.314s
user    0m0.440s
sys     0m14.000s

real    0m0.143s
user    0m0.000s
sys     0m0.000s
Remove copied tarball

real    6m32.163s
user    0m0.120s
sys     0m2.990s

real    0m0.235s
user    0m0.000s
sys     0m0.000s

geom.math mounting via nfs the /wd drive

This mount option:

sage:/wd/vmware   /bench   nfs    rsize=32768,wsize=32768,intr,noatime  0 0 

wstein@geom:/bench/bench$ time ./bench
Extract tarball

real    12m17.771s
user    0m0.540s
sys     0m11.950s

real    0m0.023s
user    0m0.000s
sys     0m0.010s
Remove tarball

real    0m1.021s
user    0m0.000s
sys     0m0.260s

real    0m0.002s
user    0m0.000s
sys     0m0.000s
Create tarball

real    1m50.797s
user    0m0.310s
sys     0m7.150s

real    0m0.006s
user    0m0.000s
sys     0m0.000s
Check size of tarball
1.9G    test_tar

real    0m6.299s
user    0m0.060s
sys     0m1.060s

real    0m0.018s
user    0m0.010s
sys     0m0.010s
Find test
./devel/sage-main/build/sage/structure/sage_object.pyx
./devel/sage-main/sage/structure/sage_object.pyx

real    0m1.015s
user    0m0.060s
sys     0m0.240s

real    0m0.019s
user    0m0.000s
sys     0m0.000s
Copy extracted tarball

real    9m59.754s
user    0m0.380s
sys     0m13.390s

real    0m0.031s
user    0m0.000s
sys     0m0.000s
Remove copied tarball

real    4m19.422s
user    0m0.080s
sys     0m2.680s

real    0m0.054s
user    0m0.010s
sys     0m0.010s
Testing doctesting of sage_object.pyx
./bench: line 59: cd: tar_test: No such file or directory
Doctesting sage_object.pyx twice

real    0m0.001s
user    0m0.000s
sys     0m0.000s

real    0m0.000s
user    0m0.000s
sys     0m0.000s
Doctesting entire sage tree using 20 threads.

real    0m0.000s
user    0m0.000s
sys     0m0.000s

real    28m36.262s
user    0m1.460s
sys     0m36.760s

2013-05-11 18:34