<div dir="ltr">I build one cluster by GlusterFS 1.3 with 3 node for brick<br><br>Xeon QuardCore 2.33GHz, 32G RAM, 3G Network (bonding), Raid 6 with 12HDD 1T + 2 HotSpare<br clear="all"><br>On this cluster i have 3M File with 16TB .<br>
<br>Client <br>5 node Xeon QuarCore 2.33Ghz, 8GRAM, 2G Network (bonding). Runing Lighttpd with 600 Concurent Connection for streaming FLV<br><br><br>I run streaming 5Gbps normal, but if run 'ls -l' in directory mount glusterfs then all system very slow.<br>
<br>Please Help me for solution<br><br>This Server Config:<br>##############################################<br>### GlusterFS Server Volume Specification ##<br>##############################################<br><br><br>
# Unify Volume 1<br>volume unify-raw1<br> type storage/posix<br> option directory /home/node1/unify/baamboo/data<br>end-volume<br><br>volume u-posix-locks1<br> type features/posix-locks<br> option mandatory on<br> subvolumes unify-raw1<br>
end-volume<br><br>volume u-io-thr1<br> type performance/io-threads<br> option thread-count 4<br> option cache-size 128MB<br> subvolumes u-posix-locks1<br>end-volume<br><br>volume u-wb1<br> type performance/write-behind<br>
option aggregate-size 1MB # default is 0bytes<br> option flush-behind on # default is 'off'<br> subvolumes u-io-thr1<br>end-volume<br><br>volume unify1<br> type performance/read-ahead<br> option page-size 512kB # 256KB is the default option<br>
option page-count 64 # 2 is default option<br> option force-atime-update off # default is off<br> subvolumes u-wb1<br>end-volume<br><br><br># Unify Volume 2<br>volume unify-raw2<br> type storage/posix<br> option directory /home/node2/unify/baamboo/data<br>
end-volume<br><br>volume u-posix-locks2<br> type features/posix-locks<br> option mandatory on<br> subvolumes unify-raw2<br>end-volume<br><br>volume u-io-thr2<br> type performance/io-threads<br> option thread-count 4<br>
option cache-size 128MB<br> subvolumes u-posix-locks2<br>end-volume<br><br>volume u-wb2<br> type performance/write-behind<br> option aggregate-size 1MB # default is 0bytes<br> option flush-behind on # default is 'off'<br>
subvolumes u-io-thr2<br>end-volume<br><br>volume unify2<br> type performance/read-ahead<br> option page-size 512kB # 256KB is the default option<br> option page-count 64 # 2 is default option<br> option force-atime-update off # default is off<br>
subvolumes u-wb2<br>end-volume<br><br><br><br><br>volume ns-raw<br> type storage/posix<br> option directory /home/node1/unify/baamboo/ns<br>end-volume<br><br>volume ns-io-thr<br> type performance/io-threads<br> option thread-count 4<br>
option cache-size 32MB<br> subvolumes ns-raw<br>end-volume<br><br>volume ns<br> type performance/read-ahead<br> option page-size 256kB # 256KB is the default option<br> option page-count 16 # 2 is default option<br>
option force-atime-update off # default is off<br> subvolumes ns-io-thr<br>end-volume<br><br><br><br>### Add network serving capability to above brick.<br>volume server<br> type protocol/server<br> option transport-type tcp/server # For TCP/IP transport<br>
option bind-address <a href="http://192.168.6.6">192.168.6.6</a> # Default is listen on all interfaces<br> option listen-port 60001 # Default is 6996<br> subvolumes unify1 unify2 ns <br> option auth.ip.unify1.allow 192.168.* # Allow access to "brick" volume<br>
option auth.ip.unify2.allow 192.168.* # Allow access to "brick" volume<br> option auth.ip.ns.allow 192.168.* # Allow access to "brick" volume<br>end-volume<br><br><br>Client Config<br><br>### file: client-volume.spec.sample<br>
<br><br>##############################################<br>### GlusterFS Client Volume Specification ##<br>##############################################<br><br>### Add client feature and attach to remote subvolume<br># volume client<br>
# type protocol/client<br># option transport-type tcp/client # for TCP/IP transport<br># option ib-verbs-work-request-send-size 1048576<br># option ib-verbs-work-request-send-count 16<br># option ib-verbs-work-request-recv-size 1048576<br>
# option ib-verbs-work-request-recv-count 16<br># option transport-type ib-sdp/client # for Infiniband transport<br># option transport-type ib-verbs/client # for ib-verbs transport <br># option remote-host <a href="http://127.0.0.1">127.0.0.1</a> # IP address of the remote brick<br>
# option remote-port 6996 # default server port is 6996<br><br># option transport-timeout 30 # seconds to wait for a reply<br> # from server for each request<br>
# option remote-subvolume brick # name of the remote volume<br># end-volume<br><br><br># Volume for Node 1<br>volume unify_1<br> type protocol/client<br> option transport-type tcp/client<br> option remote-host <a href="http://192.168.6.6">192.168.6.6</a><br>
option remote-port 60001<br> option remote-subvolume unify1<br> option transport-timeout 600 # seconds to wait for a reply<br>end-volume<br><br># Volume for Node 2<br>volume unify_2<br> type protocol/client<br>
option transport-type tcp/client<br> option remote-host <a href="http://192.168.6.6">192.168.6.6</a><br> option remote-port 60001<br> option remote-subvolume unify2<br> option transport-timeout 600 # seconds to wait for a reply<br>
end-volume<br><br># Volume for Node 3<br>volume unify_3<br> type protocol/client<br> option transport-type tcp/client<br> option remote-host <a href="http://192.168.6.8">192.168.6.8</a><br> option remote-port 60001<br>
option remote-subvolume unify1<br> option transport-timeout 600 # seconds to wait for a reply<br>end-volume<br><br># Volume for Node 4<br>volume unify_4<br> type protocol/client<br> option transport-type tcp/client<br>
option remote-host <a href="http://192.168.6.8">192.168.6.8</a><br> option remote-port 60001<br> option remote-subvolume unify2<br> option transport-timeout 600 # seconds to wait for a reply<br>end-volume<br>
<br><br># Volume for Node 5<br>volume unify_5<br> type protocol/client<br> option transport-type tcp/client<br> option remote-host <a href="http://192.168.6.4">192.168.6.4</a><br> option remote-port 60001<br> option remote-subvolume unify1<br>
option transport-timeout 600 # seconds to wait for a reply<br>end-volume<br><br><br># Volume for Node 6<br>volume unify_6<br> type protocol/client<br> option transport-type tcp/client<br> option remote-host <a href="http://192.168.6.4">192.168.6.4</a><br>
option remote-port 60001<br> option remote-subvolume unify2<br> option transport-timeout 600 # seconds to wait for a reply<br>end-volume<br><br><br># Volume for Node NS<br># volume unify_ns1<br># type protocol/client<br>
# option transport-type tcp/client<br># option remote-host <a href="http://192.168.6.6">192.168.6.6</a><br># option remote-port 60001<br># option remote-subvolume ns<br># option transport-timeout 600 # seconds to wait for a reply<br>
# end-volume<br><br># Volume for Node NS<br>volume unify_ns_raw # unify_ns2<br> type protocol/client<br> option transport-type tcp/client<br> option remote-host <a href="http://192.168.6.8">192.168.6.8</a><br> option remote-port 60001<br>
option remote-subvolume ns<br> option transport-timeout 600 # seconds to wait for a reply<br>end-volume<br><br># Volume for Node NS<br># volume unify_ns3<br># type protocol/client<br># option transport-type tcp/client<br>
# option remote-host <a href="http://192.168.6.4">192.168.6.4</a><br># option remote-port 60001<br># option remote-subvolume ns<br># option transport-timeout 600 # seconds to wait for a reply<br># end-volume<br>
<br># Volume AFR node 1-2-3<br># volume unify_ns_raw<br># type cluster/afr<br># subvolumes unify_ns1 unify_ns2 unify_ns3<br># end-volume<br><br><br>volume ns_iot<br> type performance/io-threads<br> option thread-count 4<br>
option cache-size 256MB<br> subvolumes unify_ns_raw<br>end-volume<br><br><br># Add readahead feature<br>volume ns_readahead<br> type performance/read-ahead<br> option page-size 128kB # unit in bytes<br> option page-count 16 # cache per file = (page-count x page-size)<br>
subvolumes ns_iot<br>end-volume<br><br>volume unify_ns # ns_ioc<br> type performance/io-cache<br> option cache-size 128MB # default is 32MB<br> option page-size 128KB #128KB is default option<br>
# option priority *.h:3,*.html:2,*:1 # default is '*:0'<br> option force-revalidate-timeout 2 # default is 1<br> subvolumes ns_readahead<br>end-volume<br><br><br><br>volume unify<br> type cluster/unify<br> subvolumes unify_1 unify_2 unify_3 unify_4 unify_5 unify_6<br>
option namespace unify_ns<br> option scheduler rr<br> option rr.limits.min-free-disk 25%<br> option rr.refresh-interval 10<br>end-volume<br><br>volume iot<br> type performance/io-threads<br> option thread-count 4<br>
option cache-size 256MB<br> subvolumes unify<br>end-volume<br><br>volume wb<br> type performance/write-behind<br> option aggregate-size 1MB # default is 0bytes<br> option flush-behind on # default is 'off'<br>
subvolumes iot<br>end-volume<br><br>volume readahead<br> type performance/read-ahead<br> option page-size 512kB # unit in bytes<br> option page-count 64 # cache per file = (page-count x page-size)<br> subvolumes wb<br>
end-volume<br><br>volume ioc<br> type performance/io-cache<br> option cache-size 1024MB # default is 32MB<br> option page-size 1MB #128KB is default option<br> # option priority *.h:3,*.html:2,*:1 # default is '*:0'<br>
option force-revalidate-timeout 2 # default is 1<br> subvolumes readahead<br>end-volume<br><br><br>Thanks!<br><br><br><br>-- <br>Le Minh Duc<br>Email: <a href="http://duclm.vn" target="_blank">duclm.vn</a>@<a href="http://gmail.com" target="_blank">gmail.com</a><br>
</div>