<html>
<head>
<meta content="text/html; charset=GB2312" http-equiv="Content-Type">
</head>
<body bgcolor="#FFFFFF" text="#000000">
<div class="moz-cite-prefix">On 09/13/2013 06:03 PM, kane wrote:<br>
</div>
<blockquote cite="mid:59C78E49-85AE-4AFD-BFCC-6192A19679BD@163.com"
type="cite">
<meta http-equiv="Content-Type" content="text/html;
charset=GB2312">
Hi
<div><br>
</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>We
use samba gluster vfs in IO test, but meet with gluster server
smbd oom killer, </div>
<div>The smbd process spend over 15g RES with top command show, in
the end is our simple test code:</div>
<div><br>
</div>
<div>gluster server vfs --> smbd --> client mount dir
"/mnt/vfs"--> execute vfs test program "$ ./vfs 1000"</div>
<div><br>
</div>
<div>then we can watch gluster server smbd RES with top command.</div>
<div>
<div><br>
</div>
<div> PID USER PR NI VIRT RES SHR S %CPU %MEM TIME+
COMMAND
</div>
<div> 4000 soul 20 0 5486m 4.9g 10m R 108.4 31.5
111:07.07 smbd
</div>
<div> 3447 root 20 0 1408m 44m 2428 S 44.4 0.3
59:11.55 glusterfsd </div>
</div>
<div><br>
</div>
<div>io test code:</div>
<div>=======================================</div>
<div>
<div>#define _LARGEFILE64_SOURCE</div>
<div><br>
</div>
<div>#include <stdio.h></div>
<div>#include <unistd.h></div>
<div>#include <string.h></div>
<div>#include <pthread.h></div>
<div>#include <stdlib.h></div>
<div>#include <fcntl.h></div>
<div>#include <sys/types.h></div>
<div><br>
</div>
<div>int WT = 1;</div>
<div><br>
</div>
<div>#define RND(x) ((x>0)?(genrand() % (x)):0)</div>
<div>extern unsigned long genrand();</div>
<div>extern void sgenrand();</div>
<div><br>
</div>
<div>/* Period parameters */ </div>
<div>#define N 624</div>
<div>#define M 397</div>
<div>#define MATRIX_A 0x9908b0df /* constant vector a */</div>
<div>#define UPPER_MASK 0x80000000 /* most significant w-r bits
*/</div>
<div>#define LOWER_MASK 0x7fffffff /* least significant r bits
*/</div>
<div><br>
</div>
<div>/* Tempering parameters */ </div>
<div>#define TEMPERING_MASK_B 0x9d2c5680</div>
<div>#define TEMPERING_MASK_C 0xefc60000</div>
<div>#define TEMPERING_SHIFT_U(y) (y >> 11)</div>
<div>#define TEMPERING_SHIFT_S(y) (y << 7)</div>
<div>#define TEMPERING_SHIFT_T(y) (y << 15)</div>
<div>#define TEMPERING_SHIFT_L(y) (y >> 18)</div>
<div><br>
</div>
<div>static unsigned long mt[N]; /* the array for the state
vector */</div>
<div>static int mti=N+1; /* mti==N+1 means mt[N] is not
initialized */</div>
<div><br>
</div>
<div>/* Initializing the array with a seed */</div>
<div>void</div>
<div>sgenrand(seed)</div>
<div> unsigned long seed;<span class="Apple-tab-span"
style="white-space:pre"> </span></div>
<div>{</div>
<div> int i;</div>
<div><br>
</div>
<div> for (i=0;i<N;i++) {</div>
<div> mt[i] = seed & 0xffff0000;</div>
<div> seed = 69069 * seed + 1;</div>
<div> mt[i] |= (seed & 0xffff0000) >> 16;</div>
<div> seed = 69069 * seed + 1;</div>
<div> }</div>
<div> mti = N;</div>
<div>}</div>
<div><br>
</div>
<div>unsigned long </div>
<div>genrand()</div>
<div>{</div>
<div> unsigned long y;</div>
<div> static unsigned long mag01[2]={0x0, MATRIX_A};</div>
<div> /* mag01[x] = x * MATRIX_A for x=0,1 */</div>
<div><br>
</div>
<div> if (mti >= N) { /* generate N words at one time */</div>
<div> int kk;</div>
<div><br>
</div>
<div> if (mti == N+1) /* if sgenrand() has not been
called, */</div>
<div> sgenrand(4357); /* a default initial seed is
used */</div>
<div><br>
</div>
<div> for (kk=0;kk<N-M;kk++) {</div>
<div> y =
(mt[kk]&UPPER_MASK)|(mt[kk+1]&LOWER_MASK);</div>
<div> mt[kk] = mt[kk+M] ^ (y >> 1) ^ mag01[y
& 0x1];</div>
<div> }</div>
<div> for (;kk<N-1;kk++) {</div>
<div> y =
(mt[kk]&UPPER_MASK)|(mt[kk+1]&LOWER_MASK);</div>
<div> mt[kk] = mt[kk+(M-N)] ^ (y >> 1) ^
mag01[y & 0x1];</div>
<div> }</div>
<div> y =
(mt[N-1]&UPPER_MASK)|(mt[0]&LOWER_MASK);</div>
<div> mt[N-1] = mt[M-1] ^ (y >> 1) ^ mag01[y &
0x1];</div>
<div><br>
</div>
<div> mti = 0;</div>
<div> }</div>
<div> </div>
<div> y = mt[mti++];</div>
<div> y ^= TEMPERING_SHIFT_U(y);</div>
<div> y ^= TEMPERING_SHIFT_S(y) & TEMPERING_MASK_B;</div>
<div> y ^= TEMPERING_SHIFT_T(y) & TEMPERING_MASK_C;</div>
<div> y ^= TEMPERING_SHIFT_L(y);</div>
<div><br>
</div>
<div> return y; </div>
<div>}</div>
<div><br>
</div>
<div>char *initialize_file_source(int size)</div>
<div>{</div>
<div> char *new_source;</div>
<div> int i;</div>
<div><br>
</div>
<div> if ((new_source=(char *)malloc(size))==NULL) /* allocate
buffer */</div>
<div> fprintf(stderr,"Error: failed to allocate source file
of size %d\n",size);</div>
<div> else</div>
<div> for (i=0; i<size; i++) /* file buffer with junk */</div>
<div> new_source[i]=32+RND(95);</div>
<div><br>
</div>
<div> return(new_source);</div>
<div>}</div>
<div><br>
</div>
<div>void *tran_file(void *map)</div>
<div>{</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>int
block_size = 512;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>char
*read_buffer; /* temporary space for reading file data into */</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>int
fd = open((char *)map, O_RDWR | O_CREAT | O_TRUNC, 0644);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>if(fd
== -1) {</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>perror("open");</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>return
;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div>
<div><br>
</div>
<div>//<span class="Apple-tab-span" style="white-space:pre"> </span>read_buffer=(char
*)malloc(block_size);</div>
<div>//<span class="Apple-tab-span" style="white-space:pre"> </span>memset(read_buffer,
0x56, block_size);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>read_buffer
= initialize_file_source(block_size);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>while(WT){</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>sleep(1);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>pwrite(fd,
read_buffer, block_size, 0);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>pread(fd,
read_buffer, block_size, 0);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>free(read_buffer);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>close(fd);</div>
<div>}</div>
<div><br>
</div>
<div>int main(int argc, char *argv[])</div>
<div>{</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>if(argc
== 1) {</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>printf("Set
file name and file size in GB!\n");</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>return
-1;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div>
<div><br>
</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>unsigned
long long length = atoll(argv[1]) * 1024 * 1024 * 1024;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>unsigned
long long index = 0;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>unsigned
long long count = 0;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>int
fd = 0;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>char
path[32];</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>char
*swap = NULL;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>int
psize = 65536;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>int
ret = 0;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>int
ret_i = 0;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>void
*tret;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>pthread_t
rtid_1;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>pthread_t
rtid_2;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>pthread_t
rtid_3;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>pthread_t
rtid_4;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>int
fd_i = 0;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>int
a_size = 256;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>char
*append_buffer; /* temporary space for reading file data into
*/</div>
<div><br>
</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>swap
= (char *)malloc(sizeof(char) * psize);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>if(swap
== NULL) {</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>printf("Malloc
Swap Failed!\n");</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>close(fd);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>return
-1;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>memset(swap,
0x55, psize);</div>
<div><br>
</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>append_buffer
= initialize_file_source(a_size);</div>
<div><br>
</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>pthread_create(&rtid_1,
NULL, tran_file, "/mnt/vfs/a.map");</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>pthread_create(&rtid_2,
NULL, tran_file, "/mnt/vfs/b.map");</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>pthread_create(&rtid_3,
NULL, tran_file, "/mnt/vfs/c.map");</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>pthread_create(&rtid_4,
NULL, tran_file, "/mnt/vfs/d.map");</div>
<div><br>
</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>while(index
< length)</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>{</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>if((index
% (1024*1024*1024)) == 0) {</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>count++;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>sprintf(path,
"/mnt/vfs/data/%08lld.dat", count);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>if((fd
= open(path, O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR))
< 0)</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>{</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>printf("Can
not open io file: %s\n", path);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>perror("open");</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>return
-1;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>sprintf(path,
"/mnt/vfs/data/%08lld.index", count);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>if((fd_i
= open(path, O_RDWR | O_CREAT | O_APPEND, S_IRUSR | S_IWUSR))
< 0)</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>{</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>printf("Can
not open io file: %s\n", path);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>perror("open");</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>return
-1;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>printf("...1GiB\n");</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div>
<div><br>
</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>ret
= pwrite(fd, swap, psize, (index % (1024*1024*1024)));</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>if(ret
!= psize) {</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>perror("write");</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>printf("Write
data file failed!\n");</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>continue;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>lseek64(fd_i,
((index*a_size)/65536), SEEK_SET);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>ret_i
= write(fd_i, append_buffer, a_size);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>if(ret_i
!= a_size) {</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>perror("write");</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>printf("Write
index file failed!\n");</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>continue;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div>
<div><br>
</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>index
+= ret;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>if((index
% (1024*1024*1024)) == 0) {</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>close(fd);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>close(fd_i);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>}</div>
<div><br>
</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>WT
= 0;</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>pthread_join(rtid_1,
&tret);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>pthread_join(rtid_2,
&tret);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>pthread_join(rtid_3,
&tret);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>pthread_join(rtid_4,
&tret);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>printf("%s
GB write\n", argv[1]);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>sync();</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>free(swap);</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>free(append_buffer);</div>
<div><br>
</div>
<div><span class="Apple-tab-span" style="white-space:pre"> </span>return
0;</div>
<div>}</div>
</div>
<div><br>
</div>
<div>=======================================</div>
<div><br>
</div>
<div><br>
</div>
<div><br>
</div>
<div>
<div apple-content-edited="true">
<div class="" style="color: rgb(0, 0, 0); font-family:
Helvetica; font-size: medium; font-style: normal;
font-variant: normal; font-weight: normal; letter-spacing:
normal; line-height: normal; orphans: 2; text-align:
-webkit-auto; text-indent: 0px; text-transform: none;
white-space: normal; widows: 2; word-spacing: 0px;
-webkit-text-size-adjust: auto; -webkit-text-stroke-width:
0px; word-wrap: break-word; -webkit-nbsp-mode: space;
-webkit-line-break: after-white-space; "><br>
</div>
<div class="" style="color: rgb(0, 0, 0); font-family:
Helvetica; font-size: medium; font-style: normal;
font-variant: normal; font-weight: normal; letter-spacing:
normal; line-height: normal; orphans: 2; text-align:
-webkit-auto; text-indent: 0px; text-transform: none;
white-space: normal; widows: 2; word-spacing: 0px;
-webkit-text-size-adjust: auto; -webkit-text-stroke-width:
0px; word-wrap: break-word; -webkit-nbsp-mode: space;
-webkit-line-break: after-white-space; ">kane </div>
<div class="" style="color: rgb(0, 0, 0); font-family:
Helvetica; font-size: medium; font-style: normal;
font-variant: normal; font-weight: normal; letter-spacing:
normal; line-height: normal; orphans: 2; text-align:
-webkit-auto; text-indent: 0px; text-transform: none;
white-space: normal; widows: 2; word-spacing: 0px;
-webkit-text-size-adjust: auto; -webkit-text-stroke-width:
0px; word-wrap: break-word; -webkit-nbsp-mode: space;
-webkit-line-break: after-white-space; ">----------------------------------------------------------------</div>
<div class="" style="color: rgb(0, 0, 0); font-family:
Helvetica; font-size: medium; font-style: normal;
font-variant: normal; font-weight: normal; letter-spacing:
normal; line-height: normal; orphans: 2; text-align:
-webkit-auto; text-indent: 0px; text-transform: none;
white-space: normal; widows: 2; word-spacing: 0px;
-webkit-text-size-adjust: auto; -webkit-text-stroke-width:
0px; word-wrap: break-word; -webkit-nbsp-mode: space;
-webkit-line-break: after-white-space; ">Email: <a
moz-do-not-send="true" href="mailto:kai.zhou@soulinfo.com">kai.zhou@soulinfo.com</a><br
class="">
µç»°: 0510-85385788-616</div>
<span style="color: rgb(0, 0, 0); font-family: Helvetica;
font-size: medium; font-style: normal; font-variant: normal;
font-weight: normal; letter-spacing: normal; line-height:
normal; orphans: 2; text-align: -webkit-auto; text-indent:
0px; text-transform: none; white-space: normal; widows: 2;
word-spacing: 0px; -webkit-text-size-adjust: auto;
-webkit-text-stroke-width: 0px; "><br>
</span></div>
<br>
</div>
</blockquote>
Thanks for the code. I will try to reproduce the issue using your
code.<br>
<br>
-Lala<br>
</body>
</html>