root@typhoon-base-unit1:/root> SUCCESS [2013-06-21 13:47:00.986740] E [rpcsvc.c:448:rpcsvc_check_and_reply_error] 0-rpcsvc: rpc actor failed to complete successfully [2013-06-21 13:47:00.987968] I [glusterd-handshake.c:428:glusterd_mgmt_hndsk_versions_ack] 0-management: using the op-version 1 [2013-06-21 13:47:00.988028] D [glusterd-store.c:1430:glusterd_store_retrieve_value] 0-: key UUID read [2013-06-21 13:47:00.988040] D [glusterd-store.c:1433:glusterd_store_retrieve_value] 0-: key UUID found [2013-06-21 13:47:00.988058] D [glusterd-store.c:1790:glusterd_retrieve_uuid] 0-: Returning 0 [2013-06-21 13:47:00.988072] I [glusterd.c:125:glusterd_uuid_init] 0-management: retrieved UUID: 3a686c46-1b4f-4de6-9825-a0abcf3ff9ac [2013-06-21 13:47:00.988109] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:00.988125] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:00.988649] I [glusterd-handler.c:2013:glusterd_handle_probe_query] 0-glusterd: Received probe from uuid: bcf4310c-5275-42a2-b89d-3e7b6f788f03 [2013-06-21 13:47:00.988672] D [glusterd-utils.c:4678:glusterd_friend_find_by_uuid] 0-management: Friend with uuid: bcf4310c-5275-42a2-b89d-3e7b6f788f03, not found [2013-06-21 13:47:00.988683] D [glusterd-handler.c:430:glusterd_friend_find] 0-management: Unable to find peer by uuid: bcf4310c-5275-42a2-b89d-3e7b6f788f03 [2013-06-21 13:47:00.989168] D [glusterd-utils.c:4764:glusterd_friend_find_by_hostname] 0-management: Unable to find friend: 172.17.23.117 [2013-06-21 13:47:00.989185] D [glusterd-handler.c:442:glusterd_friend_find] 0-management: Unable to find hostname: 172.17.23.117 [2013-06-21 13:47:00.989192] I [glusterd-handler.c:2041:glusterd_handle_probe_query] 0-glusterd: Unable to find peerinfo for host: 172.17.23.117 (24007) [2013-06-21 13:47:00.989205] D [glusterd-utils.c:5290:glusterd_sm_tr_log_init] 0-: returning 0 [2013-06-21 13:47:00.989212] D [glusterd-utils.c:5385:glusterd_peerinfo_new] 0-: returning 0 [2013-06-21 13:47:00.989223] D [glusterd-store.c:2802:glusterd_store_create_peer_dir] 0-: Returning with 0 [2013-06-21 13:47:00.989245] D [glusterd-store.c:1541:glusterd_store_handle_new] 0-management: Returning 0 [2013-06-21 13:47:00.989275] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:00.989290] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:00.989302] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:00.989308] D [glusterd-store.c:2919:glusterd_store_peer_write] 0-: Returning with 0 [2013-06-21 13:47:00.989327] D [glusterd-store.c:2947:glusterd_store_perform_peer_store] 0-: Returning 0 [2013-06-21 13:47:00.989333] D [glusterd-store.c:2968:glusterd_store_peerinfo] 0-: Returning with 0 [2013-06-21 13:47:00.989360] D [glusterd-handler.c:2432:glusterd_transport_inet_options_build] 0-glusterd: Returning 0 [2013-06-21 13:47:00.989386] I [rpc-clnt.c:961:rpc_clnt_connection_init] 0-management: setting frame-timeout to 600 [2013-06-21 13:47:00.989397] D [rpc-transport.c:248:rpc_transport_load] 0-rpc-transport: attempt to load file /usr/lib64/glusterfs/3.4.0alpha/rpc-transport/socket.so [2013-06-21 13:47:00.989443] I [socket.c:3459:socket_init] 0-management: SSL support is NOT enabled [2013-06-21 13:47:00.989451] I [socket.c:3474:socket_init] 0-management: using system polling thread [2013-06-21 13:47:00.989947] D [common-utils.c:207:gf_resolve_ip6] 0-resolver: returning ip-172.17.23.117 (port-24007) for hostname: 172.17.23.117 and port: 24007 [2013-06-21 13:47:00.990019] W [common-utils.c:2217:gf_get_reserved_ports] 0-glusterfs: could not open the file /proc/sys/net/ipv4/ip_local_reserved_ports for getting reserved ports info (No such file or directory) [2013-06-21 13:47:00.990030] W [common-utils.c:2250:gf_process_reserved_ports] 0-glusterfs: Not able to get reserved ports, hence there is a possibility that glusterfs may consume reserved port [2013-06-21 13:47:00.990466] D [glusterd-handler.c:2371:glusterd_rpc_create] 0-management: returning 0 [2013-06-21 13:47:00.990479] I [glusterd-handler.c:2525:glusterd_friend_add] 0-management: connect returned 0 [2013-06-21 13:47:00.990501] E [glusterd-utils.c:505:glusterd_serialize_reply] 0-: Failed to encode message [2013-06-21 13:47:00.990512] E [glusterd-utils.c:546:glusterd_submit_reply] 0-: Failed to serialize reply [2013-06-21 13:47:00.990614] I [glusterd-handler.c:2064:glusterd_handle_probe_query] 0-glusterd: Responded to 172.17.23.117, op_ret: 0, op_errno: 0, ret: 0 [2013-06-21 13:47:00.990639] D [glusterd-handler.c:3206:glusterd_peer_rpc_notify] 0-management: got RPC_CLNT_CONNECT [2013-06-21 13:47:00.991716] D [glusterd-handshake.c:956:glusterd_peer_dump_version_cbk] 0-management: Proceeding to op-version handshake with peer 172.17.23.117 [2013-06-21 13:47:00.992750] D [glusterd-handshake.c:619:gd_validate_peer_op_version] 0-management: Peer 172.17.23.117 accepted [2013-06-21 13:47:00.994700] D [glusterd-sm.c:934:glusterd_friend_sm_inject_event] 0-glusterd: Enqueue event: 'GD_FRIEND_EVENT_CONNECTED' [2013-06-21 13:47:00.994716] D [glusterd-handshake.c:562:glusterd_event_connected_inject] 0-: returning 0 [2013-06-21 13:47:00.994723] D [glusterd-sm.c:1012:glusterd_friend_sm] 0-: Dequeued event of type: 'GD_FRIEND_EVENT_CONNECTED' [2013-06-21 13:47:00.994730] D [glusterd-sm.c:188:glusterd_ac_none] 0-: Returning with 0 [2013-06-21 13:47:00.994741] D [glusterd-utils.c:5339:glusterd_sm_tr_log_transition_add] 0-management: Transitioning from 'Probe Received from peer' to 'Connected to Peer' due to event 'GD_FRIEND_EVENT_CONNECTED' [2013-06-21 13:47:00.994750] D [glusterd-utils.c:5341:glusterd_sm_tr_log_transition_add] 0-management: returning 0 [2013-06-21 13:47:00.994760] D [glusterd-store.c:2802:glusterd_store_create_peer_dir] 0-: Returning with 0 [2013-06-21 13:47:00.994795] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:00.994809] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:00.994820] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:00.994826] D [glusterd-store.c:2919:glusterd_store_peer_write] 0-: Returning with 0 [2013-06-21 13:47:00.994858] D [glusterd-store.c:2947:glusterd_store_perform_peer_store] 0-: Returning 0 [2013-06-21 13:47:00.994866] D [glusterd-store.c:2968:glusterd_store_peerinfo] 0-: Returning with 0 [2013-06-21 13:47:01.003858] I [glusterd-handler.c:1739:glusterd_handle_incoming_friend_req] 0-glusterd: Received probe from uuid: bcf4310c-5275-42a2-b89d-3e7b6f788f03 [2013-06-21 13:47:01.003878] D [glusterd-utils.c:4678:glusterd_friend_find_by_uuid] 0-management: Friend with uuid: bcf4310c-5275-42a2-b89d-3e7b6f788f03, not found [2013-06-21 13:47:01.003887] D [glusterd-handler.c:430:glusterd_friend_find] 0-management: Unable to find peer by uuid: bcf4310c-5275-42a2-b89d-3e7b6f788f03 [2013-06-21 13:47:01.003893] D [glusterd-utils.c:4715:glusterd_friend_find_by_hostname] 0-management: Friend 172.17.23.117 found.. state: 9 [2013-06-21 13:47:01.003905] D [glusterd-sm.c:934:glusterd_friend_sm_inject_event] 0-glusterd: Enqueue event: 'GD_FRIEND_EVENT_RCVD_FRIEND_REQ' [2013-06-21 13:47:01.003912] D [glusterd-sm.c:1012:glusterd_friend_sm] 0-: Dequeued event of type: 'GD_FRIEND_EVENT_RCVD_FRIEND_REQ' [2013-06-21 13:47:01.003924] D [glusterd-utils.c:3171:glusterd_compare_friend_data] 0-: Returning with ret: 0, status: 0 [2013-06-21 13:47:01.003931] D [glusterd-sm.c:934:glusterd_friend_sm_inject_event] 0-glusterd: Enqueue event: 'GD_FRIEND_EVENT_LOCAL_ACC' [2013-06-21 13:47:01.003958] I [glusterd-handler.c:2694:glusterd_xfer_friend_add_resp] 0-glusterd: Responded to 172.17.23.119 (0), ret: 0 [2013-06-21 13:47:01.003967] D [glusterd-sm.c:693:glusterd_ac_handle_friend_add_req] 0-: Returning with 0 [2013-06-21 13:47:01.003974] D [glusterd-utils.c:5339:glusterd_sm_tr_log_transition_add] 0-management: Transitioning from 'Connected to Peer' to 'Connected to Peer' due to event 'GD_FRIEND_EVENT_RCVD_FRIEND_REQ' [2013-06-21 13:47:01.003981] D [glusterd-utils.c:5341:glusterd_sm_tr_log_transition_add] 0-management: returning 0 [2013-06-21 13:47:01.003991] D [glusterd-store.c:2802:glusterd_store_create_peer_dir] 0-: Returning with 0 [2013-06-21 13:47:01.004006] D [glusterd-store.c:1580:glusterd_store_handle_destroy] 0-management: Returning 0 [2013-06-21 13:47:01.004036] D [glusterd-store.c:1541:glusterd_store_handle_new] 0-management: Returning 0 [2013-06-21 13:47:01.004062] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:01.004075] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:01.004087] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:01.004093] D [glusterd-store.c:2919:glusterd_store_peer_write] 0-: Returning with 0 [2013-06-21 13:47:01.004112] D [glusterd-store.c:2947:glusterd_store_perform_peer_store] 0-: Returning 0 [2013-06-21 13:47:01.004119] D [glusterd-store.c:2968:glusterd_store_peerinfo] 0-: Returning with 0 [2013-06-21 13:47:01.004126] D [glusterd-sm.c:1012:glusterd_friend_sm] 0-: Dequeued event of type: 'GD_FRIEND_EVENT_LOCAL_ACC' [2013-06-21 13:47:01.004135] D [glusterd-sm.c:934:glusterd_friend_sm_inject_event] 0-glusterd: Enqueue event: 'GD_FRIEND_EVENT_PROBE' [2013-06-21 13:47:01.004141] D [glusterd-sm.c:251:glusterd_ac_reverse_probe_begin] 0-: returning with 0 [2013-06-21 13:47:01.004150] D [glusterd-utils.c:5339:glusterd_sm_tr_log_transition_add] 0-management: Transitioning from 'Connected to Peer' to 'Peer is connected and Accepted' due to event 'GD_FRIEND_EVENT_LOCAL_ACC' [2013-06-21 13:47:01.004159] D [glusterd-utils.c:5341:glusterd_sm_tr_log_transition_add] 0-management: returning 0 [2013-06-21 13:47:01.004167] D [glusterd-store.c:2802:glusterd_store_create_peer_dir] 0-: Returning with 0 [2013-06-21 13:47:01.004196] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:01.004212] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:01.004223] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:01.004229] D [glusterd-store.c:2919:glusterd_store_peer_write] 0-: Returning with 0 [2013-06-21 13:47:01.004251] D [glusterd-store.c:2947:glusterd_store_perform_peer_store] 0-: Returning 0 [2013-06-21 13:47:01.004257] D [glusterd-store.c:2968:glusterd_store_peerinfo] 0-: Returning with 0 [2013-06-21 13:47:01.004263] D [glusterd-sm.c:1012:glusterd_friend_sm] 0-: Dequeued event of type: 'GD_FRIEND_EVENT_PROBE' [2013-06-21 13:47:01.004270] D [glusterd-utils.c:4715:glusterd_friend_find_by_hostname] 0-management: Friend 172.17.23.117 found.. state: 10 [2013-06-21 13:47:01.004311] D [glusterd-rpc-ops.c:959:glusterd_rpc_probe] 0-glusterd: Returning 0 [2013-06-21 13:47:01.004320] D [glusterd-sm.c:358:glusterd_ac_friend_probe] 0-: Returning with 0 [2013-06-21 13:47:01.004327] D [glusterd-utils.c:5339:glusterd_sm_tr_log_transition_add] 0-management: Transitioning from 'Peer is connected and Accepted' to 'Peer is connected and Accepted' due to event 'GD_FRIEND_EVENT_PROBE' [2013-06-21 13:47:01.004339] D [glusterd-utils.c:5341:glusterd_sm_tr_log_transition_add] 0-management: returning 0 [2013-06-21 13:47:01.004348] D [glusterd-store.c:2802:glusterd_store_create_peer_dir] 0-: Returning with 0 [2013-06-21 13:47:01.004379] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:01.004392] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:01.004404] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:01.004409] D [glusterd-store.c:2919:glusterd_store_peer_write] 0-: Returning with 0 [2013-06-21 13:47:01.004430] D [glusterd-store.c:2947:glusterd_store_perform_peer_store] 0-: Returning 0 [2013-06-21 13:47:01.004436] D [glusterd-store.c:2968:glusterd_store_peerinfo] 0-: Returning with 0 [2013-06-21 13:47:01.017346] I [glusterd-rpc-ops.c:211:glusterd_probe_cbk] 0-glusterd: Received probe resp from uuid: bcf4310c-5275-42a2-b89d-3e7b6f788f03, host: 172.17.23.117 [2013-06-21 13:47:01.017360] D [glusterd-utils.c:4671:glusterd_friend_find_by_uuid] 0-management: Friend found... state: Peer is connected and Accepted [2013-06-21 13:47:01.017368] D [glusterd-sm.c:934:glusterd_friend_sm_inject_event] 0-glusterd: Enqueue event: 'GD_FRIEND_EVENT_INIT_FRIEND_REQ' [2013-06-21 13:47:01.017375] D [glusterd-sm.c:1012:glusterd_friend_sm] 0-: Dequeued event of type: 'GD_FRIEND_EVENT_INIT_FRIEND_REQ' [2013-06-21 13:47:01.017395] D [glusterd-utils.c:2077:glusterd_build_volume_dict] 0-: Returning with 0 [2013-06-21 13:47:01.017424] D [glusterd-rpc-ops.c:1013:glusterd_rpc_friend_add] 0-glusterd: Returning 0 [2013-06-21 13:47:01.017432] D [glusterd-sm.c:286:glusterd_ac_friend_add] 0-: Returning with 0 [2013-06-21 13:47:01.017438] D [glusterd-utils.c:5339:glusterd_sm_tr_log_transition_add] 0-management: Transitioning from 'Peer is connected and Accepted' to 'Sent and Received peer request' due to event 'GD_FRIEND_EVENT_INIT_FRIEND_REQ' [2013-06-21 13:47:01.017446] D [glusterd-utils.c:5341:glusterd_sm_tr_log_transition_add] 0-management: returning 0 [2013-06-21 13:47:01.017456] D [glusterd-store.c:2802:glusterd_store_create_peer_dir] 0-: Returning with 0 [2013-06-21 13:47:01.017490] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:01.017505] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:01.017517] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:01.017523] D [glusterd-store.c:2919:glusterd_store_peer_write] 0-: Returning with 0 [2013-06-21 13:47:01.017552] D [glusterd-store.c:2947:glusterd_store_perform_peer_store] 0-: Returning 0 [2013-06-21 13:47:01.017559] D [glusterd-store.c:2968:glusterd_store_peerinfo] 0-: Returning with 0 [2013-06-21 13:47:01.017565] I [glusterd-rpc-ops.c:281:glusterd_probe_cbk] 0-glusterd: Received resp to probe req [2013-06-21 13:47:01.017818] I [glusterd-rpc-ops.c:322:glusterd_friend_add_cbk] 0-glusterd: Received ACC from uuid: bcf4310c-5275-42a2-b89d-3e7b6f788f03, host: 172.17.23.117, port: 0 [2013-06-21 13:47:01.017829] D [glusterd-utils.c:4671:glusterd_friend_find_by_uuid] 0-management: Friend found... state: Sent and Received peer request [2013-06-21 13:47:01.017836] D [glusterd-sm.c:934:glusterd_friend_sm_inject_event] 0-glusterd: Enqueue event: 'GD_FRIEND_EVENT_RCVD_ACC' [2013-06-21 13:47:01.017842] D [glusterd-sm.c:1012:glusterd_friend_sm] 0-: Dequeued event of type: 'GD_FRIEND_EVENT_RCVD_ACC' [2013-06-21 13:47:01.017855] I [glusterd-sm.c:494:glusterd_ac_send_friend_update] 0-: Added uuid: bcf4310c-5275-42a2-b89d-3e7b6f788f03, host: 172.17.23.117 [2013-06-21 13:47:01.017885] D [glusterd-rpc-ops.c:1092:glusterd_rpc_friend_update] 0-glusterd: Returning 0 [2013-06-21 13:47:01.017894] D [glusterd-sm.c:520:glusterd_ac_send_friend_update] 0-: Returning with 0 [2013-06-21 13:47:01.017901] D [glusterd-utils.c:5339:glusterd_sm_tr_log_transition_add] 0-management: Transitioning from 'Sent and Received peer request' to 'Peer in Cluster' due to event 'GD_FRIEND_EVENT_RCVD_ACC' [2013-06-21 13:47:01.017910] D [glusterd-utils.c:5341:glusterd_sm_tr_log_transition_add] 0-management: returning 0 [2013-06-21 13:47:01.017919] D [glusterd-store.c:2802:glusterd_store_create_peer_dir] 0-: Returning with 0 [2013-06-21 13:47:01.017951] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:01.017964] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:01.017975] D [glusterd-store.c:1492:glusterd_store_save_value] 0-management: returning: 0 [2013-06-21 13:47:01.017981] D [glusterd-store.c:2919:glusterd_store_peer_write] 0-: Returning with 0 [2013-06-21 13:47:01.018003] D [glusterd-store.c:2947:glusterd_store_perform_peer_store] 0-: Returning 0 [2013-06-21 13:47:01.018014] D [glusterd-store.c:2968:glusterd_store_peerinfo] 0-: Returning with 0 [2013-06-21 13:47:01.018023] D [glusterd-utils.c:336:glusterd_lock] 0-management: Cluster lock held by 3a686c46-1b4f-4de6-9825-a0abcf3ff9ac [2013-06-21 13:47:01.018054] D [glusterd-utils.c:4671:glusterd_friend_find_by_uuid] 0-management: Friend found... state: Peer in Cluster [2013-06-21 13:47:01.018064] I [glusterd-handler.c:1886:glusterd_handle_friend_update] 0-glusterd: Received friend update from uuid: bcf4310c-5275-42a2-b89d-3e7b6f788f03 [2013-06-21 13:47:01.018080] I [glusterd-handler.c:1931:glusterd_handle_friend_update] 0-: Received uuid: 3a686c46-1b4f-4de6-9825-a0abcf3ff9ac, hostname:172.17.23.119 [2013-06-21 13:47:01.018087] I [glusterd-handler.c:1940:glusterd_handle_friend_update] 0-: Received my uuid as Friend [2013-06-21 13:47:01.018311] I [glusterd-rpc-ops.c:503:glusterd_friend_update_cbk] 0-management: Received ACC from uuid: bcf4310c-5275-42a2-b89d-3e7b6f788f03 ^C