[Openais] [PATCH corosync 4/4] CTS: add quorum notifications to test agents.
Angus Salkeld
asalkeld at redhat.com
Mon Apr 12 20:42:08 PDT 2010
Test that we are getting quorum & votequorum notifications.
-Angus
Signed-off-by: Angus Salkeld <asalkeld at redhat.com>
---
cts/agents/votequorum_test_agent.c | 178 +++++++++++++++++++++++++++---------
cts/corosync.py | 1 +
cts/corotests.py | 68 ++++++--------
3 files changed, 161 insertions(+), 86 deletions(-)
diff --git a/cts/agents/votequorum_test_agent.c b/cts/agents/votequorum_test_agent.c
index ab9d96e..52512dc 100644
--- a/cts/agents/votequorum_test_agent.c
+++ b/cts/agents/votequorum_test_agent.c
@@ -46,32 +46,146 @@
#include <netdb.h>
#include <sys/un.h>
#include <syslog.h>
+#include <poll.h>
+#include <corosync/totem/coropoll.h>
#include <corosync/corotypes.h>
#include <corosync/votequorum.h>
#include <corosync/quorum.h>
#include "common_test_agent.h"
+static quorum_handle_t q_handle = 0;
+static votequorum_handle_t vq_handle = 0;
-static void getinfo (int sock)
+static void votequorum_notification_fn(
+ votequorum_handle_t handle,
+ uint64_t context,
+ uint32_t quorate,
+ uint32_t node_list_entries,
+ votequorum_node_t node_list[])
+{
+ syslog (LOG_INFO, "VQ notification quorate: %d", quorate);
+}
+
+static void quorum_notification_fn(
+ quorum_handle_t handle,
+ uint32_t quorate,
+ uint64_t ring_id,
+ uint32_t view_list_entries,
+ uint32_t *view_list)
+{
+ syslog (LOG_INFO, "NQ notification quorate: %d", quorate);
+}
+
+
+static int vq_dispatch_wrapper_fn (hdb_handle_t handle,
+ int fd,
+ int revents,
+ void *data)
+{
+ cs_error_t error = votequorum_dispatch (vq_handle, CS_DISPATCH_ALL);
+ if (error == CS_ERR_LIBRARY) {
+ syslog (LOG_ERR, "%s() got LIB error disconnecting from corosync.", __func__);
+ poll_dispatch_delete (ta_poll_handle_get(), fd);
+ close (fd);
+ }
+ return 0;
+}
+
+static int q_dispatch_wrapper_fn (hdb_handle_t handle,
+ int fd,
+ int revents,
+ void *data)
+{
+ cs_error_t error = quorum_dispatch (q_handle, CS_DISPATCH_ALL);
+ if (error == CS_ERR_LIBRARY) {
+ syslog (LOG_ERR, "%s() got LIB error disconnecting from corosync.", __func__);
+ poll_dispatch_delete (ta_poll_handle_get(), fd);
+ close (fd);
+ }
+ return 0;
+}
+
+static int q_lib_init(void)
+{
+ votequorum_callbacks_t vq_callbacks;
+ quorum_callbacks_t q_callbacks;
+ int ret = 0;
+ int fd;
+
+ if (vq_handle == 0) {
+ syslog (LOG_INFO, "votequorum_initialize");
+ vq_callbacks.votequorum_notify_fn = votequorum_notification_fn;
+ vq_callbacks.votequorum_expectedvotes_notify_fn = NULL;
+ ret = CS_ERR_NOT_EXIST;
+ while (ret == CS_ERR_NOT_EXIST) {
+ ret = votequorum_initialize (&vq_handle, &vq_callbacks);
+ sleep (1);
+ }
+ if (ret != CS_OK) {
+ syslog (LOG_ERR, "votequorum_initialize FAILED: %d\n", ret);
+ vq_handle = 0;
+ }
+ else {
+ ret = votequorum_trackstart (vq_handle, vq_handle, CS_TRACK_CHANGES);
+ if (ret != CS_OK) {
+ syslog (LOG_ERR, "votequorum_trackstart FAILED: %d\n", ret);
+ }
+
+ votequorum_fd_get (vq_handle, &fd);
+ poll_dispatch_add (ta_poll_handle_get(), fd,
+ POLLIN|POLLNVAL, NULL, vq_dispatch_wrapper_fn);
+ }
+ }
+ if (q_handle == 0) {
+ syslog (LOG_INFO, "quorum_initialize");
+ q_callbacks.quorum_notify_fn = quorum_notification_fn;
+ ret = quorum_initialize (&q_handle, &q_callbacks);
+ if (ret != CS_OK) {
+ syslog (LOG_ERR, "quorum_initialize FAILED: %d\n", ret);
+ q_handle = 0;
+ }
+ else {
+ ret = quorum_trackstart (q_handle, CS_TRACK_CHANGES);
+ if (ret != CS_OK) {
+ syslog (LOG_ERR, "quorum_trackstart FAILED: %d\n", ret);
+ }
+ quorum_fd_get (q_handle, &fd);
+ poll_dispatch_add (ta_poll_handle_get(), fd,
+ POLLIN|POLLNVAL, NULL, q_dispatch_wrapper_fn);
+ }
+ }
+ return ret;
+}
+
+static void lib_init (int sock)
{
- votequorum_callbacks_t callbacks;
int ret;
- struct votequorum_info info;
char response[100];
- votequorum_handle_t g_handle;
- callbacks.votequorum_notify_fn = NULL;
- callbacks.votequorum_expectedvotes_notify_fn = NULL;
+ ret = q_lib_init ();
- ret = votequorum_initialize(&g_handle, &callbacks);
if (ret != CS_OK) {
snprintf (response, 100, "%s", FAIL_STR);
- syslog (LOG_ERR, "votequorum_initialize FAILED: %d\n", ret);
+ syslog (LOG_ERR, "q_lib_init FAILED: %d\n", ret);
goto send_response;
}
- ret = votequorum_getinfo(g_handle, 0, &info);
+ snprintf (response, 100, "%s", OK_STR);
+
+send_response:
+ send (sock, response, strlen (response), 0);
+}
+
+static void getinfo (int sock)
+{
+ int ret;
+ struct votequorum_info info;
+ char response[100];
+
+ q_lib_init ();
+
+ ret = votequorum_getinfo(vq_handle, 0, &info);
if (ret != CS_OK) {
snprintf (response, 100, "%s", FAIL_STR);
syslog (LOG_ERR, "votequorum_getinfo FAILED: %d\n", ret);
@@ -86,29 +200,18 @@ static void getinfo (int sock)
info.quorum);
send_response:
- votequorum_finalize (g_handle);
send (sock, response, strlen (response), 0);
}
static void setexpected (int sock, char *arg)
{
- votequorum_callbacks_t callbacks;
int ret;
char response[100];
- votequorum_handle_t g_handle;
- callbacks.votequorum_notify_fn = NULL;
- callbacks.votequorum_expectedvotes_notify_fn = NULL;
+ q_lib_init ();
- ret = votequorum_initialize(&g_handle, &callbacks);
- if (ret != CS_OK) {
- snprintf (response, 100, "%s", FAIL_STR);
- syslog (LOG_ERR, "votequorum_initialize FAILED: %d\n", ret);
- goto send_response;
- }
-
- ret = votequorum_setexpected (g_handle, atoi(arg));
+ ret = votequorum_setexpected (vq_handle, atoi(arg));
if (ret != CS_OK) {
snprintf (response, 100, "%s", FAIL_STR);
syslog (LOG_ERR, "set expected votes FAILED: %d\n", ret);
@@ -118,28 +221,18 @@ static void setexpected (int sock, char *arg)
snprintf (response, 100, "%s", OK_STR);
send_response:
- votequorum_finalize (g_handle);
+ votequorum_finalize (vq_handle);
send (sock, response, strlen (response) + 1, 0);
}
static void setvotes (int sock, char *arg)
{
- votequorum_callbacks_t callbacks;
int ret;
char response[100];
- votequorum_handle_t g_handle;
- callbacks.votequorum_notify_fn = NULL;
- callbacks.votequorum_expectedvotes_notify_fn = NULL;
-
- ret = votequorum_initialize(&g_handle, &callbacks);
- if (ret != CS_OK) {
- snprintf (response, 100, "%s", FAIL_STR);
- syslog (LOG_ERR, "votequorum_initialize FAILED: %d\n", ret);
- goto send_response;
- }
+ q_lib_init ();
- ret = votequorum_setvotes (g_handle, 0, atoi(arg));
+ ret = votequorum_setvotes (vq_handle, 0, atoi(arg));
if (ret != CS_OK) {
snprintf (response, 100, "%s", FAIL_STR);
syslog (LOG_ERR, "set votes FAILED: %d\n", ret);
@@ -149,7 +242,7 @@ static void setvotes (int sock, char *arg)
snprintf (response, 100, "%s", OK_STR);
send_response:
- votequorum_finalize (g_handle);
+ votequorum_finalize (vq_handle);
send (sock, response, strlen (response), 0);
}
@@ -159,16 +252,10 @@ static void getquorate (int sock)
int ret;
int quorate;
char response[100];
- quorum_handle_t handle;
- ret = quorum_initialize (&handle, NULL);
- if (ret != CS_OK) {
- snprintf (response, 100, "%s", FAIL_STR);
- syslog (LOG_ERR, "quorum_initialize FAILED: %d\n", ret);
- goto send_response;
- }
+ q_lib_init ();
- ret = quorum_getquorate (handle, &quorate);
+ ret = quorum_getquorate (q_handle, &quorate);
if (ret != CS_OK) {
snprintf (response, 100, "%s", FAIL_STR);
syslog (LOG_ERR, "getquorate FAILED: %d\n", ret);
@@ -178,7 +265,6 @@ static void getquorate (int sock)
snprintf (response, 100, "%d", quorate);
send_response:
- quorum_finalize (handle);
send (sock, response, strlen (response), 0);
}
@@ -197,6 +283,8 @@ static void do_command (int sock, char* func, char*args[], int num_args)
setexpected (sock, args[0]);
} else if (strcmp ("quorum_getquorate", func) == 0) {
getquorate (sock);
+ } else if (strcmp ("init", func) == 0) {
+ lib_init (sock);
} else {
syslog (LOG_ERR,"%s RPC:%s not supported!", __func__, func);
snprintf (response, 100, "%s", NOT_SUPPORTED_STR);
diff --git a/cts/corosync.py b/cts/corosync.py
index 626f8dd..6fa7d6a 100644
--- a/cts/corosync.py
+++ b/cts/corosync.py
@@ -583,6 +583,7 @@ class VoteQuorumTestAgent(TestAgent):
self.initialized = False
self.nodeid = None
self.send_recv = True
+ self.send (['init'])
def cpg_local_get(self):
return 1
diff --git a/cts/corotests.py b/cts/corotests.py
index 7a96497..f13829c 100644
--- a/cts/corotests.py
+++ b/cts/corotests.py
@@ -654,6 +654,7 @@ class QuorumState(object):
def __init__(self, cm, node):
self.node = node
self.CM = cm
+ self.CM.votequorum_agent[self.node].init()
def refresh(self):
info = self.CM.votequorum_agent[self.node].votequorum_getinfo()
@@ -690,10 +691,8 @@ class VoteQuorumBase(CoroTest):
self.CM.cpg_agent[n].cpg_join(self.name)
self.id_map[n] = self.CM.cpg_agent[n].cpg_local_get()
- #self.CM.votequorum_agent[self.listener].record_events()
return ret
-
def config_valid(self, config):
if config.has_key('totem/rrp_mode'):
return False
@@ -701,45 +700,6 @@ class VoteQuorumBase(CoroTest):
return True
- def wait_for_quorum_change(self):
- found = False
- max_timeout = 5 * 60
- waited = 0
-
- printit = 0
- self.CM.log("Waiting for quorum event on " + self.listener)
- while not found:
- try:
- event = self.CM.votequorum_agent[self.listener].read_event()
- except:
- return self.failure('connection to test agent failed.')
- if not event == None:
- self.CM.debug("RECEIVED: " + str(event))
- if event == None:
- if waited >= max_timeout:
- return self.failure("timedout(" + str(waited) + " sec) == no event!")
- else:
- time.sleep(1)
- waited = waited + 1
- printit = printit + 1
- if printit is 60:
- print 'waited 60 seconds'
- printit = 0
-
- elif str(event.node_id) in str(self.wobbly_id) and not event.is_member:
- self.CM.log("Got the config change in " + str(waited) + " seconds")
- found = True
- else:
- self.CM.debug("No match")
- self.CM.debug("wobbly nodeid:" + str(self.wobbly_id))
- self.CM.debug("event nodeid:" + str(event.node_id))
- self.CM.debug("event.is_member:" + str(event.is_member))
-
- if found:
- return self.success()
-
-# repeat below with equal and uneven votes
-
###################################################################
class VoteQuorumGoDown(VoteQuorumBase):
# all up
@@ -760,7 +720,14 @@ class VoteQuorumGoDown(VoteQuorumBase):
def __call__(self, node):
self.incr("calls")
+ pats = []
+ pats.append("%s .*VQ notification quorate: 0" % self.listener)
+ pats.append("%s .*NQ notification quorate: 0" % self.listener)
+ quorum = self.create_watch(pats, 30)
+ quorum.setwatch()
+
state = QuorumState(self.CM, self.listener)
+ state.refresh()
for n in self.CM.Env["nodes"]:
if n is self.listener:
continue
@@ -768,6 +735,8 @@ class VoteQuorumGoDown(VoteQuorumBase):
self.victims.append(n)
self.CM.StopaCM(n)
+ #if not self.wait_for_quorum_change():
+ # return self.failure(self.error_message)
nodes_alive = len(self.CM.Env["nodes"]) - len(self.victims)
state.refresh()
#self.expected = self.expected - 1
@@ -793,6 +762,10 @@ class VoteQuorumGoDown(VoteQuorumBase):
if state.quorate == 0:
self.failure('we should have quorum(%d) %d <= %d' % (state.quorate, state.quorum, nodes_alive))
+ if not quorum.lookforall():
+ self.CM.log("Patterns not found: " + repr(quorum.unmatched))
+ return self.failure('quorm event not found')
+
return self.success()
@@ -821,6 +794,12 @@ class VoteQuorumGoUp(VoteQuorumBase):
def __call__(self, node):
self.incr("calls")
+ pats = []
+ pats.append("%s .*VQ notification quorate: 1" % self.listener)
+ pats.append("%s .*NQ notification quorate: 1" % self.listener)
+ quorum = self.create_watch(pats, 30)
+ quorum.setwatch()
+
self.CM.StartaCM(self.listener)
nodes_alive = 1
state = QuorumState(self.CM, self.listener)
@@ -830,6 +809,9 @@ class VoteQuorumGoUp(VoteQuorumBase):
if n is self.listener:
continue
+ #if not self.wait_for_quorum_change():
+ # return self.failure(self.error_message)
+
if state.node_votes != 1:
self.failure('unexpected number of node_votes')
@@ -855,6 +837,10 @@ class VoteQuorumGoUp(VoteQuorumBase):
nodes_alive = nodes_alive + 1
state.refresh()
+ if not quorum.lookforall():
+ self.CM.log("Patterns not found: " + repr(quorum.unmatched))
+ return self.failure('quorm event not found')
+
return self.success()
--
1.6.6.1
More information about the Openais
mailing list