fix: ignore garbage in zk active nodes

internal/app/app_dcs.go

@@ -15,7 +15,7 @@ func (app *App) GetActiveNodes() ([]string, error) {
 	var activeNodes []string
 	err := app.dcs.Get(pathActiveNodes, &activeNodes)
 	if err != nil {
-		if err == dcs.ErrNotFound {
+		if err == dcs.ErrNotFound || err == dcs.ErrMalformed {
 			return nil, nil
 		}
 		return nil, fmt.Errorf("failed to get active nodes from zk %v", err)

tests/features/active_nodes.feature

@@ -154,3 +154,52 @@ Feature: mysync saves quorum hosts in zk
     """
     ["mysql1","mysql2","mysql3"]
     """
+
+  Scenario: mysync recovers from malformed active nodes after restart
+    Given cluster is up and running
+    And zookeeper node "/test/active_nodes" should match json_exactly within "30" seconds
+    """
+    ["mysql1","mysql2","mysql3"]
+    """
+    When I run command on host "mysql1" with timeout "20" seconds
+      """
+      supervisorctl stop mysync
+      """
+    Then command return code should be "0"
+    When I run command on host "mysql2" with timeout "20" seconds
+      """
+      supervisorctl stop mysync
+      """
+    Then command return code should be "0"
+    When I run command on host "mysql3" with timeout "20" seconds
+      """
+      supervisorctl stop mysync
+      """
+    Then command return code should be "0"
+    When I set raw zookeeper node "/test/active_nodes" to
+      """
+      [[[]garbage
+      """
+    When I run command on host "mysql1" with timeout "20" seconds
+      """
+      supervisorctl start mysync
+      """
+    Then command return code should be "0"
+    When I run command on host "mysql2" with timeout "20" seconds
+      """
+      supervisorctl start mysync
+      """
+    Then command return code should be "0"
+    When I run command on host "mysql3" with timeout "20" seconds
+      """
+      supervisorctl start mysync
+      """
+    Then command return code should be "0"
+    Then mysql host "mysql1" should be master
+    And mysql host "mysql2" should be replica of "mysql1"
+    And mysql host "mysql3" should be replica of "mysql1"
+    And mysql host "mysql1" should be writable
+    And zookeeper node "/test/active_nodes" should match json_exactly within "30" seconds
+    """
+    ["mysql1","mysql2","mysql3"]
+    """

tests/mysync_test.go

@@ -1060,11 +1060,7 @@ func (tctx *testContext) createZookeeperNode(node string) error {
 	return nil
 }
 
-func (tctx *testContext) stepISetZookeeperNode(node string, body *godog.DocString) error {
-	data := []byte(strings.TrimSpace(body.Content))
-	if !json.Valid(data) {
-		return fmt.Errorf("node value is not valid json")
-	}
+func (tctx *testContext) setZookeeperNode(node string, data []byte) error {
 	_, stat, err := tctx.zk.Get(node)
 	if err != nil && err != zk.ErrNoNode {
 		return err
@@ -1077,6 +1073,19 @@ func (tctx *testContext) stepISetZookeeperNode(node string, body *godog.DocStrin
 	return err
 }
 
+func (tctx *testContext) stepISetZookeeperNode(node string, body *godog.DocString) error {
+	data := []byte(strings.TrimSpace(body.Content))
+	if !json.Valid(data) {
+		return fmt.Errorf("node value is not valid json")
+	}
+	return tctx.setZookeeperNode(node, data)
+}
+
+func (tctx *testContext) stepISetRawZookeeperNode(node string, body *godog.DocString) error {
+	data := []byte(strings.TrimSpace(body.Content))
+	return tctx.setZookeeperNode(node, data)
+}
+
 func (tctx *testContext) stepIDeleteZookeeperNode(node string) error {
 	_, stat, err := tctx.zk.Get(node)
 	if err != nil {
@@ -1681,6 +1690,7 @@ func InitializeScenario(s *godog.ScenarioContext) {
 	// zookeeper manipulation
 	s.Step(`^I get zookeeper node "([^"]*)"$`, tctx.stepIGetZookeeperNode)
 	s.Step(`^I set zookeeper node "([^"]*)" to$`, tctx.stepISetZookeeperNode)
+	s.Step(`^I set raw zookeeper node "([^"]*)" to$`, tctx.stepISetRawZookeeperNode)
 	s.Step(`^I delete zookeeper node "([^"]*)"$`, tctx.stepIDeleteZookeeperNode)
 
 	// zookeeper checking