From d89aae8d2c57e04021f2e07061962c82baa95be8 Mon Sep 17 00:00:00 2001 From: Andrianto Lie Date: Tue, 5 Mar 2019 17:52:44 +0800 Subject: [PATCH] Add additional test case for #6704 --- tests/Cluster.py | 2 +- tests/nodeos_irreversible_mode_test.py | 110 +++++++++++++++++++++---- 2 files changed, 96 insertions(+), 16 deletions(-) diff --git a/tests/Cluster.py b/tests/Cluster.py index 8363e41781e..e7740d19cd3 100644 --- a/tests/Cluster.py +++ b/tests/Cluster.py @@ -1569,4 +1569,4 @@ def stripValues(lowestMaxes,greaterThan): @staticmethod def getDataDir(nodeId): - return os.path.join(Cluster.__dataDir, "node_%02d" % (nodeId)) + return os.path.abspath(os.path.join(Cluster.__dataDir, "node_%02d" % (nodeId))) diff --git a/tests/nodeos_irreversible_mode_test.py b/tests/nodeos_irreversible_mode_test.py index 09a55165947..73796eb1f4c 100755 --- a/tests/nodeos_irreversible_mode_test.py +++ b/tests/nodeos_irreversible_mode_test.py @@ -29,7 +29,7 @@ cmdError = Utils.cmdError relaunchTimeout = 5 numOfProducers = 4 -totalNodes = 9 +totalNodes = 10 # Parse command line arguments args = TestHelper.parse_args({"-v","--clean-run","--dump-error-details","--leave-running","--keep-logs"}) @@ -46,11 +46,42 @@ cluster=Cluster(walletd=True) cluster.setWalletMgr(walletMgr) +def makeSnapshot(nodeId): + req = urllib.request.Request("http://127.0.0.1:{}/v1/producer/create_snapshot".format(8888 + int(nodeId))) + urllib.request.urlopen(req) + +def backupBlksDir(nodeId): + dataDir = Cluster.getDataDir(nodeId) + sourceDir = os.path.join(dataDir, "blocks") + destinationDir = os.path.join(os.path.dirname(dataDir), os.path.basename(dataDir) + "-backup", "blocks") + shutil.copytree(sourceDir, destinationDir) + +def recoverBackedupBlksDir(nodeId): + dataDir = Cluster.getDataDir(nodeId) + # Delete existing one and copy backed up one + existingBlocksDir = os.path.join(dataDir, "blocks") + backedupBlocksDir = os.path.join(os.path.dirname(dataDir), os.path.basename(dataDir) + "-backup", "blocks") + shutil.rmtree(existingBlocksDir, ignore_errors=True) + shutil.copytree(backedupBlocksDir, existingBlocksDir) + +def getLatestSnapshot(nodeId): + snapshotDir = os.path.join(Cluster.getDataDir(nodeId), "snapshots") + snapshotDirContents = os.listdir(snapshotDir) + assert len(snapshotDirContents) > 0 + snapshotDirContents.sort() + return os.path.join(snapshotDir, snapshotDirContents[-1]) + + def removeReversibleBlks(nodeId): dataDir = Cluster.getDataDir(nodeId) reversibleBlks = os.path.join(dataDir, "blocks", "reversible") shutil.rmtree(reversibleBlks, ignore_errors=True) +def removeState(nodeId): + dataDir = Cluster.getDataDir(nodeId) + state = os.path.join(dataDir, "state") + shutil.rmtree(state, ignore_errors=True) + def getHeadLibAndForkDbHead(node: Node): info = node.getInfo() assert info is not None, "Fail to retrieve info from the node, the node is currently having a problem" @@ -135,7 +166,8 @@ def relaunchNode(node: Node, nodeId, chainArg="", addOrSwapFlags=None, relaunchA specificExtraNodeosArgs={ 0:"--enable-stale-production", 4:"--read-mode irreversible", - 6:"--read-mode irreversible"}) + 6:"--read-mode irreversible", + 9:"--plugin eosio::producer_api_plugin"}) producingNodeId = 0 producingNode = cluster.getNode(producingNodeId) @@ -167,13 +199,13 @@ def executeTest(nodeIdOfNodeToTest, runTestScenario): # Run test scenario runTestScenario(nodeIdOfNodeToTest, nodeToTest) - - # Kill node after use - if not nodeToTest.killed: nodeToTest.kill(signal.SIGTERM) testResultMsgs.append("!!!TEST CASE #{} ({}) IS SUCCESSFUL".format(nodeIdOfNodeToTest, runTestScenario.__name__)) testResult = True except Exception as e: testResultMsgs.append("!!!BUG IS CONFIRMED ON TEST CASE #{} ({}): {}".format(nodeIdOfNodeToTest, runTestScenario.__name__, e)) + finally: + # Kill node after use + if not nodeToTest.killed: nodeToTest.kill(signal.SIGTERM) return testResult # 1st test case: Replay in irreversible mode with reversible blks @@ -313,18 +345,66 @@ def replayInIrrModeWithoutRevBlksAndConnectedToProdNode(nodeIdOfNodeToTest, node finally: stopProdNode() + # 9th test case: Switch to speculative mode while using irreversible mode snapshots and using backed up speculative blocks + # Expectation: Node replays and launches successfully + # and the head and lib should be advancing after some blocks produced + # and forkdb head, head, and lib should stay the same after relaunch + # Current Bug: Nothing + def switchToSpecModeWithIrrModeSnapshot(nodeIdOfNodeToTest, nodeToTest): + try: + # Kill node and backup blocks directory of speculative mode + headLibAndForkDbHeadBeforeShutdown = getHeadLibAndForkDbHead(nodeToTest) + nodeToTest.kill(signal.SIGTERM) + backupBlksDir(nodeIdOfNodeToTest) + + # Relaunch in irreversible mode and create the snapshot + relaunchNode(nodeToTest, nodeIdOfNodeToTest, chainArg=" --read-mode irreversible") + confirmHeadLibAndForkDbHeadOfIrrMode(nodeToTest) + makeSnapshot(nodeIdOfNodeToTest) + nodeToTest.kill(signal.SIGTERM) + + # Start from clean data dir, recover back up blocks, and then relaunch with irreversible snapshot + removeState(nodeIdOfNodeToTest) + recoverBackedupBlksDir(nodeIdOfNodeToTest) # this function will delete the existing blocks dir first + relaunchNode(nodeToTest, nodeIdOfNodeToTest, chainArg=" --snapshot {}".format(getLatestSnapshot(nodeIdOfNodeToTest)), addOrSwapFlags={"--read-mode": "speculative"}) + confirmHeadLibAndForkDbHeadOfSpecMode(nodeToTest) + # Ensure it automatically replays "reversible blocks", i.e. head lib and fork db should be the same + headLibAndForkDbHeadAfterRelaunch = getHeadLibAndForkDbHead(nodeToTest) + assert headLibAndForkDbHeadBeforeShutdown == headLibAndForkDbHeadAfterRelaunch, \ + "Head, Lib, and Fork Db after relaunch is different {} vs {}".format(headLibAndForkDbHeadBeforeShutdown, headLibAndForkDbHeadAfterRelaunch) + + # Start production and wait until lib advance, ensure everything is alright + startProdNode() + ensureHeadLibAndForkDbHeadIsAdvancing(nodeToTest) + + # Note the head, lib and fork db head + stopProdNode() + headLibAndForkDbHeadBeforeShutdown = getHeadLibAndForkDbHead(nodeToTest) + nodeToTest.kill(signal.SIGTERM) + + # Relaunch the node again (using the same snapshot) + # This time ensure it automatically replays both "irreversible blocks" and "reversible blocks", i.e. the end result should be the same as before shutdown + removeState(nodeIdOfNodeToTest) + relaunchNode(nodeToTest, nodeIdOfNodeToTest) + headLibAndForkDbHeadAfterRelaunch = getHeadLibAndForkDbHead(nodeToTest) + assert headLibAndForkDbHeadBeforeShutdown == headLibAndForkDbHeadAfterRelaunch, \ + "Head, Lib, and Fork Db after relaunch is different {} vs {}".format(headLibAndForkDbHeadBeforeShutdown, headLibAndForkDbHeadAfterRelaunch) + finally: + stopProdNode() # Start executing test cases here - testResult1 = executeTest(1, replayInIrrModeWithRevBlks) - testResult2 = executeTest(2, replayInIrrModeWithoutRevBlks) - testResult3 = executeTest(3, switchSpecToIrrMode) - testResult4 = executeTest(4, switchIrrToSpecMode) - testResult5 = executeTest(5, switchSpecToIrrModeWithConnectedToProdNode) - testResult6 = executeTest(6, switchIrrToSpecModeWithConnectedToProdNode) - testResult7 = executeTest(7, replayInIrrModeWithRevBlksAndConnectedToProdNode) - testResult8 = executeTest(8, replayInIrrModeWithoutRevBlksAndConnectedToProdNode) - - testSuccessful = testResult1 and testResult2 and testResult3 and testResult4 and testResult5 and testResult6 and testResult7 and testResult8 + testResults = [] + testResults.append( executeTest(1, replayInIrrModeWithRevBlks) ) + testResults.append( executeTest(2, replayInIrrModeWithoutRevBlks) ) + testResults.append( executeTest(3, switchSpecToIrrMode) ) + testResults.append( executeTest(4, switchIrrToSpecMode) ) + testResults.append( executeTest(5, switchSpecToIrrModeWithConnectedToProdNode) ) + testResults.append( executeTest(6, switchIrrToSpecModeWithConnectedToProdNode) ) + testResults.append( executeTest(7, replayInIrrModeWithRevBlksAndConnectedToProdNode) ) + testResults.append( executeTest(8, replayInIrrModeWithoutRevBlksAndConnectedToProdNode) ) + testResults.append( executeTest(9, switchToSpecModeWithIrrModeSnapshot) ) + + testSuccessful = all(testResults) finally: TestHelper.shutdown(cluster, walletMgr, testSuccessful, killEosInstances, killWallet, keepLogs, killAll, dumpErrorDetails) # Print test result