From 1b77e821b534f2628ee59c418658a617038bc9e5 Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Wed, 12 Sep 2018 08:00:42 -0700 Subject: [PATCH 01/41] Enable Joint Consensus. This commit enables Joint Consensus as described by the Raft paper (https://raft.github.io/raft.pdf). As of this commit it is possible to undergo arbitrary peer membership changes in a safe way. Notably, this gives us a resilient "Replace Node" functionality, which is able to progress in the situation of a loss of both the old (removed) and new (added) peers go down. This is not possible with our previous one-at-a-time strategy. Unfortunately, this feature is fairly large in scope. Thankfully it's mostly testing code! There is some new API surface, notably the `Raft::begin_membership_change` function. There is also some moved API surface. The `RaftLog::applied_to` function has been moved to `Raft::commit_apply`. The old function, while it still works, carries a depreaction warning. In the future we should make it a `pub(crate)` function. Finally, many tests were added to ensure Joint Consensus can work. TODOs: - Consider renaming `progress::Configuration` to `progress::Topology`. - Test interaction between Joint Consensus and one-by-one method. - Think of more cases we might need. - Documentation update. Signed-off-by: Hoverbear --- Cargo.toml | 1 + benches/suites/progress_set.rs | 29 +- benches/suites/raw_node.rs | 3 +- proto/eraftpb.proto | 5 + src/eraftpb.rs | 466 +++--- src/errors.rs | 9 + src/lib.rs | 4 +- src/progress.rs | 534 ++++++- src/raft.rs | 253 ++- src/raft_log.rs | 23 +- src/raw_node.rs | 18 +- tests/integration_cases/mod.rs | 1 + .../test_membership_changes.rs | 1352 +++++++++++++++++ tests/integration_cases/test_raft.rs | 12 +- tests/integration_cases/test_raft_paper.rs | 2 +- tests/test_util/mod.rs | 28 +- 16 files changed, 2444 insertions(+), 296 deletions(-) create mode 100644 tests/integration_cases/test_membership_changes.rs diff --git a/Cargo.toml b/Cargo.toml index a22f58afd..e7023ff93 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ quick-error = "1.2.2" rand = "0.5.4" fxhash = "0.2.1" fail = { version = "0.2", optional = true } +getset = "0.0.6" [dev-dependencies] env_logger = "0.5" diff --git a/benches/suites/progress_set.rs b/benches/suites/progress_set.rs index 82bc044bf..e87c55335 100644 --- a/benches/suites/progress_set.rs +++ b/benches/suites/progress_set.rs @@ -11,7 +11,8 @@ pub fn bench_progress_set(c: &mut Criterion) { bench_progress_set_remove(c); bench_progress_set_iter(c); bench_progress_set_get(c); - bench_progress_set_nodes(c); + bench_progress_set_voters(c); + bench_progress_set_learners(c); } fn quick_progress_set(voters: usize, learners: usize) -> ProgressSet { @@ -28,7 +29,7 @@ fn quick_progress_set(voters: usize, learners: usize) -> ProgressSet { pub fn bench_progress_set_new(c: &mut Criterion) { let bench = |b: &mut Bencher| { // No setup. - b.iter(|| ProgressSet::new()); + b.iter(|| ProgressSet::new); }; c.bench_function("ProgressSet::new", bench); @@ -146,14 +147,32 @@ pub fn bench_progress_set_iter(c: &mut Criterion) { }); } -pub fn bench_progress_set_nodes(c: &mut Criterion) { +pub fn bench_progress_set_voters(c: &mut Criterion) { let bench = |voters, learners| { move |b: &mut Bencher| { let set = quick_progress_set(voters, learners); b.iter(|| { let set = set.clone(); - let agg = set.iter().all(|_| true); - agg + set.voters().for_each(|_| {}); + }); + } + }; + + DEFAULT_RAFT_SETS.iter().for_each(|(voters, learners)| { + c.bench_function( + &format!("ProgressSet::nodes ({}, {})", voters, learners), + bench(*voters, *learners), + ); + }); +} + +pub fn bench_progress_set_learners(c: &mut Criterion) { + let bench = |voters, learners| { + move |b: &mut Bencher| { + let set = quick_progress_set(voters, learners); + b.iter(|| { + let set = set.clone(); + set.voters().for_each(|_| {}); }); } }; diff --git a/benches/suites/raw_node.rs b/benches/suites/raw_node.rs index b657d208f..917e0ed79 100644 --- a/benches/suites/raw_node.rs +++ b/benches/suites/raw_node.rs @@ -10,8 +10,7 @@ fn quick_raw_node() -> RawNode { let peers = vec![]; let storage = MemStorage::default(); let config = Config::new(id); - let node = RawNode::new(&config, storage, peers).unwrap(); - node + RawNode::new(&config, storage, peers).unwrap() } pub fn bench_raw_node_new(c: &mut Criterion) { diff --git a/proto/eraftpb.proto b/proto/eraftpb.proto index e7c25f50c..1edf77ca8 100644 --- a/proto/eraftpb.proto +++ b/proto/eraftpb.proto @@ -91,11 +91,16 @@ enum ConfChangeType { AddNode = 0; RemoveNode = 1; AddLearnerNode = 2; + BeginConfChange = 3; + FinalizeConfChange = 4; } message ConfChange { uint64 id = 1; ConfChangeType change_type = 2; + // Used in `AddNode`, `RemoveNode`, and `AddLearnerNode`. uint64 node_id = 3; bytes context = 4; + // Used in `BeginConfChange` and `FinalizeConfChange`. + ConfState configuration = 5; } diff --git a/src/eraftpb.rs b/src/eraftpb.rs index f65846554..699ec4e37 100644 --- a/src/eraftpb.rs +++ b/src/eraftpb.rs @@ -1,9 +1,9 @@ -// This file is generated by rust-protobuf 2.0.2. Do not edit +// This file is generated by rust-protobuf 2.0.4. Do not edit // @generated // https://github.com/Manishearth/rust-clippy/issues/702 #![allow(unknown_lints)] -#![cfg_attr(feature = "cargo-clippy", allow(clippy))] +#![allow(clippy)] #![cfg_attr(rustfmt, rustfmt_skip)] @@ -163,7 +163,7 @@ impl ::protobuf::Message for Entry { let (field_number, wire_type) = is.read_tag_unpack()?; match field_number { 1 => { - if wire_type == ::protobuf::wire_format::WireTypeVarint {self.entry_type = is.read_enum()?;} else { return ::std::result::Result::Err(::protobuf::rt::unexpected_wire_type(wire_type)); } + ::protobuf::rt::read_proto3_enum_with_unknown_fields_into(wire_type, is, &mut self.entry_type, 1, &mut self.unknown_fields)? }, 2 => { if wire_type != ::protobuf::wire_format::WireTypeVarint { @@ -1095,7 +1095,7 @@ impl ::protobuf::Message for Message { let (field_number, wire_type) = is.read_tag_unpack()?; match field_number { 1 => { - if wire_type == ::protobuf::wire_format::WireTypeVarint {self.msg_type = is.read_enum()?;} else { return ::std::result::Result::Err(::protobuf::rt::unexpected_wire_type(wire_type)); } + ::protobuf::rt::read_proto3_enum_with_unknown_fields_into(wire_type, is, &mut self.msg_type, 1, &mut self.unknown_fields)? }, 2 => { if wire_type != ::protobuf::wire_format::WireTypeVarint { @@ -1846,6 +1846,7 @@ pub struct ConfChange { pub change_type: ConfChangeType, pub node_id: u64, pub context: ::std::vec::Vec, + pub configuration: ::protobuf::SingularPtrField, // special fields unknown_fields: ::protobuf::UnknownFields, cached_size: ::protobuf::CachedSize, @@ -1926,10 +1927,48 @@ impl ConfChange { pub fn get_context(&self) -> &[u8] { &self.context } + + // .eraftpb.ConfState configuration = 5; + + pub fn clear_configuration(&mut self) { + self.configuration.clear(); + } + + pub fn has_configuration(&self) -> bool { + self.configuration.is_some() + } + + // Param is passed by value, moved + pub fn set_configuration(&mut self, v: ConfState) { + self.configuration = ::protobuf::SingularPtrField::some(v); + } + + // Mutable pointer to the field. + // If field is not initialized, it is initialized with default value first. + pub fn mut_configuration(&mut self) -> &mut ConfState { + if self.configuration.is_none() { + self.configuration.set_default(); + } + self.configuration.as_mut().unwrap() + } + + // Take field + pub fn take_configuration(&mut self) -> ConfState { + self.configuration.take().unwrap_or_else(|| ConfState::new()) + } + + pub fn get_configuration(&self) -> &ConfState { + self.configuration.as_ref().unwrap_or_else(|| ConfState::default_instance()) + } } impl ::protobuf::Message for ConfChange { fn is_initialized(&self) -> bool { + for v in &self.configuration { + if !v.is_initialized() { + return false; + } + }; true } @@ -1945,7 +1984,7 @@ impl ::protobuf::Message for ConfChange { self.id = tmp; }, 2 => { - if wire_type == ::protobuf::wire_format::WireTypeVarint {self.change_type = is.read_enum()?;} else { return ::std::result::Result::Err(::protobuf::rt::unexpected_wire_type(wire_type)); } + ::protobuf::rt::read_proto3_enum_with_unknown_fields_into(wire_type, is, &mut self.change_type, 2, &mut self.unknown_fields)? }, 3 => { if wire_type != ::protobuf::wire_format::WireTypeVarint { @@ -1957,6 +1996,9 @@ impl ::protobuf::Message for ConfChange { 4 => { ::protobuf::rt::read_singular_proto3_bytes_into(wire_type, is, &mut self.context)?; }, + 5 => { + ::protobuf::rt::read_singular_message_into(wire_type, is, &mut self.configuration)?; + }, _ => { ::protobuf::rt::read_unknown_or_skip_group(field_number, wire_type, is, self.mut_unknown_fields())?; }, @@ -1981,6 +2023,10 @@ impl ::protobuf::Message for ConfChange { if !self.context.is_empty() { my_size += ::protobuf::rt::bytes_size(4, &self.context); } + if let Some(ref v) = self.configuration.as_ref() { + let len = v.compute_size(); + my_size += 1 + ::protobuf::rt::compute_raw_varint32_size(len) + len; + } my_size += ::protobuf::rt::unknown_fields_size(self.get_unknown_fields()); self.cached_size.set(my_size); my_size @@ -1999,6 +2045,11 @@ impl ::protobuf::Message for ConfChange { if !self.context.is_empty() { os.write_bytes(4, &self.context)?; } + if let Some(ref v) = self.configuration.as_ref() { + os.write_tag(5, ::protobuf::wire_format::WireTypeLengthDelimited)?; + os.write_raw_varint32(v.get_cached_size())?; + v.write_to_with_cached_sizes(os)?; + } os.write_unknown_fields(self.get_unknown_fields())?; ::std::result::Result::Ok(()) } @@ -2061,6 +2112,11 @@ impl ::protobuf::Message for ConfChange { |m: &ConfChange| { &m.context }, |m: &mut ConfChange| { &mut m.context }, )); + fields.push(::protobuf::reflect::accessor::make_singular_ptr_field_accessor::<_, ::protobuf::types::ProtobufTypeMessage>( + "configuration", + |m: &ConfChange| { &m.configuration }, + |m: &mut ConfChange| { &mut m.configuration }, + )); ::protobuf::reflect::MessageDescriptor::new::( "ConfChange", fields, @@ -2087,6 +2143,7 @@ impl ::protobuf::Clear for ConfChange { self.clear_change_type(); self.clear_node_id(); self.clear_context(); + self.clear_configuration(); self.unknown_fields.clear(); } } @@ -2269,6 +2326,8 @@ pub enum ConfChangeType { AddNode = 0, RemoveNode = 1, AddLearnerNode = 2, + BeginConfChange = 3, + FinalizeConfChange = 4, } impl ::protobuf::ProtobufEnum for ConfChangeType { @@ -2281,6 +2340,8 @@ impl ::protobuf::ProtobufEnum for ConfChangeType { 0 => ::std::option::Option::Some(ConfChangeType::AddNode), 1 => ::std::option::Option::Some(ConfChangeType::RemoveNode), 2 => ::std::option::Option::Some(ConfChangeType::AddLearnerNode), + 3 => ::std::option::Option::Some(ConfChangeType::BeginConfChange), + 4 => ::std::option::Option::Some(ConfChangeType::FinalizeConfChange), _ => ::std::option::Option::None } } @@ -2290,6 +2351,8 @@ impl ::protobuf::ProtobufEnum for ConfChangeType { ConfChangeType::AddNode, ConfChangeType::RemoveNode, ConfChangeType::AddLearnerNode, + ConfChangeType::BeginConfChange, + ConfChangeType::FinalizeConfChange, ]; values } @@ -2347,192 +2410,217 @@ static file_descriptor_proto_data: &'static [u8] = b"\ \x04term\x12\x12\n\x04vote\x18\x02\x20\x01(\x04R\x04vote\x12\x16\n\x06co\ mmit\x18\x03\x20\x01(\x04R\x06commit\"=\n\tConfState\x12\x14\n\x05nodes\ \x18\x01\x20\x03(\x04R\x05nodes\x12\x1a\n\x08learners\x18\x02\x20\x03(\ - \x04R\x08learners\"\x89\x01\n\nConfChange\x12\x0e\n\x02id\x18\x01\x20\ + \x04R\x08learners\"\xc3\x01\n\nConfChange\x12\x0e\n\x02id\x18\x01\x20\ \x01(\x04R\x02id\x128\n\x0bchange_type\x18\x02\x20\x01(\x0e2\x17.eraftpb\ .ConfChangeTypeR\nchangeType\x12\x17\n\x07node_id\x18\x03\x20\x01(\x04R\ - \x06nodeId\x12\x18\n\x07context\x18\x04\x20\x01(\x0cR\x07context*1\n\tEn\ - tryType\x12\x0f\n\x0bEntryNormal\x10\0\x12\x13\n\x0fEntryConfChange\x10\ - \x01*\x8c\x03\n\x0bMessageType\x12\n\n\x06MsgHup\x10\0\x12\x0b\n\x07MsgB\ - eat\x10\x01\x12\x0e\n\nMsgPropose\x10\x02\x12\r\n\tMsgAppend\x10\x03\x12\ - \x15\n\x11MsgAppendResponse\x10\x04\x12\x12\n\x0eMsgRequestVote\x10\x05\ - \x12\x1a\n\x16MsgRequestVoteResponse\x10\x06\x12\x0f\n\x0bMsgSnapshot\ - \x10\x07\x12\x10\n\x0cMsgHeartbeat\x10\x08\x12\x18\n\x14MsgHeartbeatResp\ - onse\x10\t\x12\x12\n\x0eMsgUnreachable\x10\n\x12\x11\n\rMsgSnapStatus\ - \x10\x0b\x12\x12\n\x0eMsgCheckQuorum\x10\x0c\x12\x15\n\x11MsgTransferLea\ - der\x10\r\x12\x11\n\rMsgTimeoutNow\x10\x0e\x12\x10\n\x0cMsgReadIndex\x10\ - \x0f\x12\x14\n\x10MsgReadIndexResp\x10\x10\x12\x15\n\x11MsgRequestPreVot\ - e\x10\x11\x12\x1d\n\x19MsgRequestPreVoteResponse\x10\x12*A\n\x0eConfChan\ - geType\x12\x0b\n\x07AddNode\x10\0\x12\x0e\n\nRemoveNode\x10\x01\x12\x12\ - \n\x0eAddLearnerNode\x10\x02J\x83\x1c\n\x06\x12\x04\0\0Z\x01\n\x08\n\x01\ - \x0c\x12\x03\0\0\x12\n\x08\n\x01\x02\x12\x03\x01\x08\x0f\n\n\n\x02\x05\0\ - \x12\x04\x03\0\x06\x01\n\n\n\x03\x05\0\x01\x12\x03\x03\x05\x0e\n\x0b\n\ - \x04\x05\0\x02\0\x12\x03\x04\x04\x14\n\x0c\n\x05\x05\0\x02\0\x01\x12\x03\ - \x04\x04\x0f\n\x0c\n\x05\x05\0\x02\0\x02\x12\x03\x04\x12\x13\n\x0b\n\x04\ - \x05\0\x02\x01\x12\x03\x05\x04\x18\n\x0c\n\x05\x05\0\x02\x01\x01\x12\x03\ - \x05\x04\x13\n\x0c\n\x05\x05\0\x02\x01\x02\x12\x03\x05\x16\x17\n\n\n\x02\ - \x04\0\x12\x04\x08\0\x12\x01\n\n\n\x03\x04\0\x01\x12\x03\x08\x08\r\n\x0b\ - \n\x04\x04\0\x02\0\x12\x03\t\x04\x1d\n\r\n\x05\x04\0\x02\0\x04\x12\x04\t\ - \x04\x08\x0f\n\x0c\n\x05\x04\0\x02\0\x06\x12\x03\t\x04\r\n\x0c\n\x05\x04\ - \0\x02\0\x01\x12\x03\t\x0e\x18\n\x0c\n\x05\x04\0\x02\0\x03\x12\x03\t\x1b\ - \x1c\n\x0b\n\x04\x04\0\x02\x01\x12\x03\n\x04\x14\n\r\n\x05\x04\0\x02\x01\ - \x04\x12\x04\n\x04\t\x1d\n\x0c\n\x05\x04\0\x02\x01\x05\x12\x03\n\x04\n\n\ - \x0c\n\x05\x04\0\x02\x01\x01\x12\x03\n\x0b\x0f\n\x0c\n\x05\x04\0\x02\x01\ - \x03\x12\x03\n\x12\x13\n\x0b\n\x04\x04\0\x02\x02\x12\x03\x0b\x04\x15\n\r\ - \n\x05\x04\0\x02\x02\x04\x12\x04\x0b\x04\n\x14\n\x0c\n\x05\x04\0\x02\x02\ - \x05\x12\x03\x0b\x04\n\n\x0c\n\x05\x04\0\x02\x02\x01\x12\x03\x0b\x0b\x10\ - \n\x0c\n\x05\x04\0\x02\x02\x03\x12\x03\x0b\x13\x14\n\x0b\n\x04\x04\0\x02\ - \x03\x12\x03\x0c\x04\x13\n\r\n\x05\x04\0\x02\x03\x04\x12\x04\x0c\x04\x0b\ - \x15\n\x0c\n\x05\x04\0\x02\x03\x05\x12\x03\x0c\x04\t\n\x0c\n\x05\x04\0\ - \x02\x03\x01\x12\x03\x0c\n\x0e\n\x0c\n\x05\x04\0\x02\x03\x03\x12\x03\x0c\ - \x11\x12\n\x0b\n\x04\x04\0\x02\x04\x12\x03\r\x04\x16\n\r\n\x05\x04\0\x02\ - \x04\x04\x12\x04\r\x04\x0c\x13\n\x0c\n\x05\x04\0\x02\x04\x05\x12\x03\r\ - \x04\t\n\x0c\n\x05\x04\0\x02\x04\x01\x12\x03\r\n\x11\n\x0c\n\x05\x04\0\ - \x02\x04\x03\x12\x03\r\x14\x15\nm\n\x04\x04\0\x02\x05\x12\x03\x11\x04\ - \x16\x1a`\x20Deprecated!\x20It\x20is\x20kept\x20for\x20backward\x20compa\ - tibility.\n\x20TODO:\x20remove\x20it\x20in\x20the\x20next\x20major\x20re\ - lease.\n\n\r\n\x05\x04\0\x02\x05\x04\x12\x04\x11\x04\r\x16\n\x0c\n\x05\ - \x04\0\x02\x05\x05\x12\x03\x11\x04\x08\n\x0c\n\x05\x04\0\x02\x05\x01\x12\ - \x03\x11\t\x11\n\x0c\n\x05\x04\0\x02\x05\x03\x12\x03\x11\x14\x15\n\n\n\ - \x02\x04\x01\x12\x04\x14\0\x18\x01\n\n\n\x03\x04\x01\x01\x12\x03\x14\x08\ - \x18\n\x0b\n\x04\x04\x01\x02\0\x12\x03\x15\x04\x1d\n\r\n\x05\x04\x01\x02\ - \0\x04\x12\x04\x15\x04\x14\x1a\n\x0c\n\x05\x04\x01\x02\0\x06\x12\x03\x15\ - \x04\r\n\x0c\n\x05\x04\x01\x02\0\x01\x12\x03\x15\x0e\x18\n\x0c\n\x05\x04\ - \x01\x02\0\x03\x12\x03\x15\x1b\x1c\n\x0b\n\x04\x04\x01\x02\x01\x12\x03\ - \x16\x04\x15\n\r\n\x05\x04\x01\x02\x01\x04\x12\x04\x16\x04\x15\x1d\n\x0c\ - \n\x05\x04\x01\x02\x01\x05\x12\x03\x16\x04\n\n\x0c\n\x05\x04\x01\x02\x01\ - \x01\x12\x03\x16\x0b\x10\n\x0c\n\x05\x04\x01\x02\x01\x03\x12\x03\x16\x13\ - \x14\n\x0b\n\x04\x04\x01\x02\x02\x12\x03\x17\x04\x14\n\r\n\x05\x04\x01\ - \x02\x02\x04\x12\x04\x17\x04\x16\x15\n\x0c\n\x05\x04\x01\x02\x02\x05\x12\ - \x03\x17\x04\n\n\x0c\n\x05\x04\x01\x02\x02\x01\x12\x03\x17\x0b\x0f\n\x0c\ - \n\x05\x04\x01\x02\x02\x03\x12\x03\x17\x12\x13\n\n\n\x02\x04\x02\x12\x04\ - \x1a\0\x1d\x01\n\n\n\x03\x04\x02\x01\x12\x03\x1a\x08\x10\n\x0b\n\x04\x04\ - \x02\x02\0\x12\x03\x1b\x04\x13\n\r\n\x05\x04\x02\x02\0\x04\x12\x04\x1b\ - \x04\x1a\x12\n\x0c\n\x05\x04\x02\x02\0\x05\x12\x03\x1b\x04\t\n\x0c\n\x05\ - \x04\x02\x02\0\x01\x12\x03\x1b\n\x0e\n\x0c\n\x05\x04\x02\x02\0\x03\x12\ - \x03\x1b\x11\x12\n\x0b\n\x04\x04\x02\x02\x01\x12\x03\x1c\x04\"\n\r\n\x05\ - \x04\x02\x02\x01\x04\x12\x04\x1c\x04\x1b\x13\n\x0c\n\x05\x04\x02\x02\x01\ - \x06\x12\x03\x1c\x04\x14\n\x0c\n\x05\x04\x02\x02\x01\x01\x12\x03\x1c\x15\ - \x1d\n\x0c\n\x05\x04\x02\x02\x01\x03\x12\x03\x1c\x20!\n\n\n\x02\x05\x01\ - \x12\x04\x1f\03\x01\n\n\n\x03\x05\x01\x01\x12\x03\x1f\x05\x10\n\x0b\n\ - \x04\x05\x01\x02\0\x12\x03\x20\x04\x0f\n\x0c\n\x05\x05\x01\x02\0\x01\x12\ - \x03\x20\x04\n\n\x0c\n\x05\x05\x01\x02\0\x02\x12\x03\x20\r\x0e\n\x0b\n\ - \x04\x05\x01\x02\x01\x12\x03!\x04\x10\n\x0c\n\x05\x05\x01\x02\x01\x01\ - \x12\x03!\x04\x0b\n\x0c\n\x05\x05\x01\x02\x01\x02\x12\x03!\x0e\x0f\n\x0b\ - \n\x04\x05\x01\x02\x02\x12\x03\"\x04\x13\n\x0c\n\x05\x05\x01\x02\x02\x01\ - \x12\x03\"\x04\x0e\n\x0c\n\x05\x05\x01\x02\x02\x02\x12\x03\"\x11\x12\n\ - \x0b\n\x04\x05\x01\x02\x03\x12\x03#\x04\x12\n\x0c\n\x05\x05\x01\x02\x03\ - \x01\x12\x03#\x04\r\n\x0c\n\x05\x05\x01\x02\x03\x02\x12\x03#\x10\x11\n\ - \x0b\n\x04\x05\x01\x02\x04\x12\x03$\x04\x1a\n\x0c\n\x05\x05\x01\x02\x04\ - \x01\x12\x03$\x04\x15\n\x0c\n\x05\x05\x01\x02\x04\x02\x12\x03$\x18\x19\n\ - \x0b\n\x04\x05\x01\x02\x05\x12\x03%\x04\x17\n\x0c\n\x05\x05\x01\x02\x05\ - \x01\x12\x03%\x04\x12\n\x0c\n\x05\x05\x01\x02\x05\x02\x12\x03%\x15\x16\n\ - \x0b\n\x04\x05\x01\x02\x06\x12\x03&\x04\x1f\n\x0c\n\x05\x05\x01\x02\x06\ - \x01\x12\x03&\x04\x1a\n\x0c\n\x05\x05\x01\x02\x06\x02\x12\x03&\x1d\x1e\n\ - \x0b\n\x04\x05\x01\x02\x07\x12\x03'\x04\x14\n\x0c\n\x05\x05\x01\x02\x07\ - \x01\x12\x03'\x04\x0f\n\x0c\n\x05\x05\x01\x02\x07\x02\x12\x03'\x12\x13\n\ - \x0b\n\x04\x05\x01\x02\x08\x12\x03(\x04\x15\n\x0c\n\x05\x05\x01\x02\x08\ - \x01\x12\x03(\x04\x10\n\x0c\n\x05\x05\x01\x02\x08\x02\x12\x03(\x13\x14\n\ - \x0b\n\x04\x05\x01\x02\t\x12\x03)\x04\x1d\n\x0c\n\x05\x05\x01\x02\t\x01\ - \x12\x03)\x04\x18\n\x0c\n\x05\x05\x01\x02\t\x02\x12\x03)\x1b\x1c\n\x0b\n\ - \x04\x05\x01\x02\n\x12\x03*\x04\x18\n\x0c\n\x05\x05\x01\x02\n\x01\x12\ - \x03*\x04\x12\n\x0c\n\x05\x05\x01\x02\n\x02\x12\x03*\x15\x17\n\x0b\n\x04\ - \x05\x01\x02\x0b\x12\x03+\x04\x17\n\x0c\n\x05\x05\x01\x02\x0b\x01\x12\ - \x03+\x04\x11\n\x0c\n\x05\x05\x01\x02\x0b\x02\x12\x03+\x14\x16\n\x0b\n\ - \x04\x05\x01\x02\x0c\x12\x03,\x04\x18\n\x0c\n\x05\x05\x01\x02\x0c\x01\ - \x12\x03,\x04\x12\n\x0c\n\x05\x05\x01\x02\x0c\x02\x12\x03,\x15\x17\n\x0b\ - \n\x04\x05\x01\x02\r\x12\x03-\x04\x1b\n\x0c\n\x05\x05\x01\x02\r\x01\x12\ - \x03-\x04\x15\n\x0c\n\x05\x05\x01\x02\r\x02\x12\x03-\x18\x1a\n\x0b\n\x04\ - \x05\x01\x02\x0e\x12\x03.\x04\x17\n\x0c\n\x05\x05\x01\x02\x0e\x01\x12\ - \x03.\x04\x11\n\x0c\n\x05\x05\x01\x02\x0e\x02\x12\x03.\x14\x16\n\x0b\n\ - \x04\x05\x01\x02\x0f\x12\x03/\x04\x16\n\x0c\n\x05\x05\x01\x02\x0f\x01\ - \x12\x03/\x04\x10\n\x0c\n\x05\x05\x01\x02\x0f\x02\x12\x03/\x13\x15\n\x0b\ - \n\x04\x05\x01\x02\x10\x12\x030\x04\x1a\n\x0c\n\x05\x05\x01\x02\x10\x01\ - \x12\x030\x04\x14\n\x0c\n\x05\x05\x01\x02\x10\x02\x12\x030\x17\x19\n\x0b\ - \n\x04\x05\x01\x02\x11\x12\x031\x04\x1b\n\x0c\n\x05\x05\x01\x02\x11\x01\ - \x12\x031\x04\x15\n\x0c\n\x05\x05\x01\x02\x11\x02\x12\x031\x18\x1a\n\x0b\ - \n\x04\x05\x01\x02\x12\x12\x032\x04#\n\x0c\n\x05\x05\x01\x02\x12\x01\x12\ - \x032\x04\x1d\n\x0c\n\x05\x05\x01\x02\x12\x02\x12\x032\x20\"\n\n\n\x02\ - \x04\x03\x12\x045\0B\x01\n\n\n\x03\x04\x03\x01\x12\x035\x08\x0f\n\x0b\n\ - \x04\x04\x03\x02\0\x12\x036\x04\x1d\n\r\n\x05\x04\x03\x02\0\x04\x12\x046\ - \x045\x11\n\x0c\n\x05\x04\x03\x02\0\x06\x12\x036\x04\x0f\n\x0c\n\x05\x04\ - \x03\x02\0\x01\x12\x036\x10\x18\n\x0c\n\x05\x04\x03\x02\0\x03\x12\x036\ - \x1b\x1c\n\x0b\n\x04\x04\x03\x02\x01\x12\x037\x04\x12\n\r\n\x05\x04\x03\ - \x02\x01\x04\x12\x047\x046\x1d\n\x0c\n\x05\x04\x03\x02\x01\x05\x12\x037\ - \x04\n\n\x0c\n\x05\x04\x03\x02\x01\x01\x12\x037\x0b\r\n\x0c\n\x05\x04\ - \x03\x02\x01\x03\x12\x037\x10\x11\n\x0b\n\x04\x04\x03\x02\x02\x12\x038\ - \x04\x14\n\r\n\x05\x04\x03\x02\x02\x04\x12\x048\x047\x12\n\x0c\n\x05\x04\ - \x03\x02\x02\x05\x12\x038\x04\n\n\x0c\n\x05\x04\x03\x02\x02\x01\x12\x038\ - \x0b\x0f\n\x0c\n\x05\x04\x03\x02\x02\x03\x12\x038\x12\x13\n\x0b\n\x04\ - \x04\x03\x02\x03\x12\x039\x04\x14\n\r\n\x05\x04\x03\x02\x03\x04\x12\x049\ - \x048\x14\n\x0c\n\x05\x04\x03\x02\x03\x05\x12\x039\x04\n\n\x0c\n\x05\x04\ - \x03\x02\x03\x01\x12\x039\x0b\x0f\n\x0c\n\x05\x04\x03\x02\x03\x03\x12\ - \x039\x12\x13\n\x0b\n\x04\x04\x03\x02\x04\x12\x03:\x04\x18\n\r\n\x05\x04\ - \x03\x02\x04\x04\x12\x04:\x049\x14\n\x0c\n\x05\x04\x03\x02\x04\x05\x12\ - \x03:\x04\n\n\x0c\n\x05\x04\x03\x02\x04\x01\x12\x03:\x0b\x13\n\x0c\n\x05\ - \x04\x03\x02\x04\x03\x12\x03:\x16\x17\n\x0b\n\x04\x04\x03\x02\x05\x12\ - \x03;\x04\x15\n\r\n\x05\x04\x03\x02\x05\x04\x12\x04;\x04:\x18\n\x0c\n\ - \x05\x04\x03\x02\x05\x05\x12\x03;\x04\n\n\x0c\n\x05\x04\x03\x02\x05\x01\ - \x12\x03;\x0b\x10\n\x0c\n\x05\x04\x03\x02\x05\x03\x12\x03;\x13\x14\n\x0b\ - \n\x04\x04\x03\x02\x06\x12\x03<\x04\x1f\n\x0c\n\x05\x04\x03\x02\x06\x04\ - \x12\x03<\x04\x0c\n\x0c\n\x05\x04\x03\x02\x06\x06\x12\x03<\r\x12\n\x0c\n\ - \x05\x04\x03\x02\x06\x01\x12\x03<\x13\x1a\n\x0c\n\x05\x04\x03\x02\x06\ - \x03\x12\x03<\x1d\x1e\n\x0b\n\x04\x04\x03\x02\x07\x12\x03=\x04\x16\n\r\n\ - \x05\x04\x03\x02\x07\x04\x12\x04=\x04<\x1f\n\x0c\n\x05\x04\x03\x02\x07\ - \x05\x12\x03=\x04\n\n\x0c\n\x05\x04\x03\x02\x07\x01\x12\x03=\x0b\x11\n\ - \x0c\n\x05\x04\x03\x02\x07\x03\x12\x03=\x14\x15\n\x0b\n\x04\x04\x03\x02\ - \x08\x12\x03>\x04\x1a\n\r\n\x05\x04\x03\x02\x08\x04\x12\x04>\x04=\x16\n\ - \x0c\n\x05\x04\x03\x02\x08\x06\x12\x03>\x04\x0c\n\x0c\n\x05\x04\x03\x02\ - \x08\x01\x12\x03>\r\x15\n\x0c\n\x05\x04\x03\x02\x08\x03\x12\x03>\x18\x19\ - \n\x0b\n\x04\x04\x03\x02\t\x12\x03?\x04\x15\n\r\n\x05\x04\x03\x02\t\x04\ - \x12\x04?\x04>\x1a\n\x0c\n\x05\x04\x03\x02\t\x05\x12\x03?\x04\x08\n\x0c\ - \n\x05\x04\x03\x02\t\x01\x12\x03?\t\x0f\n\x0c\n\x05\x04\x03\x02\t\x03\ - \x12\x03?\x12\x14\n\x0b\n\x04\x04\x03\x02\n\x12\x03@\x04\x1c\n\r\n\x05\ - \x04\x03\x02\n\x04\x12\x04@\x04?\x15\n\x0c\n\x05\x04\x03\x02\n\x05\x12\ - \x03@\x04\n\n\x0c\n\x05\x04\x03\x02\n\x01\x12\x03@\x0b\x16\n\x0c\n\x05\ - \x04\x03\x02\n\x03\x12\x03@\x19\x1b\n\x0b\n\x04\x04\x03\x02\x0b\x12\x03A\ - \x04\x17\n\r\n\x05\x04\x03\x02\x0b\x04\x12\x04A\x04@\x1c\n\x0c\n\x05\x04\ - \x03\x02\x0b\x05\x12\x03A\x04\t\n\x0c\n\x05\x04\x03\x02\x0b\x01\x12\x03A\ - \n\x11\n\x0c\n\x05\x04\x03\x02\x0b\x03\x12\x03A\x14\x16\n\n\n\x02\x04\ - \x04\x12\x04D\0H\x01\n\n\n\x03\x04\x04\x01\x12\x03D\x08\x11\n\x0b\n\x04\ - \x04\x04\x02\0\x12\x03E\x04\x14\n\r\n\x05\x04\x04\x02\0\x04\x12\x04E\x04\ - D\x13\n\x0c\n\x05\x04\x04\x02\0\x05\x12\x03E\x04\n\n\x0c\n\x05\x04\x04\ - \x02\0\x01\x12\x03E\x0b\x0f\n\x0c\n\x05\x04\x04\x02\0\x03\x12\x03E\x12\ - \x13\n\x0b\n\x04\x04\x04\x02\x01\x12\x03F\x04\x14\n\r\n\x05\x04\x04\x02\ - \x01\x04\x12\x04F\x04E\x14\n\x0c\n\x05\x04\x04\x02\x01\x05\x12\x03F\x04\ - \n\n\x0c\n\x05\x04\x04\x02\x01\x01\x12\x03F\x0b\x0f\n\x0c\n\x05\x04\x04\ - \x02\x01\x03\x12\x03F\x12\x13\n\x0b\n\x04\x04\x04\x02\x02\x12\x03G\x04\ - \x16\n\r\n\x05\x04\x04\x02\x02\x04\x12\x04G\x04F\x14\n\x0c\n\x05\x04\x04\ - \x02\x02\x05\x12\x03G\x04\n\n\x0c\n\x05\x04\x04\x02\x02\x01\x12\x03G\x0b\ - \x11\n\x0c\n\x05\x04\x04\x02\x02\x03\x12\x03G\x14\x15\n\n\n\x02\x04\x05\ - \x12\x04J\0M\x01\n\n\n\x03\x04\x05\x01\x12\x03J\x08\x11\n\x0b\n\x04\x04\ - \x05\x02\0\x12\x03K\x04\x1e\n\x0c\n\x05\x04\x05\x02\0\x04\x12\x03K\x04\ - \x0c\n\x0c\n\x05\x04\x05\x02\0\x05\x12\x03K\r\x13\n\x0c\n\x05\x04\x05\ - \x02\0\x01\x12\x03K\x14\x19\n\x0c\n\x05\x04\x05\x02\0\x03\x12\x03K\x1c\ - \x1d\n\x0b\n\x04\x04\x05\x02\x01\x12\x03L\x04!\n\x0c\n\x05\x04\x05\x02\ - \x01\x04\x12\x03L\x04\x0c\n\x0c\n\x05\x04\x05\x02\x01\x05\x12\x03L\r\x13\ - \n\x0c\n\x05\x04\x05\x02\x01\x01\x12\x03L\x14\x1c\n\x0c\n\x05\x04\x05\ - \x02\x01\x03\x12\x03L\x1f\x20\n\n\n\x02\x05\x02\x12\x04O\0S\x01\n\n\n\ - \x03\x05\x02\x01\x12\x03O\x05\x13\n\x0b\n\x04\x05\x02\x02\0\x12\x03P\x04\ - \x13\n\x0c\n\x05\x05\x02\x02\0\x01\x12\x03P\x04\x0b\n\x0c\n\x05\x05\x02\ - \x02\0\x02\x12\x03P\x11\x12\n\x0b\n\x04\x05\x02\x02\x01\x12\x03Q\x04\x13\ - \n\x0c\n\x05\x05\x02\x02\x01\x01\x12\x03Q\x04\x0e\n\x0c\n\x05\x05\x02\ - \x02\x01\x02\x12\x03Q\x11\x12\n\x0b\n\x04\x05\x02\x02\x02\x12\x03R\x04\ - \x17\n\x0c\n\x05\x05\x02\x02\x02\x01\x12\x03R\x04\x12\n\x0c\n\x05\x05\ - \x02\x02\x02\x02\x12\x03R\x15\x16\n\n\n\x02\x04\x06\x12\x04U\0Z\x01\n\n\ - \n\x03\x04\x06\x01\x12\x03U\x08\x12\n\x0b\n\x04\x04\x06\x02\0\x12\x03V\ - \x04\x12\n\r\n\x05\x04\x06\x02\0\x04\x12\x04V\x04U\x14\n\x0c\n\x05\x04\ - \x06\x02\0\x05\x12\x03V\x04\n\n\x0c\n\x05\x04\x06\x02\0\x01\x12\x03V\x0b\ - \r\n\x0c\n\x05\x04\x06\x02\0\x03\x12\x03V\x10\x11\n\x0b\n\x04\x04\x06\ - \x02\x01\x12\x03W\x04#\n\r\n\x05\x04\x06\x02\x01\x04\x12\x04W\x04V\x12\n\ - \x0c\n\x05\x04\x06\x02\x01\x06\x12\x03W\x04\x12\n\x0c\n\x05\x04\x06\x02\ - \x01\x01\x12\x03W\x13\x1e\n\x0c\n\x05\x04\x06\x02\x01\x03\x12\x03W!\"\n\ - \x0b\n\x04\x04\x06\x02\x02\x12\x03X\x04\x17\n\r\n\x05\x04\x06\x02\x02\ - \x04\x12\x04X\x04W#\n\x0c\n\x05\x04\x06\x02\x02\x05\x12\x03X\x04\n\n\x0c\ - \n\x05\x04\x06\x02\x02\x01\x12\x03X\x0b\x12\n\x0c\n\x05\x04\x06\x02\x02\ - \x03\x12\x03X\x15\x16\n\x0b\n\x04\x04\x06\x02\x03\x12\x03Y\x04\x16\n\r\n\ - \x05\x04\x06\x02\x03\x04\x12\x04Y\x04X\x17\n\x0c\n\x05\x04\x06\x02\x03\ - \x05\x12\x03Y\x04\t\n\x0c\n\x05\x04\x06\x02\x03\x01\x12\x03Y\n\x11\n\x0c\ - \n\x05\x04\x06\x02\x03\x03\x12\x03Y\x14\x15b\x06proto3\ + \x06nodeId\x12\x18\n\x07context\x18\x04\x20\x01(\x0cR\x07context\x128\n\ + \rconfiguration\x18\x05\x20\x01(\x0b2\x12.eraftpb.ConfStateR\rconfigurat\ + ion*1\n\tEntryType\x12\x0f\n\x0bEntryNormal\x10\0\x12\x13\n\x0fEntryConf\ + Change\x10\x01*\x8c\x03\n\x0bMessageType\x12\n\n\x06MsgHup\x10\0\x12\x0b\ + \n\x07MsgBeat\x10\x01\x12\x0e\n\nMsgPropose\x10\x02\x12\r\n\tMsgAppend\ + \x10\x03\x12\x15\n\x11MsgAppendResponse\x10\x04\x12\x12\n\x0eMsgRequestV\ + ote\x10\x05\x12\x1a\n\x16MsgRequestVoteResponse\x10\x06\x12\x0f\n\x0bMsg\ + Snapshot\x10\x07\x12\x10\n\x0cMsgHeartbeat\x10\x08\x12\x18\n\x14MsgHeart\ + beatResponse\x10\t\x12\x12\n\x0eMsgUnreachable\x10\n\x12\x11\n\rMsgSnapS\ + tatus\x10\x0b\x12\x12\n\x0eMsgCheckQuorum\x10\x0c\x12\x15\n\x11MsgTransf\ + erLeader\x10\r\x12\x11\n\rMsgTimeoutNow\x10\x0e\x12\x10\n\x0cMsgReadInde\ + x\x10\x0f\x12\x14\n\x10MsgReadIndexResp\x10\x10\x12\x15\n\x11MsgRequestP\ + reVote\x10\x11\x12\x1d\n\x19MsgRequestPreVoteResponse\x10\x12*n\n\x0eCon\ + fChangeType\x12\x0b\n\x07AddNode\x10\0\x12\x0e\n\nRemoveNode\x10\x01\x12\ + \x12\n\x0eAddLearnerNode\x10\x02\x12\x13\n\x0fBeginConfChange\x10\x03\ + \x12\x16\n\x12FinalizeConfChange\x10\x04J\xe0\"\n\x06\x12\x04\0\0i\x01\n\ + \x08\n\x01\x0c\x12\x03\0\0\x12\n\x08\n\x01\x02\x12\x03\x01\x08\x0f\n\n\n\ + \x02\x05\0\x12\x04\x03\0\x06\x01\n\n\n\x03\x05\0\x01\x12\x03\x03\x05\x0e\ + \n\x0b\n\x04\x05\0\x02\0\x12\x03\x04\x04\x14\n\x0c\n\x05\x05\0\x02\0\x01\ + \x12\x03\x04\x04\x0f\n\x0c\n\x05\x05\0\x02\0\x02\x12\x03\x04\x12\x13\n\ + \x0b\n\x04\x05\0\x02\x01\x12\x03\x05\x04\x18\n\x0c\n\x05\x05\0\x02\x01\ + \x01\x12\x03\x05\x04\x13\n\x0c\n\x05\x05\0\x02\x01\x02\x12\x03\x05\x16\ + \x17\n\xdd\x04\n\x02\x04\0\x12\x04\x12\0\x1c\x01\x1a\xd0\x04\x20The\x20e\ + ntry\x20is\x20a\x20type\x20of\x20change\x20that\x20needs\x20to\x20be\x20\ + applied.\x20It\x20contains\x20two\x20data\x20fields.\n\x20While\x20the\ + \x20fields\x20are\x20built\x20into\x20the\x20model;\x20their\x20usage\ + \x20is\x20determined\x20by\x20the\x20entry_type.\n\n\x20For\x20normal\ + \x20entries,\x20the\x20data\x20field\x20should\x20contain\x20the\x20data\ + \x20change\x20that\x20should\x20be\x20applied.\n\x20The\x20context\x20fi\ + eld\x20can\x20be\x20used\x20for\x20any\x20contextual\x20data\x20that\x20\ + might\x20be\x20relevant\x20to\x20the\n\x20application\x20of\x20the\x20da\ + ta.\n\n\x20For\x20configuration\x20changes,\x20the\x20data\x20will\x20co\ + ntain\x20the\x20ConfChange\x20message\x20and\x20the\n\x20context\x20will\ + \x20provide\x20anything\x20needed\x20to\x20assist\x20the\x20configuratio\ + n\x20change.\x20The\x20context\n\x20if\x20for\x20the\x20user\x20to\x20se\ + t\x20and\x20use\x20in\x20this\x20case.\n\n\n\n\x03\x04\0\x01\x12\x03\x12\ + \x08\r\n\x0b\n\x04\x04\0\x02\0\x12\x03\x13\x04\x1d\n\r\n\x05\x04\0\x02\0\ + \x04\x12\x04\x13\x04\x12\x0f\n\x0c\n\x05\x04\0\x02\0\x06\x12\x03\x13\x04\ + \r\n\x0c\n\x05\x04\0\x02\0\x01\x12\x03\x13\x0e\x18\n\x0c\n\x05\x04\0\x02\ + \0\x03\x12\x03\x13\x1b\x1c\n\x0b\n\x04\x04\0\x02\x01\x12\x03\x14\x04\x14\ + \n\r\n\x05\x04\0\x02\x01\x04\x12\x04\x14\x04\x13\x1d\n\x0c\n\x05\x04\0\ + \x02\x01\x05\x12\x03\x14\x04\n\n\x0c\n\x05\x04\0\x02\x01\x01\x12\x03\x14\ + \x0b\x0f\n\x0c\n\x05\x04\0\x02\x01\x03\x12\x03\x14\x12\x13\n\x0b\n\x04\ + \x04\0\x02\x02\x12\x03\x15\x04\x15\n\r\n\x05\x04\0\x02\x02\x04\x12\x04\ + \x15\x04\x14\x14\n\x0c\n\x05\x04\0\x02\x02\x05\x12\x03\x15\x04\n\n\x0c\n\ + \x05\x04\0\x02\x02\x01\x12\x03\x15\x0b\x10\n\x0c\n\x05\x04\0\x02\x02\x03\ + \x12\x03\x15\x13\x14\n\x0b\n\x04\x04\0\x02\x03\x12\x03\x16\x04\x13\n\r\n\ + \x05\x04\0\x02\x03\x04\x12\x04\x16\x04\x15\x15\n\x0c\n\x05\x04\0\x02\x03\ + \x05\x12\x03\x16\x04\t\n\x0c\n\x05\x04\0\x02\x03\x01\x12\x03\x16\n\x0e\n\ + \x0c\n\x05\x04\0\x02\x03\x03\x12\x03\x16\x11\x12\n\x0b\n\x04\x04\0\x02\ + \x04\x12\x03\x17\x04\x16\n\r\n\x05\x04\0\x02\x04\x04\x12\x04\x17\x04\x16\ + \x13\n\x0c\n\x05\x04\0\x02\x04\x05\x12\x03\x17\x04\t\n\x0c\n\x05\x04\0\ + \x02\x04\x01\x12\x03\x17\n\x11\n\x0c\n\x05\x04\0\x02\x04\x03\x12\x03\x17\ + \x14\x15\nm\n\x04\x04\0\x02\x05\x12\x03\x1b\x04\x16\x1a`\x20Deprecated!\ + \x20It\x20is\x20kept\x20for\x20backward\x20compatibility.\n\x20TODO:\x20\ + remove\x20it\x20in\x20the\x20next\x20major\x20release.\n\n\r\n\x05\x04\0\ + \x02\x05\x04\x12\x04\x1b\x04\x17\x16\n\x0c\n\x05\x04\0\x02\x05\x05\x12\ + \x03\x1b\x04\x08\n\x0c\n\x05\x04\0\x02\x05\x01\x12\x03\x1b\t\x11\n\x0c\n\ + \x05\x04\0\x02\x05\x03\x12\x03\x1b\x14\x15\n\n\n\x02\x04\x01\x12\x04\x1e\ + \0\"\x01\n\n\n\x03\x04\x01\x01\x12\x03\x1e\x08\x18\n\x0b\n\x04\x04\x01\ + \x02\0\x12\x03\x1f\x04\x1d\n\r\n\x05\x04\x01\x02\0\x04\x12\x04\x1f\x04\ + \x1e\x1a\n\x0c\n\x05\x04\x01\x02\0\x06\x12\x03\x1f\x04\r\n\x0c\n\x05\x04\ + \x01\x02\0\x01\x12\x03\x1f\x0e\x18\n\x0c\n\x05\x04\x01\x02\0\x03\x12\x03\ + \x1f\x1b\x1c\n\x0b\n\x04\x04\x01\x02\x01\x12\x03\x20\x04\x15\n\r\n\x05\ + \x04\x01\x02\x01\x04\x12\x04\x20\x04\x1f\x1d\n\x0c\n\x05\x04\x01\x02\x01\ + \x05\x12\x03\x20\x04\n\n\x0c\n\x05\x04\x01\x02\x01\x01\x12\x03\x20\x0b\ + \x10\n\x0c\n\x05\x04\x01\x02\x01\x03\x12\x03\x20\x13\x14\n\x0b\n\x04\x04\ + \x01\x02\x02\x12\x03!\x04\x14\n\r\n\x05\x04\x01\x02\x02\x04\x12\x04!\x04\ + \x20\x15\n\x0c\n\x05\x04\x01\x02\x02\x05\x12\x03!\x04\n\n\x0c\n\x05\x04\ + \x01\x02\x02\x01\x12\x03!\x0b\x0f\n\x0c\n\x05\x04\x01\x02\x02\x03\x12\ + \x03!\x12\x13\n\n\n\x02\x04\x02\x12\x04$\0'\x01\n\n\n\x03\x04\x02\x01\ + \x12\x03$\x08\x10\n\x0b\n\x04\x04\x02\x02\0\x12\x03%\x04\x13\n\r\n\x05\ + \x04\x02\x02\0\x04\x12\x04%\x04$\x12\n\x0c\n\x05\x04\x02\x02\0\x05\x12\ + \x03%\x04\t\n\x0c\n\x05\x04\x02\x02\0\x01\x12\x03%\n\x0e\n\x0c\n\x05\x04\ + \x02\x02\0\x03\x12\x03%\x11\x12\n\x0b\n\x04\x04\x02\x02\x01\x12\x03&\x04\ + \"\n\r\n\x05\x04\x02\x02\x01\x04\x12\x04&\x04%\x13\n\x0c\n\x05\x04\x02\ + \x02\x01\x06\x12\x03&\x04\x14\n\x0c\n\x05\x04\x02\x02\x01\x01\x12\x03&\ + \x15\x1d\n\x0c\n\x05\x04\x02\x02\x01\x03\x12\x03&\x20!\n\n\n\x02\x05\x01\ + \x12\x04)\0=\x01\n\n\n\x03\x05\x01\x01\x12\x03)\x05\x10\n\x0b\n\x04\x05\ + \x01\x02\0\x12\x03*\x04\x0f\n\x0c\n\x05\x05\x01\x02\0\x01\x12\x03*\x04\n\ + \n\x0c\n\x05\x05\x01\x02\0\x02\x12\x03*\r\x0e\n\x0b\n\x04\x05\x01\x02\ + \x01\x12\x03+\x04\x10\n\x0c\n\x05\x05\x01\x02\x01\x01\x12\x03+\x04\x0b\n\ + \x0c\n\x05\x05\x01\x02\x01\x02\x12\x03+\x0e\x0f\n\x0b\n\x04\x05\x01\x02\ + \x02\x12\x03,\x04\x13\n\x0c\n\x05\x05\x01\x02\x02\x01\x12\x03,\x04\x0e\n\ + \x0c\n\x05\x05\x01\x02\x02\x02\x12\x03,\x11\x12\n\x0b\n\x04\x05\x01\x02\ + \x03\x12\x03-\x04\x12\n\x0c\n\x05\x05\x01\x02\x03\x01\x12\x03-\x04\r\n\ + \x0c\n\x05\x05\x01\x02\x03\x02\x12\x03-\x10\x11\n\x0b\n\x04\x05\x01\x02\ + \x04\x12\x03.\x04\x1a\n\x0c\n\x05\x05\x01\x02\x04\x01\x12\x03.\x04\x15\n\ + \x0c\n\x05\x05\x01\x02\x04\x02\x12\x03.\x18\x19\n\x0b\n\x04\x05\x01\x02\ + \x05\x12\x03/\x04\x17\n\x0c\n\x05\x05\x01\x02\x05\x01\x12\x03/\x04\x12\n\ + \x0c\n\x05\x05\x01\x02\x05\x02\x12\x03/\x15\x16\n\x0b\n\x04\x05\x01\x02\ + \x06\x12\x030\x04\x1f\n\x0c\n\x05\x05\x01\x02\x06\x01\x12\x030\x04\x1a\n\ + \x0c\n\x05\x05\x01\x02\x06\x02\x12\x030\x1d\x1e\n\x0b\n\x04\x05\x01\x02\ + \x07\x12\x031\x04\x14\n\x0c\n\x05\x05\x01\x02\x07\x01\x12\x031\x04\x0f\n\ + \x0c\n\x05\x05\x01\x02\x07\x02\x12\x031\x12\x13\n\x0b\n\x04\x05\x01\x02\ + \x08\x12\x032\x04\x15\n\x0c\n\x05\x05\x01\x02\x08\x01\x12\x032\x04\x10\n\ + \x0c\n\x05\x05\x01\x02\x08\x02\x12\x032\x13\x14\n\x0b\n\x04\x05\x01\x02\ + \t\x12\x033\x04\x1d\n\x0c\n\x05\x05\x01\x02\t\x01\x12\x033\x04\x18\n\x0c\ + \n\x05\x05\x01\x02\t\x02\x12\x033\x1b\x1c\n\x0b\n\x04\x05\x01\x02\n\x12\ + \x034\x04\x18\n\x0c\n\x05\x05\x01\x02\n\x01\x12\x034\x04\x12\n\x0c\n\x05\ + \x05\x01\x02\n\x02\x12\x034\x15\x17\n\x0b\n\x04\x05\x01\x02\x0b\x12\x035\ + \x04\x17\n\x0c\n\x05\x05\x01\x02\x0b\x01\x12\x035\x04\x11\n\x0c\n\x05\ + \x05\x01\x02\x0b\x02\x12\x035\x14\x16\n\x0b\n\x04\x05\x01\x02\x0c\x12\ + \x036\x04\x18\n\x0c\n\x05\x05\x01\x02\x0c\x01\x12\x036\x04\x12\n\x0c\n\ + \x05\x05\x01\x02\x0c\x02\x12\x036\x15\x17\n\x0b\n\x04\x05\x01\x02\r\x12\ + \x037\x04\x1b\n\x0c\n\x05\x05\x01\x02\r\x01\x12\x037\x04\x15\n\x0c\n\x05\ + \x05\x01\x02\r\x02\x12\x037\x18\x1a\n\x0b\n\x04\x05\x01\x02\x0e\x12\x038\ + \x04\x17\n\x0c\n\x05\x05\x01\x02\x0e\x01\x12\x038\x04\x11\n\x0c\n\x05\ + \x05\x01\x02\x0e\x02\x12\x038\x14\x16\n\x0b\n\x04\x05\x01\x02\x0f\x12\ + \x039\x04\x16\n\x0c\n\x05\x05\x01\x02\x0f\x01\x12\x039\x04\x10\n\x0c\n\ + \x05\x05\x01\x02\x0f\x02\x12\x039\x13\x15\n\x0b\n\x04\x05\x01\x02\x10\ + \x12\x03:\x04\x1a\n\x0c\n\x05\x05\x01\x02\x10\x01\x12\x03:\x04\x14\n\x0c\ + \n\x05\x05\x01\x02\x10\x02\x12\x03:\x17\x19\n\x0b\n\x04\x05\x01\x02\x11\ + \x12\x03;\x04\x1b\n\x0c\n\x05\x05\x01\x02\x11\x01\x12\x03;\x04\x15\n\x0c\ + \n\x05\x05\x01\x02\x11\x02\x12\x03;\x18\x1a\n\x0b\n\x04\x05\x01\x02\x12\ + \x12\x03<\x04#\n\x0c\n\x05\x05\x01\x02\x12\x01\x12\x03<\x04\x1d\n\x0c\n\ + \x05\x05\x01\x02\x12\x02\x12\x03<\x20\"\n\n\n\x02\x04\x03\x12\x04?\0L\ + \x01\n\n\n\x03\x04\x03\x01\x12\x03?\x08\x0f\n\x0b\n\x04\x04\x03\x02\0\ + \x12\x03@\x04\x1d\n\r\n\x05\x04\x03\x02\0\x04\x12\x04@\x04?\x11\n\x0c\n\ + \x05\x04\x03\x02\0\x06\x12\x03@\x04\x0f\n\x0c\n\x05\x04\x03\x02\0\x01\ + \x12\x03@\x10\x18\n\x0c\n\x05\x04\x03\x02\0\x03\x12\x03@\x1b\x1c\n\x0b\n\ + \x04\x04\x03\x02\x01\x12\x03A\x04\x12\n\r\n\x05\x04\x03\x02\x01\x04\x12\ + \x04A\x04@\x1d\n\x0c\n\x05\x04\x03\x02\x01\x05\x12\x03A\x04\n\n\x0c\n\ + \x05\x04\x03\x02\x01\x01\x12\x03A\x0b\r\n\x0c\n\x05\x04\x03\x02\x01\x03\ + \x12\x03A\x10\x11\n\x0b\n\x04\x04\x03\x02\x02\x12\x03B\x04\x14\n\r\n\x05\ + \x04\x03\x02\x02\x04\x12\x04B\x04A\x12\n\x0c\n\x05\x04\x03\x02\x02\x05\ + \x12\x03B\x04\n\n\x0c\n\x05\x04\x03\x02\x02\x01\x12\x03B\x0b\x0f\n\x0c\n\ + \x05\x04\x03\x02\x02\x03\x12\x03B\x12\x13\n\x0b\n\x04\x04\x03\x02\x03\ + \x12\x03C\x04\x14\n\r\n\x05\x04\x03\x02\x03\x04\x12\x04C\x04B\x14\n\x0c\ + \n\x05\x04\x03\x02\x03\x05\x12\x03C\x04\n\n\x0c\n\x05\x04\x03\x02\x03\ + \x01\x12\x03C\x0b\x0f\n\x0c\n\x05\x04\x03\x02\x03\x03\x12\x03C\x12\x13\n\ + \x0b\n\x04\x04\x03\x02\x04\x12\x03D\x04\x18\n\r\n\x05\x04\x03\x02\x04\ + \x04\x12\x04D\x04C\x14\n\x0c\n\x05\x04\x03\x02\x04\x05\x12\x03D\x04\n\n\ + \x0c\n\x05\x04\x03\x02\x04\x01\x12\x03D\x0b\x13\n\x0c\n\x05\x04\x03\x02\ + \x04\x03\x12\x03D\x16\x17\n\x0b\n\x04\x04\x03\x02\x05\x12\x03E\x04\x15\n\ + \r\n\x05\x04\x03\x02\x05\x04\x12\x04E\x04D\x18\n\x0c\n\x05\x04\x03\x02\ + \x05\x05\x12\x03E\x04\n\n\x0c\n\x05\x04\x03\x02\x05\x01\x12\x03E\x0b\x10\ + \n\x0c\n\x05\x04\x03\x02\x05\x03\x12\x03E\x13\x14\n\x0b\n\x04\x04\x03\ + \x02\x06\x12\x03F\x04\x1f\n\x0c\n\x05\x04\x03\x02\x06\x04\x12\x03F\x04\ + \x0c\n\x0c\n\x05\x04\x03\x02\x06\x06\x12\x03F\r\x12\n\x0c\n\x05\x04\x03\ + \x02\x06\x01\x12\x03F\x13\x1a\n\x0c\n\x05\x04\x03\x02\x06\x03\x12\x03F\ + \x1d\x1e\n\x0b\n\x04\x04\x03\x02\x07\x12\x03G\x04\x16\n\r\n\x05\x04\x03\ + \x02\x07\x04\x12\x04G\x04F\x1f\n\x0c\n\x05\x04\x03\x02\x07\x05\x12\x03G\ + \x04\n\n\x0c\n\x05\x04\x03\x02\x07\x01\x12\x03G\x0b\x11\n\x0c\n\x05\x04\ + \x03\x02\x07\x03\x12\x03G\x14\x15\n\x0b\n\x04\x04\x03\x02\x08\x12\x03H\ + \x04\x1a\n\r\n\x05\x04\x03\x02\x08\x04\x12\x04H\x04G\x16\n\x0c\n\x05\x04\ + \x03\x02\x08\x06\x12\x03H\x04\x0c\n\x0c\n\x05\x04\x03\x02\x08\x01\x12\ + \x03H\r\x15\n\x0c\n\x05\x04\x03\x02\x08\x03\x12\x03H\x18\x19\n\x0b\n\x04\ + \x04\x03\x02\t\x12\x03I\x04\x15\n\r\n\x05\x04\x03\x02\t\x04\x12\x04I\x04\ + H\x1a\n\x0c\n\x05\x04\x03\x02\t\x05\x12\x03I\x04\x08\n\x0c\n\x05\x04\x03\ + \x02\t\x01\x12\x03I\t\x0f\n\x0c\n\x05\x04\x03\x02\t\x03\x12\x03I\x12\x14\ + \n\x0b\n\x04\x04\x03\x02\n\x12\x03J\x04\x1c\n\r\n\x05\x04\x03\x02\n\x04\ + \x12\x04J\x04I\x15\n\x0c\n\x05\x04\x03\x02\n\x05\x12\x03J\x04\n\n\x0c\n\ + \x05\x04\x03\x02\n\x01\x12\x03J\x0b\x16\n\x0c\n\x05\x04\x03\x02\n\x03\ + \x12\x03J\x19\x1b\n\x0b\n\x04\x04\x03\x02\x0b\x12\x03K\x04\x17\n\r\n\x05\ + \x04\x03\x02\x0b\x04\x12\x04K\x04J\x1c\n\x0c\n\x05\x04\x03\x02\x0b\x05\ + \x12\x03K\x04\t\n\x0c\n\x05\x04\x03\x02\x0b\x01\x12\x03K\n\x11\n\x0c\n\ + \x05\x04\x03\x02\x0b\x03\x12\x03K\x14\x16\n\n\n\x02\x04\x04\x12\x04N\0R\ + \x01\n\n\n\x03\x04\x04\x01\x12\x03N\x08\x11\n\x0b\n\x04\x04\x04\x02\0\ + \x12\x03O\x04\x14\n\r\n\x05\x04\x04\x02\0\x04\x12\x04O\x04N\x13\n\x0c\n\ + \x05\x04\x04\x02\0\x05\x12\x03O\x04\n\n\x0c\n\x05\x04\x04\x02\0\x01\x12\ + \x03O\x0b\x0f\n\x0c\n\x05\x04\x04\x02\0\x03\x12\x03O\x12\x13\n\x0b\n\x04\ + \x04\x04\x02\x01\x12\x03P\x04\x14\n\r\n\x05\x04\x04\x02\x01\x04\x12\x04P\ + \x04O\x14\n\x0c\n\x05\x04\x04\x02\x01\x05\x12\x03P\x04\n\n\x0c\n\x05\x04\ + \x04\x02\x01\x01\x12\x03P\x0b\x0f\n\x0c\n\x05\x04\x04\x02\x01\x03\x12\ + \x03P\x12\x13\n\x0b\n\x04\x04\x04\x02\x02\x12\x03Q\x04\x16\n\r\n\x05\x04\ + \x04\x02\x02\x04\x12\x04Q\x04P\x14\n\x0c\n\x05\x04\x04\x02\x02\x05\x12\ + \x03Q\x04\n\n\x0c\n\x05\x04\x04\x02\x02\x01\x12\x03Q\x0b\x11\n\x0c\n\x05\ + \x04\x04\x02\x02\x03\x12\x03Q\x14\x15\n\n\n\x02\x04\x05\x12\x04T\0W\x01\ + \n\n\n\x03\x04\x05\x01\x12\x03T\x08\x11\n\x0b\n\x04\x04\x05\x02\0\x12\ + \x03U\x04\x1e\n\x0c\n\x05\x04\x05\x02\0\x04\x12\x03U\x04\x0c\n\x0c\n\x05\ + \x04\x05\x02\0\x05\x12\x03U\r\x13\n\x0c\n\x05\x04\x05\x02\0\x01\x12\x03U\ + \x14\x19\n\x0c\n\x05\x04\x05\x02\0\x03\x12\x03U\x1c\x1d\n\x0b\n\x04\x04\ + \x05\x02\x01\x12\x03V\x04!\n\x0c\n\x05\x04\x05\x02\x01\x04\x12\x03V\x04\ + \x0c\n\x0c\n\x05\x04\x05\x02\x01\x05\x12\x03V\r\x13\n\x0c\n\x05\x04\x05\ + \x02\x01\x01\x12\x03V\x14\x1c\n\x0c\n\x05\x04\x05\x02\x01\x03\x12\x03V\ + \x1f\x20\n\n\n\x02\x05\x02\x12\x04Y\0_\x01\n\n\n\x03\x05\x02\x01\x12\x03\ + Y\x05\x13\n\x0b\n\x04\x05\x02\x02\0\x12\x03Z\x04\x13\n\x0c\n\x05\x05\x02\ + \x02\0\x01\x12\x03Z\x04\x0b\n\x0c\n\x05\x05\x02\x02\0\x02\x12\x03Z\x11\ + \x12\n\x0b\n\x04\x05\x02\x02\x01\x12\x03[\x04\x13\n\x0c\n\x05\x05\x02\ + \x02\x01\x01\x12\x03[\x04\x0e\n\x0c\n\x05\x05\x02\x02\x01\x02\x12\x03[\ + \x11\x12\n\x0b\n\x04\x05\x02\x02\x02\x12\x03\\\x04\x17\n\x0c\n\x05\x05\ + \x02\x02\x02\x01\x12\x03\\\x04\x12\n\x0c\n\x05\x05\x02\x02\x02\x02\x12\ + \x03\\\x15\x16\n\x0b\n\x04\x05\x02\x02\x03\x12\x03]\x04\x18\n\x0c\n\x05\ + \x05\x02\x02\x03\x01\x12\x03]\x04\x13\n\x0c\n\x05\x05\x02\x02\x03\x02\ + \x12\x03]\x16\x17\n\x0b\n\x04\x05\x02\x02\x04\x12\x03^\x04\x1b\n\x0c\n\ + \x05\x05\x02\x02\x04\x01\x12\x03^\x04\x16\n\x0c\n\x05\x05\x02\x02\x04\ + \x02\x12\x03^\x19\x1a\n\n\n\x02\x04\x06\x12\x04a\0i\x01\n\n\n\x03\x04\ + \x06\x01\x12\x03a\x08\x12\n\x0b\n\x04\x04\x06\x02\0\x12\x03b\x04\x12\n\r\ + \n\x05\x04\x06\x02\0\x04\x12\x04b\x04a\x14\n\x0c\n\x05\x04\x06\x02\0\x05\ + \x12\x03b\x04\n\n\x0c\n\x05\x04\x06\x02\0\x01\x12\x03b\x0b\r\n\x0c\n\x05\ + \x04\x06\x02\0\x03\x12\x03b\x10\x11\n\x0b\n\x04\x04\x06\x02\x01\x12\x03c\ + \x04#\n\r\n\x05\x04\x06\x02\x01\x04\x12\x04c\x04b\x12\n\x0c\n\x05\x04\ + \x06\x02\x01\x06\x12\x03c\x04\x12\n\x0c\n\x05\x04\x06\x02\x01\x01\x12\ + \x03c\x13\x1e\n\x0c\n\x05\x04\x06\x02\x01\x03\x12\x03c!\"\nE\n\x04\x04\ + \x06\x02\x02\x12\x03e\x04\x17\x1a8\x20Used\x20in\x20`AddNode`,\x20`Remov\ + eNode`,\x20and\x20`AddLearnerNode`.\n\n\r\n\x05\x04\x06\x02\x02\x04\x12\ + \x04e\x04c#\n\x0c\n\x05\x04\x06\x02\x02\x05\x12\x03e\x04\n\n\x0c\n\x05\ + \x04\x06\x02\x02\x01\x12\x03e\x0b\x12\n\x0c\n\x05\x04\x06\x02\x02\x03\ + \x12\x03e\x15\x16\n\x0b\n\x04\x04\x06\x02\x03\x12\x03f\x04\x16\n\r\n\x05\ + \x04\x06\x02\x03\x04\x12\x04f\x04e\x17\n\x0c\n\x05\x04\x06\x02\x03\x05\ + \x12\x03f\x04\t\n\x0c\n\x05\x04\x06\x02\x03\x01\x12\x03f\n\x11\n\x0c\n\ + \x05\x04\x06\x02\x03\x03\x12\x03f\x14\x15\nB\n\x04\x04\x06\x02\x04\x12\ + \x03h\x04\x20\x1a5\x20Used\x20in\x20`BeginConfChange`\x20and\x20`Finaliz\ + eConfChange`.\n\n\r\n\x05\x04\x06\x02\x04\x04\x12\x04h\x04f\x16\n\x0c\n\ + \x05\x04\x06\x02\x04\x06\x12\x03h\x04\r\n\x0c\n\x05\x04\x06\x02\x04\x01\ + \x12\x03h\x0e\x1b\n\x0c\n\x05\x04\x06\x02\x04\x03\x12\x03h\x1e\x1fb\x06p\ + roto3\ "; static mut file_descriptor_proto_lazy: ::protobuf::lazy::Lazy<::protobuf::descriptor::FileDescriptorProto> = ::protobuf::lazy::Lazy { diff --git a/src/errors.rs b/src/errors.rs index 2ad61aecd..b30c5ccc3 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -13,6 +13,7 @@ use std::error; use std::{cmp, io, result}; +use StateRole; use protobuf::ProtobufError; @@ -63,6 +64,14 @@ quick_error! { NotExists(id: u64, set: &'static str) { display("The node {} is not in the {} set.", id, set) } + /// The action give requires the node to be in a particular state role. + InvalidState(role: StateRole) { + display("Cannot complete that action while in {:?} role.", role) + } + /// The node attempted to transition to a new membership configuration while there was none pending. + NoPendingTransition { + display("No pending membership transition. Create a pending transition with `Raft.begin_config_transition`") + } } } diff --git a/src/lib.rs b/src/lib.rs index df01a2b18..560b52858 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -275,6 +275,8 @@ extern crate quick_error; #[cfg(test)] extern crate env_logger; extern crate rand; +#[macro_use] +extern crate getset; mod config; /// This module supplies the needed message types. However, it is autogenerated and thus cannot be @@ -297,7 +299,7 @@ pub mod util; pub use self::config::Config; pub use self::errors::{Error, Result, StorageError}; pub use self::log_unstable::Unstable; -pub use self::progress::{Inflights, Progress, ProgressSet, ProgressState}; +pub use self::progress::{Configuration, Inflights, Progress, ProgressSet, ProgressState}; pub use self::raft::{vote_resp_msg_type, Raft, SoftState, StateRole, INVALID_ID, INVALID_INDEX}; pub use self::raft_log::{RaftLog, NO_LIMIT}; pub use self::raw_node::{is_empty_snap, Peer, RawNode, Ready, SnapshotStatus}; diff --git a/src/progress.rs b/src/progress.rs index 723c5a0fa..126477899 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -25,6 +25,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use eraftpb::ConfState; use errors::Error; use fxhash::{FxBuildHasher, FxHashMap, FxHashSet}; use std::cell::RefCell; @@ -54,12 +55,95 @@ impl Default for ProgressState { } } -#[derive(Clone, Debug, Default)] -struct Configuration { +#[derive(Clone, Debug, Default, PartialEq, Getters)] +/// A Raft internal representation of a Configuration. This is corallary to a ConfState, but optimized for `contains` calls. +pub struct Configuration { + /// The voter set. + #[get = "pub"] voters: FxHashSet, + /// The learner set. + #[get = "pub"] learners: FxHashSet, } +impl Configuration { + /// Create a new configuration with the given configuration. + pub fn new( + voters: impl IntoIterator, + learners: impl IntoIterator, + ) -> Self { + Self { + voters: voters.into_iter().collect(), + learners: learners.into_iter().collect(), + } + } +} + +impl<'a, I> From<(I, I)> for Configuration +where + I: IntoIterator, +{ + fn from((voters, learners): (I, I)) -> Self { + Self { + voters: voters.into_iter().cloned().collect(), + learners: learners.into_iter().cloned().collect(), + } + } +} + +impl<'a> From<&'a ConfState> for Configuration { + fn from(conf_state: &'a ConfState) -> Self { + Self { + voters: conf_state.get_nodes().iter().cloned().collect(), + learners: conf_state.get_learners().iter().cloned().collect(), + } + } +} + +impl Into for Configuration { + fn into(self) -> ConfState { + let mut state = ConfState::new(); + state.set_nodes(self.voters.iter().cloned().collect()); + state.set_learners(self.learners.iter().cloned().collect()); + state + } +} + +impl Configuration { + fn with_capacity(voters: usize, learners: usize) -> Self { + Self { + voters: HashSet::with_capacity_and_hasher(voters, FxBuildHasher::default()), + learners: HashSet::with_capacity_and_hasher(learners, FxBuildHasher::default()), + } + } + + /// Validates that the configuration not problematic. + /// + /// Namely: + /// * There can be no overlap of voters and learners. + /// * There must be at least one voter. + pub fn valid(&self) -> Result<(), Error> { + if let Some(id) = self.voters.intersection(&self.learners).next() { + Err(Error::Exists(*id, "learners"))?; + } else if self.voters.is_empty() { + Err(Error::ConfigInvalid( + "There must be at least one voter.".into(), + ))?; + } + Ok(()) + } + + fn has_quorum(&self, potential_quorum: &FxHashSet) -> bool { + self.voters.intersection(potential_quorum).count() >= majority(self.voters.len()) + } + + /// Returns whether or not the given `id` is a member of this configuration. + #[allow(trivially_copy_pass_by_ref)] // Allowed to maintain API consistency. + pub fn contains(&self, id: &u64) -> bool { + self.voters.contains(id) || self.learners.contains(id) + } +} + /// The status of an election according to a Candidate node. /// /// This is returned by `progress_set.election_status(vote_map)` @@ -75,10 +159,16 @@ pub enum CandidacyStatus { /// `ProgressSet` contains several `Progress`es, /// which could be `Leader`, `Follower` and `Learner`. -#[derive(Default, Clone)] +#[derive(Default, Clone, Getters)] pub struct ProgressSet { progress: FxHashMap, + /// The current configuration state of the cluster. + #[get = "pub"] configuration: Configuration, + /// The pending configuration, which will be adopted after the Finalize entryr are applied + #[get = "pub"] + next_configuration: Option, + configuration_capacity: (usize, usize), // A preallocated buffer for sorting in the minimally_commited_index function. // You should not depend on these values unless you just set them. // We use a cell to avoid taking a `&mut self`. @@ -88,29 +178,27 @@ pub struct ProgressSet { impl ProgressSet { /// Creates a new ProgressSet. pub fn new() -> Self { - ProgressSet { - progress: Default::default(), - configuration: Default::default(), - sort_buffer: Default::default(), - } + Self::with_capacity(0, 0) } - /// Create a progress sete with the specified sizes already reserved. + /// Create a progress set with the specified sizes already reserved. pub fn with_capacity(voters: usize, learners: usize) -> Self { ProgressSet { progress: HashMap::with_capacity_and_hasher( voters + learners, FxBuildHasher::default(), ), - configuration: Configuration { - voters: HashSet::with_capacity_and_hasher(voters, FxBuildHasher::default()), - learners: HashSet::with_capacity_and_hasher(learners, FxBuildHasher::default()), - }, - sort_buffer: Default::default(), + sort_buffer: RefCell::from(Vec::with_capacity(voters)), + configuration_capacity: (voters, learners), + configuration: Configuration::with_capacity(voters, learners), + next_configuration: Option::default(), } } /// Returns the status of voters. + /// + /// **Note:** Do not use this for majority/quorum calculation. The Raft node may be + /// transitioning to a new configuration and have two qourums. Use `has_quorum` instead. #[inline] pub fn voters(&self) -> impl Iterator { let set = self.voter_ids(); @@ -118,6 +206,9 @@ impl ProgressSet { } /// Returns the status of learners. + /// + /// **Note:** Do not use this for majority/quorum calculation. The Raft node may be + /// transitioning to a new configuration and have two qourums. Use `has_quorum` instead. #[inline] pub fn learners(&self) -> impl Iterator { let set = self.learner_ids(); @@ -125,33 +216,61 @@ impl ProgressSet { } /// Returns the mutable status of voters. + /// + /// **Note:** Do not use this for majority/quorum calculation. The Raft node may be + /// transitioning to a new configuration and have two qourums. Use `has_quorum` instead. #[inline] pub fn voters_mut(&mut self) -> impl Iterator { - let ids = &self.configuration.voters; + let ids = self.voter_ids(); self.progress .iter_mut() .filter(move |(k, _)| ids.contains(k)) } /// Returns the mutable status of learners. + /// + /// **Note:** Do not use this for majority/quorum calculation. The Raft node may be + /// transitioning to a new configuration and have two qourums. Use `has_quorum` instead. #[inline] pub fn learners_mut(&mut self) -> impl Iterator { - let ids = &self.configuration.learners; + let ids = self.learner_ids(); self.progress .iter_mut() .filter(move |(k, _)| ids.contains(k)) } /// Returns the ids of all known voters. + /// + /// **Note:** Do not use this for majority/quorum calculation. The Raft node may be + /// transitioning to a new configuration and have two qourums. Use `has_quorum` instead. #[inline] - pub fn voter_ids(&self) -> &FxHashSet { - &self.configuration.voters + pub fn voter_ids(&self) -> FxHashSet { + match self.next_configuration { + Some(ref next) => self + .configuration + .voters + .union(&next.voters) + .cloned() + .collect::>(), + None => self.configuration.voters.clone(), + } } /// Returns the ids of all known learners. + /// + /// **Note:** Do not use this for majority/quorum calculation. The Raft node may be + /// transitioning to a new configuration and have two qourums. Use `has_quorum` instead. #[inline] - pub fn learner_ids(&self) -> &FxHashSet { - &self.configuration.learners + pub fn learner_ids(&self) -> FxHashSet { + match self.next_configuration { + Some(ref next) => self + .configuration + .learners + .union(&next.learners) + .cloned() + .collect::>(), + None => self.configuration.learners.clone(), + } } /// Grabs a reference to the progress of a node. @@ -167,12 +286,18 @@ impl ProgressSet { } /// Returns an iterator across all the nodes and their progress. + /// + /// **Note:** Do not use this for majority/quorum calculation. The Raft node may be + /// transitioning to a new configuration and have two qourums. Use `has_quorum` instead. #[inline] pub fn iter(&self) -> impl ExactSizeIterator { self.progress.iter() } /// Returns a mutable iterator across all the nodes and their progress. + /// + /// **Note:** Do not use this for majority/quorum calculation. The Raft node may be + /// transitioning to a new configuration and have two qourums. Use `has_quorum` instead. #[inline] pub fn iter_mut(&mut self) -> impl ExactSizeIterator { self.progress.iter_mut() @@ -180,15 +305,18 @@ impl ProgressSet { /// Adds a voter node pub fn insert_voter(&mut self, id: u64, pr: Progress) -> Result<(), Error> { - // If the progress exists already this is in error. - if self.progress.contains_key(&id) { - // Determine the correct error to return. - if self.learner_ids().contains(&id) { - return Err(Error::Exists(id, "learners")); - } - return Err(Error::Exists(id, "voters")); + debug!("Inserting voter with id {}.", id); + + if self.learner_ids().contains(&id) { + Err(Error::Exists(id, "learners"))?; + } else if self.voter_ids().contains(&id) { + Err(Error::Exists(id, "voters"))?; } self.configuration.voters.insert(id); + self.next_configuration + .as_mut() + .map(|config| config.voters.insert(id)); + self.progress.insert(id, pr); self.assert_progress_and_configuration_consistent(); Ok(()) @@ -196,15 +324,18 @@ impl ProgressSet { /// Adds a learner to the cluster pub fn insert_learner(&mut self, id: u64, pr: Progress) -> Result<(), Error> { - // If the progress exists already this is in error. - if self.progress.contains_key(&id) { - // Determine the correct error to return. - if self.learner_ids().contains(&id) { - return Err(Error::Exists(id, "learners")); - } - return Err(Error::Exists(id, "voters")); + debug!("Inserting learner with id {}.", id); + + if self.learner_ids().contains(&id) { + Err(Error::Exists(id, "learners"))?; + } else if self.voter_ids().contains(&id) { + Err(Error::Exists(id, "voters"))?; } self.configuration.learners.insert(id); + if let Some(ref mut next) = self.next_configuration { + next.learners.insert(id); + } + self.progress.insert(id, pr); self.assert_progress_and_configuration_consistent(); Ok(()) @@ -212,8 +343,15 @@ impl ProgressSet { /// Removes the peer from the set of voters or learners. pub fn remove(&mut self, id: u64) -> Option { - self.configuration.voters.remove(&id); + debug!("Removing peer with id {}.", id); + self.configuration.learners.remove(&id); + self.configuration.voters.remove(&id); + if let Some(ref mut next) = self.next_configuration { + next.learners.remove(&id); + next.voters.remove(&id); + }; + let removed = self.progress.remove(&id); self.assert_progress_and_configuration_consistent(); removed @@ -221,6 +359,8 @@ impl ProgressSet { /// Promote a learner to a peer. pub fn promote_learner(&mut self, id: u64) -> Result<(), Error> { + debug!("Promote learner with id {}.", id); + if !self.configuration.learners.remove(&id) { // Wasn't already a learner. We can't promote what doesn't exist. return Err(Error::NotExists(id, "learners")); @@ -229,6 +369,23 @@ impl ProgressSet { // Already existed, the caller should know this was a noop. return Err(Error::Exists(id, "voters")); } + if let Some(ref mut next) = self.next_configuration { + if !next.learners.remove(&id) { + // Wasn't already a voter. We can't promote what doesn't exist. + // Rollback change above. + self.configuration.voters.remove(&id); + self.configuration.learners.insert(id); + return Err(Error::Exists(id, "next learners")); + } + if !next.voters.insert(id) { + // Already existed, the caller should know this was a noop. + // Rollback change above. + self.configuration.voters.remove(&id); + self.configuration.learners.insert(id); + return Err(Error::Exists(id, "next voters")); + } + } + self.assert_progress_and_configuration_consistent(); Ok(()) } @@ -236,19 +393,17 @@ impl ProgressSet { #[inline(always)] fn assert_progress_and_configuration_consistent(&self) { debug_assert!( - self.configuration - .voters - .union(&self.configuration.learners) + self.voter_ids() + .union(&self.learner_ids()) .all(|v| self.progress.contains_key(v)) ); debug_assert!( self.progress .keys() - .all(|v| self.configuration.learners.contains(v) - || self.configuration.voters.contains(v)) + .all(|v| self.learner_ids().contains(v) || self.voter_ids().contains(v)) ); assert_eq!( - self.configuration.voters.len() + self.configuration.learners.len(), + self.voter_ids().len() + self.learner_ids().len(), self.progress.len() ); } @@ -259,13 +414,30 @@ impl ProgressSet { pub fn maximal_committed_index(&self) -> u64 { let mut matched = self.sort_buffer.borrow_mut(); matched.clear(); - self.voters().for_each(|(_id, peer)| { + self.configuration.voters().iter().for_each(|id| { + let peer = &self.progress[id]; matched.push(peer.matched); }); // Reverse sort. matched.sort_by(|a, b| b.cmp(a)); - // Smallest that the majority has commited. - matched[matched.len() / 2] + let mut mci = matched[matched.len() / 2]; + + if let Some(next) = &self.next_configuration { + let mut matched = next + .voters() + .iter() + .map(|id| { + let peer = &self.progress[id]; + peer.matched + }).collect::>(); + matched.sort_by(|a, b| b.cmp(a)); + // Smallest that the majority has commited. + let next_mci = matched[matched.len() / 2]; + if next_mci < mci { + mci = next_mci; + } + } + mci } /// Returns the Candidate's eligibility in the current election. @@ -275,34 +447,33 @@ impl ProgressSet { /// or `Ineligible`, meaning the election can be concluded. pub fn candidacy_status<'a>( &self, - id: u64, votes: impl IntoIterator, ) -> CandidacyStatus { - let (accepted, total) = - votes - .into_iter() - .fold((0, 0), |(mut accepted, mut total), (_, nominated)| { - if *nominated { - accepted += 1; - } - total += 1; - (accepted, total) - }); - let quorum = majority(self.voter_ids().len()); - let rejected = total - accepted; - - info!( - "{} [quorum: {}] has received {} votes and {} vote rejections", - id, quorum, accepted, rejected, + let (accepts, rejects) = votes.into_iter().fold( + (FxHashSet::default(), FxHashSet::default()), + |(mut accepts, mut rejects), (&id, &accepted)| { + if accepted { + accepts.insert(id); + } else { + rejects.insert(id); + } + (accepts, rejects) + }, ); - - if accepted >= quorum { - CandidacyStatus::Elected - } else if rejected == quorum { - CandidacyStatus::Ineligible - } else { - CandidacyStatus::Eligible + if self.configuration.has_quorum(&accepts) { + return CandidacyStatus::Elected; + } else if self.configuration.has_quorum(&rejects) { + return CandidacyStatus::Ineligible; + } + if let Some(ref next) = self.next_configuration { + if next.has_quorum(&accepts) { + return CandidacyStatus::Elected; + } else if next.has_quorum(&rejects) { + return CandidacyStatus::Ineligible; + } } + + CandidacyStatus::Eligible } /// Determines if the current quorum is active according to the this raft node. @@ -310,26 +481,120 @@ impl ProgressSet { /// /// This should only be called by the leader. pub fn quorum_recently_active(&mut self, perspective_of: u64) -> bool { - let mut active = 0; + let mut active = FxHashSet::default(); for (&id, pr) in self.voters_mut() { if id == perspective_of { - active += 1; + active.insert(id); continue; } if pr.recent_active { - active += 1; + active.insert(id); } pr.recent_active = false; } for (&_id, pr) in self.learners_mut() { pr.recent_active = false; } - active >= majority(self.voter_ids().len()) + self.configuration.has_quorum(&active) && + // If `next` is `None` we don't consider it, so just `true` it. + self.next_configuration.as_ref().map(|next| next.has_quorum(&active)).unwrap_or(true) } /// Determine if a quorum is formed from the given set of nodes. + /// + /// This is the only correct way to verify you have reached a quorum for the whole group. + #[inline] pub fn has_quorum(&self, potential_quorum: &FxHashSet) -> bool { - potential_quorum.len() >= majority(self.voter_ids().len()) + self.configuration.has_quorum(potential_quorum) && self + .next_configuration + .as_ref() + .map(|next| next.has_quorum(potential_quorum)) + // If `next` is `None` we don't consider it, so just `true` it. + .unwrap_or(true) + } + + /// Determine if the ProgressSet is represented by a transition state under Joint Consensus. + #[inline] + pub fn is_in_transition(&self) -> bool { + self.next_configuration.is_some() + } + + /// Enter a joint consensus state to transition to the specified configuration. + /// + /// The `next` provided should be derived from the `ConfChange` message. `progress` is used as a basis for created peer `Progress` values. You are only expected to set `ins` from the `raft.max_inflights` value. + /// + /// Once this state is entered the leader should replicate the `ConfChange` message. After the + /// majority of nodes, in both the current and the `next`, have committed the union state. At + /// this point the leader can call `finalize_config_transition` and replicate a message + /// commiting the change. + /// + /// Valid transitions: + /// * Non-existing -> Learner + /// * Non-existing -> Voter + /// * Learner -> Voter + /// * Learner -> Non-existing + /// * Voter -> Non-existing + /// + /// Errors: + /// * Voter -> Learner + /// * Member as voter and learner. + /// * Empty voter set. + pub fn begin_config_transition( + &mut self, + next: impl Into, + mut progress: Progress, + ) -> Result<(), Error> { + let next = next.into(); + next.valid()?; + // Demotion check. + if let Some(&demoted) = self + .configuration + .voters + .intersection(&next.learners) + .next() + { + Err(Error::Exists(demoted, "learners"))?; + } + debug!("Beginning member configuration transition. End state will be voters {:?}, Learners: {:?}", next.voters, next.learners); + + // When a node is first added/promoted, we should mark it as recently active. + // Otherwise, check_quorum may cause us to step down if it is invoked + // before the added node has a chance to commuicate with us. + progress.recent_active = true; + progress.paused = false; + for id in next.voters.iter().chain(&next.learners) { + self.progress.entry(*id).or_insert_with(|| progress.clone()); + } + self.next_configuration = Some(next); + // Now we create progresses for any that do not exist. + Ok(()) + } + + /// Finalizes the joint consensus state and transitions solely to the new state. + /// + /// This should be called only after calling `begin_config_transition` and the the majority + /// of nodes in both the `current` and the `next` state have commited the changes. + pub fn finalize_config_transition(&mut self) -> Result<(), Error> { + let next = self.next_configuration.take(); + match next { + None => Err(Error::NoPendingTransition)?, + Some(next) => { + { + let pending = self + .configuration + .voters() + .difference(next.voters()) + .chain(self.configuration.learners().difference(next.learners())) + .cloned(); + for id in pending { + self.progress.remove(&id); + } + } + self.configuration = next; + debug!("Executed finalize member configration transition command. State is Voters: {:?}, Learners: {:?}", self.configuration.voters, self.configuration.learners); + } + } + Ok(()) } } @@ -514,7 +779,7 @@ impl Progress { } /// A buffer of inflight messages. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, PartialEq)] pub struct Inflights { // the starting index in the buffer start: usize, @@ -525,6 +790,19 @@ pub struct Inflights { buffer: Vec, } +// The `buffer` must have it's capacity set correctly on clone, normally it does not. +impl Clone for Inflights { + fn clone(&self) -> Self { + let mut buffer = self.buffer.clone(); + buffer.reserve(self.buffer.capacity() - self.buffer.len()); + Inflights { + start: self.start, + count: self.count, + buffer, + } + } +} + impl Inflights { /// Creates a new buffer for inflight messages. pub fn new(cap: usize) -> Inflights { @@ -745,8 +1023,8 @@ mod test { // See https://github.com/pingcap/raft-rs/issues/125 #[cfg(test)] mod test_progress_set { - use Result; - use {Progress, ProgressSet}; + use fxhash::FxHashSet; + use {progress::Configuration, Progress, ProgressSet, Result}; const CANARY: u64 = 123; @@ -843,4 +1121,108 @@ mod test_progress_set { assert_eq!(pre, *set.get(1).expect("Peer should not have been deleted")); Ok(()) } + + #[test] + fn test_config_transition_remove_voter() -> Result<()> { + check_set_nodes(&vec![1, 2], &vec![], &vec![1], &vec![]) + } + + #[test] + fn test_config_transition_remove_learner() -> Result<()> { + check_set_nodes(&vec![1], &vec![2], &vec![1], &vec![]) + } + + #[test] + fn test_config_transition_conflicting_sets() { + assert!(check_set_nodes(&vec![1], &vec![], &vec![1], &vec![1]).is_err()) + } + + #[test] + fn test_config_transition_empty_sets() { + assert!(check_set_nodes(&vec![], &vec![], &vec![], &vec![]).is_err()) + } + + #[test] + fn test_config_transition_empty_voters() { + assert!(check_set_nodes(&vec![1], &vec![], &vec![], &vec![]).is_err()) + } + + #[test] + fn test_config_transition_add_voter() -> Result<()> { + check_set_nodes(&vec![1], &vec![], &vec![1, 2], &vec![]) + } + + #[test] + fn test_config_transition_add_learner() -> Result<()> { + check_set_nodes(&vec![1], &vec![], &vec![1], &vec![2]) + } + + #[test] + fn test_config_transition_promote_learner() -> Result<()> { + check_set_nodes(&vec![1], &vec![2], &vec![1, 2], &vec![]) + } + + fn check_set_nodes<'a>( + start_voters: impl IntoIterator, + start_learners: impl IntoIterator, + end_voters: impl IntoIterator, + end_learners: impl IntoIterator, + ) -> Result<()> { + let start_voters = start_voters + .into_iter() + .cloned() + .collect::>(); + let start_learners = start_learners + .into_iter() + .cloned() + .collect::>(); + let end_voters = end_voters.into_iter().cloned().collect::>(); + let end_learners = end_learners + .into_iter() + .cloned() + .collect::>(); + let transition_voters = start_voters + .union(&end_voters) + .cloned() + .collect::>(); + let transition_learners = start_learners + .union(&end_learners) + .cloned() + .collect::>(); + + let mut set = ProgressSet::default(); + let default_progress = Progress::new(0, 10); + + for starter in start_voters { + set.insert_voter(starter, default_progress.clone())?; + } + for starter in start_learners { + set.insert_learner(starter, default_progress.clone())?; + } + set.begin_config_transition( + Configuration::new(end_voters.clone(), end_learners.clone()), + default_progress, + )?; + assert!(set.is_in_transition()); + assert_eq!( + set.voter_ids(), + transition_voters, + "Transition state voters inaccurate" + ); + assert_eq!( + set.learner_ids(), + transition_learners, + "Transition state learners inaccurate." + ); + + set.finalize_config_transition()?; + assert!(!set.is_in_transition()); + assert_eq!(set.voter_ids(), end_voters, "End state voters inaccurate"); + assert_eq!( + set.learner_ids(), + end_learners, + "End state learners inaccurate" + ); + Ok(()) + } } diff --git a/src/raft.rs b/src/raft.rs index 12c9ed398..d14d6e26d 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -27,13 +27,17 @@ use std::cmp; -use eraftpb::{Entry, EntryType, HardState, Message, MessageType, Snapshot}; +use eraftpb::{ + ConfChange, ConfChangeType, ConfState, Entry, EntryType, HardState, Message, MessageType, + Snapshot, +}; use fxhash::{FxHashMap, FxHashSet}; +use protobuf; use protobuf::RepeatedField; use rand::{self, Rng}; use super::errors::{Error, Result, StorageError}; -use super::progress::{CandidacyStatus, Progress, ProgressSet, ProgressState}; +use super::progress::{CandidacyStatus, Configuration, Progress, ProgressSet, ProgressState}; use super::raft_log::{self, RaftLog}; use super::read_only::{ReadOnly, ReadOnlyOption, ReadState}; use super::storage::Storage; @@ -142,6 +146,10 @@ pub struct Raft { /// value. pub pending_conf_index: u64, + /// The last BeginConfChange entry. Once we commit this entry we can exit the joint state. + //TODO: Ensure this is initialized like `pending_conf_index`, also attempt to merge them. + began_set_nodes_at: Option, + /// The queue of read-only requests. pub read_only: ReadOnly, @@ -246,6 +254,7 @@ impl Raft { term: Default::default(), election_elapsed: Default::default(), pending_conf_index: Default::default(), + began_set_nodes_at: Default::default(), vote: Default::default(), heartbeat_elapsed: Default::default(), randomized_election_timeout: 0, @@ -274,7 +283,7 @@ impl Raft { r.load_state(&rs.hard_state); } if c.applied > 0 { - r.raft_log.applied_to(c.applied); + r.commit_apply(c.applied); } let term = r.term; r.become_follower(term, INVALID_ID); @@ -361,6 +370,11 @@ impl Raft { self.heartbeat_timeout } + /// Fetch the number of ticks elapsed since last heartbeat. + pub fn get_heartbeat_elapsed(&self) -> usize { + self.heartbeat_elapsed + } + /// Return the length of the current randomized election timeout. pub fn get_randomized_election_timeout(&self) -> usize { self.randomized_election_timeout @@ -374,6 +388,7 @@ impl Raft { // send persists state to stable storage and then sends to its mailbox. fn send(&mut self, mut m: Message) { + debug!("Sending from {} to {}: {:?}", self.id, m.get_to(), m); m.set_from(self.id); if m.get_msg_type() == MessageType::MsgRequestVote || m.get_msg_type() == MessageType::MsgRequestPreVote @@ -502,6 +517,7 @@ impl Raft { /// Sends RPC, with entries to the given peer. pub fn send_append(&mut self, to: u64, pr: &mut Progress) { if pr.is_paused() { + trace!("Skipping sending to {}, it's paused. {:?}", to, pr); return; } let term = self.raft_log.term(pr.next_idx - 1); @@ -510,6 +526,7 @@ impl Raft { m.set_to(to); if term.is_err() || ents.is_err() { // send snapshot if we failed to get term or entries + trace!("Skipping sending to {}, term or ents is_err()", to); if !self.prepare_send_snapshot(&mut m, pr, to) { return; } @@ -572,6 +589,42 @@ impl Raft { self.raft_log.maybe_commit(mci, self.term) } + /// Commit that the Raft peer has applied up to the given index. + /// + /// Registers the new applied index to the Raft log, then checks to see if it's time to finalize a Joint Consensus state. + pub fn commit_apply(&mut self, applied: u64) { + trace!("{:?}, Enter commit_apply(applied: {:?})", self.id, applied); + #[allow(deprecated)] + self.raft_log.applied_to(applied); + + // Check to see if we need to finalize a Joint Consensus state now. + if let Some(index) = self.began_set_nodes_at { + // Invariant: We know that if we have commited past some index, we can also commit that index. + if applied >= index { + // Ensure we reset this on *any* node, since the leader might have failed + // and we don't want to finalize twice. + self.began_set_nodes_at = None; + if self.state == StateRole::Leader { + // We must replicate the commit entry. + self.append_finalize_conf_change_entry(); + } + } + } + trace!("Exit commit_apply(applied: {:?})", applied); + } + + fn append_finalize_conf_change_entry(&mut self) { + let mut conf_change = ConfChange::new(); + conf_change.set_change_type(ConfChangeType::FinalizeConfChange); + let data = protobuf::Message::write_to_bytes(&conf_change).unwrap(); + let mut entry = Entry::new(); + entry.set_entry_type(EntryType::EntryConfChange); + entry.set_data(data); + // Index/Term set here. + self.append_entry(&mut [entry]); + self.bcast_append(); + } + /// Resets the current node to a given term. pub fn reset(&mut self, term: u64) { if self.term != term { @@ -618,14 +671,17 @@ impl Raft { self.maybe_commit(); } - /// Returns true to indicate that there will probably be some readiness need to be handled. + ///maybe_commit Returns true to indicate that there will probably be some readiness need to be handled. pub fn tick(&mut self) -> bool { - match self.state { + trace!("{} enter tick()", self.id); + let result = match self.state { StateRole::Follower | StateRole::PreCandidate | StateRole::Candidate => { self.tick_election() } StateRole::Leader => self.tick_heartbeat(), - } + }; + trace!("{} exit tick()", self.id); + result } // TODO: revoke pub when there is a better way to test. @@ -779,7 +835,7 @@ impl Raft { self.id, vote_msg, self_id, self.term ); self.register_vote(self_id, acceptance); - if let CandidacyStatus::Elected = self.prs().candidacy_status(self.id, &self.votes) { + if let CandidacyStatus::Elected = self.prs().candidacy_status(&self.votes) { // We won the election after voting for ourselves (which must mean that // this is a single-node cluster). Advance to the next state. if campaign_type == CAMPAIGN_PRE_ELECTION { @@ -841,6 +897,10 @@ impl Raft { // if a server receives RequestVote request within the minimum election // timeout of hearing from a current leader, it does not update its term // or grant its vote + // + // This is included in the 3rd concern for Joint Consensus, where if another + // peer is removed from the cluster it may try to hold elections and disrupt + // stability. info!( "{} [logterm: {}, index: {}, vote: {}] ignored {:?} vote from \ {} [logterm: {}, index: {}] at term {}: lease is not expired \ @@ -1032,7 +1092,113 @@ impl Raft { StateRole::Leader => self.step_leader(m)?, }, } + Ok(()) + } + /// Called to apply a `BeginConfChange` entry. + /// + /// + /// When a Raft node applies this variant of a configuration change it will adopt a joint configuration state until the membership change is finalized. + /// + /// During this time the `Raft` will have two, possibly overlapping, cooperating quorums for both elections and log replication. + /// + /// # Implementation notes + /// + /// This uses a slightly modified "Joint Consensus" algorithm as detailed in Section 6 of the Raft paper. + /// + /// The modification is as follows: We apply the change when a node *applies* the entry, not when the entry is received. + /// + /// # Panics + /// + /// This **must** only be called on `Entry` which holds an `Entry` of with `EntryType::ConfChange` and the `data` field being a serialized `ConfChange`. + /// + /// This `ConfChange` must be of variant `BeginConfChange` and contain a `configuration` value. + // TODO: Make this return a result instead of panic. + #[inline(always)] + pub fn begin_membership_change(&mut self, entry: &Entry) -> Result<()> { + trace!( + "{} enter begin_membership_change(entry: {:?})", + self.id, + entry + ); + // TODO: Check if this should be rejected for normal reasons. + // Notably, if another is happening now. + assert_eq!(entry.get_entry_type(), EntryType::EntryConfChange); + let conf_change = protobuf::parse_from_bytes::(entry.get_data())?; + assert_eq!( + conf_change.get_change_type(), + ConfChangeType::BeginConfChange + ); + let configuration = conf_change.get_configuration(); + self.began_set_nodes_at = Some(entry.get_index()); + let max_inflights = self.max_inflight; + self.mut_prs() + .begin_config_transition(configuration, Progress::new(1, max_inflights))?; + trace!( + "{} exit begin_membership_change(entry: {:?})", + self.id, + entry + ); + Ok(()) + } + + /// Called to apply a `FinalizeConfChange` entry. + /// + /// When a Raft node applies this variant of a configuration change it will finalize the transition begun by [`begin_membership_change`] + /// + /// Once this is called the Raft will no longer have two, possibly overlapping, cooperating qourums. + /// + /// # Implementation notes + /// + /// This uses a slightly modified "Joint Consensus" algorithm as detailed in Section 6 of the Raft paper. + /// + /// The modification is as follows: We apply the change when a node *applies* the entry, not when the entry is received. + /// + /// # Panics + /// + /// This **must** only be called on `Entry` which holds an `Entry` of with `EntryType::ConfChange` and the `data` field being a serialized `ConfChange`. + /// + /// This `ConfChange` must be of variant `FinalizeConfChange` and contain no `configuration` value. + // TODO: Make this return a result instead of panic. + #[inline(always)] + pub fn finalize_membership_change(&mut self, entry: &Entry) -> Result<()> { + trace!( + "{} enter finalize_membership_change(entry: {:?})", + self.id, + entry + ); + // TODO: Check if this should be rejected for normal reasons. + // Notably, if another is happening now. + + assert_eq!(entry.get_entry_type(), EntryType::EntryConfChange); + let conf_change = protobuf::parse_from_bytes::(entry.get_data())?; + assert_eq!( + conf_change.get_change_type(), + ConfChangeType::FinalizeConfChange + ); + // Joint Consensus, in the Raft paper, states the leader should step down and become a follower if it is removed during a transition. + let leader_in_new_set = self + .prs() + .next_configuration() + .as_ref() + .map(|config| config.contains(&self.leader_id)) + .ok_or_else(|| Error::NoPendingTransition)?; + if !leader_in_new_set { + let last_term = self.raft_log.last_term(); + if self.state == StateRole::Leader { + self.become_follower(last_term, INVALID_ID); + } else { + // It's no longer safe to lookup the ID in the ProgressSet, remove it. + self.leader_id = INVALID_ID; + } + } + + self.mut_prs().finalize_config_transition()?; + trace!( + "{} exit finalize_membership_change(entry: {:?})", + self.id, + entry + ); Ok(()) } @@ -1148,6 +1314,7 @@ impl Raft { send_append: &mut bool, more_to_send: &mut Option, ) { + trace!("{} enter handle_heartbeat_response(m: {:?}, prs: ..., send_append: {:?}, more_to_send: {:?}", self.id, m, send_append, more_to_send); // Update the node. Drop the value explicitly since we'll check the qourum after. { let pr = prs.get_mut(m.get_from()).unwrap(); @@ -1163,11 +1330,15 @@ impl Raft { } if self.read_only.option != ReadOnlyOption::Safe || m.get_context().is_empty() { + debug!("Early exit due to read_only being not Safe (is {:?}), or no context. (is {:?})", self.read_only.option, m.get_context()); + trace!("{} exit handle_heartbeat_response(m: {:?}, prs: ..., send_append: {:?}, more_to_send: {:?}", self.id, m, send_append, more_to_send); return; } } if !prs.has_quorum(&self.read_only.recv_ack(m)) { + debug!("Early exit due to !has_quorum"); + trace!("{} exit handle_heartbeat_response(m: {:?}, prs: ..., send_append: {:?}, more_to_send: {:?}", self.id, m, send_append, more_to_send); return; } @@ -1190,6 +1361,7 @@ impl Raft { *more_to_send = Some(to_send); } } + trace!("{} exit handle_heartbeat_response(m: {:?}, prs: ..., send_append: {:?}, more_to_send: {:?}", self.id, m, send_append, more_to_send); } fn handle_transfer_leader(&mut self, m: &Message, pr: &mut Progress) { @@ -1361,6 +1533,7 @@ impl Raft { for (i, e) in m.mut_entries().iter_mut().enumerate() { if e.get_entry_type() == EntryType::EntryConfChange { + // TODO: This could be very dangerous with set_nodes. if self.has_pending_conf() { info!( "propose conf {:?} ignored since pending unapplied \ @@ -1518,7 +1691,7 @@ impl Raft { self.term ); self.register_vote(from_id, acceptance); - match self.prs().candidacy_status(self.id, &self.votes) { + match self.prs().candidacy_status(&self.votes) { CandidacyStatus::Elected => { if self.state == StateRole::PreCandidate { self.campaign(CAMPAIGN_ELECTION); @@ -1644,6 +1817,7 @@ impl Raft { /// For a given message, append the entries to the log. pub fn handle_append_entries(&mut self, m: &Message) { if m.get_index() < self.raft_log.committed { + debug!("Got message with lower index than committed."); let mut to_send = Message::new(); to_send.set_to(m.get_from()); to_send.set_msg_type(MessageType::MsgAppendResponse); @@ -1808,7 +1982,7 @@ impl Raft { /// This method can be false positive. #[inline] pub fn has_pending_conf(&self) -> bool { - self.pending_conf_index > self.raft_log.applied + self.pending_conf_index > self.raft_log.applied || self.began_set_nodes_at.is_some() } /// Specifies if the commit should be broadcast. @@ -1822,6 +1996,62 @@ impl Raft { self.prs().voter_ids().contains(&self.id) } + /// Begin the process of changing the cluster's peer configuration to a new one. + /// + /// This should only be called on the leader. + /// + /// ```rust + /// use raft::{Raft, Config, storage::MemStorage, eraftpb::ConfState}; + /// let config = Config { + /// id: 1, + /// peers: vec![1], + /// ..Default::default() + /// }; + /// let mut raft: Raft = Raft::new(&config, Default::default()).unwrap(); + /// raft.become_candidate(); + /// raft.become_leader(); + /// + /// let mut conf = ConfState::default(); + /// conf.set_nodes(vec![1,2,3]); + /// conf.set_learners(vec![4]); + /// if let Err(e) = raft.propose_config_transition(&conf) { + /// panic!("{}", e); + /// } + /// ``` + /// + /// # Errors + /// + /// * `voters` and `learners` are not mutually exclusive. + /// * `voters` is empty. + pub fn propose_config_transition(&mut self, conf_state: &ConfState) -> Result<()> { + if self.state != StateRole::Leader { + Err(Error::InvalidState(self.state))?; + } + let config = Configuration::from(conf_state); + config.valid()?; + debug!( + "Replicating SetNodes with voters ({:?}), learners ({:?}).", + config.voters(), + config.learners() + ); + // Prep a configuration change to append. + let mut conf_change = ConfChange::new(); + conf_change.set_change_type(ConfChangeType::BeginConfChange); + conf_change.set_configuration((*conf_state).clone()); + let data = protobuf::Message::write_to_bytes(&conf_change)?; + let mut entry = Entry::new(); + entry.set_entry_type(EntryType::EntryConfChange); + entry.set_data(data); + let mut message = Message::new(); + message.set_msg_type(MessageType::MsgPropose); + message.set_from(self.id); + message.set_index(self.raft_log.last_index() + 1); + message.set_entries(RepeatedField::from_vec(vec![entry])); + // `append_entry` sets term, index for us. + self.step(message)?; + Ok(()) + } + fn add_voter_or_learner(&mut self, id: u64, learner: bool) -> Result<()> { debug!( "Adding node (learner: {}) with ID {} to peers.", @@ -1974,4 +2204,9 @@ impl Raft { pub fn abort_leader_transfer(&mut self) { self.lead_transferee = None; } + + /// Determine if the Raft is in a transition state under Joint Consensus. + pub fn is_in_transition(&self) -> bool { + self.prs().is_in_transition() + } } diff --git a/src/raft_log.rs b/src/raft_log.rs index facc04977..f521a18f6 100644 --- a/src/raft_log.rs +++ b/src/raft_log.rs @@ -260,6 +260,8 @@ impl RaftLog { /// # Panics /// /// Panics if the value passed in is not new or known. + #[deprecated = "Call raft::commit_apply(idx) instead. Joint Consensus requires an on-apply hook to + finalize a configuration change. This will become internal API in future versions."] pub fn applied_to(&mut self, idx: u64) { if idx == 0 { return; @@ -416,12 +418,25 @@ impl RaftLog { /// Attempts to commit the index and term and returns whether it did. pub fn maybe_commit(&mut self, max_index: u64, term: u64) -> bool { - if max_index > self.committed && self.term(max_index).unwrap_or(0) == term { + trace!( + "Enter maybe_commit(max_index: {}, term: {})", + max_index, + term + ); + let result = if max_index > self.committed && self.term(max_index).unwrap_or(0) == term { + debug!("Committing index {}", max_index); self.commit_to(max_index); true } else { false - } + }; + trace!( + "Exit maybe_commit(max_index: {}, term: {}) -> {}", + max_index, + term, + result + ); + result } /// Grabs a slice of entries from the raft. Unlike a rust slice pointer, these are @@ -667,6 +682,7 @@ mod test { "maybe_commit return false" ); let committed = raft_log.committed; + #[allow(deprecated)] raft_log.applied_to(committed); let offset = 500u64; raft_log.store.wl().compact(offset).expect("compact failed"); @@ -916,6 +932,7 @@ mod test { let mut raft_log = new_raft_log(store); raft_log.append(&ents); raft_log.maybe_commit(5, 1); + #[allow(deprecated)] raft_log.applied_to(applied); let next_entries = raft_log.next_entries(); @@ -940,6 +957,7 @@ mod test { let mut raft_log = new_raft_log(store); raft_log.append(&ents); raft_log.maybe_commit(5, 1); + #[allow(deprecated)] raft_log.applied_to(applied); let actual_has_next = raft_log.has_next_entries(); @@ -1318,6 +1336,7 @@ mod test { let mut raft_log = new_raft_log(store); raft_log.maybe_commit(last_index, 0); let committed = raft_log.committed; + #[allow(deprecated)] raft_log.applied_to(committed); for (j, idx) in compact.into_iter().enumerate() { diff --git a/src/raw_node.rs b/src/raw_node.rs index aaecaf5f4..1e7b7893c 100644 --- a/src/raw_node.rs +++ b/src/raw_node.rs @@ -285,7 +285,7 @@ impl RawNode { } fn commit_apply(&mut self, applied: u64) { - self.raft.raft_log.applied_to(applied); + self.raft.commit_apply(applied); } /// Tick advances the internal logical clock by a single tick. @@ -331,7 +331,9 @@ impl RawNode { /// Takes the conf change and applies it. pub fn apply_conf_change(&mut self, cc: &ConfChange) -> ConfState { - if cc.get_node_id() == INVALID_ID { + warn!("Got ConfChange"); + if cc.get_node_id() == INVALID_ID && cc.get_change_type() != ConfChangeType::BeginConfChange + { let mut cs = ConfState::new(); cs.set_nodes(self.raft.prs().voter_ids().iter().cloned().collect()); cs.set_learners(self.raft.prs().learner_ids().iter().cloned().collect()); @@ -342,6 +344,13 @@ impl RawNode { ConfChangeType::AddNode => self.raft.add_node(nid), ConfChangeType::AddLearnerNode => self.raft.add_learner(nid), ConfChangeType::RemoveNode => self.raft.remove_node(nid), + ConfChangeType::BeginConfChange => self + .raft + .propose_config_transition(cc.get_configuration()) + .unwrap(), + ConfChangeType::FinalizeConfChange => { + self.raft.mut_prs().finalize_config_transition().unwrap(); + } } let mut cs = ConfState::new(); cs.set_nodes(self.raft.prs().voter_ids().iter().cloned().collect()); @@ -407,6 +416,7 @@ impl RawNode { /// HasReady called when RawNode user need to check if any Ready pending. /// Checking logic in this method should be consistent with Ready.containsUpdates(). + #[inline] pub fn has_ready(&self) -> bool { self.has_ready_since(None) } @@ -421,7 +431,6 @@ impl RawNode { /// last Ready results. pub fn advance(&mut self, rd: Ready) { self.advance_append(rd); - let commit_idx = self.prev_hs.get_commit(); if commit_idx != 0 { // In most cases, prevHardSt and rd.HardState will be the same @@ -438,16 +447,19 @@ impl RawNode { } /// Appends and commits the ready value. + #[inline] pub fn advance_append(&mut self, rd: Ready) { self.commit_ready(rd); } /// Advance apply to the passed index. + #[inline] pub fn advance_apply(&mut self, applied: u64) { self.commit_apply(applied); } /// Status returns the current status of the given group. + #[inline] pub fn status(&self) -> Status { Status::new(&self.raft) } diff --git a/tests/integration_cases/mod.rs b/tests/integration_cases/mod.rs index 1bed08b3f..da64313c5 100644 --- a/tests/integration_cases/mod.rs +++ b/tests/integration_cases/mod.rs @@ -11,6 +11,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod test_membership_changes; mod test_raft; mod test_raft_flow_control; mod test_raft_paper; diff --git a/tests/integration_cases/test_membership_changes.rs b/tests/integration_cases/test_membership_changes.rs new file mode 100644 index 000000000..9da43e714 --- /dev/null +++ b/tests/integration_cases/test_membership_changes.rs @@ -0,0 +1,1352 @@ +// Copyright 2016 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. +// +// +use fxhash::{FxHashMap, FxHashSet}; +use protobuf::{self, RepeatedField}; +use raft::{ + eraftpb::{ConfChange, ConfChangeType, ConfState, Entry, EntryType, Message, MessageType}, + storage::MemStorage, + Config, Configuration, Raft, Result, INVALID_ID, NO_LIMIT, +}; +use std::ops::{Deref, DerefMut}; +use test_util::{new_message, setup_for_test, Network}; + +// Test that the API itself works. +// +// * Errors are returned from misuse. +// * Happy path returns happy values. +mod api { + use super::*; + // Test that the cluster can transition from a single node to a whole cluster. + #[test] + fn can_transition() -> Result<()> { + setup_for_test(); + let mut raft = Raft::new( + &Config { + id: 1, + tag: "1".into(), + peers: vec![1], + learners: vec![], + ..Default::default() + }, + MemStorage::new(), + )?; + let begin_entry = begin_entry(&[1, 2, 3], &[4], raft.raft_log.last_index() + 1); + raft.begin_membership_change(&begin_entry)?; + let finalize_entry = finalize_entry(raft.raft_log.last_index() + 1); + raft.finalize_membership_change(&finalize_entry)?; + Ok(()) + } + + // Test if the process rejects an overlapping voter and learner set. + #[test] + fn checks_for_overlapping_membership() -> Result<()> { + setup_for_test(); + let mut raft = Raft::new( + &Config { + id: 1, + tag: "1".into(), + peers: vec![1], + learners: vec![], + ..Default::default() + }, + MemStorage::new(), + )?; + let begin_entry = begin_entry(&[1, 2, 3], &[1, 2, 3], raft.raft_log.last_index() + 1); + assert!(raft.begin_membership_change(&begin_entry).is_err()); + Ok(()) + } + + // Test if the process rejects an voter demotion. + #[test] + fn checks_for_voter_demotion() -> Result<()> { + setup_for_test(); + let mut raft = Raft::new( + &Config { + id: 1, + tag: "1".into(), + peers: vec![1, 2, 3], + learners: vec![4], + ..Default::default() + }, + MemStorage::new(), + )?; + let begin_entry = begin_entry(&[1, 2], &[3, 4], raft.raft_log.last_index() + 1); + assert!(raft.begin_membership_change(&begin_entry).is_err()); + Ok(()) + } + + // Test if the process rejects an voter demotion. + #[test] + fn finalize_before_begin_fails_gracefully() -> Result<()> { + setup_for_test(); + let mut raft = Raft::new( + &Config { + id: 1, + tag: "1".into(), + peers: vec![1, 2, 3], + learners: vec![4], + ..Default::default() + }, + MemStorage::new(), + )?; + let finalize_entry = finalize_entry(raft.raft_log.last_index() + 1); + assert!(raft.finalize_membership_change(&finalize_entry).is_err()); + Ok(()) + } +} + +// Test that small cluster is able to progress through adding a voter. +mod three_peers_add_voter { + use super::*; + + /// In a steady state transition should proceed without issue. + #[test] + fn stable() -> Result<()> { + setup_for_test(); + let leader = 1; + let old_configuration = ([1, 2, 3], []); + let new_configuration = ([1, 2, 3, 4], []); + let mut scenario = Scenario::new( + leader, + (old_configuration.0.as_ref(), old_configuration.1.as_ref()), + (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + )?; + scenario.spawn_new_peers()?; + scenario.propose_change_message()?; + + info!("Allowing quorum to commit"); + scenario.expect_read_and_dispatch_messages_from(&[1, 2, 3])?; + + info!("Advancing leader, now entered the joint"); + scenario.assert_can_apply_transition_entry_at_index( + &[1], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1]); + + info!("Leader replicates the commit and finalize entry."); + scenario.expect_read_and_dispatch_messages_from(&[1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[2, 3], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2, 3]); + + info!("Allowing new peers to catch up."); + scenario.expect_read_and_dispatch_messages_from(&[4, 1, 4])?; + scenario.assert_can_apply_transition_entry_at_index( + &[4], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2, 3, 4]); + + info!("Cluster leaving the joint."); + scenario.expect_read_and_dispatch_messages_from(&[3, 2, 1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[1, 2, 3, 4], + 3, + ConfChangeType::FinalizeConfChange, + ); + scenario.assert_not_in_transition(&[1, 2, 3, 4]); + + Ok(()) + } +} + +// Test that small cluster is able to progress through adding a learner. +mod three_peers_add_learner { + use super::*; + + /// In a steady state transition should proceed without issue. + #[test] + fn stable() -> Result<()> { + setup_for_test(); + let leader = 1; + let old_configuration = ([1, 2, 3], []); + let new_configuration = ([1, 2, 3], [4]); + let mut scenario = Scenario::new( + leader, + (old_configuration.0.as_ref(), old_configuration.1.as_ref()), + (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + )?; + scenario.spawn_new_peers()?; + scenario.propose_change_message()?; + + info!("Allowing quorum to commit"); + scenario.expect_read_and_dispatch_messages_from(&[1, 2, 3])?; + + info!("Advancing leader, now entered the joint"); + scenario.assert_can_apply_transition_entry_at_index( + &[1], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1]); + + info!("Leader replicates the commit and finalize entry."); + scenario.expect_read_and_dispatch_messages_from(&[1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[2, 3], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2, 3]); + + info!("Allowing new peers to catch up."); + scenario.expect_read_and_dispatch_messages_from(&[4, 1, 4])?; + scenario.assert_can_apply_transition_entry_at_index( + &[4], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2, 3, 4]); + + info!("Cluster leaving the joint."); + scenario.expect_read_and_dispatch_messages_from(&[3, 2, 1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[1, 2, 3, 4], + 3, + ConfChangeType::FinalizeConfChange, + ); + scenario.assert_not_in_transition(&[1, 2, 3, 4]); + + Ok(()) + } +} + +// Test that small cluster is able to progress through removing a learner. +mod remove_learner { + use super::*; + + /// In a steady state transition should proceed without issue. + #[test] + fn stable() -> Result<()> { + setup_for_test(); + let leader = 1; + let old_configuration = ([1, 2, 3], [4]); + let new_configuration = ([1, 2, 3], []); + let mut scenario = Scenario::new( + leader, + (old_configuration.0.as_ref(), old_configuration.1.as_ref()), + (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + )?; + scenario.spawn_new_peers()?; + scenario.propose_change_message()?; + + info!("Allowing quorum to commit"); + scenario.expect_read_and_dispatch_messages_from(&[1, 2, 3, 4])?; + + info!("Advancing leader, now entered the joint"); + scenario.assert_can_apply_transition_entry_at_index( + &[1], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1]); + + info!("Leader replicates the commit and finalize entry."); + scenario.expect_read_and_dispatch_messages_from(&[1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[2, 3, 4], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2, 3]); + + info!("Cluster leaving the joint."); + scenario.expect_read_and_dispatch_messages_from(&[4, 3, 2, 1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[1, 2, 3, 4], + 3, + ConfChangeType::FinalizeConfChange, + ); + scenario.assert_not_in_transition(&[1, 2, 3, 4]); + + Ok(()) + } +} + +// Test that small cluster is able to progress through removing a voter. +mod remove_voter { + use super::*; + + /// In a steady state transition should proceed without issue. + #[test] + fn stable() -> Result<()> { + setup_for_test(); + let leader = 1; + let old_configuration = ([1, 2, 3], []); + let new_configuration = ([1, 2], []); + let mut scenario = Scenario::new( + leader, + (old_configuration.0.as_ref(), old_configuration.1.as_ref()), + (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + )?; + scenario.spawn_new_peers()?; + scenario.propose_change_message()?; + + info!("Allowing quorum to commit"); + scenario.expect_read_and_dispatch_messages_from(&[1, 2, 3])?; + + info!("Advancing leader, now entered the joint"); + scenario.assert_can_apply_transition_entry_at_index( + &[1], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1]); + + info!("Leader replicates the commit and finalize entry."); + scenario.expect_read_and_dispatch_messages_from(&[1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[2, 3], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2, 3]); + + info!("Cluster leaving the joint."); + scenario.expect_read_and_dispatch_messages_from(&[2, 1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[1, 2], + 3, + ConfChangeType::FinalizeConfChange, + ); + scenario.assert_not_in_transition(&[1, 2]); + + Ok(()) + } +} + +// Test that small cluster is able to progress through removing a leader. +mod remove_leader { + use super::*; + + /// In a steady state transition should proceed without issue. + #[test] + fn stable() -> Result<()> { + setup_for_test(); + let leader = 1; + let old_configuration = ([1, 2, 3], []); + let new_configuration = ([2, 3], []); + let mut scenario = Scenario::new( + leader, + (old_configuration.0.as_ref(), old_configuration.1.as_ref()), + (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + )?; + scenario.spawn_new_peers()?; + scenario.propose_change_message()?; + + info!("Allowing quorum to commit"); + scenario.expect_read_and_dispatch_messages_from(&[1, 2, 3])?; + + info!("Advancing leader, now entered the joint"); + scenario.assert_can_apply_transition_entry_at_index( + &[1], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1]); + + info!("Leader replicates the commit and finalize entry."); + scenario.expect_read_and_dispatch_messages_from(&[1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[2, 3], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2, 3]); + + info!("Cluster leaving the joint."); + scenario.expect_read_and_dispatch_messages_from(&[2, 3, 1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[1, 2, 3], + 3, + ConfChangeType::FinalizeConfChange, + ); + scenario.assert_not_in_transition(&[1, 2, 3]); + let peer_leaders = scenario.peer_leaders(); + for id in 1..=3 { + assert_eq!(peer_leaders[&id], INVALID_ID, "peer {}", id); + } + + info!("Prompting a new election."); + { + let new_leader = scenario.peers.get_mut(&2).unwrap(); + for _ in + new_leader.election_elapsed..=(new_leader.get_randomized_election_timeout() + 1) + { + new_leader.tick(); + } + } + let messages = scenario.read_messages(); + scenario.send(messages); + + info!("Verifying that all peers have the right peer group."); + for (_, peer) in scenario.peers.iter() { + assert_eq!( + peer.prs().configuration(), + &(new_configuration.0.as_ref(), new_configuration.1.as_ref()).into() + ); + } + + info!("Verifying that old leader cannot disrupt the cluster."); + { + let old_leader = scenario.peers.get_mut(&1).unwrap(); + for _ in old_leader.get_heartbeat_elapsed()..=(old_leader.get_heartbeat_timeout() + 1) { + old_leader.tick(); + } + } + let messages = scenario.read_messages(); + scenario.send(messages); + Ok(()) + } + + /// If the leader fails after the `Begin`, then recovers after the `Finalize`, the group should ignore it. + #[test] + fn leader_fails_and_recovers() -> Result<()> { + setup_for_test(); + let leader = 1; + let old_configuration = ([1, 2, 3], []); + let new_configuration = ([2, 3], []); + let mut scenario = Scenario::new( + leader, + (old_configuration.0.as_ref(), old_configuration.1.as_ref()), + (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + )?; + scenario.spawn_new_peers()?; + scenario.propose_change_message()?; + + info!("Allowing quorum to commit"); + scenario.expect_read_and_dispatch_messages_from(&[1, 2, 3])?; + + info!("Advancing leader, now entered the joint"); + scenario.assert_can_apply_transition_entry_at_index( + &[1], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1]); + + info!("Leader replicates the commit and finalize entry."); + scenario.expect_read_and_dispatch_messages_from(&[1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[2, 3], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2, 3]); + + info!("Cluster leaving the joint."); + scenario.expect_read_and_dispatch_messages_from(&[2, 3, 1])?; + + scenario.isolate(1); // Simulate the leader failing. + + scenario.assert_can_apply_transition_entry_at_index( + &[2, 3], + 3, + ConfChangeType::FinalizeConfChange, + ); + scenario.assert_not_in_transition(&[2, 3]); + + // At this point, 1 thinks it is a leader, but actually it isn't anymore. + + info!("Prompting a new election."); + { + let new_leader = scenario.peers.get_mut(&2).unwrap(); + for _ in + new_leader.election_elapsed..=(new_leader.get_randomized_election_timeout() + 1) + { + new_leader.tick(); + } + } + let messages = scenario.read_messages(); + scenario.send(messages); + + scenario.recover(); + // Here we note that the old leader (1) has NOT applied the finalize operation and thus thinks it is still leader. + // + // The Raft paper notes that a removed leader should not disrupt the cluster. + // It suggests doing this by ignoring any `RequestVote` when it has heard from the leader within the minimum election timeout. + + info!("Verifying that old leader cannot disrupt the cluster."); + { + let old_leader = scenario.peers.get_mut(&1).unwrap(); + for _ in old_leader.get_heartbeat_elapsed()..=(old_leader.get_heartbeat_timeout() + 1) { + old_leader.tick(); + } + } + let messages = scenario.read_messages(); + scenario.send(messages); + + let peer_leader = scenario.peer_leaders(); + assert!(peer_leader[&2] != 1); + assert!(peer_leader[&3] != 1); + Ok(()) + } +} + +// Test that small cluster is able to progress through replacing a voter. +mod three_peers_replace_voter { + use super::*; + + /// In a steady state transition should proceed without issue. + #[test] + fn stable() -> Result<()> { + setup_for_test(); + let leader = 1; + let old_configuration = ([1, 2, 3], []); + let new_configuration = ([1, 2, 4], []); + let mut scenario = Scenario::new( + leader, + (old_configuration.0.as_ref(), old_configuration.1.as_ref()), + (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + )?; + scenario.spawn_new_peers()?; + scenario.propose_change_message()?; + + info!("Allowing quorum to commit"); + scenario.expect_read_and_dispatch_messages_from(&[1, 2, 3])?; + + info!("Advancing leader, now entered the joint"); + scenario.assert_can_apply_transition_entry_at_index( + &[1], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1]); + + info!("Leader replicates the commit and finalize entry."); + scenario.expect_read_and_dispatch_messages_from(&[1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[2], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2]); + + info!("Allowing new peers to catch up."); + scenario.expect_read_and_dispatch_messages_from(&[4, 1, 4])?; + scenario.assert_can_apply_transition_entry_at_index( + &[4], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2, 4]); + + info!("Cluster leaving the joint."); + scenario.expect_read_and_dispatch_messages_from(&[2, 1, 4])?; + scenario.assert_can_apply_transition_entry_at_index( + &[1, 2, 4], + 3, + ConfChangeType::FinalizeConfChange, + ); + scenario.assert_not_in_transition(&[1, 2, 4]); + + Ok(()) + } + + // Ensure if a peer in the old quorum fails, but the quorum is still big enough, it's ok. + #[test] + fn pending_delete_fails_after_begin() -> Result<()> { + setup_for_test(); + let leader = 1; + let old_configuration = ([1, 2, 3], []); + let new_configuration = ([1, 2, 4], []); + let mut scenario = Scenario::new( + leader, + (old_configuration.0.as_ref(), old_configuration.1.as_ref()), + (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + )?; + scenario.spawn_new_peers()?; + scenario.propose_change_message()?; + + info!("Allowing quorum to commit"); + scenario.expect_read_and_dispatch_messages_from(&[1, 2, 3])?; + + info!("Advancing leader, now entered the joint"); + scenario.assert_can_apply_transition_entry_at_index( + &[1], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1]); + + scenario.isolate(3); // Take 3 down. + + info!("Leader replicates the commit and finalize entry."); + scenario.expect_read_and_dispatch_messages_from(&[1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[2], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2]); + + info!("Allowing new peers to catch up."); + scenario.expect_read_and_dispatch_messages_from(&[4, 1, 4])?; + scenario.assert_can_apply_transition_entry_at_index( + &[4], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2, 4]); + + info!("Cluster leaving the joint."); + scenario.expect_read_and_dispatch_messages_from(&[2, 1, 4])?; + scenario.assert_can_apply_transition_entry_at_index( + &[1, 2, 4], + 3, + ConfChangeType::FinalizeConfChange, + ); + scenario.assert_not_in_transition(&[1, 2, 4]); + + Ok(()) + } + + // Ensure if a peer in the new quorum fails, but the quorum is still big enough, it's ok. + #[test] + fn pending_create_with_quorum_fails_after_begin() -> Result<()> { + setup_for_test(); + let leader = 1; + let old_configuration = ([1, 2, 3], []); + let new_configuration = ([1, 2, 4], []); + let mut scenario = Scenario::new( + leader, + (old_configuration.0.as_ref(), old_configuration.1.as_ref()), + (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + )?; + scenario.spawn_new_peers()?; + scenario.propose_change_message()?; + + info!("Allowing quorum to commit"); + scenario.expect_read_and_dispatch_messages_from(&[1, 2, 3])?; + + info!("Advancing leader, now entered the joint"); + scenario.assert_can_apply_transition_entry_at_index( + &[1], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1]); + + scenario.isolate(4); // Take 4 down. + + info!("Leader replicates the commit and finalize entry."); + scenario.expect_read_and_dispatch_messages_from(&[1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[2, 3], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2, 3]); + + info!("Cluster leaving the joint."); + scenario.expect_read_and_dispatch_messages_from(&[2, 1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[1, 2, 3], + 3, + ConfChangeType::FinalizeConfChange, + ); + scenario.assert_not_in_transition(&[1, 2, 3]); + + Ok(()) + } + + // Ensure if the peer pending a deletion and the peer pending a creation both fail it's still ok (so long as both quorums hold). + #[test] + fn pending_create_and_destroy_both_fail() -> Result<()> { + setup_for_test(); + let leader = 1; + let old_configuration = ([1, 2, 3], []); + let new_configuration = ([1, 2, 4], []); + let mut scenario = Scenario::new( + leader, + (old_configuration.0.as_ref(), old_configuration.1.as_ref()), + (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + )?; + scenario.spawn_new_peers()?; + scenario.propose_change_message()?; + + info!("Allowing quorum to commit"); + scenario.expect_read_and_dispatch_messages_from(&[1, 2, 3])?; + + info!("Advancing leader, now entered the joint"); + scenario.assert_can_apply_transition_entry_at_index( + &[1], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1]); + + scenario.isolate(3); // Take 3 down. + scenario.isolate(4); // Take 4 down. + + info!("Leader replicates the commit and finalize entry."); + scenario.expect_read_and_dispatch_messages_from(&[1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[2], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2]); + + info!("Cluster leaving the joint."); + scenario.expect_read_and_dispatch_messages_from(&[2, 1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[1, 2], + 3, + ConfChangeType::FinalizeConfChange, + ); + scenario.assert_not_in_transition(&[1, 2]); + + Ok(()) + } + + // Ensure if the old quorum fails during the joint state progress will halt until the peer group is recovered. + #[test] + fn old_quorum_fails() -> Result<()> { + setup_for_test(); + let leader = 1; + let old_configuration = ([1, 2, 3], []); + let new_configuration = ([1, 2, 4], []); + let mut scenario = Scenario::new( + leader, + (old_configuration.0.as_ref(), old_configuration.1.as_ref()), + (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + )?; + scenario.spawn_new_peers()?; + scenario.propose_change_message()?; + + info!("Allowing quorum to commit"); + scenario.expect_read_and_dispatch_messages_from(&[1, 2, 3])?; + + info!("Advancing leader, now entered the joint"); + scenario.assert_can_apply_transition_entry_at_index( + &[1], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1]); + + info!("Old quorum fails."); + scenario.isolate(3); // Take 3 down. + scenario.isolate(2); // Take 2 down. + + info!("Leader replicates the commit and finalize entry."); + scenario.expect_read_and_dispatch_messages_from(&[1, 4, 1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[4], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 4]); + scenario.assert_not_in_transition(&[2, 3]); + + info!("Spinning for awhile to ensure nothing spectacular happens"); + for _ in scenario.peers[&leader].get_heartbeat_elapsed() + ..=scenario.peers[&leader].get_heartbeat_timeout() + { + scenario.peers.iter_mut().for_each(|(_, peer)| { + peer.tick(); + }); + let messages = scenario.read_messages(); + scenario.dispatch(messages)?; + } + + scenario.assert_in_transition(&[1, 4]); + scenario.assert_not_in_transition(&[2, 3]); + + info!("Recovering old qourum."); + scenario.recover(); + + for _ in scenario.peers[&leader].get_heartbeat_elapsed() + ..=scenario.peers[&leader].get_heartbeat_timeout() + { + scenario.peers.iter_mut().for_each(|(_, peer)| { + peer.tick(); + }); + } + + info!("Giving the peer group time to recover."); + scenario.expect_read_and_dispatch_messages_from(&[1, 2, 3, 4, 1, 2, 3, 1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[2, 3], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2, 3, 4]); + + info!("Failed peers confirming they have commited the begin."); + scenario.expect_read_and_dispatch_messages_from(&[2, 3])?; + + info!("Cluster leaving the joint."); + scenario.expect_read_and_dispatch_messages_from(&[1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[1, 2, 3, 4], + 3, + ConfChangeType::FinalizeConfChange, + ); + scenario.assert_not_in_transition(&[1, 2, 3, 4]); + + Ok(()) + } + + // Ensure if the new quorum fails during the joint state progress will halt until the peer group is recovered. + #[test] + fn new_quorum_fails() -> Result<()> { + setup_for_test(); + let leader = 1; + let old_configuration = ([1, 2, 3], []); + let new_configuration = ([1, 2, 4], []); + let mut scenario = Scenario::new( + leader, + (old_configuration.0.as_ref(), old_configuration.1.as_ref()), + (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + )?; + scenario.spawn_new_peers()?; + scenario.propose_change_message()?; + + info!("Allowing quorum to commit"); + scenario.expect_read_and_dispatch_messages_from(&[1, 2, 3])?; + + info!("Advancing leader, now entered the joint"); + scenario.assert_can_apply_transition_entry_at_index( + &[1], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1]); + + info!("New quorum fails."); + scenario.isolate(4); // Take 4 down. + scenario.isolate(2); // Take 2 down. + + info!("Leader replicates the commit and finalize entry."); + scenario.expect_read_and_dispatch_messages_from(&[1, 3])?; + + info!("Leader waits to let the new quorum apply this before progressing."); + scenario.assert_in_transition(&[1]); + scenario.assert_not_in_transition(&[2, 3, 4]); + + info!("Spinning for awhile to ensure nothing spectacular happens"); + for _ in scenario.peers[&leader].get_heartbeat_elapsed() + ..=scenario.peers[&leader].get_heartbeat_timeout() + { + scenario.peers.iter_mut().for_each(|(_, peer)| { + peer.tick(); + }); + let messages = scenario.read_messages(); + scenario.dispatch(messages)?; + } + + scenario.assert_in_transition(&[1]); + scenario.assert_not_in_transition(&[2, 3, 4]); + + info!("Recovering new qourum."); + scenario.recover(); + + for _ in scenario.peers[&leader].get_heartbeat_elapsed() + ..=scenario.peers[&leader].get_heartbeat_timeout() + { + scenario.peers.iter_mut().for_each(|(_, peer)| { + peer.tick(); + }); + } + + info!("Giving the peer group time to recover."); + scenario.expect_read_and_dispatch_messages_from(&[1, 2, 3, 4, 1, 2, 4, 1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[2, 3, 4], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2, 3, 4]); + + info!("Failed peers confirming they have commited the begin."); + scenario.expect_read_and_dispatch_messages_from(&[2, 4])?; + + info!("Cluster leaving the joint."); + scenario.expect_read_and_dispatch_messages_from(&[1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[1, 2, 3, 4], + 3, + ConfChangeType::FinalizeConfChange, + ); + scenario.assert_not_in_transition(&[1, 2, 3, 4]); + + Ok(()) + } +} + +// Test that small cluster is able to progress through adding a more with a learner. +mod three_peers_to_five_with_learner { + use super::*; + + /// In a steady state transition should proceed without issue. + #[test] + fn stable() -> Result<()> { + setup_for_test(); + let leader = 1; + let old_configuration = ([1, 2, 3], []); + let new_configuration = ([1, 2, 3, 4, 5], [6]); + let mut scenario = Scenario::new( + leader, + (old_configuration.0.as_ref(), old_configuration.1.as_ref()), + (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + )?; + scenario.spawn_new_peers()?; + scenario.propose_change_message()?; + + info!("Allowing quorum to commit"); + scenario.expect_read_and_dispatch_messages_from(&[1, 2, 3])?; + + info!("Advancing leader, now entered the joint"); + scenario.assert_can_apply_transition_entry_at_index( + &[1], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1]); + + info!("Leader replicates the commit and finalize entry."); + scenario.expect_read_and_dispatch_messages_from(&[1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[2, 3], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2, 3]); + + info!("Allowing new peers to catch up."); + scenario.expect_read_and_dispatch_messages_from(&[4, 5, 6, 1, 4, 5, 6])?; + scenario.assert_can_apply_transition_entry_at_index( + &[4, 5, 6], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2, 3, 4, 5, 6]); + + info!("Cluster leaving the joint."); + scenario.expect_read_and_dispatch_messages_from(&[3, 2, 1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[1, 2, 3, 4, 5, 6], + 3, + ConfChangeType::FinalizeConfChange, + ); + scenario.assert_not_in_transition(&[1, 2, 3, 4, 5, 6]); + + Ok(()) + } + + /// In this, a single node (of 3) halts during the transition. + #[test] + fn minority_old_followers_halt_at_start() -> Result<()> { + setup_for_test(); + let leader = 1; + let old_configuration = ([1, 2, 3], []); + let new_configuration = ([1, 2, 3, 4, 5], [6]); + let mut scenario = Scenario::new( + leader, + (old_configuration.0.as_ref(), old_configuration.1.as_ref()), + (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + )?; + scenario.spawn_new_peers()?; + scenario.isolate(3); + scenario.propose_change_message()?; + + info!("Allowing quorum to commit"); + scenario.expect_read_and_dispatch_messages_from(&[1, 2])?; + + info!("Advancing leader, now entered the joint"); + scenario.assert_can_apply_transition_entry_at_index( + &[1], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1]); + + info!("Leader replicates the commit and finalize entry."); + scenario.expect_read_and_dispatch_messages_from(&[1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[2], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2]); + scenario.assert_not_in_transition(&[3]); + + info!("Allowing new peers to catch up."); + scenario.expect_read_and_dispatch_messages_from(&[4, 5, 6, 1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[4, 5, 6], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2, 4, 5, 6]); + scenario.assert_not_in_transition(&[3]); + + scenario.expect_read_and_dispatch_messages_from(&[4, 5, 6])?; + + info!("Cluster leaving the joint."); + { + let leader = scenario.peers.get_mut(&1).unwrap(); + let ticks = leader.get_heartbeat_timeout(); + for _ in 0..=ticks { + leader.tick(); + } + } + scenario.expect_read_and_dispatch_messages_from(&[2, 1, 4, 5, 6, 1, 4, 5, 6, 1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[1, 2, 4, 5], + 3, + ConfChangeType::FinalizeConfChange, + ); + scenario.assert_not_in_transition(&[1, 2, 4, 5]); + scenario.assert_not_in_transition(&[3]); + + Ok(()) + } +} + +/// A test harness providing some useful utility and shorthand functions appropriate for this test suite. +/// +/// Since it derefs into `Network` it can be used the same way. So it acts as a transparent set of utilities over the standard `Network`. +/// The goal here is to boil down the test suite for Joint Consensus into the simplest terms possible, while allowing for control. +struct Scenario { + old_configuration: Configuration, + old_leader: u64, + new_configuration: Configuration, + network: Network, +} +impl Deref for Scenario { + type Target = Network; + fn deref(&self) -> &Network { + &self.network + } +} +impl DerefMut for Scenario { + fn deref_mut(&mut self) -> &mut Network { + &mut self.network + } +} + +// TODO: Explore moving some functionality to `Network`. +impl Scenario { + /// Create a new scenario with the given state. + fn new( + leader: u64, + old_configuration: impl Into, + new_configuration: impl Into, + ) -> Result { + let old_configuration = old_configuration.into(); + let new_configuration = new_configuration.into(); + info!( + "Beginning scenario, old: {:?}, new: {:?}", + old_configuration, new_configuration + ); + let starting_peers = old_configuration + .voters() + .iter() + .chain(old_configuration.learners().iter()) + .map(|&id| { + Some( + Raft::new( + &Config { + id, + peers: old_configuration.voters().iter().cloned().collect(), + learners: old_configuration.learners().iter().cloned().collect(), + ..Default::default() + }, + MemStorage::new(), + ).unwrap() + .into(), + ) + }).collect(); + let mut scenario = Scenario { + old_leader: leader, + old_configuration, + new_configuration, + network: Network::new(starting_peers), + }; + // Elect the leader. + info!("Sending MsgHup to predetermined leader ({})", leader); + let message = new_message(leader, leader, MessageType::MsgHup, 0); + scenario.send(vec![message]); + Ok(scenario) + } + + /// Creates any peers which are pending creation. + /// + /// This *only* creates the peers and adds them to the `Network`. It does not take other action. Newly created peers are only aware of the leader and themself. + fn spawn_new_peers(&mut self) -> Result<()> { + let storage = MemStorage::new(); + let new_peers = self.new_peers(); + info!("Creating new peers. {:?}", new_peers); + for &id in new_peers.voters() { + let raft = Raft::new( + &Config { + id, + peers: vec![self.old_leader, id], + learners: vec![], + ..Default::default() + }, + storage.clone(), + )?; + self.peers.insert(id, raft.into()); + } + for &id in new_peers.learners() { + let raft = Raft::new( + &Config { + id, + peers: vec![self.old_leader], + learners: vec![id], + ..Default::default() + }, + storage.clone(), + )?; + self.peers.insert(id, raft.into()); + } + Ok(()) + } + + /// Return the leader id according to each peer. + fn peer_leaders(&self) -> FxHashMap { + self.peers + .iter() + .map(|(&id, peer)| (id, peer.leader_id)) + .collect() + } + + /// Return a configuration containing only the peers pending creation. + fn new_peers(&self) -> Configuration { + let all_old = self + .old_configuration + .voters() + .union(&self.old_configuration.learners()) + .cloned() + .collect::>(); + Configuration::new( + self.new_configuration + .voters() + .difference(&all_old) + .cloned(), + self.new_configuration + .learners() + .difference(&all_old) + .cloned(), + ) + } + + /// Send the message which proposes the configuration change. + fn propose_change_message(&mut self) -> Result<()> { + info!( + "Proposing change message. Target: {:?}", + self.new_configuration + ); + let message = propose_change_message( + self.old_leader, + self.new_configuration.voters(), + self.new_configuration.learners(), + self.peers[&1].raft_log.last_index() + 1, + ); + self.dispatch(vec![message]) + } + + /// Checks that the given peers are not in a transition state. + fn assert_not_in_transition<'a>(&self, peers: impl IntoIterator) { + for peer in peers.into_iter().map(|id| &self.peers[id]) { + assert!( + !peer.is_in_transition(), + "Peer {} should not have been in transition.", + peer.id + ); + } + } + + // Checks that the given peers are in a transition state. + fn assert_in_transition<'a>(&self, peers: impl IntoIterator) { + for peer in peers.into_iter().map(|id| &self.peers[id]) { + assert!( + peer.is_in_transition(), + "Peer {} should have been in transition.", + peer.id + ); + } + } + + /// Reads the pending entries to be applied to a raft peer, checks one is of the expected variant, and applies it. Then, it advances the node to that point in the configuration change. + fn expect_apply_transition_entry<'a>( + &mut self, + peers: impl IntoIterator, + entry_type: ConfChangeType, + ) -> Result<()> { + for peer in peers { + debug!( + "Advancing peer {}, expecting a {:?} entry.", + peer, entry_type + ); + let peer = self.network.peers.get_mut(peer).unwrap(); + if let Some(entries) = peer.raft_log.next_entries() { + peer.mut_store().wl().append(&entries).unwrap(); + let mut found = false; + for entry in &entries { + if entry.get_entry_type() == EntryType::EntryConfChange { + let conf_change = + protobuf::parse_from_bytes::(entry.get_data())?; + if conf_change.get_change_type() == entry_type { + found = true; + if entry_type == ConfChangeType::BeginConfChange { + peer.begin_membership_change(&entry)?; + } else { + peer.finalize_membership_change(&entry)?; + } + } + } + if found { + peer.raft_log.stable_to(entry.get_index(), entry.get_term()); + peer.raft_log.commit_to(entry.get_index()); + peer.commit_apply(entry.get_index()); + peer.tick(); + break; + } + } + assert!( + found, + "{:?} message not found for peer {}. Got: {:?}", + entry_type, peer.id, entries + ); + } else { + panic!("Didn't have any entries {}", peer.id); + } + } + Ok(()) + } + + /// Reads messages from each peer in a given list, and dispatches their message before moving to the next peer. + /// + /// Expects each peer to have a message. If the message is not defintely sent use `read_and_dispatch_messages_from`. + fn expect_read_and_dispatch_messages_from<'a>( + &mut self, + peers: impl IntoIterator, + ) -> Result<()> { + let peers = peers.into_iter().cloned(); + for (step, peer) in peers.enumerate() { + info!( + "Expecting and dispatching messages from {} at step {}.", + peer, step + ); + let messages = self.peers.get_mut(&peer).unwrap().read_messages(); + trace!("{} sends messages: {:?}", peer, messages); + assert!( + !messages.is_empty(), + "Expected peer {} to have messages at step {}.", + peer, + step + ); + self.dispatch(messages)?; + } + Ok(()) + } + + // Verify there is a transition entry at the given index of the given variant. + fn assert_transition_entry_at<'a>( + &self, + peers: impl IntoIterator, + index: u64, + entry_type: ConfChangeType, + ) { + let peers = peers.into_iter().cloned(); + for peer in peers { + let entry = &self.peers[&peer] + .raft_log + .slice(index, index + 1, NO_LIMIT) + .unwrap()[0]; + assert_eq!(entry.get_entry_type(), EntryType::EntryConfChange); + let conf_change = protobuf::parse_from_bytes::(entry.get_data()).unwrap(); + assert_eq!(conf_change.get_change_type(), entry_type); + } + } + + fn assert_can_apply_transition_entry_at_index<'a>( + &mut self, + peers: impl IntoIterator, + index: u64, + entry_type: ConfChangeType, + ) { + let peers = peers.into_iter().collect::>(); + self.expect_apply_transition_entry(peers.clone(), entry_type) + .unwrap(); + self.assert_transition_entry_at(peers, index, entry_type) + } +} + +fn conf_state<'a>( + voters: impl IntoIterator, + learners: impl IntoIterator, +) -> ConfState { + let voters = voters.into_iter().cloned().collect::>(); + let learners = learners.into_iter().cloned().collect::>(); + let mut conf_state = ConfState::new(); + conf_state.set_nodes(voters); + conf_state.set_learners(learners); + conf_state +} + +fn begin_entry<'a>( + voters: impl IntoIterator, + learners: impl IntoIterator, + index: u64, +) -> Entry { + let conf_state = conf_state(voters, learners); + let mut conf_change = ConfChange::new(); + conf_change.set_change_type(ConfChangeType::BeginConfChange); + conf_change.set_configuration(conf_state); + let data = protobuf::Message::write_to_bytes(&conf_change).unwrap(); + let mut entry = Entry::new(); + entry.set_entry_type(EntryType::EntryConfChange); + entry.set_data(data); + entry.set_index(index); + entry +} + +fn finalize_entry(index: u64) -> Entry { + let mut conf_change = ConfChange::new(); + conf_change.set_change_type(ConfChangeType::FinalizeConfChange); + let data = protobuf::Message::write_to_bytes(&conf_change).unwrap(); + let mut entry = Entry::new(); + entry.set_entry_type(EntryType::EntryConfChange); + entry.set_index(index); + entry.set_data(data); + entry +} + +fn propose_change_message<'a>( + recipient: u64, + voters: impl IntoIterator, + learners: impl IntoIterator, + index: u64, +) -> Message { + let begin_entry = begin_entry(voters, learners, index); + let mut message = Message::new(); + message.set_to(recipient); + message.set_msg_type(MessageType::MsgPropose); + message.set_index(index); + message.set_entries(RepeatedField::from_vec(vec![begin_entry])); + message +} diff --git a/tests/integration_cases/test_raft.rs b/tests/integration_cases/test_raft.rs index 433cde1ef..cb69f0416 100644 --- a/tests/integration_cases/test_raft.rs +++ b/tests/integration_cases/test_raft.rs @@ -91,7 +91,7 @@ fn next_ents(r: &mut Raft, s: &MemStorage) -> Vec { r.raft_log.stable_to(last_idx, last_term); let ents = r.raft_log.next_entries(); let committed = r.raft_log.committed; - r.raft_log.applied_to(committed); + r.commit_apply(committed); ents.unwrap_or_else(Vec::new) } @@ -1128,7 +1128,7 @@ fn test_commit() { let mut sm = new_test_raft(1, vec![1], 5, 1, store); for (j, &v) in matches.iter().enumerate() { let id = j as u64 + 1; - if !sm.prs().get(id).is_some() { + if sm.prs().get(id).is_none() { sm.set_progress(id, v, v + 1, false); } } @@ -2717,7 +2717,7 @@ fn test_restore() { ); assert_eq!( sm.prs().voter_ids(), - &s.get_metadata() + s.get_metadata() .get_conf_state() .get_nodes() .iter() @@ -2932,7 +2932,7 @@ fn test_add_node() { r.add_node(2); assert_eq!( r.prs().voter_ids(), - &vec![1, 2].into_iter().collect::>() + vec![1, 2].into_iter().collect::>() ); } @@ -3009,7 +3009,7 @@ fn test_raft_nodes() { let r = new_test_raft(1, ids, 10, 1, new_storage()); let voter_ids = r.prs().voter_ids(); let wids = wids.into_iter().collect::>(); - if voter_ids != &wids { + if voter_ids != wids { panic!("#{}: nodes = {:?}, want {:?}", i, voter_ids, wids); } } @@ -3705,7 +3705,7 @@ fn test_learner_receive_snapshot() { n1.restore(s); let committed = n1.raft_log.committed; - n1.raft_log.applied_to(committed); + n1.commit_apply(committed); let mut network = Network::new(vec![Some(n1), Some(n2)]); diff --git a/tests/integration_cases/test_raft_paper.rs b/tests/integration_cases/test_raft_paper.rs index d5bd026bb..2f0403700 100644 --- a/tests/integration_cases/test_raft_paper.rs +++ b/tests/integration_cases/test_raft_paper.rs @@ -48,7 +48,7 @@ fn commit_noop_entry(r: &mut Interface, s: &MemStorage) { .append(r.raft_log.unstable_entries().unwrap_or(&[])) .expect(""); let committed = r.raft_log.committed; - r.raft_log.applied_to(committed); + r.commit_apply(committed); let (last_index, last_term) = (r.raft_log.last_index(), r.raft_log.last_term()); r.raft_log.stable_to(last_index, last_term); } diff --git a/tests/test_util/mod.rs b/tests/test_util/mod.rs index 17cbf4678..56d4ccbac 100644 --- a/tests/test_util/mod.rs +++ b/tests/test_util/mod.rs @@ -77,6 +77,12 @@ pub struct Interface { pub raft: Option>, } +impl From> for Interface { + fn from(value: Raft) -> Self { + Interface::new(value) + } +} + impl Interface { pub fn new(r: Raft) -> Interface { Interface { raft: Some(r) } @@ -281,8 +287,8 @@ impl Network { self.ignorem.insert(t, true); } - pub fn filter(&self, mut msgs: Vec) -> Vec { - msgs.drain(..) + pub fn filter(&self, msgs: impl IntoIterator) -> Vec { + msgs.into_iter() .filter(|m| { if self .ignorem @@ -305,6 +311,13 @@ impl Network { }).collect() } + pub fn read_messages(&mut self) -> Vec { + self.peers + .iter_mut() + .flat_map(|(_peer, progress)| progress.read_messages()) + .collect() + } + pub fn send(&mut self, msgs: Vec) { let mut msgs = msgs; while !msgs.is_empty() { @@ -321,6 +334,17 @@ impl Network { } } + /// Dispatches the given messages to the appropriate peers. + /// Unlike `send` this does not gather and send any responses. + pub fn dispatch(&mut self, messages: impl IntoIterator) -> Result<()> { + for message in self.filter(messages) { + let to = message.get_to(); + let peer = self.peers.get_mut(&to).unwrap(); + peer.step(message)?; + } + Ok(()) + } + pub fn drop(&mut self, from: u64, to: u64, perc: f64) { self.dropm.insert(Connem { from, to }, perc); } From 21068ff1621148177dd46c552d6791c58e0c0c21 Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Wed, 21 Nov 2018 06:46:07 -0800 Subject: [PATCH 02/41] Reduce trace messages --- src/raft.rs | 37 ++++--------------------------------- src/raft_log.rs | 16 ++-------------- 2 files changed, 6 insertions(+), 47 deletions(-) diff --git a/src/raft.rs b/src/raft.rs index d14d6e26d..6a3dbfe92 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -517,7 +517,7 @@ impl Raft { /// Sends RPC, with entries to the given peer. pub fn send_append(&mut self, to: u64, pr: &mut Progress) { if pr.is_paused() { - trace!("Skipping sending to {}, it's paused. {:?}", to, pr); + trace!("Skipping sending to {}, it's paused. Progress: {:?}", to, pr); return; } let term = self.raft_log.term(pr.next_idx - 1); @@ -526,7 +526,7 @@ impl Raft { m.set_to(to); if term.is_err() || ents.is_err() { // send snapshot if we failed to get term or entries - trace!("Skipping sending to {}, term or ents is_err()", to); + trace!("Skipping sending to {}, term: {:?}, ents: {:?}", to, term.is_err(), ents.is_err()); if !self.prepare_send_snapshot(&mut m, pr, to) { return; } @@ -593,7 +593,6 @@ impl Raft { /// /// Registers the new applied index to the Raft log, then checks to see if it's time to finalize a Joint Consensus state. pub fn commit_apply(&mut self, applied: u64) { - trace!("{:?}, Enter commit_apply(applied: {:?})", self.id, applied); #[allow(deprecated)] self.raft_log.applied_to(applied); @@ -610,7 +609,6 @@ impl Raft { } } } - trace!("Exit commit_apply(applied: {:?})", applied); } fn append_finalize_conf_change_entry(&mut self) { @@ -673,15 +671,12 @@ impl Raft { ///maybe_commit Returns true to indicate that there will probably be some readiness need to be handled. pub fn tick(&mut self) -> bool { - trace!("{} enter tick()", self.id); - let result = match self.state { + match self.state { StateRole::Follower | StateRole::PreCandidate | StateRole::Candidate => { self.tick_election() } StateRole::Leader => self.tick_heartbeat(), - }; - trace!("{} exit tick()", self.id); - result + } } // TODO: revoke pub when there is a better way to test. @@ -1116,11 +1111,6 @@ impl Raft { // TODO: Make this return a result instead of panic. #[inline(always)] pub fn begin_membership_change(&mut self, entry: &Entry) -> Result<()> { - trace!( - "{} enter begin_membership_change(entry: {:?})", - self.id, - entry - ); // TODO: Check if this should be rejected for normal reasons. // Notably, if another is happening now. assert_eq!(entry.get_entry_type(), EntryType::EntryConfChange); @@ -1134,11 +1124,6 @@ impl Raft { let max_inflights = self.max_inflight; self.mut_prs() .begin_config_transition(configuration, Progress::new(1, max_inflights))?; - trace!( - "{} exit begin_membership_change(entry: {:?})", - self.id, - entry - ); Ok(()) } @@ -1162,11 +1147,6 @@ impl Raft { // TODO: Make this return a result instead of panic. #[inline(always)] pub fn finalize_membership_change(&mut self, entry: &Entry) -> Result<()> { - trace!( - "{} enter finalize_membership_change(entry: {:?})", - self.id, - entry - ); // TODO: Check if this should be rejected for normal reasons. // Notably, if another is happening now. @@ -1194,11 +1174,6 @@ impl Raft { } self.mut_prs().finalize_config_transition()?; - trace!( - "{} exit finalize_membership_change(entry: {:?})", - self.id, - entry - ); Ok(()) } @@ -1314,7 +1289,6 @@ impl Raft { send_append: &mut bool, more_to_send: &mut Option, ) { - trace!("{} enter handle_heartbeat_response(m: {:?}, prs: ..., send_append: {:?}, more_to_send: {:?}", self.id, m, send_append, more_to_send); // Update the node. Drop the value explicitly since we'll check the qourum after. { let pr = prs.get_mut(m.get_from()).unwrap(); @@ -1331,14 +1305,12 @@ impl Raft { if self.read_only.option != ReadOnlyOption::Safe || m.get_context().is_empty() { debug!("Early exit due to read_only being not Safe (is {:?}), or no context. (is {:?})", self.read_only.option, m.get_context()); - trace!("{} exit handle_heartbeat_response(m: {:?}, prs: ..., send_append: {:?}, more_to_send: {:?}", self.id, m, send_append, more_to_send); return; } } if !prs.has_quorum(&self.read_only.recv_ack(m)) { debug!("Early exit due to !has_quorum"); - trace!("{} exit handle_heartbeat_response(m: {:?}, prs: ..., send_append: {:?}, more_to_send: {:?}", self.id, m, send_append, more_to_send); return; } @@ -1361,7 +1333,6 @@ impl Raft { *more_to_send = Some(to_send); } } - trace!("{} exit handle_heartbeat_response(m: {:?}, prs: ..., send_append: {:?}, more_to_send: {:?}", self.id, m, send_append, more_to_send); } fn handle_transfer_leader(&mut self, m: &Message, pr: &mut Progress) { diff --git a/src/raft_log.rs b/src/raft_log.rs index f521a18f6..887edb1c6 100644 --- a/src/raft_log.rs +++ b/src/raft_log.rs @@ -418,25 +418,13 @@ impl RaftLog { /// Attempts to commit the index and term and returns whether it did. pub fn maybe_commit(&mut self, max_index: u64, term: u64) -> bool { - trace!( - "Enter maybe_commit(max_index: {}, term: {})", - max_index, - term - ); - let result = if max_index > self.committed && self.term(max_index).unwrap_or(0) == term { + if max_index > self.committed && self.term(max_index).unwrap_or(0) == term { debug!("Committing index {}", max_index); self.commit_to(max_index); true } else { false - }; - trace!( - "Exit maybe_commit(max_index: {}, term: {}) -> {}", - max_index, - term, - result - ); - result + } } /// Grabs a slice of entries from the raft. Unlike a rust slice pointer, these are From 79df099a6150f30930c6ac0a413ed7c291653b51 Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Wed, 21 Nov 2018 18:11:48 -0800 Subject: [PATCH 03/41] Test and bugfix intermingled conf changes --- src/raft.rs | 56 ++++++--- .../test_membership_changes.rs | 111 +++++++++++++++++- tests/test_util/mod.rs | 6 +- 3 files changed, 154 insertions(+), 19 deletions(-) diff --git a/src/raft.rs b/src/raft.rs index 6a3dbfe92..019df4cbd 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -144,11 +144,19 @@ pub struct Raft { /// configuration change (if any). Config changes are only allowed to /// be proposed if the leader's applied index is greater than this /// value. + /// + /// This value is conservatively set in cases where there may be a configuration change pending, but scanning the log is possibly expensive. This implies that the index stated here may not necessarily be a config pub pending_conf_index: u64, /// The last BeginConfChange entry. Once we commit this entry we can exit the joint state. - //TODO: Ensure this is initialized like `pending_conf_index`, also attempt to merge them. - began_set_nodes_at: Option, + /// + /// This is different than `pending_conf_index` since it is more specific, and also exact. + /// While `pending_conf_index` is conservatively set at times to ensure safety in the one-by-one change method, in joint consensus based changes we track the state exactly. The index here **must** only be set when a `BeginConfChange` is present at that index. + /// + /// # Caveats + /// + /// It is important that whenever this is set that `pending_conf_index` is also set to the value if it is greater than the existing value. + began_conf_change_at: Option, /// The queue of read-only requests. pub read_only: ReadOnly, @@ -254,7 +262,7 @@ impl Raft { term: Default::default(), election_elapsed: Default::default(), pending_conf_index: Default::default(), - began_set_nodes_at: Default::default(), + began_conf_change_at: Default::default(), vote: Default::default(), heartbeat_elapsed: Default::default(), randomized_election_timeout: 0, @@ -386,6 +394,20 @@ impl Raft { self.skip_bcast_commit = skip; } + /// Set when the peer began a joint consensus change. This will also set `pending_conf_index` if it is larger than the existing number. + #[inline] + fn set_began_conf_change_at(&mut self, maybe_index: impl Into>) { + let maybe_index = maybe_index.into(); + if let Some(index) = maybe_index { + assert!(self.began_conf_change_at.is_none()); + if index > self.pending_conf_index { + self.pending_conf_index = index; + } + } + self.began_conf_change_at = maybe_index; + } + + // send persists state to stable storage and then sends to its mailbox. fn send(&mut self, mut m: Message) { debug!("Sending from {} to {}: {:?}", self.id, m.get_to(), m); @@ -597,12 +619,20 @@ impl Raft { self.raft_log.applied_to(applied); // Check to see if we need to finalize a Joint Consensus state now. - if let Some(index) = self.began_set_nodes_at { + if let Some(index) = self.began_conf_change_at { + // Invariant: We know that the index stored at `began_conf_change_at` should be a `BeginConfChange`. + // Check this in debug mode for safety while testing, but skip it in production since those bugs should have been caught. + debug_assert_eq!( + self.raft_log.entries(index, 1).ok().and_then(|vec| + vec.get(0).and_then(|entry| + protobuf::parse_from_bytes::(entry.get_data()).ok()) + ).map(|conf_change| + conf_change.get_change_type() + ), + Some(ConfChangeType::BeginConfChange), + ); // Invariant: We know that if we have commited past some index, we can also commit that index. if applied >= index { - // Ensure we reset this on *any* node, since the leader might have failed - // and we don't want to finalize twice. - self.began_set_nodes_at = None; if self.state == StateRole::Leader { // We must replicate the commit entry. self.append_finalize_conf_change_entry(); @@ -1120,7 +1150,7 @@ impl Raft { ConfChangeType::BeginConfChange ); let configuration = conf_change.get_configuration(); - self.began_set_nodes_at = Some(entry.get_index()); + self.set_began_conf_change_at(entry.get_index()); let max_inflights = self.max_inflight; self.mut_prs() .begin_config_transition(configuration, Progress::new(1, max_inflights))?; @@ -1144,12 +1174,8 @@ impl Raft { /// This **must** only be called on `Entry` which holds an `Entry` of with `EntryType::ConfChange` and the `data` field being a serialized `ConfChange`. /// /// This `ConfChange` must be of variant `FinalizeConfChange` and contain no `configuration` value. - // TODO: Make this return a result instead of panic. #[inline(always)] pub fn finalize_membership_change(&mut self, entry: &Entry) -> Result<()> { - // TODO: Check if this should be rejected for normal reasons. - // Notably, if another is happening now. - assert_eq!(entry.get_entry_type(), EntryType::EntryConfChange); let conf_change = protobuf::parse_from_bytes::(entry.get_data())?; assert_eq!( @@ -1174,6 +1200,9 @@ impl Raft { } self.mut_prs().finalize_config_transition()?; + // Ensure we reset this on *any* node, since the leader might have failed + // and we don't want to finalize twice. + self.set_began_conf_change_at(None); Ok(()) } @@ -1504,7 +1533,6 @@ impl Raft { for (i, e) in m.mut_entries().iter_mut().enumerate() { if e.get_entry_type() == EntryType::EntryConfChange { - // TODO: This could be very dangerous with set_nodes. if self.has_pending_conf() { info!( "propose conf {:?} ignored since pending unapplied \ @@ -1953,7 +1981,7 @@ impl Raft { /// This method can be false positive. #[inline] pub fn has_pending_conf(&self) -> bool { - self.pending_conf_index > self.raft_log.applied || self.began_set_nodes_at.is_some() + self.pending_conf_index > self.raft_log.applied || self.began_conf_change_at.is_some() } /// Specifies if the commit should be broadcast. diff --git a/tests/integration_cases/test_membership_changes.rs b/tests/integration_cases/test_membership_changes.rs index 9da43e714..229cfb27a 100644 --- a/tests/integration_cases/test_membership_changes.rs +++ b/tests/integration_cases/test_membership_changes.rs @@ -1024,6 +1024,70 @@ mod three_peers_to_five_with_learner { } } +mod intermingled_config_changes { + use super::*; + + // In this test, we make sure that if the peer group is sent a `BeginConfChange`, then immediately a `AddNode` entry, that the `AddNode` is rejected by the leader. + #[test] + fn begin_then_add_node() -> Result<()> { + setup_for_test(); + let leader = 1; + let old_configuration = ([1, 2, 3], []); + let new_configuration = ([1, 2, 3, 4], []); + let mut scenario = Scenario::new( + leader, + (old_configuration.0.as_ref(), old_configuration.1.as_ref()), + (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + )?; + scenario.spawn_new_peers()?; + scenario.propose_change_message()?; + + info!("Allowing quorum to commit"); + scenario.expect_read_and_dispatch_messages_from(&[1, 2, 3])?; + + info!("Advancing leader, now entered the joint"); + scenario.assert_can_apply_transition_entry_at_index( + &[1], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1]); + + info!("Leader recieves an add node proposal, which it rejects since it is already in transition."); + scenario.propose_add_node_message(4).is_err(); + assert_eq!(scenario.peers[&scenario.old_leader].raft_log.entries(4, 1).unwrap()[0].get_entry_type(), EntryType::EntryNormal); + + info!("Leader replicates the commit and finalize entry."); + scenario.expect_read_and_dispatch_messages_from(&[1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[2, 3], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2, 3]); + + info!("Allowing new peers to catch up."); + scenario.expect_read_and_dispatch_messages_from(&[4, 1, 4])?; + scenario.assert_can_apply_transition_entry_at_index( + &[4], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_transition(&[1, 2, 3, 4]); + + info!("Cluster leaving the joint."); + scenario.expect_read_and_dispatch_messages_from(&[3, 2, 1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[1, 2, 3, 4], + 3, + ConfChangeType::FinalizeConfChange, + ); + scenario.assert_not_in_transition(&[1, 2, 3, 4]); + + Ok(()) + } +} + /// A test harness providing some useful utility and shorthand functions appropriate for this test suite. /// /// Since it derefs into `Network` it can be used the same way. So it acts as a transparent set of utilities over the standard `Network`. @@ -1153,13 +1217,28 @@ impl Scenario { ) } + /// Send a message proposing a "one-by-one" style AddNode configuration. + /// If the peers are in the midst joint consensus style (Begin/FinalizeConfChange) change they should reject it. + fn propose_add_node_message(&mut self, id: u64) -> Result<()> { + info!( + "Proposing add_node message. Target: {:?}", + id, + ); + let message = build_propose_add_node_message( + self.old_leader, + id, + self.peers[&1].raft_log.last_index() + 1, + ); + self.dispatch(vec![message]) + } + /// Send the message which proposes the configuration change. fn propose_change_message(&mut self) -> Result<()> { info!( "Proposing change message. Target: {:?}", self.new_configuration ); - let message = propose_change_message( + let message = build_propose_change_message( self.old_leader, self.new_configuration.voters(), self.new_configuration.learners(), @@ -1213,8 +1292,10 @@ impl Scenario { found = true; if entry_type == ConfChangeType::BeginConfChange { peer.begin_membership_change(&entry)?; - } else { + } else if entry_type == ConfChangeType::FinalizeConfChange { peer.finalize_membership_change(&entry)?; + } else if entry_type == ConfChangeType::AddNode { + peer.add_node(conf_change.get_node_id()); } } } @@ -1336,7 +1417,7 @@ fn finalize_entry(index: u64) -> Entry { entry } -fn propose_change_message<'a>( +fn build_propose_change_message<'a>( recipient: u64, voters: impl IntoIterator, learners: impl IntoIterator, @@ -1350,3 +1431,27 @@ fn propose_change_message<'a>( message.set_entries(RepeatedField::from_vec(vec![begin_entry])); message } + +fn build_propose_add_node_message( + recipient: u64, + added_id: u64, + index: u64, +) -> Message { + let add_nodes_entry = { + let mut conf_change = ConfChange::new(); + conf_change.set_change_type(ConfChangeType::AddNode); + conf_change.set_node_id(added_id); + let data = protobuf::Message::write_to_bytes(&conf_change).unwrap(); + let mut entry = Entry::new(); + entry.set_entry_type(EntryType::EntryConfChange); + entry.set_data(data); + entry.set_index(index); + entry + }; + let mut message = Message::new(); + message.set_to(recipient); + message.set_msg_type(MessageType::MsgPropose); + message.set_index(index); + message.set_entries(RepeatedField::from_vec(vec![add_nodes_entry])); + message +} \ No newline at end of file diff --git a/tests/test_util/mod.rs b/tests/test_util/mod.rs index 56d4ccbac..089c99201 100644 --- a/tests/test_util/mod.rs +++ b/tests/test_util/mod.rs @@ -161,6 +161,7 @@ pub fn new_test_raft_with_prevote( ) -> Interface { let mut config = new_test_config(id, peers, election, heartbeat); config.pre_vote = pre_vote; + config.tag = format!("{}", id); new_test_raft_with_config(&config, storage) } @@ -335,12 +336,13 @@ impl Network { } /// Dispatches the given messages to the appropriate peers. - /// Unlike `send` this does not gather and send any responses. + /// Unlike `send` this does not gather and send any responses. It also does not ignore errors. pub fn dispatch(&mut self, messages: impl IntoIterator) -> Result<()> { for message in self.filter(messages) { let to = message.get_to(); let peer = self.peers.get_mut(&to).unwrap(); - peer.step(message)?; + let result = peer.step(message); + result?; } Ok(()) } From cd93df05cbfe05c1ac86f7d6d03b717855bafaf5 Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Thu, 22 Nov 2018 09:15:35 -0800 Subject: [PATCH 04/41] Harmonize function names, documentation example. --- src/lib.rs | 113 ++++++- src/progress.rs | 34 +- src/raft.rs | 54 ++-- src/raw_node.rs | 4 +- .../test_membership_changes.rs | 292 +++++++++--------- 5 files changed, 297 insertions(+), 200 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 560b52858..401b9bb16 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -29,7 +29,10 @@ ## Creating a Raft node -You can use [`RawNode::new`](raw_node/struct.RawNode.html#method.new) to create the Raft node. To create the Raft node, you need to provide a [`Storage`](storage/trait.Storage.html) component, and a [`Config`](struct.Config.html) to the [`RawNode::new`](raw_node/struct.RawNode.html#method.new) function. +You can use [`RawNode::new`](raw_node/struct.RawNode.html#method.new) to create the Raft node. To +create the Raft node, you need to provide a [`Storage`](storage/trait.Storage.html) component, and +a [`Config`](struct.Config.html) to the [`RawNode::new`](raw_node/struct.RawNode.html#method.new) +function. ```rust use raft::{ @@ -58,7 +61,9 @@ node.raft.become_leader(); ## Ticking the Raft node -Use a timer to tick the Raft node at regular intervals. See the following example using Rust channel `recv_timeout` to drive the Raft node at least every 100ms, calling [`tick()`](raw_node/struct.RawNode.html#method.tick) each time. +Use a timer to tick the Raft node at regular intervals. See the following example using Rust +channel `recv_timeout` to drive the Raft node at least every 100ms, calling +[`tick()`](raw_node/struct.RawNode.html#method.tick) each time. ```rust # use raft::{Config, storage::MemStorage, raw_node::RawNode}; @@ -101,11 +106,17 @@ loop { ## Proposing to, and stepping the Raft node -Using the `propose` function you can drive the Raft node when the client sends a request to the Raft server. You can call `propose` to add the request to the Raft log explicitly. +Using the `propose` function you can drive the Raft node when the client sends a request to the +Raft server. You can call `propose` to add the request to the Raft log explicitly. -In most cases, the client needs to wait for a response for the request. For example, if the client writes a value to a key and wants to know whether the write succeeds or not, but the write flow is asynchronous in Raft, so the write log entry must be replicated to other followers, then committed and at last applied to the state machine, so here we need a way to notify the client after the write is finished. +In most cases, the client needs to wait for a response for the request. For example, if the +client writes a value to a key and wants to know whether the write succeeds or not, but the +write flow is asynchronous in Raft, so the write log entry must be replicated to other followers, +then committed and at last applied to the state machine, so here we need a way to notify the client +after the write is finished. -One simple way is to use a unique ID for the client request, and save the associated callback function in a hash map. When the log entry is applied, we can get the ID from the decoded entry, call the corresponding callback, and notify the client. +One simple way is to use a unique ID for the client request, and save the associated callback +function in a hash map. When the log entry is applied, we can get the ID from the decoded entry, call the corresponding callback, and notify the client. You can call the `step` function when you receive the Raft messages from other nodes. @@ -165,11 +176,15 @@ loop { } ``` -In the above example, we use a channel to receive the `propose` and `step` messages. We only propose the request ID to the Raft log. In your own practice, you can embed the ID in your request and propose the encoded binary request data. +In the above example, we use a channel to receive the `propose` and `step` messages. We only +propose the request ID to the Raft log. In your own practice, you can embed the ID in your request +and propose the encoded binary request data. ## Processing the `Ready` State -When your Raft node is ticked and running, Raft should enter a `Ready` state. You need to first use `has_ready` to check whether Raft is ready. If yes, use the `ready` function to get a `Ready` state: +When your Raft node is ticked and running, Raft should enter a `Ready` state. You need to first use +`has_ready` to check whether Raft is ready. If yes, use the `ready` function to get a `Ready` +state: ```rust,ignore if !node.has_ready() { @@ -180,9 +195,11 @@ if !node.has_ready() { let mut ready = node.ready(); ``` -The `Ready` state contains quite a bit of information, and you need to check and process them one by one: +The `Ready` state contains quite a bit of information, and you need to check and process them one +by one: -1. Check whether `snapshot` is empty or not. If not empty, it means that the Raft node has received a Raft snapshot from the leader and we must apply the snapshot: +1. Check whether `snapshot` is empty or not. If not empty, it means that the Raft node has received +a Raft snapshot from the leader and we must apply the snapshot: ```rust,ignore if !raft::is_empty_snap(ready.snapshot()) { @@ -195,7 +212,8 @@ The `Ready` state contains quite a bit of information, and you need to check and ``` -2. Check whether `entries` is empty or not. If not empty, it means that there are newly added entries but has not been committed yet, we must append the entries to the Raft log: +2. Check whether `entries` is empty or not. If not empty, it means that there are newly added +entries but has not been committed yet, we must append the entries to the Raft log: ```rust,ignore if !ready.entries.is_empty() { @@ -205,7 +223,8 @@ The `Ready` state contains quite a bit of information, and you need to check and ``` -3. Check whether `hs` is empty or not. If not empty, it means that the `HardState` of the node has changed. For example, the node may vote for a new leader, or the commit index has been increased. We must persist the changed `HardState`: +3. Check whether `hs` is empty or not. If not empty, it means that the `HardState` of the node has +changed. For example, the node may vote for a new leader, or the commit index has been increased. We must persist the changed `HardState`: ```rust,ignore if let Some(hs) = ready.hs() { @@ -214,7 +233,10 @@ The `Ready` state contains quite a bit of information, and you need to check and } ``` -4. Check whether `messages` is empty or not. If not, it means that the node will send messages to other nodes. There has been an optimization for sending messages: if the node is a leader, this can be done together with step 1 in parallel; if the node is not a leader, it needs to reply the messages to the leader after appending the Raft entries: +4. Check whether `messages` is empty or not. If not, it means that the node will send messages to +other nodes. There has been an optimization for sending messages: if the node is a leader, this can +be done together with step 1 in parallel; if the node is not a leader, it needs to reply the +messages to the leader after appending the Raft entries: ```rust,ignore if !is_leader { @@ -227,7 +249,9 @@ The `Ready` state contains quite a bit of information, and you need to check and } ``` -5. Check whether `committed_entires` is empty or not. If not, it means that there are some newly committed log entries which you must apply to the state machine. Of course, after applying, you need to update the applied index and resume `apply` later: +5. Check whether `committed_entires` is empty or not. If not, it means that there are some newly +committed log entries which you must apply to the state machine. Of course, after applying, you +need to update the applied index and resume `apply` later: ```rust,ignore if let Some(committed_entries) = ready.committed_entries.take() { @@ -258,6 +282,69 @@ The `Ready` state contains quite a bit of information, and you need to check and For more information, check out an [example](examples/single_mem_node/main.rs#L113-L179). +## Membership Changes + +When building a resilient, scalable distributed system there is a strong need to be able to change +the membership of a peer group *dynamically, without downtime.* This Raft crate supports this via +**Joint Consensus** +([Raft paper, section 6](https://web.stanford.edu/~ouster/cgi-bin/papers/raft-atc14)). + +It permits resilient arbitrary dynamic membership changes. A membership change can do any or all of +the following: + +* Add peer (learner or voter) *n* to the group. +* Remove peer *n* from the group. +* Remove a leader (unmanaged, via stepdown) +* Promote a learner to a voter. + +It (currently) does not: + +* Allow control of the replacement leader during a stepdown. +* Optionally roll back a change during a peer group pause where the new peer group configuration +fails. + +This means it's possible to do: + +```rust +use raft::{Config, storage::MemStorage, raw_node::RawNode, eraftpb::Message}; +let config = Config { id: 1, peers: vec![1], ..Default::default() }; +let mut node = RawNode::new(&config, MemStorage::default(), vec![]).unwrap(); +node.raft.become_candidate(); +node.raft.become_leader(); + +// Call this on the leader, or send the command via a normal `MsgPropose`. +node.raft.propose_membership_change(( + vec![1,2,3], // Any IntoIterator. + vec![4,5,6], +)).unwrap(); + +// ...Later when the begin entry is ready to apply: +let entry = &node.raft.raft_log.entries(2, 1).unwrap()[0]; +node.raft.begin_membership_change(entry).unwrap(); +assert!(node.raft.is_in_membership_change()); +# +# // We hide this since the user isn't really encouraged to blindly call this, but we'd like a short +# // example. +# node.raft.commit_apply(2); +# +// ...Later, when the finalize entry is ready to apply: +let entry = &node.raft.raft_log.entries(3, 1).unwrap()[0]; +node.raft.finalize_membership_change(entry).unwrap(); +assert!(node.raft.prs().voter_ids().contains(&2)); +assert!(!node.raft.is_in_membership_change()); +``` + + + +PRs to enable these are welcome! We'd love to mentor/support you through implementing it. + +This process is a two-phase process, during the midst of it the peer group's leader is managing +**two independent, possibly overlapping peer sets**. + +> In order to maintain resiliency gaurantees (progress while a majority of both peer sets is +active), it is very important to wait until the entire peer group has exited the transition phase +before taking old, removed peers offline. + */ #![cfg_attr(not(feature = "cargo-clippy"), allow(unknown_lints))] diff --git a/src/progress.rs b/src/progress.rs index 126477899..2d3faa24b 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -79,20 +79,20 @@ impl Configuration { } } -impl<'a, I> From<(I, I)> for Configuration +impl From<(I, I)> for Configuration where - I: IntoIterator, + I: IntoIterator, { fn from((voters, learners): (I, I)) -> Self { Self { - voters: voters.into_iter().cloned().collect(), - learners: learners.into_iter().cloned().collect(), + voters: voters.into_iter().collect(), + learners: learners.into_iter().collect(), } } } -impl<'a> From<&'a ConfState> for Configuration { - fn from(conf_state: &'a ConfState) -> Self { +impl From for Configuration { + fn from(conf_state: ConfState) -> Self {; Self { voters: conf_state.get_nodes().iter().cloned().collect(), learners: conf_state.get_learners().iter().cloned().collect(), @@ -515,7 +515,7 @@ impl ProgressSet { /// Determine if the ProgressSet is represented by a transition state under Joint Consensus. #[inline] - pub fn is_in_transition(&self) -> bool { + pub fn is_in_membership_change(&self) -> bool { self.next_configuration.is_some() } @@ -539,7 +539,7 @@ impl ProgressSet { /// * Voter -> Learner /// * Member as voter and learner. /// * Empty voter set. - pub fn begin_config_transition( + pub fn begin_membership_change( &mut self, next: impl Into, mut progress: Progress, @@ -557,9 +557,9 @@ impl ProgressSet { } debug!("Beginning member configuration transition. End state will be voters {:?}, Learners: {:?}", next.voters, next.learners); - // When a node is first added/promoted, we should mark it as recently active. + // When a peer is first added/promoted, we should mark it as recently active. // Otherwise, check_quorum may cause us to step down if it is invoked - // before the added node has a chance to commuicate with us. + // before the added peer has a chance to communicate with us. progress.recent_active = true; progress.paused = false; for id in next.voters.iter().chain(&next.learners) { @@ -572,9 +572,9 @@ impl ProgressSet { /// Finalizes the joint consensus state and transitions solely to the new state. /// - /// This should be called only after calling `begin_config_transition` and the the majority - /// of nodes in both the `current` and the `next` state have commited the changes. - pub fn finalize_config_transition(&mut self) -> Result<(), Error> { + /// This should be called only after calling `begin_membership_change` and the the majority + /// of peers in both the `current` and the `next` state have commited the changes. + pub fn finalize_membership_change(&mut self) -> Result<(), Error> { let next = self.next_configuration.take(); match next { None => Err(Error::NoPendingTransition)?, @@ -1199,11 +1199,11 @@ mod test_progress_set { for starter in start_learners { set.insert_learner(starter, default_progress.clone())?; } - set.begin_config_transition( + set.begin_membership_change( Configuration::new(end_voters.clone(), end_learners.clone()), default_progress, )?; - assert!(set.is_in_transition()); + assert!(set.is_in_membership_change()); assert_eq!( set.voter_ids(), transition_voters, @@ -1215,8 +1215,8 @@ mod test_progress_set { "Transition state learners inaccurate." ); - set.finalize_config_transition()?; - assert!(!set.is_in_transition()); + set.finalize_membership_change()?; + assert!(!set.is_in_membership_change()); assert_eq!(set.voter_ids(), end_voters, "End state voters inaccurate"); assert_eq!( set.learner_ids(), diff --git a/src/raft.rs b/src/raft.rs index 019df4cbd..609063291 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -144,7 +144,7 @@ pub struct Raft { /// configuration change (if any). Config changes are only allowed to /// be proposed if the leader's applied index is greater than this /// value. - /// + /// /// This value is conservatively set in cases where there may be a configuration change pending, but scanning the log is possibly expensive. This implies that the index stated here may not necessarily be a config pub pending_conf_index: u64, @@ -152,9 +152,9 @@ pub struct Raft { /// /// This is different than `pending_conf_index` since it is more specific, and also exact. /// While `pending_conf_index` is conservatively set at times to ensure safety in the one-by-one change method, in joint consensus based changes we track the state exactly. The index here **must** only be set when a `BeginConfChange` is present at that index. - /// + /// /// # Caveats - /// + /// /// It is important that whenever this is set that `pending_conf_index` is also set to the value if it is greater than the existing value. began_conf_change_at: Option, @@ -407,7 +407,6 @@ impl Raft { self.began_conf_change_at = maybe_index; } - // send persists state to stable storage and then sends to its mailbox. fn send(&mut self, mut m: Message) { debug!("Sending from {} to {}: {:?}", self.id, m.get_to(), m); @@ -539,7 +538,11 @@ impl Raft { /// Sends RPC, with entries to the given peer. pub fn send_append(&mut self, to: u64, pr: &mut Progress) { if pr.is_paused() { - trace!("Skipping sending to {}, it's paused. Progress: {:?}", to, pr); + trace!( + "Skipping sending to {}, it's paused. Progress: {:?}", + to, + pr + ); return; } let term = self.raft_log.term(pr.next_idx - 1); @@ -548,7 +551,12 @@ impl Raft { m.set_to(to); if term.is_err() || ents.is_err() { // send snapshot if we failed to get term or entries - trace!("Skipping sending to {}, term: {:?}, ents: {:?}", to, term.is_err(), ents.is_err()); + trace!( + "Skipping sending to {}, term: {:?}, ents: {:?}", + to, + term.is_err(), + ents.is_err() + ); if !self.prepare_send_snapshot(&mut m, pr, to) { return; } @@ -623,12 +631,14 @@ impl Raft { // Invariant: We know that the index stored at `began_conf_change_at` should be a `BeginConfChange`. // Check this in debug mode for safety while testing, but skip it in production since those bugs should have been caught. debug_assert_eq!( - self.raft_log.entries(index, 1).ok().and_then(|vec| - vec.get(0).and_then(|entry| - protobuf::parse_from_bytes::(entry.get_data()).ok()) - ).map(|conf_change| - conf_change.get_change_type() - ), + self.raft_log + .entries(index, 1) + .ok() + .and_then( + |vec| vec.get(0).and_then(|entry| { + protobuf::parse_from_bytes::(entry.get_data()).ok() + }) + ).map(|conf_change| conf_change.get_change_type()), Some(ConfChangeType::BeginConfChange), ); // Invariant: We know that if we have commited past some index, we can also commit that index. @@ -1144,16 +1154,16 @@ impl Raft { // TODO: Check if this should be rejected for normal reasons. // Notably, if another is happening now. assert_eq!(entry.get_entry_type(), EntryType::EntryConfChange); - let conf_change = protobuf::parse_from_bytes::(entry.get_data())?; + let mut conf_change = protobuf::parse_from_bytes::(entry.get_data())?; assert_eq!( conf_change.get_change_type(), ConfChangeType::BeginConfChange ); - let configuration = conf_change.get_configuration(); + let configuration = conf_change.take_configuration(); self.set_began_conf_change_at(entry.get_index()); let max_inflights = self.max_inflight; self.mut_prs() - .begin_config_transition(configuration, Progress::new(1, max_inflights))?; + .begin_membership_change(configuration, Progress::new(1, max_inflights))?; Ok(()) } @@ -1199,7 +1209,7 @@ impl Raft { } } - self.mut_prs().finalize_config_transition()?; + self.mut_prs().finalize_membership_change()?; // Ensure we reset this on *any* node, since the leader might have failed // and we don't want to finalize twice. self.set_began_conf_change_at(None); @@ -2013,7 +2023,7 @@ impl Raft { /// let mut conf = ConfState::default(); /// conf.set_nodes(vec![1,2,3]); /// conf.set_learners(vec![4]); - /// if let Err(e) = raft.propose_config_transition(&conf) { + /// if let Err(e) = raft.propose_membership_change(conf) { /// panic!("{}", e); /// } /// ``` @@ -2022,11 +2032,11 @@ impl Raft { /// /// * `voters` and `learners` are not mutually exclusive. /// * `voters` is empty. - pub fn propose_config_transition(&mut self, conf_state: &ConfState) -> Result<()> { + pub fn propose_membership_change(&mut self, config: impl Into) -> Result<()> { if self.state != StateRole::Leader { Err(Error::InvalidState(self.state))?; } - let config = Configuration::from(conf_state); + let config = config.into(); config.valid()?; debug!( "Replicating SetNodes with voters ({:?}), learners ({:?}).", @@ -2036,7 +2046,7 @@ impl Raft { // Prep a configuration change to append. let mut conf_change = ConfChange::new(); conf_change.set_change_type(ConfChangeType::BeginConfChange); - conf_change.set_configuration((*conf_state).clone()); + conf_change.set_configuration(config.into()); let data = protobuf::Message::write_to_bytes(&conf_change)?; let mut entry = Entry::new(); entry.set_entry_type(EntryType::EntryConfChange); @@ -2205,7 +2215,7 @@ impl Raft { } /// Determine if the Raft is in a transition state under Joint Consensus. - pub fn is_in_transition(&self) -> bool { - self.prs().is_in_transition() + pub fn is_in_membership_change(&self) -> bool { + self.prs().is_in_membership_change() } } diff --git a/src/raw_node.rs b/src/raw_node.rs index 1e7b7893c..5629f198a 100644 --- a/src/raw_node.rs +++ b/src/raw_node.rs @@ -346,10 +346,10 @@ impl RawNode { ConfChangeType::RemoveNode => self.raft.remove_node(nid), ConfChangeType::BeginConfChange => self .raft - .propose_config_transition(cc.get_configuration()) + .propose_membership_change(cc.get_configuration().clone()) .unwrap(), ConfChangeType::FinalizeConfChange => { - self.raft.mut_prs().finalize_config_transition().unwrap(); + self.raft.mut_prs().finalize_membership_change().unwrap(); } } let mut cs = ConfState::new(); diff --git a/tests/integration_cases/test_membership_changes.rs b/tests/integration_cases/test_membership_changes.rs index 229cfb27a..b174f996a 100644 --- a/tests/integration_cases/test_membership_changes.rs +++ b/tests/integration_cases/test_membership_changes.rs @@ -116,12 +116,12 @@ mod three_peers_add_voter { fn stable() -> Result<()> { setup_for_test(); let leader = 1; - let old_configuration = ([1, 2, 3], []); - let new_configuration = ([1, 2, 3, 4], []); + let old_configuration = (vec![1, 2, 3], vec![]); + let new_configuration = (vec![1, 2, 3, 4], vec![]); let mut scenario = Scenario::new( leader, - (old_configuration.0.as_ref(), old_configuration.1.as_ref()), - (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + old_configuration, + new_configuration, )?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -135,7 +135,7 @@ mod three_peers_add_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1]); + scenario.assert_in_membership_change(&[1]); info!("Leader replicates the commit and finalize entry."); scenario.expect_read_and_dispatch_messages_from(&[1])?; @@ -144,7 +144,7 @@ mod three_peers_add_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2, 3]); + scenario.assert_in_membership_change(&[1, 2, 3]); info!("Allowing new peers to catch up."); scenario.expect_read_and_dispatch_messages_from(&[4, 1, 4])?; @@ -153,7 +153,7 @@ mod three_peers_add_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2, 3, 4]); + scenario.assert_in_membership_change(&[1, 2, 3, 4]); info!("Cluster leaving the joint."); scenario.expect_read_and_dispatch_messages_from(&[3, 2, 1])?; @@ -162,7 +162,7 @@ mod three_peers_add_voter { 3, ConfChangeType::FinalizeConfChange, ); - scenario.assert_not_in_transition(&[1, 2, 3, 4]); + scenario.assert_not_in_membership_change(&[1, 2, 3, 4]); Ok(()) } @@ -177,12 +177,12 @@ mod three_peers_add_learner { fn stable() -> Result<()> { setup_for_test(); let leader = 1; - let old_configuration = ([1, 2, 3], []); - let new_configuration = ([1, 2, 3], [4]); + let old_configuration = (vec![1, 2, 3], vec![]); + let new_configuration = (vec![1, 2, 3], vec![4]); let mut scenario = Scenario::new( leader, - (old_configuration.0.as_ref(), old_configuration.1.as_ref()), - (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + old_configuration, + new_configuration, )?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -196,7 +196,7 @@ mod three_peers_add_learner { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1]); + scenario.assert_in_membership_change(&[1]); info!("Leader replicates the commit and finalize entry."); scenario.expect_read_and_dispatch_messages_from(&[1])?; @@ -205,7 +205,7 @@ mod three_peers_add_learner { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2, 3]); + scenario.assert_in_membership_change(&[1, 2, 3]); info!("Allowing new peers to catch up."); scenario.expect_read_and_dispatch_messages_from(&[4, 1, 4])?; @@ -214,7 +214,7 @@ mod three_peers_add_learner { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2, 3, 4]); + scenario.assert_in_membership_change(&[1, 2, 3, 4]); info!("Cluster leaving the joint."); scenario.expect_read_and_dispatch_messages_from(&[3, 2, 1])?; @@ -223,7 +223,7 @@ mod three_peers_add_learner { 3, ConfChangeType::FinalizeConfChange, ); - scenario.assert_not_in_transition(&[1, 2, 3, 4]); + scenario.assert_not_in_membership_change(&[1, 2, 3, 4]); Ok(()) } @@ -238,12 +238,12 @@ mod remove_learner { fn stable() -> Result<()> { setup_for_test(); let leader = 1; - let old_configuration = ([1, 2, 3], [4]); - let new_configuration = ([1, 2, 3], []); + let old_configuration = (vec![1, 2, 3], vec![4]); + let new_configuration = (vec![1, 2, 3], vec![]); let mut scenario = Scenario::new( leader, - (old_configuration.0.as_ref(), old_configuration.1.as_ref()), - (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + old_configuration, + new_configuration, )?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -257,7 +257,7 @@ mod remove_learner { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1]); + scenario.assert_in_membership_change(&[1]); info!("Leader replicates the commit and finalize entry."); scenario.expect_read_and_dispatch_messages_from(&[1])?; @@ -266,7 +266,7 @@ mod remove_learner { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2, 3]); + scenario.assert_in_membership_change(&[1, 2, 3]); info!("Cluster leaving the joint."); scenario.expect_read_and_dispatch_messages_from(&[4, 3, 2, 1])?; @@ -275,7 +275,7 @@ mod remove_learner { 3, ConfChangeType::FinalizeConfChange, ); - scenario.assert_not_in_transition(&[1, 2, 3, 4]); + scenario.assert_not_in_membership_change(&[1, 2, 3, 4]); Ok(()) } @@ -290,12 +290,12 @@ mod remove_voter { fn stable() -> Result<()> { setup_for_test(); let leader = 1; - let old_configuration = ([1, 2, 3], []); - let new_configuration = ([1, 2], []); + let old_configuration = (vec![1, 2, 3], vec![]); + let new_configuration = (vec![1, 2], vec![]); let mut scenario = Scenario::new( leader, - (old_configuration.0.as_ref(), old_configuration.1.as_ref()), - (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + old_configuration, + new_configuration, )?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -309,7 +309,7 @@ mod remove_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1]); + scenario.assert_in_membership_change(&[1]); info!("Leader replicates the commit and finalize entry."); scenario.expect_read_and_dispatch_messages_from(&[1])?; @@ -318,7 +318,7 @@ mod remove_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2, 3]); + scenario.assert_in_membership_change(&[1, 2, 3]); info!("Cluster leaving the joint."); scenario.expect_read_and_dispatch_messages_from(&[2, 1])?; @@ -327,7 +327,7 @@ mod remove_voter { 3, ConfChangeType::FinalizeConfChange, ); - scenario.assert_not_in_transition(&[1, 2]); + scenario.assert_not_in_membership_change(&[1, 2]); Ok(()) } @@ -342,12 +342,12 @@ mod remove_leader { fn stable() -> Result<()> { setup_for_test(); let leader = 1; - let old_configuration = ([1, 2, 3], []); - let new_configuration = ([2, 3], []); + let old_configuration = (vec![1, 2, 3], vec![]); + let new_configuration = (vec![2, 3], vec![]); let mut scenario = Scenario::new( leader, - (old_configuration.0.as_ref(), old_configuration.1.as_ref()), - (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + old_configuration, + new_configuration.clone(), )?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -361,7 +361,7 @@ mod remove_leader { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1]); + scenario.assert_in_membership_change(&[1]); info!("Leader replicates the commit and finalize entry."); scenario.expect_read_and_dispatch_messages_from(&[1])?; @@ -370,7 +370,7 @@ mod remove_leader { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2, 3]); + scenario.assert_in_membership_change(&[1, 2, 3]); info!("Cluster leaving the joint."); scenario.expect_read_and_dispatch_messages_from(&[2, 3, 1])?; @@ -379,7 +379,7 @@ mod remove_leader { 3, ConfChangeType::FinalizeConfChange, ); - scenario.assert_not_in_transition(&[1, 2, 3]); + scenario.assert_not_in_membership_change(&[1, 2, 3]); let peer_leaders = scenario.peer_leaders(); for id in 1..=3 { assert_eq!(peer_leaders[&id], INVALID_ID, "peer {}", id); @@ -401,7 +401,7 @@ mod remove_leader { for (_, peer) in scenario.peers.iter() { assert_eq!( peer.prs().configuration(), - &(new_configuration.0.as_ref(), new_configuration.1.as_ref()).into() + &new_configuration.clone().into(), ); } @@ -422,12 +422,12 @@ mod remove_leader { fn leader_fails_and_recovers() -> Result<()> { setup_for_test(); let leader = 1; - let old_configuration = ([1, 2, 3], []); - let new_configuration = ([2, 3], []); + let old_configuration = (vec![1, 2, 3], vec![]); + let new_configuration = (vec![2, 3], vec![]); let mut scenario = Scenario::new( leader, - (old_configuration.0.as_ref(), old_configuration.1.as_ref()), - (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + old_configuration, + new_configuration, )?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -441,7 +441,7 @@ mod remove_leader { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1]); + scenario.assert_in_membership_change(&[1]); info!("Leader replicates the commit and finalize entry."); scenario.expect_read_and_dispatch_messages_from(&[1])?; @@ -450,7 +450,7 @@ mod remove_leader { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2, 3]); + scenario.assert_in_membership_change(&[1, 2, 3]); info!("Cluster leaving the joint."); scenario.expect_read_and_dispatch_messages_from(&[2, 3, 1])?; @@ -462,7 +462,7 @@ mod remove_leader { 3, ConfChangeType::FinalizeConfChange, ); - scenario.assert_not_in_transition(&[2, 3]); + scenario.assert_not_in_membership_change(&[2, 3]); // At this point, 1 thinks it is a leader, but actually it isn't anymore. @@ -510,12 +510,12 @@ mod three_peers_replace_voter { fn stable() -> Result<()> { setup_for_test(); let leader = 1; - let old_configuration = ([1, 2, 3], []); - let new_configuration = ([1, 2, 4], []); + let old_configuration = (vec![1, 2, 3], vec![]); + let new_configuration = (vec![1, 2, 4], vec![]); let mut scenario = Scenario::new( leader, - (old_configuration.0.as_ref(), old_configuration.1.as_ref()), - (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + old_configuration, + new_configuration, )?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -529,7 +529,7 @@ mod three_peers_replace_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1]); + scenario.assert_in_membership_change(&[1]); info!("Leader replicates the commit and finalize entry."); scenario.expect_read_and_dispatch_messages_from(&[1])?; @@ -538,7 +538,7 @@ mod three_peers_replace_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2]); + scenario.assert_in_membership_change(&[1, 2]); info!("Allowing new peers to catch up."); scenario.expect_read_and_dispatch_messages_from(&[4, 1, 4])?; @@ -547,7 +547,7 @@ mod three_peers_replace_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2, 4]); + scenario.assert_in_membership_change(&[1, 2, 4]); info!("Cluster leaving the joint."); scenario.expect_read_and_dispatch_messages_from(&[2, 1, 4])?; @@ -556,7 +556,7 @@ mod three_peers_replace_voter { 3, ConfChangeType::FinalizeConfChange, ); - scenario.assert_not_in_transition(&[1, 2, 4]); + scenario.assert_not_in_membership_change(&[1, 2, 4]); Ok(()) } @@ -566,12 +566,12 @@ mod three_peers_replace_voter { fn pending_delete_fails_after_begin() -> Result<()> { setup_for_test(); let leader = 1; - let old_configuration = ([1, 2, 3], []); - let new_configuration = ([1, 2, 4], []); + let old_configuration = (vec![1, 2, 3], vec![]); + let new_configuration = (vec![1, 2, 4], vec![]); let mut scenario = Scenario::new( leader, - (old_configuration.0.as_ref(), old_configuration.1.as_ref()), - (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + old_configuration, + new_configuration, )?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -585,7 +585,7 @@ mod three_peers_replace_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1]); + scenario.assert_in_membership_change(&[1]); scenario.isolate(3); // Take 3 down. @@ -596,7 +596,7 @@ mod three_peers_replace_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2]); + scenario.assert_in_membership_change(&[1, 2]); info!("Allowing new peers to catch up."); scenario.expect_read_and_dispatch_messages_from(&[4, 1, 4])?; @@ -605,7 +605,7 @@ mod three_peers_replace_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2, 4]); + scenario.assert_in_membership_change(&[1, 2, 4]); info!("Cluster leaving the joint."); scenario.expect_read_and_dispatch_messages_from(&[2, 1, 4])?; @@ -614,7 +614,7 @@ mod three_peers_replace_voter { 3, ConfChangeType::FinalizeConfChange, ); - scenario.assert_not_in_transition(&[1, 2, 4]); + scenario.assert_not_in_membership_change(&[1, 2, 4]); Ok(()) } @@ -624,12 +624,12 @@ mod three_peers_replace_voter { fn pending_create_with_quorum_fails_after_begin() -> Result<()> { setup_for_test(); let leader = 1; - let old_configuration = ([1, 2, 3], []); - let new_configuration = ([1, 2, 4], []); + let old_configuration = (vec![1, 2, 3], vec![]); + let new_configuration = (vec![1, 2, 4], vec![]); let mut scenario = Scenario::new( leader, - (old_configuration.0.as_ref(), old_configuration.1.as_ref()), - (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + old_configuration, + new_configuration, )?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -643,7 +643,7 @@ mod three_peers_replace_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1]); + scenario.assert_in_membership_change(&[1]); scenario.isolate(4); // Take 4 down. @@ -654,7 +654,7 @@ mod three_peers_replace_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2, 3]); + scenario.assert_in_membership_change(&[1, 2, 3]); info!("Cluster leaving the joint."); scenario.expect_read_and_dispatch_messages_from(&[2, 1])?; @@ -663,7 +663,7 @@ mod three_peers_replace_voter { 3, ConfChangeType::FinalizeConfChange, ); - scenario.assert_not_in_transition(&[1, 2, 3]); + scenario.assert_not_in_membership_change(&[1, 2, 3]); Ok(()) } @@ -673,12 +673,12 @@ mod three_peers_replace_voter { fn pending_create_and_destroy_both_fail() -> Result<()> { setup_for_test(); let leader = 1; - let old_configuration = ([1, 2, 3], []); - let new_configuration = ([1, 2, 4], []); + let old_configuration = (vec![1, 2, 3], vec![]); + let new_configuration = (vec![1, 2, 4], vec![]); let mut scenario = Scenario::new( leader, - (old_configuration.0.as_ref(), old_configuration.1.as_ref()), - (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + old_configuration, + new_configuration, )?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -692,7 +692,7 @@ mod three_peers_replace_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1]); + scenario.assert_in_membership_change(&[1]); scenario.isolate(3); // Take 3 down. scenario.isolate(4); // Take 4 down. @@ -704,7 +704,7 @@ mod three_peers_replace_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2]); + scenario.assert_in_membership_change(&[1, 2]); info!("Cluster leaving the joint."); scenario.expect_read_and_dispatch_messages_from(&[2, 1])?; @@ -713,7 +713,7 @@ mod three_peers_replace_voter { 3, ConfChangeType::FinalizeConfChange, ); - scenario.assert_not_in_transition(&[1, 2]); + scenario.assert_not_in_membership_change(&[1, 2]); Ok(()) } @@ -723,12 +723,12 @@ mod three_peers_replace_voter { fn old_quorum_fails() -> Result<()> { setup_for_test(); let leader = 1; - let old_configuration = ([1, 2, 3], []); - let new_configuration = ([1, 2, 4], []); + let old_configuration = (vec![1, 2, 3], vec![]); + let new_configuration = (vec![1, 2, 4], vec![]); let mut scenario = Scenario::new( leader, - (old_configuration.0.as_ref(), old_configuration.1.as_ref()), - (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + old_configuration, + new_configuration, )?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -742,7 +742,7 @@ mod three_peers_replace_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1]); + scenario.assert_in_membership_change(&[1]); info!("Old quorum fails."); scenario.isolate(3); // Take 3 down. @@ -755,8 +755,8 @@ mod three_peers_replace_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 4]); - scenario.assert_not_in_transition(&[2, 3]); + scenario.assert_in_membership_change(&[1, 4]); + scenario.assert_not_in_membership_change(&[2, 3]); info!("Spinning for awhile to ensure nothing spectacular happens"); for _ in scenario.peers[&leader].get_heartbeat_elapsed() @@ -769,8 +769,8 @@ mod three_peers_replace_voter { scenario.dispatch(messages)?; } - scenario.assert_in_transition(&[1, 4]); - scenario.assert_not_in_transition(&[2, 3]); + scenario.assert_in_membership_change(&[1, 4]); + scenario.assert_not_in_membership_change(&[2, 3]); info!("Recovering old qourum."); scenario.recover(); @@ -790,7 +790,7 @@ mod three_peers_replace_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2, 3, 4]); + scenario.assert_in_membership_change(&[1, 2, 3, 4]); info!("Failed peers confirming they have commited the begin."); scenario.expect_read_and_dispatch_messages_from(&[2, 3])?; @@ -802,7 +802,7 @@ mod three_peers_replace_voter { 3, ConfChangeType::FinalizeConfChange, ); - scenario.assert_not_in_transition(&[1, 2, 3, 4]); + scenario.assert_not_in_membership_change(&[1, 2, 3, 4]); Ok(()) } @@ -812,12 +812,12 @@ mod three_peers_replace_voter { fn new_quorum_fails() -> Result<()> { setup_for_test(); let leader = 1; - let old_configuration = ([1, 2, 3], []); - let new_configuration = ([1, 2, 4], []); + let old_configuration = (vec![1, 2, 3], vec![]); + let new_configuration = (vec![1, 2, 4], vec![]); let mut scenario = Scenario::new( leader, - (old_configuration.0.as_ref(), old_configuration.1.as_ref()), - (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + old_configuration, + new_configuration, )?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -831,7 +831,7 @@ mod three_peers_replace_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1]); + scenario.assert_in_membership_change(&[1]); info!("New quorum fails."); scenario.isolate(4); // Take 4 down. @@ -841,8 +841,8 @@ mod three_peers_replace_voter { scenario.expect_read_and_dispatch_messages_from(&[1, 3])?; info!("Leader waits to let the new quorum apply this before progressing."); - scenario.assert_in_transition(&[1]); - scenario.assert_not_in_transition(&[2, 3, 4]); + scenario.assert_in_membership_change(&[1]); + scenario.assert_not_in_membership_change(&[2, 3, 4]); info!("Spinning for awhile to ensure nothing spectacular happens"); for _ in scenario.peers[&leader].get_heartbeat_elapsed() @@ -855,8 +855,8 @@ mod three_peers_replace_voter { scenario.dispatch(messages)?; } - scenario.assert_in_transition(&[1]); - scenario.assert_not_in_transition(&[2, 3, 4]); + scenario.assert_in_membership_change(&[1]); + scenario.assert_not_in_membership_change(&[2, 3, 4]); info!("Recovering new qourum."); scenario.recover(); @@ -876,7 +876,7 @@ mod three_peers_replace_voter { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2, 3, 4]); + scenario.assert_in_membership_change(&[1, 2, 3, 4]); info!("Failed peers confirming they have commited the begin."); scenario.expect_read_and_dispatch_messages_from(&[2, 4])?; @@ -888,7 +888,7 @@ mod three_peers_replace_voter { 3, ConfChangeType::FinalizeConfChange, ); - scenario.assert_not_in_transition(&[1, 2, 3, 4]); + scenario.assert_not_in_membership_change(&[1, 2, 3, 4]); Ok(()) } @@ -903,12 +903,12 @@ mod three_peers_to_five_with_learner { fn stable() -> Result<()> { setup_for_test(); let leader = 1; - let old_configuration = ([1, 2, 3], []); - let new_configuration = ([1, 2, 3, 4, 5], [6]); + let old_configuration = (vec![1, 2, 3], vec![]); + let new_configuration = (vec![1, 2, 3, 4, 5], vec![6]); let mut scenario = Scenario::new( leader, - (old_configuration.0.as_ref(), old_configuration.1.as_ref()), - (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + old_configuration, + new_configuration, )?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -922,7 +922,7 @@ mod three_peers_to_five_with_learner { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1]); + scenario.assert_in_membership_change(&[1]); info!("Leader replicates the commit and finalize entry."); scenario.expect_read_and_dispatch_messages_from(&[1])?; @@ -931,7 +931,7 @@ mod three_peers_to_five_with_learner { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2, 3]); + scenario.assert_in_membership_change(&[1, 2, 3]); info!("Allowing new peers to catch up."); scenario.expect_read_and_dispatch_messages_from(&[4, 5, 6, 1, 4, 5, 6])?; @@ -940,7 +940,7 @@ mod three_peers_to_five_with_learner { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2, 3, 4, 5, 6]); + scenario.assert_in_membership_change(&[1, 2, 3, 4, 5, 6]); info!("Cluster leaving the joint."); scenario.expect_read_and_dispatch_messages_from(&[3, 2, 1])?; @@ -949,7 +949,7 @@ mod three_peers_to_five_with_learner { 3, ConfChangeType::FinalizeConfChange, ); - scenario.assert_not_in_transition(&[1, 2, 3, 4, 5, 6]); + scenario.assert_not_in_membership_change(&[1, 2, 3, 4, 5, 6]); Ok(()) } @@ -959,12 +959,12 @@ mod three_peers_to_five_with_learner { fn minority_old_followers_halt_at_start() -> Result<()> { setup_for_test(); let leader = 1; - let old_configuration = ([1, 2, 3], []); - let new_configuration = ([1, 2, 3, 4, 5], [6]); + let old_configuration = (vec![1, 2, 3], vec![]); + let new_configuration = (vec![1, 2, 3, 4, 5], vec![6]); let mut scenario = Scenario::new( leader, - (old_configuration.0.as_ref(), old_configuration.1.as_ref()), - (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + old_configuration, + new_configuration, )?; scenario.spawn_new_peers()?; scenario.isolate(3); @@ -979,7 +979,7 @@ mod three_peers_to_five_with_learner { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1]); + scenario.assert_in_membership_change(&[1]); info!("Leader replicates the commit and finalize entry."); scenario.expect_read_and_dispatch_messages_from(&[1])?; @@ -988,8 +988,8 @@ mod three_peers_to_five_with_learner { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2]); - scenario.assert_not_in_transition(&[3]); + scenario.assert_in_membership_change(&[1, 2]); + scenario.assert_not_in_membership_change(&[3]); info!("Allowing new peers to catch up."); scenario.expect_read_and_dispatch_messages_from(&[4, 5, 6, 1])?; @@ -998,8 +998,8 @@ mod three_peers_to_five_with_learner { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2, 4, 5, 6]); - scenario.assert_not_in_transition(&[3]); + scenario.assert_in_membership_change(&[1, 2, 4, 5, 6]); + scenario.assert_not_in_membership_change(&[3]); scenario.expect_read_and_dispatch_messages_from(&[4, 5, 6])?; @@ -1017,8 +1017,8 @@ mod three_peers_to_five_with_learner { 3, ConfChangeType::FinalizeConfChange, ); - scenario.assert_not_in_transition(&[1, 2, 4, 5]); - scenario.assert_not_in_transition(&[3]); + scenario.assert_not_in_membership_change(&[1, 2, 4, 5]); + scenario.assert_not_in_membership_change(&[3]); Ok(()) } @@ -1026,18 +1026,18 @@ mod three_peers_to_five_with_learner { mod intermingled_config_changes { use super::*; - + // In this test, we make sure that if the peer group is sent a `BeginConfChange`, then immediately a `AddNode` entry, that the `AddNode` is rejected by the leader. #[test] fn begin_then_add_node() -> Result<()> { setup_for_test(); let leader = 1; - let old_configuration = ([1, 2, 3], []); - let new_configuration = ([1, 2, 3, 4], []); + let old_configuration = (vec![1, 2, 3], vec![]); + let new_configuration = (vec![1, 2, 3, 4], vec![]); let mut scenario = Scenario::new( leader, - (old_configuration.0.as_ref(), old_configuration.1.as_ref()), - (new_configuration.0.as_ref(), new_configuration.1.as_ref()), + old_configuration, + new_configuration, )?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -1051,11 +1051,18 @@ mod intermingled_config_changes { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1]); + scenario.assert_in_membership_change(&[1]); info!("Leader recieves an add node proposal, which it rejects since it is already in transition."); scenario.propose_add_node_message(4).is_err(); - assert_eq!(scenario.peers[&scenario.old_leader].raft_log.entries(4, 1).unwrap()[0].get_entry_type(), EntryType::EntryNormal); + assert_eq!( + scenario.peers[&scenario.old_leader] + .raft_log + .entries(4, 1) + .unwrap()[0] + .get_entry_type(), + EntryType::EntryNormal + ); info!("Leader replicates the commit and finalize entry."); scenario.expect_read_and_dispatch_messages_from(&[1])?; @@ -1064,7 +1071,7 @@ mod intermingled_config_changes { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2, 3]); + scenario.assert_in_membership_change(&[1, 2, 3]); info!("Allowing new peers to catch up."); scenario.expect_read_and_dispatch_messages_from(&[4, 1, 4])?; @@ -1073,7 +1080,7 @@ mod intermingled_config_changes { 2, ConfChangeType::BeginConfChange, ); - scenario.assert_in_transition(&[1, 2, 3, 4]); + scenario.assert_in_membership_change(&[1, 2, 3, 4]); info!("Cluster leaving the joint."); scenario.expect_read_and_dispatch_messages_from(&[3, 2, 1])?; @@ -1082,7 +1089,7 @@ mod intermingled_config_changes { 3, ConfChangeType::FinalizeConfChange, ); - scenario.assert_not_in_transition(&[1, 2, 3, 4]); + scenario.assert_not_in_membership_change(&[1, 2, 3, 4]); Ok(()) } @@ -1220,10 +1227,7 @@ impl Scenario { /// Send a message proposing a "one-by-one" style AddNode configuration. /// If the peers are in the midst joint consensus style (Begin/FinalizeConfChange) change they should reject it. fn propose_add_node_message(&mut self, id: u64) -> Result<()> { - info!( - "Proposing add_node message. Target: {:?}", - id, - ); + info!("Proposing add_node message. Target: {:?}", id,); let message = build_propose_add_node_message( self.old_leader, id, @@ -1248,29 +1252,29 @@ impl Scenario { } /// Checks that the given peers are not in a transition state. - fn assert_not_in_transition<'a>(&self, peers: impl IntoIterator) { + fn assert_not_in_membership_change<'a>(&self, peers: impl IntoIterator) { for peer in peers.into_iter().map(|id| &self.peers[id]) { assert!( - !peer.is_in_transition(), - "Peer {} should not have been in transition.", + !peer.is_in_membership_change(), + "Peer {} should not have been in a membership change.", peer.id ); } } // Checks that the given peers are in a transition state. - fn assert_in_transition<'a>(&self, peers: impl IntoIterator) { + fn assert_in_membership_change<'a>(&self, peers: impl IntoIterator) { for peer in peers.into_iter().map(|id| &self.peers[id]) { assert!( - peer.is_in_transition(), - "Peer {} should have been in transition.", + peer.is_in_membership_change(), + "Peer {} should have been in a membership change.", peer.id ); } } /// Reads the pending entries to be applied to a raft peer, checks one is of the expected variant, and applies it. Then, it advances the node to that point in the configuration change. - fn expect_apply_transition_entry<'a>( + fn expect_apply_membership_change_entry<'a>( &mut self, peers: impl IntoIterator, entry_type: ConfChangeType, @@ -1346,7 +1350,7 @@ impl Scenario { } // Verify there is a transition entry at the given index of the given variant. - fn assert_transition_entry_at<'a>( + fn assert_membership_change_entry_at<'a>( &self, peers: impl IntoIterator, index: u64, @@ -1371,9 +1375,9 @@ impl Scenario { entry_type: ConfChangeType, ) { let peers = peers.into_iter().collect::>(); - self.expect_apply_transition_entry(peers.clone(), entry_type) + self.expect_apply_membership_change_entry(peers.clone(), entry_type) .unwrap(); - self.assert_transition_entry_at(peers, index, entry_type) + self.assert_membership_change_entry_at(peers, index, entry_type) } } @@ -1432,11 +1436,7 @@ fn build_propose_change_message<'a>( message } -fn build_propose_add_node_message( - recipient: u64, - added_id: u64, - index: u64, -) -> Message { +fn build_propose_add_node_message(recipient: u64, added_id: u64, index: u64) -> Message { let add_nodes_entry = { let mut conf_change = ConfChange::new(); conf_change.set_change_type(ConfChangeType::AddNode); @@ -1454,4 +1454,4 @@ fn build_propose_add_node_message( message.set_index(index); message.set_entries(RepeatedField::from_vec(vec![add_nodes_entry])); message -} \ No newline at end of file +} From c62aee7f8c464e783f6f7814c8036dd5e5b14199 Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Thu, 22 Nov 2018 09:17:22 -0800 Subject: [PATCH 05/41] fmt --- src/progress.rs | 2 +- src/raft.rs | 11 +-- .../test_membership_changes.rs | 90 ++++--------------- 3 files changed, 20 insertions(+), 83 deletions(-) diff --git a/src/progress.rs b/src/progress.rs index 2d3faa24b..c26019743 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -92,7 +92,7 @@ where } impl From for Configuration { - fn from(conf_state: ConfState) -> Self {; + fn from(conf_state: ConfState) -> Self { Self { voters: conf_state.get_nodes().iter().cloned().collect(), learners: conf_state.get_learners().iter().cloned().collect(), diff --git a/src/raft.rs b/src/raft.rs index 609063291..01cf27f09 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -28,8 +28,7 @@ use std::cmp; use eraftpb::{ - ConfChange, ConfChangeType, ConfState, Entry, EntryType, HardState, Message, MessageType, - Snapshot, + ConfChange, ConfChangeType, Entry, EntryType, HardState, Message, MessageType, Snapshot, }; use fxhash::{FxHashMap, FxHashSet}; use protobuf; @@ -642,11 +641,9 @@ impl Raft { Some(ConfChangeType::BeginConfChange), ); // Invariant: We know that if we have commited past some index, we can also commit that index. - if applied >= index { - if self.state == StateRole::Leader { - // We must replicate the commit entry. - self.append_finalize_conf_change_entry(); - } + if applied >= index && self.state == StateRole::Leader { + // We must replicate the commit entry. + self.append_finalize_conf_change_entry(); } } } diff --git a/tests/integration_cases/test_membership_changes.rs b/tests/integration_cases/test_membership_changes.rs index b174f996a..72f8e62b7 100644 --- a/tests/integration_cases/test_membership_changes.rs +++ b/tests/integration_cases/test_membership_changes.rs @@ -118,11 +118,7 @@ mod three_peers_add_voter { let leader = 1; let old_configuration = (vec![1, 2, 3], vec![]); let new_configuration = (vec![1, 2, 3, 4], vec![]); - let mut scenario = Scenario::new( - leader, - old_configuration, - new_configuration, - )?; + let mut scenario = Scenario::new(leader, old_configuration, new_configuration)?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -179,11 +175,7 @@ mod three_peers_add_learner { let leader = 1; let old_configuration = (vec![1, 2, 3], vec![]); let new_configuration = (vec![1, 2, 3], vec![4]); - let mut scenario = Scenario::new( - leader, - old_configuration, - new_configuration, - )?; + let mut scenario = Scenario::new(leader, old_configuration, new_configuration)?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -240,11 +232,7 @@ mod remove_learner { let leader = 1; let old_configuration = (vec![1, 2, 3], vec![4]); let new_configuration = (vec![1, 2, 3], vec![]); - let mut scenario = Scenario::new( - leader, - old_configuration, - new_configuration, - )?; + let mut scenario = Scenario::new(leader, old_configuration, new_configuration)?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -292,11 +280,7 @@ mod remove_voter { let leader = 1; let old_configuration = (vec![1, 2, 3], vec![]); let new_configuration = (vec![1, 2], vec![]); - let mut scenario = Scenario::new( - leader, - old_configuration, - new_configuration, - )?; + let mut scenario = Scenario::new(leader, old_configuration, new_configuration)?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -344,11 +328,7 @@ mod remove_leader { let leader = 1; let old_configuration = (vec![1, 2, 3], vec![]); let new_configuration = (vec![2, 3], vec![]); - let mut scenario = Scenario::new( - leader, - old_configuration, - new_configuration.clone(), - )?; + let mut scenario = Scenario::new(leader, old_configuration, new_configuration.clone())?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -424,11 +404,7 @@ mod remove_leader { let leader = 1; let old_configuration = (vec![1, 2, 3], vec![]); let new_configuration = (vec![2, 3], vec![]); - let mut scenario = Scenario::new( - leader, - old_configuration, - new_configuration, - )?; + let mut scenario = Scenario::new(leader, old_configuration, new_configuration)?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -512,11 +488,7 @@ mod three_peers_replace_voter { let leader = 1; let old_configuration = (vec![1, 2, 3], vec![]); let new_configuration = (vec![1, 2, 4], vec![]); - let mut scenario = Scenario::new( - leader, - old_configuration, - new_configuration, - )?; + let mut scenario = Scenario::new(leader, old_configuration, new_configuration)?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -568,11 +540,7 @@ mod three_peers_replace_voter { let leader = 1; let old_configuration = (vec![1, 2, 3], vec![]); let new_configuration = (vec![1, 2, 4], vec![]); - let mut scenario = Scenario::new( - leader, - old_configuration, - new_configuration, - )?; + let mut scenario = Scenario::new(leader, old_configuration, new_configuration)?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -626,11 +594,7 @@ mod three_peers_replace_voter { let leader = 1; let old_configuration = (vec![1, 2, 3], vec![]); let new_configuration = (vec![1, 2, 4], vec![]); - let mut scenario = Scenario::new( - leader, - old_configuration, - new_configuration, - )?; + let mut scenario = Scenario::new(leader, old_configuration, new_configuration)?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -675,11 +639,7 @@ mod three_peers_replace_voter { let leader = 1; let old_configuration = (vec![1, 2, 3], vec![]); let new_configuration = (vec![1, 2, 4], vec![]); - let mut scenario = Scenario::new( - leader, - old_configuration, - new_configuration, - )?; + let mut scenario = Scenario::new(leader, old_configuration, new_configuration)?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -725,11 +685,7 @@ mod three_peers_replace_voter { let leader = 1; let old_configuration = (vec![1, 2, 3], vec![]); let new_configuration = (vec![1, 2, 4], vec![]); - let mut scenario = Scenario::new( - leader, - old_configuration, - new_configuration, - )?; + let mut scenario = Scenario::new(leader, old_configuration, new_configuration)?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -814,11 +770,7 @@ mod three_peers_replace_voter { let leader = 1; let old_configuration = (vec![1, 2, 3], vec![]); let new_configuration = (vec![1, 2, 4], vec![]); - let mut scenario = Scenario::new( - leader, - old_configuration, - new_configuration, - )?; + let mut scenario = Scenario::new(leader, old_configuration, new_configuration)?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -905,11 +857,7 @@ mod three_peers_to_five_with_learner { let leader = 1; let old_configuration = (vec![1, 2, 3], vec![]); let new_configuration = (vec![1, 2, 3, 4, 5], vec![6]); - let mut scenario = Scenario::new( - leader, - old_configuration, - new_configuration, - )?; + let mut scenario = Scenario::new(leader, old_configuration, new_configuration)?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; @@ -961,11 +909,7 @@ mod three_peers_to_five_with_learner { let leader = 1; let old_configuration = (vec![1, 2, 3], vec![]); let new_configuration = (vec![1, 2, 3, 4, 5], vec![6]); - let mut scenario = Scenario::new( - leader, - old_configuration, - new_configuration, - )?; + let mut scenario = Scenario::new(leader, old_configuration, new_configuration)?; scenario.spawn_new_peers()?; scenario.isolate(3); scenario.propose_change_message()?; @@ -1034,11 +978,7 @@ mod intermingled_config_changes { let leader = 1; let old_configuration = (vec![1, 2, 3], vec![]); let new_configuration = (vec![1, 2, 3, 4], vec![]); - let mut scenario = Scenario::new( - leader, - old_configuration, - new_configuration, - )?; + let mut scenario = Scenario::new(leader, old_configuration, new_configuration)?; scenario.spawn_new_peers()?; scenario.propose_change_message()?; From d3f4f7d822a5e6178aadafbb4a78e4463dbdb93d Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Thu, 22 Nov 2018 09:23:25 -0800 Subject: [PATCH 06/41] Harmonize error message name --- src/errors.rs | 4 ++-- src/progress.rs | 2 +- src/raft.rs | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/errors.rs b/src/errors.rs index b30c5ccc3..bd1edec41 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -69,8 +69,8 @@ quick_error! { display("Cannot complete that action while in {:?} role.", role) } /// The node attempted to transition to a new membership configuration while there was none pending. - NoPendingTransition { - display("No pending membership transition. Create a pending transition with `Raft.begin_config_transition`") + NoPendingMembershipChange { + display("No pending membership change. Create a pending transition with `Raft::propose_membership_change` on the leader.") } } } diff --git a/src/progress.rs b/src/progress.rs index c26019743..6cd67cb88 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -577,7 +577,7 @@ impl ProgressSet { pub fn finalize_membership_change(&mut self) -> Result<(), Error> { let next = self.next_configuration.take(); match next { - None => Err(Error::NoPendingTransition)?, + None => Err(Error::NoPendingMembershipChange)?, Some(next) => { { let pending = self diff --git a/src/raft.rs b/src/raft.rs index 01cf27f09..799bbaa5f 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -1195,7 +1195,7 @@ impl Raft { .next_configuration() .as_ref() .map(|config| config.contains(&self.leader_id)) - .ok_or_else(|| Error::NoPendingTransition)?; + .ok_or_else(|| Error::NoPendingMembershipChange)?; if !leader_in_new_set { let last_term = self.raft_log.last_term(); if self.state == StateRole::Leader { From 2f3113b27817295dc4396f419db44d830ae760c8 Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Thu, 22 Nov 2018 09:25:39 -0800 Subject: [PATCH 07/41] Touchup docs --- src/lib.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 401b9bb16..f125b9859 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -303,6 +303,8 @@ It (currently) does not: * Optionally roll back a change during a peer group pause where the new peer group configuration fails. +> PRs to enable these are welcome! We'd love to mentor/support you through implementing it. + This means it's possible to do: ```rust @@ -334,10 +336,6 @@ assert!(node.raft.prs().voter_ids().contains(&2)); assert!(!node.raft.is_in_membership_change()); ``` - - -PRs to enable these are welcome! We'd love to mentor/support you through implementing it. - This process is a two-phase process, during the midst of it the peer group's leader is managing **two independent, possibly overlapping peer sets**. From 4385ea9c9442a87f479cae6f9b7ddcb9a4a23bf5 Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Thu, 22 Nov 2018 12:17:45 -0800 Subject: [PATCH 08/41] Doc refinement and correct candidacy_status bug --- src/lib.rs | 11 +++++++---- src/progress.rs | 36 ++++++++++++++++++++++-------------- 2 files changed, 29 insertions(+), 18 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index f125b9859..8349eafab 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -316,12 +316,15 @@ node.raft.become_leader(); // Call this on the leader, or send the command via a normal `MsgPropose`. node.raft.propose_membership_change(( - vec![1,2,3], // Any IntoIterator. + // Any IntoIterator. + // Voters + vec![1,2,3], + // Learners vec![4,5,6], )).unwrap(); +# let entry = &node.raft.raft_log.entries(2, 1).unwrap()[0]; // ...Later when the begin entry is ready to apply: -let entry = &node.raft.raft_log.entries(2, 1).unwrap()[0]; node.raft.begin_membership_change(entry).unwrap(); assert!(node.raft.is_in_membership_change()); # @@ -329,8 +332,8 @@ assert!(node.raft.is_in_membership_change()); # // example. # node.raft.commit_apply(2); # +# let entry = &node.raft.raft_log.entries(3, 1).unwrap()[0]; // ...Later, when the finalize entry is ready to apply: -let entry = &node.raft.raft_log.entries(3, 1).unwrap()[0]; node.raft.finalize_membership_change(entry).unwrap(); assert!(node.raft.prs().voter_ids().contains(&2)); assert!(!node.raft.is_in_membership_change()); @@ -339,7 +342,7 @@ assert!(!node.raft.is_in_membership_change()); This process is a two-phase process, during the midst of it the peer group's leader is managing **two independent, possibly overlapping peer sets**. -> In order to maintain resiliency gaurantees (progress while a majority of both peer sets is +> **Note:** In order to maintain resiliency gaurantees (progress while a majority of both peer sets is active), it is very important to wait until the entire peer group has exited the transition phase before taking old, removed peers offline. diff --git a/src/progress.rs b/src/progress.rs index 6cd67cb88..faccb11c3 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -56,7 +56,9 @@ impl Default for ProgressState { } #[derive(Clone, Debug, Default, PartialEq, Getters)] -/// A Raft internal representation of a Configuration. This is corallary to a ConfState, but optimized for `contains` calls. +/// A Raft internal representation of a Configuration. +/// +/// This is corollary to a ConfState, but optimized for `contains` calls. pub struct Configuration { /// The voter set. #[get = "pub"] @@ -117,7 +119,7 @@ impl Configuration { } } - /// Validates that the configuration not problematic. + /// Validates that the configuration is not problematic. /// /// Namely: /// * There can be no overlap of voters and learners. @@ -138,7 +140,8 @@ impl Configuration { } /// Returns whether or not the given `id` is a member of this configuration. - #[allow(trivially_copy_pass_by_ref)] // Allowed to maintain API consistency. + // Allowed to maintain API consistency. See `HashSet::contains(&u64)`. + #[allow(trivially_copy_pass_by_ref)] pub fn contains(&self, id: &u64) -> bool { self.voters.contains(id) || self.learners.contains(id) } @@ -460,18 +463,23 @@ impl ProgressSet { (accepts, rejects) }, ); - if self.configuration.has_quorum(&accepts) { - return CandidacyStatus::Elected; - } else if self.configuration.has_quorum(&rejects) { - return CandidacyStatus::Ineligible; - } - if let Some(ref next) = self.next_configuration { - if next.has_quorum(&accepts) { - return CandidacyStatus::Elected; - } else if next.has_quorum(&rejects) { - return CandidacyStatus::Ineligible; + + match self.next_configuration { + Some(ref next) => { + if next.has_quorum(&accepts) && self.configuration.has_quorum(&accepts) { + return CandidacyStatus::Elected; + } else if next.has_quorum(&rejects) || self.configuration.has_quorum(&rejects) { + return CandidacyStatus::Ineligible; + } + }, + None => { + if self.configuration.has_quorum(&accepts) { + return CandidacyStatus::Elected; + } else if self.configuration.has_quorum(&rejects) { + return CandidacyStatus::Ineligible; + } } - } + }; CandidacyStatus::Eligible } From 907617198da80f6f953ea5566c562852720e8ec9 Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Thu, 22 Nov 2018 12:19:52 -0800 Subject: [PATCH 09/41] Add pitfall doc note --- src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/lib.rs b/src/lib.rs index 8349eafab..873acb13a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -302,6 +302,7 @@ It (currently) does not: * Allow control of the replacement leader during a stepdown. * Optionally roll back a change during a peer group pause where the new peer group configuration fails. +* Provide automated promotion of newly added voters from learner to voter when they are caught up. This must be done as a two stage process for now. > PRs to enable these are welcome! We'd love to mentor/support you through implementing it. From 8a31c83a04143bb967d809c5bc8aa3ad8a9d0589 Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Thu, 22 Nov 2018 13:43:39 -0800 Subject: [PATCH 10/41] Docs, make progress tests nicer --- src/progress.rs | 89 ++++++++++++------- .../test_membership_changes.rs | 3 +- 2 files changed, 58 insertions(+), 34 deletions(-) diff --git a/src/progress.rs b/src/progress.rs index faccb11c3..67dc6b336 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -168,7 +168,7 @@ pub struct ProgressSet { /// The current configuration state of the cluster. #[get = "pub"] configuration: Configuration, - /// The pending configuration, which will be adopted after the Finalize entryr are applied + /// The pending configuration, which will be adopted after the Finalize entry is applied. #[get = "pub"] next_configuration: Option, configuration_capacity: (usize, usize), @@ -529,7 +529,9 @@ impl ProgressSet { /// Enter a joint consensus state to transition to the specified configuration. /// - /// The `next` provided should be derived from the `ConfChange` message. `progress` is used as a basis for created peer `Progress` values. You are only expected to set `ins` from the `raft.max_inflights` value. + /// The `next` provided should be derived from the `ConfChange` message. `progress` is used as + /// a basis for created peer `Progress` values. You are only expected to set `ins` from the + /// `raft.max_inflights` value. /// /// Once this state is entered the leader should replicate the `ConfChange` message. After the /// majority of nodes, in both the current and the `next`, have committed the union state. At @@ -563,7 +565,7 @@ impl ProgressSet { { Err(Error::Exists(demoted, "learners"))?; } - debug!("Beginning member configuration transition. End state will be voters {:?}, Learners: {:?}", next.voters, next.learners); + debug!("Beginning membership change. End configuration will be {:?}", next); // When a peer is first added/promoted, we should mark it as recently active. // Otherwise, check_quorum may cause us to step down if it is invoked @@ -580,7 +582,7 @@ impl ProgressSet { /// Finalizes the joint consensus state and transitions solely to the new state. /// - /// This should be called only after calling `begin_membership_change` and the the majority + /// This must be called only after calling `begin_membership_change` and after the majority /// of peers in both the `current` and the `next` state have commited the changes. pub fn finalize_membership_change(&mut self) -> Result<(), Error> { let next = self.next_configuration.take(); @@ -599,7 +601,7 @@ impl ProgressSet { } } self.configuration = next; - debug!("Executed finalize member configration transition command. State is Voters: {:?}, Learners: {:?}", self.configuration.voters, self.configuration.learners); + debug!("Finalizing membership change. Config is {:?}", self.configuration); } } Ok(()) @@ -1131,63 +1133,84 @@ mod test_progress_set { } #[test] - fn test_config_transition_remove_voter() -> Result<()> { - check_set_nodes(&vec![1, 2], &vec![], &vec![1], &vec![]) + fn test_membership_change_configuration_remove_voter() -> Result<()> { + check_membership_change_configuration( + (vec![1, 2], vec![]), + (vec![1], vec![]) + ) } #[test] - fn test_config_transition_remove_learner() -> Result<()> { - check_set_nodes(&vec![1], &vec![2], &vec![1], &vec![]) + fn test_membership_change_configuration_remove_learner() -> Result<()> { + check_membership_change_configuration( + (vec![1], vec![2]), + (vec![1], vec![]), + ) } #[test] - fn test_config_transition_conflicting_sets() { - assert!(check_set_nodes(&vec![1], &vec![], &vec![1], &vec![1]).is_err()) + fn test_membership_change_configuration_conflicting_sets() { + assert!(check_membership_change_configuration( + (vec![1], vec![]), + (vec![1], vec![1]), + ).is_err()) } #[test] - fn test_config_transition_empty_sets() { - assert!(check_set_nodes(&vec![], &vec![], &vec![], &vec![]).is_err()) + fn test_membership_change_configuration_empty_sets() { + assert!(check_membership_change_configuration( + (vec![], vec![]), + (vec![], vec![]) + ).is_err()) } #[test] - fn test_config_transition_empty_voters() { - assert!(check_set_nodes(&vec![1], &vec![], &vec![], &vec![]).is_err()) + fn test_membership_change_configuration_empty_voters() { + assert!(check_membership_change_configuration( + (vec![1], vec![]), + (vec![], vec![]), + ).is_err()) } #[test] - fn test_config_transition_add_voter() -> Result<()> { - check_set_nodes(&vec![1], &vec![], &vec![1, 2], &vec![]) + fn test_membership_change_configuration_add_voter() -> Result<()> { + check_membership_change_configuration( + (vec![1], vec![]), + (vec![1, 2], vec![]), + ) } #[test] - fn test_config_transition_add_learner() -> Result<()> { - check_set_nodes(&vec![1], &vec![], &vec![1], &vec![2]) + fn test_membership_change_configuration_add_learner() -> Result<()> { + check_membership_change_configuration( + (vec![1], vec![]), + (vec![1], vec![2]), + ) } #[test] - fn test_config_transition_promote_learner() -> Result<()> { - check_set_nodes(&vec![1], &vec![2], &vec![1, 2], &vec![]) + fn test_membership_change_configuration_promote_learner() -> Result<()> { + check_membership_change_configuration( + (vec![1], vec![2]), + (vec![1, 2], vec![]), + ) } - fn check_set_nodes<'a>( - start_voters: impl IntoIterator, - start_learners: impl IntoIterator, - end_voters: impl IntoIterator, - end_learners: impl IntoIterator, + fn check_membership_change_configuration( + start: (impl IntoIterator, impl IntoIterator), + end: (impl IntoIterator, impl IntoIterator), ) -> Result<()> { - let start_voters = start_voters + let start_voters = start.1 .into_iter() - .cloned() .collect::>(); - let start_learners = start_learners + let start_learners = start.0 .into_iter() - .cloned() .collect::>(); - let end_voters = end_voters.into_iter().cloned().collect::>(); - let end_learners = end_learners + let end_voters = end.1 + .into_iter() + .collect::>(); + let end_learners = end.0 .into_iter() - .cloned() .collect::>(); let transition_voters = start_voters .union(&end_voters) diff --git a/tests/integration_cases/test_membership_changes.rs b/tests/integration_cases/test_membership_changes.rs index 72f8e62b7..a990958b1 100644 --- a/tests/integration_cases/test_membership_changes.rs +++ b/tests/integration_cases/test_membership_changes.rs @@ -1104,7 +1104,8 @@ impl Scenario { /// Creates any peers which are pending creation. /// - /// This *only* creates the peers and adds them to the `Network`. It does not take other action. Newly created peers are only aware of the leader and themself. + /// This *only* creates the peers and adds them to the `Network`. It does not take other + /// action. Newly created peers are only aware of the leader and themself. fn spawn_new_peers(&mut self) -> Result<()> { let storage = MemStorage::new(); let new_peers = self.new_peers(); From 827fe30256dd508082647adacf9a89623b45e946 Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Thu, 22 Nov 2018 14:01:06 -0800 Subject: [PATCH 11/41] Documentation refinements --- src/raft.rs | 73 +++++++++++++++++++++++++++++++------------------ src/raw_node.rs | 8 +++++- 2 files changed, 54 insertions(+), 27 deletions(-) diff --git a/src/raft.rs b/src/raft.rs index 799bbaa5f..a63bce7c4 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -144,17 +144,25 @@ pub struct Raft { /// be proposed if the leader's applied index is greater than this /// value. /// - /// This value is conservatively set in cases where there may be a configuration change pending, but scanning the log is possibly expensive. This implies that the index stated here may not necessarily be a config + /// This value is conservatively set in cases where there may be a configuration change pending, + /// but scanning the log is possibly expensive. This implies that the index stated here may not + /// necessarily be a config change entry, and it may not be a `BeginConfChange` entry, even if + /// we set this to one. pub pending_conf_index: u64, /// The last BeginConfChange entry. Once we commit this entry we can exit the joint state. /// /// This is different than `pending_conf_index` since it is more specific, and also exact. - /// While `pending_conf_index` is conservatively set at times to ensure safety in the one-by-one change method, in joint consensus based changes we track the state exactly. The index here **must** only be set when a `BeginConfChange` is present at that index. + /// While `pending_conf_index` is conservatively set at times to ensure safety in the + /// one-by-one change method, in joint consensus based changes we track the state exactly. The + /// index here **must** only be set when a `BeginConfChange` is present at that index. /// /// # Caveats /// - /// It is important that whenever this is set that `pending_conf_index` is also set to the value if it is greater than the existing value. + /// It is important that whenever this is set that `pending_conf_index` is also set to the + /// value if it is greater than the existing value. + /// + /// **Use `Raft::set_began_conf_change_at()` to change this value.** began_conf_change_at: Option, /// The queue of read-only requests. @@ -393,7 +401,8 @@ impl Raft { self.skip_bcast_commit = skip; } - /// Set when the peer began a joint consensus change. This will also set `pending_conf_index` if it is larger than the existing number. + /// Set when the peer began a joint consensus change. This will also set `pending_conf_index` + /// if it is larger than the existing number. #[inline] fn set_began_conf_change_at(&mut self, maybe_index: impl Into>) { let maybe_index = maybe_index.into(); @@ -620,15 +629,18 @@ impl Raft { /// Commit that the Raft peer has applied up to the given index. /// - /// Registers the new applied index to the Raft log, then checks to see if it's time to finalize a Joint Consensus state. + /// Registers the new applied index to the Raft log, then checks to see if it's time to + /// finalize a Joint Consensus state. pub fn commit_apply(&mut self, applied: u64) { #[allow(deprecated)] self.raft_log.applied_to(applied); // Check to see if we need to finalize a Joint Consensus state now. if let Some(index) = self.began_conf_change_at { - // Invariant: We know that the index stored at `began_conf_change_at` should be a `BeginConfChange`. - // Check this in debug mode for safety while testing, but skip it in production since those bugs should have been caught. + // Invariant: We know that the index stored at `began_conf_change_at` should be a + // `BeginConfChange`. + // Check this in debug mode for safety while testing, but skip it in production since + // those bugs should have been caught, and doing this can be expensive. debug_assert_eq!( self.raft_log .entries(index, 1) @@ -706,7 +718,7 @@ impl Raft { self.maybe_commit(); } - ///maybe_commit Returns true to indicate that there will probably be some readiness need to be handled. + /// Returns true to indicate that there will probably be some readiness need to be handled. pub fn tick(&mut self) -> bool { match self.state { StateRole::Follower | StateRole::PreCandidate | StateRole::Candidate => { @@ -1127,22 +1139,26 @@ impl Raft { Ok(()) } - /// Called to apply a `BeginConfChange` entry. - /// + /// Apply a `BeginConfChange` entry. /// - /// When a Raft node applies this variant of a configuration change it will adopt a joint configuration state until the membership change is finalized. + /// When a Raft node applies this variant of a configuration change it will adopt a joint + /// configuration state until the membership change is finalized. /// - /// During this time the `Raft` will have two, possibly overlapping, cooperating quorums for both elections and log replication. + /// During this time the `Raft` will have two, possibly overlapping, cooperating quorums for + /// both elections and log replication. /// /// # Implementation notes /// - /// This uses a slightly modified "Joint Consensus" algorithm as detailed in Section 6 of the Raft paper. + /// This uses a slightly modified "Joint Consensus" algorithm as detailed in Section 6 of the + /// Raft paper. /// - /// The modification is as follows: We apply the change when a node *applies* the entry, not when the entry is received. + /// The modification is as follows: We apply the change when a node *applies* the entry, not + /// when the entry is received. /// /// # Panics /// - /// This **must** only be called on `Entry` which holds an `Entry` of with `EntryType::ConfChange` and the `data` field being a serialized `ConfChange`. + /// This **must** only be called on `Entry` which holds an `Entry` of with + /// `EntryType::ConfChange` and the `data` field being a serialized `ConfChange`. /// /// This `ConfChange` must be of variant `BeginConfChange` and contain a `configuration` value. // TODO: Make this return a result instead of panic. @@ -1164,23 +1180,29 @@ impl Raft { Ok(()) } - /// Called to apply a `FinalizeConfChange` entry. + /// Apply a `FinalizeConfChange` entry. /// - /// When a Raft node applies this variant of a configuration change it will finalize the transition begun by [`begin_membership_change`] + /// When a Raft node applies this variant of a configuration change it will finalize the + /// transition begun by [`begin_membership_change`]. /// - /// Once this is called the Raft will no longer have two, possibly overlapping, cooperating qourums. + /// Once this is called the Raft will no longer have two, possibly overlapping, cooperating + /// qourums. /// /// # Implementation notes /// - /// This uses a slightly modified "Joint Consensus" algorithm as detailed in Section 6 of the Raft paper. + /// This uses a slightly modified "Joint Consensus" algorithm as detailed in Section 6 of the + /// Raft paper. /// - /// The modification is as follows: We apply the change when a node *applies* the entry, not when the entry is received. + /// The modification is as follows: We apply the change when a node *applies* the entry, not + /// when the entry is received. /// /// # Panics /// - /// This **must** only be called on `Entry` which holds an `Entry` of with `EntryType::ConfChange` and the `data` field being a serialized `ConfChange`. + /// This **must** only be called on `Entry` which holds an `Entry` of with + /// `EntryType::ConfChange` and the `data` field being a serialized `ConfChange`. /// - /// This `ConfChange` must be of variant `FinalizeConfChange` and contain no `configuration` value. + /// This `ConfChange` must be of variant `FinalizeConfChange` and contain no `configuration` + /// value. #[inline(always)] pub fn finalize_membership_change(&mut self, entry: &Entry) -> Result<()> { assert_eq!(entry.get_entry_type(), EntryType::EntryConfChange); @@ -1189,7 +1211,8 @@ impl Raft { conf_change.get_change_type(), ConfChangeType::FinalizeConfChange ); - // Joint Consensus, in the Raft paper, states the leader should step down and become a follower if it is removed during a transition. + // Joint Consensus, in the Raft paper, states the leader should step down and become a + // follower if it is removed during a transition. let leader_in_new_set = self .prs() .next_configuration() @@ -1340,13 +1363,11 @@ impl Raft { } if self.read_only.option != ReadOnlyOption::Safe || m.get_context().is_empty() { - debug!("Early exit due to read_only being not Safe (is {:?}), or no context. (is {:?})", self.read_only.option, m.get_context()); return; } } if !prs.has_quorum(&self.read_only.recv_ack(m)) { - debug!("Early exit due to !has_quorum"); return; } @@ -2031,7 +2052,7 @@ impl Raft { /// * `voters` is empty. pub fn propose_membership_change(&mut self, config: impl Into) -> Result<()> { if self.state != StateRole::Leader { - Err(Error::InvalidState(self.state))?; + return Err(Error::InvalidState(self.state)); } let config = config.into(); config.valid()?; diff --git a/src/raw_node.rs b/src/raw_node.rs index 5629f198a..2d5d5c1e1 100644 --- a/src/raw_node.rs +++ b/src/raw_node.rs @@ -330,8 +330,14 @@ impl RawNode { } /// Takes the conf change and applies it. + /// + /// # Panics + /// + /// In the case of `BeginConfChange` or `FinalizeConfChange` returning errors this will panic. + /// + /// For a safe interface for these directly call `this.raft.begin_membership_change(entry)` or + /// `this.raft.finalize_membership_change(entry)` respectively. pub fn apply_conf_change(&mut self, cc: &ConfChange) -> ConfState { - warn!("Got ConfChange"); if cc.get_node_id() == INVALID_ID && cc.get_change_type() != ConfChangeType::BeginConfChange { let mut cs = ConfState::new(); From 8379622af2a1a9948cc15cb3e705577b26f5c240 Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Thu, 22 Nov 2018 14:59:55 -0800 Subject: [PATCH 12/41] Lower asserts to result returns on public Raft functions --- src/errors.rs | 6 ++++- src/raft.rs | 74 +++++++++++++++++++++++++++++++-------------------- 2 files changed, 50 insertions(+), 30 deletions(-) diff --git a/src/errors.rs b/src/errors.rs index bd1edec41..7f7dfcdc2 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -64,7 +64,7 @@ quick_error! { NotExists(id: u64, set: &'static str) { display("The node {} is not in the {} set.", id, set) } - /// The action give requires the node to be in a particular state role. + /// The action given requires the node to be in a particular state role. InvalidState(role: StateRole) { display("Cannot complete that action while in {:?} role.", role) } @@ -72,6 +72,10 @@ quick_error! { NoPendingMembershipChange { display("No pending membership change. Create a pending transition with `Raft::propose_membership_change` on the leader.") } + /// An argument violates a calling contract. + ViolatesContract(contract: String) { + display("An argument violate a calling contract: {}", contract) + } } } diff --git a/src/raft.rs b/src/raft.rs index a63bce7c4..a72874853 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -1152,27 +1152,34 @@ impl Raft { /// This uses a slightly modified "Joint Consensus" algorithm as detailed in Section 6 of the /// Raft paper. /// - /// The modification is as follows: We apply the change when a node *applies* the entry, not - /// when the entry is received. + /// We apply the change when a node *applies* the entry, not when the entry is received. /// - /// # Panics - /// - /// This **must** only be called on `Entry` which holds an `Entry` of with - /// `EntryType::ConfChange` and the `data` field being a serialized `ConfChange`. + /// # Contracts /// - /// This `ConfChange` must be of variant `BeginConfChange` and contain a `configuration` value. - // TODO: Make this return a result instead of panic. + /// * The `entry: Entry` is of type `EntryType::ConfChange` and the `data` field is a + /// `ConfChange` serialized by `protobuf::Message::write_to_bytes`. + /// * The `ConfChange` must be of type `BeginConfChange` and contain a `configuration` value. #[inline(always)] pub fn begin_membership_change(&mut self, entry: &Entry) -> Result<()> { - // TODO: Check if this should be rejected for normal reasons. - // Notably, if another is happening now. - assert_eq!(entry.get_entry_type(), EntryType::EntryConfChange); + if entry.get_entry_type() != EntryType::EntryConfChange { + return Err(Error::ViolatesContract( + format!("{:?} != EntryConfChange", entry.get_entry_type()) + )); + } let mut conf_change = protobuf::parse_from_bytes::(entry.get_data())?; - assert_eq!( - conf_change.get_change_type(), - ConfChangeType::BeginConfChange - ); - let configuration = conf_change.take_configuration(); + if conf_change.get_change_type() != ConfChangeType::BeginConfChange { + return Err(Error::ViolatesContract( + format!("{:?} != BeginConfChange", conf_change.get_change_type()) + )); + } + let configuration = if conf_change.has_configuration() { + conf_change.take_configuration() + } else { + return Err(Error::ViolatesContract( + "!ConfChange::has_configuration()".into() + )); + }; + self.set_began_conf_change_at(entry.get_index()); let max_inflights = self.max_inflight; self.mut_prs() @@ -1193,24 +1200,33 @@ impl Raft { /// This uses a slightly modified "Joint Consensus" algorithm as detailed in Section 6 of the /// Raft paper. /// - /// The modification is as follows: We apply the change when a node *applies* the entry, not - /// when the entry is received. + /// We apply the change when a node *applies* the entry, not when the entry is received. /// - /// # Panics - /// - /// This **must** only be called on `Entry` which holds an `Entry` of with - /// `EntryType::ConfChange` and the `data` field being a serialized `ConfChange`. + /// # Contracts /// - /// This `ConfChange` must be of variant `FinalizeConfChange` and contain no `configuration` - /// value. + /// * The `entry: Entry` is of type `EntryType::ConfChange` and the `data` field is a + /// `ConfChange` serialized by `protobuf::Message::write_to_bytes`. + /// * The `ConfChange` must be of type `FinalizeConfChange` and **not** contain a + /// `configuration` value. #[inline(always)] pub fn finalize_membership_change(&mut self, entry: &Entry) -> Result<()> { - assert_eq!(entry.get_entry_type(), EntryType::EntryConfChange); + if entry.get_entry_type() != EntryType::EntryConfChange { + return Err(Error::ViolatesContract( + format!("{:?} != EntryConfChange", entry.get_entry_type()) + )); + } let conf_change = protobuf::parse_from_bytes::(entry.get_data())?; - assert_eq!( - conf_change.get_change_type(), - ConfChangeType::FinalizeConfChange - ); + if conf_change.get_change_type() != ConfChangeType::BeginConfChange { + return Err(Error::ViolatesContract( + format!("{:?} != BeginConfChange", conf_change.get_change_type()) + )); + } + if conf_change.has_configuration() { + return Err(Error::ViolatesContract( + "ConfChange::has_configuration()".into() + )); + }; + // Joint Consensus, in the Raft paper, states the leader should step down and become a // follower if it is removed during a transition. let leader_in_new_set = self From c51e5cecaecb19bfff109b264276fe58650c03f6 Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Thu, 22 Nov 2018 15:12:07 -0800 Subject: [PATCH 13/41] Further documentation clarity --- src/progress.rs | 8 ++++---- src/raft.rs | 22 ++++++++++++---------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/progress.rs b/src/progress.rs index 67dc6b336..0f8ad87de 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -1200,16 +1200,16 @@ mod test_progress_set { start: (impl IntoIterator, impl IntoIterator), end: (impl IntoIterator, impl IntoIterator), ) -> Result<()> { - let start_voters = start.1 + let start_voters = start.0 .into_iter() .collect::>(); - let start_learners = start.0 + let start_learners = start.1 .into_iter() .collect::>(); - let end_voters = end.1 + let end_voters = end.0 .into_iter() .collect::>(); - let end_learners = end.0 + let end_learners = end.1 .into_iter() .collect::>(); let transition_voters = start_voters diff --git a/src/raft.rs b/src/raft.rs index a72874853..f4d299efc 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -562,8 +562,8 @@ impl Raft { trace!( "Skipping sending to {}, term: {:?}, ents: {:?}", to, - term.is_err(), - ents.is_err() + term, + ents, ); if !self.prepare_send_snapshot(&mut m, pr, to) { return; @@ -629,8 +629,11 @@ impl Raft { /// Commit that the Raft peer has applied up to the given index. /// - /// Registers the new applied index to the Raft log, then checks to see if it's time to - /// finalize a Joint Consensus state. + /// Registers the new applied index to the Raft log. + /// + /// # Hooks + /// + /// * Post: Checks to see if it's time to finalize a Joint Consensus state. pub fn commit_apply(&mut self, applied: u64) { #[allow(deprecated)] self.raft_log.applied_to(applied); @@ -1216,7 +1219,7 @@ impl Raft { )); } let conf_change = protobuf::parse_from_bytes::(entry.get_data())?; - if conf_change.get_change_type() != ConfChangeType::BeginConfChange { + if conf_change.get_change_type() != ConfChangeType::FinalizeConfChange { return Err(Error::ViolatesContract( format!("{:?} != BeginConfChange", conf_change.get_change_type()) )); @@ -2039,9 +2042,7 @@ impl Raft { self.prs().voter_ids().contains(&self.id) } - /// Begin the process of changing the cluster's peer configuration to a new one. - /// - /// This should only be called on the leader. + /// Propose that the peer group change it's active set to a new set. /// /// ```rust /// use raft::{Raft, Config, storage::MemStorage, eraftpb::ConfState}; @@ -2050,9 +2051,9 @@ impl Raft { /// peers: vec![1], /// ..Default::default() /// }; - /// let mut raft: Raft = Raft::new(&config, Default::default()).unwrap(); + /// let mut raft = Raft::new(&config, MemStorage::default()).unwrap(); /// raft.become_candidate(); - /// raft.become_leader(); + /// raft.become_leader(); // It must be a leader! /// /// let mut conf = ConfState::default(); /// conf.set_nodes(vec![1,2,3]); @@ -2064,6 +2065,7 @@ impl Raft { /// /// # Errors /// + /// * Peer this is called on is not leader. /// * `voters` and `learners` are not mutually exclusive. /// * `voters` is empty. pub fn propose_membership_change(&mut self, config: impl Into) -> Result<()> { From 617858fc19bc7397da02d284261fc9f25c30a29e Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Thu, 22 Nov 2018 15:21:22 -0800 Subject: [PATCH 14/41] fmt --- src/progress.rs | 78 +++++++++++++++--------------------------- src/raft.rs | 42 +++++++++++++---------- src/raw_node.rs | 6 ++-- tests/test_util/mod.rs | 1 + 4 files changed, 55 insertions(+), 72 deletions(-) diff --git a/src/progress.rs b/src/progress.rs index 0f8ad87de..e19913fb7 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -57,7 +57,7 @@ impl Default for ProgressState { #[derive(Clone, Debug, Default, PartialEq, Getters)] /// A Raft internal representation of a Configuration. -/// +/// /// This is corollary to a ConfState, but optimized for `contains` calls. pub struct Configuration { /// The voter set. @@ -141,7 +141,7 @@ impl Configuration { /// Returns whether or not the given `id` is a member of this configuration. // Allowed to maintain API consistency. See `HashSet::contains(&u64)`. - #[allow(trivially_copy_pass_by_ref)] + #[allow(trivially_copy_pass_by_ref)] pub fn contains(&self, id: &u64) -> bool { self.voters.contains(id) || self.learners.contains(id) } @@ -463,7 +463,7 @@ impl ProgressSet { (accepts, rejects) }, ); - + match self.next_configuration { Some(ref next) => { if next.has_quorum(&accepts) && self.configuration.has_quorum(&accepts) { @@ -471,7 +471,7 @@ impl ProgressSet { } else if next.has_quorum(&rejects) || self.configuration.has_quorum(&rejects) { return CandidacyStatus::Ineligible; } - }, + } None => { if self.configuration.has_quorum(&accepts) { return CandidacyStatus::Elected; @@ -565,7 +565,10 @@ impl ProgressSet { { Err(Error::Exists(demoted, "learners"))?; } - debug!("Beginning membership change. End configuration will be {:?}", next); + debug!( + "Beginning membership change. End configuration will be {:?}", + next + ); // When a peer is first added/promoted, we should mark it as recently active. // Otherwise, check_quorum may cause us to step down if it is invoked @@ -601,7 +604,10 @@ impl ProgressSet { } } self.configuration = next; - debug!("Finalizing membership change. Config is {:?}", self.configuration); + debug!( + "Finalizing membership change. Config is {:?}", + self.configuration + ); } } Ok(()) @@ -1134,84 +1140,56 @@ mod test_progress_set { #[test] fn test_membership_change_configuration_remove_voter() -> Result<()> { - check_membership_change_configuration( - (vec![1, 2], vec![]), - (vec![1], vec![]) - ) + check_membership_change_configuration((vec![1, 2], vec![]), (vec![1], vec![])) } #[test] fn test_membership_change_configuration_remove_learner() -> Result<()> { - check_membership_change_configuration( - (vec![1], vec![2]), - (vec![1], vec![]), - ) + check_membership_change_configuration((vec![1], vec![2]), (vec![1], vec![])) } #[test] fn test_membership_change_configuration_conflicting_sets() { - assert!(check_membership_change_configuration( - (vec![1], vec![]), - (vec![1], vec![1]), - ).is_err()) + assert!( + check_membership_change_configuration((vec![1], vec![]), (vec![1], vec![1]),).is_err() + ) } #[test] fn test_membership_change_configuration_empty_sets() { - assert!(check_membership_change_configuration( - (vec![], vec![]), - (vec![], vec![]) - ).is_err()) + assert!(check_membership_change_configuration((vec![], vec![]), (vec![], vec![])).is_err()) } #[test] fn test_membership_change_configuration_empty_voters() { - assert!(check_membership_change_configuration( - (vec![1], vec![]), - (vec![], vec![]), - ).is_err()) + assert!( + check_membership_change_configuration((vec![1], vec![]), (vec![], vec![]),).is_err() + ) } #[test] fn test_membership_change_configuration_add_voter() -> Result<()> { - check_membership_change_configuration( - (vec![1], vec![]), - (vec![1, 2], vec![]), - ) + check_membership_change_configuration((vec![1], vec![]), (vec![1, 2], vec![])) } #[test] fn test_membership_change_configuration_add_learner() -> Result<()> { - check_membership_change_configuration( - (vec![1], vec![]), - (vec![1], vec![2]), - ) + check_membership_change_configuration((vec![1], vec![]), (vec![1], vec![2])) } #[test] fn test_membership_change_configuration_promote_learner() -> Result<()> { - check_membership_change_configuration( - (vec![1], vec![2]), - (vec![1, 2], vec![]), - ) + check_membership_change_configuration((vec![1], vec![2]), (vec![1, 2], vec![])) } fn check_membership_change_configuration( start: (impl IntoIterator, impl IntoIterator), end: (impl IntoIterator, impl IntoIterator), ) -> Result<()> { - let start_voters = start.0 - .into_iter() - .collect::>(); - let start_learners = start.1 - .into_iter() - .collect::>(); - let end_voters = end.0 - .into_iter() - .collect::>(); - let end_learners = end.1 - .into_iter() - .collect::>(); + let start_voters = start.0.into_iter().collect::>(); + let start_learners = start.1.into_iter().collect::>(); + let end_voters = end.0.into_iter().collect::>(); + let end_learners = end.1.into_iter().collect::>(); let transition_voters = start_voters .union(&end_voters) .cloned() diff --git a/src/raft.rs b/src/raft.rs index f4d299efc..3a909fe3b 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -160,8 +160,8 @@ pub struct Raft { /// # Caveats /// /// It is important that whenever this is set that `pending_conf_index` is also set to the - /// value if it is greater than the existing value. - /// + /// value if it is greater than the existing value. + /// /// **Use `Raft::set_began_conf_change_at()` to change this value.** began_conf_change_at: Option, @@ -630,9 +630,9 @@ impl Raft { /// Commit that the Raft peer has applied up to the given index. /// /// Registers the new applied index to the Raft log. - /// - /// # Hooks - /// + /// + /// # Hooks + /// /// * Post: Checks to see if it's time to finalize a Joint Consensus state. pub fn commit_apply(&mut self, applied: u64) { #[allow(deprecated)] @@ -1165,21 +1165,23 @@ impl Raft { #[inline(always)] pub fn begin_membership_change(&mut self, entry: &Entry) -> Result<()> { if entry.get_entry_type() != EntryType::EntryConfChange { - return Err(Error::ViolatesContract( - format!("{:?} != EntryConfChange", entry.get_entry_type()) - )); + return Err(Error::ViolatesContract(format!( + "{:?} != EntryConfChange", + entry.get_entry_type() + ))); } let mut conf_change = protobuf::parse_from_bytes::(entry.get_data())?; if conf_change.get_change_type() != ConfChangeType::BeginConfChange { - return Err(Error::ViolatesContract( - format!("{:?} != BeginConfChange", conf_change.get_change_type()) - )); + return Err(Error::ViolatesContract(format!( + "{:?} != BeginConfChange", + conf_change.get_change_type() + ))); } let configuration = if conf_change.has_configuration() { conf_change.take_configuration() } else { return Err(Error::ViolatesContract( - "!ConfChange::has_configuration()".into() + "!ConfChange::has_configuration()".into(), )); }; @@ -1214,19 +1216,21 @@ impl Raft { #[inline(always)] pub fn finalize_membership_change(&mut self, entry: &Entry) -> Result<()> { if entry.get_entry_type() != EntryType::EntryConfChange { - return Err(Error::ViolatesContract( - format!("{:?} != EntryConfChange", entry.get_entry_type()) - )); + return Err(Error::ViolatesContract(format!( + "{:?} != EntryConfChange", + entry.get_entry_type() + ))); } let conf_change = protobuf::parse_from_bytes::(entry.get_data())?; if conf_change.get_change_type() != ConfChangeType::FinalizeConfChange { - return Err(Error::ViolatesContract( - format!("{:?} != BeginConfChange", conf_change.get_change_type()) - )); + return Err(Error::ViolatesContract(format!( + "{:?} != BeginConfChange", + conf_change.get_change_type() + ))); } if conf_change.has_configuration() { return Err(Error::ViolatesContract( - "ConfChange::has_configuration()".into() + "ConfChange::has_configuration()".into(), )); }; diff --git a/src/raw_node.rs b/src/raw_node.rs index 2d5d5c1e1..df45c200b 100644 --- a/src/raw_node.rs +++ b/src/raw_node.rs @@ -330,11 +330,11 @@ impl RawNode { } /// Takes the conf change and applies it. - /// + /// /// # Panics - /// + /// /// In the case of `BeginConfChange` or `FinalizeConfChange` returning errors this will panic. - /// + /// /// For a safe interface for these directly call `this.raft.begin_membership_change(entry)` or /// `this.raft.finalize_membership_change(entry)` respectively. pub fn apply_conf_change(&mut self, cc: &ConfChange) -> ConfState { diff --git a/tests/test_util/mod.rs b/tests/test_util/mod.rs index 089c99201..aad445b31 100644 --- a/tests/test_util/mod.rs +++ b/tests/test_util/mod.rs @@ -336,6 +336,7 @@ impl Network { } /// Dispatches the given messages to the appropriate peers. + /// /// Unlike `send` this does not gather and send any responses. It also does not ignore errors. pub fn dispatch(&mut self, messages: impl IntoIterator) -> Result<()> { for message in self.filter(messages) { From 7f35bd19e42cf4c8b53581f3f442bae185c5052d Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Thu, 29 Nov 2018 08:28:39 -0800 Subject: [PATCH 15/41] Resolve nits --- src/lib.rs | 9 ++++++--- src/progress.rs | 2 +- tests/test_util/mod.rs | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 873acb13a..0b8221418 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -116,7 +116,8 @@ then committed and at last applied to the state machine, so here we need a way t after the write is finished. One simple way is to use a unique ID for the client request, and save the associated callback -function in a hash map. When the log entry is applied, we can get the ID from the decoded entry, call the corresponding callback, and notify the client. +function in a hash map. When the log entry is applied, we can get the ID from the decoded entry, +call the corresponding callback, and notify the client. You can call the `step` function when you receive the Raft messages from other nodes. @@ -224,7 +225,8 @@ entries but has not been committed yet, we must append the entries to the Raft l ``` 3. Check whether `hs` is empty or not. If not empty, it means that the `HardState` of the node has -changed. For example, the node may vote for a new leader, or the commit index has been increased. We must persist the changed `HardState`: +changed. For example, the node may vote for a new leader, or the commit index has been increased. +We must persist the changed `HardState`: ```rust,ignore if let Some(hs) = ready.hs() { @@ -302,7 +304,8 @@ It (currently) does not: * Allow control of the replacement leader during a stepdown. * Optionally roll back a change during a peer group pause where the new peer group configuration fails. -* Provide automated promotion of newly added voters from learner to voter when they are caught up. This must be done as a two stage process for now. +* Provide automated promotion of newly added voters from learner to voter when they are caught up. +This must be done as a two stage process for now. > PRs to enable these are welcome! We'd love to mentor/support you through implementing it. diff --git a/src/progress.rs b/src/progress.rs index e19913fb7..a20e3342f 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -55,10 +55,10 @@ impl Default for ProgressState { } } -#[derive(Clone, Debug, Default, PartialEq, Getters)] /// A Raft internal representation of a Configuration. /// /// This is corollary to a ConfState, but optimized for `contains` calls. +#[derive(Clone, Debug, Default, PartialEq, Getters)] pub struct Configuration { /// The voter set. #[get = "pub"] diff --git a/tests/test_util/mod.rs b/tests/test_util/mod.rs index aad445b31..87a8b54a2 100644 --- a/tests/test_util/mod.rs +++ b/tests/test_util/mod.rs @@ -342,8 +342,8 @@ impl Network { for message in self.filter(messages) { let to = message.get_to(); let peer = self.peers.get_mut(&to).unwrap(); - let result = peer.step(message); - result?; + let result = peer.step(message)?; + result; } Ok(()) } From 0a5011f61fa5d0fa10b944ae604cdecbc2d77444 Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Fri, 30 Nov 2018 11:10:10 -0800 Subject: [PATCH 16/41] Resolve warning --- tests/test_util/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/test_util/mod.rs b/tests/test_util/mod.rs index 87a8b54a2..5a04d79c0 100644 --- a/tests/test_util/mod.rs +++ b/tests/test_util/mod.rs @@ -342,8 +342,7 @@ impl Network { for message in self.filter(messages) { let to = message.get_to(); let peer = self.peers.get_mut(&to).unwrap(); - let result = peer.step(message)?; - result; + peer.step(message)?; } Ok(()) } From 8964a1c4a052ec8eeb3e9bc144a0b3e1d551e736 Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Thu, 6 Dec 2018 09:18:38 -0800 Subject: [PATCH 17/41] Resolve minor nits. --- src/progress.rs | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/src/progress.rs b/src/progress.rs index a20e3342f..1f04a8bce 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -81,11 +81,12 @@ impl Configuration { } } -impl From<(I, I)> for Configuration +impl From<(Iter1, Iter2)> for Configuration where - I: IntoIterator, + Iter1: IntoIterator, + Iter2: IntoIterator, { - fn from((voters, learners): (I, I)) -> Self { + fn from((voters, learners): (Iter1, Iter2)) -> Self { Self { voters: voters.into_iter().collect(), learners: learners.into_iter().collect(), @@ -426,15 +427,13 @@ impl ProgressSet { let mut mci = matched[matched.len() / 2]; if let Some(next) = &self.next_configuration { - let mut matched = next - .voters() - .iter() - .map(|id| { - let peer = &self.progress[id]; - peer.matched - }).collect::>(); + matched.clear(); + next.voters().iter().for_each(|id| { + let peer = &self.progress[id]; + matched.push(peer.matched); + }); + // Reverse sort. matched.sort_by(|a, b| b.cmp(a)); - // Smallest that the majority has commited. let next_mci = matched[matched.len() / 2]; if next_mci < mci { mci = next_mci; @@ -563,7 +562,7 @@ impl ProgressSet { .intersection(&next.learners) .next() { - Err(Error::Exists(demoted, "learners"))?; + return Err(Error::Exists(demoted, "learners")); } debug!( "Beginning membership change. End configuration will be {:?}", From 77169b53b6f754019bff15ec15565a78bc721e56 Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Thu, 6 Dec 2018 09:40:31 -0800 Subject: [PATCH 18/41] Persist began_conf_change_at to hard state Add hard state test --- proto/eraftpb.proto | 3 + src/eraftpb.rs | 138 ++++++++++++------ src/raft.rs | 29 +++- .../test_membership_changes.rs | 110 +++++++++++++- 4 files changed, 217 insertions(+), 63 deletions(-) diff --git a/proto/eraftpb.proto b/proto/eraftpb.proto index 1edf77ca8..a656bde1f 100644 --- a/proto/eraftpb.proto +++ b/proto/eraftpb.proto @@ -80,6 +80,9 @@ message HardState { uint64 term = 1; uint64 vote = 2; uint64 commit = 3; + // HACK: Our current Protobuf uses `0` as `None`. + // Good thing we know that index 0 of the raft log is always not important in this context. + uint64 began_conf_change_at = 4; } message ConfState { diff --git a/src/eraftpb.rs b/src/eraftpb.rs index 699ec4e37..fb99eb4c4 100644 --- a/src/eraftpb.rs +++ b/src/eraftpb.rs @@ -163,7 +163,7 @@ impl ::protobuf::Message for Entry { let (field_number, wire_type) = is.read_tag_unpack()?; match field_number { 1 => { - ::protobuf::rt::read_proto3_enum_with_unknown_fields_into(wire_type, is, &mut self.entry_type, 1, &mut self.unknown_fields)? + if wire_type == ::protobuf::wire_format::WireTypeVarint {self.entry_type = is.read_enum()?;} else { return ::std::result::Result::Err(::protobuf::rt::unexpected_wire_type(wire_type)); } }, 2 => { if wire_type != ::protobuf::wire_format::WireTypeVarint { @@ -1095,7 +1095,7 @@ impl ::protobuf::Message for Message { let (field_number, wire_type) = is.read_tag_unpack()?; match field_number { 1 => { - ::protobuf::rt::read_proto3_enum_with_unknown_fields_into(wire_type, is, &mut self.msg_type, 1, &mut self.unknown_fields)? + if wire_type == ::protobuf::wire_format::WireTypeVarint {self.msg_type = is.read_enum()?;} else { return ::std::result::Result::Err(::protobuf::rt::unexpected_wire_type(wire_type)); } }, 2 => { if wire_type != ::protobuf::wire_format::WireTypeVarint { @@ -1416,6 +1416,7 @@ pub struct HardState { pub term: u64, pub vote: u64, pub commit: u64, + pub began_conf_change_at: u64, // special fields unknown_fields: ::protobuf::UnknownFields, cached_size: ::protobuf::CachedSize, @@ -1470,6 +1471,21 @@ impl HardState { pub fn get_commit(&self) -> u64 { self.commit } + + // uint64 began_conf_change_at = 4; + + pub fn clear_began_conf_change_at(&mut self) { + self.began_conf_change_at = 0; + } + + // Param is passed by value, moved + pub fn set_began_conf_change_at(&mut self, v: u64) { + self.began_conf_change_at = v; + } + + pub fn get_began_conf_change_at(&self) -> u64 { + self.began_conf_change_at + } } impl ::protobuf::Message for HardState { @@ -1502,6 +1518,13 @@ impl ::protobuf::Message for HardState { let tmp = is.read_uint64()?; self.commit = tmp; }, + 4 => { + if wire_type != ::protobuf::wire_format::WireTypeVarint { + return ::std::result::Result::Err(::protobuf::rt::unexpected_wire_type(wire_type)); + } + let tmp = is.read_uint64()?; + self.began_conf_change_at = tmp; + }, _ => { ::protobuf::rt::read_unknown_or_skip_group(field_number, wire_type, is, self.mut_unknown_fields())?; }, @@ -1523,6 +1546,9 @@ impl ::protobuf::Message for HardState { if self.commit != 0 { my_size += ::protobuf::rt::value_size(3, self.commit, ::protobuf::wire_format::WireTypeVarint); } + if self.began_conf_change_at != 0 { + my_size += ::protobuf::rt::value_size(4, self.began_conf_change_at, ::protobuf::wire_format::WireTypeVarint); + } my_size += ::protobuf::rt::unknown_fields_size(self.get_unknown_fields()); self.cached_size.set(my_size); my_size @@ -1538,6 +1564,9 @@ impl ::protobuf::Message for HardState { if self.commit != 0 { os.write_uint64(3, self.commit)?; } + if self.began_conf_change_at != 0 { + os.write_uint64(4, self.began_conf_change_at)?; + } os.write_unknown_fields(self.get_unknown_fields())?; ::std::result::Result::Ok(()) } @@ -1595,6 +1624,11 @@ impl ::protobuf::Message for HardState { |m: &HardState| { &m.commit }, |m: &mut HardState| { &mut m.commit }, )); + fields.push(::protobuf::reflect::accessor::make_simple_field_accessor::<_, ::protobuf::types::ProtobufTypeUint64>( + "began_conf_change_at", + |m: &HardState| { &m.began_conf_change_at }, + |m: &mut HardState| { &mut m.began_conf_change_at }, + )); ::protobuf::reflect::MessageDescriptor::new::( "HardState", fields, @@ -1620,6 +1654,7 @@ impl ::protobuf::Clear for HardState { self.clear_term(); self.clear_vote(); self.clear_commit(); + self.clear_began_conf_change_at(); self.unknown_fields.clear(); } } @@ -1984,7 +2019,7 @@ impl ::protobuf::Message for ConfChange { self.id = tmp; }, 2 => { - ::protobuf::rt::read_proto3_enum_with_unknown_fields_into(wire_type, is, &mut self.change_type, 2, &mut self.unknown_fields)? + if wire_type == ::protobuf::wire_format::WireTypeVarint {self.change_type = is.read_enum()?;} else { return ::std::result::Result::Err(::protobuf::rt::unexpected_wire_type(wire_type)); } }, 3 => { if wire_type != ::protobuf::wire_format::WireTypeVarint { @@ -2406,10 +2441,11 @@ static file_descriptor_proto_data: &'static [u8] = b"\ \x08snapshot\x18\t\x20\x01(\x0b2\x11.eraftpb.SnapshotR\x08snapshot\x12\ \x16\n\x06reject\x18\n\x20\x01(\x08R\x06reject\x12\x1f\n\x0breject_hint\ \x18\x0b\x20\x01(\x04R\nrejectHint\x12\x18\n\x07context\x18\x0c\x20\x01(\ - \x0cR\x07context\"K\n\tHardState\x12\x12\n\x04term\x18\x01\x20\x01(\x04R\ + \x0cR\x07context\"|\n\tHardState\x12\x12\n\x04term\x18\x01\x20\x01(\x04R\ \x04term\x12\x12\n\x04vote\x18\x02\x20\x01(\x04R\x04vote\x12\x16\n\x06co\ - mmit\x18\x03\x20\x01(\x04R\x06commit\"=\n\tConfState\x12\x14\n\x05nodes\ - \x18\x01\x20\x03(\x04R\x05nodes\x12\x1a\n\x08learners\x18\x02\x20\x03(\ + mmit\x18\x03\x20\x01(\x04R\x06commit\x12/\n\x14began_conf_change_at\x18\ + \x04\x20\x01(\x04R\x11beganConfChangeAt\"=\n\tConfState\x12\x14\n\x05nod\ + es\x18\x01\x20\x03(\x04R\x05nodes\x12\x1a\n\x08learners\x18\x02\x20\x03(\ \x04R\x08learners\"\xc3\x01\n\nConfChange\x12\x0e\n\x02id\x18\x01\x20\ \x01(\x04R\x02id\x128\n\x0bchange_type\x18\x02\x20\x01(\x0e2\x17.eraftpb\ .ConfChangeTypeR\nchangeType\x12\x17\n\x07node_id\x18\x03\x20\x01(\x04R\ @@ -2428,7 +2464,7 @@ static file_descriptor_proto_data: &'static [u8] = b"\ reVote\x10\x11\x12\x1d\n\x19MsgRequestPreVoteResponse\x10\x12*n\n\x0eCon\ fChangeType\x12\x0b\n\x07AddNode\x10\0\x12\x0e\n\nRemoveNode\x10\x01\x12\ \x12\n\x0eAddLearnerNode\x10\x02\x12\x13\n\x0fBeginConfChange\x10\x03\ - \x12\x16\n\x12FinalizeConfChange\x10\x04J\xe0\"\n\x06\x12\x04\0\0i\x01\n\ + \x12\x16\n\x12FinalizeConfChange\x10\x04J\xb4$\n\x06\x12\x04\0\0l\x01\n\ \x08\n\x01\x0c\x12\x03\0\0\x12\n\x08\n\x01\x02\x12\x03\x01\x08\x0f\n\n\n\ \x02\x05\0\x12\x04\x03\0\x06\x01\n\n\n\x03\x05\0\x01\x12\x03\x03\x05\x0e\ \n\x0b\n\x04\x05\0\x02\0\x12\x03\x04\x04\x14\n\x0c\n\x05\x05\0\x02\0\x01\ @@ -2570,7 +2606,7 @@ static file_descriptor_proto_data: &'static [u8] = b"\ \x12\x03J\x19\x1b\n\x0b\n\x04\x04\x03\x02\x0b\x12\x03K\x04\x17\n\r\n\x05\ \x04\x03\x02\x0b\x04\x12\x04K\x04J\x1c\n\x0c\n\x05\x04\x03\x02\x0b\x05\ \x12\x03K\x04\t\n\x0c\n\x05\x04\x03\x02\x0b\x01\x12\x03K\n\x11\n\x0c\n\ - \x05\x04\x03\x02\x0b\x03\x12\x03K\x14\x16\n\n\n\x02\x04\x04\x12\x04N\0R\ + \x05\x04\x03\x02\x0b\x03\x12\x03K\x14\x16\n\n\n\x02\x04\x04\x12\x04N\0U\ \x01\n\n\n\x03\x04\x04\x01\x12\x03N\x08\x11\n\x0b\n\x04\x04\x04\x02\0\ \x12\x03O\x04\x14\n\r\n\x05\x04\x04\x02\0\x04\x12\x04O\x04N\x13\n\x0c\n\ \x05\x04\x04\x02\0\x05\x12\x03O\x04\n\n\x0c\n\x05\x04\x04\x02\0\x01\x12\ @@ -2581,46 +2617,52 @@ static file_descriptor_proto_data: &'static [u8] = b"\ \x03P\x12\x13\n\x0b\n\x04\x04\x04\x02\x02\x12\x03Q\x04\x16\n\r\n\x05\x04\ \x04\x02\x02\x04\x12\x04Q\x04P\x14\n\x0c\n\x05\x04\x04\x02\x02\x05\x12\ \x03Q\x04\n\n\x0c\n\x05\x04\x04\x02\x02\x01\x12\x03Q\x0b\x11\n\x0c\n\x05\ - \x04\x04\x02\x02\x03\x12\x03Q\x14\x15\n\n\n\x02\x04\x05\x12\x04T\0W\x01\ - \n\n\n\x03\x04\x05\x01\x12\x03T\x08\x11\n\x0b\n\x04\x04\x05\x02\0\x12\ - \x03U\x04\x1e\n\x0c\n\x05\x04\x05\x02\0\x04\x12\x03U\x04\x0c\n\x0c\n\x05\ - \x04\x05\x02\0\x05\x12\x03U\r\x13\n\x0c\n\x05\x04\x05\x02\0\x01\x12\x03U\ - \x14\x19\n\x0c\n\x05\x04\x05\x02\0\x03\x12\x03U\x1c\x1d\n\x0b\n\x04\x04\ - \x05\x02\x01\x12\x03V\x04!\n\x0c\n\x05\x04\x05\x02\x01\x04\x12\x03V\x04\ - \x0c\n\x0c\n\x05\x04\x05\x02\x01\x05\x12\x03V\r\x13\n\x0c\n\x05\x04\x05\ - \x02\x01\x01\x12\x03V\x14\x1c\n\x0c\n\x05\x04\x05\x02\x01\x03\x12\x03V\ - \x1f\x20\n\n\n\x02\x05\x02\x12\x04Y\0_\x01\n\n\n\x03\x05\x02\x01\x12\x03\ - Y\x05\x13\n\x0b\n\x04\x05\x02\x02\0\x12\x03Z\x04\x13\n\x0c\n\x05\x05\x02\ - \x02\0\x01\x12\x03Z\x04\x0b\n\x0c\n\x05\x05\x02\x02\0\x02\x12\x03Z\x11\ - \x12\n\x0b\n\x04\x05\x02\x02\x01\x12\x03[\x04\x13\n\x0c\n\x05\x05\x02\ - \x02\x01\x01\x12\x03[\x04\x0e\n\x0c\n\x05\x05\x02\x02\x01\x02\x12\x03[\ - \x11\x12\n\x0b\n\x04\x05\x02\x02\x02\x12\x03\\\x04\x17\n\x0c\n\x05\x05\ - \x02\x02\x02\x01\x12\x03\\\x04\x12\n\x0c\n\x05\x05\x02\x02\x02\x02\x12\ - \x03\\\x15\x16\n\x0b\n\x04\x05\x02\x02\x03\x12\x03]\x04\x18\n\x0c\n\x05\ - \x05\x02\x02\x03\x01\x12\x03]\x04\x13\n\x0c\n\x05\x05\x02\x02\x03\x02\ - \x12\x03]\x16\x17\n\x0b\n\x04\x05\x02\x02\x04\x12\x03^\x04\x1b\n\x0c\n\ - \x05\x05\x02\x02\x04\x01\x12\x03^\x04\x16\n\x0c\n\x05\x05\x02\x02\x04\ - \x02\x12\x03^\x19\x1a\n\n\n\x02\x04\x06\x12\x04a\0i\x01\n\n\n\x03\x04\ - \x06\x01\x12\x03a\x08\x12\n\x0b\n\x04\x04\x06\x02\0\x12\x03b\x04\x12\n\r\ - \n\x05\x04\x06\x02\0\x04\x12\x04b\x04a\x14\n\x0c\n\x05\x04\x06\x02\0\x05\ - \x12\x03b\x04\n\n\x0c\n\x05\x04\x06\x02\0\x01\x12\x03b\x0b\r\n\x0c\n\x05\ - \x04\x06\x02\0\x03\x12\x03b\x10\x11\n\x0b\n\x04\x04\x06\x02\x01\x12\x03c\ - \x04#\n\r\n\x05\x04\x06\x02\x01\x04\x12\x04c\x04b\x12\n\x0c\n\x05\x04\ - \x06\x02\x01\x06\x12\x03c\x04\x12\n\x0c\n\x05\x04\x06\x02\x01\x01\x12\ - \x03c\x13\x1e\n\x0c\n\x05\x04\x06\x02\x01\x03\x12\x03c!\"\nE\n\x04\x04\ - \x06\x02\x02\x12\x03e\x04\x17\x1a8\x20Used\x20in\x20`AddNode`,\x20`Remov\ - eNode`,\x20and\x20`AddLearnerNode`.\n\n\r\n\x05\x04\x06\x02\x02\x04\x12\ - \x04e\x04c#\n\x0c\n\x05\x04\x06\x02\x02\x05\x12\x03e\x04\n\n\x0c\n\x05\ - \x04\x06\x02\x02\x01\x12\x03e\x0b\x12\n\x0c\n\x05\x04\x06\x02\x02\x03\ - \x12\x03e\x15\x16\n\x0b\n\x04\x04\x06\x02\x03\x12\x03f\x04\x16\n\r\n\x05\ - \x04\x06\x02\x03\x04\x12\x04f\x04e\x17\n\x0c\n\x05\x04\x06\x02\x03\x05\ - \x12\x03f\x04\t\n\x0c\n\x05\x04\x06\x02\x03\x01\x12\x03f\n\x11\n\x0c\n\ - \x05\x04\x06\x02\x03\x03\x12\x03f\x14\x15\nB\n\x04\x04\x06\x02\x04\x12\ - \x03h\x04\x20\x1a5\x20Used\x20in\x20`BeginConfChange`\x20and\x20`Finaliz\ - eConfChange`.\n\n\r\n\x05\x04\x06\x02\x04\x04\x12\x04h\x04f\x16\n\x0c\n\ - \x05\x04\x06\x02\x04\x06\x12\x03h\x04\r\n\x0c\n\x05\x04\x06\x02\x04\x01\ - \x12\x03h\x0e\x1b\n\x0c\n\x05\x04\x06\x02\x04\x03\x12\x03h\x1e\x1fb\x06p\ - roto3\ + \x04\x04\x02\x02\x03\x12\x03Q\x14\x15\n\x98\x01\n\x04\x04\x04\x02\x03\ + \x12\x03T\x04$\x1a\x8a\x01\x20HACK:\x20Our\x20current\x20Protobuf\x20use\ + s\x20`0`\x20as\x20`None`.\n\x20Good\x20thing\x20we\x20know\x20that\x20in\ + dex\x200\x20of\x20the\x20raft\x20log\x20is\x20always\x20not\x20important\ + \x20in\x20this\x20context.\n\n\r\n\x05\x04\x04\x02\x03\x04\x12\x04T\x04Q\ + \x16\n\x0c\n\x05\x04\x04\x02\x03\x05\x12\x03T\x04\n\n\x0c\n\x05\x04\x04\ + \x02\x03\x01\x12\x03T\x0b\x1f\n\x0c\n\x05\x04\x04\x02\x03\x03\x12\x03T\"\ + #\n\n\n\x02\x04\x05\x12\x04W\0Z\x01\n\n\n\x03\x04\x05\x01\x12\x03W\x08\ + \x11\n\x0b\n\x04\x04\x05\x02\0\x12\x03X\x04\x1e\n\x0c\n\x05\x04\x05\x02\ + \0\x04\x12\x03X\x04\x0c\n\x0c\n\x05\x04\x05\x02\0\x05\x12\x03X\r\x13\n\ + \x0c\n\x05\x04\x05\x02\0\x01\x12\x03X\x14\x19\n\x0c\n\x05\x04\x05\x02\0\ + \x03\x12\x03X\x1c\x1d\n\x0b\n\x04\x04\x05\x02\x01\x12\x03Y\x04!\n\x0c\n\ + \x05\x04\x05\x02\x01\x04\x12\x03Y\x04\x0c\n\x0c\n\x05\x04\x05\x02\x01\ + \x05\x12\x03Y\r\x13\n\x0c\n\x05\x04\x05\x02\x01\x01\x12\x03Y\x14\x1c\n\ + \x0c\n\x05\x04\x05\x02\x01\x03\x12\x03Y\x1f\x20\n\n\n\x02\x05\x02\x12\ + \x04\\\0b\x01\n\n\n\x03\x05\x02\x01\x12\x03\\\x05\x13\n\x0b\n\x04\x05\ + \x02\x02\0\x12\x03]\x04\x13\n\x0c\n\x05\x05\x02\x02\0\x01\x12\x03]\x04\ + \x0b\n\x0c\n\x05\x05\x02\x02\0\x02\x12\x03]\x11\x12\n\x0b\n\x04\x05\x02\ + \x02\x01\x12\x03^\x04\x13\n\x0c\n\x05\x05\x02\x02\x01\x01\x12\x03^\x04\ + \x0e\n\x0c\n\x05\x05\x02\x02\x01\x02\x12\x03^\x11\x12\n\x0b\n\x04\x05\ + \x02\x02\x02\x12\x03_\x04\x17\n\x0c\n\x05\x05\x02\x02\x02\x01\x12\x03_\ + \x04\x12\n\x0c\n\x05\x05\x02\x02\x02\x02\x12\x03_\x15\x16\n\x0b\n\x04\ + \x05\x02\x02\x03\x12\x03`\x04\x18\n\x0c\n\x05\x05\x02\x02\x03\x01\x12\ + \x03`\x04\x13\n\x0c\n\x05\x05\x02\x02\x03\x02\x12\x03`\x16\x17\n\x0b\n\ + \x04\x05\x02\x02\x04\x12\x03a\x04\x1b\n\x0c\n\x05\x05\x02\x02\x04\x01\ + \x12\x03a\x04\x16\n\x0c\n\x05\x05\x02\x02\x04\x02\x12\x03a\x19\x1a\n\n\n\ + \x02\x04\x06\x12\x04d\0l\x01\n\n\n\x03\x04\x06\x01\x12\x03d\x08\x12\n\ + \x0b\n\x04\x04\x06\x02\0\x12\x03e\x04\x12\n\r\n\x05\x04\x06\x02\0\x04\ + \x12\x04e\x04d\x14\n\x0c\n\x05\x04\x06\x02\0\x05\x12\x03e\x04\n\n\x0c\n\ + \x05\x04\x06\x02\0\x01\x12\x03e\x0b\r\n\x0c\n\x05\x04\x06\x02\0\x03\x12\ + \x03e\x10\x11\n\x0b\n\x04\x04\x06\x02\x01\x12\x03f\x04#\n\r\n\x05\x04\ + \x06\x02\x01\x04\x12\x04f\x04e\x12\n\x0c\n\x05\x04\x06\x02\x01\x06\x12\ + \x03f\x04\x12\n\x0c\n\x05\x04\x06\x02\x01\x01\x12\x03f\x13\x1e\n\x0c\n\ + \x05\x04\x06\x02\x01\x03\x12\x03f!\"\nE\n\x04\x04\x06\x02\x02\x12\x03h\ + \x04\x17\x1a8\x20Used\x20in\x20`AddNode`,\x20`RemoveNode`,\x20and\x20`Ad\ + dLearnerNode`.\n\n\r\n\x05\x04\x06\x02\x02\x04\x12\x04h\x04f#\n\x0c\n\ + \x05\x04\x06\x02\x02\x05\x12\x03h\x04\n\n\x0c\n\x05\x04\x06\x02\x02\x01\ + \x12\x03h\x0b\x12\n\x0c\n\x05\x04\x06\x02\x02\x03\x12\x03h\x15\x16\n\x0b\ + \n\x04\x04\x06\x02\x03\x12\x03i\x04\x16\n\r\n\x05\x04\x06\x02\x03\x04\ + \x12\x04i\x04h\x17\n\x0c\n\x05\x04\x06\x02\x03\x05\x12\x03i\x04\t\n\x0c\ + \n\x05\x04\x06\x02\x03\x01\x12\x03i\n\x11\n\x0c\n\x05\x04\x06\x02\x03\ + \x03\x12\x03i\x14\x15\nB\n\x04\x04\x06\x02\x04\x12\x03k\x04\x20\x1a5\x20\ + Used\x20in\x20`BeginConfChange`\x20and\x20`FinalizeConfChange`.\n\n\r\n\ + \x05\x04\x06\x02\x04\x04\x12\x04k\x04i\x16\n\x0c\n\x05\x04\x06\x02\x04\ + \x06\x12\x03k\x04\r\n\x0c\n\x05\x04\x06\x02\x04\x01\x12\x03k\x0e\x1b\n\ + \x0c\n\x05\x04\x06\x02\x04\x03\x12\x03k\x1e\x1fb\x06proto3\ "; static mut file_descriptor_proto_lazy: ::protobuf::lazy::Lazy<::protobuf::descriptor::FileDescriptorProto> = ::protobuf::lazy::Lazy { diff --git a/src/raft.rs b/src/raft.rs index da406784f..2c69cb10e 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -37,7 +37,7 @@ use rand::{self, Rng}; use super::errors::{Error, Result, StorageError}; use super::progress::{CandidacyStatus, Configuration, Progress, ProgressSet, ProgressState}; -use super::raft_log::{self, RaftLog}; +use super::raft_log::{self, RaftLog, NO_LIMIT}; use super::read_only::{ReadOnly, ReadOnlyOption, ReadState}; use super::storage::Storage; use super::Config; @@ -87,7 +87,7 @@ pub struct SoftState { /// A struct that represents the raft consensus itself. Stores details concerning the current /// and possible state the system can take. -#[derive(Default)] +#[derive(Default, Getters)] pub struct Raft { /// The current election term. pub term: u64, @@ -163,6 +163,7 @@ pub struct Raft { /// value if it is greater than the existing value. /// /// **Use `Raft::set_began_conf_change_at()` to change this value.** + #[get = "pub"] began_conf_change_at: Option, /// The queue of read-only requests. @@ -230,8 +231,8 @@ impl Raft { /// Creates a new raft for use on the node. pub fn new(c: &Config, store: T) -> Result> { c.validate()?; - let rs = store.initial_state()?; - let conf_state = &rs.conf_state; + let raft_state = store.initial_state()?; + let conf_state = &raft_state.conf_state; let raft_log = RaftLog::new(store, c.tag.clone()); let mut peers: &[u64] = &c.peers; let mut learners: &[u64] = &c.learners; @@ -294,8 +295,8 @@ impl Raft { } } - if rs.hard_state != HardState::new() { - r.load_state(&rs.hard_state); + if raft_state.hard_state != HardState::new() { + r.load_state(&raft_state.hard_state); } if c.applied > 0 { r.commit_apply(c.applied); @@ -360,6 +361,11 @@ impl Raft { hs.set_term(self.term); hs.set_vote(self.vote); hs.set_commit(self.raft_log.committed); + // HACK: Our current Protobuf uses `0` as `None`. + // Good thing we know that index 0 of the raft log is always not important in this context. + if let Some(began_conf_change_at) = self.began_conf_change_at() { + hs.set_began_conf_change_at(*began_conf_change_at); + } hs } @@ -415,7 +421,7 @@ impl Raft { self.began_conf_change_at = maybe_index; } - // send persists state to stable storage and then sends to its mailbox. + /// send persists state to stable storage and then sends to its mailbox. fn send(&mut self, mut m: Message) { debug!("Sending from {} to {}: {:?}", self.id, m.get_to(), m); m.set_from(self.id); @@ -2216,6 +2222,15 @@ impl Raft { self.raft_log.committed = hs.get_commit(); self.term = hs.get_term(); self.vote = hs.get_vote(); + // HACK: Our current Protobuf uses `0` as `None`. + // Good thing we know that index 0 of the raft log is always not important in this context. + let began_conf_change_at = hs.get_began_conf_change_at(); + if began_conf_change_at != 0 { + let entry = &self.get_store().entries(began_conf_change_at, began_conf_change_at + 1, NO_LIMIT) + .expect("Expected to find entry at location of last membership change.")[0]; + self.begin_membership_change(entry) + .expect("Expected the membership change already record to be valid."); + } } /// `pass_election_timeout` returns true iff `election_elapsed` is greater diff --git a/tests/integration_cases/test_membership_changes.rs b/tests/integration_cases/test_membership_changes.rs index a990958b1..b7c0a6fec 100644 --- a/tests/integration_cases/test_membership_changes.rs +++ b/tests/integration_cases/test_membership_changes.rs @@ -16,7 +16,7 @@ use fxhash::{FxHashMap, FxHashSet}; use protobuf::{self, RepeatedField}; use raft::{ eraftpb::{ConfChange, ConfChangeType, ConfState, Entry, EntryType, Message, MessageType}, - storage::MemStorage, + storage::{Storage, MemStorage}, Config, Configuration, Raft, Result, INVALID_ID, NO_LIMIT, }; use std::ops::{Deref, DerefMut}; @@ -533,6 +533,73 @@ mod three_peers_replace_voter { Ok(()) } + /// The leader power cycles before actually sending the messages. + #[test] + fn leader_power_cycles() -> Result<()> { + setup_for_test(); + let leader = 1; + let old_configuration = (vec![1, 2, 3], vec![]); + let new_configuration = (vec![1, 2, 4], vec![]); + let mut scenario = Scenario::new(leader, old_configuration, new_configuration)?; + scenario.spawn_new_peers()?; + scenario.propose_change_message()?; + + info!("Allowing quorum to commit"); + scenario.expect_read_and_dispatch_messages_from(&[1, 2, 3])?; + + info!("Advancing leader, now entered the joint"); + scenario.assert_can_apply_transition_entry_at_index( + &[1], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_membership_change(&[1]); + + info!("Leader replicates the commit and finalize entry."); + scenario.expect_read_and_dispatch_messages_from(&[1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[2, 3], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_membership_change(&[1, 2, 3]); + + info!("Leader power cycles."); + assert_eq!(scenario.peers[&1].began_conf_change_at(), &Some(2)); + scenario.power_cycle(&[1]); + assert_eq!(scenario.peers[&1].began_conf_change_at(), &Some(2)); + scenario.assert_in_membership_change(&[1]); + { + let peer = scenario.peers.get_mut(&1).unwrap(); + peer.become_candidate(); + peer.become_leader(); + for _ in peer.get_heartbeat_elapsed()..=(peer.get_heartbeat_timeout() + 1) { + peer.tick(); + } + } + + info!("Allowing new peers to catch up."); + scenario.expect_read_and_dispatch_messages_from(&[4, 1, 4, 1, 4, 1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[4], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_membership_change(&[1, 2, 3, 4]); + + info!("Cluster leaving the joint."); + scenario.expect_read_and_dispatch_messages_from(&[4, 3, 2, 1, 4, 3, 2])?; + assert_eq!(scenario.peers[&1].began_conf_change_at(), &Some(2)); + scenario.assert_can_apply_transition_entry_at_index( + &[1], + 3, + ConfChangeType::FinalizeConfChange, + ); + scenario.assert_not_in_membership_change(&[1, 2, 4]); + + Ok(()) + } + // Ensure if a peer in the old quorum fails, but the quorum is still big enough, it's ok. #[test] fn pending_delete_fails_after_begin() -> Result<()> { @@ -1235,19 +1302,23 @@ impl Scenario { protobuf::parse_from_bytes::(entry.get_data())?; if conf_change.get_change_type() == entry_type { found = true; - if entry_type == ConfChangeType::BeginConfChange { - peer.begin_membership_change(&entry)?; - } else if entry_type == ConfChangeType::FinalizeConfChange { - peer.finalize_membership_change(&entry)?; - } else if entry_type == ConfChangeType::AddNode { - peer.add_node(conf_change.get_node_id()); - } + match entry_type { + ConfChangeType::BeginConfChange => + peer.begin_membership_change(&entry)?, + ConfChangeType::FinalizeConfChange => + peer.finalize_membership_change(&entry)?, + ConfChangeType::AddNode => + peer.add_node(conf_change.get_node_id()), + _ => panic!("Unexpected conf change"), + }; } } if found { peer.raft_log.stable_to(entry.get_index(), entry.get_term()); peer.raft_log.commit_to(entry.get_index()); peer.commit_apply(entry.get_index()); + let hs = peer.hard_state(); + peer.mut_store().wl().set_hardstate(hs); peer.tick(); break; } @@ -1290,6 +1361,29 @@ impl Scenario { Ok(()) } + /// Simulate a power cycle in the given nodes. + /// + /// This means that the MemStorage is kept, but nothing else. + fn power_cycle<'a>( + &mut self, + peers: impl IntoIterator, + ) { + let peers = peers.into_iter().cloned(); + for id in peers { + debug!("Power cycling {}.", id); + let mut peer = self.peers.remove(&id).expect("Peer did not exist."); + let store = peer.mut_store().clone(); + let prs = peer.prs(); + let mut peer = Raft::new(&Config { + id, + peers: prs.voter_ids().iter().cloned().collect(), + learners: prs.learner_ids().iter().cloned().collect(), + ..Default::default() + }, store).expect("Could not create new Raft"); + self.peers.insert(id, peer.into()); + } + } + // Verify there is a transition entry at the given index of the given variant. fn assert_membership_change_entry_at<'a>( &self, From 146d74947414995a37d383d6cfb5cdf324e0814c Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Fri, 28 Dec 2018 14:16:27 -0800 Subject: [PATCH 19/41] Fix power cycle failure --- src/raft.rs | 15 +++++++++++++++ src/raft_log.rs | 1 + .../integration_cases/test_membership_changes.rs | 6 +++--- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/src/raft.rs b/src/raft.rs index 2c69cb10e..b39ec5962 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -800,6 +800,7 @@ impl Raft { /// /// Panics if a leader already exists. pub fn become_candidate(&mut self) { + trace!("ENTER become_candidate"); assert_ne!( self.state, StateRole::Leader, @@ -811,6 +812,7 @@ impl Raft { self.vote = id; self.state = StateRole::Candidate; info!("{} became candidate at term {}", self.tag, self.term); + trace!("EXIT become_candidate"); } /// Converts this node to a pre-candidate @@ -842,6 +844,7 @@ impl Raft { /// /// Panics if this is a follower node. pub fn become_leader(&mut self) { + trace!("ENTER become_leader"); assert_ne!( self.state, StateRole::Follower, @@ -867,7 +870,19 @@ impl Raft { self.pending_conf_index = self.raft_log.last_index(); self.append_entry(&mut [Entry::new()]); + // In most cases, we append only a new entry marked with an index and term. + // In the specific case of a node recovering while in the middle of a membership change, + // and the finalization entry may have been lost, we must also append that, since it + // would be overwritten by the term change. + if let Some(began) = self.began_conf_change_at { + trace!("Checking if we need to finalize again..., began: {}, applied: {}, committed: {}", began, self.raft_log.applied, self.raft_log.committed); + if began <= self.raft_log.committed { + self.append_finalize_conf_change_entry(); + } + } + info!("{} became leader at term {}", self.tag, self.term); + trace!("EXIT become_leader"); } fn num_pending_conf(&self, ents: &[Entry]) -> usize { diff --git a/src/raft_log.rs b/src/raft_log.rs index 887edb1c6..3a24027d1 100644 --- a/src/raft_log.rs +++ b/src/raft_log.rs @@ -297,6 +297,7 @@ impl RaftLog { /// Appends a set of entries to the unstable list. pub fn append(&mut self, ents: &[Entry]) -> u64 { + trace!("Entries being appended to unstable list: {:?}", ents); if ents.is_empty() { return self.last_index(); } diff --git a/tests/integration_cases/test_membership_changes.rs b/tests/integration_cases/test_membership_changes.rs index b7c0a6fec..4008be632 100644 --- a/tests/integration_cases/test_membership_changes.rs +++ b/tests/integration_cases/test_membership_changes.rs @@ -588,11 +588,11 @@ mod three_peers_replace_voter { scenario.assert_in_membership_change(&[1, 2, 3, 4]); info!("Cluster leaving the joint."); - scenario.expect_read_and_dispatch_messages_from(&[4, 3, 2, 1, 4, 3, 2])?; + scenario.expect_read_and_dispatch_messages_from(&[4, 3, 2, 1, 4, 3, 2, 1])?; assert_eq!(scenario.peers[&1].began_conf_change_at(), &Some(2)); scenario.assert_can_apply_transition_entry_at_index( - &[1], - 3, + &[1, 2, 3, 4], + 4, ConfChangeType::FinalizeConfChange, ); scenario.assert_not_in_membership_change(&[1, 2, 4]); From 42a14e15e58045cda85c158eb4fcb064ebab1fd9 Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Thu, 3 Jan 2019 13:32:55 -0800 Subject: [PATCH 20/41] Use ConfChange instead of Entry. Signed-off-by: Hoverbear --- proto/eraftpb.proto | 4 + src/eraftpb.rs | 490 ++++++++++-------- src/lib.rs | 8 +- src/raft.rs | 70 ++- src/raw_node.rs | 10 +- .../test_membership_changes.rs | 47 +- 6 files changed, 347 insertions(+), 282 deletions(-) diff --git a/proto/eraftpb.proto b/proto/eraftpb.proto index a656bde1f..7fb8805fb 100644 --- a/proto/eraftpb.proto +++ b/proto/eraftpb.proto @@ -106,4 +106,8 @@ message ConfChange { bytes context = 4; // Used in `BeginConfChange` and `FinalizeConfChange`. ConfState configuration = 5; + // Used in `BeginConfChange` and `FinalizeConfChange`. + // Because `RawNode::apply_conf_change` takes a `ConfChange` instead of an `Entry` we must + // include this index so it can be known. + uint64 start_index = 6; } diff --git a/src/eraftpb.rs b/src/eraftpb.rs index fb99eb4c4..ba107f3c3 100644 --- a/src/eraftpb.rs +++ b/src/eraftpb.rs @@ -1,4 +1,4 @@ -// This file is generated by rust-protobuf 2.0.4. Do not edit +// This file is generated by rust-protobuf 2.2.2. Do not edit // @generated // https://github.com/Manishearth/rust-clippy/issues/702 @@ -31,8 +31,8 @@ pub struct Entry { pub context: ::std::vec::Vec, pub sync_log: bool, // special fields - unknown_fields: ::protobuf::UnknownFields, - cached_size: ::protobuf::CachedSize, + pub unknown_fields: ::protobuf::UnknownFields, + pub cached_size: ::protobuf::CachedSize, } impl Entry { @@ -369,8 +369,8 @@ pub struct SnapshotMetadata { pub index: u64, pub term: u64, // special fields - unknown_fields: ::protobuf::UnknownFields, - cached_size: ::protobuf::CachedSize, + pub unknown_fields: ::protobuf::UnknownFields, + pub cached_size: ::protobuf::CachedSize, } impl SnapshotMetadata { @@ -616,8 +616,8 @@ pub struct Snapshot { pub data: ::std::vec::Vec, pub metadata: ::protobuf::SingularPtrField, // special fields - unknown_fields: ::protobuf::UnknownFields, - cached_size: ::protobuf::CachedSize, + pub unknown_fields: ::protobuf::UnknownFields, + pub cached_size: ::protobuf::CachedSize, } impl Snapshot { @@ -846,8 +846,8 @@ pub struct Message { pub reject_hint: u64, pub context: ::std::vec::Vec, // special fields - unknown_fields: ::protobuf::UnknownFields, - cached_size: ::protobuf::CachedSize, + pub unknown_fields: ::protobuf::UnknownFields, + pub cached_size: ::protobuf::CachedSize, } impl Message { @@ -1418,8 +1418,8 @@ pub struct HardState { pub commit: u64, pub began_conf_change_at: u64, // special fields - unknown_fields: ::protobuf::UnknownFields, - cached_size: ::protobuf::CachedSize, + pub unknown_fields: ::protobuf::UnknownFields, + pub cached_size: ::protobuf::CachedSize, } impl HardState { @@ -1677,8 +1677,8 @@ pub struct ConfState { pub nodes: ::std::vec::Vec, pub learners: ::std::vec::Vec, // special fields - unknown_fields: ::protobuf::UnknownFields, - cached_size: ::protobuf::CachedSize, + pub unknown_fields: ::protobuf::UnknownFields, + pub cached_size: ::protobuf::CachedSize, } impl ConfState { @@ -1882,9 +1882,10 @@ pub struct ConfChange { pub node_id: u64, pub context: ::std::vec::Vec, pub configuration: ::protobuf::SingularPtrField, + pub start_index: u64, // special fields - unknown_fields: ::protobuf::UnknownFields, - cached_size: ::protobuf::CachedSize, + pub unknown_fields: ::protobuf::UnknownFields, + pub cached_size: ::protobuf::CachedSize, } impl ConfChange { @@ -1995,6 +1996,21 @@ impl ConfChange { pub fn get_configuration(&self) -> &ConfState { self.configuration.as_ref().unwrap_or_else(|| ConfState::default_instance()) } + + // uint64 start_index = 6; + + pub fn clear_start_index(&mut self) { + self.start_index = 0; + } + + // Param is passed by value, moved + pub fn set_start_index(&mut self, v: u64) { + self.start_index = v; + } + + pub fn get_start_index(&self) -> u64 { + self.start_index + } } impl ::protobuf::Message for ConfChange { @@ -2034,6 +2050,13 @@ impl ::protobuf::Message for ConfChange { 5 => { ::protobuf::rt::read_singular_message_into(wire_type, is, &mut self.configuration)?; }, + 6 => { + if wire_type != ::protobuf::wire_format::WireTypeVarint { + return ::std::result::Result::Err(::protobuf::rt::unexpected_wire_type(wire_type)); + } + let tmp = is.read_uint64()?; + self.start_index = tmp; + }, _ => { ::protobuf::rt::read_unknown_or_skip_group(field_number, wire_type, is, self.mut_unknown_fields())?; }, @@ -2062,6 +2085,9 @@ impl ::protobuf::Message for ConfChange { let len = v.compute_size(); my_size += 1 + ::protobuf::rt::compute_raw_varint32_size(len) + len; } + if self.start_index != 0 { + my_size += ::protobuf::rt::value_size(6, self.start_index, ::protobuf::wire_format::WireTypeVarint); + } my_size += ::protobuf::rt::unknown_fields_size(self.get_unknown_fields()); self.cached_size.set(my_size); my_size @@ -2085,6 +2111,9 @@ impl ::protobuf::Message for ConfChange { os.write_raw_varint32(v.get_cached_size())?; v.write_to_with_cached_sizes(os)?; } + if self.start_index != 0 { + os.write_uint64(6, self.start_index)?; + } os.write_unknown_fields(self.get_unknown_fields())?; ::std::result::Result::Ok(()) } @@ -2152,6 +2181,11 @@ impl ::protobuf::Message for ConfChange { |m: &ConfChange| { &m.configuration }, |m: &mut ConfChange| { &mut m.configuration }, )); + fields.push(::protobuf::reflect::accessor::make_simple_field_accessor::<_, ::protobuf::types::ProtobufTypeUint64>( + "start_index", + |m: &ConfChange| { &m.start_index }, + |m: &mut ConfChange| { &mut m.start_index }, + )); ::protobuf::reflect::MessageDescriptor::new::( "ConfChange", fields, @@ -2179,6 +2213,7 @@ impl ::protobuf::Clear for ConfChange { self.clear_node_id(); self.clear_context(); self.clear_configuration(); + self.clear_start_index(); self.unknown_fields.clear(); } } @@ -2446,223 +2481,232 @@ static file_descriptor_proto_data: &'static [u8] = b"\ mmit\x18\x03\x20\x01(\x04R\x06commit\x12/\n\x14began_conf_change_at\x18\ \x04\x20\x01(\x04R\x11beganConfChangeAt\"=\n\tConfState\x12\x14\n\x05nod\ es\x18\x01\x20\x03(\x04R\x05nodes\x12\x1a\n\x08learners\x18\x02\x20\x03(\ - \x04R\x08learners\"\xc3\x01\n\nConfChange\x12\x0e\n\x02id\x18\x01\x20\ + \x04R\x08learners\"\xe4\x01\n\nConfChange\x12\x0e\n\x02id\x18\x01\x20\ \x01(\x04R\x02id\x128\n\x0bchange_type\x18\x02\x20\x01(\x0e2\x17.eraftpb\ .ConfChangeTypeR\nchangeType\x12\x17\n\x07node_id\x18\x03\x20\x01(\x04R\ \x06nodeId\x12\x18\n\x07context\x18\x04\x20\x01(\x0cR\x07context\x128\n\ \rconfiguration\x18\x05\x20\x01(\x0b2\x12.eraftpb.ConfStateR\rconfigurat\ - ion*1\n\tEntryType\x12\x0f\n\x0bEntryNormal\x10\0\x12\x13\n\x0fEntryConf\ - Change\x10\x01*\x8c\x03\n\x0bMessageType\x12\n\n\x06MsgHup\x10\0\x12\x0b\ - \n\x07MsgBeat\x10\x01\x12\x0e\n\nMsgPropose\x10\x02\x12\r\n\tMsgAppend\ - \x10\x03\x12\x15\n\x11MsgAppendResponse\x10\x04\x12\x12\n\x0eMsgRequestV\ - ote\x10\x05\x12\x1a\n\x16MsgRequestVoteResponse\x10\x06\x12\x0f\n\x0bMsg\ - Snapshot\x10\x07\x12\x10\n\x0cMsgHeartbeat\x10\x08\x12\x18\n\x14MsgHeart\ - beatResponse\x10\t\x12\x12\n\x0eMsgUnreachable\x10\n\x12\x11\n\rMsgSnapS\ - tatus\x10\x0b\x12\x12\n\x0eMsgCheckQuorum\x10\x0c\x12\x15\n\x11MsgTransf\ - erLeader\x10\r\x12\x11\n\rMsgTimeoutNow\x10\x0e\x12\x10\n\x0cMsgReadInde\ - x\x10\x0f\x12\x14\n\x10MsgReadIndexResp\x10\x10\x12\x15\n\x11MsgRequestP\ - reVote\x10\x11\x12\x1d\n\x19MsgRequestPreVoteResponse\x10\x12*n\n\x0eCon\ - fChangeType\x12\x0b\n\x07AddNode\x10\0\x12\x0e\n\nRemoveNode\x10\x01\x12\ - \x12\n\x0eAddLearnerNode\x10\x02\x12\x13\n\x0fBeginConfChange\x10\x03\ - \x12\x16\n\x12FinalizeConfChange\x10\x04J\xb4$\n\x06\x12\x04\0\0l\x01\n\ - \x08\n\x01\x0c\x12\x03\0\0\x12\n\x08\n\x01\x02\x12\x03\x01\x08\x0f\n\n\n\ - \x02\x05\0\x12\x04\x03\0\x06\x01\n\n\n\x03\x05\0\x01\x12\x03\x03\x05\x0e\ - \n\x0b\n\x04\x05\0\x02\0\x12\x03\x04\x04\x14\n\x0c\n\x05\x05\0\x02\0\x01\ + ion\x12\x1f\n\x0bstart_index\x18\x06\x20\x01(\x04R\nstartIndex*1\n\tEntr\ + yType\x12\x0f\n\x0bEntryNormal\x10\0\x12\x13\n\x0fEntryConfChange\x10\ + \x01*\x8c\x03\n\x0bMessageType\x12\n\n\x06MsgHup\x10\0\x12\x0b\n\x07MsgB\ + eat\x10\x01\x12\x0e\n\nMsgPropose\x10\x02\x12\r\n\tMsgAppend\x10\x03\x12\ + \x15\n\x11MsgAppendResponse\x10\x04\x12\x12\n\x0eMsgRequestVote\x10\x05\ + \x12\x1a\n\x16MsgRequestVoteResponse\x10\x06\x12\x0f\n\x0bMsgSnapshot\ + \x10\x07\x12\x10\n\x0cMsgHeartbeat\x10\x08\x12\x18\n\x14MsgHeartbeatResp\ + onse\x10\t\x12\x12\n\x0eMsgUnreachable\x10\n\x12\x11\n\rMsgSnapStatus\ + \x10\x0b\x12\x12\n\x0eMsgCheckQuorum\x10\x0c\x12\x15\n\x11MsgTransferLea\ + der\x10\r\x12\x11\n\rMsgTimeoutNow\x10\x0e\x12\x10\n\x0cMsgReadIndex\x10\ + \x0f\x12\x14\n\x10MsgReadIndexResp\x10\x10\x12\x15\n\x11MsgRequestPreVot\ + e\x10\x11\x12\x1d\n\x19MsgRequestPreVoteResponse\x10\x12*n\n\x0eConfChan\ + geType\x12\x0b\n\x07AddNode\x10\0\x12\x0e\n\nRemoveNode\x10\x01\x12\x12\ + \n\x0eAddLearnerNode\x10\x02\x12\x13\n\x0fBeginConfChange\x10\x03\x12\ + \x16\n\x12FinalizeConfChange\x10\x04J\xc7&\n\x06\x12\x04\0\0p\x01\n\x08\ + \n\x01\x0c\x12\x03\0\0\x12\n\x08\n\x01\x02\x12\x03\x01\x08\x0f\n\n\n\x02\ + \x05\0\x12\x04\x03\0\x06\x01\n\n\n\x03\x05\0\x01\x12\x03\x03\x05\x0e\n\ + \x0b\n\x04\x05\0\x02\0\x12\x03\x04\x04\x14\n\x0c\n\x05\x05\0\x02\0\x01\ \x12\x03\x04\x04\x0f\n\x0c\n\x05\x05\0\x02\0\x02\x12\x03\x04\x12\x13\n\ \x0b\n\x04\x05\0\x02\x01\x12\x03\x05\x04\x18\n\x0c\n\x05\x05\0\x02\x01\ \x01\x12\x03\x05\x04\x13\n\x0c\n\x05\x05\0\x02\x01\x02\x12\x03\x05\x16\ - \x17\n\xdd\x04\n\x02\x04\0\x12\x04\x12\0\x1c\x01\x1a\xd0\x04\x20The\x20e\ + \x17\n\xe7\x04\n\x02\x04\0\x12\x04\x12\0\x1c\x01\x1a\xda\x04\x20The\x20e\ ntry\x20is\x20a\x20type\x20of\x20change\x20that\x20needs\x20to\x20be\x20\ - applied.\x20It\x20contains\x20two\x20data\x20fields.\n\x20While\x20the\ + applied.\x20It\x20contains\x20two\x20data\x20fields.\r\n\x20While\x20the\ \x20fields\x20are\x20built\x20into\x20the\x20model;\x20their\x20usage\ - \x20is\x20determined\x20by\x20the\x20entry_type.\n\n\x20For\x20normal\ - \x20entries,\x20the\x20data\x20field\x20should\x20contain\x20the\x20data\ - \x20change\x20that\x20should\x20be\x20applied.\n\x20The\x20context\x20fi\ - eld\x20can\x20be\x20used\x20for\x20any\x20contextual\x20data\x20that\x20\ - might\x20be\x20relevant\x20to\x20the\n\x20application\x20of\x20the\x20da\ - ta.\n\n\x20For\x20configuration\x20changes,\x20the\x20data\x20will\x20co\ - ntain\x20the\x20ConfChange\x20message\x20and\x20the\n\x20context\x20will\ - \x20provide\x20anything\x20needed\x20to\x20assist\x20the\x20configuratio\ - n\x20change.\x20The\x20context\n\x20if\x20for\x20the\x20user\x20to\x20se\ - t\x20and\x20use\x20in\x20this\x20case.\n\n\n\n\x03\x04\0\x01\x12\x03\x12\ - \x08\r\n\x0b\n\x04\x04\0\x02\0\x12\x03\x13\x04\x1d\n\r\n\x05\x04\0\x02\0\ - \x04\x12\x04\x13\x04\x12\x0f\n\x0c\n\x05\x04\0\x02\0\x06\x12\x03\x13\x04\ - \r\n\x0c\n\x05\x04\0\x02\0\x01\x12\x03\x13\x0e\x18\n\x0c\n\x05\x04\0\x02\ - \0\x03\x12\x03\x13\x1b\x1c\n\x0b\n\x04\x04\0\x02\x01\x12\x03\x14\x04\x14\ - \n\r\n\x05\x04\0\x02\x01\x04\x12\x04\x14\x04\x13\x1d\n\x0c\n\x05\x04\0\ - \x02\x01\x05\x12\x03\x14\x04\n\n\x0c\n\x05\x04\0\x02\x01\x01\x12\x03\x14\ - \x0b\x0f\n\x0c\n\x05\x04\0\x02\x01\x03\x12\x03\x14\x12\x13\n\x0b\n\x04\ - \x04\0\x02\x02\x12\x03\x15\x04\x15\n\r\n\x05\x04\0\x02\x02\x04\x12\x04\ - \x15\x04\x14\x14\n\x0c\n\x05\x04\0\x02\x02\x05\x12\x03\x15\x04\n\n\x0c\n\ - \x05\x04\0\x02\x02\x01\x12\x03\x15\x0b\x10\n\x0c\n\x05\x04\0\x02\x02\x03\ - \x12\x03\x15\x13\x14\n\x0b\n\x04\x04\0\x02\x03\x12\x03\x16\x04\x13\n\r\n\ - \x05\x04\0\x02\x03\x04\x12\x04\x16\x04\x15\x15\n\x0c\n\x05\x04\0\x02\x03\ - \x05\x12\x03\x16\x04\t\n\x0c\n\x05\x04\0\x02\x03\x01\x12\x03\x16\n\x0e\n\ - \x0c\n\x05\x04\0\x02\x03\x03\x12\x03\x16\x11\x12\n\x0b\n\x04\x04\0\x02\ - \x04\x12\x03\x17\x04\x16\n\r\n\x05\x04\0\x02\x04\x04\x12\x04\x17\x04\x16\ - \x13\n\x0c\n\x05\x04\0\x02\x04\x05\x12\x03\x17\x04\t\n\x0c\n\x05\x04\0\ - \x02\x04\x01\x12\x03\x17\n\x11\n\x0c\n\x05\x04\0\x02\x04\x03\x12\x03\x17\ - \x14\x15\nm\n\x04\x04\0\x02\x05\x12\x03\x1b\x04\x16\x1a`\x20Deprecated!\ - \x20It\x20is\x20kept\x20for\x20backward\x20compatibility.\n\x20TODO:\x20\ - remove\x20it\x20in\x20the\x20next\x20major\x20release.\n\n\r\n\x05\x04\0\ - \x02\x05\x04\x12\x04\x1b\x04\x17\x16\n\x0c\n\x05\x04\0\x02\x05\x05\x12\ - \x03\x1b\x04\x08\n\x0c\n\x05\x04\0\x02\x05\x01\x12\x03\x1b\t\x11\n\x0c\n\ - \x05\x04\0\x02\x05\x03\x12\x03\x1b\x14\x15\n\n\n\x02\x04\x01\x12\x04\x1e\ - \0\"\x01\n\n\n\x03\x04\x01\x01\x12\x03\x1e\x08\x18\n\x0b\n\x04\x04\x01\ - \x02\0\x12\x03\x1f\x04\x1d\n\r\n\x05\x04\x01\x02\0\x04\x12\x04\x1f\x04\ - \x1e\x1a\n\x0c\n\x05\x04\x01\x02\0\x06\x12\x03\x1f\x04\r\n\x0c\n\x05\x04\ - \x01\x02\0\x01\x12\x03\x1f\x0e\x18\n\x0c\n\x05\x04\x01\x02\0\x03\x12\x03\ - \x1f\x1b\x1c\n\x0b\n\x04\x04\x01\x02\x01\x12\x03\x20\x04\x15\n\r\n\x05\ - \x04\x01\x02\x01\x04\x12\x04\x20\x04\x1f\x1d\n\x0c\n\x05\x04\x01\x02\x01\ - \x05\x12\x03\x20\x04\n\n\x0c\n\x05\x04\x01\x02\x01\x01\x12\x03\x20\x0b\ - \x10\n\x0c\n\x05\x04\x01\x02\x01\x03\x12\x03\x20\x13\x14\n\x0b\n\x04\x04\ - \x01\x02\x02\x12\x03!\x04\x14\n\r\n\x05\x04\x01\x02\x02\x04\x12\x04!\x04\ - \x20\x15\n\x0c\n\x05\x04\x01\x02\x02\x05\x12\x03!\x04\n\n\x0c\n\x05\x04\ - \x01\x02\x02\x01\x12\x03!\x0b\x0f\n\x0c\n\x05\x04\x01\x02\x02\x03\x12\ - \x03!\x12\x13\n\n\n\x02\x04\x02\x12\x04$\0'\x01\n\n\n\x03\x04\x02\x01\ - \x12\x03$\x08\x10\n\x0b\n\x04\x04\x02\x02\0\x12\x03%\x04\x13\n\r\n\x05\ - \x04\x02\x02\0\x04\x12\x04%\x04$\x12\n\x0c\n\x05\x04\x02\x02\0\x05\x12\ - \x03%\x04\t\n\x0c\n\x05\x04\x02\x02\0\x01\x12\x03%\n\x0e\n\x0c\n\x05\x04\ - \x02\x02\0\x03\x12\x03%\x11\x12\n\x0b\n\x04\x04\x02\x02\x01\x12\x03&\x04\ - \"\n\r\n\x05\x04\x02\x02\x01\x04\x12\x04&\x04%\x13\n\x0c\n\x05\x04\x02\ - \x02\x01\x06\x12\x03&\x04\x14\n\x0c\n\x05\x04\x02\x02\x01\x01\x12\x03&\ - \x15\x1d\n\x0c\n\x05\x04\x02\x02\x01\x03\x12\x03&\x20!\n\n\n\x02\x05\x01\ - \x12\x04)\0=\x01\n\n\n\x03\x05\x01\x01\x12\x03)\x05\x10\n\x0b\n\x04\x05\ - \x01\x02\0\x12\x03*\x04\x0f\n\x0c\n\x05\x05\x01\x02\0\x01\x12\x03*\x04\n\ - \n\x0c\n\x05\x05\x01\x02\0\x02\x12\x03*\r\x0e\n\x0b\n\x04\x05\x01\x02\ - \x01\x12\x03+\x04\x10\n\x0c\n\x05\x05\x01\x02\x01\x01\x12\x03+\x04\x0b\n\ - \x0c\n\x05\x05\x01\x02\x01\x02\x12\x03+\x0e\x0f\n\x0b\n\x04\x05\x01\x02\ - \x02\x12\x03,\x04\x13\n\x0c\n\x05\x05\x01\x02\x02\x01\x12\x03,\x04\x0e\n\ - \x0c\n\x05\x05\x01\x02\x02\x02\x12\x03,\x11\x12\n\x0b\n\x04\x05\x01\x02\ - \x03\x12\x03-\x04\x12\n\x0c\n\x05\x05\x01\x02\x03\x01\x12\x03-\x04\r\n\ - \x0c\n\x05\x05\x01\x02\x03\x02\x12\x03-\x10\x11\n\x0b\n\x04\x05\x01\x02\ - \x04\x12\x03.\x04\x1a\n\x0c\n\x05\x05\x01\x02\x04\x01\x12\x03.\x04\x15\n\ - \x0c\n\x05\x05\x01\x02\x04\x02\x12\x03.\x18\x19\n\x0b\n\x04\x05\x01\x02\ - \x05\x12\x03/\x04\x17\n\x0c\n\x05\x05\x01\x02\x05\x01\x12\x03/\x04\x12\n\ - \x0c\n\x05\x05\x01\x02\x05\x02\x12\x03/\x15\x16\n\x0b\n\x04\x05\x01\x02\ - \x06\x12\x030\x04\x1f\n\x0c\n\x05\x05\x01\x02\x06\x01\x12\x030\x04\x1a\n\ - \x0c\n\x05\x05\x01\x02\x06\x02\x12\x030\x1d\x1e\n\x0b\n\x04\x05\x01\x02\ - \x07\x12\x031\x04\x14\n\x0c\n\x05\x05\x01\x02\x07\x01\x12\x031\x04\x0f\n\ - \x0c\n\x05\x05\x01\x02\x07\x02\x12\x031\x12\x13\n\x0b\n\x04\x05\x01\x02\ - \x08\x12\x032\x04\x15\n\x0c\n\x05\x05\x01\x02\x08\x01\x12\x032\x04\x10\n\ - \x0c\n\x05\x05\x01\x02\x08\x02\x12\x032\x13\x14\n\x0b\n\x04\x05\x01\x02\ - \t\x12\x033\x04\x1d\n\x0c\n\x05\x05\x01\x02\t\x01\x12\x033\x04\x18\n\x0c\ - \n\x05\x05\x01\x02\t\x02\x12\x033\x1b\x1c\n\x0b\n\x04\x05\x01\x02\n\x12\ - \x034\x04\x18\n\x0c\n\x05\x05\x01\x02\n\x01\x12\x034\x04\x12\n\x0c\n\x05\ - \x05\x01\x02\n\x02\x12\x034\x15\x17\n\x0b\n\x04\x05\x01\x02\x0b\x12\x035\ - \x04\x17\n\x0c\n\x05\x05\x01\x02\x0b\x01\x12\x035\x04\x11\n\x0c\n\x05\ - \x05\x01\x02\x0b\x02\x12\x035\x14\x16\n\x0b\n\x04\x05\x01\x02\x0c\x12\ - \x036\x04\x18\n\x0c\n\x05\x05\x01\x02\x0c\x01\x12\x036\x04\x12\n\x0c\n\ - \x05\x05\x01\x02\x0c\x02\x12\x036\x15\x17\n\x0b\n\x04\x05\x01\x02\r\x12\ - \x037\x04\x1b\n\x0c\n\x05\x05\x01\x02\r\x01\x12\x037\x04\x15\n\x0c\n\x05\ - \x05\x01\x02\r\x02\x12\x037\x18\x1a\n\x0b\n\x04\x05\x01\x02\x0e\x12\x038\ - \x04\x17\n\x0c\n\x05\x05\x01\x02\x0e\x01\x12\x038\x04\x11\n\x0c\n\x05\ - \x05\x01\x02\x0e\x02\x12\x038\x14\x16\n\x0b\n\x04\x05\x01\x02\x0f\x12\ - \x039\x04\x16\n\x0c\n\x05\x05\x01\x02\x0f\x01\x12\x039\x04\x10\n\x0c\n\ - \x05\x05\x01\x02\x0f\x02\x12\x039\x13\x15\n\x0b\n\x04\x05\x01\x02\x10\ - \x12\x03:\x04\x1a\n\x0c\n\x05\x05\x01\x02\x10\x01\x12\x03:\x04\x14\n\x0c\ - \n\x05\x05\x01\x02\x10\x02\x12\x03:\x17\x19\n\x0b\n\x04\x05\x01\x02\x11\ - \x12\x03;\x04\x1b\n\x0c\n\x05\x05\x01\x02\x11\x01\x12\x03;\x04\x15\n\x0c\ - \n\x05\x05\x01\x02\x11\x02\x12\x03;\x18\x1a\n\x0b\n\x04\x05\x01\x02\x12\ - \x12\x03<\x04#\n\x0c\n\x05\x05\x01\x02\x12\x01\x12\x03<\x04\x1d\n\x0c\n\ - \x05\x05\x01\x02\x12\x02\x12\x03<\x20\"\n\n\n\x02\x04\x03\x12\x04?\0L\ - \x01\n\n\n\x03\x04\x03\x01\x12\x03?\x08\x0f\n\x0b\n\x04\x04\x03\x02\0\ - \x12\x03@\x04\x1d\n\r\n\x05\x04\x03\x02\0\x04\x12\x04@\x04?\x11\n\x0c\n\ - \x05\x04\x03\x02\0\x06\x12\x03@\x04\x0f\n\x0c\n\x05\x04\x03\x02\0\x01\ - \x12\x03@\x10\x18\n\x0c\n\x05\x04\x03\x02\0\x03\x12\x03@\x1b\x1c\n\x0b\n\ - \x04\x04\x03\x02\x01\x12\x03A\x04\x12\n\r\n\x05\x04\x03\x02\x01\x04\x12\ - \x04A\x04@\x1d\n\x0c\n\x05\x04\x03\x02\x01\x05\x12\x03A\x04\n\n\x0c\n\ - \x05\x04\x03\x02\x01\x01\x12\x03A\x0b\r\n\x0c\n\x05\x04\x03\x02\x01\x03\ - \x12\x03A\x10\x11\n\x0b\n\x04\x04\x03\x02\x02\x12\x03B\x04\x14\n\r\n\x05\ - \x04\x03\x02\x02\x04\x12\x04B\x04A\x12\n\x0c\n\x05\x04\x03\x02\x02\x05\ - \x12\x03B\x04\n\n\x0c\n\x05\x04\x03\x02\x02\x01\x12\x03B\x0b\x0f\n\x0c\n\ - \x05\x04\x03\x02\x02\x03\x12\x03B\x12\x13\n\x0b\n\x04\x04\x03\x02\x03\ - \x12\x03C\x04\x14\n\r\n\x05\x04\x03\x02\x03\x04\x12\x04C\x04B\x14\n\x0c\ - \n\x05\x04\x03\x02\x03\x05\x12\x03C\x04\n\n\x0c\n\x05\x04\x03\x02\x03\ - \x01\x12\x03C\x0b\x0f\n\x0c\n\x05\x04\x03\x02\x03\x03\x12\x03C\x12\x13\n\ - \x0b\n\x04\x04\x03\x02\x04\x12\x03D\x04\x18\n\r\n\x05\x04\x03\x02\x04\ - \x04\x12\x04D\x04C\x14\n\x0c\n\x05\x04\x03\x02\x04\x05\x12\x03D\x04\n\n\ - \x0c\n\x05\x04\x03\x02\x04\x01\x12\x03D\x0b\x13\n\x0c\n\x05\x04\x03\x02\ - \x04\x03\x12\x03D\x16\x17\n\x0b\n\x04\x04\x03\x02\x05\x12\x03E\x04\x15\n\ - \r\n\x05\x04\x03\x02\x05\x04\x12\x04E\x04D\x18\n\x0c\n\x05\x04\x03\x02\ - \x05\x05\x12\x03E\x04\n\n\x0c\n\x05\x04\x03\x02\x05\x01\x12\x03E\x0b\x10\ - \n\x0c\n\x05\x04\x03\x02\x05\x03\x12\x03E\x13\x14\n\x0b\n\x04\x04\x03\ - \x02\x06\x12\x03F\x04\x1f\n\x0c\n\x05\x04\x03\x02\x06\x04\x12\x03F\x04\ - \x0c\n\x0c\n\x05\x04\x03\x02\x06\x06\x12\x03F\r\x12\n\x0c\n\x05\x04\x03\ - \x02\x06\x01\x12\x03F\x13\x1a\n\x0c\n\x05\x04\x03\x02\x06\x03\x12\x03F\ - \x1d\x1e\n\x0b\n\x04\x04\x03\x02\x07\x12\x03G\x04\x16\n\r\n\x05\x04\x03\ - \x02\x07\x04\x12\x04G\x04F\x1f\n\x0c\n\x05\x04\x03\x02\x07\x05\x12\x03G\ - \x04\n\n\x0c\n\x05\x04\x03\x02\x07\x01\x12\x03G\x0b\x11\n\x0c\n\x05\x04\ - \x03\x02\x07\x03\x12\x03G\x14\x15\n\x0b\n\x04\x04\x03\x02\x08\x12\x03H\ - \x04\x1a\n\r\n\x05\x04\x03\x02\x08\x04\x12\x04H\x04G\x16\n\x0c\n\x05\x04\ - \x03\x02\x08\x06\x12\x03H\x04\x0c\n\x0c\n\x05\x04\x03\x02\x08\x01\x12\ - \x03H\r\x15\n\x0c\n\x05\x04\x03\x02\x08\x03\x12\x03H\x18\x19\n\x0b\n\x04\ - \x04\x03\x02\t\x12\x03I\x04\x15\n\r\n\x05\x04\x03\x02\t\x04\x12\x04I\x04\ - H\x1a\n\x0c\n\x05\x04\x03\x02\t\x05\x12\x03I\x04\x08\n\x0c\n\x05\x04\x03\ - \x02\t\x01\x12\x03I\t\x0f\n\x0c\n\x05\x04\x03\x02\t\x03\x12\x03I\x12\x14\ - \n\x0b\n\x04\x04\x03\x02\n\x12\x03J\x04\x1c\n\r\n\x05\x04\x03\x02\n\x04\ - \x12\x04J\x04I\x15\n\x0c\n\x05\x04\x03\x02\n\x05\x12\x03J\x04\n\n\x0c\n\ - \x05\x04\x03\x02\n\x01\x12\x03J\x0b\x16\n\x0c\n\x05\x04\x03\x02\n\x03\ - \x12\x03J\x19\x1b\n\x0b\n\x04\x04\x03\x02\x0b\x12\x03K\x04\x17\n\r\n\x05\ - \x04\x03\x02\x0b\x04\x12\x04K\x04J\x1c\n\x0c\n\x05\x04\x03\x02\x0b\x05\ - \x12\x03K\x04\t\n\x0c\n\x05\x04\x03\x02\x0b\x01\x12\x03K\n\x11\n\x0c\n\ - \x05\x04\x03\x02\x0b\x03\x12\x03K\x14\x16\n\n\n\x02\x04\x04\x12\x04N\0U\ - \x01\n\n\n\x03\x04\x04\x01\x12\x03N\x08\x11\n\x0b\n\x04\x04\x04\x02\0\ - \x12\x03O\x04\x14\n\r\n\x05\x04\x04\x02\0\x04\x12\x04O\x04N\x13\n\x0c\n\ - \x05\x04\x04\x02\0\x05\x12\x03O\x04\n\n\x0c\n\x05\x04\x04\x02\0\x01\x12\ - \x03O\x0b\x0f\n\x0c\n\x05\x04\x04\x02\0\x03\x12\x03O\x12\x13\n\x0b\n\x04\ - \x04\x04\x02\x01\x12\x03P\x04\x14\n\r\n\x05\x04\x04\x02\x01\x04\x12\x04P\ - \x04O\x14\n\x0c\n\x05\x04\x04\x02\x01\x05\x12\x03P\x04\n\n\x0c\n\x05\x04\ - \x04\x02\x01\x01\x12\x03P\x0b\x0f\n\x0c\n\x05\x04\x04\x02\x01\x03\x12\ - \x03P\x12\x13\n\x0b\n\x04\x04\x04\x02\x02\x12\x03Q\x04\x16\n\r\n\x05\x04\ - \x04\x02\x02\x04\x12\x04Q\x04P\x14\n\x0c\n\x05\x04\x04\x02\x02\x05\x12\ - \x03Q\x04\n\n\x0c\n\x05\x04\x04\x02\x02\x01\x12\x03Q\x0b\x11\n\x0c\n\x05\ - \x04\x04\x02\x02\x03\x12\x03Q\x14\x15\n\x98\x01\n\x04\x04\x04\x02\x03\ - \x12\x03T\x04$\x1a\x8a\x01\x20HACK:\x20Our\x20current\x20Protobuf\x20use\ - s\x20`0`\x20as\x20`None`.\n\x20Good\x20thing\x20we\x20know\x20that\x20in\ - dex\x200\x20of\x20the\x20raft\x20log\x20is\x20always\x20not\x20important\ - \x20in\x20this\x20context.\n\n\r\n\x05\x04\x04\x02\x03\x04\x12\x04T\x04Q\ - \x16\n\x0c\n\x05\x04\x04\x02\x03\x05\x12\x03T\x04\n\n\x0c\n\x05\x04\x04\ - \x02\x03\x01\x12\x03T\x0b\x1f\n\x0c\n\x05\x04\x04\x02\x03\x03\x12\x03T\"\ - #\n\n\n\x02\x04\x05\x12\x04W\0Z\x01\n\n\n\x03\x04\x05\x01\x12\x03W\x08\ - \x11\n\x0b\n\x04\x04\x05\x02\0\x12\x03X\x04\x1e\n\x0c\n\x05\x04\x05\x02\ - \0\x04\x12\x03X\x04\x0c\n\x0c\n\x05\x04\x05\x02\0\x05\x12\x03X\r\x13\n\ - \x0c\n\x05\x04\x05\x02\0\x01\x12\x03X\x14\x19\n\x0c\n\x05\x04\x05\x02\0\ - \x03\x12\x03X\x1c\x1d\n\x0b\n\x04\x04\x05\x02\x01\x12\x03Y\x04!\n\x0c\n\ - \x05\x04\x05\x02\x01\x04\x12\x03Y\x04\x0c\n\x0c\n\x05\x04\x05\x02\x01\ - \x05\x12\x03Y\r\x13\n\x0c\n\x05\x04\x05\x02\x01\x01\x12\x03Y\x14\x1c\n\ - \x0c\n\x05\x04\x05\x02\x01\x03\x12\x03Y\x1f\x20\n\n\n\x02\x05\x02\x12\ - \x04\\\0b\x01\n\n\n\x03\x05\x02\x01\x12\x03\\\x05\x13\n\x0b\n\x04\x05\ - \x02\x02\0\x12\x03]\x04\x13\n\x0c\n\x05\x05\x02\x02\0\x01\x12\x03]\x04\ - \x0b\n\x0c\n\x05\x05\x02\x02\0\x02\x12\x03]\x11\x12\n\x0b\n\x04\x05\x02\ - \x02\x01\x12\x03^\x04\x13\n\x0c\n\x05\x05\x02\x02\x01\x01\x12\x03^\x04\ - \x0e\n\x0c\n\x05\x05\x02\x02\x01\x02\x12\x03^\x11\x12\n\x0b\n\x04\x05\ - \x02\x02\x02\x12\x03_\x04\x17\n\x0c\n\x05\x05\x02\x02\x02\x01\x12\x03_\ - \x04\x12\n\x0c\n\x05\x05\x02\x02\x02\x02\x12\x03_\x15\x16\n\x0b\n\x04\ - \x05\x02\x02\x03\x12\x03`\x04\x18\n\x0c\n\x05\x05\x02\x02\x03\x01\x12\ - \x03`\x04\x13\n\x0c\n\x05\x05\x02\x02\x03\x02\x12\x03`\x16\x17\n\x0b\n\ - \x04\x05\x02\x02\x04\x12\x03a\x04\x1b\n\x0c\n\x05\x05\x02\x02\x04\x01\ - \x12\x03a\x04\x16\n\x0c\n\x05\x05\x02\x02\x04\x02\x12\x03a\x19\x1a\n\n\n\ - \x02\x04\x06\x12\x04d\0l\x01\n\n\n\x03\x04\x06\x01\x12\x03d\x08\x12\n\ - \x0b\n\x04\x04\x06\x02\0\x12\x03e\x04\x12\n\r\n\x05\x04\x06\x02\0\x04\ - \x12\x04e\x04d\x14\n\x0c\n\x05\x04\x06\x02\0\x05\x12\x03e\x04\n\n\x0c\n\ - \x05\x04\x06\x02\0\x01\x12\x03e\x0b\r\n\x0c\n\x05\x04\x06\x02\0\x03\x12\ - \x03e\x10\x11\n\x0b\n\x04\x04\x06\x02\x01\x12\x03f\x04#\n\r\n\x05\x04\ - \x06\x02\x01\x04\x12\x04f\x04e\x12\n\x0c\n\x05\x04\x06\x02\x01\x06\x12\ - \x03f\x04\x12\n\x0c\n\x05\x04\x06\x02\x01\x01\x12\x03f\x13\x1e\n\x0c\n\ - \x05\x04\x06\x02\x01\x03\x12\x03f!\"\nE\n\x04\x04\x06\x02\x02\x12\x03h\ - \x04\x17\x1a8\x20Used\x20in\x20`AddNode`,\x20`RemoveNode`,\x20and\x20`Ad\ - dLearnerNode`.\n\n\r\n\x05\x04\x06\x02\x02\x04\x12\x04h\x04f#\n\x0c\n\ - \x05\x04\x06\x02\x02\x05\x12\x03h\x04\n\n\x0c\n\x05\x04\x06\x02\x02\x01\ - \x12\x03h\x0b\x12\n\x0c\n\x05\x04\x06\x02\x02\x03\x12\x03h\x15\x16\n\x0b\ - \n\x04\x04\x06\x02\x03\x12\x03i\x04\x16\n\r\n\x05\x04\x06\x02\x03\x04\ - \x12\x04i\x04h\x17\n\x0c\n\x05\x04\x06\x02\x03\x05\x12\x03i\x04\t\n\x0c\ - \n\x05\x04\x06\x02\x03\x01\x12\x03i\n\x11\n\x0c\n\x05\x04\x06\x02\x03\ - \x03\x12\x03i\x14\x15\nB\n\x04\x04\x06\x02\x04\x12\x03k\x04\x20\x1a5\x20\ - Used\x20in\x20`BeginConfChange`\x20and\x20`FinalizeConfChange`.\n\n\r\n\ - \x05\x04\x06\x02\x04\x04\x12\x04k\x04i\x16\n\x0c\n\x05\x04\x06\x02\x04\ - \x06\x12\x03k\x04\r\n\x0c\n\x05\x04\x06\x02\x04\x01\x12\x03k\x0e\x1b\n\ - \x0c\n\x05\x04\x06\x02\x04\x03\x12\x03k\x1e\x1fb\x06proto3\ + \x20is\x20determined\x20by\x20the\x20entry_type.\r\n\r\n\x20For\x20norma\ + l\x20entries,\x20the\x20data\x20field\x20should\x20contain\x20the\x20dat\ + a\x20change\x20that\x20should\x20be\x20applied.\r\n\x20The\x20context\ + \x20field\x20can\x20be\x20used\x20for\x20any\x20contextual\x20data\x20th\ + at\x20might\x20be\x20relevant\x20to\x20the\r\n\x20application\x20of\x20t\ + he\x20data.\r\n\r\n\x20For\x20configuration\x20changes,\x20the\x20data\ + \x20will\x20contain\x20the\x20ConfChange\x20message\x20and\x20the\r\n\ + \x20context\x20will\x20provide\x20anything\x20needed\x20to\x20assist\x20\ + the\x20configuration\x20change.\x20The\x20context\r\n\x20if\x20for\x20th\ + e\x20user\x20to\x20set\x20and\x20use\x20in\x20this\x20case.\r\n\n\n\n\ + \x03\x04\0\x01\x12\x03\x12\x08\r\n\x0b\n\x04\x04\0\x02\0\x12\x03\x13\x04\ + \x1d\n\r\n\x05\x04\0\x02\0\x04\x12\x04\x13\x04\x12\x0f\n\x0c\n\x05\x04\0\ + \x02\0\x06\x12\x03\x13\x04\r\n\x0c\n\x05\x04\0\x02\0\x01\x12\x03\x13\x0e\ + \x18\n\x0c\n\x05\x04\0\x02\0\x03\x12\x03\x13\x1b\x1c\n\x0b\n\x04\x04\0\ + \x02\x01\x12\x03\x14\x04\x14\n\r\n\x05\x04\0\x02\x01\x04\x12\x04\x14\x04\ + \x13\x1d\n\x0c\n\x05\x04\0\x02\x01\x05\x12\x03\x14\x04\n\n\x0c\n\x05\x04\ + \0\x02\x01\x01\x12\x03\x14\x0b\x0f\n\x0c\n\x05\x04\0\x02\x01\x03\x12\x03\ + \x14\x12\x13\n\x0b\n\x04\x04\0\x02\x02\x12\x03\x15\x04\x15\n\r\n\x05\x04\ + \0\x02\x02\x04\x12\x04\x15\x04\x14\x14\n\x0c\n\x05\x04\0\x02\x02\x05\x12\ + \x03\x15\x04\n\n\x0c\n\x05\x04\0\x02\x02\x01\x12\x03\x15\x0b\x10\n\x0c\n\ + \x05\x04\0\x02\x02\x03\x12\x03\x15\x13\x14\n\x0b\n\x04\x04\0\x02\x03\x12\ + \x03\x16\x04\x13\n\r\n\x05\x04\0\x02\x03\x04\x12\x04\x16\x04\x15\x15\n\ + \x0c\n\x05\x04\0\x02\x03\x05\x12\x03\x16\x04\t\n\x0c\n\x05\x04\0\x02\x03\ + \x01\x12\x03\x16\n\x0e\n\x0c\n\x05\x04\0\x02\x03\x03\x12\x03\x16\x11\x12\ + \n\x0b\n\x04\x04\0\x02\x04\x12\x03\x17\x04\x16\n\r\n\x05\x04\0\x02\x04\ + \x04\x12\x04\x17\x04\x16\x13\n\x0c\n\x05\x04\0\x02\x04\x05\x12\x03\x17\ + \x04\t\n\x0c\n\x05\x04\0\x02\x04\x01\x12\x03\x17\n\x11\n\x0c\n\x05\x04\0\ + \x02\x04\x03\x12\x03\x17\x14\x15\no\n\x04\x04\0\x02\x05\x12\x03\x1b\x04\ + \x16\x1ab\x20Deprecated!\x20It\x20is\x20kept\x20for\x20backward\x20compa\ + tibility.\r\n\x20TODO:\x20remove\x20it\x20in\x20the\x20next\x20major\x20\ + release.\r\n\n\r\n\x05\x04\0\x02\x05\x04\x12\x04\x1b\x04\x17\x16\n\x0c\n\ + \x05\x04\0\x02\x05\x05\x12\x03\x1b\x04\x08\n\x0c\n\x05\x04\0\x02\x05\x01\ + \x12\x03\x1b\t\x11\n\x0c\n\x05\x04\0\x02\x05\x03\x12\x03\x1b\x14\x15\n\n\ + \n\x02\x04\x01\x12\x04\x1e\0\"\x01\n\n\n\x03\x04\x01\x01\x12\x03\x1e\x08\ + \x18\n\x0b\n\x04\x04\x01\x02\0\x12\x03\x1f\x04\x1d\n\r\n\x05\x04\x01\x02\ + \0\x04\x12\x04\x1f\x04\x1e\x1a\n\x0c\n\x05\x04\x01\x02\0\x06\x12\x03\x1f\ + \x04\r\n\x0c\n\x05\x04\x01\x02\0\x01\x12\x03\x1f\x0e\x18\n\x0c\n\x05\x04\ + \x01\x02\0\x03\x12\x03\x1f\x1b\x1c\n\x0b\n\x04\x04\x01\x02\x01\x12\x03\ + \x20\x04\x15\n\r\n\x05\x04\x01\x02\x01\x04\x12\x04\x20\x04\x1f\x1d\n\x0c\ + \n\x05\x04\x01\x02\x01\x05\x12\x03\x20\x04\n\n\x0c\n\x05\x04\x01\x02\x01\ + \x01\x12\x03\x20\x0b\x10\n\x0c\n\x05\x04\x01\x02\x01\x03\x12\x03\x20\x13\ + \x14\n\x0b\n\x04\x04\x01\x02\x02\x12\x03!\x04\x14\n\r\n\x05\x04\x01\x02\ + \x02\x04\x12\x04!\x04\x20\x15\n\x0c\n\x05\x04\x01\x02\x02\x05\x12\x03!\ + \x04\n\n\x0c\n\x05\x04\x01\x02\x02\x01\x12\x03!\x0b\x0f\n\x0c\n\x05\x04\ + \x01\x02\x02\x03\x12\x03!\x12\x13\n\n\n\x02\x04\x02\x12\x04$\0'\x01\n\n\ + \n\x03\x04\x02\x01\x12\x03$\x08\x10\n\x0b\n\x04\x04\x02\x02\0\x12\x03%\ + \x04\x13\n\r\n\x05\x04\x02\x02\0\x04\x12\x04%\x04$\x12\n\x0c\n\x05\x04\ + \x02\x02\0\x05\x12\x03%\x04\t\n\x0c\n\x05\x04\x02\x02\0\x01\x12\x03%\n\ + \x0e\n\x0c\n\x05\x04\x02\x02\0\x03\x12\x03%\x11\x12\n\x0b\n\x04\x04\x02\ + \x02\x01\x12\x03&\x04\"\n\r\n\x05\x04\x02\x02\x01\x04\x12\x04&\x04%\x13\ + \n\x0c\n\x05\x04\x02\x02\x01\x06\x12\x03&\x04\x14\n\x0c\n\x05\x04\x02\ + \x02\x01\x01\x12\x03&\x15\x1d\n\x0c\n\x05\x04\x02\x02\x01\x03\x12\x03&\ + \x20!\n\n\n\x02\x05\x01\x12\x04)\0=\x01\n\n\n\x03\x05\x01\x01\x12\x03)\ + \x05\x10\n\x0b\n\x04\x05\x01\x02\0\x12\x03*\x04\x0f\n\x0c\n\x05\x05\x01\ + \x02\0\x01\x12\x03*\x04\n\n\x0c\n\x05\x05\x01\x02\0\x02\x12\x03*\r\x0e\n\ + \x0b\n\x04\x05\x01\x02\x01\x12\x03+\x04\x10\n\x0c\n\x05\x05\x01\x02\x01\ + \x01\x12\x03+\x04\x0b\n\x0c\n\x05\x05\x01\x02\x01\x02\x12\x03+\x0e\x0f\n\ + \x0b\n\x04\x05\x01\x02\x02\x12\x03,\x04\x13\n\x0c\n\x05\x05\x01\x02\x02\ + \x01\x12\x03,\x04\x0e\n\x0c\n\x05\x05\x01\x02\x02\x02\x12\x03,\x11\x12\n\ + \x0b\n\x04\x05\x01\x02\x03\x12\x03-\x04\x12\n\x0c\n\x05\x05\x01\x02\x03\ + \x01\x12\x03-\x04\r\n\x0c\n\x05\x05\x01\x02\x03\x02\x12\x03-\x10\x11\n\ + \x0b\n\x04\x05\x01\x02\x04\x12\x03.\x04\x1a\n\x0c\n\x05\x05\x01\x02\x04\ + \x01\x12\x03.\x04\x15\n\x0c\n\x05\x05\x01\x02\x04\x02\x12\x03.\x18\x19\n\ + \x0b\n\x04\x05\x01\x02\x05\x12\x03/\x04\x17\n\x0c\n\x05\x05\x01\x02\x05\ + \x01\x12\x03/\x04\x12\n\x0c\n\x05\x05\x01\x02\x05\x02\x12\x03/\x15\x16\n\ + \x0b\n\x04\x05\x01\x02\x06\x12\x030\x04\x1f\n\x0c\n\x05\x05\x01\x02\x06\ + \x01\x12\x030\x04\x1a\n\x0c\n\x05\x05\x01\x02\x06\x02\x12\x030\x1d\x1e\n\ + \x0b\n\x04\x05\x01\x02\x07\x12\x031\x04\x14\n\x0c\n\x05\x05\x01\x02\x07\ + \x01\x12\x031\x04\x0f\n\x0c\n\x05\x05\x01\x02\x07\x02\x12\x031\x12\x13\n\ + \x0b\n\x04\x05\x01\x02\x08\x12\x032\x04\x15\n\x0c\n\x05\x05\x01\x02\x08\ + \x01\x12\x032\x04\x10\n\x0c\n\x05\x05\x01\x02\x08\x02\x12\x032\x13\x14\n\ + \x0b\n\x04\x05\x01\x02\t\x12\x033\x04\x1d\n\x0c\n\x05\x05\x01\x02\t\x01\ + \x12\x033\x04\x18\n\x0c\n\x05\x05\x01\x02\t\x02\x12\x033\x1b\x1c\n\x0b\n\ + \x04\x05\x01\x02\n\x12\x034\x04\x18\n\x0c\n\x05\x05\x01\x02\n\x01\x12\ + \x034\x04\x12\n\x0c\n\x05\x05\x01\x02\n\x02\x12\x034\x15\x17\n\x0b\n\x04\ + \x05\x01\x02\x0b\x12\x035\x04\x17\n\x0c\n\x05\x05\x01\x02\x0b\x01\x12\ + \x035\x04\x11\n\x0c\n\x05\x05\x01\x02\x0b\x02\x12\x035\x14\x16\n\x0b\n\ + \x04\x05\x01\x02\x0c\x12\x036\x04\x18\n\x0c\n\x05\x05\x01\x02\x0c\x01\ + \x12\x036\x04\x12\n\x0c\n\x05\x05\x01\x02\x0c\x02\x12\x036\x15\x17\n\x0b\ + \n\x04\x05\x01\x02\r\x12\x037\x04\x1b\n\x0c\n\x05\x05\x01\x02\r\x01\x12\ + \x037\x04\x15\n\x0c\n\x05\x05\x01\x02\r\x02\x12\x037\x18\x1a\n\x0b\n\x04\ + \x05\x01\x02\x0e\x12\x038\x04\x17\n\x0c\n\x05\x05\x01\x02\x0e\x01\x12\ + \x038\x04\x11\n\x0c\n\x05\x05\x01\x02\x0e\x02\x12\x038\x14\x16\n\x0b\n\ + \x04\x05\x01\x02\x0f\x12\x039\x04\x16\n\x0c\n\x05\x05\x01\x02\x0f\x01\ + \x12\x039\x04\x10\n\x0c\n\x05\x05\x01\x02\x0f\x02\x12\x039\x13\x15\n\x0b\ + \n\x04\x05\x01\x02\x10\x12\x03:\x04\x1a\n\x0c\n\x05\x05\x01\x02\x10\x01\ + \x12\x03:\x04\x14\n\x0c\n\x05\x05\x01\x02\x10\x02\x12\x03:\x17\x19\n\x0b\ + \n\x04\x05\x01\x02\x11\x12\x03;\x04\x1b\n\x0c\n\x05\x05\x01\x02\x11\x01\ + \x12\x03;\x04\x15\n\x0c\n\x05\x05\x01\x02\x11\x02\x12\x03;\x18\x1a\n\x0b\ + \n\x04\x05\x01\x02\x12\x12\x03<\x04#\n\x0c\n\x05\x05\x01\x02\x12\x01\x12\ + \x03<\x04\x1d\n\x0c\n\x05\x05\x01\x02\x12\x02\x12\x03<\x20\"\n\n\n\x02\ + \x04\x03\x12\x04?\0L\x01\n\n\n\x03\x04\x03\x01\x12\x03?\x08\x0f\n\x0b\n\ + \x04\x04\x03\x02\0\x12\x03@\x04\x1d\n\r\n\x05\x04\x03\x02\0\x04\x12\x04@\ + \x04?\x11\n\x0c\n\x05\x04\x03\x02\0\x06\x12\x03@\x04\x0f\n\x0c\n\x05\x04\ + \x03\x02\0\x01\x12\x03@\x10\x18\n\x0c\n\x05\x04\x03\x02\0\x03\x12\x03@\ + \x1b\x1c\n\x0b\n\x04\x04\x03\x02\x01\x12\x03A\x04\x12\n\r\n\x05\x04\x03\ + \x02\x01\x04\x12\x04A\x04@\x1d\n\x0c\n\x05\x04\x03\x02\x01\x05\x12\x03A\ + \x04\n\n\x0c\n\x05\x04\x03\x02\x01\x01\x12\x03A\x0b\r\n\x0c\n\x05\x04\ + \x03\x02\x01\x03\x12\x03A\x10\x11\n\x0b\n\x04\x04\x03\x02\x02\x12\x03B\ + \x04\x14\n\r\n\x05\x04\x03\x02\x02\x04\x12\x04B\x04A\x12\n\x0c\n\x05\x04\ + \x03\x02\x02\x05\x12\x03B\x04\n\n\x0c\n\x05\x04\x03\x02\x02\x01\x12\x03B\ + \x0b\x0f\n\x0c\n\x05\x04\x03\x02\x02\x03\x12\x03B\x12\x13\n\x0b\n\x04\ + \x04\x03\x02\x03\x12\x03C\x04\x14\n\r\n\x05\x04\x03\x02\x03\x04\x12\x04C\ + \x04B\x14\n\x0c\n\x05\x04\x03\x02\x03\x05\x12\x03C\x04\n\n\x0c\n\x05\x04\ + \x03\x02\x03\x01\x12\x03C\x0b\x0f\n\x0c\n\x05\x04\x03\x02\x03\x03\x12\ + \x03C\x12\x13\n\x0b\n\x04\x04\x03\x02\x04\x12\x03D\x04\x18\n\r\n\x05\x04\ + \x03\x02\x04\x04\x12\x04D\x04C\x14\n\x0c\n\x05\x04\x03\x02\x04\x05\x12\ + \x03D\x04\n\n\x0c\n\x05\x04\x03\x02\x04\x01\x12\x03D\x0b\x13\n\x0c\n\x05\ + \x04\x03\x02\x04\x03\x12\x03D\x16\x17\n\x0b\n\x04\x04\x03\x02\x05\x12\ + \x03E\x04\x15\n\r\n\x05\x04\x03\x02\x05\x04\x12\x04E\x04D\x18\n\x0c\n\ + \x05\x04\x03\x02\x05\x05\x12\x03E\x04\n\n\x0c\n\x05\x04\x03\x02\x05\x01\ + \x12\x03E\x0b\x10\n\x0c\n\x05\x04\x03\x02\x05\x03\x12\x03E\x13\x14\n\x0b\ + \n\x04\x04\x03\x02\x06\x12\x03F\x04\x1f\n\x0c\n\x05\x04\x03\x02\x06\x04\ + \x12\x03F\x04\x0c\n\x0c\n\x05\x04\x03\x02\x06\x06\x12\x03F\r\x12\n\x0c\n\ + \x05\x04\x03\x02\x06\x01\x12\x03F\x13\x1a\n\x0c\n\x05\x04\x03\x02\x06\ + \x03\x12\x03F\x1d\x1e\n\x0b\n\x04\x04\x03\x02\x07\x12\x03G\x04\x16\n\r\n\ + \x05\x04\x03\x02\x07\x04\x12\x04G\x04F\x1f\n\x0c\n\x05\x04\x03\x02\x07\ + \x05\x12\x03G\x04\n\n\x0c\n\x05\x04\x03\x02\x07\x01\x12\x03G\x0b\x11\n\ + \x0c\n\x05\x04\x03\x02\x07\x03\x12\x03G\x14\x15\n\x0b\n\x04\x04\x03\x02\ + \x08\x12\x03H\x04\x1a\n\r\n\x05\x04\x03\x02\x08\x04\x12\x04H\x04G\x16\n\ + \x0c\n\x05\x04\x03\x02\x08\x06\x12\x03H\x04\x0c\n\x0c\n\x05\x04\x03\x02\ + \x08\x01\x12\x03H\r\x15\n\x0c\n\x05\x04\x03\x02\x08\x03\x12\x03H\x18\x19\ + \n\x0b\n\x04\x04\x03\x02\t\x12\x03I\x04\x15\n\r\n\x05\x04\x03\x02\t\x04\ + \x12\x04I\x04H\x1a\n\x0c\n\x05\x04\x03\x02\t\x05\x12\x03I\x04\x08\n\x0c\ + \n\x05\x04\x03\x02\t\x01\x12\x03I\t\x0f\n\x0c\n\x05\x04\x03\x02\t\x03\ + \x12\x03I\x12\x14\n\x0b\n\x04\x04\x03\x02\n\x12\x03J\x04\x1c\n\r\n\x05\ + \x04\x03\x02\n\x04\x12\x04J\x04I\x15\n\x0c\n\x05\x04\x03\x02\n\x05\x12\ + \x03J\x04\n\n\x0c\n\x05\x04\x03\x02\n\x01\x12\x03J\x0b\x16\n\x0c\n\x05\ + \x04\x03\x02\n\x03\x12\x03J\x19\x1b\n\x0b\n\x04\x04\x03\x02\x0b\x12\x03K\ + \x04\x17\n\r\n\x05\x04\x03\x02\x0b\x04\x12\x04K\x04J\x1c\n\x0c\n\x05\x04\ + \x03\x02\x0b\x05\x12\x03K\x04\t\n\x0c\n\x05\x04\x03\x02\x0b\x01\x12\x03K\ + \n\x11\n\x0c\n\x05\x04\x03\x02\x0b\x03\x12\x03K\x14\x16\n\n\n\x02\x04\ + \x04\x12\x04N\0U\x01\n\n\n\x03\x04\x04\x01\x12\x03N\x08\x11\n\x0b\n\x04\ + \x04\x04\x02\0\x12\x03O\x04\x14\n\r\n\x05\x04\x04\x02\0\x04\x12\x04O\x04\ + N\x13\n\x0c\n\x05\x04\x04\x02\0\x05\x12\x03O\x04\n\n\x0c\n\x05\x04\x04\ + \x02\0\x01\x12\x03O\x0b\x0f\n\x0c\n\x05\x04\x04\x02\0\x03\x12\x03O\x12\ + \x13\n\x0b\n\x04\x04\x04\x02\x01\x12\x03P\x04\x14\n\r\n\x05\x04\x04\x02\ + \x01\x04\x12\x04P\x04O\x14\n\x0c\n\x05\x04\x04\x02\x01\x05\x12\x03P\x04\ + \n\n\x0c\n\x05\x04\x04\x02\x01\x01\x12\x03P\x0b\x0f\n\x0c\n\x05\x04\x04\ + \x02\x01\x03\x12\x03P\x12\x13\n\x0b\n\x04\x04\x04\x02\x02\x12\x03Q\x04\ + \x16\n\r\n\x05\x04\x04\x02\x02\x04\x12\x04Q\x04P\x14\n\x0c\n\x05\x04\x04\ + \x02\x02\x05\x12\x03Q\x04\n\n\x0c\n\x05\x04\x04\x02\x02\x01\x12\x03Q\x0b\ + \x11\n\x0c\n\x05\x04\x04\x02\x02\x03\x12\x03Q\x14\x15\n\x9a\x01\n\x04\ + \x04\x04\x02\x03\x12\x03T\x04$\x1a\x8c\x01\x20HACK:\x20Our\x20current\ + \x20Protobuf\x20uses\x20`0`\x20as\x20`None`.\r\n\x20Good\x20thing\x20we\ + \x20know\x20that\x20index\x200\x20of\x20the\x20raft\x20log\x20is\x20alwa\ + ys\x20not\x20important\x20in\x20this\x20context.\r\n\n\r\n\x05\x04\x04\ + \x02\x03\x04\x12\x04T\x04Q\x16\n\x0c\n\x05\x04\x04\x02\x03\x05\x12\x03T\ + \x04\n\n\x0c\n\x05\x04\x04\x02\x03\x01\x12\x03T\x0b\x1f\n\x0c\n\x05\x04\ + \x04\x02\x03\x03\x12\x03T\"#\n\n\n\x02\x04\x05\x12\x04W\0Z\x01\n\n\n\x03\ + \x04\x05\x01\x12\x03W\x08\x11\n\x0b\n\x04\x04\x05\x02\0\x12\x03X\x04\x1e\ + \n\x0c\n\x05\x04\x05\x02\0\x04\x12\x03X\x04\x0c\n\x0c\n\x05\x04\x05\x02\ + \0\x05\x12\x03X\r\x13\n\x0c\n\x05\x04\x05\x02\0\x01\x12\x03X\x14\x19\n\ + \x0c\n\x05\x04\x05\x02\0\x03\x12\x03X\x1c\x1d\n\x0b\n\x04\x04\x05\x02\ + \x01\x12\x03Y\x04!\n\x0c\n\x05\x04\x05\x02\x01\x04\x12\x03Y\x04\x0c\n\ + \x0c\n\x05\x04\x05\x02\x01\x05\x12\x03Y\r\x13\n\x0c\n\x05\x04\x05\x02\ + \x01\x01\x12\x03Y\x14\x1c\n\x0c\n\x05\x04\x05\x02\x01\x03\x12\x03Y\x1f\ + \x20\n\n\n\x02\x05\x02\x12\x04\\\0b\x01\n\n\n\x03\x05\x02\x01\x12\x03\\\ + \x05\x13\n\x0b\n\x04\x05\x02\x02\0\x12\x03]\x04\x13\n\x0c\n\x05\x05\x02\ + \x02\0\x01\x12\x03]\x04\x0b\n\x0c\n\x05\x05\x02\x02\0\x02\x12\x03]\x11\ + \x12\n\x0b\n\x04\x05\x02\x02\x01\x12\x03^\x04\x13\n\x0c\n\x05\x05\x02\ + \x02\x01\x01\x12\x03^\x04\x0e\n\x0c\n\x05\x05\x02\x02\x01\x02\x12\x03^\ + \x11\x12\n\x0b\n\x04\x05\x02\x02\x02\x12\x03_\x04\x17\n\x0c\n\x05\x05\ + \x02\x02\x02\x01\x12\x03_\x04\x12\n\x0c\n\x05\x05\x02\x02\x02\x02\x12\ + \x03_\x15\x16\n\x0b\n\x04\x05\x02\x02\x03\x12\x03`\x04\x18\n\x0c\n\x05\ + \x05\x02\x02\x03\x01\x12\x03`\x04\x13\n\x0c\n\x05\x05\x02\x02\x03\x02\ + \x12\x03`\x16\x17\n\x0b\n\x04\x05\x02\x02\x04\x12\x03a\x04\x1b\n\x0c\n\ + \x05\x05\x02\x02\x04\x01\x12\x03a\x04\x16\n\x0c\n\x05\x05\x02\x02\x04\ + \x02\x12\x03a\x19\x1a\n\n\n\x02\x04\x06\x12\x04d\0p\x01\n\n\n\x03\x04\ + \x06\x01\x12\x03d\x08\x12\n\x0b\n\x04\x04\x06\x02\0\x12\x03e\x04\x12\n\r\ + \n\x05\x04\x06\x02\0\x04\x12\x04e\x04d\x14\n\x0c\n\x05\x04\x06\x02\0\x05\ + \x12\x03e\x04\n\n\x0c\n\x05\x04\x06\x02\0\x01\x12\x03e\x0b\r\n\x0c\n\x05\ + \x04\x06\x02\0\x03\x12\x03e\x10\x11\n\x0b\n\x04\x04\x06\x02\x01\x12\x03f\ + \x04#\n\r\n\x05\x04\x06\x02\x01\x04\x12\x04f\x04e\x12\n\x0c\n\x05\x04\ + \x06\x02\x01\x06\x12\x03f\x04\x12\n\x0c\n\x05\x04\x06\x02\x01\x01\x12\ + \x03f\x13\x1e\n\x0c\n\x05\x04\x06\x02\x01\x03\x12\x03f!\"\nF\n\x04\x04\ + \x06\x02\x02\x12\x03h\x04\x17\x1a9\x20Used\x20in\x20`AddNode`,\x20`Remov\ + eNode`,\x20and\x20`AddLearnerNode`.\r\n\n\r\n\x05\x04\x06\x02\x02\x04\ + \x12\x04h\x04f#\n\x0c\n\x05\x04\x06\x02\x02\x05\x12\x03h\x04\n\n\x0c\n\ + \x05\x04\x06\x02\x02\x01\x12\x03h\x0b\x12\n\x0c\n\x05\x04\x06\x02\x02\ + \x03\x12\x03h\x15\x16\n\x0b\n\x04\x04\x06\x02\x03\x12\x03i\x04\x16\n\r\n\ + \x05\x04\x06\x02\x03\x04\x12\x04i\x04h\x17\n\x0c\n\x05\x04\x06\x02\x03\ + \x05\x12\x03i\x04\t\n\x0c\n\x05\x04\x06\x02\x03\x01\x12\x03i\n\x11\n\x0c\ + \n\x05\x04\x06\x02\x03\x03\x12\x03i\x14\x15\nC\n\x04\x04\x06\x02\x04\x12\ + \x03k\x04\x20\x1a6\x20Used\x20in\x20`BeginConfChange`\x20and\x20`Finaliz\ + eConfChange`.\r\n\n\r\n\x05\x04\x06\x02\x04\x04\x12\x04k\x04i\x16\n\x0c\ + \n\x05\x04\x06\x02\x04\x06\x12\x03k\x04\r\n\x0c\n\x05\x04\x06\x02\x04\ + \x01\x12\x03k\x0e\x1b\n\x0c\n\x05\x04\x06\x02\x04\x03\x12\x03k\x1e\x1f\n\ + \xc7\x01\n\x04\x04\x06\x02\x05\x12\x03o\x04\x1b\x1a\xb9\x01\x20Used\x20i\ + n\x20`BeginConfChange`\x20and\x20`FinalizeConfChange`.\r\n\x20Because\ + \x20`RawNode::apply_conf_change`\x20takes\x20a\x20`ConfChange`\x20instea\ + d\x20of\x20an\x20`Entry`\x20we\x20must\r\n\x20include\x20this\x20index\ + \x20so\x20it\x20can\x20be\x20known.\r\n\n\r\n\x05\x04\x06\x02\x05\x04\ + \x12\x04o\x04k\x20\n\x0c\n\x05\x04\x06\x02\x05\x05\x12\x03o\x04\n\n\x0c\ + \n\x05\x04\x06\x02\x05\x01\x12\x03o\x0b\x16\n\x0c\n\x05\x04\x06\x02\x05\ + \x03\x12\x03o\x19\x1ab\x06proto3\ "; static mut file_descriptor_proto_lazy: ::protobuf::lazy::Lazy<::protobuf::descriptor::FileDescriptorProto> = ::protobuf::lazy::Lazy { diff --git a/src/lib.rs b/src/lib.rs index 0b8221418..9974eb8ea 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -312,7 +312,7 @@ This must be done as a two stage process for now. This means it's possible to do: ```rust -use raft::{Config, storage::MemStorage, raw_node::RawNode, eraftpb::Message}; +use raft::{Config, storage::MemStorage, raw_node::RawNode, eraftpb::{Message, ConfChange}}; let config = Config { id: 1, peers: vec![1], ..Default::default() }; let mut node = RawNode::new(&config, MemStorage::default(), vec![]).unwrap(); node.raft.become_candidate(); @@ -329,7 +329,8 @@ node.raft.propose_membership_change(( # let entry = &node.raft.raft_log.entries(2, 1).unwrap()[0]; // ...Later when the begin entry is ready to apply: -node.raft.begin_membership_change(entry).unwrap(); +let conf_change = protobuf::parse_from_bytes::(entry.get_data()).unwrap(); +node.raft.begin_membership_change(&conf_change).unwrap(); assert!(node.raft.is_in_membership_change()); # # // We hide this since the user isn't really encouraged to blindly call this, but we'd like a short @@ -338,7 +339,8 @@ assert!(node.raft.is_in_membership_change()); # # let entry = &node.raft.raft_log.entries(3, 1).unwrap()[0]; // ...Later, when the finalize entry is ready to apply: -node.raft.finalize_membership_change(entry).unwrap(); +let conf_change = protobuf::parse_from_bytes::(entry.get_data()).unwrap(); +node.raft.finalize_membership_change(&conf_change).unwrap(); assert!(node.raft.prs().voter_ids().contains(&2)); assert!(!node.raft.is_in_membership_change()); ``` diff --git a/src/raft.rs b/src/raft.rs index b39ec5962..f2de04f53 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -1170,7 +1170,7 @@ impl Raft { Ok(()) } - /// Apply a `BeginConfChange` entry. + /// Apply a `BeginConfChange` variant `ConfChange`. /// /// When a Raft node applies this variant of a configuration change it will adopt a joint /// configuration state until the membership change is finalized. @@ -1187,18 +1187,12 @@ impl Raft { /// /// # Contracts /// - /// * The `entry: Entry` is of type `EntryType::ConfChange` and the `data` field is a - /// `ConfChange` serialized by `protobuf::Message::write_to_bytes`. - /// * The `ConfChange` must be of type `BeginConfChange` and contain a `configuration` value. + /// * The `ConfChange.change_type` must be a `BeginConfChange` + /// * The `ConfChange.configuration` value must exist. + /// * The `ConfChange.start_index` value must exist. It should equal the index of the + /// corresponding entry. #[inline(always)] - pub fn begin_membership_change(&mut self, entry: &Entry) -> Result<()> { - if entry.get_entry_type() != EntryType::EntryConfChange { - return Err(Error::ViolatesContract(format!( - "{:?} != EntryConfChange", - entry.get_entry_type() - ))); - } - let mut conf_change = protobuf::parse_from_bytes::(entry.get_data())?; + pub fn begin_membership_change(&mut self, conf_change: &ConfChange) -> Result<()> { if conf_change.get_change_type() != ConfChangeType::BeginConfChange { return Err(Error::ViolatesContract(format!( "{:?} != BeginConfChange", @@ -1206,21 +1200,28 @@ impl Raft { ))); } let configuration = if conf_change.has_configuration() { - conf_change.take_configuration() + conf_change.get_configuration().clone() } else { return Err(Error::ViolatesContract( "!ConfChange::has_configuration()".into(), )); }; + let start_index = if conf_change.get_start_index() != 0 { + conf_change.get_start_index() + } else { + return Err(Error::ViolatesContract( + "!ConfChange::has_start_index()".into(), + )); + }; - self.set_began_conf_change_at(entry.get_index()); + self.set_began_conf_change_at(start_index); let max_inflights = self.max_inflight; self.mut_prs() .begin_membership_change(configuration, Progress::new(1, max_inflights))?; Ok(()) } - /// Apply a `FinalizeConfChange` entry. + /// Apply a `FinalizeConfChange` variant `ConfChange`. /// /// When a Raft node applies this variant of a configuration change it will finalize the /// transition begun by [`begin_membership_change`]. @@ -1235,21 +1236,14 @@ impl Raft { /// /// We apply the change when a node *applies* the entry, not when the entry is received. /// - /// # Contracts + /// # Contracts /// - /// * The `entry: Entry` is of type `EntryType::ConfChange` and the `data` field is a - /// `ConfChange` serialized by `protobuf::Message::write_to_bytes`. - /// * The `ConfChange` must be of type `FinalizeConfChange` and **not** contain a - /// `configuration` value. + /// * The Raft should already have started a configuration change with `begin_membership_change`. + /// * The `ConfChange.change_type` must be a `FinalizeConfChange`. + /// * The `ConfChange.configuration` value should not exist. (Panics in debug mode.) + /// * The `ConfChange.start_index` value should not exist. (Panics in debug mode.) #[inline(always)] - pub fn finalize_membership_change(&mut self, entry: &Entry) -> Result<()> { - if entry.get_entry_type() != EntryType::EntryConfChange { - return Err(Error::ViolatesContract(format!( - "{:?} != EntryConfChange", - entry.get_entry_type() - ))); - } - let conf_change = protobuf::parse_from_bytes::(entry.get_data())?; + pub fn finalize_membership_change(&mut self, conf_change: &ConfChange) -> Result<()> { if conf_change.get_change_type() != ConfChangeType::FinalizeConfChange { return Err(Error::ViolatesContract(format!( "{:?} != BeginConfChange", @@ -1261,15 +1255,15 @@ impl Raft { "ConfChange::has_configuration()".into(), )); }; - - // Joint Consensus, in the Raft paper, states the leader should step down and become a - // follower if it is removed during a transition. - let leader_in_new_set = self + let leader_in_new_set = self .prs() .next_configuration() .as_ref() .map(|config| config.contains(&self.leader_id)) .ok_or_else(|| Error::NoPendingMembershipChange)?; + + // Joint Consensus, in the Raft paper, states the leader should step down and become a + // follower if it is removed during a transition. if !leader_in_new_set { let last_term = self.raft_log.last_term(); if self.state == StateRole::Leader { @@ -2071,7 +2065,7 @@ impl Raft { self.prs().voter_ids().contains(&self.id) } - /// Propose that the peer group change it's active set to a new set. + /// Propose that the peer group change its active set to a new set. /// /// ```rust /// use raft::{Raft, Config, storage::MemStorage, eraftpb::ConfState}; @@ -2108,10 +2102,12 @@ impl Raft { config.voters(), config.learners() ); + let destination_index = self.raft_log.last_index() + 1; // Prep a configuration change to append. let mut conf_change = ConfChange::new(); conf_change.set_change_type(ConfChangeType::BeginConfChange); conf_change.set_configuration(config.into()); + conf_change.set_start_index(destination_index); let data = protobuf::Message::write_to_bytes(&conf_change)?; let mut entry = Entry::new(); entry.set_entry_type(EntryType::EntryConfChange); @@ -2119,7 +2115,7 @@ impl Raft { let mut message = Message::new(); message.set_msg_type(MessageType::MsgPropose); message.set_from(self.id); - message.set_index(self.raft_log.last_index() + 1); + message.set_index(destination_index); message.set_entries(RepeatedField::from_vec(vec![entry])); // `append_entry` sets term, index for us. self.step(message)?; @@ -2243,8 +2239,10 @@ impl Raft { if began_conf_change_at != 0 { let entry = &self.get_store().entries(began_conf_change_at, began_conf_change_at + 1, NO_LIMIT) .expect("Expected to find entry at location of last membership change.")[0]; - self.begin_membership_change(entry) - .expect("Expected the membership change already record to be valid."); + let conf_change = protobuf::parse_from_bytes::(entry.get_data()) + .expect("Expected the membership change already recorded to be valid."); + self.begin_membership_change(&conf_change) + .expect("Expected the membership change already recorded to be valid."); } } diff --git a/src/raw_node.rs b/src/raw_node.rs index df45c200b..66753f627 100644 --- a/src/raw_node.rs +++ b/src/raw_node.rs @@ -350,10 +350,12 @@ impl RawNode { ConfChangeType::AddNode => self.raft.add_node(nid), ConfChangeType::AddLearnerNode => self.raft.add_learner(nid), ConfChangeType::RemoveNode => self.raft.remove_node(nid), - ConfChangeType::BeginConfChange => self - .raft - .propose_membership_change(cc.get_configuration().clone()) - .unwrap(), + ConfChangeType::BeginConfChange => { + self + .raft + .begin_membership_change(cc) + .unwrap() + }, ConfChangeType::FinalizeConfChange => { self.raft.mut_prs().finalize_membership_change().unwrap(); } diff --git a/tests/integration_cases/test_membership_changes.rs b/tests/integration_cases/test_membership_changes.rs index 4008be632..76594876e 100644 --- a/tests/integration_cases/test_membership_changes.rs +++ b/tests/integration_cases/test_membership_changes.rs @@ -42,10 +42,10 @@ mod api { }, MemStorage::new(), )?; - let begin_entry = begin_entry(&[1, 2, 3], &[4], raft.raft_log.last_index() + 1); - raft.begin_membership_change(&begin_entry)?; - let finalize_entry = finalize_entry(raft.raft_log.last_index() + 1); - raft.finalize_membership_change(&finalize_entry)?; + let begin_conf_change = begin_conf_change(&[1, 2, 3], &[4], raft.raft_log.last_index() + 1); + raft.begin_membership_change(&begin_conf_change)?; + let finalize_conf_change = finalize_conf_change(); + raft.finalize_membership_change(&finalize_conf_change)?; Ok(()) } @@ -63,8 +63,8 @@ mod api { }, MemStorage::new(), )?; - let begin_entry = begin_entry(&[1, 2, 3], &[1, 2, 3], raft.raft_log.last_index() + 1); - assert!(raft.begin_membership_change(&begin_entry).is_err()); + let begin_conf_change = begin_conf_change(&[1, 2, 3], &[1, 2, 3], raft.raft_log.last_index() + 1); + assert!(raft.begin_membership_change(&begin_conf_change).is_err()); Ok(()) } @@ -82,8 +82,8 @@ mod api { }, MemStorage::new(), )?; - let begin_entry = begin_entry(&[1, 2], &[3, 4], raft.raft_log.last_index() + 1); - assert!(raft.begin_membership_change(&begin_entry).is_err()); + let begin_conf_change = begin_conf_change(&[1, 2], &[3, 4], raft.raft_log.last_index() + 1); + assert!(raft.begin_membership_change(&begin_conf_change).is_err()); Ok(()) } @@ -101,8 +101,8 @@ mod api { }, MemStorage::new(), )?; - let finalize_entry = finalize_entry(raft.raft_log.last_index() + 1); - assert!(raft.finalize_membership_change(&finalize_entry).is_err()); + let finalize_conf_change = finalize_conf_change(); + assert!(raft.finalize_membership_change(&finalize_conf_change).is_err()); Ok(()) } } @@ -1304,9 +1304,9 @@ impl Scenario { found = true; match entry_type { ConfChangeType::BeginConfChange => - peer.begin_membership_change(&entry)?, + peer.begin_membership_change(&conf_change)?, ConfChangeType::FinalizeConfChange => - peer.finalize_membership_change(&entry)?, + peer.finalize_membership_change(&conf_change)?, ConfChangeType::AddNode => peer.add_node(conf_change.get_node_id()), _ => panic!("Unexpected conf change"), @@ -1428,15 +1428,31 @@ fn conf_state<'a>( conf_state } -fn begin_entry<'a>( +fn begin_conf_change<'a>( voters: impl IntoIterator, learners: impl IntoIterator, index: u64, -) -> Entry { +) -> ConfChange { let conf_state = conf_state(voters, learners); let mut conf_change = ConfChange::new(); conf_change.set_change_type(ConfChangeType::BeginConfChange); conf_change.set_configuration(conf_state); + conf_change.set_start_index(index); + conf_change +} + +fn finalize_conf_change<'a>() -> ConfChange { + let mut conf_change = ConfChange::new(); + conf_change.set_change_type(ConfChangeType::FinalizeConfChange); + conf_change +} + +fn begin_entry<'a>( + voters: impl IntoIterator, + learners: impl IntoIterator, + index: u64, +) -> Entry { + let conf_change = begin_conf_change(voters, learners, index); let data = protobuf::Message::write_to_bytes(&conf_change).unwrap(); let mut entry = Entry::new(); entry.set_entry_type(EntryType::EntryConfChange); @@ -1446,8 +1462,7 @@ fn begin_entry<'a>( } fn finalize_entry(index: u64) -> Entry { - let mut conf_change = ConfChange::new(); - conf_change.set_change_type(ConfChangeType::FinalizeConfChange); + let mut conf_change = finalize_conf_change(); let data = protobuf::Message::write_to_bytes(&conf_change).unwrap(); let mut entry = Entry::new(); entry.set_entry_type(EntryType::EntryConfChange); From bacf6e3717258608a0c74b09454b7c92acbda8bf Mon Sep 17 00:00:00 2001 From: Hoverbear Date: Fri, 4 Jan 2019 09:52:36 -0800 Subject: [PATCH 21/41] wip --- benches/benches.rs | 2 +- proto/eraftpb.proto | 6 +- src/eraftpb.rs | 610 ++++++++++-------- src/progress.rs | 10 +- src/raft.rs | 94 +-- src/raw_node.rs | 5 +- src/storage.rs | 8 +- src/util.rs | 8 + .../test_membership_changes.rs | 182 +++++- tests/integration_cases/test_raft.rs | 4 +- 10 files changed, 584 insertions(+), 345 deletions(-) diff --git a/benches/benches.rs b/benches/benches.rs index 2746c63a0..ea2be50a0 100644 --- a/benches/benches.rs +++ b/benches/benches.rs @@ -1,5 +1,5 @@ #![allow(dead_code)] // Due to criterion we need this to avoid warnings. -#![cfg_attr(feature = "cargo-clippy", allow(let_and_return))] // Benches often artificially return values. Allow it. +#![cfg_attr(feature = "cargo-clippy", allow(clippy::let_and_return))] // Benches often artificially return values. Allow it. extern crate criterion; extern crate env_logger; diff --git a/proto/eraftpb.proto b/proto/eraftpb.proto index 7fb8805fb..ada4472ca 100644 --- a/proto/eraftpb.proto +++ b/proto/eraftpb.proto @@ -30,6 +30,7 @@ message Entry { message SnapshotMetadata { ConfState conf_state = 1; + ConfChange pending_membership_change = 4; uint64 index = 2; uint64 term = 3; } @@ -80,9 +81,8 @@ message HardState { uint64 term = 1; uint64 vote = 2; uint64 commit = 3; - // HACK: Our current Protobuf uses `0` as `None`. - // Good thing we know that index 0 of the raft log is always not important in this context. - uint64 began_conf_change_at = 4; + // If the peer goes down we need to be aware of a partially completed membership change. + ConfChange pending_membership_change = 4; } message ConfState { diff --git a/src/eraftpb.rs b/src/eraftpb.rs index 8b9812471..beca6cb85 100644 --- a/src/eraftpb.rs +++ b/src/eraftpb.rs @@ -3,7 +3,7 @@ // https://github.com/Manishearth/rust-clippy/issues/702 #![allow(unknown_lints)] -#![cfg_attr(feature = "cargo-clippy", allow(clippy::all))] +#![allow(clippy)] #![cfg_attr(rustfmt, rustfmt_skip)] @@ -366,6 +366,7 @@ impl ::protobuf::reflect::ProtobufValue for Entry { pub struct SnapshotMetadata { // message fields pub conf_state: ::protobuf::SingularPtrField, + pub pending_membership_change: ::protobuf::SingularPtrField, pub index: u64, pub term: u64, // special fields @@ -411,6 +412,39 @@ impl SnapshotMetadata { self.conf_state.as_ref().unwrap_or_else(|| ConfState::default_instance()) } + // .eraftpb.ConfChange pending_membership_change = 4; + + pub fn clear_pending_membership_change(&mut self) { + self.pending_membership_change.clear(); + } + + pub fn has_pending_membership_change(&self) -> bool { + self.pending_membership_change.is_some() + } + + // Param is passed by value, moved + pub fn set_pending_membership_change(&mut self, v: ConfChange) { + self.pending_membership_change = ::protobuf::SingularPtrField::some(v); + } + + // Mutable pointer to the field. + // If field is not initialized, it is initialized with default value first. + pub fn mut_pending_membership_change(&mut self) -> &mut ConfChange { + if self.pending_membership_change.is_none() { + self.pending_membership_change.set_default(); + } + self.pending_membership_change.as_mut().unwrap() + } + + // Take field + pub fn take_pending_membership_change(&mut self) -> ConfChange { + self.pending_membership_change.take().unwrap_or_else(|| ConfChange::new()) + } + + pub fn get_pending_membership_change(&self) -> &ConfChange { + self.pending_membership_change.as_ref().unwrap_or_else(|| ConfChange::default_instance()) + } + // uint64 index = 2; pub fn clear_index(&mut self) { @@ -449,6 +483,11 @@ impl ::protobuf::Message for SnapshotMetadata { return false; } }; + for v in &self.pending_membership_change { + if !v.is_initialized() { + return false; + } + }; true } @@ -459,6 +498,9 @@ impl ::protobuf::Message for SnapshotMetadata { 1 => { ::protobuf::rt::read_singular_message_into(wire_type, is, &mut self.conf_state)?; }, + 4 => { + ::protobuf::rt::read_singular_message_into(wire_type, is, &mut self.pending_membership_change)?; + }, 2 => { if wire_type != ::protobuf::wire_format::WireTypeVarint { return ::std::result::Result::Err(::protobuf::rt::unexpected_wire_type(wire_type)); @@ -489,6 +531,10 @@ impl ::protobuf::Message for SnapshotMetadata { let len = v.compute_size(); my_size += 1 + ::protobuf::rt::compute_raw_varint32_size(len) + len; } + if let Some(ref v) = self.pending_membership_change.as_ref() { + let len = v.compute_size(); + my_size += 1 + ::protobuf::rt::compute_raw_varint32_size(len) + len; + } if self.index != 0 { my_size += ::protobuf::rt::value_size(2, self.index, ::protobuf::wire_format::WireTypeVarint); } @@ -506,6 +552,11 @@ impl ::protobuf::Message for SnapshotMetadata { os.write_raw_varint32(v.get_cached_size())?; v.write_to_with_cached_sizes(os)?; } + if let Some(ref v) = self.pending_membership_change.as_ref() { + os.write_tag(4, ::protobuf::wire_format::WireTypeLengthDelimited)?; + os.write_raw_varint32(v.get_cached_size())?; + v.write_to_with_cached_sizes(os)?; + } if self.index != 0 { os.write_uint64(2, self.index)?; } @@ -559,6 +610,11 @@ impl ::protobuf::Message for SnapshotMetadata { |m: &SnapshotMetadata| { &m.conf_state }, |m: &mut SnapshotMetadata| { &mut m.conf_state }, )); + fields.push(::protobuf::reflect::accessor::make_singular_ptr_field_accessor::<_, ::protobuf::types::ProtobufTypeMessage>( + "pending_membership_change", + |m: &SnapshotMetadata| { &m.pending_membership_change }, + |m: &mut SnapshotMetadata| { &mut m.pending_membership_change }, + )); fields.push(::protobuf::reflect::accessor::make_simple_field_accessor::<_, ::protobuf::types::ProtobufTypeUint64>( "index", |m: &SnapshotMetadata| { &m.index }, @@ -592,6 +648,7 @@ impl ::protobuf::Message for SnapshotMetadata { impl ::protobuf::Clear for SnapshotMetadata { fn clear(&mut self) { self.clear_conf_state(); + self.clear_pending_membership_change(); self.clear_index(); self.clear_term(); self.unknown_fields.clear(); @@ -1416,7 +1473,7 @@ pub struct HardState { pub term: u64, pub vote: u64, pub commit: u64, - pub began_conf_change_at: u64, + pub pending_membership_change: ::protobuf::SingularPtrField, // special fields pub unknown_fields: ::protobuf::UnknownFields, pub cached_size: ::protobuf::CachedSize, @@ -1472,24 +1529,47 @@ impl HardState { self.commit } - // uint64 began_conf_change_at = 4; + // .eraftpb.ConfChange pending_membership_change = 4; - pub fn clear_began_conf_change_at(&mut self) { - self.began_conf_change_at = 0; + pub fn clear_pending_membership_change(&mut self) { + self.pending_membership_change.clear(); + } + + pub fn has_pending_membership_change(&self) -> bool { + self.pending_membership_change.is_some() } // Param is passed by value, moved - pub fn set_began_conf_change_at(&mut self, v: u64) { - self.began_conf_change_at = v; + pub fn set_pending_membership_change(&mut self, v: ConfChange) { + self.pending_membership_change = ::protobuf::SingularPtrField::some(v); + } + + // Mutable pointer to the field. + // If field is not initialized, it is initialized with default value first. + pub fn mut_pending_membership_change(&mut self) -> &mut ConfChange { + if self.pending_membership_change.is_none() { + self.pending_membership_change.set_default(); + } + self.pending_membership_change.as_mut().unwrap() + } + + // Take field + pub fn take_pending_membership_change(&mut self) -> ConfChange { + self.pending_membership_change.take().unwrap_or_else(|| ConfChange::new()) } - pub fn get_began_conf_change_at(&self) -> u64 { - self.began_conf_change_at + pub fn get_pending_membership_change(&self) -> &ConfChange { + self.pending_membership_change.as_ref().unwrap_or_else(|| ConfChange::default_instance()) } } impl ::protobuf::Message for HardState { fn is_initialized(&self) -> bool { + for v in &self.pending_membership_change { + if !v.is_initialized() { + return false; + } + }; true } @@ -1519,11 +1599,7 @@ impl ::protobuf::Message for HardState { self.commit = tmp; }, 4 => { - if wire_type != ::protobuf::wire_format::WireTypeVarint { - return ::std::result::Result::Err(::protobuf::rt::unexpected_wire_type(wire_type)); - } - let tmp = is.read_uint64()?; - self.began_conf_change_at = tmp; + ::protobuf::rt::read_singular_message_into(wire_type, is, &mut self.pending_membership_change)?; }, _ => { ::protobuf::rt::read_unknown_or_skip_group(field_number, wire_type, is, self.mut_unknown_fields())?; @@ -1546,8 +1622,9 @@ impl ::protobuf::Message for HardState { if self.commit != 0 { my_size += ::protobuf::rt::value_size(3, self.commit, ::protobuf::wire_format::WireTypeVarint); } - if self.began_conf_change_at != 0 { - my_size += ::protobuf::rt::value_size(4, self.began_conf_change_at, ::protobuf::wire_format::WireTypeVarint); + if let Some(ref v) = self.pending_membership_change.as_ref() { + let len = v.compute_size(); + my_size += 1 + ::protobuf::rt::compute_raw_varint32_size(len) + len; } my_size += ::protobuf::rt::unknown_fields_size(self.get_unknown_fields()); self.cached_size.set(my_size); @@ -1564,8 +1641,10 @@ impl ::protobuf::Message for HardState { if self.commit != 0 { os.write_uint64(3, self.commit)?; } - if self.began_conf_change_at != 0 { - os.write_uint64(4, self.began_conf_change_at)?; + if let Some(ref v) = self.pending_membership_change.as_ref() { + os.write_tag(4, ::protobuf::wire_format::WireTypeLengthDelimited)?; + os.write_raw_varint32(v.get_cached_size())?; + v.write_to_with_cached_sizes(os)?; } os.write_unknown_fields(self.get_unknown_fields())?; ::std::result::Result::Ok(()) @@ -1624,10 +1703,10 @@ impl ::protobuf::Message for HardState { |m: &HardState| { &m.commit }, |m: &mut HardState| { &mut m.commit }, )); - fields.push(::protobuf::reflect::accessor::make_simple_field_accessor::<_, ::protobuf::types::ProtobufTypeUint64>( - "began_conf_change_at", - |m: &HardState| { &m.began_conf_change_at }, - |m: &mut HardState| { &mut m.began_conf_change_at }, + fields.push(::protobuf::reflect::accessor::make_singular_ptr_field_accessor::<_, ::protobuf::types::ProtobufTypeMessage>( + "pending_membership_change", + |m: &HardState| { &m.pending_membership_change }, + |m: &mut HardState| { &mut m.pending_membership_change }, )); ::protobuf::reflect::MessageDescriptor::new::( "HardState", @@ -1654,7 +1733,7 @@ impl ::protobuf::Clear for HardState { self.clear_term(); self.clear_vote(); self.clear_commit(); - self.clear_began_conf_change_at(); + self.clear_pending_membership_change(); self.unknown_fields.clear(); } } @@ -2461,252 +2540,255 @@ static file_descriptor_proto_data: &'static [u8] = b"\ \n\x04term\x18\x02\x20\x01(\x04R\x04term\x12\x14\n\x05index\x18\x03\x20\ \x01(\x04R\x05index\x12\x12\n\x04data\x18\x04\x20\x01(\x0cR\x04data\x12\ \x18\n\x07context\x18\x06\x20\x01(\x0cR\x07context\x12\x19\n\x08sync_log\ - \x18\x05\x20\x01(\x08R\x07syncLog\"o\n\x10SnapshotMetadata\x121\n\nconf_\ - state\x18\x01\x20\x01(\x0b2\x12.eraftpb.ConfStateR\tconfState\x12\x14\n\ - \x05index\x18\x02\x20\x01(\x04R\x05index\x12\x12\n\x04term\x18\x03\x20\ - \x01(\x04R\x04term\"U\n\x08Snapshot\x12\x12\n\x04data\x18\x01\x20\x01(\ - \x0cR\x04data\x125\n\x08metadata\x18\x02\x20\x01(\x0b2\x19.eraftpb.Snaps\ - hotMetadataR\x08metadata\"\xe7\x02\n\x07Message\x12/\n\x08msg_type\x18\ - \x01\x20\x01(\x0e2\x14.eraftpb.MessageTypeR\x07msgType\x12\x0e\n\x02to\ - \x18\x02\x20\x01(\x04R\x02to\x12\x12\n\x04from\x18\x03\x20\x01(\x04R\x04\ - from\x12\x12\n\x04term\x18\x04\x20\x01(\x04R\x04term\x12\x19\n\x08log_te\ - rm\x18\x05\x20\x01(\x04R\x07logTerm\x12\x14\n\x05index\x18\x06\x20\x01(\ - \x04R\x05index\x12(\n\x07entries\x18\x07\x20\x03(\x0b2\x0e.eraftpb.Entry\ - R\x07entries\x12\x16\n\x06commit\x18\x08\x20\x01(\x04R\x06commit\x12-\n\ - \x08snapshot\x18\t\x20\x01(\x0b2\x11.eraftpb.SnapshotR\x08snapshot\x12\ - \x16\n\x06reject\x18\n\x20\x01(\x08R\x06reject\x12\x1f\n\x0breject_hint\ - \x18\x0b\x20\x01(\x04R\nrejectHint\x12\x18\n\x07context\x18\x0c\x20\x01(\ - \x0cR\x07context\"|\n\tHardState\x12\x12\n\x04term\x18\x01\x20\x01(\x04R\ - \x04term\x12\x12\n\x04vote\x18\x02\x20\x01(\x04R\x04vote\x12\x16\n\x06co\ - mmit\x18\x03\x20\x01(\x04R\x06commit\x12/\n\x14began_conf_change_at\x18\ - \x04\x20\x01(\x04R\x11beganConfChangeAt\"=\n\tConfState\x12\x14\n\x05nod\ - es\x18\x01\x20\x03(\x04R\x05nodes\x12\x1a\n\x08learners\x18\x02\x20\x03(\ - \x04R\x08learners\"\xe4\x01\n\nConfChange\x12\x0e\n\x02id\x18\x01\x20\ - \x01(\x04R\x02id\x128\n\x0bchange_type\x18\x02\x20\x01(\x0e2\x17.eraftpb\ - .ConfChangeTypeR\nchangeType\x12\x17\n\x07node_id\x18\x03\x20\x01(\x04R\ - \x06nodeId\x12\x18\n\x07context\x18\x04\x20\x01(\x0cR\x07context\x128\n\ - \rconfiguration\x18\x05\x20\x01(\x0b2\x12.eraftpb.ConfStateR\rconfigurat\ - ion\x12\x1f\n\x0bstart_index\x18\x06\x20\x01(\x04R\nstartIndex*1\n\tEntr\ - yType\x12\x0f\n\x0bEntryNormal\x10\0\x12\x13\n\x0fEntryConfChange\x10\ - \x01*\x8c\x03\n\x0bMessageType\x12\n\n\x06MsgHup\x10\0\x12\x0b\n\x07MsgB\ - eat\x10\x01\x12\x0e\n\nMsgPropose\x10\x02\x12\r\n\tMsgAppend\x10\x03\x12\ - \x15\n\x11MsgAppendResponse\x10\x04\x12\x12\n\x0eMsgRequestVote\x10\x05\ - \x12\x1a\n\x16MsgRequestVoteResponse\x10\x06\x12\x0f\n\x0bMsgSnapshot\ - \x10\x07\x12\x10\n\x0cMsgHeartbeat\x10\x08\x12\x18\n\x14MsgHeartbeatResp\ - onse\x10\t\x12\x12\n\x0eMsgUnreachable\x10\n\x12\x11\n\rMsgSnapStatus\ - \x10\x0b\x12\x12\n\x0eMsgCheckQuorum\x10\x0c\x12\x15\n\x11MsgTransferLea\ - der\x10\r\x12\x11\n\rMsgTimeoutNow\x10\x0e\x12\x10\n\x0cMsgReadIndex\x10\ - \x0f\x12\x14\n\x10MsgReadIndexResp\x10\x10\x12\x15\n\x11MsgRequestPreVot\ - e\x10\x11\x12\x1d\n\x19MsgRequestPreVoteResponse\x10\x12*n\n\x0eConfChan\ - geType\x12\x0b\n\x07AddNode\x10\0\x12\x0e\n\nRemoveNode\x10\x01\x12\x12\ - \n\x0eAddLearnerNode\x10\x02\x12\x13\n\x0fBeginConfChange\x10\x03\x12\ - \x16\n\x12FinalizeConfChange\x10\x04J\xc7&\n\x06\x12\x04\0\0p\x01\n\x08\ - \n\x01\x0c\x12\x03\0\0\x12\n\x08\n\x01\x02\x12\x03\x01\x08\x0f\n\n\n\x02\ - \x05\0\x12\x04\x03\0\x06\x01\n\n\n\x03\x05\0\x01\x12\x03\x03\x05\x0e\n\ - \x0b\n\x04\x05\0\x02\0\x12\x03\x04\x04\x14\n\x0c\n\x05\x05\0\x02\0\x01\ + \x18\x05\x20\x01(\x08R\x07syncLog\"\xc0\x01\n\x10SnapshotMetadata\x121\n\ + \nconf_state\x18\x01\x20\x01(\x0b2\x12.eraftpb.ConfStateR\tconfState\x12\ + O\n\x19pending_membership_change\x18\x04\x20\x01(\x0b2\x13.eraftpb.ConfC\ + hangeR\x17pendingMembershipChange\x12\x14\n\x05index\x18\x02\x20\x01(\ + \x04R\x05index\x12\x12\n\x04term\x18\x03\x20\x01(\x04R\x04term\"U\n\x08S\ + napshot\x12\x12\n\x04data\x18\x01\x20\x01(\x0cR\x04data\x125\n\x08metada\ + ta\x18\x02\x20\x01(\x0b2\x19.eraftpb.SnapshotMetadataR\x08metadata\"\xe7\ + \x02\n\x07Message\x12/\n\x08msg_type\x18\x01\x20\x01(\x0e2\x14.eraftpb.M\ + essageTypeR\x07msgType\x12\x0e\n\x02to\x18\x02\x20\x01(\x04R\x02to\x12\ + \x12\n\x04from\x18\x03\x20\x01(\x04R\x04from\x12\x12\n\x04term\x18\x04\ + \x20\x01(\x04R\x04term\x12\x19\n\x08log_term\x18\x05\x20\x01(\x04R\x07lo\ + gTerm\x12\x14\n\x05index\x18\x06\x20\x01(\x04R\x05index\x12(\n\x07entrie\ + s\x18\x07\x20\x03(\x0b2\x0e.eraftpb.EntryR\x07entries\x12\x16\n\x06commi\ + t\x18\x08\x20\x01(\x04R\x06commit\x12-\n\x08snapshot\x18\t\x20\x01(\x0b2\ + \x11.eraftpb.SnapshotR\x08snapshot\x12\x16\n\x06reject\x18\n\x20\x01(\ + \x08R\x06reject\x12\x1f\n\x0breject_hint\x18\x0b\x20\x01(\x04R\nrejectHi\ + nt\x12\x18\n\x07context\x18\x0c\x20\x01(\x0cR\x07context\"\x9c\x01\n\tHa\ + rdState\x12\x12\n\x04term\x18\x01\x20\x01(\x04R\x04term\x12\x12\n\x04vot\ + e\x18\x02\x20\x01(\x04R\x04vote\x12\x16\n\x06commit\x18\x03\x20\x01(\x04\ + R\x06commit\x12O\n\x19pending_membership_change\x18\x04\x20\x01(\x0b2\ + \x13.eraftpb.ConfChangeR\x17pendingMembershipChange\"=\n\tConfState\x12\ + \x14\n\x05nodes\x18\x01\x20\x03(\x04R\x05nodes\x12\x1a\n\x08learners\x18\ + \x02\x20\x03(\x04R\x08learners\"\xe4\x01\n\nConfChange\x12\x0e\n\x02id\ + \x18\x01\x20\x01(\x04R\x02id\x128\n\x0bchange_type\x18\x02\x20\x01(\x0e2\ + \x17.eraftpb.ConfChangeTypeR\nchangeType\x12\x17\n\x07node_id\x18\x03\ + \x20\x01(\x04R\x06nodeId\x12\x18\n\x07context\x18\x04\x20\x01(\x0cR\x07c\ + ontext\x128\n\rconfiguration\x18\x05\x20\x01(\x0b2\x12.eraftpb.ConfState\ + R\rconfiguration\x12\x1f\n\x0bstart_index\x18\x06\x20\x01(\x04R\nstartIn\ + dex*1\n\tEntryType\x12\x0f\n\x0bEntryNormal\x10\0\x12\x13\n\x0fEntryConf\ + Change\x10\x01*\x8c\x03\n\x0bMessageType\x12\n\n\x06MsgHup\x10\0\x12\x0b\ + \n\x07MsgBeat\x10\x01\x12\x0e\n\nMsgPropose\x10\x02\x12\r\n\tMsgAppend\ + \x10\x03\x12\x15\n\x11MsgAppendResponse\x10\x04\x12\x12\n\x0eMsgRequestV\ + ote\x10\x05\x12\x1a\n\x16MsgRequestVoteResponse\x10\x06\x12\x0f\n\x0bMsg\ + Snapshot\x10\x07\x12\x10\n\x0cMsgHeartbeat\x10\x08\x12\x18\n\x14MsgHeart\ + beatResponse\x10\t\x12\x12\n\x0eMsgUnreachable\x10\n\x12\x11\n\rMsgSnapS\ + tatus\x10\x0b\x12\x12\n\x0eMsgCheckQuorum\x10\x0c\x12\x15\n\x11MsgTransf\ + erLeader\x10\r\x12\x11\n\rMsgTimeoutNow\x10\x0e\x12\x10\n\x0cMsgReadInde\ + x\x10\x0f\x12\x14\n\x10MsgReadIndexResp\x10\x10\x12\x15\n\x11MsgRequestP\ + reVote\x10\x11\x12\x1d\n\x19MsgRequestPreVoteResponse\x10\x12*n\n\x0eCon\ + fChangeType\x12\x0b\n\x07AddNode\x10\0\x12\x0e\n\nRemoveNode\x10\x01\x12\ + \x12\n\x0eAddLearnerNode\x10\x02\x12\x13\n\x0fBeginConfChange\x10\x03\ + \x12\x16\n\x12FinalizeConfChange\x10\x04J\xc5&\n\x06\x12\x04\0\0p\x01\n\ + \x08\n\x01\x0c\x12\x03\0\0\x12\n\x08\n\x01\x02\x12\x03\x01\x08\x0f\n\n\n\ + \x02\x05\0\x12\x04\x03\0\x06\x01\n\n\n\x03\x05\0\x01\x12\x03\x03\x05\x0e\ + \n\x0b\n\x04\x05\0\x02\0\x12\x03\x04\x04\x14\n\x0c\n\x05\x05\0\x02\0\x01\ \x12\x03\x04\x04\x0f\n\x0c\n\x05\x05\0\x02\0\x02\x12\x03\x04\x12\x13\n\ \x0b\n\x04\x05\0\x02\x01\x12\x03\x05\x04\x18\n\x0c\n\x05\x05\0\x02\x01\ \x01\x12\x03\x05\x04\x13\n\x0c\n\x05\x05\0\x02\x01\x02\x12\x03\x05\x16\ - \x17\n\xe7\x04\n\x02\x04\0\x12\x04\x12\0\x1c\x01\x1a\xda\x04\x20The\x20e\ + \x17\n\xdd\x04\n\x02\x04\0\x12\x04\x12\0\x1c\x01\x1a\xd0\x04\x20The\x20e\ ntry\x20is\x20a\x20type\x20of\x20change\x20that\x20needs\x20to\x20be\x20\ - applied.\x20It\x20contains\x20two\x20data\x20fields.\r\n\x20While\x20the\ + applied.\x20It\x20contains\x20two\x20data\x20fields.\n\x20While\x20the\ \x20fields\x20are\x20built\x20into\x20the\x20model;\x20their\x20usage\ - \x20is\x20determined\x20by\x20the\x20entry_type.\r\n\r\n\x20For\x20norma\ - l\x20entries,\x20the\x20data\x20field\x20should\x20contain\x20the\x20dat\ - a\x20change\x20that\x20should\x20be\x20applied.\r\n\x20The\x20context\ - \x20field\x20can\x20be\x20used\x20for\x20any\x20contextual\x20data\x20th\ - at\x20might\x20be\x20relevant\x20to\x20the\r\n\x20application\x20of\x20t\ - he\x20data.\r\n\r\n\x20For\x20configuration\x20changes,\x20the\x20data\ - \x20will\x20contain\x20the\x20ConfChange\x20message\x20and\x20the\r\n\ - \x20context\x20will\x20provide\x20anything\x20needed\x20to\x20assist\x20\ - the\x20configuration\x20change.\x20The\x20context\r\n\x20if\x20for\x20th\ - e\x20user\x20to\x20set\x20and\x20use\x20in\x20this\x20case.\r\n\n\n\n\ - \x03\x04\0\x01\x12\x03\x12\x08\r\n\x0b\n\x04\x04\0\x02\0\x12\x03\x13\x04\ - \x1d\n\r\n\x05\x04\0\x02\0\x04\x12\x04\x13\x04\x12\x0f\n\x0c\n\x05\x04\0\ - \x02\0\x06\x12\x03\x13\x04\r\n\x0c\n\x05\x04\0\x02\0\x01\x12\x03\x13\x0e\ - \x18\n\x0c\n\x05\x04\0\x02\0\x03\x12\x03\x13\x1b\x1c\n\x0b\n\x04\x04\0\ - \x02\x01\x12\x03\x14\x04\x14\n\r\n\x05\x04\0\x02\x01\x04\x12\x04\x14\x04\ - \x13\x1d\n\x0c\n\x05\x04\0\x02\x01\x05\x12\x03\x14\x04\n\n\x0c\n\x05\x04\ - \0\x02\x01\x01\x12\x03\x14\x0b\x0f\n\x0c\n\x05\x04\0\x02\x01\x03\x12\x03\ - \x14\x12\x13\n\x0b\n\x04\x04\0\x02\x02\x12\x03\x15\x04\x15\n\r\n\x05\x04\ - \0\x02\x02\x04\x12\x04\x15\x04\x14\x14\n\x0c\n\x05\x04\0\x02\x02\x05\x12\ - \x03\x15\x04\n\n\x0c\n\x05\x04\0\x02\x02\x01\x12\x03\x15\x0b\x10\n\x0c\n\ - \x05\x04\0\x02\x02\x03\x12\x03\x15\x13\x14\n\x0b\n\x04\x04\0\x02\x03\x12\ - \x03\x16\x04\x13\n\r\n\x05\x04\0\x02\x03\x04\x12\x04\x16\x04\x15\x15\n\ - \x0c\n\x05\x04\0\x02\x03\x05\x12\x03\x16\x04\t\n\x0c\n\x05\x04\0\x02\x03\ - \x01\x12\x03\x16\n\x0e\n\x0c\n\x05\x04\0\x02\x03\x03\x12\x03\x16\x11\x12\ - \n\x0b\n\x04\x04\0\x02\x04\x12\x03\x17\x04\x16\n\r\n\x05\x04\0\x02\x04\ - \x04\x12\x04\x17\x04\x16\x13\n\x0c\n\x05\x04\0\x02\x04\x05\x12\x03\x17\ - \x04\t\n\x0c\n\x05\x04\0\x02\x04\x01\x12\x03\x17\n\x11\n\x0c\n\x05\x04\0\ - \x02\x04\x03\x12\x03\x17\x14\x15\no\n\x04\x04\0\x02\x05\x12\x03\x1b\x04\ - \x16\x1ab\x20Deprecated!\x20It\x20is\x20kept\x20for\x20backward\x20compa\ - tibility.\r\n\x20TODO:\x20remove\x20it\x20in\x20the\x20next\x20major\x20\ - release.\r\n\n\r\n\x05\x04\0\x02\x05\x04\x12\x04\x1b\x04\x17\x16\n\x0c\n\ - \x05\x04\0\x02\x05\x05\x12\x03\x1b\x04\x08\n\x0c\n\x05\x04\0\x02\x05\x01\ - \x12\x03\x1b\t\x11\n\x0c\n\x05\x04\0\x02\x05\x03\x12\x03\x1b\x14\x15\n\n\ - \n\x02\x04\x01\x12\x04\x1e\0\"\x01\n\n\n\x03\x04\x01\x01\x12\x03\x1e\x08\ - \x18\n\x0b\n\x04\x04\x01\x02\0\x12\x03\x1f\x04\x1d\n\r\n\x05\x04\x01\x02\ - \0\x04\x12\x04\x1f\x04\x1e\x1a\n\x0c\n\x05\x04\x01\x02\0\x06\x12\x03\x1f\ - \x04\r\n\x0c\n\x05\x04\x01\x02\0\x01\x12\x03\x1f\x0e\x18\n\x0c\n\x05\x04\ - \x01\x02\0\x03\x12\x03\x1f\x1b\x1c\n\x0b\n\x04\x04\x01\x02\x01\x12\x03\ - \x20\x04\x15\n\r\n\x05\x04\x01\x02\x01\x04\x12\x04\x20\x04\x1f\x1d\n\x0c\ - \n\x05\x04\x01\x02\x01\x05\x12\x03\x20\x04\n\n\x0c\n\x05\x04\x01\x02\x01\ - \x01\x12\x03\x20\x0b\x10\n\x0c\n\x05\x04\x01\x02\x01\x03\x12\x03\x20\x13\ - \x14\n\x0b\n\x04\x04\x01\x02\x02\x12\x03!\x04\x14\n\r\n\x05\x04\x01\x02\ - \x02\x04\x12\x04!\x04\x20\x15\n\x0c\n\x05\x04\x01\x02\x02\x05\x12\x03!\ - \x04\n\n\x0c\n\x05\x04\x01\x02\x02\x01\x12\x03!\x0b\x0f\n\x0c\n\x05\x04\ - \x01\x02\x02\x03\x12\x03!\x12\x13\n\n\n\x02\x04\x02\x12\x04$\0'\x01\n\n\ - \n\x03\x04\x02\x01\x12\x03$\x08\x10\n\x0b\n\x04\x04\x02\x02\0\x12\x03%\ - \x04\x13\n\r\n\x05\x04\x02\x02\0\x04\x12\x04%\x04$\x12\n\x0c\n\x05\x04\ - \x02\x02\0\x05\x12\x03%\x04\t\n\x0c\n\x05\x04\x02\x02\0\x01\x12\x03%\n\ - \x0e\n\x0c\n\x05\x04\x02\x02\0\x03\x12\x03%\x11\x12\n\x0b\n\x04\x04\x02\ - \x02\x01\x12\x03&\x04\"\n\r\n\x05\x04\x02\x02\x01\x04\x12\x04&\x04%\x13\ - \n\x0c\n\x05\x04\x02\x02\x01\x06\x12\x03&\x04\x14\n\x0c\n\x05\x04\x02\ - \x02\x01\x01\x12\x03&\x15\x1d\n\x0c\n\x05\x04\x02\x02\x01\x03\x12\x03&\ - \x20!\n\n\n\x02\x05\x01\x12\x04)\0=\x01\n\n\n\x03\x05\x01\x01\x12\x03)\ - \x05\x10\n\x0b\n\x04\x05\x01\x02\0\x12\x03*\x04\x0f\n\x0c\n\x05\x05\x01\ - \x02\0\x01\x12\x03*\x04\n\n\x0c\n\x05\x05\x01\x02\0\x02\x12\x03*\r\x0e\n\ - \x0b\n\x04\x05\x01\x02\x01\x12\x03+\x04\x10\n\x0c\n\x05\x05\x01\x02\x01\ - \x01\x12\x03+\x04\x0b\n\x0c\n\x05\x05\x01\x02\x01\x02\x12\x03+\x0e\x0f\n\ - \x0b\n\x04\x05\x01\x02\x02\x12\x03,\x04\x13\n\x0c\n\x05\x05\x01\x02\x02\ - \x01\x12\x03,\x04\x0e\n\x0c\n\x05\x05\x01\x02\x02\x02\x12\x03,\x11\x12\n\ - \x0b\n\x04\x05\x01\x02\x03\x12\x03-\x04\x12\n\x0c\n\x05\x05\x01\x02\x03\ - \x01\x12\x03-\x04\r\n\x0c\n\x05\x05\x01\x02\x03\x02\x12\x03-\x10\x11\n\ - \x0b\n\x04\x05\x01\x02\x04\x12\x03.\x04\x1a\n\x0c\n\x05\x05\x01\x02\x04\ - \x01\x12\x03.\x04\x15\n\x0c\n\x05\x05\x01\x02\x04\x02\x12\x03.\x18\x19\n\ - \x0b\n\x04\x05\x01\x02\x05\x12\x03/\x04\x17\n\x0c\n\x05\x05\x01\x02\x05\ - \x01\x12\x03/\x04\x12\n\x0c\n\x05\x05\x01\x02\x05\x02\x12\x03/\x15\x16\n\ - \x0b\n\x04\x05\x01\x02\x06\x12\x030\x04\x1f\n\x0c\n\x05\x05\x01\x02\x06\ - \x01\x12\x030\x04\x1a\n\x0c\n\x05\x05\x01\x02\x06\x02\x12\x030\x1d\x1e\n\ - \x0b\n\x04\x05\x01\x02\x07\x12\x031\x04\x14\n\x0c\n\x05\x05\x01\x02\x07\ - \x01\x12\x031\x04\x0f\n\x0c\n\x05\x05\x01\x02\x07\x02\x12\x031\x12\x13\n\ - \x0b\n\x04\x05\x01\x02\x08\x12\x032\x04\x15\n\x0c\n\x05\x05\x01\x02\x08\ - \x01\x12\x032\x04\x10\n\x0c\n\x05\x05\x01\x02\x08\x02\x12\x032\x13\x14\n\ - \x0b\n\x04\x05\x01\x02\t\x12\x033\x04\x1d\n\x0c\n\x05\x05\x01\x02\t\x01\ - \x12\x033\x04\x18\n\x0c\n\x05\x05\x01\x02\t\x02\x12\x033\x1b\x1c\n\x0b\n\ - \x04\x05\x01\x02\n\x12\x034\x04\x18\n\x0c\n\x05\x05\x01\x02\n\x01\x12\ - \x034\x04\x12\n\x0c\n\x05\x05\x01\x02\n\x02\x12\x034\x15\x17\n\x0b\n\x04\ - \x05\x01\x02\x0b\x12\x035\x04\x17\n\x0c\n\x05\x05\x01\x02\x0b\x01\x12\ - \x035\x04\x11\n\x0c\n\x05\x05\x01\x02\x0b\x02\x12\x035\x14\x16\n\x0b\n\ - \x04\x05\x01\x02\x0c\x12\x036\x04\x18\n\x0c\n\x05\x05\x01\x02\x0c\x01\ - \x12\x036\x04\x12\n\x0c\n\x05\x05\x01\x02\x0c\x02\x12\x036\x15\x17\n\x0b\ - \n\x04\x05\x01\x02\r\x12\x037\x04\x1b\n\x0c\n\x05\x05\x01\x02\r\x01\x12\ - \x037\x04\x15\n\x0c\n\x05\x05\x01\x02\r\x02\x12\x037\x18\x1a\n\x0b\n\x04\ - \x05\x01\x02\x0e\x12\x038\x04\x17\n\x0c\n\x05\x05\x01\x02\x0e\x01\x12\ - \x038\x04\x11\n\x0c\n\x05\x05\x01\x02\x0e\x02\x12\x038\x14\x16\n\x0b\n\ - \x04\x05\x01\x02\x0f\x12\x039\x04\x16\n\x0c\n\x05\x05\x01\x02\x0f\x01\ - \x12\x039\x04\x10\n\x0c\n\x05\x05\x01\x02\x0f\x02\x12\x039\x13\x15\n\x0b\ - \n\x04\x05\x01\x02\x10\x12\x03:\x04\x1a\n\x0c\n\x05\x05\x01\x02\x10\x01\ - \x12\x03:\x04\x14\n\x0c\n\x05\x05\x01\x02\x10\x02\x12\x03:\x17\x19\n\x0b\ - \n\x04\x05\x01\x02\x11\x12\x03;\x04\x1b\n\x0c\n\x05\x05\x01\x02\x11\x01\ - \x12\x03;\x04\x15\n\x0c\n\x05\x05\x01\x02\x11\x02\x12\x03;\x18\x1a\n\x0b\ - \n\x04\x05\x01\x02\x12\x12\x03<\x04#\n\x0c\n\x05\x05\x01\x02\x12\x01\x12\ - \x03<\x04\x1d\n\x0c\n\x05\x05\x01\x02\x12\x02\x12\x03<\x20\"\n\n\n\x02\ - \x04\x03\x12\x04?\0L\x01\n\n\n\x03\x04\x03\x01\x12\x03?\x08\x0f\n\x0b\n\ - \x04\x04\x03\x02\0\x12\x03@\x04\x1d\n\r\n\x05\x04\x03\x02\0\x04\x12\x04@\ - \x04?\x11\n\x0c\n\x05\x04\x03\x02\0\x06\x12\x03@\x04\x0f\n\x0c\n\x05\x04\ - \x03\x02\0\x01\x12\x03@\x10\x18\n\x0c\n\x05\x04\x03\x02\0\x03\x12\x03@\ - \x1b\x1c\n\x0b\n\x04\x04\x03\x02\x01\x12\x03A\x04\x12\n\r\n\x05\x04\x03\ - \x02\x01\x04\x12\x04A\x04@\x1d\n\x0c\n\x05\x04\x03\x02\x01\x05\x12\x03A\ - \x04\n\n\x0c\n\x05\x04\x03\x02\x01\x01\x12\x03A\x0b\r\n\x0c\n\x05\x04\ - \x03\x02\x01\x03\x12\x03A\x10\x11\n\x0b\n\x04\x04\x03\x02\x02\x12\x03B\ - \x04\x14\n\r\n\x05\x04\x03\x02\x02\x04\x12\x04B\x04A\x12\n\x0c\n\x05\x04\ - \x03\x02\x02\x05\x12\x03B\x04\n\n\x0c\n\x05\x04\x03\x02\x02\x01\x12\x03B\ - \x0b\x0f\n\x0c\n\x05\x04\x03\x02\x02\x03\x12\x03B\x12\x13\n\x0b\n\x04\ - \x04\x03\x02\x03\x12\x03C\x04\x14\n\r\n\x05\x04\x03\x02\x03\x04\x12\x04C\ - \x04B\x14\n\x0c\n\x05\x04\x03\x02\x03\x05\x12\x03C\x04\n\n\x0c\n\x05\x04\ - \x03\x02\x03\x01\x12\x03C\x0b\x0f\n\x0c\n\x05\x04\x03\x02\x03\x03\x12\ - \x03C\x12\x13\n\x0b\n\x04\x04\x03\x02\x04\x12\x03D\x04\x18\n\r\n\x05\x04\ - \x03\x02\x04\x04\x12\x04D\x04C\x14\n\x0c\n\x05\x04\x03\x02\x04\x05\x12\ - \x03D\x04\n\n\x0c\n\x05\x04\x03\x02\x04\x01\x12\x03D\x0b\x13\n\x0c\n\x05\ - \x04\x03\x02\x04\x03\x12\x03D\x16\x17\n\x0b\n\x04\x04\x03\x02\x05\x12\ - \x03E\x04\x15\n\r\n\x05\x04\x03\x02\x05\x04\x12\x04E\x04D\x18\n\x0c\n\ - \x05\x04\x03\x02\x05\x05\x12\x03E\x04\n\n\x0c\n\x05\x04\x03\x02\x05\x01\ - \x12\x03E\x0b\x10\n\x0c\n\x05\x04\x03\x02\x05\x03\x12\x03E\x13\x14\n\x0b\ - \n\x04\x04\x03\x02\x06\x12\x03F\x04\x1f\n\x0c\n\x05\x04\x03\x02\x06\x04\ - \x12\x03F\x04\x0c\n\x0c\n\x05\x04\x03\x02\x06\x06\x12\x03F\r\x12\n\x0c\n\ - \x05\x04\x03\x02\x06\x01\x12\x03F\x13\x1a\n\x0c\n\x05\x04\x03\x02\x06\ - \x03\x12\x03F\x1d\x1e\n\x0b\n\x04\x04\x03\x02\x07\x12\x03G\x04\x16\n\r\n\ - \x05\x04\x03\x02\x07\x04\x12\x04G\x04F\x1f\n\x0c\n\x05\x04\x03\x02\x07\ - \x05\x12\x03G\x04\n\n\x0c\n\x05\x04\x03\x02\x07\x01\x12\x03G\x0b\x11\n\ - \x0c\n\x05\x04\x03\x02\x07\x03\x12\x03G\x14\x15\n\x0b\n\x04\x04\x03\x02\ - \x08\x12\x03H\x04\x1a\n\r\n\x05\x04\x03\x02\x08\x04\x12\x04H\x04G\x16\n\ - \x0c\n\x05\x04\x03\x02\x08\x06\x12\x03H\x04\x0c\n\x0c\n\x05\x04\x03\x02\ - \x08\x01\x12\x03H\r\x15\n\x0c\n\x05\x04\x03\x02\x08\x03\x12\x03H\x18\x19\ - \n\x0b\n\x04\x04\x03\x02\t\x12\x03I\x04\x15\n\r\n\x05\x04\x03\x02\t\x04\ - \x12\x04I\x04H\x1a\n\x0c\n\x05\x04\x03\x02\t\x05\x12\x03I\x04\x08\n\x0c\ - \n\x05\x04\x03\x02\t\x01\x12\x03I\t\x0f\n\x0c\n\x05\x04\x03\x02\t\x03\ - \x12\x03I\x12\x14\n\x0b\n\x04\x04\x03\x02\n\x12\x03J\x04\x1c\n\r\n\x05\ - \x04\x03\x02\n\x04\x12\x04J\x04I\x15\n\x0c\n\x05\x04\x03\x02\n\x05\x12\ - \x03J\x04\n\n\x0c\n\x05\x04\x03\x02\n\x01\x12\x03J\x0b\x16\n\x0c\n\x05\ - \x04\x03\x02\n\x03\x12\x03J\x19\x1b\n\x0b\n\x04\x04\x03\x02\x0b\x12\x03K\ - \x04\x17\n\r\n\x05\x04\x03\x02\x0b\x04\x12\x04K\x04J\x1c\n\x0c\n\x05\x04\ - \x03\x02\x0b\x05\x12\x03K\x04\t\n\x0c\n\x05\x04\x03\x02\x0b\x01\x12\x03K\ - \n\x11\n\x0c\n\x05\x04\x03\x02\x0b\x03\x12\x03K\x14\x16\n\n\n\x02\x04\ - \x04\x12\x04N\0U\x01\n\n\n\x03\x04\x04\x01\x12\x03N\x08\x11\n\x0b\n\x04\ - \x04\x04\x02\0\x12\x03O\x04\x14\n\r\n\x05\x04\x04\x02\0\x04\x12\x04O\x04\ - N\x13\n\x0c\n\x05\x04\x04\x02\0\x05\x12\x03O\x04\n\n\x0c\n\x05\x04\x04\ - \x02\0\x01\x12\x03O\x0b\x0f\n\x0c\n\x05\x04\x04\x02\0\x03\x12\x03O\x12\ - \x13\n\x0b\n\x04\x04\x04\x02\x01\x12\x03P\x04\x14\n\r\n\x05\x04\x04\x02\ - \x01\x04\x12\x04P\x04O\x14\n\x0c\n\x05\x04\x04\x02\x01\x05\x12\x03P\x04\ - \n\n\x0c\n\x05\x04\x04\x02\x01\x01\x12\x03P\x0b\x0f\n\x0c\n\x05\x04\x04\ - \x02\x01\x03\x12\x03P\x12\x13\n\x0b\n\x04\x04\x04\x02\x02\x12\x03Q\x04\ - \x16\n\r\n\x05\x04\x04\x02\x02\x04\x12\x04Q\x04P\x14\n\x0c\n\x05\x04\x04\ - \x02\x02\x05\x12\x03Q\x04\n\n\x0c\n\x05\x04\x04\x02\x02\x01\x12\x03Q\x0b\ - \x11\n\x0c\n\x05\x04\x04\x02\x02\x03\x12\x03Q\x14\x15\n\x9a\x01\n\x04\ - \x04\x04\x02\x03\x12\x03T\x04$\x1a\x8c\x01\x20HACK:\x20Our\x20current\ - \x20Protobuf\x20uses\x20`0`\x20as\x20`None`.\r\n\x20Good\x20thing\x20we\ - \x20know\x20that\x20index\x200\x20of\x20the\x20raft\x20log\x20is\x20alwa\ - ys\x20not\x20important\x20in\x20this\x20context.\r\n\n\r\n\x05\x04\x04\ - \x02\x03\x04\x12\x04T\x04Q\x16\n\x0c\n\x05\x04\x04\x02\x03\x05\x12\x03T\ - \x04\n\n\x0c\n\x05\x04\x04\x02\x03\x01\x12\x03T\x0b\x1f\n\x0c\n\x05\x04\ - \x04\x02\x03\x03\x12\x03T\"#\n\n\n\x02\x04\x05\x12\x04W\0Z\x01\n\n\n\x03\ - \x04\x05\x01\x12\x03W\x08\x11\n\x0b\n\x04\x04\x05\x02\0\x12\x03X\x04\x1e\ - \n\x0c\n\x05\x04\x05\x02\0\x04\x12\x03X\x04\x0c\n\x0c\n\x05\x04\x05\x02\ - \0\x05\x12\x03X\r\x13\n\x0c\n\x05\x04\x05\x02\0\x01\x12\x03X\x14\x19\n\ - \x0c\n\x05\x04\x05\x02\0\x03\x12\x03X\x1c\x1d\n\x0b\n\x04\x04\x05\x02\ - \x01\x12\x03Y\x04!\n\x0c\n\x05\x04\x05\x02\x01\x04\x12\x03Y\x04\x0c\n\ - \x0c\n\x05\x04\x05\x02\x01\x05\x12\x03Y\r\x13\n\x0c\n\x05\x04\x05\x02\ - \x01\x01\x12\x03Y\x14\x1c\n\x0c\n\x05\x04\x05\x02\x01\x03\x12\x03Y\x1f\ - \x20\n\n\n\x02\x05\x02\x12\x04\\\0b\x01\n\n\n\x03\x05\x02\x01\x12\x03\\\ - \x05\x13\n\x0b\n\x04\x05\x02\x02\0\x12\x03]\x04\x13\n\x0c\n\x05\x05\x02\ - \x02\0\x01\x12\x03]\x04\x0b\n\x0c\n\x05\x05\x02\x02\0\x02\x12\x03]\x11\ - \x12\n\x0b\n\x04\x05\x02\x02\x01\x12\x03^\x04\x13\n\x0c\n\x05\x05\x02\ - \x02\x01\x01\x12\x03^\x04\x0e\n\x0c\n\x05\x05\x02\x02\x01\x02\x12\x03^\ - \x11\x12\n\x0b\n\x04\x05\x02\x02\x02\x12\x03_\x04\x17\n\x0c\n\x05\x05\ - \x02\x02\x02\x01\x12\x03_\x04\x12\n\x0c\n\x05\x05\x02\x02\x02\x02\x12\ - \x03_\x15\x16\n\x0b\n\x04\x05\x02\x02\x03\x12\x03`\x04\x18\n\x0c\n\x05\ - \x05\x02\x02\x03\x01\x12\x03`\x04\x13\n\x0c\n\x05\x05\x02\x02\x03\x02\ - \x12\x03`\x16\x17\n\x0b\n\x04\x05\x02\x02\x04\x12\x03a\x04\x1b\n\x0c\n\ - \x05\x05\x02\x02\x04\x01\x12\x03a\x04\x16\n\x0c\n\x05\x05\x02\x02\x04\ - \x02\x12\x03a\x19\x1a\n\n\n\x02\x04\x06\x12\x04d\0p\x01\n\n\n\x03\x04\ - \x06\x01\x12\x03d\x08\x12\n\x0b\n\x04\x04\x06\x02\0\x12\x03e\x04\x12\n\r\ - \n\x05\x04\x06\x02\0\x04\x12\x04e\x04d\x14\n\x0c\n\x05\x04\x06\x02\0\x05\ - \x12\x03e\x04\n\n\x0c\n\x05\x04\x06\x02\0\x01\x12\x03e\x0b\r\n\x0c\n\x05\ - \x04\x06\x02\0\x03\x12\x03e\x10\x11\n\x0b\n\x04\x04\x06\x02\x01\x12\x03f\ - \x04#\n\r\n\x05\x04\x06\x02\x01\x04\x12\x04f\x04e\x12\n\x0c\n\x05\x04\ - \x06\x02\x01\x06\x12\x03f\x04\x12\n\x0c\n\x05\x04\x06\x02\x01\x01\x12\ - \x03f\x13\x1e\n\x0c\n\x05\x04\x06\x02\x01\x03\x12\x03f!\"\nF\n\x04\x04\ - \x06\x02\x02\x12\x03h\x04\x17\x1a9\x20Used\x20in\x20`AddNode`,\x20`Remov\ - eNode`,\x20and\x20`AddLearnerNode`.\r\n\n\r\n\x05\x04\x06\x02\x02\x04\ - \x12\x04h\x04f#\n\x0c\n\x05\x04\x06\x02\x02\x05\x12\x03h\x04\n\n\x0c\n\ - \x05\x04\x06\x02\x02\x01\x12\x03h\x0b\x12\n\x0c\n\x05\x04\x06\x02\x02\ - \x03\x12\x03h\x15\x16\n\x0b\n\x04\x04\x06\x02\x03\x12\x03i\x04\x16\n\r\n\ - \x05\x04\x06\x02\x03\x04\x12\x04i\x04h\x17\n\x0c\n\x05\x04\x06\x02\x03\ - \x05\x12\x03i\x04\t\n\x0c\n\x05\x04\x06\x02\x03\x01\x12\x03i\n\x11\n\x0c\ - \n\x05\x04\x06\x02\x03\x03\x12\x03i\x14\x15\nC\n\x04\x04\x06\x02\x04\x12\ - \x03k\x04\x20\x1a6\x20Used\x20in\x20`BeginConfChange`\x20and\x20`Finaliz\ - eConfChange`.\r\n\n\r\n\x05\x04\x06\x02\x04\x04\x12\x04k\x04i\x16\n\x0c\ - \n\x05\x04\x06\x02\x04\x06\x12\x03k\x04\r\n\x0c\n\x05\x04\x06\x02\x04\ - \x01\x12\x03k\x0e\x1b\n\x0c\n\x05\x04\x06\x02\x04\x03\x12\x03k\x1e\x1f\n\ - \xc7\x01\n\x04\x04\x06\x02\x05\x12\x03o\x04\x1b\x1a\xb9\x01\x20Used\x20i\ - n\x20`BeginConfChange`\x20and\x20`FinalizeConfChange`.\r\n\x20Because\ - \x20`RawNode::apply_conf_change`\x20takes\x20a\x20`ConfChange`\x20instea\ - d\x20of\x20an\x20`Entry`\x20we\x20must\r\n\x20include\x20this\x20index\ - \x20so\x20it\x20can\x20be\x20known.\r\n\n\r\n\x05\x04\x06\x02\x05\x04\ - \x12\x04o\x04k\x20\n\x0c\n\x05\x04\x06\x02\x05\x05\x12\x03o\x04\n\n\x0c\ - \n\x05\x04\x06\x02\x05\x01\x12\x03o\x0b\x16\n\x0c\n\x05\x04\x06\x02\x05\ - \x03\x12\x03o\x19\x1ab\x06proto3\ + \x20is\x20determined\x20by\x20the\x20entry_type.\n\n\x20For\x20normal\ + \x20entries,\x20the\x20data\x20field\x20should\x20contain\x20the\x20data\ + \x20change\x20that\x20should\x20be\x20applied.\n\x20The\x20context\x20fi\ + eld\x20can\x20be\x20used\x20for\x20any\x20contextual\x20data\x20that\x20\ + might\x20be\x20relevant\x20to\x20the\n\x20application\x20of\x20the\x20da\ + ta.\n\n\x20For\x20configuration\x20changes,\x20the\x20data\x20will\x20co\ + ntain\x20the\x20ConfChange\x20message\x20and\x20the\n\x20context\x20will\ + \x20provide\x20anything\x20needed\x20to\x20assist\x20the\x20configuratio\ + n\x20change.\x20The\x20context\n\x20if\x20for\x20the\x20user\x20to\x20se\ + t\x20and\x20use\x20in\x20this\x20case.\n\n\n\n\x03\x04\0\x01\x12\x03\x12\ + \x08\r\n\x0b\n\x04\x04\0\x02\0\x12\x03\x13\x04\x1d\n\r\n\x05\x04\0\x02\0\ + \x04\x12\x04\x13\x04\x12\x0f\n\x0c\n\x05\x04\0\x02\0\x06\x12\x03\x13\x04\ + \r\n\x0c\n\x05\x04\0\x02\0\x01\x12\x03\x13\x0e\x18\n\x0c\n\x05\x04\0\x02\ + \0\x03\x12\x03\x13\x1b\x1c\n\x0b\n\x04\x04\0\x02\x01\x12\x03\x14\x04\x14\ + \n\r\n\x05\x04\0\x02\x01\x04\x12\x04\x14\x04\x13\x1d\n\x0c\n\x05\x04\0\ + \x02\x01\x05\x12\x03\x14\x04\n\n\x0c\n\x05\x04\0\x02\x01\x01\x12\x03\x14\ + \x0b\x0f\n\x0c\n\x05\x04\0\x02\x01\x03\x12\x03\x14\x12\x13\n\x0b\n\x04\ + \x04\0\x02\x02\x12\x03\x15\x04\x15\n\r\n\x05\x04\0\x02\x02\x04\x12\x04\ + \x15\x04\x14\x14\n\x0c\n\x05\x04\0\x02\x02\x05\x12\x03\x15\x04\n\n\x0c\n\ + \x05\x04\0\x02\x02\x01\x12\x03\x15\x0b\x10\n\x0c\n\x05\x04\0\x02\x02\x03\ + \x12\x03\x15\x13\x14\n\x0b\n\x04\x04\0\x02\x03\x12\x03\x16\x04\x13\n\r\n\ + \x05\x04\0\x02\x03\x04\x12\x04\x16\x04\x15\x15\n\x0c\n\x05\x04\0\x02\x03\ + \x05\x12\x03\x16\x04\t\n\x0c\n\x05\x04\0\x02\x03\x01\x12\x03\x16\n\x0e\n\ + \x0c\n\x05\x04\0\x02\x03\x03\x12\x03\x16\x11\x12\n\x0b\n\x04\x04\0\x02\ + \x04\x12\x03\x17\x04\x16\n\r\n\x05\x04\0\x02\x04\x04\x12\x04\x17\x04\x16\ + \x13\n\x0c\n\x05\x04\0\x02\x04\x05\x12\x03\x17\x04\t\n\x0c\n\x05\x04\0\ + \x02\x04\x01\x12\x03\x17\n\x11\n\x0c\n\x05\x04\0\x02\x04\x03\x12\x03\x17\ + \x14\x15\nm\n\x04\x04\0\x02\x05\x12\x03\x1b\x04\x16\x1a`\x20Deprecated!\ + \x20It\x20is\x20kept\x20for\x20backward\x20compatibility.\n\x20TODO:\x20\ + remove\x20it\x20in\x20the\x20next\x20major\x20release.\n\n\r\n\x05\x04\0\ + \x02\x05\x04\x12\x04\x1b\x04\x17\x16\n\x0c\n\x05\x04\0\x02\x05\x05\x12\ + \x03\x1b\x04\x08\n\x0c\n\x05\x04\0\x02\x05\x01\x12\x03\x1b\t\x11\n\x0c\n\ + \x05\x04\0\x02\x05\x03\x12\x03\x1b\x14\x15\n\n\n\x02\x04\x01\x12\x04\x1e\ + \0#\x01\n\n\n\x03\x04\x01\x01\x12\x03\x1e\x08\x18\n\x0b\n\x04\x04\x01\ + \x02\0\x12\x03\x1f\x04\x1d\n\r\n\x05\x04\x01\x02\0\x04\x12\x04\x1f\x04\ + \x1e\x1a\n\x0c\n\x05\x04\x01\x02\0\x06\x12\x03\x1f\x04\r\n\x0c\n\x05\x04\ + \x01\x02\0\x01\x12\x03\x1f\x0e\x18\n\x0c\n\x05\x04\x01\x02\0\x03\x12\x03\ + \x1f\x1b\x1c\n\x0b\n\x04\x04\x01\x02\x01\x12\x03\x20\x04-\n\r\n\x05\x04\ + \x01\x02\x01\x04\x12\x04\x20\x04\x1f\x1d\n\x0c\n\x05\x04\x01\x02\x01\x06\ + \x12\x03\x20\x04\x0e\n\x0c\n\x05\x04\x01\x02\x01\x01\x12\x03\x20\x0f(\n\ + \x0c\n\x05\x04\x01\x02\x01\x03\x12\x03\x20+,\n\x0b\n\x04\x04\x01\x02\x02\ + \x12\x03!\x04\x15\n\r\n\x05\x04\x01\x02\x02\x04\x12\x04!\x04\x20-\n\x0c\ + \n\x05\x04\x01\x02\x02\x05\x12\x03!\x04\n\n\x0c\n\x05\x04\x01\x02\x02\ + \x01\x12\x03!\x0b\x10\n\x0c\n\x05\x04\x01\x02\x02\x03\x12\x03!\x13\x14\n\ + \x0b\n\x04\x04\x01\x02\x03\x12\x03\"\x04\x14\n\r\n\x05\x04\x01\x02\x03\ + \x04\x12\x04\"\x04!\x15\n\x0c\n\x05\x04\x01\x02\x03\x05\x12\x03\"\x04\n\ + \n\x0c\n\x05\x04\x01\x02\x03\x01\x12\x03\"\x0b\x0f\n\x0c\n\x05\x04\x01\ + \x02\x03\x03\x12\x03\"\x12\x13\n\n\n\x02\x04\x02\x12\x04%\0(\x01\n\n\n\ + \x03\x04\x02\x01\x12\x03%\x08\x10\n\x0b\n\x04\x04\x02\x02\0\x12\x03&\x04\ + \x13\n\r\n\x05\x04\x02\x02\0\x04\x12\x04&\x04%\x12\n\x0c\n\x05\x04\x02\ + \x02\0\x05\x12\x03&\x04\t\n\x0c\n\x05\x04\x02\x02\0\x01\x12\x03&\n\x0e\n\ + \x0c\n\x05\x04\x02\x02\0\x03\x12\x03&\x11\x12\n\x0b\n\x04\x04\x02\x02\ + \x01\x12\x03'\x04\"\n\r\n\x05\x04\x02\x02\x01\x04\x12\x04'\x04&\x13\n\ + \x0c\n\x05\x04\x02\x02\x01\x06\x12\x03'\x04\x14\n\x0c\n\x05\x04\x02\x02\ + \x01\x01\x12\x03'\x15\x1d\n\x0c\n\x05\x04\x02\x02\x01\x03\x12\x03'\x20!\ + \n\n\n\x02\x05\x01\x12\x04*\0>\x01\n\n\n\x03\x05\x01\x01\x12\x03*\x05\ + \x10\n\x0b\n\x04\x05\x01\x02\0\x12\x03+\x04\x0f\n\x0c\n\x05\x05\x01\x02\ + \0\x01\x12\x03+\x04\n\n\x0c\n\x05\x05\x01\x02\0\x02\x12\x03+\r\x0e\n\x0b\ + \n\x04\x05\x01\x02\x01\x12\x03,\x04\x10\n\x0c\n\x05\x05\x01\x02\x01\x01\ + \x12\x03,\x04\x0b\n\x0c\n\x05\x05\x01\x02\x01\x02\x12\x03,\x0e\x0f\n\x0b\ + \n\x04\x05\x01\x02\x02\x12\x03-\x04\x13\n\x0c\n\x05\x05\x01\x02\x02\x01\ + \x12\x03-\x04\x0e\n\x0c\n\x05\x05\x01\x02\x02\x02\x12\x03-\x11\x12\n\x0b\ + \n\x04\x05\x01\x02\x03\x12\x03.\x04\x12\n\x0c\n\x05\x05\x01\x02\x03\x01\ + \x12\x03.\x04\r\n\x0c\n\x05\x05\x01\x02\x03\x02\x12\x03.\x10\x11\n\x0b\n\ + \x04\x05\x01\x02\x04\x12\x03/\x04\x1a\n\x0c\n\x05\x05\x01\x02\x04\x01\ + \x12\x03/\x04\x15\n\x0c\n\x05\x05\x01\x02\x04\x02\x12\x03/\x18\x19\n\x0b\ + \n\x04\x05\x01\x02\x05\x12\x030\x04\x17\n\x0c\n\x05\x05\x01\x02\x05\x01\ + \x12\x030\x04\x12\n\x0c\n\x05\x05\x01\x02\x05\x02\x12\x030\x15\x16\n\x0b\ + \n\x04\x05\x01\x02\x06\x12\x031\x04\x1f\n\x0c\n\x05\x05\x01\x02\x06\x01\ + \x12\x031\x04\x1a\n\x0c\n\x05\x05\x01\x02\x06\x02\x12\x031\x1d\x1e\n\x0b\ + \n\x04\x05\x01\x02\x07\x12\x032\x04\x14\n\x0c\n\x05\x05\x01\x02\x07\x01\ + \x12\x032\x04\x0f\n\x0c\n\x05\x05\x01\x02\x07\x02\x12\x032\x12\x13\n\x0b\ + \n\x04\x05\x01\x02\x08\x12\x033\x04\x15\n\x0c\n\x05\x05\x01\x02\x08\x01\ + \x12\x033\x04\x10\n\x0c\n\x05\x05\x01\x02\x08\x02\x12\x033\x13\x14\n\x0b\ + \n\x04\x05\x01\x02\t\x12\x034\x04\x1d\n\x0c\n\x05\x05\x01\x02\t\x01\x12\ + \x034\x04\x18\n\x0c\n\x05\x05\x01\x02\t\x02\x12\x034\x1b\x1c\n\x0b\n\x04\ + \x05\x01\x02\n\x12\x035\x04\x18\n\x0c\n\x05\x05\x01\x02\n\x01\x12\x035\ + \x04\x12\n\x0c\n\x05\x05\x01\x02\n\x02\x12\x035\x15\x17\n\x0b\n\x04\x05\ + \x01\x02\x0b\x12\x036\x04\x17\n\x0c\n\x05\x05\x01\x02\x0b\x01\x12\x036\ + \x04\x11\n\x0c\n\x05\x05\x01\x02\x0b\x02\x12\x036\x14\x16\n\x0b\n\x04\ + \x05\x01\x02\x0c\x12\x037\x04\x18\n\x0c\n\x05\x05\x01\x02\x0c\x01\x12\ + \x037\x04\x12\n\x0c\n\x05\x05\x01\x02\x0c\x02\x12\x037\x15\x17\n\x0b\n\ + \x04\x05\x01\x02\r\x12\x038\x04\x1b\n\x0c\n\x05\x05\x01\x02\r\x01\x12\ + \x038\x04\x15\n\x0c\n\x05\x05\x01\x02\r\x02\x12\x038\x18\x1a\n\x0b\n\x04\ + \x05\x01\x02\x0e\x12\x039\x04\x17\n\x0c\n\x05\x05\x01\x02\x0e\x01\x12\ + \x039\x04\x11\n\x0c\n\x05\x05\x01\x02\x0e\x02\x12\x039\x14\x16\n\x0b\n\ + \x04\x05\x01\x02\x0f\x12\x03:\x04\x16\n\x0c\n\x05\x05\x01\x02\x0f\x01\ + \x12\x03:\x04\x10\n\x0c\n\x05\x05\x01\x02\x0f\x02\x12\x03:\x13\x15\n\x0b\ + \n\x04\x05\x01\x02\x10\x12\x03;\x04\x1a\n\x0c\n\x05\x05\x01\x02\x10\x01\ + \x12\x03;\x04\x14\n\x0c\n\x05\x05\x01\x02\x10\x02\x12\x03;\x17\x19\n\x0b\ + \n\x04\x05\x01\x02\x11\x12\x03<\x04\x1b\n\x0c\n\x05\x05\x01\x02\x11\x01\ + \x12\x03<\x04\x15\n\x0c\n\x05\x05\x01\x02\x11\x02\x12\x03<\x18\x1a\n\x0b\ + \n\x04\x05\x01\x02\x12\x12\x03=\x04#\n\x0c\n\x05\x05\x01\x02\x12\x01\x12\ + \x03=\x04\x1d\n\x0c\n\x05\x05\x01\x02\x12\x02\x12\x03=\x20\"\n\n\n\x02\ + \x04\x03\x12\x04@\0M\x01\n\n\n\x03\x04\x03\x01\x12\x03@\x08\x0f\n\x0b\n\ + \x04\x04\x03\x02\0\x12\x03A\x04\x1d\n\r\n\x05\x04\x03\x02\0\x04\x12\x04A\ + \x04@\x11\n\x0c\n\x05\x04\x03\x02\0\x06\x12\x03A\x04\x0f\n\x0c\n\x05\x04\ + \x03\x02\0\x01\x12\x03A\x10\x18\n\x0c\n\x05\x04\x03\x02\0\x03\x12\x03A\ + \x1b\x1c\n\x0b\n\x04\x04\x03\x02\x01\x12\x03B\x04\x12\n\r\n\x05\x04\x03\ + \x02\x01\x04\x12\x04B\x04A\x1d\n\x0c\n\x05\x04\x03\x02\x01\x05\x12\x03B\ + \x04\n\n\x0c\n\x05\x04\x03\x02\x01\x01\x12\x03B\x0b\r\n\x0c\n\x05\x04\ + \x03\x02\x01\x03\x12\x03B\x10\x11\n\x0b\n\x04\x04\x03\x02\x02\x12\x03C\ + \x04\x14\n\r\n\x05\x04\x03\x02\x02\x04\x12\x04C\x04B\x12\n\x0c\n\x05\x04\ + \x03\x02\x02\x05\x12\x03C\x04\n\n\x0c\n\x05\x04\x03\x02\x02\x01\x12\x03C\ + \x0b\x0f\n\x0c\n\x05\x04\x03\x02\x02\x03\x12\x03C\x12\x13\n\x0b\n\x04\ + \x04\x03\x02\x03\x12\x03D\x04\x14\n\r\n\x05\x04\x03\x02\x03\x04\x12\x04D\ + \x04C\x14\n\x0c\n\x05\x04\x03\x02\x03\x05\x12\x03D\x04\n\n\x0c\n\x05\x04\ + \x03\x02\x03\x01\x12\x03D\x0b\x0f\n\x0c\n\x05\x04\x03\x02\x03\x03\x12\ + \x03D\x12\x13\n\x0b\n\x04\x04\x03\x02\x04\x12\x03E\x04\x18\n\r\n\x05\x04\ + \x03\x02\x04\x04\x12\x04E\x04D\x14\n\x0c\n\x05\x04\x03\x02\x04\x05\x12\ + \x03E\x04\n\n\x0c\n\x05\x04\x03\x02\x04\x01\x12\x03E\x0b\x13\n\x0c\n\x05\ + \x04\x03\x02\x04\x03\x12\x03E\x16\x17\n\x0b\n\x04\x04\x03\x02\x05\x12\ + \x03F\x04\x15\n\r\n\x05\x04\x03\x02\x05\x04\x12\x04F\x04E\x18\n\x0c\n\ + \x05\x04\x03\x02\x05\x05\x12\x03F\x04\n\n\x0c\n\x05\x04\x03\x02\x05\x01\ + \x12\x03F\x0b\x10\n\x0c\n\x05\x04\x03\x02\x05\x03\x12\x03F\x13\x14\n\x0b\ + \n\x04\x04\x03\x02\x06\x12\x03G\x04\x1f\n\x0c\n\x05\x04\x03\x02\x06\x04\ + \x12\x03G\x04\x0c\n\x0c\n\x05\x04\x03\x02\x06\x06\x12\x03G\r\x12\n\x0c\n\ + \x05\x04\x03\x02\x06\x01\x12\x03G\x13\x1a\n\x0c\n\x05\x04\x03\x02\x06\ + \x03\x12\x03G\x1d\x1e\n\x0b\n\x04\x04\x03\x02\x07\x12\x03H\x04\x16\n\r\n\ + \x05\x04\x03\x02\x07\x04\x12\x04H\x04G\x1f\n\x0c\n\x05\x04\x03\x02\x07\ + \x05\x12\x03H\x04\n\n\x0c\n\x05\x04\x03\x02\x07\x01\x12\x03H\x0b\x11\n\ + \x0c\n\x05\x04\x03\x02\x07\x03\x12\x03H\x14\x15\n\x0b\n\x04\x04\x03\x02\ + \x08\x12\x03I\x04\x1a\n\r\n\x05\x04\x03\x02\x08\x04\x12\x04I\x04H\x16\n\ + \x0c\n\x05\x04\x03\x02\x08\x06\x12\x03I\x04\x0c\n\x0c\n\x05\x04\x03\x02\ + \x08\x01\x12\x03I\r\x15\n\x0c\n\x05\x04\x03\x02\x08\x03\x12\x03I\x18\x19\ + \n\x0b\n\x04\x04\x03\x02\t\x12\x03J\x04\x15\n\r\n\x05\x04\x03\x02\t\x04\ + \x12\x04J\x04I\x1a\n\x0c\n\x05\x04\x03\x02\t\x05\x12\x03J\x04\x08\n\x0c\ + \n\x05\x04\x03\x02\t\x01\x12\x03J\t\x0f\n\x0c\n\x05\x04\x03\x02\t\x03\ + \x12\x03J\x12\x14\n\x0b\n\x04\x04\x03\x02\n\x12\x03K\x04\x1c\n\r\n\x05\ + \x04\x03\x02\n\x04\x12\x04K\x04J\x15\n\x0c\n\x05\x04\x03\x02\n\x05\x12\ + \x03K\x04\n\n\x0c\n\x05\x04\x03\x02\n\x01\x12\x03K\x0b\x16\n\x0c\n\x05\ + \x04\x03\x02\n\x03\x12\x03K\x19\x1b\n\x0b\n\x04\x04\x03\x02\x0b\x12\x03L\ + \x04\x17\n\r\n\x05\x04\x03\x02\x0b\x04\x12\x04L\x04K\x1c\n\x0c\n\x05\x04\ + \x03\x02\x0b\x05\x12\x03L\x04\t\n\x0c\n\x05\x04\x03\x02\x0b\x01\x12\x03L\ + \n\x11\n\x0c\n\x05\x04\x03\x02\x0b\x03\x12\x03L\x14\x16\n\n\n\x02\x04\ + \x04\x12\x04O\0U\x01\n\n\n\x03\x04\x04\x01\x12\x03O\x08\x11\n\x0b\n\x04\ + \x04\x04\x02\0\x12\x03P\x04\x14\n\r\n\x05\x04\x04\x02\0\x04\x12\x04P\x04\ + O\x13\n\x0c\n\x05\x04\x04\x02\0\x05\x12\x03P\x04\n\n\x0c\n\x05\x04\x04\ + \x02\0\x01\x12\x03P\x0b\x0f\n\x0c\n\x05\x04\x04\x02\0\x03\x12\x03P\x12\ + \x13\n\x0b\n\x04\x04\x04\x02\x01\x12\x03Q\x04\x14\n\r\n\x05\x04\x04\x02\ + \x01\x04\x12\x04Q\x04P\x14\n\x0c\n\x05\x04\x04\x02\x01\x05\x12\x03Q\x04\ + \n\n\x0c\n\x05\x04\x04\x02\x01\x01\x12\x03Q\x0b\x0f\n\x0c\n\x05\x04\x04\ + \x02\x01\x03\x12\x03Q\x12\x13\n\x0b\n\x04\x04\x04\x02\x02\x12\x03R\x04\ + \x16\n\r\n\x05\x04\x04\x02\x02\x04\x12\x04R\x04Q\x14\n\x0c\n\x05\x04\x04\ + \x02\x02\x05\x12\x03R\x04\n\n\x0c\n\x05\x04\x04\x02\x02\x01\x12\x03R\x0b\ + \x11\n\x0c\n\x05\x04\x04\x02\x02\x03\x12\x03R\x14\x15\nd\n\x04\x04\x04\ + \x02\x03\x12\x03T\x04-\x1aW\x20If\x20the\x20peer\x20goes\x20down\x20we\ + \x20need\x20to\x20be\x20aware\x20of\x20a\x20partially\x20completed\x20me\ + mbership\x20change.\n\n\r\n\x05\x04\x04\x02\x03\x04\x12\x04T\x04R\x16\n\ + \x0c\n\x05\x04\x04\x02\x03\x06\x12\x03T\x04\x0e\n\x0c\n\x05\x04\x04\x02\ + \x03\x01\x12\x03T\x0f(\n\x0c\n\x05\x04\x04\x02\x03\x03\x12\x03T+,\n\n\n\ + \x02\x04\x05\x12\x04W\0Z\x01\n\n\n\x03\x04\x05\x01\x12\x03W\x08\x11\n\ + \x0b\n\x04\x04\x05\x02\0\x12\x03X\x04\x1e\n\x0c\n\x05\x04\x05\x02\0\x04\ + \x12\x03X\x04\x0c\n\x0c\n\x05\x04\x05\x02\0\x05\x12\x03X\r\x13\n\x0c\n\ + \x05\x04\x05\x02\0\x01\x12\x03X\x14\x19\n\x0c\n\x05\x04\x05\x02\0\x03\ + \x12\x03X\x1c\x1d\n\x0b\n\x04\x04\x05\x02\x01\x12\x03Y\x04!\n\x0c\n\x05\ + \x04\x05\x02\x01\x04\x12\x03Y\x04\x0c\n\x0c\n\x05\x04\x05\x02\x01\x05\ + \x12\x03Y\r\x13\n\x0c\n\x05\x04\x05\x02\x01\x01\x12\x03Y\x14\x1c\n\x0c\n\ + \x05\x04\x05\x02\x01\x03\x12\x03Y\x1f\x20\n\n\n\x02\x05\x02\x12\x04\\\0b\ + \x01\n\n\n\x03\x05\x02\x01\x12\x03\\\x05\x13\n\x0b\n\x04\x05\x02\x02\0\ + \x12\x03]\x04\x13\n\x0c\n\x05\x05\x02\x02\0\x01\x12\x03]\x04\x0b\n\x0c\n\ + \x05\x05\x02\x02\0\x02\x12\x03]\x11\x12\n\x0b\n\x04\x05\x02\x02\x01\x12\ + \x03^\x04\x13\n\x0c\n\x05\x05\x02\x02\x01\x01\x12\x03^\x04\x0e\n\x0c\n\ + \x05\x05\x02\x02\x01\x02\x12\x03^\x11\x12\n\x0b\n\x04\x05\x02\x02\x02\ + \x12\x03_\x04\x17\n\x0c\n\x05\x05\x02\x02\x02\x01\x12\x03_\x04\x12\n\x0c\ + \n\x05\x05\x02\x02\x02\x02\x12\x03_\x15\x16\n\x0b\n\x04\x05\x02\x02\x03\ + \x12\x03`\x04\x18\n\x0c\n\x05\x05\x02\x02\x03\x01\x12\x03`\x04\x13\n\x0c\ + \n\x05\x05\x02\x02\x03\x02\x12\x03`\x16\x17\n\x0b\n\x04\x05\x02\x02\x04\ + \x12\x03a\x04\x1b\n\x0c\n\x05\x05\x02\x02\x04\x01\x12\x03a\x04\x16\n\x0c\ + \n\x05\x05\x02\x02\x04\x02\x12\x03a\x19\x1a\n\n\n\x02\x04\x06\x12\x04d\0\ + p\x01\n\n\n\x03\x04\x06\x01\x12\x03d\x08\x12\n\x0b\n\x04\x04\x06\x02\0\ + \x12\x03e\x04\x12\n\r\n\x05\x04\x06\x02\0\x04\x12\x04e\x04d\x14\n\x0c\n\ + \x05\x04\x06\x02\0\x05\x12\x03e\x04\n\n\x0c\n\x05\x04\x06\x02\0\x01\x12\ + \x03e\x0b\r\n\x0c\n\x05\x04\x06\x02\0\x03\x12\x03e\x10\x11\n\x0b\n\x04\ + \x04\x06\x02\x01\x12\x03f\x04#\n\r\n\x05\x04\x06\x02\x01\x04\x12\x04f\ + \x04e\x12\n\x0c\n\x05\x04\x06\x02\x01\x06\x12\x03f\x04\x12\n\x0c\n\x05\ + \x04\x06\x02\x01\x01\x12\x03f\x13\x1e\n\x0c\n\x05\x04\x06\x02\x01\x03\ + \x12\x03f!\"\nE\n\x04\x04\x06\x02\x02\x12\x03h\x04\x17\x1a8\x20Used\x20i\ + n\x20`AddNode`,\x20`RemoveNode`,\x20and\x20`AddLearnerNode`.\n\n\r\n\x05\ + \x04\x06\x02\x02\x04\x12\x04h\x04f#\n\x0c\n\x05\x04\x06\x02\x02\x05\x12\ + \x03h\x04\n\n\x0c\n\x05\x04\x06\x02\x02\x01\x12\x03h\x0b\x12\n\x0c\n\x05\ + \x04\x06\x02\x02\x03\x12\x03h\x15\x16\n\x0b\n\x04\x04\x06\x02\x03\x12\ + \x03i\x04\x16\n\r\n\x05\x04\x06\x02\x03\x04\x12\x04i\x04h\x17\n\x0c\n\ + \x05\x04\x06\x02\x03\x05\x12\x03i\x04\t\n\x0c\n\x05\x04\x06\x02\x03\x01\ + \x12\x03i\n\x11\n\x0c\n\x05\x04\x06\x02\x03\x03\x12\x03i\x14\x15\nB\n\ + \x04\x04\x06\x02\x04\x12\x03k\x04\x20\x1a5\x20Used\x20in\x20`BeginConfCh\ + ange`\x20and\x20`FinalizeConfChange`.\n\n\r\n\x05\x04\x06\x02\x04\x04\ + \x12\x04k\x04i\x16\n\x0c\n\x05\x04\x06\x02\x04\x06\x12\x03k\x04\r\n\x0c\ + \n\x05\x04\x06\x02\x04\x01\x12\x03k\x0e\x1b\n\x0c\n\x05\x04\x06\x02\x04\ + \x03\x12\x03k\x1e\x1f\n\xc4\x01\n\x04\x04\x06\x02\x05\x12\x03o\x04\x1b\ + \x1a\xb6\x01\x20Used\x20in\x20`BeginConfChange`\x20and\x20`FinalizeConfC\ + hange`.\n\x20Because\x20`RawNode::apply_conf_change`\x20takes\x20a\x20`C\ + onfChange`\x20instead\x20of\x20an\x20`Entry`\x20we\x20must\n\x20include\ + \x20this\x20index\x20so\x20it\x20can\x20be\x20known.\n\n\r\n\x05\x04\x06\ + \x02\x05\x04\x12\x04o\x04k\x20\n\x0c\n\x05\x04\x06\x02\x05\x05\x12\x03o\ + \x04\n\n\x0c\n\x05\x04\x06\x02\x05\x01\x12\x03o\x0b\x16\n\x0c\n\x05\x04\ + \x06\x02\x05\x03\x12\x03o\x19\x1ab\x06proto3\ "; static mut file_descriptor_proto_lazy: ::protobuf::lazy::Lazy<::protobuf::descriptor::FileDescriptorProto> = ::protobuf::lazy::Lazy { diff --git a/src/progress.rs b/src/progress.rs index 6ac489d36..360b14702 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -103,11 +103,11 @@ impl From for Configuration { } } -impl Into for Configuration { - fn into(self) -> ConfState { - let mut state = ConfState::new(); - state.set_nodes(self.voters.iter().cloned().collect()); - state.set_learners(self.learners.iter().cloned().collect()); +impl From for ConfState { + fn from(conf: Configuration) -> Self { + let mut state = ConfState::default(); + state.set_nodes(conf.voters.iter().cloned().collect()); + state.set_learners(conf.learners.iter().cloned().collect()); state } } diff --git a/src/raft.rs b/src/raft.rs index 35e62dbae..8a864368c 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -162,9 +162,9 @@ pub struct Raft { /// It is important that whenever this is set that `pending_conf_index` is also set to the /// value if it is greater than the existing value. /// - /// **Use `Raft::set_began_conf_change_at()` to change this value.** + /// **Use `Raft::set_pending_membership_change()` to change this value.** #[get = "pub"] - began_conf_change_at: Option, + pending_membership_change: Option, /// The queue of read-only requests. pub read_only: ReadOnly, @@ -271,7 +271,7 @@ impl Raft { term: Default::default(), election_elapsed: Default::default(), pending_conf_index: Default::default(), - began_conf_change_at: Default::default(), + pending_membership_change: Default::default(), vote: Default::default(), heartbeat_elapsed: Default::default(), randomized_election_timeout: 0, @@ -362,10 +362,8 @@ impl Raft { hs.set_term(self.term); hs.set_vote(self.vote); hs.set_commit(self.raft_log.committed); - // HACK: Our current Protobuf uses `0` as `None`. - // Good thing we know that index 0 of the raft log is always not important in this context. - if let Some(began_conf_change_at) = self.began_conf_change_at() { - hs.set_began_conf_change_at(*began_conf_change_at); + if let Some(pending_membership_change) = self.pending_membership_change() { + hs.set_pending_membership_change(pending_membership_change.clone()); } hs } @@ -408,18 +406,26 @@ impl Raft { self.skip_bcast_commit = skip; } - /// Set when the peer began a joint consensus change. This will also set `pending_conf_index` - /// if it is larger than the existing number. + /// Set when the peer began a joint consensus change. + /// + /// This will also set `pending_conf_index` if it is larger than the existing number. #[inline] - fn set_began_conf_change_at(&mut self, maybe_index: impl Into>) { - let maybe_index = maybe_index.into(); - if let Some(index) = maybe_index { - assert!(self.began_conf_change_at.is_none()); + fn set_pending_membership_change(&mut self, maybe_change: impl Into>) { + let maybe_change = maybe_change.into(); + if let Some(ref change) = maybe_change { + assert!(self.pending_membership_change.is_none()); + let index = change.get_start_index(); if index > self.pending_conf_index { self.pending_conf_index = index; } } - self.began_conf_change_at = maybe_index; + self.pending_membership_change = maybe_change.clone(); + } + + /// Get the index which the pending membership change started at. + #[inline] + pub fn began_membership_change_at(&self) -> Option { + self.pending_membership_change.as_ref().map(|v| v.get_start_index()) } /// send persists state to stable storage and then sends to its mailbox. @@ -646,23 +652,12 @@ impl Raft { self.raft_log.applied_to(applied); // Check to see if we need to finalize a Joint Consensus state now. - if let Some(index) = self.began_conf_change_at { - // Invariant: We know that the index stored at `began_conf_change_at` should be a - // `BeginConfChange`. - // Check this in debug mode for safety while testing, but skip it in production since - // those bugs should have been caught, and doing this can be expensive. - debug_assert_eq!( - self.raft_log - .entries(index, 1) - .ok() - .and_then( - |vec| vec.get(0).and_then(|entry| { - protobuf::parse_from_bytes::(entry.get_data()).ok() - }) - ) - .map(|conf_change| conf_change.get_change_type()), - Some(ConfChangeType::BeginConfChange), - ); + let start_index = self.pending_membership_change + .as_ref() + .map(|v| Some(v.get_start_index())) + .unwrap_or(None); + + if let Some(index) = start_index { // Invariant: We know that if we have commited past some index, we can also commit that index. if applied >= index && self.state == StateRole::Leader { // We must replicate the commit entry. @@ -872,18 +867,23 @@ impl Raft { self.pending_conf_index = self.raft_log.last_index(); self.append_entry(&mut [Entry::new()]); + // In most cases, we append only a new entry marked with an index and term. // In the specific case of a node recovering while in the middle of a membership change, // and the finalization entry may have been lost, we must also append that, since it // would be overwritten by the term change. - if let Some(began) = self.began_conf_change_at { + let change_start_index = self.pending_membership_change + .as_ref() + .map(|v| Some(v.get_start_index())) + .unwrap_or(None); + if let Some(index) = change_start_index { trace!( "Checking if we need to finalize again..., began: {}, applied: {}, committed: {}", - began, + index, self.raft_log.applied, self.raft_log.committed ); - if began <= self.raft_log.committed { + if index <= self.raft_log.committed { self.append_finalize_conf_change_entry(); } } @@ -1224,7 +1224,7 @@ impl Raft { )); }; - self.set_began_conf_change_at(start_index); + self.set_pending_membership_change(conf_change.clone()); let max_inflights = self.max_inflight; self.mut_prs() .begin_membership_change(configuration, Progress::new(1, max_inflights))?; @@ -1287,7 +1287,7 @@ impl Raft { self.mut_prs().finalize_membership_change()?; // Ensure we reset this on *any* node, since the leader might have failed // and we don't want to finalize twice. - self.set_began_conf_change_at(None); + self.set_pending_membership_change(None); Ok(()) } @@ -2039,6 +2039,12 @@ impl Raft { ); } } + + if meta.has_pending_membership_change() { + let change = meta.get_pending_membership_change(); + self.begin_membership_change(change).expect("Expected already valid change to still be valid."); + } + None } @@ -2061,7 +2067,7 @@ impl Raft { /// This method can be false positive. #[inline] pub fn has_pending_conf(&self) -> bool { - self.pending_conf_index > self.raft_log.applied || self.began_conf_change_at.is_some() + self.pending_conf_index > self.raft_log.applied || self.pending_membership_change.is_some() } /// Specifies if the commit should be broadcast. @@ -2243,16 +2249,10 @@ impl Raft { self.raft_log.committed = hs.get_commit(); self.term = hs.get_term(); self.vote = hs.get_vote(); - // HACK: Our current Protobuf uses `0` as `None`. - // Good thing we know that index 0 of the raft log is always not important in this context. - let began_conf_change_at = hs.get_began_conf_change_at(); - if began_conf_change_at != 0 { - let entry = &self - .get_store() - .entries(began_conf_change_at, began_conf_change_at + 1, NO_LIMIT) - .expect("Expected to find entry at location of last membership change.")[0]; - let conf_change = protobuf::parse_from_bytes::(entry.get_data()) - .expect("Expected the membership change already recorded to be valid."); + + // See if we need to re-apply some membership change entries. + if hs.has_pending_membership_change() { + let conf_change = hs.get_pending_membership_change().clone(); self.begin_membership_change(&conf_change) .expect("Expected the membership change already recorded to be valid."); } diff --git a/src/raw_node.rs b/src/raw_node.rs index 8b31b6b4a..6c7369284 100644 --- a/src/raw_node.rs +++ b/src/raw_node.rs @@ -357,10 +357,7 @@ impl RawNode { self.raft.mut_prs().finalize_membership_change().unwrap(); } } - let mut cs = ConfState::new(); - cs.set_nodes(self.raft.prs().voter_ids().iter().cloned().collect()); - cs.set_learners(self.raft.prs().learner_ids().iter().cloned().collect()); - cs + self.raft.prs().configuration().clone().into() } /// Step advances the state machine using the given message. diff --git a/src/storage.rs b/src/storage.rs index 4e8a10761..ce02438c6 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -33,7 +33,7 @@ use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}; -use eraftpb::{ConfState, Entry, HardState, Snapshot}; +use eraftpb::{ConfState, ConfChange, Entry, HardState, Snapshot}; use errors::{Error, Result, StorageError}; use util; @@ -137,6 +137,7 @@ impl MemStorageCore { &mut self, idx: u64, cs: Option, + pending_membership_change: Option, data: Vec, ) -> Result<&Snapshot> { if idx <= self.snapshot.get_metadata().get_index() { @@ -158,6 +159,9 @@ impl MemStorageCore { if let Some(cs) = cs { self.snapshot.mut_metadata().set_conf_state(cs) } + if let Some(pending_change) = pending_membership_change { + self.snapshot.mut_metadata().set_pending_membership_change(pending_change); + } self.snapshot.set_data(data); Ok(&self.snapshot) } @@ -539,7 +543,7 @@ mod test { storage .wl() - .create_snapshot(idx, Some(cs.clone()), data.clone()) + .create_snapshot(idx, Some(cs.clone()), None, data.clone()) .expect("create snapshot failed"); let result = storage.snapshot(); if result != wresult { diff --git a/src/util.rs b/src/util.rs index f74a76390..1d3f2ec65 100644 --- a/src/util.rs +++ b/src/util.rs @@ -17,6 +17,7 @@ use std::u64; use protobuf::Message; +use eraftpb::ConfState; /// A number to represent that there is no limit. pub const NO_LIMIT: u64 = u64::MAX; @@ -69,3 +70,10 @@ pub fn limit_size(entries: &mut Vec, max: u64) { entries.truncate(limit); } + +// Bring some consistency to things. The protobuf has `nodes` and it's not really a term that's used anymore. +impl ConfState { + fn get_voters(&self) -> &[u64] { + self.get_nodes() + } +} \ No newline at end of file diff --git a/tests/integration_cases/test_membership_changes.rs b/tests/integration_cases/test_membership_changes.rs index 0cdae3919..5c6bed848 100644 --- a/tests/integration_cases/test_membership_changes.rs +++ b/tests/integration_cases/test_membership_changes.rs @@ -15,7 +15,7 @@ use fxhash::{FxHashMap, FxHashSet}; use protobuf::{self, RepeatedField}; use raft::{ - eraftpb::{ConfChange, ConfChangeType, ConfState, Entry, EntryType, Message, MessageType}, + eraftpb::{ConfChange, Snapshot, ConfChangeType, ConfState, Entry, EntryType, Message, MessageType}, storage::{MemStorage, Storage}, Config, Configuration, Raft, Result, INVALID_ID, NO_LIMIT, }; @@ -538,7 +538,7 @@ mod three_peers_replace_voter { /// The leader power cycles before actually sending the messages. #[test] - fn leader_power_cycles() -> Result<()> { + fn leader_power_cycles_no_compaction() -> Result<()> { setup_for_test(); let leader = 1; let old_configuration = (vec![1, 2, 3], vec![]); @@ -568,9 +568,9 @@ mod three_peers_replace_voter { scenario.assert_in_membership_change(&[1, 2, 3]); info!("Leader power cycles."); - assert_eq!(scenario.peers[&1].began_conf_change_at(), &Some(2)); - scenario.power_cycle(&[1]); - assert_eq!(scenario.peers[&1].began_conf_change_at(), &Some(2)); + assert_eq!(scenario.peers[&1].began_membership_change_at(), Some(2)); + scenario.power_cycle(&[1], None); + assert_eq!(scenario.peers[&1].began_membership_change_at(), Some(2)); scenario.assert_in_membership_change(&[1]); { let peer = scenario.peers.get_mut(&1).unwrap(); @@ -592,7 +592,85 @@ mod three_peers_replace_voter { info!("Cluster leaving the joint."); scenario.expect_read_and_dispatch_messages_from(&[4, 3, 2, 1, 4, 3, 2, 1])?; - assert_eq!(scenario.peers[&1].began_conf_change_at(), &Some(2)); + assert_eq!(scenario.peers[&1].began_membership_change_at(), Some(2)); + scenario.assert_can_apply_transition_entry_at_index( + &[1, 2, 3, 4], + 4, + ConfChangeType::FinalizeConfChange, + ); + scenario.assert_not_in_membership_change(&[1, 2, 4]); + + Ok(()) + } + + /// The leader power cycles before actually sending the messages. + #[test] + fn leader_power_cycles_compacted_log() -> Result<()> { + setup_for_test(); + let leader = 1; + let old_configuration = (vec![1, 2, 3], vec![]); + let new_configuration = (vec![1, 2, 4], vec![]); + let mut scenario = Scenario::new(leader, old_configuration, new_configuration)?; + scenario.spawn_new_peers()?; + scenario.propose_change_message()?; + + info!("Allowing quorum to commit"); + scenario.expect_read_and_dispatch_messages_from(&[1, 2, 3])?; + + info!("Advancing leader, now entered the joint"); + scenario.assert_can_apply_transition_entry_at_index( + &[1], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_membership_change(&[1]); + + info!("Leader replicates the commit and finalize entry."); + scenario.expect_read_and_dispatch_messages_from(&[1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[2, 3], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_membership_change(&[1, 2, 3]); + + + info!("Compacting leader's log"); + let snapshot = { + let peer = scenario.peers.get_mut(&1).unwrap(); + peer.raft_log.store.wl().create_snapshot( + 2, + ConfState::from(peer.prs().configuration().clone()).into(), + peer.pending_membership_change().clone(), + vec![] + )?; + let snapshot = peer.raft_log.snapshot()?; + peer.raft_log.store.wl().compact(2)?; + snapshot + }; + + info!("Leader power cycles."); + assert_eq!(scenario.peers[&1].began_membership_change_at(), Some(2)); + scenario.power_cycle(&[1], snapshot); + + assert_eq!(scenario.peers[&1].began_membership_change_at(), Some(2)); + scenario.assert_in_membership_change(&[1]); + { + let peer = scenario.peers.get_mut(&1).unwrap(); + peer.become_candidate(); + peer.become_leader(); + for _ in peer.get_heartbeat_elapsed()..=(peer.get_heartbeat_timeout() + 1) { + peer.tick(); + } + } + + info!("Allowing new peers to catch up."); + scenario.expect_read_and_dispatch_messages_from(&[4, 1, 4, 1, 4, 1, 4, 1])?; + scenario.assert_in_membership_change(&[1, 2, 3, 4]); + + info!("Cluster leaving the joint."); + scenario.expect_read_and_dispatch_messages_from(&[4, 3, 2, 1, 4, 3, 2, 1])?; + assert_eq!(scenario.peers[&1].began_membership_change_at(), Some(2)); scenario.assert_can_apply_transition_entry_at_index( &[1, 2, 3, 4], 4, @@ -1105,6 +1183,64 @@ mod intermingled_config_changes { } } +mod compaction { + use super::*; + + // Ensure that if a Raft compacts its log before finalizing that there are no failures. + #[test] + fn begin_compact_then_finalize() -> Result<()> { + setup_for_test(); + let leader = 1; + let old_configuration = (vec![1, 2, 3], vec![]); + let new_configuration = (vec![1, 2, 3], vec![4]); + let mut scenario = Scenario::new(leader, old_configuration, new_configuration)?; + scenario.spawn_new_peers()?; + scenario.propose_change_message()?; + + info!("Allowing quorum to commit"); + scenario.expect_read_and_dispatch_messages_from(&[1, 2, 3])?; + + info!("Advancing leader, now entered the joint"); + scenario.assert_can_apply_transition_entry_at_index( + &[1], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_membership_change(&[1]); + + info!("Leader replicates the commit and finalize entry."); + scenario.expect_read_and_dispatch_messages_from(&[1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[2, 3], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_membership_change(&[1, 2, 3]); + + info!("Allowing new peers to catch up."); + scenario.expect_read_and_dispatch_messages_from(&[4, 1, 4])?; + scenario.assert_can_apply_transition_entry_at_index( + &[4], + 2, + ConfChangeType::BeginConfChange, + ); + scenario.assert_in_membership_change(&[1, 2, 3, 4]); + + scenario.peers.get_mut(&1).unwrap().raft_log.store.wl().compact(2)?; + + info!("Cluster leaving the joint."); + scenario.expect_read_and_dispatch_messages_from(&[3, 2, 1])?; + scenario.assert_can_apply_transition_entry_at_index( + &[1, 2, 3, 4], + 3, + ConfChangeType::FinalizeConfChange, + ); + scenario.assert_not_in_membership_change(&[1, 2, 3, 4]); + + Ok(()) + } +} + /// A test harness providing some useful utility and shorthand functions appropriate for this test suite. /// /// Since it derefs into `Network` it can be used the same way. So it acts as a transparent set of utilities over the standard `Network`. @@ -1370,23 +1506,35 @@ impl Scenario { /// Simulate a power cycle in the given nodes. /// /// This means that the MemStorage is kept, but nothing else. - fn power_cycle<'a>(&mut self, peers: impl IntoIterator) { + fn power_cycle<'a>(&mut self, peers: impl IntoIterator, snapshot: impl Into>) { let peers = peers.into_iter().cloned(); + let snapshot = snapshot.into(); for id in peers { debug!("Power cycling {}.", id); let mut peer = self.peers.remove(&id).expect("Peer did not exist."); let store = peer.mut_store().clone(); let prs = peer.prs(); - let mut peer = Raft::new( - &Config { - id, - peers: prs.voter_ids().iter().cloned().collect(), - learners: prs.learner_ids().iter().cloned().collect(), - ..Default::default() - }, - store, - ) - .expect("Could not create new Raft"); + let mut peer = if let Some(ref snapshot) = snapshot { + let mut peer = Raft::new( + &Config { + id, + ..Default::default() + }, + store, + ).expect("Could not create new Raft"); + peer.restore(snapshot.clone()); + peer + } else { + Raft::new( + &Config { + id, + peers: prs.voter_ids().iter().cloned().collect(), + learners: prs.learner_ids().iter().cloned().collect(), + ..Default::default() + }, + store, + ).expect("Could not create new Raft") + }; self.peers.insert(id, peer.into()); } } diff --git a/tests/integration_cases/test_raft.rs b/tests/integration_cases/test_raft.rs index 375065ffd..1a8967a71 100644 --- a/tests/integration_cases/test_raft.rs +++ b/tests/integration_cases/test_raft.rs @@ -2849,7 +2849,7 @@ fn test_slow_node_restore() { cs.set_nodes(nt.peers[&1].prs().voter_ids().iter().cloned().collect()); nt.storage[&1] .wl() - .create_snapshot(nt.peers[&1].raft_log.applied, Some(cs), vec![]) + .create_snapshot(nt.peers[&1].raft_log.applied, Some(cs), None, vec![]) .expect(""); nt.storage[&1] .wl() @@ -3229,7 +3229,7 @@ fn test_leader_transfer_after_snapshot() { cs.set_nodes(nt.peers[&1].prs().voter_ids().iter().cloned().collect()); nt.storage[&1] .wl() - .create_snapshot(nt.peers[&1].raft_log.applied, Some(cs), vec![]) + .create_snapshot(nt.peers[&1].raft_log.applied, Some(cs), None, vec![]) .expect(""); nt.storage[&1] .wl() From 042992782e95e10bafd85c7500808cf5bb17dc05 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Tue, 8 Jan 2019 17:23:33 -0800 Subject: [PATCH 22/41] Some refinements, still problems. --- src/raft.rs | 24 +++++++++++++++++++ .../test_membership_changes.rs | 16 ++++++------- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/src/raft.rs b/src/raft.rs index 8a864368c..17163ac76 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -2042,6 +2042,30 @@ impl Raft { if meta.has_pending_membership_change() { let change = meta.get_pending_membership_change(); + + let (voters, learners) = { + let config = Configuration::from(change.get_configuration().clone()); + let voters = config.voters().difference(self.prs().configuration().voters()).cloned().collect::>(); + let learners = config.learners().difference(self.prs().configuration().learners()).cloned().collect::>(); + (voters, learners) + }; + for &(is_learner, ref nodes) in &[(false, voters), (true, learners)] { + for &n in nodes { + let next_index = self.raft_log.last_index() + 1; + let mut matched = 0; + if n == self.id { + matched = next_index - 1; + self.is_learner = is_learner; + } + self.set_progress(n, matched, next_index, is_learner); + info!( + "{} restored progress of {} [{:?}]", + self.tag, + n, + self.prs().get(n) + ); + } + } self.begin_membership_change(change).expect("Expected already valid change to still be valid."); } diff --git a/tests/integration_cases/test_membership_changes.rs b/tests/integration_cases/test_membership_changes.rs index 5c6bed848..3fa378f44 100644 --- a/tests/integration_cases/test_membership_changes.rs +++ b/tests/integration_cases/test_membership_changes.rs @@ -649,23 +649,22 @@ mod three_peers_replace_voter { snapshot }; + // At this point, there is a sentinel at index 3. + info!("Leader power cycles."); assert_eq!(scenario.peers[&1].began_membership_change_at(), Some(2)); scenario.power_cycle(&[1], snapshot); - - assert_eq!(scenario.peers[&1].began_membership_change_at(), Some(2)); - scenario.assert_in_membership_change(&[1]); { let peer = scenario.peers.get_mut(&1).unwrap(); peer.become_candidate(); peer.become_leader(); - for _ in peer.get_heartbeat_elapsed()..=(peer.get_heartbeat_timeout() + 1) { - peer.tick(); - } } + assert_eq!(scenario.peers[&1].began_membership_change_at(), Some(2)); + scenario.assert_in_membership_change(&[1]); + info!("Allowing new peers to catch up."); - scenario.expect_read_and_dispatch_messages_from(&[4, 1, 4, 1, 4, 1, 4, 1])?; + scenario.expect_read_and_dispatch_messages_from(&[1, 4, 1, 4, 1, 4, 1])?; scenario.assert_in_membership_change(&[1, 2, 3, 4]); info!("Cluster leaving the joint."); @@ -1226,10 +1225,11 @@ mod compaction { ); scenario.assert_in_membership_change(&[1, 2, 3, 4]); + info!("Compacting the leaders log"); scenario.peers.get_mut(&1).unwrap().raft_log.store.wl().compact(2)?; info!("Cluster leaving the joint."); - scenario.expect_read_and_dispatch_messages_from(&[3, 2, 1])?; + scenario.expect_read_and_dispatch_messages_from(&[4, 3, 2, 1])?; scenario.assert_can_apply_transition_entry_at_index( &[1, 2, 3, 4], 3, From 815f7d2f3c26d0c81d9c9f736fde048c553e0266 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Wed, 9 Jan 2019 14:48:29 -0800 Subject: [PATCH 23/41] Test is green! --- src/raft.rs | 14 ++++----- src/storage.rs | 3 +- .../test_membership_changes.rs | 31 +++++++++++++++---- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/src/raft.rs b/src/raft.rs index 17163ac76..43e3dde13 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -413,8 +413,8 @@ impl Raft { fn set_pending_membership_change(&mut self, maybe_change: impl Into>) { let maybe_change = maybe_change.into(); if let Some(ref change) = maybe_change { - assert!(self.pending_membership_change.is_none()); let index = change.get_start_index(); + assert!(self.pending_membership_change.is_none() || index == self.pending_conf_index); if index > self.pending_conf_index { self.pending_conf_index = index; } @@ -1216,9 +1216,7 @@ impl Raft { "!ConfChange::has_configuration()".into(), )); }; - let start_index = if conf_change.get_start_index() != 0 { - conf_change.get_start_index() - } else { + let start_index = if conf_change.get_start_index() == 0 { return Err(Error::ViolatesContract( "!ConfChange::has_start_index()".into(), )); @@ -1919,10 +1917,10 @@ impl Raft { } None => { debug!( - "{} [logterm: {}, index: {}] rejected msgApp [logterm: {}, index: {}] \ + "{} [logterm: {:?}, index: {}] rejected msgApp [logterm: {}, index: {}] \ from {}", self.tag, - self.raft_log.term(m.get_index()).unwrap_or(0), + self.raft_log.term(m.get_index()), m.get_index(), m.get_log_term(), m.get_index(), @@ -1954,8 +1952,8 @@ impl Raft { ); if self.restore(m.take_snapshot()) { info!( - "{} [commit: {}] restored snapshot [index: {}, term: {}]", - self.tag, self.raft_log.committed, sindex, sterm + "{} [commit: {}, term: {}] restored snapshot [index: {}, term: {}]", + self.tag, self.term, self.raft_log.committed, sindex, sterm ); let mut to_send = Message::new(); to_send.set_to(m.get_from()); diff --git a/src/storage.rs b/src/storage.rs index ce02438c6..b2d7216d2 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -114,6 +114,7 @@ impl MemStorageCore { /// Overwrites the contents of this Storage object with those of the given snapshot. pub fn apply_snapshot(&mut self, snapshot: Snapshot) -> Result<()> { + error!("I HAVE BEEN CALLED!!!!"); // handle check for old snapshot being applied let index = self.snapshot.get_metadata().get_index(); let snapshot_index = snapshot.get_metadata().get_index(); @@ -276,7 +277,7 @@ impl Storage for MemStorage { } if high > core.inner_last_index() + 1 { - panic!("index out of bound") + panic!("index out of bound (last: {}, high: {}", core.inner_last_index()+1, high); } // only contains dummy entries. if core.entries.len() == 1 { diff --git a/tests/integration_cases/test_membership_changes.rs b/tests/integration_cases/test_membership_changes.rs index 3fa378f44..1db1ca885 100644 --- a/tests/integration_cases/test_membership_changes.rs +++ b/tests/integration_cases/test_membership_changes.rs @@ -636,6 +636,7 @@ mod three_peers_replace_voter { info!("Compacting leader's log"); + // This snapshot has a term 1. let snapshot = { let peer = scenario.peers.get_mut(&1).unwrap(); peer.raft_log.store.wl().create_snapshot( @@ -649,11 +650,11 @@ mod three_peers_replace_voter { snapshot }; - // At this point, there is a sentinel at index 3. + // At this point, there is a sentinel at index 3, term 2. info!("Leader power cycles."); assert_eq!(scenario.peers[&1].began_membership_change_at(), Some(2)); - scenario.power_cycle(&[1], snapshot); + scenario.power_cycle(&[1], snapshot.clone()); { let peer = scenario.peers.get_mut(&1).unwrap(); peer.become_candidate(); @@ -664,18 +665,29 @@ mod three_peers_replace_voter { scenario.assert_in_membership_change(&[1]); info!("Allowing new peers to catch up."); - scenario.expect_read_and_dispatch_messages_from(&[1, 4, 1, 4, 1, 4, 1])?; + scenario.expect_read_and_dispatch_messages_from(&[1, 4, 1])?; // 1, 4, 1, 4, 1])?; scenario.assert_in_membership_change(&[1, 2, 3, 4]); + + { + assert_eq!(3, scenario.peers.get_mut(&4).unwrap() + .raft_log + .unstable + .offset); + let new_peer = scenario.peers.get_mut(&4).unwrap(); + let snap = new_peer.raft_log.snapshot().unwrap(); + new_peer.raft_log.store.wl().apply_snapshot(snap).unwrap(); + new_peer.raft_log.stable_snap_to(snapshot.get_metadata().get_index()); + } info!("Cluster leaving the joint."); - scenario.expect_read_and_dispatch_messages_from(&[4, 3, 2, 1, 4, 3, 2, 1])?; + scenario.expect_read_and_dispatch_messages_from(&[4, 1, 4, 3, 2, 1, 3, 2, 1])?; assert_eq!(scenario.peers[&1].began_membership_change_at(), Some(2)); scenario.assert_can_apply_transition_entry_at_index( &[1, 2, 3, 4], 4, ConfChangeType::FinalizeConfChange, ); - scenario.assert_not_in_membership_change(&[1, 2, 4]); + scenario.assert_not_in_membership_change(&[1, 2, 3, 4]); Ok(()) } @@ -1229,7 +1241,7 @@ mod compaction { scenario.peers.get_mut(&1).unwrap().raft_log.store.wl().compact(2)?; info!("Cluster leaving the joint."); - scenario.expect_read_and_dispatch_messages_from(&[4, 3, 2, 1])?; + scenario.expect_read_and_dispatch_messages_from(&[3, 2, 1])?; scenario.assert_can_apply_transition_entry_at_index( &[1, 2, 3, 4], 3, @@ -1288,6 +1300,7 @@ impl Scenario { id, peers: old_configuration.voters().iter().cloned().collect(), learners: old_configuration.learners().iter().cloned().collect(), + tag: format!("{}", id), ..Default::default() }, MemStorage::new(), @@ -1324,6 +1337,7 @@ impl Scenario { id, peers: vec![self.old_leader, id], learners: vec![], + tag: format!("{}", id), ..Default::default() }, storage.clone(), @@ -1336,6 +1350,7 @@ impl Scenario { id, peers: vec![self.old_leader], learners: vec![id], + tag: format!("{}", id), ..Default::default() }, storage.clone(), @@ -1511,6 +1526,7 @@ impl Scenario { let snapshot = snapshot.into(); for id in peers { debug!("Power cycling {}.", id); + let applied = self.peers[&id].raft_log.applied; let mut peer = self.peers.remove(&id).expect("Peer did not exist."); let store = peer.mut_store().clone(); let prs = peer.prs(); @@ -1518,6 +1534,8 @@ impl Scenario { let mut peer = Raft::new( &Config { id, + tag: format!("{}", id), + applied: applied, ..Default::default() }, store, @@ -1530,6 +1548,7 @@ impl Scenario { id, peers: prs.voter_ids().iter().cloned().collect(), learners: prs.learner_ids().iter().cloned().collect(), + applied: applied, ..Default::default() }, store, From c26eef46b0ccf6c540c7b09f1acd7dee3ebc4f6a Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Fri, 11 Jan 2019 12:52:48 -0800 Subject: [PATCH 24/41] lints --- src/eraftpb.rs | 2 +- src/progress.rs | 2 +- src/raft.rs | 37 ++++++++++------ src/storage.rs | 12 ++++-- src/util.rs | 8 ++-- .../test_membership_changes.rs | 42 +++++++++++++------ 6 files changed, 70 insertions(+), 33 deletions(-) diff --git a/src/eraftpb.rs b/src/eraftpb.rs index beca6cb85..f37540622 100644 --- a/src/eraftpb.rs +++ b/src/eraftpb.rs @@ -3,7 +3,7 @@ // https://github.com/Manishearth/rust-clippy/issues/702 #![allow(unknown_lints)] -#![allow(clippy)] +#![allow(clippy::all)] #![cfg_attr(rustfmt, rustfmt_skip)] diff --git a/src/progress.rs b/src/progress.rs index 360b14702..44bb30d42 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -142,7 +142,7 @@ impl Configuration { /// Returns whether or not the given `id` is a member of this configuration. // Allowed to maintain API consistency. See `HashSet::contains(&u64)`. - #[allow(trivially_copy_pass_by_ref)] + #[allow(clippy::trivially_copy_pass_by_ref)] pub fn contains(&self, id: &u64) -> bool { self.voters.contains(id) || self.learners.contains(id) } diff --git a/src/raft.rs b/src/raft.rs index 43e3dde13..81cacf51c 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -37,7 +37,7 @@ use rand::{self, Rng}; use super::errors::{Error, Result, StorageError}; use super::progress::{CandidacyStatus, Configuration, Progress, ProgressSet, ProgressState}; -use super::raft_log::{self, RaftLog, NO_LIMIT}; +use super::raft_log::{self, RaftLog}; use super::read_only::{ReadOnly, ReadOnlyOption, ReadState}; use super::storage::Storage; use super::Config; @@ -407,7 +407,7 @@ impl Raft { } /// Set when the peer began a joint consensus change. - /// + /// /// This will also set `pending_conf_index` if it is larger than the existing number. #[inline] fn set_pending_membership_change(&mut self, maybe_change: impl Into>) { @@ -425,7 +425,9 @@ impl Raft { /// Get the index which the pending membership change started at. #[inline] pub fn began_membership_change_at(&self) -> Option { - self.pending_membership_change.as_ref().map(|v| v.get_start_index()) + self.pending_membership_change + .as_ref() + .map(|v| v.get_start_index()) } /// send persists state to stable storage and then sends to its mailbox. @@ -652,7 +654,8 @@ impl Raft { self.raft_log.applied_to(applied); // Check to see if we need to finalize a Joint Consensus state now. - let start_index = self.pending_membership_change + let start_index = self + .pending_membership_change .as_ref() .map(|v| Some(v.get_start_index())) .unwrap_or(None); @@ -867,12 +870,13 @@ impl Raft { self.pending_conf_index = self.raft_log.last_index(); self.append_entry(&mut [Entry::new()]); - + // In most cases, we append only a new entry marked with an index and term. // In the specific case of a node recovering while in the middle of a membership change, // and the finalization entry may have been lost, we must also append that, since it // would be overwritten by the term change. - let change_start_index = self.pending_membership_change + let change_start_index = self + .pending_membership_change .as_ref() .map(|v| Some(v.get_start_index())) .unwrap_or(None); @@ -1216,7 +1220,7 @@ impl Raft { "!ConfChange::has_configuration()".into(), )); }; - let start_index = if conf_change.get_start_index() == 0 { + if conf_change.get_start_index() == 0 { return Err(Error::ViolatesContract( "!ConfChange::has_start_index()".into(), )); @@ -2040,11 +2044,19 @@ impl Raft { if meta.has_pending_membership_change() { let change = meta.get_pending_membership_change(); - + let (voters, learners) = { let config = Configuration::from(change.get_configuration().clone()); - let voters = config.voters().difference(self.prs().configuration().voters()).cloned().collect::>(); - let learners = config.learners().difference(self.prs().configuration().learners()).cloned().collect::>(); + let voters = config + .voters() + .difference(self.prs().configuration().voters()) + .cloned() + .collect::>(); + let learners = config + .learners() + .difference(self.prs().configuration().learners()) + .cloned() + .collect::>(); (voters, learners) }; for &(is_learner, ref nodes) in &[(false, voters), (true, learners)] { @@ -2055,7 +2067,7 @@ impl Raft { matched = next_index - 1; self.is_learner = is_learner; } - self.set_progress(n, matched, next_index, is_learner); + self.set_progress(n, matched, next_index, is_learner); info!( "{} restored progress of {} [{:?}]", self.tag, @@ -2064,7 +2076,8 @@ impl Raft { ); } } - self.begin_membership_change(change).expect("Expected already valid change to still be valid."); + self.begin_membership_change(change) + .expect("Expected already valid change to still be valid."); } None diff --git a/src/storage.rs b/src/storage.rs index b2d7216d2..400890a8a 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -33,7 +33,7 @@ use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard}; -use eraftpb::{ConfState, ConfChange, Entry, HardState, Snapshot}; +use eraftpb::{ConfChange, ConfState, Entry, HardState, Snapshot}; use errors::{Error, Result, StorageError}; use util; @@ -161,7 +161,9 @@ impl MemStorageCore { self.snapshot.mut_metadata().set_conf_state(cs) } if let Some(pending_change) = pending_membership_change { - self.snapshot.mut_metadata().set_pending_membership_change(pending_change); + self.snapshot + .mut_metadata() + .set_pending_membership_change(pending_change); } self.snapshot.set_data(data); Ok(&self.snapshot) @@ -277,7 +279,11 @@ impl Storage for MemStorage { } if high > core.inner_last_index() + 1 { - panic!("index out of bound (last: {}, high: {}", core.inner_last_index()+1, high); + panic!( + "index out of bound (last: {}, high: {}", + core.inner_last_index() + 1, + high + ); } // only contains dummy entries. if core.entries.len() == 1 { diff --git a/src/util.rs b/src/util.rs index 1d3f2ec65..9558229c0 100644 --- a/src/util.rs +++ b/src/util.rs @@ -16,8 +16,8 @@ use std::u64; -use protobuf::Message; use eraftpb::ConfState; +use protobuf::Message; /// A number to represent that there is no limit. pub const NO_LIMIT: u64 = u64::MAX; @@ -73,7 +73,9 @@ pub fn limit_size(entries: &mut Vec, max: u64) { // Bring some consistency to things. The protobuf has `nodes` and it's not really a term that's used anymore. impl ConfState { - fn get_voters(&self) -> &[u64] { + /// Get the voters. This is identical to `get_nodes()`. + #[inline] + pub fn get_voters(&self) -> &[u64] { self.get_nodes() } -} \ No newline at end of file +} diff --git a/tests/integration_cases/test_membership_changes.rs b/tests/integration_cases/test_membership_changes.rs index 1db1ca885..fdf96171f 100644 --- a/tests/integration_cases/test_membership_changes.rs +++ b/tests/integration_cases/test_membership_changes.rs @@ -15,7 +15,9 @@ use fxhash::{FxHashMap, FxHashSet}; use protobuf::{self, RepeatedField}; use raft::{ - eraftpb::{ConfChange, Snapshot, ConfChangeType, ConfState, Entry, EntryType, Message, MessageType}, + eraftpb::{ + ConfChange, ConfChangeType, ConfState, Entry, EntryType, Message, MessageType, Snapshot, + }, storage::{MemStorage, Storage}, Config, Configuration, Raft, Result, INVALID_ID, NO_LIMIT, }; @@ -634,7 +636,6 @@ mod three_peers_replace_voter { ); scenario.assert_in_membership_change(&[1, 2, 3]); - info!("Compacting leader's log"); // This snapshot has a term 1. let snapshot = { @@ -643,7 +644,7 @@ mod three_peers_replace_voter { 2, ConfState::from(peer.prs().configuration().clone()).into(), peer.pending_membership_change().clone(), - vec![] + vec![], )?; let snapshot = peer.raft_log.snapshot()?; peer.raft_log.store.wl().compact(2)?; @@ -667,16 +668,18 @@ mod three_peers_replace_voter { info!("Allowing new peers to catch up."); scenario.expect_read_and_dispatch_messages_from(&[1, 4, 1])?; // 1, 4, 1, 4, 1])?; scenario.assert_in_membership_change(&[1, 2, 3, 4]); - + { - assert_eq!(3, scenario.peers.get_mut(&4).unwrap() - .raft_log - .unstable - .offset); + assert_eq!( + 3, + scenario.peers.get_mut(&4).unwrap().raft_log.unstable.offset + ); let new_peer = scenario.peers.get_mut(&4).unwrap(); let snap = new_peer.raft_log.snapshot().unwrap(); new_peer.raft_log.store.wl().apply_snapshot(snap).unwrap(); - new_peer.raft_log.stable_snap_to(snapshot.get_metadata().get_index()); + new_peer + .raft_log + .stable_snap_to(snapshot.get_metadata().get_index()); } info!("Cluster leaving the joint."); @@ -1238,7 +1241,14 @@ mod compaction { scenario.assert_in_membership_change(&[1, 2, 3, 4]); info!("Compacting the leaders log"); - scenario.peers.get_mut(&1).unwrap().raft_log.store.wl().compact(2)?; + scenario + .peers + .get_mut(&1) + .unwrap() + .raft_log + .store + .wl() + .compact(2)?; info!("Cluster leaving the joint."); scenario.expect_read_and_dispatch_messages_from(&[3, 2, 1])?; @@ -1521,7 +1531,11 @@ impl Scenario { /// Simulate a power cycle in the given nodes. /// /// This means that the MemStorage is kept, but nothing else. - fn power_cycle<'a>(&mut self, peers: impl IntoIterator, snapshot: impl Into>) { + fn power_cycle<'a>( + &mut self, + peers: impl IntoIterator, + snapshot: impl Into>, + ) { let peers = peers.into_iter().cloned(); let snapshot = snapshot.into(); for id in peers { @@ -1539,7 +1553,8 @@ impl Scenario { ..Default::default() }, store, - ).expect("Could not create new Raft"); + ) + .expect("Could not create new Raft"); peer.restore(snapshot.clone()); peer } else { @@ -1552,7 +1567,8 @@ impl Scenario { ..Default::default() }, store, - ).expect("Could not create new Raft") + ) + .expect("Could not create new Raft") }; self.peers.insert(id, peer.into()); } From e9721e55d4ccc5afd0221ac5a0432b055d7a36c4 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Fri, 11 Jan 2019 14:33:03 -0800 Subject: [PATCH 25/41] Use appropriate name for proto Signed-off-by: Ana Hobden --- proto/eraftpb.proto | 8 +- src/eraftpb.rs | 423 +++++++++--------- src/raft.rs | 26 +- src/raw_node.rs | 8 +- src/storage.rs | 1 - .../test_membership_changes.rs | 152 +++---- 6 files changed, 304 insertions(+), 314 deletions(-) diff --git a/proto/eraftpb.proto b/proto/eraftpb.proto index ada4472ca..58b5db59e 100644 --- a/proto/eraftpb.proto +++ b/proto/eraftpb.proto @@ -94,8 +94,8 @@ enum ConfChangeType { AddNode = 0; RemoveNode = 1; AddLearnerNode = 2; - BeginConfChange = 3; - FinalizeConfChange = 4; + BeginMembershipChange = 3; + FinalizeMembershipChange = 4; } message ConfChange { @@ -104,9 +104,9 @@ message ConfChange { // Used in `AddNode`, `RemoveNode`, and `AddLearnerNode`. uint64 node_id = 3; bytes context = 4; - // Used in `BeginConfChange` and `FinalizeConfChange`. + // Used in `BeginMembershipChange` and `FinalizeMembershipChange`. ConfState configuration = 5; - // Used in `BeginConfChange` and `FinalizeConfChange`. + // Used in `BeginMembershipChange` and `FinalizeMembershipChange`. // Because `RawNode::apply_conf_change` takes a `ConfChange` instead of an `Entry` we must // include this index so it can be known. uint64 start_index = 6; diff --git a/src/eraftpb.rs b/src/eraftpb.rs index f37540622..5b26ea094 100644 --- a/src/eraftpb.rs +++ b/src/eraftpb.rs @@ -3,7 +3,7 @@ // https://github.com/Manishearth/rust-clippy/issues/702 #![allow(unknown_lints)] -#![allow(clippy::all)] +#![allow(clippy)] #![cfg_attr(rustfmt, rustfmt_skip)] @@ -2475,8 +2475,8 @@ pub enum ConfChangeType { AddNode = 0, RemoveNode = 1, AddLearnerNode = 2, - BeginConfChange = 3, - FinalizeConfChange = 4, + BeginMembershipChange = 3, + FinalizeMembershipChange = 4, } impl ::protobuf::ProtobufEnum for ConfChangeType { @@ -2489,8 +2489,8 @@ impl ::protobuf::ProtobufEnum for ConfChangeType { 0 => ::std::option::Option::Some(ConfChangeType::AddNode), 1 => ::std::option::Option::Some(ConfChangeType::RemoveNode), 2 => ::std::option::Option::Some(ConfChangeType::AddLearnerNode), - 3 => ::std::option::Option::Some(ConfChangeType::BeginConfChange), - 4 => ::std::option::Option::Some(ConfChangeType::FinalizeConfChange), + 3 => ::std::option::Option::Some(ConfChangeType::BeginMembershipChange), + 4 => ::std::option::Option::Some(ConfChangeType::FinalizeMembershipChange), _ => ::std::option::Option::None } } @@ -2500,8 +2500,8 @@ impl ::protobuf::ProtobufEnum for ConfChangeType { ConfChangeType::AddNode, ConfChangeType::RemoveNode, ConfChangeType::AddLearnerNode, - ConfChangeType::BeginConfChange, - ConfChangeType::FinalizeConfChange, + ConfChangeType::BeginMembershipChange, + ConfChangeType::FinalizeMembershipChange, ]; values } @@ -2578,211 +2578,212 @@ static file_descriptor_proto_data: &'static [u8] = b"\ tatus\x10\x0b\x12\x12\n\x0eMsgCheckQuorum\x10\x0c\x12\x15\n\x11MsgTransf\ erLeader\x10\r\x12\x11\n\rMsgTimeoutNow\x10\x0e\x12\x10\n\x0cMsgReadInde\ x\x10\x0f\x12\x14\n\x10MsgReadIndexResp\x10\x10\x12\x15\n\x11MsgRequestP\ - reVote\x10\x11\x12\x1d\n\x19MsgRequestPreVoteResponse\x10\x12*n\n\x0eCon\ + reVote\x10\x11\x12\x1d\n\x19MsgRequestPreVoteResponse\x10\x12*z\n\x0eCon\ fChangeType\x12\x0b\n\x07AddNode\x10\0\x12\x0e\n\nRemoveNode\x10\x01\x12\ - \x12\n\x0eAddLearnerNode\x10\x02\x12\x13\n\x0fBeginConfChange\x10\x03\ - \x12\x16\n\x12FinalizeConfChange\x10\x04J\xc5&\n\x06\x12\x04\0\0p\x01\n\ - \x08\n\x01\x0c\x12\x03\0\0\x12\n\x08\n\x01\x02\x12\x03\x01\x08\x0f\n\n\n\ - \x02\x05\0\x12\x04\x03\0\x06\x01\n\n\n\x03\x05\0\x01\x12\x03\x03\x05\x0e\ - \n\x0b\n\x04\x05\0\x02\0\x12\x03\x04\x04\x14\n\x0c\n\x05\x05\0\x02\0\x01\ - \x12\x03\x04\x04\x0f\n\x0c\n\x05\x05\0\x02\0\x02\x12\x03\x04\x12\x13\n\ - \x0b\n\x04\x05\0\x02\x01\x12\x03\x05\x04\x18\n\x0c\n\x05\x05\0\x02\x01\ - \x01\x12\x03\x05\x04\x13\n\x0c\n\x05\x05\0\x02\x01\x02\x12\x03\x05\x16\ - \x17\n\xdd\x04\n\x02\x04\0\x12\x04\x12\0\x1c\x01\x1a\xd0\x04\x20The\x20e\ - ntry\x20is\x20a\x20type\x20of\x20change\x20that\x20needs\x20to\x20be\x20\ - applied.\x20It\x20contains\x20two\x20data\x20fields.\n\x20While\x20the\ - \x20fields\x20are\x20built\x20into\x20the\x20model;\x20their\x20usage\ - \x20is\x20determined\x20by\x20the\x20entry_type.\n\n\x20For\x20normal\ - \x20entries,\x20the\x20data\x20field\x20should\x20contain\x20the\x20data\ - \x20change\x20that\x20should\x20be\x20applied.\n\x20The\x20context\x20fi\ - eld\x20can\x20be\x20used\x20for\x20any\x20contextual\x20data\x20that\x20\ - might\x20be\x20relevant\x20to\x20the\n\x20application\x20of\x20the\x20da\ - ta.\n\n\x20For\x20configuration\x20changes,\x20the\x20data\x20will\x20co\ - ntain\x20the\x20ConfChange\x20message\x20and\x20the\n\x20context\x20will\ - \x20provide\x20anything\x20needed\x20to\x20assist\x20the\x20configuratio\ - n\x20change.\x20The\x20context\n\x20if\x20for\x20the\x20user\x20to\x20se\ - t\x20and\x20use\x20in\x20this\x20case.\n\n\n\n\x03\x04\0\x01\x12\x03\x12\ - \x08\r\n\x0b\n\x04\x04\0\x02\0\x12\x03\x13\x04\x1d\n\r\n\x05\x04\0\x02\0\ - \x04\x12\x04\x13\x04\x12\x0f\n\x0c\n\x05\x04\0\x02\0\x06\x12\x03\x13\x04\ - \r\n\x0c\n\x05\x04\0\x02\0\x01\x12\x03\x13\x0e\x18\n\x0c\n\x05\x04\0\x02\ - \0\x03\x12\x03\x13\x1b\x1c\n\x0b\n\x04\x04\0\x02\x01\x12\x03\x14\x04\x14\ - \n\r\n\x05\x04\0\x02\x01\x04\x12\x04\x14\x04\x13\x1d\n\x0c\n\x05\x04\0\ - \x02\x01\x05\x12\x03\x14\x04\n\n\x0c\n\x05\x04\0\x02\x01\x01\x12\x03\x14\ - \x0b\x0f\n\x0c\n\x05\x04\0\x02\x01\x03\x12\x03\x14\x12\x13\n\x0b\n\x04\ - \x04\0\x02\x02\x12\x03\x15\x04\x15\n\r\n\x05\x04\0\x02\x02\x04\x12\x04\ - \x15\x04\x14\x14\n\x0c\n\x05\x04\0\x02\x02\x05\x12\x03\x15\x04\n\n\x0c\n\ - \x05\x04\0\x02\x02\x01\x12\x03\x15\x0b\x10\n\x0c\n\x05\x04\0\x02\x02\x03\ - \x12\x03\x15\x13\x14\n\x0b\n\x04\x04\0\x02\x03\x12\x03\x16\x04\x13\n\r\n\ - \x05\x04\0\x02\x03\x04\x12\x04\x16\x04\x15\x15\n\x0c\n\x05\x04\0\x02\x03\ - \x05\x12\x03\x16\x04\t\n\x0c\n\x05\x04\0\x02\x03\x01\x12\x03\x16\n\x0e\n\ - \x0c\n\x05\x04\0\x02\x03\x03\x12\x03\x16\x11\x12\n\x0b\n\x04\x04\0\x02\ - \x04\x12\x03\x17\x04\x16\n\r\n\x05\x04\0\x02\x04\x04\x12\x04\x17\x04\x16\ - \x13\n\x0c\n\x05\x04\0\x02\x04\x05\x12\x03\x17\x04\t\n\x0c\n\x05\x04\0\ - \x02\x04\x01\x12\x03\x17\n\x11\n\x0c\n\x05\x04\0\x02\x04\x03\x12\x03\x17\ - \x14\x15\nm\n\x04\x04\0\x02\x05\x12\x03\x1b\x04\x16\x1a`\x20Deprecated!\ - \x20It\x20is\x20kept\x20for\x20backward\x20compatibility.\n\x20TODO:\x20\ - remove\x20it\x20in\x20the\x20next\x20major\x20release.\n\n\r\n\x05\x04\0\ - \x02\x05\x04\x12\x04\x1b\x04\x17\x16\n\x0c\n\x05\x04\0\x02\x05\x05\x12\ - \x03\x1b\x04\x08\n\x0c\n\x05\x04\0\x02\x05\x01\x12\x03\x1b\t\x11\n\x0c\n\ - \x05\x04\0\x02\x05\x03\x12\x03\x1b\x14\x15\n\n\n\x02\x04\x01\x12\x04\x1e\ - \0#\x01\n\n\n\x03\x04\x01\x01\x12\x03\x1e\x08\x18\n\x0b\n\x04\x04\x01\ - \x02\0\x12\x03\x1f\x04\x1d\n\r\n\x05\x04\x01\x02\0\x04\x12\x04\x1f\x04\ - \x1e\x1a\n\x0c\n\x05\x04\x01\x02\0\x06\x12\x03\x1f\x04\r\n\x0c\n\x05\x04\ - \x01\x02\0\x01\x12\x03\x1f\x0e\x18\n\x0c\n\x05\x04\x01\x02\0\x03\x12\x03\ - \x1f\x1b\x1c\n\x0b\n\x04\x04\x01\x02\x01\x12\x03\x20\x04-\n\r\n\x05\x04\ - \x01\x02\x01\x04\x12\x04\x20\x04\x1f\x1d\n\x0c\n\x05\x04\x01\x02\x01\x06\ - \x12\x03\x20\x04\x0e\n\x0c\n\x05\x04\x01\x02\x01\x01\x12\x03\x20\x0f(\n\ - \x0c\n\x05\x04\x01\x02\x01\x03\x12\x03\x20+,\n\x0b\n\x04\x04\x01\x02\x02\ - \x12\x03!\x04\x15\n\r\n\x05\x04\x01\x02\x02\x04\x12\x04!\x04\x20-\n\x0c\ - \n\x05\x04\x01\x02\x02\x05\x12\x03!\x04\n\n\x0c\n\x05\x04\x01\x02\x02\ - \x01\x12\x03!\x0b\x10\n\x0c\n\x05\x04\x01\x02\x02\x03\x12\x03!\x13\x14\n\ - \x0b\n\x04\x04\x01\x02\x03\x12\x03\"\x04\x14\n\r\n\x05\x04\x01\x02\x03\ - \x04\x12\x04\"\x04!\x15\n\x0c\n\x05\x04\x01\x02\x03\x05\x12\x03\"\x04\n\ - \n\x0c\n\x05\x04\x01\x02\x03\x01\x12\x03\"\x0b\x0f\n\x0c\n\x05\x04\x01\ - \x02\x03\x03\x12\x03\"\x12\x13\n\n\n\x02\x04\x02\x12\x04%\0(\x01\n\n\n\ - \x03\x04\x02\x01\x12\x03%\x08\x10\n\x0b\n\x04\x04\x02\x02\0\x12\x03&\x04\ - \x13\n\r\n\x05\x04\x02\x02\0\x04\x12\x04&\x04%\x12\n\x0c\n\x05\x04\x02\ - \x02\0\x05\x12\x03&\x04\t\n\x0c\n\x05\x04\x02\x02\0\x01\x12\x03&\n\x0e\n\ - \x0c\n\x05\x04\x02\x02\0\x03\x12\x03&\x11\x12\n\x0b\n\x04\x04\x02\x02\ - \x01\x12\x03'\x04\"\n\r\n\x05\x04\x02\x02\x01\x04\x12\x04'\x04&\x13\n\ - \x0c\n\x05\x04\x02\x02\x01\x06\x12\x03'\x04\x14\n\x0c\n\x05\x04\x02\x02\ - \x01\x01\x12\x03'\x15\x1d\n\x0c\n\x05\x04\x02\x02\x01\x03\x12\x03'\x20!\ - \n\n\n\x02\x05\x01\x12\x04*\0>\x01\n\n\n\x03\x05\x01\x01\x12\x03*\x05\ - \x10\n\x0b\n\x04\x05\x01\x02\0\x12\x03+\x04\x0f\n\x0c\n\x05\x05\x01\x02\ - \0\x01\x12\x03+\x04\n\n\x0c\n\x05\x05\x01\x02\0\x02\x12\x03+\r\x0e\n\x0b\ - \n\x04\x05\x01\x02\x01\x12\x03,\x04\x10\n\x0c\n\x05\x05\x01\x02\x01\x01\ - \x12\x03,\x04\x0b\n\x0c\n\x05\x05\x01\x02\x01\x02\x12\x03,\x0e\x0f\n\x0b\ - \n\x04\x05\x01\x02\x02\x12\x03-\x04\x13\n\x0c\n\x05\x05\x01\x02\x02\x01\ - \x12\x03-\x04\x0e\n\x0c\n\x05\x05\x01\x02\x02\x02\x12\x03-\x11\x12\n\x0b\ - \n\x04\x05\x01\x02\x03\x12\x03.\x04\x12\n\x0c\n\x05\x05\x01\x02\x03\x01\ - \x12\x03.\x04\r\n\x0c\n\x05\x05\x01\x02\x03\x02\x12\x03.\x10\x11\n\x0b\n\ - \x04\x05\x01\x02\x04\x12\x03/\x04\x1a\n\x0c\n\x05\x05\x01\x02\x04\x01\ - \x12\x03/\x04\x15\n\x0c\n\x05\x05\x01\x02\x04\x02\x12\x03/\x18\x19\n\x0b\ - \n\x04\x05\x01\x02\x05\x12\x030\x04\x17\n\x0c\n\x05\x05\x01\x02\x05\x01\ - \x12\x030\x04\x12\n\x0c\n\x05\x05\x01\x02\x05\x02\x12\x030\x15\x16\n\x0b\ - \n\x04\x05\x01\x02\x06\x12\x031\x04\x1f\n\x0c\n\x05\x05\x01\x02\x06\x01\ - \x12\x031\x04\x1a\n\x0c\n\x05\x05\x01\x02\x06\x02\x12\x031\x1d\x1e\n\x0b\ - \n\x04\x05\x01\x02\x07\x12\x032\x04\x14\n\x0c\n\x05\x05\x01\x02\x07\x01\ - \x12\x032\x04\x0f\n\x0c\n\x05\x05\x01\x02\x07\x02\x12\x032\x12\x13\n\x0b\ - \n\x04\x05\x01\x02\x08\x12\x033\x04\x15\n\x0c\n\x05\x05\x01\x02\x08\x01\ - \x12\x033\x04\x10\n\x0c\n\x05\x05\x01\x02\x08\x02\x12\x033\x13\x14\n\x0b\ - \n\x04\x05\x01\x02\t\x12\x034\x04\x1d\n\x0c\n\x05\x05\x01\x02\t\x01\x12\ - \x034\x04\x18\n\x0c\n\x05\x05\x01\x02\t\x02\x12\x034\x1b\x1c\n\x0b\n\x04\ - \x05\x01\x02\n\x12\x035\x04\x18\n\x0c\n\x05\x05\x01\x02\n\x01\x12\x035\ - \x04\x12\n\x0c\n\x05\x05\x01\x02\n\x02\x12\x035\x15\x17\n\x0b\n\x04\x05\ - \x01\x02\x0b\x12\x036\x04\x17\n\x0c\n\x05\x05\x01\x02\x0b\x01\x12\x036\ - \x04\x11\n\x0c\n\x05\x05\x01\x02\x0b\x02\x12\x036\x14\x16\n\x0b\n\x04\ - \x05\x01\x02\x0c\x12\x037\x04\x18\n\x0c\n\x05\x05\x01\x02\x0c\x01\x12\ - \x037\x04\x12\n\x0c\n\x05\x05\x01\x02\x0c\x02\x12\x037\x15\x17\n\x0b\n\ - \x04\x05\x01\x02\r\x12\x038\x04\x1b\n\x0c\n\x05\x05\x01\x02\r\x01\x12\ - \x038\x04\x15\n\x0c\n\x05\x05\x01\x02\r\x02\x12\x038\x18\x1a\n\x0b\n\x04\ - \x05\x01\x02\x0e\x12\x039\x04\x17\n\x0c\n\x05\x05\x01\x02\x0e\x01\x12\ - \x039\x04\x11\n\x0c\n\x05\x05\x01\x02\x0e\x02\x12\x039\x14\x16\n\x0b\n\ - \x04\x05\x01\x02\x0f\x12\x03:\x04\x16\n\x0c\n\x05\x05\x01\x02\x0f\x01\ - \x12\x03:\x04\x10\n\x0c\n\x05\x05\x01\x02\x0f\x02\x12\x03:\x13\x15\n\x0b\ - \n\x04\x05\x01\x02\x10\x12\x03;\x04\x1a\n\x0c\n\x05\x05\x01\x02\x10\x01\ - \x12\x03;\x04\x14\n\x0c\n\x05\x05\x01\x02\x10\x02\x12\x03;\x17\x19\n\x0b\ - \n\x04\x05\x01\x02\x11\x12\x03<\x04\x1b\n\x0c\n\x05\x05\x01\x02\x11\x01\ - \x12\x03<\x04\x15\n\x0c\n\x05\x05\x01\x02\x11\x02\x12\x03<\x18\x1a\n\x0b\ - \n\x04\x05\x01\x02\x12\x12\x03=\x04#\n\x0c\n\x05\x05\x01\x02\x12\x01\x12\ - \x03=\x04\x1d\n\x0c\n\x05\x05\x01\x02\x12\x02\x12\x03=\x20\"\n\n\n\x02\ - \x04\x03\x12\x04@\0M\x01\n\n\n\x03\x04\x03\x01\x12\x03@\x08\x0f\n\x0b\n\ - \x04\x04\x03\x02\0\x12\x03A\x04\x1d\n\r\n\x05\x04\x03\x02\0\x04\x12\x04A\ - \x04@\x11\n\x0c\n\x05\x04\x03\x02\0\x06\x12\x03A\x04\x0f\n\x0c\n\x05\x04\ - \x03\x02\0\x01\x12\x03A\x10\x18\n\x0c\n\x05\x04\x03\x02\0\x03\x12\x03A\ - \x1b\x1c\n\x0b\n\x04\x04\x03\x02\x01\x12\x03B\x04\x12\n\r\n\x05\x04\x03\ - \x02\x01\x04\x12\x04B\x04A\x1d\n\x0c\n\x05\x04\x03\x02\x01\x05\x12\x03B\ - \x04\n\n\x0c\n\x05\x04\x03\x02\x01\x01\x12\x03B\x0b\r\n\x0c\n\x05\x04\ - \x03\x02\x01\x03\x12\x03B\x10\x11\n\x0b\n\x04\x04\x03\x02\x02\x12\x03C\ - \x04\x14\n\r\n\x05\x04\x03\x02\x02\x04\x12\x04C\x04B\x12\n\x0c\n\x05\x04\ - \x03\x02\x02\x05\x12\x03C\x04\n\n\x0c\n\x05\x04\x03\x02\x02\x01\x12\x03C\ - \x0b\x0f\n\x0c\n\x05\x04\x03\x02\x02\x03\x12\x03C\x12\x13\n\x0b\n\x04\ - \x04\x03\x02\x03\x12\x03D\x04\x14\n\r\n\x05\x04\x03\x02\x03\x04\x12\x04D\ - \x04C\x14\n\x0c\n\x05\x04\x03\x02\x03\x05\x12\x03D\x04\n\n\x0c\n\x05\x04\ - \x03\x02\x03\x01\x12\x03D\x0b\x0f\n\x0c\n\x05\x04\x03\x02\x03\x03\x12\ - \x03D\x12\x13\n\x0b\n\x04\x04\x03\x02\x04\x12\x03E\x04\x18\n\r\n\x05\x04\ - \x03\x02\x04\x04\x12\x04E\x04D\x14\n\x0c\n\x05\x04\x03\x02\x04\x05\x12\ - \x03E\x04\n\n\x0c\n\x05\x04\x03\x02\x04\x01\x12\x03E\x0b\x13\n\x0c\n\x05\ - \x04\x03\x02\x04\x03\x12\x03E\x16\x17\n\x0b\n\x04\x04\x03\x02\x05\x12\ - \x03F\x04\x15\n\r\n\x05\x04\x03\x02\x05\x04\x12\x04F\x04E\x18\n\x0c\n\ - \x05\x04\x03\x02\x05\x05\x12\x03F\x04\n\n\x0c\n\x05\x04\x03\x02\x05\x01\ - \x12\x03F\x0b\x10\n\x0c\n\x05\x04\x03\x02\x05\x03\x12\x03F\x13\x14\n\x0b\ - \n\x04\x04\x03\x02\x06\x12\x03G\x04\x1f\n\x0c\n\x05\x04\x03\x02\x06\x04\ - \x12\x03G\x04\x0c\n\x0c\n\x05\x04\x03\x02\x06\x06\x12\x03G\r\x12\n\x0c\n\ - \x05\x04\x03\x02\x06\x01\x12\x03G\x13\x1a\n\x0c\n\x05\x04\x03\x02\x06\ - \x03\x12\x03G\x1d\x1e\n\x0b\n\x04\x04\x03\x02\x07\x12\x03H\x04\x16\n\r\n\ - \x05\x04\x03\x02\x07\x04\x12\x04H\x04G\x1f\n\x0c\n\x05\x04\x03\x02\x07\ - \x05\x12\x03H\x04\n\n\x0c\n\x05\x04\x03\x02\x07\x01\x12\x03H\x0b\x11\n\ - \x0c\n\x05\x04\x03\x02\x07\x03\x12\x03H\x14\x15\n\x0b\n\x04\x04\x03\x02\ - \x08\x12\x03I\x04\x1a\n\r\n\x05\x04\x03\x02\x08\x04\x12\x04I\x04H\x16\n\ - \x0c\n\x05\x04\x03\x02\x08\x06\x12\x03I\x04\x0c\n\x0c\n\x05\x04\x03\x02\ - \x08\x01\x12\x03I\r\x15\n\x0c\n\x05\x04\x03\x02\x08\x03\x12\x03I\x18\x19\ - \n\x0b\n\x04\x04\x03\x02\t\x12\x03J\x04\x15\n\r\n\x05\x04\x03\x02\t\x04\ - \x12\x04J\x04I\x1a\n\x0c\n\x05\x04\x03\x02\t\x05\x12\x03J\x04\x08\n\x0c\ - \n\x05\x04\x03\x02\t\x01\x12\x03J\t\x0f\n\x0c\n\x05\x04\x03\x02\t\x03\ - \x12\x03J\x12\x14\n\x0b\n\x04\x04\x03\x02\n\x12\x03K\x04\x1c\n\r\n\x05\ - \x04\x03\x02\n\x04\x12\x04K\x04J\x15\n\x0c\n\x05\x04\x03\x02\n\x05\x12\ - \x03K\x04\n\n\x0c\n\x05\x04\x03\x02\n\x01\x12\x03K\x0b\x16\n\x0c\n\x05\ - \x04\x03\x02\n\x03\x12\x03K\x19\x1b\n\x0b\n\x04\x04\x03\x02\x0b\x12\x03L\ - \x04\x17\n\r\n\x05\x04\x03\x02\x0b\x04\x12\x04L\x04K\x1c\n\x0c\n\x05\x04\ - \x03\x02\x0b\x05\x12\x03L\x04\t\n\x0c\n\x05\x04\x03\x02\x0b\x01\x12\x03L\ - \n\x11\n\x0c\n\x05\x04\x03\x02\x0b\x03\x12\x03L\x14\x16\n\n\n\x02\x04\ - \x04\x12\x04O\0U\x01\n\n\n\x03\x04\x04\x01\x12\x03O\x08\x11\n\x0b\n\x04\ - \x04\x04\x02\0\x12\x03P\x04\x14\n\r\n\x05\x04\x04\x02\0\x04\x12\x04P\x04\ - O\x13\n\x0c\n\x05\x04\x04\x02\0\x05\x12\x03P\x04\n\n\x0c\n\x05\x04\x04\ - \x02\0\x01\x12\x03P\x0b\x0f\n\x0c\n\x05\x04\x04\x02\0\x03\x12\x03P\x12\ - \x13\n\x0b\n\x04\x04\x04\x02\x01\x12\x03Q\x04\x14\n\r\n\x05\x04\x04\x02\ - \x01\x04\x12\x04Q\x04P\x14\n\x0c\n\x05\x04\x04\x02\x01\x05\x12\x03Q\x04\ - \n\n\x0c\n\x05\x04\x04\x02\x01\x01\x12\x03Q\x0b\x0f\n\x0c\n\x05\x04\x04\ - \x02\x01\x03\x12\x03Q\x12\x13\n\x0b\n\x04\x04\x04\x02\x02\x12\x03R\x04\ - \x16\n\r\n\x05\x04\x04\x02\x02\x04\x12\x04R\x04Q\x14\n\x0c\n\x05\x04\x04\ - \x02\x02\x05\x12\x03R\x04\n\n\x0c\n\x05\x04\x04\x02\x02\x01\x12\x03R\x0b\ - \x11\n\x0c\n\x05\x04\x04\x02\x02\x03\x12\x03R\x14\x15\nd\n\x04\x04\x04\ - \x02\x03\x12\x03T\x04-\x1aW\x20If\x20the\x20peer\x20goes\x20down\x20we\ - \x20need\x20to\x20be\x20aware\x20of\x20a\x20partially\x20completed\x20me\ - mbership\x20change.\n\n\r\n\x05\x04\x04\x02\x03\x04\x12\x04T\x04R\x16\n\ - \x0c\n\x05\x04\x04\x02\x03\x06\x12\x03T\x04\x0e\n\x0c\n\x05\x04\x04\x02\ - \x03\x01\x12\x03T\x0f(\n\x0c\n\x05\x04\x04\x02\x03\x03\x12\x03T+,\n\n\n\ - \x02\x04\x05\x12\x04W\0Z\x01\n\n\n\x03\x04\x05\x01\x12\x03W\x08\x11\n\ - \x0b\n\x04\x04\x05\x02\0\x12\x03X\x04\x1e\n\x0c\n\x05\x04\x05\x02\0\x04\ - \x12\x03X\x04\x0c\n\x0c\n\x05\x04\x05\x02\0\x05\x12\x03X\r\x13\n\x0c\n\ - \x05\x04\x05\x02\0\x01\x12\x03X\x14\x19\n\x0c\n\x05\x04\x05\x02\0\x03\ - \x12\x03X\x1c\x1d\n\x0b\n\x04\x04\x05\x02\x01\x12\x03Y\x04!\n\x0c\n\x05\ - \x04\x05\x02\x01\x04\x12\x03Y\x04\x0c\n\x0c\n\x05\x04\x05\x02\x01\x05\ - \x12\x03Y\r\x13\n\x0c\n\x05\x04\x05\x02\x01\x01\x12\x03Y\x14\x1c\n\x0c\n\ - \x05\x04\x05\x02\x01\x03\x12\x03Y\x1f\x20\n\n\n\x02\x05\x02\x12\x04\\\0b\ - \x01\n\n\n\x03\x05\x02\x01\x12\x03\\\x05\x13\n\x0b\n\x04\x05\x02\x02\0\ - \x12\x03]\x04\x13\n\x0c\n\x05\x05\x02\x02\0\x01\x12\x03]\x04\x0b\n\x0c\n\ - \x05\x05\x02\x02\0\x02\x12\x03]\x11\x12\n\x0b\n\x04\x05\x02\x02\x01\x12\ - \x03^\x04\x13\n\x0c\n\x05\x05\x02\x02\x01\x01\x12\x03^\x04\x0e\n\x0c\n\ - \x05\x05\x02\x02\x01\x02\x12\x03^\x11\x12\n\x0b\n\x04\x05\x02\x02\x02\ - \x12\x03_\x04\x17\n\x0c\n\x05\x05\x02\x02\x02\x01\x12\x03_\x04\x12\n\x0c\ - \n\x05\x05\x02\x02\x02\x02\x12\x03_\x15\x16\n\x0b\n\x04\x05\x02\x02\x03\ - \x12\x03`\x04\x18\n\x0c\n\x05\x05\x02\x02\x03\x01\x12\x03`\x04\x13\n\x0c\ - \n\x05\x05\x02\x02\x03\x02\x12\x03`\x16\x17\n\x0b\n\x04\x05\x02\x02\x04\ - \x12\x03a\x04\x1b\n\x0c\n\x05\x05\x02\x02\x04\x01\x12\x03a\x04\x16\n\x0c\ - \n\x05\x05\x02\x02\x04\x02\x12\x03a\x19\x1a\n\n\n\x02\x04\x06\x12\x04d\0\ - p\x01\n\n\n\x03\x04\x06\x01\x12\x03d\x08\x12\n\x0b\n\x04\x04\x06\x02\0\ - \x12\x03e\x04\x12\n\r\n\x05\x04\x06\x02\0\x04\x12\x04e\x04d\x14\n\x0c\n\ - \x05\x04\x06\x02\0\x05\x12\x03e\x04\n\n\x0c\n\x05\x04\x06\x02\0\x01\x12\ - \x03e\x0b\r\n\x0c\n\x05\x04\x06\x02\0\x03\x12\x03e\x10\x11\n\x0b\n\x04\ - \x04\x06\x02\x01\x12\x03f\x04#\n\r\n\x05\x04\x06\x02\x01\x04\x12\x04f\ - \x04e\x12\n\x0c\n\x05\x04\x06\x02\x01\x06\x12\x03f\x04\x12\n\x0c\n\x05\ - \x04\x06\x02\x01\x01\x12\x03f\x13\x1e\n\x0c\n\x05\x04\x06\x02\x01\x03\ - \x12\x03f!\"\nE\n\x04\x04\x06\x02\x02\x12\x03h\x04\x17\x1a8\x20Used\x20i\ - n\x20`AddNode`,\x20`RemoveNode`,\x20and\x20`AddLearnerNode`.\n\n\r\n\x05\ - \x04\x06\x02\x02\x04\x12\x04h\x04f#\n\x0c\n\x05\x04\x06\x02\x02\x05\x12\ - \x03h\x04\n\n\x0c\n\x05\x04\x06\x02\x02\x01\x12\x03h\x0b\x12\n\x0c\n\x05\ - \x04\x06\x02\x02\x03\x12\x03h\x15\x16\n\x0b\n\x04\x04\x06\x02\x03\x12\ - \x03i\x04\x16\n\r\n\x05\x04\x06\x02\x03\x04\x12\x04i\x04h\x17\n\x0c\n\ - \x05\x04\x06\x02\x03\x05\x12\x03i\x04\t\n\x0c\n\x05\x04\x06\x02\x03\x01\ - \x12\x03i\n\x11\n\x0c\n\x05\x04\x06\x02\x03\x03\x12\x03i\x14\x15\nB\n\ - \x04\x04\x06\x02\x04\x12\x03k\x04\x20\x1a5\x20Used\x20in\x20`BeginConfCh\ - ange`\x20and\x20`FinalizeConfChange`.\n\n\r\n\x05\x04\x06\x02\x04\x04\ - \x12\x04k\x04i\x16\n\x0c\n\x05\x04\x06\x02\x04\x06\x12\x03k\x04\r\n\x0c\ - \n\x05\x04\x06\x02\x04\x01\x12\x03k\x0e\x1b\n\x0c\n\x05\x04\x06\x02\x04\ - \x03\x12\x03k\x1e\x1f\n\xc4\x01\n\x04\x04\x06\x02\x05\x12\x03o\x04\x1b\ - \x1a\xb6\x01\x20Used\x20in\x20`BeginConfChange`\x20and\x20`FinalizeConfC\ + \x12\n\x0eAddLearnerNode\x10\x02\x12\x19\n\x15BeginMembershipChange\x10\ + \x03\x12\x1c\n\x18FinalizeMembershipChange\x10\x04J\xdd&\n\x06\x12\x04\0\ + \0p\x01\n\x08\n\x01\x0c\x12\x03\0\0\x12\n\x08\n\x01\x02\x12\x03\x01\x08\ + \x0f\n\n\n\x02\x05\0\x12\x04\x03\0\x06\x01\n\n\n\x03\x05\0\x01\x12\x03\ + \x03\x05\x0e\n\x0b\n\x04\x05\0\x02\0\x12\x03\x04\x04\x14\n\x0c\n\x05\x05\ + \0\x02\0\x01\x12\x03\x04\x04\x0f\n\x0c\n\x05\x05\0\x02\0\x02\x12\x03\x04\ + \x12\x13\n\x0b\n\x04\x05\0\x02\x01\x12\x03\x05\x04\x18\n\x0c\n\x05\x05\0\ + \x02\x01\x01\x12\x03\x05\x04\x13\n\x0c\n\x05\x05\0\x02\x01\x02\x12\x03\ + \x05\x16\x17\n\xdd\x04\n\x02\x04\0\x12\x04\x12\0\x1c\x01\x1a\xd0\x04\x20\ + The\x20entry\x20is\x20a\x20type\x20of\x20change\x20that\x20needs\x20to\ + \x20be\x20applied.\x20It\x20contains\x20two\x20data\x20fields.\n\x20Whil\ + e\x20the\x20fields\x20are\x20built\x20into\x20the\x20model;\x20their\x20\ + usage\x20is\x20determined\x20by\x20the\x20entry_type.\n\n\x20For\x20norm\ + al\x20entries,\x20the\x20data\x20field\x20should\x20contain\x20the\x20da\ + ta\x20change\x20that\x20should\x20be\x20applied.\n\x20The\x20context\x20\ + field\x20can\x20be\x20used\x20for\x20any\x20contextual\x20data\x20that\ + \x20might\x20be\x20relevant\x20to\x20the\n\x20application\x20of\x20the\ + \x20data.\n\n\x20For\x20configuration\x20changes,\x20the\x20data\x20will\ + \x20contain\x20the\x20ConfChange\x20message\x20and\x20the\n\x20context\ + \x20will\x20provide\x20anything\x20needed\x20to\x20assist\x20the\x20conf\ + iguration\x20change.\x20The\x20context\n\x20if\x20for\x20the\x20user\x20\ + to\x20set\x20and\x20use\x20in\x20this\x20case.\n\n\n\n\x03\x04\0\x01\x12\ + \x03\x12\x08\r\n\x0b\n\x04\x04\0\x02\0\x12\x03\x13\x04\x1d\n\r\n\x05\x04\ + \0\x02\0\x04\x12\x04\x13\x04\x12\x0f\n\x0c\n\x05\x04\0\x02\0\x06\x12\x03\ + \x13\x04\r\n\x0c\n\x05\x04\0\x02\0\x01\x12\x03\x13\x0e\x18\n\x0c\n\x05\ + \x04\0\x02\0\x03\x12\x03\x13\x1b\x1c\n\x0b\n\x04\x04\0\x02\x01\x12\x03\ + \x14\x04\x14\n\r\n\x05\x04\0\x02\x01\x04\x12\x04\x14\x04\x13\x1d\n\x0c\n\ + \x05\x04\0\x02\x01\x05\x12\x03\x14\x04\n\n\x0c\n\x05\x04\0\x02\x01\x01\ + \x12\x03\x14\x0b\x0f\n\x0c\n\x05\x04\0\x02\x01\x03\x12\x03\x14\x12\x13\n\ + \x0b\n\x04\x04\0\x02\x02\x12\x03\x15\x04\x15\n\r\n\x05\x04\0\x02\x02\x04\ + \x12\x04\x15\x04\x14\x14\n\x0c\n\x05\x04\0\x02\x02\x05\x12\x03\x15\x04\n\ + \n\x0c\n\x05\x04\0\x02\x02\x01\x12\x03\x15\x0b\x10\n\x0c\n\x05\x04\0\x02\ + \x02\x03\x12\x03\x15\x13\x14\n\x0b\n\x04\x04\0\x02\x03\x12\x03\x16\x04\ + \x13\n\r\n\x05\x04\0\x02\x03\x04\x12\x04\x16\x04\x15\x15\n\x0c\n\x05\x04\ + \0\x02\x03\x05\x12\x03\x16\x04\t\n\x0c\n\x05\x04\0\x02\x03\x01\x12\x03\ + \x16\n\x0e\n\x0c\n\x05\x04\0\x02\x03\x03\x12\x03\x16\x11\x12\n\x0b\n\x04\ + \x04\0\x02\x04\x12\x03\x17\x04\x16\n\r\n\x05\x04\0\x02\x04\x04\x12\x04\ + \x17\x04\x16\x13\n\x0c\n\x05\x04\0\x02\x04\x05\x12\x03\x17\x04\t\n\x0c\n\ + \x05\x04\0\x02\x04\x01\x12\x03\x17\n\x11\n\x0c\n\x05\x04\0\x02\x04\x03\ + \x12\x03\x17\x14\x15\nm\n\x04\x04\0\x02\x05\x12\x03\x1b\x04\x16\x1a`\x20\ + Deprecated!\x20It\x20is\x20kept\x20for\x20backward\x20compatibility.\n\ + \x20TODO:\x20remove\x20it\x20in\x20the\x20next\x20major\x20release.\n\n\ + \r\n\x05\x04\0\x02\x05\x04\x12\x04\x1b\x04\x17\x16\n\x0c\n\x05\x04\0\x02\ + \x05\x05\x12\x03\x1b\x04\x08\n\x0c\n\x05\x04\0\x02\x05\x01\x12\x03\x1b\t\ + \x11\n\x0c\n\x05\x04\0\x02\x05\x03\x12\x03\x1b\x14\x15\n\n\n\x02\x04\x01\ + \x12\x04\x1e\0#\x01\n\n\n\x03\x04\x01\x01\x12\x03\x1e\x08\x18\n\x0b\n\ + \x04\x04\x01\x02\0\x12\x03\x1f\x04\x1d\n\r\n\x05\x04\x01\x02\0\x04\x12\ + \x04\x1f\x04\x1e\x1a\n\x0c\n\x05\x04\x01\x02\0\x06\x12\x03\x1f\x04\r\n\ + \x0c\n\x05\x04\x01\x02\0\x01\x12\x03\x1f\x0e\x18\n\x0c\n\x05\x04\x01\x02\ + \0\x03\x12\x03\x1f\x1b\x1c\n\x0b\n\x04\x04\x01\x02\x01\x12\x03\x20\x04-\ + \n\r\n\x05\x04\x01\x02\x01\x04\x12\x04\x20\x04\x1f\x1d\n\x0c\n\x05\x04\ + \x01\x02\x01\x06\x12\x03\x20\x04\x0e\n\x0c\n\x05\x04\x01\x02\x01\x01\x12\ + \x03\x20\x0f(\n\x0c\n\x05\x04\x01\x02\x01\x03\x12\x03\x20+,\n\x0b\n\x04\ + \x04\x01\x02\x02\x12\x03!\x04\x15\n\r\n\x05\x04\x01\x02\x02\x04\x12\x04!\ + \x04\x20-\n\x0c\n\x05\x04\x01\x02\x02\x05\x12\x03!\x04\n\n\x0c\n\x05\x04\ + \x01\x02\x02\x01\x12\x03!\x0b\x10\n\x0c\n\x05\x04\x01\x02\x02\x03\x12\ + \x03!\x13\x14\n\x0b\n\x04\x04\x01\x02\x03\x12\x03\"\x04\x14\n\r\n\x05\ + \x04\x01\x02\x03\x04\x12\x04\"\x04!\x15\n\x0c\n\x05\x04\x01\x02\x03\x05\ + \x12\x03\"\x04\n\n\x0c\n\x05\x04\x01\x02\x03\x01\x12\x03\"\x0b\x0f\n\x0c\ + \n\x05\x04\x01\x02\x03\x03\x12\x03\"\x12\x13\n\n\n\x02\x04\x02\x12\x04%\ + \0(\x01\n\n\n\x03\x04\x02\x01\x12\x03%\x08\x10\n\x0b\n\x04\x04\x02\x02\0\ + \x12\x03&\x04\x13\n\r\n\x05\x04\x02\x02\0\x04\x12\x04&\x04%\x12\n\x0c\n\ + \x05\x04\x02\x02\0\x05\x12\x03&\x04\t\n\x0c\n\x05\x04\x02\x02\0\x01\x12\ + \x03&\n\x0e\n\x0c\n\x05\x04\x02\x02\0\x03\x12\x03&\x11\x12\n\x0b\n\x04\ + \x04\x02\x02\x01\x12\x03'\x04\"\n\r\n\x05\x04\x02\x02\x01\x04\x12\x04'\ + \x04&\x13\n\x0c\n\x05\x04\x02\x02\x01\x06\x12\x03'\x04\x14\n\x0c\n\x05\ + \x04\x02\x02\x01\x01\x12\x03'\x15\x1d\n\x0c\n\x05\x04\x02\x02\x01\x03\ + \x12\x03'\x20!\n\n\n\x02\x05\x01\x12\x04*\0>\x01\n\n\n\x03\x05\x01\x01\ + \x12\x03*\x05\x10\n\x0b\n\x04\x05\x01\x02\0\x12\x03+\x04\x0f\n\x0c\n\x05\ + \x05\x01\x02\0\x01\x12\x03+\x04\n\n\x0c\n\x05\x05\x01\x02\0\x02\x12\x03+\ + \r\x0e\n\x0b\n\x04\x05\x01\x02\x01\x12\x03,\x04\x10\n\x0c\n\x05\x05\x01\ + \x02\x01\x01\x12\x03,\x04\x0b\n\x0c\n\x05\x05\x01\x02\x01\x02\x12\x03,\ + \x0e\x0f\n\x0b\n\x04\x05\x01\x02\x02\x12\x03-\x04\x13\n\x0c\n\x05\x05\ + \x01\x02\x02\x01\x12\x03-\x04\x0e\n\x0c\n\x05\x05\x01\x02\x02\x02\x12\ + \x03-\x11\x12\n\x0b\n\x04\x05\x01\x02\x03\x12\x03.\x04\x12\n\x0c\n\x05\ + \x05\x01\x02\x03\x01\x12\x03.\x04\r\n\x0c\n\x05\x05\x01\x02\x03\x02\x12\ + \x03.\x10\x11\n\x0b\n\x04\x05\x01\x02\x04\x12\x03/\x04\x1a\n\x0c\n\x05\ + \x05\x01\x02\x04\x01\x12\x03/\x04\x15\n\x0c\n\x05\x05\x01\x02\x04\x02\ + \x12\x03/\x18\x19\n\x0b\n\x04\x05\x01\x02\x05\x12\x030\x04\x17\n\x0c\n\ + \x05\x05\x01\x02\x05\x01\x12\x030\x04\x12\n\x0c\n\x05\x05\x01\x02\x05\ + \x02\x12\x030\x15\x16\n\x0b\n\x04\x05\x01\x02\x06\x12\x031\x04\x1f\n\x0c\ + \n\x05\x05\x01\x02\x06\x01\x12\x031\x04\x1a\n\x0c\n\x05\x05\x01\x02\x06\ + \x02\x12\x031\x1d\x1e\n\x0b\n\x04\x05\x01\x02\x07\x12\x032\x04\x14\n\x0c\ + \n\x05\x05\x01\x02\x07\x01\x12\x032\x04\x0f\n\x0c\n\x05\x05\x01\x02\x07\ + \x02\x12\x032\x12\x13\n\x0b\n\x04\x05\x01\x02\x08\x12\x033\x04\x15\n\x0c\ + \n\x05\x05\x01\x02\x08\x01\x12\x033\x04\x10\n\x0c\n\x05\x05\x01\x02\x08\ + \x02\x12\x033\x13\x14\n\x0b\n\x04\x05\x01\x02\t\x12\x034\x04\x1d\n\x0c\n\ + \x05\x05\x01\x02\t\x01\x12\x034\x04\x18\n\x0c\n\x05\x05\x01\x02\t\x02\ + \x12\x034\x1b\x1c\n\x0b\n\x04\x05\x01\x02\n\x12\x035\x04\x18\n\x0c\n\x05\ + \x05\x01\x02\n\x01\x12\x035\x04\x12\n\x0c\n\x05\x05\x01\x02\n\x02\x12\ + \x035\x15\x17\n\x0b\n\x04\x05\x01\x02\x0b\x12\x036\x04\x17\n\x0c\n\x05\ + \x05\x01\x02\x0b\x01\x12\x036\x04\x11\n\x0c\n\x05\x05\x01\x02\x0b\x02\ + \x12\x036\x14\x16\n\x0b\n\x04\x05\x01\x02\x0c\x12\x037\x04\x18\n\x0c\n\ + \x05\x05\x01\x02\x0c\x01\x12\x037\x04\x12\n\x0c\n\x05\x05\x01\x02\x0c\ + \x02\x12\x037\x15\x17\n\x0b\n\x04\x05\x01\x02\r\x12\x038\x04\x1b\n\x0c\n\ + \x05\x05\x01\x02\r\x01\x12\x038\x04\x15\n\x0c\n\x05\x05\x01\x02\r\x02\ + \x12\x038\x18\x1a\n\x0b\n\x04\x05\x01\x02\x0e\x12\x039\x04\x17\n\x0c\n\ + \x05\x05\x01\x02\x0e\x01\x12\x039\x04\x11\n\x0c\n\x05\x05\x01\x02\x0e\ + \x02\x12\x039\x14\x16\n\x0b\n\x04\x05\x01\x02\x0f\x12\x03:\x04\x16\n\x0c\ + \n\x05\x05\x01\x02\x0f\x01\x12\x03:\x04\x10\n\x0c\n\x05\x05\x01\x02\x0f\ + \x02\x12\x03:\x13\x15\n\x0b\n\x04\x05\x01\x02\x10\x12\x03;\x04\x1a\n\x0c\ + \n\x05\x05\x01\x02\x10\x01\x12\x03;\x04\x14\n\x0c\n\x05\x05\x01\x02\x10\ + \x02\x12\x03;\x17\x19\n\x0b\n\x04\x05\x01\x02\x11\x12\x03<\x04\x1b\n\x0c\ + \n\x05\x05\x01\x02\x11\x01\x12\x03<\x04\x15\n\x0c\n\x05\x05\x01\x02\x11\ + \x02\x12\x03<\x18\x1a\n\x0b\n\x04\x05\x01\x02\x12\x12\x03=\x04#\n\x0c\n\ + \x05\x05\x01\x02\x12\x01\x12\x03=\x04\x1d\n\x0c\n\x05\x05\x01\x02\x12\ + \x02\x12\x03=\x20\"\n\n\n\x02\x04\x03\x12\x04@\0M\x01\n\n\n\x03\x04\x03\ + \x01\x12\x03@\x08\x0f\n\x0b\n\x04\x04\x03\x02\0\x12\x03A\x04\x1d\n\r\n\ + \x05\x04\x03\x02\0\x04\x12\x04A\x04@\x11\n\x0c\n\x05\x04\x03\x02\0\x06\ + \x12\x03A\x04\x0f\n\x0c\n\x05\x04\x03\x02\0\x01\x12\x03A\x10\x18\n\x0c\n\ + \x05\x04\x03\x02\0\x03\x12\x03A\x1b\x1c\n\x0b\n\x04\x04\x03\x02\x01\x12\ + \x03B\x04\x12\n\r\n\x05\x04\x03\x02\x01\x04\x12\x04B\x04A\x1d\n\x0c\n\ + \x05\x04\x03\x02\x01\x05\x12\x03B\x04\n\n\x0c\n\x05\x04\x03\x02\x01\x01\ + \x12\x03B\x0b\r\n\x0c\n\x05\x04\x03\x02\x01\x03\x12\x03B\x10\x11\n\x0b\n\ + \x04\x04\x03\x02\x02\x12\x03C\x04\x14\n\r\n\x05\x04\x03\x02\x02\x04\x12\ + \x04C\x04B\x12\n\x0c\n\x05\x04\x03\x02\x02\x05\x12\x03C\x04\n\n\x0c\n\ + \x05\x04\x03\x02\x02\x01\x12\x03C\x0b\x0f\n\x0c\n\x05\x04\x03\x02\x02\ + \x03\x12\x03C\x12\x13\n\x0b\n\x04\x04\x03\x02\x03\x12\x03D\x04\x14\n\r\n\ + \x05\x04\x03\x02\x03\x04\x12\x04D\x04C\x14\n\x0c\n\x05\x04\x03\x02\x03\ + \x05\x12\x03D\x04\n\n\x0c\n\x05\x04\x03\x02\x03\x01\x12\x03D\x0b\x0f\n\ + \x0c\n\x05\x04\x03\x02\x03\x03\x12\x03D\x12\x13\n\x0b\n\x04\x04\x03\x02\ + \x04\x12\x03E\x04\x18\n\r\n\x05\x04\x03\x02\x04\x04\x12\x04E\x04D\x14\n\ + \x0c\n\x05\x04\x03\x02\x04\x05\x12\x03E\x04\n\n\x0c\n\x05\x04\x03\x02\ + \x04\x01\x12\x03E\x0b\x13\n\x0c\n\x05\x04\x03\x02\x04\x03\x12\x03E\x16\ + \x17\n\x0b\n\x04\x04\x03\x02\x05\x12\x03F\x04\x15\n\r\n\x05\x04\x03\x02\ + \x05\x04\x12\x04F\x04E\x18\n\x0c\n\x05\x04\x03\x02\x05\x05\x12\x03F\x04\ + \n\n\x0c\n\x05\x04\x03\x02\x05\x01\x12\x03F\x0b\x10\n\x0c\n\x05\x04\x03\ + \x02\x05\x03\x12\x03F\x13\x14\n\x0b\n\x04\x04\x03\x02\x06\x12\x03G\x04\ + \x1f\n\x0c\n\x05\x04\x03\x02\x06\x04\x12\x03G\x04\x0c\n\x0c\n\x05\x04\ + \x03\x02\x06\x06\x12\x03G\r\x12\n\x0c\n\x05\x04\x03\x02\x06\x01\x12\x03G\ + \x13\x1a\n\x0c\n\x05\x04\x03\x02\x06\x03\x12\x03G\x1d\x1e\n\x0b\n\x04\ + \x04\x03\x02\x07\x12\x03H\x04\x16\n\r\n\x05\x04\x03\x02\x07\x04\x12\x04H\ + \x04G\x1f\n\x0c\n\x05\x04\x03\x02\x07\x05\x12\x03H\x04\n\n\x0c\n\x05\x04\ + \x03\x02\x07\x01\x12\x03H\x0b\x11\n\x0c\n\x05\x04\x03\x02\x07\x03\x12\ + \x03H\x14\x15\n\x0b\n\x04\x04\x03\x02\x08\x12\x03I\x04\x1a\n\r\n\x05\x04\ + \x03\x02\x08\x04\x12\x04I\x04H\x16\n\x0c\n\x05\x04\x03\x02\x08\x06\x12\ + \x03I\x04\x0c\n\x0c\n\x05\x04\x03\x02\x08\x01\x12\x03I\r\x15\n\x0c\n\x05\ + \x04\x03\x02\x08\x03\x12\x03I\x18\x19\n\x0b\n\x04\x04\x03\x02\t\x12\x03J\ + \x04\x15\n\r\n\x05\x04\x03\x02\t\x04\x12\x04J\x04I\x1a\n\x0c\n\x05\x04\ + \x03\x02\t\x05\x12\x03J\x04\x08\n\x0c\n\x05\x04\x03\x02\t\x01\x12\x03J\t\ + \x0f\n\x0c\n\x05\x04\x03\x02\t\x03\x12\x03J\x12\x14\n\x0b\n\x04\x04\x03\ + \x02\n\x12\x03K\x04\x1c\n\r\n\x05\x04\x03\x02\n\x04\x12\x04K\x04J\x15\n\ + \x0c\n\x05\x04\x03\x02\n\x05\x12\x03K\x04\n\n\x0c\n\x05\x04\x03\x02\n\ + \x01\x12\x03K\x0b\x16\n\x0c\n\x05\x04\x03\x02\n\x03\x12\x03K\x19\x1b\n\ + \x0b\n\x04\x04\x03\x02\x0b\x12\x03L\x04\x17\n\r\n\x05\x04\x03\x02\x0b\ + \x04\x12\x04L\x04K\x1c\n\x0c\n\x05\x04\x03\x02\x0b\x05\x12\x03L\x04\t\n\ + \x0c\n\x05\x04\x03\x02\x0b\x01\x12\x03L\n\x11\n\x0c\n\x05\x04\x03\x02\ + \x0b\x03\x12\x03L\x14\x16\n\n\n\x02\x04\x04\x12\x04O\0U\x01\n\n\n\x03\ + \x04\x04\x01\x12\x03O\x08\x11\n\x0b\n\x04\x04\x04\x02\0\x12\x03P\x04\x14\ + \n\r\n\x05\x04\x04\x02\0\x04\x12\x04P\x04O\x13\n\x0c\n\x05\x04\x04\x02\0\ + \x05\x12\x03P\x04\n\n\x0c\n\x05\x04\x04\x02\0\x01\x12\x03P\x0b\x0f\n\x0c\ + \n\x05\x04\x04\x02\0\x03\x12\x03P\x12\x13\n\x0b\n\x04\x04\x04\x02\x01\ + \x12\x03Q\x04\x14\n\r\n\x05\x04\x04\x02\x01\x04\x12\x04Q\x04P\x14\n\x0c\ + \n\x05\x04\x04\x02\x01\x05\x12\x03Q\x04\n\n\x0c\n\x05\x04\x04\x02\x01\ + \x01\x12\x03Q\x0b\x0f\n\x0c\n\x05\x04\x04\x02\x01\x03\x12\x03Q\x12\x13\n\ + \x0b\n\x04\x04\x04\x02\x02\x12\x03R\x04\x16\n\r\n\x05\x04\x04\x02\x02\ + \x04\x12\x04R\x04Q\x14\n\x0c\n\x05\x04\x04\x02\x02\x05\x12\x03R\x04\n\n\ + \x0c\n\x05\x04\x04\x02\x02\x01\x12\x03R\x0b\x11\n\x0c\n\x05\x04\x04\x02\ + \x02\x03\x12\x03R\x14\x15\nd\n\x04\x04\x04\x02\x03\x12\x03T\x04-\x1aW\ + \x20If\x20the\x20peer\x20goes\x20down\x20we\x20need\x20to\x20be\x20aware\ + \x20of\x20a\x20partially\x20completed\x20membership\x20change.\n\n\r\n\ + \x05\x04\x04\x02\x03\x04\x12\x04T\x04R\x16\n\x0c\n\x05\x04\x04\x02\x03\ + \x06\x12\x03T\x04\x0e\n\x0c\n\x05\x04\x04\x02\x03\x01\x12\x03T\x0f(\n\ + \x0c\n\x05\x04\x04\x02\x03\x03\x12\x03T+,\n\n\n\x02\x04\x05\x12\x04W\0Z\ + \x01\n\n\n\x03\x04\x05\x01\x12\x03W\x08\x11\n\x0b\n\x04\x04\x05\x02\0\ + \x12\x03X\x04\x1e\n\x0c\n\x05\x04\x05\x02\0\x04\x12\x03X\x04\x0c\n\x0c\n\ + \x05\x04\x05\x02\0\x05\x12\x03X\r\x13\n\x0c\n\x05\x04\x05\x02\0\x01\x12\ + \x03X\x14\x19\n\x0c\n\x05\x04\x05\x02\0\x03\x12\x03X\x1c\x1d\n\x0b\n\x04\ + \x04\x05\x02\x01\x12\x03Y\x04!\n\x0c\n\x05\x04\x05\x02\x01\x04\x12\x03Y\ + \x04\x0c\n\x0c\n\x05\x04\x05\x02\x01\x05\x12\x03Y\r\x13\n\x0c\n\x05\x04\ + \x05\x02\x01\x01\x12\x03Y\x14\x1c\n\x0c\n\x05\x04\x05\x02\x01\x03\x12\ + \x03Y\x1f\x20\n\n\n\x02\x05\x02\x12\x04\\\0b\x01\n\n\n\x03\x05\x02\x01\ + \x12\x03\\\x05\x13\n\x0b\n\x04\x05\x02\x02\0\x12\x03]\x04\x13\n\x0c\n\ + \x05\x05\x02\x02\0\x01\x12\x03]\x04\x0b\n\x0c\n\x05\x05\x02\x02\0\x02\ + \x12\x03]\x11\x12\n\x0b\n\x04\x05\x02\x02\x01\x12\x03^\x04\x13\n\x0c\n\ + \x05\x05\x02\x02\x01\x01\x12\x03^\x04\x0e\n\x0c\n\x05\x05\x02\x02\x01\ + \x02\x12\x03^\x11\x12\n\x0b\n\x04\x05\x02\x02\x02\x12\x03_\x04\x17\n\x0c\ + \n\x05\x05\x02\x02\x02\x01\x12\x03_\x04\x12\n\x0c\n\x05\x05\x02\x02\x02\ + \x02\x12\x03_\x15\x16\n\x0b\n\x04\x05\x02\x02\x03\x12\x03`\x04\x1e\n\x0c\ + \n\x05\x05\x02\x02\x03\x01\x12\x03`\x04\x19\n\x0c\n\x05\x05\x02\x02\x03\ + \x02\x12\x03`\x1c\x1d\n\x0b\n\x04\x05\x02\x02\x04\x12\x03a\x04!\n\x0c\n\ + \x05\x05\x02\x02\x04\x01\x12\x03a\x04\x1c\n\x0c\n\x05\x05\x02\x02\x04\ + \x02\x12\x03a\x1f\x20\n\n\n\x02\x04\x06\x12\x04d\0p\x01\n\n\n\x03\x04\ + \x06\x01\x12\x03d\x08\x12\n\x0b\n\x04\x04\x06\x02\0\x12\x03e\x04\x12\n\r\ + \n\x05\x04\x06\x02\0\x04\x12\x04e\x04d\x14\n\x0c\n\x05\x04\x06\x02\0\x05\ + \x12\x03e\x04\n\n\x0c\n\x05\x04\x06\x02\0\x01\x12\x03e\x0b\r\n\x0c\n\x05\ + \x04\x06\x02\0\x03\x12\x03e\x10\x11\n\x0b\n\x04\x04\x06\x02\x01\x12\x03f\ + \x04#\n\r\n\x05\x04\x06\x02\x01\x04\x12\x04f\x04e\x12\n\x0c\n\x05\x04\ + \x06\x02\x01\x06\x12\x03f\x04\x12\n\x0c\n\x05\x04\x06\x02\x01\x01\x12\ + \x03f\x13\x1e\n\x0c\n\x05\x04\x06\x02\x01\x03\x12\x03f!\"\nE\n\x04\x04\ + \x06\x02\x02\x12\x03h\x04\x17\x1a8\x20Used\x20in\x20`AddNode`,\x20`Remov\ + eNode`,\x20and\x20`AddLearnerNode`.\n\n\r\n\x05\x04\x06\x02\x02\x04\x12\ + \x04h\x04f#\n\x0c\n\x05\x04\x06\x02\x02\x05\x12\x03h\x04\n\n\x0c\n\x05\ + \x04\x06\x02\x02\x01\x12\x03h\x0b\x12\n\x0c\n\x05\x04\x06\x02\x02\x03\ + \x12\x03h\x15\x16\n\x0b\n\x04\x04\x06\x02\x03\x12\x03i\x04\x16\n\r\n\x05\ + \x04\x06\x02\x03\x04\x12\x04i\x04h\x17\n\x0c\n\x05\x04\x06\x02\x03\x05\ + \x12\x03i\x04\t\n\x0c\n\x05\x04\x06\x02\x03\x01\x12\x03i\n\x11\n\x0c\n\ + \x05\x04\x06\x02\x03\x03\x12\x03i\x14\x15\nN\n\x04\x04\x06\x02\x04\x12\ + \x03k\x04\x20\x1aA\x20Used\x20in\x20`BeginMembershipChange`\x20and\x20`F\ + inalizeMembershipChange`.\n\n\r\n\x05\x04\x06\x02\x04\x04\x12\x04k\x04i\ + \x16\n\x0c\n\x05\x04\x06\x02\x04\x06\x12\x03k\x04\r\n\x0c\n\x05\x04\x06\ + \x02\x04\x01\x12\x03k\x0e\x1b\n\x0c\n\x05\x04\x06\x02\x04\x03\x12\x03k\ + \x1e\x1f\n\xd0\x01\n\x04\x04\x06\x02\x05\x12\x03o\x04\x1b\x1a\xc2\x01\ + \x20Used\x20in\x20`BeginMembershipChange`\x20and\x20`FinalizeMembershipC\ hange`.\n\x20Because\x20`RawNode::apply_conf_change`\x20takes\x20a\x20`C\ onfChange`\x20instead\x20of\x20an\x20`Entry`\x20we\x20must\n\x20include\ \x20this\x20index\x20so\x20it\x20can\x20be\x20known.\n\n\r\n\x05\x04\x06\ diff --git a/src/raft.rs b/src/raft.rs index 81cacf51c..4efcc067c 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -146,16 +146,16 @@ pub struct Raft { /// /// This value is conservatively set in cases where there may be a configuration change pending, /// but scanning the log is possibly expensive. This implies that the index stated here may not - /// necessarily be a config change entry, and it may not be a `BeginConfChange` entry, even if + /// necessarily be a config change entry, and it may not be a `BeginMembershipChange` entry, even if /// we set this to one. pub pending_conf_index: u64, - /// The last BeginConfChange entry. Once we commit this entry we can exit the joint state. + /// The last BeginMembershipChange entry. Once we commit this entry we can exit the joint state. /// /// This is different than `pending_conf_index` since it is more specific, and also exact. /// While `pending_conf_index` is conservatively set at times to ensure safety in the /// one-by-one change method, in joint consensus based changes we track the state exactly. The - /// index here **must** only be set when a `BeginConfChange` is present at that index. + /// index here **must** only be set when a `BeginMembershipChange` is present at that index. /// /// # Caveats /// @@ -671,7 +671,7 @@ impl Raft { fn append_finalize_conf_change_entry(&mut self) { let mut conf_change = ConfChange::new(); - conf_change.set_change_type(ConfChangeType::FinalizeConfChange); + conf_change.set_change_type(ConfChangeType::FinalizeMembershipChange); let data = protobuf::Message::write_to_bytes(&conf_change).unwrap(); let mut entry = Entry::new(); entry.set_entry_type(EntryType::EntryConfChange); @@ -1184,7 +1184,7 @@ impl Raft { Ok(()) } - /// Apply a `BeginConfChange` variant `ConfChange`. + /// Apply a `BeginMembershipChange` variant `ConfChange`. /// /// When a Raft node applies this variant of a configuration change it will adopt a joint /// configuration state until the membership change is finalized. @@ -1201,15 +1201,15 @@ impl Raft { /// /// # Contracts /// - /// * The `ConfChange.change_type` must be a `BeginConfChange` + /// * The `ConfChange.change_type` must be a `BeginMembershipChange` /// * The `ConfChange.configuration` value must exist. /// * The `ConfChange.start_index` value must exist. It should equal the index of the /// corresponding entry. #[inline(always)] pub fn begin_membership_change(&mut self, conf_change: &ConfChange) -> Result<()> { - if conf_change.get_change_type() != ConfChangeType::BeginConfChange { + if conf_change.get_change_type() != ConfChangeType::BeginMembershipChange { return Err(Error::ViolatesContract(format!( - "{:?} != BeginConfChange", + "{:?} != BeginMembershipChange", conf_change.get_change_type() ))); } @@ -1233,7 +1233,7 @@ impl Raft { Ok(()) } - /// Apply a `FinalizeConfChange` variant `ConfChange`. + /// Apply a `FinalizeMembershipChange` variant `ConfChange`. /// /// When a Raft node applies this variant of a configuration change it will finalize the /// transition begun by [`begin_membership_change`]. @@ -1251,14 +1251,14 @@ impl Raft { /// # Contracts /// /// * The Raft should already have started a configuration change with `begin_membership_change`. - /// * The `ConfChange.change_type` must be a `FinalizeConfChange`. + /// * The `ConfChange.change_type` must be a `FinalizeMembershipChange`. /// * The `ConfChange.configuration` value should not exist. (Panics in debug mode.) /// * The `ConfChange.start_index` value should not exist. (Panics in debug mode.) #[inline(always)] pub fn finalize_membership_change(&mut self, conf_change: &ConfChange) -> Result<()> { - if conf_change.get_change_type() != ConfChangeType::FinalizeConfChange { + if conf_change.get_change_type() != ConfChangeType::FinalizeMembershipChange { return Err(Error::ViolatesContract(format!( - "{:?} != BeginConfChange", + "{:?} != BeginMembershipChange", conf_change.get_change_type() ))); } @@ -2156,7 +2156,7 @@ impl Raft { let destination_index = self.raft_log.last_index() + 1; // Prep a configuration change to append. let mut conf_change = ConfChange::new(); - conf_change.set_change_type(ConfChangeType::BeginConfChange); + conf_change.set_change_type(ConfChangeType::BeginMembershipChange); conf_change.set_configuration(config.into()); conf_change.set_start_index(destination_index); let data = protobuf::Message::write_to_bytes(&conf_change)?; diff --git a/src/raw_node.rs b/src/raw_node.rs index 6c7369284..a123dbf08 100644 --- a/src/raw_node.rs +++ b/src/raw_node.rs @@ -335,12 +335,12 @@ impl RawNode { /// /// # Panics /// - /// In the case of `BeginConfChange` or `FinalizeConfChange` returning errors this will panic. + /// In the case of `BeginMembershipChange` or `FinalizeMembershipChange` returning errors this will panic. /// /// For a safe interface for these directly call `this.raft.begin_membership_change(entry)` or /// `this.raft.finalize_membership_change(entry)` respectively. pub fn apply_conf_change(&mut self, cc: &ConfChange) -> ConfState { - if cc.get_node_id() == INVALID_ID && cc.get_change_type() != ConfChangeType::BeginConfChange + if cc.get_node_id() == INVALID_ID && cc.get_change_type() != ConfChangeType::BeginMembershipChange { let mut cs = ConfState::new(); cs.set_nodes(self.raft.prs().voter_ids().iter().cloned().collect()); @@ -352,8 +352,8 @@ impl RawNode { ConfChangeType::AddNode => self.raft.add_node(nid), ConfChangeType::AddLearnerNode => self.raft.add_learner(nid), ConfChangeType::RemoveNode => self.raft.remove_node(nid), - ConfChangeType::BeginConfChange => self.raft.begin_membership_change(cc).unwrap(), - ConfChangeType::FinalizeConfChange => { + ConfChangeType::BeginMembershipChange => self.raft.begin_membership_change(cc).unwrap(), + ConfChangeType::FinalizeMembershipChange => { self.raft.mut_prs().finalize_membership_change().unwrap(); } } diff --git a/src/storage.rs b/src/storage.rs index 400890a8a..9038a6366 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -114,7 +114,6 @@ impl MemStorageCore { /// Overwrites the contents of this Storage object with those of the given snapshot. pub fn apply_snapshot(&mut self, snapshot: Snapshot) -> Result<()> { - error!("I HAVE BEEN CALLED!!!!"); // handle check for old snapshot being applied let index = self.snapshot.get_metadata().get_index(); let snapshot_index = snapshot.get_metadata().get_index(); diff --git a/tests/integration_cases/test_membership_changes.rs b/tests/integration_cases/test_membership_changes.rs index fdf96171f..3672f6881 100644 --- a/tests/integration_cases/test_membership_changes.rs +++ b/tests/integration_cases/test_membership_changes.rs @@ -18,7 +18,7 @@ use raft::{ eraftpb::{ ConfChange, ConfChangeType, ConfState, Entry, EntryType, Message, MessageType, Snapshot, }, - storage::{MemStorage, Storage}, + storage::{MemStorage}, Config, Configuration, Raft, Result, INVALID_ID, NO_LIMIT, }; use std::ops::{Deref, DerefMut}; @@ -134,7 +134,7 @@ mod three_peers_add_voter { scenario.assert_can_apply_transition_entry_at_index( &[1], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1]); @@ -143,7 +143,7 @@ mod three_peers_add_voter { scenario.assert_can_apply_transition_entry_at_index( &[2, 3], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3]); @@ -152,7 +152,7 @@ mod three_peers_add_voter { scenario.assert_can_apply_transition_entry_at_index( &[4], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3, 4]); @@ -161,7 +161,7 @@ mod three_peers_add_voter { scenario.assert_can_apply_transition_entry_at_index( &[1, 2, 3, 4], 3, - ConfChangeType::FinalizeConfChange, + ConfChangeType::FinalizeMembershipChange, ); scenario.assert_not_in_membership_change(&[1, 2, 3, 4]); @@ -191,7 +191,7 @@ mod three_peers_add_learner { scenario.assert_can_apply_transition_entry_at_index( &[1], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1]); @@ -200,7 +200,7 @@ mod three_peers_add_learner { scenario.assert_can_apply_transition_entry_at_index( &[2, 3], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3]); @@ -209,7 +209,7 @@ mod three_peers_add_learner { scenario.assert_can_apply_transition_entry_at_index( &[4], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3, 4]); @@ -218,7 +218,7 @@ mod three_peers_add_learner { scenario.assert_can_apply_transition_entry_at_index( &[1, 2, 3, 4], 3, - ConfChangeType::FinalizeConfChange, + ConfChangeType::FinalizeMembershipChange, ); scenario.assert_not_in_membership_change(&[1, 2, 3, 4]); @@ -248,7 +248,7 @@ mod remove_learner { scenario.assert_can_apply_transition_entry_at_index( &[1], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1]); @@ -257,7 +257,7 @@ mod remove_learner { scenario.assert_can_apply_transition_entry_at_index( &[2, 3, 4], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3]); @@ -266,7 +266,7 @@ mod remove_learner { scenario.assert_can_apply_transition_entry_at_index( &[1, 2, 3, 4], 3, - ConfChangeType::FinalizeConfChange, + ConfChangeType::FinalizeMembershipChange, ); scenario.assert_not_in_membership_change(&[1, 2, 3, 4]); @@ -296,7 +296,7 @@ mod remove_voter { scenario.assert_can_apply_transition_entry_at_index( &[1], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1]); @@ -305,7 +305,7 @@ mod remove_voter { scenario.assert_can_apply_transition_entry_at_index( &[2, 3], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3]); @@ -314,7 +314,7 @@ mod remove_voter { scenario.assert_can_apply_transition_entry_at_index( &[1, 2], 3, - ConfChangeType::FinalizeConfChange, + ConfChangeType::FinalizeMembershipChange, ); scenario.assert_not_in_membership_change(&[1, 2]); @@ -344,7 +344,7 @@ mod remove_leader { scenario.assert_can_apply_transition_entry_at_index( &[1], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1]); @@ -353,7 +353,7 @@ mod remove_leader { scenario.assert_can_apply_transition_entry_at_index( &[2, 3], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3]); @@ -362,7 +362,7 @@ mod remove_leader { scenario.assert_can_apply_transition_entry_at_index( &[1, 2, 3], 3, - ConfChangeType::FinalizeConfChange, + ConfChangeType::FinalizeMembershipChange, ); scenario.assert_not_in_membership_change(&[1, 2, 3]); let peer_leaders = scenario.peer_leaders(); @@ -420,7 +420,7 @@ mod remove_leader { scenario.assert_can_apply_transition_entry_at_index( &[1], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1]); @@ -429,7 +429,7 @@ mod remove_leader { scenario.assert_can_apply_transition_entry_at_index( &[2, 3], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3]); @@ -441,7 +441,7 @@ mod remove_leader { scenario.assert_can_apply_transition_entry_at_index( &[2, 3], 3, - ConfChangeType::FinalizeConfChange, + ConfChangeType::FinalizeMembershipChange, ); scenario.assert_not_in_membership_change(&[2, 3]); @@ -504,7 +504,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[1], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1]); @@ -513,7 +513,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[2], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2]); @@ -522,7 +522,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[4], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 4]); @@ -531,7 +531,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[1, 2, 4], 3, - ConfChangeType::FinalizeConfChange, + ConfChangeType::FinalizeMembershipChange, ); scenario.assert_not_in_membership_change(&[1, 2, 4]); @@ -556,7 +556,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[1], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1]); @@ -565,7 +565,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[2, 3], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3]); @@ -588,7 +588,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[4], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3, 4]); @@ -598,7 +598,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[1, 2, 3, 4], 4, - ConfChangeType::FinalizeConfChange, + ConfChangeType::FinalizeMembershipChange, ); scenario.assert_not_in_membership_change(&[1, 2, 4]); @@ -623,7 +623,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[1], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1]); @@ -632,7 +632,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[2, 3], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3]); @@ -688,7 +688,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[1, 2, 3, 4], 4, - ConfChangeType::FinalizeConfChange, + ConfChangeType::FinalizeMembershipChange, ); scenario.assert_not_in_membership_change(&[1, 2, 3, 4]); @@ -713,7 +713,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[1], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1]); @@ -724,7 +724,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[2], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2]); @@ -733,7 +733,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[4], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 4]); @@ -742,7 +742,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[1, 2, 4], 3, - ConfChangeType::FinalizeConfChange, + ConfChangeType::FinalizeMembershipChange, ); scenario.assert_not_in_membership_change(&[1, 2, 4]); @@ -767,7 +767,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[1], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1]); @@ -778,7 +778,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[2, 3], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3]); @@ -787,7 +787,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[1, 2, 3], 3, - ConfChangeType::FinalizeConfChange, + ConfChangeType::FinalizeMembershipChange, ); scenario.assert_not_in_membership_change(&[1, 2, 3]); @@ -812,7 +812,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[1], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1]); @@ -824,7 +824,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[2], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2]); @@ -833,7 +833,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[1, 2], 3, - ConfChangeType::FinalizeConfChange, + ConfChangeType::FinalizeMembershipChange, ); scenario.assert_not_in_membership_change(&[1, 2]); @@ -858,7 +858,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[1], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1]); @@ -871,7 +871,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[4], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 4]); scenario.assert_not_in_membership_change(&[2, 3]); @@ -906,7 +906,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[2, 3], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3, 4]); @@ -918,7 +918,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[1, 2, 3, 4], 3, - ConfChangeType::FinalizeConfChange, + ConfChangeType::FinalizeMembershipChange, ); scenario.assert_not_in_membership_change(&[1, 2, 3, 4]); @@ -943,7 +943,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[1], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1]); @@ -988,7 +988,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[2, 3, 4], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3, 4]); @@ -1000,7 +1000,7 @@ mod three_peers_replace_voter { scenario.assert_can_apply_transition_entry_at_index( &[1, 2, 3, 4], 3, - ConfChangeType::FinalizeConfChange, + ConfChangeType::FinalizeMembershipChange, ); scenario.assert_not_in_membership_change(&[1, 2, 3, 4]); @@ -1030,7 +1030,7 @@ mod three_peers_to_five_with_learner { scenario.assert_can_apply_transition_entry_at_index( &[1], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1]); @@ -1039,7 +1039,7 @@ mod three_peers_to_five_with_learner { scenario.assert_can_apply_transition_entry_at_index( &[2, 3], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3]); @@ -1048,7 +1048,7 @@ mod three_peers_to_five_with_learner { scenario.assert_can_apply_transition_entry_at_index( &[4, 5, 6], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3, 4, 5, 6]); @@ -1057,7 +1057,7 @@ mod three_peers_to_five_with_learner { scenario.assert_can_apply_transition_entry_at_index( &[1, 2, 3, 4, 5, 6], 3, - ConfChangeType::FinalizeConfChange, + ConfChangeType::FinalizeMembershipChange, ); scenario.assert_not_in_membership_change(&[1, 2, 3, 4, 5, 6]); @@ -1083,7 +1083,7 @@ mod three_peers_to_five_with_learner { scenario.assert_can_apply_transition_entry_at_index( &[1], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1]); @@ -1092,7 +1092,7 @@ mod three_peers_to_five_with_learner { scenario.assert_can_apply_transition_entry_at_index( &[2], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2]); scenario.assert_not_in_membership_change(&[3]); @@ -1102,7 +1102,7 @@ mod three_peers_to_five_with_learner { scenario.assert_can_apply_transition_entry_at_index( &[4, 5, 6], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 4, 5, 6]); scenario.assert_not_in_membership_change(&[3]); @@ -1121,7 +1121,7 @@ mod three_peers_to_five_with_learner { scenario.assert_can_apply_transition_entry_at_index( &[1, 2, 4, 5], 3, - ConfChangeType::FinalizeConfChange, + ConfChangeType::FinalizeMembershipChange, ); scenario.assert_not_in_membership_change(&[1, 2, 4, 5]); scenario.assert_not_in_membership_change(&[3]); @@ -1133,7 +1133,7 @@ mod three_peers_to_five_with_learner { mod intermingled_config_changes { use super::*; - // In this test, we make sure that if the peer group is sent a `BeginConfChange`, then immediately a `AddNode` entry, that the `AddNode` is rejected by the leader. + // In this test, we make sure that if the peer group is sent a `BeginMembershipChange`, then immediately a `AddNode` entry, that the `AddNode` is rejected by the leader. #[test] fn begin_then_add_node() -> Result<()> { setup_for_test(); @@ -1151,7 +1151,7 @@ mod intermingled_config_changes { scenario.assert_can_apply_transition_entry_at_index( &[1], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1]); @@ -1171,7 +1171,7 @@ mod intermingled_config_changes { scenario.assert_can_apply_transition_entry_at_index( &[2, 3], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3]); @@ -1180,7 +1180,7 @@ mod intermingled_config_changes { scenario.assert_can_apply_transition_entry_at_index( &[4], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3, 4]); @@ -1189,7 +1189,7 @@ mod intermingled_config_changes { scenario.assert_can_apply_transition_entry_at_index( &[1, 2, 3, 4], 3, - ConfChangeType::FinalizeConfChange, + ConfChangeType::FinalizeMembershipChange, ); scenario.assert_not_in_membership_change(&[1, 2, 3, 4]); @@ -1218,7 +1218,7 @@ mod compaction { scenario.assert_can_apply_transition_entry_at_index( &[1], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1]); @@ -1227,7 +1227,7 @@ mod compaction { scenario.assert_can_apply_transition_entry_at_index( &[2, 3], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3]); @@ -1236,7 +1236,7 @@ mod compaction { scenario.assert_can_apply_transition_entry_at_index( &[4], 2, - ConfChangeType::BeginConfChange, + ConfChangeType::BeginMembershipChange, ); scenario.assert_in_membership_change(&[1, 2, 3, 4]); @@ -1255,7 +1255,7 @@ mod compaction { scenario.assert_can_apply_transition_entry_at_index( &[1, 2, 3, 4], 3, - ConfChangeType::FinalizeConfChange, + ConfChangeType::FinalizeMembershipChange, ); scenario.assert_not_in_membership_change(&[1, 2, 3, 4]); @@ -1399,7 +1399,7 @@ impl Scenario { } /// Send a message proposing a "one-by-one" style AddNode configuration. - /// If the peers are in the midst joint consensus style (Begin/FinalizeConfChange) change they should reject it. + /// If the peers are in the midst joint consensus style (Begin/FinalizeMembershipChange) change they should reject it. fn propose_add_node_message(&mut self, id: u64) -> Result<()> { info!("Proposing add_node message. Target: {:?}", id,); let message = build_propose_add_node_message( @@ -1469,10 +1469,10 @@ impl Scenario { if conf_change.get_change_type() == entry_type { found = true; match entry_type { - ConfChangeType::BeginConfChange => { + ConfChangeType::BeginMembershipChange => { peer.begin_membership_change(&conf_change)? } - ConfChangeType::FinalizeConfChange => { + ConfChangeType::FinalizeMembershipChange => { peer.finalize_membership_change(&conf_change)? } ConfChangeType::AddNode => peer.add_node(conf_change.get_node_id()), @@ -1625,7 +1625,7 @@ fn begin_conf_change<'a>( ) -> ConfChange { let conf_state = conf_state(voters, learners); let mut conf_change = ConfChange::new(); - conf_change.set_change_type(ConfChangeType::BeginConfChange); + conf_change.set_change_type(ConfChangeType::BeginMembershipChange); conf_change.set_configuration(conf_state); conf_change.set_start_index(index); conf_change @@ -1633,7 +1633,7 @@ fn begin_conf_change<'a>( fn finalize_conf_change<'a>() -> ConfChange { let mut conf_change = ConfChange::new(); - conf_change.set_change_type(ConfChangeType::FinalizeConfChange); + conf_change.set_change_type(ConfChangeType::FinalizeMembershipChange); conf_change } @@ -1651,16 +1651,6 @@ fn begin_entry<'a>( entry } -fn finalize_entry(index: u64) -> Entry { - let mut conf_change = finalize_conf_change(); - let data = protobuf::Message::write_to_bytes(&conf_change).unwrap(); - let mut entry = Entry::new(); - entry.set_entry_type(EntryType::EntryConfChange); - entry.set_index(index); - entry.set_data(data); - entry -} - fn build_propose_change_message<'a>( recipient: u64, voters: impl IntoIterator, From 8c4ba9d732883a1a365d83f54941b46708015dbe Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Fri, 11 Jan 2019 15:04:56 -0800 Subject: [PATCH 26/41] Add/remove/promote error in progress on pending change. Signed-off-by: Ana Hobden --- src/progress.rs | 100 ++++++++++-------- src/raft.rs | 2 +- src/raw_node.rs | 3 +- .../test_membership_changes.rs | 2 +- 4 files changed, 60 insertions(+), 47 deletions(-) diff --git a/src/progress.rs b/src/progress.rs index 44bb30d42..361738517 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -26,7 +26,7 @@ // limitations under the License. use eraftpb::ConfState; -use errors::Error; +use errors::{Error, Result}; use fxhash::{FxBuildHasher, FxHashMap, FxHashSet}; use std::cell::RefCell; use std::cmp; @@ -125,7 +125,7 @@ impl Configuration { /// Namely: /// * There can be no overlap of voters and learners. /// * There must be at least one voter. - pub fn valid(&self) -> Result<(), Error> { + pub fn valid(&self) -> Result<()> { if let Some(id) = self.voters.intersection(&self.learners).next() { Err(Error::Exists(*id, "learners"))?; } else if self.voters.is_empty() { @@ -307,64 +307,92 @@ impl ProgressSet { self.progress.iter_mut() } - /// Adds a voter node - pub fn insert_voter(&mut self, id: u64, pr: Progress) -> Result<(), Error> { + /// Adds a voter to the group. + /// + /// # Contracts + /// + /// * The `id` does not exist in the voter set. + /// * The `id` does not exist in the learner set. + /// * There is no pending membership change. + pub fn insert_voter(&mut self, id: u64, pr: Progress) -> Result<()> { debug!("Inserting voter with id {}.", id); if self.learner_ids().contains(&id) { - Err(Error::Exists(id, "learners"))?; + return Err(Error::Exists(id, "learners")); } else if self.voter_ids().contains(&id) { - Err(Error::Exists(id, "voters"))?; + return Err(Error::Exists(id, "voters")); + } else if self.is_in_membership_change() { + return Err(Error::ViolatesContract( + "There is a pending membership change.".into(), + )); } - self.configuration.voters.insert(id); - self.next_configuration - .as_mut() - .map(|config| config.voters.insert(id)); + self.configuration.voters.insert(id); self.progress.insert(id, pr); + self.assert_progress_and_configuration_consistent(); Ok(()) } - /// Adds a learner to the cluster - pub fn insert_learner(&mut self, id: u64, pr: Progress) -> Result<(), Error> { + /// Adds a learner to the group. + /// + /// # Contracts + /// + /// * The `id` does not exist in the voter set. + /// * The `id` does not exist in the learner set. + /// * There is no pending membership change. + pub fn insert_learner(&mut self, id: u64, pr: Progress) -> Result<()> { debug!("Inserting learner with id {}.", id); if self.learner_ids().contains(&id) { - Err(Error::Exists(id, "learners"))?; + return Err(Error::Exists(id, "learners")); } else if self.voter_ids().contains(&id) { - Err(Error::Exists(id, "voters"))?; - } - self.configuration.learners.insert(id); - if let Some(ref mut next) = self.next_configuration { - next.learners.insert(id); + return Err(Error::Exists(id, "voters")); + } else if self.is_in_membership_change() { + return Err(Error::ViolatesContract( + "There is a pending membership change.".into(), + )); } + self.configuration.learners.insert(id); self.progress.insert(id, pr); + self.assert_progress_and_configuration_consistent(); Ok(()) } /// Removes the peer from the set of voters or learners. - pub fn remove(&mut self, id: u64) -> Option { + /// + /// # Contracts + /// + /// * There is no pending membership change. + pub fn remove(&mut self, id: u64) -> Result> { debug!("Removing peer with id {}.", id); + if self.is_in_membership_change() { + return Err(Error::ViolatesContract( + "There is a pending membership change.".into(), + )); + } + self.configuration.learners.remove(&id); self.configuration.voters.remove(&id); - if let Some(ref mut next) = self.next_configuration { - next.learners.remove(&id); - next.voters.remove(&id); - }; - let removed = self.progress.remove(&id); + self.assert_progress_and_configuration_consistent(); - removed + Ok(removed) } /// Promote a learner to a peer. - pub fn promote_learner(&mut self, id: u64) -> Result<(), Error> { + pub fn promote_learner(&mut self, id: u64) -> Result<()> { debug!("Promote learner with id {}.", id); + if self.is_in_membership_change() { + return Err(Error::ViolatesContract( + "There is a pending membership change.".into(), + )); + } + if !self.configuration.learners.remove(&id) { // Wasn't already a learner. We can't promote what doesn't exist. return Err(Error::NotExists(id, "learners")); @@ -373,22 +401,6 @@ impl ProgressSet { // Already existed, the caller should know this was a noop. return Err(Error::Exists(id, "voters")); } - if let Some(ref mut next) = self.next_configuration { - if !next.learners.remove(&id) { - // Wasn't already a voter. We can't promote what doesn't exist. - // Rollback change above. - self.configuration.voters.remove(&id); - self.configuration.learners.insert(id); - return Err(Error::Exists(id, "next learners")); - } - if !next.voters.insert(id) { - // Already existed, the caller should know this was a noop. - // Rollback change above. - self.configuration.voters.remove(&id); - self.configuration.learners.insert(id); - return Err(Error::Exists(id, "next voters")); - } - } self.assert_progress_and_configuration_consistent(); Ok(()) @@ -553,7 +565,7 @@ impl ProgressSet { &mut self, next: impl Into, mut progress: Progress, - ) -> Result<(), Error> { + ) -> Result<()> { let next = next.into(); next.valid()?; // Demotion check. @@ -587,7 +599,7 @@ impl ProgressSet { /// /// This must be called only after calling `begin_membership_change` and after the majority /// of peers in both the `current` and the `next` state have commited the changes. - pub fn finalize_membership_change(&mut self) -> Result<(), Error> { + pub fn finalize_membership_change(&mut self) -> Result<()> { let next = self.next_configuration.take(); match next { None => Err(Error::NoPendingMembershipChange)?, diff --git a/src/raft.rs b/src/raft.rs index 4efcc067c..8100cf0cc 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -2217,7 +2217,7 @@ impl Raft { /// Removes a node from the raft. pub fn remove_node(&mut self, id: u64) { - self.mut_prs().remove(id); + self.mut_prs().remove(id).ok(); // do not try to commit or abort transferring if there are no nodes in the cluster. if self.prs().voter_ids().is_empty() && self.prs().learner_ids().is_empty() { diff --git a/src/raw_node.rs b/src/raw_node.rs index a123dbf08..d9bb026c8 100644 --- a/src/raw_node.rs +++ b/src/raw_node.rs @@ -340,7 +340,8 @@ impl RawNode { /// For a safe interface for these directly call `this.raft.begin_membership_change(entry)` or /// `this.raft.finalize_membership_change(entry)` respectively. pub fn apply_conf_change(&mut self, cc: &ConfChange) -> ConfState { - if cc.get_node_id() == INVALID_ID && cc.get_change_type() != ConfChangeType::BeginMembershipChange + if cc.get_node_id() == INVALID_ID + && cc.get_change_type() != ConfChangeType::BeginMembershipChange { let mut cs = ConfState::new(); cs.set_nodes(self.raft.prs().voter_ids().iter().cloned().collect()); diff --git a/tests/integration_cases/test_membership_changes.rs b/tests/integration_cases/test_membership_changes.rs index 3672f6881..82dc258da 100644 --- a/tests/integration_cases/test_membership_changes.rs +++ b/tests/integration_cases/test_membership_changes.rs @@ -18,7 +18,7 @@ use raft::{ eraftpb::{ ConfChange, ConfChangeType, ConfState, Entry, EntryType, Message, MessageType, Snapshot, }, - storage::{MemStorage}, + storage::MemStorage, Config, Configuration, Raft, Result, INVALID_ID, NO_LIMIT, }; use std::ops::{Deref, DerefMut}; From bc66fd705c25b4dcc60acd97986d4e6ee900fb78 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Fri, 11 Jan 2019 15:23:49 -0800 Subject: [PATCH 27/41] Raft layer insert/remove/promote report errors Signed-off-by: Ana Hobden --- src/eraftpb.rs | 2 +- src/raft.rs | 18 +++-- src/raw_node.rs | 16 ++-- .../test_membership_changes.rs | 4 +- tests/integration_cases/test_raft.rs | 78 ++++++++++++------- tests/test_util/mod.rs | 2 +- 6 files changed, 77 insertions(+), 43 deletions(-) diff --git a/src/eraftpb.rs b/src/eraftpb.rs index 5b26ea094..bc9f381a2 100644 --- a/src/eraftpb.rs +++ b/src/eraftpb.rs @@ -3,7 +3,7 @@ // https://github.com/Manishearth/rust-clippy/issues/702 #![allow(unknown_lints)] -#![allow(clippy)] +#![allow(clippy::all)] #![cfg_attr(rustfmt, rustfmt_skip)] diff --git a/src/raft.rs b/src/raft.rs index 8100cf0cc..6a73175e3 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -2205,23 +2205,23 @@ impl Raft { /// Adds a new node to the cluster. // TODO: Return an error on a redundant insert. - pub fn add_node(&mut self, id: u64) { - self.add_voter_or_learner(id, false).ok(); + pub fn add_node(&mut self, id: u64) -> Result<()> { + self.add_voter_or_learner(id, false) } /// Adds a learner node. // TODO: Return an error on a redundant insert. - pub fn add_learner(&mut self, id: u64) { - self.add_voter_or_learner(id, true).ok(); + pub fn add_learner(&mut self, id: u64) -> Result<()> { + self.add_voter_or_learner(id, true) } /// Removes a node from the raft. - pub fn remove_node(&mut self, id: u64) { - self.mut_prs().remove(id).ok(); + pub fn remove_node(&mut self, id: u64) -> Result<()> { + self.mut_prs().remove(id)?; // do not try to commit or abort transferring if there are no nodes in the cluster. if self.prs().voter_ids().is_empty() && self.prs().learner_ids().is_empty() { - return; + return Ok(()); } // The quorum size is now smaller, so see if any pending entries can @@ -2231,8 +2231,10 @@ impl Raft { } // If the removed node is the lead_transferee, then abort the leadership transferring. if self.state == StateRole::Leader && self.lead_transferee == Some(id) { - self.abort_leader_transfer() + self.abort_leader_transfer(); } + + Ok(()) } /// Updates the progress of the learner or voter. diff --git a/src/raw_node.rs b/src/raw_node.rs index d9bb026c8..fbed17f4f 100644 --- a/src/raw_node.rs +++ b/src/raw_node.rs @@ -251,7 +251,7 @@ impl RawNode { rn.raft.raft_log.append(&ents); rn.raft.raft_log.committed = ents.len() as u64; for peer in peers { - rn.raft.add_node(peer.id); + rn.raft.add_node(peer.id)?; } } rn.prev_ss = rn.raft.soft_state(); @@ -335,7 +335,7 @@ impl RawNode { /// /// # Panics /// - /// In the case of `BeginMembershipChange` or `FinalizeMembershipChange` returning errors this will panic. + /// In the case of `BeginMembershipChange` or `FinalizeConfChange` returning errors this will panic. /// /// For a safe interface for these directly call `this.raft.begin_membership_change(entry)` or /// `this.raft.finalize_membership_change(entry)` respectively. @@ -349,15 +349,19 @@ impl RawNode { return cs; } let nid = cc.get_node_id(); - match cc.get_change_type() { + let result = match cc.get_change_type() { ConfChangeType::AddNode => self.raft.add_node(nid), ConfChangeType::AddLearnerNode => self.raft.add_learner(nid), ConfChangeType::RemoveNode => self.raft.remove_node(nid), - ConfChangeType::BeginMembershipChange => self.raft.begin_membership_change(cc).unwrap(), + ConfChangeType::BeginMembershipChange => self.raft.begin_membership_change(cc), ConfChangeType::FinalizeMembershipChange => { - self.raft.mut_prs().finalize_membership_change().unwrap(); + self.raft.mut_prs().finalize_membership_change() } - } + }; + + // TODO: Remove this. + result.ok(); + self.raft.prs().configuration().clone().into() } diff --git a/tests/integration_cases/test_membership_changes.rs b/tests/integration_cases/test_membership_changes.rs index 82dc258da..17a1ef5dd 100644 --- a/tests/integration_cases/test_membership_changes.rs +++ b/tests/integration_cases/test_membership_changes.rs @@ -1475,7 +1475,9 @@ impl Scenario { ConfChangeType::FinalizeMembershipChange => { peer.finalize_membership_change(&conf_change)? } - ConfChangeType::AddNode => peer.add_node(conf_change.get_node_id()), + ConfChangeType::AddNode => { + peer.add_node(conf_change.get_node_id())? + } _ => panic!("Unexpected conf change"), }; } diff --git a/tests/integration_cases/test_raft.rs b/tests/integration_cases/test_raft.rs index 1a8967a71..e5130057c 100644 --- a/tests/integration_cases/test_raft.rs +++ b/tests/integration_cases/test_raft.rs @@ -2950,18 +2950,21 @@ fn test_new_leader_pending_config() { // test_add_node tests that add_node could update nodes correctly. #[test] -fn test_add_node() { +fn test_add_node() -> Result<()> { setup_for_test(); + let mut r = new_test_raft(1, vec![1], 10, 1, new_storage()); - r.add_node(2); + r.add_node(2)?; assert_eq!( r.prs().voter_ids(), vec![1, 2].into_iter().collect::>() ); + + Ok(()) } #[test] -fn test_add_node_check_quorum() { +fn test_add_node_check_quorum() -> Result<()> { setup_for_test(); let mut r = new_test_raft(1, vec![1], 10, 1, new_storage()); r.check_quorum = true; @@ -2973,7 +2976,7 @@ fn test_add_node_check_quorum() { r.tick(); } - r.add_node(2); + r.add_node(2)?; // This tick will reach electionTimeout, which triggers a quorum check. r.tick(); @@ -2988,20 +2991,24 @@ fn test_add_node_check_quorum() { } assert_eq!(r.state, StateRole::Follower); + + Ok(()) } // test_remove_node tests that removeNode could update pendingConf, nodes and // and removed list correctly. #[test] -fn test_remove_node() { +fn test_remove_node() -> Result<()> { setup_for_test(); + let mut r = new_test_raft(1, vec![1, 2], 10, 1, new_storage()); - r.remove_node(2); + r.remove_node(2)?; assert_eq!(r.prs().voter_ids().iter().next().unwrap(), &1); - // remove all nodes from cluster - r.remove_node(1); + r.remove_node(1)?; assert!(r.prs().voter_ids().is_empty()); + + Ok(()) } #[test] @@ -3067,8 +3074,9 @@ fn test_campaign_while_leader_with_pre_vote(pre_vote: bool) { // test_commit_after_remove_node verifies that pending commands can become // committed when a config change reduces the quorum requirements. #[test] -fn test_commit_after_remove_node() { +fn test_commit_after_remove_node() -> Result<()> { setup_for_test(); + // Create a cluster with two nodes. let s = new_storage(); let mut r = new_test_raft(1, vec![1, 2], 5, 1, s.clone()); @@ -3108,11 +3116,13 @@ fn test_commit_after_remove_node() { // Apply the config change. This reduces quorum requirements so the // pending command can now commit. - r.remove_node(2); + r.remove_node(2)?; let ents = next_ents(&mut r, &s); assert_eq!(ents.len(), 1); assert_eq!(ents[0].get_entry_type(), EntryType::EntryNormal); assert_eq!(ents[0].get_data(), b"hello"); + + Ok(()) } // test_leader_transfer_to_uptodate_node verifies transferring should succeed @@ -3346,7 +3356,7 @@ fn test_leader_transfer_receive_higher_term_vote() { } #[test] -fn test_leader_transfer_remove_node() { +fn test_leader_transfer_remove_node() -> Result<()> { setup_for_test(); let mut nt = Network::new(vec![None, None, None]); nt.send(vec![new_message(1, 1, MessageType::MsgHup, 0)]); @@ -3357,9 +3367,11 @@ fn test_leader_transfer_remove_node() { nt.send(vec![new_message(3, 1, MessageType::MsgTransferLeader, 0)]); assert_eq!(nt.peers[&1].lead_transferee.unwrap(), 3); - nt.peers.get_mut(&1).unwrap().remove_node(3); + nt.peers.get_mut(&1).unwrap().remove_node(3)?; check_leader_transfer_state(&nt.peers[&1], StateRole::Leader, 1); + + Ok(()) } // test_leader_transfer_back verifies leadership can transfer @@ -3558,8 +3570,9 @@ fn test_learner_election_timeout() { // TestLearnerPromotion verifies that the leaner should not election until // it is promoted to a normal peer. #[test] -fn test_learner_promotion() { +fn test_learner_promotion() -> Result<()> { setup_for_test(); + let mut n1 = new_test_learner_raft(1, vec![1], vec![2], 10, 1, new_storage()); n1.become_follower(1, INVALID_ID); @@ -3586,8 +3599,8 @@ fn test_learner_promotion() { network.send(vec![heart_beat.clone()]); // Promote n2 from learner to follower. - network.peers.get_mut(&1).unwrap().add_node(2); - network.peers.get_mut(&2).unwrap().add_node(2); + network.peers.get_mut(&1).unwrap().add_node(2)?; + network.peers.get_mut(&2).unwrap().add_node(2)?; assert_eq!(network.peers[&2].state, StateRole::Follower); assert!(!network.peers[&2].is_learner); @@ -3606,6 +3619,8 @@ fn test_learner_promotion() { network.send(vec![heart_beat]); assert_eq!(network.peers[&1].state, StateRole::Follower); assert_eq!(network.peers[&2].state, StateRole::Leader); + + Ok(()) } // TestLearnerLogReplication tests that a learner can receive entries from the leader. @@ -3757,44 +3772,52 @@ fn test_learner_receive_snapshot() { // TestAddLearner tests that addLearner could update nodes correctly. #[test] -fn test_add_learner() { +fn test_add_learner() -> Result<()> { setup_for_test(); let mut n1 = new_test_raft(1, vec![1], 10, 1, new_storage()); - n1.add_learner(2); + n1.add_learner(2)?; assert_eq!(*n1.prs().learner_ids().iter().next().unwrap(), 2); assert!(n1.prs().learner_ids().contains(&2)); + + Ok(()) } // Ensure when add_voter is called on a peers own ID that it will be promoted. // When the action fails, ensure it doesn't mutate the raft state. #[test] -fn test_add_voter_peer_promotes_self_sets_is_learner() { +fn test_add_voter_peer_promotes_self_sets_is_learner() -> Result<()> { setup_for_test(); + let mut n1 = new_test_raft(1, vec![1], 10, 1, new_storage()); // Node is already voter. - n1.add_learner(1); + n1.add_learner(1).ok(); assert_eq!(n1.is_learner, false); assert!(n1.prs().voter_ids().contains(&1)); - n1.remove_node(1); - n1.add_learner(1); + n1.remove_node(1)?; + n1.add_learner(1)?; assert_eq!(n1.is_learner, true); assert!(n1.prs().learner_ids().contains(&1)); + + Ok(()) } // TestRemoveLearner tests that removeNode could update nodes and // and removed list correctly. #[test] -fn test_remove_learner() { +fn test_remove_learner() -> Result<()> { setup_for_test(); + let mut n1 = new_test_learner_raft(1, vec![1], vec![2], 10, 1, new_storage()); - n1.remove_node(2); + n1.remove_node(2)?; assert_eq!(n1.prs().voter_ids().iter().next().unwrap(), &1); assert!(n1.prs().learner_ids().is_empty()); - n1.remove_node(1); + n1.remove_node(1)?; assert!(n1.prs().voter_ids().is_empty()); assert_eq!(n1.prs().learner_ids().len(), 0); + + Ok(()) } // simulate rolling update a cluster for Pre-Vote. cluster has 3 nodes [n1, n2, n3]. @@ -3908,8 +3931,9 @@ fn test_prevote_migration_with_free_stuck_pre_candidate() { } #[test] -fn test_learner_respond_vote() { +fn test_learner_respond_vote() -> Result<()> { setup_for_test(); + let mut n1 = new_test_learner_raft(1, vec![1, 2], vec![3], 10, 1, new_storage()); n1.become_follower(1, INVALID_ID); n1.reset_randomized_election_timeout(); @@ -3931,9 +3955,11 @@ fn test_learner_respond_vote() { assert_eq!(network.peers[&1].state, StateRole::Candidate); // After promote 3 to voter, election should success. - network.peers.get_mut(&1).unwrap().add_node(3); + network.peers.get_mut(&1).unwrap().add_node(3)?; do_campaign(&mut network); assert_eq!(network.peers[&1].state, StateRole::Leader); + + Ok(()) } #[test] diff --git a/tests/test_util/mod.rs b/tests/test_util/mod.rs index b6632ee84..1e21cf66e 100644 --- a/tests/test_util/mod.rs +++ b/tests/test_util/mod.rs @@ -228,7 +228,7 @@ struct Connem { to: u64, } -#[allow(declare_interior_mutable_const)] +#[allow(clippy::declare_interior_mutable_const)] pub const NOP_STEPPER: Option = Some(Interface { raft: None }); #[derive(Default)] From 74c77aa7cd1f13c64043c4299646563bc6f196d1 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Fri, 11 Jan 2019 15:28:21 -0800 Subject: [PATCH 28/41] apply_conf_change in rawnode gives errors --- src/raw_node.rs | 23 +++++++++-------------- tests/integration_cases/test_raw_node.rs | 6 ++++-- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/src/raw_node.rs b/src/raw_node.rs index fbed17f4f..d817db6e3 100644 --- a/src/raw_node.rs +++ b/src/raw_node.rs @@ -339,30 +339,25 @@ impl RawNode { /// /// For a safe interface for these directly call `this.raft.begin_membership_change(entry)` or /// `this.raft.finalize_membership_change(entry)` respectively. - pub fn apply_conf_change(&mut self, cc: &ConfChange) -> ConfState { + pub fn apply_conf_change(&mut self, cc: &ConfChange) -> Result { if cc.get_node_id() == INVALID_ID && cc.get_change_type() != ConfChangeType::BeginMembershipChange { let mut cs = ConfState::new(); cs.set_nodes(self.raft.prs().voter_ids().iter().cloned().collect()); cs.set_learners(self.raft.prs().learner_ids().iter().cloned().collect()); - return cs; + return Ok(cs); } let nid = cc.get_node_id(); - let result = match cc.get_change_type() { - ConfChangeType::AddNode => self.raft.add_node(nid), - ConfChangeType::AddLearnerNode => self.raft.add_learner(nid), - ConfChangeType::RemoveNode => self.raft.remove_node(nid), - ConfChangeType::BeginMembershipChange => self.raft.begin_membership_change(cc), - ConfChangeType::FinalizeMembershipChange => { - self.raft.mut_prs().finalize_membership_change() - } + match cc.get_change_type() { + ConfChangeType::AddNode => self.raft.add_node(nid)?, + ConfChangeType::AddLearnerNode => self.raft.add_learner(nid)?, + ConfChangeType::RemoveNode => self.raft.remove_node(nid)?, + ConfChangeType::BeginMembershipChange => self.raft.begin_membership_change(cc)?, + ConfChangeType::FinalizeMembershipChange => self.raft.mut_prs().finalize_membership_change()?, }; - // TODO: Remove this. - result.ok(); - - self.raft.prs().configuration().clone().into() + Ok(self.raft.prs().configuration().clone().into()) } /// Step advances the state machine using the given message. diff --git a/tests/integration_cases/test_raw_node.rs b/tests/integration_cases/test_raw_node.rs index f75c251e0..a27584728 100644 --- a/tests/integration_cases/test_raw_node.rs +++ b/tests/integration_cases/test_raw_node.rs @@ -274,7 +274,7 @@ fn test_raw_node_propose_add_duplicate_node() { } #[test] -fn test_raw_node_propose_add_learner_node() { +fn test_raw_node_propose_add_learner_node() -> Result<()> { setup_for_test(); let s = new_storage(); let mut raw_node = new_raw_node(1, vec![], 10, 1, s.clone(), vec![new_peer(1)]); @@ -307,9 +307,11 @@ fn test_raw_node_propose_add_learner_node() { let e = &rd.committed_entries.as_ref().unwrap()[0]; let conf_change = protobuf::parse_from_bytes(e.get_data()).unwrap(); - let conf_state = raw_node.apply_conf_change(&conf_change); + let conf_state = raw_node.apply_conf_change(&conf_change)?; assert_eq!(conf_state.nodes, vec![1]); assert_eq!(conf_state.learners, vec![2]); + + Ok(()) } // test_raw_node_read_index ensures that RawNode.read_index sends the MsgReadIndex message From 3e7ed81a2bd021d2cd22eb9397b457a6197a2d25 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Fri, 11 Jan 2019 15:42:26 -0800 Subject: [PATCH 29/41] Contracts -> Errors --- src/progress.rs | 20 ++++++++++---------- src/raft.rs | 44 ++++++++++++++++++++++++++++++++------------ 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/src/progress.rs b/src/progress.rs index 361738517..12c13e854 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -309,11 +309,11 @@ impl ProgressSet { /// Adds a voter to the group. /// - /// # Contracts + /// # Errors /// - /// * The `id` does not exist in the voter set. - /// * The `id` does not exist in the learner set. - /// * There is no pending membership change. + /// * `id` is in the voter set. + /// * `id` is in the learner set. + /// * There is a pending membership change. pub fn insert_voter(&mut self, id: u64, pr: Progress) -> Result<()> { debug!("Inserting voter with id {}.", id); @@ -336,11 +336,11 @@ impl ProgressSet { /// Adds a learner to the group. /// - /// # Contracts + /// # Errors /// - /// * The `id` does not exist in the voter set. - /// * The `id` does not exist in the learner set. - /// * There is no pending membership change. + /// * `id` is in the voter set. + /// * `id` is in the learner set. + /// * There is a pending membership change. pub fn insert_learner(&mut self, id: u64, pr: Progress) -> Result<()> { debug!("Inserting learner with id {}.", id); @@ -363,9 +363,9 @@ impl ProgressSet { /// Removes the peer from the set of voters or learners. /// - /// # Contracts + /// # Errors /// - /// * There is no pending membership change. + /// * There is a pending membership change. pub fn remove(&mut self, id: u64) -> Result> { debug!("Removing peer with id {}.", id); diff --git a/src/raft.rs b/src/raft.rs index 6a73175e3..3d22de2bb 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -1199,11 +1199,11 @@ impl Raft { /// /// We apply the change when a node *applies* the entry, not when the entry is received. /// - /// # Contracts + /// # Errors /// - /// * The `ConfChange.change_type` must be a `BeginMembershipChange` - /// * The `ConfChange.configuration` value must exist. - /// * The `ConfChange.start_index` value must exist. It should equal the index of the + /// * `ConfChange.change_type` is not `BeginMembershipChange` + /// * `ConfChange.configuration` does not exist. + /// * `ConfChange.start_index` does not exist. It **must** equal the index of the /// corresponding entry. #[inline(always)] pub fn begin_membership_change(&mut self, conf_change: &ConfChange) -> Result<()> { @@ -1248,12 +1248,12 @@ impl Raft { /// /// We apply the change when a node *applies* the entry, not when the entry is received. /// - /// # Contracts + /// # Errors /// - /// * The Raft should already have started a configuration change with `begin_membership_change`. - /// * The `ConfChange.change_type` must be a `FinalizeMembershipChange`. - /// * The `ConfChange.configuration` value should not exist. (Panics in debug mode.) - /// * The `ConfChange.start_index` value should not exist. (Panics in debug mode.) + /// * This Raft is not in a configuration change via `begin_membership_change`. + /// * `ConfChange.change_type` is not a `FinalizeMembershipChange`. + /// * `ConfChange.configuration` value should not exist. + /// * `ConfChange.start_index` value should not exist. #[inline(always)] pub fn finalize_membership_change(&mut self, conf_change: &ConfChange) -> Result<()> { if conf_change.get_change_type() != ConfChangeType::FinalizeMembershipChange { @@ -2139,7 +2139,7 @@ impl Raft { /// /// # Errors /// - /// * Peer this is called on is not leader. + /// * This Peer is not leader. /// * `voters` and `learners` are not mutually exclusive. /// * `voters` is empty. pub fn propose_membership_change(&mut self, config: impl Into) -> Result<()> { @@ -2173,6 +2173,11 @@ impl Raft { Ok(()) } + /// # Errors + /// + /// * `id` is already a voter. + /// * `id` is already a learner. + /// * There is a pending membership change. (See `is_in_membership_change()`) fn add_voter_or_learner(&mut self, id: u64, learner: bool) -> Result<()> { debug!( "Adding node (learner: {}) with ID {} to peers.", @@ -2204,18 +2209,33 @@ impl Raft { } /// Adds a new node to the cluster. - // TODO: Return an error on a redundant insert. + /// + /// # Errors + /// + /// * `id` is already a voter. + /// * `id` is already a learner. + /// * There is a pending membership change. (See `is_in_membership_change()`) pub fn add_node(&mut self, id: u64) -> Result<()> { self.add_voter_or_learner(id, false) } /// Adds a learner node. - // TODO: Return an error on a redundant insert. + /// + /// # Errors + /// + /// * `id` is already a voter. + /// * `id` is already a learner. + /// * There is a pending membership change. (See `is_in_membership_change()`) pub fn add_learner(&mut self, id: u64) -> Result<()> { self.add_voter_or_learner(id, true) } /// Removes a node from the raft. + /// + /// # Errors + /// + /// * `id` is not a voter or learner. + /// * There is a pending membership change. (See `is_in_membership_change()`) pub fn remove_node(&mut self, id: u64) -> Result<()> { self.mut_prs().remove(id)?; From 9d9a89260478c8d65cce2e24aeedba1c1c78e647 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Fri, 11 Jan 2019 15:43:52 -0800 Subject: [PATCH 30/41] Fix test_raw_node_propose_add_duplicate_node test --- tests/integration_cases/test_raw_node.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_cases/test_raw_node.rs b/tests/integration_cases/test_raw_node.rs index a27584728..abe8503c8 100644 --- a/tests/integration_cases/test_raw_node.rs +++ b/tests/integration_cases/test_raw_node.rs @@ -246,7 +246,7 @@ fn test_raw_node_propose_add_duplicate_node() { for e in rd.committed_entries.as_ref().unwrap() { if e.get_entry_type() == EntryType::EntryConfChange { let conf_change = protobuf::parse_from_bytes(e.get_data()).unwrap(); - raw_node.apply_conf_change(&conf_change); + raw_node.apply_conf_change(&conf_change).ok(); } } raw_node.advance(rd); From 115f72b0465eecb1abbbf425d6a15df6818bfa49 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Mon, 14 Jan 2019 12:42:12 -0800 Subject: [PATCH 31/41] fmt Signed-off-by: Ana Hobden --- src/raft.rs | 14 +++++++------- src/raw_node.rs | 4 +++- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/raft.rs b/src/raft.rs index 3d22de2bb..9a0658825 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -2174,7 +2174,7 @@ impl Raft { } /// # Errors - /// + /// /// * `id` is already a voter. /// * `id` is already a learner. /// * There is a pending membership change. (See `is_in_membership_change()`) @@ -2209,9 +2209,9 @@ impl Raft { } /// Adds a new node to the cluster. - /// + /// /// # Errors - /// + /// /// * `id` is already a voter. /// * `id` is already a learner. /// * There is a pending membership change. (See `is_in_membership_change()`) @@ -2220,9 +2220,9 @@ impl Raft { } /// Adds a learner node. - /// + /// /// # Errors - /// + /// /// * `id` is already a voter. /// * `id` is already a learner. /// * There is a pending membership change. (See `is_in_membership_change()`) @@ -2231,9 +2231,9 @@ impl Raft { } /// Removes a node from the raft. - /// + /// /// # Errors - /// + /// /// * `id` is not a voter or learner. /// * There is a pending membership change. (See `is_in_membership_change()`) pub fn remove_node(&mut self, id: u64) -> Result<()> { diff --git a/src/raw_node.rs b/src/raw_node.rs index d817db6e3..f95b4b0a3 100644 --- a/src/raw_node.rs +++ b/src/raw_node.rs @@ -354,7 +354,9 @@ impl RawNode { ConfChangeType::AddLearnerNode => self.raft.add_learner(nid)?, ConfChangeType::RemoveNode => self.raft.remove_node(nid)?, ConfChangeType::BeginMembershipChange => self.raft.begin_membership_change(cc)?, - ConfChangeType::FinalizeMembershipChange => self.raft.mut_prs().finalize_membership_change()?, + ConfChangeType::FinalizeMembershipChange => { + self.raft.mut_prs().finalize_membership_change()? + } }; Ok(self.raft.prs().configuration().clone().into()) From 4d6ee8116dfcc6eb8ff010eef9b23a487e0a905f Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Mon, 14 Jan 2019 13:05:50 -0800 Subject: [PATCH 32/41] lints --- benches/suites/raft.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benches/suites/raft.rs b/benches/suites/raft.rs index 934d8a795..67214f30e 100644 --- a/benches/suites/raft.rs +++ b/benches/suites/raft.rs @@ -13,10 +13,10 @@ fn quick_raft(voters: usize, learners: usize) -> Raft { let config = Config::new(id); let mut raft = Raft::new(&config, storage).unwrap(); (0..voters).for_each(|id| { - raft.add_node(id as u64); + raft.add_node(id as u64).unwrap(); }); (voters..learners).for_each(|id| { - raft.add_learner(id as u64); + raft.add_learner(id as u64).unwrap(); }); raft } From 15f19ab09bc342fb25c8ca9eff95cd40896ee859 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Thu, 17 Jan 2019 21:23:29 -0800 Subject: [PATCH 33/41] contains doesn't borrow --- src/progress.rs | 6 ++---- src/raft.rs | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/progress.rs b/src/progress.rs index 12c13e854..9fefc71ea 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -141,10 +141,8 @@ impl Configuration { } /// Returns whether or not the given `id` is a member of this configuration. - // Allowed to maintain API consistency. See `HashSet::contains(&u64)`. - #[allow(clippy::trivially_copy_pass_by_ref)] - pub fn contains(&self, id: &u64) -> bool { - self.voters.contains(id) || self.learners.contains(id) + pub fn contains(&self, id: u64) -> bool { + self.voters.contains(&id) || self.learners.contains(&id) } } diff --git a/src/raft.rs b/src/raft.rs index 9a0658825..daceae575 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -1271,7 +1271,7 @@ impl Raft { .prs() .next_configuration() .as_ref() - .map(|config| config.contains(&self.leader_id)) + .map(|config| config.contains(self.leader_id)) .ok_or_else(|| Error::NoPendingMembershipChange)?; // Joint Consensus, in the Raft paper, states the leader should step down and become a From 99f47e60215740861f1e6cb3226ebefcd9b3bed4 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Thu, 17 Jan 2019 21:57:48 -0800 Subject: [PATCH 34/41] Ensure benches aren't noops Signed-off-by: Ana Hobden --- benches/suites/progress_set.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/benches/suites/progress_set.rs b/benches/suites/progress_set.rs index b6e23b200..60e15fb5d 100644 --- a/benches/suites/progress_set.rs +++ b/benches/suites/progress_set.rs @@ -153,7 +153,8 @@ pub fn bench_progress_set_voters(c: &mut Criterion) { let set = quick_progress_set(voters, learners); b.iter(|| { let set = set.clone(); - set.voters().for_each(|_| {}); + let sum = set.voters().fold(0, |mut sum, _| { sum += 1; sum }); + sum }); } }; @@ -172,7 +173,8 @@ pub fn bench_progress_set_learners(c: &mut Criterion) { let set = quick_progress_set(voters, learners); b.iter(|| { let set = set.clone(); - set.voters().for_each(|_| {}); + let sum = set.voters().fold(0, |mut sum, _| { sum += 1; sum }); + sum }); } }; From dfea37358756c03abd55d49430b65369fc403cac Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Mon, 28 Jan 2019 02:39:40 -0800 Subject: [PATCH 35/41] Reflect some of @qupeng's feedback. One test still fails. --- proto/eraftpb.proto | 5 +- src/eraftpb.rs | 514 +++++++++--------- src/lib.rs | 2 +- src/progress.rs | 6 +- src/raft.rs | 52 +- src/raft_log.rs | 2 +- src/storage.rs | 28 +- src/util.rs | 12 +- .../test_membership_changes.rs | 2 + 9 files changed, 320 insertions(+), 303 deletions(-) diff --git a/proto/eraftpb.proto b/proto/eraftpb.proto index 58b5db59e..664a4fea7 100644 --- a/proto/eraftpb.proto +++ b/proto/eraftpb.proto @@ -30,7 +30,8 @@ message Entry { message SnapshotMetadata { ConfState conf_state = 1; - ConfChange pending_membership_change = 4; + ConfState pending_membership_change = 4; + uint64 pending_membership_change_index = 5; uint64 index = 2; uint64 term = 3; } @@ -81,8 +82,6 @@ message HardState { uint64 term = 1; uint64 vote = 2; uint64 commit = 3; - // If the peer goes down we need to be aware of a partially completed membership change. - ConfChange pending_membership_change = 4; } message ConfState { diff --git a/src/eraftpb.rs b/src/eraftpb.rs index bc9f381a2..1f3c36de5 100644 --- a/src/eraftpb.rs +++ b/src/eraftpb.rs @@ -1,9 +1,9 @@ -// This file is generated by rust-protobuf 2.2.2. Do not edit +// This file is generated by rust-protobuf 2.2.4. Do not edit // @generated // https://github.com/Manishearth/rust-clippy/issues/702 #![allow(unknown_lints)] -#![allow(clippy::all)] +#![allow(clippy)] #![cfg_attr(rustfmt, rustfmt_skip)] @@ -366,7 +366,8 @@ impl ::protobuf::reflect::ProtobufValue for Entry { pub struct SnapshotMetadata { // message fields pub conf_state: ::protobuf::SingularPtrField, - pub pending_membership_change: ::protobuf::SingularPtrField, + pub pending_membership_change: ::protobuf::SingularPtrField, + pub pending_membership_change_index: u64, pub index: u64, pub term: u64, // special fields @@ -412,7 +413,7 @@ impl SnapshotMetadata { self.conf_state.as_ref().unwrap_or_else(|| ConfState::default_instance()) } - // .eraftpb.ConfChange pending_membership_change = 4; + // .eraftpb.ConfState pending_membership_change = 4; pub fn clear_pending_membership_change(&mut self) { self.pending_membership_change.clear(); @@ -423,13 +424,13 @@ impl SnapshotMetadata { } // Param is passed by value, moved - pub fn set_pending_membership_change(&mut self, v: ConfChange) { + pub fn set_pending_membership_change(&mut self, v: ConfState) { self.pending_membership_change = ::protobuf::SingularPtrField::some(v); } // Mutable pointer to the field. // If field is not initialized, it is initialized with default value first. - pub fn mut_pending_membership_change(&mut self) -> &mut ConfChange { + pub fn mut_pending_membership_change(&mut self) -> &mut ConfState { if self.pending_membership_change.is_none() { self.pending_membership_change.set_default(); } @@ -437,12 +438,27 @@ impl SnapshotMetadata { } // Take field - pub fn take_pending_membership_change(&mut self) -> ConfChange { - self.pending_membership_change.take().unwrap_or_else(|| ConfChange::new()) + pub fn take_pending_membership_change(&mut self) -> ConfState { + self.pending_membership_change.take().unwrap_or_else(|| ConfState::new()) } - pub fn get_pending_membership_change(&self) -> &ConfChange { - self.pending_membership_change.as_ref().unwrap_or_else(|| ConfChange::default_instance()) + pub fn get_pending_membership_change(&self) -> &ConfState { + self.pending_membership_change.as_ref().unwrap_or_else(|| ConfState::default_instance()) + } + + // uint64 pending_membership_change_index = 5; + + pub fn clear_pending_membership_change_index(&mut self) { + self.pending_membership_change_index = 0; + } + + // Param is passed by value, moved + pub fn set_pending_membership_change_index(&mut self, v: u64) { + self.pending_membership_change_index = v; + } + + pub fn get_pending_membership_change_index(&self) -> u64 { + self.pending_membership_change_index } // uint64 index = 2; @@ -501,6 +517,13 @@ impl ::protobuf::Message for SnapshotMetadata { 4 => { ::protobuf::rt::read_singular_message_into(wire_type, is, &mut self.pending_membership_change)?; }, + 5 => { + if wire_type != ::protobuf::wire_format::WireTypeVarint { + return ::std::result::Result::Err(::protobuf::rt::unexpected_wire_type(wire_type)); + } + let tmp = is.read_uint64()?; + self.pending_membership_change_index = tmp; + }, 2 => { if wire_type != ::protobuf::wire_format::WireTypeVarint { return ::std::result::Result::Err(::protobuf::rt::unexpected_wire_type(wire_type)); @@ -535,6 +558,9 @@ impl ::protobuf::Message for SnapshotMetadata { let len = v.compute_size(); my_size += 1 + ::protobuf::rt::compute_raw_varint32_size(len) + len; } + if self.pending_membership_change_index != 0 { + my_size += ::protobuf::rt::value_size(5, self.pending_membership_change_index, ::protobuf::wire_format::WireTypeVarint); + } if self.index != 0 { my_size += ::protobuf::rt::value_size(2, self.index, ::protobuf::wire_format::WireTypeVarint); } @@ -557,6 +583,9 @@ impl ::protobuf::Message for SnapshotMetadata { os.write_raw_varint32(v.get_cached_size())?; v.write_to_with_cached_sizes(os)?; } + if self.pending_membership_change_index != 0 { + os.write_uint64(5, self.pending_membership_change_index)?; + } if self.index != 0 { os.write_uint64(2, self.index)?; } @@ -610,11 +639,16 @@ impl ::protobuf::Message for SnapshotMetadata { |m: &SnapshotMetadata| { &m.conf_state }, |m: &mut SnapshotMetadata| { &mut m.conf_state }, )); - fields.push(::protobuf::reflect::accessor::make_singular_ptr_field_accessor::<_, ::protobuf::types::ProtobufTypeMessage>( + fields.push(::protobuf::reflect::accessor::make_singular_ptr_field_accessor::<_, ::protobuf::types::ProtobufTypeMessage>( "pending_membership_change", |m: &SnapshotMetadata| { &m.pending_membership_change }, |m: &mut SnapshotMetadata| { &mut m.pending_membership_change }, )); + fields.push(::protobuf::reflect::accessor::make_simple_field_accessor::<_, ::protobuf::types::ProtobufTypeUint64>( + "pending_membership_change_index", + |m: &SnapshotMetadata| { &m.pending_membership_change_index }, + |m: &mut SnapshotMetadata| { &mut m.pending_membership_change_index }, + )); fields.push(::protobuf::reflect::accessor::make_simple_field_accessor::<_, ::protobuf::types::ProtobufTypeUint64>( "index", |m: &SnapshotMetadata| { &m.index }, @@ -649,6 +683,7 @@ impl ::protobuf::Clear for SnapshotMetadata { fn clear(&mut self) { self.clear_conf_state(); self.clear_pending_membership_change(); + self.clear_pending_membership_change_index(); self.clear_index(); self.clear_term(); self.unknown_fields.clear(); @@ -1473,7 +1508,6 @@ pub struct HardState { pub term: u64, pub vote: u64, pub commit: u64, - pub pending_membership_change: ::protobuf::SingularPtrField, // special fields pub unknown_fields: ::protobuf::UnknownFields, pub cached_size: ::protobuf::CachedSize, @@ -1528,48 +1562,10 @@ impl HardState { pub fn get_commit(&self) -> u64 { self.commit } - - // .eraftpb.ConfChange pending_membership_change = 4; - - pub fn clear_pending_membership_change(&mut self) { - self.pending_membership_change.clear(); - } - - pub fn has_pending_membership_change(&self) -> bool { - self.pending_membership_change.is_some() - } - - // Param is passed by value, moved - pub fn set_pending_membership_change(&mut self, v: ConfChange) { - self.pending_membership_change = ::protobuf::SingularPtrField::some(v); - } - - // Mutable pointer to the field. - // If field is not initialized, it is initialized with default value first. - pub fn mut_pending_membership_change(&mut self) -> &mut ConfChange { - if self.pending_membership_change.is_none() { - self.pending_membership_change.set_default(); - } - self.pending_membership_change.as_mut().unwrap() - } - - // Take field - pub fn take_pending_membership_change(&mut self) -> ConfChange { - self.pending_membership_change.take().unwrap_or_else(|| ConfChange::new()) - } - - pub fn get_pending_membership_change(&self) -> &ConfChange { - self.pending_membership_change.as_ref().unwrap_or_else(|| ConfChange::default_instance()) - } } impl ::protobuf::Message for HardState { fn is_initialized(&self) -> bool { - for v in &self.pending_membership_change { - if !v.is_initialized() { - return false; - } - }; true } @@ -1598,9 +1594,6 @@ impl ::protobuf::Message for HardState { let tmp = is.read_uint64()?; self.commit = tmp; }, - 4 => { - ::protobuf::rt::read_singular_message_into(wire_type, is, &mut self.pending_membership_change)?; - }, _ => { ::protobuf::rt::read_unknown_or_skip_group(field_number, wire_type, is, self.mut_unknown_fields())?; }, @@ -1622,10 +1615,6 @@ impl ::protobuf::Message for HardState { if self.commit != 0 { my_size += ::protobuf::rt::value_size(3, self.commit, ::protobuf::wire_format::WireTypeVarint); } - if let Some(ref v) = self.pending_membership_change.as_ref() { - let len = v.compute_size(); - my_size += 1 + ::protobuf::rt::compute_raw_varint32_size(len) + len; - } my_size += ::protobuf::rt::unknown_fields_size(self.get_unknown_fields()); self.cached_size.set(my_size); my_size @@ -1641,11 +1630,6 @@ impl ::protobuf::Message for HardState { if self.commit != 0 { os.write_uint64(3, self.commit)?; } - if let Some(ref v) = self.pending_membership_change.as_ref() { - os.write_tag(4, ::protobuf::wire_format::WireTypeLengthDelimited)?; - os.write_raw_varint32(v.get_cached_size())?; - v.write_to_with_cached_sizes(os)?; - } os.write_unknown_fields(self.get_unknown_fields())?; ::std::result::Result::Ok(()) } @@ -1703,11 +1687,6 @@ impl ::protobuf::Message for HardState { |m: &HardState| { &m.commit }, |m: &mut HardState| { &mut m.commit }, )); - fields.push(::protobuf::reflect::accessor::make_singular_ptr_field_accessor::<_, ::protobuf::types::ProtobufTypeMessage>( - "pending_membership_change", - |m: &HardState| { &m.pending_membership_change }, - |m: &mut HardState| { &mut m.pending_membership_change }, - )); ::protobuf::reflect::MessageDescriptor::new::( "HardState", fields, @@ -1733,7 +1712,6 @@ impl ::protobuf::Clear for HardState { self.clear_term(); self.clear_vote(); self.clear_commit(); - self.clear_pending_membership_change(); self.unknown_fields.clear(); } } @@ -2540,52 +2518,52 @@ static file_descriptor_proto_data: &'static [u8] = b"\ \n\x04term\x18\x02\x20\x01(\x04R\x04term\x12\x14\n\x05index\x18\x03\x20\ \x01(\x04R\x05index\x12\x12\n\x04data\x18\x04\x20\x01(\x0cR\x04data\x12\ \x18\n\x07context\x18\x06\x20\x01(\x0cR\x07context\x12\x19\n\x08sync_log\ - \x18\x05\x20\x01(\x08R\x07syncLog\"\xc0\x01\n\x10SnapshotMetadata\x121\n\ + \x18\x05\x20\x01(\x08R\x07syncLog\"\x86\x02\n\x10SnapshotMetadata\x121\n\ \nconf_state\x18\x01\x20\x01(\x0b2\x12.eraftpb.ConfStateR\tconfState\x12\ - O\n\x19pending_membership_change\x18\x04\x20\x01(\x0b2\x13.eraftpb.ConfC\ - hangeR\x17pendingMembershipChange\x12\x14\n\x05index\x18\x02\x20\x01(\ - \x04R\x05index\x12\x12\n\x04term\x18\x03\x20\x01(\x04R\x04term\"U\n\x08S\ - napshot\x12\x12\n\x04data\x18\x01\x20\x01(\x0cR\x04data\x125\n\x08metada\ - ta\x18\x02\x20\x01(\x0b2\x19.eraftpb.SnapshotMetadataR\x08metadata\"\xe7\ - \x02\n\x07Message\x12/\n\x08msg_type\x18\x01\x20\x01(\x0e2\x14.eraftpb.M\ - essageTypeR\x07msgType\x12\x0e\n\x02to\x18\x02\x20\x01(\x04R\x02to\x12\ - \x12\n\x04from\x18\x03\x20\x01(\x04R\x04from\x12\x12\n\x04term\x18\x04\ - \x20\x01(\x04R\x04term\x12\x19\n\x08log_term\x18\x05\x20\x01(\x04R\x07lo\ - gTerm\x12\x14\n\x05index\x18\x06\x20\x01(\x04R\x05index\x12(\n\x07entrie\ - s\x18\x07\x20\x03(\x0b2\x0e.eraftpb.EntryR\x07entries\x12\x16\n\x06commi\ - t\x18\x08\x20\x01(\x04R\x06commit\x12-\n\x08snapshot\x18\t\x20\x01(\x0b2\ - \x11.eraftpb.SnapshotR\x08snapshot\x12\x16\n\x06reject\x18\n\x20\x01(\ - \x08R\x06reject\x12\x1f\n\x0breject_hint\x18\x0b\x20\x01(\x04R\nrejectHi\ - nt\x12\x18\n\x07context\x18\x0c\x20\x01(\x0cR\x07context\"\x9c\x01\n\tHa\ - rdState\x12\x12\n\x04term\x18\x01\x20\x01(\x04R\x04term\x12\x12\n\x04vot\ - e\x18\x02\x20\x01(\x04R\x04vote\x12\x16\n\x06commit\x18\x03\x20\x01(\x04\ - R\x06commit\x12O\n\x19pending_membership_change\x18\x04\x20\x01(\x0b2\ - \x13.eraftpb.ConfChangeR\x17pendingMembershipChange\"=\n\tConfState\x12\ - \x14\n\x05nodes\x18\x01\x20\x03(\x04R\x05nodes\x12\x1a\n\x08learners\x18\ - \x02\x20\x03(\x04R\x08learners\"\xe4\x01\n\nConfChange\x12\x0e\n\x02id\ - \x18\x01\x20\x01(\x04R\x02id\x128\n\x0bchange_type\x18\x02\x20\x01(\x0e2\ - \x17.eraftpb.ConfChangeTypeR\nchangeType\x12\x17\n\x07node_id\x18\x03\ - \x20\x01(\x04R\x06nodeId\x12\x18\n\x07context\x18\x04\x20\x01(\x0cR\x07c\ - ontext\x128\n\rconfiguration\x18\x05\x20\x01(\x0b2\x12.eraftpb.ConfState\ - R\rconfiguration\x12\x1f\n\x0bstart_index\x18\x06\x20\x01(\x04R\nstartIn\ - dex*1\n\tEntryType\x12\x0f\n\x0bEntryNormal\x10\0\x12\x13\n\x0fEntryConf\ - Change\x10\x01*\x8c\x03\n\x0bMessageType\x12\n\n\x06MsgHup\x10\0\x12\x0b\ - \n\x07MsgBeat\x10\x01\x12\x0e\n\nMsgPropose\x10\x02\x12\r\n\tMsgAppend\ - \x10\x03\x12\x15\n\x11MsgAppendResponse\x10\x04\x12\x12\n\x0eMsgRequestV\ - ote\x10\x05\x12\x1a\n\x16MsgRequestVoteResponse\x10\x06\x12\x0f\n\x0bMsg\ - Snapshot\x10\x07\x12\x10\n\x0cMsgHeartbeat\x10\x08\x12\x18\n\x14MsgHeart\ - beatResponse\x10\t\x12\x12\n\x0eMsgUnreachable\x10\n\x12\x11\n\rMsgSnapS\ - tatus\x10\x0b\x12\x12\n\x0eMsgCheckQuorum\x10\x0c\x12\x15\n\x11MsgTransf\ - erLeader\x10\r\x12\x11\n\rMsgTimeoutNow\x10\x0e\x12\x10\n\x0cMsgReadInde\ - x\x10\x0f\x12\x14\n\x10MsgReadIndexResp\x10\x10\x12\x15\n\x11MsgRequestP\ - reVote\x10\x11\x12\x1d\n\x19MsgRequestPreVoteResponse\x10\x12*z\n\x0eCon\ - fChangeType\x12\x0b\n\x07AddNode\x10\0\x12\x0e\n\nRemoveNode\x10\x01\x12\ - \x12\n\x0eAddLearnerNode\x10\x02\x12\x19\n\x15BeginMembershipChange\x10\ - \x03\x12\x1c\n\x18FinalizeMembershipChange\x10\x04J\xdd&\n\x06\x12\x04\0\ - \0p\x01\n\x08\n\x01\x0c\x12\x03\0\0\x12\n\x08\n\x01\x02\x12\x03\x01\x08\ - \x0f\n\n\n\x02\x05\0\x12\x04\x03\0\x06\x01\n\n\n\x03\x05\0\x01\x12\x03\ - \x03\x05\x0e\n\x0b\n\x04\x05\0\x02\0\x12\x03\x04\x04\x14\n\x0c\n\x05\x05\ - \0\x02\0\x01\x12\x03\x04\x04\x0f\n\x0c\n\x05\x05\0\x02\0\x02\x12\x03\x04\ + N\n\x19pending_membership_change\x18\x04\x20\x01(\x0b2\x12.eraftpb.ConfS\ + tateR\x17pendingMembershipChange\x12E\n\x1fpending_membership_change_ind\ + ex\x18\x05\x20\x01(\x04R\x1cpendingMembershipChangeIndex\x12\x14\n\x05in\ + dex\x18\x02\x20\x01(\x04R\x05index\x12\x12\n\x04term\x18\x03\x20\x01(\ + \x04R\x04term\"U\n\x08Snapshot\x12\x12\n\x04data\x18\x01\x20\x01(\x0cR\ + \x04data\x125\n\x08metadata\x18\x02\x20\x01(\x0b2\x19.eraftpb.SnapshotMe\ + tadataR\x08metadata\"\xe7\x02\n\x07Message\x12/\n\x08msg_type\x18\x01\ + \x20\x01(\x0e2\x14.eraftpb.MessageTypeR\x07msgType\x12\x0e\n\x02to\x18\ + \x02\x20\x01(\x04R\x02to\x12\x12\n\x04from\x18\x03\x20\x01(\x04R\x04from\ + \x12\x12\n\x04term\x18\x04\x20\x01(\x04R\x04term\x12\x19\n\x08log_term\ + \x18\x05\x20\x01(\x04R\x07logTerm\x12\x14\n\x05index\x18\x06\x20\x01(\ + \x04R\x05index\x12(\n\x07entries\x18\x07\x20\x03(\x0b2\x0e.eraftpb.Entry\ + R\x07entries\x12\x16\n\x06commit\x18\x08\x20\x01(\x04R\x06commit\x12-\n\ + \x08snapshot\x18\t\x20\x01(\x0b2\x11.eraftpb.SnapshotR\x08snapshot\x12\ + \x16\n\x06reject\x18\n\x20\x01(\x08R\x06reject\x12\x1f\n\x0breject_hint\ + \x18\x0b\x20\x01(\x04R\nrejectHint\x12\x18\n\x07context\x18\x0c\x20\x01(\ + \x0cR\x07context\"K\n\tHardState\x12\x12\n\x04term\x18\x01\x20\x01(\x04R\ + \x04term\x12\x12\n\x04vote\x18\x02\x20\x01(\x04R\x04vote\x12\x16\n\x06co\ + mmit\x18\x03\x20\x01(\x04R\x06commit\"=\n\tConfState\x12\x14\n\x05nodes\ + \x18\x01\x20\x03(\x04R\x05nodes\x12\x1a\n\x08learners\x18\x02\x20\x03(\ + \x04R\x08learners\"\xe4\x01\n\nConfChange\x12\x0e\n\x02id\x18\x01\x20\ + \x01(\x04R\x02id\x128\n\x0bchange_type\x18\x02\x20\x01(\x0e2\x17.eraftpb\ + .ConfChangeTypeR\nchangeType\x12\x17\n\x07node_id\x18\x03\x20\x01(\x04R\ + \x06nodeId\x12\x18\n\x07context\x18\x04\x20\x01(\x0cR\x07context\x128\n\ + \rconfiguration\x18\x05\x20\x01(\x0b2\x12.eraftpb.ConfStateR\rconfigurat\ + ion\x12\x1f\n\x0bstart_index\x18\x06\x20\x01(\x04R\nstartIndex*1\n\tEntr\ + yType\x12\x0f\n\x0bEntryNormal\x10\0\x12\x13\n\x0fEntryConfChange\x10\ + \x01*\x8c\x03\n\x0bMessageType\x12\n\n\x06MsgHup\x10\0\x12\x0b\n\x07MsgB\ + eat\x10\x01\x12\x0e\n\nMsgPropose\x10\x02\x12\r\n\tMsgAppend\x10\x03\x12\ + \x15\n\x11MsgAppendResponse\x10\x04\x12\x12\n\x0eMsgRequestVote\x10\x05\ + \x12\x1a\n\x16MsgRequestVoteResponse\x10\x06\x12\x0f\n\x0bMsgSnapshot\ + \x10\x07\x12\x10\n\x0cMsgHeartbeat\x10\x08\x12\x18\n\x14MsgHeartbeatResp\ + onse\x10\t\x12\x12\n\x0eMsgUnreachable\x10\n\x12\x11\n\rMsgSnapStatus\ + \x10\x0b\x12\x12\n\x0eMsgCheckQuorum\x10\x0c\x12\x15\n\x11MsgTransferLea\ + der\x10\r\x12\x11\n\rMsgTimeoutNow\x10\x0e\x12\x10\n\x0cMsgReadIndex\x10\ + \x0f\x12\x14\n\x10MsgReadIndexResp\x10\x10\x12\x15\n\x11MsgRequestPreVot\ + e\x10\x11\x12\x1d\n\x19MsgRequestPreVoteResponse\x10\x12*z\n\x0eConfChan\ + geType\x12\x0b\n\x07AddNode\x10\0\x12\x0e\n\nRemoveNode\x10\x01\x12\x12\ + \n\x0eAddLearnerNode\x10\x02\x12\x19\n\x15BeginMembershipChange\x10\x03\ + \x12\x1c\n\x18FinalizeMembershipChange\x10\x04J\x84&\n\x06\x12\x04\0\0o\ + \x01\n\x08\n\x01\x0c\x12\x03\0\0\x12\n\x08\n\x01\x02\x12\x03\x01\x08\x0f\ + \n\n\n\x02\x05\0\x12\x04\x03\0\x06\x01\n\n\n\x03\x05\0\x01\x12\x03\x03\ + \x05\x0e\n\x0b\n\x04\x05\0\x02\0\x12\x03\x04\x04\x14\n\x0c\n\x05\x05\0\ + \x02\0\x01\x12\x03\x04\x04\x0f\n\x0c\n\x05\x05\0\x02\0\x02\x12\x03\x04\ \x12\x13\n\x0b\n\x04\x05\0\x02\x01\x12\x03\x05\x04\x18\n\x0c\n\x05\x05\0\ \x02\x01\x01\x12\x03\x05\x04\x13\n\x0c\n\x05\x05\0\x02\x01\x02\x12\x03\ \x05\x16\x17\n\xdd\x04\n\x02\x04\0\x12\x04\x12\0\x1c\x01\x1a\xd0\x04\x20\ @@ -2625,171 +2603,169 @@ static file_descriptor_proto_data: &'static [u8] = b"\ \r\n\x05\x04\0\x02\x05\x04\x12\x04\x1b\x04\x17\x16\n\x0c\n\x05\x04\0\x02\ \x05\x05\x12\x03\x1b\x04\x08\n\x0c\n\x05\x04\0\x02\x05\x01\x12\x03\x1b\t\ \x11\n\x0c\n\x05\x04\0\x02\x05\x03\x12\x03\x1b\x14\x15\n\n\n\x02\x04\x01\ - \x12\x04\x1e\0#\x01\n\n\n\x03\x04\x01\x01\x12\x03\x1e\x08\x18\n\x0b\n\ + \x12\x04\x1e\0$\x01\n\n\n\x03\x04\x01\x01\x12\x03\x1e\x08\x18\n\x0b\n\ \x04\x04\x01\x02\0\x12\x03\x1f\x04\x1d\n\r\n\x05\x04\x01\x02\0\x04\x12\ \x04\x1f\x04\x1e\x1a\n\x0c\n\x05\x04\x01\x02\0\x06\x12\x03\x1f\x04\r\n\ \x0c\n\x05\x04\x01\x02\0\x01\x12\x03\x1f\x0e\x18\n\x0c\n\x05\x04\x01\x02\ - \0\x03\x12\x03\x1f\x1b\x1c\n\x0b\n\x04\x04\x01\x02\x01\x12\x03\x20\x04-\ + \0\x03\x12\x03\x1f\x1b\x1c\n\x0b\n\x04\x04\x01\x02\x01\x12\x03\x20\x04,\ \n\r\n\x05\x04\x01\x02\x01\x04\x12\x04\x20\x04\x1f\x1d\n\x0c\n\x05\x04\ - \x01\x02\x01\x06\x12\x03\x20\x04\x0e\n\x0c\n\x05\x04\x01\x02\x01\x01\x12\ - \x03\x20\x0f(\n\x0c\n\x05\x04\x01\x02\x01\x03\x12\x03\x20+,\n\x0b\n\x04\ - \x04\x01\x02\x02\x12\x03!\x04\x15\n\r\n\x05\x04\x01\x02\x02\x04\x12\x04!\ - \x04\x20-\n\x0c\n\x05\x04\x01\x02\x02\x05\x12\x03!\x04\n\n\x0c\n\x05\x04\ - \x01\x02\x02\x01\x12\x03!\x0b\x10\n\x0c\n\x05\x04\x01\x02\x02\x03\x12\ - \x03!\x13\x14\n\x0b\n\x04\x04\x01\x02\x03\x12\x03\"\x04\x14\n\r\n\x05\ - \x04\x01\x02\x03\x04\x12\x04\"\x04!\x15\n\x0c\n\x05\x04\x01\x02\x03\x05\ - \x12\x03\"\x04\n\n\x0c\n\x05\x04\x01\x02\x03\x01\x12\x03\"\x0b\x0f\n\x0c\ - \n\x05\x04\x01\x02\x03\x03\x12\x03\"\x12\x13\n\n\n\x02\x04\x02\x12\x04%\ - \0(\x01\n\n\n\x03\x04\x02\x01\x12\x03%\x08\x10\n\x0b\n\x04\x04\x02\x02\0\ - \x12\x03&\x04\x13\n\r\n\x05\x04\x02\x02\0\x04\x12\x04&\x04%\x12\n\x0c\n\ - \x05\x04\x02\x02\0\x05\x12\x03&\x04\t\n\x0c\n\x05\x04\x02\x02\0\x01\x12\ - \x03&\n\x0e\n\x0c\n\x05\x04\x02\x02\0\x03\x12\x03&\x11\x12\n\x0b\n\x04\ - \x04\x02\x02\x01\x12\x03'\x04\"\n\r\n\x05\x04\x02\x02\x01\x04\x12\x04'\ - \x04&\x13\n\x0c\n\x05\x04\x02\x02\x01\x06\x12\x03'\x04\x14\n\x0c\n\x05\ - \x04\x02\x02\x01\x01\x12\x03'\x15\x1d\n\x0c\n\x05\x04\x02\x02\x01\x03\ - \x12\x03'\x20!\n\n\n\x02\x05\x01\x12\x04*\0>\x01\n\n\n\x03\x05\x01\x01\ - \x12\x03*\x05\x10\n\x0b\n\x04\x05\x01\x02\0\x12\x03+\x04\x0f\n\x0c\n\x05\ - \x05\x01\x02\0\x01\x12\x03+\x04\n\n\x0c\n\x05\x05\x01\x02\0\x02\x12\x03+\ - \r\x0e\n\x0b\n\x04\x05\x01\x02\x01\x12\x03,\x04\x10\n\x0c\n\x05\x05\x01\ - \x02\x01\x01\x12\x03,\x04\x0b\n\x0c\n\x05\x05\x01\x02\x01\x02\x12\x03,\ - \x0e\x0f\n\x0b\n\x04\x05\x01\x02\x02\x12\x03-\x04\x13\n\x0c\n\x05\x05\ - \x01\x02\x02\x01\x12\x03-\x04\x0e\n\x0c\n\x05\x05\x01\x02\x02\x02\x12\ - \x03-\x11\x12\n\x0b\n\x04\x05\x01\x02\x03\x12\x03.\x04\x12\n\x0c\n\x05\ - \x05\x01\x02\x03\x01\x12\x03.\x04\r\n\x0c\n\x05\x05\x01\x02\x03\x02\x12\ - \x03.\x10\x11\n\x0b\n\x04\x05\x01\x02\x04\x12\x03/\x04\x1a\n\x0c\n\x05\ - \x05\x01\x02\x04\x01\x12\x03/\x04\x15\n\x0c\n\x05\x05\x01\x02\x04\x02\ - \x12\x03/\x18\x19\n\x0b\n\x04\x05\x01\x02\x05\x12\x030\x04\x17\n\x0c\n\ - \x05\x05\x01\x02\x05\x01\x12\x030\x04\x12\n\x0c\n\x05\x05\x01\x02\x05\ - \x02\x12\x030\x15\x16\n\x0b\n\x04\x05\x01\x02\x06\x12\x031\x04\x1f\n\x0c\ - \n\x05\x05\x01\x02\x06\x01\x12\x031\x04\x1a\n\x0c\n\x05\x05\x01\x02\x06\ - \x02\x12\x031\x1d\x1e\n\x0b\n\x04\x05\x01\x02\x07\x12\x032\x04\x14\n\x0c\ - \n\x05\x05\x01\x02\x07\x01\x12\x032\x04\x0f\n\x0c\n\x05\x05\x01\x02\x07\ - \x02\x12\x032\x12\x13\n\x0b\n\x04\x05\x01\x02\x08\x12\x033\x04\x15\n\x0c\ - \n\x05\x05\x01\x02\x08\x01\x12\x033\x04\x10\n\x0c\n\x05\x05\x01\x02\x08\ - \x02\x12\x033\x13\x14\n\x0b\n\x04\x05\x01\x02\t\x12\x034\x04\x1d\n\x0c\n\ - \x05\x05\x01\x02\t\x01\x12\x034\x04\x18\n\x0c\n\x05\x05\x01\x02\t\x02\ - \x12\x034\x1b\x1c\n\x0b\n\x04\x05\x01\x02\n\x12\x035\x04\x18\n\x0c\n\x05\ - \x05\x01\x02\n\x01\x12\x035\x04\x12\n\x0c\n\x05\x05\x01\x02\n\x02\x12\ - \x035\x15\x17\n\x0b\n\x04\x05\x01\x02\x0b\x12\x036\x04\x17\n\x0c\n\x05\ - \x05\x01\x02\x0b\x01\x12\x036\x04\x11\n\x0c\n\x05\x05\x01\x02\x0b\x02\ - \x12\x036\x14\x16\n\x0b\n\x04\x05\x01\x02\x0c\x12\x037\x04\x18\n\x0c\n\ - \x05\x05\x01\x02\x0c\x01\x12\x037\x04\x12\n\x0c\n\x05\x05\x01\x02\x0c\ - \x02\x12\x037\x15\x17\n\x0b\n\x04\x05\x01\x02\r\x12\x038\x04\x1b\n\x0c\n\ - \x05\x05\x01\x02\r\x01\x12\x038\x04\x15\n\x0c\n\x05\x05\x01\x02\r\x02\ - \x12\x038\x18\x1a\n\x0b\n\x04\x05\x01\x02\x0e\x12\x039\x04\x17\n\x0c\n\ - \x05\x05\x01\x02\x0e\x01\x12\x039\x04\x11\n\x0c\n\x05\x05\x01\x02\x0e\ - \x02\x12\x039\x14\x16\n\x0b\n\x04\x05\x01\x02\x0f\x12\x03:\x04\x16\n\x0c\ - \n\x05\x05\x01\x02\x0f\x01\x12\x03:\x04\x10\n\x0c\n\x05\x05\x01\x02\x0f\ - \x02\x12\x03:\x13\x15\n\x0b\n\x04\x05\x01\x02\x10\x12\x03;\x04\x1a\n\x0c\ - \n\x05\x05\x01\x02\x10\x01\x12\x03;\x04\x14\n\x0c\n\x05\x05\x01\x02\x10\ - \x02\x12\x03;\x17\x19\n\x0b\n\x04\x05\x01\x02\x11\x12\x03<\x04\x1b\n\x0c\ - \n\x05\x05\x01\x02\x11\x01\x12\x03<\x04\x15\n\x0c\n\x05\x05\x01\x02\x11\ - \x02\x12\x03<\x18\x1a\n\x0b\n\x04\x05\x01\x02\x12\x12\x03=\x04#\n\x0c\n\ - \x05\x05\x01\x02\x12\x01\x12\x03=\x04\x1d\n\x0c\n\x05\x05\x01\x02\x12\ - \x02\x12\x03=\x20\"\n\n\n\x02\x04\x03\x12\x04@\0M\x01\n\n\n\x03\x04\x03\ - \x01\x12\x03@\x08\x0f\n\x0b\n\x04\x04\x03\x02\0\x12\x03A\x04\x1d\n\r\n\ - \x05\x04\x03\x02\0\x04\x12\x04A\x04@\x11\n\x0c\n\x05\x04\x03\x02\0\x06\ - \x12\x03A\x04\x0f\n\x0c\n\x05\x04\x03\x02\0\x01\x12\x03A\x10\x18\n\x0c\n\ - \x05\x04\x03\x02\0\x03\x12\x03A\x1b\x1c\n\x0b\n\x04\x04\x03\x02\x01\x12\ - \x03B\x04\x12\n\r\n\x05\x04\x03\x02\x01\x04\x12\x04B\x04A\x1d\n\x0c\n\ - \x05\x04\x03\x02\x01\x05\x12\x03B\x04\n\n\x0c\n\x05\x04\x03\x02\x01\x01\ - \x12\x03B\x0b\r\n\x0c\n\x05\x04\x03\x02\x01\x03\x12\x03B\x10\x11\n\x0b\n\ - \x04\x04\x03\x02\x02\x12\x03C\x04\x14\n\r\n\x05\x04\x03\x02\x02\x04\x12\ - \x04C\x04B\x12\n\x0c\n\x05\x04\x03\x02\x02\x05\x12\x03C\x04\n\n\x0c\n\ - \x05\x04\x03\x02\x02\x01\x12\x03C\x0b\x0f\n\x0c\n\x05\x04\x03\x02\x02\ - \x03\x12\x03C\x12\x13\n\x0b\n\x04\x04\x03\x02\x03\x12\x03D\x04\x14\n\r\n\ - \x05\x04\x03\x02\x03\x04\x12\x04D\x04C\x14\n\x0c\n\x05\x04\x03\x02\x03\ - \x05\x12\x03D\x04\n\n\x0c\n\x05\x04\x03\x02\x03\x01\x12\x03D\x0b\x0f\n\ - \x0c\n\x05\x04\x03\x02\x03\x03\x12\x03D\x12\x13\n\x0b\n\x04\x04\x03\x02\ - \x04\x12\x03E\x04\x18\n\r\n\x05\x04\x03\x02\x04\x04\x12\x04E\x04D\x14\n\ - \x0c\n\x05\x04\x03\x02\x04\x05\x12\x03E\x04\n\n\x0c\n\x05\x04\x03\x02\ - \x04\x01\x12\x03E\x0b\x13\n\x0c\n\x05\x04\x03\x02\x04\x03\x12\x03E\x16\ - \x17\n\x0b\n\x04\x04\x03\x02\x05\x12\x03F\x04\x15\n\r\n\x05\x04\x03\x02\ - \x05\x04\x12\x04F\x04E\x18\n\x0c\n\x05\x04\x03\x02\x05\x05\x12\x03F\x04\ - \n\n\x0c\n\x05\x04\x03\x02\x05\x01\x12\x03F\x0b\x10\n\x0c\n\x05\x04\x03\ - \x02\x05\x03\x12\x03F\x13\x14\n\x0b\n\x04\x04\x03\x02\x06\x12\x03G\x04\ - \x1f\n\x0c\n\x05\x04\x03\x02\x06\x04\x12\x03G\x04\x0c\n\x0c\n\x05\x04\ - \x03\x02\x06\x06\x12\x03G\r\x12\n\x0c\n\x05\x04\x03\x02\x06\x01\x12\x03G\ - \x13\x1a\n\x0c\n\x05\x04\x03\x02\x06\x03\x12\x03G\x1d\x1e\n\x0b\n\x04\ - \x04\x03\x02\x07\x12\x03H\x04\x16\n\r\n\x05\x04\x03\x02\x07\x04\x12\x04H\ - \x04G\x1f\n\x0c\n\x05\x04\x03\x02\x07\x05\x12\x03H\x04\n\n\x0c\n\x05\x04\ - \x03\x02\x07\x01\x12\x03H\x0b\x11\n\x0c\n\x05\x04\x03\x02\x07\x03\x12\ - \x03H\x14\x15\n\x0b\n\x04\x04\x03\x02\x08\x12\x03I\x04\x1a\n\r\n\x05\x04\ - \x03\x02\x08\x04\x12\x04I\x04H\x16\n\x0c\n\x05\x04\x03\x02\x08\x06\x12\ - \x03I\x04\x0c\n\x0c\n\x05\x04\x03\x02\x08\x01\x12\x03I\r\x15\n\x0c\n\x05\ - \x04\x03\x02\x08\x03\x12\x03I\x18\x19\n\x0b\n\x04\x04\x03\x02\t\x12\x03J\ - \x04\x15\n\r\n\x05\x04\x03\x02\t\x04\x12\x04J\x04I\x1a\n\x0c\n\x05\x04\ - \x03\x02\t\x05\x12\x03J\x04\x08\n\x0c\n\x05\x04\x03\x02\t\x01\x12\x03J\t\ - \x0f\n\x0c\n\x05\x04\x03\x02\t\x03\x12\x03J\x12\x14\n\x0b\n\x04\x04\x03\ - \x02\n\x12\x03K\x04\x1c\n\r\n\x05\x04\x03\x02\n\x04\x12\x04K\x04J\x15\n\ - \x0c\n\x05\x04\x03\x02\n\x05\x12\x03K\x04\n\n\x0c\n\x05\x04\x03\x02\n\ - \x01\x12\x03K\x0b\x16\n\x0c\n\x05\x04\x03\x02\n\x03\x12\x03K\x19\x1b\n\ - \x0b\n\x04\x04\x03\x02\x0b\x12\x03L\x04\x17\n\r\n\x05\x04\x03\x02\x0b\ - \x04\x12\x04L\x04K\x1c\n\x0c\n\x05\x04\x03\x02\x0b\x05\x12\x03L\x04\t\n\ - \x0c\n\x05\x04\x03\x02\x0b\x01\x12\x03L\n\x11\n\x0c\n\x05\x04\x03\x02\ - \x0b\x03\x12\x03L\x14\x16\n\n\n\x02\x04\x04\x12\x04O\0U\x01\n\n\n\x03\ - \x04\x04\x01\x12\x03O\x08\x11\n\x0b\n\x04\x04\x04\x02\0\x12\x03P\x04\x14\ - \n\r\n\x05\x04\x04\x02\0\x04\x12\x04P\x04O\x13\n\x0c\n\x05\x04\x04\x02\0\ - \x05\x12\x03P\x04\n\n\x0c\n\x05\x04\x04\x02\0\x01\x12\x03P\x0b\x0f\n\x0c\ - \n\x05\x04\x04\x02\0\x03\x12\x03P\x12\x13\n\x0b\n\x04\x04\x04\x02\x01\ - \x12\x03Q\x04\x14\n\r\n\x05\x04\x04\x02\x01\x04\x12\x04Q\x04P\x14\n\x0c\ - \n\x05\x04\x04\x02\x01\x05\x12\x03Q\x04\n\n\x0c\n\x05\x04\x04\x02\x01\ - \x01\x12\x03Q\x0b\x0f\n\x0c\n\x05\x04\x04\x02\x01\x03\x12\x03Q\x12\x13\n\ - \x0b\n\x04\x04\x04\x02\x02\x12\x03R\x04\x16\n\r\n\x05\x04\x04\x02\x02\ - \x04\x12\x04R\x04Q\x14\n\x0c\n\x05\x04\x04\x02\x02\x05\x12\x03R\x04\n\n\ - \x0c\n\x05\x04\x04\x02\x02\x01\x12\x03R\x0b\x11\n\x0c\n\x05\x04\x04\x02\ - \x02\x03\x12\x03R\x14\x15\nd\n\x04\x04\x04\x02\x03\x12\x03T\x04-\x1aW\ - \x20If\x20the\x20peer\x20goes\x20down\x20we\x20need\x20to\x20be\x20aware\ - \x20of\x20a\x20partially\x20completed\x20membership\x20change.\n\n\r\n\ - \x05\x04\x04\x02\x03\x04\x12\x04T\x04R\x16\n\x0c\n\x05\x04\x04\x02\x03\ - \x06\x12\x03T\x04\x0e\n\x0c\n\x05\x04\x04\x02\x03\x01\x12\x03T\x0f(\n\ - \x0c\n\x05\x04\x04\x02\x03\x03\x12\x03T+,\n\n\n\x02\x04\x05\x12\x04W\0Z\ - \x01\n\n\n\x03\x04\x05\x01\x12\x03W\x08\x11\n\x0b\n\x04\x04\x05\x02\0\ - \x12\x03X\x04\x1e\n\x0c\n\x05\x04\x05\x02\0\x04\x12\x03X\x04\x0c\n\x0c\n\ - \x05\x04\x05\x02\0\x05\x12\x03X\r\x13\n\x0c\n\x05\x04\x05\x02\0\x01\x12\ - \x03X\x14\x19\n\x0c\n\x05\x04\x05\x02\0\x03\x12\x03X\x1c\x1d\n\x0b\n\x04\ - \x04\x05\x02\x01\x12\x03Y\x04!\n\x0c\n\x05\x04\x05\x02\x01\x04\x12\x03Y\ - \x04\x0c\n\x0c\n\x05\x04\x05\x02\x01\x05\x12\x03Y\r\x13\n\x0c\n\x05\x04\ - \x05\x02\x01\x01\x12\x03Y\x14\x1c\n\x0c\n\x05\x04\x05\x02\x01\x03\x12\ - \x03Y\x1f\x20\n\n\n\x02\x05\x02\x12\x04\\\0b\x01\n\n\n\x03\x05\x02\x01\ - \x12\x03\\\x05\x13\n\x0b\n\x04\x05\x02\x02\0\x12\x03]\x04\x13\n\x0c\n\ - \x05\x05\x02\x02\0\x01\x12\x03]\x04\x0b\n\x0c\n\x05\x05\x02\x02\0\x02\ - \x12\x03]\x11\x12\n\x0b\n\x04\x05\x02\x02\x01\x12\x03^\x04\x13\n\x0c\n\ - \x05\x05\x02\x02\x01\x01\x12\x03^\x04\x0e\n\x0c\n\x05\x05\x02\x02\x01\ - \x02\x12\x03^\x11\x12\n\x0b\n\x04\x05\x02\x02\x02\x12\x03_\x04\x17\n\x0c\ - \n\x05\x05\x02\x02\x02\x01\x12\x03_\x04\x12\n\x0c\n\x05\x05\x02\x02\x02\ - \x02\x12\x03_\x15\x16\n\x0b\n\x04\x05\x02\x02\x03\x12\x03`\x04\x1e\n\x0c\ - \n\x05\x05\x02\x02\x03\x01\x12\x03`\x04\x19\n\x0c\n\x05\x05\x02\x02\x03\ - \x02\x12\x03`\x1c\x1d\n\x0b\n\x04\x05\x02\x02\x04\x12\x03a\x04!\n\x0c\n\ - \x05\x05\x02\x02\x04\x01\x12\x03a\x04\x1c\n\x0c\n\x05\x05\x02\x02\x04\ - \x02\x12\x03a\x1f\x20\n\n\n\x02\x04\x06\x12\x04d\0p\x01\n\n\n\x03\x04\ - \x06\x01\x12\x03d\x08\x12\n\x0b\n\x04\x04\x06\x02\0\x12\x03e\x04\x12\n\r\ - \n\x05\x04\x06\x02\0\x04\x12\x04e\x04d\x14\n\x0c\n\x05\x04\x06\x02\0\x05\ - \x12\x03e\x04\n\n\x0c\n\x05\x04\x06\x02\0\x01\x12\x03e\x0b\r\n\x0c\n\x05\ - \x04\x06\x02\0\x03\x12\x03e\x10\x11\n\x0b\n\x04\x04\x06\x02\x01\x12\x03f\ - \x04#\n\r\n\x05\x04\x06\x02\x01\x04\x12\x04f\x04e\x12\n\x0c\n\x05\x04\ - \x06\x02\x01\x06\x12\x03f\x04\x12\n\x0c\n\x05\x04\x06\x02\x01\x01\x12\ - \x03f\x13\x1e\n\x0c\n\x05\x04\x06\x02\x01\x03\x12\x03f!\"\nE\n\x04\x04\ - \x06\x02\x02\x12\x03h\x04\x17\x1a8\x20Used\x20in\x20`AddNode`,\x20`Remov\ + \x01\x02\x01\x06\x12\x03\x20\x04\r\n\x0c\n\x05\x04\x01\x02\x01\x01\x12\ + \x03\x20\x0e'\n\x0c\n\x05\x04\x01\x02\x01\x03\x12\x03\x20*+\n\x0b\n\x04\ + \x04\x01\x02\x02\x12\x03!\x04/\n\r\n\x05\x04\x01\x02\x02\x04\x12\x04!\ + \x04\x20,\n\x0c\n\x05\x04\x01\x02\x02\x05\x12\x03!\x04\n\n\x0c\n\x05\x04\ + \x01\x02\x02\x01\x12\x03!\x0b*\n\x0c\n\x05\x04\x01\x02\x02\x03\x12\x03!-\ + .\n\x0b\n\x04\x04\x01\x02\x03\x12\x03\"\x04\x15\n\r\n\x05\x04\x01\x02\ + \x03\x04\x12\x04\"\x04!/\n\x0c\n\x05\x04\x01\x02\x03\x05\x12\x03\"\x04\n\ + \n\x0c\n\x05\x04\x01\x02\x03\x01\x12\x03\"\x0b\x10\n\x0c\n\x05\x04\x01\ + \x02\x03\x03\x12\x03\"\x13\x14\n\x0b\n\x04\x04\x01\x02\x04\x12\x03#\x04\ + \x14\n\r\n\x05\x04\x01\x02\x04\x04\x12\x04#\x04\"\x15\n\x0c\n\x05\x04\ + \x01\x02\x04\x05\x12\x03#\x04\n\n\x0c\n\x05\x04\x01\x02\x04\x01\x12\x03#\ + \x0b\x0f\n\x0c\n\x05\x04\x01\x02\x04\x03\x12\x03#\x12\x13\n\n\n\x02\x04\ + \x02\x12\x04&\0)\x01\n\n\n\x03\x04\x02\x01\x12\x03&\x08\x10\n\x0b\n\x04\ + \x04\x02\x02\0\x12\x03'\x04\x13\n\r\n\x05\x04\x02\x02\0\x04\x12\x04'\x04\ + &\x12\n\x0c\n\x05\x04\x02\x02\0\x05\x12\x03'\x04\t\n\x0c\n\x05\x04\x02\ + \x02\0\x01\x12\x03'\n\x0e\n\x0c\n\x05\x04\x02\x02\0\x03\x12\x03'\x11\x12\ + \n\x0b\n\x04\x04\x02\x02\x01\x12\x03(\x04\"\n\r\n\x05\x04\x02\x02\x01\ + \x04\x12\x04(\x04'\x13\n\x0c\n\x05\x04\x02\x02\x01\x06\x12\x03(\x04\x14\ + \n\x0c\n\x05\x04\x02\x02\x01\x01\x12\x03(\x15\x1d\n\x0c\n\x05\x04\x02\ + \x02\x01\x03\x12\x03(\x20!\n\n\n\x02\x05\x01\x12\x04+\0?\x01\n\n\n\x03\ + \x05\x01\x01\x12\x03+\x05\x10\n\x0b\n\x04\x05\x01\x02\0\x12\x03,\x04\x0f\ + \n\x0c\n\x05\x05\x01\x02\0\x01\x12\x03,\x04\n\n\x0c\n\x05\x05\x01\x02\0\ + \x02\x12\x03,\r\x0e\n\x0b\n\x04\x05\x01\x02\x01\x12\x03-\x04\x10\n\x0c\n\ + \x05\x05\x01\x02\x01\x01\x12\x03-\x04\x0b\n\x0c\n\x05\x05\x01\x02\x01\ + \x02\x12\x03-\x0e\x0f\n\x0b\n\x04\x05\x01\x02\x02\x12\x03.\x04\x13\n\x0c\ + \n\x05\x05\x01\x02\x02\x01\x12\x03.\x04\x0e\n\x0c\n\x05\x05\x01\x02\x02\ + \x02\x12\x03.\x11\x12\n\x0b\n\x04\x05\x01\x02\x03\x12\x03/\x04\x12\n\x0c\ + \n\x05\x05\x01\x02\x03\x01\x12\x03/\x04\r\n\x0c\n\x05\x05\x01\x02\x03\ + \x02\x12\x03/\x10\x11\n\x0b\n\x04\x05\x01\x02\x04\x12\x030\x04\x1a\n\x0c\ + \n\x05\x05\x01\x02\x04\x01\x12\x030\x04\x15\n\x0c\n\x05\x05\x01\x02\x04\ + \x02\x12\x030\x18\x19\n\x0b\n\x04\x05\x01\x02\x05\x12\x031\x04\x17\n\x0c\ + \n\x05\x05\x01\x02\x05\x01\x12\x031\x04\x12\n\x0c\n\x05\x05\x01\x02\x05\ + \x02\x12\x031\x15\x16\n\x0b\n\x04\x05\x01\x02\x06\x12\x032\x04\x1f\n\x0c\ + \n\x05\x05\x01\x02\x06\x01\x12\x032\x04\x1a\n\x0c\n\x05\x05\x01\x02\x06\ + \x02\x12\x032\x1d\x1e\n\x0b\n\x04\x05\x01\x02\x07\x12\x033\x04\x14\n\x0c\ + \n\x05\x05\x01\x02\x07\x01\x12\x033\x04\x0f\n\x0c\n\x05\x05\x01\x02\x07\ + \x02\x12\x033\x12\x13\n\x0b\n\x04\x05\x01\x02\x08\x12\x034\x04\x15\n\x0c\ + \n\x05\x05\x01\x02\x08\x01\x12\x034\x04\x10\n\x0c\n\x05\x05\x01\x02\x08\ + \x02\x12\x034\x13\x14\n\x0b\n\x04\x05\x01\x02\t\x12\x035\x04\x1d\n\x0c\n\ + \x05\x05\x01\x02\t\x01\x12\x035\x04\x18\n\x0c\n\x05\x05\x01\x02\t\x02\ + \x12\x035\x1b\x1c\n\x0b\n\x04\x05\x01\x02\n\x12\x036\x04\x18\n\x0c\n\x05\ + \x05\x01\x02\n\x01\x12\x036\x04\x12\n\x0c\n\x05\x05\x01\x02\n\x02\x12\ + \x036\x15\x17\n\x0b\n\x04\x05\x01\x02\x0b\x12\x037\x04\x17\n\x0c\n\x05\ + \x05\x01\x02\x0b\x01\x12\x037\x04\x11\n\x0c\n\x05\x05\x01\x02\x0b\x02\ + \x12\x037\x14\x16\n\x0b\n\x04\x05\x01\x02\x0c\x12\x038\x04\x18\n\x0c\n\ + \x05\x05\x01\x02\x0c\x01\x12\x038\x04\x12\n\x0c\n\x05\x05\x01\x02\x0c\ + \x02\x12\x038\x15\x17\n\x0b\n\x04\x05\x01\x02\r\x12\x039\x04\x1b\n\x0c\n\ + \x05\x05\x01\x02\r\x01\x12\x039\x04\x15\n\x0c\n\x05\x05\x01\x02\r\x02\ + \x12\x039\x18\x1a\n\x0b\n\x04\x05\x01\x02\x0e\x12\x03:\x04\x17\n\x0c\n\ + \x05\x05\x01\x02\x0e\x01\x12\x03:\x04\x11\n\x0c\n\x05\x05\x01\x02\x0e\ + \x02\x12\x03:\x14\x16\n\x0b\n\x04\x05\x01\x02\x0f\x12\x03;\x04\x16\n\x0c\ + \n\x05\x05\x01\x02\x0f\x01\x12\x03;\x04\x10\n\x0c\n\x05\x05\x01\x02\x0f\ + \x02\x12\x03;\x13\x15\n\x0b\n\x04\x05\x01\x02\x10\x12\x03<\x04\x1a\n\x0c\ + \n\x05\x05\x01\x02\x10\x01\x12\x03<\x04\x14\n\x0c\n\x05\x05\x01\x02\x10\ + \x02\x12\x03<\x17\x19\n\x0b\n\x04\x05\x01\x02\x11\x12\x03=\x04\x1b\n\x0c\ + \n\x05\x05\x01\x02\x11\x01\x12\x03=\x04\x15\n\x0c\n\x05\x05\x01\x02\x11\ + \x02\x12\x03=\x18\x1a\n\x0b\n\x04\x05\x01\x02\x12\x12\x03>\x04#\n\x0c\n\ + \x05\x05\x01\x02\x12\x01\x12\x03>\x04\x1d\n\x0c\n\x05\x05\x01\x02\x12\ + \x02\x12\x03>\x20\"\n\n\n\x02\x04\x03\x12\x04A\0N\x01\n\n\n\x03\x04\x03\ + \x01\x12\x03A\x08\x0f\n\x0b\n\x04\x04\x03\x02\0\x12\x03B\x04\x1d\n\r\n\ + \x05\x04\x03\x02\0\x04\x12\x04B\x04A\x11\n\x0c\n\x05\x04\x03\x02\0\x06\ + \x12\x03B\x04\x0f\n\x0c\n\x05\x04\x03\x02\0\x01\x12\x03B\x10\x18\n\x0c\n\ + \x05\x04\x03\x02\0\x03\x12\x03B\x1b\x1c\n\x0b\n\x04\x04\x03\x02\x01\x12\ + \x03C\x04\x12\n\r\n\x05\x04\x03\x02\x01\x04\x12\x04C\x04B\x1d\n\x0c\n\ + \x05\x04\x03\x02\x01\x05\x12\x03C\x04\n\n\x0c\n\x05\x04\x03\x02\x01\x01\ + \x12\x03C\x0b\r\n\x0c\n\x05\x04\x03\x02\x01\x03\x12\x03C\x10\x11\n\x0b\n\ + \x04\x04\x03\x02\x02\x12\x03D\x04\x14\n\r\n\x05\x04\x03\x02\x02\x04\x12\ + \x04D\x04C\x12\n\x0c\n\x05\x04\x03\x02\x02\x05\x12\x03D\x04\n\n\x0c\n\ + \x05\x04\x03\x02\x02\x01\x12\x03D\x0b\x0f\n\x0c\n\x05\x04\x03\x02\x02\ + \x03\x12\x03D\x12\x13\n\x0b\n\x04\x04\x03\x02\x03\x12\x03E\x04\x14\n\r\n\ + \x05\x04\x03\x02\x03\x04\x12\x04E\x04D\x14\n\x0c\n\x05\x04\x03\x02\x03\ + \x05\x12\x03E\x04\n\n\x0c\n\x05\x04\x03\x02\x03\x01\x12\x03E\x0b\x0f\n\ + \x0c\n\x05\x04\x03\x02\x03\x03\x12\x03E\x12\x13\n\x0b\n\x04\x04\x03\x02\ + \x04\x12\x03F\x04\x18\n\r\n\x05\x04\x03\x02\x04\x04\x12\x04F\x04E\x14\n\ + \x0c\n\x05\x04\x03\x02\x04\x05\x12\x03F\x04\n\n\x0c\n\x05\x04\x03\x02\ + \x04\x01\x12\x03F\x0b\x13\n\x0c\n\x05\x04\x03\x02\x04\x03\x12\x03F\x16\ + \x17\n\x0b\n\x04\x04\x03\x02\x05\x12\x03G\x04\x15\n\r\n\x05\x04\x03\x02\ + \x05\x04\x12\x04G\x04F\x18\n\x0c\n\x05\x04\x03\x02\x05\x05\x12\x03G\x04\ + \n\n\x0c\n\x05\x04\x03\x02\x05\x01\x12\x03G\x0b\x10\n\x0c\n\x05\x04\x03\ + \x02\x05\x03\x12\x03G\x13\x14\n\x0b\n\x04\x04\x03\x02\x06\x12\x03H\x04\ + \x1f\n\x0c\n\x05\x04\x03\x02\x06\x04\x12\x03H\x04\x0c\n\x0c\n\x05\x04\ + \x03\x02\x06\x06\x12\x03H\r\x12\n\x0c\n\x05\x04\x03\x02\x06\x01\x12\x03H\ + \x13\x1a\n\x0c\n\x05\x04\x03\x02\x06\x03\x12\x03H\x1d\x1e\n\x0b\n\x04\ + \x04\x03\x02\x07\x12\x03I\x04\x16\n\r\n\x05\x04\x03\x02\x07\x04\x12\x04I\ + \x04H\x1f\n\x0c\n\x05\x04\x03\x02\x07\x05\x12\x03I\x04\n\n\x0c\n\x05\x04\ + \x03\x02\x07\x01\x12\x03I\x0b\x11\n\x0c\n\x05\x04\x03\x02\x07\x03\x12\ + \x03I\x14\x15\n\x0b\n\x04\x04\x03\x02\x08\x12\x03J\x04\x1a\n\r\n\x05\x04\ + \x03\x02\x08\x04\x12\x04J\x04I\x16\n\x0c\n\x05\x04\x03\x02\x08\x06\x12\ + \x03J\x04\x0c\n\x0c\n\x05\x04\x03\x02\x08\x01\x12\x03J\r\x15\n\x0c\n\x05\ + \x04\x03\x02\x08\x03\x12\x03J\x18\x19\n\x0b\n\x04\x04\x03\x02\t\x12\x03K\ + \x04\x15\n\r\n\x05\x04\x03\x02\t\x04\x12\x04K\x04J\x1a\n\x0c\n\x05\x04\ + \x03\x02\t\x05\x12\x03K\x04\x08\n\x0c\n\x05\x04\x03\x02\t\x01\x12\x03K\t\ + \x0f\n\x0c\n\x05\x04\x03\x02\t\x03\x12\x03K\x12\x14\n\x0b\n\x04\x04\x03\ + \x02\n\x12\x03L\x04\x1c\n\r\n\x05\x04\x03\x02\n\x04\x12\x04L\x04K\x15\n\ + \x0c\n\x05\x04\x03\x02\n\x05\x12\x03L\x04\n\n\x0c\n\x05\x04\x03\x02\n\ + \x01\x12\x03L\x0b\x16\n\x0c\n\x05\x04\x03\x02\n\x03\x12\x03L\x19\x1b\n\ + \x0b\n\x04\x04\x03\x02\x0b\x12\x03M\x04\x17\n\r\n\x05\x04\x03\x02\x0b\ + \x04\x12\x04M\x04L\x1c\n\x0c\n\x05\x04\x03\x02\x0b\x05\x12\x03M\x04\t\n\ + \x0c\n\x05\x04\x03\x02\x0b\x01\x12\x03M\n\x11\n\x0c\n\x05\x04\x03\x02\ + \x0b\x03\x12\x03M\x14\x16\n\n\n\x02\x04\x04\x12\x04P\0T\x01\n\n\n\x03\ + \x04\x04\x01\x12\x03P\x08\x11\n\x0b\n\x04\x04\x04\x02\0\x12\x03Q\x04\x14\ + \n\r\n\x05\x04\x04\x02\0\x04\x12\x04Q\x04P\x13\n\x0c\n\x05\x04\x04\x02\0\ + \x05\x12\x03Q\x04\n\n\x0c\n\x05\x04\x04\x02\0\x01\x12\x03Q\x0b\x0f\n\x0c\ + \n\x05\x04\x04\x02\0\x03\x12\x03Q\x12\x13\n\x0b\n\x04\x04\x04\x02\x01\ + \x12\x03R\x04\x14\n\r\n\x05\x04\x04\x02\x01\x04\x12\x04R\x04Q\x14\n\x0c\ + \n\x05\x04\x04\x02\x01\x05\x12\x03R\x04\n\n\x0c\n\x05\x04\x04\x02\x01\ + \x01\x12\x03R\x0b\x0f\n\x0c\n\x05\x04\x04\x02\x01\x03\x12\x03R\x12\x13\n\ + \x0b\n\x04\x04\x04\x02\x02\x12\x03S\x04\x16\n\r\n\x05\x04\x04\x02\x02\ + \x04\x12\x04S\x04R\x14\n\x0c\n\x05\x04\x04\x02\x02\x05\x12\x03S\x04\n\n\ + \x0c\n\x05\x04\x04\x02\x02\x01\x12\x03S\x0b\x11\n\x0c\n\x05\x04\x04\x02\ + \x02\x03\x12\x03S\x14\x15\n\n\n\x02\x04\x05\x12\x04V\0Y\x01\n\n\n\x03\ + \x04\x05\x01\x12\x03V\x08\x11\n\x0b\n\x04\x04\x05\x02\0\x12\x03W\x04\x1e\ + \n\x0c\n\x05\x04\x05\x02\0\x04\x12\x03W\x04\x0c\n\x0c\n\x05\x04\x05\x02\ + \0\x05\x12\x03W\r\x13\n\x0c\n\x05\x04\x05\x02\0\x01\x12\x03W\x14\x19\n\ + \x0c\n\x05\x04\x05\x02\0\x03\x12\x03W\x1c\x1d\n\x0b\n\x04\x04\x05\x02\ + \x01\x12\x03X\x04!\n\x0c\n\x05\x04\x05\x02\x01\x04\x12\x03X\x04\x0c\n\ + \x0c\n\x05\x04\x05\x02\x01\x05\x12\x03X\r\x13\n\x0c\n\x05\x04\x05\x02\ + \x01\x01\x12\x03X\x14\x1c\n\x0c\n\x05\x04\x05\x02\x01\x03\x12\x03X\x1f\ + \x20\n\n\n\x02\x05\x02\x12\x04[\0a\x01\n\n\n\x03\x05\x02\x01\x12\x03[\ + \x05\x13\n\x0b\n\x04\x05\x02\x02\0\x12\x03\\\x04\x13\n\x0c\n\x05\x05\x02\ + \x02\0\x01\x12\x03\\\x04\x0b\n\x0c\n\x05\x05\x02\x02\0\x02\x12\x03\\\x11\ + \x12\n\x0b\n\x04\x05\x02\x02\x01\x12\x03]\x04\x13\n\x0c\n\x05\x05\x02\ + \x02\x01\x01\x12\x03]\x04\x0e\n\x0c\n\x05\x05\x02\x02\x01\x02\x12\x03]\ + \x11\x12\n\x0b\n\x04\x05\x02\x02\x02\x12\x03^\x04\x17\n\x0c\n\x05\x05\ + \x02\x02\x02\x01\x12\x03^\x04\x12\n\x0c\n\x05\x05\x02\x02\x02\x02\x12\ + \x03^\x15\x16\n\x0b\n\x04\x05\x02\x02\x03\x12\x03_\x04\x1e\n\x0c\n\x05\ + \x05\x02\x02\x03\x01\x12\x03_\x04\x19\n\x0c\n\x05\x05\x02\x02\x03\x02\ + \x12\x03_\x1c\x1d\n\x0b\n\x04\x05\x02\x02\x04\x12\x03`\x04!\n\x0c\n\x05\ + \x05\x02\x02\x04\x01\x12\x03`\x04\x1c\n\x0c\n\x05\x05\x02\x02\x04\x02\ + \x12\x03`\x1f\x20\n\n\n\x02\x04\x06\x12\x04c\0o\x01\n\n\n\x03\x04\x06\ + \x01\x12\x03c\x08\x12\n\x0b\n\x04\x04\x06\x02\0\x12\x03d\x04\x12\n\r\n\ + \x05\x04\x06\x02\0\x04\x12\x04d\x04c\x14\n\x0c\n\x05\x04\x06\x02\0\x05\ + \x12\x03d\x04\n\n\x0c\n\x05\x04\x06\x02\0\x01\x12\x03d\x0b\r\n\x0c\n\x05\ + \x04\x06\x02\0\x03\x12\x03d\x10\x11\n\x0b\n\x04\x04\x06\x02\x01\x12\x03e\ + \x04#\n\r\n\x05\x04\x06\x02\x01\x04\x12\x04e\x04d\x12\n\x0c\n\x05\x04\ + \x06\x02\x01\x06\x12\x03e\x04\x12\n\x0c\n\x05\x04\x06\x02\x01\x01\x12\ + \x03e\x13\x1e\n\x0c\n\x05\x04\x06\x02\x01\x03\x12\x03e!\"\nE\n\x04\x04\ + \x06\x02\x02\x12\x03g\x04\x17\x1a8\x20Used\x20in\x20`AddNode`,\x20`Remov\ eNode`,\x20and\x20`AddLearnerNode`.\n\n\r\n\x05\x04\x06\x02\x02\x04\x12\ - \x04h\x04f#\n\x0c\n\x05\x04\x06\x02\x02\x05\x12\x03h\x04\n\n\x0c\n\x05\ - \x04\x06\x02\x02\x01\x12\x03h\x0b\x12\n\x0c\n\x05\x04\x06\x02\x02\x03\ - \x12\x03h\x15\x16\n\x0b\n\x04\x04\x06\x02\x03\x12\x03i\x04\x16\n\r\n\x05\ - \x04\x06\x02\x03\x04\x12\x04i\x04h\x17\n\x0c\n\x05\x04\x06\x02\x03\x05\ - \x12\x03i\x04\t\n\x0c\n\x05\x04\x06\x02\x03\x01\x12\x03i\n\x11\n\x0c\n\ - \x05\x04\x06\x02\x03\x03\x12\x03i\x14\x15\nN\n\x04\x04\x06\x02\x04\x12\ - \x03k\x04\x20\x1aA\x20Used\x20in\x20`BeginMembershipChange`\x20and\x20`F\ - inalizeMembershipChange`.\n\n\r\n\x05\x04\x06\x02\x04\x04\x12\x04k\x04i\ - \x16\n\x0c\n\x05\x04\x06\x02\x04\x06\x12\x03k\x04\r\n\x0c\n\x05\x04\x06\ - \x02\x04\x01\x12\x03k\x0e\x1b\n\x0c\n\x05\x04\x06\x02\x04\x03\x12\x03k\ - \x1e\x1f\n\xd0\x01\n\x04\x04\x06\x02\x05\x12\x03o\x04\x1b\x1a\xc2\x01\ + \x04g\x04e#\n\x0c\n\x05\x04\x06\x02\x02\x05\x12\x03g\x04\n\n\x0c\n\x05\ + \x04\x06\x02\x02\x01\x12\x03g\x0b\x12\n\x0c\n\x05\x04\x06\x02\x02\x03\ + \x12\x03g\x15\x16\n\x0b\n\x04\x04\x06\x02\x03\x12\x03h\x04\x16\n\r\n\x05\ + \x04\x06\x02\x03\x04\x12\x04h\x04g\x17\n\x0c\n\x05\x04\x06\x02\x03\x05\ + \x12\x03h\x04\t\n\x0c\n\x05\x04\x06\x02\x03\x01\x12\x03h\n\x11\n\x0c\n\ + \x05\x04\x06\x02\x03\x03\x12\x03h\x14\x15\nN\n\x04\x04\x06\x02\x04\x12\ + \x03j\x04\x20\x1aA\x20Used\x20in\x20`BeginMembershipChange`\x20and\x20`F\ + inalizeMembershipChange`.\n\n\r\n\x05\x04\x06\x02\x04\x04\x12\x04j\x04h\ + \x16\n\x0c\n\x05\x04\x06\x02\x04\x06\x12\x03j\x04\r\n\x0c\n\x05\x04\x06\ + \x02\x04\x01\x12\x03j\x0e\x1b\n\x0c\n\x05\x04\x06\x02\x04\x03\x12\x03j\ + \x1e\x1f\n\xd0\x01\n\x04\x04\x06\x02\x05\x12\x03n\x04\x1b\x1a\xc2\x01\ \x20Used\x20in\x20`BeginMembershipChange`\x20and\x20`FinalizeMembershipC\ hange`.\n\x20Because\x20`RawNode::apply_conf_change`\x20takes\x20a\x20`C\ onfChange`\x20instead\x20of\x20an\x20`Entry`\x20we\x20must\n\x20include\ \x20this\x20index\x20so\x20it\x20can\x20be\x20known.\n\n\r\n\x05\x04\x06\ - \x02\x05\x04\x12\x04o\x04k\x20\n\x0c\n\x05\x04\x06\x02\x05\x05\x12\x03o\ - \x04\n\n\x0c\n\x05\x04\x06\x02\x05\x01\x12\x03o\x0b\x16\n\x0c\n\x05\x04\ - \x06\x02\x05\x03\x12\x03o\x19\x1ab\x06proto3\ + \x02\x05\x04\x12\x04n\x04j\x20\n\x0c\n\x05\x04\x06\x02\x05\x05\x12\x03n\ + \x04\n\n\x0c\n\x05\x04\x06\x02\x05\x01\x12\x03n\x0b\x16\n\x0c\n\x05\x04\ + \x06\x02\x05\x03\x12\x03n\x19\x1ab\x06proto3\ "; static mut file_descriptor_proto_lazy: ::protobuf::lazy::Lazy<::protobuf::descriptor::FileDescriptorProto> = ::protobuf::lazy::Lazy { diff --git a/src/lib.rs b/src/lib.rs index 3d311a8a1..770cc7372 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -348,7 +348,7 @@ assert!(!node.raft.is_in_membership_change()); This process is a two-phase process, during the midst of it the peer group's leader is managing **two independent, possibly overlapping peer sets**. -> **Note:** In order to maintain resiliency gaurantees (progress while a majority of both peer sets is +> **Note:** In order to maintain resiliency guarantees (progress while a majority of both peer sets is active), it is very important to wait until the entire peer group has exited the transition phase before taking old, removed peers offline. diff --git a/src/progress.rs b/src/progress.rs index 9fefc71ea..7bbc4bad5 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -348,7 +348,7 @@ impl ProgressSet { return Err(Error::Exists(id, "voters")); } else if self.is_in_membership_change() { return Err(Error::ViolatesContract( - "There is a pending membership change.".into(), + "There is a pending membership change".into(), )); } @@ -559,7 +559,7 @@ impl ProgressSet { /// * Voter -> Learner /// * Member as voter and learner. /// * Empty voter set. - pub fn begin_membership_change( + pub(crate) fn begin_membership_change( &mut self, next: impl Into, mut progress: Progress, @@ -586,10 +586,10 @@ impl ProgressSet { progress.recent_active = true; progress.paused = false; for id in next.voters.iter().chain(&next.learners) { + // Now we create progresses for any that do not exist. self.progress.entry(*id).or_insert_with(|| progress.clone()); } self.next_configuration = Some(next); - // Now we create progresses for any that do not exist. Ok(()) } diff --git a/src/raft.rs b/src/raft.rs index daceae575..cf776bf15 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -304,16 +304,29 @@ impl Raft { } let term = r.term; r.become_follower(term, INVALID_ID); + + // Used to resume Joint Consensus Changes + let pending_conf_state = raft_state.pending_conf_state(); + let pending_conf_state_start_index = raft_state.pending_conf_state_start_index(); + match (pending_conf_state, pending_conf_state_start_index) { + (Some(state), Some(idx)) => { + r.begin_membership_change(&ConfChange::from((idx.clone(), state.clone())))?; + }, + (None, None) => (), + _ => unreachable!("Should never find pending_conf_change without an index."), + }; + info!( "{} newRaft [peers: {:?}, term: {:?}, commit: {}, applied: {}, last_index: {}, \ - last_term: {}]", + last_term: {}, pending_membership_change: {:?}]", r.tag, r.prs().voters().collect::>(), r.term, r.raft_log.committed, r.raft_log.get_applied(), r.raft_log.last_index(), - r.raft_log.last_term() + r.raft_log.last_term(), + r.pending_membership_change(), ); Ok(r) } @@ -362,9 +375,6 @@ impl Raft { hs.set_term(self.term); hs.set_vote(self.vote); hs.set_commit(self.raft_log.committed); - if let Some(pending_membership_change) = self.pending_membership_change() { - hs.set_pending_membership_change(pending_membership_change.clone()); - } hs } @@ -575,7 +585,8 @@ impl Raft { if term.is_err() || ents.is_err() { // send snapshot if we failed to get term or entries trace!( - "Skipping sending to {}, term: {:?}, ents: {:?}", + "{} Skipping sending to {}, term: {:?}, ents: {:?}", + self.tag, to, term, ents, @@ -800,7 +811,6 @@ impl Raft { /// /// Panics if a leader already exists. pub fn become_candidate(&mut self) { - trace!("ENTER become_candidate"); assert_ne!( self.state, StateRole::Leader, @@ -812,7 +822,6 @@ impl Raft { self.vote = id; self.state = StateRole::Candidate; info!("{} became candidate at term {}", self.tag, self.term); - trace!("EXIT become_candidate"); } /// Converts this node to a pre-candidate @@ -1898,7 +1907,7 @@ impl Raft { /// For a given message, append the entries to the log. pub fn handle_append_entries(&mut self, m: &Message) { if m.get_index() < self.raft_log.committed { - debug!("Got message with lower index than committed."); + debug!("{} Got message with lower index than committed.", self.tag); let mut to_send = Message::new(); to_send.set_to(m.get_from()); to_send.set_msg_type(MessageType::MsgAppendResponse); @@ -1979,6 +1988,16 @@ impl Raft { fn restore_raft(&mut self, snap: &Snapshot) -> Option { let meta = snap.get_metadata(); + + if snap.get_metadata().has_pending_membership_change() { + let meta = snap.get_metadata(); + let change = meta.get_pending_membership_change().clone(); + let index = meta.get_pending_membership_change_index(); + let change = ConfChange::from((index, change)); + // We already started this change, so it must be safe. We can't bail here. + self.begin_membership_change(&change).unwrap(); + } + if self.raft_log.match_term(meta.get_index(), meta.get_term()) { info!( "{} [commit: {}, lastindex: {}, lastterm: {}] fast-forwarded commit to \ @@ -2043,10 +2062,12 @@ impl Raft { } if meta.has_pending_membership_change() { - let change = meta.get_pending_membership_change(); + let state = meta.get_pending_membership_change(); + let start_index = meta.get_pending_membership_change_index(); + let change = ConfChange::from((start_index, state.clone())); + let config = Configuration::from(state.clone()); let (voters, learners) = { - let config = Configuration::from(change.get_configuration().clone()); let voters = config .voters() .difference(self.prs().configuration().voters()) @@ -2076,7 +2097,7 @@ impl Raft { ); } } - self.begin_membership_change(change) + self.begin_membership_change(&change) .expect("Expected already valid change to still be valid."); } @@ -2306,13 +2327,6 @@ impl Raft { self.raft_log.committed = hs.get_commit(); self.term = hs.get_term(); self.vote = hs.get_vote(); - - // See if we need to re-apply some membership change entries. - if hs.has_pending_membership_change() { - let conf_change = hs.get_pending_membership_change().clone(); - self.begin_membership_change(&conf_change) - .expect("Expected the membership change already recorded to be valid."); - } } /// `pass_election_timeout` returns true iff `election_elapsed` is greater diff --git a/src/raft_log.rs b/src/raft_log.rs index 7c2d5c265..773fd2a6b 100644 --- a/src/raft_log.rs +++ b/src/raft_log.rs @@ -297,7 +297,7 @@ impl RaftLog { /// Appends a set of entries to the unstable list. pub fn append(&mut self, ents: &[Entry]) -> u64 { - trace!("Entries being appended to unstable list: {:?}", ents); + trace!("{} Entries being appended to unstable list: {:?}", self.tag, ents); if ents.is_empty() { return self.last_index(); } diff --git a/src/storage.rs b/src/storage.rs index 9038a6366..fd5857b69 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -40,12 +40,21 @@ use util; /// Holds both the hard state (commit index, vote leader, term) and the configuration state /// (Current node IDs) -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Getters, Setters)] pub struct RaftState { /// Contains the last meta information including commit index, the vote leader, and the vote term. pub hard_state: HardState, /// Records the current node IDs like `[1, 2, 3]` in the cluster. Every Raft node must have a unique ID in the cluster; pub conf_state: ConfState, + /// If this peer is in the middle of a membership change (The period between + /// `BeginMembershipChange` and `FinalizeMembershipChange`) this will hold the final desired + /// state. + #[get = "pub"] #[set] + pending_conf_state: Option, + /// If `pending_conf_state` exists this will contain the index of the `BeginMembershipChange` + /// entry. + #[get = "pub"] #[set] + pending_conf_state_start_index: Option, } /// Storage saves all the information about the current Raft implementation, including Raft Log, commit index, the leader to vote for, etc. @@ -160,9 +169,9 @@ impl MemStorageCore { self.snapshot.mut_metadata().set_conf_state(cs) } if let Some(pending_change) = pending_membership_change { - self.snapshot - .mut_metadata() - .set_pending_membership_change(pending_change); + let meta = self.snapshot.mut_metadata(); + meta.set_pending_membership_change(pending_change.get_configuration().clone()); + meta.set_pending_membership_change_index(pending_change.get_start_index()); } self.snapshot.set_data(data); Ok(&self.snapshot) @@ -263,10 +272,17 @@ impl Storage for MemStorage { /// Implements the Storage trait. fn initial_state(&self) -> Result { let core = self.rl(); - Ok(RaftState { + let mut state = RaftState { hard_state: core.hard_state.clone(), conf_state: core.snapshot.get_metadata().get_conf_state().clone(), - }) + pending_conf_state: None, + pending_conf_state_start_index: None, + }; + if core.snapshot.get_metadata().has_pending_membership_change() { + state.pending_conf_state = core.snapshot.get_metadata().get_pending_membership_change().clone().into(); + state.pending_conf_state_start_index = core.snapshot.get_metadata().get_pending_membership_change_index().clone().into(); + } + Ok(state) } /// Implements the Storage trait. diff --git a/src/util.rs b/src/util.rs index 9558229c0..e09b2e071 100644 --- a/src/util.rs +++ b/src/util.rs @@ -16,7 +16,7 @@ use std::u64; -use eraftpb::ConfState; +use eraftpb::{ConfState, ConfChange, ConfChangeType}; use protobuf::Message; /// A number to represent that there is no limit. @@ -79,3 +79,13 @@ impl ConfState { self.get_nodes() } } + +impl From<(u64, ConfState)> for ConfChange { + fn from((start_index, state): (u64, ConfState)) -> Self { + let mut change = ConfChange::new(); + change.set_change_type(ConfChangeType::BeginMembershipChange); + change.set_configuration(state); + change.set_start_index(start_index); + change + } +} \ No newline at end of file diff --git a/tests/integration_cases/test_membership_changes.rs b/tests/integration_cases/test_membership_changes.rs index 17a1ef5dd..84aebab5e 100644 --- a/tests/integration_cases/test_membership_changes.rs +++ b/tests/integration_cases/test_membership_changes.rs @@ -640,6 +640,7 @@ mod three_peers_replace_voter { // This snapshot has a term 1. let snapshot = { let peer = scenario.peers.get_mut(&1).unwrap(); + warn!("BLAH {:?}", peer.pending_membership_change().clone()); peer.raft_log.store.wl().create_snapshot( 2, ConfState::from(peer.prs().configuration().clone()).into(), @@ -647,6 +648,7 @@ mod three_peers_replace_voter { vec![], )?; let snapshot = peer.raft_log.snapshot()?; + warn!("BLAH {:?}", snapshot.get_metadata()); peer.raft_log.store.wl().compact(2)?; snapshot }; From 7b84b24b8dea597468fc9205ddf043ff882aca37 Mon Sep 17 00:00:00 2001 From: qupeng Date: Tue, 29 Jan 2019 20:14:18 +0800 Subject: [PATCH 36/41] fix a test case about power cycle. (#173) Signed-off-by: qupeng --- benches/suites/progress_set.rs | 10 +++- src/raft.rs | 4 +- src/raft_log.rs | 6 ++- src/storage.rs | 39 +++++++++++++-- src/util.rs | 4 +- .../test_membership_changes.rs | 48 +++++++++---------- 6 files changed, 74 insertions(+), 37 deletions(-) diff --git a/benches/suites/progress_set.rs b/benches/suites/progress_set.rs index 60e15fb5d..3c7b4bcc0 100644 --- a/benches/suites/progress_set.rs +++ b/benches/suites/progress_set.rs @@ -153,7 +153,10 @@ pub fn bench_progress_set_voters(c: &mut Criterion) { let set = quick_progress_set(voters, learners); b.iter(|| { let set = set.clone(); - let sum = set.voters().fold(0, |mut sum, _| { sum += 1; sum }); + let sum = set.voters().fold(0, |mut sum, _| { + sum += 1; + sum + }); sum }); } @@ -173,7 +176,10 @@ pub fn bench_progress_set_learners(c: &mut Criterion) { let set = quick_progress_set(voters, learners); b.iter(|| { let set = set.clone(); - let sum = set.voters().fold(0, |mut sum, _| { sum += 1; sum }); + let sum = set.voters().fold(0, |mut sum, _| { + sum += 1; + sum + }); sum }); } diff --git a/src/raft.rs b/src/raft.rs index cf776bf15..51e0b163a 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -304,14 +304,14 @@ impl Raft { } let term = r.term; r.become_follower(term, INVALID_ID); - + // Used to resume Joint Consensus Changes let pending_conf_state = raft_state.pending_conf_state(); let pending_conf_state_start_index = raft_state.pending_conf_state_start_index(); match (pending_conf_state, pending_conf_state_start_index) { (Some(state), Some(idx)) => { r.begin_membership_change(&ConfChange::from((idx.clone(), state.clone())))?; - }, + } (None, None) => (), _ => unreachable!("Should never find pending_conf_change without an index."), }; diff --git a/src/raft_log.rs b/src/raft_log.rs index 773fd2a6b..b348f57a2 100644 --- a/src/raft_log.rs +++ b/src/raft_log.rs @@ -297,7 +297,11 @@ impl RaftLog { /// Appends a set of entries to the unstable list. pub fn append(&mut self, ents: &[Entry]) -> u64 { - trace!("{} Entries being appended to unstable list: {:?}", self.tag, ents); + trace!( + "{} Entries being appended to unstable list: {:?}", + self.tag, + ents + ); if ents.is_empty() { return self.last_index(); } diff --git a/src/storage.rs b/src/storage.rs index fd5857b69..0553b2ee1 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -46,14 +46,16 @@ pub struct RaftState { pub hard_state: HardState, /// Records the current node IDs like `[1, 2, 3]` in the cluster. Every Raft node must have a unique ID in the cluster; pub conf_state: ConfState, - /// If this peer is in the middle of a membership change (The period between + /// If this peer is in the middle of a membership change (The period between /// `BeginMembershipChange` and `FinalizeMembershipChange`) this will hold the final desired /// state. - #[get = "pub"] #[set] + #[get = "pub"] + #[set] pending_conf_state: Option, /// If `pending_conf_state` exists this will contain the index of the `BeginMembershipChange` /// entry. - #[get = "pub"] #[set] + #[get = "pub"] + #[set] pending_conf_state_start_index: Option, } @@ -117,6 +119,23 @@ impl MemStorageCore { self.hard_state = hs; } + /// Saves the current conf state. + pub fn set_conf_state( + &mut self, + cs: ConfState, + pending_membership_change: Option<(ConfState, u64)>, + ) { + self.snapshot.mut_metadata().set_conf_state(cs); + if let Some((cs, idx)) = pending_membership_change { + self.snapshot + .mut_metadata() + .set_pending_membership_change(cs); + self.snapshot + .mut_metadata() + .set_pending_membership_change_index(idx); + } + } + fn inner_last_index(&self) -> u64 { self.entries[0].get_index() + self.entries.len() as u64 - 1 } @@ -279,8 +298,18 @@ impl Storage for MemStorage { pending_conf_state_start_index: None, }; if core.snapshot.get_metadata().has_pending_membership_change() { - state.pending_conf_state = core.snapshot.get_metadata().get_pending_membership_change().clone().into(); - state.pending_conf_state_start_index = core.snapshot.get_metadata().get_pending_membership_change_index().clone().into(); + state.pending_conf_state = core + .snapshot + .get_metadata() + .get_pending_membership_change() + .clone() + .into(); + state.pending_conf_state_start_index = core + .snapshot + .get_metadata() + .get_pending_membership_change_index() + .clone() + .into(); } Ok(state) } diff --git a/src/util.rs b/src/util.rs index e09b2e071..cfddf2ea6 100644 --- a/src/util.rs +++ b/src/util.rs @@ -16,7 +16,7 @@ use std::u64; -use eraftpb::{ConfState, ConfChange, ConfChangeType}; +use eraftpb::{ConfChange, ConfChangeType, ConfState}; use protobuf::Message; /// A number to represent that there is no limit. @@ -88,4 +88,4 @@ impl From<(u64, ConfState)> for ConfChange { change.set_start_index(start_index); change } -} \ No newline at end of file +} diff --git a/tests/integration_cases/test_membership_changes.rs b/tests/integration_cases/test_membership_changes.rs index 84aebab5e..867ed4bba 100644 --- a/tests/integration_cases/test_membership_changes.rs +++ b/tests/integration_cases/test_membership_changes.rs @@ -571,6 +571,16 @@ mod three_peers_replace_voter { info!("Leader power cycles."); assert_eq!(scenario.peers[&1].began_membership_change_at(), Some(2)); + + if let Some(idx) = scenario.peers[&1].began_membership_change_at() { + let raft = scenario.peers.get_mut(&1).unwrap(); + let conf_state: ConfState = raft.prs().configuration().clone().into(); + let new_conf_state: ConfState = raft.prs().next_configuration().clone().unwrap().into(); + raft.mut_store() + .wl() + .set_conf_state(conf_state, Some((new_conf_state, idx))); + } + scenario.power_cycle(&[1], None); assert_eq!(scenario.peers[&1].began_membership_change_at(), Some(2)); scenario.assert_in_membership_change(&[1]); @@ -1547,32 +1557,20 @@ impl Scenario { let applied = self.peers[&id].raft_log.applied; let mut peer = self.peers.remove(&id).expect("Peer did not exist."); let store = peer.mut_store().clone(); - let prs = peer.prs(); - let mut peer = if let Some(ref snapshot) = snapshot { - let mut peer = Raft::new( - &Config { - id, - tag: format!("{}", id), - applied: applied, - ..Default::default() - }, - store, - ) - .expect("Could not create new Raft"); + + let mut peer = Raft::new( + &Config { + id, + tag: format!("{}", id), + applied: applied, + ..Default::default() + }, + store, + ) + .expect("Could not create new Raft"); + + if let Some(ref snapshot) = snapshot { peer.restore(snapshot.clone()); - peer - } else { - Raft::new( - &Config { - id, - peers: prs.voter_ids().iter().cloned().collect(), - learners: prs.learner_ids().iter().cloned().collect(), - applied: applied, - ..Default::default() - }, - store, - ) - .expect("Could not create new Raft") }; self.peers.insert(id, peer.into()); } From 39cb541b93ae52a98c0998e26682ac3668915921 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Wed, 30 Jan 2019 01:09:32 -0800 Subject: [PATCH 37/41] Fix nits --- tests/integration_cases/test_membership_changes.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration_cases/test_membership_changes.rs b/tests/integration_cases/test_membership_changes.rs index b147bb0e3..d23ac17c3 100644 --- a/tests/integration_cases/test_membership_changes.rs +++ b/tests/integration_cases/test_membership_changes.rs @@ -1,4 +1,4 @@ -// Copyright 2016 PingCAP, Inc. +// Copyright 2019 PingCAP, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -1417,7 +1417,7 @@ impl Scenario { let message = build_propose_add_node_message( self.old_leader, id, - self.peers[&1].raft_log.last_index() + 1, + self.peers[&id].raft_log.last_index() + 1, ); self.dispatch(vec![message]) } From ff73fe9a333eb51a86b13bc6583a98b29ad1b8b2 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Thu, 31 Jan 2019 11:47:20 -0800 Subject: [PATCH 38/41] Clippy/fmt --- src/eraftpb.rs | 2 +- src/raft.rs | 2 +- src/storage.rs | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/eraftpb.rs b/src/eraftpb.rs index 27610997b..dd3008ccc 100644 --- a/src/eraftpb.rs +++ b/src/eraftpb.rs @@ -3,7 +3,7 @@ // https://github.com/Manishearth/rust-clippy/issues/702 #![allow(unknown_lints)] -#![allow(clippy)] +#![allow(clippy::all)] #![cfg_attr(rustfmt, rustfmt_skip)] diff --git a/src/raft.rs b/src/raft.rs index 05d39fb52..50bbe6490 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -309,7 +309,7 @@ impl Raft { let pending_conf_state_start_index = raft_state.pending_conf_state_start_index(); match (pending_conf_state, pending_conf_state_start_index) { (Some(state), Some(idx)) => { - r.begin_membership_change(&ConfChange::from((idx.clone(), state.clone())))?; + r.begin_membership_change(&ConfChange::from((*idx, state.clone())))?; } (None, None) => (), _ => unreachable!("Should never find pending_conf_change without an index."), diff --git a/src/storage.rs b/src/storage.rs index 0553b2ee1..67df549de 100644 --- a/src/storage.rs +++ b/src/storage.rs @@ -308,7 +308,6 @@ impl Storage for MemStorage { .snapshot .get_metadata() .get_pending_membership_change_index() - .clone() .into(); } Ok(state) From 356abd74c437e5b3fa7eb1f67483e6a1859b3d47 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Thu, 31 Jan 2019 12:10:57 -0800 Subject: [PATCH 39/41] Documentation touchup --- src/lib.rs | 27 ++++++++++++++++++--------- src/raft.rs | 26 ++++++++++---------------- 2 files changed, 28 insertions(+), 25 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index f5f56db1c..6a63c87ea 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -284,7 +284,9 @@ need to update the applied index and resume `apply` later: For more information, check out an [example](examples/single_mem_node/main.rs#L113-L179). -## Membership Changes +## Arbitrary Membership Changes + +> **Note:** This is an experimental feature. When building a resilient, scalable distributed system there is a strong need to be able to change the membership of a peer group *dynamically, without downtime.* This Raft crate supports this via @@ -298,6 +300,7 @@ the following: * Remove peer *n* from the group. * Remove a leader (unmanaged, via stepdown) * Promote a learner to a voter. +* Replace a node *n* with another node *m*. It (currently) does not: @@ -313,7 +316,7 @@ This means it's possible to do: ```rust use raft::{Config, storage::MemStorage, raw_node::RawNode, eraftpb::{Message, ConfChange}}; -let config = Config { id: 1, peers: vec![1], ..Default::default() }; +let config = Config { id: 1, peers: vec![1, 2], ..Default::default() }; let mut node = RawNode::new(&config, MemStorage::default(), vec![]).unwrap(); node.raft.become_candidate(); node.raft.become_leader(); @@ -322,26 +325,32 @@ node.raft.become_leader(); node.raft.propose_membership_change(( // Any IntoIterator. // Voters - vec![1,2,3], + vec![1,3], // Remove 2, add 3. // Learners - vec![4,5,6], + vec![4,5,6], // Add 4, 5, 6. )).unwrap(); # let entry = &node.raft.raft_log.entries(2, 1).unwrap()[0]; -// ...Later when the begin entry is ready to apply: -let conf_change = protobuf::parse_from_bytes::(entry.get_data()).unwrap(); +// ...Later when the begin entry is recieved from a `ready()` in the `entries` field... +let conf_change = protobuf::parse_from_bytes::(entry.get_data()) + .unwrap(); node.raft.begin_membership_change(&conf_change).unwrap(); assert!(node.raft.is_in_membership_change()); +assert!(node.raft.prs().voter_ids().contains(&2)); +assert!(node.raft.prs().voter_ids().contains(&3)); # # // We hide this since the user isn't really encouraged to blindly call this, but we'd like a short # // example. +# node.raft.raft_log.commit_to(2); # node.raft.commit_apply(2); # # let entry = &node.raft.raft_log.entries(3, 1).unwrap()[0]; -// ...Later, when the finalize entry is ready to apply: -let conf_change = protobuf::parse_from_bytes::(entry.get_data()).unwrap(); +// ...Later, when the finalize entry is recieved from a `ready()` in the `entries` field... +let conf_change = protobuf::parse_from_bytes::(entry.get_data()) + .unwrap(); node.raft.finalize_membership_change(&conf_change).unwrap(); -assert!(node.raft.prs().voter_ids().contains(&2)); +assert!(!node.raft.prs().voter_ids().contains(&2)); +assert!(node.raft.prs().voter_ids().contains(&3)); assert!(!node.raft.is_in_membership_change()); ``` diff --git a/src/raft.rs b/src/raft.rs index 50bbe6490..cc419f50f 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -138,7 +138,7 @@ pub struct Raft { pub lead_transferee: Option, /// Only one conf change may be pending (in the log, but not yet - /// applied) at a time. This is enforced via pending_conf_index, which + /// applied) at a time. This is enforced via `pending_conf_index`, which /// is set to a value >= the log index of the latest pending /// configuration change (if any). Config changes are only allowed to /// be proposed if the leader's applied index is greater than this @@ -150,7 +150,7 @@ pub struct Raft { /// we set this to one. pub pending_conf_index: u64, - /// The last BeginMembershipChange entry. Once we commit this entry we can exit the joint state. + /// The last `BeginMembershipChange` entry. Once we make this change we exit the joint state. /// /// This is different than `pending_conf_index` since it is more specific, and also exact. /// While `pending_conf_index` is conservatively set at times to ensure safety in the @@ -432,6 +432,8 @@ impl Raft { } /// Get the index which the pending membership change started at. + /// + /// > **Note:** This is an experimental feature. #[inline] pub fn began_membership_change_at(&self) -> Option { self.pending_membership_change @@ -1193,6 +1195,8 @@ impl Raft { } /// Apply a `BeginMembershipChange` variant `ConfChange`. + /// + /// > **Note:** This is an experimental feature. /// /// When a Raft node applies this variant of a configuration change it will adopt a joint /// configuration state until the membership change is finalized. @@ -1200,13 +1204,6 @@ impl Raft { /// During this time the `Raft` will have two, possibly overlapping, cooperating quorums for /// both elections and log replication. /// - /// # Implementation notes - /// - /// This uses a slightly modified "Joint Consensus" algorithm as detailed in Section 6 of the - /// Raft paper. - /// - /// We apply the change when a node *applies* the entry, not when the entry is received. - /// /// # Errors /// /// * `ConfChange.change_type` is not `BeginMembershipChange` @@ -1242,6 +1239,8 @@ impl Raft { } /// Apply a `FinalizeMembershipChange` variant `ConfChange`. + /// + /// > **Note:** This is an experimental feature. /// /// When a Raft node applies this variant of a configuration change it will finalize the /// transition begun by [`begin_membership_change`]. @@ -1249,13 +1248,6 @@ impl Raft { /// Once this is called the Raft will no longer have two, possibly overlapping, cooperating /// qourums. /// - /// # Implementation notes - /// - /// This uses a slightly modified "Joint Consensus" algorithm as detailed in Section 6 of the - /// Raft paper. - /// - /// We apply the change when a node *applies* the entry, not when the entry is received. - /// /// # Errors /// /// * This Raft is not in a configuration change via `begin_membership_change`. @@ -2137,6 +2129,8 @@ impl Raft { } /// Propose that the peer group change its active set to a new set. + /// + /// > **Note:** This is an experimental feature. /// /// ```rust /// use raft::{Raft, Config, storage::MemStorage, eraftpb::ConfState}; From 016e4a7f891f1aa25dcbbe9c1d66950430ae8d48 Mon Sep 17 00:00:00 2001 From: Ana Hobden Date: Thu, 31 Jan 2019 23:28:17 -0800 Subject: [PATCH 40/41] fmt --- src/raft.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/raft.rs b/src/raft.rs index cc419f50f..097ee05a2 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -432,7 +432,7 @@ impl Raft { } /// Get the index which the pending membership change started at. - /// + /// /// > **Note:** This is an experimental feature. #[inline] pub fn began_membership_change_at(&self) -> Option { @@ -1195,7 +1195,7 @@ impl Raft { } /// Apply a `BeginMembershipChange` variant `ConfChange`. - /// + /// /// > **Note:** This is an experimental feature. /// /// When a Raft node applies this variant of a configuration change it will adopt a joint @@ -1239,7 +1239,7 @@ impl Raft { } /// Apply a `FinalizeMembershipChange` variant `ConfChange`. - /// + /// /// > **Note:** This is an experimental feature. /// /// When a Raft node applies this variant of a configuration change it will finalize the @@ -2129,7 +2129,7 @@ impl Raft { } /// Propose that the peer group change its active set to a new set. - /// + /// /// > **Note:** This is an experimental feature. /// /// ```rust From 55385dd0820c3047fc4804fdf36e6a1488976895 Mon Sep 17 00:00:00 2001 From: qupeng Date: Sat, 2 Feb 2019 13:39:42 +0800 Subject: [PATCH 41/41] simplify snapshot restore. (#176) --- src/progress.rs | 50 +++++++++++++++++++++++++++-- src/raft.rs | 83 ++++++++----------------------------------------- 2 files changed, 61 insertions(+), 72 deletions(-) diff --git a/src/progress.rs b/src/progress.rs index c1b30ca70..b12beabaf 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -25,7 +25,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use eraftpb::ConfState; +use eraftpb::{ConfState, SnapshotMetadata}; use errors::{Error, Result}; use hashbrown::hash_map::DefaultHashBuilder; use hashbrown::{HashMap, HashSet}; @@ -197,6 +197,44 @@ impl ProgressSet { } } + pub(crate) fn restore_snapmeta( + meta: &SnapshotMetadata, + next_idx: u64, + max_inflight: usize, + ) -> Self { + let mut prs = ProgressSet::new(); + let pr = Progress::new(next_idx, max_inflight); + meta.get_conf_state().get_nodes().iter().for_each(|id| { + prs.progress.insert(*id, pr.clone()); + prs.configuration.voters.insert(*id); + }); + meta.get_conf_state().get_learners().iter().for_each(|id| { + prs.progress.insert(*id, pr.clone()); + prs.configuration.learners.insert(*id); + }); + + if meta.pending_membership_change_index != 0 { + let mut next_configuration = Configuration::with_capacity(0, 0); + meta.get_pending_membership_change() + .get_nodes() + .iter() + .for_each(|id| { + prs.progress.insert(*id, pr.clone()); + next_configuration.voters.insert(*id); + }); + meta.get_pending_membership_change() + .get_learners() + .iter() + .for_each(|id| { + prs.progress.insert(*id, pr.clone()); + next_configuration.learners.insert(*id); + }); + prs.next_configuration = Some(next_configuration); + } + prs.assert_progress_and_configuration_consistent(); + prs + } + /// Returns the status of voters. /// /// **Note:** Do not use this for majority/quorum calculation. The Raft node may be @@ -415,7 +453,15 @@ impl ProgressSet { .progress .keys() .all(|v| self.configuration.learners.contains(v) - || self.configuration.voters.contains(v))); + || self.configuration.voters.contains(v) + || self + .next_configuration + .as_ref() + .map_or(false, |c| c.learners.contains(v)) + || self + .next_configuration + .as_ref() + .map_or(false, |c| c.voters.contains(v)))); assert_eq!( self.voter_ids().len() + self.learner_ids().len(), self.progress.len() diff --git a/src/raft.rs b/src/raft.rs index 097ee05a2..50cf0b308 100644 --- a/src/raft.rs +++ b/src/raft.rs @@ -1979,16 +1979,6 @@ impl Raft { fn restore_raft(&mut self, snap: &Snapshot) -> Option { let meta = snap.get_metadata(); - - if snap.get_metadata().has_pending_membership_change() { - let meta = snap.get_metadata(); - let change = meta.get_pending_membership_change().clone(); - let index = meta.get_pending_membership_change_index(); - let change = ConfChange::from((index, change)); - // We already started this change, so it must be safe. We can't bail here. - self.begin_membership_change(&change).unwrap(); - } - if self.raft_log.match_term(meta.get_index(), meta.get_term()) { info!( "{} [commit: {}, lastindex: {}, lastterm: {}] fast-forwarded commit to \ @@ -2030,68 +2020,21 @@ impl Raft { meta.get_term() ); - let nodes = meta.get_conf_state().get_nodes(); - let learners = meta.get_conf_state().get_learners(); - self.prs = Some(ProgressSet::with_capacity(nodes.len(), learners.len())); - - for &(is_learner, nodes) in &[(false, nodes), (true, learners)] { - for &n in nodes { - let next_index = self.raft_log.last_index() + 1; - let mut matched = 0; - if n == self.id { - matched = next_index - 1; - self.is_learner = is_learner; - } - self.set_progress(n, matched, next_index, is_learner); - info!( - "{} restored progress of {} [{:?}]", - self.tag, - n, - self.prs().get(n) - ); - } + let next_idx = self.raft_log.last_index() + 1; + let mut prs = ProgressSet::restore_snapmeta(meta, next_idx, self.max_inflight); + prs.get_mut(self.id).unwrap().matched = next_idx - 1; + if self.is_learner && prs.configuration().voters().contains(&self.id) { + self.is_learner = false; } - - if meta.has_pending_membership_change() { - let state = meta.get_pending_membership_change(); - let start_index = meta.get_pending_membership_change_index(); - let change = ConfChange::from((start_index, state.clone())); - let config = Configuration::from(state.clone()); - - let (voters, learners) = { - let voters = config - .voters() - .difference(self.prs().configuration().voters()) - .cloned() - .collect::>(); - let learners = config - .learners() - .difference(self.prs().configuration().learners()) - .cloned() - .collect::>(); - (voters, learners) - }; - for &(is_learner, ref nodes) in &[(false, voters), (true, learners)] { - for &n in nodes { - let next_index = self.raft_log.last_index() + 1; - let mut matched = 0; - if n == self.id { - matched = next_index - 1; - self.is_learner = is_learner; - } - self.set_progress(n, matched, next_index, is_learner); - info!( - "{} restored progress of {} [{:?}]", - self.tag, - n, - self.prs().get(n) - ); - } - } - self.begin_membership_change(&change) - .expect("Expected already valid change to still be valid."); + self.prs = Some(prs); + if meta.get_pending_membership_change_index() > 0 { + let cs = meta.get_pending_membership_change().clone(); + let mut conf_change = ConfChange::new(); + conf_change.set_change_type(ConfChangeType::BeginMembershipChange); + conf_change.set_configuration(cs); + conf_change.set_start_index(meta.get_pending_membership_change_index()); + self.pending_membership_change = Some(conf_change); } - None }