Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support configuring election timeout range #63

Merged
merged 6 commits into from
May 20, 2018
Merged

Support configuring election timeout range #63

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift click to select a range
b1fab32
support least election timeout
BusyJay May 18, 2018
2a13355
support larger timeout
BusyJay May 19, 2018
58290c4
generalize the API
BusyJay May 19, 2018
7b781aa
address comment
BusyJay May 20, 2018
4dfc7d8
Merge branch 'master' into add-least-timeout
BusyJay May 20, 2018
4bb7282
Merge branch 'master' into add-least-timeout
BusyJay May 20, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
generalize the API
  • Loading branch information
@BusyJay
BusyJay committed May 19, 2018
commit 58290c40911936b25a7f924be3d353f119443775
53 changes: 39 additions & 14 deletions src/raft.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -124,10 124,13 @@ pub struct Config {
/// rejoins the cluster.
pub pre_vote: bool,

/// The least election timeout. In some cases, we hope some nodes has less possibility
/// The range of election timeout. In some cases, we hope some nodes has less possibility
/// to become leader. This configuration ensures that the randomized election_timeout
/// always not less than this value.
pub least_election_timeout_tick: usize,
/// will always be suit in [min_election_tick, max_election_tick).
/// If it is None, then election_tick will be chosen.
pub min_election_tick: Option<usize>,
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you can remove Option here, you can set min to election timeout and max to 2 x election timeout explicitly if they are 0 at initialization.

/// If it is None, then 2 * election_tick will be chosen.
pub max_election_tick: Option<usize>,

/// read_only_option specifies how the read only request is processed.
pub read_only_option: ReadOnlyOption,
Expand All @@ -142,6 145,17 @@ pub struct Config {
}

impl Config {
#[inline]
pub fn min_election_tick(&self) -> usize {
self.min_election_tick.unwrap_or(self.election_tick)
}

#[inline]
pub fn max_election_tick(&self) -> usize {
self.max_election_tick
.unwrap_or_else(|| 2 * self.election_tick)
}

pub fn validate(&self) -> Result<()> {
if self.id == INVALID_ID {
return Err(Error::ConfigInvalid("invalid node id".to_owned()));
Expand All @@ -159,10 173,20 @@ impl Config {
));
}

if self.least_election_timeout_tick < self.election_tick {
return Err(Error::ConfigInvalid(
"lease election timeout tick must not less than election_tick".to_owned(),
));
let min_timeout = self.min_election_tick();
let max_timeout = self.max_election_tick();
if min_timeout < self.election_tick {
return Err(Error::ConfigInvalid(format!(
"min election tick {} must not be less than election_tick {}",
min_timeout, self.election_tick
)));
}

if min_timeout >= max_timeout {
return Err(Error::ConfigInvalid(format!(
"min election tick {} should be less than max election tick {}",
min_timeout, max_timeout
)));
}

if self.max_inflight_msgs == 0 {
Expand Down Expand Up @@ -243,10 267,11 @@ pub struct Raft<T: Storage> {
election_timeout: usize,

// randomized_election_timeout is a random number between
// [election_timeout, 2 * election_timeout - 1]. It gets reset
// [min_election_timeout, max_election_timeout - 1]. It gets reset
// when raft changes its state to follower or candidate.
randomized_election_timeout: usize,
least_election_timeout: usize,
min_election_timeout: usize,
max_election_timeout: usize,

/// Will be called when step** is about to be called.
/// return false will skip step**.
Expand Down Expand Up @@ -334,7 359,8 @@ impl<T: Storage> Raft<T> {
vote: Default::default(),
heartbeat_elapsed: Default::default(),
randomized_election_timeout: 0,
least_election_timeout: c.least_election_timeout_tick,
min_election_timeout: c.min_election_tick(),
max_election_timeout: c.max_election_tick(),
skip_bcast_commit: c.skip_bcast_commit,
tag: c.tag.to_owned(),
};
Expand Down Expand Up @@ -423,7 449,7 @@ impl<T: Storage> Raft<T> {

// for testing leader lease
pub fn set_randomized_election_timeout(&mut self, t: usize) {
assert!(t >= self.least_election_timeout);
assert!(self.min_election_timeout <= t && t < self.max_election_timeout);
self.randomized_election_timeout = t;
}

Expand Down Expand Up @@ -1986,9 2012,8 @@ impl<T: Storage> Raft<T> {

pub fn reset_randomized_election_timeout(&mut self) {
let prev_timeout = self.randomized_election_timeout;
let max_timeout =
(self.least_election_timeout / self.election_timeout 1) * self.election_timeout;
let timeout = rand::thread_rng().gen_range(self.least_election_timeout, max_timeout);
let timeout =
rand::thread_rng().gen_range(self.min_election_timeout, self.max_election_timeout);
debug!(
"{} reset election timeout {} -> {} at {}",
self.tag, prev_timeout, timeout, self.election_elapsed
Expand Down
34 changes: 33 additions & 1 deletion tests/cases/test_raft.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 74,6 @@ pub fn new_test_config(id: u64, peers: Vec<u64>, election: usize, heartbeat: usi
id: id,
peers: peers,
election_tick: election,
least_election_timeout_tick: election,
heartbeat_tick: heartbeat,
max_size_per_msg: NO_LIMIT,
max_inflight_msgs: 256,
Expand Down Expand Up @@ -4086,3 4085,36 @@ fn test_learner_respond_vote() {
do_campaign(&mut network);
assert_eq!(network.peers[&1].state, StateRole::Leader);
}

#[test]
fn test_election_tick_range() {
let mut cfg = new_test_config(1, vec![1, 2, 3], 10, 1);
let mut raft = Raft::new(&cfg, new_storage());
for _ in 0..1000 {
raft.reset_randomized_election_timeout();
let randomized_timeout = raft.get_randomized_election_timeout();
assert!(
cfg.election_tick <= randomized_timeout && randomized_timeout < 2 * cfg.election_tick
);
}

cfg.min_election_tick = Some(cfg.election_tick);
cfg.validate().unwrap();

// Too small election tick.
cfg.min_election_tick = Some(cfg.election_tick - 1);
cfg.validate().unwrap_err();

// max_election_tick should be larger than min_election_tick
cfg.min_election_tick = Some(cfg.election_tick);
cfg.max_election_tick = Some(cfg.election_tick);
cfg.validate().unwrap_err();

cfg.max_election_tick = Some(cfg.election_tick 1);
raft = Raft::new(&cfg, new_storage());
for _ in 0..100 {
raft.reset_randomized_election_timeout();
let randomized_timeout = raft.get_randomized_election_timeout();
assert_eq!(randomized_timeout, cfg.election_tick);
}
}