Introduce benchmarking using Criterion.

tikv · Aug 28, 2018 · 58246d2 · 58246d2
1 parent 8b547c5
commit 58246d2
Show file tree

Hide file tree

Showing 10 changed files with 346 additions and 3 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -18,6  18,13 @@ quick-error = "1.2.2"
 rand = "0.5.4"
 fxhash = "0.2.1"
 
 [dev-dependencies]
 criterion = ">0.2.4"
 
 [[bench]]
 name = "benches"
 harness = false
 
 [badges]
 travis-ci = { repository = "pingcap/raft-rs" }
 

diff --git a/README.md b/README.md
@@ -38,20  38,26 @@ Using `rustup` you can get started this way:
 ```bash
 rustup override set stable
 rustup toolchain install nightly
 rustup component add clippy-review --nightly
 rustup component add rustfmt-preview
 ```
 
 In order to have your PR merged running the following must finish without error:
 
 ```bash
-cargo  nightly test --features dev
 cargo test --all && \
 cargo  nightly clippy --all && \
 cargo fmt --all -- --check
 ```
 
 You may optionally want to install `cargo-watch` to allow for automated rebuilding while editing:
 
 ```bash
-cargo watch -s "cargo check --features dev"
 cargo watch -s "cargo check"
 ```
 
 ### Modifying Protobufs
 
 If proto file `eraftpb.proto` changed, run the command to regenerate `eraftpb.rs`:
 
 ```bash
@@ -60,6  66,31 @@ protoc proto/eraftpb.proto --rust_out=src
 
 You can check `Cargo.toml` to find which version of `protobuf-codegen` is required.
 
 ### Benchmarks
 
 We use [Criterion](https://github.com/japaric/criterion.rs) for benchmarking.
 
 > It's currently an ongoing effort to build an appropriate benchmarking suite. If you'd like to help out please let us know! [Interested?](https://github.com/pingcap/raft-rs/issues/109)
 
 You can run the benchmarks by installing `gnuplot` then running:
 
 ```bash
 cargo bench
 ```
 
 You can check `target/criterion/report/index.html` for plots and charts relating to the benchmarks.
 
 You can check the performance between two branches:
 
 ```bash
 git checkout master
 cargo bench --bench benches -- --save-baseline master
 git checkout other
 cargo bench --bench benches -- --baseline master
 ```
 
 This will report relative increases or decreased for each benchmark.
 
 ## Acknowledgments
 
 Thanks [etcd](https://github.com/coreos/etcd) for providing the amazing Go implementation!

diff --git a/benches/benches.rs b/benches/benches.rs
@@ -0,0  1,15 @@
 #![allow(dead_code)] // Due to criterion we need this to avoid warnings.
 
 #[macro_use]
 extern crate criterion;
 extern crate raft;
 
 mod suites;
 use suites::*;
 
 criterion_main!(
     bench_raft,
     bench_raw_node,
     bench_progress,
     bench_progress_set,
 );
diff --git a/benches/suites/mod.rs b/benches/suites/mod.rs
@@ -0,0  1,8 @@
 mod raft;
 pub use self::raft::*;
 mod raw_node;
 pub use self::raw_node::*;
 mod progress;
 pub use self::progress::*;
 mod progress_set;
 pub use self::progress_set::*;
diff --git a/benches/suites/progress.rs b/benches/suites/progress.rs
@@ -0,0  1,13 @@
 use criterion::{Bencher, Criterion};
 use raft::Progress;
 
 criterion_group!(bench_progress, bench_progress_default);
 
 pub fn bench_progress_default(c: &mut Criterion) {
     let bench = |b: &mut Bencher| {
         // No setup.
         b.iter(|| Progress::default());
     };
 
     c.bench_function("Progress::default", bench);
 }
diff --git a/benches/suites/progress_set.rs b/benches/suites/progress_set.rs
@@ -0,0  1,155 @@
 use criterion::{Bencher, Criterion};
 use raft::ProgressSet;
 
 criterion_group!(
     bench_progress_set,
     bench_progress_set_new,
     bench_progress_set_insert_voter,
     bench_progress_set_insert_learner,
     bench_progress_set_promote_learner,
     bench_progress_set_remove,
     bench_progress_set_iter,
     bench_progress_set_get,
     bench_progress_set_nodes,
 );
 
 fn quick_progress_set(voters: usize, learners: usize) -> ProgressSet {
     let mut set = ProgressSet::new(voters, learners);
     (0..voters).for_each(|id| {
         set.insert_voter(id as u64, Default::default()).ok();
     });
     (voters..learners).for_each(|id| {
         set.insert_learner(id as u64, Default::default()).ok();
     });
     set
 }
 
 pub fn bench_progress_set_new(c: &mut Criterion) {
     let bench = |voters, learners| {
         move |b: &mut Bencher| {
             // No setup.
             b.iter(|| ProgressSet::new(voters, learners));
         }
     };
 
     c.bench_function("ProgressSet::new (0, 0)", bench(0, 0));
     c.bench_function("ProgressSet::new (3, 1)", bench(3, 1));
     c.bench_function("ProgressSet::new (5, 2)", bench(5, 2));
     c.bench_function("ProgressSet::new (7, 3)", bench(7, 3));
 }
 
 pub fn bench_progress_set_insert_voter(c: &mut Criterion) {
     let bench = |voters, learners| {
         move |b: &mut Bencher| {
             let set = quick_progress_set(voters, learners);
             b.iter(|| {
                 let mut set = set.clone();
                 set.insert_voter(99, Default::default()).ok()
             });
         }
     };
     c.bench_function("ProgressSet::insert_voter (0, 0)", bench(0, 0));
     c.bench_function("ProgressSet::insert_voter (3, 1)", bench(3, 1));
     c.bench_function("ProgressSet::insert_voter (5, 2)", bench(5, 2));
     c.bench_function("ProgressSet::insert_voter (7, 3)", bench(7, 3));
 }
 
 pub fn bench_progress_set_insert_learner(c: &mut Criterion) {
     let bench = |voters, learners| {
         move |b: &mut Bencher| {
             let set = quick_progress_set(voters, learners);
             b.iter(|| {
                 let mut set = set.clone();
                 set.insert_learner(99, Default::default()).ok()
             });
         }
     };
     c.bench_function("ProgressSet::insert_learner (0, 0)", bench(0, 0));
     c.bench_function("ProgressSet::insert_learner (3, 1)", bench(3, 1));
     c.bench_function("ProgressSet::insert_learner (5, 2)", bench(5, 2));
     c.bench_function("ProgressSet::insert_learner (7, 3)", bench(7, 3));
 }
 
 pub fn bench_progress_set_remove(c: &mut Criterion) {
     let bench = |voters, learners| {
         move |b: &mut Bencher| {
             let set = quick_progress_set(voters, learners);
             b.iter(|| {
                 let mut set = set.clone();
                 set.remove(3)
             });
         }
     };
     c.bench_function("ProgressSet::remove (0, 0)", bench(0, 0));
     c.bench_function("ProgressSet::remove (3, 1)", bench(3, 1));
     c.bench_function("ProgressSet::remove (5, 2)", bench(5, 2));
     c.bench_function("ProgressSet::remove (7, 3)", bench(7, 3));
 }
 
 pub fn bench_progress_set_promote_learner(c: &mut Criterion) {
     let bench = |voters, learners| {
         move |b: &mut Bencher| {
             let set = quick_progress_set(voters, learners);
             b.iter(|| {
                 let mut set = set.clone();
                 set.promote_learner(3)
             });
         }
     };
     c.bench_function("ProgressSet::promote (0, 0)", bench(0, 0));
     c.bench_function("ProgressSet::promote (3, 1)", bench(3, 1));
     c.bench_function("ProgressSet::promote (5, 2)", bench(5, 2));
     c.bench_function("ProgressSet::promote (7, 3)", bench(7, 3));
 }
 
 pub fn bench_progress_set_iter(c: &mut Criterion) {
     let bench = |voters, learners| {
         move |b: &mut Bencher| {
             let set = quick_progress_set(voters, learners);
             b.iter(|| {
                 let set = set.clone();
                 let agg = set.iter().all(|_| true);
                 agg
             });
         }
     };
     c.bench_function("ProgressSet::iter (0, 0)", bench(0, 0));
     c.bench_function("ProgressSet::iter (3, 1)", bench(3, 1));
     c.bench_function("ProgressSet::iter (5, 2)", bench(5, 2));
     c.bench_function("ProgressSet::iter (7, 3)", bench(7, 3));
 }
 
 pub fn bench_progress_set_nodes(c: &mut Criterion) {
     let bench = |voters, learners| {
         move |b: &mut Bencher| {
             let set = quick_progress_set(voters, learners);
             b.iter(|| {
                 let set = set.clone();
                 let agg = set.iter().all(|_| true);
                 agg
             });
         }
     };
     c.bench_function("ProgressSet::nodes (0, 0)", bench(0, 0));
     c.bench_function("ProgressSet::nodes (3, 1)", bench(3, 1));
     c.bench_function("ProgressSet::nodes (5, 2)", bench(5, 2));
     c.bench_function("ProgressSet::nodes (7, 3)", bench(7, 3));
 }
 
 pub fn bench_progress_set_get(c: &mut Criterion) {
     let bench = |voters, learners| {
         move |b: &mut Bencher| {
             let set = quick_progress_set(voters, learners);
             b.iter(|| {
                 let set = set.clone();
                 {
                     set.get(1);
                 }
             });
         }
     };
     c.bench_function("ProgressSet::get (0, 0)", bench(0, 0));
     c.bench_function("ProgressSet::get (3, 1)", bench(3, 1));
     c.bench_function("ProgressSet::get (5, 2)", bench(5, 2));
     c.bench_function("ProgressSet::get (7, 3)", bench(7, 3));
 }
diff --git a/benches/suites/raft.rs b/benches/suites/raft.rs
@@ -0,0  1,86 @@
 use criterion::{Bencher, Criterion};
 use raft::{storage::MemStorage, Config, Raft};
 
 criterion_group!(bench_raft, bench_raft_new, bench_raft_campaign,);
 
 fn quick_raft(voters: usize, learners: usize) -> Raft<MemStorage> {
     let id = 1;
     let storage = MemStorage::default();
     let config = Config::new(id);
     let mut raft = Raft::new(&config, storage);
     (0..voters).for_each(|id| {
         raft.add_node(id as u64);
     });
     (voters..learners).for_each(|id| {
         raft.add_learner(id as u64);
     });
     raft
 }
 
 pub fn bench_raft_new(c: &mut Criterion) {
     let bench = |voters, learners| {
         move |b: &mut Bencher| {
             // No setup.
             b.iter(|| quick_raft(voters, learners));
         }
     };
 
     c.bench_function("Raft::new (0, 0)", bench(0, 0));
     c.bench_function("Raft::new (3, 1)", bench(3, 1));
     c.bench_function("Raft::new (5, 2)", bench(5, 2));
     c.bench_function("Raft::new (7, 3)", bench(7, 3));
 }
 
 pub fn bench_raft_campaign(c: &mut Criterion) {
     let bench = |voters, learners, variant| {
         move |b: &mut Bencher| {
             b.iter(|| {
                 // TODO: Make raft clone somehow.
                 let mut raft = quick_raft(voters, learners);
                 raft.campaign(variant)
             })
         }
     };
 
     // We don't want to make `raft::raft` public at this point.
     let pre_election = b"CampaignPreElection";
     let election = b"CampaignElection";
     let transfer = b"CampaignTransfer";
 
     c.bench_function(
         "Raft::campaign (3, 1, pre_election)",
         bench(3, 1, &pre_election[..]),
     );
     c.bench_function(
         "Raft::campaign (3, 1, election)",
         bench(3, 1, &election[..]),
     );
     c.bench_function(
         "Raft::campaign (3, 1, transfer)",
         bench(3, 1, &transfer[..]),
     );
     c.bench_function(
         "Raft::campaign (5, 2, pre_election)",
         bench(5, 2, &pre_election[..]),
     );
     c.bench_function(
         "Raft::campaign (5, 2, election)",
         bench(5, 2, &election[..]),
     );
     c.bench_function(
         "Raft::campaign (5, 2, transfer)",
         bench(5, 2, &transfer[..]),
     );
     c.bench_function(
         "Raft::campaign (7, 3, pre_election)",
         bench(7, 3, &pre_election[..]),
     );
     c.bench_function(
         "Raft::campaign (7, 3, election)",
         bench(7, 3, &election[..]),
     );
     c.bench_function(
         "Raft::campaign (7, 3, transfer)",
         bench(7, 3, &transfer[..]),
     );
 }
diff --git a/benches/suites/raw_node.rs b/benches/suites/raw_node.rs
@@ -0,0  1,22 @@
 use criterion::{Bencher, Criterion};
 use raft::{storage::MemStorage, Config, RawNode};
 
 criterion_group!(bench_raw_node, bench_raw_node_new,);
 
 fn quick_raw_node() -> RawNode<MemStorage> {
     let id = 1;
     let peers = vec![];
     let storage = MemStorage::default();
     let config = Config::new(id);
     let node = RawNode::new(&config, storage, peers).unwrap();
     node
 }
 
 pub fn bench_raw_node_new(c: &mut Criterion) {
     let bench = |b: &mut Bencher| {
         // No setup.
         b.iter(|| quick_raw_node());
     };
 
     c.bench_function("RawNode::new", bench);
 }
diff --git a/src/lib.rs b/src/lib.rs
@@ -232,6  232,9 @@ pub mod eraftpb;
 mod errors;
 mod log_unstable;
 mod progress;
 #[cfg(test)]
 pub mod raft;
 #[cfg(not(test))]
 mod raft;
 mod raft_log;
 pub mod raw_node;