narwhal: manually build grpc interface

Manually build grpc interface instead of leveraging protoc in order to eliminate needing to build the expensive protoc compiler.
MystenLabs · Aug 1, 2024 · d3c7e0e · d3c7e0e
1 parent 44150d4
commit d3c7e0e
Show file tree

Hide file tree

Showing 8 changed files with 36 additions and 104 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -112,13  112,6 @@ jobs:
         if: ${{ matrix.os == 'windows-ghcloud' && env.s3_archive_exist == '' }}
         uses: taiki-e/install-action@33022ba120c3f523d134bbbee12278fc11a3df1a # pin@nextest
 
-      - name: Setup protoc (Windows)
-        if: ${{ matrix.os == 'windows-ghcloud' && env.s3_archive_exist == '' }}
-        uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # [email protected]
-        # this avoids rate-limiting
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
-
       - name: Install postgres (Windows)
         if: ${{ matrix.os == 'windows-ghcloud' && env.s3_archive_exist == '' }}
         shell: bash

diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -200,10  200,6 @@ jobs:
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
       - uses: taiki-e/install-action@nextest
-      - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # [email protected]
-        # this avoids rate-limiting
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Install postgres (Windows)
         shell: bash
@@ -343,10  339,6 @@ jobs:
     if: needs.diff.outputs.isRust == 'true'
     runs-on: [ ubuntu-ghcloud ]
     steps:
-      - uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # [email protected]
-        # this avoids rate-limiting
-        with:
-          repo-token: ${{ secrets.GITHUB_TOKEN }}
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # Pin v4.1.1
       - run: rustup component add clippy
       # TODO(bradh): debug and re-enable this; the caching is breaking the clippy build

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -408,7  408,6 @@ proptest-derive = "0.3.0"
 prost = "0.13"
 prost-build = "0.13"
 protobuf = { version = "2.28", features = ["with-bytes"] }
-protobuf-src = "1.1.0"
 quinn-proto = "0.11"
 quote = "1.0.23"
 rand = "0.8.5"

diff --git a/narwhal/types/Cargo.toml b/narwhal/types/Cargo.toml
@@ -47,12  47,7 @@ criterion.workspace = true
 serde_test.workspace = true
 test-utils = { path = "../test-utils", package = "narwhal-test-utils" }
 
-[target.'cfg(not(target_env = "msvc"))'.build-dependencies]
-protobuf-src.workspace = true
-
 [build-dependencies]
-prost-build.workspace = true
-rustversion.workspace = true
 tonic-build.workspace = true
 anemo-build.workspace = true
 

diff --git a/narwhal/types/build.rs b/narwhal/types/build.rs
@@ -9,36  9,47 @@ use std::{
 type Result<T> = std::result::Result<T, Box<dyn std::error::Error   Send   Sync>>;
 
 fn main() -> Result<()> {
-    #[cfg(not(target_env = "msvc"))]
-    std::env::set_var("PROTOC", protobuf_src::protoc());
-
     let out_dir = if env::var("DUMP_GENERATED_GRPC").is_ok() {
         PathBuf::from("")
     } else {
         PathBuf::from(env::var("OUT_DIR")?)
     };
 
-    let proto_files = &["proto/narwhal.proto"];
-    let dirs = &["proto"];
     let codec_path = "tonic::codec::ProstCodec";
 
-    // Use `Bytes` instead of `Vec<u8>` for bytes fields
-    let mut config = prost_build::Config::new();
-    config.bytes(["."]);
     let service = tonic_build::manual::Service::builder()
         .name("Transactions")
         .package("narwhal")
         .method(
             tonic_build::manual::Method::builder()
                 .name("submit_transaction")
                 .route_name("SubmitTransaction")
                 .input_type("crate::proto::narwhal::Transaction")
                 .output_type("crate::proto::narwhal::Empty")
                 .codec_path(codec_path)
                 .build(),
         )
         .method(
             tonic_build::manual::Method::builder()
                 .name("submit_transaction_stream")
                 .route_name("SubmitTransactionStream")
                 .input_type("crate::proto::narwhal::Transaction")
                 .output_type("crate::proto::narwhal::Empty")
                 .codec_path(codec_path)
                 .client_streaming()
                 .build(),
         )
         .build();
 
-    tonic_build::configure()
     tonic_build::manual::Builder::new()
         .out_dir(&out_dir)
-        .compile_with_config(config, proto_files, dirs)?;
         .compile(&[service]);
 
     build_anemo_services(&out_dir);
 
     println!("cargo:rerun-if-changed=build.rs");
-    println!("cargo:rerun-if-changed=proto");
     println!("cargo:rerun-if-env-changed=DUMP_GENERATED_GRPC");
 
-    nightly();
-    beta();
-    stable();
-
     Ok(())
 }
 
@@ -162,27  173,3 @@ fn build_anemo_services(out_dir: &Path) {
             worker_to_worker,
         ]);
 }
-
-#[rustversion::nightly]
-fn nightly() {
-    println!("cargo:rustc-cfg=nightly");
-}
-
-#[rustversion::not(nightly)]
-fn nightly() {}
-
-#[rustversion::beta]
-fn beta() {
-    println!("cargo:rustc-cfg=beta");
-}
-
-#[rustversion::not(beta)]
-fn beta() {}
-
-#[rustversion::stable]
-fn stable() {
-    println!("cargo:rustc-cfg=stable");
-}
-
-#[rustversion::not(stable)]
-fn stable() {}
diff --git a/narwhal/types/proto/narwhal.proto b/narwhal/types/proto/narwhal.proto
diff --git a/narwhal/types/src/proto.rs b/narwhal/types/src/proto.rs
@@ -1,8  1,16 @@
 // Copyright (c) Mysten Labs, Inc.
 // SPDX-License-Identifier: Apache-2.0
 mod narwhal {
-    #![allow(clippy::derive_partial_eq_without_eq)]
-    tonic::include_proto!("narwhal");
     #[derive(Clone, PartialEq, ::prost::Message)]
     pub struct Transaction {
         #[prost(bytes = "bytes", repeated, tag = "1")]
         pub transactions: ::prost::alloc::vec::Vec<::prost::bytes::Bytes>,
     }
     /// Empty message for when we don't have anything to return
     #[derive(Clone, Copy, PartialEq, ::prost::Message)]
     pub struct Empty {}
 
     include!(concat!(env!("OUT_DIR"), "/narwhal.Transactions.rs"));
 
     include!(concat!(env!("OUT_DIR"), "/narwhal.PrimaryToPrimary.rs"));
     include!(concat!(env!("OUT_DIR"), "/narwhal.PrimaryToWorker.rs"));