Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Parallel formatting to increase speed #6095

Open
wants to merge 23 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Compiles
  • Loading branch information
MarcusGrass committed Feb 26, 2024
commit 7456b978e4caa4da464052b7856753cf66120acf
50 changes: 36 additions & 14 deletions src/bin/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@ use std::str::FromStr;
use std::thread::JoinHandle;

use getopts::{Matches, Options};
use rustfmt_nightly::buf_eprintln;
use rustfmt_nightly::print::Printer;

use crate::rustfmt::{
load_config, CliOptions, Color, Config, Edition, EmitMode, FileLines, FileName,
Expand Down Expand Up @@ -290,7 +292,8 @@ fn format_string(input: String, options: GetOptsOptions) -> Result<i32> {
}

let out = &mut stdout();
let mut session = Session::new(config, Some(out));
let printer = Printer::new(config.color());
let mut session = Session::new(config, Some(out), &printer);
format_and_emit_report(&mut session, Input::Text(input));

let exit_code = if session.has_operational_errors() || session.has_parsing_errors() {
Expand All @@ -301,6 +304,13 @@ fn format_string(input: String, options: GetOptsOptions) -> Result<i32> {
Ok(exit_code)
}

struct ThreadedFileOutput {
session_result: Result<Vec<u8>, std::io::Error>,
id: i32,
exit_code: i32,
printer: Printer,
}

fn format(
files: Vec<PathBuf>,
minimal_config_path: Option<String>,
Expand All @@ -326,17 +336,18 @@ fn format(
let mut id = 0;
let mut handles: HashMap<i32, JoinHandle<_>> = HashMap::new();
let check = options.check;

let color = config.color();
for file in files {
let cfg = config.clone();
let opts = options.clone();
let s = send.clone();
let handle = std::thread::spawn(move || {
let my_id = id;
let mut session_out = Vec::new();
let mut session = Session::new(cfg, Some(&mut session_out));
let printer = Printer::new(color);
let mut session = Session::new(cfg, Some(&mut session_out), &printer);
if !file.exists() {
eprintln!("Error: file `{}` does not exist", file.to_str().unwrap());
buf_eprintln!(printer, "Error: file `{}` does not exist", file.to_str().unwrap());
session.add_operational_error();
} else if file.is_dir() {
eprintln!("Error: `{}` is a directory", file.to_str().unwrap());
Expand All @@ -348,7 +359,13 @@ fn format(
match load_config(Some(file.parent().unwrap()), Some(opts)) {
Ok((lc, cf)) => (lc, cf),
Err(e) => {
let _ = s.send((my_id, Err(e)));
drop(session);
let _ = s.send(ThreadedFileOutput {
session_result: Err(e),
id,
exit_code,
printer,
});
return Ok::<_, std::io::Error>(my_id);
}
};
Expand Down Expand Up @@ -378,20 +395,25 @@ fn format(
0
};
drop(session);
let _ = s.send((my_id, Ok((session_out, exit_code))));
let _ = s.send(ThreadedFileOutput {
session_result: Ok(session_out),
id,
exit_code,
printer,
});
Ok(my_id)
});
handles.insert(id, handle);
id += 1;
outstanding += 1;
if outstanding >= num_cpus {
if let Ok((id, res)) = recv.recv() {
if let Ok(thread_out) = recv.recv() {
handles.remove(&id).unwrap().join().unwrap().unwrap();
let (output, exit) = res?;
let output = thread_out.session_result?;
let out = &mut stdout();
out.write_all(&output).unwrap();
if exit != 0 {
exit_code = exit;
if thread_out.exit_code != 0 {
exit_code = thread_out.exit_code;
}
outstanding -= 1;
} else {
Expand All @@ -401,12 +423,12 @@ fn format(
}
drop(send);
let out = &mut stdout();
while let Ok((id, res)) = recv.recv() {
while let Ok(thread_out) = recv.recv() {
handles.remove(&id).unwrap().join().unwrap().unwrap();
let (output, exit) = res?;
let output = thread_out.session_result?;
out.write_all(&output).unwrap();
if exit != 0 {
exit_code = exit;
if thread_out.exit_code != 0 {
exit_code = thread_out.exit_code;
}
}
// These have errors
Expand Down
4 changes: 3 additions & 1 deletion src/git-rustfmt/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ use std::str::FromStr;
use getopts::{Matches, Options};
use rustfmt_nightly as rustfmt;
use tracing_subscriber::EnvFilter;
use rustfmt_nightly::print::Printer;

use crate::rustfmt::{load_config, CliOptions, FormatReportFormatterBuilder, Input, Session};

Expand Down Expand Up @@ -63,7 +64,8 @@ fn fmt_files(files: &[&str]) -> i32 {

let mut exit_code = 0;
let mut out = stdout();
let mut session = Session::new(config, Some(&mut out));
let printer = Printer::new(config.color());
let mut session = Session::new(config, Some(&mut out), &printer);
for file in files {
let report = session.format(Input::File(PathBuf::from(file))).unwrap();
if report.has_warnings() {
Expand Down
2 changes: 1 addition & 1 deletion src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ mod test;
mod types;
mod vertical;
pub(crate) mod visitor;
mod print;
pub mod print;

/// The various errors that can occur during formatting. Note that not all of
/// these can currently be propagated to clients.
Expand Down
15 changes: 12 additions & 3 deletions src/print.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,15 @@ struct PrinterInner {
}

impl Printer {

pub fn new(term_output_color: Color) -> Self {
Self {
inner: Mutex::new(PrinterInner {
color: term_output_color,
messages: vec![],
}),
}
}
#[inline]
pub fn push_msg(&self, msg: PrintMessage) {
self.inner.lock().unwrap().messages.push(msg);
Expand All @@ -22,7 +31,7 @@ macro_rules! buf_println {
($pb: expr, $($arg:tt)*) => {{
let mut msg_buf = Vec::new();
let _ = writeln!(&mut msg_buf, $($arg)*);
$pb.push_msg(PrintMessage::Stdout(msg_buf));
$pb.push_msg($crate::print::PrintMessage::Stdout(msg_buf));
}};
}

Expand All @@ -31,7 +40,7 @@ macro_rules! buf_eprintln {
($pb: expr, $($arg:tt)*) => {{
let mut msg_buf = Vec::new();
let _ = writeln!(&mut msg_buf, $($arg)*);
$pb.push_msg(PrintMessage::StdErr(msg_buf));
$pb.push_msg($crate::print::PrintMessage::StdErr(msg_buf));
}};
}

Expand All @@ -40,7 +49,7 @@ macro_rules! buf_term_println {
($pb: expr, $col:expr, $($arg:tt)*) => {{
let mut msg_buf = Vec::new();
let _ = writeln!(&mut msg_buf, $($arg)*);
$pb.push_msg(PrintMessage::Term(TermMessage::new(msg_buf, $col)));
$pb.push_msg($crate::print::PrintMessage::Term(TermMessage::new(msg_buf, $col)));
}};
}

Expand Down