Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add typst-kit crate #4540

Merged
merged 4 commits into from
Aug 5, 2024
Merged

Add typst-kit crate #4540

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift click to select a range
47c62db
Add typst-kit crate
tingerrr Jul 11, 2024
1998356
Move font searching to typst-kit
tingerrr Jul 11, 2024
77fe1a5
Move package downloads to typst-kit
tingerrr Jul 11, 2024
690d7d4
Add feature flags to typst-kit
tingerrr Jul 19, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 21 additions & 4 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 19,7 @@ readme = "README.md"
typst = { path = "crates/typst", version = "0.11.0" }
typst-cli = { path = "crates/typst-cli", version = "0.11.0" }
typst-ide = { path = "crates/typst-ide", version = "0.11.0" }
typst-kit = { path = "crates/typst-kit", version = "0.11.0" }
typst-macros = { path = "crates/typst-macros", version = "0.11.0" }
typst-pdf = { path = "crates/typst-pdf", version = "0.11.0" }
typst-render = { path = "crates/typst-render", version = "0.11.0" }
Expand Down
18 changes: 4 additions & 14 deletions crates/typst-cli/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 20,7 @@ doc = false
[dependencies]
typst = { workspace = true }
typst-assets = { workspace = true, features = ["fonts"] }
typst-kit = { workspace = true }
typst-macros = { workspace = true }
typst-pdf = { workspace = true }
typst-render = { workspace = true }
Expand All @@ -31,9 32,6 @@ codespan-reporting = { workspace = true }
comemo = { workspace = true }
dirs = { workspace = true }
ecow = { workspace = true }
env_proxy = { workspace = true }
flate2 = { workspace = true }
fontdb = { workspace = true, features = ["memmap", "fontconfig"] }
fs_extra = { workspace = true }
native-tls = { workspace = true }
notify = { workspace = true }
Expand All @@ -56,11 54,6 @@ ureq = { workspace = true }
xz2 = { workspace = true, optional = true }
zip = { workspace = true, optional = true }

# Explicitly depend on OpenSSL if applicable, so that we can add the
# `openssl/vendored` feature to it if `vendor-openssl` is enabled.
[target.'cfg(not(any(target_os = "windows", target_os = "macos", target_os = "ios", target_os = "watchos", target_os = "tvos")))'.dependencies]
openssl = { workspace = true }

[build-dependencies]
chrono = { workspace = true }
clap = { workspace = true, features = ["string"] }
Expand All @@ -71,17 64,14 @@ semver = { workspace = true }
[features]
default = ["embed-fonts"]

# Embeds some fonts into the binary:
# - For text: Linux Libertine, New Computer Modern
# - For math: New Computer Modern Math
# - For code: Deja Vu Sans Mono
embed-fonts = []
# Embeds some fonts into the binary, see typst-kit
embed-fonts = ["typst-kit/embed-fonts"]

# Permits the CLI to update itself without a package manager.
self-update = ["dep:self-replace", "dep:xz2", "dep:zip"]

# Whether to vendor OpenSSL. Not applicable to Windows and macOS builds.
vendor-openssl = ["openssl/vendored"]
vendor-openssl = ["typst-kit/vendor-openssl"]

[lints]
workspace = true
Expand Down
263 changes: 74 additions & 189 deletions crates/typst-cli/src/download.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,207 1,92 @@
// Acknowledgement:
// Closely modelled after rustup's `DownloadTracker`.
// https://github.com/rust-lang/rustup/blob/master/src/cli/download_tracker.rs

use std::collections::VecDeque;
use std::io::{self, ErrorKind, Read, Write};
use std::sync::Arc;
use std::fmt::Display;
use std::io;
use std::io::Write;
use std::time::{Duration, Instant};

use native_tls::{Certificate, TlsConnector};
use once_cell::sync::OnceCell;
use ureq::Response;

use crate::terminal;

/// Keep track of this many download speed samples.
const SPEED_SAMPLES: usize = 5;

/// Load a certificate from the file system if the `--cert` argument or
/// `TYPST_CERT` environment variable is present. The certificate is cached for
/// efficiency.
///
/// - Returns `None` if `--cert` and `TYPST_CERT` are not set.
/// - Returns `Some(Ok(cert))` if the certificate was loaded successfully.
/// - Returns `Some(Err(err))` if an error occurred while loading the certificate.
fn cert() -> Option<Result<&'static Certificate, io::Error>> {
static CERT: OnceCell<Certificate> = OnceCell::new();
crate::ARGS.cert.as_ref().map(|path| {
CERT.get_or_try_init(|| {
let pem = std::fs::read(path)?;
Certificate::from_pem(&pem).map_err(io::Error::other)
})
})
}
use codespan_reporting::term;
use codespan_reporting::term::termcolor::WriteColor;
use typst_kit::download::{DownloadState, Downloader, Progress};

/// Download binary data and display its progress.
#[allow(clippy::result_large_err)]
pub fn download_with_progress(url: &str) -> Result<Vec<u8>, ureq::Error> {
let response = download(url)?;
Ok(RemoteReader::from_response(response).download()?)
}
use crate::terminal::{self, TermOut};
use crate::ARGS;

/// Download from a URL.
#[allow(clippy::result_large_err)]
pub fn download(url: &str) -> Result<ureq::Response, ureq::Error> {
let mut builder = ureq::AgentBuilder::new();
let mut tls = TlsConnector::builder();

// Set user agent.
builder = builder.user_agent(concat!("typst/", env!("CARGO_PKG_VERSION")));

// Get the network proxy config from the environment and apply it.
if let Some(proxy) = env_proxy::for_url_str(url)
.to_url()
.and_then(|url| ureq::Proxy::new(url).ok())
{
builder = builder.proxy(proxy);
}
/// Prints download progress by writing `downloading {0}` followed by repeatedly
/// updating the last terminal line.
pub struct PrintDownload<T>(pub T);

// Apply a custom CA certificate if present.
if let Some(cert) = cert() {
tls.add_root_certificate(cert?.clone());
}
impl<T: Display> Progress for PrintDownload<T> {
fn print_start(&mut self) {
// Print that a package downloading is happening.
let styles = term::Styles::default();

// Configure native TLS.
let connector =
tls.build().map_err(|err| io::Error::new(io::ErrorKind::Other, err))?;
builder = builder.tls_connector(Arc::new(connector));

builder.build().get(url).call()
}

/// A wrapper around [`ureq::Response`] that reads the response body in chunks
/// over a websocket and displays statistics about its progress.
///
/// Downloads will _never_ fail due to statistics failing to print, print errors
/// are silently ignored.
struct RemoteReader {
reader: Box<dyn Read Send Sync 'static>,
content_len: Option<usize>,
total_downloaded: usize,
downloaded_this_sec: usize,
downloaded_last_few_secs: VecDeque<usize>,
start_time: Instant,
last_print: Option<Instant>,
}
let mut out = terminal::out();
let _ = out.set_color(&styles.header_help);
let _ = write!(out, "downloading");

impl RemoteReader {
/// Wraps a [`ureq::Response`] and prepares it for downloading.
///
/// The 'Content-Length' header is used as a size hint for read
/// optimization, if present.
pub fn from_response(response: Response) -> Self {
let content_len: Option<usize> = response
.header("Content-Length")
.and_then(|header| header.parse().ok());

Self {
reader: response.into_reader(),
content_len,
total_downloaded: 0,
downloaded_this_sec: 0,
downloaded_last_few_secs: VecDeque::with_capacity(SPEED_SAMPLES),
start_time: Instant::now(),
last_print: None,
}
let _ = out.reset();
let _ = writeln!(out, " {}", self.0);
}

/// Download the bodies content as raw bytes while attempting to print
/// download statistics to standard error. Download progress gets displayed
/// and updated every second.
///
/// These statistics will never prevent a download from completing, errors
/// are silently ignored.
pub fn download(mut self) -> io::Result<Vec<u8>> {
let mut buffer = vec![0; 8192];
let mut data = match self.content_len {
Some(content_len) => Vec::with_capacity(content_len),
None => Vec::with_capacity(8192),
};

loop {
let read = match self.reader.read(&mut buffer) {
Ok(0) => break,
Ok(n) => n,
// If the data is not yet ready but will be available eventually
// keep trying until we either get an actual error, receive data
// or an Ok(0).
Err(ref e) if e.kind() == ErrorKind::Interrupted => continue,
Err(e) => return Err(e),
};

data.extend(&buffer[..read]);

let last_printed = match self.last_print {
Some(prev) => prev,
None => {
let current_time = Instant::now();
self.last_print = Some(current_time);
current_time
}
};
let elapsed = Instant::now().saturating_duration_since(last_printed);

self.total_downloaded = read;
self.downloaded_this_sec = read;

if elapsed >= Duration::from_secs(1) {
if self.downloaded_last_few_secs.len() == SPEED_SAMPLES {
self.downloaded_last_few_secs.pop_back();
}

self.downloaded_last_few_secs.push_front(self.downloaded_this_sec);
self.downloaded_this_sec = 0;

terminal::out().clear_last_line()?;
self.display()?;
self.last_print = Some(Instant::now());
}
}

self.display()?;
writeln!(&mut terminal::out())?;
fn print_progress(&mut self, state: &DownloadState) {
let mut out = terminal::out();
let _ = out.clear_last_line();
let _ = display_download_progress(&mut out, state);
}

Ok(data)
fn print_finish(&mut self, state: &DownloadState) {
let mut out = terminal::out();
let _ = display_download_progress(&mut out, state);
let _ = writeln!(out);
}
}

/// Compile and format several download statistics and make an attempt at
/// displaying them on standard error.
fn display(&mut self) -> io::Result<()> {
let sum: usize = self.downloaded_last_few_secs.iter().sum();
let len = self.downloaded_last_few_secs.len();
let speed = if len > 0 { sum / len } else { self.content_len.unwrap_or(0) };

let total_downloaded = as_bytes_unit(self.total_downloaded);
let speed_h = as_throughput_unit(speed);
let elapsed =
time_suffix(Instant::now().saturating_duration_since(self.start_time));

match self.content_len {
Some(content_len) => {
let percent = (self.total_downloaded as f64 / content_len as f64) * 100.;
let remaining = content_len - self.total_downloaded;

let download_size = as_bytes_unit(content_len);
let eta = time_suffix(Duration::from_secs(if speed == 0 {
0
} else {
(remaining / speed) as u64
}));
writeln!(
terminal::out(),
"{total_downloaded} / {download_size} ({percent:3.0} %) {speed_h} in {elapsed} ETA: {eta}",
)?;
}
None => writeln!(
terminal::out(),
"Total downloaded: {total_downloaded} Speed: {speed_h} Elapsed: {elapsed}",
)?,
};
Ok(())
/// Returns a new downloader.
pub fn downloader() -> Downloader {
let user_agent = concat!("typst/", env!("CARGO_PKG_VERSION"));
match ARGS.cert.clone() {
Some(cert) => Downloader::with_path(user_agent, cert),
None => Downloader::new(user_agent),
}
}

/// Compile and format several download statistics and make and attempt at
/// displaying them on standard error.
pub fn display_download_progress(
out: &mut TermOut,
state: &DownloadState,
) -> io::Result<()> {
let sum: usize = state.bytes_per_second.iter().sum();
let len = state.bytes_per_second.len();
let speed = if len > 0 { sum / len } else { state.content_len.unwrap_or(0) };

let total_downloaded = as_bytes_unit(state.total_downloaded);
let speed_h = as_throughput_unit(speed);
let elapsed = time_suffix(Instant::now().saturating_duration_since(state.start_time));

match state.content_len {
Some(content_len) => {
let percent = (state.total_downloaded as f64 / content_len as f64) * 100.;
let remaining = content_len - state.total_downloaded;

let download_size = as_bytes_unit(content_len);
let eta = time_suffix(Duration::from_secs(if speed == 0 {
0
} else {
(remaining / speed) as u64
}));
writeln!(
out,
"{total_downloaded} / {download_size} ({percent:3.0} %) {speed_h} in {elapsed} ETA: {eta}",
)?;
}
None => writeln!(
out,
"Total downloaded: {total_downloaded} Speed: {speed_h} Elapsed: {elapsed}",
)?,
};
Ok(())
}

/// Append a unit-of-time suffix.
fn time_suffix(duration: Duration) -> String {
let secs = duration.as_secs();
Expand Down
Loading