Add basic zstd support (#44)

* add zstd support on basic module * Bump version to 2.4.0
luizirber · Dec 25, 2021 · ff028d6 · ff028d6
1 parent 8ecd60f
commit ff028d6
Show file tree

Hide file tree

Showing 5 changed files with 115 additions and 4 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
 ## [2.4.0] - 2021-12-25
 
 ### Added
 
 - Zstd basic support (#44)
 - GitHub issues templates (#47)
 
 ### Changed
 
 - Bump MSRV to 1.51 (#48)
 
 ## [2.3.2] - 2021-05-27
 
 ### Changed
@@ -78,7 89,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Rename crate from `ocf` to `niffler`
 - Import codebase from sourmash repo (which copied it from the yacrd repo)
 
-[unreleased]: https://github.com/luizirber/niffler/compare/v2.3.0...HEAD
 [unreleased]: https://github.com/luizirber/niffler/compare/v2.4.0...HEAD
 [2.4.0]: https://github.com/luizirber/niffler/compare/v2.3.2..v2.4.0
 [2.3.2]: https://github.com/luizirber/niffler/compare/v2.3.1..v2.3.2
 [2.3.1]: https://github.com/luizirber/niffler/compare/v2.3.0..v2.3.1
 [2.3.0]: https://github.com/luizirber/niffler/compare/v2.2.0..v2.3.0
 [2.2.0]: https://github.com/luizirber/niffler/compare/v2.0.1..v2.2.0
 [2.0.1]: https://github.com/luizirber/niffler/compare/v2.0.0..v2.0.1

diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 1,6 @@
 [package]
 name = "niffler"
-version = "2.3.2"
 version = "2.4.0"
 authors = ["Pierre Marijon <[email protected]>", "Luiz Irber <[email protected]>"]
 description = "Simple and transparent support for compressed files"
 license = "MIT/Apache-2.0"
@@ -11,7 11,7 @@ documentation = "https://docs.rs/niffler"
 edition = "2018"
 
 [features]
-default = ["bz2", "lzma", "xz", "gz", "bgz"]
 default = ["bz2", "lzma", "xz", "gz", "bgz", "zstd"]
 bz2 = ["bzip2"]
 lzma = ["xz2"]
 gz = ["flate2"]
@@ -25,6 25,7 @@ bzip2 = { version = "0.4.1", optional = true }
 flate2 = { version = "1.0", optional = true }
 xz2 = { version = "0.1", optional = true }
 bgzip = { version = "0.2.0", optional = true }
 zstd = { version = "0.7.0", optional = true }
 
 [dev-dependencies]
 tempfile = "3"

diff --git a/src/basic/compression.rs b/src/basic/compression.rs
@@ -19,6 19,7 @@ pub enum Format {
  Gzip,
  Bzip,
  Lzma,
  Zstd,
  No,
 }
 
@@ -41,6 42,7 @@ pub(crate) fn bytes2type(bytes: [u8; 5]) -> Format {
  match bytes {
  [0x1f, 0x8b, ..] => Format::Gzip,
  [0x42, 0x5a, ..] => Format::Bzip,
  [0x28, 0xb5, 0x2f, 0xfd, ..] => Format::Zstd,
  [0xfd, 0x37, 0x7a, 0x58, 0x5a] => Format::Lzma,
  _ => Format::No,
  }
@@ -126,3 128,30 @@ cfg_if! {
  }
  }
 }
 
 cfg_if! {
  if #[cfg(feature = "zstd")] {
  pub(crate) fn new_zstd_encoder<'a>(out: Box<dyn io::Write 'a>, level: Level) -> Result<Box<dyn io::Write 'a>, Error> {
  Ok(Box::new(zstd::stream::write::Encoder::new(
  out,
  level.into(),
  )?.auto_finish()))
  }
 
  pub(crate) fn new_zstd_decoder<'a>(
  inp: Box<dyn io::Read 'a>,
  ) -> Result<(Box<dyn io::Read 'a>, Format), Error> {
  Ok((Box::new(zstd::stream::read::Decoder::new(inp)?),
  Format::Zstd,
  ))
  }
  } else {
  pub(crate) fn new_zstd_encoder<'a>(_: Box<dyn io::Write 'a>, _: Level) -> Result<Box<dyn io::Write 'a>, Error> {
  Err(Error::FeatureDisabled)
  }
 
  pub(crate) fn new_zstd_decoder<'a>(_: Box<dyn io::Read 'a>) -> Result<(Box<dyn io::Read 'a>, Format), Error> {
  Err(Error::FeatureDisabled)
  }
  }
 }
diff --git a/src/basic/mod.rs b/src/basic/mod.rs
@@ -43,7 43,8 @@ pub fn sniff<'a>(
  match compression::bytes2type(first_bytes) {
  e @ compression::Format::Gzip
  | e @ compression::Format::Bzip
- | e @ compression::Format::Lzma => Ok((Box::new(cursor.chain(in_stream)), e)),
  | e @ compression::Format::Lzma
  | e @ compression::Format::Zstd => Ok((Box::new(cursor.chain(in_stream)), e)),
  _ => Ok((Box::new(cursor.chain(in_stream)), compression::Format::No)),
  }
 }
@@ -85,6 86,7 @@ pub fn get_reader<'a>(
  compression::Format::Gzip => compression::new_gz_decoder(in_stream),
  compression::Format::Bzip => compression::new_bz2_decoder(in_stream),
  compression::Format::Lzma => compression::new_lzma_decoder(in_stream),
  compression::Format::Zstd => compression::new_zstd_decoder(in_stream),
  compression::Format::No => Ok((in_stream, compression::Format::No)),
  }
 }
@@ -125,6 127,7 @@ pub fn get_writer<'a>(
  compression::Format::Gzip => compression::new_gz_encoder(out_stream, level),
  compression::Format::Bzip => compression::new_bz2_encoder(out_stream, level),
  compression::Format::Lzma => compression::new_lzma_encoder(out_stream, level),
  compression::Format::Zstd => compression::new_zstd_encoder(out_stream, level),
  compression::Format::No => Ok(Box::new(out_stream)),
  }
 }
@@ -199,6 202,7 @@ mod test {
  pub(crate) const GZIP_FILE: &'static [u8] = &[0o037, 0o213, 0o0, 0o0, 0o0];
  pub(crate) const BZIP_FILE: &'static [u8] = &[0o102, 0o132, 0o0, 0o0, 0o0];
  pub(crate) const LZMA_FILE: &'static [u8] = &[0o375, 0o067, 0o172, 0o130, 0o132];
  pub(crate) const ZSTD_FILE: &'static [u8] = &[0x28, 0xb5, 0x2f, 0xfd, 0];
  pub(crate) const LOREM_IPSUM: &'static [u8] = b"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Ut ultricies scelerisque diam, a scelerisque enim sagittis at.";
 
  mod compress_uncompress {
@@ -379,6 383,47 @@ mod test {
  .expect("Error during reading");
  assert_eq!(LOREM_IPSUM, buffer.as_slice());
  }
 
  #[test]
  #[cfg(not(feature = "zstd"))]
  fn no_zstd_feature() {
  assert!(
  get_writer(Box::new(vec![]), compression::Format::Zstd, Level::Six).is_err(),
  "zstd disabled, this assertion should fail"
  );
 
  assert!(
  get_reader(Box::new(&ZSTD_FILE[..])).is_err(),
  "zstd disabled, this assertion should fail"
  );
  }
 
  #[cfg(feature = "zstd")]
  #[test]
  fn zstd() {
  let ofile = NamedTempFile::new().expect("Can't create tmpfile");
 
  {
  let wfile = ofile.reopen().expect("Can't create tmpfile");
  let mut writer =
  get_writer(Box::new(wfile), compression::Format::Zstd, Level::Six).unwrap();
  writer
  .write_all(LOREM_IPSUM)
  .expect("Error during write of data");
  }
 
  let rfile = ofile.reopen().expect("Can't create tmpfile");
  let (mut reader, compression) =
  get_reader(Box::new(rfile)).expect("Error reading from tmpfile");
 
  assert_eq!(compression, compression::Format::Zstd);
 
  let mut buffer = Vec::new();
  reader
  .read_to_end(&mut buffer)
  .expect("Error during reading");
  assert_eq!(LOREM_IPSUM, buffer.as_slice());
  }
  }
 
  mod compression_format_detection {
@@ -408,6 453,12 @@ mod test {
  assert_eq!(compression, compression::Format::Xz);
  }
 
  #[test]
  fn zstd() {
  let (_, compression) = sniff(Box::new(ZSTD_FILE)).expect("Error in read file");
  assert_eq!(compression, compression::Format::Zstd);
  }
 
  #[test]
  fn too_short() {
  let result = sniff(Box::new(SHORT_FILE));

diff --git a/src/level.rs b/src/level.rs
@@ -35,6 35,22 @@ impl From<Level> for u32 {
  }
 }
 
 impl From<Level> for i32 {
  fn from(level: Level) -> Self {
  match level {
  Level::One => 1,
  Level::Two => 2,
  Level::Three => 3,
  Level::Four => 4,
  Level::Five => 5,
  Level::Six => 6,
  Level::Seven => 7,
  Level::Eight => 8,
  Level::Nine => 9,
  }
  }
 }
 
 #[cfg(feature = "gz")]
 impl From<Level> for flate2::Compression {
  fn from(level: Level) -> Self {