code.octet-stream.net Git - hashgood/blob - src/calculate.rs

   1 use super::Algorithm;
   2 use crossbeam_channel::bounded;
   3 use crossbeam_channel::Receiver;
   4 use md5::{Digest, Md5};
   5 use sha1::Sha1;
   6 use sha2::Sha256;
   7 use std::error::Error;
   8 use std::fs::File;
   9 use std::io::prelude::*;
  10 use std::path::Path;
  11 use std::sync::Arc;
  12 use std::thread;
  13 use std::thread::JoinHandle;
  14
  15 pub type CalculateResult = Result<Vec<(Algorithm, Vec<u8>)>, Box<dyn Error>>;
  16
  17 /// For a given path to the input (may be "-" for STDIN), try to obtain a reader for the data within it.
  18 pub fn get_input_reader(input: &Path) -> Result<Box<dyn Read>, String> {
  19     if input.to_str() == Some("-") {
  20         // Special case: standard input
  21         return Ok(Box::new(std::io::stdin()));
  22     }
  23     if !input.exists() {
  24         return Err(format!(
  25             "The path '{}' does not exist.",
  26             input.to_string_lossy()
  27         ));
  28     }
  29     if !input.is_file() {
  30         return Err(format!(
  31             "The path '{}' is not a regular file.",
  32             input.to_string_lossy()
  33         ));
  34     }
  35     match File::open(input) {
  36         Ok(f) => Ok(Box::new(f)),
  37         Err(e) => Err(format!("File open: {}", e)),
  38     }
  39 }
  40
  41 /// For the given input stream, calculate all requested digest types
  42 pub fn create_digests(algorithms: &[Algorithm], mut input: Box<dyn Read>) -> CalculateResult {
  43     let mut senders = vec![];
  44     let mut handles = vec![];
  45
  46     if algorithms.contains(&Algorithm::Md5) {
  47         let (s, r) = bounded::<Arc<Vec<u8>>>(1);
  48         senders.push(s);
  49         handles.push(md5_digest(r));
  50     }
  51     if algorithms.contains(&Algorithm::Sha1) {
  52         let (s, r) = bounded::<Arc<Vec<u8>>>(1);
  53         senders.push(s);
  54         handles.push(sha1_digest(r));
  55     }
  56     if algorithms.contains(&Algorithm::Sha256) {
  57         let (s, r) = bounded::<Arc<Vec<u8>>>(1);
  58         senders.push(s);
  59         handles.push(sha256_digest(r));
  60     }
  61
  62     // 64 KB chunks will be read from the input at 64 KB and supplied to all hashing threads at once
  63     // Right now that could be up to three threads. If CPU-bound, the other threads will mostly block while the slowest one finishes
  64     const BUF_SIZE: usize = 1024 * 64;
  65     let mut buf = [0; BUF_SIZE];
  66     while let Ok(size) = input.read(&mut buf) {
  67         if size == 0 {
  68             break;
  69         } else {
  70             // Create a shared read-only copy for the hashers to take as input
  71             // buf is freed up for more reading
  72             let chunk = Arc::new(buf[0..size].to_vec());
  73             for s in &senders {
  74                 s.send(chunk.clone())?;
  75             }
  76         }
  77     }
  78     drop(senders);
  79     // Once all data has been sent we just have to wait for the digests to fall out
  80     Ok(handles.into_iter().map(|h| h.join().unwrap()).collect())
  81 }
  82
  83 /// Calculate the md5 digest of some data on the given channel
  84 fn md5_digest(rx: Receiver<Arc<Vec<u8>>>) -> JoinHandle<(Algorithm, Vec<u8>)> {
  85     thread::spawn(move || {
  86         let mut md5 = Md5::new();
  87         while let Ok(chunk) = rx.recv() {
  88             md5.update(&*chunk);
  89         }
  90         let result = md5.finalize();
  91         (Algorithm::Md5, result.to_vec())
  92     })
  93 }
  94
  95 /// Calculate the sha1 digest of some data on the given channel
  96 fn sha1_digest(rx: Receiver<Arc<Vec<u8>>>) -> JoinHandle<(Algorithm, Vec<u8>)> {
  97     thread::spawn(move || {
  98         let mut sha1 = Sha1::new();
  99         while let Ok(chunk) = rx.recv() {
 100             sha1.update(&*chunk);
 101         }
 102         let result = sha1.finalize();
 103         (Algorithm::Sha1, result.to_vec())
 104     })
 105 }
 106
 107 /// Calculate the sha256 digest of some data on the given channel
 108 fn sha256_digest(rx: Receiver<Arc<Vec<u8>>>) -> JoinHandle<(Algorithm, Vec<u8>)> {
 109     thread::spawn(move || {
 110         let mut sha256 = Sha256::new();
 111         while let Ok(chunk) = rx.recv() {
 112             sha256.update(&*chunk);
 113         }
 114         let result = sha256.finalize();
 115         (Algorithm::Sha256, result.to_vec())
 116     })
 117 }
 118
 119 #[cfg(test)]
 120 mod tests {
 121     use super::*;
 122     use std::io::Cursor;
 123
 124     static SMALL_DATA: [u8; 10] = [b'A'; 10];
 125     // python3 -c 'print ("A"*10, end="", flush=True)' | md5sum
 126     static SMALL_DATA_MD5: &str = "16c52c6e8326c071da771e66dc6e9e57";
 127     // python3 -c 'print ("A"*10, end="", flush=True)' | sha1sum
 128     static SMALL_DATA_SHA1: &str = "c71613a7386fd67995708464bf0223c0d78225c4";
 129     // python3 -c 'print ("A"*10, end="", flush=True)' | sha256sum
 130     static SMALL_DATA_SHA256: &str =
 131         "1d65bf29403e4fb1767522a107c827b8884d16640cf0e3b18c4c1dd107e0d49d";
 132
 133     static LARGE_DATA: [u8; 1_000_000] = [b'B'; 1_000_000];
 134     // python3 -c 'print ("B"*1000000, end="", flush=True)' | md5sum
 135     static LARGE_DATA_MD5: &str = "9171f6d67a87ca649a702434a03458a1";
 136     // python3 -c 'print ("B"*1000000, end="", flush=True)' | sha1sum
 137     static LARGE_DATA_SHA1: &str = "cfae4cebfd01884111bdede7cf983626bb249c94";
 138     // python3 -c 'print ("B"*1000000, end="", flush=True)' | sha256sum
 139     static LARGE_DATA_SHA256: &str =
 140         "b9193853f7798e92e2f6b82eda336fa7d6fc0fa90fdefe665f372b0bad8cdf8c";
 141
 142     fn verify_digest(alg: Algorithm, data: &'static [u8], hash: &str) {
 143         let reader = Cursor::new(&*data);
 144         let digests = create_digests(&[alg], Box::new(reader)).unwrap();
 145         assert_eq!(digests.len(), 1);
 146         assert_eq!(digests[0], (alg, hex::decode(hash).unwrap()));
 147     }
 148
 149     /// Assert that digests for all algorithms are calculated correctly for a small piece
 150     /// of test data (single block).
 151     #[test]
 152     fn small_digests() {
 153         verify_digest(Algorithm::Md5, &SMALL_DATA, SMALL_DATA_MD5);
 154         verify_digest(Algorithm::Sha1, &SMALL_DATA, SMALL_DATA_SHA1);
 155         verify_digest(Algorithm::Sha256, &SMALL_DATA, SMALL_DATA_SHA256);
 156     }
 157
 158     /// Assert that digests for all algorithms are calculated correctly for a large piece
 159     /// of test data. For our purposes, "large" means that it spans several of the 64 KB
 160     /// blocks used to break up the input processing. Using one million bytes instead of
 161     /// 1 MiB means that the final block will be slightly smaller than the others.
 162     #[test]
 163     fn large_digests() {
 164         verify_digest(Algorithm::Md5, &LARGE_DATA, LARGE_DATA_MD5);
 165         verify_digest(Algorithm::Sha1, &LARGE_DATA, LARGE_DATA_SHA1);
 166         verify_digest(Algorithm::Sha256, &LARGE_DATA, LARGE_DATA_SHA256);
 167     }
 168 }