code.octet-stream.net Git - hashgood/blob - src/calculate.rs

   1 use super::Algorithm;
   2 use md5::{Digest, Md5};
   3 use sha1::Sha1;
   4 use sha2::Sha256;
   5 use std::error::Error;
   6 use std::fs::File;
   7 use std::io::prelude::*;
   8 use std::path::Path;
   9 use std::sync::mpsc::{channel, Receiver};
  10 use std::sync::Arc;
  11 use std::thread;
  12 use std::thread::JoinHandle;
  13
  14 pub type CalculateResult = Result<Vec<(Algorithm, Vec<u8>)>, Box<dyn Error>>;
  15
  16 /// For a given path to the input (may be "-" for STDIN), try to obtain a reader for the data within it.
  17 pub fn get_input_reader(input: &Path) -> Result<Box<dyn Read>, String> {
  18     if input.to_str() == Some("-") {
  19         // Special case: standard input
  20         return Ok(Box::new(std::io::stdin()));
  21     }
  22     if !input.exists() {
  23         return Err(format!(
  24             "The path '{}' does not exist.",
  25             input.to_string_lossy()
  26         ));
  27     }
  28     if !input.is_file() {
  29         return Err(format!(
  30             "The path '{}' is not a regular file.",
  31             input.to_string_lossy()
  32         ));
  33     }
  34     match File::open(input) {
  35         Ok(f) => Ok(Box::new(f)),
  36         Err(e) => Err(format!("File open: {}", e)),
  37     }
  38 }
  39
  40 /// For the given input stream, calculate all requested digest types
  41 pub fn create_digests(algorithms: &[Algorithm], mut input: Box<dyn Read>) -> CalculateResult {
  42     let mut senders = vec![];
  43     let mut handles = vec![];
  44
  45     if algorithms.contains(&Algorithm::Md5) {
  46         let (s, r) = channel();
  47         senders.push(s);
  48         handles.push(md5_digest(r));
  49     }
  50     if algorithms.contains(&Algorithm::Sha1) {
  51         let (s, r) = channel();
  52         senders.push(s);
  53         handles.push(sha1_digest(r));
  54     }
  55     if algorithms.contains(&Algorithm::Sha256) {
  56         let (s, r) = channel();
  57         senders.push(s);
  58         handles.push(sha256_digest(r));
  59     }
  60
  61     // 64 KB chunks will be read from the input at 64 KB and supplied to all hashing threads at once
  62     // Right now that could be up to three threads. If CPU-bound, the other threads will mostly block while the slowest one finishes
  63     const BUF_SIZE: usize = 1024 * 64;
  64     let mut buf = [0; BUF_SIZE];
  65     while let Ok(size) = input.read(&mut buf) {
  66         if size == 0 {
  67             break;
  68         } else {
  69             // Create a shared read-only copy for the hashers to take as input
  70             // buf is freed up for more reading
  71             let chunk = Arc::new(buf[0..size].to_vec());
  72             for s in &senders {
  73                 s.send(chunk.clone())?;
  74             }
  75         }
  76     }
  77     drop(senders);
  78     // Once all data has been sent we just have to wait for the digests to fall out
  79     Ok(handles.into_iter().map(|h| h.join().unwrap()).collect())
  80 }
  81
  82 /// Calculate the md5 digest of some data on the given channel
  83 fn md5_digest(rx: Receiver<Arc<Vec<u8>>>) -> JoinHandle<(Algorithm, Vec<u8>)> {
  84     thread::spawn(move || {
  85         let mut md5 = Md5::new();
  86         while let Ok(chunk) = rx.recv() {
  87             md5.update(&*chunk);
  88         }
  89         let result = md5.finalize();
  90         (Algorithm::Md5, result.to_vec())
  91     })
  92 }
  93
  94 /// Calculate the sha1 digest of some data on the given channel
  95 fn sha1_digest(rx: Receiver<Arc<Vec<u8>>>) -> JoinHandle<(Algorithm, Vec<u8>)> {
  96     thread::spawn(move || {
  97         let mut sha1 = Sha1::new();
  98         while let Ok(chunk) = rx.recv() {
  99             sha1.update(&*chunk);
 100         }
 101         let result = sha1.finalize();
 102         (Algorithm::Sha1, result.to_vec())
 103     })
 104 }
 105
 106 /// Calculate the sha256 digest of some data on the given channel
 107 fn sha256_digest(rx: Receiver<Arc<Vec<u8>>>) -> JoinHandle<(Algorithm, Vec<u8>)> {
 108     thread::spawn(move || {
 109         let mut sha256 = Sha256::new();
 110         while let Ok(chunk) = rx.recv() {
 111             sha256.update(&*chunk);
 112         }
 113         let result = sha256.finalize();
 114         (Algorithm::Sha256, result.to_vec())
 115     })
 116 }
 117
 118 #[cfg(test)]
 119 mod tests {
 120     use super::*;
 121     use std::io::Cursor;
 122
 123     static SMALL_DATA: [u8; 10] = [b'A'; 10];
 124     // python3 -c 'print ("A"*10, end="", flush=True)' | md5sum
 125     static SMALL_DATA_MD5: &str = "16c52c6e8326c071da771e66dc6e9e57";
 126     // python3 -c 'print ("A"*10, end="", flush=True)' | sha1sum
 127     static SMALL_DATA_SHA1: &str = "c71613a7386fd67995708464bf0223c0d78225c4";
 128     // python3 -c 'print ("A"*10, end="", flush=True)' | sha256sum
 129     static SMALL_DATA_SHA256: &str =
 130         "1d65bf29403e4fb1767522a107c827b8884d16640cf0e3b18c4c1dd107e0d49d";
 131
 132     static LARGE_DATA: [u8; 1_000_000] = [b'B'; 1_000_000];
 133     // python3 -c 'print ("B"*1000000, end="", flush=True)' | md5sum
 134     static LARGE_DATA_MD5: &str = "9171f6d67a87ca649a702434a03458a1";
 135     // python3 -c 'print ("B"*1000000, end="", flush=True)' | sha1sum
 136     static LARGE_DATA_SHA1: &str = "cfae4cebfd01884111bdede7cf983626bb249c94";
 137     // python3 -c 'print ("B"*1000000, end="", flush=True)' | sha256sum
 138     static LARGE_DATA_SHA256: &str =
 139         "b9193853f7798e92e2f6b82eda336fa7d6fc0fa90fdefe665f372b0bad8cdf8c";
 140
 141     fn verify_digest(alg: Algorithm, data: &'static [u8], hash: &str) {
 142         let reader = Cursor::new(&*data);
 143         let digests = create_digests(&[alg], Box::new(reader)).unwrap();
 144         assert_eq!(digests.len(), 1);
 145         assert_eq!(digests[0], (alg, hex::decode(hash).unwrap()));
 146     }
 147
 148     /// Assert that digests for all algorithms are calculated correctly for a small piece
 149     /// of test data (single block).
 150     #[test]
 151     fn small_digests() {
 152         verify_digest(Algorithm::Md5, &SMALL_DATA, SMALL_DATA_MD5);
 153         verify_digest(Algorithm::Sha1, &SMALL_DATA, SMALL_DATA_SHA1);
 154         verify_digest(Algorithm::Sha256, &SMALL_DATA, SMALL_DATA_SHA256);
 155     }
 156
 157     /// Assert that digests for all algorithms are calculated correctly for a large piece
 158     /// of test data. For our purposes, "large" means that it spans several of the 64 KB
 159     /// blocks used to break up the input processing. Using one million bytes instead of
 160     /// 1 MiB means that the final block will be slightly smaller than the others.
 161     #[test]
 162     fn large_digests() {
 163         verify_digest(Algorithm::Md5, &LARGE_DATA, LARGE_DATA_MD5);
 164         verify_digest(Algorithm::Sha1, &LARGE_DATA, LARGE_DATA_SHA1);
 165         verify_digest(Algorithm::Sha256, &LARGE_DATA, LARGE_DATA_SHA256);
 166     }
 167 }