nardl: fix some warnings

rust/nardl/Cargo.lock

@@ -26,6 +26,21 @@ dependencies = [
  "memchr",
 ]
 
+[[package]]
+name = "alloc-no-stdlib"
+version = "2.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
+
+[[package]]
+name = "alloc-stdlib"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece"
+dependencies = [
+ "alloc-no-stdlib",
+]
+
 [[package]]
 name = "anstream"
 version = "0.6.14"
@@ -81,14 +96,26 @@ version = "0.4.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5"
 dependencies = [
+ "brotli",
+ "bzip2",
  "futures-core",
  "memchr",
  "pin-project-lite",
  "tokio",
+ "xz2",
  "zstd",
  "zstd-safe",
 ]
 
+[[package]]
+name = "async-tempfile"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "acb90d9834a8015109afc79f1f548223a0614edcbab62fb35b62d4b707e975e7"
+dependencies = [
+ "tokio",
+]
+
 [[package]]
 name = "atomic-waker"
 version = "1.1.2"
@@ -143,6 +170,27 @@ dependencies = [
  "generic-array",
 ]
 
+[[package]]
+name = "brotli"
+version = "6.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+ "brotli-decompressor",
+]
+
+[[package]]
+name = "brotli-decompressor"
+version = "4.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362"
+dependencies = [
+ "alloc-no-stdlib",
+ "alloc-stdlib",
+]
+
 [[package]]
 name = "bumpalo"
 version = "3.16.0"
@@ -155,6 +203,27 @@ version = "1.6.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
 
+[[package]]
+name = "bzip2"
+version = "0.4.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8"
+dependencies = [
+ "bzip2-sys",
+ "libc",
+]
+
+[[package]]
+name = "bzip2-sys"
+version = "0.1.11+1.0.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+]
+
 [[package]]
 name = "camino"
 version = "1.1.7"
@@ -298,6 +367,12 @@ dependencies = [
  "syn 1.0.109",
 ]
 
+[[package]]
+name = "data-encoding"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2"
+
 [[package]]
 name = "der"
 version = "0.7.9"
@@ -448,6 +523,23 @@ version = "0.3.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
 
+[[package]]
+name = "futures-io"
+version = "0.3.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
+
+[[package]]
+name = "futures-macro"
+version = "0.3.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.69",
+]
+
 [[package]]
 name = "futures-sink"
 version = "0.3.30"
@@ -467,9 +559,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
 dependencies = [
  "futures-core",
+ "futures-io",
+ "futures-macro",
+ "futures-sink",
  "futures-task",
+ "memchr",
  "pin-project-lite",
  "pin-utils",
+ "slab",
 ]
 
 [[package]]
@@ -739,6 +836,17 @@ version = "0.4.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
 
+[[package]]
+name = "lzma-sys"
+version = "0.1.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+]
+
 [[package]]
 name = "memchr"
 version = "2.7.4"
@@ -775,15 +883,21 @@ dependencies = [
 name = "nardl"
 version = "0.1.0"
 dependencies = [
- "base64",
+ "async-compression",
+ "async-tempfile",
  "color-eyre",
+ "data-encoding",
  "ed25519-dalek",
  "env_logger",
  "log",
  "narinfo",
+ "nix-base32",
  "nix-nar",
  "reqwest",
+ "sha2",
  "tokio",
+ "tokio-stream",
+ "tokio-util",
 ]
 
 [[package]]
@@ -795,6 +909,12 @@ dependencies = [
  "derive_builder",
 ]
 
+[[package]]
+name = "nix-base32"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8548db8274cf1b2b4c093557783f99e9ad64ffdaaa29a6c1af0abc9895c15612"
+
 [[package]]
 name = "nix-nar"
 version = "0.3.0"
@@ -1094,6 +1214,7 @@ dependencies = [
  "url",
  "wasm-bindgen",
  "wasm-bindgen-futures",
+ "wasm-streams",
  "web-sys",
  "webpki-roots",
  "winreg",
@@ -1448,6 +1569,17 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tokio-stream"
+version = "0.1.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af"
+dependencies = [
+ "futures-core",
+ "pin-project-lite",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-util"
 version = "0.7.11"
@@ -1690,6 +1822,19 @@ version = "0.2.92"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
 
+[[package]]
+name = "wasm-streams"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129"
+dependencies = [
+ "futures-util",
+ "js-sys",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
 [[package]]
 name = "web-sys"
 version = "0.3.69"
@@ -1880,6 +2025,15 @@ dependencies = [
  "windows-sys 0.48.0",
 ]
 
+[[package]]
+name = "xz2"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2"
+dependencies = [
+ "lzma-sys",
+]
+
 [[package]]
 name = "zeroize"
 version = "1.8.1"

rust/nardl/Cargo.toml

@@ -4,12 +4,18 @@ version = "0.1.0"
 edition = "2021"
 
 [dependencies]
-base64 = "0.22.1"
+async-compression = { version = "0.4.11", features = [ "tokio", "xz", "bzip2", "zstd", "brotli" ] }
+async-tempfile = "0.6.0"
 color-eyre = "0.6"
+data-encoding = "2.6.0"
 ed25519-dalek = "2.1.1"
 env_logger = "0.11"
 log = "0.4"
 narinfo = "1.0.1"
+nix-base32 = "0.1.1"
 nix-nar = "0.3.0"
-reqwest = { version = "0.12.5", features = ["http2", "rustls-tls", "zstd"], default-features = false }
+reqwest = { version = "0.12.5", features = ["http2", "stream", "rustls-tls", "zstd"], default-features = false }
+sha2 = "0.10.8"
 tokio = { version = "1.38.0", features = ["full"] }
+tokio-util = { version = "0.7.11", features = ["io-util"] }
+tokio-stream = "0.1.15"

rust/nardl/src/main.rs

@@ -1,20 +1,16 @@
 #![allow(dead_code)]
-use base64::{
+use color_eyre::eyre::{self, Context, OptionExt};
-    engine::general_purpose::{
-        STANDARD as BASE64_STANDARD, STANDARD_NO_PAD as BASE64_STANDARD_NO_PAD,
-    },
-    Engine,
-};
-use color_eyre::{
-    eyre::{self, Context, OptionExt},
-    owo_colors::styles::ReversedDisplay,
-};
 use ed25519_dalek::{Signature, VerifyingKey};
-use reqwest::ResponseBuilderExt;
+use sha2::Digest;
 use std::{
     collections::{HashMap, HashSet},
+    os::unix::fs::MetadataExt,
     path::Path,
+    pin::Pin,
 };
+use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt};
+use tokio_stream::StreamExt;
+use tokio_util::io::StreamReader;
 
 // oh no i'm too lazy to add arguments for now
 const SUBSTITUERS: [&str; 2] = ["https://cache.nixos.org", "https://cache.lix.systems"];
@@ -26,6 +22,8 @@ const KEYS: [&str; 2] = [
 
 const OUTPUT: &str = "/nix/store/n50jk09x9hshwx1lh6k3qaiygc7yxbv9-lix-2.90.0-rc1";
 
+const TEST_PREFIX: &str = "/tmp/nardl";
+
 #[tokio::main]
 async fn main() -> eyre::Result<()> {
     env_logger::init();
@@ -33,7 +31,7 @@ async fn main() -> eyre::Result<()> {
 
     let cache_base_urls = SUBSTITUERS
         .iter()
-        .map(|url| reqwest::Url::parse(*url))
+        .map(|url| reqwest::Url::parse(url))
         .collect::<Result<Vec<_>, _>>()?;
 
     let output_path = Path::new(OUTPUT);
@@ -54,9 +52,9 @@ async fn main() -> eyre::Result<()> {
     let trusted_keys = KEYS
         .iter()
         .map(|key| -> eyre::Result<_> {
-            let (name, public_str) = key.split_once(":").ok_or_eyre("Key has no name")?;
+            let (name, public_str) = key.split_once(':').ok_or_eyre("Key has no name")?;
-            let public_bytes = BASE64_STANDARD_NO_PAD
+            let public_bytes = data_encoding::BASE64_NOPAD
-                .decode(public_str.trim_end_matches("="))
+                .decode(public_str.trim_end_matches('=').as_bytes())
                 .wrap_err("Invalid base64 in key")?;
             Ok((
                 name,
@@ -65,8 +63,19 @@ async fn main() -> eyre::Result<()> {
         })
         .collect::<eyre::Result<HashMap<&str, VerifyingKey>>>()?;
 
+    let temp_dir = async_tempfile::TempDir::new().await?;
+
     let client = reqwest::Client::new();
 
+    // We need a no-compression client so that we don't try to decompress
+    // twice if someone sends a Content-Encoding header from their cache
+    let client_no_compression = reqwest::Client::builder()
+        .no_gzip()
+        .no_brotli()
+        .no_zstd()
+        .no_deflate()
+        .build()?;
+
     // TODO: Handle priority here
     for cache in cache_base_urls.iter() {
         let info_str = client
@@ -80,7 +89,7 @@ async fn main() -> eyre::Result<()> {
 
         log::trace!("narinfo {}: {:?}", cache, parsed);
 
-        if &parsed.store_dir != store_dir.to_str().unwrap() {
+        if parsed.store_dir != store_dir.to_str().unwrap() {
             eyre::bail!("Cache {} has a different store directory", cache);
         }
     }
@@ -89,6 +98,7 @@ async fn main() -> eyre::Result<()> {
     let mut outputs_done = HashSet::new();
 
     loop {
+        // Basic variables
         let Some(output) = outputs_remaining.pop() else {
             break;
         };
@@ -97,12 +107,14 @@ async fn main() -> eyre::Result<()> {
         log::debug!("Requesting output {}", output);
 
         let (fingerprint, _) = &output
-            .split_once("-")
+            .split_once('-')
             .ok_or_else(|| eyre::eyre!("Invalid output name {}", output))?;
 
-        let narinfo_text = get_narinfo(client.clone(), cache_base_urls.as_slice(), &fingerprint)
+        // Parse, verify, and handle narinfo
-            .await
+        let (narinfo_text, cache_base_url) =
-            .wrap_err_with(|| format!("While processing {}", output))?;
+            get_narinfo(client.clone(), cache_base_urls.as_slice(), fingerprint)
+                .await
+                .wrap_err_with(|| format!("While processing {}", output))?;
         let narinfo_parsed = narinfo::NarInfo::parse(&narinfo_text).unwrap();
 
         verify_signature(&narinfo_parsed, &trusted_keys, store_dir, &output)
@@ -114,17 +126,90 @@ async fn main() -> eyre::Result<()> {
             }
             outputs_remaining.push(reference.to_string());
         }
+
+        // Download nar
+        let nar_url = cache_base_url.join(narinfo_parsed.url)?;
+        log::trace!("Found nar url {}", nar_url);
+
+        let response = client_no_compression
+            .get(nar_url)
+            .send()
+            .await?
+            .error_for_status()?;
+
+        let reader = StreamReader::new(response.bytes_stream().map(|result| {
+            result.map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err))
+        }));
+
+        let mut decompressed_stream: Pin<Box<dyn AsyncRead>> =
+            if let Some(compression) = narinfo_parsed.compression {
+                match compression.as_ref() {
+                    "zstd" => Box::pin(async_compression::tokio::bufread::ZstdDecoder::new(reader)),
+                    "xz" => Box::pin(async_compression::tokio::bufread::XzDecoder::new(reader)),
+                    "bzip2" => Box::pin(async_compression::tokio::bufread::BzDecoder::new(reader)),
+                    "br" => Box::pin(async_compression::tokio::bufread::BrotliDecoder::new(
+                        reader,
+                    )),
+                    unknown => eyre::bail!("Unknown compression {} in {}", unknown, output),
+                }
+            } else {
+                Box::pin(reader)
+            };
+
+        let mut out_file = tokio::fs::OpenOptions::new()
+            .read(true)
+            .write(true)
+            .truncate(true)
+            .create(true)
+            .open(temp_dir.join("temp.nar"))
+            .await?;
+        tokio::io::copy(&mut decompressed_stream, &mut out_file).await?;
+
+        out_file.seek(std::io::SeekFrom::Start(0)).await?;
+
+        // Verify nar
+        let found_size = out_file.metadata().await?.size();
+        if found_size != narinfo_parsed.nar_size as u64 {
+            eyre::bail!("Wrong nar size for {}", output);
+        }
+
+        let (hash_algorithm, hash_expected) = narinfo_parsed
+            .nar_hash
+            .as_ref()
+            .split_once(':')
+            .ok_or_eyre("Invalid hash in nar")?;
+
+        if hash_algorithm != "sha256" {
+            eyre::bail!("who is using hashes other than sha256????");
+        }
+
+        log::trace!("expected hash: {}", hash_expected);
+
+        let mut buf = [0u8; 1024];
+        let mut hasher = sha2::Sha256::new();
+        loop {
+            let num_read = out_file.read(&mut buf).await?;
+            if num_read == 0 {
+                break;
+            }
+            hasher.update(&buf[0..num_read]);
+        }
+
+        let hash_found = hasher.finalize();
+
+        if nix_base32::to_nix_base32(hash_found.as_ref()) != hash_expected {
+            eyre::bail!("Incorrect hash when downloading {}", output)
+        }
     }
 
     Ok(())
 }
 
-#[must_use]
+async fn get_narinfo<'a>(
-async fn get_narinfo(
     client: reqwest::Client,
-    cache_base_urls: &[reqwest::Url],
+    cache_base_urls: &'a [reqwest::Url],
     fingerprint: &str,
-) -> eyre::Result<String> {
+) -> eyre::Result<(String, &'a reqwest::Url)> {
     for cache in cache_base_urls.iter() {
         let response = client
             .get(cache.join(&format!("{}.narinfo", fingerprint))?)
@@ -134,7 +219,13 @@ async fn get_narinfo(
             continue;
         }
 
-        return response.text().await.wrap_err("Could not download narinfo");
+        return Ok((
+            response
+                .text()
+                .await
+                .wrap_err("Could not download narinfo")?,
+            cache,
+        ));
     }
 
     eyre::bail!("No cache has fingerprint {}", fingerprint);
@@ -150,7 +241,7 @@ fn verify_signature(
         let Some(key) = trusted_keys.get(sig_info.key_name.as_ref()) else {
             continue;
         };
-        let signature_bytes = BASE64_STANDARD.decode(sig_info.sig.as_ref())?;
+        let signature_bytes = data_encoding::BASE64.decode(sig_info.sig.as_ref().as_bytes())?;
 
         let signature = Signature::from_bytes(
             signature_bytes
@@ -159,18 +250,26 @@ fn verify_signature(
                 .wrap_err("Invalid signature length")?,
         );
 
+        let expected_out_path = store_dir
+            .join(output_name)
+            .to_str()
+            .ok_or_eyre("Path not valid UTF-8")?
+            .to_owned();
+
+        if expected_out_path != info.store_path {
+            eyre::bail!("narinfo describes path we weren't expecting");
+        }
+
         // no one documents it, but this is all that's actually signed:
         // https://git.lix.systems/lix-project/lix/src/commit/d461cc1d7b2f489c3886f147166ba5b5e0e37541/src/libstore/path-info.cc#L25
         let fingerprint = format!(
             "1;{};{};{};{}",
-            store_dir
+            expected_out_path,
-                .join(output_name)
-                .to_str()
-                .ok_or_eyre("Path not valid UTF-8")?,
             info.nar_hash,
             info.nar_size,
             info.references
                 .iter()
+                // our narinfo parser sucks and returns empty strings. i should write a new one
                 .filter(|reference| !reference.is_empty())
                 .map(|reference| store_dir
                     .join(reference.as_ref())
@@ -182,7 +281,7 @@ fn verify_signature(
         );
         log::trace!("narinfo fingerprint: `{}`", fingerprint);
 
-        key.verify_strict(&fingerprint.as_bytes(), &signature)
+        key.verify_strict(fingerprint.as_bytes(), &signature)
             .wrap_err("Invalid signature")?;
         return Ok(());
     }