Compare commits

...

3 commits

Author SHA1 Message Date
Artemis Tosini 279dea8167
nardl: fix some warnings 2024-07-08 03:26:02 +00:00
Artemis Tosini afe1f86e1c
nardl: fix hashing 2024-07-08 03:25:13 +00:00
Artemis Tosini 25a01aad83
nardl: download nars 2024-07-08 02:45:07 +00:00
3 changed files with 293 additions and 34 deletions

156
rust/nardl/Cargo.lock generated
View file

@ -26,6 +26,21 @@ dependencies = [
"memchr",
]
[[package]]
name = "alloc-no-stdlib"
version = "2.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
[[package]]
name = "alloc-stdlib"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece"
dependencies = [
"alloc-no-stdlib",
]
[[package]]
name = "anstream"
version = "0.6.14"
@ -81,14 +96,26 @@ version = "0.4.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5"
dependencies = [
"brotli",
"bzip2",
"futures-core",
"memchr",
"pin-project-lite",
"tokio",
"xz2",
"zstd",
"zstd-safe",
]
[[package]]
name = "async-tempfile"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acb90d9834a8015109afc79f1f548223a0614edcbab62fb35b62d4b707e975e7"
dependencies = [
"tokio",
]
[[package]]
name = "atomic-waker"
version = "1.1.2"
@ -143,6 +170,27 @@ dependencies = [
"generic-array",
]
[[package]]
name = "brotli"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
"brotli-decompressor",
]
[[package]]
name = "brotli-decompressor"
version = "4.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
]
[[package]]
name = "bumpalo"
version = "3.16.0"
@ -155,6 +203,27 @@ version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
[[package]]
name = "bzip2"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8"
dependencies = [
"bzip2-sys",
"libc",
]
[[package]]
name = "bzip2-sys"
version = "0.1.11+1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc"
dependencies = [
"cc",
"libc",
"pkg-config",
]
[[package]]
name = "camino"
version = "1.1.7"
@ -298,6 +367,12 @@ dependencies = [
"syn 1.0.109",
]
[[package]]
name = "data-encoding"
version = "2.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8566979429cf69b49a5c740c60791108e86440e8be149bbea4fe54d2c32d6e2"
[[package]]
name = "der"
version = "0.7.9"
@ -448,6 +523,23 @@ version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
[[package]]
name = "futures-io"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
[[package]]
name = "futures-macro"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.69",
]
[[package]]
name = "futures-sink"
version = "0.3.30"
@ -467,9 +559,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
dependencies = [
"futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task",
"memchr",
"pin-project-lite",
"pin-utils",
"slab",
]
[[package]]
@ -739,6 +836,17 @@ version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
[[package]]
name = "lzma-sys"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27"
dependencies = [
"cc",
"libc",
"pkg-config",
]
[[package]]
name = "memchr"
version = "2.7.4"
@ -775,15 +883,21 @@ dependencies = [
name = "nardl"
version = "0.1.0"
dependencies = [
"base64",
"async-compression",
"async-tempfile",
"color-eyre",
"data-encoding",
"ed25519-dalek",
"env_logger",
"log",
"narinfo",
"nix-base32",
"nix-nar",
"reqwest",
"sha2",
"tokio",
"tokio-stream",
"tokio-util",
]
[[package]]
@ -795,6 +909,12 @@ dependencies = [
"derive_builder",
]
[[package]]
name = "nix-base32"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8548db8274cf1b2b4c093557783f99e9ad64ffdaaa29a6c1af0abc9895c15612"
[[package]]
name = "nix-nar"
version = "0.3.0"
@ -1094,6 +1214,7 @@ dependencies = [
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-streams",
"web-sys",
"webpki-roots",
"winreg",
@ -1448,6 +1569,17 @@ dependencies = [
"tokio",
]
[[package]]
name = "tokio-stream"
version = "0.1.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af"
dependencies = [
"futures-core",
"pin-project-lite",
"tokio",
]
[[package]]
name = "tokio-util"
version = "0.7.11"
@ -1690,6 +1822,19 @@ version = "0.2.92"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
[[package]]
name = "wasm-streams"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129"
dependencies = [
"futures-util",
"js-sys",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]]
name = "web-sys"
version = "0.3.69"
@ -1880,6 +2025,15 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "xz2"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2"
dependencies = [
"lzma-sys",
]
[[package]]
name = "zeroize"
version = "1.8.1"

View file

@ -4,12 +4,18 @@ version = "0.1.0"
edition = "2021"
[dependencies]
base64 = "0.22.1"
async-compression = { version = "0.4.11", features = [ "tokio", "xz", "bzip2", "zstd", "brotli" ] }
async-tempfile = "0.6.0"
color-eyre = "0.6"
data-encoding = "2.6.0"
ed25519-dalek = "2.1.1"
env_logger = "0.11"
log = "0.4"
narinfo = "1.0.1"
nix-base32 = "0.1.1"
nix-nar = "0.3.0"
reqwest = { version = "0.12.5", features = ["http2", "rustls-tls", "zstd"], default-features = false }
reqwest = { version = "0.12.5", features = ["http2", "stream", "rustls-tls", "zstd"], default-features = false }
sha2 = "0.10.8"
tokio = { version = "1.38.0", features = ["full"] }
tokio-util = { version = "0.7.11", features = ["io-util"] }
tokio-stream = "0.1.15"

View file

@ -1,20 +1,16 @@
#![allow(dead_code)]
use base64::{
engine::general_purpose::{
STANDARD as BASE64_STANDARD, STANDARD_NO_PAD as BASE64_STANDARD_NO_PAD,
},
Engine,
};
use color_eyre::{
eyre::{self, Context, OptionExt},
owo_colors::styles::ReversedDisplay,
};
use color_eyre::eyre::{self, Context, OptionExt};
use ed25519_dalek::{Signature, VerifyingKey};
use reqwest::ResponseBuilderExt;
use sha2::Digest;
use std::{
collections::{HashMap, HashSet},
os::unix::fs::MetadataExt,
path::Path,
pin::Pin,
};
use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt};
use tokio_stream::StreamExt;
use tokio_util::io::StreamReader;
// oh no i'm too lazy to add arguments for now
const SUBSTITUERS: [&str; 2] = ["https://cache.nixos.org", "https://cache.lix.systems"];
@ -26,6 +22,8 @@ const KEYS: [&str; 2] = [
const OUTPUT: &str = "/nix/store/n50jk09x9hshwx1lh6k3qaiygc7yxbv9-lix-2.90.0-rc1";
const TEST_PREFIX: &str = "/tmp/nardl";
#[tokio::main]
async fn main() -> eyre::Result<()> {
env_logger::init();
@ -33,7 +31,7 @@ async fn main() -> eyre::Result<()> {
let cache_base_urls = SUBSTITUERS
.iter()
.map(|url| reqwest::Url::parse(*url))
.map(|url| reqwest::Url::parse(url))
.collect::<Result<Vec<_>, _>>()?;
let output_path = Path::new(OUTPUT);
@ -54,9 +52,9 @@ async fn main() -> eyre::Result<()> {
let trusted_keys = KEYS
.iter()
.map(|key| -> eyre::Result<_> {
let (name, public_str) = key.split_once(":").ok_or_eyre("Key has no name")?;
let public_bytes = BASE64_STANDARD_NO_PAD
.decode(public_str.trim_end_matches("="))
let (name, public_str) = key.split_once(':').ok_or_eyre("Key has no name")?;
let public_bytes = data_encoding::BASE64_NOPAD
.decode(public_str.trim_end_matches('=').as_bytes())
.wrap_err("Invalid base64 in key")?;
Ok((
name,
@ -65,8 +63,19 @@ async fn main() -> eyre::Result<()> {
})
.collect::<eyre::Result<HashMap<&str, VerifyingKey>>>()?;
let temp_dir = async_tempfile::TempDir::new().await?;
let client = reqwest::Client::new();
// We need a no-compression client so that we don't try to decompress
// twice if someone sends a Content-Encoding header from their cache
let client_no_compression = reqwest::Client::builder()
.no_gzip()
.no_brotli()
.no_zstd()
.no_deflate()
.build()?;
// TODO: Handle priority here
for cache in cache_base_urls.iter() {
let info_str = client
@ -80,7 +89,7 @@ async fn main() -> eyre::Result<()> {
log::trace!("narinfo {}: {:?}", cache, parsed);
if &parsed.store_dir != store_dir.to_str().unwrap() {
if parsed.store_dir != store_dir.to_str().unwrap() {
eyre::bail!("Cache {} has a different store directory", cache);
}
}
@ -89,6 +98,7 @@ async fn main() -> eyre::Result<()> {
let mut outputs_done = HashSet::new();
loop {
// Basic variables
let Some(output) = outputs_remaining.pop() else {
break;
};
@ -97,12 +107,14 @@ async fn main() -> eyre::Result<()> {
log::debug!("Requesting output {}", output);
let (fingerprint, _) = &output
.split_once("-")
.split_once('-')
.ok_or_else(|| eyre::eyre!("Invalid output name {}", output))?;
let narinfo_text = get_narinfo(client.clone(), cache_base_urls.as_slice(), &fingerprint)
.await
.wrap_err_with(|| format!("While processing {}", output))?;
// Parse, verify, and handle narinfo
let (narinfo_text, cache_base_url) =
get_narinfo(client.clone(), cache_base_urls.as_slice(), fingerprint)
.await
.wrap_err_with(|| format!("While processing {}", output))?;
let narinfo_parsed = narinfo::NarInfo::parse(&narinfo_text).unwrap();
verify_signature(&narinfo_parsed, &trusted_keys, store_dir, &output)
@ -114,17 +126,90 @@ async fn main() -> eyre::Result<()> {
}
outputs_remaining.push(reference.to_string());
}
// Download nar
let nar_url = cache_base_url.join(narinfo_parsed.url)?;
log::trace!("Found nar url {}", nar_url);
let response = client_no_compression
.get(nar_url)
.send()
.await?
.error_for_status()?;
let reader = StreamReader::new(response.bytes_stream().map(|result| {
result.map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err))
}));
let mut decompressed_stream: Pin<Box<dyn AsyncRead>> =
if let Some(compression) = narinfo_parsed.compression {
match compression.as_ref() {
"zstd" => Box::pin(async_compression::tokio::bufread::ZstdDecoder::new(reader)),
"xz" => Box::pin(async_compression::tokio::bufread::XzDecoder::new(reader)),
"bzip2" => Box::pin(async_compression::tokio::bufread::BzDecoder::new(reader)),
"br" => Box::pin(async_compression::tokio::bufread::BrotliDecoder::new(
reader,
)),
unknown => eyre::bail!("Unknown compression {} in {}", unknown, output),
}
} else {
Box::pin(reader)
};
let mut out_file = tokio::fs::OpenOptions::new()
.read(true)
.write(true)
.truncate(true)
.create(true)
.open(temp_dir.join("temp.nar"))
.await?;
tokio::io::copy(&mut decompressed_stream, &mut out_file).await?;
out_file.seek(std::io::SeekFrom::Start(0)).await?;
// Verify nar
let found_size = out_file.metadata().await?.size();
if found_size != narinfo_parsed.nar_size as u64 {
eyre::bail!("Wrong nar size for {}", output);
}
let (hash_algorithm, hash_expected) = narinfo_parsed
.nar_hash
.as_ref()
.split_once(':')
.ok_or_eyre("Invalid hash in nar")?;
if hash_algorithm != "sha256" {
eyre::bail!("who is using hashes other than sha256????");
}
log::trace!("expected hash: {}", hash_expected);
let mut buf = [0u8; 1024];
let mut hasher = sha2::Sha256::new();
loop {
let num_read = out_file.read(&mut buf).await?;
if num_read == 0 {
break;
}
hasher.update(&buf[0..num_read]);
}
let hash_found = hasher.finalize();
if nix_base32::to_nix_base32(hash_found.as_ref()) != hash_expected {
eyre::bail!("Incorrect hash when downloading {}", output)
}
}
Ok(())
}
#[must_use]
async fn get_narinfo(
async fn get_narinfo<'a>(
client: reqwest::Client,
cache_base_urls: &[reqwest::Url],
cache_base_urls: &'a [reqwest::Url],
fingerprint: &str,
) -> eyre::Result<String> {
) -> eyre::Result<(String, &'a reqwest::Url)> {
for cache in cache_base_urls.iter() {
let response = client
.get(cache.join(&format!("{}.narinfo", fingerprint))?)
@ -134,7 +219,13 @@ async fn get_narinfo(
continue;
}
return response.text().await.wrap_err("Could not download narinfo");
return Ok((
response
.text()
.await
.wrap_err("Could not download narinfo")?,
cache,
));
}
eyre::bail!("No cache has fingerprint {}", fingerprint);
@ -150,7 +241,7 @@ fn verify_signature(
let Some(key) = trusted_keys.get(sig_info.key_name.as_ref()) else {
continue;
};
let signature_bytes = BASE64_STANDARD.decode(sig_info.sig.as_ref())?;
let signature_bytes = data_encoding::BASE64.decode(sig_info.sig.as_ref().as_bytes())?;
let signature = Signature::from_bytes(
signature_bytes
@ -159,18 +250,26 @@ fn verify_signature(
.wrap_err("Invalid signature length")?,
);
let expected_out_path = store_dir
.join(output_name)
.to_str()
.ok_or_eyre("Path not valid UTF-8")?
.to_owned();
if expected_out_path != info.store_path {
eyre::bail!("narinfo describes path we weren't expecting");
}
// no one documents it, but this is all that's actually signed:
// https://git.lix.systems/lix-project/lix/src/commit/d461cc1d7b2f489c3886f147166ba5b5e0e37541/src/libstore/path-info.cc#L25
let fingerprint = format!(
"1;{};{};{};{}",
store_dir
.join(output_name)
.to_str()
.ok_or_eyre("Path not valid UTF-8")?,
expected_out_path,
info.nar_hash,
info.nar_size,
info.references
.iter()
// our narinfo parser sucks and returns empty strings. i should write a new one
.filter(|reference| !reference.is_empty())
.map(|reference| store_dir
.join(reference.as_ref())
@ -182,7 +281,7 @@ fn verify_signature(
);
log::trace!("narinfo fingerprint: `{}`", fingerprint);
key.verify_strict(&fingerprint.as_bytes(), &signature)
key.verify_strict(fingerprint.as_bytes(), &signature)
.wrap_err("Invalid signature")?;
return Ok(());
}