nardl: download nars

This commit is contained in:
Artemis Tosini 2024-07-08 02:45:07 +00:00
parent 21d5613c4d
commit 25a01aad83
Signed by: artemist
GPG key ID: EE5227935FE3FF18
3 changed files with 247 additions and 17 deletions

141
rust/nardl/Cargo.lock generated
View file

@ -26,6 +26,21 @@ dependencies = [
"memchr", "memchr",
] ]
[[package]]
name = "alloc-no-stdlib"
version = "2.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3"
[[package]]
name = "alloc-stdlib"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece"
dependencies = [
"alloc-no-stdlib",
]
[[package]] [[package]]
name = "anstream" name = "anstream"
version = "0.6.14" version = "0.6.14"
@ -81,14 +96,26 @@ version = "0.4.11"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5" checksum = "cd066d0b4ef8ecb03a55319dc13aa6910616d0f44008a045bb1835af830abff5"
dependencies = [ dependencies = [
"brotli",
"bzip2",
"futures-core", "futures-core",
"memchr", "memchr",
"pin-project-lite", "pin-project-lite",
"tokio", "tokio",
"xz2",
"zstd", "zstd",
"zstd-safe", "zstd-safe",
] ]
[[package]]
name = "async-tempfile"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "acb90d9834a8015109afc79f1f548223a0614edcbab62fb35b62d4b707e975e7"
dependencies = [
"tokio",
]
[[package]] [[package]]
name = "atomic-waker" name = "atomic-waker"
version = "1.1.2" version = "1.1.2"
@ -143,6 +170,27 @@ dependencies = [
"generic-array", "generic-array",
] ]
[[package]]
name = "brotli"
version = "6.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74f7971dbd9326d58187408ab83117d8ac1bb9c17b085fdacd1cf2f598719b6b"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
"brotli-decompressor",
]
[[package]]
name = "brotli-decompressor"
version = "4.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362"
dependencies = [
"alloc-no-stdlib",
"alloc-stdlib",
]
[[package]] [[package]]
name = "bumpalo" name = "bumpalo"
version = "3.16.0" version = "3.16.0"
@ -155,6 +203,27 @@ version = "1.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9"
[[package]]
name = "bzip2"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8"
dependencies = [
"bzip2-sys",
"libc",
]
[[package]]
name = "bzip2-sys"
version = "0.1.11+1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "736a955f3fa7875102d57c82b8cac37ec45224a07fd32d58f9f7a186b6cd4cdc"
dependencies = [
"cc",
"libc",
"pkg-config",
]
[[package]] [[package]]
name = "camino" name = "camino"
version = "1.1.7" version = "1.1.7"
@ -448,6 +517,23 @@ version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
[[package]]
name = "futures-io"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
[[package]]
name = "futures-macro"
version = "0.3.30"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.69",
]
[[package]] [[package]]
name = "futures-sink" name = "futures-sink"
version = "0.3.30" version = "0.3.30"
@ -467,9 +553,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
dependencies = [ dependencies = [
"futures-core", "futures-core",
"futures-io",
"futures-macro",
"futures-sink",
"futures-task", "futures-task",
"memchr",
"pin-project-lite", "pin-project-lite",
"pin-utils", "pin-utils",
"slab",
] ]
[[package]] [[package]]
@ -739,6 +830,17 @@ version = "0.4.22"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
[[package]]
name = "lzma-sys"
version = "0.1.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fda04ab3764e6cde78b9974eec4f779acaba7c4e84b36eca3cf77c581b85d27"
dependencies = [
"cc",
"libc",
"pkg-config",
]
[[package]] [[package]]
name = "memchr" name = "memchr"
version = "2.7.4" version = "2.7.4"
@ -775,6 +877,8 @@ dependencies = [
name = "nardl" name = "nardl"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"async-compression",
"async-tempfile",
"base64", "base64",
"color-eyre", "color-eyre",
"ed25519-dalek", "ed25519-dalek",
@ -783,7 +887,10 @@ dependencies = [
"narinfo", "narinfo",
"nix-nar", "nix-nar",
"reqwest", "reqwest",
"sha2",
"tokio", "tokio",
"tokio-stream",
"tokio-util",
] ]
[[package]] [[package]]
@ -1094,6 +1201,7 @@ dependencies = [
"url", "url",
"wasm-bindgen", "wasm-bindgen",
"wasm-bindgen-futures", "wasm-bindgen-futures",
"wasm-streams",
"web-sys", "web-sys",
"webpki-roots", "webpki-roots",
"winreg", "winreg",
@ -1448,6 +1556,17 @@ dependencies = [
"tokio", "tokio",
] ]
[[package]]
name = "tokio-stream"
version = "0.1.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af"
dependencies = [
"futures-core",
"pin-project-lite",
"tokio",
]
[[package]] [[package]]
name = "tokio-util" name = "tokio-util"
version = "0.7.11" version = "0.7.11"
@ -1690,6 +1809,19 @@ version = "0.2.92"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
[[package]]
name = "wasm-streams"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129"
dependencies = [
"futures-util",
"js-sys",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]] [[package]]
name = "web-sys" name = "web-sys"
version = "0.3.69" version = "0.3.69"
@ -1880,6 +2012,15 @@ dependencies = [
"windows-sys 0.48.0", "windows-sys 0.48.0",
] ]
[[package]]
name = "xz2"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "388c44dc09d76f1536602ead6d325eb532f5c122f17782bd57fb47baeeb767e2"
dependencies = [
"lzma-sys",
]
[[package]] [[package]]
name = "zeroize" name = "zeroize"
version = "1.8.1" version = "1.8.1"

View file

@ -4,6 +4,8 @@ version = "0.1.0"
edition = "2021" edition = "2021"
[dependencies] [dependencies]
async-compression = { version = "0.4.11", features = [ "tokio", "xz", "bzip2", "zstd", "brotli" ] }
async-tempfile = "0.6.0"
base64 = "0.22.1" base64 = "0.22.1"
color-eyre = "0.6" color-eyre = "0.6"
ed25519-dalek = "2.1.1" ed25519-dalek = "2.1.1"
@ -11,5 +13,8 @@ env_logger = "0.11"
log = "0.4" log = "0.4"
narinfo = "1.0.1" narinfo = "1.0.1"
nix-nar = "0.3.0" nix-nar = "0.3.0"
reqwest = { version = "0.12.5", features = ["http2", "rustls-tls", "zstd"], default-features = false } reqwest = { version = "0.12.5", features = ["http2", "stream", "rustls-tls", "zstd"], default-features = false }
sha2 = "0.10.8"
tokio = { version = "1.38.0", features = ["full"] } tokio = { version = "1.38.0", features = ["full"] }
tokio-util = { version = "0.7.11", features = ["io-util"] }
tokio-stream = "0.1.15"

View file

@ -5,16 +5,17 @@ use base64::{
}, },
Engine, Engine,
}; };
use color_eyre::{ use color_eyre::eyre::{self, Context, OptionExt};
eyre::{self, Context, OptionExt},
owo_colors::styles::ReversedDisplay,
};
use ed25519_dalek::{Signature, VerifyingKey}; use ed25519_dalek::{Signature, VerifyingKey};
use reqwest::ResponseBuilderExt; use sha2::Digest;
use std::{ use std::{
collections::{HashMap, HashSet}, collections::{HashMap, HashSet},
path::Path, path::Path,
pin::Pin,
}; };
use tokio::io::{AsyncRead, AsyncReadExt, AsyncSeekExt};
use tokio_stream::StreamExt;
use tokio_util::io::StreamReader;
// oh no i'm too lazy to add arguments for now // oh no i'm too lazy to add arguments for now
const SUBSTITUERS: [&str; 2] = ["https://cache.nixos.org", "https://cache.lix.systems"]; const SUBSTITUERS: [&str; 2] = ["https://cache.nixos.org", "https://cache.lix.systems"];
@ -26,6 +27,8 @@ const KEYS: [&str; 2] = [
const OUTPUT: &str = "/nix/store/n50jk09x9hshwx1lh6k3qaiygc7yxbv9-lix-2.90.0-rc1"; const OUTPUT: &str = "/nix/store/n50jk09x9hshwx1lh6k3qaiygc7yxbv9-lix-2.90.0-rc1";
const TEST_PREFIX: &str = "/tmp/nardl";
#[tokio::main] #[tokio::main]
async fn main() -> eyre::Result<()> { async fn main() -> eyre::Result<()> {
env_logger::init(); env_logger::init();
@ -65,8 +68,19 @@ async fn main() -> eyre::Result<()> {
}) })
.collect::<eyre::Result<HashMap<&str, VerifyingKey>>>()?; .collect::<eyre::Result<HashMap<&str, VerifyingKey>>>()?;
let temp_dir = async_tempfile::TempDir::new().await?;
let client = reqwest::Client::new(); let client = reqwest::Client::new();
// We need a no-compression client so that we don't try to decompress
// twice if someone sends a Content-Encoding header from their cache
let client_no_compression = reqwest::Client::builder()
.no_gzip()
.no_brotli()
.no_zstd()
.no_deflate()
.build()?;
// TODO: Handle priority here // TODO: Handle priority here
for cache in cache_base_urls.iter() { for cache in cache_base_urls.iter() {
let info_str = client let info_str = client
@ -89,6 +103,7 @@ async fn main() -> eyre::Result<()> {
let mut outputs_done = HashSet::new(); let mut outputs_done = HashSet::new();
loop { loop {
// Basic variables
let Some(output) = outputs_remaining.pop() else { let Some(output) = outputs_remaining.pop() else {
break; break;
}; };
@ -100,7 +115,9 @@ async fn main() -> eyre::Result<()> {
.split_once("-") .split_once("-")
.ok_or_else(|| eyre::eyre!("Invalid output name {}", output))?; .ok_or_else(|| eyre::eyre!("Invalid output name {}", output))?;
let narinfo_text = get_narinfo(client.clone(), cache_base_urls.as_slice(), &fingerprint) // Parse, verify, and handle narinfo
let (narinfo_text, cache_base_url) =
get_narinfo(client.clone(), cache_base_urls.as_slice(), &fingerprint)
.await .await
.wrap_err_with(|| format!("While processing {}", output))?; .wrap_err_with(|| format!("While processing {}", output))?;
let narinfo_parsed = narinfo::NarInfo::parse(&narinfo_text).unwrap(); let narinfo_parsed = narinfo::NarInfo::parse(&narinfo_text).unwrap();
@ -114,17 +131,70 @@ async fn main() -> eyre::Result<()> {
} }
outputs_remaining.push(reference.to_string()); outputs_remaining.push(reference.to_string());
} }
// Download nar
let nar_url = cache_base_url.join(narinfo_parsed.url)?;
log::trace!("Found nar url {}", nar_url);
let response = client_no_compression
.get(nar_url)
.send()
.await?
.error_for_status()?;
let reader = StreamReader::new(response.bytes_stream().map(|result| {
result.map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err))
}));
let mut decompressed_stream: Pin<Box<dyn AsyncRead>> =
if let Some(compression) = narinfo_parsed.compression {
match compression.as_ref() {
"zstd" => Box::pin(async_compression::tokio::bufread::ZstdDecoder::new(reader)),
"xz" => Box::pin(async_compression::tokio::bufread::XzDecoder::new(reader)),
"bzip2" => Box::pin(async_compression::tokio::bufread::BzDecoder::new(reader)),
"br" => Box::pin(async_compression::tokio::bufread::BrotliDecoder::new(
reader,
)),
unknown => eyre::bail!("Unknown compression {} in {}", unknown, output),
}
} else {
Box::pin(reader)
};
let mut out_file = tokio::fs::OpenOptions::new()
.read(true)
.write(true)
.truncate(true)
.create(true)
.open(temp_dir.as_ref().join("temp.nar"))
.await?;
tokio::io::copy(&mut decompressed_stream, &mut out_file).await?;
out_file.seek(std::io::SeekFrom::Start(0)).await?;
let mut buf = [0u8; 1024];
let mut hasher = sha2::Sha256::new();
loop {
let num_read = out_file.read(&mut buf).await?;
if num_read == 0 {
break;
}
hasher.update(&buf[0..num_read]);
}
let found_hash = hasher.finalize();
log::trace!("Got hash {:?}", found_hash);
} }
Ok(()) Ok(())
} }
#[must_use] #[must_use]
async fn get_narinfo( async fn get_narinfo<'a>(
client: reqwest::Client, client: reqwest::Client,
cache_base_urls: &[reqwest::Url], cache_base_urls: &'a [reqwest::Url],
fingerprint: &str, fingerprint: &str,
) -> eyre::Result<String> { ) -> eyre::Result<(String, &'a reqwest::Url)> {
for cache in cache_base_urls.iter() { for cache in cache_base_urls.iter() {
let response = client let response = client
.get(cache.join(&format!("{}.narinfo", fingerprint))?) .get(cache.join(&format!("{}.narinfo", fingerprint))?)
@ -134,7 +204,13 @@ async fn get_narinfo(
continue; continue;
} }
return response.text().await.wrap_err("Could not download narinfo"); return Ok((
response
.text()
.await
.wrap_err("Could not download narinfo")?,
cache,
));
} }
eyre::bail!("No cache has fingerprint {}", fingerprint); eyre::bail!("No cache has fingerprint {}", fingerprint);
@ -159,18 +235,26 @@ fn verify_signature(
.wrap_err("Invalid signature length")?, .wrap_err("Invalid signature length")?,
); );
let expected_out_path = store_dir
.join(output_name)
.to_str()
.ok_or_eyre("Path not valid UTF-8")?
.to_owned();
if expected_out_path != info.store_path {
eyre::bail!("narinfo describes path we weren't expecting");
}
// no one documents it, but this is all that's actually signed: // no one documents it, but this is all that's actually signed:
// https://git.lix.systems/lix-project/lix/src/commit/d461cc1d7b2f489c3886f147166ba5b5e0e37541/src/libstore/path-info.cc#L25 // https://git.lix.systems/lix-project/lix/src/commit/d461cc1d7b2f489c3886f147166ba5b5e0e37541/src/libstore/path-info.cc#L25
let fingerprint = format!( let fingerprint = format!(
"1;{};{};{};{}", "1;{};{};{};{}",
store_dir expected_out_path,
.join(output_name)
.to_str()
.ok_or_eyre("Path not valid UTF-8")?,
info.nar_hash, info.nar_hash,
info.nar_size, info.nar_size,
info.references info.references
.iter() .iter()
// our narinfo parser sucks and returns empty strings. i should write a new one
.filter(|reference| !reference.is_empty()) .filter(|reference| !reference.is_empty())
.map(|reference| store_dir .map(|reference| store_dir
.join(reference.as_ref()) .join(reference.as_ref())